diff --git a/Documentation/ABI/testing/sysfs-driver-intel-xe-sriov b/Documentation/ABI/testing/sysfs-driver-intel-xe-sriov index 2fd7e9b7bacc08..7f5ef9eada531d 100644 --- a/Documentation/ABI/testing/sysfs-driver-intel-xe-sriov +++ b/Documentation/ABI/testing/sysfs-driver-intel-xe-sriov @@ -119,7 +119,7 @@ Description: The GT preemption timeout (PT) in [us] to be applied to all functions. See sriov_admin/{pf,vf}/profile/preempt_timeout_us for more details. - sched_priority: (RW/RO) string + sched_priority: (WO) string The GT scheduling priority to be applied for all functions. See sriov_admin/{pf,vf}/profile/sched_priority for more details. diff --git a/Documentation/PCI/endpoint/pci-vntb-howto.rst b/Documentation/PCI/endpoint/pci-vntb-howto.rst index 9a7a2f0a68498e..3679f5c3025494 100644 --- a/Documentation/PCI/endpoint/pci-vntb-howto.rst +++ b/Documentation/PCI/endpoint/pci-vntb-howto.rst @@ -52,14 +52,14 @@ pci-epf-vntb device, the following commands can be used:: # cd /sys/kernel/config/pci_ep/ # mkdir functions/pci_epf_vntb/func1 -The "mkdir func1" above creates the pci-epf-ntb function device that will +The "mkdir func1" above creates the pci-epf-vntb function device that will be probed by pci_epf_vntb driver. The PCI endpoint framework populates the directory with the following configurable fields:: - # ls functions/pci_epf_ntb/func1 - baseclass_code deviceid msi_interrupts pci-epf-ntb.0 + # ls functions/pci_epf_vntb/func1 + baseclass_code deviceid msi_interrupts pci-epf-vntb.0 progif_code secondary subsys_id vendorid cache_line_size interrupt_pin msix_interrupts primary revid subclass_code subsys_vendor_id @@ -111,13 +111,13 @@ A sample configuration for virtual NTB driver for virtual PCI bus:: # echo 0x080A > functions/pci_epf_vntb/func1/pci_epf_vntb.0/vntb_pid # echo 0x10 > functions/pci_epf_vntb/func1/pci_epf_vntb.0/vbus_number -Binding pci-epf-ntb Device to EP Controller +Binding pci-epf-vntb Device to EP Controller -------------------------------------------- NTB function device should be attached to PCI endpoint controllers connected to the host. - # ln -s controllers/5f010000.pcie_ep functions/pci-epf-ntb/func1/primary + # ln -s controllers/5f010000.pcie_ep functions/pci_epf_vntb/func1/primary Once the above step is completed, the PCI endpoint controllers are ready to establish a link with the host. @@ -139,7 +139,7 @@ lspci Output at Host side ------------------------- Note that the devices listed here correspond to the values populated in -"Creating pci-epf-ntb Device" section above:: +"Creating pci-epf-vntb Device" section above:: # lspci 00:00.0 PCI bridge: Freescale Semiconductor Inc Device 0000 (rev 01) @@ -152,7 +152,7 @@ lspci Output at EP Side / Virtual PCI bus ----------------------------------------- Note that the devices listed here correspond to the values populated in -"Creating pci-epf-ntb Device" section above:: +"Creating pci-epf-vntb Device" section above:: # lspci 10:00.0 Unassigned class [ffff]: Dawicontrol Computersysteme GmbH Device 1234 (rev ff) diff --git a/Documentation/admin-guide/cgroup-v2.rst b/Documentation/admin-guide/cgroup-v2.rst index 7f5b59d95fce5c..510df2461aff23 100644 --- a/Documentation/admin-guide/cgroup-v2.rst +++ b/Documentation/admin-guide/cgroup-v2.rst @@ -2561,10 +2561,10 @@ Cpuset Interface Files Users can manually set it to a value that is different from "cpuset.cpus". One constraint in setting it is that the list of CPUs must be exclusive with respect to "cpuset.cpus.exclusive" - of its sibling. If "cpuset.cpus.exclusive" of a sibling cgroup - isn't set, its "cpuset.cpus" value, if set, cannot be a subset - of it to leave at least one CPU available when the exclusive - CPUs are taken away. + and "cpuset.cpus.exclusive.effective" of its siblings. Another + constraint is that it cannot be a superset of "cpuset.cpus" + of its sibling in order to leave at least one CPU available to + that sibling when the exclusive CPUs are taken away. For a parent cgroup, any one of its exclusive CPUs can only be distributed to at most one of its child cgroups. Having an diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index aa0031108bc1da..f31e9e4c598fc7 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -8090,6 +8090,9 @@ Kernel parameters p = USB_QUIRK_SHORT_SET_ADDRESS_REQ_TIMEOUT (Reduce timeout of the SET_ADDRESS request from 5000 ms to 500 ms); + q = USB_QUIRK_FORCE_ONE_CONFIG (Device + claims zero configurations, + forcing to 1); Example: quirks=0781:5580:bk,0a5c:5834:gij usbhid.mousepoll= diff --git a/Documentation/admin-guide/mm/damon/reclaim.rst b/Documentation/admin-guide/mm/damon/reclaim.rst index 8eba3da8dcee0a..1d68db2aa27ea1 100644 --- a/Documentation/admin-guide/mm/damon/reclaim.rst +++ b/Documentation/admin-guide/mm/damon/reclaim.rst @@ -71,6 +71,10 @@ of parametrs except ``enabled`` again. Once the re-reading is done, this parameter is set as ``N``. If invalid parameters are found while the re-reading, DAMON_RECLAIM will be disabled. +Once ``Y`` is written to this parameter, the user must not write to any +parameters until reading ``commit_inputs`` again returns ``N``. If users +violate this rule, the kernel may exhibit undefined behavior. + min_age ------- diff --git a/Documentation/devicetree/bindings/arm/apple.yaml b/Documentation/devicetree/bindings/arm/apple.yaml index 5c2629ec3d4cbc..49e9563e202cb7 100644 --- a/Documentation/devicetree/bindings/arm/apple.yaml +++ b/Documentation/devicetree/bindings/arm/apple.yaml @@ -96,6 +96,13 @@ description: | - MacBook Pro (13-inch, M2, 2022) - Mac mini (M2, 2023) + Devices based on the "M3" SoC: + + - MacBook Air (13-inch, M3, 2024) + - MacBook Air (15-inch, M3, 2024) + - MacBook Pro (14-inch, M3, 2023) + - iMac (24-inch, M3, 2023) + Devices based on the "M1 Pro", "M1 Max" and "M1 Ultra" SoCs: - MacBook Pro (14-inch, M1 Pro, 2021) @@ -116,6 +123,14 @@ description: | - Mac Studio (M2 Ultra, 2023) - Mac Pro (M2 Ultra, 2023) + Devices based on the "M3 Pro", "M3 Max" and "M3 Ultra" SoCs: + + - MacBook Pro (14-inch, M3 Pro, 2023) + - MacBook Pro (14-inch, M3 Max, 2023) + - MacBook Pro (16-inch, M3 Pro, 2023) + - MacBook Pro (16-inch, M3 Max, 2023) + - Mac Studio (M3 Ultra, 2025) + The compatible property should follow this format: compatible = "apple,", "apple,", "apple,arm-platform"; @@ -297,6 +312,17 @@ properties: - const: apple,t8112 - const: apple,arm-platform + - description: Apple M3 SoC based platforms + items: + - enum: + - apple,j433 # iMac (24-inch, 2x USB-C, M3, 2023) + - apple,j434 # iMac (24-inch, 4x USB-C, M3, 2023) + - apple,j504 # MacBook Pro (14-inch, M3, 2023) + - apple,j613 # MacBook Air (13-inch, M3, 2024) + - apple,j615 # MacBook Air (15-inch, M3, 2024) + - const: apple,t8122 + - const: apple,arm-platform + - description: Apple M1 Pro SoC based platforms items: - enum: @@ -347,6 +373,36 @@ properties: - const: apple,t6022 - const: apple,arm-platform + - description: Apple M3 Pro SoC based platforms + items: + - enum: + - apple,j514s # MacBook Pro (14-inch, M3 Pro, 2023) + - apple,j516s # MacBook Pro (16-inch, M3 Pro, 2023) + - const: apple,t6030 + - const: apple,arm-platform + + - description: Apple M3 Max SoC based platforms + oneOf: + - items: + - enum: + - apple,j514c # MacBook Pro (14-inch, M3 Max, 16 cores, 2023) + - apple,j516c # MacBook Pro (16-inch, M3 Max, 16 cores, 2023) + - const: apple,t6031 + - const: apple,arm-platform + - items: + - enum: + - apple,j514m # MacBook Pro (14-inch, M3 Max, 14 cores, 2023) + - apple,j516m # MacBook Pro (16-inch, M3 Max, 14 cores, 2023) + - const: apple,t6034 + - const: apple,arm-platform + + - description: Apple M3 Ultra SoC based platforms + items: + - enum: + - apple,j575d # Mac Studio (M3 Ultra, 2025) + - const: apple,t6032 + - const: apple,arm-platform + additionalProperties: true ... diff --git a/Documentation/devicetree/bindings/arm/apple/apple,pmgr.yaml b/Documentation/devicetree/bindings/arm/apple/apple,pmgr.yaml index b88f41a225a385..44f1bd3e9e2491 100644 --- a/Documentation/devicetree/bindings/arm/apple/apple,pmgr.yaml +++ b/Documentation/devicetree/bindings/arm/apple/apple,pmgr.yaml @@ -36,7 +36,10 @@ properties: - const: syscon - const: simple-mfd - items: - - const: apple,t6020-pmgr + - enum: + - apple,t6020-pmgr + - apple,t6030-pmgr + - apple,t8122-pmgr - const: apple,t8103-pmgr - const: syscon - const: simple-mfd diff --git a/Documentation/devicetree/bindings/arm/cpus.yaml b/Documentation/devicetree/bindings/arm/cpus.yaml index 736b7ab1bd0a02..e1fd73281657df 100644 --- a/Documentation/devicetree/bindings/arm/cpus.yaml +++ b/Documentation/devicetree/bindings/arm/cpus.yaml @@ -85,11 +85,13 @@ properties: - apple,avalanche - apple,blizzard - apple,cyclone + - apple,everest - apple,firestorm - apple,hurricane-zephyr - apple,icestorm - apple,mistral - apple,monsoon + - apple,sawtooth - apple,twister - apple,typhoon - arm,arm710t diff --git a/Documentation/devicetree/bindings/auxdisplay/holtek,ht16k33.yaml b/Documentation/devicetree/bindings/auxdisplay/holtek,ht16k33.yaml index b90eec2077b4bd..fe1272e86467eb 100644 --- a/Documentation/devicetree/bindings/auxdisplay/holtek,ht16k33.yaml +++ b/Documentation/devicetree/bindings/auxdisplay/holtek,ht16k33.yaml @@ -66,7 +66,7 @@ then: required: - refresh-rate-hz -additionalProperties: false +unevaluatedProperties: false examples: - | diff --git a/Documentation/devicetree/bindings/connector/usb-connector.yaml b/Documentation/devicetree/bindings/connector/usb-connector.yaml index 11e40d225b9f3a..d97b29e49bf55b 100644 --- a/Documentation/devicetree/bindings/connector/usb-connector.yaml +++ b/Documentation/devicetree/bindings/connector/usb-connector.yaml @@ -301,6 +301,7 @@ properties: maxItems: 4 dependencies: + pd-disable: [typec-power-opmode] sink-vdos-v1: [ sink-vdos ] sink-vdos: [ sink-vdos-v1 ] diff --git a/Documentation/devicetree/bindings/display/msm/dp-controller.yaml b/Documentation/devicetree/bindings/display/msm/dp-controller.yaml index ebda78db87a68b..02ddfaab5f56d0 100644 --- a/Documentation/devicetree/bindings/display/msm/dp-controller.yaml +++ b/Documentation/devicetree/bindings/display/msm/dp-controller.yaml @@ -253,7 +253,6 @@ allOf: enum: # these platforms support 2 streams MST on some interfaces, # others are SST only - - qcom,glymur-dp - qcom,sc8280xp-dp - qcom,x1e80100-dp then: @@ -310,6 +309,26 @@ allOf: minItems: 6 maxItems: 8 + - if: + properties: + compatible: + contains: + enum: + # these platforms support 2 streams MST on some interfaces, + # others are SST only, but all controllers have 4 ports + - qcom,glymur-dp + then: + properties: + reg: + minItems: 9 + maxItems: 9 + clocks: + minItems: 5 + maxItems: 6 + clocks-names: + minItems: 5 + maxItems: 6 + unevaluatedProperties: false examples: diff --git a/Documentation/devicetree/bindings/display/msm/qcom,glymur-mdss.yaml b/Documentation/devicetree/bindings/display/msm/qcom,glymur-mdss.yaml index 2329ed96e6cb37..64dde43373ac76 100644 --- a/Documentation/devicetree/bindings/display/msm/qcom,glymur-mdss.yaml +++ b/Documentation/devicetree/bindings/display/msm/qcom,glymur-mdss.yaml @@ -176,13 +176,17 @@ examples: }; }; - displayport-controller@ae90000 { + displayport-controller@af54000 { compatible = "qcom,glymur-dp"; - reg = <0xae90000 0x200>, - <0xae90200 0x200>, - <0xae90400 0x600>, - <0xae91000 0x400>, - <0xae91400 0x400>; + reg = <0xaf54000 0x200>, + <0xaf54200 0x200>, + <0xaf55000 0xc00>, + <0xaf56000 0x400>, + <0xaf57000 0x400>, + <0xaf58000 0x400>, + <0xaf59000 0x400>, + <0xaf5a000 0x600>, + <0xaf5b000 0x600>; interrupt-parent = <&mdss>; interrupts = <12>; diff --git a/Documentation/devicetree/bindings/display/msm/qcom,sm8750-mdss.yaml b/Documentation/devicetree/bindings/display/msm/qcom,sm8750-mdss.yaml index d55fda9a523e27..a38c2261ef1ac5 100644 --- a/Documentation/devicetree/bindings/display/msm/qcom,sm8750-mdss.yaml +++ b/Documentation/devicetree/bindings/display/msm/qcom,sm8750-mdss.yaml @@ -10,7 +10,7 @@ maintainers: - Krzysztof Kozlowski description: - SM8650 MSM Mobile Display Subsystem(MDSS), which encapsulates sub-blocks like + SM8750 MSM Mobile Display Subsystem(MDSS), which encapsulates sub-blocks like DPU display controller, DSI and DP interfaces etc. $ref: /schemas/display/msm/mdss-common.yaml# diff --git a/Documentation/devicetree/bindings/dma/apple,sio.yaml b/Documentation/devicetree/bindings/dma/apple,sio.yaml new file mode 100644 index 00000000000000..0e3780ad9dd79a --- /dev/null +++ b/Documentation/devicetree/bindings/dma/apple,sio.yaml @@ -0,0 +1,111 @@ +# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/dma/apple,sio.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple SIO Coprocessor + +description: + SIO is a coprocessor on Apple M1 and later chips (and maybe also on earlier + chips). Its role is to offload SPI, UART and DisplayPort audio transfers, + being a pretend DMA controller. + +maintainers: + - Martin Povišer + +allOf: + - $ref: dma-controller.yaml# + +properties: + compatible: + items: + - enum: + - apple,t6000-sio + - apple,t8103-sio + - const: apple,sio + + reg: + maxItems: 1 + + '#dma-cells': + const: 1 + description: + DMA clients specify a single cell that corresponds to the RTKit endpoint + number used for arranging the transfers in question + + dma-channels: + maximum: 128 + + mboxes: + maxItems: 1 + + iommus: + maxItems: 1 + + power-domains: + maxItems: 1 + + memory-region: + minItems: 2 + maxItems: 8 + description: + A number of references to reserved memory regions among which are the DATA/TEXT + sections of coprocessor executable firmware and also auxiliary firmware data + describing the available DMA-enabled peripherals + + apple,sio-firmware-params: + $ref: /schemas/types.yaml#/definitions/uint32-array + description: | + Parameters in the form of opaque key/value pairs that are to be sent to the SIO + coprocesssor once it boots. These parameters can point into the reserved memory + regions (in device address space). + + Note that unlike Apple's firmware, we treat the parameters, and the data they + refer to, as opaque. Apple embed short data blobs into their SIO devicetree node + that describe the DMA-enabled peripherals (presumably with defined semantics). + Their driver processes those blobs and sets up data structure in mapped device + memory, then references this memory in the parameters sent to the SIO. At the + level of description we are opting for in this binding, we assume the job of + constructing those data structures has been done in advance, leaving behind an + opaque list of key/value parameter pairs to be sent by a prospective driver. + + This approach is chosen for two reasons: + + - It means we don't need to try to understand the semantics of Apple's blobs + as long as we know the transformation we need to do from Apple's devicetree + data to SIO data (which can be shoved away into a loader). It also means the + semantics of Apple's blobs (or of something to replace them) need not be part + of the binding and be kept up with Apple's firmware changes in the future. + + - It leaves less work for the driver attaching on this binding. Instead the work + is done upfront in the loader which can be better suited for keeping up with + Apple's firmware changes. + +required: + - compatible + - reg + - '#dma-cells' + - dma-channels + - mboxes + - iommus + - power-domains + +additionalProperties: false + +examples: + - | + sio: dma-controller@36400000 { + compatible = "apple,t8103-sio", "apple,sio"; + reg = <0x36400000 0x8000>; + dma-channels = <128>; + #dma-cells = <1>; + mboxes = <&sio_mbox>; + iommus = <&sio_dart 0>; + power-domains = <&ps_sio_cpu>; + memory-region = <&sio_text>, <&sio_data>, + <&sio_auxdata1>, <&sio_auxdata2>; /* Filled by loader */ + apple,sio-firmware-params = <0xb 0x10>, <0xc 0x1b80>, <0xf 0x14>, + <0x10 0x1e000>, <0x30d 0x34>, <0x30e 0x4000>, + <0x1a 0x38>, <0x1b 0x50>; /* Filled by loader */ + }; diff --git a/Documentation/devicetree/bindings/gpio/microchip,mpfs-gpio.yaml b/Documentation/devicetree/bindings/gpio/microchip,mpfs-gpio.yaml index 184432d24ea181..f42c54653d5217 100644 --- a/Documentation/devicetree/bindings/gpio/microchip,mpfs-gpio.yaml +++ b/Documentation/devicetree/bindings/gpio/microchip,mpfs-gpio.yaml @@ -37,7 +37,7 @@ properties: const: 2 "#interrupt-cells": - const: 1 + const: 2 ngpios: description: @@ -86,7 +86,7 @@ examples: gpio-controller; #gpio-cells = <2>; interrupt-controller; - #interrupt-cells = <1>; + #interrupt-cells = <2>; interrupts = <53>, <53>, <53>, <53>, <53>, <53>, <53>, <53>, <53>, <53>, <53>, <53>, diff --git a/Documentation/devicetree/bindings/hwmon/apple,smc-hwmon.yaml b/Documentation/devicetree/bindings/hwmon/apple,smc-hwmon.yaml new file mode 100644 index 00000000000000..2eec317bc4b3e6 --- /dev/null +++ b/Documentation/devicetree/bindings/hwmon/apple,smc-hwmon.yaml @@ -0,0 +1,86 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/hwmon/apple,smc-hwmon.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple SMC Hardware Monitoring + +description: + Apple's System Management Controller (SMC) exposes a vast array of + hardware monitoring sensors, including temperature probes, current and + voltage sense, power meters, and fan speeds. It also provides endpoints + to manually control the speed of each fan individually. Each Apple + Silicon device exposes a different set of endpoints via SMC keys. This + is true even when two machines share an SoC. The CPU core temperature + sensor keys on an M1 Mac mini are different to those on an M1 MacBook + Pro, for example. + +maintainers: + - James Calligeros + +$defs: + sensor: + type: object + + properties: + apple,key-id: + $ref: /schemas/types.yaml#/definitions/string + pattern: "^[A-Za-z0-9]{4}$" + description: The SMC FourCC key of the desired sensor. + Must match the node's suffix. + + label: + description: Human-readable name for the sensor + + required: + - apple,key-id + +properties: + compatible: + const: apple,smc-hwmon + +patternProperties: + "^current-[A-Za-z0-9]{4}$": + $ref: "#/$defs/sensor" + unevaluatedProperties: false + + "^fan-[A-Za-z0-9]{4}$": + $ref: "#/$defs/sensor" + unevaluatedProperties: false + + properties: + apple,fan-minimum: + $ref: /schemas/types.yaml#/definitions/string + pattern: "^[A-Za-z0-9]{4}$" + description: SMC key containing the fan's minimum speed + + apple,fan-maximum: + $ref: /schemas/types.yaml#/definitions/string + pattern: "^[A-Za-z0-9]{4}$" + description: SMC key containing the fan's maximum speed + + apple,fan-target: + $ref: /schemas/types.yaml#/definitions/string + pattern: "^[A-Za-z0-9]{4}$" + description: Writeable endpoint for setting desired fan speed + + apple,fan-mode: + $ref: /schemas/types.yaml#/definitions/string + pattern: "^[A-Za-z0-9]{4}$" + description: Writeable key to enable/disable manual fan control + + + "^power-[A-Za-z0-9]{4}$": + $ref: "#/$defs/sensor" + unevaluatedProperties: false + + "^temperature-[A-Za-z0-9]{4}$": + $ref: "#/$defs/sensor" + unevaluatedProperties: false + + "^voltage-[A-Za-z0-9]{4}$": + $ref: "#/$defs/sensor" + unevaluatedProperties: false + +additionalProperties: false diff --git a/Documentation/devicetree/bindings/i2c/apple,i2c.yaml b/Documentation/devicetree/bindings/i2c/apple,i2c.yaml index 500a965bdb7a84..ea320e8a22b02e 100644 --- a/Documentation/devicetree/bindings/i2c/apple,i2c.yaml +++ b/Documentation/devicetree/bindings/i2c/apple,i2c.yaml @@ -22,7 +22,10 @@ properties: compatible: oneOf: - items: - - const: apple,t6020-i2c + - enum: + - apple,t6020-i2c + - apple,t6030-i2c + - apple,t8122-i2c - const: apple,t8103-i2c - items: - enum: diff --git a/Documentation/devicetree/bindings/interrupt-controller/apple,aic2.yaml b/Documentation/devicetree/bindings/interrupt-controller/apple,aic2.yaml index ee5a0dfff43781..d0d9a90e96e7ba 100644 --- a/Documentation/devicetree/bindings/interrupt-controller/apple,aic2.yaml +++ b/Documentation/devicetree/bindings/interrupt-controller/apple,aic2.yaml @@ -4,10 +4,10 @@ $id: http://devicetree.org/schemas/interrupt-controller/apple,aic2.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Apple Interrupt Controller 2 +title: Apple Interrupt Controller 2 and 3 maintainers: - - Hector Martin + - Janne Grunau description: | The Apple Interrupt Controller 2 is a simple interrupt controller present on @@ -28,14 +28,24 @@ description: | which do not go through a discrete interrupt controller. It also handles FIQ-based Fast IPIs. + The Apple Interrupt Controller 3 is in its base functionality very similar to + the Apple Interrupt Controller 2 and uses the same device tree bindings. It is + found on Apple ARM SoCs platforms starting with t8122 (M3). + properties: compatible: - items: - - enum: - - apple,t8112-aic - - apple,t6000-aic - - apple,t6020-aic - - const: apple,aic2 + oneOf: + - items: + - enum: + - apple,t6000-aic + - apple,t6020-aic + - apple,t8112-aic + - const: apple,aic2 + - items: + - enum: + - apple,t6030-aic3 + - const: apple,t8122-aic3 + - const: apple,t8122-aic3 interrupt-controller: true @@ -117,7 +127,9 @@ allOf: properties: compatible: contains: - const: apple,t8112-aic + enum: + - apple,t8112-aic + - apple,t8122-aic3 then: properties: '#interrupt-cells': diff --git a/Documentation/devicetree/bindings/iommu/apple,dart.yaml b/Documentation/devicetree/bindings/iommu/apple,dart.yaml index 47ec7fa52c3ac6..e179199dbd3b54 100644 --- a/Documentation/devicetree/bindings/iommu/apple,dart.yaml +++ b/Documentation/devicetree/bindings/iommu/apple,dart.yaml @@ -29,7 +29,9 @@ properties: - apple,t8110-dart - apple,t6000-dart - items: - - const: apple,t6020-dart + - enum: + - apple,t6020-dart + - apple,t8122-dart - const: apple,t8110-dart reg: diff --git a/Documentation/devicetree/bindings/iommu/apple,sart.yaml b/Documentation/devicetree/bindings/iommu/apple,sart.yaml index 88e66d4b13c6bc..8fd7775704ab7d 100644 --- a/Documentation/devicetree/bindings/iommu/apple,sart.yaml +++ b/Documentation/devicetree/bindings/iommu/apple,sart.yaml @@ -33,6 +33,7 @@ properties: - enum: - apple,t6020-sart - apple,t8112-sart + - apple,t8122-sart - const: apple,t6000-sart - enum: - apple,t6000-sart diff --git a/Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml b/Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml index 28985cc62c2539..946c909c692275 100644 --- a/Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml +++ b/Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml @@ -30,6 +30,7 @@ properties: - enum: - apple,t8103-asc-mailbox - apple,t8112-asc-mailbox + - apple,t8122-asc-mailbox - apple,t6000-asc-mailbox - apple,t6020-asc-mailbox - const: apple,asc-mailbox-v4 diff --git a/Documentation/devicetree/bindings/media/qcom,qcs8300-camss.yaml b/Documentation/devicetree/bindings/media/qcom,qcs8300-camss.yaml index 80a4540a22dc23..e5f170aa4d9ee7 100644 --- a/Documentation/devicetree/bindings/media/qcom,qcs8300-camss.yaml +++ b/Documentation/devicetree/bindings/media/qcom,qcs8300-camss.yaml @@ -120,6 +120,14 @@ properties: items: - const: top + vdda-phy-supply: + description: + Phandle to a 0.88V regulator supply to CSI PHYs. + + vdda-pll-supply: + description: + Phandle to 1.2V regulator supply to CSI PHYs pll block. + ports: $ref: /schemas/graph.yaml#/properties/ports @@ -160,6 +168,8 @@ required: - power-domains - power-domain-names - ports + - vdda-phy-supply + - vdda-pll-supply additionalProperties: false @@ -328,6 +338,9 @@ examples: power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; power-domain-names = "top"; + vdda-phy-supply = <&vreg_l4a_0p88>; + vdda-pll-supply = <&vreg_l1c_1p2>; + ports { #address-cells = <1>; #size-cells = <0>; diff --git a/Documentation/devicetree/bindings/mfd/apple,smc.yaml b/Documentation/devicetree/bindings/mfd/apple,smc.yaml index 0410e712c900a7..d5d30c05384585 100644 --- a/Documentation/devicetree/bindings/mfd/apple,smc.yaml +++ b/Documentation/devicetree/bindings/mfd/apple,smc.yaml @@ -17,7 +17,9 @@ properties: compatible: oneOf: - items: - - const: apple,t6020-smc + - enum: + - apple,t6020-smc + - apple,t8122-smc - const: apple,t8103-smc - items: - enum: @@ -49,6 +51,9 @@ properties: rtc: $ref: /schemas/rtc/apple,smc-rtc.yaml + hwmon: + $ref: /schemas/hwmon/apple,smc-hwmon.yaml + additionalProperties: false required: @@ -89,5 +94,38 @@ examples: nvmem-cells = <&rtc_offset>; nvmem-cell-names = "rtc_offset"; }; + + hwmon { + compatible = "apple,smc-hwmon"; + + current-ID0R { + apple,key-id = "ID0R"; + label = "AC Input Current"; + }; + + fan-F0Ac { + apple,key-id = "F0Ac"; + apple,fan-minimum = "F0Mn"; + apple,fan-maximum = "F0Mx"; + apple,fan-target = "F0Tg"; + apple,fan-mode = "F0Md"; + label = "Fan 1"; + }; + + power-PSTR { + apple,key-id = "PSTR"; + label = "Total System Power"; + }; + + temperature-TW0P { + apple,key-id = "TW0P"; + label = "WiFi/BT Module Temperature"; + }; + + voltage-VD0R { + apple,key-id = "VD0R"; + label = "AC Input Voltage"; + }; + }; }; }; diff --git a/Documentation/devicetree/bindings/net/nvidia,tegra234-mgbe.yaml b/Documentation/devicetree/bindings/net/nvidia,tegra234-mgbe.yaml index 2bd3efff2485ef..215f14d1897d21 100644 --- a/Documentation/devicetree/bindings/net/nvidia,tegra234-mgbe.yaml +++ b/Documentation/devicetree/bindings/net/nvidia,tegra234-mgbe.yaml @@ -42,7 +42,7 @@ properties: - const: mgbe - const: mac - const: mac-divider - - const: ptp-ref + - const: ptp_ref - const: rx-input-m - const: rx-input - const: tx @@ -133,7 +133,7 @@ examples: <&bpmp TEGRA234_CLK_MGBE0_RX_PCS_M>, <&bpmp TEGRA234_CLK_MGBE0_RX_PCS>, <&bpmp TEGRA234_CLK_MGBE0_TX_PCS>; - clock-names = "mgbe", "mac", "mac-divider", "ptp-ref", "rx-input-m", + clock-names = "mgbe", "mac", "mac-divider", "ptp_ref", "rx-input-m", "rx-input", "tx", "eee-pcs", "rx-pcs-input", "rx-pcs-m", "rx-pcs", "tx-pcs"; resets = <&bpmp TEGRA234_RESET_MGBE0_MAC>, diff --git a/Documentation/devicetree/bindings/nvme/apple,nvme-ans.yaml b/Documentation/devicetree/bindings/nvme/apple,nvme-ans.yaml index 4c0b1f90aff846..352251c801f5a3 100644 --- a/Documentation/devicetree/bindings/nvme/apple,nvme-ans.yaml +++ b/Documentation/devicetree/bindings/nvme/apple,nvme-ans.yaml @@ -14,7 +14,9 @@ properties: oneOf: - const: apple,t8015-nvme-ans2 - items: - - const: apple,t6020-nvme-ans2 + - enum: + - apple,t6020-nvme-ans2 + - apple,t8122-nvme-ans2 - const: apple,t8103-nvme-ans2 - items: - enum: diff --git a/Documentation/devicetree/bindings/pci/apple,pcie.yaml b/Documentation/devicetree/bindings/pci/apple,pcie.yaml index c0852be04f6ded..0bed9e126fdaa4 100644 --- a/Documentation/devicetree/bindings/pci/apple,pcie.yaml +++ b/Documentation/devicetree/bindings/pci/apple,pcie.yaml @@ -41,6 +41,9 @@ properties: - apple,t8112-pcie - apple,t6000-pcie - const: apple,pcie + - items: + - const: apple,t8122-pcie + - const: apple,t6020-pcie - const: apple,t6020-pcie reg: @@ -82,6 +85,27 @@ properties: power-domains: maxItems: 1 +patternProperties: + "^pci@": + $ref: /schemas/pci/pci-bus.yaml# + type: object + description: A single PCI root port + + properties: + reg: + maxItems: 1 + + pwren-gpios: + description: Optional GPIO to power on the device + maxItems: 1 + + required: + - reset-gpios + - interrupt-controller + - "#interrupt-cells" + - interrupt-map-mask + - interrupt-map + required: - compatible - reg @@ -161,7 +185,7 @@ examples: pinctrl-0 = <&pcie_pins>; pinctrl-names = "default"; - pci@0,0 { + port00: pci@0,0 { device_type = "pci"; reg = <0x0 0x0 0x0 0x0 0x0>; reset-gpios = <&pinctrl_ap 152 0>; @@ -169,9 +193,17 @@ examples: #address-cells = <3>; #size-cells = <2>; ranges; + + interrupt-controller; + #interrupt-cells = <1>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &port00 0 0 0 0>, + <0 0 0 2 &port00 0 0 0 1>, + <0 0 0 3 &port00 0 0 0 2>, + <0 0 0 4 &port00 0 0 0 3>; }; - pci@1,0 { + port01: pci@1,0 { device_type = "pci"; reg = <0x800 0x0 0x0 0x0 0x0>; reset-gpios = <&pinctrl_ap 153 0>; @@ -179,9 +211,17 @@ examples: #address-cells = <3>; #size-cells = <2>; ranges; + + interrupt-controller; + #interrupt-cells = <1>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &port01 0 0 0 0>, + <0 0 0 2 &port01 0 0 0 1>, + <0 0 0 3 &port01 0 0 0 2>, + <0 0 0 4 &port01 0 0 0 3>; }; - pci@2,0 { + port02: pci@2,0 { device_type = "pci"; reg = <0x1000 0x0 0x0 0x0 0x0>; reset-gpios = <&pinctrl_ap 33 0>; @@ -189,6 +229,14 @@ examples: #address-cells = <3>; #size-cells = <2>; ranges; + + interrupt-controller; + #interrupt-cells = <1>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &port02 0 0 0 0>, + <0 0 0 2 &port02 0 0 0 1>, + <0 0 0 3 &port02 0 0 0 2>, + <0 0 0 4 &port02 0 0 0 3>; }; }; }; diff --git a/Documentation/devicetree/bindings/phy/apple,atcphy.yaml b/Documentation/devicetree/bindings/phy/apple,atcphy.yaml new file mode 100644 index 00000000000000..0acac7e3ee67ef --- /dev/null +++ b/Documentation/devicetree/bindings/phy/apple,atcphy.yaml @@ -0,0 +1,222 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/phy/apple,atcphy.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple Type-C PHY (ATCPHY) + +maintainers: + - Sven Peter + +description: > + The Apple Type-C PHY (ATCPHY) is a combined PHY for USB 2.0, USB 3.x, + USB4/Thunderbolt, and DisplayPort connectivity via Type-C ports found in + Apple Silicon SoCs. + + The PHY handles muxing between these different protocols and also provides the + reset controller for the attached DWC3 USB controller. + + It is designed for USB4 operation and does not handle individual differential + pairs as distinct DisplayPort lanes. Any reference to lane in this binding + hence refers to two differential pairs (RX and TX) as used in USB terminology. + + In order to correctly setup these lanes for the various modes calibration + values copied from Apple's firmware and converted to the format described + below by our bootloader m1n1 are required. Without these only USB2 operation + is possible. + +allOf: + - $ref: /schemas/usb/usb-switch.yaml# + +$defs: + apple,tunable: + $ref: /schemas/types.yaml#/definitions/uint32-matrix + items: + items: + - description: Register offset + - description: Mask to be applied to the register value + - description: Bits to be set after applying the mask + description: > + List of (register offset, mask, value) tuples copied from Apple's Device + Tree by our bootloader m1n1 and used to configure the PHY. These values + even vary for a single product/device and likely contain calibration + values determined by Apple at manufacturing time. + Unless otherwise noted these tunables are always applied to the core + register region. + +properties: + compatible: + oneOf: + - items: + - enum: + - apple,t6000-atcphy + - apple,t6020-atcphy + - apple,t8112-atcphy + - const: apple,t8103-atcphy + - const: apple,t8103-atcphy + + reg: + items: + - description: Common controls for all PHYs (USB2/3/4, DisplayPort, TBT) + - description: DisplayPort Alternate Mode PHY specific controls + - description: Type-C PHY AXI to Apple Fabric interconnect controls + - description: USB2 PHY specific controls + - description: USB3 PIPE interface controls + + reg-names: + items: + - const: core + - const: lpdptx + - const: axi2af + - const: usb2phy + - const: pipehandler + + "#phy-cells": + const: 1 + + "#reset-cells": + const: 0 + + mode-switch: true + orientation-switch: true + + power-domains: + maxItems: 1 + + ports: + $ref: /schemas/graph.yaml#/properties/ports + properties: + port@0: + $ref: /schemas/graph.yaml#/properties/port + description: Outgoing connection to the SS port of the Type-C connector. + + port@1: + $ref: /schemas/graph.yaml#/properties/port + description: Incoming endpoint from the USB3 controller. + + port@2: + $ref: /schemas/graph.yaml#/properties/port + description: Incoming endpoint from the DisplayPort controller. + + port@3: + $ref: /schemas/graph.yaml#/properties/port + description: Incoming endpoint from the USB4/Thunderbolt controller. + + apple,tunable-common-a: + $ref: "#/$defs/apple,tunable" + description: > + Common tunables required for all modes, applied before tunable-axi2af. + + apple,tunable-axi2af: + $ref: "#/$defs/apple,tunable" + description: > + AXI to Apple Fabric tunables, required for all modes. Unlike all other + tunables these are applied to the axi2af region. + + apple,tunable-common-b: + $ref: "#/$defs/apple,tunable" + description: > + Common tunables required for all modes, applied after tunable-axi2af. + + apple,tunable-lane0-usb: + $ref: "#/$defs/apple,tunable" + description: USB3 tunables for lane 0. + + apple,tunable-lane1-usb: + $ref: "#/$defs/apple,tunable" + description: USB3 tunables for lane 1. + + apple,tunable-lane0-cio: + $ref: "#/$defs/apple,tunable" + description: USB4/Thunderbolt ("Converged IO") tunables for lane 0. + + apple,tunable-lane1-cio: + $ref: "#/$defs/apple,tunable" + description: USB4/Thunderbolt ("Converged IO") tunables for lane 1. + + apple,tunable-lane0-dp: + $ref: "#/$defs/apple,tunable" + description: > + DisplayPort tunables for lane 0. + + Note that lane here refers to a USB RX and TX pair re-used for DisplayPort + and not to an individual DisplayPort differential lane. + + apple,tunable-lane1-dp: + $ref: "#/$defs/apple,tunable" + description: > + DisplayPort tunables for lane 1. + + Note that lane here refers to a USB RX and TX pair re-used for DisplayPort + and not to an individual DisplayPort differential lane. + +required: + - compatible + - reg + - reg-names + - "#phy-cells" + - "#reset-cells" + - orientation-switch + - mode-switch + - power-domains + - ports + +additionalProperties: false + +examples: + - | + phy@83000000 { + compatible = "apple,t8103-atcphy"; + reg = <0x83000000 0x4c000>, + <0x83050000 0x8000>, + <0x80000000 0x4000>, + <0x82a90000 0x4000>, + <0x82a84000 0x4000>; + reg-names = "core", "lpdptx", "axi2af", "usb2phy", + "pipehandler"; + + #phy-cells = <1>; + #reset-cells = <0>; + + orientation-switch; + mode-switch; + power-domains = <&ps_atc0_usb>; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + endpoint { + remote-endpoint = <&typec_connector_ss>; + }; + }; + + port@1 { + reg = <1>; + + endpoint { + remote-endpoint = <&dwc3_ss_out>; + }; + }; + + port@2 { + reg = <2>; + + endpoint { + remote-endpoint = <&dcp_dp_out>; + }; + }; + + port@3 { + reg = <3>; + + endpoint { + remote-endpoint = <&acio_tbt_out>; + }; + }; + }; + }; diff --git a/Documentation/devicetree/bindings/phy/qcom,edp-phy.yaml b/Documentation/devicetree/bindings/phy/qcom,edp-phy.yaml index eb97181cbb9579..bfc4d75f50ff9e 100644 --- a/Documentation/devicetree/bindings/phy/qcom,edp-phy.yaml +++ b/Documentation/devicetree/bindings/phy/qcom,edp-phy.yaml @@ -37,12 +37,15 @@ properties: - description: PLL register block clocks: - maxItems: 2 + minItems: 2 + maxItems: 3 clock-names: + minItems: 2 items: - const: aux - const: cfg_ahb + - const: ref "#clock-cells": const: 1 @@ -64,6 +67,29 @@ required: - "#clock-cells" - "#phy-cells" +allOf: + - if: + properties: + compatible: + enum: + - qcom,x1e80100-dp-phy + then: + properties: + clocks: + minItems: 3 + maxItems: 3 + clock-names: + minItems: 3 + maxItems: 3 + else: + properties: + clocks: + minItems: 2 + maxItems: 2 + clock-names: + minItems: 2 + maxItems: 2 + additionalProperties: false examples: diff --git a/Documentation/devicetree/bindings/pinctrl/apple,pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/apple,pinctrl.yaml index 665ec79a69f1db..1d9e101745fbc7 100644 --- a/Documentation/devicetree/bindings/pinctrl/apple,pinctrl.yaml +++ b/Documentation/devicetree/bindings/pinctrl/apple,pinctrl.yaml @@ -18,7 +18,10 @@ properties: compatible: oneOf: - items: - - const: apple,t6020-pinctrl + - enum: + - apple,t6020-pinctrl + - apple,t6030-pinctrl + - apple,t8122-pinctrl - const: apple,t8103-pinctrl - items: # Do not add additional SoC to this list. diff --git a/Documentation/devicetree/bindings/power/apple,pmgr-pwrstate.yaml b/Documentation/devicetree/bindings/power/apple,pmgr-pwrstate.yaml index caf15188099921..9e3b394bf64d00 100644 --- a/Documentation/devicetree/bindings/power/apple,pmgr-pwrstate.yaml +++ b/Documentation/devicetree/bindings/power/apple,pmgr-pwrstate.yaml @@ -43,7 +43,10 @@ properties: - apple,t6000-pmgr-pwrstate - const: apple,pmgr-pwrstate - items: - - const: apple,t6020-pmgr-pwrstate + - enum: + - apple,t6020-pmgr-pwrstate + - apple,t6030-pmgr-pwrstate + - apple,t8122-pmgr-pwrstate - const: apple,t8103-pmgr-pwrstate reg: @@ -80,6 +83,18 @@ properties: minimum: 0 maximum: 15 + apple,force-disable: + description: + Forces this device to be disabled (bus access blocked) when the power + domain is powered down. + type: boolean + + apple,force-reset: + description: + Forces a reset/error recovery of the power control logic when the power + domain is powered down. + type: boolean + required: - compatible - reg diff --git a/Documentation/devicetree/bindings/pwm/apple,s5l-fpwm.yaml b/Documentation/devicetree/bindings/pwm/apple,s5l-fpwm.yaml index 04519b0c581d0e..dffb72896da47b 100644 --- a/Documentation/devicetree/bindings/pwm/apple,s5l-fpwm.yaml +++ b/Documentation/devicetree/bindings/pwm/apple,s5l-fpwm.yaml @@ -18,8 +18,10 @@ properties: - enum: - apple,t8103-fpwm - apple,t8112-fpwm + - apple,t8122-fpwm - apple,t6000-fpwm - apple,t6020-fpwm + - apple,t6030-fpwm - const: apple,s5l-fpwm reg: diff --git a/Documentation/devicetree/bindings/sound/apple,macaudio.yaml b/Documentation/devicetree/bindings/sound/apple,macaudio.yaml new file mode 100644 index 00000000000000..8fe22dec3015d6 --- /dev/null +++ b/Documentation/devicetree/bindings/sound/apple,macaudio.yaml @@ -0,0 +1,162 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/sound/apple,macaudio.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple Silicon Macs integrated sound peripherals + +description: + This binding represents the overall machine-level integration of sound + peripherals on 'Apple Silicon' machines by Apple. + +maintainers: + - Martin Povišer + +properties: + compatible: + items: + - enum: + - apple,j274-macaudio + - apple,j293-macaudio + - apple,j314-macaudio + - const: apple,macaudio + + "#address-cells": + const: 1 + + "#size-cells": + const: 0 + + model: + description: + Model name for presentation to users + $ref: /schemas/types.yaml#/definitions/string + +patternProperties: + "^dai-link(@[0-9a-f]+)?$": + description: | + Node for each sound peripheral such as the speaker array, headphones jack, + or microphone. + type: object + + additionalProperties: false + + properties: + reg: + maxItems: 1 + + link-name: + description: | + Name for the peripheral, expecting 'Speaker' or 'Speakers' if this is + the speaker array. + $ref: /schemas/types.yaml#/definitions/string + + cpu: + type: object + + properties: + sound-dai: + description: | + DAI list with CPU-side I2S ports involved in this peripheral. + minItems: 1 + maxItems: 2 + + required: + - sound-dai + + codec: + type: object + + properties: + sound-dai: + minItems: 1 + maxItems: 8 + description: | + DAI list with the CODEC-side DAIs connected to the above CPU-side + DAIs and involved in this sound peripheral. + + The list is in left/right order if applicable. If there are more + than one CPU-side DAIs (there can be two), the CODECs must be + listed first those connected to the first CPU, then those + connected to the second. + + In addition, on some machines with many speaker codecs, the CODECs + are listed in this fixed order: + + J293: Left Front, Left Rear, Right Front, Right Rear + J314: Left Woofer 1, Left Tweeter, Left Woofer 2, + Right Woofer 1, Right Tweeter, Right Woofer 2 + + required: + - sound-dai + + required: + - reg + - cpu + - codec + +required: + - compatible + - model + +additionalProperties: false + +examples: + - | + mca: mca@9b600000 { + compatible = "apple,t6000-mca", "apple,mca"; + reg = <0x9b600000 0x10000>, + <0x9b500000 0x20000>; + + clocks = <&nco 0>, <&nco 1>, <&nco 2>, <&nco 3>; + power-domains = <&ps_audio_p>, <&ps_mca0>, <&ps_mca1>, + <&ps_mca2>, <&ps_mca3>; + dmas = <&admac 0>, <&admac 1>, <&admac 2>, <&admac 3>, + <&admac 4>, <&admac 5>, <&admac 6>, <&admac 7>, + <&admac 8>, <&admac 9>, <&admac 10>, <&admac 11>, + <&admac 12>, <&admac 13>, <&admac 14>, <&admac 15>; + dma-names = "tx0a", "rx0a", "tx0b", "rx0b", + "tx1a", "rx1a", "tx1b", "rx1b", + "tx2a", "rx2a", "tx2b", "rx2b", + "tx3a", "rx3a", "tx3b", "rx3b"; + + #sound-dai-cells = <1>; + }; + + sound { + compatible = "apple,j314-macaudio", "apple,macaudio"; + model = "MacBook Pro J314 integrated audio"; + + #address-cells = <1>; + #size-cells = <0>; + + dai-link@0 { + reg = <0>; + link-name = "Speakers"; + + cpu { + sound-dai = <&mca 0>, <&mca 1>; + }; + codec { + sound-dai = <&speaker_left_woof1>, + <&speaker_left_tweet>, + <&speaker_left_woof2>, + <&speaker_right_woof1>, + <&speaker_right_tweet>, + <&speaker_right_woof2>; + }; + }; + + dai-link@1 { + reg = <1>; + link-name = "Headphones Jack"; + + cpu { + sound-dai = <&mca 2>; + }; + codec { + sound-dai = <&jack_codec>; + }; + }; + }; diff --git a/Documentation/devicetree/bindings/sound/asahi-kasei,ak4458.yaml b/Documentation/devicetree/bindings/sound/asahi-kasei,ak4458.yaml index 1fdbeecc5eff9d..3a3313ea0890a4 100644 --- a/Documentation/devicetree/bindings/sound/asahi-kasei,ak4458.yaml +++ b/Documentation/devicetree/bindings/sound/asahi-kasei,ak4458.yaml @@ -21,10 +21,10 @@ properties: reg: maxItems: 1 - avdd-supply: + AVDD-supply: description: Analog power supply - dvdd-supply: + DVDD-supply: description: Digital power supply reset-gpios: @@ -60,7 +60,7 @@ allOf: properties: dsd-path: false -additionalProperties: false +unevaluatedProperties: false examples: - | diff --git a/Documentation/devicetree/bindings/sound/asahi-kasei,ak5558.yaml b/Documentation/devicetree/bindings/sound/asahi-kasei,ak5558.yaml index d3d494ae8abfeb..dc8f85f266bf30 100644 --- a/Documentation/devicetree/bindings/sound/asahi-kasei,ak5558.yaml +++ b/Documentation/devicetree/bindings/sound/asahi-kasei,ak5558.yaml @@ -19,10 +19,10 @@ properties: reg: maxItems: 1 - avdd-supply: + AVDD-supply: description: A 1.8V supply that powers up the AVDD pin. - dvdd-supply: + DVDD-supply: description: A 1.2V supply that powers up the DVDD pin. reset-gpios: diff --git a/Documentation/devicetree/bindings/sound/st,stm32-sai.yaml b/Documentation/devicetree/bindings/sound/st,stm32-sai.yaml index 4a7129d0b15747..551edf39e76630 100644 --- a/Documentation/devicetree/bindings/sound/st,stm32-sai.yaml +++ b/Documentation/devicetree/bindings/sound/st,stm32-sai.yaml @@ -164,7 +164,7 @@ allOf: properties: compatible: contains: - const: st,stm32mph7-sai + const: st,stm32h7-sai then: properties: clocks: diff --git a/Documentation/devicetree/bindings/spmi/apple,spmi.yaml b/Documentation/devicetree/bindings/spmi/apple,spmi.yaml index ba524f1eb7049d..d5779ccd58461b 100644 --- a/Documentation/devicetree/bindings/spmi/apple,spmi.yaml +++ b/Documentation/devicetree/bindings/spmi/apple,spmi.yaml @@ -22,6 +22,7 @@ properties: - apple,t6020-spmi - apple,t8012-spmi - apple,t8015-spmi + - apple,t8122-spmi - const: apple,t8103-spmi - items: - enum: diff --git a/Documentation/devicetree/bindings/watchdog/apple,wdt.yaml b/Documentation/devicetree/bindings/watchdog/apple,wdt.yaml index 05602678c070d7..db0d11aa0a3e15 100644 --- a/Documentation/devicetree/bindings/watchdog/apple,wdt.yaml +++ b/Documentation/devicetree/bindings/watchdog/apple,wdt.yaml @@ -16,7 +16,10 @@ properties: compatible: oneOf: - items: - - const: apple,t6020-wdt + - enum: + - apple,t6020-wdt + - apple,t6030-wdt + - apple,t8122-wdt - const: apple,t8103-wdt - items: - enum: diff --git a/Documentation/filesystems/overlayfs.rst b/Documentation/filesystems/overlayfs.rst index ab989807a2cb69..c493fcbe0df4a4 100644 --- a/Documentation/filesystems/overlayfs.rst +++ b/Documentation/filesystems/overlayfs.rst @@ -783,6 +783,56 @@ controlled by the "uuid" mount option, which supports these values: mounted with "uuid=on". +Durability and copy up +---------------------- + +The fsync(2) system call ensures that the data and metadata of a file +are safely written to the backing storage, which is expected to +guarantee the existence of the information post system crash. + +Without an fsync(2) call, there is no guarantee that the observed +data after a system crash will be either the old or the new data, but +in practice, the observed data after crash is often the old or new data +or a mix of both. + +When an overlayfs file is modified for the first time, copy up will +create a copy of the lower file and its parent directories in the upper +layer. Since the Linux filesystem API does not enforce any particular +ordering on storing changes without explicit fsync(2) calls, in case +of a system crash, the upper file could end up with no data at all +(i.e. zeros), which would be an unusual outcome. To avoid this +experience, overlayfs calls fsync(2) on the upper file before completing +data copy up with rename(2) or link(2) to make the copy up "atomic". + +By default, overlayfs does not explicitly call fsync(2) on copied up +directories or on metadata-only copy up, so it provides no guarantee to +persist the user's modification unless the user calls fsync(2). +The fsync during copy up only guarantees that if a copy up is observed +after a crash, the observed data is not zeroes or intermediate values +from the copy up staging area. + +On traditional local filesystems with a single journal (e.g. ext4, xfs), +fsync on a file also persists the parent directory changes, because they +are usually modified in the same transaction, so metadata durability during +data copy up effectively comes for free. Overlayfs further limits risk by +disallowing network filesystems as upper layer. + +Overlayfs can be tuned to prefer performance or durability when storing +to the underlying upper layer. This is controlled by the "fsync" mount +option, which supports these values: + +- "auto": (default) + Call fsync(2) on upper file before completion of data copy up. + No explicit fsync(2) on directory or metadata-only copy up. +- "strict": + Call fsync(2) on upper file and directories before completion of any + copy up. +- "volatile": [*] + Prefer performance over durability (see `Volatile mount`_) + +[*] The mount option "volatile" is an alias to "fsync=volatile". + + Volatile mount -------------- diff --git a/Documentation/hwmon/adm1177.rst b/Documentation/hwmon/adm1177.rst index 1c85a2af92bf74..375f6d6e03a7d3 100644 --- a/Documentation/hwmon/adm1177.rst +++ b/Documentation/hwmon/adm1177.rst @@ -27,10 +27,10 @@ for details. Sysfs entries ------------- -The following attributes are supported. Current maxim attribute +The following attributes are supported. Current maximum attribute is read-write, all other attributes are read-only. -in0_input Measured voltage in microvolts. +in0_input Measured voltage in millivolts. -curr1_input Measured current in microamperes. -curr1_max_alarm Overcurrent alarm in microamperes. +curr1_input Measured current in milliamperes. +curr1_max Overcurrent shutdown threshold in milliamperes. diff --git a/Documentation/hwmon/asus_ec_sensors.rst b/Documentation/hwmon/asus_ec_sensors.rst index 232885f24430d0..b5e1bc7ac06431 100644 --- a/Documentation/hwmon/asus_ec_sensors.rst +++ b/Documentation/hwmon/asus_ec_sensors.rst @@ -10,6 +10,7 @@ Supported boards: * PRIME X670E-PRO WIFI * PRIME Z270-A * Pro WS TRX50-SAGE WIFI + * Pro WS TRX50-SAGE WIFI A * Pro WS X570-ACE * Pro WS WRX90E-SAGE SE * ProArt X570-CREATOR WIFI diff --git a/Documentation/hwmon/nct6683.rst b/Documentation/hwmon/nct6683.rst index 3e549ba95a15a6..45eec9dd349aaa 100644 --- a/Documentation/hwmon/nct6683.rst +++ b/Documentation/hwmon/nct6683.rst @@ -65,6 +65,7 @@ AMD BC-250 NCT6686D EC firmware version 1.0 build 07/28/21 ASRock X570 NCT6683D EC firmware version 1.0 build 06/28/19 ASRock X670E NCT6686D EC firmware version 1.0 build 05/19/22 ASRock B650 Steel Legend WiFi NCT6686D EC firmware version 1.0 build 11/09/23 +ASRock Z590 Taichi NCT6686D EC firmware version 1.0 build 01/25/21 MSI B550 NCT6687D EC firmware version 1.0 build 05/07/20 MSI X670-P NCT6687D EC firmware version 0.0 build 09/27/22 MSI X870E NCT6687D EC firmware version 0.0 build 11/13/24 diff --git a/Documentation/hwmon/peci-cputemp.rst b/Documentation/hwmon/peci-cputemp.rst index fe0422248dc5ea..266b62a46f49c2 100644 --- a/Documentation/hwmon/peci-cputemp.rst +++ b/Documentation/hwmon/peci-cputemp.rst @@ -51,8 +51,9 @@ temp1_max Provides thermal control temperature of the CPU package temp1_crit Provides shutdown temperature of the CPU package which is also known as the maximum processor junction temperature, Tjmax or Tprochot. -temp1_crit_hyst Provides the hysteresis value from Tcontrol to Tjmax of - the CPU package. +temp1_crit_hyst Provides the hysteresis temperature of the CPU + package. Returns Tcontrol, the temperature at which + the critical condition clears. temp2_label "DTS" temp2_input Provides current temperature of the CPU package scaled @@ -62,8 +63,9 @@ temp2_max Provides thermal control temperature of the CPU package temp2_crit Provides shutdown temperature of the CPU package which is also known as the maximum processor junction temperature, Tjmax or Tprochot. -temp2_crit_hyst Provides the hysteresis value from Tcontrol to Tjmax of - the CPU package. +temp2_crit_hyst Provides the hysteresis temperature of the CPU + package. Returns Tcontrol, the temperature at which + the critical condition clears. temp3_label "Tcontrol" temp3_input Provides current Tcontrol temperature of the CPU diff --git a/Documentation/netlink/specs/net_shaper.yaml b/Documentation/netlink/specs/net_shaper.yaml index 0b1b54be48f92c..3f2ad772b64b15 100644 --- a/Documentation/netlink/specs/net_shaper.yaml +++ b/Documentation/netlink/specs/net_shaper.yaml @@ -247,8 +247,8 @@ operations: flags: [admin-perm] do: - pre: net-shaper-nl-pre-doit - post: net-shaper-nl-post-doit + pre: net-shaper-nl-pre-doit-write + post: net-shaper-nl-post-doit-write request: attributes: - ifindex @@ -278,8 +278,8 @@ operations: flags: [admin-perm] do: - pre: net-shaper-nl-pre-doit - post: net-shaper-nl-post-doit + pre: net-shaper-nl-pre-doit-write + post: net-shaper-nl-post-doit-write request: attributes: *ns-binding @@ -309,8 +309,8 @@ operations: flags: [admin-perm] do: - pre: net-shaper-nl-pre-doit - post: net-shaper-nl-post-doit + pre: net-shaper-nl-pre-doit-write + post: net-shaper-nl-post-doit-write request: attributes: - ifindex diff --git a/Documentation/networking/ip-sysctl.rst b/Documentation/networking/ip-sysctl.rst index bc9a01606daf5a..2c65d57103fb1e 100644 --- a/Documentation/networking/ip-sysctl.rst +++ b/Documentation/networking/ip-sysctl.rst @@ -3232,12 +3232,13 @@ enhanced_dad - BOOLEAN =========== ratelimit - INTEGER - Limit the maximal rates for sending ICMPv6 messages. + Limit the maximal rates for sending ICMPv6 messages to a particular + peer. 0 to disable any limiting, - otherwise the minimal space between responses in milliseconds. + otherwise the space between responses in milliseconds. - Default: 1000 + Default: 100 ratemask - list of comma separated ranges For ICMPv6 message types matching the ranges in the ratemask, limit diff --git a/Documentation/sound/alsa-configuration.rst b/Documentation/sound/alsa-configuration.rst index 0a4eaa7d66ddd0..55b845d382368f 100644 --- a/Documentation/sound/alsa-configuration.rst +++ b/Documentation/sound/alsa-configuration.rst @@ -2372,6 +2372,10 @@ quirk_flags audible volume * bit 25: ``mixer_capture_min_mute`` Similar to bit 24 but for capture streams + * bit 26: ``skip_iface_setup`` + Skip the probe-time interface setup (usb_set_interface, + init_pitch, init_sample_rate); redundant with + snd_usb_endpoint_prepare() at stream-open time This module supports multiple devices, autoprobe and hotplugging. diff --git a/Documentation/trace/events-pci.rst b/Documentation/trace/events-pci.rst new file mode 100644 index 00000000000000..03ff4ad30ddfa1 --- /dev/null +++ b/Documentation/trace/events-pci.rst @@ -0,0 +1,74 @@ +.. SPDX-License-Identifier: GPL-2.0 + +=========================== +Subsystem Trace Points: PCI +=========================== + +Overview +======== +The PCI tracing system provides tracepoints to monitor critical hardware events +that can impact system performance and reliability. These events normally show +up here: + + /sys/kernel/tracing/events/pci + +Cf. include/trace/events/pci.h for the events definitions. + +Available Tracepoints +===================== + +pci_hp_event +------------ + +Monitors PCI hotplug events including card insertion/removal and link +state changes. +:: + + pci_hp_event "%s slot:%s, event:%s\n" + +**Event Types**: + +* ``LINK_UP`` - PCIe link established +* ``LINK_DOWN`` - PCIe link lost +* ``CARD_PRESENT`` - Card detected in slot +* ``CARD_NOT_PRESENT`` - Card removed from slot + +**Example Usage**:: + + # Enable the tracepoint + echo 1 > /sys/kernel/debug/tracing/events/pci/pci_hp_event/enable + + # Monitor events (the following output is generated when a device is hotplugged) + cat /sys/kernel/debug/tracing/trace_pipe + irq/51-pciehp-88 [001] ..... 1311.177459: pci_hp_event: 0000:00:02.0 slot:10, event:CARD_PRESENT + + irq/51-pciehp-88 [001] ..... 1311.177566: pci_hp_event: 0000:00:02.0 slot:10, event:LINK_UP + +pcie_link_event +--------------- + +Monitors PCIe link speed changes and provides detailed link status information. +:: + + pcie_link_event "%s type:%d, reason:%d, cur_bus_speed:%d, max_bus_speed:%d, width:%u, flit_mode:%u, status:%s\n" + +**Parameters**: + +* ``type`` - PCIe device type (4=Root Port, etc.) +* ``reason`` - Reason for link change: + + - ``0`` - Link retrain + - ``1`` - Bus enumeration + - ``2`` - Bandwidth notification enable + - ``3`` - Bandwidth notification IRQ + - ``4`` - Hotplug event + + +**Example Usage**:: + + # Enable the tracepoint + echo 1 > /sys/kernel/debug/tracing/events/pci/pcie_link_event/enable + + # Monitor events (the following output is generated when a device is hotplugged) + cat /sys/kernel/debug/tracing/trace_pipe + irq/51-pciehp-88 [001] ..... 381.545386: pcie_link_event: 0000:00:02.0 type:4, reason:4, cur_bus_speed:20, max_bus_speed:23, width:1, flit_mode:0, status:DLLLA diff --git a/Documentation/trace/index.rst b/Documentation/trace/index.rst index b4a429dc4f7ad6..0a40bfabcf19bf 100644 --- a/Documentation/trace/index.rst +++ b/Documentation/trace/index.rst @@ -54,6 +54,7 @@ applications. events-power events-nmi events-msr + events-pci boottime-trace histogram histogram-design diff --git a/Documentation/virt/kvm/api.rst b/Documentation/virt/kvm/api.rst index 01a3abef8abb91..b1008283e11de5 100644 --- a/Documentation/virt/kvm/api.rst +++ b/Documentation/virt/kvm/api.rst @@ -7835,8 +7835,10 @@ Will return -EBUSY if a VCPU has already been created. Valid feature flags in args[0] are:: - #define KVM_X2APIC_API_USE_32BIT_IDS (1ULL << 0) - #define KVM_X2APIC_API_DISABLE_BROADCAST_QUIRK (1ULL << 1) + #define KVM_X2APIC_API_USE_32BIT_IDS (1ULL << 0) + #define KVM_X2APIC_API_DISABLE_BROADCAST_QUIRK (1ULL << 1) + #define KVM_X2APIC_ENABLE_SUPPRESS_EOI_BROADCAST (1ULL << 2) + #define KVM_X2APIC_DISABLE_SUPPRESS_EOI_BROADCAST (1ULL << 3) Enabling KVM_X2APIC_API_USE_32BIT_IDS changes the behavior of KVM_SET_GSI_ROUTING, KVM_SIGNAL_MSI, KVM_SET_LAPIC, and KVM_GET_LAPIC, @@ -7849,6 +7851,28 @@ as a broadcast even in x2APIC mode in order to support physical x2APIC without interrupt remapping. This is undesirable in logical mode, where 0xff represents CPUs 0-7 in cluster 0. +Setting KVM_X2APIC_ENABLE_SUPPRESS_EOI_BROADCAST instructs KVM to enable +Suppress EOI Broadcasts. KVM will advertise support for Suppress EOI +Broadcast to the guest and suppress LAPIC EOI broadcasts when the guest +sets the Suppress EOI Broadcast bit in the SPIV register. This flag is +supported only when using a split IRQCHIP. + +Setting KVM_X2APIC_DISABLE_SUPPRESS_EOI_BROADCAST disables support for +Suppress EOI Broadcasts entirely, i.e. instructs KVM to NOT advertise +support to the guest. + +Modern VMMs should either enable KVM_X2APIC_ENABLE_SUPPRESS_EOI_BROADCAST +or KVM_X2APIC_DISABLE_SUPPRESS_EOI_BROADCAST. If not, legacy quirky +behavior will be used by KVM: in split IRQCHIP mode, KVM will advertise +support for Suppress EOI Broadcasts but not actually suppress EOI +broadcasts; for in-kernel IRQCHIP mode, KVM will not advertise support for +Suppress EOI Broadcasts. + +Setting both KVM_X2APIC_ENABLE_SUPPRESS_EOI_BROADCAST and +KVM_X2APIC_DISABLE_SUPPRESS_EOI_BROADCAST will fail with an EINVAL error, +as will setting KVM_X2APIC_ENABLE_SUPPRESS_EOI_BROADCAST without a split +IRCHIP. + 7.8 KVM_CAP_S390_USER_INSTR0 ---------------------------- @@ -8414,6 +8438,14 @@ KVM_X86_QUIRK_IGNORE_GUEST_PAT By default, on Intel platforms, KVM ignores guest software, for example if it does not expose a bochs graphics device (which is known to have had a buggy driver). + +KVM_X86_QUIRK_VMCS12_ALLOW_FREEZE_IN_SMM By default, KVM relaxes the consistency + check for GUEST_IA32_DEBUGCTL in vmcs12 + to allow FREEZE_IN_SMM to be set. When + this quirk is disabled, KVM requires this + bit to be cleared. Note that the vmcs02 + bit is still completely controlled by the + host, regardless of the quirk setting. =================================== ============================================ 7.32 KVM_CAP_MAX_VCPU_ID diff --git a/MAINTAINERS b/MAINTAINERS index e0876732376362..11dbc902ada896 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -1884,6 +1884,13 @@ L: linux-input@vger.kernel.org S: Odd fixes F: drivers/input/mouse/bcm5974.c +APPLE DRM DISPLAY DRIVER +M: Janne Grunau +L: dri-devel@lists.freedesktop.org +S: Maintained +T: git git://anongit.freedesktop.org/drm/drm-misc +F: drivers/gpu/drm/apple/ + APPLE PCIE CONTROLLER DRIVER M: Marc Zyngier L: linux-pci@vger.kernel.org @@ -2434,9 +2441,11 @@ M: Martin Povišer L: asahi@lists.linux.dev L: linux-sound@vger.kernel.org S: Maintained +F: Documentation/devicetree/bindings/dma/apple,sio.yaml F: Documentation/devicetree/bindings/sound/apple,* F: Documentation/devicetree/bindings/sound/cirrus,cs42l84.yaml F: Documentation/devicetree/bindings/sound/trivial-codec.yaml +F: drivers/dma/apple-sio.c F: sound/soc/apple/* F: sound/soc/codecs/cs42l83-i2c.c F: sound/soc/codecs/cs42l84.* @@ -2460,6 +2469,7 @@ F: Documentation/devicetree/bindings/cpufreq/apple,cluster-cpufreq.yaml F: Documentation/devicetree/bindings/dma/apple,admac.yaml F: Documentation/devicetree/bindings/gpio/apple,smc-gpio.yaml F: Documentation/devicetree/bindings/gpu/apple,agx.yaml +F: Documentation/devicetree/bindings/hwmon/apple,smc-hwmon.yaml F: Documentation/devicetree/bindings/i2c/apple,i2c.yaml F: Documentation/devicetree/bindings/input/touchscreen/apple,z2-multitouch.yaml F: Documentation/devicetree/bindings/interrupt-controller/apple,* @@ -2473,6 +2483,7 @@ F: Documentation/devicetree/bindings/nvme/apple,nvme-ans.yaml F: Documentation/devicetree/bindings/nvmem/apple,efuses.yaml F: Documentation/devicetree/bindings/nvmem/apple,spmi-nvmem.yaml F: Documentation/devicetree/bindings/pci/apple,pcie.yaml +F: Documentation/devicetree/bindings/phy/apple,atcphy.yaml F: Documentation/devicetree/bindings/pinctrl/apple,pinctrl.yaml F: Documentation/devicetree/bindings/power/apple* F: Documentation/devicetree/bindings/power/reset/apple,smc-reboot.yaml @@ -2493,6 +2504,7 @@ F: drivers/hwmon/macsmc-hwmon.c F: drivers/pmdomain/apple/ F: drivers/i2c/busses/i2c-pasemi-core.c F: drivers/i2c/busses/i2c-pasemi-platform.c +F: drivers/input/misc/macsmc-input.c F: drivers/input/touchscreen/apple_z2.c F: drivers/iommu/apple-dart.c F: drivers/iommu/io-pgtable-dart.c @@ -2501,6 +2513,7 @@ F: drivers/mfd/macsmc.c F: drivers/nvme/host/apple.c F: drivers/nvmem/apple-efuses.c F: drivers/nvmem/apple-spmi-nvmem.c +F: drivers/phy/apple/ F: drivers/pinctrl/pinctrl-apple-gpio.c F: drivers/power/reset/macsmc-reboot.c F: drivers/pwm/pwm-apple.c @@ -2516,6 +2529,8 @@ F: include/dt-bindings/pinctrl/apple.h F: include/linux/mfd/macsmc.h F: include/linux/soc/apple/* F: include/uapi/drm/asahi_drm.h +F: net/bluetooth/brcm.c +F: net/bluetooth/brcm.h ARM/ARTPEC MACHINE SUPPORT M: Jesper Nilsson diff --git a/Makefile b/Makefile index d3a8482bdbd0de..cb2ba18730acde 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ # SPDX-License-Identifier: GPL-2.0 VERSION = 6 PATCHLEVEL = 19 -SUBLEVEL = 0 +SUBLEVEL = 14 EXTRAVERSION = NAME = Baby Opossum Posse @@ -473,6 +473,7 @@ KBUILD_USERLDFLAGS := $(USERLDFLAGS) export rust_common_flags := --edition=2021 \ -Zbinary_dep_depinfo=y \ -Astable_features \ + -Aunused_features \ -Dnon_ascii_idents \ -Dunsafe_op_in_unsafe_fn \ -Wmissing_docs \ @@ -1474,6 +1475,15 @@ ifneq ($(wildcard $(resolve_btfids_O)),) $(Q)$(MAKE) -sC $(srctree)/tools/bpf/resolve_btfids O=$(resolve_btfids_O) clean endif +PHONY += objtool_clean objtool_mrproper + +objtool_O = $(abspath $(objtree))/tools/objtool + +objtool_clean objtool_mrproper: +ifneq ($(wildcard $(objtool_O)),) + $(Q)$(MAKE) -sC $(abs_srctree)/tools/objtool O=$(objtool_O) srctree=$(abs_srctree) $(patsubst objtool_%,%,$@) +endif + tools/: FORCE $(Q)mkdir -p $(objtree)/tools $(Q)$(MAKE) O=$(abspath $(objtree)) subdir=tools -C $(srctree)/tools/ @@ -1612,7 +1622,7 @@ CLEAN_FILES += vmlinux.symvers modules-only.symvers \ modules.builtin.ranges vmlinux.o.map vmlinux.unstripped \ compile_commands.json rust/test \ rust-project.json .vmlinux.objs .vmlinux.export.c \ - .builtin-dtbs-list .builtin-dtb.S + .builtin-dtbs-list .builtin-dtbs.S # Directories & files removed with 'make mrproper' MRPROPER_FILES += include/config include/generated \ @@ -1637,7 +1647,7 @@ vmlinuxclean: $(Q)$(CONFIG_SHELL) $(srctree)/scripts/link-vmlinux.sh clean $(Q)$(if $(ARCH_POSTLINK), $(MAKE) -f $(ARCH_POSTLINK) clean) -clean: archclean vmlinuxclean resolve_btfids_clean +clean: archclean vmlinuxclean resolve_btfids_clean objtool_clean # mrproper - Delete all generated files, including .config # @@ -1648,7 +1658,7 @@ PHONY += $(mrproper-dirs) mrproper $(mrproper-dirs): $(Q)$(MAKE) $(clean)=$(patsubst _mrproper_%,%,$@) -mrproper: clean $(mrproper-dirs) +mrproper: clean objtool_mrproper $(mrproper-dirs) $(call cmd,rmfiles) @find . $(RCS_FIND_IGNORE) \ \( -name '*.rmeta' \) \ diff --git a/arch/alpha/include/asm/pgtable.h b/arch/alpha/include/asm/pgtable.h index 90e7a953910228..c9508ec37efc43 100644 --- a/arch/alpha/include/asm/pgtable.h +++ b/arch/alpha/include/asm/pgtable.h @@ -17,6 +17,7 @@ #include /* For TASK_SIZE */ #include #include +#include struct mm_struct; struct vm_area_struct; @@ -183,6 +184,9 @@ extern inline void pud_set(pud_t * pudp, pmd_t * pmdp) { pud_val(*pudp) = _PAGE_TABLE | ((((unsigned long) pmdp) - PAGE_OFFSET) << (32-PAGE_SHIFT)); } +extern void migrate_flush_tlb_page(struct vm_area_struct *vma, + unsigned long addr); + extern inline unsigned long pmd_page_vaddr(pmd_t pmd) { @@ -202,7 +206,7 @@ extern inline int pte_none(pte_t pte) { return !pte_val(pte); } extern inline int pte_present(pte_t pte) { return pte_val(pte) & _PAGE_VALID; } extern inline void pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep) { - pte_val(*ptep) = 0; + WRITE_ONCE(pte_val(*ptep), 0); } extern inline int pmd_none(pmd_t pmd) { return !pmd_val(pmd); } @@ -264,6 +268,33 @@ extern inline pte_t * pte_offset_kernel(pmd_t * dir, unsigned long address) extern pgd_t swapper_pg_dir[1024]; +#ifdef CONFIG_COMPACTION +#define __HAVE_ARCH_PTEP_GET_AND_CLEAR + +static inline pte_t ptep_get_and_clear(struct mm_struct *mm, + unsigned long address, + pte_t *ptep) +{ + pte_t pte = READ_ONCE(*ptep); + + pte_clear(mm, address, ptep); + return pte; +} + +#define __HAVE_ARCH_PTEP_CLEAR_FLUSH + +static inline pte_t ptep_clear_flush(struct vm_area_struct *vma, + unsigned long addr, pte_t *ptep) +{ + struct mm_struct *mm = vma->vm_mm; + pte_t pte = ptep_get_and_clear(mm, addr, ptep); + + page_table_check_pte_clear(mm, pte); + migrate_flush_tlb_page(vma, addr); + return pte; +} + +#endif /* * The Alpha doesn't have any external MMU info: the kernel page * tables contain all the necessary information. diff --git a/arch/alpha/include/asm/tlbflush.h b/arch/alpha/include/asm/tlbflush.h index ba4b359d6c395d..0c8529997f54e7 100644 --- a/arch/alpha/include/asm/tlbflush.h +++ b/arch/alpha/include/asm/tlbflush.h @@ -58,7 +58,9 @@ flush_tlb_other(struct mm_struct *mm) unsigned long *mmc = &mm->context[smp_processor_id()]; /* Check it's not zero first to avoid cacheline ping pong when possible. */ - if (*mmc) *mmc = 0; + + if (READ_ONCE(*mmc)) + WRITE_ONCE(*mmc, 0); } #ifndef CONFIG_SMP diff --git a/arch/alpha/kernel/vmlinux.lds.S b/arch/alpha/kernel/vmlinux.lds.S index 2efa7dfc798a95..2d136c63db161a 100644 --- a/arch/alpha/kernel/vmlinux.lds.S +++ b/arch/alpha/kernel/vmlinux.lds.S @@ -71,6 +71,7 @@ SECTIONS STABS_DEBUG DWARF_DEBUG + MODINFO ELF_DETAILS DISCARDS diff --git a/arch/alpha/mm/Makefile b/arch/alpha/mm/Makefile index 101dbd06b4ceb6..2d05664058f64e 100644 --- a/arch/alpha/mm/Makefile +++ b/arch/alpha/mm/Makefile @@ -3,4 +3,4 @@ # Makefile for the linux alpha-specific parts of the memory manager. # -obj-y := init.o fault.o +obj-y := init.o fault.o tlbflush.o diff --git a/arch/alpha/mm/tlbflush.c b/arch/alpha/mm/tlbflush.c new file mode 100644 index 00000000000000..ccbc317b9a3483 --- /dev/null +++ b/arch/alpha/mm/tlbflush.c @@ -0,0 +1,112 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Alpha TLB shootdown helpers + * + * Copyright (C) 2025 Magnus Lindholm + * + * Alpha-specific TLB flush helpers that cannot be expressed purely + * as inline functions. + * + * These helpers provide combined MM context handling (ASN rollover) + * and immediate TLB invalidation for page migration and memory + * compaction paths, where lazy shootdowns are insufficient. + */ + +#include +#include +#include +#include +#include +#include + +#define asn_locked() (cpu_data[smp_processor_id()].asn_lock) + +/* + * Migration/compaction helper: combine mm context (ASN) handling with an + * immediate per-page TLB invalidate and (for exec) an instruction barrier. + * + * This mirrors the SMP combined IPI handler semantics, but runs locally on UP. + */ +#ifndef CONFIG_SMP +void migrate_flush_tlb_page(struct vm_area_struct *vma, + unsigned long addr) +{ + struct mm_struct *mm = vma->vm_mm; + int tbi_type = (vma->vm_flags & VM_EXEC) ? 3 : 2; + + /* + * First do the mm-context side: + * If we're currently running this mm, reload a fresh context ASN. + * Otherwise, mark context invalid. + * + * On UP, this is mostly about matching the SMP semantics and ensuring + * exec/i-cache tagging assumptions hold when compaction migrates pages. + */ + if (mm == current->active_mm) + flush_tlb_current(mm); + else + flush_tlb_other(mm); + + /* + * Then do the immediate translation kill for this VA. + * For exec mappings, order instruction fetch after invalidation. + */ + tbi(tbi_type, addr); +} + +#else +struct tlb_mm_and_addr { + struct mm_struct *mm; + unsigned long addr; + int tbi_type; /* 2 = DTB, 3 = ITB+DTB */ +}; + +static void ipi_flush_mm_and_page(void *x) +{ + struct tlb_mm_and_addr *d = x; + + /* Part 1: mm context side (Alpha uses ASN/context as a key mechanism). */ + if (d->mm == current->active_mm && !asn_locked()) + __load_new_mm_context(d->mm); + else + flush_tlb_other(d->mm); + + /* Part 2: immediate per-VA invalidation on this CPU. */ + tbi(d->tbi_type, d->addr); +} + +void migrate_flush_tlb_page(struct vm_area_struct *vma, unsigned long addr) +{ + struct mm_struct *mm = vma->vm_mm; + struct tlb_mm_and_addr d = { + .mm = mm, + .addr = addr, + .tbi_type = (vma->vm_flags & VM_EXEC) ? 3 : 2, + }; + + /* + * One synchronous rendezvous: every CPU runs ipi_flush_mm_and_page(). + * This is the "combined" version of flush_tlb_mm + per-page invalidate. + */ + preempt_disable(); + on_each_cpu(ipi_flush_mm_and_page, &d, 1); + + /* + * mimic flush_tlb_mm()'s mm_users<=1 optimization. + */ + if (atomic_read(&mm->mm_users) <= 1) { + + int cpu, this_cpu; + this_cpu = smp_processor_id(); + + for (cpu = 0; cpu < NR_CPUS; cpu++) { + if (!cpu_online(cpu) || cpu == this_cpu) + continue; + if (READ_ONCE(mm->context[cpu])) + WRITE_ONCE(mm->context[cpu], 0); + } + } + preempt_enable(); +} + +#endif diff --git a/arch/arc/kernel/vmlinux.lds.S b/arch/arc/kernel/vmlinux.lds.S index 61a1b2b96e1d81..6af63084ff2854 100644 --- a/arch/arc/kernel/vmlinux.lds.S +++ b/arch/arc/kernel/vmlinux.lds.S @@ -123,6 +123,7 @@ SECTIONS _end = . ; STABS_DEBUG + MODINFO ELF_DETAILS DISCARDS diff --git a/arch/arm/boot/compressed/vmlinux.lds.S b/arch/arm/boot/compressed/vmlinux.lds.S index d411abd4310ea9..2d916647df03ca 100644 --- a/arch/arm/boot/compressed/vmlinux.lds.S +++ b/arch/arm/boot/compressed/vmlinux.lds.S @@ -21,6 +21,7 @@ SECTIONS COMMON_DISCARDS *(.ARM.exidx*) *(.ARM.extab*) + *(.modinfo) *(.note.*) *(.rel.*) *(.printk_index) diff --git a/arch/arm/boot/dts/allwinner/sun5i-a13-utoo-p66.dts b/arch/arm/boot/dts/allwinner/sun5i-a13-utoo-p66.dts index be486d28d04fae..428cab5a0e906e 100644 --- a/arch/arm/boot/dts/allwinner/sun5i-a13-utoo-p66.dts +++ b/arch/arm/boot/dts/allwinner/sun5i-a13-utoo-p66.dts @@ -102,6 +102,7 @@ /* The P66 uses a different EINT then the reference design */ interrupts = <6 9 IRQ_TYPE_EDGE_FALLING>; /* EINT9 (PG9) */ /* The icn8318 binding expects wake-gpios instead of power-gpios */ + /delete-property/ power-gpios; wake-gpios = <&pio 1 3 GPIO_ACTIVE_HIGH>; /* PB3 */ touchscreen-size-x = <800>; touchscreen-size-y = <480>; diff --git a/arch/arm/boot/dts/microchip/sam9x7.dtsi b/arch/arm/boot/dts/microchip/sam9x7.dtsi index 46dacbbd201ddb..d242d7a934d0fa 100644 --- a/arch/arm/boot/dts/microchip/sam9x7.dtsi +++ b/arch/arm/boot/dts/microchip/sam9x7.dtsi @@ -1226,7 +1226,7 @@ interrupt-controller; #gpio-cells = <2>; gpio-controller; - #gpio-lines = <26>; + #gpio-lines = <27>; clocks = <&pmc PMC_TYPE_PERIPHERAL 3>; }; diff --git a/arch/arm/boot/dts/nxp/lpc/lpc32xx.dtsi b/arch/arm/boot/dts/nxp/lpc/lpc32xx.dtsi index 2236901a003130..8e9ed93da129e8 100644 --- a/arch/arm/boot/dts/nxp/lpc/lpc32xx.dtsi +++ b/arch/arm/boot/dts/nxp/lpc/lpc32xx.dtsi @@ -302,6 +302,7 @@ mpwm: pwm@400e8000 { compatible = "nxp,lpc3220-motor-pwm"; reg = <0x400e8000 0x78>; + clocks = <&clk LPC32XX_CLK_MCPWM>; #pwm-cells = <3>; status = "disabled"; }; diff --git a/arch/arm/include/asm/string.h b/arch/arm/include/asm/string.h index c35250c4991bc7..96fc6cf460ecbe 100644 --- a/arch/arm/include/asm/string.h +++ b/arch/arm/include/asm/string.h @@ -39,13 +39,17 @@ static inline void *memset32(uint32_t *p, uint32_t v, __kernel_size_t n) } #define __HAVE_ARCH_MEMSET64 -extern void *__memset64(uint64_t *, uint32_t low, __kernel_size_t, uint32_t hi); +extern void *__memset64(uint64_t *, uint32_t first, __kernel_size_t, uint32_t second); static inline void *memset64(uint64_t *p, uint64_t v, __kernel_size_t n) { - if (IS_ENABLED(CONFIG_CPU_LITTLE_ENDIAN)) - return __memset64(p, v, n * 8, v >> 32); - else - return __memset64(p, v >> 32, n * 8, v); + union { + uint64_t val; + struct { + uint32_t first, second; + }; + } word = { .val = v }; + + return __memset64(p, word.first, n * 8, word.second); } /* diff --git a/arch/arm/kernel/vdso.c b/arch/arm/kernel/vdso.c index e38a30477f3d70..566c40f0f7c77b 100644 --- a/arch/arm/kernel/vdso.c +++ b/arch/arm/kernel/vdso.c @@ -161,6 +161,7 @@ static void __init patch_vdso(void *ehdr) vdso_nullpatch_one(&einfo, "__vdso_gettimeofday"); vdso_nullpatch_one(&einfo, "__vdso_clock_gettime"); vdso_nullpatch_one(&einfo, "__vdso_clock_gettime64"); + vdso_nullpatch_one(&einfo, "__vdso_clock_getres"); } } diff --git a/arch/arm/kernel/vmlinux-xip.lds.S b/arch/arm/kernel/vmlinux-xip.lds.S index f2e8d4fac06871..5afb725998ec0a 100644 --- a/arch/arm/kernel/vmlinux-xip.lds.S +++ b/arch/arm/kernel/vmlinux-xip.lds.S @@ -154,6 +154,7 @@ SECTIONS STABS_DEBUG DWARF_DEBUG + MODINFO ARM_DETAILS ARM_ASSERTS diff --git a/arch/arm/kernel/vmlinux.lds.S b/arch/arm/kernel/vmlinux.lds.S index d592a203f9c6b2..c07843c3c53d3b 100644 --- a/arch/arm/kernel/vmlinux.lds.S +++ b/arch/arm/kernel/vmlinux.lds.S @@ -153,6 +153,7 @@ SECTIONS STABS_DEBUG DWARF_DEBUG + MODINFO ARM_DETAILS ARM_ASSERTS diff --git a/arch/arm/mach-omap2/control.c b/arch/arm/mach-omap2/control.c index 79860b23030de3..eb6fc7c61b6e08 100644 --- a/arch/arm/mach-omap2/control.c +++ b/arch/arm/mach-omap2/control.c @@ -732,7 +732,7 @@ int __init omap2_control_base_init(void) */ int __init omap_control_init(void) { - struct device_node *np, *scm_conf; + struct device_node *np, *scm_conf, *clocks_node; const struct of_device_id *match; const struct omap_prcm_init_data *data; int ret; @@ -753,16 +753,19 @@ int __init omap_control_init(void) if (IS_ERR(syscon)) { ret = PTR_ERR(syscon); - goto of_node_put; + goto err_put_scm_conf; } - if (of_get_child_by_name(scm_conf, "clocks")) { + clocks_node = of_get_child_by_name(scm_conf, "clocks"); + if (clocks_node) { + of_node_put(clocks_node); ret = omap2_clk_provider_init(scm_conf, data->index, syscon, NULL); if (ret) - goto of_node_put; + goto err_put_scm_conf; } + of_node_put(scm_conf); } else { /* No scm_conf found, direct access */ ret = omap2_clk_provider_init(np, data->index, NULL, @@ -780,6 +783,9 @@ int __init omap_control_init(void) return 0; +err_put_scm_conf: + if (scm_conf) + of_node_put(scm_conf); of_node_put: of_node_put(np); return ret; diff --git a/arch/arm/mm/physaddr.c b/arch/arm/mm/physaddr.c index 3f263c840ebc46..1a37ebfacbba96 100644 --- a/arch/arm/mm/physaddr.c +++ b/arch/arm/mm/physaddr.c @@ -38,7 +38,7 @@ static inline bool __virt_addr_valid(unsigned long x) phys_addr_t __virt_to_phys(unsigned long x) { WARN(!__virt_addr_valid(x), - "virt_to_phys used for non-linear address: %pK (%pS)\n", + "virt_to_phys used for non-linear address: %px (%pS)\n", (void *)x, (void *)x); return __virt_to_phys_nodebug(x); diff --git a/arch/arm64/Kbuild b/arch/arm64/Kbuild index 5bfbf7d79c99be..d876bc0e542110 100644 --- a/arch/arm64/Kbuild +++ b/arch/arm64/Kbuild @@ -1,4 +1,8 @@ # SPDX-License-Identifier: GPL-2.0-only + +# Branch profiling isn't noinstr-safe +subdir-ccflags-$(CONFIG_TRACE_BRANCH_PROFILING) += -DDISABLE_BRANCH_PROFILING + obj-y += kernel/ mm/ net/ obj-$(CONFIG_KVM) += kvm/ obj-$(CONFIG_XEN) += xen/ diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 93173f0a09c7de..5fede9c28b4897 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -444,6 +444,9 @@ config KASAN_SHADOW_OFFSET config UNWIND_TABLES bool +config ARM64_ACTLR_STATE + bool + source "arch/arm64/Kconfig.platforms" menu "Kernel Features" @@ -2294,6 +2297,16 @@ config ARM64_DEBUG_PRIORITY_MASKING If unsure, say N endif # ARM64_PSEUDO_NMI +config ARM64_MEMORY_MODEL_CONTROL + bool "Runtime memory model control" + select ARM64_ACTLR_STATE + help + Some ARM64 CPUs support runtime switching of the CPU memory + model, which can be useful to emulate other CPU architectures + which have different memory models. Say Y to enable support + for the PR_SET_MEM_MODEL/PR_GET_MEM_MODEL prctl() calls on + CPUs with this feature. + config RELOCATABLE bool "Build a relocatable kernel image" if EXPERT select ARCH_HAS_RELR diff --git a/arch/arm64/boot/dts/amlogic/amlogic-c3.dtsi b/arch/arm64/boot/dts/amlogic/amlogic-c3.dtsi index 13b7ac03f9b201..885d9d64e9e4ec 100644 --- a/arch/arm64/boot/dts/amlogic/amlogic-c3.dtsi +++ b/arch/arm64/boot/dts/amlogic/amlogic-c3.dtsi @@ -969,6 +969,10 @@ no-sd; resets = <&reset RESET_SD_EMMC_A>; status = "disabled"; + + assigned-clocks = <&clkc_periphs CLKID_SD_EMMC_A>; + assigned-clock-rates = <24000000>; + }; sd: mmc@8a000 { @@ -984,6 +988,9 @@ no-sdio; resets = <&reset RESET_SD_EMMC_B>; status = "disabled"; + + assigned-clocks = <&clkc_periphs CLKID_SD_EMMC_B>; + assigned-clock-rates = <24000000>; }; nand: nand-controller@8d000 { diff --git a/arch/arm64/boot/dts/amlogic/meson-axg.dtsi b/arch/arm64/boot/dts/amlogic/meson-axg.dtsi index e95c91894968b2..cc72491eaf6f52 100644 --- a/arch/arm64/boot/dts/amlogic/meson-axg.dtsi +++ b/arch/arm64/boot/dts/amlogic/meson-axg.dtsi @@ -1960,6 +1960,9 @@ <&clkc CLKID_FCLK_DIV2>; clock-names = "core", "clkin0", "clkin1"; resets = <&reset RESET_SD_EMMC_B>; + + assigned-clocks = <&clkc CLKID_SD_EMMC_B_CLK0>; + assigned-clock-rates = <24000000>; }; sd_emmc_c: mmc@7000 { @@ -1972,6 +1975,9 @@ <&clkc CLKID_FCLK_DIV2>; clock-names = "core", "clkin0", "clkin1"; resets = <&reset RESET_SD_EMMC_C>; + + assigned-clocks = <&clkc CLKID_SD_EMMC_C_CLK0>; + assigned-clock-rates = <24000000>; }; nfc: nand-controller@7800 { diff --git a/arch/arm64/boot/dts/amlogic/meson-g12-common.dtsi b/arch/arm64/boot/dts/amlogic/meson-g12-common.dtsi index ca455f634834b5..00609d2da67437 100644 --- a/arch/arm64/boot/dts/amlogic/meson-g12-common.dtsi +++ b/arch/arm64/boot/dts/amlogic/meson-g12-common.dtsi @@ -2431,6 +2431,9 @@ <&clkc CLKID_FCLK_DIV2>; clock-names = "core", "clkin0", "clkin1"; resets = <&reset RESET_SD_EMMC_A>; + + assigned-clocks = <&clkc CLKID_SD_EMMC_A_CLK0>; + assigned-clock-rates = <24000000>; }; sd_emmc_b: mmc@ffe05000 { @@ -2443,6 +2446,9 @@ <&clkc CLKID_FCLK_DIV2>; clock-names = "core", "clkin0", "clkin1"; resets = <&reset RESET_SD_EMMC_B>; + + assigned-clocks = <&clkc CLKID_SD_EMMC_B_CLK0>; + assigned-clock-rates = <24000000>; }; sd_emmc_c: mmc@ffe07000 { @@ -2455,6 +2461,9 @@ <&clkc CLKID_FCLK_DIV2>; clock-names = "core", "clkin0", "clkin1"; resets = <&reset RESET_SD_EMMC_C>; + + assigned-clocks = <&clkc CLKID_SD_EMMC_C_CLK0>; + assigned-clock-rates = <24000000>; }; usb: usb@ffe09000 { diff --git a/arch/arm64/boot/dts/amlogic/meson-gxbb.dtsi b/arch/arm64/boot/dts/amlogic/meson-gxbb.dtsi index f69923da07febd..a9c830a570cc6c 100644 --- a/arch/arm64/boot/dts/amlogic/meson-gxbb.dtsi +++ b/arch/arm64/boot/dts/amlogic/meson-gxbb.dtsi @@ -824,6 +824,9 @@ <&clkc CLKID_FCLK_DIV2>; clock-names = "core", "clkin0", "clkin1"; resets = <&reset RESET_SD_EMMC_A>; + + assigned-clocks = <&clkc CLKID_SD_EMMC_A_CLK0>; + assigned-clock-rates = <24000000>; }; &sd_emmc_b { @@ -832,6 +835,9 @@ <&clkc CLKID_FCLK_DIV2>; clock-names = "core", "clkin0", "clkin1"; resets = <&reset RESET_SD_EMMC_B>; + + assigned-clocks = <&clkc CLKID_SD_EMMC_B_CLK0>; + assigned-clock-rates = <24000000>; }; &sd_emmc_c { @@ -840,6 +846,9 @@ <&clkc CLKID_FCLK_DIV2>; clock-names = "core", "clkin0", "clkin1"; resets = <&reset RESET_SD_EMMC_C>; + + assigned-clocks = <&clkc CLKID_SD_EMMC_C_CLK0>; + assigned-clock-rates = <24000000>; }; &simplefb_hdmi { diff --git a/arch/arm64/boot/dts/amlogic/meson-gxl.dtsi b/arch/arm64/boot/dts/amlogic/meson-gxl.dtsi index ba535010a3c91d..e202d84f067205 100644 --- a/arch/arm64/boot/dts/amlogic/meson-gxl.dtsi +++ b/arch/arm64/boot/dts/amlogic/meson-gxl.dtsi @@ -894,6 +894,9 @@ <&clkc CLKID_FCLK_DIV2>; clock-names = "core", "clkin0", "clkin1"; resets = <&reset RESET_SD_EMMC_A>; + + assigned-clocks = <&clkc CLKID_SD_EMMC_A_CLK0>; + assigned-clock-rates = <24000000>; }; &sd_emmc_b { @@ -902,6 +905,9 @@ <&clkc CLKID_FCLK_DIV2>; clock-names = "core", "clkin0", "clkin1"; resets = <&reset RESET_SD_EMMC_B>; + + assigned-clocks = <&clkc CLKID_SD_EMMC_B_CLK0>; + assigned-clock-rates = <24000000>; }; &sd_emmc_c { @@ -910,6 +916,9 @@ <&clkc CLKID_FCLK_DIV2>; clock-names = "core", "clkin0", "clkin1"; resets = <&reset RESET_SD_EMMC_C>; + + assigned-clocks = <&clkc CLKID_SD_EMMC_C_CLK0>; + assigned-clock-rates = <24000000>; }; &simplefb_hdmi { diff --git a/arch/arm64/boot/dts/amlogic/meson-s4.dtsi b/arch/arm64/boot/dts/amlogic/meson-s4.dtsi index 9d99ed2994dfa2..dfc0a30a6e61be 100644 --- a/arch/arm64/boot/dts/amlogic/meson-s4.dtsi +++ b/arch/arm64/boot/dts/amlogic/meson-s4.dtsi @@ -819,13 +819,16 @@ reg = <0x0 0xfe088000 0x0 0x800>; interrupts = ; clocks = <&clkc_periphs CLKID_SDEMMC_A>, - <&xtal>, + <&clkc_periphs CLKID_SD_EMMC_A>, <&clkc_pll CLKID_FCLK_DIV2>; clock-names = "core", "clkin0", "clkin1"; resets = <&reset RESET_SD_EMMC_A>; cap-sdio-irq; keep-power-in-suspend; status = "disabled"; + + assigned-clocks = <&clkc_periphs CLKID_SD_EMMC_A>; + assigned-clock-rates = <24000000>; }; sd: mmc@fe08a000 { @@ -838,6 +841,9 @@ clock-names = "core", "clkin0", "clkin1"; resets = <&reset RESET_SD_EMMC_B>; status = "disabled"; + + assigned-clocks = <&clkc_periphs CLKID_SD_EMMC_B>; + assigned-clock-rates = <24000000>; }; emmc: mmc@fe08c000 { @@ -845,13 +851,16 @@ reg = <0x0 0xfe08c000 0x0 0x800>; interrupts = ; clocks = <&clkc_periphs CLKID_NAND>, - <&xtal>, + <&clkc_periphs CLKID_SD_EMMC_C>, <&clkc_pll CLKID_FCLK_DIV2>; clock-names = "core", "clkin0", "clkin1"; resets = <&reset RESET_NAND_EMMC>; no-sdio; no-sd; status = "disabled"; + + assigned-clocks = <&clkc_periphs CLKID_SD_EMMC_C>; + assigned-clock-rates = <24000000>; }; }; }; diff --git a/arch/arm64/boot/dts/amlogic/meson-sm1-odroid-hc4.dts b/arch/arm64/boot/dts/amlogic/meson-sm1-odroid-hc4.dts index 0170139b8d32f4..3ece30a0a1fff7 100644 --- a/arch/arm64/boot/dts/amlogic/meson-sm1-odroid-hc4.dts +++ b/arch/arm64/boot/dts/amlogic/meson-sm1-odroid-hc4.dts @@ -52,6 +52,7 @@ gpio = <&gpio GPIOH_8 GPIO_OPEN_DRAIN>; enable-active-high; + regulator-boot-on; regulator-always-on; }; @@ -65,6 +66,7 @@ gpio = <&gpio GPIOH_8 GPIO_OPEN_DRAIN>; enable-active-high; + regulator-boot-on; regulator-always-on; }; diff --git a/arch/arm64/boot/dts/amlogic/meson-sm1-odroid.dtsi b/arch/arm64/boot/dts/amlogic/meson-sm1-odroid.dtsi index c4524eb4f0996d..0bce4e8d965f2c 100644 --- a/arch/arm64/boot/dts/amlogic/meson-sm1-odroid.dtsi +++ b/arch/arm64/boot/dts/amlogic/meson-sm1-odroid.dtsi @@ -37,6 +37,7 @@ gpio = <&gpio_ao GPIOAO_3 GPIO_OPEN_DRAIN>; enable-active-high; + regulator-boot-on; regulator-always-on; }; @@ -50,6 +51,7 @@ enable-gpios = <&gpio_ao GPIOE_2 GPIO_OPEN_DRAIN>; enable-active-high; + regulator-boot-on; regulator-always-on; gpios = <&gpio_ao GPIOAO_6 GPIO_OPEN_SOURCE>; @@ -81,6 +83,7 @@ regulator-name = "5V"; regulator-min-microvolt = <5000000>; regulator-max-microvolt = <5000000>; + regulator-boot-on; regulator-always-on; vin-supply = <&main_12v>; gpio = <&gpio GPIOH_8 GPIO_OPEN_DRAIN>; diff --git a/arch/arm64/boot/dts/apple/Makefile b/arch/arm64/boot/dts/apple/Makefile index 4eebcd85c90fcf..6fc3349a58428f 100644 --- a/arch/arm64/boot/dts/apple/Makefile +++ b/arch/arm64/boot/dts/apple/Makefile @@ -91,3 +91,8 @@ dtb-$(CONFIG_ARCH_APPLE) += t8112-j413.dtb dtb-$(CONFIG_ARCH_APPLE) += t8112-j415.dtb dtb-$(CONFIG_ARCH_APPLE) += t8112-j473.dtb dtb-$(CONFIG_ARCH_APPLE) += t8112-j493.dtb +dtb-$(CONFIG_ARCH_APPLE) += t8122-j433.dtb +dtb-$(CONFIG_ARCH_APPLE) += t8122-j434.dtb +dtb-$(CONFIG_ARCH_APPLE) += t8122-j504.dtb +dtb-$(CONFIG_ARCH_APPLE) += t8122-j613.dtb +dtb-$(CONFIG_ARCH_APPLE) += t8122-j615.dtb diff --git a/arch/arm64/boot/dts/apple/hwmon-common.dtsi b/arch/arm64/boot/dts/apple/hwmon-common.dtsi new file mode 100644 index 00000000000000..2a74d9a114abb6 --- /dev/null +++ b/arch/arm64/boot/dts/apple/hwmon-common.dtsi @@ -0,0 +1,33 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * hwmon sensors expected on all systems + * + * Copyright The Asahi Linux Contributors + */ + +&smc_hwmon { + power-PSTR { + apple,key-id = "PSTR"; + label = "Total System Power"; + }; + power-PDTR { + apple,key-id = "PDTR"; + label = "AC Input Power"; + }; + power-PMVR { + apple,key-id = "PMVR"; + label = "3.8 V Rail Power"; + }; + temperature-TH0x { + apple,key-id = "TH0x"; + label = "NAND Flash Temperature"; + }; + voltage-VD0R { + apple,key-id = "VD0R"; + label = "AC Input Voltage"; + }; + current-ID0R { + apple,key-id = "ID0R"; + label = "AC Input Current"; + }; +}; diff --git a/arch/arm64/boot/dts/apple/hwmon-fan-dual.dtsi b/arch/arm64/boot/dts/apple/hwmon-fan-dual.dtsi new file mode 100644 index 00000000000000..61c34692f1cd5a --- /dev/null +++ b/arch/arm64/boot/dts/apple/hwmon-fan-dual.dtsi @@ -0,0 +1,22 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * Copyright The Asahi Linux Contributors + * + * Fan hwmon sensors for machines with 2 fan. + */ + +#include "hwmon-fan.dtsi" + +&smc_hwmon { + fan-F0Ac { + label = "Fan 1"; + }; + fan-F1Ac { + apple,key-id = "F1Ac"; + label = "Fan 2"; + apple,fan-minimum = "F1Mn"; + apple,fan-maximum = "F1Mx"; + apple,fan-target = "F1Tg"; + apple,fan-mode = "F1Md"; + }; +}; diff --git a/arch/arm64/boot/dts/apple/hwmon-fan.dtsi b/arch/arm64/boot/dts/apple/hwmon-fan.dtsi new file mode 100644 index 00000000000000..180eb8d7441f44 --- /dev/null +++ b/arch/arm64/boot/dts/apple/hwmon-fan.dtsi @@ -0,0 +1,17 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * Copyright The Asahi Linux Contributors + * + * Fan hwmon sensors for machines with a single fan. + */ + +&smc_hwmon { + fan-F0Ac { + apple,key-id = "F0Ac"; + label = "Fan"; + apple,fan-minimum = "F0Mn"; + apple,fan-maximum = "F0Mx"; + apple,fan-target = "F0Tg"; + apple,fan-mode = "F0Md"; + }; +}; diff --git a/arch/arm64/boot/dts/apple/hwmon-laptop.dtsi b/arch/arm64/boot/dts/apple/hwmon-laptop.dtsi new file mode 100644 index 00000000000000..4afb91ee69fe76 --- /dev/null +++ b/arch/arm64/boot/dts/apple/hwmon-laptop.dtsi @@ -0,0 +1,33 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * hwmon sensors expected on all laptops + * + * Copyright The Asahi Linux Contributors + */ + +&smc_hwmon { + power-PHPC { + apple,key-id = "PHPC"; + label = "Heatpipe Power"; + }; + temperature-TB0T { + apple,key-id = "TB0T"; + label = "Battery Hotspot"; + }; + temperature-TCHP { + apple,key-id = "TCHP"; + label = "Charge Regulator Temp"; + }; + temperature-TW0P { + apple,key-id = "TW0P"; + label = "WiFi/BT Module Temp"; + }; + voltage-SBAV { + apple,key-id = "SBAV"; + label = "Battery Voltage"; + }; + voltage-VD0R { + apple,key-id = "VD0R"; + label = "Charger Input Voltage"; + }; +}; diff --git a/arch/arm64/boot/dts/apple/hwmon-mini.dtsi b/arch/arm64/boot/dts/apple/hwmon-mini.dtsi new file mode 100644 index 00000000000000..7fd86e911acfe7 --- /dev/null +++ b/arch/arm64/boot/dts/apple/hwmon-mini.dtsi @@ -0,0 +1,16 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * hwmon sensors common to the Mac mini desktop + * models, but not the Studio or Pro. + * + * Copyright The Asahi Linux Contributors + */ + +#include "hwmon-fan.dtsi" + +&smc_hwmon { + temperature-TW0P { + apple,key-id = "TW0P"; + label = "WiFi/BT Module Temp"; + }; +}; diff --git a/arch/arm64/boot/dts/apple/isp-common.dtsi b/arch/arm64/boot/dts/apple/isp-common.dtsi new file mode 100644 index 00000000000000..739e6e9e66e740 --- /dev/null +++ b/arch/arm64/boot/dts/apple/isp-common.dtsi @@ -0,0 +1,47 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * Common ISP configuration for Apple silicon platforms. + * + * Copyright The Asahi Linux Contributors + */ + +/ { + aliases { + isp = &isp; + }; + + reserved-memory { + #address-cells = <2>; + #size-cells = <2>; + ranges; + + isp_heap: isp-heap { + compatible = "apple,asc-mem"; + /* Filled in by bootloder */ + reg = <0 0 0 0>; + no-map; + }; + }; +}; + +&isp { + memory-region = <&isp_heap>; + memory-region-names = "heap"; + status = "okay"; +}; + +&isp_dart0 { + status = "okay"; +}; + +&isp_dart1 { + status = "okay"; +}; + +&isp_dart2 { + status = "okay"; +}; + +&ps_isp_sys { + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/apple/isp-imx248.dtsi b/arch/arm64/boot/dts/apple/isp-imx248.dtsi new file mode 100644 index 00000000000000..0a4ac1a0152c2c --- /dev/null +++ b/arch/arm64/boot/dts/apple/isp-imx248.dtsi @@ -0,0 +1,57 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * ISP configuration for platforms with IMX248 sensor. + * + * Copyright The Asahi Linux Contributors + */ + +#include "isp-common.dtsi" + +&isp { + apple,temporal-filter = <0>; + + sensor-presets { + /* 1280x720 */ + preset0 { + apple,config-index = <0>; + apple,input-size = <1296 736>; + apple,output-size = <1280 720>; + apple,crop = <8 8 1280 720>; + }; + /* 960x720 (4:3) */ + preset1 { + apple,config-index = <0>; + apple,input-size = <1296 736>; + apple,output-size = <960 720>; + apple,crop = <168 8 960 720>; + }; + /* 960x540 (16:9) */ + preset2 { + apple,config-index = <0>; + apple,input-size = <1296 736>; + apple,output-size = <960 540>; + apple,crop = <8 8 1280 720>; + }; + /* 640x480 (4:3) */ + preset3 { + apple,config-index = <0>; + apple,input-size = <1296 736>; + apple,output-size = <640 480>; + apple,crop = <168 8 960 720>; + }; + /* 640x360 (16:9) */ + preset4 { + apple,config-index = <0>; + apple,input-size = <1296 736>; + apple,output-size = <640 360>; + apple,crop = <8 8 1280 720>; + }; + /* 320x180 (16:9) */ + preset5 { + apple,config-index = <0>; + apple,input-size = <1296 736>; + apple,output-size = <320 180>; + apple,crop = <8 8 1280 720>; + }; + }; +}; diff --git a/arch/arm64/boot/dts/apple/isp-imx364.dtsi b/arch/arm64/boot/dts/apple/isp-imx364.dtsi new file mode 100644 index 00000000000000..55484d86523657 --- /dev/null +++ b/arch/arm64/boot/dts/apple/isp-imx364.dtsi @@ -0,0 +1,71 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * ISP configuration for platforms with IMX364 sensor. + * + * Copyright The Asahi Linux Contributors + */ + +#include "isp-common.dtsi" + +&isp { + apple,temporal-filter = <0>; + + sensor-presets { + /* 1920x1080 */ + preset0 { + apple,config-index = <0>; + apple,input-size = <1920 1080>; + apple,output-size = <1920 1080>; + apple,crop = <0 0 1920 1080>; + }; + /* 1440x720 (4:3) */ + preset1 { + apple,config-index = <0>; + apple,input-size = <1920 1080>; + apple,output-size = <1440 1080>; + apple,crop = <240 0 1440 1080>; + }; + /* 1280x720 (16:9) */ + preset2 { + apple,config-index = <0>; + apple,input-size = <1920 1080>; + apple,output-size = <1280 720>; + apple,crop = <0 0 1920 1080>; + }; + /* 960x720 (4:3) */ + preset3{ + apple,config-index = <0>; + apple,input-size = <1920 1080>; + apple,output-size = <960 720>; + apple,crop = <240 0 1440 1080>; + }; + /* 960x540 (16:9) */ + preset4 { + apple,config-index = <0>; + apple,input-size = <1920 1080>; + apple,output-size = <960 540>; + apple,crop = <0 0 1920 1080>; + }; + /* 640x480 (4:3) */ + preset5 { + apple,config-index = <0>; + apple,input-size = <1920 1080>; + apple,output-size = <640 480>; + apple,crop = <240 0 1440 1080>; + }; + /* 640x360 (16:9) */ + preset6 { + apple,config-index = <0>; + apple,input-size = <1920 1080>; + apple,output-size = <640 360>; + apple,crop = <0 0 1920 1080>; + }; + /* 320x180 (16:9) */ + preset7 { + apple,config-index = <0>; + apple,input-size = <1920 1080>; + apple,output-size = <320 180>; + apple,crop = <0 0 1920 1080>; + }; + }; +}; diff --git a/arch/arm64/boot/dts/apple/isp-imx558-cfg0.dtsi b/arch/arm64/boot/dts/apple/isp-imx558-cfg0.dtsi new file mode 100644 index 00000000000000..729b97829cbb7e --- /dev/null +++ b/arch/arm64/boot/dts/apple/isp-imx558-cfg0.dtsi @@ -0,0 +1,92 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * ISP configuration for platforms with IMX558 sensor in + * config #0 mode. + * + * These platforms enable MLVNR for all configs except + * #0, which we don't support. Config #0 is an uncropped + * square 1920x1920 sensor, with dark corners. + * Therefore, we synthesize common resolutions by using + * crop/scale while always choosing config #0. + * + * Copyright The Asahi Linux Contributors + */ + +#include "isp-common.dtsi" + +&isp { + apple,temporal-filter = <0>; + + sensor-presets { + /* 1920x1080 */ + preset0 { + apple,config-index = <0>; + apple,input-size = <1920 1920>; + apple,output-size = <1920 1080>; + apple,crop = <0 420 1920 1080>; + }; + /* 1080x1920 */ + preset1 { + apple,config-index = <0>; + apple,input-size = <1920 1920>; + apple,output-size = <1080 1920>; + apple,crop = <420 0 1080 1920>; + }; + /* 1920x1440 */ + preset2 { + apple,config-index = <0>; + apple,input-size = <1920 1920>; + apple,output-size = <1920 1440>; + apple,crop = <0 240 1920 1440>; + }; + /* 1440x1920 */ + preset3 { + apple,config-index = <0>; + apple,input-size = <1920 1920>; + apple,output-size = <1440 1920>; + apple,crop = <240 0 1440 1920>; + }; + /* 1280x720 */ + preset4 { + apple,config-index = <0>; + apple,input-size = <1920 1920>; + apple,output-size = <1280 720>; + apple,crop = <0 420 1920 1080>; + }; + /* 720x1280 */ + preset5 { + apple,config-index = <0>; + apple,input-size = <1920 1920>; + apple,output-size = <720 1280>; + apple,crop = <420 0 1080 1920>; + }; + /* 1280x960 */ + preset6 { + apple,config-index = <0>; + apple,input-size = <1920 1920>; + apple,output-size = <1280 960>; + apple,crop = <0 240 1920 1440>; + }; + /* 960x1280 */ + preset7 { + apple,config-index = <0>; + apple,input-size = <1920 1920>; + apple,output-size = <960 1280>; + apple,crop = <240 0 1440 1920>; + }; + /* 640x480 */ + preset8 { + apple,config-index = <0>; + apple,input-size = <1920 1920>; + apple,output-size = <640 480>; + apple,crop = <0 240 1920 1440>; + }; + /* 480x640 */ + preset9 { + apple,config-index = <0>; + apple,input-size = <1920 1920>; + apple,output-size = <480 640>; + apple,crop = <240 0 1440 1920>; + }; + }; +}; diff --git a/arch/arm64/boot/dts/apple/isp-imx558.dtsi b/arch/arm64/boot/dts/apple/isp-imx558.dtsi new file mode 100644 index 00000000000000..d55854c883f5b6 --- /dev/null +++ b/arch/arm64/boot/dts/apple/isp-imx558.dtsi @@ -0,0 +1,92 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * ISP configuration for platforms with IMX558 sensor. + * + * Copyright The Asahi Linux Contributors + */ + +#include "isp-common.dtsi" + +&isp { + apple,temporal-filter = <0>; + + sensor-presets { + /* 1920x1080 */ + preset0 { + apple,config-index = <1>; + apple,input-size = <1920 1080>; + apple,output-size = <1920 1080>; + apple,crop = <0 0 1920 1080>; + }; + /* 1080x1920 */ + preset1 { + apple,config-index = <2>; + apple,input-size = <1080 1920>; + apple,output-size = <1080 1920>; + apple,crop = <0 0 1080 1920>; + }; + /* 1760x1328 */ + preset2 { + apple,config-index = <3>; + apple,input-size = <1760 1328>; + apple,output-size = <1760 1328>; + apple,crop = <0 0 1760 1328>; + }; + /* 1328x1760 */ + preset3 { + apple,config-index = <4>; + apple,input-size = <1328 1760>; + apple,output-size = < 1328 1760>; + apple,crop = <0 0 1328 1760>; + }; + /* 1152x1152 */ + preset4 { + apple,config-index = <5>; + apple,input-size = <1152 1152>; + apple,output-size = <1152 1152>; + apple,crop = <0 0 1152 1152>; + }; + /* 1280x720 */ + preset5 { + apple,config-index = <1>; + apple,input-size = <1920 1080>; + apple,output-size = <1280 720>; + apple,crop = <0 0 1920 1080>; + }; + /* 720x1280 */ + preset6 { + apple,config-index = <2>; + apple,input-size = <1080 1920>; + apple,output-size = <720 1280>; + apple,crop = <0 0 1080 1920>; + }; + /* 1280x960 */ + preset7 { + apple,config-index = <3>; + apple,input-size = <1760 1328>; + apple,output-size = <1280 960>; + apple,crop = <0 4 1760 1320>; + }; + /* 960x1280 */ + preset8 { + apple,config-index = <4>; + apple,input-size = <1328 1760>; + apple,output-size = <960 1280>; + apple,crop = <4 0 1320 1760>; + }; + /* 640x480 */ + preset9 { + apple,config-index = <3>; + apple,input-size = <1760 1328>; + apple,output-size = <640 480>; + apple,crop = <0 4 1760 1320>; + }; + /* 480x640 */ + preset10 { + apple,config-index = <4>; + apple,input-size = <1328 1760>; + apple,output-size = <480 640>; + apple,crop = <4 0 1320 1760>; + }; + }; +}; diff --git a/arch/arm64/boot/dts/apple/s8001-j98a-j99a.dtsi b/arch/arm64/boot/dts/apple/s8001-j98a-j99a.dtsi index e66a4c1c138fe8..67633c56a72389 100644 --- a/arch/arm64/boot/dts/apple/s8001-j98a-j99a.dtsi +++ b/arch/arm64/boot/dts/apple/s8001-j98a-j99a.dtsi @@ -9,6 +9,10 @@ * Copyright (c) 2024, Nick Chan */ +&dwi_bl { + status = "okay"; +}; + &ps_dcs4 { apple,always-on; /* LPDDR4 interface */ }; diff --git a/arch/arm64/boot/dts/apple/s8001.dtsi b/arch/arm64/boot/dts/apple/s8001.dtsi index b5b00dca6ffa4c..209c7dd19b7c28 100644 --- a/arch/arm64/boot/dts/apple/s8001.dtsi +++ b/arch/arm64/boot/dts/apple/s8001.dtsi @@ -209,6 +209,13 @@ power-domains = <&ps_aic>; }; + dwi_bl: backlight@20e200080 { + compatible = "apple,s8000-dwi-bl", "apple,dwi-bl"; + reg = <0x2 0x0e200080 0x0 0x8>; + power-domains = <&ps_dwi>; + status = "disabled"; + }; + pinctrl_ap: pinctrl@20f100000 { compatible = "apple,s8000-pinctrl", "apple,pinctrl"; reg = <0x2 0x0f100000 0x0 0x100000>; diff --git a/arch/arm64/boot/dts/apple/t6000-j314s.dts b/arch/arm64/boot/dts/apple/t6000-j314s.dts index 1430b91ff1b152..afa86668440f04 100644 --- a/arch/arm64/boot/dts/apple/t6000-j314s.dts +++ b/arch/arm64/boot/dts/apple/t6000-j314s.dts @@ -24,3 +24,20 @@ &bluetooth0 { brcm,board-type = "apple,maldives"; }; + +&panel { + compatible = "apple,panel-j314", "apple,panel-mini-led", "apple,panel"; + width-mm = <302>; + height-mm = <196>; + adj-height-mm = <189>; +}; + +&aop_audio { + apple,chassis-name = "J314"; + apple,machine-kind = "MacBook Pro"; +}; + +&sound { + compatible = "apple,j314-macaudio", "apple,macaudio"; + model = "MacBook Pro J314"; +}; diff --git a/arch/arm64/boot/dts/apple/t6000-j316s.dts b/arch/arm64/boot/dts/apple/t6000-j316s.dts index da0cbe7d96736b..ddfc3c530923c7 100644 --- a/arch/arm64/boot/dts/apple/t6000-j316s.dts +++ b/arch/arm64/boot/dts/apple/t6000-j316s.dts @@ -24,3 +24,20 @@ &bluetooth0 { brcm,board-type = "apple,madagascar"; }; + +&panel { + compatible = "apple,panel-j316", "apple,panel-mini-led", "apple,panel"; + width-mm = <346>; + height-mm = <223>; + adj-height-mm = <216>; +}; + +&aop_audio { + apple,chassis-name = "J316"; + apple,machine-kind = "MacBook Pro"; +}; + +&sound { + compatible = "apple,j316-macaudio", "apple,macaudio"; + model = "MacBook Pro J316"; +}; diff --git a/arch/arm64/boot/dts/apple/t6000.dtsi b/arch/arm64/boot/dts/apple/t6000.dtsi index 0ad77c98073fe6..db7ea6eac5a6f1 100644 --- a/arch/arm64/boot/dts/apple/t6000.dtsi +++ b/arch/arm64/boot/dts/apple/t6000.dtsi @@ -9,6 +9,8 @@ /* This chip is just a cut down version of t6001, so include it and disable the missing parts */ +#define GPU_REPEAT(x) + #include "t6001.dtsi" / { @@ -16,7 +18,24 @@ }; /delete-node/ &pmgr_south; +/delete-node/ &pmp_report_dispext2; +/delete-node/ &pmp_report_dispext3; +/delete-node/ &pmp_report_venc1; +/delete-node/ &pmp_report_msr1; +/delete-node/ &pmp_report_prores; +/delete-node/ &pmp_report_afnc4_ioa; +/delete-node/ &pmp_report_afnc5_ioa; + +&pmp { + apple,pio-ranges = <0x2 0x82000000 0x0 0x1000000>, + <0x3 0x4000000 0x0 0x1000000>, + <0x3 0x83000000 0x0 0x1000000>, + <0x4 0x2000000 0x0 0x1000000>, + <0x2 0x10e70000 0x0 0x90000>, + <0x2 0x11e70000 0x0 0x90000>, + <0x2 0x12e70000 0x0 0x90000>; +}; &gpu { - compatible = "apple,agx-g13s"; + compatible = "apple,agx-t6000", "apple,agx-g13x"; }; diff --git a/arch/arm64/boot/dts/apple/t6001-j314c.dts b/arch/arm64/boot/dts/apple/t6001-j314c.dts index c37097dcfdb304..245df6d03ee422 100644 --- a/arch/arm64/boot/dts/apple/t6001-j314c.dts +++ b/arch/arm64/boot/dts/apple/t6001-j314c.dts @@ -24,3 +24,20 @@ &bluetooth0 { brcm,board-type = "apple,maldives"; }; + +&panel { + compatible = "apple,panel-j314", "apple,panel-mini-led", "apple,panel"; + width-mm = <302>; + height-mm = <196>; + adj-height-mm = <189>; +}; + +&aop_audio { + apple,chassis-name = "J314"; + apple,machine-kind = "MacBook Pro"; +}; + +&sound { + compatible = "apple,j314-macaudio", "apple,macaudio"; + model = "MacBook Pro J314"; +}; diff --git a/arch/arm64/boot/dts/apple/t6001-j316c.dts b/arch/arm64/boot/dts/apple/t6001-j316c.dts index 3bc6e0c3294cf9..a000d497b705fa 100644 --- a/arch/arm64/boot/dts/apple/t6001-j316c.dts +++ b/arch/arm64/boot/dts/apple/t6001-j316c.dts @@ -24,3 +24,20 @@ &bluetooth0 { brcm,board-type = "apple,madagascar"; }; + +&panel { + compatible = "apple,panel-j316", "apple,panel-mini-led", "apple,panel"; + width-mm = <346>; + height-mm = <223>; + adj-height-mm = <216>; +}; + +&aop_audio { + apple,chassis-name = "J316"; + apple,machine-kind = "MacBook Pro"; +}; + +&sound { + compatible = "apple,j316-macaudio", "apple,macaudio"; + model = "MacBook Pro J316"; +}; diff --git a/arch/arm64/boot/dts/apple/t6001-j375c.dts b/arch/arm64/boot/dts/apple/t6001-j375c.dts index 2e7c23714d4d00..f3f98f03800908 100644 --- a/arch/arm64/boot/dts/apple/t6001-j375c.dts +++ b/arch/arm64/boot/dts/apple/t6001-j375c.dts @@ -24,3 +24,39 @@ &bluetooth0 { brcm,board-type = "apple,okinawa"; }; + +&dpaudio0 { + status = "okay"; +}; + +&sound { + compatible = "apple,j375-macaudio", "apple,macaudio"; + model = "Mac Studio J375"; +}; + +&pinctrl_ap { + usb_hub_oe-hog { + gpio-hog; + gpios = <230 0>; + input; + line-name = "usb-hub-oe"; + }; + + usb_hub_rst-hog { + gpio-hog; + gpios = <231 GPIO_ACTIVE_LOW>; + output-low; + line-name = "usb-hub-rst"; + }; +}; + +&gpu { + apple,avg-power-ki-only = <0.6375>; + apple,avg-power-kp = <0.58>; + apple,avg-power-target-filter-tc = <1>; + apple,perf-base-pstate = <3>; + apple,ppm-ki = <5.8>; + apple,ppm-kp = <0.355>; +}; + +#include "hwmon-fan-dual.dtsi" diff --git a/arch/arm64/boot/dts/apple/t6001.dtsi b/arch/arm64/boot/dts/apple/t6001.dtsi index ffbe823b71bc8d..73053599d71d81 100644 --- a/arch/arm64/boot/dts/apple/t6001.dtsi +++ b/arch/arm64/boot/dts/apple/t6001.dtsi @@ -11,10 +11,15 @@ #include #include #include +#include #include #include "multi-die-cpp.h" +#ifndef GPU_REPEAT +# define GPU_REPEAT(x) +#endif + #include "t600x-common.dtsi" / { @@ -27,6 +32,8 @@ ranges; nonposted-mmio; + /* Required to get >32-bit DMA via DARTs */ + dma-ranges = <0 0 0 0 0xffffffff 0xffffc000>; // filled via templated includes at the end of the file }; @@ -63,6 +70,18 @@ }; }; +&pmp { + apple,pio-ranges = <0x2 0x82000000 0x0 0x1000000>, + <0x3 0x4000000 0x0 0x1000000>, + <0x3 0x83000000 0x0 0x1000000>, + <0x4 0x2000000 0x0 0x1000000>, + <0x2 0x10e70000 0x0 0x90000>, + <0x2 0x11e70000 0x0 0x90000>, + <0x2 0x12e70000 0x0 0x90000>, + <0x4 0x82000000 0x0 0x1000000>, + <0x5 0x2000000 0x0 0x1000000>; +}; + &gpu { - compatible = "apple,agx-g13c", "apple,agx-g13s"; + compatible = "apple,agx-t6001", "apple,agx-g13c", "apple,agx-g13s"; }; diff --git a/arch/arm64/boot/dts/apple/t6002-j375d.dts b/arch/arm64/boot/dts/apple/t6002-j375d.dts index 2b7f80119618ad..5cf30cd162b679 100644 --- a/arch/arm64/boot/dts/apple/t6002-j375d.dts +++ b/arch/arm64/boot/dts/apple/t6002-j375d.dts @@ -15,6 +15,19 @@ / { compatible = "apple,j375d", "apple,t6002", "apple,arm-platform"; model = "Apple Mac Studio (M1 Ultra, 2022)"; + aliases { + atcphy4 = &atcphy0_die1; + atcphy5 = &atcphy1_die1; + }; +}; + +&dpaudio0 { + status = "okay"; +}; + +&sound { + compatible = "apple,j375-macaudio", "apple,macaudio"; + model = "Mac Studio J375"; }; /* USB Type C */ @@ -26,6 +39,30 @@ interrupt-parent = <&pinctrl_ap>; interrupts = <174 IRQ_TYPE_LEVEL_LOW>; interrupt-names = "irq"; + + typec4: connector { + compatible = "usb-c-connector"; + label = "USB-C Front Right"; + power-role = "dual"; + data-role = "dual"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + typec4_connector_hs: endpoint { + remote-endpoint = <&dwc3_4_hs>; + }; + }; + port@1 { + reg = <1>; + typec4_connector_ss: endpoint { + remote-endpoint = <&atcphy4_typec_lanes>; + }; + }; + }; + }; }; /* front-left */ @@ -35,6 +72,30 @@ interrupt-parent = <&pinctrl_ap>; interrupts = <174 IRQ_TYPE_LEVEL_LOW>; interrupt-names = "irq"; + + typec5: connector { + compatible = "usb-c-connector"; + label = "USB-C Front Left"; + power-role = "dual"; + data-role = "dual"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + typec5_connector_hs: endpoint { + remote-endpoint = <&dwc3_5_hs>; + }; + }; + port@1 { + reg = <1>; + typec5_connector_ss: endpoint { + remote-endpoint = <&atcphy5_typec_lanes>; + }; + }; + }; + }; }; }; @@ -46,6 +107,106 @@ brcm,board-type = "apple,okinawa"; }; +/* USB controllers on die 1 */ +&dwc3_0_die1 { + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + dwc3_4_hs: endpoint { + remote-endpoint = <&typec4_connector_hs>; + }; + }; + + port@1 { + reg = <1>; + dwc3_4_ss: endpoint { + remote-endpoint = <&atcphy4_usb3>; + }; + }; + }; +}; + +&dwc3_1_die1 { + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + dwc3_5_hs: endpoint { + remote-endpoint = <&typec5_connector_hs>; + }; + }; + + port@1 { + reg = <1>; + dwc3_5_ss: endpoint { + remote-endpoint = <&atcphy5_usb3>; + }; + }; + }; +}; + +/* Type-C PHYs */ +&atcphy0_die1 { + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + atcphy4_typec_lanes: endpoint { + remote-endpoint = <&typec4_connector_ss>; + }; + }; + + port@1 { + reg = <1>; + atcphy4_usb3: endpoint { + remote-endpoint = <&dwc3_4_ss>; + }; + }; + }; +}; + +&atcphy1_die1 { + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + atcphy5_typec_lanes: endpoint { + remote-endpoint = <&typec5_connector_ss>; + }; + }; + + port@1 { + reg = <1>; + atcphy5_usb3: endpoint { + remote-endpoint = <&dwc3_5_ss>; + }; + }; + }; +}; + +/* delete unused USB nodes on die 1 */ + +/delete-node/ &dwc3_2_dart_0_die1; +/delete-node/ &dwc3_2_dart_1_die1; +/delete-node/ &dwc3_2_die1; +/delete-node/ &atcphy2_die1; +/delete-node/ &atcphy2_xbar_die1; + +/delete-node/ &dwc3_3_dart_0_die1; +/delete-node/ &dwc3_3_dart_1_die1; +/delete-node/ &dwc3_3_die1; +/delete-node/ &atcphy3_die1; +/delete-node/ &atcphy3_xbar_die1; + /* delete unused always-on power-domains on die 1 */ /delete-node/ &ps_atc2_usb_aon_die1; @@ -56,3 +217,14 @@ /delete-node/ &ps_disp0_cpu0_die1; /delete-node/ &ps_disp0_fe_die1; + +&gpu { + apple,avg-power-ki-only = <0.6375>; + apple,avg-power-kp = <0.58>; + apple,avg-power-target-filter-tc = <1>; + apple,perf-base-pstate = <3>; + apple,ppm-ki = <5.8>; + apple,ppm-kp = <0.355>; +}; + +#include "hwmon-fan-dual.dtsi" diff --git a/arch/arm64/boot/dts/apple/t6002.dtsi b/arch/arm64/boot/dts/apple/t6002.dtsi index 8fb648836b538b..b5f175d71ee69a 100644 --- a/arch/arm64/boot/dts/apple/t6002.dtsi +++ b/arch/arm64/boot/dts/apple/t6002.dtsi @@ -11,10 +11,15 @@ #include #include #include +#include #include #include "multi-die-cpp.h" +#ifndef GPU_REPEAT +# define GPU_REPEAT(x) +#endif + #include "t600x-common.dtsi" / { @@ -235,6 +240,8 @@ <0x5 0x80000000 0x5 0x80000000 0x1 0x80000000>, <0x7 0x0 0x7 0x0 0xf 0x80000000>; nonposted-mmio; + /* Required to get >32-bit DMA via DARTs */ + dma-ranges = <0 0 0 0 0xffffffff 0xffffc000>; // filled via templated includes at the end of the file }; @@ -246,6 +253,8 @@ ranges = <0x2 0x0 0x22 0x0 0x4 0x0>, <0x7 0x0 0x27 0x0 0xf 0x80000000>; nonposted-mmio; + /* Required to get >32-bit DMA via DARTs */ + dma-ranges = <0 0 0 0 0xffffffff 0xffffc000>; // filled via templated includes at the end of the file }; @@ -296,11 +305,78 @@ }; }; +&dcpext0_die1 { + // TODO: verify + apple,bw-scratch = <&pmgr_dcp 0 4 0x9c0>; +}; + +&dcpext1_die1 { + // TODO: verify + apple,bw-scratch = <&pmgr_dcp 0 4 0x9c8>; +}; + &ps_gfx { // On t6002, the die0 GPU power domain needs both AFR power domains power-domains = <&ps_afr>, <&ps_afr_die1>; }; +&pmp { + apple,pio-ranges = <0x2 0x82000000 0x0 0x1000000>, + <0x3 0x4000000 0x0 0x1000000>, + <0x3 0x83000000 0x0 0x1000000>, + <0x4 0x2000000 0x0 0x1000000>, + <0x2 0x10e70000 0x0 0x90000>, + <0x2 0x11e70000 0x0 0x90000>, + <0x2 0x12e70000 0x0 0x90000>, + <0x4 0x82000000 0x0 0x1000000>, + <0x5 0x2000000 0x0 0x1000000>, + <0x22 0x82000000 0x0 0x1000000>, + <0x23 0x4000000 0x0 0x1000000>, + <0x23 0x83000000 0x0 0x1000000>, + <0x24 0x2000000 0x0 0x1000000>, + <0x24 0x82000000 0x0 0x1000000>, + <0x25 0x2000000 0x0 0x1000000>, + <0x22 0x10e70000 0x0 0x90000>, + <0x22 0x11e70000 0x0 0x90000>, + <0x22 0x12e70000 0x0 0x90000>; +}; + +&pmp_report { + pmp_report_dispext0_die1: report@33 { + compatible = "apple,t6000-pmp-v2-report-entry"; + reg = <0x33>; + label = "pmp-dispext0_die1"; + #power-domain-cells = <0>; + power-domains = <&ps_dispext0_cpu0_die1>; + }; + + pmp_report_dispext1_die1: report@34 { + compatible = "apple,t6000-pmp-v2-report-entry"; + reg = <0x34>; + label = "pmp-dispext1_die1"; + #power-domain-cells = <0>; + power-domains = <&ps_dispext1_cpu0_die1>; + }; + + pmp_report_dispext2_die1: report@35 { + compatible = "apple,t6000-pmp-v2-report-entry"; + reg = <0x35>; + label = "pmp-dispext2_die1"; + #power-domain-cells = <0>; + power-domains = <&ps_dispext1_cpu0_die1>; + status = "disabled"; + }; + + pmp_report_dispext3_die1: report@36 { + compatible = "apple,t6000-pmp-v2-report-entry"; + reg = <0x36>; + label = "pmp-dispext3_die1"; + #power-domain-cells = <0>; + power-domains = <&ps_dispext1_cpu0_die1>; + status = "disabled"; + }; +}; + &gpu { - compatible = "apple,agx-g13d", "apple,agx-g13s"; + compatible = "apple,agx-t6002", "apple,agx-g13d", "apple,agx-g13s"; }; diff --git a/arch/arm64/boot/dts/apple/t600x-common.dtsi b/arch/arm64/boot/dts/apple/t600x-common.dtsi index e20234ef213538..a7f25092533e7d 100644 --- a/arch/arm64/boot/dts/apple/t600x-common.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-common.dtsi @@ -13,6 +13,9 @@ aliases { gpu = &gpu; + #ifdef APPLE_USE_PMP + pmp = &pmp; + #endif }; cpus { @@ -229,26 +232,31 @@ opp-hz = /bits/ 64 <600000000>; opp-level = <1>; clock-latency-ns = <7500>; + opp-microwatt = <47296>; }; opp02 { opp-hz = /bits/ 64 <972000000>; opp-level = <2>; clock-latency-ns = <23000>; + opp-microwatt = <99715>; }; opp03 { opp-hz = /bits/ 64 <1332000000>; opp-level = <3>; clock-latency-ns = <29000>; + opp-microwatt = <188860>; }; opp04 { opp-hz = /bits/ 64 <1704000000>; opp-level = <4>; clock-latency-ns = <40000>; + opp-microwatt = <288891>; }; opp05 { opp-hz = /bits/ 64 <2064000000>; opp-level = <5>; clock-latency-ns = <50000>; + opp-microwatt = <412979>; }; }; @@ -259,82 +267,139 @@ opp-hz = /bits/ 64 <600000000>; opp-level = <1>; clock-latency-ns = <8000>; + opp-microwatt = <290230>; }; opp02 { opp-hz = /bits/ 64 <828000000>; opp-level = <2>; clock-latency-ns = <18000>; + opp-microwatt = <449013>; }; opp03 { opp-hz = /bits/ 64 <1056000000>; opp-level = <3>; clock-latency-ns = <19000>; + opp-microwatt = <647097>; }; opp04 { opp-hz = /bits/ 64 <1296000000>; opp-level = <4>; clock-latency-ns = <23000>; + opp-microwatt = <865620>; }; opp05 { opp-hz = /bits/ 64 <1524000000>; opp-level = <5>; clock-latency-ns = <24000>; + opp-microwatt = <1112838>; }; opp06 { opp-hz = /bits/ 64 <1752000000>; opp-level = <6>; clock-latency-ns = <28000>; + opp-microwatt = <1453271>; }; opp07 { opp-hz = /bits/ 64 <1980000000>; opp-level = <7>; clock-latency-ns = <31000>; + opp-microwatt = <1776667>; }; opp08 { opp-hz = /bits/ 64 <2208000000>; opp-level = <8>; clock-latency-ns = <45000>; + opp-microwatt = <2366690>; }; opp09 { opp-hz = /bits/ 64 <2448000000>; opp-level = <9>; clock-latency-ns = <49000>; + opp-microwatt = <2892193>; }; opp10 { opp-hz = /bits/ 64 <2676000000>; opp-level = <10>; clock-latency-ns = <53000>; + opp-microwatt = <3475417>; }; opp11 { opp-hz = /bits/ 64 <2904000000>; opp-level = <11>; clock-latency-ns = <56000>; + opp-microwatt = <3959410>; }; opp12 { opp-hz = /bits/ 64 <3036000000>; opp-level = <12>; clock-latency-ns = <56000>; + opp-microwatt = <4540620>; }; - /* Not available until CPU deep sleep is implemented opp13 { opp-hz = /bits/ 64 <3132000000>; opp-level = <13>; clock-latency-ns = <56000>; + opp-microwatt = <4745031>; turbo-mode; }; opp14 { opp-hz = /bits/ 64 <3168000000>; opp-level = <14>; clock-latency-ns = <56000>; + opp-microwatt = <4822390>; turbo-mode; }; opp15 { opp-hz = /bits/ 64 <3228000000>; opp-level = <15>; clock-latency-ns = <56000>; + opp-microwatt = <4951324>; turbo-mode; }; - */ + }; + + gpu_opp: opp-table-gpu { + compatible = "operating-points-v2"; + + /* + * NOTE: The voltage and power values are device-specific and + * must be filled in by the bootloader. + */ + opp00 { + opp-hz = /bits/ 64 <0>; + opp-microvolt = GPU_REPEAT(400000); + opp-microwatt = <0>; + }; + opp01 { + opp-hz = /bits/ 64 <388800000>; + opp-microvolt = GPU_REPEAT(634000); + opp-microwatt = <25011450>; + }; + opp02 { + opp-hz = /bits/ 64 <486000000>; + opp-microvolt = GPU_REPEAT(650000); + opp-microwatt = <31681170>; + }; + opp03 { + opp-hz = /bits/ 64 <648000000>; + opp-microvolt = GPU_REPEAT(668000); + opp-microwatt = <41685750>; + }; + opp04 { + opp-hz = /bits/ 64 <777600000>; + opp-microvolt = GPU_REPEAT(715000); + opp-microwatt = <56692620>; + }; + opp05 { + opp-hz = /bits/ 64 <972000000>; + opp-microvolt = GPU_REPEAT(778000); + opp-microwatt = <83371500>; + }; + opp06 { + opp-hz = /bits/ 64 <1296000000>; + opp-microvolt = GPU_REPEAT(903000); + opp-microwatt = <166743000>; + }; }; pmu-e { @@ -373,6 +438,40 @@ clock-output-names = "clk_200m"; }; + clk_disp0: clock-disp0 { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <237333328>; + clock-output-names = "clk_disp0"; + }; + + clk_dispext0: clock-dispext0 { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <0>; + clock-output-names = "clk_dispext0"; + }; + + clk_dispext0_die1: clock-dispext0_die1 { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <0>; + clock-output-names = "clk_dispext0_die1"; + }; + + clk_dispext1: clock-dispext1 { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <0>; + clock-output-names = "clk_dispext1"; + }; + + clk_dispext1_die1: clock-dispext1_die1 { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <0>; + clock-output-names = "clk_dispext1_die1"; + }; /* * This is a fabulated representation of the input clock * to NCO since we don't know the true clock tree. @@ -401,15 +500,15 @@ }; uat_handoff: uat-handoff { - status = "disabled"; + reg = <0 0 0 0>; }; uat_pagetables: uat-pagetables { - status = "disabled"; + reg = <0 0 0 0>; }; uat_ttbs: uat-ttbs { - status = "disabled"; + reg = <0 0 0 0>; }; }; }; diff --git a/arch/arm64/boot/dts/apple/t600x-die0.dtsi b/arch/arm64/boot/dts/apple/t600x-die0.dtsi index 3603b276a2abcf..6bf9802ed4dc8b 100644 --- a/arch/arm64/boot/dts/apple/t600x-die0.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-die0.dtsi @@ -24,6 +24,215 @@ power-domains = <&ps_aic>; }; + pmgr_misc: power-management@28e20c000 { + compatible = "apple,t6000-pmgr-misc"; + #address-cells = <1>; + #size-cells = <1>; + reg = <0x2 0x8e20c000 0 0x400>, + <0x2 0x8e20c800 0 0x400>; + reg-names = "fabric-ps", "dcs-ps"; + apple,dcs-min-ps = <7>; + }; + + pmp_dart: iommu@28e300000 { + compatible = "apple,t6000-dart"; + reg = <0x2 0x8e300000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_pmp>; + }; + + pmp_report: pmp_report@28e3c0000 { + compatible = "apple,t6000-pmp-v2-report"; + reg = <0x2 0x8e3c0000 0x0 0x20000>; + power-domains = <&ps_pms_sram>; + #address-cells = <1>; + #size-cells = <0>; + + pmp_report_ane_sys: report@a { + compatible = "apple,t6000-pmp-v2-report-entry"; + reg = <0xa>; + label = "pmp-ane-sys"; + #power-domain-cells = <0>; + power-domains = <&ps_ane_sys>; + status = "disabled"; + }; + + pmp_report_isp_sys: report@b { + compatible = "apple,t6000-pmp-v2-report-entry"; + reg = <0xb>; + label = "pmp-isp-sys"; + #power-domain-cells = <0>; + power-domains = <&ps_isp_sys>; + status = "disabled"; + }; + + pmp_report_disp0: report@c { + compatible = "apple,t6000-pmp-v2-report-entry"; + reg = <0xc>; + label = "pmp-disp0"; + #power-domain-cells = <0>; + power-domains = <&ps_disp0_cpu0>; + apple,always-on; + }; + + pmp_report_dispext0: report@d { + compatible = "apple,t6000-pmp-v2-report-entry"; + reg = <0xd>; + label = "pmp-dispext0"; + #power-domain-cells = <0>; + power-domains = <&ps_dispext0_cpu0>; + apple,always-on; + }; + + pmp_report_dispext1: report@e { + compatible = "apple,t6000-pmp-v2-report-entry"; + reg = <0xe>; + label = "pmp-dispext1"; + #power-domain-cells = <0>; + power-domains = <&ps_dispext1_cpu0>; + }; + + pmp_report_venc_sys: report@10 { + compatible = "apple,t6000-pmp-v2-report-entry"; + reg = <0x10>; + label = "pmp-venc-sys"; + #power-domain-cells = <0>; + power-domains = <&ps_venc_sys>; + status = "disabled"; + }; + + pmp_report_avd_sys: report@11 { + compatible = "apple,t6000-pmp-v2-report-entry"; + reg = <0x11>; + label = "pmp-avd-sys"; + #power-domain-cells = <0>; + power-domains = <&ps_avd_sys>; + status = "disabled"; + }; + + pmp_report_msr0: report@12 { + compatible = "apple,t6000-pmp-v2-report-entry"; + reg = <0x12>; + label = "pmp-msr0"; + #power-domain-cells = <0>; + power-domains = <&ps_msr0>; + status = "disabled"; + }; + + pmp_report_jpg: report@13 { + compatible = "apple,t6000-pmp-v2-report-entry"; + reg = <0x13>; + label = "pmp-jpg"; + #power-domain-cells = <0>; + power-domains = <&ps_jpg>; + status = "disabled"; + }; + + pmp_report_scodec: report@14 { + compatible = "apple,t6000-pmp-v2-report-entry"; + reg = <0x14>; + label = "pmp-scodec"; + #power-domain-cells = <0>; + power-domains = <&ps_scodec>; + status = "disabled"; + }; + + pmp_report_afnc4_ioa: report@1d { + compatible = "apple,t6000-pmp-v2-report-entry"; + reg = <0x1d>; + label = "pmp-afnc4-ioa"; + #power-domain-cells = <0>; + apple,always-on; + }; + + pmp_report_afnc5_ioa: report@1e { + compatible = "apple,t6000-pmp-v2-report-entry"; + reg = <0x1e>; + label = "pmp-afnc5-ioa"; + #power-domain-cells = <0>; + apple,always-on; + }; + + pmp_report_dispext2: report@20 { + compatible = "apple,t6000-pmp-v2-report-entry"; + reg = <0x20>; + label = "pmp-dispext2"; + #power-domain-cells = <0>; + power-domains = <&ps_dispext2_cpu0>; + status = "disabled"; + }; + + pmp_report_dispext3: report@21 { + compatible = "apple,t6000-pmp-v2-report-entry"; + reg = <0x21>; + label = "pmp-dispext3"; + #power-domain-cells = <0>; + power-domains = <&ps_dispext3_cpu0>; + status = "disabled"; + }; + + pmp_report_venc1: report@22 { + compatible = "apple,t6000-pmp-v2-report-entry"; + reg = <0x22>; + label = "pmp-venc1"; + #power-domain-cells = <0>; + power-domains = <&ps_venc1_sys>; + status = "disabled"; + }; + + pmp_report_msr1: report@23 { + compatible = "apple,t6000-pmp-v2-report-entry"; + reg = <0x23>; + label = "pmp-msr1"; + #power-domain-cells = <0>; + power-domains = <&ps_msr1>; + status = "disabled"; + }; + + pmp_report_prores: report@24 { + compatible = "apple,t6000-pmp-v2-report-entry"; + reg = <0x24>; + label = "pmp-prores"; + #power-domain-cells = <0>; + power-domains = <&ps_prores>; + status = "disabled"; + }; + }; + + pmgr_dcp: power-management@28e3d0000 { + reg = <0x2 0x8e3d0000 0x0 0x4000>; + reg-names = "dcp-fw-pmgr"; + #apple,bw-scratch-cells = <3>; + }; + + pmp: pmp@28e700000 { + compatible = "apple,t6000-pmp-v2"; + reg = <0x2 0x8e700000 0x0 0x100000>, + <0x2 0x8ec00000 0x0 0x4000>; + reg-names = "pmp", "asc"; + mboxes = <&pmp_mbox>; + mbox-names = "mbox"; + iommus = <&pmp_dart 0>; + power-domains = <&ps_pmp>; + status = "disabled"; + }; + + pmp_mbox: mbox@28ec08000 { + compatible = "apple,t6000-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x2 0x8ec08000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + power-domains = <&ps_pmp>, <&ps_pms_sram>; + }; + smc: smc@290400000 { compatible = "apple,t6000-smc", "apple,smc"; reg = <0x2 0x90400000 0x0 0x4000>, @@ -37,12 +246,22 @@ #gpio-cells = <2>; }; + smc_hwmon: hwmon { + compatible = "apple,smc-hwmon"; + }; + smc_reboot: reboot { compatible = "apple,smc-reboot"; nvmem-cells = <&shutdown_flag>, <&boot_stage>, - <&boot_error_count>, <&panic_count>; + <&boot_error_count>, <&panic_count>, <&pm_setting>; nvmem-cell-names = "shutdown_flag", "boot_stage", - "boot_error_count", "panic_count"; + "boot_error_count", "panic_count", "pm_setting"; + }; + + rtc { + compatible = "apple,smc-rtc"; + nvmem-cells = <&rtc_offset>; + nvmem-cell-names = "rtc_offset"; }; }; @@ -145,22 +364,202 @@ interrupts = ; }; - sio_dart_0: iommu@39b004000 { + aop_mbox: mbox@293408000 { + compatible = "apple,t6000-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x2 0x93408000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + status = "disabled"; + }; + + aop_dart: iommu@293808000 { compatible = "apple,t6000-dart"; - reg = <0x3 0x9b004000 0x0 0x4000>; + reg = <0x2 0x93808000 0x0 0x4000>; + #iommu-cells = <1>; interrupt-parent = <&aic>; - interrupts = ; + interrupts = ; + status = "disabled"; + }; + + aop_admac: dma-controller@293980000 { + compatible = "apple,t6000-admac", "apple,admac"; + reg = <0x2 0x93980000 0x0 0x34000>; + #dma-cells = <1>; + dma-channels = <16>; + interrupts-extended = <0>, + <0>, + <&aic AIC_IRQ 0 600 IRQ_TYPE_LEVEL_HIGH>, + <0>; + iommus = <&aop_dart 7>; + status = "disabled"; + }; + + aop: aop@293c00000 { + compatible = "apple,t6000-aop"; + reg = <0x2 0x93c00000 0x0 0x250000>, + <0x2 0x93400000 0x0 0x6c000>; + mboxes = <&aop_mbox>; + mbox-names = "mbox"; + iommus = <&aop_dart 0>; + + status = "disabled"; + + aop_audio: audio { + compatible = "apple,t6000-aop-audio", "apple,aop-audio"; + dmas = <&aop_admac 1>; + dma-names = "dma"; + }; + + aop_als: als { + compatible = "apple,t6000-aop-als", "apple,aop-als"; + // intentionally empty + }; + + las { + compatible = "apple,t6000-aop-las", "apple,aop-las"; + }; + }; + + disp0_dart: iommu@38b304000 { + compatible = "apple,t6000-dart"; + reg = <0x3 0x8b304000 0x0 0x4000>; #iommu-cells = <1>; - power-domains = <&ps_sio_cpu>; + interrupt-parent = <&aic>; + interrupts = ; + status = "disabled"; + power-domains = <&ps_disp0_cpu0>; + apple,dma-range = <0x0 0x0 0x0 0xfc000000>; }; - sio_dart_1: iommu@39b008000 { + dcp_dart: iommu@38b30c000 { compatible = "apple,t6000-dart"; - reg = <0x3 0x9b008000 0x0 0x8000>; + reg = <0x3 0x8b30c000 0x0 0x4000>; + #iommu-cells = <1>; interrupt-parent = <&aic>; - interrupts = ; + interrupts = ; + power-domains = <&ps_disp0_cpu0>; + apple,dma-range = <0x1f0 0x0 0x0 0xfc000000>; + }; + + sep_dart: iommu@3952c0000 { + compatible = "apple,t6000-dart"; + reg = <0x3 0x952c0000 0x0 0x4000>; #iommu-cells = <1>; - power-domains = <&ps_sio_cpu>; + interrupt-parent = <&aic>; + interrupts = ; + }; + + sep: sep@396400000 { + compatible = "apple,sep"; + reg = <0x3 0x96400000 0x0 0x6C000>; + mboxes = <&sep_mbox>; + mbox-names = "mbox"; + iommus = <&sep_dart 0>; + power-domains = <&ps_sep>; + status = "disabled"; + }; + + sep_mbox: mbox@396408000 { + compatible = "apple,t6000-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x3 0x96408000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + }; + + dpaudio0: audio-controller@39b500000 { + compatible = "apple,t6000-dpaudio", "apple,dpaudio"; + reg = <0x3 0x9b500000 0x0 0x4000>; + dmas = <&sio 0x64>; + dma-names = "tx"; + power-domains = <&ps_dpa0>; + reset-domains = <&ps_dpa0>; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + dpaudio0_dcp: endpoint { + remote-endpoint = <&dcp_audio>; + }; + }; + }; + }; + + dcp_mbox: mbox@38bc08000 { + compatible = "apple,t6000-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x3 0x8bc08000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + power-domains = <&ps_disp0_cpu0>; + }; + + dcp: dcp@38bc00000 { + compatible = "apple,t6000-dcp", "apple,dcp"; + mboxes = <&dcp_mbox>; + mbox-names = "mbox"; + iommus = <&dcp_dart 0>; + + reg-names = "coproc", "disp-0", "disp-1", "disp-2", "disp-3"; + reg = <0x3 0x8bc00000 0x0 0x4000>, + <0x3 0x8a000000 0x0 0x3000000>, + <0x3 0x8b320000 0x0 0x4000>, + <0x3 0x8b344000 0x0 0x4000>, + <0x3 0x8b800000 0x0 0x800000>; + apple,bw-scratch = <&pmgr_dcp 0 4 0x988>; + #ifdef APPLE_USE_PMP + power-domains = <&pmp_report_disp0>; + #else + power-domains = <&ps_disp0_cpu0>; + #endif + resets = <&ps_disp0_cpu0>; + clocks = <&clk_disp0>; + phandle = <&dcp>; + // required bus properties for 'piodma' subdevice + #address-cells = <2>; + #size-cells = <2>; + + disp0_piodma: piodma { + iommus = <&disp0_dart 4>; + phandle = <&disp0_piodma>; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + dcp_audio: endpoint { + remote-endpoint = <&dpaudio0_dcp>; + }; + }; + }; + }; + + display: display-subsystem { + compatible = "apple,display-subsystem"; + iommus = <&disp0_dart 0>; + /* generate phandle explicitly for use in loader */ + phandle = <&display>; }; fpwm0: pwm@39b030000 { @@ -327,10 +726,10 @@ "tx2a", "rx2a", "tx2b", "rx2b", "tx3a", "rx3a", "tx3b", "rx3b"; interrupt-parent = <&aic>; - interrupts = , + interrupts = , + , , - , - ; + ; power-domains = <&ps_audio_p>, <&ps_mca0>, <&ps_mca1>, <&ps_mca2>, <&ps_mca3>; resets = <&ps_audio_p>; @@ -350,6 +749,43 @@ "hw-cal-a", "hw-cal-b", "globals"; apple,firmware-abi = <0 0 0>; + + apple,firmware-version = <12 3 0>; + apple,firmware-compat = <12 3 0>; + + operating-points-v2 = <&gpu_opp>; + apple,perf-base-pstate = <1>; + apple,min-sram-microvolt = <790000>; + apple,avg-power-filter-tc-ms = <1000>; + apple,avg-power-ki-only = <2.4>; + apple,avg-power-kp = <1.5>; + apple,avg-power-min-duty-cycle = <40>; + apple,avg-power-target-filter-tc = <125>; + apple,fast-die0-integral-gain = <500.0>; + apple,fast-die0-proportional-gain = <72.0>; + apple,perf-boost-ce-step = <50>; + apple,perf-boost-min-util = <90>; + apple,perf-filter-drop-threshold = <0>; + apple,perf-filter-time-constant = <5>; + apple,perf-filter-time-constant2 = <50>; + apple,perf-integral-gain = <6.3>; + apple,perf-integral-gain2 = <0.197392>; + apple,perf-integral-min-clamp = <0>; + apple,perf-proportional-gain = <15.75>; + apple,perf-proportional-gain2 = <6.853981>; + apple,perf-tgt-utilization = <85>; + apple,power-sample-period = <8>; + apple,ppm-filter-time-constant-ms = <100>; + apple,ppm-ki = <30.0>; + apple,ppm-kp = <1.5>; + apple,pwr-filter-time-constant = <313>; + apple,pwr-integral-gain = <0.0202129>; + apple,pwr-integral-min-clamp = <0>; + apple,pwr-min-duty-cycle = <40>; + apple,pwr-proportional-gain = <5.2831855>; + + apple,core-leak-coef = GPU_REPEAT(1200.0); + apple,sram-leak-coef = GPU_REPEAT(20.0); }; agx_mbox: mbox@406408000 { @@ -365,6 +801,59 @@ #mbox-cells = <0>; }; + isp_dart0: iommu@3860e8000 { + compatible = "apple,t6000-dart"; + reg = <0x3 0x860e8000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_isp_sys>; + status = "disabled"; + }; + + isp_dart1: iommu@3860f4000 { + compatible = "apple,t6000-dart"; + reg = <0x3 0x860f4000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_isp_sys>; + status = "disabled"; + }; + + isp_dart2: iommu@3860fc000 { + compatible = "apple,t6000-dart"; + reg = <0x3 0x860fc000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_isp_sys>; + status = "disabled"; + }; + + isp: isp@384000000 { + compatible = "apple,t6000-isp", "apple,isp"; + iommus = <&isp_dart0 0>, <&isp_dart1 0>, <&isp_dart2 0>; + reg-names = "coproc", "mbox", "gpio", "mbox2"; + reg = <0x3 0x84000000 0x0 0x2000000>, + <0x3 0x86104000 0x0 0x100>, + <0x3 0x86104170 0x0 0x100>, + <0x3 0x861043f0 0x0 0x100>; + interrupt-parent = <&aic>; + interrupts = ; + #ifdef APPLE_USE_PMP + power-domains = <&pmp_report_isp_sys>, <&ps_isp_set0>, + #else + power-domains = <&ps_isp_sys>, <&ps_isp_set0>, + #endif + <&ps_isp_set1>, <&ps_isp_fe>, <&ps_isp_set3>, + <&ps_isp_set4>, <&ps_isp_set5>, <&ps_isp_set6>, + <&ps_isp_set7>, <&ps_isp_set8>; + apple,dart-vm-size = <0x0 0xa0000000>; + + status = "disabled"; + }; + pcie0_dart_0: iommu@581008000 { compatible = "apple,t6000-dart"; reg = <0x5 0x81008000 0x0 0x4000>; @@ -442,6 +931,8 @@ pinctrl-0 = <&pcie_pins>; pinctrl-names = "default"; + dma-coherent; + port00: pci@0,0 { device_type = "pci"; reg = <0x0 0x0 0x0 0x0 0x0>; diff --git a/arch/arm64/boot/dts/apple/t600x-dieX.dtsi b/arch/arm64/boot/dts/apple/t600x-dieX.dtsi index a32ff0c9d7b0c2..121d158ff1b3ea 100644 --- a/arch/arm64/boot/dts/apple/t600x-dieX.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-dieX.dtsi @@ -24,6 +24,168 @@ #performance-domain-cells = <0>; }; + DIE_NODE(dispext0_dart): iommu@289304000 { + compatible = "apple,t6000-dart"; + reg = <0x2 0x89304000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&DIE_NODE(ps_dispext0_cpu0)>; + apple,dma-range = <0x0 0x0 0x0 0xfc000000>; + status = "disabled"; + }; + + DIE_NODE(dcpext0_dart): iommu@28930c000 { + compatible = "apple,t6000-dart"; + reg = <0x2 0x8930c000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&DIE_NODE(ps_dispext0_cpu0)>; + apple,dma-range = <0x1f0 0x0 0x0 0xfc000000>; + status = "disabled"; + }; + + DIE_NODE(dcpext0_mbox): mbox@289c08000 { + compatible = "apple,t6000-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x2 0x89c08000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + power-domains = <&DIE_NODE(ps_dispext0_cpu0)>; + resets = <&DIE_NODE(ps_dispext0_cpu0)>; + status = "disabled"; + }; + + DIE_NODE(dcpext0): dcp@289c00000 { + compatible = "apple,t6000-dcpext", "apple,dcpext"; + mboxes = <&DIE_NODE(dcpext0_mbox)>; + mbox-names = "mbox"; + iommus = <&DIE_NODE(dcpext0_dart) 0>; + + reg-names = "coproc", "disp-0", "disp-1", "disp-2", "disp-3"; + reg = <0x2 0x89c00000 0x0 0x4000>, + <0x2 0x88000000 0x0 0x3000000>, + <0x2 0x89320000 0x0 0x4000>, + <0x2 0x89344000 0x0 0x4000>, + <0x2 0x89800000 0x0 0x800000>; + apple,bw-scratch = <&pmgr_dcp 0 4 0x990>; + #ifdef APPLE_USE_PMP + power-domains = <&DIE_NODE(pmp_report_dispext0)>; + #else + power-domains = <&DIE_NODE(ps_dispext0_cpu0)>; + #endif + resets = <&DIE_NODE(ps_dispext0_cpu0)>; + clocks = <&DIE_NODE(clk_dispext0)>; + phandle = <&DIE_NODE(dcpext0)>; + apple,dcp-index = <1>; + status = "disabled"; + // required bus properties for 'piodma' subdevice + #address-cells = <2>; + #size-cells = <2>; + + piodma { + iommus = <&DIE_NODE(dispext0_dart) 4>; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + DIE_NODE(dcpext0_audio): endpoint { + remote-endpoint = <&DIE_NODE(dpaudio1_dcp)>; + }; + }; + }; + }; + + DIE_NODE(dispext1_dart): iommu@28c304000 { + compatible = "apple,t6000-dart", "apple,t8110-dart"; + reg = <0x2 0x8c304000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&DIE_NODE(ps_dispext1_cpu0)>; + apple,dma-range = <0x0 0x0 0x0 0xfc000000>; + status = "disabled"; + }; + + DIE_NODE(dcpext1_dart): iommu@28c30c000 { + compatible = "apple,t6000-dart", "apple,t8110-dart"; + reg = <0x2 0x8c30c000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&DIE_NODE(ps_dispext1_cpu0)>; + apple,dma-range = <0x1f0 0x0 0x0 0xfc000000>; + status = "disabled"; + }; + + DIE_NODE(dcpext1_mbox): mbox@28cc08000 { + compatible = "apple,t6000-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x2 0x8cc08000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + power-domains = <&DIE_NODE(ps_dispext1_cpu0)>; + resets = <&DIE_NODE(ps_dispext1_cpu0)>; + status = "disabled"; + }; + + DIE_NODE(dcpext1): dcp@28cc00000 { + compatible = "apple,t6000-dcpext", "apple,dcpext"; + mboxes = <&DIE_NODE(dcpext1_mbox)>; + mbox-names = "mbox"; + iommus = <&DIE_NODE(dcpext1_dart) 0>; + + reg-names = "coproc", "disp-0", "disp-1", "disp-2", "disp-3"; + reg = <0x2 0x8cc00000 0x0 0x4000>, + <0x2 0x8b000000 0x0 0x3000000>, + <0x2 0x8c320000 0x0 0x4000>, + <0x2 0x8c344000 0x0 0x4000>, + <0x2 0x8c800000 0x0 0x800000>; + apple,bw-scratch = <&pmgr_dcp 0 4 0x998>; + #ifdef APPLE_USE_PMP + power-domains = <&DIE_NODE(pmp_report_dispext1)>; + #else + power-domains = <&DIE_NODE(ps_dispext1_cpu0)>; + #endif + resets = <&DIE_NODE(ps_dispext1_cpu0)>; + clocks = <&DIE_NODE(clk_dispext1)>; + phandle = <&DIE_NODE(dcpext1)>; + apple,dcp-index = <2>; + status = "disabled"; + // required bus properties for 'piodma' subdevice + #address-cells = <2>; + #size-cells = <2>; + + piodma { + iommus = <&DIE_NODE(dispext1_dart) 4>; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + DIE_NODE(dcpext1_audio): endpoint { + remote-endpoint = <&DIE_NODE(dpaudio2_dcp)>; + }; + }; + }; + }; + DIE_NODE(pmgr): power-management@28e080000 { compatible = "apple,t6000-pmgr", "apple,pmgr", "syscon", "simple-mfd"; #address-cells = <1>; @@ -95,6 +257,24 @@ ; }; + DIE_NODE(sio_dart_0): iommu@39b004000 { + compatible = "apple,t6000-dart"; + reg = <0x3 0x9b004000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + #iommu-cells = <1>; + power-domains = <&DIE_NODE(ps_sio_cpu)>; + }; + + DIE_NODE(sio_dart_1): iommu@39b008000 { + compatible = "apple,t6000-dart"; + reg = <0x3 0x9b008000 0x0 0x8000>; + interrupt-parent = <&aic>; + interrupts = ; + #iommu-cells = <1>; + power-domains = <&DIE_NODE(ps_sio_cpu)>; + }; + DIE_NODE(pinctrl_ap): pinctrl@39b028000 { compatible = "apple,t6000-pinctrl", "apple,pinctrl"; reg = <0x3 0x9b028000 0x0 0x4000>; @@ -119,3 +299,361 @@ interrupt-controller; #interrupt-cells = <2>; }; + + DIE_NODE(sio_mbox): mbox@39bc08000 { + compatible = "apple,t6000-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x3 0x9bc08000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + power-domains = <&DIE_NODE(ps_sio_cpu)>; + }; + + DIE_NODE(sio): sio@39bc00000 { + compatible = "apple,t6000-sio", "apple,sio"; + reg = <0x3 0x9bc00000 0x0 0x8000>; + dma-channels = <128>; + #dma-cells = <1>; + mboxes = <&DIE_NODE(sio_mbox)>; + iommus = <&DIE_NODE(sio_dart_0) 0>, <&DIE_NODE(sio_dart_1) 0>; + power-domains = <&DIE_NODE(ps_sio_cpu)>; + resets = <&DIE_NODE(ps_sio)>; /* TODO: verify reset does something */ + status = "disabled"; + }; + + DIE_NODE(dpaudio1): audio-controller@39b504000 { + compatible = "apple,t6000-dpaudio", "apple,dpaudio"; + reg = <0x3 0x9b540000 0x0 0x4000>; + dmas = <&DIE_NODE(sio) 0x66>; + dma-names = "tx"; + power-domains = <&DIE_NODE(ps_dpa1)>; + reset-domains = <&DIE_NODE(ps_dpa1)>; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + DIE_NODE(dpaudio1_dcp): endpoint { + remote-endpoint = <&DIE_NODE(dcpext0_audio)>; + }; + }; + }; + }; + + DIE_NODE(dpaudio2): audio-controller@39b508000 { + compatible = "apple,t6000-dpaudio", "apple,dpaudio"; + reg = <0x3 0x9b580000 0x0 0x4000>; + dmas = <&DIE_NODE(sio) 0x68>; + dma-names = "tx"; + power-domains = <&DIE_NODE(ps_dpa2)>; + reset-domains = <&DIE_NODE(ps_dpa2)>; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + DIE_NODE(dpaudio2_dcp): endpoint { + remote-endpoint = <&DIE_NODE(dcpext1_audio)>; + }; + }; + }; + }; + + /* + * omit dpaudio3 / 4 as long as the linked dcpext nodes don't exist + * + DIE_NODE(dpaudio3): audio-controller@39b50c000 { + compatible = "apple,t6000-dpaudio", "apple,dpaudio"; + reg = <0x3 0x9b5c0000 0x0 0x4000>; + dmas = <&DIE_NODE(sio) 0x6a>; + dma-names = "tx"; + power-domains = <&DIE_NODE(ps_dpa3)>; + reset-domains = <&DIE_NODE(ps_dpa3)>; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + DIE_NODE(dpaudio3_dcp): endpoint { + remote-endpoint = <&DIE_NODE(dcpext2_audio)>; + }; + }; + }; + }; + + DIE_NODE(dpaudio4): audio-controller@39b510000 { + compatible = "apple,t6000-dpaudio", "apple,dpaudio"; + reg = <0x3 0x9b500000 0x0 0x4000>; + dmas = <&DIE_NODE(sio) 0x6c>; + dma-names = "tx"; + power-domains = <&DIE_NODE(ps_dpa4)>; + reset-domains = <&DIE_NODE(ps_dpa4)>; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + DIE_NODE(dpaudio4_dcp): endpoint { + remote-endpoint = <&DIE_NODE(dcpext3_audio)>; + }; + }; + }; + }; + */ + + DIE_NODE(dwc3_0): usb@702280000 { + compatible = "apple,t6000-dwc3", "apple,t8103-dwc3"; + reg = <0x7 0x02280000 0x0 0xcd00>, <0x7 0x0228cd00 0x0 0x3200>; + reg-names = "dwc3-core", "dwc3-apple"; + interrupt-parent = <&aic>; + interrupts = ; + dr_mode = "otg"; + usb-role-switch; + role-switch-default-mode = "host"; + iommus = <&DIE_NODE(dwc3_0_dart_0) 0>, + <&DIE_NODE(dwc3_0_dart_1) 1>; + power-domains = <&DIE_NODE(ps_atc0_usb)>; + resets = <&DIE_NODE(atcphy0)>; + phys = <&DIE_NODE(atcphy0) PHY_TYPE_USB2>, <&DIE_NODE(atcphy0) PHY_TYPE_USB3>; + phy-names = "usb2-phy", "usb3-phy"; + }; + + DIE_NODE(dwc3_0_dart_0): iommu@702f00000 { + compatible = "apple,t6000-dart"; + reg = <0x7 0x02f00000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&DIE_NODE(ps_atc0_usb)>; + #iommu-cells = <1>; + }; + + DIE_NODE(dwc3_0_dart_1): iommu@702f80000 { + compatible = "apple,t6000-dart"; + reg = <0x7 0x02f80000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&DIE_NODE(ps_atc0_usb)>; + #iommu-cells = <1>; + }; + + DIE_NODE(atcphy0): phy@703000000 { + compatible = "apple,t6000-atcphy", "apple,t8103-atcphy"; + reg = <0x7 0x03000000 0x0 0x4c000>, + <0x7 0x03050000 0x0 0x8000>, + <0x7 0x00000000 0x0 0x4000>, + <0x7 0x02a90000 0x0 0x4000>, + <0x7 0x02a84000 0x0 0x4000>; + reg-names = "core", "lpdptx", "axi2af", "usb2phy", + "pipehandler"; + + #phy-cells = <1>; + #reset-cells = <0>; + + orientation-switch; + mode-switch; + power-domains = <&DIE_NODE(ps_atc0_usb)>; + }; + + DIE_NODE(atcphy0_xbar): mux@70304c000 { + compatible = "apple,t6000-display-crossbar"; + reg = <0x7 0x0304c000 0x0 0x4000>; + #mux-control-cells = <1>; + power-domains = <&DIE_NODE(ps_atc0_usb)>; + status = "disabled"; + }; + + DIE_NODE(dwc3_1): usb@b02280000 { + compatible = "apple,t6000-dwc3", "apple,t8103-dwc3"; + reg = <0xb 0x02280000 0x0 0xcd00>, <0xb 0x0228cd00 0x0 0x3200>; + reg-names = "dwc3-core", "dwc3-apple"; + interrupt-parent = <&aic>; + interrupts = ; + dr_mode = "otg"; + usb-role-switch; + role-switch-default-mode = "host"; + iommus = <&DIE_NODE(dwc3_1_dart_0) 0>, + <&DIE_NODE(dwc3_1_dart_1) 1>; + power-domains = <&DIE_NODE(ps_atc1_usb)>; + resets = <&DIE_NODE(atcphy1)>; + phys = <&DIE_NODE(atcphy1) PHY_TYPE_USB2>, <&DIE_NODE(atcphy1) PHY_TYPE_USB3>; + phy-names = "usb2-phy", "usb3-phy"; + }; + + DIE_NODE(dwc3_1_dart_0): iommu@b02f00000 { + compatible = "apple,t6000-dart"; + reg = <0xb 0x02f00000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&DIE_NODE(ps_atc1_usb)>; + #iommu-cells = <1>; + }; + + DIE_NODE(dwc3_1_dart_1): iommu@b02f80000 { + compatible = "apple,t6000-dart"; + reg = <0xb 0x02f80000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&DIE_NODE(ps_atc1_usb)>; + #iommu-cells = <1>; + }; + + DIE_NODE(atcphy1): phy@b03000000 { + compatible = "apple,t6000-atcphy", "apple,t8103-atcphy"; + reg = <0xb 0x03000000 0x0 0x4c000>, + <0xb 0x03050000 0x0 0x8000>, + <0xb 0x00000000 0x0 0x4000>, + <0xb 0x02a90000 0x0 0x4000>, + <0xb 0x02a84000 0x0 0x4000>; + reg-names = "core", "lpdptx", "axi2af", "usb2phy", + "pipehandler"; + + #phy-cells = <1>; + #reset-cells = <0>; + + orientation-switch; + mode-switch; + power-domains = <&DIE_NODE(ps_atc1_usb)>; + }; + + DIE_NODE(atcphy1_xbar): mux@b0304c000 { + compatible = "apple,t6000-display-crossbar"; + reg = <0xb 0x0304c000 0x0 0x4000>; + #mux-control-cells = <1>; + power-domains = <&DIE_NODE(ps_atc1_usb)>; + status = "disabled"; + }; + + DIE_NODE(dwc3_2): usb@f02280000 { + compatible = "apple,t6000-dwc3", "apple,t8103-dwc3"; + reg = <0xf 0x02280000 0x0 0xcd00>, <0xf 0x0228cd00 0x0 0x3200>; + reg-names = "dwc3-core", "dwc3-apple"; + interrupt-parent = <&aic>; + interrupts = ; + dr_mode = "otg"; + usb-role-switch; + role-switch-default-mode = "host"; + iommus = <&DIE_NODE(dwc3_2_dart_0) 0>, + <&DIE_NODE(dwc3_2_dart_1) 1>; + power-domains = <&DIE_NODE(ps_atc2_usb)>; + resets = <&DIE_NODE(atcphy2)>; + phys = <&DIE_NODE(atcphy2) PHY_TYPE_USB2>, <&DIE_NODE(atcphy2) PHY_TYPE_USB3>; + phy-names = "usb2-phy", "usb3-phy"; + }; + + DIE_NODE(dwc3_2_dart_0): iommu@f02f00000 { + compatible = "apple,t6000-dart"; + reg = <0xf 0x02f00000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&DIE_NODE(ps_atc2_usb)>; + #iommu-cells = <1>; + }; + + DIE_NODE(dwc3_2_dart_1): iommu@f02f80000 { + compatible = "apple,t6000-dart"; + reg = <0xf 0x02f80000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&DIE_NODE(ps_atc2_usb)>; + #iommu-cells = <1>; + }; + + DIE_NODE(atcphy2): phy@f03000000 { + compatible = "apple,t6000-atcphy", "apple,t8103-atcphy"; + reg = <0xf 0x03000000 0x0 0x4c000>, + <0xf 0x03050000 0x0 0x8000>, + <0xf 0x00000000 0x0 0x4000>, + <0xf 0x02a90000 0x0 0x4000>, + <0xf 0x02a84000 0x0 0x4000>; + reg-names = "core", "lpdptx", "axi2af", "usb2phy", + "pipehandler"; + + #phy-cells = <1>; + #reset-cells = <0>; + + orientation-switch; + mode-switch; + power-domains = <&DIE_NODE(ps_atc2_usb)>; + }; + + DIE_NODE(atcphy2_xbar): mux@f0304c000 { + compatible = "apple,t6000-display-crossbar"; + reg = <0xf 0x0304c000 0x0 0x4000>; + #mux-control-cells = <1>; + power-domains = <&DIE_NODE(ps_atc2_usb)>; + status = "disabled"; + }; + + DIE_NODE(dwc3_3): usb@1302280000 { + compatible = "apple,t6000-dwc3", "apple,t8103-dwc3"; + reg = <0x13 0x02280000 0x0 0xcd00>, <0x13 0x0228cd00 0x0 0x3200>; + reg-names = "dwc3-core", "dwc3-apple"; + interrupt-parent = <&aic>; + interrupts = ; + dr_mode = "otg"; + usb-role-switch; + role-switch-default-mode = "host"; + iommus = <&DIE_NODE(dwc3_3_dart_0) 0>, + <&DIE_NODE(dwc3_3_dart_1) 1>; + power-domains = <&DIE_NODE(ps_atc3_usb)>; + resets = <&DIE_NODE(atcphy3)>; + phys = <&DIE_NODE(atcphy3) PHY_TYPE_USB2>, <&DIE_NODE(atcphy3) PHY_TYPE_USB3>; + phy-names = "usb2-phy", "usb3-phy"; + }; + + DIE_NODE(dwc3_3_dart_0): iommu@1302f00000 { + compatible = "apple,t6000-dart"; + reg = <0x13 0x02f00000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&DIE_NODE(ps_atc3_usb)>; + #iommu-cells = <1>; + }; + + DIE_NODE(dwc3_3_dart_1): iommu@1302f80000 { + compatible = "apple,t6000-dart"; + reg = <0x13 0x02f80000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&DIE_NODE(ps_atc3_usb)>; + #iommu-cells = <1>; + }; + + DIE_NODE(atcphy3): phy@1303000000 { + compatible = "apple,t6000-atcphy", "apple,t8103-atcphy"; + reg = <0x13 0x03000000 0x0 0x4c000>, + <0x13 0x03050000 0x0 0x8000>, + <0x13 0x00000000 0x0 0x4000>, + <0x13 0x02a90000 0x0 0x4000>, + <0x13 0x02a84000 0x0 0x4000>; + reg-names = "core", "lpdptx", "axi2af", "usb2phy", + "pipehandler"; + + #phy-cells = <1>; + #reset-cells = <0>; + + orientation-switch; + mode-switch; + power-domains = <&DIE_NODE(ps_atc3_usb)>; + }; + + DIE_NODE(atcphy3_xbar): mux@130304c000 { + compatible = "apple,t6000-display-crossbar"; + reg = <0x13 0x0304c000 0x0 0x4000>; + #mux-control-cells = <1>; + power-domains = <&DIE_NODE(ps_atc3_usb)>; + status = "disabled"; + }; diff --git a/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi b/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi index c0aac59a6fae4f..cea603e5c2d9a0 100644 --- a/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi @@ -12,9 +12,20 @@ #include / { + chassis-type = "laptop"; + aliases { + atcphy0 = &atcphy0; + atcphy1 = &atcphy1; + atcphy2 = &atcphy2; + atcphy3 = &atcphy3; bluetooth0 = &bluetooth0; + dcp = &dcp; + dcpext0 = &dcpext0; + disp0 = &display; + disp0_piodma = &disp0_piodma; serial0 = &serial0; + sio = &sio; wifi0 = &wifi0; }; @@ -30,6 +41,9 @@ reg = <0 0 0 0>; /* To be filled by loader */ /* Format properties will be added by loader */ status = "disabled"; + panel = <&panel>; + post-init-providers = <&panel>; + power-domains = <&ps_disp0_cpu0>; }; }; @@ -55,6 +69,76 @@ status = "okay"; }; +&dcp { + panel: panel { + apple,max-brightness = <500>; + }; +}; + +&display { + iommus = <&disp0_dart 0>, <&dispext0_dart 0>; +}; + +&dispext0_dart { + status = "okay"; +}; + +&dcpext0_dart { + status = "okay"; +}; + +&dcpext0_mbox { + status = "okay"; +}; + +&dcpext0 { + /* enabled by the loader */ + apple,connector-type = "HDMI-A"; + + /* HDMI HPD gpio, used as interrupt*/ + hdmi-hpd-gpios = <&pinctrl_nub 15 GPIO_ACTIVE_HIGH>; + + hdmi-pwren-gpios = <&smc_gpio 23 GPIO_ACTIVE_HIGH>; + dp2hdmi-pwren-gpios = <&smc_gpio 6 GPIO_ACTIVE_HIGH>; + + phys = <&atcphy3 PHY_TYPE_DP>; + phy-names = "dp-phy"; + mux-controls = <&atcphy3_xbar 0>; + mux-control-names = "dp-xbar"; + mux-index = <0>; + apple,dptx-phy = <3>; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@1 { + reg = <1>; + + decpext0_dpout: endpoint { + remote-endpoint = <&atcphy3_dp>; + }; + }; + }; +}; + +/* remove once m1n1 enables sio nodes after setup */ +&sio { + status = "okay"; +}; + +&dpaudio1 { + status = "okay"; +}; + +&atcphy3_xbar { + status = "okay"; +}; + +&ps_atc3_common { + apple,always-on; /* Needs to stay on for HDMI resume */ +}; + /* USB Type C */ &i2c0 { hpm0: usb-pd@38 { @@ -63,6 +147,30 @@ interrupt-parent = <&pinctrl_ap>; interrupts = <174 IRQ_TYPE_LEVEL_LOW>; interrupt-names = "irq"; + + typec0: connector { + compatible = "usb-c-connector"; + label = "USB-C Left Rear"; + power-role = "dual"; + data-role = "dual"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + typec0_connector_hs: endpoint { + remote-endpoint = <&dwc3_0_hs>; + }; + }; + port@1 { + reg = <1>; + typec0_connector_ss: endpoint { + remote-endpoint = <&atcphy0_typec_lanes>; + }; + }; + }; + }; }; hpm1: usb-pd@3f { @@ -71,6 +179,30 @@ interrupt-parent = <&pinctrl_ap>; interrupts = <174 IRQ_TYPE_LEVEL_LOW>; interrupt-names = "irq"; + + typec1: connector { + compatible = "usb-c-connector"; + label = "USB-C Left Front"; + power-role = "dual"; + data-role = "dual"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + typec1_connector_hs: endpoint { + remote-endpoint = <&dwc3_1_hs>; + }; + }; + port@1 { + reg = <1>; + typec1_connector_ss: endpoint { + remote-endpoint = <&atcphy1_typec_lanes>; + }; + }; + }; + }; }; hpm2: usb-pd@3b { @@ -79,6 +211,30 @@ interrupt-parent = <&pinctrl_ap>; interrupts = <174 IRQ_TYPE_LEVEL_LOW>; interrupt-names = "irq"; + + typec2: connector { + compatible = "usb-c-connector"; + label = "USB-C Right"; + power-role = "dual"; + data-role = "dual"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + typec2_connector_hs: endpoint { + remote-endpoint = <&dwc3_2_hs>; + }; + }; + port@1 { + reg = <1>; + typec2_connector_ss: endpoint { + remote-endpoint = <&atcphy2_typec_lanes>; + }; + }; + }; + }; }; /* MagSafe port */ @@ -91,14 +247,138 @@ }; }; +/* Virtual regulator representing the shared shutdown GPIO */ +/ { + speaker_sdz: fixed-regulator-sn012776-sdz { + compatible = "regulator-fixed"; + regulator-name = "sn012776-sdz"; + startup-delay-us = <5000>; + gpios = <&pinctrl_ap 178 GPIO_ACTIVE_HIGH>; + enable-active-high; + }; +}; + +&i2c1 { + status = "okay"; + + speaker_left_tweet: codec@3a { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x3a>; + SDZ-supply = <&speaker_sdz>; + #sound-dai-cells = <0>; + sound-name-prefix = "Left Tweeter"; + interrupts-extended = <&pinctrl_ap 179 IRQ_TYPE_LEVEL_LOW>; + ti,imon-slot-no = <8>; + ti,vmon-slot-no = <10>; + }; + + speaker_left_woof1: codec@38 { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x38>; + SDZ-supply = <&speaker_sdz>; + #sound-dai-cells = <0>; + sound-name-prefix = "Left Woofer 1"; + interrupts-extended = <&pinctrl_ap 179 IRQ_TYPE_LEVEL_LOW>; + ti,imon-slot-no = <0>; + ti,vmon-slot-no = <2>; + ti,sdout-force-zero-mask = <0xf0f0f0>; + }; + + speaker_left_woof2: codec@39 { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x39>; + SDZ-supply = <&speaker_sdz>; + #sound-dai-cells = <0>; + sound-name-prefix = "Left Woofer 2"; + interrupts-extended = <&pinctrl_ap 179 IRQ_TYPE_LEVEL_LOW>; + ti,imon-slot-no = <16>; + ti,vmon-slot-no = <18>; + }; +}; + +&i2c2 { + status = "okay"; + + jack_codec: codec@4b { + compatible = "cirrus,cs42l84"; + reg = <0x4b>; + reset-gpios = <&pinctrl_nub 4 GPIO_ACTIVE_HIGH>; + #sound-dai-cells = <0>; + interrupts-extended = <&pinctrl_ap 180 IRQ_TYPE_LEVEL_LOW>; + sound-name-prefix = "Jack"; + }; +}; + +&i2c3 { + status = "okay"; + + speaker_right_tweet: codec@3d { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x3d>; + SDZ-supply = <&speaker_sdz>; + #sound-dai-cells = <0>; + sound-name-prefix = "Right Tweeter"; + interrupts-extended = <&pinctrl_ap 179 IRQ_TYPE_LEVEL_LOW>; + ti,imon-slot-no = <12>; + ti,vmon-slot-no = <14>; + }; + + speaker_right_woof1: codec@3b { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x3b>; + SDZ-supply = <&speaker_sdz>; + #sound-dai-cells = <0>; + sound-name-prefix = "Right Woofer 1"; + interrupts-extended = <&pinctrl_ap 179 IRQ_TYPE_LEVEL_LOW>; + ti,imon-slot-no = <4>; + ti,vmon-slot-no = <6>; + ti,sdout-force-zero-mask = <0x0f0f0f>; + }; + + speaker_right_woof2: codec@3c { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x3c>; + SDZ-supply = <&speaker_sdz>; + #sound-dai-cells = <0>; + sound-name-prefix = "Right Woofer 2"; + interrupts-extended = <&pinctrl_ap 179 IRQ_TYPE_LEVEL_LOW>; + ti,imon-slot-no = <20>; + ti,vmon-slot-no = <22>; + }; +}; + &nco_clkref { clock-frequency = <1068000000>; }; +#ifndef NO_SPI_TRACKPAD +&spi3 { + status = "okay"; + + hid-transport@0 { + compatible = "apple,spi-hid-transport"; + reg = <0>; + spi-max-frequency = <8000000>; + /* + * Apple's ADT specifies 20us CS change delays, and the + * SPI HID interface metadata specifies 45us. Using either + * seems not to be reliable, but adding both works, so + * best guess is they are cumulative. + */ + spi-cs-setup-delay-ns = <65000>; + spi-cs-hold-delay-ns = <65000>; + spi-cs-inactive-delay-ns = <250000>; + spien-gpios = <&pinctrl_ap 194 0>; + interrupts-extended = <&pinctrl_nub 6 IRQ_TYPE_LEVEL_LOW>; + }; +}; +#endif + /* PCIe devices */ &port00 { /* WLAN */ bus-range = <1 1>; + pwren-gpios = <&smc_gpio 13 GPIO_ACTIVE_HIGH>; wifi0: wifi@0,0 { compatible = "pci14e4,4433"; reg = <0x10000 0x0 0x0 0x0 0x0>; @@ -118,6 +398,8 @@ &port01 { /* SD card reader */ bus-range = <2 2>; + pwren-gpios = <&smc_gpio 26 GPIO_ACTIVE_HIGH>; + status = "okay"; sdhci0: mmc@0,0 { compatible = "pci17a0,9755"; reg = <0x20000 0x0 0x0 0x0 0x0>; @@ -130,4 +412,240 @@ status = "okay"; }; +&pcie0_dart_1 { + status = "okay"; +}; + +/* USB controllers */ +&dwc3_0 { + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + dwc3_0_hs: endpoint { + remote-endpoint = <&typec0_connector_hs>; + }; + }; + + port@1 { + reg = <1>; + dwc3_0_ss: endpoint { + remote-endpoint = <&atcphy0_usb3>; + }; + }; + }; +}; + +&dwc3_1 { + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + dwc3_1_hs: endpoint { + remote-endpoint = <&typec1_connector_hs>; + }; + }; + + port@1 { + reg = <1>; + dwc3_1_ss: endpoint { + remote-endpoint = <&atcphy1_usb3>; + }; + }; + }; +}; + +&dwc3_2 { + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + dwc3_2_hs: endpoint { + remote-endpoint = <&typec2_connector_hs>; + }; + }; + + port@1 { + reg = <1>; + dwc3_2_ss: endpoint { + remote-endpoint = <&atcphy2_usb3>; + }; + }; + }; +}; + +/* + * ps_atc3_usb_aon power-domain is always-on to keep dwc3 working over suspend. + * atc3 is used exclusively for the DP-to-HDMI so do not keep this always on. + * On t600x it is required to keep atc DP state over suspend. + */ +// &ps_atc3_usb_aon { +// /delete-property/ apple,always-on; +// }; + +/* ATC3 is used for DisplayPort -> HDMI only */ +&dwc3_3_dart_0 { + status = "disabled"; +}; + +&dwc3_3_dart_1 { + status = "disabled"; +}; + +&dwc3_3 { + status = "disabled"; +}; + +/* Delete unused dwc3_3 to prevent dt_disable_missing_devs() from disabling + * atcphy3 via phandle references from a disablecd device. + */ +/delete-node/ &dwc3_3; + +/* Type-C PHYs */ +&atcphy0 { + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + atcphy0_typec_lanes: endpoint { + remote-endpoint = <&typec0_connector_ss>; + }; + }; + + port@1 { + reg = <1>; + atcphy0_usb3: endpoint { + remote-endpoint = <&dwc3_0_ss>; + }; + }; + }; +}; + +&atcphy1 { + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + atcphy1_typec_lanes: endpoint { + remote-endpoint = <&typec1_connector_ss>; + }; + }; + + port@1 { + reg = <1>; + atcphy1_usb3: endpoint { + remote-endpoint = <&dwc3_1_ss>; + }; + }; + }; +}; + +&atcphy2 { + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + atcphy2_typec_lanes: endpoint { + remote-endpoint = <&typec2_connector_ss>; + }; + }; + + port@1 { + reg = <1>; + atcphy2_usb3: endpoint { + remote-endpoint = <&dwc3_2_ss>; + }; + }; + }; +}; + +&atcphy3 { + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@2 { + reg = <2>; + atcphy3_dp: endpoint { + remote-endpoint = <&decpext0_dpout>; + }; + }; + }; +}; + +&aop_mbox { + status = "okay"; +}; + +&aop_dart { + status = "okay"; +}; + +&aop_admac { + status = "okay"; +}; + +&aop { + status = "okay"; +}; + +/ { + sound: sound { + /* compatible is set per machine */ + + dai-link@0 { + link-name = "Speakers"; + + cpu { + sound-dai = <&mca 0>, <&mca 1>; + }; + codec { + sound-dai = <&speaker_left_woof1>, + <&speaker_left_tweet>, + <&speaker_left_woof2>, + <&speaker_right_woof1>, + <&speaker_right_tweet>, + <&speaker_right_woof2>; + }; + }; + + dai-link@1 { + link-name = "Headphone Jack"; + + cpu { + sound-dai = <&mca 2>; + }; + codec { + sound-dai = <&jack_codec>; + }; + }; + }; +}; + #include "spi1-nvram.dtsi" + +#include "isp-imx558.dtsi" + +&isp { + apple,platform-id = <3>; +}; + +&pmp_report_isp_sys { + status = "okay"; +}; + +#include "hwmon-common.dtsi" +#include "hwmon-fan-dual.dtsi" +#include "hwmon-laptop.dtsi" diff --git a/arch/arm64/boot/dts/apple/t600x-j375.dtsi b/arch/arm64/boot/dts/apple/t600x-j375.dtsi index c0fb93ae72f4d4..ce339c0855bc98 100644 --- a/arch/arm64/boot/dts/apple/t600x-j375.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-j375.dtsi @@ -10,10 +10,22 @@ */ / { + chassis-type = "desktop"; + aliases { + atcphy0 = &atcphy0; + atcphy1 = &atcphy1; + atcphy2 = &atcphy2; + atcphy3 = &atcphy3; bluetooth0 = &bluetooth0; + #ifndef NO_DCP + dcp = &dcp; + disp0 = &display; + disp0_piodma = &disp0_piodma; + #endif ethernet0 = ðernet0; serial0 = &serial0; + sio = &sio; wifi0 = &wifi0; }; @@ -29,6 +41,7 @@ reg = <0 0 0 0>; /* To be filled by loader */ /* Format properties will be added by loader */ status = "disabled"; + power-domains = <&ps_disp0_cpu0>; }; }; @@ -42,6 +55,15 @@ status = "okay"; }; +&dcp { + apple,connector-type = "HDMI-A"; +}; + +/* remove once m1n1 enables sio nodes after setup */ +&sio { + status = "okay"; +}; + /* USB Type C */ &i2c0 { hpm0: usb-pd@38 { @@ -50,6 +72,30 @@ interrupt-parent = <&pinctrl_ap>; interrupts = <174 IRQ_TYPE_LEVEL_LOW>; interrupt-names = "irq"; + + typec0: connector { + compatible = "usb-c-connector"; + label = "USB-C Back Left"; + power-role = "dual"; + data-role = "dual"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + typec0_connector_hs: endpoint { + remote-endpoint = <&dwc3_0_hs>; + }; + }; + port@1 { + reg = <1>; + typec0_connector_ss: endpoint { + remote-endpoint = <&atcphy0_typec_lanes>; + }; + }; + }; + }; }; hpm1: usb-pd@3f { @@ -58,6 +104,30 @@ interrupt-parent = <&pinctrl_ap>; interrupts = <174 IRQ_TYPE_LEVEL_LOW>; interrupt-names = "irq"; + + typec1: connector { + compatible = "usb-c-connector"; + label = "USB-C Back Left Middle"; + power-role = "dual"; + data-role = "dual"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + typec1_connector_hs: endpoint { + remote-endpoint = <&dwc3_1_hs>; + }; + }; + port@1 { + reg = <1>; + typec1_connector_ss: endpoint { + remote-endpoint = <&atcphy1_typec_lanes>; + }; + }; + }; + }; }; hpm2: usb-pd@3b { @@ -66,6 +136,30 @@ interrupt-parent = <&pinctrl_ap>; interrupts = <174 IRQ_TYPE_LEVEL_LOW>; interrupt-names = "irq"; + + typec2: connector { + compatible = "usb-c-connector"; + label = "USB-C Back Right Middle"; + power-role = "dual"; + data-role = "dual"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + typec2_connector_hs: endpoint { + remote-endpoint = <&dwc3_2_hs>; + }; + }; + port@1 { + reg = <1>; + typec2_connector_ss: endpoint { + remote-endpoint = <&atcphy2_typec_lanes>; + }; + }; + }; + }; }; hpm3: usb-pd@3c { @@ -74,6 +168,228 @@ interrupt-parent = <&pinctrl_ap>; interrupts = <174 IRQ_TYPE_LEVEL_LOW>; interrupt-names = "irq"; + + typec3: connector { + compatible = "usb-c-connector"; + label = "USB-C Back Right"; + power-role = "dual"; + data-role = "dual"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + typec3_connector_hs: endpoint { + remote-endpoint = <&dwc3_3_hs>; + }; + }; + port@1 { + reg = <1>; + typec3_connector_ss: endpoint { + remote-endpoint = <&atcphy3_typec_lanes>; + }; + }; + }; + }; + }; +}; + +/* USB controllers */ +&dwc3_0 { + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + dwc3_0_hs: endpoint { + remote-endpoint = <&typec0_connector_hs>; + }; + }; + + port@1 { + reg = <1>; + dwc3_0_ss: endpoint { + remote-endpoint = <&atcphy0_usb3>; + }; + }; + }; +}; + +&dwc3_1 { + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + dwc3_1_hs: endpoint { + remote-endpoint = <&typec1_connector_hs>; + }; + }; + + port@1 { + reg = <1>; + dwc3_1_ss: endpoint { + remote-endpoint = <&atcphy1_usb3>; + }; + }; + }; +}; + +&dwc3_2 { + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + dwc3_2_hs: endpoint { + remote-endpoint = <&typec2_connector_hs>; + }; + }; + + port@1 { + reg = <1>; + dwc3_2_ss: endpoint { + remote-endpoint = <&atcphy2_usb3>; + }; + }; + }; +}; + +&dwc3_3 { + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + dwc3_3_hs: endpoint { + remote-endpoint = <&typec3_connector_hs>; + }; + }; + + port@1 { + reg = <1>; + dwc3_3_ss: endpoint { + remote-endpoint = <&atcphy3_usb3>; + }; + }; + }; +}; + +/* Type-C PHYs */ +&atcphy0 { + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + atcphy0_typec_lanes: endpoint { + remote-endpoint = <&typec0_connector_ss>; + }; + }; + + port@1 { + reg = <1>; + atcphy0_usb3: endpoint { + remote-endpoint = <&dwc3_0_ss>; + }; + }; + }; +}; + +&atcphy1 { + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + atcphy1_typec_lanes: endpoint { + remote-endpoint = <&typec1_connector_ss>; + }; + }; + + port@1 { + reg = <1>; + atcphy1_usb3: endpoint { + remote-endpoint = <&dwc3_1_ss>; + }; + }; + }; +}; + +&atcphy2 { + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + atcphy2_typec_lanes: endpoint { + remote-endpoint = <&typec2_connector_ss>; + }; + }; + + port@1 { + reg = <1>; + atcphy2_usb3: endpoint { + remote-endpoint = <&dwc3_2_ss>; + }; + }; + }; +}; + +&atcphy3 { + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + atcphy3_typec_lanes: endpoint { + remote-endpoint = <&typec3_connector_ss>; + }; + }; + + port@1 { + reg = <1>; + atcphy3_usb3: endpoint { + remote-endpoint = <&dwc3_3_ss>; + }; + }; + }; +}; + +/* Audio */ +&i2c1 { + status = "okay"; + + speaker: codec@38 { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x38>; + shutdown-gpios = <&pinctrl_ap 178 GPIO_ACTIVE_HIGH>; + #sound-dai-cells = <0>; + interrupts-extended = <&pinctrl_ap 179 IRQ_TYPE_LEVEL_LOW>; + ti,imon-slot-no = <0>; + ti,vmon-slot-no = <2>; + }; +}; + +&i2c2 { + status = "okay"; + + jack_codec: codec@4b { + compatible = "cirrus,cs42l84"; + reg = <0x4b>; + reset-gpios = <&pinctrl_nub 4 GPIO_ACTIVE_HIGH>; + #sound-dai-cells = <0>; + interrupts-extended = <&pinctrl_ap 180 IRQ_TYPE_LEVEL_LOW>; + sound-name-prefix = "Jack"; }; }; @@ -81,10 +397,39 @@ clock-frequency = <1068000000>; }; +/ { + sound: sound { + /* compatible is set per machine */ + + dai-link@0 { + link-name = "Speaker"; + + cpu { + sound-dai = <&mca 0>; + }; + codec { + sound-dai = <&speaker>; + }; + }; + + dai-link@1 { + link-name = "Headphone Jack"; + + cpu { + sound-dai = <&mca 2>; + }; + codec { + sound-dai = <&jack_codec>; + }; + }; + }; +}; + /* PCIe devices */ &port00 { /* WLAN */ bus-range = <1 1>; + pwren-gpios = <&smc_gpio 13 GPIO_ACTIVE_HIGH>; wifi0: wifi@0,0 { compatible = "pci14e4,4433"; reg = <0x10000 0x0 0x0 0x0 0x0>; @@ -104,6 +449,7 @@ &port01 { /* SD card reader */ bus-range = <2 2>; + pwren-gpios = <&smc_gpio 26 GPIO_ACTIVE_HIGH>; sdhci0: mmc@0,0 { compatible = "pci17a0,9755"; reg = <0x20000 0x0 0x0 0x0 0x0>; @@ -126,6 +472,7 @@ &port03 { /* USB xHCI */ bus-range = <4 4>; + pwren-gpios = <&smc_gpio 20 GPIO_ACTIVE_HIGH>; status = "okay"; }; @@ -139,3 +486,5 @@ }; #include "spi1-nvram.dtsi" + +#include "hwmon-common.dtsi" diff --git a/arch/arm64/boot/dts/apple/t600x-pmgr.dtsi b/arch/arm64/boot/dts/apple/t600x-pmgr.dtsi index 0bd44753b76a0c..833a59ecf37922 100644 --- a/arch/arm64/boot/dts/apple/t600x-pmgr.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-pmgr.dtsi @@ -396,6 +396,7 @@ #reset-cells = <0>; label = DIE_LABEL(dispext0_cpu0); power-domains = <&DIE_NODE(ps_dispext0_fe)>; + apple,min-state = <4>; }; DIE_NODE(ps_dispext1_cpu0): power-controller@2a8 { @@ -405,6 +406,7 @@ #reset-cells = <0>; label = DIE_LABEL(dispext1_cpu0); power-domains = <&DIE_NODE(ps_dispext1_fe)>; + apple,min-state = <4>; }; DIE_NODE(ps_ane_sys_cpu): power-controller@2c8 { @@ -424,6 +426,7 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = DIE_LABEL(pmp); + apple,always-on; }; #endif @@ -433,6 +436,7 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = DIE_LABEL(pms_sram); + apple,always-on; }; DIE_NODE(ps_apcie_st_sys): power-controller@2e8 { @@ -824,7 +828,7 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = DIE_LABEL(sio_cpu); - power-domains = <&DIE_NODE(ps_sio)>; + power-domains = <&DIE_NODE(ps_sio) &DIE_NODE(ps_uart_p) &DIE_NODE(ps_spi_p) &DIE_NODE(ps_audio_p)>; }; DIE_NODE(ps_fpwm0): power-controller@190 { @@ -1113,6 +1117,7 @@ #reset-cells = <0>; label = DIE_LABEL(mca0); power-domains = <&DIE_NODE(ps_audio_p)>, <&DIE_NODE(ps_sio_adma)>; + apple,externally-clocked; }; DIE_NODE(ps_mca1): power-controller@290 { @@ -1122,6 +1127,7 @@ #reset-cells = <0>; label = DIE_LABEL(mca1); power-domains = <&DIE_NODE(ps_audio_p)>, <&DIE_NODE(ps_sio_adma)>; + apple,externally-clocked; }; DIE_NODE(ps_mca2): power-controller@298 { @@ -1131,6 +1137,7 @@ #reset-cells = <0>; label = DIE_LABEL(mca2); power-domains = <&DIE_NODE(ps_audio_p)>, <&DIE_NODE(ps_sio_adma)>; + apple,externally-clocked; }; DIE_NODE(ps_mca3): power-controller@2a0 { @@ -1140,6 +1147,7 @@ #reset-cells = <0>; label = DIE_LABEL(mca3); power-domains = <&DIE_NODE(ps_audio_p)>, <&DIE_NODE(ps_sio_adma)>; + apple,externally-clocked; }; DIE_NODE(ps_dpa0): power-controller@2a8 { @@ -1293,7 +1301,6 @@ #reset-cells = <0>; label = DIE_LABEL(disp0_fe); power-domains = <&DIE_NODE(ps_afnc2_lw0)>; - apple,always-on; /* TODO: figure out if we can enable PM here */ }; DIE_NODE(ps_disp0_cpu0): power-controller@350 { @@ -1303,7 +1310,6 @@ #reset-cells = <0>; label = DIE_LABEL(disp0_cpu0); power-domains = <&DIE_NODE(ps_disp0_fe)>; - apple,always-on; /* TODO: figure out if we can enable PM here */ apple,min-state = <4>; }; @@ -1368,6 +1374,7 @@ #reset-cells = <0>; label = DIE_LABEL(isp_sys); power-domains = <&DIE_NODE(ps_afnc2_lw1)>; + status = "disabled"; }; DIE_NODE(ps_venc_sys): power-controller@3b0 { @@ -1385,12 +1392,6 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = DIE_LABEL(ans2); - /* - * The ADT makes ps_apcie_st[1]_sys depend on ps_ans2 instead, - * but we'd rather have a single power domain for the downstream - * device to depend on, so use this node as the child. - * This makes more sense anyway (since ANS2 uses APCIE_ST). - */ power-domains = <&DIE_NODE(ps_afnc2_lw0)>; }; @@ -1456,6 +1457,86 @@ label = DIE_LABEL(venc_me1); power-domains = <&DIE_NODE(ps_venc_me0)>; }; + + /* There is a dependency tree involved with these PDs, + * but we do not express it here since the ISP driver + * is supposed to sequence them in the right order anyway + * (and we do not know the exact tree structure). + * + * This also works around spurious parent PD activation + * on machines with ISP disabled (desktops). + */ + DIE_NODE(ps_isp_set0): power-controller@4000 { + compatible = "apple,t6000-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4000 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(isp_set0); + }; + + DIE_NODE(ps_isp_set1): power-controller@4010 { + compatible = "apple,t6000-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4010 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(isp_set1); + }; + + DIE_NODE(ps_isp_fe): power-controller@4008 { + compatible = "apple,t6000-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4008 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(ps_isp_fe); + }; + + DIE_NODE(ps_isp_set3): power-controller@4028 { + compatible = "apple,t6000-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4028 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(isp_set3); + }; + + DIE_NODE(ps_isp_set4): power-controller@4020 { + compatible = "apple,t6000-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4020 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(isp_set4); + }; + + DIE_NODE(ps_isp_set5): power-controller@4030 { + compatible = "apple,t6000-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4030 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(isp_set5); + }; + + DIE_NODE(ps_isp_set6): power-controller@4018 { + compatible = "apple,t6000-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4018 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(isp_set6); + }; + + DIE_NODE(ps_isp_set7): power-controller@4038 { + compatible = "apple,t6000-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4038 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(isp_set7); + }; + + DIE_NODE(ps_isp_set8): power-controller@4040 { + compatible = "apple,t6000-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4040 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(isp_set8); + }; }; &DIE_NODE(pmgr_south) { @@ -1715,6 +1796,7 @@ #reset-cells = <0>; label = DIE_LABEL(dispext2_cpu0); power-domains = <&DIE_NODE(ps_dispext2_fe)>; + apple,min-state = <4>; }; DIE_NODE(ps_dispext3_fe): power-controller@210 { @@ -1733,6 +1815,7 @@ #reset-cells = <0>; label = DIE_LABEL(dispext3_cpu0); power-domains = <&DIE_NODE(ps_dispext3_fe)>; + apple,min-state = <4>; }; DIE_NODE(ps_msr1): power-controller@250 { @@ -1881,6 +1964,7 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = DIE_LABEL(msg); + apple,always-on; /* Core AON device? */ }; DIE_NODE(ps_nub_gpio): power-controller@80 { diff --git a/arch/arm64/boot/dts/apple/t6020-j414s.dts b/arch/arm64/boot/dts/apple/t6020-j414s.dts index 631c54c5f03dee..a227069727dd8f 100644 --- a/arch/arm64/boot/dts/apple/t6020-j414s.dts +++ b/arch/arm64/boot/dts/apple/t6020-j414s.dts @@ -24,3 +24,24 @@ &bluetooth0 { brcm,board-type = "apple,tokara"; }; + +&panel { + compatible = "apple,panel-j414", "apple,panel-mini-led", "apple,panel"; + width-mm = <302>; + height-mm = <196>; + adj-height-mm = <189>; +}; + +&aop_audio { + apple,chassis-name = "J414"; + apple,machine-kind = "MacBook Pro"; +}; + +&sound { + compatible = "apple,j414-macaudio", "apple,j314-macaudio", "apple,macaudio"; + model = "MacBook Pro J414"; +}; + +&mtp_mt { + firmware-name = "apple/tpmtfw-j414s.bin"; +}; diff --git a/arch/arm64/boot/dts/apple/t6020-j416s.dts b/arch/arm64/boot/dts/apple/t6020-j416s.dts index c277ed5889a214..3ea2b1d52593e2 100644 --- a/arch/arm64/boot/dts/apple/t6020-j416s.dts +++ b/arch/arm64/boot/dts/apple/t6020-j416s.dts @@ -24,3 +24,24 @@ &bluetooth0 { brcm,board-type = "apple,amami"; }; + +&panel { + compatible = "apple,panel-j416", "apple,panel-mini-led", "apple,panel"; + width-mm = <346>; + height-mm = <223>; + adj-height-mm = <216>; +}; + +&aop_audio { + apple,chassis-name = "J416"; + apple,machine-kind = "MacBook Pro"; +}; + +&sound { + compatible = "apple,j416-macaudio", "apple,j316-macaudio", "apple,macaudio"; + model = "MacBook Pro J416"; +}; + +&mtp_mt { + firmware-name = "apple/tpmtfw-j416s.bin"; +}; diff --git a/arch/arm64/boot/dts/apple/t6020-j474s.dts b/arch/arm64/boot/dts/apple/t6020-j474s.dts index 7c7ad5b8ad189e..12dfe9693502ad 100644 --- a/arch/arm64/boot/dts/apple/t6020-j474s.dts +++ b/arch/arm64/boot/dts/apple/t6020-j474s.dts @@ -45,3 +45,84 @@ <0x200 &pcie0_dart_2 1 1>, <0x300 &pcie0_dart_3 1 1>; }; + +&port02 { + bus-range = <2 2>; +}; + +ðernet0 { + reg = <0x20000 0x0 0x0 0x0 0x0>; +}; + +&port03 { + bus-range = <3 3>; +}; + +&sound { + compatible = "apple,j474-macaudio", "apple,j473-macaudio", "apple,macaudio"; + model = "Mac mini J474"; +}; + +&lpdptxphy { + status = "okay"; +}; + +#define USE_DCPEXT0 1 + +#if USE_DCPEXT0 +/ { + aliases { + dcpext0 = &dcpext0; + /delete-property/ dcp; + }; +}; + +&framebuffer0 { + power-domains = <&ps_dispext0_cpu0>, <&ps_dptx_phy_ps>; +}; + +&dcp { + status = "disabled"; +}; +&display { + iommus = <&dispext0_dart 0>; +}; +&dispext0_dart { + status = "okay"; +}; +&dcpext0_dart { + status = "okay"; +}; +&dcpext0_mbox { + status = "okay"; +}; +&dpaudio1 { + status = "okay"; +}; +&dcpext0 { +#else +&dpaudio0 { + status = "okay"; +}; +&dcp { +#endif + status = "okay"; + apple,connector-type = "HDMI-A"; + + /* HDMI HPD gpio, used as interrupt*/ + hdmi-hpd-gpios = <&pinctrl_aop 25 GPIO_ACTIVE_HIGH>; + + hdmi-pwren-gpios = <&smc_gpio 23 GPIO_ACTIVE_HIGH>; + dp2hdmi-pwren-gpios = <&smc_gpio 25 GPIO_ACTIVE_HIGH>; + + phys = <&lpdptxphy>; + phy-names = "dp-phy"; + apple,dptx-phy = <4>; +}; + +&gpu { + /* Apple does not do this, but they probably should */ + apple,perf-base-pstate = <3>; +}; + +#include "hwmon-mini.dtsi" diff --git a/arch/arm64/boot/dts/apple/t6020.dtsi b/arch/arm64/boot/dts/apple/t6020.dtsi index bffa66a3ffff3f..500726e697a894 100644 --- a/arch/arm64/boot/dts/apple/t6020.dtsi +++ b/arch/arm64/boot/dts/apple/t6020.dtsi @@ -9,6 +9,8 @@ /* This chip is just a cut down version of t6021, so include it and disable the missing parts */ +#define GPU_REPEAT(x) + #include "t6021.dtsi" / { @@ -16,7 +18,21 @@ }; /delete-node/ &pmgr_south; +/delete-node/ &pmp_report_dispext2; +/delete-node/ &pmp_report_dispext3; +/delete-node/ &pmp_report_venc1; +/delete-node/ &pmp_report_msr1; +/delete-node/ &pmp_report_prores; +/delete-node/ &pmp_report_afnc4_ioa; +/delete-node/ &pmp_report_afnc5_ioa; &gpu { - compatible = "apple,agx-g14s"; + compatible = "apple,agx-t6020", "apple,agx-g14x", "apple,agx-g14s"; + + apple,avg-power-filter-tc-ms = <302>; + apple,avg-power-ki-only = <2.6375>; + apple,avg-power-kp = <0.18>; + apple,fast-die0-integral-gain = <1350.0>; + apple,ppm-filter-time-constant-ms = <32>; + apple,ppm-ki = <28.0>; }; diff --git a/arch/arm64/boot/dts/apple/t6021-j414c.dts b/arch/arm64/boot/dts/apple/t6021-j414c.dts index cdcf0740714dcf..fab3b03ff3c452 100644 --- a/arch/arm64/boot/dts/apple/t6021-j414c.dts +++ b/arch/arm64/boot/dts/apple/t6021-j414c.dts @@ -24,3 +24,24 @@ &bluetooth0 { brcm,board-type = "apple,tokara"; }; + +&panel { + compatible = "apple,panel-j414", "apple,panel-mini-led", "apple,panel"; + width-mm = <302>; + height-mm = <196>; + adj-height-mm = <189>; +}; + +&aop_audio { + apple,chassis-name = "J414"; + apple,machine-kind = "MacBook Pro"; +}; + +&sound { + compatible = "apple,j414-macaudio", "apple,j314-macaudio", "apple,macaudio"; + model = "MacBook Pro J414"; +}; + +&mtp_mt { + firmware-name = "apple/tpmtfw-j414c.bin"; +}; diff --git a/arch/arm64/boot/dts/apple/t6021-j416c.dts b/arch/arm64/boot/dts/apple/t6021-j416c.dts index 6d8146b9417036..b476e235639ffc 100644 --- a/arch/arm64/boot/dts/apple/t6021-j416c.dts +++ b/arch/arm64/boot/dts/apple/t6021-j416c.dts @@ -17,6 +17,26 @@ model = "Apple MacBook Pro (16-inch, M2 Max, 2023)"; }; +/* This machine model (only) has two extra boost CPU P-states * + * Disabled: Only the highest CPU bin (38 GPU cores) has this. + * Keep this disabled until m1n1 learns how to remove these OPPs + * for unsupported machines, otherwise it breaks cpufreq. +&avalanche_opp { + opp18 { + opp-hz = /bits/ 64 <3528000000>; + opp-level = <18>; + clock-latency-ns = <67000>; + turbo-mode; + }; + opp19 { + opp-hz = /bits/ 64 <3696000000>; + opp-level = <19>; + clock-latency-ns = <67000>; + turbo-mode; + }; +}; +*/ + &wifi0 { brcm,board-type = "apple,amami"; }; @@ -24,3 +44,24 @@ &bluetooth0 { brcm,board-type = "apple,amami"; }; + +&panel { + compatible = "apple,panel-j416", "apple,panel-mini-led", "apple,panel"; + width-mm = <346>; + height-mm = <223>; + adj-height-mm = <216>; +}; + +&aop_audio { + apple,chassis-name = "J416"; + apple,machine-kind = "MacBook Pro"; +}; + +&sound { + compatible = "apple,j416-macaudio", "apple,j316-macaudio", "apple,macaudio"; + model = "MacBook Pro J416"; +}; + +&mtp_mt { + firmware-name = "apple/tpmtfw-j416c.bin"; +}; diff --git a/arch/arm64/boot/dts/apple/t6021-j475c.dts b/arch/arm64/boot/dts/apple/t6021-j475c.dts index 533e3577487469..e4321cfc556838 100644 --- a/arch/arm64/boot/dts/apple/t6021-j475c.dts +++ b/arch/arm64/boot/dts/apple/t6021-j475c.dts @@ -29,9 +29,99 @@ /* enable PCIe port01 with SDHCI */ &port01 { + pwren-gpios = <&smc_gpio 22 GPIO_ACTIVE_HIGH>; status = "okay"; }; &pcie0_dart_1 { status = "okay"; }; + +&pinctrl_ap { + usb_hub_oe-hog { + gpio-hog; + gpios = <231 0>; + input; + line-name = "usb-hub-oe"; + }; + + usb_hub_rst-hog { + gpio-hog; + gpios = <232 GPIO_ACTIVE_LOW>; + output-low; + line-name = "usb-hub-rst"; + }; +}; + +&sound { + compatible = "apple,j475-macaudio", "apple,j375-macaudio", "apple,macaudio"; + model = "Mac Studio J475"; +}; + +&lpdptxphy { + status = "okay"; +}; + + +#define USE_DCPEXT0 1 + +#if USE_DCPEXT0 +/ { + aliases { + dcpext0 = &dcpext0; + /delete-property/ dcp; + }; +}; + +&framebuffer0 { + power-domains = <&ps_dispext0_cpu0>, <&ps_dptx_phy_ps>; +}; + +&dcp { + status = "disabled"; +}; +&display { + iommus = <&dispext0_dart 0>; +}; +&dispext0_dart { + status = "okay"; +}; +&dcpext0_dart { + status = "okay"; +}; +&dcpext0_mbox { + status = "okay"; +}; +&dpaudio1 { + status = "okay"; +}; +&dcpext0 { +#else +&dpaudio0 { + status = "okay"; +}; +&dcp { +#endif + status = "okay"; + apple,connector-type = "HDMI-A"; + + /* HDMI HPD gpio, used as interrupt*/ + hdmi-hpd-gpios = <&pinctrl_aop 25 GPIO_ACTIVE_HIGH>; + + hdmi-pwren-gpios = <&smc_gpio 23 GPIO_ACTIVE_HIGH>; + dp2hdmi-pwren-gpios = <&smc_gpio 25 GPIO_ACTIVE_HIGH>; + + phys = <&lpdptxphy>; + phy-names = "dp-phy"; + apple,dptx-phy = <4>; +}; + +&gpu { + apple,idleoff-standby-timer = <3000>; + apple,perf-base-pstate = <5>; + apple,perf-boost-ce-step = <100>; + apple,perf-boost-min-util = <75>; + apple,perf-tgt-utilization = <70>; +}; + +#include "hwmon-fan-dual.dtsi" diff --git a/arch/arm64/boot/dts/apple/t6021.dtsi b/arch/arm64/boot/dts/apple/t6021.dtsi index 62907ad6a54683..bb0e66851f1b59 100644 --- a/arch/arm64/boot/dts/apple/t6021.dtsi +++ b/arch/arm64/boot/dts/apple/t6021.dtsi @@ -16,6 +16,13 @@ #include "multi-die-cpp.h" +#ifndef GPU_REPEAT +# define GPU_REPEAT(x) +#endif +#ifndef GPU_DIE_REPEAT +# define GPU_DIE_REPEAT(x) +#endif + #include "t602x-common.dtsi" / { @@ -28,6 +35,8 @@ ranges; nonposted-mmio; + /* Required to get >32-bit DMA via DARTs */ + dma-ranges = <0 0 0 0 0xffffffff 0xffffc000>; // filled via templated includes at the end of the file }; @@ -65,5 +74,12 @@ }; &gpu { - compatible = "apple,agx-g14c", "apple,agx-g14s"; + compatible = "apple,agx-t6021", "apple,agx-g14x", "apple,agx-g14c", "apple,agx-g14s"; + + apple,avg-power-filter-tc-ms = <300>; + apple,avg-power-ki-only = <1.5125>; + apple,avg-power-kp = <0.38>; + apple,fast-die0-integral-gain = <700.0>; + apple,ppm-filter-time-constant-ms = <34>; + apple,ppm-ki = <18.0>; }; diff --git a/arch/arm64/boot/dts/apple/t6022-j180d.dts b/arch/arm64/boot/dts/apple/t6022-j180d.dts index dca6bd167c225a..4a528c28031805 100644 --- a/arch/arm64/boot/dts/apple/t6022-j180d.dts +++ b/arch/arm64/boot/dts/apple/t6022-j180d.dts @@ -11,13 +11,28 @@ #include "t6022.dtsi" #include "t6022-jxxxd.dtsi" +#include "t6022-pcie-ge.dtsi" / { compatible = "apple,j180d", "apple,t6022", "apple,arm-platform"; model = "Apple Mac Pro (M2 Ultra, 2023)"; + chassis-type = "server"; + aliases { - nvram = &nvram; + atcphy0 = &atcphy0; + atcphy1 = &atcphy1; + atcphy2 = &atcphy2; + atcphy3 = &atcphy3; + atcphy4 = &atcphy0_die1; + atcphy5 = &atcphy1_die1; + atcphy6 = &atcphy2_die1; + atcphy7 = &atcphy3_die1; + bluetooth0 = &bluetooth0; + dcpext0 = &dcpext0; + ethernet0 = ðernet0; + ethernet1 = ðernet1; serial0 = &serial0; + wifi0 = &wifi0; }; chosen { @@ -36,6 +51,13 @@ }; }; + reserved-memory { + #address-cells = <2>; + #size-cells = <2>; + ranges; + /* To be filled by loader */ + }; + memory@10000000000 { device_type = "memory"; reg = <0x100 0 0x2 0>; /* To be filled by loader */ @@ -46,6 +68,46 @@ status = "okay"; }; +&lpdptxphy { + status = "okay"; +}; + +&display { + iommus = <&dispext0_dart_die1 0>, <&dispext0_dart 0>; +}; + +&dispext0_dart { + status = "okay"; +}; + +&dcpext0_dart { + status = "okay"; +}; + +&dcpext0_mbox { + status = "okay"; +}; + +&dcpext0 { + status = "okay"; + apple,connector-type = "HDMI-A"; + + /* HDMI HPD gpio, used as interrupt*/ + hdmi-hpd-gpios = <&pinctrl_aop 25 GPIO_ACTIVE_HIGH>; + + // shared between dp2hdmi-gpio0 / dp2hdmi-gpio1 + // hdmi-pwren-gpios = <&smc_gpio 23 GPIO_ACTIVE_HIGH>; + + phys = <&lpdptxphy>; + phy-names = "dp-phy"; + apple,dptx-phy = <4>; + apple,dptx-die = <0>; +}; + +&dpaudio1 { + status = "okay"; +}; + /* USB Type C Rear */ &i2c0 { hpm2: usb-pd@3b { @@ -54,6 +116,30 @@ interrupt-parent = <&pinctrl_ap>; interrupts = <44 IRQ_TYPE_LEVEL_LOW>; interrupt-names = "irq"; + + typec2: connector { + compatible = "usb-c-connector"; + label = "USB-C Back 1"; + power-role = "dual"; + data-role = "dual"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + typec2_connector_hs: endpoint { + remote-endpoint = <&dwc3_2_hs>; + }; + }; + port@1 { + reg = <1>; + typec2_connector_ss: endpoint { + remote-endpoint = <&atcphy2_typec_lanes>; + }; + }; + }; + }; }; hpm3: usb-pd@3c { @@ -62,6 +148,30 @@ interrupt-parent = <&pinctrl_ap>; interrupts = <44 IRQ_TYPE_LEVEL_LOW>; interrupt-names = "irq"; + + typec3: connector { + compatible = "usb-c-connector"; + label = "USB-C Back 2"; + power-role = "dual"; + data-role = "dual"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + typec3_connector_hs: endpoint { + remote-endpoint = <&dwc3_3_hs>; + }; + }; + port@1 { + reg = <1>; + typec3_connector_ss: endpoint { + remote-endpoint = <&atcphy3_typec_lanes>; + }; + }; + }; + }; }; /* hpm4 and hpm5 included from t6022-jxxxd.dtsi */ @@ -72,6 +182,30 @@ interrupt-parent = <&pinctrl_ap>; interrupts = <44 IRQ_TYPE_LEVEL_LOW>; interrupt-names = "irq"; + + typec6: connector { + compatible = "usb-c-connector"; + label = "USB-C Back 5"; + power-role = "dual"; + data-role = "dual"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + typec6_connector_hs: endpoint { + remote-endpoint = <&dwc3_6_hs>; + }; + }; + port@1 { + reg = <1>; + typec6_connector_ss: endpoint { + remote-endpoint = <&atcphy6_typec_lanes>; + }; + }; + }; + }; }; hpm7: usb-pd@3e { @@ -80,9 +214,41 @@ interrupt-parent = <&pinctrl_ap>; interrupts = <44 IRQ_TYPE_LEVEL_LOW>; interrupt-names = "irq"; + + typec7: connector { + compatible = "usb-c-connector"; + label = "USB-C Back 6"; + power-role = "dual"; + data-role = "dual"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + typec7_connector_hs: endpoint { + remote-endpoint = <&dwc3_7_hs>; + }; + }; + port@1 { + reg = <1>; + typec7_connector_ss: endpoint { + remote-endpoint = <&atcphy7_typec_lanes>; + }; + }; + }; + }; }; }; +&typec4 { + label = "USB-C Back 3"; +}; + +&typec5 { + label = "USB-C Back 4"; +}; + /* USB Type C Front */ &i2c3 { status = "okay"; @@ -93,6 +259,30 @@ interrupt-parent = <&pinctrl_ap>; interrupts = <60 IRQ_TYPE_LEVEL_LOW>; interrupt-names = "irq"; + + typec0: connector { + compatible = "usb-c-connector"; + label = "USB-C Top Right"; + power-role = "dual"; + data-role = "dual"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + typec0_connector_hs: endpoint { + remote-endpoint = <&dwc3_0_hs>; + }; + }; + port@1 { + reg = <1>; + typec0_connector_ss: endpoint { + remote-endpoint = <&atcphy0_typec_lanes>; + }; + }; + }; + }; }; hpm1: usb-pd@3f { @@ -101,9 +291,711 @@ interrupt-parent = <&pinctrl_ap>; interrupts = <60 IRQ_TYPE_LEVEL_LOW>; interrupt-names = "irq"; + + typec1: connector { + compatible = "usb-c-connector"; + label = "USB-C Top Left"; + power-role = "dual"; + data-role = "dual"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + typec1_connector_hs: endpoint { + remote-endpoint = <&dwc3_1_hs>; + }; + }; + port@1 { + reg = <1>; + typec1_connector_ss: endpoint { + remote-endpoint = <&atcphy1_typec_lanes>; + }; + }; + }; + }; + }; +}; + +/* USB controllers */ +&dwc3_0 { + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + dwc3_0_hs: endpoint { + remote-endpoint = <&typec0_connector_hs>; + }; + }; + + port@1 { + reg = <1>; + dwc3_0_ss: endpoint { + remote-endpoint = <&atcphy0_usb3>; + }; + }; + }; +}; + +&dwc3_1 { + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + dwc3_1_hs: endpoint { + remote-endpoint = <&typec1_connector_hs>; + }; + }; + + port@1 { + reg = <1>; + dwc3_1_ss: endpoint { + remote-endpoint = <&atcphy1_usb3>; + }; + }; + }; +}; + +&dwc3_2 { + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + dwc3_2_hs: endpoint { + remote-endpoint = <&typec2_connector_hs>; + }; + }; + + port@1 { + reg = <1>; + dwc3_2_ss: endpoint { + remote-endpoint = <&atcphy2_usb3>; + }; + }; + }; +}; + +&dwc3_3 { + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + dwc3_3_hs: endpoint { + remote-endpoint = <&typec3_connector_hs>; + }; + }; + + port@1 { + reg = <1>; + dwc3_3_ss: endpoint { + remote-endpoint = <&atcphy3_usb3>; + }; + }; }; }; +/* USB controllers on die 1 */ +&dwc3_2_die1 { + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + dwc3_6_hs: endpoint { + remote-endpoint = <&typec6_connector_hs>; + }; + }; + + port@1 { + reg = <1>; + dwc3_6_ss: endpoint { + remote-endpoint = <&atcphy6_usb3>; + }; + }; + }; +}; + +&dwc3_3_die1 { + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + dwc3_7_hs: endpoint { + remote-endpoint = <&typec7_connector_hs>; + }; + }; + + port@1 { + reg = <1>; + dwc3_7_ss: endpoint { + remote-endpoint = <&atcphy7_usb3>; + }; + }; + }; +}; + +/* Type-C PHYs */ +&atcphy0 { + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + atcphy0_typec_lanes: endpoint { + remote-endpoint = <&typec0_connector_ss>; + }; + }; + + port@1 { + reg = <1>; + atcphy0_usb3: endpoint { + remote-endpoint = <&dwc3_0_ss>; + }; + }; + }; +}; + +&atcphy1 { + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + atcphy1_typec_lanes: endpoint { + remote-endpoint = <&typec1_connector_ss>; + }; + }; + + port@1 { + reg = <1>; + atcphy1_usb3: endpoint { + remote-endpoint = <&dwc3_1_ss>; + }; + }; + }; +}; + +&atcphy2 { + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + atcphy2_typec_lanes: endpoint { + remote-endpoint = <&typec2_connector_ss>; + }; + }; + + port@1 { + reg = <1>; + atcphy2_usb3: endpoint { + remote-endpoint = <&dwc3_2_ss>; + }; + }; + }; +}; + +&atcphy3 { + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + atcphy3_typec_lanes: endpoint { + remote-endpoint = <&typec3_connector_ss>; + }; + }; + + port@1 { + reg = <1>; + atcphy3_usb3: endpoint { + remote-endpoint = <&dwc3_3_ss>; + }; + }; + }; +}; + +&atcphy2_die1 { + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + atcphy6_typec_lanes: endpoint { + remote-endpoint = <&typec6_connector_ss>; + }; + }; + + port@1 { + reg = <1>; + atcphy6_usb3: endpoint { + remote-endpoint = <&dwc3_6_ss>; + }; + }; + }; +}; + +&atcphy3_die1 { + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + atcphy7_typec_lanes: endpoint { + remote-endpoint = <&typec7_connector_ss>; + }; + }; + + port@1 { + reg = <1>; + atcphy7_usb3: endpoint { + remote-endpoint = <&dwc3_7_ss>; + }; + }; + }; +}; + +/* Audio */ +&i2c1 { + status = "okay"; + + speaker_tweeter: codec@38 { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x38>; + shutdown-gpios = <&pinctrl_ap 57 GPIO_ACTIVE_HIGH>; + #sound-dai-cells = <0>; + sound-name-prefix = "Tweeter"; + interrupts-extended = <&pinctrl_ap 58 IRQ_TYPE_LEVEL_LOW>; + ti,imon-slot-no = <4>; + ti,vmon-slot-no = <6>; + }; + + speaker_woofer: codec@39 { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x39>; + shutdown-gpios = <&pinctrl_ap 57 GPIO_ACTIVE_HIGH>; + #sound-dai-cells = <0>; + sound-name-prefix = "Woofer"; + interrupts-extended = <&pinctrl_ap 58 IRQ_TYPE_LEVEL_LOW>; + ti,imon-slot-no = <0>; + ti,vmon-slot-no = <2>; + }; +}; + +&i2c2 { + status = "okay"; + + jack_codec: codec@4b { + compatible = "cirrus,cs42l84"; + reg = <0x4b>; + reset-gpios = <&pinctrl_nub 8 GPIO_ACTIVE_HIGH>; + #sound-dai-cells = <0>; + interrupts-extended = <&pinctrl_ap 59 IRQ_TYPE_LEVEL_LOW>; + sound-name-prefix = "Jack"; + }; +}; + +&nco_clkref { + clock-frequency = <1068000000>; +}; + +/ { + sound: sound { + compatible = "apple,j180-macaudio", "apple,macaudio"; + model = "Mac Pro J180"; + + dai-link@0 { + link-name = "Speakers"; + /* + * DANGER ZONE: You can blow your speakers! + * + * The drivers are not ready, and unless you are careful + * to attenuate the audio stream, you run the risk of + * blowing your speakers. + */ + status = "disabled"; + cpu { + sound-dai = <&mca 0>; + }; + codec { + sound-dai = <&speaker_woofer>, <&speaker_tweeter>; + }; + }; + + dai-link@1 { + link-name = "Headphone Jack"; + + cpu { + sound-dai = <&mca 2>; + }; + codec { + sound-dai = <&jack_codec>; + }; + }; + }; +}; + +/* PCIe devices */ +&port_ge00 { + bus-range = <0x01 0x09>; + + pci@0,0 { + // compatible = "pci11f8,4000", "pciclass,060400", "pciclass,0604"; + device_type = "pci"; + reg = <0x10000 0x00 0x00 0x00 0x00>; + bus-range = <0x02 0x07>; + + #address-cells = <0x03>; + #size-cells = <0x02>; + ranges = <0x82010000 0x00 0x80000000 0x82010000 0x00 0x80000000 0x00 0x500000>, + <0xc3010000 0x18 0x00000000 0xc3010000 0x18 0x00000000 0x00 0x500000>; + + #interrupt-cells = <0x01>; + interrupt-map-mask = <0xffff00 0x00 0x00 0x07>; + interrupt-map = <0x20000 0x00 0x00 0x01 &port_ge00 0x00 0x00 0x00 0x00>, + <0x20000 0x00 0x00 0x02 &port_ge00 0x00 0x00 0x00 0x01>, + <0x20000 0x00 0x00 0x03 &port_ge00 0x00 0x00 0x00 0x02>, + <0x20000 0x00 0x00 0x04 &port_ge00 0x00 0x00 0x00 0x03>, + <0x20800 0x00 0x00 0x01 &port_ge00 0x00 0x00 0x00 0x01>, + <0x20800 0x00 0x00 0x02 &port_ge00 0x00 0x00 0x00 0x02>, + <0x20800 0x00 0x00 0x03 &port_ge00 0x00 0x00 0x00 0x03>, + <0x20800 0x00 0x00 0x04 &port_ge00 0x00 0x00 0x00 0x00>, + <0x21000 0x00 0x00 0x01 &port_ge00 0x00 0x00 0x00 0x02>, + <0x21000 0x00 0x00 0x02 &port_ge00 0x00 0x00 0x00 0x03>, + <0x21000 0x00 0x00 0x03 &port_ge00 0x00 0x00 0x00 0x00>, + <0x21000 0x00 0x00 0x04 &port_ge00 0x00 0x00 0x00 0x01>, + <0x21800 0x00 0x00 0x01 &port_ge00 0x00 0x00 0x00 0x03>, + <0x21800 0x00 0x00 0x02 &port_ge00 0x00 0x00 0x00 0x00>, + <0x21800 0x00 0x00 0x03 &port_ge00 0x00 0x00 0x00 0x01>, + <0x21800 0x00 0x00 0x04 &port_ge00 0x00 0x00 0x00 0x02>, + <0x22000 0x00 0x00 0x01 &port_ge00 0x00 0x00 0x00 0x00>, + <0x22000 0x00 0x00 0x02 &port_ge00 0x00 0x00 0x00 0x01>, + <0x22000 0x00 0x00 0x03 &port_ge00 0x00 0x00 0x00 0x02>, + <0x22000 0x00 0x00 0x04 &port_ge00 0x00 0x00 0x00 0x03>; + + /* pci-slot1-dsp, PCIe slot-1 */ + pci@0,0 { + compatible = "pci11f8,4000", "pciclass,060400", "pciclass,0604"; + device_type = "pci"; + reg = <0x20000 0x00 0x00 0x00 0x00>; + bus-range = <0x03 0x03>; + + #address-cells = <0x03>; + #size-cells = <0x02>; + ranges; + }; + + /* pci-slot2-dsp, PCIe slot-2 */ + pci@1,0 { + compatible = "pci11f8,4000", "pciclass,060400", "pciclass,0604"; + device_type = "pci"; + reg = <0x20800 0x00 0x00 0x00 0x00>; + bus-range = <0x04 0x04>; + + #address-cells = <0x03>; + #size-cells = <0x02>; + ranges; + }; + + /* pci-slot3-dsp, PCIe slot-3 */ + pci@2,0 { + compatible = "pci11f8,4000", "pciclass,060400", "pciclass,0604"; + device_type = "pci"; + reg = <0x21000 0x00 0x00 0x00 0x00>; + bus-range = <0x05 0x05>; + + #address-cells = <0x03>; + #size-cells = <0x02>; + ranges; + }; + + /* pci-slot4-dsp, PCIe slot-4 */ + pci@3,0 { + compatible = "pci11f8,4000", "pciclass,060400", "pciclass,0604"; + device_type = "pci"; + reg = <0x21800 0x00 0x00 0x00 0x00>; + bus-range = <0x06 0x06>; + + #address-cells = <0x03>; + #size-cells = <0x02>; + ranges; + }; + + /* pci-slot5-dsp, PCIe slot-5 */ + pci@4,0 { + compatible = "pci11f8,4000", "pciclass,060400", "pciclass,0604"; + device_type = "pci"; + reg = <0x22000 0x00 0x00 0x00 0x00>; + bus-range = <0x07 0x07>; + + #address-cells = <0x03>; + #size-cells = <0x02>; + ranges; + }; + + }; +}; + +&port_ge00_die1 { + bus-range = <0x01 0x09>; + + /* + * Add mulptiple "reset-gpios" since there is no mechanismen to access + * PERST# for devices behind the PCIe switch. + * The "pwren" GPIO is from the wifi/bt chip which faces the same + * problem without pci-pwrctrl integration. + */ + reset-gpios = <&pinctrl_ap 4 GPIO_ACTIVE_LOW>, + <&pinctrl_ap 6 GPIO_ACTIVE_LOW>, + <&pinctrl_ap 7 GPIO_ACTIVE_LOW>, + <&pinctrl_ap_die1 9 GPIO_ACTIVE_LOW>; + pwren-gpios = <&smc_gpio 13 GPIO_ACTIVE_HIGH>; + + pci@0,0 { + device_type = "pci"; + reg = <0x10000 0x00 0x00 0x00 0x00>; + bus-range = <0x02 0x09>; + + #address-cells = <3>; + #size-cells = <2>; + ranges; + + interrupt-controller; + #interrupt-cells = <1>; + interrupt-map-mask = <0xffff00 0x00 0x00 0x07>; + interrupt-map = <0x20000 0x00 0x00 0x01 &port_ge00_die1 0x00 0x00 0x00 0x00>, + <0x20000 0x00 0x00 0x02 &port_ge00_die1 0x00 0x00 0x00 0x01>, + <0x20000 0x00 0x00 0x03 &port_ge00_die1 0x00 0x00 0x00 0x02>, + <0x20000 0x00 0x00 0x04 &port_ge00_die1 0x00 0x00 0x00 0x03>, + <0x20800 0x00 0x00 0x01 &port_ge00_die1 0x00 0x00 0x00 0x01>, + <0x20800 0x00 0x00 0x02 &port_ge00_die1 0x00 0x00 0x00 0x02>, + <0x20800 0x00 0x00 0x03 &port_ge00_die1 0x00 0x00 0x00 0x03>, + <0x20800 0x00 0x00 0x04 &port_ge00_die1 0x00 0x00 0x00 0x00>, + <0x21000 0x00 0x00 0x01 &port_ge00_die1 0x00 0x00 0x00 0x02>, + <0x21000 0x00 0x00 0x02 &port_ge00_die1 0x00 0x00 0x00 0x03>, + <0x21000 0x00 0x00 0x03 &port_ge00_die1 0x00 0x00 0x00 0x00>, + <0x21000 0x00 0x00 0x04 &port_ge00_die1 0x00 0x00 0x00 0x01>, + <0x21800 0x00 0x00 0x01 &port_ge00_die1 0x00 0x00 0x00 0x03>, + <0x21800 0x00 0x00 0x02 &port_ge00_die1 0x00 0x00 0x00 0x00>, + <0x21800 0x00 0x00 0x03 &port_ge00_die1 0x00 0x00 0x00 0x01>, + <0x21800 0x00 0x00 0x04 &port_ge00_die1 0x00 0x00 0x00 0x02>, + <0x22000 0x00 0x00 0x01 &port_ge00_die1 0x00 0x00 0x00 0x00>, + <0x22000 0x00 0x00 0x02 &port_ge00_die1 0x00 0x00 0x00 0x01>, + <0x22000 0x00 0x00 0x03 &port_ge00_die1 0x00 0x00 0x00 0x02>, + <0x22000 0x00 0x00 0x04 &port_ge00_die1 0x00 0x00 0x00 0x03>, + <0x22800 0x00 0x00 0x01 &port_ge00_die1 0x00 0x00 0x00 0x01>, + <0x22800 0x00 0x00 0x02 &port_ge00_die1 0x00 0x00 0x00 0x02>, + <0x22800 0x00 0x00 0x03 &port_ge00_die1 0x00 0x00 0x00 0x03>, + <0x22800 0x00 0x00 0x04 &port_ge00_die1 0x00 0x00 0x00 0x00>, + <0x23000 0x00 0x00 0x01 &port_ge00_die1 0x00 0x00 0x00 0x02>, + <0x23000 0x00 0x00 0x02 &port_ge00_die1 0x00 0x00 0x00 0x03>, + <0x23000 0x00 0x00 0x03 &port_ge00_die1 0x00 0x00 0x00 0x00>, + <0x23000 0x00 0x00 0x04 &port_ge00_die1 0x00 0x00 0x00 0x01>; + + /* pci-usba-dsp, internal USB-A port */ + pci@0,0 { + compatible = "pci11f8,4000", "pciclass,060400", "pciclass,0604"; + device_type = "pci"; + reg = <0x20000 0x00 0x00 0x00 0x00>; + bus-range = <0x03 0x03>; + + #address-cells = <3>; + #size-cells = <2>; + ranges; + + interrupt-controller; + #interrupt-cells = <1>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0x30000 0x00 0x00 0x02 &port_ge00_die1 0x00 0x00 0x00 0x00>, + <0x30000 0x00 0x00 0x03 &port_ge00_die1 0x00 0x00 0x00 0x01>, + <0x30000 0x00 0x00 0x04 &port_ge00_die1 0x00 0x00 0x00 0x02>; + + /* temporarily handled in the root port */ + // reset-gpios = <&pinctrl_ap 6 GPIO_ACTIVE_LOW>; + }; + + /* pci-sata-dsp, internal AHCI controller */ + pci@1,0 { + compatible = "pci11f8,4000", "pciclass,060400", "pciclass,0604"; + device_type = "pci"; + reg = <0x20800 0x00 0x00 0x00 0x00>; + bus-range = <0x04 0x04>; + + #address-cells = <3>; + #size-cells = <2>; + ranges; + + interrupt-controller; + #interrupt-cells = <1>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0x40000 0x00 0x00 0x01 &port_ge00_die1 0x00 0x00 0x00 0x00>, + <0x40000 0x00 0x00 0x02 &port_ge00_die1 0x00 0x00 0x00 0x01>, + <0x40000 0x00 0x00 0x03 &port_ge00_die1 0x00 0x00 0x00 0x02>; + }; + + /* pci-bio-dsp, I/O board USB-A ports */ + pci@2,0 { + compatible = "pci11f8,4000", "pciclass,060400", "pciclass,0604"; + device_type = "pci"; + reg = <0x21000 0x00 0x00 0x00 0x00>; + bus-range = <0x05 0x05>; + + #address-cells = <3>; + #size-cells = <2>; + ranges; + + interrupt-controller; + #interrupt-cells = <1>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0x50000 0x00 0x00 0x01 &port_ge00_die1 0x00 0x00 0x00 0x01>, + <0x50000 0x00 0x00 0x02 &port_ge00_die1 0x00 0x00 0x00 0x02>, + <0x50000 0x00 0x00 0x04 &port_ge00_die1 0x00 0x00 0x00 0x00>; + + /* temporarily handled in the root port */ + // reset-gpios = <&pinctrl_ap 7 GPIO_ACTIVE_LOW>; + }; + + /* pci-lan-dsp, Qtion AQC113 10G etherner controller (0) */ + pci@3,0 { + compatible = "pci11f8,4000", "pciclass,060400", "pciclass,0604"; + device_type = "pci"; + reg = <0x21800 0x00 0x00 0x00 0x00>; + bus-range = <0x06 0x06>; + + #address-cells = <3>; + #size-cells = <2>; + ranges; + + interrupt-controller; + #interrupt-cells = <1>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0x60000 0x00 0x00 0x01 &port_ge00_die1 0x00 0x00 0x00 0x02>, + <0x60000 0x00 0x00 0x03 &port_ge00_die1 0x00 0x00 0x00 0x00>, + <0x60000 0x00 0x00 0x04 &port_ge00_die1 0x00 0x00 0x00 0x01>; + + ethernet0: ethernet@0,0 { + reg = <0x60000 0x0 0x0 0x0 0x0>; + /* To be filled by the loader */ + local-mac-address = [00 10 18 00 00 00]; + }; + }; + + /* pci-lan-b-dsp, Qtion AQC113 10G etherner controller (1) */ + pci@4,0 { + compatible = "pci11f8,4000", "pciclass,060400", "pciclass,0604"; + device_type = "pci"; + reg = <0x22000 0x00 0x00 0x00 0x00>; + bus-range = <0x07 0x07>; + + #address-cells = <3>; + #size-cells = <2>; + ranges; + + interrupt-controller; + #interrupt-cells = <1>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0x70000 0x00 0x00 0x02 &port_ge00_die1 0x00 0x00 0x00 0x00>, + <0x70000 0x00 0x00 0x03 &port_ge00_die1 0x00 0x00 0x00 0x01>, + <0x70000 0x00 0x00 0x04 &port_ge00_die1 0x00 0x00 0x00 0x02>; + + ethernet1: ethernet@0,0 { + reg = <0x70000 0x0 0x0 0x0 0x0>; + /* To be filled by the loader */ + local-mac-address = [00 10 18 00 00 00]; + }; + }; + + /* pci-wifibt-dsp, Broadcom BCM4388 Wlan/BT */ + pci@5,0 { + compatible = "pci11f8,4000", "pciclass,060400", "pciclass,0604"; + device_type = "pci"; + reg = <0x22800 0x00 0x00 0x00 0x00>; + bus-range = <0x08 0x08>; + + #address-cells = <3>; + #size-cells = <2>; + ranges; + + interrupt-controller; + #interrupt-cells = <1>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0x80000 0x00 0x00 0x01 &port_ge00_die1 0x00 0x00 0x00 0x00>, + <0x80000 0x00 0x00 0x02 &port_ge00_die1 0x00 0x00 0x00 0x01>, + <0x80000 0x00 0x00 0x03 &port_ge00_die1 0x00 0x00 0x00 0x02>, + <0x80100 0x00 0x00 0x01 &port_ge00_die1 0x00 0x00 0x00 0x00>, + <0x80100 0x00 0x00 0x02 &port_ge00_die1 0x00 0x00 0x00 0x01>, + <0x80100 0x00 0x00 0x03 &port_ge00_die1 0x00 0x00 0x00 0x02>; + + /* temporarily handled in the root port */ + // reset-gpios = <&pinctrl_ap 4 GPIO_ACTIVE_LOW>; + // pwren-gpios = <&smc_gpio 13 GPIO_ACTIVE_HIGH>; + + wifi0: wifi@0,0 { + reg = <0x80000 0x0 0x0 0x0 0x0>; + compatible = "pci14e4,4433"; + brcm,board-type = "apple,sumatra"; + apple,antenna-sku = "XX"; + /* To be filled by the loader */ + local-mac-address = [00 10 18 00 00 10]; + }; + + bluetooth0: network@0,1 { + compatible = "pci14e4,5f71"; + brcm,board-type = "apple,sumatra"; + // reg = <0x80100 0x0 0x0 0x0 0x0>; + /* To be filled by the loader */ + local-bd-address = [00 00 00 00 00 00]; + }; + }; + + /* pci-slot6-dsp, PCIe slot-6 */ + pci@6,0 { + compatible = "pci11f8,4000", "pciclass,060400", "pciclass,0604"; + device_type = "pci"; + reg = <0x23000 0x00 0x00 0x00 0x00>; + bus-range = <0x09 0x09>; + + #address-cells = <3>; + #size-cells = <2>; + ranges; + }; + }; +}; + +&pcie_ge { + status = "ok"; +}; + +&pcie_ge_dart { + status = "ok"; +}; + +&pcie_ge_die1 { + status = "ok"; +}; + +&pcie_ge_dart_die1 { + status = "ok"; +}; + /* * Delete unused PCIe nodes, the Mac Pro uses slightly different PCIe * controllers with a single port connected to a PM40100 PCIe switch diff --git a/arch/arm64/boot/dts/apple/t6022-j475d.dts b/arch/arm64/boot/dts/apple/t6022-j475d.dts index 736594544f79b5..cdfc78a1703c7a 100644 --- a/arch/arm64/boot/dts/apple/t6022-j475d.dts +++ b/arch/arm64/boot/dts/apple/t6022-j475d.dts @@ -9,6 +9,8 @@ /dts-v1/; +#define NO_DCP + #include "t6022.dtsi" #include "t602x-j474-j475.dtsi" #include "t6022-jxxxd.dtsi" @@ -16,14 +18,31 @@ / { compatible = "apple,j475d", "apple,t6022", "apple,arm-platform"; model = "Apple Mac Studio (M2 Ultra, 2023)"; + + aliases { + atcphy4 = &atcphy0_die1; + atcphy5 = &atcphy1_die1; + /delete-property/ dcp; + /delete-property/ sio; + }; +}; + +&sio { + status = "disabled"; }; &framebuffer0 { power-domains = <&ps_dispext0_cpu0_die1>, <&ps_dptx_phy_ps_die1>; }; +&dcpext0_die1 { + // J180 misses "function-dp2hdmi_pwr_en" + dp2hdmi-pwren-gpios = <&smc_gpio 25 GPIO_ACTIVE_HIGH>; +}; + /* enable PCIe port01 with SDHCI */ &port01 { + pwren-gpios = <&smc_gpio 22 GPIO_ACTIVE_HIGH>; status = "okay"; }; @@ -31,6 +50,34 @@ status = "okay"; }; +&typec4 { + label = "USB-C Front Right"; +}; + +&typec5 { + label = "USB-C Front Left"; +}; + +/* delete unused USB nodes on die 1 */ +/delete-node/ &dwc3_2_dart_0_die1; +/delete-node/ &dwc3_2_dart_1_die1; +/delete-node/ &dwc3_2_die1; +/delete-node/ &atcphy2_die1; +/delete-node/ &atcphy2_xbar_die1; + +/delete-node/ &dwc3_3_dart_0_die1; +/delete-node/ &dwc3_3_dart_1_die1; +/delete-node/ &dwc3_3_die1; +/delete-node/ &atcphy3_die1; +/delete-node/ &atcphy3_xbar_die1; + +/* delete unused always-on power-domains on die 1 */ +/delete-node/ &ps_atc2_usb_aon_die1; +/delete-node/ &ps_atc2_usb_die1; + +/delete-node/ &ps_atc3_usb_aon_die1; +/delete-node/ &ps_atc3_usb_die1; + &wifi0 { compatible = "pci14e4,4434"; brcm,board-type = "apple,canary"; @@ -40,3 +87,10 @@ compatible = "pci14e4,5f72"; brcm,board-type = "apple,canary"; }; + +&sound { + compatible = "apple,j475-macaudio", "apple,j375-macaudio", "apple,macaudio"; + model = "Mac Studio J475"; +}; + +#include "hwmon-fan-dual.dtsi" diff --git a/arch/arm64/boot/dts/apple/t6022-jxxxd.dtsi b/arch/arm64/boot/dts/apple/t6022-jxxxd.dtsi index 4f7bf2ebfe397d..fa0183441d791b 100644 --- a/arch/arm64/boot/dts/apple/t6022-jxxxd.dtsi +++ b/arch/arm64/boot/dts/apple/t6022-jxxxd.dtsi @@ -9,9 +9,66 @@ * Copyright The Asahi Linux Contributors */ +/ { + aliases { + dcpext4 = &dcpext0_die1; + disp0 = &display; + sio1 = &sio_die1; + }; +}; + +&lpdptxphy_die1 { + status = "okay"; +}; + +&display { + iommus = <&dispext0_dart_die1 0>; +}; + +&dispext0_dart_die1 { + status = "okay"; +}; + +&dcpext0_dart_die1 { + status = "okay"; +}; + +&dcpext0_mbox_die1 { + status = "okay"; +}; + +&dcpext0_die1 { + status = "okay"; + apple,connector-type = "HDMI-A"; + + /* HDMI HPD gpio, used as interrupt*/ + hdmi-hpd-gpios = <&pinctrl_aop 41 GPIO_ACTIVE_HIGH>; + + hdmi-pwren-gpios = <&smc_gpio 23 GPIO_ACTIVE_HIGH>; + // J180 misses "function-dp2hdmi_pwr_en" + // dp2hdmi-pwren-gpios = <&smc_gpio 25 GPIO_ACTIVE_HIGH>; + + phys = <&lpdptxphy_die1>; + phy-names = "dp-phy"; + apple,dptx-phy = <4>; + apple,dptx-die = <1>; +}; + +&dpaudio1_die1 { + status = "okay"; +}; + +/* delete missing dcp0/disp0 */ +/delete-node/ &disp0_dart; +/delete-node/ &dcp_dart; +/delete-node/ &dcp_mbox; +/delete-node/ &dcp; +/delete-node/ &dpaudio0; + /* delete power-domains for missing disp0 / disp0_die1 */ /delete-node/ &ps_disp0_cpu0; /delete-node/ &ps_disp0_fe; +/delete-node/ &pmp_report_disp0; /delete-node/ &ps_disp0_cpu0_die1; /delete-node/ &ps_disp0_fe_die1; @@ -25,6 +82,29 @@ interrupt-parent = <&pinctrl_ap>; interrupts = <44 IRQ_TYPE_LEVEL_LOW>; interrupt-names = "irq"; + + typec4: connector { + compatible = "usb-c-connector"; + power-role = "dual"; + data-role = "dual"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + typec4_connector_hs: endpoint { + remote-endpoint = <&dwc3_4_hs>; + }; + }; + port@1 { + reg = <1>; + typec4_connector_ss: endpoint { + remote-endpoint = <&atcphy4_typec_lanes>; + }; + }; + }; + }; }; /* front-left */ @@ -34,5 +114,115 @@ interrupt-parent = <&pinctrl_ap>; interrupts = <44 IRQ_TYPE_LEVEL_LOW>; interrupt-names = "irq"; + + typec5: connector { + compatible = "usb-c-connector"; + power-role = "dual"; + data-role = "dual"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + typec5_connector_hs: endpoint { + remote-endpoint = <&dwc3_5_hs>; + }; + }; + port@1 { + reg = <1>; + typec5_connector_ss: endpoint { + remote-endpoint = <&atcphy5_typec_lanes>; + }; + }; + }; + }; + }; +}; + + +/* USB controllers on die 1 */ +&dwc3_0_die1 { + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + dwc3_4_hs: endpoint { + remote-endpoint = <&typec4_connector_hs>; + }; + }; + + port@1 { + reg = <1>; + dwc3_4_ss: endpoint { + remote-endpoint = <&atcphy4_usb3>; + }; + }; + }; +}; + +&dwc3_1_die1 { + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + dwc3_5_hs: endpoint { + remote-endpoint = <&typec5_connector_hs>; + }; + }; + + port@1 { + reg = <1>; + dwc3_5_ss: endpoint { + remote-endpoint = <&atcphy5_usb3>; + }; + }; + }; +}; + +/* Type-C PHYs */ +&atcphy0_die1 { + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + atcphy4_typec_lanes: endpoint { + remote-endpoint = <&typec4_connector_ss>; + }; + }; + + port@1 { + reg = <1>; + atcphy4_usb3: endpoint { + remote-endpoint = <&dwc3_4_ss>; + }; + }; + }; +}; + +&atcphy1_die1 { + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + atcphy5_typec_lanes: endpoint { + remote-endpoint = <&typec5_connector_ss>; + }; + }; + + port@1 { + reg = <1>; + atcphy5_usb3: endpoint { + remote-endpoint = <&dwc3_5_ss>; + }; + }; }; }; diff --git a/arch/arm64/boot/dts/apple/t6022-pcie-ge.dtsi b/arch/arm64/boot/dts/apple/t6022-pcie-ge.dtsi new file mode 100644 index 00000000000000..f78c483c29133f --- /dev/null +++ b/arch/arm64/boot/dts/apple/t6022-pcie-ge.dtsi @@ -0,0 +1,27 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * Include PCIe-GE nodes presen on both dies of T6022 (M2 Ultra) in the + * Mac Pro (2023). + * + * Copyright The Asahi Linux Contributors + */ + +#define DIE +#define DIE_NO 0 + +&die0 { + #include "t602x-pcie-ge.dtsi" +}; + +#undef DIE +#undef DIE_NO + +#define DIE _die1 +#define DIE_NO 1 + +&die1 { + #include "t602x-pcie-ge.dtsi" +}; + +#undef DIE +#undef DIE_NO diff --git a/arch/arm64/boot/dts/apple/t6022.dtsi b/arch/arm64/boot/dts/apple/t6022.dtsi index e73bf2f7510ae2..fe065d48726a3d 100644 --- a/arch/arm64/boot/dts/apple/t6022.dtsi +++ b/arch/arm64/boot/dts/apple/t6022.dtsi @@ -16,6 +16,13 @@ #include "multi-die-cpp.h" +#ifndef GPU_REPEAT +# define GPU_REPEAT(x) +#endif +#ifndef GPU_DIE_REPEAT +# define GPU_DIE_REPEAT(x) +#endif + #include "t602x-common.dtsi" / { @@ -339,11 +346,58 @@ }; }; +&dcpext0_die1 { + apple,bw-scratch = <&pmgr_dcp 0 4 0x1240>; +}; + +&dcpext1_die1 { + apple,bw-scratch = <&pmgr_dcp 0 4 0x1248>; +}; + &ps_gfx { // On t6022, the die0 GPU power domain needs both AFR power domains power-domains = <&ps_afr>, <&ps_afr_die1>; }; +&pmp_report { + pmp_report_dispext0_die1: report@1f { + compatible = "apple,t6020-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0x1f>; + label = "pmp-dispext0_die1"; + #power-domain-cells = <0>; + power-domains = <&ps_dispext0_cpu0_die1>; + }; + + pmp_report_dispext1_die1: report@20 { + compatible = "apple,t6020-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0x20>; + label = "pmp-dispext1_die1"; + #power-domain-cells = <0>; + power-domains = <&ps_dispext1_cpu0_die1>; + }; +}; + &gpu { - compatible = "apple,agx-g14d", "apple,agx-g14s"; + compatible = "apple,agx-t6022", "apple,agx-g14x", "apple,agx-g14d", "apple,agx-g14s"; + + apple,avg-power-filter-tc-ms = <302>; + apple,avg-power-ki-only = <1.0125>; + apple,avg-power-kp = <0.15>; + apple,fast-die0-integral-gain = <9.6>; + apple,fast-die0-proportional-gain = <24.0>; + apple,idleoff-standby-timer = <3000>; + apple,perf-base-pstate = <5>; + apple,perf-boost-ce-step = <100>; + apple,perf-boost-min-util = <75>; + apple,perf-tgt-utilization = <70>; + apple,ppm-ki = <11.0>; + apple,ppm-kp = <0.15>; +}; + +&pinctrl_ap_die1 { + pcie_ge_pins_die1: pcie-ge1-pins { + pinmux = ; + }; }; diff --git a/arch/arm64/boot/dts/apple/t602x-common.dtsi b/arch/arm64/boot/dts/apple/t602x-common.dtsi index 9c800a391e7e87..2fa6b93a14165f 100644 --- a/arch/arm64/boot/dts/apple/t602x-common.dtsi +++ b/arch/arm64/boot/dts/apple/t602x-common.dtsi @@ -13,6 +13,9 @@ aliases { gpu = &gpu; + #ifdef APPLE_USE_PMP + pmp = &pmp; + #endif }; cpus { @@ -387,6 +390,134 @@ }; }; + gpu_opp: opp-table-gpu { + compatible = "operating-points-v2"; + + /* + * NOTE: The voltage and power values are device-specific and + * must be filled in by the bootloader. + */ + opp00 { + opp-hz = /bits/ 64 <0>; + opp-microvolt = GPU_REPEAT(400000); + opp-microwatt = <0>; + }; + opp01 { + opp-hz = /bits/ 64 <444000000>; + opp-microvolt = GPU_REPEAT(637000); + opp-microwatt = <4295000>; + }; + opp02 { + opp-hz = /bits/ 64 <612000000>; + opp-microvolt = GPU_REPEAT(656000); + opp-microwatt = <6251000>; + }; + opp03 { + opp-hz = /bits/ 64 <808000000>; + opp-microvolt = GPU_REPEAT(687000); + opp-microwatt = <8625000>; + }; + opp04 { + opp-hz = /bits/ 64 <968000000>; + opp-microvolt = GPU_REPEAT(725000); + opp-microwatt = <11948000>; + }; + opp05 { + opp-hz = /bits/ 64 <1110000000>; + opp-microvolt = GPU_REPEAT(790000); + opp-microwatt = <15071000>; + }; + opp06 { + opp-hz = /bits/ 64 <1236000000>; + opp-microvolt = GPU_REPEAT(843000); + opp-microwatt = <18891000>; + }; + opp07 { + opp-hz = /bits/ 64 <1338000000>; + opp-microvolt = GPU_REPEAT(887000); + opp-microwatt = <21960000>; + }; + opp08 { + opp-hz = /bits/ 64 <1398000000>; + opp-microvolt = GPU_REPEAT(918000); + opp-microwatt = <22800000>; + }; + }; + + gpu_cs_opp: opp-table-gpu-cs { + compatible = "operating-points-v2"; + + /* + * NOTE: The voltage and power values are device-specific and + * must be filled in by the bootloader. + */ + opp00 { + opp-hz = /bits/ 64 <24>; + opp-microvolt = GPU_DIE_REPEAT(668000); + }; + opp01 { + opp-hz = /bits/ 64 <444000000>; + opp-microvolt = GPU_DIE_REPEAT(668000); + }; + opp02 { + opp-hz = /bits/ 64 <612000000>; + opp-microvolt = GPU_DIE_REPEAT(678000); + }; + opp03 { + opp-hz = /bits/ 64 <808000000>; + opp-microvolt = GPU_DIE_REPEAT(737000); + }; + opp04 { + opp-hz = /bits/ 64 <1024000000>; + opp-microvolt = GPU_DIE_REPEAT(815000); + }; + opp05 { + opp-hz = /bits/ 64 <1140000000>; + opp-microvolt = GPU_DIE_REPEAT(862000); + }; + opp06 { + opp-hz = /bits/ 64 <1236000000>; + opp-microvolt = GPU_DIE_REPEAT(893000); + }; + }; + + gpu_afr_opp: opp-table-gpu-afr { + compatible = "operating-points-v2"; + + /* + * NOTE: The voltage and power values are device-specific and + * must be filled in by the bootloader. + */ + opp00 { + opp-hz = /bits/ 64 <24>; + opp-microvolt = GPU_DIE_REPEAT(668000); + }; + opp01 { + opp-hz = /bits/ 64 <400000000>; + opp-microvolt = GPU_DIE_REPEAT(668000); + }; + opp02 { + opp-hz = /bits/ 64 <552000000>; + opp-microvolt = GPU_DIE_REPEAT(678000); + }; + opp03 { + opp-hz = /bits/ 64 <760000000>; + opp-microvolt = GPU_DIE_REPEAT(737000); + }; + opp04 { + opp-hz = /bits/ 64 <980000000>; + opp-microvolt = GPU_DIE_REPEAT(815000); + }; + opp05 { + opp-hz = /bits/ 64 <1098000000>; + opp-microvolt = GPU_DIE_REPEAT(862000); + }; + opp06 { + opp-hz = /bits/ 64 <1200000000>; + opp-microvolt = GPU_DIE_REPEAT(893000); + }; + }; + pmu-e { compatible = "apple,blizzard-pmu"; interrupt-parent = <&aic>; @@ -423,6 +554,41 @@ clock-output-names = "clk_200m"; }; + clk_disp0: clock-disp0 { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <257142848>; /* TODO: check */ + clock-output-names = "clk_disp0"; + }; + + clk_dispext0: clock-dispext0 { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <0>; + clock-output-names = "clk_dispext0"; + }; + + clk_dispext0_die1: clock-dispext0_die1 { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <0>; + clock-output-names = "clk_dispext0_die1"; + }; + + clk_dispext1: clock-dispext1 { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <0>; + clock-output-names = "clk_dispext1"; + }; + + clk_dispext1_die1: clock-dispext1_die1 { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <0>; + clock-output-names = "clk_dispext1_die1"; + }; + /* * This is a fabulated representation of the input clock * to NCO since we don't know the true clock tree. @@ -451,15 +617,15 @@ }; uat_handoff: uat-handoff { - status = "disabled"; + reg = <0 0 0 0>; }; uat_pagetables: uat-pagetables { - status = "disabled"; + reg = <0 0 0 0>; }; uat_ttbs: uat-ttbs { - status = "disabled"; + reg = <0 0 0 0>; }; }; }; diff --git a/arch/arm64/boot/dts/apple/t602x-die0.dtsi b/arch/arm64/boot/dts/apple/t602x-die0.dtsi index 2e7d2bf08ddc82..c0c0626249a5a6 100644 --- a/arch/arm64/boot/dts/apple/t602x-die0.dtsi +++ b/arch/arm64/boot/dts/apple/t602x-die0.dtsi @@ -7,7 +7,7 @@ */ nco: clock-controller@28e03c000 { - compatible = "apple,t6020-nco", "apple,t8103-nco"; + compatible = "apple,t6020-nco", "apple,t8103-nco", "apple,nco"; reg = <0x2 0x8e03c000 0x0 0x14000>; clocks = <&nco_clkref>; #clock-cells = <1>; @@ -23,8 +23,234 @@ power-domains = <&ps_aic>; }; + pmgr_misc: power-management@28e20c000 { + compatible = "apple,t6020-pmgr-misc", "apple,t6000-pmgr-misc"; + #address-cells = <1>; + #size-cells = <1>; + reg = <0x2 0x8e20c000 0 0x400>, + <0x2 0x8e20c400 0 0x400>; + reg-names = "fabric-ps", "dcs-ps"; + }; + + pmp_dart: iommu@28e300000 { + compatible = "apple,t6020-dart", "apple,t8110-dart"; + reg = <0x2 0x8e300000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_pmp>; + }; + + pmp_report: pmp_report@28e3c0000 { + compatible = "apple,t6020-pmp-v2-report"; + reg = <0x2 0x8e3c0000 0x0 0x20000>; + power-domains = <&ps_pms_sram>; + #address-cells = <1>; + #size-cells = <0>; + + pmp_report_ane_sys: report@b { + compatible = "apple,t6020-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0xb>; + label = "pmp-ane-sys"; + #power-domain-cells = <0>; + power-domains = <&ps_ane_sys>; + status = "disabled"; + }; + + pmp_report_isp_sys: report@c { + compatible = "apple,t6020-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0xc>; + label = "pmp-isp-sys"; + #power-domain-cells = <0>; + power-domains = <&ps_isp_sys>; + status = "disabled"; + }; + + pmp_report_disp0: report@d { + compatible = "apple,t6020-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0xd>; + label = "pmp-disp0"; + #power-domain-cells = <0>; + power-domains = <&ps_disp0_cpu0>; + apple,always-on; + }; + + pmp_report_dispext0: report@e { + compatible = "apple,t6020-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0xe>; + label = "pmp-dispext0"; + #power-domain-cells = <0>; + power-domains = <&ps_dispext0_cpu0>; + apple,always-on; + }; + + pmp_report_dispext1: report@f { + compatible = "apple,t6020-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0xf>; + label = "pmp-dispext1"; + #power-domain-cells = <0>; + power-domains = <&ps_dispext1_cpu0>; + apple,always-on; + }; + + pmp_report_venc_sys: report@10 { + compatible = "apple,t6020-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0x10>; + label = "pmp-venc-sys"; + #power-domain-cells = <0>; + power-domains = <&ps_venc_sys>; + status = "disabled"; + }; + + pmp_report_avd_sys: report@11 { + compatible = "apple,t6020-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0x11>; + label = "pmp-avd-sys"; + #power-domain-cells = <0>; + power-domains = <&ps_avd_sys>; + status = "disabled"; + }; + + pmp_report_msr0: report@12 { + compatible = "apple,t6020-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0x12>; + label = "pmp-msr0"; + #power-domain-cells = <0>; + power-domains = <&ps_msr0>; + status = "disabled"; + }; + + pmp_report_jpg: report@13 { + compatible = "apple,t6020-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0x13>; + label = "pmp-jpg"; + #power-domain-cells = <0>; + power-domains = <&ps_jpg>; + status = "disabled"; + }; + + pmp_report_scodec: report@14 { + compatible = "apple,t6020-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0x14>; + label = "pmp-scodec"; + #power-domain-cells = <0>; + power-domains = <&ps_scodec>; + status = "disabled"; + }; + + pmp_report_afnc4_ioa: report@1d { + compatible = "apple,t6020-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0x1d>; + label = "pmp-afnc4-ioa"; + #power-domain-cells = <0>; + apple,always-on; + }; + + pmp_report_afnc5_ioa: report@1e { + compatible = "apple,t6020-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0x1e>; + label = "pmp-afnc5-ioa"; + #power-domain-cells = <0>; + apple,always-on; + }; + + pmp_report_dispext2: report@1f { + compatible = "apple,t6020-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0x1f>; + label = "pmp-dispext2"; + #power-domain-cells = <0>; + power-domains = <&ps_dispext2_cpu0>; + status = "disabled"; + }; + + pmp_report_dispext3: report@20 { + compatible = "apple,t6020-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0x20>; + label = "pmp-dispext3"; + #power-domain-cells = <0>; + power-domains = <&ps_dispext3_cpu0>; + status = "disabled"; + }; + + pmp_report_venc1: report@21 { + compatible = "apple,t6020-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0x21>; + label = "pmp-venc1"; + #power-domain-cells = <0>; + power-domains = <&ps_venc1_sys>; + status = "disabled"; + }; + + pmp_report_msr1: report@22 { + compatible = "apple,t6020-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0x22>; + label = "pmp-msr1"; + #power-domain-cells = <0>; + power-domains = <&ps_msr1>; + status = "disabled"; + }; + + pmp_report_prores: report@23 { + compatible = "apple,t6020-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0x23>; + label = "pmp-prores"; + #power-domain-cells = <0>; + power-domains = <&ps_prores>; + status = "disabled"; + }; + }; + + pmgr_dcp: power-management@28e3d0000 { + reg = <0x2 0x8e3d0000 0x0 0x4000>; + reg-names = "dcp-fw-pmgr"; + #apple,bw-scratch-cells = <3>; + }; + + pmp: pmp@28e700000 { + compatible = "apple,t6020-pmp-v2", "apple,t6000-pmp-v2"; + reg = <0x2 0x8e700000 0x0 0x100000>, + <0x2 0x8ec00000 0x0 0x4000>; + reg-names = "pmp", "asc"; + mboxes = <&pmp_mbox>; + mbox-names = "mbox"; + iommus = <&pmp_dart 0>; + power-domains = <&ps_pmp>; + status = "disabled"; + }; + + pmp_mbox: mbox@28ec08000 { + compatible = "apple,t6020-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x2 0x8ec08000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + power-domains = <&ps_pmp>, <&ps_pms_sram>; + }; + nub_spmi0: spmi@29e114000 { - compatible = "apple,t6020-spmi", "apple,t8103-spmi"; + compatible = "apple,t6020-spmi", "apple,t8103-spmi", "apple,spmi"; reg = <0x2 0x9e114000 0x0 0x100>; #address-cells = <2>; #size-cells = <0>; @@ -81,7 +307,7 @@ }; wdt: watchdog@29e2c4000 { - compatible = "apple,t6020-wdt", "apple,t8103-wdt"; + compatible = "apple,t6020-wdt", "apple,t8103-wdt", "apple,wdt"; reg = <0x2 0x9e2c4000 0x0 0x4000>; clocks = <&clkref>; interrupt-parent = <&aic>; @@ -102,7 +328,7 @@ }; smc: smc@2a2400000 { - compatible = "apple,t6020-smc", "apple,t8103-smc"; + compatible = "apple,t6020-smc", "apple,t8103-smc", "apple,smc"; reg = <0x2 0xa2400000 0x0 0x4000>, <0x2 0xa3e00000 0x0 0x100000>; reg-names = "smc", "sram"; @@ -114,17 +340,27 @@ #gpio-cells = <2>; }; + smc_hwmon: hwmon { + compatible = "apple,smc-hwmon"; + }; + smc_reboot: reboot { compatible = "apple,smc-reboot"; nvmem-cells = <&shutdown_flag>, <&boot_stage>, - <&boot_error_count>, <&panic_count>; + <&boot_error_count>, <&panic_count>, <&pm_setting>; nvmem-cell-names = "shutdown_flag", "boot_stage", - "boot_error_count", "panic_count"; + "boot_error_count", "panic_count", "pm_setting"; + }; + + rtc { + compatible = "apple,smc-rtc"; + nvmem-cells = <&rtc_offset>; + nvmem-cell-names = "rtc_offset"; }; }; pinctrl_smc: pinctrl@2a2820000 { - compatible = "apple,t6020-pinctrl", "apple,t8103-pinctrl"; + compatible = "apple,t6020-pinctrl", "apple,t8103-pinctrl", "apple,pinctrl"; reg = <0x2 0xa2820000 0x0 0x4000>; gpio-controller; @@ -144,13 +380,317 @@ ; }; - sio_dart: iommu@39b008000 { + aop_mbox: mbox@2a6408000 { + compatible = "apple,t6020-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x2 0xa6408000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + status = "disabled"; + }; + + aop_dart: iommu@2a6808000 { compatible = "apple,t6020-dart", "apple,t8110-dart"; - reg = <0x3 0x9b008000 0x0 0x8000>; + reg = <0x2 0xa6808000 0x0 0x4000>; + #iommu-cells = <1>; interrupt-parent = <&aic>; - interrupts = ; + interrupts = ; + status = "disabled"; + // apple,dma-range = <0x100 0x0 0x300 0x0>; + }; + + aop_admac: dma-controller@2a6980000 { + compatible = "apple,t6020-admac", "apple,admac"; + reg = <0x2 0xa6980000 0x0 0x34000>; + #dma-cells = <1>; + dma-channels = <16>; + interrupts-extended = <0>, + <0>, + <&aic AIC_IRQ 0 631 IRQ_TYPE_LEVEL_HIGH>, + <0>; + iommus = <&aop_dart 10>; + status = "disabled"; + }; + + aop: aop@2a6c00000 { + compatible = "apple,t6020-aop"; + reg = <0x2 0xa6c00000 0x0 0x250000>, + <0x2 0xa6400000 0x0 0x6c000>; + mboxes = <&aop_mbox>; + mbox-names = "mbox"; + iommus = <&aop_dart 0>; + + status = "disabled"; + + aop_audio: audio { + compatible = "apple,t6020-aop-audio", "apple,aop-audio"; + dmas = <&aop_admac 1>; + dma-names = "dma"; + }; + + aop_als: als { + compatible = "apple,t6020-aop-als", "apple,aop-als"; + // intentionally empty + }; + + las { + compatible = "apple,t6020-aop-las", "apple,aop-las"; + }; + }; + + mtp: mtp@2a9400000 { + compatible = "apple,t6020-mtp", "apple,t6020-rtk-helper-asc4", "apple,mtp", "apple,rtk-helper-asc4"; + reg = <0x2 0xa9400000 0x0 0x4000>, + <0x2 0xa9c00000 0x0 0x100000>; + reg-names = "asc", "sram"; + mboxes = <&mtp_mbox>; + iommus = <&mtp_dart 1>; + #helper-cells = <0>; + + status = "disabled"; + }; + + mtp_mbox: mbox@2a9408000 { + compatible = "apple,t6020-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x2 0xa9408000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + + status = "disabled"; + }; + + mtp_dart: iommu@2a9808000 { + compatible = "apple,t6020-dart", "apple,t8110-dart"; + reg = <0x2 0xa9808000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + #iommu-cells = <1>; + + apple,dma-range = <0x100 0x0 0x1 0x0>; + + status = "disabled"; + }; + + mtp_dockchannel: fifo@2a9b14000 { + compatible = "apple,t6020-dockchannel", "apple,dockchannel"; + reg = <0x2 0xa9b14000 0x0 0x4000>; + reg-names = "irq"; + interrupt-parent = <&aic>; + interrupts = ; + + ranges = <0 0x2 0xa9b28000 0x20000>; + nonposted-mmio; + #address-cells = <1>; + #size-cells = <1>; + + interrupt-controller; + #interrupt-cells = <2>; + + status = "disabled"; + + mtp_hid: input@8000 { + compatible = "apple,dockchannel-hid"; + reg = <0x8000 0x4000>, + <0xc000 0x4000>, + <0x0000 0x4000>, + <0x4000 0x4000>; + reg-names = "config", "data", + "rmt-config", "rmt-data"; + iommus = <&mtp_dart 1>; + interrupt-parent = <&mtp_dockchannel>; + interrupts = <2 IRQ_TYPE_LEVEL_HIGH>, + <3 IRQ_TYPE_LEVEL_HIGH>; + interrupt-names = "tx", "rx"; + + apple,fifo-size = <0x800>; + apple,helper-cpu = <&mtp>; + }; + + }; + + isp_dart0: iommu@3860e8000 { + compatible = "apple,t6020-dart", "apple,t8110-dart"; + reg = <0x3 0x860e8000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + #ifdef APPLE_USE_PMP + power-domains = <&pmp_report_isp_sys>; + #else + power-domains = <&ps_isp_sys>; + #endif + + apple,dma-range = <0x100 0x0 0x1 0x0>; + status = "disabled"; + }; + + isp_dart1: iommu@3860f4000 { + compatible = "apple,t6020-dart", "apple,t8110-dart"; + reg = <0x3 0x860f4000 0x0 0x4000>; #iommu-cells = <1>; - power-domains = <&ps_sio_cpu>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_isp_sys>; + + apple,dma-range = <0x100 0x0 0x1 0x0>; + status = "disabled"; + }; + + isp_dart2: iommu@3860fc000 { + compatible = "apple,t6020-dart", "apple,t8110-dart"; + reg = <0x3 0x860fc000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_isp_sys>; + + apple,dma-range = <0x100 0x0 0x1 0x0>; + status = "disabled"; + }; + + isp: isp@384000000 { + compatible = "apple,t6020-isp", "apple,isp"; + iommus = <&isp_dart0 0>, <&isp_dart1 0>, <&isp_dart2 0>; + reg-names = "coproc", "mbox", "gpio", "mbox2"; + reg = <0x3 0x84000000 0x0 0x2000000>, + <0x3 0x86104000 0x0 0x100>, + <0x3 0x86104170 0x0 0x100>, + <0x3 0x861043f0 0x0 0x100>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_isp_cpu>, <&ps_isp_fe>, + <&ps_dprx>, <&ps_isp_vis>, <&ps_isp_be>, + <&ps_isp_clr>, <&ps_isp_raw>; + apple,dart-vm-size = <0x0 0xa0000000>; + + status = "disabled"; + }; + + disp0_dart: iommu@389304000 { + compatible = "apple,t6020-dart", "apple,t8110-dart"; + reg = <0x3 0x89304000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + status = "disabled"; + power-domains = <&ps_disp0_cpu0>; + apple,dma-range = <0x100 0x0 0x10 0x0>; + }; + + dcp_dart: iommu@38930c000 { + compatible = "apple,t6020-dart", "apple,t8110-dart"; + reg = <0x3 0x8930c000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_disp0_cpu0>; + apple,dma-range = <0x100 0x0 0x10 0x0>; + }; + + dcp_mbox: mbox@389c08000 { + compatible = "apple,t6020-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x3 0x89c08000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + power-domains = <&ps_disp0_cpu0>; + }; + + dcp: dcp@389c00000 { + compatible = "apple,t6020-dcp", "apple,dcp"; + mboxes = <&dcp_mbox>; + mbox-names = "mbox"; + iommus = <&dcp_dart 5>; + + reg-names = "coproc", "disp-0", "disp-1", "disp-2", "disp-3"; + reg = <0x3 0x89c00000 0x0 0x4000>, // check? + <0x3 0x88000000 0x0 0x61c000>, + <0x3 0x89320000 0x0 0x4000>, + <0x3 0x89344000 0x0 0x4000>, + <0x3 0x89800000 0x0 0x800000>; + apple,bw-scratch = <&pmgr_dcp 0 4 0x1208>; + #ifdef APPLE_USE_PMP + power-domains = <&pmp_report_disp0>; + #else + power-domains = <&ps_disp0_cpu0>; + #endif + resets = <&ps_disp0_cpu0>; + clocks = <&clk_disp0>; + phandle = <&dcp>; + // required bus properties for 'piodma' subdevice + #address-cells = <2>; + #size-cells = <2>; + + disp0_piodma: piodma { + iommus = <&disp0_dart 4>; + phandle = <&disp0_piodma>; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + dcp_audio: endpoint { + remote-endpoint = <&dpaudio0_dcp>; + }; + }; + }; + }; + + display: display-subsystem { + compatible = "apple,display-subsystem"; + iommus = <&disp0_dart 0>; + /* generate phandle explicitly for use in loader */ + phandle = <&display>; + }; + + sep_dart: iommu@394ac0000 { + compatible = "apple,t6020-dart", "apple,t8110-dart"; + reg = <0x3 0x94ac0000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + status = "disabled"; + }; + + sep: sep@396400000 { + compatible = "apple,sep"; + reg = <0x3 0x96400000 0x0 0x6C000>; + mboxes = <&sep_mbox>; + mbox-names = "mbox"; + iommus = <&sep_dart 0>; + power-domains = <&ps_sep>; + status = "disabled"; + }; + + sep_mbox: mbox@396408000 { + compatible = "apple,t6020-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x3 0x96408000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; }; fpwm0: pwm@39b030000 { @@ -163,7 +703,7 @@ }; i2c0: i2c@39b040000 { - compatible = "apple,t6020-i2c", "apple,t8103-i2c"; + compatible = "apple,t6020-i2c", "apple,t8103-i2c", "apple,i2c"; reg = <0x3 0x9b040000 0x0 0x4000>; clocks = <&clkref>; interrupt-parent = <&aic>; @@ -176,7 +716,7 @@ }; i2c1: i2c@39b044000 { - compatible = "apple,t6020-i2c", "apple,t8103-i2c"; + compatible = "apple,t6020-i2c", "apple,t8103-i2c", "apple,i2c"; reg = <0x3 0x9b044000 0x0 0x4000>; clocks = <&clkref>; interrupt-parent = <&aic>; @@ -190,7 +730,7 @@ }; i2c2: i2c@39b048000 { - compatible = "apple,t6020-i2c", "apple,t8103-i2c"; + compatible = "apple,t6020-i2c", "apple,t8103-i2c", "apple,i2c"; reg = <0x3 0x9b048000 0x0 0x4000>; clocks = <&clkref>; interrupt-parent = <&aic>; @@ -204,7 +744,7 @@ }; i2c3: i2c@39b04c000 { - compatible = "apple,t6020-i2c", "apple,t8103-i2c"; + compatible = "apple,t6020-i2c", "apple,t8103-i2c", "apple,i2c"; reg = <0x3 0x9b04c000 0x0 0x4000>; clocks = <&clkref>; interrupt-parent = <&aic>; @@ -218,7 +758,7 @@ }; i2c4: i2c@39b050000 { - compatible = "apple,t6020-i2c", "apple,t8103-i2c"; + compatible = "apple,t6020-i2c", "apple,t8103-i2c", "apple,i2c"; reg = <0x3 0x9b050000 0x0 0x4000>; clocks = <&clkref>; interrupt-parent = <&aic>; @@ -232,7 +772,7 @@ }; i2c5: i2c@39b054000 { - compatible = "apple,t6020-i2c", "apple,t8103-i2c"; + compatible = "apple,t6020-i2c", "apple,t8103-i2c", "apple,i2c"; reg = <0x3 0x9b054000 0x0 0x4000>; clocks = <&clkref>; interrupt-parent = <&aic>; @@ -246,7 +786,7 @@ }; i2c6: i2c@39b054000 { - compatible = "apple,t6020-i2c", "apple,t8103-i2c"; + compatible = "apple,t6020-i2c", "apple,t8103-i2c", "apple,i2c"; reg = <0x3 0x9b054000 0x0 0x4000>; clocks = <&clkref>; interrupt-parent = <&aic>; @@ -260,7 +800,7 @@ }; i2c7: i2c@39b054000 { - compatible = "apple,t6020-i2c", "apple,t8103-i2c"; + compatible = "apple,t6020-i2c", "apple,t8103-i2c", "apple,i2c"; reg = <0x3 0x9b054000 0x0 0x4000>; clocks = <&clkref>; interrupt-parent = <&aic>; @@ -274,7 +814,7 @@ }; i2c8: i2c@39b054000 { - compatible = "apple,t6020-i2c", "apple,t8103-i2c"; + compatible = "apple,t6020-i2c", "apple,t8103-i2c", "apple,i2c"; reg = <0x3 0x9b054000 0x0 0x4000>; clocks = <&clkref>; interrupt-parent = <&aic>; @@ -288,7 +828,7 @@ }; spi1: spi@39b104000 { - compatible = "apple,t6020-spi", "apple,t8103-spi"; + compatible = "apple,t6020-spi", "apple,t8103-spi", "apple,spi"; reg = <0x3 0x9b104000 0x0 0x4000>; interrupt-parent = <&aic>; interrupts = ; @@ -302,7 +842,7 @@ }; spi2: spi@39b108000 { - compatible = "apple,t6020-spi", "apple,t8103-spi"; + compatible = "apple,t6020-spi", "apple,t8103-spi", "apple,spi"; reg = <0x3 0x9b108000 0x0 0x4000>; interrupt-parent = <&aic>; interrupts = ; @@ -316,7 +856,7 @@ }; spi4: spi@39b110000 { - compatible = "apple,t6020-spi", "apple,t8103-spi"; + compatible = "apple,t6020-spi", "apple,t8103-spi", "apple,spi"; reg = <0x3 0x9b110000 0x0 0x4000>; interrupt-parent = <&aic>; interrupts = ; @@ -346,7 +886,7 @@ }; admac: dma-controller@39b400000 { - compatible = "apple,t6020-admac", "apple,t8103-admac"; + compatible = "apple,t6020-admac", "apple,t8103-admac", "apple,admac"; reg = <0x3 0x9b400000 0x0 0x34000>; #dma-cells = <1>; dma-channels = <16>; @@ -359,8 +899,29 @@ resets = <&ps_audio_p>; }; + dpaudio0: audio-controller@39b500000 { + compatible = "apple,t6020-dpaudio", "apple,dpaudio"; + reg = <0x3 0x9b500000 0x0 0x4000>; + dmas = <&sio 0x64>; + dma-names = "tx"; + power-domains = <&ps_dpa0>; + reset-domains = <&ps_dpa0>; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + dpaudio0_dcp: endpoint { + remote-endpoint = <&dcp_audio>; + }; + }; + }; + }; + mca: mca@39b600000 { - compatible = "apple,t6020-mca", "apple,t8103-mca"; + compatible = "apple,t6020-mca", "apple,t8103-mca", "apple,mca"; reg = <0x3 0x9b600000 0x0 0x10000>, <0x3 0x9b500000 0x0 0x20000>; clocks = <&nco 0>, <&nco 1>, <&nco 2>, <&nco 3>; @@ -388,6 +949,14 @@ reg = <0x4 0x6400000 0 0x40000>, <0x4 0x4000000 0 0x1000000>; reg-names = "asc", "sgx"; + interrupt-parent = <&aic>; + interrupts = , + , + , + , + , + , + ; mboxes = <&agx_mbox>; power-domains = <&ps_gfx>; memory-region = <&uat_ttbs>, <&uat_pagetables>, <&uat_handoff>, @@ -396,6 +965,55 @@ "hw-cal-a", "hw-cal-b", "globals"; apple,firmware-abi = <0 0 0>; + apple,firmware-version = <0 0 0>; + apple,firmware-compat = <0 0 0>; + + operating-points-v2 = <&gpu_opp>; + apple,cs-opp = <&gpu_cs_opp>; + apple,afr-opp = <&gpu_afr_opp>; + + apple,min-sram-microvolt = <790000>; + apple,csafr-min-sram-microvolt = <812000>; + apple,perf-base-pstate = <1>; + + apple,avg-power-min-duty-cycle = <40>; + apple,avg-power-target-filter-tc = <1>; + apple,fast-die0-proportional-gain = <34.0>; + apple,perf-boost-ce-step = <50>; + apple,perf-boost-min-util = <90>; + apple,perf-filter-drop-threshold = <0>; + apple,perf-filter-time-constant = <5>; + apple,perf-filter-time-constant2 = <200>; + apple,perf-integral-gain = <1.62>; + apple,perf-integral-gain2 = <1.62>; + apple,perf-integral-min-clamp = <0>; + apple,perf-proportional-gain2 = <5.4>; + apple,perf-proportional-gain = <5.4>; + apple,perf-tgt-utilization = <85>; + apple,power-sample-period = <8>; + apple,ppm-filter-time-constant-ms = <34>; + apple,ppm-ki = <18.0>; + apple,ppm-kp = <0.1>; + apple,pwr-filter-time-constant = <313>; + apple,pwr-integral-gain = <0.0202129>; + apple,pwr-integral-min-clamp = <0>; + apple,pwr-min-duty-cycle = <40>; + apple,pwr-proportional-gain = <5.2831855>; + apple,pwr-sample-period-aic-clks = <200000>; + apple,se-engagement-criteria = <700>; + apple,se-filter-time-constant = <9>; + apple,se-filter-time-constant-1 = <3>; + apple,se-inactive-threshold = <2500>; + apple,se-ki = <-50.0>; + apple,se-ki-1 = <-100.0>; + apple,se-kp = <-5.0>; + apple,se-kp-1 = <-10.0>; + apple,se-reset-criteria = <50>; + + apple,core-leak-coef = GPU_REPEAT(1200.0); + apple,sram-leak-coef = GPU_REPEAT(20.0); + apple,cs-leak-coef = GPU_DIE_REPEAT(400.0); + apple,afr-leak-coef = GPU_DIE_REPEAT(200.0); }; agx_mbox: mbox@406408000 { @@ -455,6 +1073,8 @@ pinctrl-0 = <&pcie_pins>; pinctrl-names = "default"; + dma-coherent; + port00: pci@0,0 { device_type = "pci"; reg = <0x0 0x0 0x0 0x0 0x0>; diff --git a/arch/arm64/boot/dts/apple/t602x-dieX.dtsi b/arch/arm64/boot/dts/apple/t602x-dieX.dtsi index cb07fd82b32e67..ae0038a4c28710 100644 --- a/arch/arm64/boot/dts/apple/t602x-dieX.dtsi +++ b/arch/arm64/boot/dts/apple/t602x-dieX.dtsi @@ -23,6 +23,87 @@ #performance-domain-cells = <0>; }; + DIE_NODE(dispext0_dart): iommu@289304000 { + compatible = "apple,t6020-dart", "apple,t8110-dart"; + reg = <0x2 0x89304000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&DIE_NODE(ps_dispext0_cpu0)>; + apple,dma-range = <0x100 0x0 0x10 0x0>; + status = "disabled"; + }; + + DIE_NODE(dcpext0_dart): iommu@28930c000 { + compatible = "apple,t6020-dart", "apple,t8110-dart"; + reg = <0x2 0x8930c000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&DIE_NODE(ps_dispext0_cpu0)>; + apple,dma-range = <0x100 0x0 0x10 0x0>; + status = "disabled"; + }; + + DIE_NODE(dcpext0_mbox): mbox@289c08000 { + compatible = "apple,t6020-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x2 0x89c08000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + power-domains = <&DIE_NODE(ps_dispext0_cpu0)>; + resets = <&DIE_NODE(ps_dispext0_cpu0)>; + status = "disabled"; + }; + + DIE_NODE(dcpext0): dcp@289c00000 { + compatible = "apple,t6020-dcpext", "apple,dcpext"; + mboxes = <&DIE_NODE(dcpext0_mbox)>; + mbox-names = "mbox"; + iommus = <&DIE_NODE(dcpext0_dart) 5>; + + reg-names = "coproc", "disp-0", "disp-1", "disp-2", "disp-3"; + reg = <0x2 0x89c00000 0x0 0x4000>, + <0x2 0x88000000 0x0 0x4000000>, + <0x2 0x89320000 0x0 0x4000>, + <0x2 0x89344000 0x0 0x4000>, + <0x2 0x89800000 0x0 0x800000>; + apple,bw-scratch = <&pmgr_dcp 0 4 0x1210>; + #ifdef APPLE_USE_PMP + power-domains = <&DIE_NODE(pmp_report_dispext0)>; + #else + power-domains = <&DIE_NODE(ps_dispext0_cpu0)>; + #endif + resets = <&DIE_NODE(ps_dispext0_cpu0)>; + clocks = <&DIE_NODE(clk_dispext0)>; + phandle = <&DIE_NODE(dcpext0)>; + apple,dcp-index = <1>; + status = "disabled"; + // required bus properties for 'piodma' subdevice + #address-cells = <2>; + #size-cells = <2>; + + piodma { + iommus = <&DIE_NODE(dispext0_dart) 4>; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + DIE_NODE(dcpext0_audio): endpoint { + remote-endpoint = <&DIE_NODE(dpaudio1_dcp)>; + }; + }; + }; + }; + DIE_NODE(pmgr): power-management@28e080000 { compatible = "apple,t6020-pmgr", "apple,t8103-pmgr", "syscon", "simple-mfd"; #address-cells = <1>; @@ -45,7 +126,7 @@ }; DIE_NODE(pinctrl_nub): pinctrl@29e1f0000 { - compatible = "apple,t6020-pinctrl", "apple,t8103-pinctrl"; + compatible = "apple,t6020-pinctrl", "apple,t8103-pinctrl", "apple,pinctrl"; reg = <0x2 0x9e1f0000 0x0 0x4000>; power-domains = <&DIE_NODE(ps_nub_gpio)>; @@ -74,7 +155,7 @@ }; DIE_NODE(pinctrl_aop): pinctrl@2a6820000 { - compatible = "apple,t6020-pinctrl", "apple,t8103-pinctrl"; + compatible = "apple,t6020-pinctrl", "apple,t8103-pinctrl", "apple,pinctrl"; reg = <0x2 0xa6820000 0x0 0x4000>; gpio-controller; @@ -94,8 +175,99 @@ ; }; + DIE_NODE(dispext1_dart): iommu@315304000 { + compatible = "apple,t6020-dart", "apple,t8110-dart"; + reg = <0x3 0x15304000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&DIE_NODE(ps_dispext1_cpu0)>; + apple,dma-range = <0x100 0x0 0x10 0x0>; + status = "disabled"; + }; + + DIE_NODE(dcpext1_dart): iommu@31530c000 { + compatible = "apple,t6020-dart", "apple,t8110-dart"; + reg = <0x3 0x1530c000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&DIE_NODE(ps_dispext1_cpu0)>; + apple,dma-range = <0x100 0x0 0x10 0x0>; + status = "disabled"; + }; + + DIE_NODE(dcpext1_mbox): mbox@315c08000 { + compatible = "apple,t6020-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x3 0x15c08000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + power-domains = <&DIE_NODE(ps_dispext1_cpu0)>; + resets = <&DIE_NODE(ps_dispext1_cpu0)>; + status = "disabled"; + }; + + DIE_NODE(dcpext1): dcp@315c00000 { + compatible = "apple,t6020-dcpext", "apple,dcpext"; + mboxes = <&DIE_NODE(dcpext1_mbox)>; + mbox-names = "mbox"; + iommus = <&DIE_NODE(dcpext1_dart) 5>; + + reg-names = "coproc", "disp-0", "disp-1", "disp-2", "disp-3"; + reg = <0x3 0x15c00000 0x0 0x4000>, + <0x3 0x14000000 0x0 0x4000000>, + <0x3 0x15320000 0x0 0x4000>, + <0x3 0x15344000 0x0 0x4000>, + <0x3 0x15800000 0x0 0x800000>; + apple,bw-scratch = <&pmgr_dcp 0 4 0x1218>; + #ifdef APPLE_USE_PMP + power-domains = <&DIE_NODE(pmp_report_dispext1)>; + #else + power-domains = <&DIE_NODE(ps_dispext1_cpu0)>; + #endif + resets = <&DIE_NODE(ps_dispext1_cpu0)>; + clocks = <&DIE_NODE(clk_dispext1)>; + phandle = <&DIE_NODE(dcpext1)>; + apple,dcp-index = <2>; + status = "disabled"; + // required bus properties for 'piodma' subdevice + #address-cells = <2>; + #size-cells = <2>; + + piodma { + iommus = <&DIE_NODE(dispext1_dart) 4>; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + DIE_NODE(dcpext1_audio): endpoint { + remote-endpoint = <&DIE_NODE(dpaudio2_dcp)>; + }; + }; + }; + }; + + DIE_NODE(sio_dart): iommu@39b008000 { + compatible = "apple,t6020-dart", "apple,t8110-dart"; + reg = <0x3 0x9b008000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + #iommu-cells = <1>; + power-domains = <&DIE_NODE(ps_sio)>; + //apple,dma-range = <0x100 0x0001c000 0x2ff 0xfffe4000>; + }; + DIE_NODE(pinctrl_ap): pinctrl@39b028000 { - compatible = "apple,t6020-pinctrl", "apple,t8103-pinctrl"; + compatible = "apple,t6020-pinctrl", "apple,t8103-pinctrl", "apple,pinctrl"; reg = <0x3 0x9b028000 0x0 0x4000>; interrupt-parent = <&aic>; @@ -119,6 +291,131 @@ #interrupt-cells = <2>; }; + DIE_NODE(sio_mbox): mbox@39bc08000 { + compatible = "apple,t6020-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x3 0x9bc08000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + power-domains = <&DIE_NODE(ps_sio_cpu)>; + }; + + DIE_NODE(sio): sio@39bc00000 { + compatible = "apple,t6020-sio", "apple,sio"; + reg = <0x3 0x9bc00000 0x0 0x8000>; + dma-channels = <128>; + #dma-cells = <1>; + mboxes = <&DIE_NODE(sio_mbox)>; + iommus = <&DIE_NODE(sio_dart) 0>; + power-domains = <&DIE_NODE(ps_sio_cpu)>; + resets = <&DIE_NODE(ps_sio_cpu)>; + status = "disabled"; + }; + + DIE_NODE(dpaudio1): audio-controller@39b504000 { + compatible = "apple,t6020-dpaudio", "apple,dpaudio"; + reg = <0x3 0x9b540000 0x0 0x4000>; + dmas = <&DIE_NODE(sio) 0x66>; + dma-names = "tx"; + power-domains = <&DIE_NODE(ps_dpa1)>; + reset-domains = <&DIE_NODE(ps_dpa1)>; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + DIE_NODE(dpaudio1_dcp): endpoint { + remote-endpoint = <&DIE_NODE(dcpext0_audio)>; + }; + }; + }; + }; + + DIE_NODE(dpaudio2): audio-controller@39b508000 { + compatible = "apple,t6020-dpaudio", "apple,dpaudio"; + reg = <0x3 0x9b580000 0x0 0x4000>; + dmas = <&DIE_NODE(sio) 0x68>; + dma-names = "tx"; + power-domains = <&DIE_NODE(ps_dpa2)>; + reset-domains = <&DIE_NODE(ps_dpa2)>; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + DIE_NODE(dpaudio2_dcp): endpoint { + remote-endpoint = <&DIE_NODE(dcpext1_audio)>; + }; + }; + }; + }; + + /* + * omit dpaudio3 / 4 as long as the linked dcpext nodes don't exist + * + DIE_NODE(dpaudio3): audio-controller@39b50c000 { + compatible = "apple,t6020-dpaudio", "apple,dpaudio"; + reg = <0x3 0x9b5c0000 0x0 0x4000>; + dmas = <&DIE_NODE(sio) 0x6a>; + dma-names = "tx"; + power-domains = <&DIE_NODE(ps_dpa3)>; + reset-domains = <&DIE_NODE(ps_dpa3)>; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + DIE_NODE(dpaudio3_dcp): endpoint { + remote-endpoint = <&DIE_NODE(dcpext2_audio)>; + }; + }; + }; + }; + + DIE_NODE(dpaudio4): audio-controller@39b510000 { + compatible = "apple,t6020-dpaudio", "apple,dpaudio"; + reg = <0x3 0x9b500000 0x0 0x4000>; + dmas = <&DIE_NODE(sio) 0x6c>; + dma-names = "tx"; + power-domains = <&DIE_NODE(ps_dpa4)>; + reset-domains = <&DIE_NODE(ps_dpa4)>; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + DIE_NODE(dpaudio4_dcp): endpoint { + remote-endpoint = <&DIE_NODE(dcpext3_audio)>; + }; + }; + }; + }; + */ + + DIE_NODE(lpdptxphy): phy@39c000000 { + compatible = "apple,t6020-dptx-phy", "apple,dptx-phy"; + reg = <0x3 0x9c000000 0x0 0x4000>, + <0x3 0x9c040000 0x0 0xc000>; + reg-names = "core", "dptx"; + power-domains = <&DIE_NODE(ps_dptx_phy_ps)>; + #phy-cells = <0>; + #reset-cells = <0>; + status = "disabled"; /* only exposed on desktop devices */ + }; + DIE_NODE(pmgr_gfx): power-management@404e80000 { compatible = "apple,t6020-pmgr", "apple,t8103-pmgr", "syscon", "simple-mfd"; #address-cells = <1>; @@ -126,3 +423,247 @@ reg = <0x4 0x4e80000 0 0x4000>; }; + + DIE_NODE(dwc3_0): usb@702280000 { + compatible = "apple,t6020-dwc3", "apple,t8103-dwc3"; + reg = <0x7 0x02280000 0x0 0xcd00>, <0x7 0x0228cd00 0x0 0x3200>; + reg-names = "dwc3-core", "dwc3-apple"; + interrupt-parent = <&aic>; + interrupts = ; + dr_mode = "otg"; + usb-role-switch; + role-switch-default-mode = "host"; + iommus = <&DIE_NODE(dwc3_0_dart_0) 0>, + <&DIE_NODE(dwc3_0_dart_1) 1>; + power-domains = <&DIE_NODE(ps_atc0_usb)>; + resets = <&DIE_NODE(atcphy0)>; + phys = <&DIE_NODE(atcphy0) PHY_TYPE_USB2>, <&DIE_NODE(atcphy0) PHY_TYPE_USB3>; + phy-names = "usb2-phy", "usb3-phy"; + }; + + DIE_NODE(dwc3_0_dart_0): iommu@702f00000 { + compatible = "apple,t6020-dart", "apple,t8110-dart"; + reg = <0x7 0x02f00000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&DIE_NODE(ps_atc0_usb)>; + #iommu-cells = <1>; + }; + + DIE_NODE(dwc3_0_dart_1): iommu@702f80000 { + compatible = "apple,t6020-dart", "apple,t8110-dart"; + reg = <0x7 0x02f80000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&DIE_NODE(ps_atc0_usb)>; + #iommu-cells = <1>; + }; + + DIE_NODE(atcphy0): phy@703000000 { + compatible = "apple,t6020-atcphy", "apple,t8103-atcphy"; + reg = <0x7 0x03000000 0x0 0x4c000>, + <0x7 0x03050000 0x0 0x8000>, + <0x7 0x00000000 0x0 0x4000>, + <0x7 0x02a90000 0x0 0x4000>, + <0x7 0x02a84000 0x0 0x4000>; + reg-names = "core", "lpdptx", "axi2af", "usb2phy", + "pipehandler"; + + #phy-cells = <1>; + #reset-cells = <0>; + + orientation-switch; + mode-switch; + power-domains = <&DIE_NODE(ps_atc0_usb)>; + }; + + DIE_NODE(atcphy0_xbar): mux@70304c000 { + compatible = "apple,t6020-display-crossbar"; + reg = <0x7 0x0304c000 0x0 0x4000>; + #mux-control-cells = <1>; + power-domains = <&DIE_NODE(ps_atc0_usb)>; + status = "disabled"; + }; + + DIE_NODE(dwc3_1): usb@b02280000 { + compatible = "apple,t6020-dwc3", "apple,t8103-dwc3"; + reg = <0xb 0x02280000 0x0 0xcd00>, <0xb 0x0228cd00 0x0 0x3200>; + reg-names = "dwc3-core", "dwc3-apple"; + interrupt-parent = <&aic>; + interrupts = ; + dr_mode = "otg"; + usb-role-switch; + role-switch-default-mode = "host"; + iommus = <&DIE_NODE(dwc3_1_dart_0) 0>, + <&DIE_NODE(dwc3_1_dart_1) 1>; + power-domains = <&DIE_NODE(ps_atc1_usb)>; + resets = <&DIE_NODE(atcphy1)>; + phys = <&DIE_NODE(atcphy1) PHY_TYPE_USB2>, <&DIE_NODE(atcphy1) PHY_TYPE_USB3>; + phy-names = "usb2-phy", "usb3-phy"; + }; + + DIE_NODE(dwc3_1_dart_0): iommu@b02f00000 { + compatible = "apple,t6020-dart", "apple,t8110-dart"; + reg = <0xb 0x02f00000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&DIE_NODE(ps_atc1_usb)>; + #iommu-cells = <1>; + }; + + DIE_NODE(dwc3_1_dart_1): iommu@b02f80000 { + compatible = "apple,t6020-dart", "apple,t8110-dart"; + reg = <0xb 0x02f80000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&DIE_NODE(ps_atc1_usb)>; + #iommu-cells = <1>; + }; + + DIE_NODE(atcphy1): phy@b03000000 { + compatible = "apple,t6020-atcphy", "apple,t8103-atcphy"; + reg = <0xb 0x03000000 0x0 0x4c000>, + <0xb 0x03050000 0x0 0x8000>, + <0xb 0x00000000 0x0 0x4000>, + <0xb 0x02a90000 0x0 0x4000>, + <0xb 0x02a84000 0x0 0x4000>; + reg-names = "core", "lpdptx", "axi2af", "usb2phy", + "pipehandler"; + + #phy-cells = <1>; + #reset-cells = <0>; + + orientation-switch; + mode-switch; + power-domains = <&DIE_NODE(ps_atc1_usb)>; + }; + + DIE_NODE(atcphy1_xbar): mux@b0304c000 { + compatible = "apple,t6020-display-crossbar"; + reg = <0xb 0x0304c000 0x0 0x4000>; + #mux-control-cells = <1>; + power-domains = <&DIE_NODE(ps_atc1_usb)>; + status = "disabled"; + }; + + DIE_NODE(dwc3_2): usb@f02280000 { + compatible = "apple,t6020-dwc3", "apple,t8103-dwc3"; + reg = <0xf 0x02280000 0x0 0xcd00>, <0xf 0x0228cd00 0x0 0x3200>; + reg-names = "dwc3-core", "dwc3-apple"; + interrupt-parent = <&aic>; + interrupts = ; + dr_mode = "otg"; + usb-role-switch; + role-switch-default-mode = "host"; + iommus = <&DIE_NODE(dwc3_2_dart_0) 0>, + <&DIE_NODE(dwc3_2_dart_1) 1>; + power-domains = <&DIE_NODE(ps_atc2_usb)>; + resets = <&DIE_NODE(atcphy2)>; + phys = <&DIE_NODE(atcphy2) PHY_TYPE_USB2>, <&DIE_NODE(atcphy2) PHY_TYPE_USB3>; + phy-names = "usb2-phy", "usb3-phy"; + }; + + DIE_NODE(dwc3_2_dart_0): iommu@f02f00000 { + compatible = "apple,t6020-dart", "apple,t8110-dart"; + reg = <0xf 0x02f00000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&DIE_NODE(ps_atc2_usb)>; + #iommu-cells = <1>; + }; + + DIE_NODE(dwc3_2_dart_1): iommu@f02f80000 { + compatible = "apple,t6020-dart", "apple,t8110-dart"; + reg = <0xf 0x02f80000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&DIE_NODE(ps_atc2_usb)>; + #iommu-cells = <1>; + }; + + DIE_NODE(atcphy2): phy@f03000000 { + compatible = "apple,t6020-atcphy", "apple,t8103-atcphy"; + reg = <0xf 0x03000000 0x0 0x4c000>, + <0xf 0x03050000 0x0 0x8000>, + <0xf 0x00000000 0x0 0x4000>, + <0xf 0x02a90000 0x0 0x4000>, + <0xf 0x02a84000 0x0 0x4000>; + reg-names = "core", "lpdptx", "axi2af", "usb2phy", + "pipehandler"; + + #phy-cells = <1>; + #reset-cells = <0>; + + orientation-switch; + mode-switch; + power-domains = <&DIE_NODE(ps_atc2_usb)>; + }; + + DIE_NODE(atcphy2_xbar): mux@f0304c000 { + compatible = "apple,t6020-display-crossbar"; + reg = <0xf 0x0304c000 0x0 0x4000>; + #mux-control-cells = <1>; + power-domains = <&DIE_NODE(ps_atc2_usb)>; + status = "disabled"; + }; + + DIE_NODE(dwc3_3): usb@1302280000 { + compatible = "apple,t6020-dwc3", "apple,t8103-dwc3"; + reg = <0x13 0x02280000 0x0 0xcd00>, <0x13 0x0228cd00 0x0 0x3200>; + reg-names = "dwc3-core", "dwc3-apple"; + interrupt-parent = <&aic>; + interrupts = ; + dr_mode = "otg"; + usb-role-switch; + role-switch-default-mode = "host"; + iommus = <&DIE_NODE(dwc3_3_dart_0) 0>, + <&DIE_NODE(dwc3_3_dart_1) 1>; + power-domains = <&DIE_NODE(ps_atc3_usb)>; + resets = <&DIE_NODE(atcphy3)>; + phys = <&DIE_NODE(atcphy3) PHY_TYPE_USB2>, <&DIE_NODE(atcphy3) PHY_TYPE_USB3>; + phy-names = "usb2-phy", "usb3-phy"; + }; + + DIE_NODE(dwc3_3_dart_0): iommu@1302f00000 { + compatible = "apple,t6020-dart", "apple,t8110-dart"; + reg = <0x13 0x02f00000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&DIE_NODE(ps_atc3_usb)>; + #iommu-cells = <1>; + }; + + DIE_NODE(dwc3_3_dart_1): iommu@1302f80000 { + compatible = "apple,t6020-dart", "apple,t8110-dart"; + reg = <0x13 0x02f80000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&DIE_NODE(ps_atc3_usb)>; + #iommu-cells = <1>; + }; + + DIE_NODE(atcphy3): phy@1303000000 { + compatible = "apple,t6020-atcphy", "apple,t8103-atcphy"; + reg = <0x13 0x03000000 0x0 0x4c000>, + <0x13 0x03050000 0x0 0x8000>, + <0x13 0x00000000 0x0 0x4000>, + <0x13 0x02a90000 0x0 0x4000>, + <0x13 0x02a84000 0x0 0x4000>; + reg-names = "core", "lpdptx", "axi2af", "usb2phy", + "pipehandler"; + + #phy-cells = <1>; + #reset-cells = <0>; + + orientation-switch; + mode-switch; + power-domains = <&DIE_NODE(ps_atc3_usb)>; + }; + + DIE_NODE(atcphy3_xbar): mux@130304c000 { + compatible = "apple,t6020-display-crossbar"; + reg = <0x13 0x0304c000 0x0 0x4000>; + #mux-control-cells = <1>; + power-domains = <&DIE_NODE(ps_atc3_usb)>; + status = "disabled"; + }; diff --git a/arch/arm64/boot/dts/apple/t602x-gpio-pins.dtsi b/arch/arm64/boot/dts/apple/t602x-gpio-pins.dtsi index e41b6475f79218..c5de99bd2e5cf3 100644 --- a/arch/arm64/boot/dts/apple/t602x-gpio-pins.dtsi +++ b/arch/arm64/boot/dts/apple/t602x-gpio-pins.dtsi @@ -78,4 +78,8 @@ , ; }; + + pcie_ge_pins: pcie-ge-pins { + pinmux = ; + }; }; diff --git a/arch/arm64/boot/dts/apple/t602x-j414-j416.dtsi b/arch/arm64/boot/dts/apple/t602x-j414-j416.dtsi index 0e806d8ddf81b1..0057e6a9465f9d 100644 --- a/arch/arm64/boot/dts/apple/t602x-j414-j416.dtsi +++ b/arch/arm64/boot/dts/apple/t602x-j414-j416.dtsi @@ -14,12 +14,31 @@ * the GPIO indices. */ +#define NO_SPI_TRACKPAD #include "t600x-j314-j316.dtsi" +/ { + aliases { + keyboard = &keyboard; + }; +}; + &framebuffer0 { power-domains = <&ps_disp0_cpu0>, <&ps_dptx_phy_ps>; }; +/* HACK: keep dptx_phy_ps power-domain always-on + * it is unclear how to sequence with dcp for the integrated display + */ +&ps_dptx_phy_ps { + apple,always-on; +}; + +&dcpext0 { + /* HDMI HPD gpio, used as interrupt*/ + hdmi-hpd-gpios = <&pinctrl_aop 25 GPIO_ACTIVE_HIGH>; +}; + &hpm0 { interrupts = <44 IRQ_TYPE_LEVEL_LOW>; }; @@ -36,6 +55,40 @@ interrupts = <44 IRQ_TYPE_LEVEL_LOW>; }; +/* Redefine GPIO for SDZ */ +&speaker_sdz { + gpios = <&pinctrl_ap 57 GPIO_ACTIVE_HIGH>; +}; + +&speaker_left_tweet { + interrupts-extended = <&pinctrl_ap 58 IRQ_TYPE_LEVEL_LOW>; +}; + +&speaker_left_woof1 { + interrupts-extended = <&pinctrl_ap 58 IRQ_TYPE_LEVEL_LOW>; +}; + +&speaker_left_woof2 { + interrupts-extended = <&pinctrl_ap 58 IRQ_TYPE_LEVEL_LOW>; +}; + +&speaker_right_tweet { + interrupts-extended = <&pinctrl_ap 58 IRQ_TYPE_LEVEL_LOW>; +}; + +&speaker_right_woof1 { + interrupts-extended = <&pinctrl_ap 58 IRQ_TYPE_LEVEL_LOW>; +}; + +&speaker_right_woof2 { + interrupts-extended = <&pinctrl_ap 58 IRQ_TYPE_LEVEL_LOW>; +}; + +&jack_codec { + reset-gpios = <&pinctrl_nub 8 GPIO_ACTIVE_HIGH>; + interrupts-extended = <&pinctrl_ap 59 IRQ_TYPE_LEVEL_LOW>; +}; + &wifi0 { compatible = "pci14e4,4434"; }; @@ -43,3 +96,56 @@ &bluetooth0 { compatible = "pci14e4,5f72"; }; + +&port01 { + pwren-gpios = <&smc_gpio 22 GPIO_ACTIVE_HIGH>; +}; + +&ps_mtp_fabric { + status = "okay"; +}; + +&mtp { + status = "okay"; +}; + +&mtp_mbox { + status = "okay"; +}; + +&mtp_dart { + status = "okay"; +}; + +&mtp_dockchannel { + status = "okay"; +}; + +&mtp_hid { + apple,afe-reset-gpios = <&smc_gpio 25 GPIO_ACTIVE_LOW>; + apple,stm-reset-gpios = <&smc_gpio 26 GPIO_ACTIVE_LOW>; + + mtp_mt: multi-touch { + }; + + keyboard: keyboard { + hid-country-code = <0>; + apple,keyboard-layout-id = <0>; + }; + + stm { + }; + + actuator { + }; + + tp_accel { + }; +}; + +&isp { + apple,platform-id = <7>; + /delete-node/ sensor-presets; /* Override j31[46] below */ +}; + +#include "isp-imx558-cfg0.dtsi" diff --git a/arch/arm64/boot/dts/apple/t602x-j474-j475.dtsi b/arch/arm64/boot/dts/apple/t602x-j474-j475.dtsi index ee12fea5b12cb3..287348628eb177 100644 --- a/arch/arm64/boot/dts/apple/t602x-j474-j475.dtsi +++ b/arch/arm64/boot/dts/apple/t602x-j474-j475.dtsi @@ -36,3 +36,23 @@ &hpm3 { interrupts = <44 IRQ_TYPE_LEVEL_LOW>; }; + +/* PCIe devices */ +&port00 { + pwren-gpios = <&smc_gpio 13 GPIO_ACTIVE_HIGH>; +}; + +&port03 { + /* USB xHCI */ + pwren-gpios = <&smc_gpio 19 GPIO_ACTIVE_HIGH>; +}; + +&speaker { + shutdown-gpios = <&pinctrl_ap 57 GPIO_ACTIVE_HIGH>; + interrupts-extended = <&pinctrl_ap 58 IRQ_TYPE_LEVEL_LOW>; +}; + +&jack_codec { + reset-gpios = <&pinctrl_nub 8 GPIO_ACTIVE_HIGH>; + interrupts-extended = <&pinctrl_ap 59 IRQ_TYPE_LEVEL_LOW>; +}; diff --git a/arch/arm64/boot/dts/apple/t602x-nvme.dtsi b/arch/arm64/boot/dts/apple/t602x-nvme.dtsi index 590cec8ac804c0..eb8c4e359079e5 100644 --- a/arch/arm64/boot/dts/apple/t602x-nvme.dtsi +++ b/arch/arm64/boot/dts/apple/t602x-nvme.dtsi @@ -26,7 +26,7 @@ }; DIE_NODE(nvme): nvme@34bcc0000 { - compatible = "apple,t6020-nvme-ans2", "apple,t8103-nvme-ans2"; + compatible = "apple,t6020-nvme-ans2", "apple,t8103-nvme-ans2", "apple,nvme-ans2"; reg = <0x3 0x4bcc0000 0x0 0x40000>, <0x3 0x47400000 0x0 0x4000>; reg-names = "nvme", "ans"; interrupt-parent = <&aic>; diff --git a/arch/arm64/boot/dts/apple/t602x-pcie-ge.dtsi b/arch/arm64/boot/dts/apple/t602x-pcie-ge.dtsi new file mode 100644 index 00000000000000..4a509cae0e5766 --- /dev/null +++ b/arch/arm64/boot/dts/apple/t602x-pcie-ge.dtsi @@ -0,0 +1,87 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * PCIe-GE Nodes present on both dies of a T6022 (M2 Ultra) and M2 Pro/Max but + * only used on T6022 in the Mac Pro (2023). + * + * Copyright The Asahi Linux Contributors + */ + + DIE_NODE(pcie_ge): pcie@1680000000 { + compatible = "apple,t6020-pcie-ge", "apple,t6020-pcie"; + device_type = "pci"; + + reg = <0x16 0x80000000 0x0 0x1000000>, /* config */ + <0x16 0x91000000 0x0 0x4000>, /* rc */ + <0x16 0x94008000 0x0 0x4000>, /* port0 */ + <0x16 0x9e01c000 0x0 0x4000>, /* phy0 */ + <0x16 0x9401c000 0x0 0x1000>; /* ltssm0 */ + reg-names = "config", "rc", "port0", "phy0", "ltssm0"; + + interrupt-parent = <&aic>; + interrupts = ; + + msi-controller; + msi-parent = <&DIE_NODE(pcie_ge)>; + msi-ranges = <&aic AIC_IRQ DIE_NO 1672 IRQ_TYPE_EDGE_RISING 128>; + + iommu-map = <0x000 &DIE_NODE(pcie_ge_dart) 0 0>, + <0x100 &DIE_NODE(pcie_ge_dart) 1 1>, + <0x200 &DIE_NODE(pcie_ge_dart) 2 2>, + <0x300 &DIE_NODE(pcie_ge_dart) 3 3>, + <0x400 &DIE_NODE(pcie_ge_dart) 4 4>, + <0x500 &DIE_NODE(pcie_ge_dart) 5 5>, + <0x600 &DIE_NODE(pcie_ge_dart) 6 6>, + <0x700 &DIE_NODE(pcie_ge_dart) 7 7>, + <0x800 &DIE_NODE(pcie_ge_dart) 8 8>, + <0x900 &DIE_NODE(pcie_ge_dart) 9 9>, + <0xa00 &DIE_NODE(pcie_ge_dart) 10 10>, + <0xb00 &DIE_NODE(pcie_ge_dart) 11 11>, + <0xc00 &DIE_NODE(pcie_ge_dart) 12 12>, + <0xd00 &DIE_NODE(pcie_ge_dart) 13 13>, + <0xe00 &DIE_NODE(pcie_ge_dart) 14 14>, + <0xf00 &DIE_NODE(pcie_ge_dart) 15 15>; + iommu-map-mask = <0xff00>; + + bus-range = <0 15>; + #address-cells = <3>; + #size-cells = <2>; + ranges = <0x43000000 0x18 0x00000000 0x18 0x00000000 0x4 0x00000000>, + <0x02000000 0x00 0x80000000 0x17 0x80000000 0x0 0x80000000>; + + power-domains = <&DIE_NODE(ps_apcie_ge_sys)>; + pinctrl-0 = <&DIE_NODE(pcie_ge_pins)>; + pinctrl-names = "default"; + + dma-coherent; + + status = "disabled"; + + DIE_NODE(port_ge00): pci@0,0 { + device_type = "pci"; + reg = <0x0 0x0 0x0 0x0 0x0>; + reset-gpios = <&DIE_NODE(pinctrl_ap) 9 GPIO_ACTIVE_LOW>; + + #address-cells = <3>; + #size-cells = <2>; + ranges; + + interrupt-controller; + #interrupt-cells = <1>; + + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &DIE_NODE(port_ge00) 0 0 0 0>, + <0 0 0 2 &DIE_NODE(port_ge00) 0 0 0 1>, + <0 0 0 3 &DIE_NODE(port_ge00) 0 0 0 2>, + <0 0 0 4 &DIE_NODE(port_ge00) 0 0 0 3>; + }; + }; + + DIE_NODE(pcie_ge_dart): iommu@1694000000 { + compatible = "apple,t6020-dart", "apple,t8110-dart"; + reg = <0x16 0x94000000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&DIE_NODE(ps_apcie_ge_sys)>; + status = "disabled"; + }; diff --git a/arch/arm64/boot/dts/apple/t602x-pmgr.dtsi b/arch/arm64/boot/dts/apple/t602x-pmgr.dtsi index f5382a2faf0b25..4ead781fea6893 100644 --- a/arch/arm64/boot/dts/apple/t602x-pmgr.dtsi +++ b/arch/arm64/boot/dts/apple/t602x-pmgr.dtsi @@ -7,7 +7,7 @@ &DIE_NODE(pmgr) { DIE_NODE(ps_afi): power-controller@100 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x100 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -16,7 +16,7 @@ }; DIE_NODE(ps_aic): power-controller@108 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x108 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -25,7 +25,7 @@ }; DIE_NODE(ps_dwi): power-controller@110 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x110 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -33,7 +33,7 @@ }; DIE_NODE(ps_pms): power-controller@118 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x118 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -42,7 +42,7 @@ }; DIE_NODE(ps_gpio): power-controller@120 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x120 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -51,7 +51,7 @@ }; DIE_NODE(ps_soc_dpe): power-controller@128 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x128 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -60,7 +60,7 @@ }; DIE_NODE(ps_pms_c1ppt): power-controller@130 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x130 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -69,7 +69,7 @@ }; DIE_NODE(ps_pmgr_soc_ocla): power-controller@138 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x138 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -78,7 +78,7 @@ }; DIE_NODE(ps_amcc0): power-controller@168 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x168 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -87,7 +87,7 @@ }; DIE_NODE(ps_amcc2): power-controller@170 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x170 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -96,7 +96,7 @@ }; DIE_NODE(ps_dcs_00): power-controller@178 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x178 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -105,7 +105,7 @@ }; DIE_NODE(ps_dcs_01): power-controller@180 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x180 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -114,7 +114,7 @@ }; DIE_NODE(ps_dcs_02): power-controller@188 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x188 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -123,7 +123,7 @@ }; DIE_NODE(ps_dcs_03): power-controller@190 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x190 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -132,7 +132,7 @@ }; DIE_NODE(ps_dcs_08): power-controller@198 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x198 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -141,7 +141,7 @@ }; DIE_NODE(ps_dcs_09): power-controller@1a0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x1a0 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -150,7 +150,7 @@ }; DIE_NODE(ps_dcs_10): power-controller@1a8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x1a8 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -159,7 +159,7 @@ }; DIE_NODE(ps_dcs_11): power-controller@1b0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x1b0 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -168,7 +168,7 @@ }; DIE_NODE(ps_afnc1_ioa): power-controller@1b8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x1b8 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -178,7 +178,7 @@ }; DIE_NODE(ps_afc): power-controller@1d0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x1d0 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -187,7 +187,7 @@ }; DIE_NODE(ps_afnc0_ioa): power-controller@1e8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x1e8 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -197,7 +197,7 @@ }; DIE_NODE(ps_afnc1_ls): power-controller@1f0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x1f0 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -207,7 +207,7 @@ }; DIE_NODE(ps_afnc0_ls): power-controller@1f8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x1f8 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -217,7 +217,7 @@ }; DIE_NODE(ps_afnc1_lw0): power-controller@200 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x200 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -227,7 +227,7 @@ }; DIE_NODE(ps_afnc1_lw1): power-controller@208 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x208 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -237,7 +237,7 @@ }; DIE_NODE(ps_afnc1_lw2): power-controller@210 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x210 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -247,7 +247,7 @@ }; DIE_NODE(ps_afnc0_lw0): power-controller@218 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x218 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -257,7 +257,7 @@ }; DIE_NODE(ps_scodec): power-controller@220 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x220 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -266,7 +266,7 @@ }; DIE_NODE(ps_atc0_common): power-controller@228 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x228 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -275,7 +275,7 @@ }; DIE_NODE(ps_atc1_common): power-controller@230 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x230 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -284,7 +284,7 @@ }; DIE_NODE(ps_atc2_common): power-controller@238 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x238 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -293,7 +293,7 @@ }; DIE_NODE(ps_atc3_common): power-controller@240 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x240 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -302,7 +302,7 @@ }; DIE_NODE(ps_dispext1_sys): power-controller@248 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x248 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -311,7 +311,7 @@ }; DIE_NODE(ps_pms_bridge): power-controller@250 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x250 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -321,7 +321,7 @@ }; DIE_NODE(ps_dispext0_sys): power-controller@258 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x258 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -330,7 +330,7 @@ }; DIE_NODE(ps_ane_sys): power-controller@260 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x260 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -339,7 +339,7 @@ }; DIE_NODE(ps_avd_sys): power-controller@268 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x268 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -348,7 +348,7 @@ }; DIE_NODE(ps_atc0_cio): power-controller@270 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x270 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -357,7 +357,7 @@ }; DIE_NODE(ps_atc0_pcie): power-controller@278 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x278 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -366,7 +366,7 @@ }; DIE_NODE(ps_atc1_cio): power-controller@280 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x280 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -375,7 +375,7 @@ }; DIE_NODE(ps_atc1_pcie): power-controller@288 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x288 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -384,7 +384,7 @@ }; DIE_NODE(ps_atc2_cio): power-controller@290 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x290 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -393,7 +393,7 @@ }; DIE_NODE(ps_atc2_pcie): power-controller@298 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x298 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -402,7 +402,7 @@ }; DIE_NODE(ps_atc3_cio): power-controller@2a0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x2a0 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -411,7 +411,7 @@ }; DIE_NODE(ps_atc3_pcie): power-controller@2a8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x2a8 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -420,7 +420,7 @@ }; DIE_NODE(ps_dispext1_fe): power-controller@2b0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x2b0 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -429,7 +429,7 @@ }; DIE_NODE(ps_dispext1_cpu0): power-controller@2b8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x2b8 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -439,7 +439,7 @@ }; DIE_NODE(ps_dispext0_fe): power-controller@2c0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x2c0 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -447,24 +447,27 @@ power-domains = <&DIE_NODE(ps_dispext0_sys)>; }; + /* PMP is only present on die 0 of the M2 Ultra */ DIE_NODE(ps_pmp): power-controller@2c8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x2c8 4>; #power-domain-cells = <0>; #reset-cells = <0>; label = DIE_LABEL(pmp); + apple,always-on; }; DIE_NODE(ps_pms_sram): power-controller@2d0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x2d0 4>; #power-domain-cells = <0>; #reset-cells = <0>; label = DIE_LABEL(pms_sram); + apple,always-on; }; DIE_NODE(ps_dispext0_cpu0): power-controller@2d8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x2d8 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -474,7 +477,7 @@ }; DIE_NODE(ps_ane_cpu): power-controller@2e0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x2e0 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -483,7 +486,7 @@ }; DIE_NODE(ps_atc0_cio_pcie): power-controller@2e8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x2e8 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -492,7 +495,7 @@ }; DIE_NODE(ps_atc0_cio_usb): power-controller@2f0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x2f0 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -501,7 +504,7 @@ }; DIE_NODE(ps_atc1_cio_pcie): power-controller@2f8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x2f8 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -510,7 +513,7 @@ }; DIE_NODE(ps_atc1_cio_usb): power-controller@300 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x300 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -519,7 +522,7 @@ }; DIE_NODE(ps_atc2_cio_pcie): power-controller@308 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x308 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -528,7 +531,7 @@ }; DIE_NODE(ps_atc2_cio_usb): power-controller@310 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x310 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -537,7 +540,7 @@ }; DIE_NODE(ps_atc3_cio_pcie): power-controller@318 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x318 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -546,7 +549,7 @@ }; DIE_NODE(ps_atc3_cio_usb): power-controller@320 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x320 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -555,7 +558,7 @@ }; DIE_NODE(ps_trace_fab): power-controller@390 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x390 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -563,7 +566,7 @@ }; DIE_NODE(ps_ane_sys_mpm): power-controller@4000 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x4000 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -572,7 +575,7 @@ }; DIE_NODE(ps_ane_td): power-controller@4008 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x4008 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -581,7 +584,7 @@ }; DIE_NODE(ps_ane_base): power-controller@4010 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x4010 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -590,7 +593,7 @@ }; DIE_NODE(ps_ane_set1): power-controller@4018 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x4018 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -599,7 +602,7 @@ }; DIE_NODE(ps_ane_set2): power-controller@4020 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x4020 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -608,7 +611,7 @@ }; DIE_NODE(ps_ane_set3): power-controller@4028 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x4028 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -617,7 +620,7 @@ }; DIE_NODE(ps_ane_set4): power-controller@4030 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x4030 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -628,7 +631,7 @@ &DIE_NODE(pmgr_south) { DIE_NODE(ps_amcc4): power-controller@100 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x100 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -637,7 +640,7 @@ }; DIE_NODE(ps_amcc5): power-controller@108 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x108 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -646,7 +649,7 @@ }; DIE_NODE(ps_amcc6): power-controller@110 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x110 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -655,7 +658,7 @@ }; DIE_NODE(ps_amcc7): power-controller@118 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x118 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -664,7 +667,7 @@ }; DIE_NODE(ps_dcs_16): power-controller@120 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x120 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -673,7 +676,7 @@ }; DIE_NODE(ps_dcs_17): power-controller@128 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x128 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -682,7 +685,7 @@ }; DIE_NODE(ps_dcs_18): power-controller@130 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x130 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -691,7 +694,7 @@ }; DIE_NODE(ps_dcs_19): power-controller@138 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x138 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -700,7 +703,7 @@ }; DIE_NODE(ps_dcs_20): power-controller@140 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x140 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -709,7 +712,7 @@ }; DIE_NODE(ps_dcs_21): power-controller@148 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x148 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -718,7 +721,7 @@ }; DIE_NODE(ps_dcs_22): power-controller@150 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x150 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -727,7 +730,7 @@ }; DIE_NODE(ps_dcs_23): power-controller@158 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x158 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -736,7 +739,7 @@ }; DIE_NODE(ps_dcs_24): power-controller@160 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x160 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -745,7 +748,7 @@ }; DIE_NODE(ps_dcs_25): power-controller@168 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x168 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -754,7 +757,7 @@ }; DIE_NODE(ps_dcs_26): power-controller@170 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x170 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -763,7 +766,7 @@ }; DIE_NODE(ps_dcs_27): power-controller@178 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x178 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -772,7 +775,7 @@ }; DIE_NODE(ps_dcs_28): power-controller@180 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x180 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -781,7 +784,7 @@ }; DIE_NODE(ps_dcs_29): power-controller@188 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x188 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -790,7 +793,7 @@ }; DIE_NODE(ps_dcs_30): power-controller@190 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x190 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -799,7 +802,7 @@ }; DIE_NODE(ps_dcs_31): power-controller@198 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x198 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -808,7 +811,7 @@ }; DIE_NODE(ps_afnc4_ioa): power-controller@1a0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x1a0 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -818,7 +821,7 @@ }; DIE_NODE(ps_afnc4_ls): power-controller@1a8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x1a8 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -828,7 +831,7 @@ }; DIE_NODE(ps_afnc4_lw0): power-controller@1b0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x1b0 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -838,7 +841,7 @@ }; DIE_NODE(ps_afnc5_ioa): power-controller@1b8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x1b8 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -848,7 +851,7 @@ }; DIE_NODE(ps_afnc5_ls): power-controller@1c0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x1c0 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -858,7 +861,7 @@ }; DIE_NODE(ps_afnc5_lw0): power-controller@1c8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x1c8 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -868,7 +871,7 @@ }; DIE_NODE(ps_dispext2_sys): power-controller@1d0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x1d0 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -876,7 +879,7 @@ }; DIE_NODE(ps_msr1): power-controller@1d8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x1d8 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -884,7 +887,7 @@ }; DIE_NODE(ps_dispext2_fe): power-controller@1e0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x1e0 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -893,7 +896,7 @@ }; DIE_NODE(ps_dispext2_cpu0): power-controller@1e8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x1e8 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -903,7 +906,7 @@ }; DIE_NODE(ps_msr1_ase_core): power-controller@1f0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x1f0 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -912,7 +915,7 @@ }; DIE_NODE(ps_dispext3_sys): power-controller@220 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x220 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -920,7 +923,7 @@ }; DIE_NODE(ps_venc1_sys): power-controller@228 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x228 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -928,7 +931,7 @@ }; DIE_NODE(ps_dispext3_fe): power-controller@230 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x230 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -937,7 +940,7 @@ }; DIE_NODE(ps_dispext3_cpu0): power-controller@238 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x238 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -947,7 +950,7 @@ }; DIE_NODE(ps_venc1_dma): power-controller@4000 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x4000 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -956,7 +959,7 @@ }; DIE_NODE(ps_venc1_pipe4): power-controller@4008 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x4008 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -965,7 +968,7 @@ }; DIE_NODE(ps_venc1_pipe5): power-controller@4010 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x4010 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -974,7 +977,7 @@ }; DIE_NODE(ps_venc1_me0): power-controller@4018 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x4018 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -983,7 +986,7 @@ }; DIE_NODE(ps_venc1_me1): power-controller@4020 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x4020 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -994,7 +997,7 @@ &DIE_NODE(pmgr_east) { DIE_NODE(ps_clvr_spmi0): power-controller@100 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x100 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1003,7 +1006,7 @@ }; DIE_NODE(ps_clvr_spmi1): power-controller@108 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x108 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1012,7 +1015,7 @@ }; DIE_NODE(ps_clvr_spmi2): power-controller@110 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x110 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1021,7 +1024,7 @@ }; DIE_NODE(ps_clvr_spmi3): power-controller@118 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x118 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1030,7 +1033,7 @@ }; DIE_NODE(ps_clvr_spmi4): power-controller@120 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x120 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1039,7 +1042,7 @@ }; DIE_NODE(ps_ispsens0): power-controller@128 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x128 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1047,7 +1050,7 @@ }; DIE_NODE(ps_ispsens1): power-controller@130 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x130 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1055,7 +1058,7 @@ }; DIE_NODE(ps_ispsens2): power-controller@138 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x138 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1063,7 +1066,7 @@ }; DIE_NODE(ps_ispsens3): power-controller@140 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x140 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1071,7 +1074,7 @@ }; DIE_NODE(ps_afnc6_ioa): power-controller@148 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x148 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1081,7 +1084,7 @@ }; DIE_NODE(ps_afnc6_ls): power-controller@150 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x150 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1091,7 +1094,7 @@ }; DIE_NODE(ps_afnc6_lw0): power-controller@158 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x158 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1101,7 +1104,7 @@ }; DIE_NODE(ps_afnc2_ioa): power-controller@160 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x160 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1111,7 +1114,7 @@ }; DIE_NODE(ps_afnc2_ls): power-controller@168 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x168 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1121,7 +1124,7 @@ }; DIE_NODE(ps_afnc2_lw0): power-controller@170 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x170 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1131,7 +1134,7 @@ }; DIE_NODE(ps_afnc2_lw1): power-controller@178 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x178 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1141,7 +1144,7 @@ }; DIE_NODE(ps_afnc3_ioa): power-controller@180 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x180 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1151,7 +1154,7 @@ }; DIE_NODE(ps_afnc3_ls): power-controller@188 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x188 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1161,7 +1164,7 @@ }; DIE_NODE(ps_afnc3_lw0): power-controller@190 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x190 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1171,7 +1174,7 @@ }; DIE_NODE(ps_apcie_gp): power-controller@198 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x198 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1180,7 +1183,7 @@ }; DIE_NODE(ps_apcie_st): power-controller@1a0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x1a0 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1189,7 +1192,7 @@ }; DIE_NODE(ps_ans2): power-controller@1a8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x1a8 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1198,7 +1201,7 @@ }; DIE_NODE(ps_disp0_sys): power-controller@1b0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x1b0 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1207,7 +1210,7 @@ }; DIE_NODE(ps_jpg): power-controller@1b8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x1b8 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1216,7 +1219,7 @@ }; DIE_NODE(ps_sio): power-controller@1c0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x1c0 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1225,7 +1228,7 @@ }; DIE_NODE(ps_isp_sys): power-controller@1c8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x1c8 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1235,7 +1238,7 @@ }; DIE_NODE(ps_disp0_fe): power-controller@1d0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x1d0 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1244,7 +1247,7 @@ }; DIE_NODE(ps_disp0_cpu0): power-controller@1d8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x1d8 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1254,16 +1257,16 @@ }; DIE_NODE(ps_sio_cpu): power-controller@1e0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x1e0 4>; #power-domain-cells = <0>; #reset-cells = <0>; label = DIE_LABEL(sio_cpu); - power-domains = <&DIE_NODE(ps_sio)>; + power-domains = <&DIE_NODE(ps_sio) &DIE_NODE(ps_uart_p) &DIE_NODE(ps_spi_p) &DIE_NODE(ps_audio_p)>; }; DIE_NODE(ps_fpwm0): power-controller@1e8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x1e8 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1272,7 +1275,7 @@ }; DIE_NODE(ps_fpwm1): power-controller@1f0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x1f0 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1281,7 +1284,7 @@ }; DIE_NODE(ps_fpwm2): power-controller@1f8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x1f8 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1290,7 +1293,7 @@ }; DIE_NODE(ps_i2c0): power-controller@200 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x200 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1299,7 +1302,7 @@ }; DIE_NODE(ps_i2c1): power-controller@208 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x208 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1308,7 +1311,7 @@ }; DIE_NODE(ps_i2c2): power-controller@210 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x210 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1317,7 +1320,7 @@ }; DIE_NODE(ps_i2c3): power-controller@218 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x218 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1326,7 +1329,7 @@ }; DIE_NODE(ps_i2c4): power-controller@220 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x220 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1335,7 +1338,7 @@ }; DIE_NODE(ps_i2c5): power-controller@228 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x228 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1344,7 +1347,7 @@ }; DIE_NODE(ps_i2c6): power-controller@230 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x230 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1353,7 +1356,7 @@ }; DIE_NODE(ps_i2c7): power-controller@238 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x238 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1362,7 +1365,7 @@ }; DIE_NODE(ps_i2c8): power-controller@240 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x240 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1371,7 +1374,7 @@ }; DIE_NODE(ps_spi_p): power-controller@248 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x248 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1380,7 +1383,7 @@ }; DIE_NODE(ps_sio_spmi0): power-controller@250 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x250 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1389,7 +1392,7 @@ }; DIE_NODE(ps_sio_spmi1): power-controller@258 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x258 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1398,7 +1401,7 @@ }; DIE_NODE(ps_sio_spmi2): power-controller@260 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x260 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1407,7 +1410,7 @@ }; DIE_NODE(ps_uart_p): power-controller@268 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x268 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1416,7 +1419,7 @@ }; DIE_NODE(ps_audio_p): power-controller@270 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x270 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1425,7 +1428,7 @@ }; DIE_NODE(ps_sio_adma): power-controller@278 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x278 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1434,7 +1437,7 @@ }; DIE_NODE(ps_aes): power-controller@280 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x280 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1444,7 +1447,7 @@ }; DIE_NODE(ps_dptx_phy_ps): power-controller@288 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x288 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1453,7 +1456,7 @@ }; DIE_NODE(ps_spi0): power-controller@2d8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x2d8 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1462,7 +1465,7 @@ }; DIE_NODE(ps_spi1): power-controller@2e0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x2e0 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1471,7 +1474,7 @@ }; DIE_NODE(ps_spi2): power-controller@2e8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x2e8 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1480,7 +1483,7 @@ }; DIE_NODE(ps_spi3): power-controller@2f0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x2f0 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1489,7 +1492,7 @@ }; DIE_NODE(ps_spi4): power-controller@2f8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x2f8 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1498,7 +1501,7 @@ }; DIE_NODE(ps_spi5): power-controller@300 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x300 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1507,7 +1510,7 @@ }; DIE_NODE(ps_uart_n): power-controller@308 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x308 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1516,7 +1519,7 @@ }; DIE_NODE(ps_uart0): power-controller@310 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x310 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1525,7 +1528,7 @@ }; DIE_NODE(ps_amcc1): power-controller@318 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x318 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1534,7 +1537,7 @@ }; DIE_NODE(ps_amcc3): power-controller@320 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x320 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1543,7 +1546,7 @@ }; DIE_NODE(ps_dcs_04): power-controller@328 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x328 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1552,7 +1555,7 @@ }; DIE_NODE(ps_dcs_05): power-controller@330 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x330 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1561,7 +1564,7 @@ }; DIE_NODE(ps_dcs_06): power-controller@338 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x338 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1570,7 +1573,7 @@ }; DIE_NODE(ps_dcs_07): power-controller@340 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x340 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1579,7 +1582,7 @@ }; DIE_NODE(ps_dcs_12): power-controller@348 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x348 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1588,7 +1591,7 @@ }; DIE_NODE(ps_dcs_13): power-controller@350 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x350 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1597,7 +1600,7 @@ }; DIE_NODE(ps_dcs_14): power-controller@358 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x358 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1606,7 +1609,7 @@ }; DIE_NODE(ps_dcs_15): power-controller@360 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x360 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1615,7 +1618,7 @@ }; DIE_NODE(ps_uart1): power-controller@368 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x368 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1624,7 +1627,7 @@ }; DIE_NODE(ps_uart2): power-controller@370 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x370 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1633,7 +1636,7 @@ }; DIE_NODE(ps_uart3): power-controller@378 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x378 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1642,7 +1645,7 @@ }; DIE_NODE(ps_uart4): power-controller@380 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x380 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1651,7 +1654,7 @@ }; DIE_NODE(ps_uart5): power-controller@388 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x388 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1660,7 +1663,7 @@ }; DIE_NODE(ps_uart6): power-controller@390 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x390 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1669,43 +1672,47 @@ }; DIE_NODE(ps_mca0): power-controller@398 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x398 4>; #power-domain-cells = <0>; #reset-cells = <0>; label = DIE_LABEL(mca0); power-domains = <&DIE_NODE(ps_audio_p)>, <&DIE_NODE(ps_sio_adma)>; + apple,externally-clocked; }; DIE_NODE(ps_mca1): power-controller@3a0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x3a0 4>; #power-domain-cells = <0>; #reset-cells = <0>; label = DIE_LABEL(mca1); power-domains = <&DIE_NODE(ps_audio_p)>, <&DIE_NODE(ps_sio_adma)>; + apple,externally-clocked; }; DIE_NODE(ps_mca2): power-controller@3a8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x3a8 4>; #power-domain-cells = <0>; #reset-cells = <0>; label = DIE_LABEL(mca2); power-domains = <&DIE_NODE(ps_audio_p)>, <&DIE_NODE(ps_sio_adma)>; + apple,externally-clocked; }; DIE_NODE(ps_mca3): power-controller@3b0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x3b0 4>; #power-domain-cells = <0>; #reset-cells = <0>; label = DIE_LABEL(mca3); power-domains = <&DIE_NODE(ps_audio_p)>, <&DIE_NODE(ps_sio_adma)>; + apple,externally-clocked; }; DIE_NODE(ps_dpa0): power-controller@3b8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x3b8 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1714,7 +1721,7 @@ }; DIE_NODE(ps_dpa1): power-controller@3c0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x3c0 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1723,7 +1730,7 @@ }; DIE_NODE(ps_dpa2): power-controller@3c8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x3c8 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1732,7 +1739,7 @@ }; DIE_NODE(ps_dpa3): power-controller@3d0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x3d0 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1741,7 +1748,7 @@ }; DIE_NODE(ps_msr0): power-controller@3d8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x3d8 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1749,7 +1756,7 @@ }; DIE_NODE(ps_venc_sys): power-controller@3e0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x3e0 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1757,7 +1764,7 @@ }; DIE_NODE(ps_dpa4): power-controller@3e8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x3e8 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1766,7 +1773,7 @@ }; DIE_NODE(ps_msr0_ase_core): power-controller@3f0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x3f0 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1775,7 +1782,7 @@ }; DIE_NODE(ps_apcie_gpshr_sys): power-controller@3f8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x3f8 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1784,7 +1791,7 @@ }; DIE_NODE(ps_apcie_st_sys): power-controller@408 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x408 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1793,7 +1800,7 @@ }; DIE_NODE(ps_apcie_st1_sys): power-controller@410 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x410 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1802,7 +1809,7 @@ }; DIE_NODE(ps_apcie_gp_sys): power-controller@418 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x418 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1812,7 +1819,7 @@ }; DIE_NODE(ps_apcie_ge_sys): power-controller@420 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x420 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1821,7 +1828,7 @@ }; DIE_NODE(ps_apcie_phy_sw): power-controller@428 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x428 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1830,7 +1837,7 @@ }; DIE_NODE(ps_sep): power-controller@c00 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0xc00 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1847,16 +1854,17 @@ * have to enable/disable everything in the per-model DTs. */ DIE_NODE(ps_isp_cpu): power-controller@4000 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x4000 4>; #power-domain-cells = <0>; #reset-cells = <0>; label = DIE_LABEL(isp_cpu); /* power-domains = <&DIE_NODE(ps_isp_sys)>; */ + apple,force-disable; }; DIE_NODE(ps_isp_fe): power-controller@4008 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x4008 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1865,7 +1873,7 @@ }; DIE_NODE(ps_dprx): power-controller@4010 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x4010 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1874,7 +1882,7 @@ }; DIE_NODE(ps_isp_vis): power-controller@4018 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x4018 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1883,7 +1891,7 @@ }; DIE_NODE(ps_isp_be): power-controller@4020 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x4020 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1892,7 +1900,7 @@ }; DIE_NODE(ps_isp_raw): power-controller@4028 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x4028 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1901,7 +1909,7 @@ }; DIE_NODE(ps_isp_clr): power-controller@4030 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x4030 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1910,7 +1918,7 @@ }; DIE_NODE(ps_venc_dma): power-controller@8000 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x8000 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1919,7 +1927,7 @@ }; DIE_NODE(ps_venc_pipe4): power-controller@8008 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x8008 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1928,7 +1936,7 @@ }; DIE_NODE(ps_venc_pipe5): power-controller@8010 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x8010 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1937,7 +1945,7 @@ }; DIE_NODE(ps_venc_me0): power-controller@8018 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x8018 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1946,7 +1954,7 @@ }; DIE_NODE(ps_venc_me1): power-controller@8020 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x8020 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1955,7 +1963,7 @@ }; DIE_NODE(ps_prores): power-controller@c000 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0xc000 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1966,7 +1974,7 @@ &DIE_NODE(pmgr_mini) { DIE_NODE(ps_debug): power-controller@58 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x58 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1975,7 +1983,7 @@ }; DIE_NODE(ps_nub_spmi0): power-controller@60 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x60 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1984,7 +1992,7 @@ }; DIE_NODE(ps_nub_spmi1): power-controller@68 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x68 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1993,7 +2001,7 @@ }; DIE_NODE(ps_nub_aon): power-controller@70 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x70 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -2002,7 +2010,7 @@ }; DIE_NODE(ps_msg): power-controller@78 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x78 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -2011,7 +2019,7 @@ }; DIE_NODE(ps_nub_gpio): power-controller@80 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x80 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -2020,7 +2028,7 @@ }; DIE_NODE(ps_nub_fabric): power-controller@88 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x88 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -2029,7 +2037,7 @@ }; DIE_NODE(ps_atc0_usb_aon): power-controller@90 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x90 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -2038,7 +2046,7 @@ }; DIE_NODE(ps_atc1_usb_aon): power-controller@98 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x98 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -2047,7 +2055,7 @@ }; DIE_NODE(ps_atc2_usb_aon): power-controller@a0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0xa0 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -2056,7 +2064,7 @@ }; DIE_NODE(ps_atc3_usb_aon): power-controller@a8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0xa8 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -2065,7 +2073,7 @@ }; DIE_NODE(ps_mtp_fabric): power-controller@b0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0xb0 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -2076,7 +2084,7 @@ }; DIE_NODE(ps_nub_sram): power-controller@b8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0xb8 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -2085,7 +2093,7 @@ }; DIE_NODE(ps_debug_switch): power-controller@c0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0xc0 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -2094,7 +2102,7 @@ }; DIE_NODE(ps_atc0_usb): power-controller@c8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0xc8 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -2103,7 +2111,7 @@ }; DIE_NODE(ps_atc1_usb): power-controller@d0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0xd0 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -2112,7 +2120,7 @@ }; DIE_NODE(ps_atc2_usb): power-controller@d8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0xd8 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -2121,7 +2129,7 @@ }; DIE_NODE(ps_atc3_usb): power-controller@e0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0xe0 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -2132,7 +2140,7 @@ #if 0 /* MTP stuff is self-managed */ DIE_NODE(ps_mtp_gpio): power-controller@e8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0xe8 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -2142,7 +2150,7 @@ }; DIE_NODE(ps_mtp_base): power-controller@f0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0xf0 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -2152,7 +2160,7 @@ }; DIE_NODE(ps_mtp_periph): power-controller@f8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0xf8 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -2162,7 +2170,7 @@ }; DIE_NODE(ps_mtp_spi0): power-controller@100 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x100 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -2172,7 +2180,7 @@ }; DIE_NODE(ps_mtp_i2cm0): power-controller@108 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x108 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -2182,7 +2190,7 @@ }; DIE_NODE(ps_mtp_uart0): power-controller@110 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x110 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -2192,7 +2200,7 @@ }; DIE_NODE(ps_mtp_cpu): power-controller@118 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x118 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -2202,7 +2210,7 @@ }; DIE_NODE(ps_mtp_scm_fabric): power-controller@120 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x120 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -2212,7 +2220,7 @@ }; DIE_NODE(ps_mtp_sram): power-controller@128 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x128 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -2222,7 +2230,7 @@ }; DIE_NODE(ps_mtp_dma): power-controller@130 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x130 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -2235,7 +2243,7 @@ &DIE_NODE(pmgr_gfx) { DIE_NODE(ps_gpx): power-controller@0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x0 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -2245,7 +2253,7 @@ }; DIE_NODE(ps_afr): power-controller@100 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x100 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -2255,7 +2263,7 @@ }; DIE_NODE(ps_gfx): power-controller@108 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x108 4>; #power-domain-cells = <0>; #reset-cells = <0>; diff --git a/arch/arm64/boot/dts/apple/t8103-j274.dts b/arch/arm64/boot/dts/apple/t8103-j274.dts index 1c3e37f86d46d7..f55683c48784b8 100644 --- a/arch/arm64/boot/dts/apple/t8103-j274.dts +++ b/arch/arm64/boot/dts/apple/t8103-j274.dts @@ -15,12 +15,27 @@ / { compatible = "apple,j274", "apple,t8103", "apple,arm-platform"; model = "Apple Mac mini (M1, 2020)"; + chassis-type = "desktop"; aliases { ethernet0 = ðernet0; + sio = &sio; }; }; +&dcp { + apple,connector-type = "HDMI-A"; +}; + +/* remove once m1n1 enables sio nodes after setup */ +&sio { + status = "okay"; +}; + +&dpaudio0 { + status = "okay"; +}; + &bluetooth0 { brcm,board-type = "apple,atlantisb"; }; @@ -29,6 +44,18 @@ brcm,board-type = "apple,atlantisb"; }; +/* + * Provide labels for the USB type C ports. + */ + +&typec0 { + label = "USB-C Back-left"; +}; + +&typec1 { + label = "USB-C Back-right"; +}; + /* * Force the bus number assignments so that we can declare some of the * on-board devices and properties that are populated by the bootloader @@ -58,6 +85,65 @@ status = "okay"; }; +&i2c1 { + speaker_amp: codec@31 { + compatible = "ti,tas5770l", "ti,tas2770"; + reg = <0x31>; + shutdown-gpios = <&pinctrl_ap 181 GPIO_ACTIVE_HIGH>; + #sound-dai-cells = <0>; + ti,imon-slot-no = <0>; + ti,vmon-slot-no = <2>; + ti,sdout-zero-fill; + }; +}; + &i2c2 { status = "okay"; + + jack_codec: codec@48 { + compatible = "cirrus,cs42l83"; + reg = <0x48>; + reset-gpios = <&pinctrl_nub 11 GPIO_ACTIVE_HIGH>; + interrupt-parent = <&pinctrl_ap>; + interrupts = <183 IRQ_TYPE_LEVEL_LOW>; + #sound-dai-cells = <0>; + cirrus,ts-inv = <1>; + sound-name-prefix = "Jack"; + }; }; + +/ { + sound { + compatible = "apple,j274-macaudio", "apple,macaudio"; + model = "Mac mini J274"; + + dai-link@0 { + link-name = "Speaker"; + + cpu { + sound-dai = <&mca 0>; + }; + codec { + sound-dai = <&speaker_amp>; + }; + }; + + dai-link@1 { + link-name = "Headphone Jack"; + + cpu { + sound-dai = <&mca 2>; + }; + codec { + sound-dai = <&jack_codec>; + }; + }; + + }; +}; + +&gpu { + apple,perf-base-pstate = <3>; +}; + +#include "hwmon-mini.dtsi" diff --git a/arch/arm64/boot/dts/apple/t8103-j293.dts b/arch/arm64/boot/dts/apple/t8103-j293.dts index 5b3c42e9f0e677..6fa04626b1d08b 100644 --- a/arch/arm64/boot/dts/apple/t8103-j293.dts +++ b/arch/arm64/boot/dts/apple/t8103-j293.dts @@ -16,6 +16,7 @@ / { compatible = "apple,j293", "apple,t8103", "apple,arm-platform"; model = "Apple MacBook Pro (13-inch, M1, 2020)"; + chassis-type = "laptop"; /* * All of those are used by the bootloader to pass calibration @@ -23,6 +24,7 @@ */ aliases { touchbar0 = &touchbar0; + sep = &sep; }; led-controller { @@ -38,6 +40,20 @@ }; }; +&dcp { + panel: panel { + compatible = "apple,panel-j293", "apple,panel"; + width-mm = <286>; + height-mm = <179>; + apple,max-brightness = <525>; + }; +}; + +&framebuffer0 { + panel = <&panel>; + post-init-providers = <&panel>; +}; + &bluetooth0 { brcm,board-type = "apple,honshu"; }; @@ -46,8 +62,117 @@ brcm,board-type = "apple,honshu"; }; +/* + * Provide labels for the USB type C ports. + */ + +&typec0 { + label = "USB-C Left-back"; +}; + +&typec1 { + label = "USB-C Left-front"; +}; + +&spi3 { + status = "okay"; + + hid-transport@0 { + compatible = "apple,spi-hid-transport"; + reg = <0>; + spi-max-frequency = <8000000>; + /* + * Apple's ADT specifies 20us CS change delays, and the + * SPI HID interface metadata specifies 45us. Using either + * seems not to be reliable, but adding both works, so + * best guess is they are cumulative. + */ + spi-cs-setup-delay-ns = <65000>; + spi-cs-hold-delay-ns = <65000>; + spi-cs-inactive-delay-ns = <250000>; + spien-gpios = <&pinctrl_ap 195 0>; + interrupts-extended = <&pinctrl_nub 13 IRQ_TYPE_LEVEL_LOW>; + }; +}; + +/* Virtual regulator representing the shared shutdown GPIO */ +/ { + speaker_sdz: fixed-regulator-tas5770-sdz { + compatible = "regulator-fixed"; + regulator-name = "tas5770-sdz"; + startup-delay-us = <5000>; + gpios = <&pinctrl_ap 181 GPIO_ACTIVE_HIGH>; + enable-active-high; + }; +}; + +&i2c1 { + speaker_left_rear: codec@31 { + compatible = "ti,tas5770l", "ti,tas2770"; + reg = <0x31>; + SDZ-supply = <&speaker_sdz>; + #sound-dai-cells = <0>; + sound-name-prefix = "Left Rear"; + interrupts-extended = <&pinctrl_ap 182 IRQ_TYPE_LEVEL_LOW>; + ti,imon-slot-no = <8>; + ti,vmon-slot-no = <10>; + ti,pdm-slot-no = <12>; + }; + + speaker_left_front: codec@32 { + compatible = "ti,tas5770l", "ti,tas2770"; + reg = <0x32>; + SDZ-supply = <&speaker_sdz>; + #sound-dai-cells = <0>; + sound-name-prefix = "Left Front"; + interrupts-extended = <&pinctrl_ap 182 IRQ_TYPE_LEVEL_LOW>; + ti,imon-slot-no = <0>; + ti,vmon-slot-no = <2>; + ti,pdm-slot-no = <4>; + ti,sdout-pull-down; + }; +}; + &i2c2 { status = "okay"; + + jack_codec: codec@48 { + compatible = "cirrus,cs42l83"; + reg = <0x48>; + reset-gpios = <&pinctrl_nub 11 GPIO_ACTIVE_HIGH>; + interrupt-parent = <&pinctrl_ap>; + interrupts = <183 IRQ_TYPE_LEVEL_LOW>; + #sound-dai-cells = <0>; + cirrus,ts-inv = <1>; + sound-name-prefix = "Jack"; + }; +}; + +&i2c3 { + speaker_right_rear: codec@34 { + compatible = "ti,tas5770l", "ti,tas2770"; + reg = <0x34>; + SDZ-supply = <&speaker_sdz>; + #sound-dai-cells = <0>; + sound-name-prefix = "Right Rear"; + interrupts-extended = <&pinctrl_ap 182 IRQ_TYPE_LEVEL_LOW>; + ti,imon-slot-no = <12>; + ti,vmon-slot-no = <14>; + ti,pdm-slot-no = <16>; + }; + + speaker_right_front: codec@35 { + compatible = "ti,tas5770l", "ti,tas2770"; + reg = <0x35>; + SDZ-supply = <&speaker_sdz>; + #sound-dai-cells = <0>; + sound-name-prefix = "Right Front"; + interrupts-extended = <&pinctrl_ap 182 IRQ_TYPE_LEVEL_LOW>; + ti,imon-slot-no = <4>; + ti,vmon-slot-no = <6>; + ti,pdm-slot-no = <8>; + ti,sdout-pull-down; + }; }; &i2c4 { @@ -119,3 +244,67 @@ &displaydfr_dart { status = "okay"; }; + +&aop_mbox { + status = "okay"; +}; + +&aop_dart { + status = "okay"; +}; + +&aop_admac { + status = "okay"; +}; + +&aop { + status = "okay"; +}; + +&sep { + status = "okay"; +}; + +&aop_audio { + apple,chassis-name = "J293"; + apple,machine-kind = "MacBook Pro"; +}; + +/ { + sound { + compatible = "apple,j293-macaudio", "apple,macaudio"; + model = "MacBook Pro J293"; + + dai-link@0 { + link-name = "Speakers"; + + cpu { + sound-dai = <&mca 0>, <&mca 1>; + }; + codec { + sound-dai = <&speaker_left_front>, <&speaker_left_rear>, + <&speaker_right_front>, <&speaker_right_rear>; + }; + }; + + dai-link@1 { + link-name = "Headphone Jack"; + + cpu { + sound-dai = <&mca 2>; + }; + codec { + sound-dai = <&jack_codec>; + }; + }; + }; +}; + +#include "isp-imx248.dtsi" + +&isp { + apple,platform-id = <1>; +}; + +#include "hwmon-fan.dtsi" +#include "hwmon-laptop.dtsi" diff --git a/arch/arm64/boot/dts/apple/t8103-j313.dts b/arch/arm64/boot/dts/apple/t8103-j313.dts index 97a4344d8dca68..883ba4a1f0100a 100644 --- a/arch/arm64/boot/dts/apple/t8103-j313.dts +++ b/arch/arm64/boot/dts/apple/t8103-j313.dts @@ -16,6 +16,11 @@ / { compatible = "apple,j313", "apple,t8103", "apple,arm-platform"; model = "Apple MacBook Air (M1, 2020)"; + chassis-type = "laptop"; + + aliases { + sep = &sep; + }; led-controller { compatible = "pwm-leds"; @@ -30,6 +35,20 @@ }; }; +&dcp { + panel: panel { + compatible = "apple,panel-j313", "apple,panel"; + width-mm = <286>; + height-mm = <179>; + apple,max-brightness = <420>; + }; +}; + +&framebuffer0 { + panel = <&panel>; + post-init-providers = <&panel>; +}; + &bluetooth0 { brcm,board-type = "apple,shikoku"; }; @@ -41,3 +60,148 @@ &fpwm1 { status = "okay"; }; + +/* + * Provide labels for the USB type C ports. + */ + +&typec0 { + label = "USB-C Left-back"; +}; + +&typec1 { + label = "USB-C Left-front"; +}; + +&spi3 { + status = "okay"; + + hid-transport@0 { + compatible = "apple,spi-hid-transport"; + reg = <0>; + spi-max-frequency = <8000000>; + /* + * Apple's ADT specifies 20us CS change delays, and the + * SPI HID interface metadata specifies 45us. Using either + * seems not to be reliable, but adding both works, so + * best guess is they are cumulative. + */ + spi-cs-setup-delay-ns = <65000>; + spi-cs-hold-delay-ns = <65000>; + spi-cs-inactive-delay-ns = <250000>; + spien-gpios = <&pinctrl_ap 195 0>; + interrupts-extended = <&pinctrl_nub 13 IRQ_TYPE_LEVEL_LOW>; + }; +}; + +/* Virtual regulator representing the shared shutdown GPIO */ +/ { + speaker_sdz: fixed-regulator-tas5770-sdz { + compatible = "regulator-fixed"; + regulator-name = "tas5770-sdz"; + startup-delay-us = <5000>; + gpios = <&pinctrl_ap 181 GPIO_ACTIVE_HIGH>; + enable-active-high; + }; +}; + +&i2c1 { + speaker_left: codec@31 { + compatible = "ti,tas5770l", "ti,tas2770"; + reg = <0x31>; + SDZ-supply = <&speaker_sdz>; + #sound-dai-cells = <0>; + sound-name-prefix = "Left"; + interrupts-extended = <&pinctrl_ap 182 IRQ_TYPE_LEVEL_LOW>; + ti,imon-slot-no = <0>; + ti,vmon-slot-no = <2>; + ti,sdout-zero-fill; + }; +}; + +&i2c3 { + speaker_right: codec@34 { + compatible = "ti,tas5770l", "ti,tas2770"; + reg = <0x34>; + SDZ-supply = <&speaker_sdz>; + #sound-dai-cells = <0>; + sound-name-prefix = "Right"; + interrupts-extended = <&pinctrl_ap 182 IRQ_TYPE_LEVEL_LOW>; + ti,imon-slot-no = <4>; + ti,vmon-slot-no = <6>; + ti,sdout-zero-fill; + }; + + jack_codec: codec@48 { + compatible = "cirrus,cs42l83"; + reg = <0x48>; + reset-gpios = <&pinctrl_nub 11 GPIO_ACTIVE_HIGH>; + interrupt-parent = <&pinctrl_ap>; + interrupts = <183 IRQ_TYPE_LEVEL_LOW>; + #sound-dai-cells = <0>; + cirrus,ts-inv = <1>; + sound-name-prefix = "Jack"; + }; +}; + +&aop_mbox { + status = "okay"; +}; + +&aop_dart { + status = "okay"; +}; + +&aop_admac { + status = "okay"; +}; + +&aop { + status = "okay"; +}; + +&sep { + status = "okay"; +}; + +&aop_audio { + apple,chassis-name = "J313"; + apple,machine-kind = "MacBook Air"; +}; + +/ { + sound { + compatible = "apple,j313-macaudio", "apple,macaudio"; + model = "MacBook Air J313"; + + dai-link@0 { + link-name = "Speakers"; + + cpu { + sound-dai = <&mca 0>, <&mca 1>; + }; + codec { + sound-dai = <&speaker_left>, <&speaker_right>; + }; + }; + + dai-link@1 { + link-name = "Headphone Jack"; + + cpu { + sound-dai = <&mca 2>; + }; + codec { + sound-dai = <&jack_codec>; + }; + }; + }; +}; + +#include "isp-imx248.dtsi" + +&isp { + apple,platform-id = <1>; +}; + +#include "hwmon-laptop.dtsi" diff --git a/arch/arm64/boot/dts/apple/t8103-j456.dts b/arch/arm64/boot/dts/apple/t8103-j456.dts index 58c8e43789b486..c7da4815fb94c0 100644 --- a/arch/arm64/boot/dts/apple/t8103-j456.dts +++ b/arch/arm64/boot/dts/apple/t8103-j456.dts @@ -15,12 +15,27 @@ / { compatible = "apple,j456", "apple,t8103", "apple,arm-platform"; model = "Apple iMac (24-inch, 4x USB-C, M1, 2021)"; + chassis-type = "all-in-one"; aliases { ethernet0 = ðernet0; }; }; +&dcp { + panel: panel { + compatible = "apple,panel-j456", "apple,panel"; + width-mm = <522>; + height-mm = <294>; + apple,max-brightness = <525>; + }; +}; + +&framebuffer0 { + panel = <&panel>; + post-init-providers = <&panel>; +}; + &bluetooth0 { brcm,board-type = "apple,capri"; }; @@ -47,6 +62,18 @@ }; }; +/* + * Provide labels for the USB type C ports. + */ + +&typec0 { + label = "USB-C Back-right"; +}; + +&typec1 { + label = "USB-C Back-right-middle"; +}; + /* * Force the bus number assignments so that we can declare some of the * on-board devices and properties that are populated by the bootloader @@ -75,3 +102,72 @@ &pcie0_dart_2 { status = "okay"; }; + +&i2c1 { + jack_codec: codec@48 { + compatible = "cirrus,cs42l83"; + reg = <0x48>; + reset-gpios = <&pinctrl_nub 11 GPIO_ACTIVE_HIGH>; + interrupt-parent = <&pinctrl_ap>; + interrupts = <183 IRQ_TYPE_LEVEL_LOW>; + #sound-dai-cells = <0>; + cirrus,ts-inv = <1>; + sound-name-prefix = "Jack"; + }; +}; + +&aop_mbox { + status = "okay"; +}; + +&aop_dart { + status = "okay"; +}; + +&aop_admac { + status = "okay"; +}; + +&aop { + status = "okay"; +}; + +&sep { + status = "okay"; +}; + +&aop_audio { + apple,chassis-name = "J456"; + apple,machine-kind = "iMac"; + apple,no-beamforming; +}; + +/ { + sound { + compatible = "apple,j456-macaudio", "apple,macaudio"; + model = "iMac J456"; + + dai-link@1 { + link-name = "Headphone Jack"; + + cpu { + sound-dai = <&mca 2>; + }; + codec { + sound-dai = <&jack_codec>; + }; + }; + }; +}; + +&gpu { + apple,perf-base-pstate = <3>; +}; + +#include "isp-imx364.dtsi" + +&isp { + apple,platform-id = <2>; +}; + +#include "hwmon-fan-dual.dtsi" diff --git a/arch/arm64/boot/dts/apple/t8103-j457.dts b/arch/arm64/boot/dts/apple/t8103-j457.dts index 7089ccf3ce5566..fc0f28fb1c4367 100644 --- a/arch/arm64/boot/dts/apple/t8103-j457.dts +++ b/arch/arm64/boot/dts/apple/t8103-j457.dts @@ -15,12 +15,27 @@ / { compatible = "apple,j457", "apple,t8103", "apple,arm-platform"; model = "Apple iMac (24-inch, 2x USB-C, M1, 2021)"; + chassis-type = "all-in-one"; aliases { ethernet0 = ðernet0; }; }; +&dcp { + panel: panel { + compatible = "apple,panel-j457", "apple,panel"; + width-mm = <522>; + height-mm = <294>; + apple,max-brightness = <525>; + }; +}; + +&framebuffer0 { + panel = <&panel>; + post-init-providers = <&panel>; +}; + /* * Adjust pcie0's iommu-map to account for the disabled port01. */ @@ -37,6 +52,18 @@ brcm,board-type = "apple,santorini"; }; +/* + * Provide labels for the USB type C ports. + */ + +&typec0 { + label = "USB-C Back-right"; +}; + +&typec1 { + label = "USB-C Back-left"; +}; + /* * Force the bus number assignments so that we can declare some of the * on-board devices and properties that are populated by the bootloader @@ -56,3 +83,72 @@ &pcie0_dart_2 { status = "okay"; }; + +&i2c1 { + jack_codec: codec@48 { + compatible = "cirrus,cs42l83"; + reg = <0x48>; + reset-gpios = <&pinctrl_nub 11 GPIO_ACTIVE_HIGH>; + interrupt-parent = <&pinctrl_ap>; + interrupts = <183 IRQ_TYPE_LEVEL_LOW>; + #sound-dai-cells = <0>; + cirrus,ts-inv = <1>; + sound-name-prefix = "Jack"; + }; +}; + +&aop_mbox { + status = "okay"; +}; + +&aop_dart { + status = "okay"; +}; + +&aop_admac { + status = "okay"; +}; + +&aop { + status = "okay"; +}; + +&sep { + status = "okay"; +}; + +&aop_audio { + apple,chassis-name = "J457"; + apple,machine-kind = "iMac"; + apple,no-beamforming; +}; + +/ { + sound { + compatible = "apple,j457-macaudio", "apple,macaudio"; + model = "iMac J457"; + + dai-link@1 { + link-name = "Headphone Jack"; + + cpu { + sound-dai = <&mca 2>; + }; + codec { + sound-dai = <&jack_codec>; + }; + }; + }; +}; + +&gpu { + apple,perf-base-pstate = <3>; +}; + +#include "isp-imx364.dtsi" + +&isp { + apple,platform-id = <2>; +}; + +#include "hwmon-fan.dtsi" diff --git a/arch/arm64/boot/dts/apple/t8103-jxxx.dtsi b/arch/arm64/boot/dts/apple/t8103-jxxx.dtsi index 0c8206156bfefd..67a57fc507df92 100644 --- a/arch/arm64/boot/dts/apple/t8103-jxxx.dtsi +++ b/arch/arm64/boot/dts/apple/t8103-jxxx.dtsi @@ -12,9 +12,14 @@ / { aliases { bluetooth0 = &bluetooth0; + dcp = &dcp; + disp0 = &display; + disp0_piodma = &disp0_piodma; serial0 = &serial0; serial2 = &serial2; wifi0 = &wifi0; + atcphy0 = &atcphy0; + atcphy1 = &atcphy1; }; chosen { @@ -27,11 +32,19 @@ framebuffer0: framebuffer@0 { compatible = "apple,simple-framebuffer", "simple-framebuffer"; reg = <0 0 0 0>; /* To be filled by loader */ + power-domains = <&ps_disp0_cpu0>; /* Format properties will be added by loader */ status = "disabled"; }; }; + reserved-memory { + #address-cells = <2>; + #size-cells = <2>; + ranges; + /* To be filled by loader */ + }; + memory@800000000 { device_type = "memory"; reg = <0x8 0 0x2 0>; /* To be filled by loader */ @@ -53,6 +66,29 @@ interrupt-parent = <&pinctrl_ap>; interrupts = <106 IRQ_TYPE_LEVEL_LOW>; interrupt-names = "irq"; + + typec0: connector { + compatible = "usb-c-connector"; + power-role = "dual"; + data-role = "dual"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + typec0_connector_hs: endpoint { + remote-endpoint = <&dwc3_0_hs>; + }; + }; + port@1 { + reg = <1>; + typec0_connector_ss: endpoint { + remote-endpoint = <&atcphy0_typec_lanes>; + }; + }; + }; + }; }; hpm1: usb-pd@3f { @@ -61,6 +97,115 @@ interrupt-parent = <&pinctrl_ap>; interrupts = <106 IRQ_TYPE_LEVEL_LOW>; interrupt-names = "irq"; + + typec1: connector { + compatible = "usb-c-connector"; + power-role = "dual"; + data-role = "dual"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + typec1_connector_hs: endpoint { + remote-endpoint = <&dwc3_1_hs>; + }; + }; + port@1 { + reg = <1>; + typec1_connector_ss: endpoint { + remote-endpoint = <&atcphy1_typec_lanes>; + }; + }; + }; + }; + }; +}; + +/* USB controllers */ +&dwc3_0 { + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + dwc3_0_hs: endpoint { + remote-endpoint = <&typec0_connector_hs>; + }; + }; + + port@1 { + reg = <1>; + dwc3_0_ss: endpoint { + remote-endpoint = <&atcphy0_usb3>; + }; + }; + }; +}; + +&dwc3_1 { + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + dwc3_1_hs: endpoint { + remote-endpoint = <&typec1_connector_hs>; + }; + }; + + port@1 { + reg = <1>; + dwc3_1_ss: endpoint { + remote-endpoint = <&atcphy1_usb3>; + }; + }; + }; +}; + +/* Type-C PHYs */ +&atcphy0 { + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + atcphy0_typec_lanes: endpoint { + remote-endpoint = <&typec0_connector_ss>; + }; + }; + + port@1 { + reg = <1>; + atcphy0_usb3: endpoint { + remote-endpoint = <&dwc3_0_ss>; + }; + }; + }; +}; + +&atcphy1 { + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + atcphy1_typec_lanes: endpoint { + remote-endpoint = <&typec1_connector_ss>; + }; + }; + + port@1 { + reg = <1>; + atcphy1_usb3: endpoint { + remote-endpoint = <&dwc3_1_ss>; + }; + }; }; }; @@ -71,6 +216,7 @@ */ &port00 { bus-range = <1 1>; + pwren-gpios = <&smc_gpio 13 GPIO_ACTIVE_HIGH>; wifi0: wifi@0,0 { compatible = "pci14e4,4425"; reg = <0x10000 0x0 0x0 0x0 0x0>; @@ -91,4 +237,6 @@ clock-frequency = <900000000>; }; +#include "hwmon-common.dtsi" + #include "spi1-nvram.dtsi" diff --git a/arch/arm64/boot/dts/apple/t8103-pmgr.dtsi b/arch/arm64/boot/dts/apple/t8103-pmgr.dtsi index c41c57d63997a5..5d3846d44e3578 100644 --- a/arch/arm64/boot/dts/apple/t8103-pmgr.dtsi +++ b/arch/arm64/boot/dts/apple/t8103-pmgr.dtsi @@ -234,7 +234,7 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = "sio_cpu"; - power-domains = <&ps_sio>; + power-domains = <&ps_sio &ps_uart_p &ps_spi_p &ps_dpa0>; }; ps_fpwm0: power-controller@1d8 { @@ -493,6 +493,7 @@ #reset-cells = <0>; label = "mca0"; power-domains = <&ps_audio_p>, <&ps_sio_adma>; + apple,externally-clocked; }; ps_mca1: power-controller@2c0 { @@ -502,6 +503,7 @@ #reset-cells = <0>; label = "mca1"; power-domains = <&ps_audio_p>, <&ps_sio_adma>; + apple,externally-clocked; }; ps_mca2: power-controller@2c8 { @@ -511,6 +513,7 @@ #reset-cells = <0>; label = "mca2"; power-domains = <&ps_audio_p>, <&ps_sio_adma>; + apple,externally-clocked; }; ps_mca3: power-controller@2d0 { @@ -520,6 +523,7 @@ #reset-cells = <0>; label = "mca3"; power-domains = <&ps_audio_p>, <&ps_sio_adma>; + apple,externally-clocked; }; ps_mca4: power-controller@2d8 { @@ -529,6 +533,7 @@ #reset-cells = <0>; label = "mca4"; power-domains = <&ps_audio_p>, <&ps_sio_adma>; + apple,externally-clocked; }; ps_mca5: power-controller@2e0 { @@ -538,6 +543,7 @@ #reset-cells = <0>; label = "mca5"; power-domains = <&ps_audio_p>, <&ps_sio_adma>; + apple,externally-clocked; }; ps_dpa0: power-controller@2e8 { @@ -645,8 +651,7 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = "disp0_fe"; - power-domains = <&ps_rmx>; - apple,always-on; /* TODO: figure out if we can enable PM here */ + power-domains = <&ps_rmx>, <&ps_pmp>; }; ps_dispext_fe: power-controller@368 { @@ -655,7 +660,7 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = "dispext_fe"; - power-domains = <&ps_rmx>; + power-domains = <&ps_rmx>, <&ps_pmp>; }; ps_dispext_cpu0: power-controller@378 { @@ -717,6 +722,7 @@ #reset-cells = <0>; label = "apcie_gp"; power-domains = <&ps_apcie>; + apple,always-on; /* Breaks things if shut down */ }; ps_ans2: power-controller@3f0 { @@ -733,6 +739,7 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = "gfx"; + power-domains = <&ps_pmp>; }; ps_dcs4: power-controller@320 { @@ -805,6 +812,7 @@ #reset-cells = <0>; label = "isp_sys"; power-domains = <&ps_rmx>; + status = "disabled"; }; ps_venc_sys: power-controller@408 { @@ -1000,9 +1008,125 @@ #reset-cells = <0>; label = "disp0_cpu0"; power-domains = <&ps_disp0_fe>; - apple,always-on; /* TODO: figure out if we can enable PM here */ apple,min-state = <4>; }; + + /* There is a dependency tree involved with these PDs, + * but we do not express it here since the ISP driver + * is supposed to sequence them in the right order anyway + * (and we do not know the exact tree structure). + * + * This also works around spurious parent PD activation + * on machines with ISP disabled (desktops). + */ + ps_isp_set0: power-controller@4000 { + compatible = "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4000 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_set0"; + apple,force-disable; + }; + + ps_isp_set1: power-controller@4008 { + compatible = "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4008 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_set1"; + apple,force-disable; + apple,force-reset; + }; + + ps_isp_set2: power-controller@4010 { + compatible = "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4010 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_set2"; + apple,force-disable; + apple,force-reset; + }; + + ps_isp_fe: power-controller@4018 { + compatible = "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4018 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_fe"; + }; + + ps_isp_set4: power-controller@4020 { + compatible = "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4020 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_set4"; + }; + + ps_isp_set5: power-controller@4028 { + compatible = "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4028 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_set5"; + }; + + ps_isp_set6: power-controller@4030 { + compatible = "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4030 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_set6"; + }; + + ps_isp_set7: power-controller@4038 { + compatible = "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4038 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_set7"; + }; + + ps_isp_set8: power-controller@4040 { + compatible = "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4040 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_set8"; + }; + + ps_isp_set9: power-controller@4048 { + compatible = "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4048 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_set9"; + }; + + ps_isp_set10: power-controller@4050 { + compatible = "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4050 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_set10"; + }; + + ps_isp_set11: power-controller@4058 { + compatible = "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4058 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_set11"; + }; + + ps_isp_set12: power-controller@4060 { + compatible = "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4060 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_set12"; + }; }; &pmgr_mini { @@ -1095,6 +1219,7 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = "msg"; + apple,always-on; /* Core AON device? */ }; ps_atc0_usb_aon: power-controller@88 { @@ -1103,6 +1228,7 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = "atc0_usb_aon"; + apple,always-on; /* Needs to stay on for dwc3 to work */ }; ps_atc1_usb_aon: power-controller@90 { @@ -1111,6 +1237,7 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = "atc1_usb_aon"; + apple,always-on; /* Needs to stay on for dwc3 to work */ }; ps_atc0_usb: power-controller@98 { diff --git a/arch/arm64/boot/dts/apple/t8103.dtsi b/arch/arm64/boot/dts/apple/t8103.dtsi index 8b7b2788796874..d3fc50b8f901b5 100644 --- a/arch/arm64/boot/dts/apple/t8103.dtsi +++ b/arch/arm64/boot/dts/apple/t8103.dtsi @@ -12,6 +12,7 @@ #include #include #include +#include / { compatible = "apple,t8103", "apple,arm-platform"; @@ -193,26 +194,31 @@ opp-hz = /bits/ 64 <600000000>; opp-level = <1>; clock-latency-ns = <7500>; + opp-microwatt = <47296>; }; opp02 { opp-hz = /bits/ 64 <972000000>; opp-level = <2>; clock-latency-ns = <22000>; + opp-microwatt = <99715>; }; opp03 { opp-hz = /bits/ 64 <1332000000>; opp-level = <3>; clock-latency-ns = <27000>; + opp-microwatt = <188860>; }; opp04 { opp-hz = /bits/ 64 <1704000000>; opp-level = <4>; clock-latency-ns = <33000>; + opp-microwatt = <288891>; }; opp05 { opp-hz = /bits/ 64 <2064000000>; opp-level = <5>; clock-latency-ns = <50000>; + opp-microwatt = <412979>; }; }; @@ -223,83 +229,140 @@ opp-hz = /bits/ 64 <600000000>; opp-level = <1>; clock-latency-ns = <8000>; + opp-microwatt = <290230>; }; opp02 { opp-hz = /bits/ 64 <828000000>; opp-level = <2>; clock-latency-ns = <19000>; + opp-microwatt = <449013>; }; opp03 { opp-hz = /bits/ 64 <1056000000>; opp-level = <3>; clock-latency-ns = <21000>; + opp-microwatt = <647097>; }; opp04 { opp-hz = /bits/ 64 <1284000000>; opp-level = <4>; clock-latency-ns = <23000>; + opp-microwatt = <865620>; }; opp05 { opp-hz = /bits/ 64 <1500000000>; opp-level = <5>; clock-latency-ns = <24000>; + opp-microwatt = <1112838>; }; opp06 { opp-hz = /bits/ 64 <1728000000>; opp-level = <6>; clock-latency-ns = <29000>; + opp-microwatt = <1453271>; }; opp07 { opp-hz = /bits/ 64 <1956000000>; opp-level = <7>; clock-latency-ns = <31000>; + opp-microwatt = <1776667>; }; opp08 { opp-hz = /bits/ 64 <2184000000>; opp-level = <8>; clock-latency-ns = <34000>; + opp-microwatt = <2366690>; }; opp09 { opp-hz = /bits/ 64 <2388000000>; opp-level = <9>; clock-latency-ns = <36000>; + opp-microwatt = <2892193>; }; opp10 { opp-hz = /bits/ 64 <2592000000>; opp-level = <10>; clock-latency-ns = <51000>; + opp-microwatt = <3475417>; }; opp11 { opp-hz = /bits/ 64 <2772000000>; opp-level = <11>; clock-latency-ns = <54000>; + opp-microwatt = <3959410>; }; opp12 { opp-hz = /bits/ 64 <2988000000>; opp-level = <12>; clock-latency-ns = <55000>; + opp-microwatt = <4540620>; }; -#if 0 /* Not available until CPU deep sleep is implemented */ opp13 { opp-hz = /bits/ 64 <3096000000>; opp-level = <13>; clock-latency-ns = <55000>; + opp-microwatt = <4745031>; turbo-mode; }; opp14 { opp-hz = /bits/ 64 <3144000000>; opp-level = <14>; clock-latency-ns = <56000>; + opp-microwatt = <4822390>; turbo-mode; }; opp15 { opp-hz = /bits/ 64 <3204000000>; opp-level = <15>; clock-latency-ns = <56000>; + opp-microwatt = <4951324>; turbo-mode; }; -#endif + }; + + gpu_opp: opp-table-gpu { + compatible = "operating-points-v2"; + + /* + * NOTE: The voltage and power values are device-specific and + * must be filled in by the bootloader. + */ + opp00 { + opp-hz = /bits/ 64 <0>; + opp-microvolt = <400000>; + opp-microwatt = <0>; + }; + opp01 { + opp-hz = /bits/ 64 <396000000>; + opp-microvolt = <603000>; + opp-microwatt = <3714690>; + }; + opp02 { + opp-hz = /bits/ 64 <528000000>; + opp-microvolt = <640000>; + opp-microwatt = <5083260>; + }; + opp03 { + opp-hz = /bits/ 64 <720000000>; + opp-microvolt = <690000>; + opp-microwatt = <7429380>; + }; + opp04 { + opp-hz = /bits/ 64 <924000000>; + opp-microvolt = <784000>; + opp-microwatt = <11730600>; + }; + opp05 { + opp-hz = /bits/ 64 <1128000000>; + opp-microvolt = <862000>; + opp-microwatt = <17009370>; + }; + opp06 { + opp-hz = /bits/ 64 <1278000000>; + opp-microvolt = <931000>; + opp-microwatt = <19551000>; + }; }; timer { @@ -345,6 +408,22 @@ clock-output-names = "clk_200m"; }; + /* Pixel clock? frequency in Hz (compare: 4K@60 VGA clock 533.250 MHz) */ + clk_disp0: clock-disp0 { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <533333328>; + clock-output-names = "clk_disp0"; + }; + + /* Pixel clock? frequency in Hz (compare: 4K@60 VGA clock 533.250 MHz) */ + clk_dispext0: clock-dispext0 { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <0>; + clock-output-names = "clk_dispext0"; + }; + /* * This is a fabulated representation of the input clock * to NCO since we don't know the true clock tree. @@ -373,15 +452,15 @@ }; uat_handoff: uat-handoff { - status = "disabled"; + reg = <0 0 0 0>; }; uat_pagetables: uat-pagetables { - status = "disabled"; + reg = <0 0 0 0>; }; uat_ttbs: uat-ttbs { - status = "disabled"; + reg = <0 0 0 0>; }; }; @@ -392,9 +471,11 @@ ranges; nonposted-mmio; + /* Required to get >32-bit DMA via DARTs */ + dma-ranges = <0 0 0 0 0xffffffff 0xffffc000>; gpu: gpu@206400000 { - compatible = "apple,agx-g13g"; + compatible = "apple,agx-t8103", "apple,agx-g13g"; reg = <0x2 0x6400000 0 0x40000>, <0x2 0x4000000 0 0x1000000>; reg-names = "asc", "sgx"; @@ -406,6 +487,40 @@ "hw-cal-a", "hw-cal-b", "globals"; apple,firmware-abi = <0 0 0>; + + apple,firmware-version = <12 3 0>; + apple,firmware-compat = <12 3 0>; + + operating-points-v2 = <&gpu_opp>; + apple,perf-base-pstate = <1>; + apple,min-sram-microvolt = <850000>; + apple,avg-power-filter-tc-ms = <1000>; + apple,avg-power-ki-only = <7.5>; + apple,avg-power-kp = <4.0>; + apple,avg-power-min-duty-cycle = <40>; + apple,avg-power-target-filter-tc = <125>; + apple,fast-die0-integral-gain = <200.0>; + apple,fast-die0-proportional-gain = <5.0>; + apple,perf-filter-drop-threshold = <0>; + apple,perf-filter-time-constant = <5>; + apple,perf-filter-time-constant2 = <50>; + apple,perf-integral-gain2 = <0.197392>; + apple,perf-integral-min-clamp = <0>; + apple,perf-proportional-gain2 = <6.853981>; + apple,perf-tgt-utilization = <85>; + apple,power-sample-period = <8>; + apple,power-zones = <30000 100 6875>; + apple,ppm-filter-time-constant-ms = <100>; + apple,ppm-ki = <91.5>; + apple,ppm-kp = <6.9>; + apple,pwr-filter-time-constant = <313>; + apple,pwr-integral-gain = <0.0202129>; + apple,pwr-integral-min-clamp = <0>; + apple,pwr-min-duty-cycle = <40>; + apple,pwr-proportional-gain = <5.2831855>; + + apple,core-leak-coef = <1000.0>; + apple,sram-leak-coef = <45.0>; }; agx_mbox: mbox@206408000 { @@ -492,6 +607,146 @@ }; }; + disp0_dart: iommu@231304000 { + compatible = "apple,t8103-dart"; + reg = <0x2 0x31304000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_disp0_cpu0>; + apple,dma-range = <0x0 0x0 0x0 0xfc000000>; + status = "disabled"; + }; + + dcp_dart: iommu@23130c000 { + compatible = "apple,t8103-dart"; + reg = <0x2 0x3130c000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + apple,dma-range = <0xf 0x00000000 0x0 0xfc000000>; + power-domains = <&ps_disp0_cpu0>; + }; + + dcp_mbox: mbox@231c08000 { + compatible = "apple,t8103-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x2 0x31c08000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + power-domains = <&ps_disp0_cpu0>; + resets = <&ps_disp0_cpu0>; + }; + + dcp: dcp@231c00000 { + compatible = "apple,t8103-dcp", "apple,dcp"; + mboxes = <&dcp_mbox>; + mbox-names = "mbox"; + iommus = <&dcp_dart 0>; + + reg-names = "coproc", "disp-0", "disp-1", "disp-2", + "disp-3", "disp-4"; + reg = <0x2 0x31c00000 0x0 0x4000>, + <0x2 0x30000000 0x0 0x3e8000>, + <0x2 0x31320000 0x0 0x4000>, + <0x2 0x31344000 0x0 0x4000>, + <0x2 0x31800000 0x0 0x800000>, + <0x2 0x3b3d0000 0x0 0x4000>; + apple,bw-scratch = <&pmgr_dcp 0 5 0x14>; + apple,bw-doorbell = <&pmgr_dcp 1 6>; + power-domains = <&ps_disp0_cpu0>; + resets = <&ps_disp0_cpu0>; + clocks = <&clk_disp0>; + phandle = <&dcp>; + // required bus properties for 'piodma' subdevice + #address-cells = <2>; + #size-cells = <2>; + + disp0_piodma: piodma { + iommus = <&disp0_dart 4>; + phandle = <&disp0_piodma>; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + dcp_audio: endpoint { + remote-endpoint = <&dpaudio0_dcp>; + }; + }; + }; + }; + + display: display-subsystem { + compatible = "apple,display-subsystem"; + /* disp_dart0 must be 1st since it is locked */ + iommus = <&disp0_dart 0>; + /* generate phandle explicitly for use in loader */ + phandle = <&display>; + }; + + isp_dart0: iommu@22c0e8000 { + compatible = "apple,t8103-dart"; + reg = <0x2 0x2c0e8000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_isp_sys>; + + status = "disabled"; + }; + + isp_dart1: iommu@22c0f4000 { + compatible = "apple,t8103-dart"; + reg = <0x2 0x2c0f4000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_isp_sys>; + + status = "disabled"; + }; + + isp_dart2: iommu@22c0fc000 { + compatible = "apple,t8103-dart"; + reg = <0x2 0x2c0fc000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_isp_sys>; + + status = "disabled"; + }; + + isp: isp@22a000000 { + compatible = "apple,t8103-isp", "apple,isp"; + iommus = <&isp_dart0 0>, <&isp_dart1 0>, <&isp_dart2 0>; + reg-names = "coproc", "mbox", "gpio", "mbox2"; + reg = <0x2 0x2a000000 0x0 0x2000000>, + <0x2 0x2c104000 0x0 0x100>, + <0x2 0x2c104170 0x0 0x100>, + <0x2 0x2c1043f0 0x0 0x100>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_isp_sys>, <&ps_isp_set0>, + <&ps_isp_set1>, <&ps_isp_set2>, <&ps_isp_fe>, + <&ps_isp_set4>, <&ps_isp_set5>, <&ps_isp_set6>, + <&ps_isp_set7>, <&ps_isp_set8>, <&ps_isp_set9>, + <&ps_isp_set10>, <&ps_isp_set11>, + <&ps_isp_set12>; + + apple,dart-vm-size = <0x0 0xa0000000>; + + status = "disabled"; + }; + sio_dart: iommu@235004000 { compatible = "apple,t8103-dart"; reg = <0x2 0x35004000 0x0 0x4000>; @@ -647,6 +902,32 @@ status = "disabled"; }; + sio_mbox: mbox@236408000 { + compatible = "apple,t8103-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x2 0x36408000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + power-domains = <&ps_sio>; + }; + + sio: sio@236400000 { + compatible = "apple,t8103-sio", "apple,sio"; + reg = <0x2 0x36400000 0x0 0x8000>; + dma-channels = <128>; + #dma-cells = <1>; + mboxes = <&sio_mbox>; + iommus = <&sio_dart 0>; + power-domains = <&ps_sio_cpu>; + resets = <&ps_sio>; /* TODO: verify reset does something */ + status = "disabled"; + }; + admac: dma-controller@238200000 { compatible = "apple,t8103-admac", "apple,admac"; reg = <0x2 0x38200000 0x0 0x34000>; @@ -661,6 +942,48 @@ resets = <&ps_audio_p>; }; + dpaudio0: audio-controller@238330000 { + compatible = "apple,t8103-dpaudio", "apple,dpaudio"; + reg = <0x2 0x38330000 0x0 0x4000>; + dmas = <&sio 0x64>; + dma-names = "tx"; + power-domains = <&ps_dpa0>; + reset-domains = <&ps_dpa0>; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + dpaudio0_dcp: endpoint { + remote-endpoint = <&dcp_audio>; + }; + }; + }; + }; + + dpaudio1: audio-controller@238334000 { + compatible = "apple,t8103-dpaudio", "apple,dpaudio"; + reg = <0x2 0x38334000 0x0 0x4000>; + dmas = <&sio 0x66>; + dma-names = "tx"; + power-domains = <&ps_dpa1>; + reset-domains = <&ps_dpa1>; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + dpaudio1_dcp: endpoint { + remote-endpoint = <&dcpext_audio>; + }; + }; + }; + }; + mca: i2s@238400000 { compatible = "apple,t8103-mca", "apple,mca"; reg = <0x2 0x38400000 0x0 0x18000>, @@ -729,6 +1052,14 @@ reg = <0x2 0x3b700000 0 0x14000>; }; + pmgr_dcp: power-management@23b738000 { + reg = <0x2 0x3b738000 0x0 0x1000>, + <0x2 0x3bc3c000 0x0 0x1000>; + reg-names = "dcp-bw-scratch", "dcp-bw-doorbell"; + #apple,bw-scratch-cells = <3>; + #apple,bw-doorbell-cells = <2>; + }; + pinctrl_ap: pinctrl@23c100000 { compatible = "apple,t8103-pinctrl", "apple,pinctrl"; reg = <0x2 0x3c100000 0x0 0x100000>; @@ -909,12 +1240,22 @@ #gpio-cells = <2>; }; + smc_hwmon: hwmon { + compatible = "apple,smc-hwmon"; + }; + smc_reboot: reboot { compatible = "apple,smc-reboot"; nvmem-cells = <&shutdown_flag>, <&boot_stage>, - <&boot_error_count>, <&panic_count>; + <&boot_error_count>, <&panic_count>, <&pm_setting>; nvmem-cell-names = "shutdown_flag", "boot_stage", - "boot_error_count", "panic_count"; + "boot_error_count", "panic_count", "pm_setting"; + }; + + rtc { + compatible = "apple,smc-rtc"; + nvmem-cells = <&rtc_offset>; + nvmem-cell-names = "rtc_offset"; }; }; @@ -952,6 +1293,36 @@ ; }; + sep_dart: iommu@2412c0000 { + compatible = "apple,t8103-dart"; + reg = <0x2 0x412c0000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + }; + + sep: sep@242400000 { + compatible = "apple,sep"; + reg = <0x2 0x42400000 0x0 0x6C000>; + mboxes = <&sep_mbox>; + mbox-names = "mbox"; + iommus = <&sep_dart 0>; + status = "disabled"; + }; + + sep_mbox: mbox@242408000 { + compatible = "apple,t8103-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x2 0x42408000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + }; + pinctrl_aop: pinctrl@24a820000 { compatible = "apple,t8103-pinctrl", "apple,pinctrl"; reg = <0x2 0x4a820000 0x0 0x4000>; @@ -973,6 +1344,147 @@ ; }; + aop_mbox: mbox@24a408000 { + compatible = "apple,t8103-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x2 0x4a408000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + status = "disabled"; + }; + + aop_dart: iommu@24a808000 { + compatible = "apple,t8103-dart"; + reg = <0x2 0x4a808000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + status = "disabled"; + }; + + aop_admac: dma-controller@24a980000 { + compatible = "apple,t8103-admac", "apple,admac"; + reg = <0x2 0x4a980000 0x0 0x34000>; + #dma-cells = <1>; + dma-channels = <16>; + interrupts-extended = <0>, + <0>, + <&aic AIC_IRQ 321 IRQ_TYPE_LEVEL_HIGH>, + <0>; + iommus = <&aop_dart 7>; + status = "disabled"; + }; + + aop: aop@24ac00000 { + compatible = "apple,t8103-aop"; + reg = <0x2 0x4ac00000 0x0 0x1e0000>, + <0x2 0x4a400000 0x0 0x6c000>; + mboxes = <&aop_mbox>; + mbox-names = "mbox"; + iommus = <&aop_dart 0>; + + status = "disabled"; + + aop_audio: audio { + compatible = "apple,t8103-aop-audio", "apple,aop-audio"; + dmas = <&aop_admac 1>; + dma-names = "dma"; + }; + + aop_als: als { + compatible = "apple,t8103-aop-als", "apple,aop-als"; + // intentionally empty + }; + + las { + compatible = "apple,t8103-aop-las", "apple,aop-las"; + }; + }; + + dispext0_dart: iommu@271304000 { + compatible = "apple,t8103-dart"; + reg = <0x2 0x71304000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_dispext_cpu0>; + apple,dma-range = <0x0 0x0 0x0 0xfc000000>; + status = "disabled"; + }; + + dcpext_dart: iommu@27130c000 { + compatible = "apple,t8103-dart"; + reg = <0x2 0x7130c000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_dispext_cpu0>; + apple,dma-range = <0xf 0x00000000 0x0 0xfc000000>; + status = "disabled"; + }; + + dcpext_mbox: mbox@271c08000 { + compatible = "apple,t8103-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x2 0x71c08000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + power-domains = <&ps_dispext_cpu0>; + resets = <&ps_dispext_cpu0>; + status = "disabled"; + }; + + dcpext: dcp@271c00000 { + compatible = "apple,t8103-dcpext", "apple,dcpext"; + mboxes = <&dcpext_mbox>; + mbox-names = "mbox"; + iommus = <&dcpext_dart 0>; + phandle = <&dcpext>; + + reg-names = "coproc", "disp-0", "disp-1", "disp-2", + "disp-3", "disp-4"; + reg = <0x2 0x71c00000 0x0 0x4000>, + <0x2 0x70000000 0x0 0x118000>, + <0x2 0x71320000 0x0 0x4000>, + <0x2 0x71344000 0x0 0x4000>, + <0x2 0x71800000 0x0 0x800000>, + <0x2 0x3b3d0000 0x0 0x4000>; + apple,bw-scratch = <&pmgr_dcp 0 5 0x18>; + apple,bw-doorbell = <&pmgr_dcp 1 6>; + power-domains = <&ps_dispext_cpu0>; + resets = <&ps_dispext_cpu0>; + clocks = <&clk_dispext0>; + status = "disabled"; + // required bus properties for 'piodma' subdevice + #address-cells = <2>; + #size-cells = <2>; + + piodma { + iommus = <&dispext0_dart 4>; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + dcpext_audio: endpoint { + remote-endpoint = <&dpaudio1_dcp>; + }; + }; + }; + }; + ans_mbox: mbox@277408000 { compatible = "apple,t8103-asc-mailbox", "apple,asc-mailbox-v4"; reg = <0x2 0x77408000 0x0 0x4000>; @@ -1007,6 +1519,124 @@ resets = <&ps_ans2>; }; + dwc3_0: usb@382280000 { + compatible = "apple,t8103-dwc3"; + reg = <0x3 0x82280000 0x0 0xcd00>, <0x3 0x8228cd00 0x0 0x3200>; + reg-names = "dwc3-core", "dwc3-apple"; + interrupt-parent = <&aic>; + interrupts = ; + dr_mode = "otg"; + usb-role-switch; + role-switch-default-mode = "host"; + iommus = <&dwc3_0_dart_0 0>, <&dwc3_0_dart_1 1>; + power-domains = <&ps_atc0_usb>; + resets = <&atcphy0>; + phys = <&atcphy0 PHY_TYPE_USB2>, <&atcphy0 PHY_TYPE_USB3>; + phy-names = "usb2-phy", "usb3-phy"; + }; + + dwc3_0_dart_0: iommu@382f00000 { + compatible = "apple,t8103-dart"; + reg = <0x3 0x82f00000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + #iommu-cells = <1>; + power-domains = <&ps_atc0_usb>; + }; + + dwc3_0_dart_1: iommu@382f80000 { + compatible = "apple,t8103-dart"; + reg = <0x3 0x82f80000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + #iommu-cells = <1>; + power-domains = <&ps_atc0_usb>; + }; + + atcphy0: phy@383000000 { + compatible = "apple,t8103-atcphy"; + reg = <0x3 0x83000000 0x0 0x4c000>, + <0x3 0x83050000 0x0 0x8000>, + <0x3 0x80000000 0x0 0x4000>, + <0x3 0x82a90000 0x0 0x4000>, + <0x3 0x82a84000 0x0 0x4000>; + reg-names = "core", "lpdptx", "axi2af", "usb2phy", + "pipehandler"; + + #phy-cells = <1>; + #reset-cells = <0>; + + orientation-switch; + mode-switch; + power-domains = <&ps_atc0_usb>; + }; + + atcphy0_xbar: mux@38304c000 { + compatible = "apple,t8103-display-crossbar"; + reg = <0x3 0x8304c000 0x0 0x4000>; + #mux-control-cells = <1>; + power-domains = <&ps_atc0_usb>; + }; + + dwc3_1: usb@502280000 { + compatible = "apple,t8103-dwc3"; + reg = <0x5 0x02280000 0x0 0xcd00>, <0x5 0x0228cd00 0x0 0x3200>; + reg-names = "dwc3-core", "dwc3-apple"; + interrupt-parent = <&aic>; + interrupts = ; + dr_mode = "otg"; + usb-role-switch; + role-switch-default-mode = "host"; + iommus = <&dwc3_1_dart_0 0>, <&dwc3_1_dart_1 1>; + power-domains = <&ps_atc1_usb>; + resets = <&atcphy1>; + phys = <&atcphy1 PHY_TYPE_USB2>, <&atcphy1 PHY_TYPE_USB3>; + phy-names = "usb2-phy", "usb3-phy"; + }; + + dwc3_1_dart_0: iommu@502f00000 { + compatible = "apple,t8103-dart"; + reg = <0x5 0x02f00000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + #iommu-cells = <1>; + power-domains = <&ps_atc1_usb>; + }; + + dwc3_1_dart_1: iommu@502f80000 { + compatible = "apple,t8103-dart"; + reg = <0x5 0x02f80000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + #iommu-cells = <1>; + power-domains = <&ps_atc1_usb>; + }; + + atcphy1: phy@503000000 { + compatible = "apple,t8103-atcphy"; + reg = <0x5 0x03000000 0x0 0x4c000>, + <0x5 0x03050000 0x0 0x8000>, + <0x5 0x0 0x0 0x4000>, + <0x5 0x02a90000 0x0 0x4000>, + <0x5 0x02a84000 0x0 0x4000>; + reg-names = "core", "lpdptx", "axi2af", "usb2phy", + "pipehandler"; + + #phy-cells = <1>; + #reset-cells = <0>; + + orientation-switch; + mode-switch; + power-domains = <&ps_atc1_usb>; + }; + + atcphy1_xbar: mux@50304c000 { + compatible = "apple,t8103-display-crossbar"; + reg = <0x5 0x0304c000 0x0 0x4000>; + #mux-control-cells = <1>; + power-domains = <&ps_atc1_usb>; + }; + pcie0_dart_0: iommu@681008000 { compatible = "apple,t8103-dart"; reg = <0x6 0x81008000 0x0 0x4000>; @@ -1072,6 +1702,8 @@ pinctrl-0 = <&pcie_pins>; pinctrl-names = "default"; + dma-coherent; + port00: pci@0,0 { device_type = "pci"; reg = <0x0 0x0 0x0 0x0 0x0>; diff --git a/arch/arm64/boot/dts/apple/t8112-j413.dts b/arch/arm64/boot/dts/apple/t8112-j413.dts index 6f69658623bf89..f36d40cb7fe611 100644 --- a/arch/arm64/boot/dts/apple/t8112-j413.dts +++ b/arch/arm64/boot/dts/apple/t8112-j413.dts @@ -16,9 +16,11 @@ / { compatible = "apple,j413", "apple,t8112", "apple,arm-platform"; model = "Apple MacBook Air (13-inch, M2, 2022)"; + chassis-type = "laptop"; aliases { bluetooth0 = &bluetooth0; + keyboard = &keyboard; wifi0 = &wifi0; }; @@ -35,6 +37,21 @@ }; }; +&dcp { + panel: panel { + compatible = "apple,panel-j413", "apple,panel"; + width-mm = <290>; + height-mm = <189>; + adj-height-mm = <181>; + apple,max-brightness = <525>; + }; +}; + +&framebuffer0 { + panel = <&panel>; + post-init-providers = <&panel>; +}; + /* * Force the bus number assignments so that we can declare some of the * on-board devices and properties that are populated by the bootloader @@ -42,6 +59,7 @@ */ &port00 { bus-range = <1 1>; + pwren-gpios = <&smc_gpio 13 GPIO_ACTIVE_HIGH>; wifi0: wifi@0,0 { compatible = "pci14e4,4433"; reg = <0x10000 0x0 0x0 0x0 0x0>; @@ -60,6 +78,18 @@ }; }; +/* + * Provide labels for the USB type C ports. + */ + +&typec0 { + label = "USB-C Left-back"; +}; + +&typec1 { + label = "USB-C Left-front"; +}; + &i2c0 { /* MagSafe port */ hpm5: usb-pd@3a { @@ -71,6 +101,76 @@ }; }; +/* Virtual regulator representing the shared shutdown GPIO */ +/ { + speaker_sdz: fixed-regulator-sn012776-sdz { + compatible = "regulator-fixed"; + regulator-name = "sn012776-sdz"; + startup-delay-us = <5000>; + gpios = <&pinctrl_ap 88 GPIO_ACTIVE_HIGH>; + enable-active-high; + }; +}; + +&i2c1 { + speaker_left_woof: codec@38 { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x38>; + SDZ-supply = <&speaker_sdz>; + #sound-dai-cells = <0>; + sound-name-prefix = "Left Woofer"; + interrupts-extended = <&pinctrl_ap 11 IRQ_TYPE_LEVEL_LOW>; + ti,imon-slot-no = <0>; + ti,vmon-slot-no = <2>; + ti,sdout-force-zero-mask = <0xf0f0>; + }; + + speaker_left_tweet: codec@39 { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x39>; + SDZ-supply = <&speaker_sdz>; + #sound-dai-cells = <0>; + sound-name-prefix = "Left Tweeter"; + interrupts-extended = <&pinctrl_ap 11 IRQ_TYPE_LEVEL_LOW>; + ti,imon-slot-no = <8>; + ti,vmon-slot-no = <10>; + }; +}; + +&i2c3 { + speaker_right_woof: codec@3b { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x3b>; + SDZ-supply = <&speaker_sdz>; + #sound-dai-cells = <0>; + sound-name-prefix = "Right Woofer"; + interrupts-extended = <&pinctrl_ap 11 IRQ_TYPE_LEVEL_LOW>; + ti,imon-slot-no = <4>; + ti,vmon-slot-no = <6>; + ti,sdout-force-zero-mask = <0x0f0f>; + }; + + speaker_right_tweet: codec@3c { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x3c>; + SDZ-supply = <&speaker_sdz>; + #sound-dai-cells = <0>; + sound-name-prefix = "Right Tweeter"; + interrupts-extended = <&pinctrl_ap 11 IRQ_TYPE_LEVEL_LOW>; + ti,imon-slot-no = <12>; + ti,vmon-slot-no = <14>; + }; + + jack_codec: codec@4b { + compatible = "cirrus,cs42l84"; + reg = <0x4b>; + reset-gpios = <&pinctrl_nub 12 GPIO_ACTIVE_HIGH>; + #sound-dai-cells = <0>; + interrupts-extended = <&pinctrl_ap 149 IRQ_TYPE_LEVEL_LOW>; + sound-name-prefix = "Jack"; + }; +}; + &i2c4 { status = "okay"; }; @@ -78,3 +178,102 @@ &fpwm1 { status = "okay"; }; + +&aop_mbox { + status = "okay"; +}; + +&aop_dart { + status = "okay"; +}; + +&aop_admac { + status = "okay"; +}; + +&aop { + status = "okay"; +}; + +&aop_audio { + apple,chassis-name = "J413"; + apple,machine-kind = "MacBook Air"; +}; + +/ { + sound { + compatible = "apple,j413-macaudio", "apple,macaudio"; + model = "MacBook Air J413"; + + dai-link@0 { + link-name = "Speakers"; + + cpu { + sound-dai = <&mca 0>, <&mca 1>; + }; + codec { + sound-dai = <&speaker_left_woof>, <&speaker_left_tweet>, + <&speaker_right_woof>, <&speaker_right_tweet>; + }; + }; + + dai-link@1 { + link-name = "Headphone Jack"; + + cpu { + sound-dai = <&mca 2>; + }; + codec { + sound-dai = <&jack_codec>; + }; + }; + }; +}; + +&mtp { + status = "okay"; +}; +&mtp_mbox { + status = "okay"; +}; +&mtp_dart { + status = "okay"; +}; +&mtp_dockchannel { + status = "okay"; +}; +&mtp_hid { + apple,afe-reset-gpios = <&smc_gpio 8 GPIO_ACTIVE_LOW>; + apple,stm-reset-gpios = <&smc_gpio 24 GPIO_ACTIVE_LOW>; + + multi-touch { + firmware-name = "apple/tpmtfw-j413.bin"; + }; + + keyboard: keyboard { + hid-country-code = <0>; + apple,keyboard-layout-id = <0>; + }; + + stm { + }; + + actuator { + }; + + tp_accel { + }; +}; + +#include "isp-imx558-cfg0.dtsi" + +&isp { + apple,platform-id = <14>; + apple,temporal-filter = <1>; +}; + +&pmp_report_isp_sys { + status = "okay"; +}; + +#include "hwmon-laptop.dtsi" diff --git a/arch/arm64/boot/dts/apple/t8112-j415.dts b/arch/arm64/boot/dts/apple/t8112-j415.dts index b54e218e5384ca..7f63969ede2ab9 100644 --- a/arch/arm64/boot/dts/apple/t8112-j415.dts +++ b/arch/arm64/boot/dts/apple/t8112-j415.dts @@ -16,9 +16,11 @@ / { compatible = "apple,j415", "apple,t8112", "apple,arm-platform"; model = "Apple MacBook Air (15-inch, M2, 2023)"; + chassis-type = "laptop"; aliases { bluetooth0 = &bluetooth0; + keyboard = &keyboard; wifi0 = &wifi0; }; @@ -35,6 +37,21 @@ }; }; +&dcp { + panel: panel { + compatible = "apple,panel-j415", "apple,panel"; + width-mm = <327>; + height-mm = <211>; + adj-height-mm = <204>; + apple,max-brightness = <500>; + }; +}; + +&framebuffer0 { + panel = <&panel>; + post-init-providers = <&panel>; +}; + /* * Force the bus number assignments so that we can declare some of the * on-board devices and properties that are populated by the bootloader @@ -42,6 +59,7 @@ */ &port00 { bus-range = <1 1>; + pwren-gpios = <&smc_gpio 13 GPIO_ACTIVE_HIGH>; wifi0: wifi@0,0 { compatible = "pci14e4,4433"; reg = <0x10000 0x0 0x0 0x0 0x0>; @@ -60,6 +78,18 @@ }; }; +/* + * Provide labels for the USB type C ports. + */ + +&typec0 { + label = "USB-C Left-back"; +}; + +&typec1 { + label = "USB-C Left-front"; +}; + &i2c0 { /* MagSafe port */ hpm5: usb-pd@3a { @@ -71,6 +101,98 @@ }; }; +/* Virtual regulator representing the shared shutdown GPIO */ +/ { + speaker_sdz: fixed-regulator-sn012776-sdz { + compatible = "regulator-fixed"; + regulator-name = "sn012776-sdz"; + startup-delay-us = <5000>; + gpios = <&pinctrl_ap 88 GPIO_ACTIVE_HIGH>; + enable-active-high; + }; +}; + +&i2c1 { + speaker_left_woof1: codec@38 { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x38>; + SDZ-supply = <&speaker_sdz>; + #sound-dai-cells = <0>; + sound-name-prefix = "Left Woofer 1"; + interrupts-extended = <&pinctrl_ap 11 IRQ_TYPE_LEVEL_LOW>; + ti,imon-slot-no = <0>; + ti,vmon-slot-no = <2>; + ti,sdout-force-zero-mask = <0xf0f0f0>; + }; + + speaker_left_tweet: codec@39 { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x39>; + SDZ-supply = <&speaker_sdz>; + #sound-dai-cells = <0>; + sound-name-prefix = "Left Tweeter"; + interrupts-extended = <&pinctrl_ap 11 IRQ_TYPE_LEVEL_LOW>; + ti,imon-slot-no = <8>; + ti,vmon-slot-no = <10>; + }; + + speaker_left_woof2: codec@3a { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x3a>; + SDZ-supply = <&speaker_sdz>; + #sound-dai-cells = <0>; + sound-name-prefix = "Left Woofer 2"; + interrupts-extended = <&pinctrl_ap 11 IRQ_TYPE_LEVEL_LOW>; + ti,imon-slot-no = <16>; + ti,vmon-slot-no = <18>; + }; +}; + +&i2c3 { + speaker_right_woof1: codec@3b { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x3b>; + SDZ-supply = <&speaker_sdz>; + #sound-dai-cells = <0>; + sound-name-prefix = "Right Woofer 1"; + interrupts-extended = <&pinctrl_ap 11 IRQ_TYPE_LEVEL_LOW>; + ti,imon-slot-no = <4>; + ti,vmon-slot-no = <6>; + ti,sdout-force-zero-mask = <0x0f0f0f>; + }; + + speaker_right_tweet: codec@3c { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x3c>; + SDZ-supply = <&speaker_sdz>; + #sound-dai-cells = <0>; + sound-name-prefix = "Right Tweeter"; + interrupts-extended = <&pinctrl_ap 11 IRQ_TYPE_LEVEL_LOW>; + ti,imon-slot-no = <12>; + ti,vmon-slot-no = <14>; + }; + + speaker_right_woof2: codec@3d { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x3d>; + SDZ-supply = <&speaker_sdz>; + #sound-dai-cells = <0>; + sound-name-prefix = "Right Woofer 2"; + interrupts-extended = <&pinctrl_ap 11 IRQ_TYPE_LEVEL_LOW>; + ti,imon-slot-no = <20>; + ti,vmon-slot-no = <22>; + }; + + jack_codec: codec@4b { + compatible = "cirrus,cs42l84"; + reg = <0x4b>; + reset-gpios = <&pinctrl_nub 12 GPIO_ACTIVE_HIGH>; + #sound-dai-cells = <0>; + interrupts-extended = <&pinctrl_ap 149 IRQ_TYPE_LEVEL_LOW>; + sound-name-prefix = "Jack"; + }; +}; + &i2c4 { status = "okay"; }; @@ -78,3 +200,106 @@ &fpwm1 { status = "okay"; }; + +&aop_mbox { + status = "okay"; +}; + +&aop_dart { + status = "okay"; +}; + +&aop_admac { + status = "okay"; +}; + +&aop { + status = "okay"; +}; + +&aop_audio { + apple,chassis-name = "J415"; + apple,machine-kind = "MacBook Air"; +}; + +/ { + sound { + compatible = "apple,j415-macaudio", "apple,macaudio"; + model = "MacBook Air J415"; + + dai-link@0 { + link-name = "Speakers"; + + cpu { + sound-dai = <&mca 0>, <&mca 1>; + }; + codec { + sound-dai = <&speaker_left_woof1>, + <&speaker_left_tweet>, + <&speaker_left_woof2>, + <&speaker_right_woof1>, + <&speaker_right_tweet>, + <&speaker_right_woof2>; + }; + }; + + dai-link@1 { + link-name = "Headphone Jack"; + + cpu { + sound-dai = <&mca 2>; + }; + codec { + sound-dai = <&jack_codec>; + }; + }; + }; +}; + +&mtp { + status = "okay"; +}; +&mtp_mbox { + status = "okay"; +}; +&mtp_dart { + status = "okay"; +}; +&mtp_dockchannel { + status = "okay"; +}; +&mtp_hid { + apple,afe-reset-gpios = <&smc_gpio 8 GPIO_ACTIVE_LOW>; + apple,stm-reset-gpios = <&smc_gpio 24 GPIO_ACTIVE_LOW>; + + multi-touch { + firmware-name = "apple/tpmtfw-j415.bin"; + }; + + keyboard: keyboard { + hid-country-code = <0>; + apple,keyboard-layout-id = <0>; + }; + + stm { + }; + + actuator { + }; + + tp_accel { + }; +}; + +#include "isp-imx558-cfg0.dtsi" + +&isp { + apple,platform-id = <15>; + apple,temporal-filter = <1>; +}; + +&pmp_report_isp_sys { + status = "okay"; +}; + +#include "hwmon-laptop.dtsi" diff --git a/arch/arm64/boot/dts/apple/t8112-j473.dts b/arch/arm64/boot/dts/apple/t8112-j473.dts index 06fe257f08be49..48f268bce7d32c 100644 --- a/arch/arm64/boot/dts/apple/t8112-j473.dts +++ b/arch/arm64/boot/dts/apple/t8112-j473.dts @@ -15,12 +15,104 @@ / { compatible = "apple,j473", "apple,t8112", "apple,arm-platform"; model = "Apple Mac mini (M2, 2023)"; + chassis-type = "desktop"; aliases { + bluetooth0 = &bluetooth0; + /delete-property/ dcp; + dcpext = &dcpext; ethernet0 = ðernet0; + sio = &sio; + wifi0 = &wifi0; }; }; +&framebuffer0 { + power-domains = <&ps_dispext_cpu0>, <&ps_dptx_ext_phy>; +}; + +&dptxphy { + status = "okay"; +}; + +&dcp { + status = "disabled"; +}; + +&display { + iommus = <&dispext0_dart 0>; +}; +&dispext0_dart { + status = "okay"; +}; +&dcpext_dart { + status = "okay"; +}; +&dcpext_mbox { + status = "okay"; +}; +&dcpext { + status = "okay"; + apple,connector-type = "HDMI-A"; + + /* HDMI HPD gpio, used as interrupt*/ + hdmi-hpd-gpios = <&pinctrl_aop 49 GPIO_ACTIVE_HIGH>; + + hdmi-pwren-gpios = <&smc_gpio 21 GPIO_ACTIVE_HIGH>; + dp2hdmi-pwren-gpios = <&smc_gpio 22 GPIO_ACTIVE_HIGH>; + + phys = <&dptxphy>; + phy-names = "dp-phy"; + apple,dptx-phy = <5>; +}; + +/* remove once m1n1 enables sio nodes after setup */ +&sio { + status = "okay"; +}; + +&dpaudio1 { + status = "okay"; +}; + +/* + * Keep the power-domains used for the HDMI port on. + */ +&framebuffer0 { + power-domains = <&ps_dispext_cpu0>, <&ps_dptx_ext_phy>; +}; + +/* + * The M2 Mac mini uses dispext for the HDMI output so it's not necessary to + * keep disp0 power-domains always-on. + */ +&ps_disp0_sys { + /delete-property/ apple,always-on; +}; + +&ps_disp0_fe { + /delete-property/ apple,always-on; +}; + +/* + * Keep the power-domains used for the HDMI port on. + */ +&framebuffer0 { + power-domains = <&ps_dispext_cpu0>, <&ps_dptx_ext_phy>; +}; + +/* + * The M2 Mac mini uses dispext for the HDMI output so it's not necessary to + * keep disp0 power-domains always-on. + */ +&ps_disp0_sys { + /delete-property/ apple,always-on; +}; + +&ps_disp0_fe { + /delete-property/ apple,always-on; +}; + /* * Force the bus number assignments so that we can declare some of the * on-board devices and properties that are populated by the bootloader @@ -28,10 +120,28 @@ */ &port00 { bus-range = <1 1>; + pwren-gpios = <&smc_gpio 13 GPIO_ACTIVE_HIGH>; + wifi0: wifi@0,0 { + compatible = "pci14e4,4434"; + reg = <0x10000 0x0 0x0 0x0 0x0>; + /* To be filled by the loader */ + local-mac-address = [00 10 18 00 00 10]; + apple,antenna-sku = "XX"; + brcm,board-type = "apple,miyake"; + }; + + bluetooth0: bluetooth@0,1 { + compatible = "pci14e4,5f72"; + reg = <0x10100 0x0 0x0 0x0 0x0>; + /* To be filled by the loader */ + local-bd-address = [00 00 00 00 00 00]; + brcm,board-type = "apple,miyake"; + }; }; &port01 { bus-range = <2 2>; + pwren-gpios = <&smc_gpio 24 GPIO_ACTIVE_HIGH>; status = "okay"; }; @@ -52,3 +162,75 @@ &pcie2_dart { status = "okay"; }; + +/* + * Provide labels for the USB type C ports. + */ + +&typec0 { + label = "USB-C Back-left"; +}; + +&typec1 { + label = "USB-C Back-right"; +}; + +&i2c1 { + speaker_amp: codec@38 { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x38>; + shutdown-gpios = <&pinctrl_ap 88 GPIO_ACTIVE_HIGH>; + #sound-dai-cells = <0>; + interrupt-parent = <&pinctrl_ap>; + interrupts = <11 IRQ_TYPE_LEVEL_LOW>; + ti,imon-slot-no = <0>; + ti,vmon-slot-no = <2>; + }; + + jack_codec: codec@4b { + compatible = "cirrus,cs42l84"; + reg = <0x4b>; + reset-gpios = <&pinctrl_nub 12 GPIO_ACTIVE_HIGH>; + interrupt-parent = <&pinctrl_ap>; + interrupts = <149 IRQ_TYPE_LEVEL_LOW>; + #sound-dai-cells = <0>; + cirrus,ts-inv = <1>; + sound-name-prefix = "Jack"; + }; +}; + +/ { + sound { + compatible = "apple,j473-macaudio", "apple,macaudio"; + model = "Mac mini J473"; + + dai-link@0 { + link-name = "Speaker"; + + cpu { + sound-dai = <&mca 0>; + }; + codec { + sound-dai = <&speaker_amp>; + }; + }; + + dai-link@1 { + link-name = "Headphone Jack"; + + cpu { + sound-dai = <&mca 2>; + }; + codec { + sound-dai = <&jack_codec>; + }; + }; + + }; +}; + +&gpu { + apple,perf-base-pstate = <3>; +}; + +#include "hwmon-mini.dtsi" diff --git a/arch/arm64/boot/dts/apple/t8112-j493.dts b/arch/arm64/boot/dts/apple/t8112-j493.dts index fb8ad7d4c65a8f..61bbb7a3c6f486 100644 --- a/arch/arm64/boot/dts/apple/t8112-j493.dts +++ b/arch/arm64/boot/dts/apple/t8112-j493.dts @@ -16,6 +16,7 @@ / { compatible = "apple,j493", "apple,t8112", "apple,arm-platform"; model = "Apple MacBook Pro (13-inch, M2, 2022)"; + chassis-type = "laptop"; /* * All of those are used by the bootloader to pass calibration @@ -23,6 +24,7 @@ */ aliases { bluetooth0 = &bluetooth0; + keyboard = &keyboard; touchbar0 = &touchbar0; wifi0 = &wifi0; }; @@ -50,6 +52,20 @@ apple,always-on; }; +&dcp { + panel: panel { + compatible = "apple,panel-j493", "apple,panel"; + width-mm = <286>; + height-mm = <179>; + apple,max-brightness = <525>; + }; +}; + +&framebuffer0 { + panel = <&panel>; + post-init-providers = <&panel>; +}; + &display_dfr { status = "okay"; }; @@ -90,6 +106,7 @@ */ &port00 { bus-range = <1 1>; + pwren-gpios = <&smc_gpio 13 GPIO_ACTIVE_HIGH>; wifi0: wifi@0,0 { compatible = "pci14e4,4425"; reg = <0x10000 0x0 0x0 0x0 0x0>; @@ -108,6 +125,88 @@ }; }; +/* + * Provide labels for the USB type C ports. + */ + +&typec0 { + label = "USB-C Left-back"; +}; + +&typec1 { + label = "USB-C Left-front"; +}; + +/* Virtual regulator representing the shared shutdown GPIO */ +/ { + speaker_sdz: fixed-regulator-sn012776-sdz { + compatible = "regulator-fixed"; + regulator-name = "sn012776-sdz"; + startup-delay-us = <5000>; + gpios = <&pinctrl_ap 88 GPIO_ACTIVE_HIGH>; + enable-active-high; + }; +}; + +&i2c1 { + speaker_left_rear: codec@38 { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x38>; + SDZ-supply = <&speaker_sdz>; + #sound-dai-cells = <0>; + sound-name-prefix = "Left Rear"; + interrupts-extended = <&pinctrl_ap 11 IRQ_TYPE_LEVEL_LOW>; + ti,imon-slot-no = <8>; + ti,vmon-slot-no = <10>; + }; + + speaker_left_front: codec@39 { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x39>; + SDZ-supply = <&speaker_sdz>; + #sound-dai-cells = <0>; + sound-name-prefix = "Left Front"; + interrupts-extended = <&pinctrl_ap 11 IRQ_TYPE_LEVEL_LOW>; + ti,imon-slot-no = <0>; + ti,vmon-slot-no = <2>; + ti,sdout-force-zero-mask = <0xf0f0>; + }; +}; + +&i2c3 { + speaker_right_rear: codec@3b { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x3b>; + SDZ-supply = <&speaker_sdz>; + #sound-dai-cells = <0>; + sound-name-prefix = "Right Rear"; + interrupts-extended = <&pinctrl_ap 11 IRQ_TYPE_LEVEL_LOW>; + ti,imon-slot-no = <12>; + ti,vmon-slot-no = <14>; + }; + + speaker_right_front: codec@3c { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x3c>; + SDZ-supply = <&speaker_sdz>; + #sound-dai-cells = <0>; + sound-name-prefix = "Right Front"; + interrupts-extended = <&pinctrl_ap 11 IRQ_TYPE_LEVEL_LOW>; + ti,imon-slot-no = <4>; + ti,vmon-slot-no = <6>; + ti,sdout-force-zero-mask = <0x0f0f>; + }; + + jack_codec: codec@4b { + compatible = "cirrus,cs42l84"; + reg = <0x4b>; + reset-gpios = <&pinctrl_nub 12 GPIO_ACTIVE_HIGH>; + #sound-dai-cells = <0>; + interrupts-extended = <&pinctrl_ap 149 IRQ_TYPE_LEVEL_LOW>; + sound-name-prefix = "Jack"; + }; +}; + &i2c4 { status = "okay"; }; @@ -133,3 +232,102 @@ touchscreen-inverted-y; }; }; + +&aop_mbox { + status = "okay"; +}; + +&aop_dart { + status = "okay"; +}; + +&aop_admac { + status = "okay"; +}; + +&aop { + status = "okay"; +}; + +&aop_audio { + apple,chassis-name = "J493"; + apple,machine-kind = "MacBook Pro"; +}; + +/ { + sound { + compatible = "apple,j493-macaudio", "apple,macaudio"; + model = "MacBook Pro J493"; + + dai-link@0 { + link-name = "Speakers"; + + cpu { + sound-dai = <&mca 0>, <&mca 1>; + }; + codec { + sound-dai = <&speaker_left_front>, <&speaker_left_rear>, + <&speaker_right_front>, <&speaker_right_rear>; + }; + }; + + dai-link@1 { + link-name = "Headphone Jack"; + + cpu { + sound-dai = <&mca 2>; + }; + codec { + sound-dai = <&jack_codec>; + }; + }; + }; +}; + +&mtp { + status = "okay"; +}; +&mtp_mbox { + status = "okay"; +}; +&mtp_dart { + status = "okay"; +}; +&mtp_dockchannel { + status = "okay"; +}; +&mtp_hid { + apple,afe-reset-gpios = <&smc_gpio 8 GPIO_ACTIVE_LOW>; + apple,stm-reset-gpios = <&smc_gpio 24 GPIO_ACTIVE_LOW>; + + multi-touch { + firmware-name = "apple/tpmtfw-j493.bin"; + }; + + keyboard: keyboard { + hid-country-code = <0>; + apple,keyboard-layout-id = <0>; + }; + + stm { + }; + + actuator { + }; + + tp_accel { + }; +}; + +#include "isp-imx248.dtsi" + +&isp { + apple,platform-id = <6>; +}; + +&pmp_report_isp_sys { + status = "okay"; +}; + +#include "hwmon-fan.dtsi" +#include "hwmon-laptop.dtsi" diff --git a/arch/arm64/boot/dts/apple/t8112-jxxx.dtsi b/arch/arm64/boot/dts/apple/t8112-jxxx.dtsi index 6da35496a4c88d..fb957f785d82c5 100644 --- a/arch/arm64/boot/dts/apple/t8112-jxxx.dtsi +++ b/arch/arm64/boot/dts/apple/t8112-jxxx.dtsi @@ -11,6 +11,11 @@ / { aliases { + atcphy0 = &atcphy0; + atcphy1 = &atcphy1; + dcp = &dcp; + disp0 = &display; + disp0_piodma = &disp0_piodma; serial0 = &serial0; serial2 = &serial2; }; @@ -25,6 +30,7 @@ framebuffer0: framebuffer@0 { compatible = "apple,simple-framebuffer", "simple-framebuffer"; reg = <0 0 0 0>; /* To be filled by loader */ + power-domains = <&ps_disp0_cpu0>; /* Format properties will be added by loader */ status = "disabled"; }; @@ -53,6 +59,29 @@ interrupt-parent = <&pinctrl_ap>; interrupts = <8 IRQ_TYPE_LEVEL_LOW>; interrupt-names = "irq"; + + typec0: connector { + compatible = "usb-c-connector"; + power-role = "dual"; + data-role = "dual"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + typec0_connector_hs: endpoint { + remote-endpoint = <&dwc3_0_hs>; + }; + }; + port@1 { + reg = <1>; + typec0_connector_ss: endpoint { + remote-endpoint = <&atcphy0_typec_lanes>; + }; + }; + }; + }; }; hpm1: usb-pd@3f { @@ -61,6 +90,115 @@ interrupt-parent = <&pinctrl_ap>; interrupts = <8 IRQ_TYPE_LEVEL_LOW>; interrupt-names = "irq"; + + typec1: connector { + compatible = "usb-c-connector"; + power-role = "dual"; + data-role = "dual"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + typec1_connector_hs: endpoint { + remote-endpoint = <&dwc3_1_hs>; + }; + }; + port@1 { + reg = <1>; + typec1_connector_ss: endpoint { + remote-endpoint = <&atcphy1_typec_lanes>; + }; + }; + }; + }; + }; +}; + +/* USB controllers */ +&dwc3_0 { + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + dwc3_0_hs: endpoint { + remote-endpoint = <&typec0_connector_hs>; + }; + }; + + port@1 { + reg = <1>; + dwc3_0_ss: endpoint { + remote-endpoint = <&atcphy0_usb3>; + }; + }; + }; +}; + +&dwc3_1 { + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + dwc3_1_hs: endpoint { + remote-endpoint = <&typec1_connector_hs>; + }; + }; + + port@1 { + reg = <1>; + dwc3_1_ss: endpoint { + remote-endpoint = <&atcphy1_usb3>; + }; + }; + }; +}; + +/* Type-C PHYs */ +&atcphy0 { + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + atcphy0_typec_lanes: endpoint { + remote-endpoint = <&typec0_connector_ss>; + }; + }; + + port@1 { + reg = <1>; + atcphy0_usb3: endpoint { + remote-endpoint = <&dwc3_0_ss>; + }; + }; + }; +}; + +&atcphy1 { + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + atcphy1_typec_lanes: endpoint { + remote-endpoint = <&typec1_connector_ss>; + }; + }; + + port@1 { + reg = <1>; + atcphy1_usb3: endpoint { + remote-endpoint = <&dwc3_1_ss>; + }; + }; }; }; @@ -80,4 +218,6 @@ clock-frequency = <900000000>; }; +#include "hwmon-common.dtsi" + #include "spi1-nvram.dtsi" diff --git a/arch/arm64/boot/dts/apple/t8112-pmgr.dtsi b/arch/arm64/boot/dts/apple/t8112-pmgr.dtsi index 7c050c6f2707a1..ab8ec9bd4e4401 100644 --- a/arch/arm64/boot/dts/apple/t8112-pmgr.dtsi +++ b/arch/arm64/boot/dts/apple/t8112-pmgr.dtsi @@ -176,7 +176,7 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = "sio_cpu"; - power-domains = <&ps_sio>; + power-domains = <&ps_sio &ps_uart_p &ps_spi_p &ps_dpa0>; }; ps_fpwm0: power-controller@1c8 { @@ -465,6 +465,7 @@ #reset-cells = <0>; label = "mca0"; power-domains = <&ps_sio_adma>, <&ps_audio_p>; + apple,externally-clocked; }; ps_mca1: power-controller@2c8 { @@ -474,6 +475,7 @@ #reset-cells = <0>; label = "mca1"; power-domains = <&ps_sio_adma>, <&ps_audio_p>; + apple,externally-clocked; }; ps_mca2: power-controller@2d0 { @@ -483,6 +485,7 @@ #reset-cells = <0>; label = "mca2"; power-domains = <&ps_sio_adma>, <&ps_audio_p>; + apple,externally-clocked; }; ps_mca3: power-controller@2d8 { @@ -492,6 +495,7 @@ #reset-cells = <0>; label = "mca3"; power-domains = <&ps_sio_adma>, <&ps_audio_p>; + apple,externally-clocked; }; ps_mca4: power-controller@2e0 { @@ -501,6 +505,7 @@ #reset-cells = <0>; label = "mca4"; power-domains = <&ps_sio_adma>, <&ps_audio_p>; + apple,externally-clocked; }; ps_mca5: power-controller@2e8 { @@ -510,6 +515,7 @@ #reset-cells = <0>; label = "mca5"; power-domains = <&ps_sio_adma>, <&ps_audio_p>; + apple,externally-clocked; }; ps_mcc: power-controller@2f0 { @@ -663,7 +669,6 @@ #reset-cells = <0>; label = "disp0_sys"; power-domains = <&ps_rmx1>; - apple,always-on; /* TODO: figure out if we can enable PM here */ }; ps_disp0_fe: power-controller@378 { @@ -672,8 +677,7 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = "disp0_fe"; - power-domains = <&ps_disp0_sys>; - apple,always-on; /* TODO: figure out if we can enable PM here */ + power-domains = <&ps_disp0_sys>, <&ps_pmp>; }; ps_dispext_sys: power-controller@380 { @@ -691,7 +695,7 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = "dispext_fe"; - power-domains = <&ps_dispext_sys>; + power-domains = <&ps_dispext_sys>, <&ps_pmp>; }; ps_dispext_cpu0: power-controller@3c8 { @@ -773,7 +777,6 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = "pmp"; - apple,always-on; }; ps_pms_sram: power-controller@418 { @@ -818,6 +821,7 @@ #reset-cells = <0>; label = "isp_sys"; power-domains = <&ps_rmx1>; + status = "disabled"; }; ps_venc_sys: power-controller@440 { @@ -964,6 +968,123 @@ apple,always-on; }; + /* There is a dependency tree involved with these PDs, + * but we do not express it here since the ISP driver + * is supposed to sequence them in the right order anyway + * (and we do not know the exact tree structure). + * + * This also works around spurious parent PD activation + * on machines with ISP disabled (desktops). + */ + ps_isp_set0: power-controller@4000 { + compatible = "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4000 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_set0"; + apple,force-disable; + }; + + ps_isp_set1: power-controller@4008 { + compatible = "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4008 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_set1"; + apple,force-disable; + apple,force-reset; + }; + + ps_isp_set2: power-controller@4010 { + compatible = "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4010 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_set2"; + apple,force-disable; + apple,force-reset; + }; + + ps_isp_fe: power-controller@4018 { + compatible = "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4018 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_fe"; + }; + + ps_isp_set4: power-controller@4020 { + compatible = "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4020 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_set4"; + }; + + ps_isp_set5: power-controller@4028 { + compatible = "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4028 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_set5"; + }; + + ps_isp_set6: power-controller@4030 { + compatible = "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4030 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_set6"; + }; + + ps_isp_set7: power-controller@4038 { + compatible = "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4038 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_set7"; + }; + + ps_isp_set8: power-controller@4040 { + compatible = "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4040 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_set8"; + }; + + ps_isp_set9: power-controller@4048 { + compatible = "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4048 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_set9"; + }; + + ps_isp_set12: power-controller@4050 { + compatible = "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4050 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_set10"; + }; + + ps_isp_set10: power-controller@4058 { + compatible = "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4058 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_set11"; + }; + + ps_isp_set11: power-controller@4060 { + compatible = "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4060 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_set12"; + }; + ps_venc_dma: power-controller@8000 { compatible = "apple,t8112-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x8000 4>; @@ -1064,6 +1185,7 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = "msg"; + apple,always-on; /* Core AON device? */ }; ps_nub_gpio: power-controller@80 { diff --git a/arch/arm64/boot/dts/apple/t8112.dtsi b/arch/arm64/boot/dts/apple/t8112.dtsi index 3f79878b25af1f..b667944f6dc5c3 100644 --- a/arch/arm64/boot/dts/apple/t8112.dtsi +++ b/arch/arm64/boot/dts/apple/t8112.dtsi @@ -11,6 +11,7 @@ #include #include #include +#include #include / { @@ -21,6 +22,9 @@ aliases { gpu = &gpu; + #ifdef APPLE_USE_PMP + pmp = &pmp; + #endif }; cpus { @@ -194,36 +198,43 @@ opp-hz = /bits/ 64 <600000000>; opp-level = <1>; clock-latency-ns = <7500>; + opp-microwatt = <26000>; }; opp02 { opp-hz = /bits/ 64 <912000000>; opp-level = <2>; clock-latency-ns = <20000>; + opp-microwatt = <56000>; }; opp03 { opp-hz = /bits/ 64 <1284000000>; opp-level = <3>; clock-latency-ns = <22000>; + opp-microwatt = <88000>; }; opp04 { opp-hz = /bits/ 64 <1752000000>; opp-level = <4>; clock-latency-ns = <30000>; + opp-microwatt = <155000>; }; opp05 { opp-hz = /bits/ 64 <2004000000>; opp-level = <5>; clock-latency-ns = <35000>; + opp-microwatt = <231000>; }; opp06 { opp-hz = /bits/ 64 <2256000000>; opp-level = <6>; clock-latency-ns = <39000>; + opp-microwatt = <254000>; }; opp07 { opp-hz = /bits/ 64 <2424000000>; opp-level = <7>; clock-latency-ns = <53000>; + opp-microwatt = <351000>; }; }; @@ -235,93 +246,161 @@ opp-hz = /bits/ 64 <660000000>; opp-level = <1>; clock-latency-ns = <9000>; + opp-microwatt = <133000>; }; opp02 { opp-hz = /bits/ 64 <924000000>; opp-level = <2>; clock-latency-ns = <19000>; + opp-microwatt = <212000>; }; opp03 { opp-hz = /bits/ 64 <1188000000>; opp-level = <3>; clock-latency-ns = <22000>; + opp-microwatt = <261000>; }; opp04 { opp-hz = /bits/ 64 <1452000000>; opp-level = <4>; clock-latency-ns = <24000>; + opp-microwatt = <345000>; }; opp05 { opp-hz = /bits/ 64 <1704000000>; opp-level = <5>; clock-latency-ns = <26000>; + opp-microwatt = <441000>; }; opp06 { opp-hz = /bits/ 64 <1968000000>; opp-level = <6>; clock-latency-ns = <28000>; + opp-microwatt = <619000>; }; opp07 { opp-hz = /bits/ 64 <2208000000>; opp-level = <7>; clock-latency-ns = <30000>; + opp-microwatt = <740000>; }; opp08 { opp-hz = /bits/ 64 <2400000000>; opp-level = <8>; clock-latency-ns = <33000>; + opp-microwatt = <855000>; }; opp09 { opp-hz = /bits/ 64 <2568000000>; opp-level = <9>; clock-latency-ns = <34000>; + opp-microwatt = <1006000>; }; opp10 { opp-hz = /bits/ 64 <2724000000>; opp-level = <10>; clock-latency-ns = <36000>; + opp-microwatt = <1217000>; }; opp11 { opp-hz = /bits/ 64 <2868000000>; opp-level = <11>; clock-latency-ns = <41000>; + opp-microwatt = <1534000>; }; opp12 { opp-hz = /bits/ 64 <2988000000>; opp-level = <12>; clock-latency-ns = <42000>; + opp-microwatt = <1714000>; }; opp13 { opp-hz = /bits/ 64 <3096000000>; opp-level = <13>; clock-latency-ns = <44000>; + opp-microwatt = <1877000>; }; opp14 { opp-hz = /bits/ 64 <3204000000>; opp-level = <14>; clock-latency-ns = <46000>; + opp-microwatt = <2159000>; }; - /* Not available until CPU deep sleep is implemented */ -#if 0 opp15 { opp-hz = /bits/ 64 <3324000000>; opp-level = <15>; clock-latency-ns = <62000>; + opp-microwatt = <2393000>; turbo-mode; }; opp16 { opp-hz = /bits/ 64 <3408000000>; opp-level = <16>; clock-latency-ns = <62000>; + opp-microwatt = <2497000>; turbo-mode; }; opp17 { opp-hz = /bits/ 64 <3504000000>; opp-level = <17>; clock-latency-ns = <62000>; + opp-microwatt = <2648000>; turbo-mode; }; -#endif + }; + + gpu_opp: opp-table-gpu { + compatible = "operating-points-v2"; + + /* + * NOTE: The voltage and power values are device-specific and + * must be filled in by the bootloader. + */ + opp00 { + opp-hz = /bits/ 64 <0>; + opp-microvolt = <400000>; + opp-microwatt = <0>; + }; + opp01 { + opp-hz = /bits/ 64 <444000000>; + opp-microvolt = <603000>; + opp-microwatt = <4295000>; + }; + opp02 { + opp-hz = /bits/ 64 <612000000>; + opp-microvolt = <675000>; + opp-microwatt = <6251000>; + }; + opp03 { + opp-hz = /bits/ 64 <808000000>; + opp-microvolt = <710000>; + opp-microwatt = <8625000>; + }; + opp04 { + opp-hz = /bits/ 64 <968000000>; + opp-microvolt = <775000>; + opp-microwatt = <11948000>; + }; + opp05 { + opp-hz = /bits/ 64 <1110000000>; + opp-microvolt = <820000>; + opp-microwatt = <15071000>; + }; + opp06 { + opp-hz = /bits/ 64 <1236000000>; + opp-microvolt = <875000>; + opp-microwatt = <18891000>; + }; + opp07 { + opp-hz = /bits/ 64 <1338000000>; + opp-microvolt = <915000>; + opp-microwatt = <21960000>; + }; + opp08 { + opp-hz = /bits/ 64 <1398000000>; + opp-microvolt = <950000>; + opp-microwatt = <22800000>; + }; }; timer { @@ -370,6 +449,22 @@ clock-output-names = "nco_ref"; }; + /* Pixel clock? frequency in Hz (compare: 4K@60 VGA clock 533.250 MHz) */ + clk_disp0: clock-disp0 { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <533333328>; + clock-output-names = "clk_disp0"; + }; + + /* Pixel clock? frequency in Hz (compare: 4K@60 VGA clock 533.250 MHz) */ + clk_dispext0: clock-dispext0 { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <0>; + clock-output-names = "clk_dispext0"; + }; + reserved-memory { #address-cells = <2>; #size-cells = <2>; @@ -388,15 +483,15 @@ }; uat_handoff: uat-handoff { - status = "disabled"; + reg = <0 0 0 0>; }; uat_pagetables: uat-pagetables { - status = "disabled"; + reg = <0 0 0 0>; }; uat_ttbs: uat-ttbs { - status = "disabled"; + reg = <0 0 0 0>; }; }; @@ -407,20 +502,57 @@ ranges; nonposted-mmio; + /* Required to get >32-bit DMA via DARTs */ + dma-ranges = <0 0 0 0 0xffffffff 0xffffc000>; gpu: gpu@206400000 { - compatible = "apple,agx-g14g"; + compatible = "apple,agx-t8112", "apple,agx-g14g"; reg = <0x2 0x6400000 0 0x40000>, <0x2 0x4000000 0 0x1000000>; reg-names = "asc", "sgx"; mboxes = <&agx_mbox>; + #ifdef APPLE_USE_PMP + power-domains = <&pmp_report_gfx>; + #else power-domains = <&ps_gfx>; + #endif memory-region = <&uat_ttbs>, <&uat_pagetables>, <&uat_handoff>, <&gpu_hw_cal_a>, <&gpu_hw_cal_b>, <&gpu_globals>; memory-region-names = "ttbs", "pagetables", "handoff", "hw-cal-a", "hw-cal-b", "globals"; apple,firmware-abi = <0 0 0>; + apple,firmware-version = <12 4 0>; + apple,firmware-compat = <12 4 0>; + + operating-points-v2 = <&gpu_opp>; + apple,perf-base-pstate = <1>; + apple,min-sram-microvolt = <780000>; + apple,avg-power-filter-tc-ms = <300>; + apple,avg-power-ki-only = <9.375>; + apple,avg-power-kp = <3.22>; + apple,avg-power-min-duty-cycle = <40>; + apple,avg-power-target-filter-tc = <1>; + apple,fast-die0-integral-gain = <200.0>; + apple,fast-die0-proportional-gain = <5.0>; + apple,perf-boost-ce-step = <50>; + apple,perf-boost-min-util = <90>; + apple,perf-filter-drop-threshold = <0>; + apple,perf-filter-time-constant = <5>; + apple,perf-filter-time-constant2 = <200>; + apple,perf-integral-gain = <5.94>; + apple,perf-integral-gain2 = <5.94>; + apple,perf-integral-min-clamp = <0>; + apple,perf-proportional-gain = <14.85>; + apple,perf-proportional-gain2 = <14.85>; + apple,perf-tgt-utilization = <85>; + apple,power-sample-period = <8>; + apple,ppm-filter-time-constant-ms = <34>; + apple,ppm-ki = <205.0>; + apple,ppm-kp = <0.75>; + apple,pwr-min-duty-cycle = <40>; + apple,core-leak-coef = <1920.0>; + apple,sram-leak-coef = <74.0>; }; agx_mbox: mbox@206408000 { @@ -453,7 +585,11 @@ reg = <0x2 0x28200000 0x0 0xc000>, <0x2 0x28400000 0x0 0x4000>; reg-names = "be", "fe"; + #ifdef APPLE_USE_PMP + power-domains = <&pmp_report_dispdfr>, <&ps_dispdfr_be>; + #else power-domains = <&ps_dispdfr_fe>, <&ps_dispdfr_be>; + #endif interrupt-parent = <&aic>; interrupts = , ; @@ -507,6 +643,148 @@ }; }; + isp_dart0: iommu@22c4a8000 { + compatible = "apple,t8110-dart"; + reg = <0x2 0x2c4a8000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_isp_sys>; + status = "disabled"; + }; + + isp_dart1: iommu@22c4b4000 { + compatible = "apple,t8110-dart"; + reg = <0x2 0x2c4b4000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_isp_sys>; + status = "disabled"; + }; + + isp_dart2: iommu@22c4bc000 { + compatible = "apple,t8110-dart"; + reg = <0x2 0x2c4bc000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_isp_sys>; + status = "disabled"; + }; + + isp: isp@22a000000 { + compatible = "apple,t8112-isp", "apple,isp"; + iommus = <&isp_dart0 0>, <&isp_dart1 0>, <&isp_dart2 0>; + reg-names = "coproc", "mbox", "gpio", "mbox2"; + reg = <0x2 0x2a000000 0x0 0x2000000>, + <0x2 0x2c4c4000 0x0 0x100>, + <0x2 0x2c4c41b0 0x0 0x100>, + <0x2 0x2c4c4430 0x0 0x100>; + interrupt-parent = <&aic>; + interrupts = ; + #ifdef APPLE_USE_PMP + power-domains = <&pmp_report_isp_sys>, <&ps_isp_set0>, + #else + power-domains = <&ps_isp_sys>, <&ps_isp_set0>, + #endif + <&ps_isp_set1>, <&ps_isp_set2>, <&ps_isp_fe>, + <&ps_isp_set4>, <&ps_isp_set5>, <&ps_isp_set6>, + <&ps_isp_set7>, <&ps_isp_set8>, <&ps_isp_set9>, + <&ps_isp_set10>, <&ps_isp_set11>, + <&ps_isp_set12>; + + apple,dart-vm-size = <0x0 0xa0000000>; + status = "disabled"; + }; + + disp0_dart: iommu@231304000 { + compatible = "apple,t8112-dart", "apple,t8110-dart"; + reg = <0x2 0x31304000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_disp0_cpu0>; + apple,dma-range = <0x0 0x0 0xf 0xffff0000>; + status = "disabled"; + }; + + dcp_dart: iommu@23130c000 { + compatible = "apple,t8112-dart", "apple,t8110-dart"; + reg = <0x2 0x3130c000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_disp0_cpu0>; + apple,dma-range = <0x8 0x00000000 0x7 0xffff0000>; + }; + + dcp_mbox: mbox@231c08000 { + compatible = "apple,t8112-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x2 0x31c08000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + power-domains = <&ps_disp0_cpu0>; + resets = <&ps_disp0_cpu0>; + }; + + dcp: dcp@231c00000 { + compatible = "apple,t8112-dcp", "apple,dcp"; + mboxes = <&dcp_mbox>; + mbox-names = "mbox"; + iommus = <&dcp_dart 5>; + + /* the ADT has 2 additional regs which seems to be unused */ + reg-names = "coproc", "disp-0", "disp-1", "disp-2", "disp-3"; + reg = <0x2 0x31c00000 0x0 0x4000>, + <0x2 0x30000000 0x0 0x61c000>, + <0x2 0x31320000 0x0 0x4000>, + <0x2 0x31344000 0x0 0x4000>, + <0x2 0x31800000 0x0 0x800000>; + apple,bw-scratch = <&pmgr_dcp 0 4 0x5d8>; + #ifdef APPLE_USE_PMP + power-domains = <&pmp_report_disp0>; + #else + power-domains = <&ps_disp0_cpu0>; + #endif + resets = <&ps_disp0_cpu0>; + clocks = <&clk_disp0>; + phandle = <&dcp>; + // required bus properties for 'piodma' subdevice + #address-cells = <2>; + #size-cells = <2>; + + disp0_piodma: piodma { + iommus = <&disp0_dart 4>; + phandle = <&disp0_piodma>; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + dcp_audio: endpoint { + remote-endpoint = <&dpaudio0_dcp>; + }; + }; + }; + }; + + display: display-subsystem { + compatible = "apple,display-subsystem"; + /* disp_dart0 must be 1st since it is locked */ + iommus = <&disp0_dart 0>; + /* generate phandle explicitly for use in loader */ + phandle = <&display>; + }; + sio_dart: iommu@235004000 { compatible = "apple,t8110-dart"; reg = <0x2 0x35004000 0x0 0x4000>; @@ -651,6 +929,32 @@ status = "disabled"; }; + sio_mbox: mbox@236408000 { + compatible = "apple,t8112-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x2 0x36408000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + power-domains = <&ps_sio_cpu>; + }; + + sio: sio@236400000 { + compatible = "apple,t8112-sio", "apple,sio"; + reg = <0x2 0x36400000 0x0 0x8000>; + dma-channels = <128>; + #dma-cells = <1>; + mboxes = <&sio_mbox>; + iommus = <&sio_dart 0>; + power-domains = <&ps_sio_cpu>; + resets = <&ps_sio>; /* TODO: verify reset does something */ + status = "disabled"; + }; + admac: dma-controller@238200000 { compatible = "apple,t8112-admac", "apple,admac"; reg = <0x2 0x38200000 0x0 0x34000>; @@ -665,6 +969,48 @@ resets = <&ps_audio_p>; }; + dpaudio0: audio-controller@238330000 { + compatible = "apple,t8112-dpaudio", "apple,dpaudio"; + reg = <0x2 0x38330000 0x0 0x4000>; + dmas = <&sio 0x64>; + dma-names = "tx"; + power-domains = <&ps_dpa0>; + reset-domains = <&ps_dpa0>; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + dpaudio0_dcp: endpoint { + remote-endpoint = <&dcp_audio>; + }; + }; + }; + }; + + dpaudio1: audio-controller@238334000 { + compatible = "apple,t8112-dpaudio", "apple,dpaudio"; + reg = <0x2 0x38334000 0x0 0x4000>; + dmas = <&sio 0x66>; + dma-names = "tx"; + power-domains = <&ps_dpa1>; + reset-domains = <&ps_dpa1>; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + dpaudio1_dcp: endpoint { + remote-endpoint = <&dcpext_audio>; + }; + }; + }; + }; + mca: i2s@238400000 { compatible = "apple,t8112-mca", "apple,mca"; reg = <0x2 0x38400000 0x0 0x18000>, @@ -728,6 +1074,148 @@ }; }; + pmp_dart: iommu@23b300000 { + compatible = "apple,t8110-dart"; + reg = <0x2 0x3b300000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_pmp>; + }; + + pmp_report: pmp_report@23b3c0000 { + compatible = "apple,t8112-pmp-v2-report"; + reg = <0x2 0x3b3c0000 0x0 0x20000>; + power-domains = <&ps_pms_sram>; + #address-cells = <1>; + #size-cells = <0>; + + pmp_report_gfx: report@4 { + compatible = "apple,t8112-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0x4>; + label = "pmp-gfx"; + #power-domain-cells = <0>; + power-domains = <&ps_gfx>; + }; + + pmp_report_ane_sys: report@5 { + compatible = "apple,t8112-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0x5>; + label = "pmp-ane-sys"; + #power-domain-cells = <0>; + power-domains = <&ps_ane_sys>; + status = "disabled"; + }; + + pmp_report_isp_sys: report@6 { + compatible = "apple,t8112-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0x6>; + label = "pmp-isp-sys"; + #power-domain-cells = <0>; + power-domains = <&ps_isp_sys>; + status = "disabled"; + }; + + pmp_report_disp0: report@7 { + compatible = "apple,t8112-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0x7>; + label = "pmp-disp0"; + #power-domain-cells = <0>; + power-domains = <&ps_disp0_cpu0>; + apple,always-on; + }; + + pmp_report_dispext: report@8 { + compatible = "apple,t8112-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0x8>; + label = "pmp-dispext"; + #power-domain-cells = <0>; + power-domains = <&ps_dispext_cpu0>; + }; + + pmp_report_venc_sys: report@9 { + compatible = "apple,t8112-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0x9>; + label = "pmp-venc-sys"; + #power-domain-cells = <0>; + power-domains = <&ps_venc_sys>; + status = "disabled"; + }; + + pmp_report_avd_sys: report@a { + compatible = "apple,t8112-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0xa>; + label = "pmp-avd-sys"; + #power-domain-cells = <0>; + power-domains = <&ps_avd_sys>; + status = "disabled"; + }; + + pmp_report_msr: report@b { + compatible = "apple,t8112-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0xb>; + label = "pmp-msr"; + #power-domain-cells = <0>; + power-domains = <&ps_msr>; + status = "disabled"; + }; + + pmp_report_jpg: report@c { + compatible = "apple,t8112-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0xc>; + label = "pmp-jpg"; + #power-domain-cells = <0>; + power-domains = <&ps_jpg>; + status = "disabled"; + }; + + pmp_report_scodec: report@d { + compatible = "apple,t8112-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0xd>; + label = "pmp-scodec"; + #power-domain-cells = <0>; + power-domains = <&ps_scodec>; + status = "disabled"; + }; + + pmp_report_dispdfr: report@11 { + compatible = "apple,t8112-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0x11>; + label = "pmp-dispdfr"; + #power-domain-cells = <0>; + power-domains = <&ps_dispdfr_fe>; + }; + }; + + pmgr_dcp: power-management@23b3d0000 { + reg = <0x2 0x3b3d0000 0x0 0x4000>; + reg-names = "dcp-bw-scratch"; + #apple,bw-scratch-cells = <3>; + }; + + pmp: pmp@23b500000 { + compatible = "apple,t8112-pmp-v2", "apple,t6000-pmp-v2"; + reg = <0x2 0x3b500000 0x0 0x80000>, + <0x2 0x3bc00000 0x0 0x4000>; + reg-names = "pmp", "asc"; + mboxes = <&pmp_mbox>; + mbox-names = "mbox"; + iommus = <&pmp_dart 0>; + power-domains = <&ps_pmp>; + status = "disabled"; + }; + pmgr: power-management@23b700000 { compatible = "apple,t8112-pmgr", "apple,pmgr", "syscon", "simple-mfd"; #address-cells = <1>; @@ -736,6 +1224,20 @@ /* child nodes are added in t8103-pmgr.dtsi */ }; + pmp_mbox: mbox@23bc08000 { + compatible = "apple,t8112-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x2 0x3bc08000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + power-domains = <&ps_pmp>, <&ps_pms_sram>; + }; + pinctrl_ap: pinctrl@23c100000 { compatible = "apple,t8112-pinctrl", "apple,pinctrl"; reg = <0x2 0x3c100000 0x0 0x100000>; @@ -804,6 +1306,17 @@ }; }; + dptxphy: phy@23c500000 { + compatible = "apple,t8112-dptx-phy", "apple,dptx-phy"; + reg = <0x2 0x3c500000 0x0 0x4000>, + <0x2 0x3c540000 0x0 0xc000>; + reg-names = "core", "dptx"; + power-domains = <&ps_dptx_ext_phy>; + #phy-cells = <0>; + #reset-cells = <0>; + status = "disabled"; /* only used on j473 */ + }; + pinctrl_nub: pinctrl@23d1f0000 { compatible = "apple,t8112-pinctrl", "apple,pinctrl"; reg = <0x2 0x3d1f0000 0x0 0x4000>; @@ -912,12 +1425,22 @@ #gpio-cells = <2>; }; + smc_hwmon: hwmon { + compatible = "apple,smc-hwmon"; + }; + smc_reboot: reboot { compatible = "apple,smc-reboot"; nvmem-cells = <&shutdown_flag>, <&boot_stage>, - <&boot_error_count>, <&panic_count>; + <&boot_error_count>, <&panic_count>, <&pm_setting>; nvmem-cell-names = "shutdown_flag", "boot_stage", - "boot_error_count", "panic_count"; + "boot_error_count", "panic_count", "pm_setting"; + }; + + rtc { + compatible = "apple,smc-rtc"; + nvmem-cells = <&rtc_offset>; + nvmem-cell-names = "rtc_offset"; }; }; @@ -976,6 +1499,254 @@ ; }; + aop_mbox: mbox@24a408000 { + compatible = "apple,t8112-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x2 0x4a408000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + status = "disabled"; + }; + + aop_dart: iommu@24a808000 { + compatible = "apple,t8112-dart", "apple,t8110-dart"; + reg = <0x2 0x4a808000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + status = "disabled"; + }; + + aop_admac: dma-controller@24a980000 { + compatible = "apple,t8112-admac", "apple,admac"; + reg = <0x2 0x4a980000 0x0 0x34000>; + #dma-cells = <1>; + dma-channels = <16>; + interrupts-extended = <0>, + <0>, + <&aic AIC_IRQ 359 IRQ_TYPE_LEVEL_HIGH>, + <0>; + iommus = <&aop_dart 10>; + status = "disabled"; + }; + + aop: aop@24ac00000 { + compatible = "apple,t8112-aop"; + reg = <0x2 0x4ac00000 0x0 0x1e0000>, + <0x2 0x4a400000 0x0 0x6c000>; + mboxes = <&aop_mbox>; + mbox-names = "mbox"; + iommus = <&aop_dart 0>; + + status = "disabled"; + + aop_audio: audio { + compatible = "apple,t8112-aop-audio", "apple,aop-audio"; + dmas = <&aop_admac 1>; + dma-names = "dma"; + }; + + aop_als: als { + compatible = "apple,t8112-aop-als", "apple,aop-als"; + // intentionally empty + }; + + las { + compatible = "apple,t8112-aop-las", "apple,aop-las"; + }; + }; + + mtp: mtp@24e400000 { + compatible = "apple,t8112-mtp", "apple,t8112-rtk-helper-asc4", "apple,mtp", "apple,rtk-helper-asc4"; + reg = <0x2 0x4e400000 0x0 0x4000>, + <0x2 0x4ec00000 0x0 0x100000>; + reg-names = "asc", "sram"; + mboxes = <&mtp_mbox>; + iommus = <&mtp_dart 1>; + #helper-cells = <0>; + + status = "disabled"; + }; + + mtp_mbox: mbox@24e408000 { + compatible = "apple,t8112-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x2 0x4e408000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + + status = "disabled"; + }; + + mtp_dart: iommu@24e808000 { + compatible = "apple,t8112-dart", "apple,t8110-dart"; + reg = <0x2 0x4e808000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + #iommu-cells = <1>; + + status = "disabled"; + }; + + mtp_dockchannel: fifo@24eb14000 { + compatible = "apple,t8112-dockchannel", "apple,dockchannel"; + reg = <0x2 0x4eb14000 0x0 0x4000>; + reg-names = "irq"; + interrupt-parent = <&aic>; + interrupts = ; + + ranges = <0 0x2 0x4eb28000 0x20000>; + nonposted-mmio; + #address-cells = <1>; + #size-cells = <1>; + + interrupt-controller; + #interrupt-cells = <2>; + + status = "disabled"; + + mtp_hid: input@8000 { + compatible = "apple,dockchannel-hid"; + reg = <0x8000 0x4000>, + <0xc000 0x4000>, + <0x0000 0x4000>, + <0x4000 0x4000>; + reg-names = "config", "data", + "rmt-config", "rmt-data"; + iommus = <&mtp_dart 1>; + interrupt-parent = <&mtp_dockchannel>; + interrupts = <2 IRQ_TYPE_LEVEL_HIGH>, + <3 IRQ_TYPE_LEVEL_HIGH>; + interrupt-names = "tx", "rx"; + + apple,fifo-size = <0x800>; + apple,helper-cpu = <&mtp>; + }; + + }; + + sep_dart: iommu@25d2c0000 { + compatible = "apple,t8112-dart", "apple,t8110-dart"; + reg = <0x2 0x5d2c0000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + }; + + sep: sep@25e400000 { + compatible = "apple,sep"; + reg = <0x2 0x5e400000 0x0 0x6C000>; + mboxes = <&sep_mbox>; + mbox-names = "mbox"; + iommus = <&sep_dart 0>; + status = "disabled"; + }; + + sep_mbox: mbox@25e408000 { + compatible = "apple,t8112-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x2 0x5e408000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + }; + + dispext0_dart: iommu@271304000 { + compatible = "apple,t8112-dart", "apple,t8110-dart"; + reg = <0x2 0x71304000 0x0 0x4000>; + #iommu-cells = <1>; + apple,dma-range = <0x0 0x0 0xf 0xffff0000>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_dispext_cpu0>; + status = "disabled"; + }; + + dcpext_dart: iommu@27130c000 { + compatible = "apple,t8112-dart", "apple,t8110-dart"; + reg = <0x2 0x7130c000 0x0 0x4000>; + #iommu-cells = <1>; + apple,dma-range = <0x8 0x0 0x7 0xffff0000>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_dispext_cpu0>; + status = "disabled"; + }; + + dcpext_mbox: mbox@271c08000 { + compatible = "apple,t8112-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x2 0x71c08000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + power-domains = <&ps_dispext_cpu0>; + resets = <&ps_dispext_cpu0>; + status = "disabled"; + }; + + dcpext: dcp@271c00000 { + compatible = "apple,t8112-dcpext", "apple,dcpext"; + mboxes = <&dcpext_mbox>; + mbox-names = "mbox"; + iommus = <&dcpext_dart 5>; + phandle = <&dcpext>; + + /* the ADT has 2 additional regs which seems to be unused */ + reg-names = "coproc", "disp-0", "disp-1", "disp-2", "disp-3"; + reg = <0x2 0x71c00000 0x0 0x4000>, + <0x2 0x70000000 0x0 0x61C000>, + <0x2 0x71320000 0x0 0x4000>, + <0x2 0x71344000 0x0 0x4000>, + <0x2 0x71800000 0x0 0x800000>; + apple,bw-scratch = <&pmgr_dcp 0 4 0x5e0>; + #ifdef APPLE_USE_PMP + power-domains = <&pmp_report_dispext>; + #else + power-domains = <&ps_dispext_cpu0>; + #endif + resets = <&ps_dispext_cpu0>; + clocks = <&clk_dispext0>; + apple,dcp-index = <1>; + status = "disabled"; + // required bus properties for 'piodma' subdevice + #address-cells = <2>; + #size-cells = <2>; + + piodma { + iommus = <&dispext0_dart 4>; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + dcpext_audio: endpoint { + remote-endpoint = <&dpaudio1_dcp>; + }; + }; + }; + }; + ans_mbox: mbox@277408000 { compatible = "apple,t8112-asc-mailbox", "apple,asc-mailbox-v4"; reg = <0x2 0x77408000 0x0 0x4000>; @@ -1010,6 +1781,124 @@ resets = <&ps_ans>; }; + dwc3_0: usb@382280000 { + compatible = "apple,t8112-dwc3", "apple,t8103-dwc3"; + reg = <0x3 0x82280000 0x0 0xcd00>, <0x3 0x8228cd00 0x0 0x3200>; + reg-names = "dwc3-core", "dwc3-apple"; + interrupt-parent = <&aic>; + interrupts = ; + dr_mode = "otg"; + usb-role-switch; + role-switch-default-mode = "host"; + iommus = <&dwc3_0_dart_0 0>, <&dwc3_0_dart_1 1>; + power-domains = <&ps_atc0_usb>; + resets = <&atcphy0>; + phys = <&atcphy0 PHY_TYPE_USB2>, <&atcphy0 PHY_TYPE_USB3>; + phy-names = "usb2-phy", "usb3-phy"; + }; + + dwc3_0_dart_0: iommu@382f00000 { + compatible = "apple,t8110-dart"; + reg = <0x3 0x82f00000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + #iommu-cells = <1>; + power-domains = <&ps_atc0_usb>; + }; + + dwc3_0_dart_1: iommu@382f80000 { + compatible = "apple,t8110-dart"; + reg = <0x3 0x82f80000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + #iommu-cells = <1>; + power-domains = <&ps_atc0_usb>; + }; + + atcphy0: phy@383000000 { + compatible = "apple,t8112-atcphy", "apple,t8103-atcphy"; + reg = <0x3 0x83000000 0x0 0x4c000>, + <0x3 0x83050000 0x0 0x8000>, + <0x3 0x80000000 0x0 0x4000>, + <0x3 0x82a90000 0x0 0x4000>, + <0x3 0x82a84000 0x0 0x4000>; + reg-names = "core", "lpdptx", "axi2af", "usb2phy", + "pipehandler"; + + #phy-cells = <1>; + #reset-cells = <0>; + + orientation-switch; + mode-switch; + power-domains = <&ps_atc0_usb>; + }; + + atcphy0_xbar: mux@38304c000 { + compatible = "apple,t8112-display-crossbar", "apple,t8103-display-crossbar"; + reg = <0x3 0x8304c000 0x0 0x4000>; + #mux-control-cells = <1>; + power-domains = <&ps_atc0_usb>; + }; + + dwc3_1: usb@502280000 { + compatible = "apple,t8112-dwc3", "apple,t8103-dwc3"; + reg = <0x5 0x02280000 0x0 0xcd00>, <0x5 0x0228cd00 0x0 0x3200>; + reg-names = "dwc3-core", "dwc3-apple"; + interrupt-parent = <&aic>; + interrupts = ; + dr_mode = "otg"; + usb-role-switch; + role-switch-default-mode = "host"; + iommus = <&dwc3_1_dart_0 0>, <&dwc3_1_dart_1 1>; + power-domains = <&ps_atc1_usb>; + resets = <&atcphy1>; + phys = <&atcphy1 PHY_TYPE_USB2>, <&atcphy1 PHY_TYPE_USB3>; + phy-names = "usb2-phy", "usb3-phy"; + }; + + dwc3_1_dart_0: iommu@502f00000 { + compatible = "apple,t8110-dart"; + reg = <0x5 0x02f00000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + #iommu-cells = <1>; + power-domains = <&ps_atc1_usb>; + }; + + dwc3_1_dart_1: iommu@502f80000 { + compatible = "apple,t8110-dart"; + reg = <0x5 0x02f80000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + #iommu-cells = <1>; + power-domains = <&ps_atc1_usb>; + }; + + atcphy1: phy@503000000 { + compatible = "apple,t8112-atcphy", "apple,t8103-atcphy"; + reg = <0x5 0x03000000 0x0 0x4c000>, + <0x5 0x03050000 0x0 0x8000>, + <0x5 0x0 0x0 0x4000>, + <0x5 0x02a90000 0x0 0x4000>, + <0x5 0x02a84000 0x0 0x4000>; + reg-names = "core", "lpdptx", "axi2af", "usb2phy", + "pipehandler"; + + #phy-cells = <1>; + #reset-cells = <0>; + + orientation-switch; + mode-switch; + power-domains = <&ps_atc1_usb>; + }; + + atcphy1_xbar: mux@50304c000 { + compatible = "apple,t8112-display-crossbar", "apple,t8103-display-crossbar"; + reg = <0x5 0x0304c000 0x0 0x4000>; + #mux-control-cells = <1>; + power-domains = <&ps_atc1_usb>; + }; + pcie0_dart: iommu@681008000 { compatible = "apple,t8110-dart"; reg = <0x6 0x81008000 0x0 0x4000>; @@ -1087,6 +1976,8 @@ pinctrl-0 = <&pcie_pins>; pinctrl-names = "default"; + dma-coherent; + port00: pci@0,0 { device_type = "pci"; reg = <0x0 0x0 0x0 0x0 0x0>; diff --git a/arch/arm64/boot/dts/apple/t8122-j433.dts b/arch/arm64/boot/dts/apple/t8122-j433.dts new file mode 100644 index 00000000000000..937f159741b736 --- /dev/null +++ b/arch/arm64/boot/dts/apple/t8122-j433.dts @@ -0,0 +1,45 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * Apple iMac (24-inch, 2x USB-C, M3, 2023) + * + * target-type: J433 + * + * Copyright The Asahi Linux Contributors + */ + +/dts-v1/; + +#include "t8122.dtsi" +#include "t8122-usbpd-i2c.dtsi" +#include "t8122-jxxx.dtsi" + +/ { + compatible = "apple,j433", "apple,t8122", "apple,arm-platform"; + model = "Apple iMac (24-inch, 2x USB-C, M3, 2023)"; + + aliases { + ethernet0 = ðernet0; + }; +}; + +&wifi0 { + brcm,board-type = "apple,azores"; +}; + +&bluetooth0 { + brcm,board-type = "apple,azores"; +}; + +&port01 { + bus-range = <2 2>; + status = "okay"; + ethernet0: ethernet@0,0 { + reg = <0x20000 0x0 0x0 0x0 0x0>; + /* To be filled by the loader */ + local-mac-address = [00 10 18 00 00 00]; + }; +}; + +&pcie1_dart { + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/apple/t8122-j434.dts b/arch/arm64/boot/dts/apple/t8122-j434.dts new file mode 100644 index 00000000000000..fd79ec61091391 --- /dev/null +++ b/arch/arm64/boot/dts/apple/t8122-j434.dts @@ -0,0 +1,51 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * Apple iMac (24-inch, 4x USB-C, M3, 2023) + * + * target-type: J434 + * + * Copyright The Asahi Linux Contributors + */ + +/dts-v1/; + +#include "t8122.dtsi" +#include "t8122-usbpd-i2c.dtsi" +#include "t8122-jxxx.dtsi" + +/ { + compatible = "apple,j434", "apple,t8122", "apple,arm-platform"; + model = "Apple iMac (24-inch, 4x USB-C, M3, 2023)"; +}; + +&wifi0 { + brcm,board-type = "apple,iona"; +}; + +&bluetooth0 { + brcm,board-type = "apple,iona"; +}; + +&port01 { + bus-range = <2 2>; + status = "okay"; + ethernet0: ethernet@0,0 { + reg = <0x20000 0x0 0x0 0x0 0x0>; + /* To be filled by the loader */ + local-mac-address = [00 10 18 00 00 00]; + }; +}; + +&port02 { + bus-range = <3 3>; + pwren-gpios = <&smc_gpio 14 GPIO_ACTIVE_HIGH>; + status = "okay"; +}; + +&pcie1_dart { + status = "okay"; +}; + +&pcie2_dart { + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/apple/t8122-j504.dts b/arch/arm64/boot/dts/apple/t8122-j504.dts new file mode 100644 index 00000000000000..7cc0ffc4a8a925 --- /dev/null +++ b/arch/arm64/boot/dts/apple/t8122-j504.dts @@ -0,0 +1,103 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * Apple MacBook Pro (14-inch, M3, 2023) + * + * target-type: J504 + * + * Copyright The Asahi Linux Contributors + */ + +/dts-v1/; + +#include "t8122.dtsi" +#include "t8122-usbpd-i2c.dtsi" +#include "t8122-jxxx.dtsi" +#include + +/ { + compatible = "apple,j504", "apple,t8122", "apple,arm-platform"; + model = "Apple MacBook Pro (14-inch, M3, 2023)"; + + led-controller { + compatible = "pwm-leds"; + led-0 { + pwms = <&fpwm1 0 40000>; + label = "kbd_backlight"; + function = LED_FUNCTION_KBD_BACKLIGHT; + color = ; + max-brightness = <255>; + default-state = "keep"; + }; + }; +}; + +&wifi0 { + brcm,board-type = "apple,tresco"; +}; + +&bluetooth0 { + brcm,board-type = "apple,tresco"; +}; + +&port01 { + /* SD card reader */ + bus-range = <2 2>; + pwren-gpios = <&smc_gpio 23 GPIO_ACTIVE_HIGH>; + status = "okay"; + + sdhci0: mmc@0,0 { + compatible = "pci17a0,9755"; + reg = <0x20000 0x0 0x0 0x0 0x0>; + cd-inverted; + wp-inverted; + }; +}; + +&pcie1_dart { + status = "okay"; +}; + + +&fpwm1 { + status = "okay"; +}; + +&mtp { + status = "okay"; +}; + +&mtp_mbox { + status = "okay"; +}; + +&mtp_dart { + status = "okay"; +}; + +&mtp_dockchannel { + status = "okay"; +}; + +&mtp_hid { + apple,afe-reset-gpios = <&smc_gpio 8 GPIO_ACTIVE_LOW>; + apple,stm-reset-gpios = <&smc_gpio 24 GPIO_ACTIVE_LOW>; + + multi-touch { + firmware-name = "apple/tpmtfw-j504.bin"; + }; + + keyboard: keyboard { + hid-country-code = <0>; + apple,keyboard-layout-id = <0>; + }; + + stm { + }; + + actuator { + }; + + tp_accel { + }; +}; + diff --git a/arch/arm64/boot/dts/apple/t8122-j613.dts b/arch/arm64/boot/dts/apple/t8122-j613.dts new file mode 100644 index 00000000000000..0e0ff85f7e793a --- /dev/null +++ b/arch/arm64/boot/dts/apple/t8122-j613.dts @@ -0,0 +1,82 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * Apple MacBook Air (13-inch, M3, 2024) + * + * target-type: J613 + * + * Copyright The Asahi Linux Contributors + */ + +/dts-v1/; + +#include "t8122.dtsi" +#include "t8122-jxxx.dtsi" +#include + +/ { + compatible = "apple,j613", "apple,t8122", "apple,arm-platform"; + model = "Apple MacBook Air (13-inch, M3, 2024)"; + + led-controller { + compatible = "pwm-leds"; + led-0 { + pwms = <&fpwm1 0 40000>; + label = "kbd_backlight"; + function = LED_FUNCTION_KBD_BACKLIGHT; + color = ; + max-brightness = <255>; + default-state = "keep"; + }; + }; +}; + +&wifi0 { + brcm,board-type = "apple,dnieper"; +}; + +&bluetooth0 { + brcm,board-type = "apple,dnieper"; +}; + +&fpwm1 { + status = "okay"; +}; + +&mtp { + status = "okay"; +}; + +&mtp_mbox { + status = "okay"; +}; + +&mtp_dart { + status = "okay"; +}; + +&mtp_dockchannel { + status = "okay"; +}; + +&mtp_hid { + apple,afe-reset-gpios = <&smc_gpio 8 GPIO_ACTIVE_LOW>; + apple,stm-reset-gpios = <&smc_gpio 24 GPIO_ACTIVE_LOW>; + + multi-touch { + firmware-name = "apple/tpmtfw-j613.bin"; + }; + + keyboard: keyboard { + hid-country-code = <0>; + apple,keyboard-layout-id = <0>; + }; + + stm { + }; + + actuator { + }; + + tp_accel { + }; +}; diff --git a/arch/arm64/boot/dts/apple/t8122-j615.dts b/arch/arm64/boot/dts/apple/t8122-j615.dts new file mode 100644 index 00000000000000..77b249dda6fcfc --- /dev/null +++ b/arch/arm64/boot/dts/apple/t8122-j615.dts @@ -0,0 +1,82 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * Apple MacBook Air (15-inch, M3, 2024) + * + * target-type: J615 + * + * Copyright The Asahi Linux Contributors + */ + +/dts-v1/; + +#include "t8122.dtsi" +#include "t8122-jxxx.dtsi" +#include + +/ { + compatible = "apple,j615", "apple,t8122", "apple,arm-platform"; + model = "Apple MacBook Air (15-inch, M3, 2024)"; + + led-controller { + compatible = "pwm-leds"; + led-0 { + pwms = <&fpwm1 0 40000>; + label = "kbd_backlight"; + function = LED_FUNCTION_KBD_BACKLIGHT; + color = ; + max-brightness = <255>; + default-state = "keep"; + }; + }; +}; + +&wifi0 { + brcm,board-type = "apple,tuzla"; +}; + +&bluetooth0 { + brcm,board-type = "apple,tuzla"; +}; + +&fpwm1 { + status = "okay"; +}; + +&mtp { + status = "okay"; +}; + +&mtp_mbox { + status = "okay"; +}; + +&mtp_dart { + status = "okay"; +}; + +&mtp_dockchannel { + status = "okay"; +}; + +&mtp_hid { + apple,afe-reset-gpios = <&smc_gpio 8 GPIO_ACTIVE_LOW>; + apple,stm-reset-gpios = <&smc_gpio 24 GPIO_ACTIVE_LOW>; + + multi-touch { + firmware-name = "apple/tpmtfw-j615.bin"; + }; + + keyboard: keyboard { + hid-country-code = <0>; + apple,keyboard-layout-id = <0>; + }; + + stm { + }; + + actuator { + }; + + tp_accel { + }; +}; diff --git a/arch/arm64/boot/dts/apple/t8122-jxxx.dtsi b/arch/arm64/boot/dts/apple/t8122-jxxx.dtsi new file mode 100644 index 00000000000000..3eac7384882040 --- /dev/null +++ b/arch/arm64/boot/dts/apple/t8122-jxxx.dtsi @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * Apple M3 MacBook Air/Pro and iMac (M3, 2023/2024) + * + * This file contains parts common to all Apple M3 devices using the t8122. + * + * target-type: J433, J434, J504, J613, J615 + * + * Copyright The Asahi Linux Contributors + */ + +/ { + aliases { + bluetooth0 = &bluetooth0; + serial0 = &serial0; + wifi0 = &wifi0; + }; + + chosen { + #address-cells = <2>; + #size-cells = <2>; + ranges; + + stdout-path = "serial0"; + + framebuffer0: framebuffer@0 { + compatible = "apple,simple-framebuffer", "simple-framebuffer"; + reg = <0 0 0 0>; /* To be filled by loader */ + power-domains = <&ps_disp_cpu>, <&ps_dptx_ext_phy>; + /* Format properties will be added by loader */ + status = "disabled"; + }; + }; + + reserved-memory { + #address-cells = <2>; + #size-cells = <2>; + ranges; + /* To be filled by loader */ + }; + + memory@800000000 { + device_type = "memory"; + reg = <0x8 0 0x2 0>; /* To be filled by loader */ + }; +}; + +&serial0 { + status = "okay"; +}; + +/* + * Force the bus number assignments so that we can declare some of the + * on-board devices and properties that are populated by the bootloader + * (such as MAC addresses). + */ +&port00 { + bus-range = <1 1>; + pwren-gpios = <&smc_gpio 13 GPIO_ACTIVE_HIGH>; + + wifi0: wifi@0,0 { + compatible = "pci14e4,4434"; + reg = <0x10000 0x0 0x0 0x0 0x0>; + /* To be filled by the loader */ + local-mac-address = [00 00 00 00 00 00]; + apple,antenna-sku = "XX"; + }; + + bluetooth0: bluetooth@0,1 { + compatible = "pci14e4,5f72"; + reg = <0x10100 0x0 0x0 0x0 0x0>; + /* To be filled by the loader */ + local-bd-address = [00 00 00 00 00 00]; + }; +}; diff --git a/arch/arm64/boot/dts/apple/t8122-pmgr.dtsi b/arch/arm64/boot/dts/apple/t8122-pmgr.dtsi new file mode 100644 index 00000000000000..64093792e0adeb --- /dev/null +++ b/arch/arm64/boot/dts/apple/t8122-pmgr.dtsi @@ -0,0 +1,1149 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * PMGR Power domains for the Apple T8122 "M3" SoC + * + * Copyright The Asahi Linux Contributors + */ + +&pmgr { + ps_sbr: power-controller@100 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x100 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "sbr"; + apple,always-on; /* Core device */ + }; + + ps_msg: power-controller@108 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x108 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "msg"; + }; + + ps_aic: power-controller@110 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x110 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "aic"; + apple,always-on; /* Core device */ + }; + + ps_dwi: power-controller@118 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x118 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dwi"; + }; + + ps_gpio: power-controller@120 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x120 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "gpio"; + }; + + ps_pms_busif: power-controller@128 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x128 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "pms_busif"; + apple,always-on; /* Core device */ + }; + + ps_pms: power-controller@130 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x130 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "pms"; + apple,always-on; /* Core device */ + }; + + ps_pms_fpwm0: power-controller@138 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x138 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "pms_fpwm0"; + power-domains = <&ps_pms>; + }; + + ps_pms_fpwm1: power-controller@140 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x140 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "pms_fpwm1"; + power-domains = <&ps_pms>; + }; + + ps_pms_fpwm2: power-controller@148 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x148 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "pms_fpwm2"; + power-domains = <&ps_pms>; + }; + + ps_pms_fpwm3: power-controller@150 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x150 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "pms_fpwm3"; + power-domains = <&ps_pms>; + }; + + ps_pms_fpwm4: power-controller@158 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x158 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "pms_fpwm4"; + power-domains = <&ps_pms>; + }; + + ps_pms_c1ppt: power-controller@160 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x160 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "pms_c1ppt"; + }; + + ps_soc_rc: power-controller@168 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x168 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "soc_rc"; + }; + + ps_soc_dpe: power-controller@170 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x170 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "soc_dpe"; + apple,always-on; + }; + + ps_pmgr_soc_ocla: power-controller@178 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x178 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "pmgr_soc_ocla"; + power-domains = <&ps_pms>; + }; + + ps_ispsens0: power-controller@180 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x180 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "ispsens0"; + }; + + ps_ispsens1: power-controller@188 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x188 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "ispsens1"; + }; + + ps_ispsens2: power-controller@190 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x190 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "ispsens2"; + }; + + ps_ispsens3: power-controller@198 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x198 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "ispsens3"; + }; + + ps_aft0: power-controller@1a8 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x1a8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "aft0"; + }; + + ps_ioa0: power-controller@1b0 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x1b0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "ioa0"; + apple,always-on; + }; + + ps_ap_tmm: power-controller@1b8 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x1b8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "ap_tmm"; + }; + + ps_disp_sys: power-controller@1d8 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x1d8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "disp_sys"; + apple,always-on; /* TODO: figure out if we can enable PM here */ + }; + + ps_gfx: power-controller@1e0 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x1e0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "gfx"; + }; + + ps_isp_sys: power-controller@1e8 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x1e8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_sys"; + }; + + ps_avd_sys: power-controller@1f0 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x1f0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "avd_sys"; + }; + + ps_jpg: power-controller@200 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x200 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "jpg"; + }; + + ps_disp_fe: power-controller@208 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x208 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "disp_fe"; + power-domains = <&ps_disp_sys>; + apple,always-on; /* TODO: figure out if we can enable PM here */ + }; + + ps_sio_cpu: power-controller@210 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x210 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "sio_cpu"; + }; + + ps_fpwm0: power-controller@218 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x218 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "fpwm0"; + }; + + ps_fpwm1: power-controller@220 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x220 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "fpwm1"; + }; + + ps_fpwm2: power-controller@228 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x228 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "fpwm2"; + }; + + ps_i2c0: power-controller@230 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x230 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "i2c0"; + }; + + ps_i2c1: power-controller@238 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x238 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "i2c1"; + }; + + ps_i2c2: power-controller@240 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x240 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "i2c2"; + }; + + ps_i2c3: power-controller@248 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x248 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "i2c3"; + }; + + ps_i2c4: power-controller@250 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x250 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "i2c4"; + }; + + ps_i2c5: power-controller@258 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x258 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "i2c5"; + }; + + ps_i2c6: power-controller@260 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x260 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "i2c6"; + }; + + ps_i2c7: power-controller@268 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x268 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "i2c7"; + }; + + ps_i2c8: power-controller@270 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x270 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "i2c8"; + }; + + ps_spi_p: power-controller@278 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x278 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "spi_p"; + }; + + ps_uart_p: power-controller@280 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x280 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "uart_p"; + }; + + ps_audio_p: power-controller@288 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x288 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "audio_p"; + }; + + ps_aes: power-controller@290 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x290 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "aes"; + }; + + ps_spi0: power-controller@298 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x298 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "spi0"; + power-domains = <&ps_spi_p>; + }; + + ps_spi1: power-controller@2a0 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2a0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "spi1"; + power-domains = <&ps_spi_p>; + }; + + ps_spi2: power-controller@2a8 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2a8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "spi2"; + power-domains = <&ps_spi_p>; + }; + + ps_spi3: power-controller@2b0 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2b0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "spi3"; + power-domains = <&ps_spi_p>; + }; + + ps_spi4: power-controller@2b8 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2b8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "spi4"; + power-domains = <&ps_spi_p>; + }; + + ps_spi5: power-controller@2c0 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2c0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "spi5"; + power-domains = <&ps_spi_p>; + }; + + ps_qspi: power-controller@2c8 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2c8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "qspi"; + power-domains = <&ps_spi_p>; + }; + + ps_uart_n: power-controller@2d0 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2d0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "uart_n"; + power-domains = <&ps_uart_p>; + }; + + ps_uart0: power-controller@2d8 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2d8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "uart0"; + power-domains = <&ps_uart_p>; + }; + + ps_uart1: power-controller@2e0 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2e0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "uart1"; + power-domains = <&ps_uart_p>; + }; + + ps_uart2: power-controller@2e8 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2e8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "uart2"; + power-domains = <&ps_uart_p>; + }; + + ps_uart3: power-controller@2f0 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2f0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "uart3"; + power-domains = <&ps_uart_p>; + }; + + ps_uart4: power-controller@2f8 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2f8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "uart4"; + power-domains = <&ps_uart_p>; + }; + + ps_uart5: power-controller@300 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x300 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "uart5"; + power-domains = <&ps_uart_p>; + }; + + ps_uart6: power-controller@308 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x308 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "uart6"; + power-domains = <&ps_uart_p>; + }; + + ps_sio_adma: power-controller@310 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x310 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "sio_adma"; + power-domains = <&ps_fpwm0>; + }; + + ps_dpa0: power-controller@318 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x318 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dpa0"; + power-domains = <&ps_audio_p>; + }; + + ps_dcs0: power-controller@330 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x330 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dcs0"; + apple,always-on; + }; + + ps_dcs2: power-controller@338 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x338 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dcs2"; + apple,always-on; + }; + + ps_dcs1: power-controller@340 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x340 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dcs1"; + apple,always-on; + }; + + ps_dcs3: power-controller@348 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x348 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dcs3"; + apple,always-on; + }; + + ps_dcs4: power-controller@358 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x358 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dcs4"; + apple,always-on; + }; + + ps_dcs5: power-controller@360 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x360 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dcs5"; + apple,always-on; + }; + + ps_dcs6: power-controller@368 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x368 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dcs6"; + apple,always-on; + }; + + ps_dcs7: power-controller@370 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x370 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dcs7"; + apple,always-on; + }; + + ps_dpa1: power-controller@378 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x378 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dpa1"; + power-domains = <&ps_audio_p>; + }; + + ps_dpa2: power-controller@380 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x380 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dpa2"; + power-domains = <&ps_audio_p>; + }; + + ps_dpa3: power-controller@388 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x388 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dpa3"; + power-domains = <&ps_audio_p>; + }; + + ps_dpa4: power-controller@390 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x390 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dpa4"; + power-domains = <&ps_audio_p>; + }; + + ps_mca0: power-controller@398 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x398 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "mca0"; + power-domains = <&ps_sio_adma>, <&ps_audio_p>; + }; + + ps_mca1: power-controller@3a0 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x3a0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "mca1"; + power-domains = <&ps_sio_adma>, <&ps_audio_p>; + }; + + ps_mca2: power-controller@3a8 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x3a8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "mca2"; + power-domains = <&ps_sio_adma>, <&ps_audio_p>; + }; + + ps_trace_fab: power-controller@3b0 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x3b0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "trace_fab"; + }; + + ps_mca3: power-controller@3b8 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x3b8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "mca3"; + power-domains = <&ps_sio_adma>, <&ps_audio_p>; + }; + + ps_ioa1: power-controller@3c0 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x3c0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "ioa1"; + apple,always-on; + }; + + ps_apcie: power-controller@3f0 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x3f0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "apcie"; + }; + + ps_ans: power-controller@3f8 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x3f8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "ans"; + }; + + ps_atc0_common: power-controller@400 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x400 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "atc0_common"; + }; + + ps_atc1_common: power-controller@408 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x408 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "atc1_common"; + }; + + ps_dispext_sys: power-controller@410 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x410 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dispext_sys"; + }; + + ps_venc_sys: power-controller@418 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x418 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "venc_sys"; + }; + + ps_scodec: power-controller@420 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x420 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "scodec"; + }; + + ps_msr: power-controller@428 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x428 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "msr"; + power-domains = <&ps_aft0>; + }; + + ps_dptx_ext_phy: power-controller@430 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x430 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dptx_ext_phy"; + }; + + ps_ane_sys: power-controller@438 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x438 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "ane_sys"; + }; + + ps_apcie_gp: power-controller@440 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x440 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "apcie_gp"; + power-domains = <&ps_apcie>; + }; + + ps_apcie_st: power-controller@448 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x448 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "apcie_st"; + power-domains = <&ps_ans>, <&ps_apcie>; + }; + + ps_pmp: power-controller@450 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x450 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "pmp"; + apple,always-on; + }; + + ps_pms_sram: power-controller@458 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x458 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "pms_sram"; + apple,always-on; + }; + + ps_atc0_pcie: power-controller@460 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x460 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "atc0_pcie"; + power-domains = <&ps_atc0_common>; + }; + + ps_atc0_cio: power-controller@468 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x468 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "atc0_cio"; + power-domains = <&ps_atc0_common>; + }; + + ps_atc1_pcie: power-controller@470 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x470 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "atc1_pcie"; + power-domains = <&ps_atc1_common>; + }; + + ps_atc1_cio: power-controller@478 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x478 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "atc1_cio"; + power-domains = <&ps_atc1_common>; + }; + + ps_dispext_fe: power-controller@480 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x480 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dispext_fe"; + power-domains = <&ps_dispext_sys>; + }; + + ps_dispext_cpu: power-controller@488 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x488 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dispext_cpu"; + power-domains = <&ps_dispext_fe>; + apple,min-state = <4>; + }; + + ps_scodec_stream: power-controller@490 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x490 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "scodec_stream"; + power-domains = <&ps_scodec>; + }; + + ps_msr_ase_core: power-controller@498 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x498 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "msr_ase_core"; + power-domains = <&ps_msr>; + }; + + ps_apcie_phy_sw: power-controller@4a0 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4a0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "apcie_phy_sw"; + power-domains = <&ps_apcie_st>, <&ps_apcie_gp>; + }; + + ps_atc0_cio_pcie: power-controller@4a8 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4a8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "atc0_cio_pcie"; + power-domains = <&ps_atc0_cio>; + }; + + ps_atc0_cio_usb: power-controller@4b0 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4b0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "atc0_cio_usb"; + power-domains = <&ps_atc0_cio>; + }; + + ps_atc1_cio_pcie: power-controller@4b8 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4b8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "atc1_cio_pcie"; + power-domains = <&ps_atc1_cio>; + }; + + ps_atc1_cio_usb: power-controller@4c0 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4c0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "atc1_cio_usb"; + power-domains = <&ps_atc1_cio>; + }; + + ps_sep: power-controller@c00 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0xc00 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "sep"; + apple,always-on; + }; + + ps_venc_dma: power-controller@8000 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x8000 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "venc_dma"; + power-domains = <&ps_venc_sys>; + }; + + ps_venc_pipe4: power-controller@8008 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x8008 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "venc_pipe4"; + power-domains = <&ps_venc_dma>; + }; + + ps_venc_pipe5: power-controller@8010 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x8010 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "venc_pipe5"; + power-domains = <&ps_venc_dma>; + }; + + ps_venc_me0: power-controller@8018 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x8018 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "venc_me0"; + power-domains = <&ps_venc_dma>; + }; + + ps_venc_me1: power-controller@8020 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x8020 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "venc_me1"; + power-domains = <&ps_venc_me0>; + }; + + ps_disp_cpu: power-controller@10000 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x10000 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "disp_cpu"; + power-domains = <&ps_disp_fe>; + apple,min-state = <4>; + }; +}; + +&pmgr_mini { + + ps_debug_gated: power-controller@0 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "debug_gated"; + apple,always-on; + }; + + ps_nub_spmi0: power-controller@58 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x58 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "nub_spmi0"; + apple,always-on; + }; + + ps_nub_spmi1: power-controller@60 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x60 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "nub_spmi1"; + apple,always-on; + }; + + ps_nub_spmi2: power-controller@68 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x68 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "nub_spmi2"; + apple,always-on; + }; + + ps_nub_spmi_a0: power-controller@70 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x70 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "nub_spmi_a0"; + apple,always-on; + }; + + ps_nub_aon: power-controller@78 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x78 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "nub_aon"; + apple,always-on; + }; + + ps_nub_spi0: power-controller@80 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x80 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "nub_spi0"; + apple,always-on; + }; + + ps_nub_ocla: power-controller@88 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x88 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "nub_ocla"; + apple,always-on; + }; + + ps_nub_gpio: power-controller@90 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x90 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "nub_gpio"; + apple,always-on; + }; + + ps_nub_fabric: power-controller@98 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x98 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "nub_fabric"; + apple,always-on; + }; + + ps_nub_sram: power-controller@a0 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0xa0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "nub_sram"; + apple,always-on; + }; + + ps_debug_switch: power-controller@a8 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0xa8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "debug_switch"; + apple,always-on; + }; + + ps_atc0_usb_aon: power-controller@b0 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0xb0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "atc0_usb_aon"; + }; + + ps_atc1_usb_aon: power-controller@b8 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0xb8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "atc1_usb_aon"; + }; + + ps_atc0_usb: power-controller@c0 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0xc0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "atc0_usb"; + power-domains = <&ps_atc0_usb_aon>, <&ps_atc0_common>; + }; + + ps_atc1_usb: power-controller@c8 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0xc8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "atc1_usb"; + power-domains = <&ps_atc1_usb_aon>, <&ps_atc1_common>; + }; +}; diff --git a/arch/arm64/boot/dts/apple/t8122-usbpd-i2c.dtsi b/arch/arm64/boot/dts/apple/t8122-usbpd-i2c.dtsi new file mode 100644 index 00000000000000..112c5199cabdd4 --- /dev/null +++ b/arch/arm64/boot/dts/apple/t8122-usbpd-i2c.dtsi @@ -0,0 +1,32 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * Apple M3 MacBook Pro and iMac (M3, 2023) I2C based USB PD controller nodes + * + * This file contains nodes for t8122 devices using I2C based cd321x USB Type-C + * port controllers. The are used in the M3 MacBook Pro and iMacs but not in the + * M3 Macbook Airs. + * + * target-type: J433, J434, J504 + * + * Copyright The Asahi Linux Contributors + */ + +&i2c0 { + status = "okay"; + + hpm0: usb-pd@38 { + compatible = "apple,cd321x"; + reg = <0x38>; + interrupt-parent = <&pinctrl_ap>; + interrupts = <8 IRQ_TYPE_LEVEL_LOW>; + interrupt-names = "irq"; + }; + + hpm1: usb-pd@3f { + compatible = "apple,cd321x"; + reg = <0x3f>; + interrupt-parent = <&pinctrl_ap>; + interrupts = <8 IRQ_TYPE_LEVEL_LOW>; + interrupt-names = "irq"; + }; +}; diff --git a/arch/arm64/boot/dts/apple/t8122.dtsi b/arch/arm64/boot/dts/apple/t8122.dtsi new file mode 100644 index 00000000000000..9d1a4d1ea06ce0 --- /dev/null +++ b/arch/arm64/boot/dts/apple/t8122.dtsi @@ -0,0 +1,826 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * Apple T8122 "M3" SoC + * + * Other names: H15G + * + * Copyright The Asahi Linux Contributors + */ + +#include +#include +#include +#include +#include +#include + +/ { + compatible = "apple,t8122", "apple,arm-platform"; + + #address-cells = <2>; + #size-cells = <2>; + + cpus { + #address-cells = <2>; + #size-cells = <0>; + + cpu-map { + cluster0 { + core0 { + cpu = <&cpu_e0>; + }; + core1 { + cpu = <&cpu_e1>; + }; + core2 { + cpu = <&cpu_e2>; + }; + core3 { + cpu = <&cpu_e3>; + }; + }; + + cluster1 { + core0 { + cpu = <&cpu_p0>; + }; + core1 { + cpu = <&cpu_p1>; + }; + core2 { + cpu = <&cpu_p2>; + }; + core3 { + cpu = <&cpu_p3>; + }; + }; + }; + + cpu_e0: cpu@0 { + compatible = "apple,sawtooth"; + device_type = "cpu"; + reg = <0x0 0x0>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* To be filled by loader */ + next-level-cache = <&l2_cache_0>; + i-cache-size = <0x20000>; + d-cache-size = <0x10000>; + }; + + cpu_e1: cpu@1 { + compatible = "apple,sawtooth"; + device_type = "cpu"; + reg = <0x0 0x1>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* To be filled by loader */ + next-level-cache = <&l2_cache_0>; + i-cache-size = <0x20000>; + d-cache-size = <0x10000>; + }; + + cpu_e2: cpu@2 { + compatible = "apple,sawtooth"; + device_type = "cpu"; + reg = <0x0 0x2>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* To be filled by loader */ + next-level-cache = <&l2_cache_0>; + i-cache-size = <0x20000>; + d-cache-size = <0x10000>; + }; + + cpu_e3: cpu@3 { + compatible = "apple,sawtooth"; + device_type = "cpu"; + reg = <0x0 0x3>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* To be filled by loader */ + next-level-cache = <&l2_cache_0>; + i-cache-size = <0x20000>; + d-cache-size = <0x10000>; + }; + + cpu_p0: cpu@10100 { + compatible = "apple,everest"; + device_type = "cpu"; + reg = <0x0 0x10100>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* To be filled by loader */ + next-level-cache = <&l2_cache_1>; + i-cache-size = <0x30000>; + d-cache-size = <0x20000>; + }; + + cpu_p1: cpu@10101 { + compatible = "apple,everest"; + device_type = "cpu"; + reg = <0x0 0x10101>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* To be filled by loader */ + next-level-cache = <&l2_cache_1>; + i-cache-size = <0x30000>; + d-cache-size = <0x20000>; + }; + + cpu_p2: cpu@10102 { + compatible = "apple,everest"; + device_type = "cpu"; + reg = <0x0 0x10102>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* To be filled by loader */ + next-level-cache = <&l2_cache_1>; + i-cache-size = <0x30000>; + d-cache-size = <0x20000>; + }; + + cpu_p3: cpu@10103 { + compatible = "apple,everest"; + device_type = "cpu"; + reg = <0x0 0x10103>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* To be filled by loader */ + next-level-cache = <&l2_cache_1>; + i-cache-size = <0x30000>; + d-cache-size = <0x20000>; + }; + + l2_cache_0: l2-cache-0 { + compatible = "cache"; + cache-level = <2>; + cache-unified; + cache-size = <0x400000>; + }; + + l2_cache_1: l2-cache-1 { + compatible = "cache"; + cache-level = <2>; + cache-unified; + cache-size = <0x1000000>; + }; + }; + + timer { + compatible = "arm,armv8-timer"; + interrupt-parent = <&aic>; + interrupt-names = "phys", "virt", "hyp-phys", "hyp-virt"; + interrupts = , + , + , + ; + }; + + clkref: clock-ref { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <24000000>; + clock-output-names = "clkref"; + }; + + soc { + compatible = "simple-bus"; + #address-cells = <2>; + #size-cells = <2>; + + ranges; + nonposted-mmio; + /* Required to get >32-bit DMA via DARTs */ + dma-ranges = <0 0 0 0 0xffffffff 0xffffc000>; + + i2c0: i2c@235010000 { + compatible = "apple,t8122-i2c", "apple,t8103-i2c"; + reg = <0x2 0x35010000 0x0 0x4000>; + clocks = <&clkref>; + interrupt-parent = <&aic>; + interrupts = ; + pinctrl-0 = <&i2c0_pins>; + pinctrl-names = "default"; + #address-cells = <0x1>; + #size-cells = <0x0>; + power-domains = <&ps_i2c0>; + status = "disabled"; + }; + + i2c1: i2c@235014000 { + compatible = "apple,t8122-i2c", "apple,t8103-i2c"; + reg = <0x2 0x35014000 0x0 0x4000>; + clocks = <&clkref>; + interrupt-parent = <&aic>; + interrupts = ; + pinctrl-0 = <&i2c1_pins>; + pinctrl-names = "default"; + #address-cells = <0x1>; + #size-cells = <0x0>; + power-domains = <&ps_i2c1>; + status = "disabled"; + }; + + i2c2: i2c@235018000 { + compatible = "apple,t8122-i2c", "apple,t8103-i2c"; + reg = <0x2 0x35018000 0x0 0x4000>; + clocks = <&clkref>; + interrupt-parent = <&aic>; + interrupts = ; + pinctrl-0 = <&i2c2_pins>; + pinctrl-names = "default"; + #address-cells = <0x1>; + #size-cells = <0x0>; + power-domains = <&ps_i2c2>; + status = "disabled"; + }; + + i2c3: i2c@23501c000 { + compatible = "apple,t8122-i2c", "apple,t8103-i2c"; + reg = <0x2 0x3501c000 0x0 0x4000>; + clocks = <&clkref>; + interrupt-parent = <&aic>; + interrupts = ; + pinctrl-0 = <&i2c3_pins>; + pinctrl-names = "default"; + #address-cells = <0x1>; + #size-cells = <0x0>; + power-domains = <&ps_i2c3>; + status = "disabled"; + }; + + i2c4: i2c@235020000 { + compatible = "apple,t8122-i2c", "apple,t8103-i2c"; + reg = <0x2 0x35020000 0x0 0x4000>; + clocks = <&clkref>; + interrupt-parent = <&aic>; + interrupts = ; + pinctrl-0 = <&i2c4_pins>; + pinctrl-names = "default"; + #address-cells = <0x1>; + #size-cells = <0x0>; + power-domains = <&ps_i2c4>; + status = "disabled"; + }; + + fpwm1: pwm@2a1044000 { + compatible = "apple,t8122-fpwm", "apple,s5l-fpwm"; + reg = <0x2 0xa1044000 0x0 0x4000>; + power-domains = <&ps_fpwm1>; + clocks = <&clkref>; + #pwm-cells = <2>; + status = "disabled"; + }; + + serial0: serial@2a1200000 { + compatible = "apple,s5l-uart"; + reg = <0x2 0xa1200000 0x0 0x1000>; + reg-io-width = <4>; + interrupt-parent = <&aic>; + interrupts = ; + /* + * TODO: figure out the clocking properly, there may + * be a third selectable clock. + */ + clocks = <&clkref>, <&clkref>; + clock-names = "uart", "clk_uart_baud0"; + power-domains = <&ps_uart0>; + status = "disabled"; + }; + + aic: interrupt-controller@2d1000000 { + compatible = "apple,t8122-aic3"; + #interrupt-cells = <3>; + interrupt-controller; + reg = <0x2 0xd1000000 0x0 0x184000>, + <0x2 0xd1040000 0x0 0x4>; + reg-names = "core", "event"; + power-domains = <&ps_aic>; + + affinities { + e-core-pmu-affinity { + apple,fiq-index = ; + cpus = <&cpu_e0 &cpu_e1 &cpu_e2 &cpu_e3>; + }; + + p-core-pmu-affinity { + apple,fiq-index = ; + cpus = <&cpu_p0 &cpu_p1 &cpu_p2 &cpu_p3>; + }; + }; + }; + + pmgr: power-management@2d0700000 { + compatible = "apple,t8122-pmgr", "apple,t8103-pmgr", "syscon", "simple-mfd"; + #address-cells = <1>; + #size-cells = <1>; + reg = <0x2 0xd0700000 0 0x10000>; + /* child nodes are added in t8122-pmgr.dtsi */ + }; + + pinctrl_ap: pinctrl@2c7100000 { + compatible = "apple,t8122-pinctrl", "apple,t8103-pinctrl"; + reg = <0x2 0xc7100000 0x0 0x100000>; + power-domains = <&ps_gpio>; + + gpio-controller; + #gpio-cells = <2>; + gpio-ranges = <&pinctrl_ap 0 0 224>; + apple,npins = <224>; + + interrupt-controller; + #interrupt-cells = <2>; + interrupt-parent = <&aic>; + interrupts = , + , + , + , + , + , + ; + + i2c0_pins: i2c0-pins { + pinmux = , + ; + }; + + i2c1_pins: i2c1-pins { + pinmux = , + ; + }; + + i2c2_pins: i2c2-pins { + pinmux = , + ; + }; + + i2c3_pins: i2c3-pins { + pinmux = , + ; + }; + + i2c4_pins: i2c4-pins { + pinmux = , + ; + }; + + pcie_pins: pcie-pins { + // clkreq pins + pinmux = , + , + , + ; + }; + }; + + pinctrl_nub: pinctrl@2e41f0000 { + compatible = "apple,t8122-pinctrl", "apple,t8103-pinctrl"; + reg = <0x2 0xe41f0000 0x0 0x4000>; + power-domains = <&ps_nub_gpio>; + + gpio-controller; + #gpio-cells = <2>; + gpio-ranges = <&pinctrl_nub 0 0 32>; + apple,npins = <32>; + + interrupt-controller; + #interrupt-cells = <2>; + interrupt-parent = <&aic>; + interrupts = , + , + , + , + , + , + ; + }; + + pmgr_mini: power-management@2e4280000 { + compatible = "apple,t8122-pmgr", "apple,t8103-pmgr", "syscon", "simple-mfd"; + #address-cells = <1>; + #size-cells = <1>; + reg = <0x2 0xe4280000 0 0x4000>; + /* child nodes are added in t8122-pmgr.dtsi */ + }; + + wdt: watchdog@2e42b0000 { + compatible = "apple,t8122-wdt", "apple,t8103-wdt"; + reg = <0x2 0xe42b0000 0x0 0x4000>; + clocks = <&clkref>; + interrupt-parent = <&aic>; + interrupts = ; + }; + + nub_spmi: spmi@2e4714000 { + compatible = "apple,t8122-spmi", "apple,t8103-spmi"; + reg = <0x2 0xe4714000 0x0 0x100>; + #address-cells = <2>; + #size-cells = <0>; + + pmic1: pmic@e { + compatible = "apple,stowe-pmic", "apple,spmi-nvmem"; + reg = <0xe SPMI_USID>; + + nvmem-layout { + compatible = "fixed-layout"; + #address-cells = <1>; + #size-cells = <1>; + + fault_shadow: fault-shadow@867b { + reg = <0x867b 0x10>; + }; + + socd: socd@8b00 { + reg = <0x8b00 0x400>; + }; + + boot_stage: boot-stage@f701 { + reg = <0xf701 0x1>; + }; + + boot_error_count: boot-error-count@f702,0 { + reg = <0xf702 0x1>; + bits = <0 4>; + }; + + panic_count: panic-count@f702,4 { + reg = <0xf702 0x1>; + bits = <4 4>; + }; + + boot_error_stage: boot-error-stage@f703 { + reg = <0xf703 0x1>; + }; + + shutdown_flag: shutdown-flag@f70f,3 { + reg = <0xf70f 0x1>; + bits = <3 1>; + }; + + pm_setting: pm-setting@f801 { + reg = <0xf801 0x1>; + }; + + rtc_offset: rtc-offset@f900 { + reg = <0xf900 0x6>; + }; + }; + }; + }; + + smc_mbox: mbox@2ec408000 { + compatible = "apple,t8122-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x2 0xec408000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + }; + + smc: smc@2ec400000 { + compatible = "apple,t8122-smc", "apple,t8103-smc"; + reg = <0x2 0xec400000 0x0 0x4000>, + <0x2 0xede00000 0x0 0x100000>; + reg-names = "smc", "sram"; + mboxes = <&smc_mbox>; + + smc_gpio: gpio { + compatible = "apple,smc-gpio"; + gpio-controller; + #gpio-cells = <2>; + }; + + smc_reboot: reboot { + compatible = "apple,smc-reboot"; + nvmem-cells = <&shutdown_flag>, <&boot_stage>, + <&boot_error_count>, <&panic_count>; + nvmem-cell-names = "shutdown_flag", "boot_stage", + "boot_error_count", "panic_count"; + }; + + rtc { + compatible = "apple,smc-rtc"; + nvmem-cells = <&rtc_offset>; + nvmem-cell-names = "rtc_offset"; + }; + }; + + pinctrl_smc: pinctrl@2ec820000 { + compatible = "apple,t8122-pinctrl", "apple,t8103-pinctrl"; + reg = <0x2 0xec820000 0x0 0x4000>; + + gpio-controller; + #gpio-cells = <2>; + gpio-ranges = <&pinctrl_smc 0 0 18>; + apple,npins = <18>; + + interrupt-controller; + #interrupt-cells = <2>; + interrupt-parent = <&aic>; + interrupts = , + , + , + , + , + , + ; + }; + + pinctrl_aop: pinctrl@2f4824000 { + compatible = "apple,t8122-pinctrl", "apple,t8103-pinctrl"; + reg = <0x2 0xf4824000 0x0 0x4000>; + + gpio-controller; + #gpio-cells = <2>; + gpio-ranges = <&pinctrl_aop 0 0 54>; + apple,npins = <54>; + + interrupt-controller; + #interrupt-cells = <2>; + interrupt-parent = <&aic>; + interrupts = , + , + , + , + , + , + ; + }; + + mtp: mtp@2fa400000 { + compatible = "apple,t8122-mtp", "apple,t8122-rtk-helper-asc4", "apple,mtp", "apple,rtk-helper-asc4"; + reg = <0x2 0xfa400000 0x0 0x4000>, + <0x2 0xfac00000 0x0 0x100000>; + reg-names = "asc", "sram"; + + mboxes = <&mtp_mbox>; + iommus = <&mtp_dart 1>; + #helper-cells = <0>; + + status = "disabled"; + }; + + mtp_mbox: mbox@2fa408000 { + compatible = "apple,t8122-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x2 0xfa408000 0x0 0x4000>; + + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + status = "disabled"; + }; + + mtp_dart: iommu@2fa808000 { + compatible = "apple,t8122-dart", "apple,t8110-dart"; + reg = <0x2 0xfa808000 0x0 0x4000>; + + interrupt-parent = <&aic>; + interrupts = ; + + #iommu-cells = <1>; + + status = "disabled"; + }; + + mtp_dockchannel: fifo@2fab30000 { + compatible = "apple,t8122-dockchannel", "apple,dockchannel"; + reg = <0x2 0xfab14000 0x0 0x4000>; + reg-names = "irq"; + interrupt-parent = <&aic>; + interrupts = ; + + ranges = <0 0x2 0xfab30000 0x20000>; + nonposted-mmio; + #address-cells = <1>; + #size-cells = <1>; + + interrupt-controller; + #interrupt-cells = <2>; + + status = "disabled"; + + mtp_hid: input@8000 { + compatible = "apple,dockchannel-hid"; + reg = <0x8000 0x4000>, + <0xc000 0x4000>, + <0x0000 0x4000>, + <0x4000 0x4000>; + reg-names = "rmt-config", "rmt-data", "config", "data"; + + iommus = <&mtp_dart 1>; + + interrupt-parent = <&mtp_dockchannel>; + interrupts = <2 IRQ_TYPE_LEVEL_HIGH>, + <3 IRQ_TYPE_LEVEL_HIGH>; + interrupt-names = "tx", "rx"; + + apple,fifo-size = <0x800>; + apple,helper-cpu = <&mtp>; + }; + }; + + ans_mbox: mbox@309408000 { + compatible = "apple,t8122-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x3 0x09408000 0x0 0x4000>; + + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + + #mbox-cells = <0>; + power-domains = <&ps_ans>; + }; + + sart: sart@30dc50000 { + compatible = "apple,t8122-sart", "apple,t6000-sart"; + reg = <0x3 0x0dc50000 0x0 0x10000>; + power-domains = <&ps_ans>; + }; + + nvme: nvme@30dcc0000 { + compatible = "apple,t8122-nvme-ans2", "apple,t8103-nvme-ans2"; + reg = <0x3 0x0dcc0000 0x0 0x60000>, + <0x3 0x09400000 0x0 0x4000>; + reg-names = "nvme", "ans"; + + interrupt-parent = <&aic>; + interrupts = ; + + mboxes = <&ans_mbox>; + apple,sart = <&sart>; + + power-domains = <&ps_ans>, <&ps_apcie_phy_sw>; + power-domain-names = "ans", "apcie0"; + resets = <&ps_ans>; + }; + + pcie0_dart: iommu@594000000 { + compatible = "apple,t8122-dart", "apple,t8110-dart"; + reg = <0x5 0x94000000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_apcie_gp>; + }; + + pcie1_dart: iommu@595000000 { + compatible = "apple,t8122-dart", "apple,t8110-dart"; + reg = <0x5 0x95000000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_apcie_gp>; + status = "disabled"; + }; + + pcie2_dart: iommu@596000000 { + compatible = "apple,t8122-dart", "apple,t8110-dart"; + reg = <0x6 0x96000000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_apcie_gp>; + status = "disabled"; + }; + + pcie3_dart: iommu@597000000 { + compatible = "apple,t8122-dart", "apple,t8110-dart"; + reg = <0x6 0x97000000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_apcie_gp>; + status = "disabled"; + }; + + pcie0: pcie@580000000 { + compatible = "apple,t8122-pcie", "apple,t6020-pcie"; + device_type = "pci"; + + reg = <0x5 0x80000000 0x0 0x1000000>, /* config */ + <0x5 0x91000000 0x0 0x4000>, /* rc */ + <0x5 0x94008000 0x0 0x4000>, /* port0 */ + <0x5 0x95008000 0x0 0x4000>, /* port1 */ + <0x5 0x96008000 0x0 0x4000>, /* port2 */ + <0x5 0x97008000 0x0 0x4000>, /* port3 */ + <0x5 0x9e00c000 0x0 0x4000>, /* phy0 */ + <0x5 0x9e010000 0x0 0x4000>, /* phy1 */ + <0x5 0x9e014000 0x0 0x4000>, /* phy2 */ + <0x5 0x9e018000 0x0 0x4000>; /* phy3 */ + reg-names = "config", "rc", "port0", "port1", "port2", "port3", "phy0", "phy1", "phy2", "phy3"; + + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + + msi-controller; + msi-parent = <&pcie0>; + msi-ranges = <&aic AIC_IRQ 1075 IRQ_TYPE_EDGE_RISING 32>; + + iommu-map = <0x100 &pcie0_dart 0 1>, + <0x200 &pcie1_dart 1 1>, + <0x300 &pcie2_dart 2 1>, + <0x300 &pcie3_dart 3 1>; + iommu-map-mask = <0xff00>; + + bus-range = <0 4>; + #address-cells = <3>; + #size-cells = <2>; + ranges = <0x43000000 0x5 0xa0000000 0x5 0xa0000000 0x0 0x20000000>, + <0x02000000 0x0 0xc0000000 0x5 0xc0000000 0x0 0x40000000>; + + power-domains = <&ps_apcie_gp>; + pinctrl-0 = <&pcie_pins>; + pinctrl-names = "default"; + + port00: pci@0,0 { + device_type = "pci"; + reg = <0x0 0x0 0x0 0x0 0x0>; + reset-gpios = <&pinctrl_ap 187 GPIO_ACTIVE_LOW>; // perst + + #address-cells = <3>; + #size-cells = <2>; + ranges; + + interrupt-controller; + #interrupt-cells = <1>; + + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &port00 0 0 0 0>, + <0 0 0 2 &port00 0 0 0 1>, + <0 0 0 3 &port00 0 0 0 2>, + <0 0 0 4 &port00 0 0 0 3>; + }; + + port01: pci@1,0 { + device_type = "pci"; + reg = <0x800 0x0 0x0 0x0 0x0>; + reset-gpios = <&pinctrl_ap 188 GPIO_ACTIVE_LOW>; // perst + + #address-cells = <3>; + #size-cells = <2>; + ranges; + + interrupt-controller; + #interrupt-cells = <1>; + + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &port01 0 0 0 0>, + <0 0 0 2 &port01 0 0 0 1>, + <0 0 0 3 &port01 0 0 0 2>, + <0 0 0 4 &port01 0 0 0 3>; + status = "disabled"; + }; + + port02: pci@2,0 { + device_type = "pci"; + reg = <0x1000 0x0 0x0 0x0 0x0>; + reset-gpios = <&pinctrl_ap 189 GPIO_ACTIVE_LOW>; // perst + + #address-cells = <3>; + #size-cells = <2>; + ranges; + + interrupt-controller; + #interrupt-cells = <1>; + + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &port02 0 0 0 0>, + <0 0 0 2 &port02 0 0 0 1>, + <0 0 0 3 &port02 0 0 0 2>, + <0 0 0 4 &port02 0 0 0 3>; + status = "disabled"; + }; + + port03: pci@3,0 { + device_type = "pci"; + reg = <0x1800 0x0 0x0 0x0 0x0>; + reset-gpios = <&pinctrl_ap 190 GPIO_ACTIVE_LOW>; // perst + + #address-cells = <3>; + #size-cells = <2>; + ranges; + + interrupt-controller; + #interrupt-cells = <1>; + + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &port03 0 0 0 0>, + <0 0 0 2 &port03 0 0 0 1>, + <0 0 0 3 &port03 0 0 0 2>, + <0 0 0 4 &port03 0 0 0 3>; + status = "disabled"; + }; + }; + }; +}; + +#include "t8122-pmgr.dtsi" diff --git a/arch/arm64/boot/dts/freescale/imx8mn-tqma8mqnl-mba8mx.dts b/arch/arm64/boot/dts/freescale/imx8mn-tqma8mqnl-mba8mx.dts index d7f7f9aafb7d1b..0d009f4be804e8 100644 --- a/arch/arm64/boot/dts/freescale/imx8mn-tqma8mqnl-mba8mx.dts +++ b/arch/arm64/boot/dts/freescale/imx8mn-tqma8mqnl-mba8mx.dts @@ -69,6 +69,10 @@ samsung,esc-clock-frequency = <20000000>; }; +®_usdhc2_vqmmc { + status = "okay"; +}; + &sai3 { assigned-clocks = <&clk IMX8MN_CLK_SAI3>; assigned-clock-parents = <&clk IMX8MN_AUDIO_PLL1_OUT>; @@ -216,8 +220,7 @@ , , , - , - ; + ; }; pinctrl_usdhc2_100mhz: usdhc2-100mhzgrp { @@ -226,8 +229,7 @@ , , , - , - ; + ; }; pinctrl_usdhc2_200mhz: usdhc2-200mhzgrp { @@ -236,8 +238,7 @@ , , , - , - ; + ; }; pinctrl_usdhc2_gpio: usdhc2-gpiogrp { diff --git a/arch/arm64/boot/dts/freescale/imx8mn-tqma8mqnl.dtsi b/arch/arm64/boot/dts/freescale/imx8mn-tqma8mqnl.dtsi index 1d23814e11cd30..e2ccebf6ee13f1 100644 --- a/arch/arm64/boot/dts/freescale/imx8mn-tqma8mqnl.dtsi +++ b/arch/arm64/boot/dts/freescale/imx8mn-tqma8mqnl.dtsi @@ -30,6 +30,20 @@ regulator-max-microvolt = <3300000>; }; + reg_usdhc2_vqmmc: regulator-usdhc2-vqmmc { + compatible = "regulator-gpio"; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_reg_usdhc2_vqmmc>; + regulator-name = "V_SD2"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <3300000>; + gpios = <&gpio1 4 GPIO_ACTIVE_HIGH>; + states = <1800000 0x1>, + <3300000 0x0>; + vin-supply = <&ldo5_reg>; + status = "disabled"; + }; + reserved-memory { #address-cells = <2>; #size-cells = <2>; @@ -233,6 +247,10 @@ vddio-supply = <&ldo3_reg>; }; +&usdhc2 { + vqmmc-supply = <®_usdhc2_vqmmc>; +}; + &usdhc3 { pinctrl-names = "default", "state_100mhz", "state_200mhz"; pinctrl-0 = <&pinctrl_usdhc3>; @@ -287,6 +305,10 @@ fsl,pins = ; }; + pinctrl_reg_usdhc2_vqmmc: regusdhc2vqmmcgrp { + fsl,pins = ; + }; + pinctrl_usdhc3: usdhc3grp { fsl,pins = , , diff --git a/arch/arm64/boot/dts/freescale/imx8mp-tqma8mpql-mba8mp-ras314.dts b/arch/arm64/boot/dts/freescale/imx8mp-tqma8mpql-mba8mp-ras314.dts index f7346b3d35fe53..a122f2ed5f5314 100644 --- a/arch/arm64/boot/dts/freescale/imx8mp-tqma8mpql-mba8mp-ras314.dts +++ b/arch/arm64/boot/dts/freescale/imx8mp-tqma8mpql-mba8mp-ras314.dts @@ -704,7 +704,7 @@ fsl,pins = , , , - ; + ; }; pinctrl_gpt1: gpt1grp { diff --git a/arch/arm64/boot/dts/freescale/imx8mp-tqma8mpql-mba8mpxl.dts b/arch/arm64/boot/dts/freescale/imx8mp-tqma8mpql-mba8mpxl.dts index 59642a8a2c445d..c73d40fb789f69 100644 --- a/arch/arm64/boot/dts/freescale/imx8mp-tqma8mpql-mba8mpxl.dts +++ b/arch/arm64/boot/dts/freescale/imx8mp-tqma8mpql-mba8mpxl.dts @@ -867,7 +867,7 @@ fsl,pins = , , , - ; + ; }; pinctrl_hoggpio2: hoggpio2grp { diff --git a/arch/arm64/boot/dts/freescale/imx8mq-librem5-r3.dts b/arch/arm64/boot/dts/freescale/imx8mq-librem5-r3.dts index 077c5cd2586f75..4533a84fb0b95f 100644 --- a/arch/arm64/boot/dts/freescale/imx8mq-librem5-r3.dts +++ b/arch/arm64/boot/dts/freescale/imx8mq-librem5-r3.dts @@ -7,7 +7,7 @@ &a53_opp_table { opp-1000000000 { - opp-microvolt = <950000>; + opp-microvolt = <1000000>; }; }; diff --git a/arch/arm64/boot/dts/freescale/imx8mq-librem5.dtsi b/arch/arm64/boot/dts/freescale/imx8mq-librem5.dtsi index 9e0e2d7271efbe..e5f228517b6db3 100644 --- a/arch/arm64/boot/dts/freescale/imx8mq-librem5.dtsi +++ b/arch/arm64/boot/dts/freescale/imx8mq-librem5.dtsi @@ -844,9 +844,9 @@ regulator-max-microvolt = <1300000>; regulator-boot-on; regulator-ramp-delay = <1250>; - rohm,dvs-run-voltage = <880000>; - rohm,dvs-idle-voltage = <820000>; - rohm,dvs-suspend-voltage = <810000>; + rohm,dvs-run-voltage = <900000>; + rohm,dvs-idle-voltage = <850000>; + rohm,dvs-suspend-voltage = <850000>; regulator-always-on; }; @@ -856,8 +856,8 @@ regulator-max-microvolt = <1300000>; regulator-boot-on; regulator-ramp-delay = <1250>; - rohm,dvs-run-voltage = <950000>; - rohm,dvs-idle-voltage = <850000>; + rohm,dvs-run-voltage = <1000000>; + rohm,dvs-idle-voltage = <900000>; regulator-always-on; }; @@ -866,14 +866,14 @@ regulator-min-microvolt = <700000>; regulator-max-microvolt = <1300000>; regulator-boot-on; - rohm,dvs-run-voltage = <850000>; + rohm,dvs-run-voltage = <900000>; }; buck4_reg: BUCK4 { regulator-name = "buck4"; regulator-min-microvolt = <700000>; regulator-max-microvolt = <1300000>; - rohm,dvs-run-voltage = <930000>; + rohm,dvs-run-voltage = <1000000>; }; buck5_reg: BUCK5 { @@ -1407,13 +1407,3 @@ fsl,ext-reset-output; status = "okay"; }; - -&a53_opp_table { - opp-1000000000 { - opp-microvolt = <850000>; - }; - - opp-1500000000 { - opp-microvolt = <950000>; - }; -}; diff --git a/arch/arm64/boot/dts/freescale/imx8mq.dtsi b/arch/arm64/boot/dts/freescale/imx8mq.dtsi index 607962f807bebe..6a25e219832ced 100644 --- a/arch/arm64/boot/dts/freescale/imx8mq.dtsi +++ b/arch/arm64/boot/dts/freescale/imx8mq.dtsi @@ -1632,7 +1632,7 @@ <&clk IMX8MQ_GPU_PLL_OUT>, <&clk IMX8MQ_GPU_PLL>; assigned-clock-rates = <800000000>, <800000000>, - <800000000>, <800000000>, <0>; + <800000000>, <400000000>, <0>; power-domains = <&pgc_gpu>; }; diff --git a/arch/arm64/boot/dts/freescale/imx91-tqma9131.dtsi b/arch/arm64/boot/dts/freescale/imx91-tqma9131.dtsi index 5792952b7a8e14..c99d7bc1684836 100644 --- a/arch/arm64/boot/dts/freescale/imx91-tqma9131.dtsi +++ b/arch/arm64/boot/dts/freescale/imx91-tqma9131.dtsi @@ -272,20 +272,20 @@ /* enable SION for data and cmd pad due to ERR052021 */ pinctrl_usdhc1: usdhc1grp { fsl,pins = /* PD | FSEL 3 | DSE X5 */ - , + , /* HYS | FSEL 0 | no drive */ , /* HYS | FSEL 3 | X5 */ - , + , /* HYS | FSEL 3 | X4 */ - , - , - , - , - , - , - , - ; + , + , + , + , + , + , + , + ; }; pinctrl_wdog: wdoggrp { diff --git a/arch/arm64/boot/dts/freescale/imx93-9x9-qsb.dts b/arch/arm64/boot/dts/freescale/imx93-9x9-qsb.dts index 0852067eab2cb8..197c8f8b7f6696 100644 --- a/arch/arm64/boot/dts/freescale/imx93-9x9-qsb.dts +++ b/arch/arm64/boot/dts/freescale/imx93-9x9-qsb.dts @@ -507,6 +507,7 @@ pinctrl-2 = <&pinctrl_usdhc1_200mhz>; bus-width = <8>; non-removable; + fsl,tuning-step = <1>; status = "okay"; }; @@ -519,6 +520,7 @@ vmmc-supply = <®_usdhc2_vmmc>; bus-width = <4>; no-mmc; + fsl,tuning-step = <1>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/freescale/imx93-tqma9352.dtsi b/arch/arm64/boot/dts/freescale/imx93-tqma9352.dtsi index 3a23e2eb9febe8..ce34a296495c45 100644 --- a/arch/arm64/boot/dts/freescale/imx93-tqma9352.dtsi +++ b/arch/arm64/boot/dts/freescale/imx93-tqma9352.dtsi @@ -271,21 +271,21 @@ /* enable SION for data and cmd pad due to ERR052021 */ pinctrl_usdhc1: usdhc1grp { fsl,pins = < - /* PD | FSEL 3 | DSE X5 */ - MX93_PAD_SD1_CLK__USDHC1_CLK 0x5be + /* PD | FSEL 3 | DSE X4 */ + MX93_PAD_SD1_CLK__USDHC1_CLK 0x59e /* HYS | FSEL 0 | no drive */ MX93_PAD_SD1_STROBE__USDHC1_STROBE 0x1000 - /* HYS | FSEL 3 | X5 */ - MX93_PAD_SD1_CMD__USDHC1_CMD 0x400011be - /* HYS | FSEL 3 | X4 */ - MX93_PAD_SD1_DATA0__USDHC1_DATA0 0x4000119e - MX93_PAD_SD1_DATA1__USDHC1_DATA1 0x4000119e - MX93_PAD_SD1_DATA2__USDHC1_DATA2 0x4000119e - MX93_PAD_SD1_DATA3__USDHC1_DATA3 0x4000119e - MX93_PAD_SD1_DATA4__USDHC1_DATA4 0x4000119e - MX93_PAD_SD1_DATA5__USDHC1_DATA5 0x4000119e - MX93_PAD_SD1_DATA6__USDHC1_DATA6 0x4000119e - MX93_PAD_SD1_DATA7__USDHC1_DATA7 0x4000119e + /* HYS | PU | FSEL 3 | DSE X4 */ + MX93_PAD_SD1_CMD__USDHC1_CMD 0x4000139e + /* HYS | PU | FSEL 3 | DSE X4 */ + MX93_PAD_SD1_DATA0__USDHC1_DATA0 0x4000139e + MX93_PAD_SD1_DATA1__USDHC1_DATA1 0x4000139e + MX93_PAD_SD1_DATA2__USDHC1_DATA2 0x4000139e + MX93_PAD_SD1_DATA3__USDHC1_DATA3 0x4000139e + MX93_PAD_SD1_DATA4__USDHC1_DATA4 0x4000139e + MX93_PAD_SD1_DATA5__USDHC1_DATA5 0x4000139e + MX93_PAD_SD1_DATA6__USDHC1_DATA6 0x4000139e + MX93_PAD_SD1_DATA7__USDHC1_DATA7 0x4000139e >; }; diff --git a/arch/arm64/boot/dts/freescale/imx95-clock.h b/arch/arm64/boot/dts/freescale/imx95-clock.h index e1f91203e79470..22311612e44033 100644 --- a/arch/arm64/boot/dts/freescale/imx95-clock.h +++ b/arch/arm64/boot/dts/freescale/imx95-clock.h @@ -183,5 +183,6 @@ #define IMX95_CLK_SEL_A55P (IMX95_CCM_NUM_CLK_SRC + 123 + 7) #define IMX95_CLK_SEL_DRAM (IMX95_CCM_NUM_CLK_SRC + 123 + 8) #define IMX95_CLK_SEL_TEMPSENSE (IMX95_CCM_NUM_CLK_SRC + 123 + 9) +#define IMX95_CLK_GPU_CGC (IMX95_CCM_NUM_CLK_SRC + 123 + 10) #endif /* __CLOCK_IMX95_H */ diff --git a/arch/arm64/boot/dts/freescale/imx95.dtsi b/arch/arm64/boot/dts/freescale/imx95.dtsi index a4d85481755947..55e2da094c889f 100644 --- a/arch/arm64/boot/dts/freescale/imx95.dtsi +++ b/arch/arm64/boot/dts/freescale/imx95.dtsi @@ -2164,7 +2164,7 @@ gpu: gpu@4d900000 { compatible = "nxp,imx95-mali", "arm,mali-valhall-csf"; reg = <0 0x4d900000 0 0x480000>; - clocks = <&scmi_clk IMX95_CLK_GPU>, <&scmi_clk IMX95_CLK_GPUAPB>; + clocks = <&scmi_clk IMX95_CLK_GPU_CGC>, <&scmi_clk IMX95_CLK_GPUAPB>; clock-names = "core", "coregroup"; interrupts = , , diff --git a/arch/arm64/boot/dts/hisilicon/hi3798cv200-poplar.dts b/arch/arm64/boot/dts/hisilicon/hi3798cv200-poplar.dts index 7d370dac4c8571..579d55daa7d04e 100644 --- a/arch/arm64/boot/dts/hisilicon/hi3798cv200-poplar.dts +++ b/arch/arm64/boot/dts/hisilicon/hi3798cv200-poplar.dts @@ -179,7 +179,7 @@ }; &pcie { - reset-gpios = <&gpio4 4 GPIO_ACTIVE_HIGH>; + reset-gpios = <&gpio4 4 GPIO_ACTIVE_LOW>; vpcie-supply = <®_pcie>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/hisilicon/hi3798cv200.dtsi b/arch/arm64/boot/dts/hisilicon/hi3798cv200.dtsi index f6bc001c383263..2f4ad5da5e33c7 100644 --- a/arch/arm64/boot/dts/hisilicon/hi3798cv200.dtsi +++ b/arch/arm64/boot/dts/hisilicon/hi3798cv200.dtsi @@ -122,6 +122,7 @@ #address-cells = <1>; #size-cells = <1>; ranges = <0x0 0x0 0xf0000000 0x10000000>; + dma-ranges = <0x0 0x0 0x0 0x40000000>; crg: clock-reset-controller@8a22000 { compatible = "hisilicon,hi3798cv200-crg", "syscon", "simple-mfd"; diff --git a/arch/arm64/boot/dts/mediatek/mt8183-kukui-jacuzzi-pico6.dts b/arch/arm64/boot/dts/mediatek/mt8183-kukui-jacuzzi-pico6.dts index cce326aec1aa59..40af5656d6f15f 100644 --- a/arch/arm64/boot/dts/mediatek/mt8183-kukui-jacuzzi-pico6.dts +++ b/arch/arm64/boot/dts/mediatek/mt8183-kukui-jacuzzi-pico6.dts @@ -91,7 +91,7 @@ &pio { bt_pins_wakeup: bt-pins-wakeup { - piins-bt-wakeup { + pins-bt-wakeup { pinmux = ; input-enable; }; diff --git a/arch/arm64/boot/dts/mediatek/mt8183.dtsi b/arch/arm64/boot/dts/mediatek/mt8183.dtsi index 4e20a8f2eb2580..95cc067995336c 100644 --- a/arch/arm64/boot/dts/mediatek/mt8183.dtsi +++ b/arch/arm64/boot/dts/mediatek/mt8183.dtsi @@ -1812,15 +1812,23 @@ #size-cells = <0>; port@0 { + #address-cells = <1>; + #size-cells = <0>; reg = <0>; - ovl_2l1_in: endpoint { + + ovl_2l1_in: endpoint@1 { + reg = <1>; remote-endpoint = <&mmsys_ep_ext>; }; }; port@1 { + #address-cells = <1>; + #size-cells = <0>; reg = <1>; - ovl_2l1_out: endpoint { + + ovl_2l1_out: endpoint@1 { + reg = <1>; remote-endpoint = <&rdma1_in>; }; }; @@ -1872,15 +1880,23 @@ #size-cells = <0>; port@0 { + #address-cells = <1>; + #size-cells = <0>; reg = <0>; - rdma1_in: endpoint { + + rdma1_in: endpoint@1 { + reg = <1>; remote-endpoint = <&ovl_2l1_out>; }; }; port@1 { + #address-cells = <1>; + #size-cells = <0>; reg = <1>; - rdma1_out: endpoint { + + rdma1_out: endpoint@1 { + reg = <1>; remote-endpoint = <&dpi_in>; }; }; @@ -2076,15 +2092,24 @@ #size-cells = <0>; port@0 { + #address-cells = <1>; + #size-cells = <0>; reg = <0>; - dpi_in: endpoint { + + dpi_in: endpoint@1 { + reg = <1>; remote-endpoint = <&rdma1_out>; }; }; port@1 { + #address-cells = <1>; + #size-cells = <0>; reg = <1>; - dpi_out: endpoint { }; + + dpi_out: endpoint@1 { + reg = <1>; + }; }; }; }; diff --git a/arch/arm64/boot/dts/nvidia/tegra210-smaug.dts b/arch/arm64/boot/dts/nvidia/tegra210-smaug.dts index 5aa6afd56cbc63..dfbd1c72388c12 100644 --- a/arch/arm64/boot/dts/nvidia/tegra210-smaug.dts +++ b/arch/arm64/boot/dts/nvidia/tegra210-smaug.dts @@ -1809,6 +1809,8 @@ status = "okay"; vbus-supply = <&usbc_vbus>; mode = "otg"; + usb-role-switch; + role-switch-default-mode = "host"; }; usb3-0 { diff --git a/arch/arm64/boot/dts/qcom/agatti.dtsi b/arch/arm64/boot/dts/qcom/agatti.dtsi index 8bf5c5583fc22e..969ae1378db2ce 100644 --- a/arch/arm64/boot/dts/qcom/agatti.dtsi +++ b/arch/arm64/boot/dts/qcom/agatti.dtsi @@ -1591,8 +1591,12 @@ gpu: gpu@5900000 { compatible = "qcom,adreno-07000200", "qcom,adreno"; - reg = <0x0 0x05900000 0x0 0x40000>; - reg-names = "kgsl_3d0_reg_memory"; + reg = <0x0 0x05900000 0x0 0x40000>, + <0x0 0x0599e000 0x0 0x1000>, + <0x0 0x05961000 0x0 0x800>; + reg-names = "kgsl_3d0_reg_memory", + "cx_mem", + "cx_dbgc"; interrupts = ; diff --git a/arch/arm64/boot/dts/qcom/hamoa.dtsi b/arch/arm64/boot/dts/qcom/hamoa.dtsi index a17900eacb2039..f1ebb99d942416 100644 --- a/arch/arm64/boot/dts/qcom/hamoa.dtsi +++ b/arch/arm64/boot/dts/qcom/hamoa.dtsi @@ -269,7 +269,7 @@ idle-state-name = "ret"; arm,psci-suspend-param = <0x00000004>; entry-latency-us = <180>; - exit-latency-us = <500>; + exit-latency-us = <320>; min-residency-us = <600>; }; }; @@ -791,8 +791,8 @@ #address-cells = <2>; #size-cells = <2>; - dma-ranges = <0 0 0 0 0x10 0>; - ranges = <0 0 0 0 0x10 0>; + dma-ranges = <0 0 0 0 0x100 0>; + ranges = <0 0 0 0 0x100 0>; gcc: clock-controller@100000 { compatible = "qcom,x1e80100-gcc"; @@ -2937,7 +2937,7 @@ reg = <0 0x00fda000 0 0x4000>; clocks = <&gcc GCC_USB3_SEC_PHY_AUX_CLK>, - <&rpmhcc RPMH_CXO_CLK>, + <&tcsr TCSR_USB4_1_CLKREF_EN>, <&gcc GCC_USB3_SEC_PHY_COM_AUX_CLK>, <&gcc GCC_USB3_SEC_PHY_PIPE_CLK>; clock-names = "aux", @@ -3008,7 +3008,7 @@ reg = <0 0x00fdf000 0 0x4000>; clocks = <&gcc GCC_USB3_TERT_PHY_AUX_CLK>, - <&rpmhcc RPMH_CXO_CLK>, + <&tcsr TCSR_USB4_2_CLKREF_EN>, <&gcc GCC_USB3_TERT_PHY_COM_AUX_CLK>, <&gcc GCC_USB3_TERT_PHY_PIPE_CLK>; clock-names = "aux", @@ -5896,9 +5896,11 @@ <0 0x0aec2000 0 0x1c8>; clocks = <&dispcc DISP_CC_MDSS_DPTX2_AUX_CLK>, - <&dispcc DISP_CC_MDSS_AHB_CLK>; + <&dispcc DISP_CC_MDSS_AHB_CLK>, + <&tcsr TCSR_EDP_CLKREF_EN>; clock-names = "aux", - "cfg_ahb"; + "cfg_ahb", + "ref"; power-domains = <&rpmhpd RPMHPD_MX>; @@ -5916,9 +5918,11 @@ <0 0x0aec5000 0 0x1c8>; clocks = <&dispcc DISP_CC_MDSS_DPTX3_AUX_CLK>, - <&dispcc DISP_CC_MDSS_AHB_CLK>; + <&dispcc DISP_CC_MDSS_AHB_CLK>, + <&tcsr TCSR_EDP_CLKREF_EN>; clock-names = "aux", - "cfg_ahb"; + "cfg_ahb", + "ref"; power-domains = <&rpmhpd RPMHPD_MX>; diff --git a/arch/arm64/boot/dts/qcom/monaco.dtsi b/arch/arm64/boot/dts/qcom/monaco.dtsi index 816fa2af8a9a66..a407f80bc5e1f3 100644 --- a/arch/arm64/boot/dts/qcom/monaco.dtsi +++ b/arch/arm64/boot/dts/qcom/monaco.dtsi @@ -757,6 +757,11 @@ hwlocks = <&tcsr_mutex 3>; }; + gunyah_md_mem: gunyah-md-region@91a80000 { + reg = <0x0 0x91a80000 0x0 0x80000>; + no-map; + }; + lpass_machine_learning_mem: lpass-machine-learning-region@93b00000 { reg = <0x0 0x93b00000 0x0 0xf00000>; no-map; @@ -5437,12 +5442,12 @@ }; qup_uart10_rts: qup-uart10-rts-state { - pins = "gpio84"; + pins = "gpio85"; function = "qup1_se2"; }; qup_uart10_tx: qup-uart10-tx-state { - pins = "gpio85"; + pins = "gpio86"; function = "qup1_se2"; }; diff --git a/arch/arm64/boot/dts/qcom/msm8994-msft-lumia-octagon.dtsi b/arch/arm64/boot/dts/qcom/msm8994-msft-lumia-octagon.dtsi index 4c983b10dd9252..7ace3540ef0a0e 100644 --- a/arch/arm64/boot/dts/qcom/msm8994-msft-lumia-octagon.dtsi +++ b/arch/arm64/boot/dts/qcom/msm8994-msft-lumia-octagon.dtsi @@ -378,7 +378,7 @@ status = "okay"; sideinteraction: touch@2c { - compatible = "ad,ad7147_captouch"; + compatible = "adi,ad7147_captouch"; reg = <0x2c>; pinctrl-names = "default", "sleep"; diff --git a/arch/arm64/boot/dts/qcom/qcm6490-idp.dts b/arch/arm64/boot/dts/qcom/qcm6490-idp.dts index 089a027c57d5ca..b2f00e107643dc 100644 --- a/arch/arm64/boot/dts/qcom/qcm6490-idp.dts +++ b/arch/arm64/boot/dts/qcom/qcm6490-idp.dts @@ -177,7 +177,7 @@ pinctrl-0 = <&wcd_default>; pinctrl-names = "default"; - reset-gpios = <&tlmm 83 GPIO_ACTIVE_HIGH>; + reset-gpios = <&tlmm 83 GPIO_ACTIVE_LOW>; vdd-buck-supply = <&vreg_l17b_1p7>; vdd-rxtx-supply = <&vreg_l18b_1p8>; diff --git a/arch/arm64/boot/dts/qcom/qrb4210-rb2.dts b/arch/arm64/boot/dts/qcom/qrb4210-rb2.dts index 0cd36c54632fa3..5f8613150bdd29 100644 --- a/arch/arm64/boot/dts/qcom/qrb4210-rb2.dts +++ b/arch/arm64/boot/dts/qcom/qrb4210-rb2.dts @@ -694,7 +694,7 @@ &uart3 { interrupts-extended = <&intc GIC_SPI 330 IRQ_TYPE_LEVEL_HIGH>, - <&tlmm 11 IRQ_TYPE_LEVEL_HIGH>; + <&tlmm 11 IRQ_TYPE_EDGE_FALLING>; pinctrl-0 = <&uart3_default>; pinctrl-1 = <&uart3_sleep>; pinctrl-names = "default", "sleep"; diff --git a/arch/arm64/boot/dts/qcom/sdm630.dtsi b/arch/arm64/boot/dts/qcom/sdm630.dtsi index 8b1a45a4e56ed1..876a6871745cfc 100644 --- a/arch/arm64/boot/dts/qcom/sdm630.dtsi +++ b/arch/arm64/boot/dts/qcom/sdm630.dtsi @@ -598,8 +598,8 @@ }; gpu_speed_bin: gpu-speed-bin@41a0 { - reg = <0x41a2 0x1>; - bits = <5 7>; + reg = <0x41a2 0x2>; + bits = <5 8>; }; }; @@ -1563,6 +1563,7 @@ reg-names = "mdss_phys", "vbif_phys"; power-domains = <&mmcc MDSS_GDSC>; + resets = <&mmcc MDSS_BCR>; clocks = <&mmcc MDSS_AHB_CLK>, <&mmcc MDSS_AXI_CLK>, diff --git a/arch/arm64/boot/dts/qcom/sdm845-db845c.dts b/arch/arm64/boot/dts/qcom/sdm845-db845c.dts index ce23f87e0316b6..5118b776a9bb37 100644 --- a/arch/arm64/boot/dts/qcom/sdm845-db845c.dts +++ b/arch/arm64/boot/dts/qcom/sdm845-db845c.dts @@ -379,6 +379,12 @@ regulator-initial-mode = ; }; + vreg_l23a_3p3: ldo23 { + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3312000>; + regulator-initial-mode = ; + }; + vreg_l24a_3p075: ldo24 { regulator-min-microvolt = <3088000>; regulator-max-microvolt = <3088000>; @@ -850,7 +856,6 @@ status = "okay"; pinctrl-names = "default"; pinctrl-0 = <&qup_spi0_default>; - cs-gpios = <&tlmm 3 GPIO_ACTIVE_LOW>; can@0 { compatible = "microchip,mcp2517fd"; @@ -1156,6 +1161,7 @@ vdd-1.8-xo-supply = <&vreg_l7a_1p8>; vdd-1.3-rfa-supply = <&vreg_l17a_1p3>; vdd-3.3-ch0-supply = <&vreg_l25a_3p3>; + vdd-3.3-ch1-supply = <&vreg_l23a_3p3>; qcom,snoc-host-cap-8bit-quirk; qcom,calibration-variant = "Thundercomm_DB845C"; diff --git a/arch/arm64/boot/dts/qcom/sdm845-oneplus-common.dtsi b/arch/arm64/boot/dts/qcom/sdm845-oneplus-common.dtsi index db6dd04c51bb5f..8251f5a2f94757 100644 --- a/arch/arm64/boot/dts/qcom/sdm845-oneplus-common.dtsi +++ b/arch/arm64/boot/dts/qcom/sdm845-oneplus-common.dtsi @@ -148,7 +148,6 @@ gpio = <&tlmm 88 0>; enable-active-high; - regulator-boot-on; }; panel_vci_3v3: panel-vci-3v3-regulator { @@ -273,7 +272,7 @@ regulator-min-microvolt = <1800000>; regulator-max-microvolt = <1800000>; regulator-initial-mode = ; - regulator-always-on; + regulator-boot-on; }; vreg_l17a_1p3: ldo17 { diff --git a/arch/arm64/boot/dts/qcom/sdm850-huawei-matebook-e-2019.dts b/arch/arm64/boot/dts/qcom/sdm850-huawei-matebook-e-2019.dts index 0ef9ea38a424a5..f048653818702a 100644 --- a/arch/arm64/boot/dts/qcom/sdm850-huawei-matebook-e-2019.dts +++ b/arch/arm64/boot/dts/qcom/sdm850-huawei-matebook-e-2019.dts @@ -30,9 +30,7 @@ /delete-node/ &ipa_fw_mem; /delete-node/ &ipa_gsi_mem; /delete-node/ &gpu_mem; -/delete-node/ &adsp_mem; /delete-node/ &wlan_msa_mem; -/delete-node/ &slpi_mem; / { model = "Huawei MateBook E 2019"; @@ -145,22 +143,13 @@ no-map; }; - adsp_mem: adsp@8c500000 { - reg = <0 0x8c500000 0 0x1a00000>; - no-map; - }; - ipa_fw_mem: ipa-fw@8df00000 { - reg = <0 0x8df00000 0 0x100000>; + reg = <0 0x8df00000 0 0x5a000>; no-map; }; - slpi_mem: slpi@96700000 { - reg = <0 0x96700000 0 0x1200000>; - }; - - gpu_mem: gpu@97900000 { - reg = <0 0x97900000 0 0x5000>; + gpu_mem: gpu@8df5a000 { + reg = <0 0x8df5a000 0 0x5000>; no-map; }; diff --git a/arch/arm64/boot/dts/qcom/sm6115.dtsi b/arch/arm64/boot/dts/qcom/sm6115.dtsi index 5e2032c26ea388..4dba724f2c756e 100644 --- a/arch/arm64/boot/dts/qcom/sm6115.dtsi +++ b/arch/arm64/boot/dts/qcom/sm6115.dtsi @@ -1715,8 +1715,12 @@ gpu: gpu@5900000 { compatible = "qcom,adreno-610.0", "qcom,adreno"; - reg = <0x0 0x05900000 0x0 0x40000>; - reg-names = "kgsl_3d0_reg_memory"; + reg = <0x0 0x05900000 0x0 0x40000>, + <0x0 0x0599e000 0x0 0x1000>, + <0x0 0x05961000 0x0 0x800>; + reg-names = "kgsl_3d0_reg_memory", + "cx_mem", + "cx_dbgc"; /* There's no (real) GMU, so we have to handle quite a bunch of clocks! */ clocks = <&gpucc GPU_CC_GX_GFX3D_CLK>, diff --git a/arch/arm64/boot/dts/qcom/sm8150-hdk.dts b/arch/arm64/boot/dts/qcom/sm8150-hdk.dts index 0339a572f34d01..1eea9c5c668473 100644 --- a/arch/arm64/boot/dts/qcom/sm8150-hdk.dts +++ b/arch/arm64/boot/dts/qcom/sm8150-hdk.dts @@ -387,6 +387,10 @@ status = "okay"; }; +&gpu_zap_shader { + firmware-name = "qcom/sm8150/a640_zap.mbn"; +}; + &i2c4 { clock-frequency = <100000>; diff --git a/arch/arm64/boot/dts/qcom/sm8150-mtp.dts b/arch/arm64/boot/dts/qcom/sm8150-mtp.dts index 12e8e1ada6d8bd..0f2d511624a8bf 100644 --- a/arch/arm64/boot/dts/qcom/sm8150-mtp.dts +++ b/arch/arm64/boot/dts/qcom/sm8150-mtp.dts @@ -358,6 +358,10 @@ status = "okay"; }; +&gpu_zap_shader { + firmware-name = "qcom/sm8150/a640_zap.mbn"; +}; + &pon { mode-bootloader = <0x2>; mode-recovery = <0x1>; diff --git a/arch/arm64/boot/dts/qcom/sm8250-hdk.dts b/arch/arm64/boot/dts/qcom/sm8250-hdk.dts index f5c193c6c5f9b4..3ea9d2b1a7d581 100644 --- a/arch/arm64/boot/dts/qcom/sm8250-hdk.dts +++ b/arch/arm64/boot/dts/qcom/sm8250-hdk.dts @@ -373,6 +373,10 @@ status = "okay"; }; +&gpu_zap_shader { + firmware-name = "qcom/sm8250/a650_zap.mbn"; +}; + &pon { mode-bootloader = <0x2>; mode-recovery = <0x1>; diff --git a/arch/arm64/boot/dts/qcom/sm8750.dtsi b/arch/arm64/boot/dts/qcom/sm8750.dtsi index 3f0b57f428bbb3..0efbf5e29f0f7d 100644 --- a/arch/arm64/boot/dts/qcom/sm8750.dtsi +++ b/arch/arm64/boot/dts/qcom/sm8750.dtsi @@ -2073,6 +2073,8 @@ <&apps_smmu 0x481 0>; qcom,ee = <0>; + qcom,num-ees = <4>; + num-channels = <20>; qcom,controlled-remotely; }; diff --git a/arch/arm64/boot/dts/qcom/talos.dtsi b/arch/arm64/boot/dts/qcom/talos.dtsi index 95d26e3136229f..3ef21bae317423 100644 --- a/arch/arm64/boot/dts/qcom/talos.dtsi +++ b/arch/arm64/boot/dts/qcom/talos.dtsi @@ -537,7 +537,6 @@ qup_opp_table: opp-table-qup { compatible = "operating-points-v2"; - opp-shared; opp-75000000 { opp-hz = /bits/ 64 <75000000>; diff --git a/arch/arm64/boot/dts/renesas/r8a779g3-sparrow-hawk.dts b/arch/arm64/boot/dts/renesas/r8a779g3-sparrow-hawk.dts index ff07d984cbf299..812b133cf29ed7 100644 --- a/arch/arm64/boot/dts/renesas/r8a779g3-sparrow-hawk.dts +++ b/arch/arm64/boot/dts/renesas/r8a779g3-sparrow-hawk.dts @@ -118,6 +118,17 @@ reg = <0x6 0x00000000 0x1 0x00000000>; }; + reserved-memory { + #address-cells = <2>; + #size-cells = <2>; + ranges; + + tfa@40000000 { + reg = <0x0 0x40000000 0x0 0x8000000>; + no-map; + }; + }; + /* Page 27 / DSI to Display */ dp-con { compatible = "dp-connector"; diff --git a/arch/arm64/boot/dts/renesas/r8a78000.dtsi b/arch/arm64/boot/dts/renesas/r8a78000.dtsi index 4c97298fa76348..3e1c98903cea08 100644 --- a/arch/arm64/boot/dts/renesas/r8a78000.dtsi +++ b/arch/arm64/boot/dts/renesas/r8a78000.dtsi @@ -698,7 +698,7 @@ compatible = "renesas,scif-r8a78000", "renesas,rcar-gen5-scif", "renesas,scif"; reg = <0 0xc0700000 0 0x40>; - interrupts = ; + interrupts = ; clocks = <&dummy_clk_sgasyncd16>, <&dummy_clk_sgasyncd16>, <&scif_clk>; clock-names = "fck", "brg_int", "scif_clk"; status = "disabled"; @@ -708,7 +708,7 @@ compatible = "renesas,scif-r8a78000", "renesas,rcar-gen5-scif", "renesas,scif"; reg = <0 0xc0704000 0 0x40>; - interrupts = ; + interrupts = ; clocks = <&dummy_clk_sgasyncd16>, <&dummy_clk_sgasyncd16>, <&scif_clk>; clock-names = "fck", "brg_int", "scif_clk"; status = "disabled"; @@ -718,7 +718,7 @@ compatible = "renesas,scif-r8a78000", "renesas,rcar-gen5-scif", "renesas,scif"; reg = <0 0xc0708000 0 0x40>; - interrupts = ; + interrupts = ; clocks = <&dummy_clk_sgasyncd16>, <&dummy_clk_sgasyncd16>, <&scif_clk>; clock-names = "fck", "brg_int", "scif_clk"; status = "disabled"; @@ -728,7 +728,7 @@ compatible = "renesas,scif-r8a78000", "renesas,rcar-gen5-scif", "renesas,scif"; reg = <0 0xc070c000 0 0x40>; - interrupts = ; + interrupts = ; clocks = <&dummy_clk_sgasyncd16>, <&dummy_clk_sgasyncd16>, <&scif_clk>; clock-names = "fck", "brg_int", "scif_clk"; status = "disabled"; @@ -738,7 +738,7 @@ compatible = "renesas,hscif-r8a78000", "renesas,rcar-gen5-hscif", "renesas,hscif"; reg = <0 0xc0710000 0 0x60>; - interrupts = ; + interrupts = ; clocks = <&dummy_clk_sgasyncd4>, <&dummy_clk_sgasyncd4>, <&scif_clk>; clock-names = "fck", "brg_int", "scif_clk"; status = "disabled"; @@ -748,7 +748,7 @@ compatible = "renesas,hscif-r8a78000", "renesas,rcar-gen5-hscif", "renesas,hscif"; reg = <0 0xc0714000 0 0x60>; - interrupts = ; + interrupts = ; clocks = <&dummy_clk_sgasyncd4>, <&dummy_clk_sgasyncd4>, <&scif_clk>; clock-names = "fck", "brg_int", "scif_clk"; status = "disabled"; @@ -758,7 +758,7 @@ compatible = "renesas,hscif-r8a78000", "renesas,rcar-gen5-hscif", "renesas,hscif"; reg = <0 0xc0718000 0 0x60>; - interrupts = ; + interrupts = ; clocks = <&dummy_clk_sgasyncd4>, <&dummy_clk_sgasyncd4>, <&scif_clk>; clock-names = "fck", "brg_int", "scif_clk"; status = "disabled"; @@ -768,7 +768,7 @@ compatible = "renesas,hscif-r8a78000", "renesas,rcar-gen5-hscif", "renesas,hscif"; reg = <0 0xc071c000 0 0x60>; - interrupts = ; + interrupts = ; clocks = <&dummy_clk_sgasyncd4>, <&dummy_clk_sgasyncd4>, <&scif_clk>; clock-names = "fck", "brg_int", "scif_clk"; status = "disabled"; diff --git a/arch/arm64/boot/dts/renesas/r9a09g047e57-smarc.dts b/arch/arm64/boot/dts/renesas/r9a09g047e57-smarc.dts index 08e814c03fa855..ed6fcdc337a0bd 100644 --- a/arch/arm64/boot/dts/renesas/r9a09g047e57-smarc.dts +++ b/arch/arm64/boot/dts/renesas/r9a09g047e57-smarc.dts @@ -8,7 +8,6 @@ /dts-v1/; /* Switch selection settings */ -#define SW_LCD_EN 0 #define SW_GPIO8_CAN0_STB 0 #define SW_GPIO9_CAN1_STB 0 #define SW_LCD_EN 0 diff --git a/arch/arm64/boot/dts/renesas/r9a09g057.dtsi b/arch/arm64/boot/dts/renesas/r9a09g057.dtsi index 4df32d7e999818..3d7f4dae5c1957 100644 --- a/arch/arm64/boot/dts/renesas/r9a09g057.dtsi +++ b/arch/arm64/boot/dts/renesas/r9a09g057.dtsi @@ -581,16 +581,6 @@ status = "disabled"; }; - wdt0: watchdog@11c00400 { - compatible = "renesas,r9a09g057-wdt"; - reg = <0 0x11c00400 0 0x400>; - clocks = <&cpg CPG_MOD 0x4b>, <&cpg CPG_MOD 0x4c>; - clock-names = "pclk", "oscclk"; - resets = <&cpg 0x75>; - power-domains = <&cpg>; - status = "disabled"; - }; - wdt1: watchdog@14400000 { compatible = "renesas,r9a09g057-wdt"; reg = <0 0x14400000 0 0x400>; @@ -601,26 +591,6 @@ status = "disabled"; }; - wdt2: watchdog@13000000 { - compatible = "renesas,r9a09g057-wdt"; - reg = <0 0x13000000 0 0x400>; - clocks = <&cpg CPG_MOD 0x4f>, <&cpg CPG_MOD 0x50>; - clock-names = "pclk", "oscclk"; - resets = <&cpg 0x77>; - power-domains = <&cpg>; - status = "disabled"; - }; - - wdt3: watchdog@13000400 { - compatible = "renesas,r9a09g057-wdt"; - reg = <0 0x13000400 0 0x400>; - clocks = <&cpg CPG_MOD 0x51>, <&cpg CPG_MOD 0x52>; - clock-names = "pclk", "oscclk"; - resets = <&cpg 0x78>; - power-domains = <&cpg>; - status = "disabled"; - }; - rtc: rtc@11c00800 { compatible = "renesas,r9a09g057-rtca3", "renesas,rz-rtca3"; reg = <0 0x11c00800 0 0x400>; diff --git a/arch/arm64/boot/dts/renesas/r9a09g077.dtsi b/arch/arm64/boot/dts/renesas/r9a09g077.dtsi index f5fa6ca0640972..5f4d30f75cbde4 100644 --- a/arch/arm64/boot/dts/renesas/r9a09g077.dtsi +++ b/arch/arm64/boot/dts/renesas/r9a09g077.dtsi @@ -747,8 +747,8 @@ cpg: clock-controller@80280000 { compatible = "renesas,r9a09g077-cpg-mssr"; - reg = <0 0x80280000 0 0x1000>, - <0 0x81280000 0 0x9000>; + reg = <0 0x80280000 0 0x10000>, + <0 0x81280000 0 0x10000>; clocks = <&extal_clk>; clock-names = "extal"; #clock-cells = <2>; diff --git a/arch/arm64/boot/dts/renesas/r9a09g087.dtsi b/arch/arm64/boot/dts/renesas/r9a09g087.dtsi index 361a9235f00d94..46f2b1fd98dc3f 100644 --- a/arch/arm64/boot/dts/renesas/r9a09g087.dtsi +++ b/arch/arm64/boot/dts/renesas/r9a09g087.dtsi @@ -750,8 +750,8 @@ cpg: clock-controller@80280000 { compatible = "renesas,r9a09g087-cpg-mssr"; - reg = <0 0x80280000 0 0x1000>, - <0 0x81280000 0 0x9000>; + reg = <0 0x80280000 0 0x10000>, + <0 0x81280000 0 0x10000>; clocks = <&extal_clk>; clock-names = "extal"; #clock-cells = <2>; diff --git a/arch/arm64/boot/dts/renesas/rzg3s-smarc-som.dtsi b/arch/arm64/boot/dts/renesas/rzg3s-smarc-som.dtsi index 6f25ab6179829e..fbfa6cfb192975 100644 --- a/arch/arm64/boot/dts/renesas/rzg3s-smarc-som.dtsi +++ b/arch/arm64/boot/dts/renesas/rzg3s-smarc-som.dtsi @@ -162,7 +162,7 @@ <100000000>; renesas,settings = [ 80 00 11 19 4c 42 dc 2f 06 7d 20 1a 5f 1e f2 27 - 00 40 00 00 00 00 00 00 06 0c 19 02 3f f0 90 86 + 00 40 00 00 00 00 00 00 06 0c 19 02 3b f0 90 86 a0 80 30 30 9c ]; }; diff --git a/arch/arm64/boot/dts/renesas/rzt2h-n2h-evk-common.dtsi b/arch/arm64/boot/dts/renesas/rzt2h-n2h-evk-common.dtsi index 3eed1f3948e8ea..890e4ddc1e78b0 100644 --- a/arch/arm64/boot/dts/renesas/rzt2h-n2h-evk-common.dtsi +++ b/arch/arm64/boot/dts/renesas/rzt2h-n2h-evk-common.dtsi @@ -53,6 +53,7 @@ regulator-max-microvolt = <3300000>; gpios-states = <0>; states = <3300000 0>, <1800000 1>; + regulator-ramp-delay = <60>; }; #endif @@ -224,8 +225,7 @@ ctrl-pins { pinmux = , /* SD0_CLK */ , /* SD0_CMD */ - , /* SD0_CD */ - ; /* SD0_WP */ + ; /* SD0_CD */ }; }; @@ -282,6 +282,7 @@ pinctrl-names = "default", "state_uhs"; vmmc-supply = <®_3p3v>; vqmmc-supply = <&vqmmc_sdhi0>; + wp-gpios = <&pinctrl RZT2H_GPIO(22, 6) GPIO_ACTIVE_HIGH>; bus-width = <4>; sd-uhs-sdr50; sd-uhs-sdr104; diff --git a/arch/arm64/boot/dts/renesas/rzv2-evk-cn15-sd.dtso b/arch/arm64/boot/dts/renesas/rzv2-evk-cn15-sd.dtso index 0af1e0a6c7f482..fc53c1aae3b522 100644 --- a/arch/arm64/boot/dts/renesas/rzv2-evk-cn15-sd.dtso +++ b/arch/arm64/boot/dts/renesas/rzv2-evk-cn15-sd.dtso @@ -25,6 +25,7 @@ regulator-max-microvolt = <3300000>; gpios-states = <0>; states = <3300000 0>, <1800000 1>; + regulator-ramp-delay = <60>; }; }; diff --git a/arch/arm64/boot/dts/rockchip/rk3399-pinebook-pro.dts b/arch/arm64/boot/dts/rockchip/rk3399-pinebook-pro.dts index 810ab6ff4e670b..ae937a3afa1136 100644 --- a/arch/arm64/boot/dts/rockchip/rk3399-pinebook-pro.dts +++ b/arch/arm64/boot/dts/rockchip/rk3399-pinebook-pro.dts @@ -421,10 +421,6 @@ status = "okay"; }; -&hdmi_sound { - status = "okay"; -}; - &i2c0 { clock-frequency = <400000>; i2c-scl-falling-time-ns = <4>; @@ -883,12 +879,6 @@ }; }; - wifi { - wifi_host_wake_l: wifi-host-wake-l { - rockchip,pins = <0 RK_PA3 RK_FUNC_GPIO &pcfg_pull_none>; - }; - }; - wireless-bluetooth { bt_wake_pin: bt-wake-pin { rockchip,pins = <2 RK_PD3 RK_FUNC_GPIO &pcfg_pull_none>; @@ -946,19 +936,7 @@ pinctrl-names = "default"; pinctrl-0 = <&sdio0_bus4 &sdio0_cmd &sdio0_clk>; sd-uhs-sdr104; - #address-cells = <1>; - #size-cells = <0>; status = "okay"; - - brcmf: wifi@1 { - compatible = "brcm,bcm4329-fmac"; - reg = <1>; - interrupt-parent = <&gpio0>; - interrupts = ; - interrupt-names = "host-wake"; - pinctrl-names = "default"; - pinctrl-0 = <&wifi_host_wake_l>; - }; }; &sdhci { diff --git a/arch/arm64/boot/dts/rockchip/rk3568.dtsi b/arch/arm64/boot/dts/rockchip/rk3568.dtsi index e719a3df126c59..658097ed69714a 100644 --- a/arch/arm64/boot/dts/rockchip/rk3568.dtsi +++ b/arch/arm64/boot/dts/rockchip/rk3568.dtsi @@ -185,7 +185,7 @@ <0x0 0xf2000000 0x0 0x00100000>; ranges = <0x01000000 0x0 0xf2100000 0x0 0xf2100000 0x0 0x00100000>, <0x02000000 0x0 0xf2200000 0x0 0xf2200000 0x0 0x01e00000>, - <0x03000000 0x0 0x40000000 0x3 0x40000000 0x0 0x40000000>; + <0x03000000 0x3 0x40000000 0x3 0x40000000 0x0 0x40000000>; reg-names = "dbi", "apb", "config"; resets = <&cru SRST_PCIE30X1_POWERUP>; reset-names = "pipe"; @@ -238,7 +238,7 @@ <0x0 0xf0000000 0x0 0x00100000>; ranges = <0x01000000 0x0 0xf0100000 0x0 0xf0100000 0x0 0x00100000>, <0x02000000 0x0 0xf0200000 0x0 0xf0200000 0x0 0x01e00000>, - <0x03000000 0x0 0x40000000 0x3 0x80000000 0x0 0x40000000>; + <0x03000000 0x3 0x80000000 0x3 0x80000000 0x0 0x40000000>; reg-names = "dbi", "apb", "config"; resets = <&cru SRST_PCIE30X2_POWERUP>; reset-names = "pipe"; diff --git a/arch/arm64/boot/dts/rockchip/rk356x-base.dtsi b/arch/arm64/boot/dts/rockchip/rk356x-base.dtsi index 8893b7b6cc9ff3..a2c4957a589921 100644 --- a/arch/arm64/boot/dts/rockchip/rk356x-base.dtsi +++ b/arch/arm64/boot/dts/rockchip/rk356x-base.dtsi @@ -1022,7 +1022,7 @@ power-domains = <&power RK3568_PD_PIPE>; ranges = <0x01000000 0x0 0xf4100000 0x0 0xf4100000 0x0 0x00100000>, <0x02000000 0x0 0xf4200000 0x0 0xf4200000 0x0 0x01e00000>, - <0x03000000 0x0 0x40000000 0x3 0x00000000 0x0 0x40000000>; + <0x03000000 0x3 0x00000000 0x3 0x00000000 0x0 0x40000000>; resets = <&cru SRST_PCIE20_POWERUP>; reset-names = "pipe"; #address-cells = <3>; diff --git a/arch/arm64/boot/dts/rockchip/rk3576-evb1-v10.dts b/arch/arm64/boot/dts/rockchip/rk3576-evb1-v10.dts index db8fef7a4f1b95..ffe55f970f4615 100644 --- a/arch/arm64/boot/dts/rockchip/rk3576-evb1-v10.dts +++ b/arch/arm64/boot/dts/rockchip/rk3576-evb1-v10.dts @@ -223,6 +223,18 @@ vin-supply = <&vcc_3v3_s3>; }; + vcc3v3_sd: regulator-vcc-3v3-sd { + compatible = "regulator-fixed"; + enable-active-high; + gpios = <&gpio0 RK_PB6 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&sdmmc_pwren>; + regulator-name = "vcc3v3_sd"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + vin-supply = <&vcc_3v3_s0>; + }; + vcc_ufs_s0: regulator-vcc-ufs-s0 { compatible = "regulator-fixed"; regulator-name = "vcc_ufs_s0"; @@ -810,6 +822,12 @@ }; }; + sdmmc { + sdmmc_pwren: sdmmc-pwren { + rockchip,pins = <0 RK_PB6 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + usb { usb_host_pwren: usb-host-pwren { rockchip,pins = <0 RK_PC7 RK_FUNC_GPIO &pcfg_pull_none>; @@ -851,11 +869,15 @@ bus-width = <4>; cap-mmc-highspeed; cap-sd-highspeed; + cd-gpios = <&gpio0 RK_PA7 GPIO_ACTIVE_LOW>; disable-wp; max-frequency = <200000000>; no-sdio; no-mmc; + pinctrl-names = "default"; + pinctrl-0 = <&sdmmc0_clk &sdmmc0_cmd &sdmmc0_det &sdmmc0_bus4>; sd-uhs-sdr104; + vmmc-supply = <&vcc3v3_sd>; vqmmc-supply = <&vccio_sd_s0>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/rockchip/rk3576-nanopi-r76s.dts b/arch/arm64/boot/dts/rockchip/rk3576-nanopi-r76s.dts index 31fbefaeceab49..7ec27b05ff10e6 100644 --- a/arch/arm64/boot/dts/rockchip/rk3576-nanopi-r76s.dts +++ b/arch/arm64/boot/dts/rockchip/rk3576-nanopi-r76s.dts @@ -192,6 +192,18 @@ regulator-name = "vcc_3v3_s0"; vin-supply = <&vcc_3v3_s3>; }; + + vcc3v3_sd: regulator-vcc-3v3-sd { + compatible = "regulator-fixed"; + enable-active-high; + gpios = <&gpio0 RK_PB6 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&sdmmc_pwren>; + regulator-name = "vcc3v3_sd"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + vin-supply = <&vcc_3v3_s0>; + }; }; &combphy0_ps { @@ -726,6 +738,12 @@ }; }; + sdmmc { + sdmmc_pwren: sdmmc-pwren { + rockchip,pins = <0 RK_PB6 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + usb { usb_otg0_pwren_h: usb-otg0-pwren-h { rockchip,pins = <0 RK_PD1 RK_FUNC_GPIO &pcfg_pull_none>; @@ -751,11 +769,14 @@ bus-width = <4>; cap-mmc-highspeed; cap-sd-highspeed; + cd-gpios = <&gpio0 RK_PA7 GPIO_ACTIVE_LOW>; disable-wp; no-mmc; no-sdio; + pinctrl-names = "default"; + pinctrl-0 = <&sdmmc0_clk &sdmmc0_cmd &sdmmc0_det &sdmmc0_bus4>; sd-uhs-sdr104; - vmmc-supply = <&vcc_3v3_s3>; + vmmc-supply = <&vcc3v3_sd>; vqmmc-supply = <&vccio_sd_s0>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/rockchip/rk3576-pinctrl.dtsi b/arch/arm64/boot/dts/rockchip/rk3576-pinctrl.dtsi index 0b0851a7e4ea9e..98c9f8013158ca 100644 --- a/arch/arm64/boot/dts/rockchip/rk3576-pinctrl.dtsi +++ b/arch/arm64/boot/dts/rockchip/rk3576-pinctrl.dtsi @@ -5228,6 +5228,13 @@ /* ufs_rstn */ <4 RK_PD0 1 &pcfg_pull_none>; }; + + /omit-if-no-ref/ + ufs_rstgpio: ufs-rstgpio { + rockchip,pins = + /* ufs_rstn */ + <4 RK_PD0 RK_FUNC_GPIO &pcfg_pull_down>; + }; }; ufs_testdata0 { diff --git a/arch/arm64/boot/dts/rockchip/rk3576.dtsi b/arch/arm64/boot/dts/rockchip/rk3576.dtsi index c72343e7a0456e..70e67d4dccb8a9 100644 --- a/arch/arm64/boot/dts/rockchip/rk3576.dtsi +++ b/arch/arm64/boot/dts/rockchip/rk3576.dtsi @@ -1826,7 +1826,7 @@ assigned-clock-parents = <&cru CLK_REF_MPHY_26M>; interrupts = ; power-domains = <&power RK3576_PD_USB>; - pinctrl-0 = <&ufs_refclk>; + pinctrl-0 = <&ufs_refclk &ufs_rstgpio>; pinctrl-names = "default"; resets = <&cru SRST_A_UFS_BIU>, <&cru SRST_A_UFS_SYS>, <&cru SRST_A_UFS>, <&cru SRST_P_UFS_GRF>; diff --git a/arch/arm64/boot/dts/rockchip/rk3588-base.dtsi b/arch/arm64/boot/dts/rockchip/rk3588-base.dtsi index 7ab12d1054a73b..fdb017258b7bc8 100644 --- a/arch/arm64/boot/dts/rockchip/rk3588-base.dtsi +++ b/arch/arm64/boot/dts/rockchip/rk3588-base.dtsi @@ -1955,7 +1955,7 @@ power-domains = <&power RK3588_PD_PCIE>; ranges = <0x01000000 0x0 0xf3100000 0x0 0xf3100000 0x0 0x00100000>, <0x02000000 0x0 0xf3200000 0x0 0xf3200000 0x0 0x00e00000>, - <0x03000000 0x0 0x40000000 0x9 0xc0000000 0x0 0x40000000>; + <0x03000000 0x9 0xc0000000 0x9 0xc0000000 0x0 0x40000000>; reg = <0xa 0x40c00000 0x0 0x00400000>, <0x0 0xfe180000 0x0 0x00010000>, <0x0 0xf3000000 0x0 0x00100000>; @@ -2007,7 +2007,7 @@ power-domains = <&power RK3588_PD_PCIE>; ranges = <0x01000000 0x0 0xf4100000 0x0 0xf4100000 0x0 0x00100000>, <0x02000000 0x0 0xf4200000 0x0 0xf4200000 0x0 0x00e00000>, - <0x03000000 0x0 0x40000000 0xa 0x00000000 0x0 0x40000000>; + <0x03000000 0xa 0x00000000 0xa 0x00000000 0x0 0x40000000>; reg = <0xa 0x41000000 0x0 0x00400000>, <0x0 0xfe190000 0x0 0x00010000>, <0x0 0xf4000000 0x0 0x00100000>; diff --git a/arch/arm64/boot/dts/rockchip/rk3588-extra.dtsi b/arch/arm64/boot/dts/rockchip/rk3588-extra.dtsi index 6e5a58428bbabd..a2640014ee0421 100644 --- a/arch/arm64/boot/dts/rockchip/rk3588-extra.dtsi +++ b/arch/arm64/boot/dts/rockchip/rk3588-extra.dtsi @@ -375,7 +375,7 @@ power-domains = <&power RK3588_PD_PCIE>; ranges = <0x01000000 0x0 0xf0100000 0x0 0xf0100000 0x0 0x00100000>, <0x02000000 0x0 0xf0200000 0x0 0xf0200000 0x0 0x00e00000>, - <0x03000000 0x0 0x40000000 0x9 0x00000000 0x0 0x40000000>; + <0x03000000 0x9 0x00000000 0x9 0x00000000 0x0 0x40000000>; reg = <0xa 0x40000000 0x0 0x00400000>, <0x0 0xfe150000 0x0 0x00010000>, <0x0 0xf0000000 0x0 0x00100000>; @@ -462,7 +462,7 @@ power-domains = <&power RK3588_PD_PCIE>; ranges = <0x01000000 0x0 0xf1100000 0x0 0xf1100000 0x0 0x00100000>, <0x02000000 0x0 0xf1200000 0x0 0xf1200000 0x0 0x00e00000>, - <0x03000000 0x0 0x40000000 0x9 0x40000000 0x0 0x40000000>; + <0x03000000 0x9 0x40000000 0x9 0x40000000 0x0 0x40000000>; reg = <0xa 0x40400000 0x0 0x00400000>, <0x0 0xfe160000 0x0 0x00010000>, <0x0 0xf1000000 0x0 0x00100000>; @@ -512,7 +512,7 @@ power-domains = <&power RK3588_PD_PCIE>; ranges = <0x01000000 0x0 0xf2100000 0x0 0xf2100000 0x0 0x00100000>, <0x02000000 0x0 0xf2200000 0x0 0xf2200000 0x0 0x00e00000>, - <0x03000000 0x0 0x40000000 0x9 0x80000000 0x0 0x40000000>; + <0x03000000 0x9 0x80000000 0x9 0x80000000 0x0 0x40000000>; reg = <0xa 0x40800000 0x0 0x00400000>, <0x0 0xfe170000 0x0 0x00010000>, <0x0 0xf2000000 0x0 0x00100000>; diff --git a/arch/arm64/boot/dts/ti/k3-am62p-verdin.dtsi b/arch/arm64/boot/dts/ti/k3-am62p-verdin.dtsi index 5e050cbb9eaf3b..ec9dd931fe9227 100644 --- a/arch/arm64/boot/dts/ti/k3-am62p-verdin.dtsi +++ b/arch/arm64/boot/dts/ti/k3-am62p-verdin.dtsi @@ -112,7 +112,7 @@ regulator-max-microvolt = <3300000>; regulator-min-microvolt = <3300000>; regulator-name = "+V3.3_SD"; - startup-delay-us = <2000>; + startup-delay-us = <20000>; }; reg_sd1_vqmmc: regulator-sdhci1-vqmmc { diff --git a/arch/arm64/boot/dts/ti/k3-am67a-kontron-sa67-base.dts b/arch/arm64/boot/dts/ti/k3-am67a-kontron-sa67-base.dts index 7169d934adac58..95234c8460ed0d 100644 --- a/arch/arm64/boot/dts/ti/k3-am67a-kontron-sa67-base.dts +++ b/arch/arm64/boot/dts/ti/k3-am67a-kontron-sa67-base.dts @@ -85,8 +85,7 @@ linux,cma { compatible = "shared-dma-pool"; reusable; - size = <0x10000000>; - alignment = <0x2000>; + size = <0x00 0x10000000>; linux,cma-default; }; @@ -174,6 +173,7 @@ regulator-max-microvolt = <3300000>; vin-supply = <&vcc_3p3_s0>; regulator-boot-on; + enable-active-high; enable-gpios = <&main_gpio0 7 GPIO_ACTIVE_HIGH>; gpios = <&main_gpio0 8 GPIO_ACTIVE_HIGH>; states = <3300000 0x0>, diff --git a/arch/arm64/boot/dts/ti/k3-am69-aquila-clover.dts b/arch/arm64/boot/dts/ti/k3-am69-aquila-clover.dts index 55fd214a82e44c..ec8ff458771574 100644 --- a/arch/arm64/boot/dts/ti/k3-am69-aquila-clover.dts +++ b/arch/arm64/boot/dts/ti/k3-am69-aquila-clover.dts @@ -208,7 +208,8 @@ pinctrl-0 = <&pinctrl_main_spi2>, <&pinctrl_main_spi2_cs0>, <&pinctrl_gpio_05>; - cs-gpios = <0>, <&wkup_gpio0 29 GPIO_ACTIVE_LOW>; + cs-gpios = <&main_gpio0 39 GPIO_ACTIVE_LOW>, + <&wkup_gpio0 29 GPIO_ACTIVE_LOW>; status = "okay"; tpm@1 { @@ -280,8 +281,8 @@ try-power-role = "sink"; self-powered; source-pdos = ; - sink-pdos = ; - op-sink-microwatt = <1000000>; + sink-pdos = ; + op-sink-microwatt = <0>; ports { #address-cells = <1>; diff --git a/arch/arm64/boot/dts/ti/k3-am69-aquila-dev.dts b/arch/arm64/boot/dts/ti/k3-am69-aquila-dev.dts index c7ce804eac7038..f48601ae38b7c7 100644 --- a/arch/arm64/boot/dts/ti/k3-am69-aquila-dev.dts +++ b/arch/arm64/boot/dts/ti/k3-am69-aquila-dev.dts @@ -399,8 +399,8 @@ try-power-role = "sink"; self-powered; source-pdos = ; - sink-pdos = ; - op-sink-microwatt = <1000000>; + sink-pdos = ; + op-sink-microwatt = <0>; ports { #address-cells = <1>; diff --git a/arch/arm64/boot/dts/ti/k3-am69-aquila.dtsi b/arch/arm64/boot/dts/ti/k3-am69-aquila.dtsi index 0866eb8a6f3482..5119baf62a4c23 100644 --- a/arch/arm64/boot/dts/ti/k3-am69-aquila.dtsi +++ b/arch/arm64/boot/dts/ti/k3-am69-aquila.dtsi @@ -479,7 +479,7 @@ /* Aquila SPI_2 CS */ pinctrl_main_spi0_cs0: main-spi0-cs0-default-pins { pinctrl-single,pins = < - J784S4_IOPAD(0x0cc, PIN_OUTPUT, 0) /* (AM37) SPI0_CS0 */ /* AQUILA D16 */ + J784S4_IOPAD(0x0cc, PIN_OUTPUT, 7) /* (AM37) SPI0_CS0.GPIO0_51 */ /* AQUILA D16 */ >; }; @@ -495,7 +495,7 @@ /* Aquila SPI_1 CS */ pinctrl_main_spi2_cs0: main-spi2-cs0-default-pins { pinctrl-single,pins = < - J784S4_IOPAD(0x09c, PIN_OUTPUT, 10) /* (AF35) MCASP0_AXR11.SPI2_CS1 */ /* AQUILA D9 */ + J784S4_IOPAD(0x09c, PIN_OUTPUT, 7) /* (AF35) MCASP0_AXR11.GPIO0_39 */ /* AQUILA D9 */ >; }; @@ -1204,6 +1204,7 @@ &main_spi0 { pinctrl-names = "default"; pinctrl-0 = <&pinctrl_main_spi0>, <&pinctrl_main_spi0_cs0>; + cs-gpios = <&main_gpio0 51 GPIO_ACTIVE_LOW>; status = "disabled"; }; @@ -1211,6 +1212,7 @@ &main_spi2 { pinctrl-names = "default"; pinctrl-0 = <&pinctrl_main_spi2>, <&pinctrl_main_spi2_cs0>; + cs-gpios = <&main_gpio0 39 GPIO_ACTIVE_LOW>; status = "disabled"; }; diff --git a/arch/arm64/boot/dts/ti/k3-j784s4-j742s2-main-common.dtsi b/arch/arm64/boot/dts/ti/k3-j784s4-j742s2-main-common.dtsi index 9cc0901d58fbf9..c2636e624f18ba 100644 --- a/arch/arm64/boot/dts/ti/k3-j784s4-j742s2-main-common.dtsi +++ b/arch/arm64/boot/dts/ti/k3-j784s4-j742s2-main-common.dtsi @@ -2378,42 +2378,6 @@ assigned-clock-parents = <&k3_clks 351 4>; }; - watchdog4: watchdog@2240000 { - compatible = "ti,j7-rti-wdt"; - reg = <0x00 0x2240000 0x00 0x100>; - clocks = <&k3_clks 352 0>; - power-domains = <&k3_pds 352 TI_SCI_PD_EXCLUSIVE>; - assigned-clocks = <&k3_clks 352 0>; - assigned-clock-parents = <&k3_clks 352 4>; - }; - - watchdog5: watchdog@2250000 { - compatible = "ti,j7-rti-wdt"; - reg = <0x00 0x2250000 0x00 0x100>; - clocks = <&k3_clks 353 0>; - power-domains = <&k3_pds 353 TI_SCI_PD_EXCLUSIVE>; - assigned-clocks = <&k3_clks 353 0>; - assigned-clock-parents = <&k3_clks 353 4>; - }; - - watchdog6: watchdog@2260000 { - compatible = "ti,j7-rti-wdt"; - reg = <0x00 0x2260000 0x00 0x100>; - clocks = <&k3_clks 354 0>; - power-domains = <&k3_pds 354 TI_SCI_PD_EXCLUSIVE>; - assigned-clocks = <&k3_clks 354 0>; - assigned-clock-parents = <&k3_clks 354 4>; - }; - - watchdog7: watchdog@2270000 { - compatible = "ti,j7-rti-wdt"; - reg = <0x00 0x2270000 0x00 0x100>; - clocks = <&k3_clks 355 0>; - power-domains = <&k3_pds 355 TI_SCI_PD_EXCLUSIVE>; - assigned-clocks = <&k3_clks 355 0>; - assigned-clock-parents = <&k3_clks 355 4>; - }; - /* * The following RTI instances are coupled with MCU R5Fs, c7x and * GPU so keeping them reserved as these will be used by their diff --git a/arch/arm64/boot/dts/ti/k3-j784s4-main.dtsi b/arch/arm64/boot/dts/ti/k3-j784s4-main.dtsi index 0160fe0da98388..78fcd0c40abcfa 100644 --- a/arch/arm64/boot/dts/ti/k3-j784s4-main.dtsi +++ b/arch/arm64/boot/dts/ti/k3-j784s4-main.dtsi @@ -6,17 +6,40 @@ */ &cbass_main { - c71_3: dsp@67800000 { - compatible = "ti,j721s2-c71-dsp"; - reg = <0x00 0x67800000 0x00 0x00080000>, - <0x00 0x67e00000 0x00 0x0000c000>; - reg-names = "l2sram", "l1dram"; - resets = <&k3_reset 40 1>; - firmware-name = "j784s4-c71_3-fw"; - ti,sci = <&sms>; - ti,sci-dev-id = <40>; - ti,sci-proc-ids = <0x33 0xff>; - status = "disabled"; + watchdog4: watchdog@2240000 { + compatible = "ti,j7-rti-wdt"; + reg = <0x00 0x2240000 0x00 0x100>; + clocks = <&k3_clks 352 0>; + power-domains = <&k3_pds 352 TI_SCI_PD_EXCLUSIVE>; + assigned-clocks = <&k3_clks 352 0>; + assigned-clock-parents = <&k3_clks 352 4>; + }; + + watchdog5: watchdog@2250000 { + compatible = "ti,j7-rti-wdt"; + reg = <0x00 0x2250000 0x00 0x100>; + clocks = <&k3_clks 353 0>; + power-domains = <&k3_pds 353 TI_SCI_PD_EXCLUSIVE>; + assigned-clocks = <&k3_clks 353 0>; + assigned-clock-parents = <&k3_clks 353 4>; + }; + + watchdog6: watchdog@2260000 { + compatible = "ti,j7-rti-wdt"; + reg = <0x00 0x2260000 0x00 0x100>; + clocks = <&k3_clks 354 0>; + power-domains = <&k3_pds 354 TI_SCI_PD_EXCLUSIVE>; + assigned-clocks = <&k3_clks 354 0>; + assigned-clock-parents = <&k3_clks 354 4>; + }; + + watchdog7: watchdog@2270000 { + compatible = "ti,j7-rti-wdt"; + reg = <0x00 0x2270000 0x00 0x100>; + clocks = <&k3_clks 355 0>; + power-domains = <&k3_pds 355 TI_SCI_PD_EXCLUSIVE>; + assigned-clocks = <&k3_clks 355 0>; + assigned-clock-parents = <&k3_clks 355 4>; }; pcie2_rc: pcie@2920000 { @@ -113,6 +136,19 @@ status = "disabled"; }; }; + + c71_3: dsp@67800000 { + compatible = "ti,j721s2-c71-dsp"; + reg = <0x00 0x67800000 0x00 0x00080000>, + <0x00 0x67e00000 0x00 0x0000c000>; + reg-names = "l2sram", "l1dram"; + resets = <&k3_reset 40 1>; + firmware-name = "j784s4-c71_3-fw"; + ti,sci = <&sms>; + ti,sci-dev-id = <40>; + ti,sci-proc-ids = <0x33 0xff>; + status = "disabled"; + }; }; &scm_conf { diff --git a/arch/arm64/boot/dts/xilinx/zynqmp.dtsi b/arch/arm64/boot/dts/xilinx/zynqmp.dtsi index 938b014ca9231d..b55c6b2e8e0e10 100644 --- a/arch/arm64/boot/dts/xilinx/zynqmp.dtsi +++ b/arch/arm64/boot/dts/xilinx/zynqmp.dtsi @@ -192,11 +192,6 @@ }; firmware { - optee: optee { - compatible = "linaro,optee-tz"; - method = "smc"; - }; - zynqmp_firmware: zynqmp-firmware { compatible = "xlnx,zynqmp-firmware"; #power-domain-cells = <1>; diff --git a/arch/arm64/configs/asahi.config b/arch/arm64/configs/asahi.config new file mode 100644 index 00000000000000..c76f2c727fb0dd --- /dev/null +++ b/arch/arm64/configs/asahi.config @@ -0,0 +1,77 @@ +CONFIG_RUST=y +CONFIG_ARM64_ACTLR_STATE=y +CONFIG_ARCH_APPLE=y +# CONFIG_ARM64_4K_PAGES is not set +CONFIG_ARM64_16K_PAGES=y +# CONFIG_ARM64_64K_PAGES is not set +CONFIG_ARM64_MEMORY_MODEL_CONTROL=y +CONFIG_ARM_APPLE_CPUIDLE=y +CONFIG_ARM_APPLE_SOC_CPUFREQ=m +CONFIG_BT_BRCMEXT=y +CONFIG_BT_HCIBCM4377=m +CONFIG_PCIE_APPLE=m +CONFIG_NVME_APPLE=m +CONFIG_BRCMFMAC=m +CONFIG_BRCMFMAC_PCIE=y +CONFIG_TOUCHSCREEN_APPLE_Z2=m +CONFIG_INPUT_MACSMC_INPUT=m +CONFIG_I2C_APPLE=m +CONFIG_SPI_APPLE=m +CONFIG_SPMI_APPLE=m +CONFIG_PINCTRL_APPLE_GPIO=m +CONFIG_GPIO_MACSMC=m +CONFIG_POWER_RESET_MACSMC=m +CONFIG_CHARGER_MACSMC=m +CONFIG_SENSORS_MACSMC_HWMON=m +CONFIG_APPLE_WATCHDOG=m +CONFIG_VIDEO_APPLE_ISP=m +CONFIG_DRM=y +CONFIG_DRM_ASAHI=m +CONFIG_DRM_ADP=m +CONFIG_DRM_APPLE=m +CONFIG_DRM_APPLE_AUDIO=y +CONFIG_SND_SOC_APPLE_AOP_AUDIO=m +CONFIG_SND_SOC_APPLE_MCA=m +CONFIG_SND_SOC_APPLE_MACAUDIO=m +CONFIG_SND_SOC_CS42L83=m +CONFIG_SND_SOC_CS42L84=m +CONFIG_SND_SOC_TAS2764=m +CONFIG_SND_SOC_TAS2770=m +CONFIG_HID_APPLE=m +CONFIG_HID_MAGICMOUSE=m +CONFIG_SERIAL_SAMSUNG=y +CONFIG_SERIAL_SAMSUNG_CONSOLE=y +CONFIG_HID_DOCKCHANNEL=m +CONFIG_SPI_HID_APPLE_OF=m +CONFIG_SPI_HID_APPLE_CORE=m +CONFIG_USB_DWC3_APPLE=m +CONFIG_USB_XHCI_PCI_ASMEDIA=y +CONFIG_RTC_DRV_MACSMC=m +CONFIG_APPLE_ADMAC=m +CONFIG_APPLE_SIO=m +CONFIG_MFD_MACSMC=m +CONFIG_COMMON_CLK_APPLE_NCO=m +CONFIG_APPLE_DART=m +CONFIG_APPLE_DOCKCHANNEL=m +CONFIG_APPLE_MAILBOX=y +CONFIG_APPLE_PMGR_MISC=y +CONFIG_APPLE_RTKIT=y +CONFIG_APPLE_RTKIT_HELPER=m +CONFIG_APPLE_SART=m +CONFIG_RUST_APPLE_RTKIT=y +CONFIG_APPLE_AOP=m +CONFIG_APPLE_SEP=m +CONFIG_APPLE_PMGR_PWRSTATE=y +CONFIG_APPLE_PMP=m +CONFIG_APPLE_PMP_REPORT=y +CONFIG_IIO_AOP_SENSOR_LAS=m +CONFIG_IIO_AOP_SENSOR_ALS=m +CONFIG_RUST_FW_LOADER_ABSTRACTIONS=y +CONFIG_PWM_APPLE=m +CONFIG_APPLE_AIC=y +CONFIG_PHY_APPLE_ATC=m +CONFIG_PHY_APPLE_DPTX=m +CONFIG_APPLE_M1_CPU_PMU=y +CONFIG_NVMEM_APPLE_EFUSES=m +CONFIG_NVMEM_APPLE_SPMI=m +CONFIG_MUX_APPLE_DPXBAR=m diff --git a/arch/arm64/include/asm/apple_cpufeature.h b/arch/arm64/include/asm/apple_cpufeature.h new file mode 100644 index 00000000000000..4370d91ffa3ec9 --- /dev/null +++ b/arch/arm64/include/asm/apple_cpufeature.h @@ -0,0 +1,15 @@ +// SPDX-License-Identifier: GPL-2.0 + +#ifndef __ASM_APPLE_CPUFEATURES_H +#define __ASM_APPLE_CPUFEATURES_H + +#include +#include + +#define AIDR_APPLE_TSO_SHIFT 9 +#define AIDR_APPLE_TSO BIT(9) + +#define ACTLR_APPLE_TSO_SHIFT 1 +#define ACTLR_APPLE_TSO BIT(1) + +#endif diff --git a/arch/arm64/include/asm/cpufeature.h b/arch/arm64/include/asm/cpufeature.h index 4de51f8d92cbac..a69eae348990bf 100644 --- a/arch/arm64/include/asm/cpufeature.h +++ b/arch/arm64/include/asm/cpufeature.h @@ -955,6 +955,12 @@ static inline unsigned int get_vmid_bits(u64 mmfr1) return 8; } +static __always_inline bool system_has_actlr_state(void) +{ + return IS_ENABLED(CONFIG_ARM64_ACTLR_STATE) && + alternative_has_cap_unlikely(ARM64_HAS_TSO_APPLE); +} + s64 arm64_ftr_safe_value(const struct arm64_ftr_bits *ftrp, s64 new, s64 cur); struct arm64_ftr_reg *get_arm64_ftr_reg(u32 sys_id); @@ -1078,6 +1084,10 @@ static inline bool cpu_has_lpa2(void) #endif } +void __init init_cpucap_indirect_list_impdef(void); +void __init init_cpucap_indirect_list_from_array(const struct arm64_cpu_capabilities *caps); +bool cpufeature_matches(u64 reg, const struct arm64_cpu_capabilities *entry); + #endif /* __ASSEMBLER__ */ #endif diff --git a/arch/arm64/include/asm/io.h b/arch/arm64/include/asm/io.h index 83e03abbb2ca92..8cbd1e96fd50bf 100644 --- a/arch/arm64/include/asm/io.h +++ b/arch/arm64/include/asm/io.h @@ -264,19 +264,33 @@ __iowrite64_copy(void __iomem *to, const void *from, size_t count) typedef int (*ioremap_prot_hook_t)(phys_addr_t phys_addr, size_t size, pgprot_t *prot); int arm64_ioremap_prot_hook_register(const ioremap_prot_hook_t hook); +void __iomem *__ioremap_prot(phys_addr_t phys, size_t size, pgprot_t prot); -#define ioremap_prot ioremap_prot +static inline void __iomem *ioremap_prot(phys_addr_t phys, size_t size, + pgprot_t user_prot) +{ + pgprot_t prot; + ptdesc_t user_prot_val = pgprot_val(user_prot); + + if (WARN_ON_ONCE(!(user_prot_val & PTE_USER))) + return NULL; -#define _PAGE_IOREMAP PROT_DEVICE_nGnRE + prot = __pgprot_modify(PAGE_KERNEL, PTE_ATTRINDX_MASK, + user_prot_val & PTE_ATTRINDX_MASK); + return __ioremap_prot(phys, size, prot); +} +#define ioremap_prot ioremap_prot +#define ioremap(addr, size) \ + __ioremap_prot((addr), (size), __pgprot(PROT_DEVICE_nGnRE)) #define ioremap_wc(addr, size) \ - ioremap_prot((addr), (size), __pgprot(PROT_NORMAL_NC)) + __ioremap_prot((addr), (size), __pgprot(PROT_NORMAL_NC)) #define ioremap_np(addr, size) \ - ioremap_prot((addr), (size), __pgprot(PROT_DEVICE_nGnRnE)) + __ioremap_prot((addr), (size), __pgprot(PROT_DEVICE_nGnRnE)) #define ioremap_encrypted(addr, size) \ - ioremap_prot((addr), (size), PAGE_KERNEL) + __ioremap_prot((addr), (size), PAGE_KERNEL) /* * io{read,write}{16,32,64}be() macros @@ -297,7 +311,7 @@ static inline void __iomem *ioremap_cache(phys_addr_t addr, size_t size) if (pfn_is_map_memory(__phys_to_pfn(addr))) return (void __iomem *)__phys_to_virt(addr); - return ioremap_prot(addr, size, __pgprot(PROT_NORMAL)); + return __ioremap_prot(addr, size, __pgprot(PROT_NORMAL)); } /* diff --git a/arch/arm64/include/asm/kvm_emulate.h b/arch/arm64/include/asm/kvm_emulate.h index 55d34192a8de17..b50c32b10f7829 100644 --- a/arch/arm64/include/asm/kvm_emulate.h +++ b/arch/arm64/include/asm/kvm_emulate.h @@ -103,6 +103,11 @@ static inline void vcpu_reset_hcr(struct kvm_vcpu *vcpu) { if (!vcpu_has_run_once(vcpu)) vcpu->arch.hcr_el2 = HCR_GUEST_FLAGS; + if (IS_ENABLED(CONFIG_ARM64_ACTLR_STATE) && ( + alternative_has_cap_unlikely(ARM64_HAS_ACTLR_VIRT) || + alternative_has_cap_unlikely(ARM64_HAS_ACTLR_VIRT_APPLE) + )) + vcpu->arch.hcr_el2 &= ~HCR_TACR; /* * For non-FWB CPUs, we trap VM ops (HCR_EL2.TVM) until M+C diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index ac7f970c788302..59f25b85be2b25 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -760,6 +760,9 @@ struct kvm_host_data { /* Number of debug breakpoints/watchpoints for this CPU (minus 1) */ unsigned int debug_brps; unsigned int debug_wrps; + + /* Last vgic_irq part of the AP list recorded in an LR */ + struct vgic_irq *last_lr_irq; }; struct kvm_host_psci_config { diff --git a/arch/arm64/include/asm/memory.h b/arch/arm64/include/asm/memory.h index 9d54b2ea49d66b..66383e3009feae 100644 --- a/arch/arm64/include/asm/memory.h +++ b/arch/arm64/include/asm/memory.h @@ -112,7 +112,7 @@ #define DIRECT_MAP_PHYSMEM_END __pa(PAGE_END - 1) -#define MIN_THREAD_SHIFT (14 + KASAN_THREAD_SHIFT) +#define MIN_THREAD_SHIFT (15 + KASAN_THREAD_SHIFT) /* * VMAP'd stacks are allocated at page granularity, so we must ensure that such diff --git a/arch/arm64/include/asm/pgtable-prot.h b/arch/arm64/include/asm/pgtable-prot.h index 161e8660edddc3..a2d7c17e77e01f 100644 --- a/arch/arm64/include/asm/pgtable-prot.h +++ b/arch/arm64/include/asm/pgtable-prot.h @@ -25,6 +25,8 @@ */ #define PTE_PRESENT_INVALID (PTE_NG) /* only when !PTE_VALID */ +#define PTE_PRESENT_VALID_KERNEL (PTE_VALID | PTE_MAYBE_NG) + #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP #define PTE_UFFD_WP (_AT(pteval_t, 1) << 58) /* uffd-wp tracking */ #define PTE_SWP_UFFD_WP (_AT(pteval_t, 1) << 3) /* only for swp ptes */ @@ -50,11 +52,11 @@ #define _PAGE_DEFAULT (_PROT_DEFAULT | PTE_ATTRINDX(MT_NORMAL)) -#define _PAGE_KERNEL (PROT_NORMAL) -#define _PAGE_KERNEL_RO ((PROT_NORMAL & ~PTE_WRITE) | PTE_RDONLY) -#define _PAGE_KERNEL_ROX ((PROT_NORMAL & ~(PTE_WRITE | PTE_PXN)) | PTE_RDONLY) -#define _PAGE_KERNEL_EXEC (PROT_NORMAL & ~PTE_PXN) -#define _PAGE_KERNEL_EXEC_CONT ((PROT_NORMAL & ~PTE_PXN) | PTE_CONT) +#define _PAGE_KERNEL (PROT_NORMAL | PTE_DIRTY) +#define _PAGE_KERNEL_RO ((PROT_NORMAL & ~PTE_WRITE) | PTE_RDONLY | PTE_DIRTY) +#define _PAGE_KERNEL_ROX ((PROT_NORMAL & ~(PTE_WRITE | PTE_PXN)) | PTE_RDONLY | PTE_DIRTY) +#define _PAGE_KERNEL_EXEC ((PROT_NORMAL & ~PTE_PXN) | PTE_DIRTY) +#define _PAGE_KERNEL_EXEC_CONT ((PROT_NORMAL & ~PTE_PXN) | PTE_CONT | PTE_DIRTY) #define _PAGE_SHARED (_PAGE_DEFAULT | PTE_USER | PTE_RDONLY | PTE_NG | PTE_PXN | PTE_UXN | PTE_WRITE) #define _PAGE_SHARED_EXEC (_PAGE_DEFAULT | PTE_USER | PTE_RDONLY | PTE_NG | PTE_PXN | PTE_WRITE) @@ -164,9 +166,6 @@ static inline bool __pure lpa2_is_enabled(void) #define _PAGE_GCS (_PAGE_DEFAULT | PTE_NG | PTE_UXN | PTE_WRITE | PTE_USER) #define _PAGE_GCS_RO (_PAGE_DEFAULT | PTE_NG | PTE_UXN | PTE_USER) -#define PAGE_GCS __pgprot(_PAGE_GCS) -#define PAGE_GCS_RO __pgprot(_PAGE_GCS_RO) - #define PIE_E0 ( \ PIRx_ELx_PERM_PREP(pte_pi_index(_PAGE_GCS), PIE_GCS) | \ PIRx_ELx_PERM_PREP(pte_pi_index(_PAGE_GCS_RO), PIE_R) | \ diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h index 64d5f1d9cce96c..e62e5631b6a12b 100644 --- a/arch/arm64/include/asm/pgtable.h +++ b/arch/arm64/include/asm/pgtable.h @@ -179,8 +179,6 @@ static inline pteval_t __phys_to_pte_val(phys_addr_t phys) __pte(__phys_to_pte_val((phys_addr_t)(pfn) << PAGE_SHIFT) | pgprot_val(prot)) #define pte_none(pte) (!pte_val(pte)) -#define __pte_clear(mm, addr, ptep) \ - __set_pte(ptep, __pte(0)) #define pte_page(pte) (pfn_to_page(pte_pfn(pte))) /* @@ -359,9 +357,11 @@ static inline pte_t pte_mknoncont(pte_t pte) return clear_pte_bit(pte, __pgprot(PTE_CONT)); } -static inline pte_t pte_mkvalid(pte_t pte) +static inline pte_t pte_mkvalid_k(pte_t pte) { - return set_pte_bit(pte, __pgprot(PTE_VALID)); + pte = clear_pte_bit(pte, __pgprot(PTE_PRESENT_INVALID)); + pte = set_pte_bit(pte, __pgprot(PTE_PRESENT_VALID_KERNEL)); + return pte; } static inline pte_t pte_mkinvalid(pte_t pte) @@ -631,6 +631,7 @@ static inline int pmd_protnone(pmd_t pmd) #define pmd_mkclean(pmd) pte_pmd(pte_mkclean(pmd_pte(pmd))) #define pmd_mkdirty(pmd) pte_pmd(pte_mkdirty(pmd_pte(pmd))) #define pmd_mkyoung(pmd) pte_pmd(pte_mkyoung(pmd_pte(pmd))) +#define pmd_mkvalid_k(pmd) pte_pmd(pte_mkvalid_k(pmd_pte(pmd))) #define pmd_mkinvalid(pmd) pte_pmd(pte_mkinvalid(pmd_pte(pmd))) #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP #define pmd_uffd_wp(pmd) pte_uffd_wp(pmd_pte(pmd)) @@ -672,6 +673,8 @@ static inline pmd_t pmd_mkspecial(pmd_t pmd) #define pud_young(pud) pte_young(pud_pte(pud)) #define pud_mkyoung(pud) pte_pud(pte_mkyoung(pud_pte(pud))) +#define pud_mkwrite_novma(pud) pte_pud(pte_mkwrite_novma(pud_pte(pud))) +#define pud_mkvalid_k(pud) pte_pud(pte_mkvalid_k(pud_pte(pud))) #define pud_write(pud) pte_write(pud_pte(pud)) static inline pud_t pud_mkhuge(pud_t pud) @@ -1320,6 +1323,13 @@ static inline bool pud_user_accessible_page(pud_t pud) /* * Atomic pte/pmd modifications. */ + +static inline void __pte_clear(struct mm_struct *mm, + unsigned long addr, pte_t *ptep) +{ + __set_pte(ptep, __pte(0)); +} + static inline int __ptep_test_and_clear_young(struct vm_area_struct *vma, unsigned long address, pte_t *ptep) diff --git a/arch/arm64/include/asm/processor.h b/arch/arm64/include/asm/processor.h index e30c4c8e3a7a7d..499e0a85c08828 100644 --- a/arch/arm64/include/asm/processor.h +++ b/arch/arm64/include/asm/processor.h @@ -199,6 +199,9 @@ struct thread_struct { u64 gcs_base; u64 gcs_size; #endif +#ifdef CONFIG_ARM64_ACTLR_STATE + u64 actlr; +#endif }; static inline unsigned int thread_get_vl(struct thread_struct *thread, diff --git a/arch/arm64/include/asm/rwonce.h b/arch/arm64/include/asm/rwonce.h index 78beceec10cda4..fc0fb42b0b6411 100644 --- a/arch/arm64/include/asm/rwonce.h +++ b/arch/arm64/include/asm/rwonce.h @@ -58,7 +58,7 @@ default: \ atomic = 0; \ } \ - atomic ? (typeof(*__x))__u.__val : (*(volatile typeof(__x))__x);\ + atomic ? (typeof(*__x))__u.__val : (*(volatile typeof(*__x) *)__x);\ }) #endif /* !BUILD_VDSO */ diff --git a/arch/arm64/include/asm/uaccess.h b/arch/arm64/include/asm/uaccess.h index 6490930deef84d..770bb5211b5403 100644 --- a/arch/arm64/include/asm/uaccess.h +++ b/arch/arm64/include/asm/uaccess.h @@ -480,7 +480,7 @@ extern __must_check long strnlen_user(const char __user *str, long n); #ifdef CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE extern unsigned long __must_check __copy_user_flushcache(void *to, const void __user *from, unsigned long n); -static inline int __copy_from_user_flushcache(void *dst, const void __user *src, unsigned size) +static inline size_t copy_from_user_flushcache(void *dst, const void __user *src, size_t size) { kasan_check_write(dst, size); return __copy_user_flushcache(dst, __uaccess_mask_ptr(src), size); diff --git a/arch/arm64/kernel/Makefile b/arch/arm64/kernel/Makefile index 76f32e424065e5..70bf2f40a0e4dd 100644 --- a/arch/arm64/kernel/Makefile +++ b/arch/arm64/kernel/Makefile @@ -34,6 +34,7 @@ obj-y := debug-monitors.o entry.o irq.o fpsimd.o \ cpufeature.o alternative.o cacheinfo.o \ smp.o smp_spin_table.o topology.o smccc-call.o \ syscall.o proton-pack.o idle.o patching.o pi/ \ + cpufeature_impdef.o \ rsi.o jump_label.o obj-$(CONFIG_COMPAT) += sys32.o signal32.o \ diff --git a/arch/arm64/kernel/acpi.c b/arch/arm64/kernel/acpi.c index af90128cfed56d..a9d884fd1d0019 100644 --- a/arch/arm64/kernel/acpi.c +++ b/arch/arm64/kernel/acpi.c @@ -377,7 +377,7 @@ void __iomem *acpi_os_ioremap(acpi_physical_address phys, acpi_size size) prot = __acpi_get_writethrough_mem_attribute(); } } - return ioremap_prot(phys, size, prot); + return __ioremap_prot(phys, size, prot); } /* diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index c840a93b9ef95b..ee962ae449c0a4 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -1080,7 +1080,7 @@ static void init_cpu_ftr_reg(u32 sys_reg, u64 new) extern const struct arm64_cpu_capabilities arm64_errata[]; static const struct arm64_cpu_capabilities arm64_features[]; -static void __init +void __init init_cpucap_indirect_list_from_array(const struct arm64_cpu_capabilities *caps) { for (; caps->matches; caps++) { @@ -1592,8 +1592,8 @@ has_always(const struct arm64_cpu_capabilities *entry, int scope) return true; } -static bool -feature_matches(u64 reg, const struct arm64_cpu_capabilities *entry) +bool +cpufeature_matches(u64 reg, const struct arm64_cpu_capabilities *entry) { int val, min, max; u64 tmp; @@ -1646,14 +1646,14 @@ has_user_cpuid_feature(const struct arm64_cpu_capabilities *entry, int scope) if (!mask) return false; - return feature_matches(val, entry); + return cpufeature_matches(val, entry); } static bool has_cpuid_feature(const struct arm64_cpu_capabilities *entry, int scope) { u64 val = read_scoped_sysreg(entry, scope); - return feature_matches(val, entry); + return cpufeature_matches(val, entry); } const struct cpumask *system_32bit_el0_cpumask(void) @@ -2336,6 +2336,15 @@ static bool can_trap_icv_dir_el1(const struct arm64_cpu_capabilities *entry, if (this_cpu_has_cap(ARM64_HAS_GICV5_LEGACY)) return true; + /* + * pKVM prevents late onlining of CPUs. This means that whatever + * state the capability is in after deprivilege cannot be affected + * by a new CPU booting -- this is garanteed to be a CPU we have + * already seen, and the cap is therefore unchanged. + */ + if (system_capabilities_finalized() && is_protected_kvm_enabled()) + return cpus_have_final_cap(ARM64_HAS_ICH_HCR_EL2_TDIR); + if (is_kernel_in_hyp_mode()) res.a1 = read_sysreg_s(SYS_ICH_VTR_EL2); else @@ -3888,6 +3897,7 @@ void __init setup_boot_cpu_features(void) * handle the boot CPU. */ init_cpucap_indirect_list(); + init_cpucap_indirect_list_impdef(); /* * Detect broken pseudo-NMI. Must be called _before_ the call to diff --git a/arch/arm64/kernel/cpufeature_impdef.c b/arch/arm64/kernel/cpufeature_impdef.c new file mode 100644 index 00000000000000..aee7571fbadb84 --- /dev/null +++ b/arch/arm64/kernel/cpufeature_impdef.c @@ -0,0 +1,117 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Contains implementation-defined CPU feature definitions. + */ + +#define pr_fmt(fmt) "CPU features: " fmt + +#include +#include +#include +#include +#include + +#ifdef CONFIG_ARM64_MEMORY_MODEL_CONTROL +static bool has_apple_feature(const struct arm64_cpu_capabilities *entry, int scope) +{ + u64 val; + WARN_ON(scope == SCOPE_LOCAL_CPU && preemptible()); + + if (read_cpuid_implementor() != ARM_CPU_IMP_APPLE) + return false; + + val = read_sysreg(aidr_el1); + return cpufeature_matches(val, entry); +} + +static bool has_apple_tso(const struct arm64_cpu_capabilities *entry, int scope) +{ + u64 val; + + if (!has_apple_feature(entry, scope)) + return false; + + /* + * KVM and old versions of the macOS hypervisor will advertise TSO in + * AIDR_EL1, but then ignore writes to ACTLR_EL1. Test that the bit is + * actually writable before enabling TSO. + */ + + val = read_sysreg(actlr_el1); + write_sysreg(val ^ ACTLR_APPLE_TSO, actlr_el1); + if (!((val ^ read_sysreg(actlr_el1)) & ACTLR_APPLE_TSO)) { + pr_info_once("CPU advertises Apple TSO but it is broken, ignoring\n"); + return false; + } + + write_sysreg(val, actlr_el1); + return true; +} + +static bool has_tso_fixed(const struct arm64_cpu_capabilities *entry, int scope) +{ + /* List of CPUs that always use the TSO memory model */ + static const struct midr_range fixed_tso_list[] = { + MIDR_ALL_VERSIONS(MIDR_NVIDIA_DENVER), + MIDR_ALL_VERSIONS(MIDR_NVIDIA_CARMEL), + MIDR_ALL_VERSIONS(MIDR_FUJITSU_A64FX), + { /* sentinel */ } + }; + + return is_midr_in_range_list(fixed_tso_list); +} +#endif + +static bool has_apple_actlr_virt_impdef(const struct arm64_cpu_capabilities *entry, int scope) +{ + u64 midr = read_cpuid_id() & MIDR_CPU_MODEL_MASK; + + return midr >= MIDR_APPLE_M1_ICESTORM && midr <= MIDR_APPLE_M1_FIRESTORM_MAX; +} + +static bool has_apple_actlr_virt(const struct arm64_cpu_capabilities *entry, int scope) +{ + u64 midr = read_cpuid_id() & MIDR_CPU_MODEL_MASK; + + return midr >= MIDR_APPLE_M2_BLIZZARD && midr <= MIDR_CPU_MODEL(ARM_CPU_IMP_APPLE, 0xfff); +} + +static const struct arm64_cpu_capabilities arm64_impdef_features[] = { +#ifdef CONFIG_ARM64_MEMORY_MODEL_CONTROL + { + .desc = "TSO memory model (Apple)", + .capability = ARM64_HAS_TSO_APPLE, + .type = ARM64_CPUCAP_EARLY_LOCAL_CPU_FEATURE, + .matches = has_apple_tso, + .field_pos = AIDR_APPLE_TSO_SHIFT, + .field_width = 1, + .sign = FTR_UNSIGNED, + .min_field_value = 1, + .max_field_value = 1, + }, + { + .desc = "TSO memory model (Fixed)", + .capability = ARM64_HAS_TSO_FIXED, + .type = ARM64_CPUCAP_EARLY_LOCAL_CPU_FEATURE, + .matches = has_tso_fixed, + }, +#endif + { + .desc = "ACTLR virtualization (IMPDEF, Apple)", + .capability = ARM64_HAS_ACTLR_VIRT_APPLE, + .type = ARM64_CPUCAP_EARLY_LOCAL_CPU_FEATURE, + .matches = has_apple_actlr_virt_impdef, + }, + { + .desc = "ACTLR virtualization (architectural?)", + .capability = ARM64_HAS_ACTLR_VIRT, + .type = ARM64_CPUCAP_EARLY_LOCAL_CPU_FEATURE, + .matches = has_apple_actlr_virt, + }, + {}, +}; + +void __init init_cpucap_indirect_list_impdef(void) +{ + init_cpucap_indirect_list_from_array(arm64_impdef_features); +} diff --git a/arch/arm64/kernel/kexec_image.c b/arch/arm64/kernel/kexec_image.c index 532d72ea42ee8e..b70f4df15a1ae5 100644 --- a/arch/arm64/kernel/kexec_image.c +++ b/arch/arm64/kernel/kexec_image.c @@ -41,7 +41,7 @@ static void *image_load(struct kimage *image, struct arm64_image_header *h; u64 flags, value; bool be_image, be_kernel; - struct kexec_buf kbuf; + struct kexec_buf kbuf = {}; unsigned long text_offset, kernel_segment_number; struct kexec_segment *kernel_segment; int ret; diff --git a/arch/arm64/kernel/pi/patch-scs.c b/arch/arm64/kernel/pi/patch-scs.c index bbe7d30ed12b32..dac568e4a54f23 100644 --- a/arch/arm64/kernel/pi/patch-scs.c +++ b/arch/arm64/kernel/pi/patch-scs.c @@ -192,6 +192,14 @@ static int scs_handle_fde_frame(const struct eh_frame *frame, size -= 2; break; + case DW_CFA_advance_loc4: + loc += *opcode++ * code_alignment_factor; + loc += (*opcode++ << 8) * code_alignment_factor; + loc += (*opcode++ << 16) * code_alignment_factor; + loc += (*opcode++ << 24) * code_alignment_factor; + size -= 4; + break; + case DW_CFA_def_cfa: case DW_CFA_offset_extended: size = skip_xleb128(&opcode, size); diff --git a/arch/arm64/kernel/process.c b/arch/arm64/kernel/process.c index 489554931231e6..59c416cfcdd2a4 100644 --- a/arch/arm64/kernel/process.c +++ b/arch/arm64/kernel/process.c @@ -41,8 +41,10 @@ #include #include #include +#include #include +#include #include #include #include @@ -442,6 +444,11 @@ int copy_thread(struct task_struct *p, const struct kernel_clone_args *args) if (system_supports_poe()) p->thread.por_el0 = read_sysreg_s(SYS_POR_EL0); +#ifdef CONFIG_ARM64_ACTLR_STATE + if (system_has_actlr_state()) + p->thread.actlr = read_sysreg(actlr_el1); +#endif + if (stack_start) { if (is_compat_thread(task_thread_info(p))) childregs->compat_sp = stack_start; @@ -699,6 +706,65 @@ void update_sctlr_el1(u64 sctlr) isb(); } +#ifdef CONFIG_ARM64_MEMORY_MODEL_CONTROL +int arch_prctl_mem_model_get(struct task_struct *t) +{ + if (alternative_has_cap_unlikely(ARM64_HAS_TSO_APPLE) && + t->thread.actlr & ACTLR_APPLE_TSO) + return PR_SET_MEM_MODEL_TSO; + + return PR_SET_MEM_MODEL_DEFAULT; +} + +int arch_prctl_mem_model_set(struct task_struct *t, unsigned long val) +{ + if (alternative_has_cap_unlikely(ARM64_HAS_TSO_FIXED) && + val == PR_SET_MEM_MODEL_TSO) + return 0; + + if (alternative_has_cap_unlikely(ARM64_HAS_TSO_APPLE)) { + WARN_ON(!system_has_actlr_state()); + + switch (val) { + case PR_SET_MEM_MODEL_TSO: + t->thread.actlr |= ACTLR_APPLE_TSO; + break; + case PR_SET_MEM_MODEL_DEFAULT: + t->thread.actlr &= ~ACTLR_APPLE_TSO; + break; + default: + return -EINVAL; + } + write_sysreg(t->thread.actlr, actlr_el1); + return 0; + } + + if (val == PR_SET_MEM_MODEL_DEFAULT) + return 0; + + return -EINVAL; +} +#endif + +#ifdef CONFIG_ARM64_ACTLR_STATE +/* + * IMPDEF control register ACTLR_EL1 handling. Some CPUs use this to + * expose features that can be controlled by userspace. + */ +static void actlr_thread_switch(struct task_struct *next) +{ + if (!system_has_actlr_state()) + return; + + current->thread.actlr = read_sysreg(actlr_el1); + write_sysreg(next->thread.actlr, actlr_el1); +} +#else +static inline void actlr_thread_switch(struct task_struct *next) +{ +} +#endif + /* * Thread switching. */ @@ -718,6 +784,7 @@ struct task_struct *__switch_to(struct task_struct *prev, ptrauth_thread_switch_user(next); permission_overlay_switch(next); gcs_thread_switch(next); + actlr_thread_switch(next); /* * Complete any pending TLB or cache maintenance on this CPU in case the @@ -840,6 +907,10 @@ void arch_setup_new_exec(void) arch_prctl_spec_ctrl_set(current, PR_SPEC_STORE_BYPASS, PR_SPEC_ENABLE); } + +#ifdef CONFIG_ARM64_MEMORY_MODEL_CONTROL + arch_prctl_mem_model_set(current, PR_SET_MEM_MODEL_DEFAULT); +#endif } #ifdef CONFIG_ARM64_TAGGED_ADDR_ABI diff --git a/arch/arm64/kernel/proton-pack.c b/arch/arm64/kernel/proton-pack.c index 80a580e019c501..b3801f532b10b3 100644 --- a/arch/arm64/kernel/proton-pack.c +++ b/arch/arm64/kernel/proton-pack.c @@ -887,6 +887,7 @@ static u8 spectre_bhb_loop_affected(void) MIDR_ALL_VERSIONS(MIDR_CORTEX_X2), MIDR_ALL_VERSIONS(MIDR_NEOVERSE_N2), MIDR_ALL_VERSIONS(MIDR_NEOVERSE_V1), + MIDR_ALL_VERSIONS(MIDR_HISI_TSV110), {}, }; static const struct midr_range spectre_bhb_k24_list[] = { diff --git a/arch/arm64/kernel/ptrace.c b/arch/arm64/kernel/ptrace.c index 6c5ff6807d4cc8..64ff87f0231130 100644 --- a/arch/arm64/kernel/ptrace.c +++ b/arch/arm64/kernel/ptrace.c @@ -1484,6 +1484,9 @@ static int poe_get(struct task_struct *target, if (!system_supports_poe()) return -EINVAL; + if (target == current) + current->thread.por_el0 = read_sysreg_s(SYS_POR_EL0); + return membuf_write(&to, &target->thread.por_el0, sizeof(target->thread.por_el0)); } diff --git a/arch/arm64/kernel/rsi.c b/arch/arm64/kernel/rsi.c index c64a06f58c0bc0..9e846ce4ef9ca7 100644 --- a/arch/arm64/kernel/rsi.c +++ b/arch/arm64/kernel/rsi.c @@ -12,6 +12,7 @@ #include #include +#include #include static struct realm_config config; @@ -146,7 +147,7 @@ void __init arm64_rsi_init(void) return; if (WARN_ON(rsi_get_realm_config(&config))) return; - prot_ns_shared = BIT(config.ipa_bits - 1); + prot_ns_shared = __phys_to_pte_val(BIT(config.ipa_bits - 1)); if (arm64_ioremap_prot_hook_register(realm_ioremap_hook)) return; diff --git a/arch/arm64/kernel/setup.c b/arch/arm64/kernel/setup.c index 23c05dc7a8f2ac..0fa2403c6fc0e6 100644 --- a/arch/arm64/kernel/setup.c +++ b/arch/arm64/kernel/setup.c @@ -368,6 +368,14 @@ void __init __no_sanitize_address setup_arch(char **cmdline_p) */ init_task.thread_info.ttbr0 = phys_to_ttbr(__pa_symbol(reserved_pg_dir)); #endif +#ifdef CONFIG_ARM64_ACTLR_STATE + /* Store the boot CPU ACTLR_EL1 value as the default. This will only + * be actually restored during context switching iff the platform is + * known to use ACTLR_EL1 for exposable features and its layout is + * known to be the same on all CPUs. + */ + init_task.thread.actlr = read_sysreg(actlr_el1); +#endif if (boot_args[1] || boot_args[2] || boot_args[3]) { pr_err("WARNING: x1-x3 nonzero in violation of boot protocol:\n" diff --git a/arch/arm64/kernel/vmlinux.lds.S b/arch/arm64/kernel/vmlinux.lds.S index ad6133b89e7a40..2964aad0362e4e 100644 --- a/arch/arm64/kernel/vmlinux.lds.S +++ b/arch/arm64/kernel/vmlinux.lds.S @@ -349,6 +349,7 @@ SECTIONS STABS_DEBUG DWARF_DEBUG + MODINFO ELF_DETAILS HEAD_SYMBOLS diff --git a/arch/arm64/kvm/at.c b/arch/arm64/kvm/at.c index 808d26bed1824a..492fa7ab86c397 100644 --- a/arch/arm64/kvm/at.c +++ b/arch/arm64/kvm/at.c @@ -1785,7 +1785,7 @@ int __kvm_at_swap_desc(struct kvm *kvm, gpa_t ipa, u64 old, u64 new) if (!writable) return -EPERM; - ptep = (u64 __user *)hva + offset; + ptep = (void __user *)hva + offset; if (cpus_have_final_cap(ARM64_HAS_LSE_ATOMICS)) r = __lse_swap_desc(ptep, old, new); else diff --git a/arch/arm64/kvm/emulate-nested.c b/arch/arm64/kvm/emulate-nested.c index 834f13fb1fb7d4..2d04fb56746ea9 100644 --- a/arch/arm64/kvm/emulate-nested.c +++ b/arch/arm64/kvm/emulate-nested.c @@ -2428,7 +2428,7 @@ static u64 kvm_get_sysreg_res0(struct kvm *kvm, enum vcpu_sysreg sr) masks = kvm->arch.sysreg_masks; - return masks->mask[sr - __VNCR_START__].res0; + return masks->mask[sr - __SANITISED_REG_START__].res0; } static bool check_fgt_bit(struct kvm_vcpu *vcpu, enum vcpu_sysreg sr, diff --git a/arch/arm64/kvm/guest.c b/arch/arm64/kvm/guest.c index 1c87699fd886e7..332c453b87cf8c 100644 --- a/arch/arm64/kvm/guest.c +++ b/arch/arm64/kvm/guest.c @@ -29,7 +29,7 @@ #include "trace.h" -const struct _kvm_stats_desc kvm_vm_stats_desc[] = { +const struct kvm_stats_desc kvm_vm_stats_desc[] = { KVM_GENERIC_VM_STATS() }; @@ -42,7 +42,7 @@ const struct kvm_stats_header kvm_vm_stats_header = { sizeof(kvm_vm_stats_desc), }; -const struct _kvm_stats_desc kvm_vcpu_stats_desc[] = { +const struct kvm_stats_desc kvm_vcpu_stats_desc[] = { KVM_GENERIC_VCPU_STATS(), STATS_DESC_COUNTER(VCPU, hvc_exit_stat), STATS_DESC_COUNTER(VCPU, wfe_exit_stat), diff --git a/arch/arm64/kvm/hyp/include/hyp/sysreg-sr.h b/arch/arm64/kvm/hyp/include/hyp/sysreg-sr.h index a17cbe7582de90..7c8383c809ea36 100644 --- a/arch/arm64/kvm/hyp/include/hyp/sysreg-sr.h +++ b/arch/arm64/kvm/hyp/include/hyp/sysreg-sr.h @@ -16,6 +16,9 @@ #include #include +#define SYS_IMP_APL_ACTLR_EL12 sys_reg(3, 6, 15, 14, 6) +#define SYS_ACTLR_EL12 sys_reg(3, 5, 1, 0, 1) + static inline bool ctxt_has_s1poe(struct kvm_cpu_context *ctxt); static inline struct kvm_vcpu *ctxt_to_vcpu(struct kvm_cpu_context *ctxt) @@ -172,6 +175,13 @@ static inline void __sysreg_save_el1_state(struct kvm_cpu_context *ctxt) if (ctxt_has_sctlr2(ctxt)) ctxt_sys_reg(ctxt, SCTLR2_EL1) = read_sysreg_el1(SYS_SCTLR2); + + if (IS_ENABLED(CONFIG_ARM64_ACTLR_STATE)) { + if (alternative_has_cap_unlikely(ARM64_HAS_ACTLR_VIRT)) + ctxt_sys_reg(ctxt, ACTLR_EL1) = read_sysreg_s(SYS_ACTLR_EL12); + else if (alternative_has_cap_unlikely(ARM64_HAS_ACTLR_VIRT_APPLE)) + ctxt_sys_reg(ctxt, ACTLR_EL1) = read_sysreg_s(SYS_IMP_APL_ACTLR_EL12); + } } static inline void __sysreg_save_el2_return_state(struct kvm_cpu_context *ctxt) @@ -256,6 +266,13 @@ static inline void __sysreg_restore_el1_state(struct kvm_cpu_context *ctxt, write_sysreg(ctxt_sys_reg(ctxt, PAR_EL1), par_el1); write_sysreg(ctxt_sys_reg(ctxt, TPIDR_EL1), tpidr_el1); + if (IS_ENABLED(CONFIG_ARM64_ACTLR_STATE)) { + if (alternative_has_cap_unlikely(ARM64_HAS_ACTLR_VIRT)) + write_sysreg_s(ctxt_sys_reg(ctxt, ACTLR_EL1), SYS_ACTLR_EL12); + else if (alternative_has_cap_unlikely(ARM64_HAS_ACTLR_VIRT_APPLE)) + write_sysreg_s(ctxt_sys_reg(ctxt, ACTLR_EL1), SYS_IMP_APL_ACTLR_EL12); + } + if (ctxt_has_mte(ctxt)) { write_sysreg_el1(ctxt_sys_reg(ctxt, TFSR_EL1), SYS_TFSR); write_sysreg_s(ctxt_sys_reg(ctxt, TFSRE0_EL1), SYS_TFSRE0_EL1); diff --git a/arch/arm64/kvm/hyp/nvhe/mem_protect.c b/arch/arm64/kvm/hyp/nvhe/mem_protect.c index 49db32f3ddf715..ece04bb10ab096 100644 --- a/arch/arm64/kvm/hyp/nvhe/mem_protect.c +++ b/arch/arm64/kvm/hyp/nvhe/mem_protect.c @@ -516,7 +516,7 @@ static int host_stage2_adjust_range(u64 addr, struct kvm_mem_range *range) granule = kvm_granule_size(level); cur.start = ALIGN_DOWN(addr, granule); cur.end = cur.start + granule; - if (!range_included(&cur, range)) + if (!range_included(&cur, range) && level < KVM_PGTABLE_LAST_LEVEL) continue; *range = cur; return 0; diff --git a/arch/arm64/kvm/hyp/nvhe/pkvm.c b/arch/arm64/kvm/hyp/nvhe/pkvm.c index 12b2acfbcfd14a..59a01022181896 100644 --- a/arch/arm64/kvm/hyp/nvhe/pkvm.c +++ b/arch/arm64/kvm/hyp/nvhe/pkvm.c @@ -345,6 +345,7 @@ static void pkvm_init_features_from_host(struct pkvm_hyp_vm *hyp_vm, const struc /* No restrictions for non-protected VMs. */ if (!kvm_vm_is_protected(kvm)) { hyp_vm->kvm.arch.flags = host_arch_flags; + hyp_vm->kvm.arch.flags &= ~BIT_ULL(KVM_ARCH_FLAG_ID_REGS_INITIALIZED); bitmap_copy(kvm->arch.vcpu_features, host_kvm->arch.vcpu_features, @@ -471,6 +472,35 @@ static int pkvm_vcpu_init_sve(struct pkvm_hyp_vcpu *hyp_vcpu, struct kvm_vcpu *h return ret; } +static int vm_copy_id_regs(struct pkvm_hyp_vcpu *hyp_vcpu) +{ + struct pkvm_hyp_vm *hyp_vm = pkvm_hyp_vcpu_to_hyp_vm(hyp_vcpu); + const struct kvm *host_kvm = hyp_vm->host_kvm; + struct kvm *kvm = &hyp_vm->kvm; + + if (!test_bit(KVM_ARCH_FLAG_ID_REGS_INITIALIZED, &host_kvm->arch.flags)) + return -EINVAL; + + if (test_and_set_bit(KVM_ARCH_FLAG_ID_REGS_INITIALIZED, &kvm->arch.flags)) + return 0; + + memcpy(kvm->arch.id_regs, host_kvm->arch.id_regs, sizeof(kvm->arch.id_regs)); + + return 0; +} + +static int pkvm_vcpu_init_sysregs(struct pkvm_hyp_vcpu *hyp_vcpu) +{ + int ret = 0; + + if (pkvm_hyp_vcpu_is_protected(hyp_vcpu)) + kvm_init_pvm_id_regs(&hyp_vcpu->vcpu); + else + ret = vm_copy_id_regs(hyp_vcpu); + + return ret; +} + static int init_pkvm_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu, struct pkvm_hyp_vm *hyp_vm, struct kvm_vcpu *host_vcpu) @@ -490,8 +520,9 @@ static int init_pkvm_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu, hyp_vcpu->vcpu.arch.cflags = READ_ONCE(host_vcpu->arch.cflags); hyp_vcpu->vcpu.arch.mp_state.mp_state = KVM_MP_STATE_STOPPED; - if (pkvm_hyp_vcpu_is_protected(hyp_vcpu)) - kvm_init_pvm_id_regs(&hyp_vcpu->vcpu); + ret = pkvm_vcpu_init_sysregs(hyp_vcpu); + if (ret) + goto done; ret = pkvm_vcpu_init_traps(hyp_vcpu); if (ret) diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c index 2caa97f87890fd..9f22d60b64db55 100644 --- a/arch/arm64/kvm/mmu.c +++ b/arch/arm64/kvm/mmu.c @@ -1753,14 +1753,12 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, } /* - * Both the canonical IPA and fault IPA must be hugepage-aligned to - * ensure we find the right PFN and lay down the mapping in the right - * place. + * Both the canonical IPA and fault IPA must be aligned to the + * mapping size to ensure we find the right PFN and lay down the + * mapping in the right place. */ - if (vma_pagesize == PMD_SIZE || vma_pagesize == PUD_SIZE) { - fault_ipa &= ~(vma_pagesize - 1); - ipa &= ~(vma_pagesize - 1); - } + fault_ipa = ALIGN_DOWN(fault_ipa, vma_pagesize); + ipa = ALIGN_DOWN(ipa, vma_pagesize); gfn = ipa >> PAGE_SHIFT; mte_allowed = kvm_vma_mte_allowed(vma); diff --git a/arch/arm64/kvm/reset.c b/arch/arm64/kvm/reset.c index 959532422d3a30..b963fd975aacaf 100644 --- a/arch/arm64/kvm/reset.c +++ b/arch/arm64/kvm/reset.c @@ -247,6 +247,20 @@ void kvm_reset_vcpu(struct kvm_vcpu *vcpu) kvm_vcpu_set_be(vcpu); *vcpu_pc(vcpu) = target_pc; + + /* + * We may come from a state where either a PC update was + * pending (SMC call resulting in PC being increpented to + * skip the SMC) or a pending exception. Make sure we get + * rid of all that, as this cannot be valid out of reset. + * + * Note that clearing the exception mask also clears PC + * updates, but that's an implementation detail, and we + * really want to make it explicit. + */ + vcpu_clear_flag(vcpu, PENDING_EXCEPTION); + vcpu_clear_flag(vcpu, EXCEPT_MASK); + vcpu_clear_flag(vcpu, INCREMENT_PC); vcpu_set_reg(vcpu, 0, reset_state.r0); } diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c index 88a57ca36d96c0..237e8bd1cf29c7 100644 --- a/arch/arm64/kvm/sys_regs.c +++ b/arch/arm64/kvm/sys_regs.c @@ -1816,6 +1816,9 @@ static u64 __kvm_read_sanitised_id_reg(const struct kvm_vcpu *vcpu, ID_AA64MMFR3_EL1_SCTLRX | ID_AA64MMFR3_EL1_S1POE | ID_AA64MMFR3_EL1_S1PIE; + + if (!system_supports_poe()) + val &= ~ID_AA64MMFR3_EL1_S1POE; break; case SYS_ID_MMFR4_EL1: val &= ~ID_MMFR4_EL1_CCIDX; diff --git a/arch/arm64/kvm/vgic/vgic-init.c b/arch/arm64/kvm/vgic/vgic-init.c index dc9f9db3102644..6ed01c7faa91d7 100644 --- a/arch/arm64/kvm/vgic/vgic-init.c +++ b/arch/arm64/kvm/vgic/vgic-init.c @@ -140,26 +140,9 @@ int kvm_vgic_create(struct kvm *kvm, u32 type) goto out_unlock; } - kvm_for_each_vcpu(i, vcpu, kvm) { - ret = vgic_allocate_private_irqs_locked(vcpu, type); - if (ret) - break; - } - - if (ret) { - kvm_for_each_vcpu(i, vcpu, kvm) { - struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu; - kfree(vgic_cpu->private_irqs); - vgic_cpu->private_irqs = NULL; - } - - goto out_unlock; - } - kvm->arch.vgic.in_kernel = true; kvm->arch.vgic.vgic_model = type; kvm->arch.vgic.implementation_rev = KVM_VGIC_IMP_REV_LATEST; - kvm->arch.vgic.vgic_dist_base = VGIC_ADDR_UNDEF; aa64pfr0 = kvm_read_vm_id_reg(kvm, SYS_ID_AA64PFR0_EL1) & ~ID_AA64PFR0_EL1_GIC; @@ -176,6 +159,23 @@ int kvm_vgic_create(struct kvm *kvm, u32 type) kvm_set_vm_id_reg(kvm, SYS_ID_AA64PFR0_EL1, aa64pfr0); kvm_set_vm_id_reg(kvm, SYS_ID_PFR1_EL1, pfr1); + kvm_for_each_vcpu(i, vcpu, kvm) { + ret = vgic_allocate_private_irqs_locked(vcpu, type); + if (ret) + break; + } + + if (ret) { + kvm_for_each_vcpu(i, vcpu, kvm) { + struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu; + kfree(vgic_cpu->private_irqs); + vgic_cpu->private_irqs = NULL; + } + + kvm->arch.vgic.vgic_model = 0; + goto out_unlock; + } + if (type == KVM_DEV_TYPE_ARM_VGIC_V3) kvm->arch.vgic.nassgicap = system_supports_direct_sgis(); diff --git a/arch/arm64/kvm/vgic/vgic-v2.c b/arch/arm64/kvm/vgic/vgic-v2.c index 585491fbda8077..cafa3cb32bda6f 100644 --- a/arch/arm64/kvm/vgic/vgic-v2.c +++ b/arch/arm64/kvm/vgic/vgic-v2.c @@ -115,7 +115,7 @@ void vgic_v2_fold_lr_state(struct kvm_vcpu *vcpu) struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu; struct vgic_v2_cpu_if *cpuif = &vgic_cpu->vgic_v2; u32 eoicount = FIELD_GET(GICH_HCR_EOICOUNT, cpuif->vgic_hcr); - struct vgic_irq *irq; + struct vgic_irq *irq = *host_data_ptr(last_lr_irq); DEBUG_SPINLOCK_BUG_ON(!irqs_disabled()); @@ -123,7 +123,7 @@ void vgic_v2_fold_lr_state(struct kvm_vcpu *vcpu) vgic_v2_fold_lr(vcpu, cpuif->vgic_lr[lr]); /* See the GICv3 equivalent for the EOIcount handling rationale */ - list_for_each_entry(irq, &vgic_cpu->ap_list_head, ap_list) { + list_for_each_entry_continue(irq, &vgic_cpu->ap_list_head, ap_list) { u32 lr; if (!eoicount) { diff --git a/arch/arm64/kvm/vgic/vgic-v3.c b/arch/arm64/kvm/vgic/vgic-v3.c index 1d6dd1b545bdd6..db2f220c29f76b 100644 --- a/arch/arm64/kvm/vgic/vgic-v3.c +++ b/arch/arm64/kvm/vgic/vgic-v3.c @@ -148,7 +148,7 @@ void vgic_v3_fold_lr_state(struct kvm_vcpu *vcpu) struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu; struct vgic_v3_cpu_if *cpuif = &vgic_cpu->vgic_v3; u32 eoicount = FIELD_GET(ICH_HCR_EL2_EOIcount, cpuif->vgic_hcr); - struct vgic_irq *irq; + struct vgic_irq *irq = *host_data_ptr(last_lr_irq); DEBUG_SPINLOCK_BUG_ON(!irqs_disabled()); @@ -158,12 +158,12 @@ void vgic_v3_fold_lr_state(struct kvm_vcpu *vcpu) /* * EOIMode=0: use EOIcount to emulate deactivation. We are * guaranteed to deactivate in reverse order of the activation, so - * just pick one active interrupt after the other in the ap_list, - * and replay the deactivation as if the CPU was doing it. We also - * rely on priority drop to have taken place, and the list to be - * sorted by priority. + * just pick one active interrupt after the other in the tail part + * of the ap_list, past the LRs, and replay the deactivation as if + * the CPU was doing it. We also rely on priority drop to have taken + * place, and the list to be sorted by priority. */ - list_for_each_entry(irq, &vgic_cpu->ap_list_head, ap_list) { + list_for_each_entry_continue(irq, &vgic_cpu->ap_list_head, ap_list) { u64 lr; /* diff --git a/arch/arm64/kvm/vgic/vgic.c b/arch/arm64/kvm/vgic/vgic.c index 430aa98888fda5..e22b79cfff9654 100644 --- a/arch/arm64/kvm/vgic/vgic.c +++ b/arch/arm64/kvm/vgic/vgic.c @@ -814,6 +814,9 @@ static void vgic_prune_ap_list(struct kvm_vcpu *vcpu) static inline void vgic_fold_lr_state(struct kvm_vcpu *vcpu) { + if (!*host_data_ptr(last_lr_irq)) + return; + if (kvm_vgic_global_state.type == VGIC_V2) vgic_v2_fold_lr_state(vcpu); else @@ -960,10 +963,13 @@ static void vgic_flush_lr_state(struct kvm_vcpu *vcpu) if (irqs_outside_lrs(&als)) vgic_sort_ap_list(vcpu); + *host_data_ptr(last_lr_irq) = NULL; + list_for_each_entry(irq, &vgic_cpu->ap_list_head, ap_list) { scoped_guard(raw_spinlock, &irq->irq_lock) { if (likely(vgic_target_oracle(irq) == vcpu)) { vgic_populate_lr(vcpu, irq, count++); + *host_data_ptr(last_lr_irq) = irq; } } diff --git a/arch/arm64/lib/delay.c b/arch/arm64/lib/delay.c index cb2062e7e23405..e278e060e78a9e 100644 --- a/arch/arm64/lib/delay.c +++ b/arch/arm64/lib/delay.c @@ -23,9 +23,24 @@ static inline unsigned long xloops_to_cycles(unsigned long xloops) return (xloops * loops_per_jiffy * HZ) >> 32; } +/* + * Force the use of CNTVCT_EL0 in order to have the same base as WFxT. + * This avoids some annoying issues when CNTVOFF_EL2 is not reset 0 on a + * KVM host running at EL1 until we do a vcpu_put() on the vcpu. When + * running at EL2, the effective offset is always 0. + * + * Note that userspace cannot change the offset behind our back either, + * as the vcpu mutex is held as long as KVM_RUN is in progress. + */ +static cycles_t notrace __delay_cycles(void) +{ + guard(preempt_notrace)(); + return __arch_counter_get_cntvct_stable(); +} + void __delay(unsigned long cycles) { - cycles_t start = get_cycles(); + cycles_t start = __delay_cycles(); if (alternative_has_cap_unlikely(ARM64_HAS_WFXT)) { u64 end = start + cycles; @@ -35,17 +50,17 @@ void __delay(unsigned long cycles) * early, use a WFET loop to complete the delay. */ wfit(end); - while ((get_cycles() - start) < cycles) + while ((__delay_cycles() - start) < cycles) wfet(end); } else if (arch_timer_evtstrm_available()) { const cycles_t timer_evt_period = USECS_TO_CYCLES(ARCH_TIMER_EVT_STREAM_PERIOD_US); - while ((get_cycles() - start + timer_evt_period) < cycles) + while ((__delay_cycles() - start + timer_evt_period) < cycles) wfe(); } - while ((get_cycles() - start) < cycles) + while ((__delay_cycles() - start) < cycles) cpu_relax(); } EXPORT_SYMBOL(__delay); diff --git a/arch/arm64/mm/contpte.c b/arch/arm64/mm/contpte.c index 589bcf87893884..cb6cd58deda9d3 100644 --- a/arch/arm64/mm/contpte.c +++ b/arch/arm64/mm/contpte.c @@ -581,6 +581,27 @@ void contpte_clear_young_dirty_ptes(struct vm_area_struct *vma, } EXPORT_SYMBOL_GPL(contpte_clear_young_dirty_ptes); +static bool contpte_all_subptes_match_access_flags(pte_t *ptep, pte_t entry) +{ + pte_t *cont_ptep = contpte_align_down(ptep); + /* + * PFNs differ per sub-PTE. Match only bits consumed by + * __ptep_set_access_flags(): AF, DIRTY and write permission. + */ + const pteval_t cmp_mask = PTE_RDONLY | PTE_AF | PTE_WRITE | PTE_DIRTY; + pteval_t entry_cmp = pte_val(entry) & cmp_mask; + int i; + + for (i = 0; i < CONT_PTES; i++) { + pteval_t pte_cmp = pte_val(__ptep_get(cont_ptep + i)) & cmp_mask; + + if (pte_cmp != entry_cmp) + return false; + } + + return true; +} + int contpte_ptep_set_access_flags(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep, pte_t entry, int dirty) @@ -590,13 +611,37 @@ int contpte_ptep_set_access_flags(struct vm_area_struct *vma, int i; /* - * Gather the access/dirty bits for the contiguous range. If nothing has - * changed, its a noop. + * Check whether all sub-PTEs in the CONT block already match the + * requested access flags/write permission, using raw per-PTE values + * rather than the gathered ptep_get() view. + * + * __ptep_set_access_flags() can update AF, dirty and write + * permission, but only to make the mapping more permissive. + * + * ptep_get() gathers AF/dirty state across the whole CONT block, + * which is correct for a CPU with FEAT_HAFDBS. But page-table + * walkers that evaluate each descriptor individually (e.g. a CPU + * without DBM support, or an SMMU without HTTU, or with HA/HD + * disabled in CD.TCR) can keep faulting on the target sub-PTE if + * only a sibling has been updated. Gathering can therefore cause + * false no-ops when only a sibling has been updated: + * - write faults: target still has PTE_RDONLY (needs PTE_RDONLY cleared) + * - read faults: target still lacks PTE_AF + * + * Per Arm ARM (DDI 0487) D8.7.1, any sub-PTE in a CONT range may + * become the effective cached translation, so all entries must have + * consistent attributes. Check the full CONT block before returning + * no-op, and when any sub-PTE mismatches, proceed to update the whole + * range. */ - orig_pte = pte_mknoncont(ptep_get(ptep)); - if (pte_val(orig_pte) == pte_val(entry)) + if (contpte_all_subptes_match_access_flags(ptep, entry)) return 0; + /* + * Use raw target pte (not gathered) for write-bit unfold decision. + */ + orig_pte = pte_mknoncont(__ptep_get(ptep)); + /* * We can fix up access/dirty bits without having to unfold the contig * range. But if the write bit is changing, we must unfold. diff --git a/arch/arm64/mm/gcs.c b/arch/arm64/mm/gcs.c index 6e93f78de79b17..04a23a497f2051 100644 --- a/arch/arm64/mm/gcs.c +++ b/arch/arm64/mm/gcs.c @@ -199,8 +199,8 @@ int arch_set_shadow_stack_status(struct task_struct *task, unsigned long arg) size = gcs_size(0); gcs = alloc_gcs(0, size); - if (!gcs) - return -ENOMEM; + if (IS_ERR_VALUE(gcs)) + return gcs; task->thread.gcspr_el0 = gcs + size - sizeof(u64); task->thread.gcs_base = gcs; diff --git a/arch/arm64/mm/ioremap.c b/arch/arm64/mm/ioremap.c index 10e246f1127104..1e4794a2af7d63 100644 --- a/arch/arm64/mm/ioremap.c +++ b/arch/arm64/mm/ioremap.c @@ -14,8 +14,8 @@ int arm64_ioremap_prot_hook_register(ioremap_prot_hook_t hook) return 0; } -void __iomem *ioremap_prot(phys_addr_t phys_addr, size_t size, - pgprot_t pgprot) +void __iomem *__ioremap_prot(phys_addr_t phys_addr, size_t size, + pgprot_t pgprot) { unsigned long last_addr = phys_addr + size - 1; @@ -38,7 +38,7 @@ void __iomem *ioremap_prot(phys_addr_t phys_addr, size_t size, return generic_ioremap_prot(phys_addr, size, pgprot); } -EXPORT_SYMBOL(ioremap_prot); +EXPORT_SYMBOL(__ioremap_prot); /* * Must be called after early_fixmap_init diff --git a/arch/arm64/mm/mmap.c b/arch/arm64/mm/mmap.c index 08ee177432c2f7..92b2f5097a96c1 100644 --- a/arch/arm64/mm/mmap.c +++ b/arch/arm64/mm/mmap.c @@ -34,6 +34,8 @@ static pgprot_t protection_map[16] __ro_after_init = { [VM_SHARED | VM_EXEC | VM_WRITE | VM_READ] = PAGE_SHARED_EXEC }; +static ptdesc_t gcs_page_prot __ro_after_init = _PAGE_GCS_RO; + /* * You really shouldn't be using read() or write() on /dev/mem. This might go * away in the future. @@ -73,9 +75,11 @@ static int __init adjust_protection_map(void) protection_map[VM_EXEC | VM_SHARED] = PAGE_EXECONLY; } - if (lpa2_is_enabled()) + if (lpa2_is_enabled()) { for (int i = 0; i < ARRAY_SIZE(protection_map); i++) pgprot_val(protection_map[i]) &= ~PTE_SHARED; + gcs_page_prot &= ~PTE_SHARED; + } return 0; } @@ -87,7 +91,11 @@ pgprot_t vm_get_page_prot(vm_flags_t vm_flags) /* Short circuit GCS to avoid bloating the table. */ if (system_supports_gcs() && (vm_flags & VM_SHADOW_STACK)) { - prot = _PAGE_GCS_RO; + /* Honour mprotect(PROT_NONE) on shadow stack mappings */ + if (vm_flags & VM_ACCESS_FLAGS) + prot = gcs_page_prot; + else + prot = pgprot_val(protection_map[VM_NONE]); } else { prot = pgprot_val(protection_map[vm_flags & (VM_READ|VM_WRITE|VM_EXEC|VM_SHARED)]); diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c index 8e1d80a7033e34..60fd87a92de0c1 100644 --- a/arch/arm64/mm/mmu.c +++ b/arch/arm64/mm/mmu.c @@ -602,6 +602,8 @@ static int split_pmd(pmd_t *pmdp, pmd_t pmd, gfp_t gfp, bool to_cont) tableprot |= PMD_TABLE_PXN; prot = __pgprot((pgprot_val(prot) & ~PTE_TYPE_MASK) | PTE_TYPE_PAGE); + if (!pmd_valid(pmd)) + prot = pte_pgprot(pte_mkinvalid(pfn_pte(0, prot))); prot = __pgprot(pgprot_val(prot) & ~PTE_CONT); if (to_cont) prot = __pgprot(pgprot_val(prot) | PTE_CONT); @@ -647,6 +649,8 @@ static int split_pud(pud_t *pudp, pud_t pud, gfp_t gfp, bool to_cont) tableprot |= PUD_TABLE_PXN; prot = __pgprot((pgprot_val(prot) & ~PMD_TYPE_MASK) | PMD_TYPE_SECT); + if (!pud_valid(pud)) + prot = pmd_pgprot(pmd_mkinvalid(pfn_pmd(0, prot))); prot = __pgprot(pgprot_val(prot) & ~PTE_CONT); if (to_cont) prot = __pgprot(pgprot_val(prot) | PTE_CONT); diff --git a/arch/arm64/mm/pageattr.c b/arch/arm64/mm/pageattr.c index 7176ff39cb8796..672058657514c8 100644 --- a/arch/arm64/mm/pageattr.c +++ b/arch/arm64/mm/pageattr.c @@ -25,6 +25,11 @@ static ptdesc_t set_pageattr_masks(ptdesc_t val, struct mm_walk *walk) { struct page_change_data *masks = walk->private; + /* + * Some users clear and set bits which alias each other (e.g. PTE_NG and + * PTE_PRESENT_INVALID). It is therefore important that we always clear + * first then set. + */ val &= ~(pgprot_val(masks->clear_mask)); val |= (pgprot_val(masks->set_mask)); @@ -36,7 +41,7 @@ static int pageattr_pud_entry(pud_t *pud, unsigned long addr, { pud_t val = pudp_get(pud); - if (pud_sect(val)) { + if (pud_leaf(val)) { if (WARN_ON_ONCE((next - addr) != PUD_SIZE)) return -EINVAL; val = __pud(set_pageattr_masks(pud_val(val), walk)); @@ -52,7 +57,7 @@ static int pageattr_pmd_entry(pmd_t *pmd, unsigned long addr, { pmd_t val = pmdp_get(pmd); - if (pmd_sect(val)) { + if (pmd_leaf(val)) { if (WARN_ON_ONCE((next - addr) != PMD_SIZE)) return -EINVAL; val = __pmd(set_pageattr_masks(pmd_val(val), walk)); @@ -132,11 +137,12 @@ static int __change_memory_common(unsigned long start, unsigned long size, ret = update_range_prot(start, size, set_mask, clear_mask); /* - * If the memory is being made valid without changing any other bits - * then a TLBI isn't required as a non-valid entry cannot be cached in - * the TLB. + * If the memory is being switched from present-invalid to valid without + * changing any other bits then a TLBI isn't required as a non-valid + * entry cannot be cached in the TLB. */ - if (pgprot_val(set_mask) != PTE_VALID || pgprot_val(clear_mask)) + if (pgprot_val(set_mask) != PTE_PRESENT_VALID_KERNEL || + pgprot_val(clear_mask) != PTE_PRESENT_INVALID) flush_tlb_kernel_range(start, start + size); return ret; } @@ -237,18 +243,18 @@ int set_memory_valid(unsigned long addr, int numpages, int enable) { if (enable) return __change_memory_common(addr, PAGE_SIZE * numpages, - __pgprot(PTE_VALID), - __pgprot(0)); + __pgprot(PTE_PRESENT_VALID_KERNEL), + __pgprot(PTE_PRESENT_INVALID)); else return __change_memory_common(addr, PAGE_SIZE * numpages, - __pgprot(0), - __pgprot(PTE_VALID)); + __pgprot(PTE_PRESENT_INVALID), + __pgprot(PTE_PRESENT_VALID_KERNEL)); } int set_direct_map_invalid_noflush(struct page *page) { - pgprot_t clear_mask = __pgprot(PTE_VALID); - pgprot_t set_mask = __pgprot(0); + pgprot_t clear_mask = __pgprot(PTE_PRESENT_VALID_KERNEL); + pgprot_t set_mask = __pgprot(PTE_PRESENT_INVALID); if (!can_set_direct_map()) return 0; @@ -259,8 +265,8 @@ int set_direct_map_invalid_noflush(struct page *page) int set_direct_map_default_noflush(struct page *page) { - pgprot_t set_mask = __pgprot(PTE_VALID | PTE_WRITE); - pgprot_t clear_mask = __pgprot(PTE_RDONLY); + pgprot_t set_mask = __pgprot(PTE_PRESENT_VALID_KERNEL | PTE_WRITE); + pgprot_t clear_mask = __pgprot(PTE_PRESENT_INVALID | PTE_RDONLY); if (!can_set_direct_map()) return 0; @@ -296,8 +302,8 @@ static int __set_memory_enc_dec(unsigned long addr, * entries or Synchronous External Aborts caused by RIPAS_EMPTY */ ret = __change_memory_common(addr, PAGE_SIZE * numpages, - __pgprot(set_prot), - __pgprot(clear_prot | PTE_VALID)); + __pgprot(set_prot | PTE_PRESENT_INVALID), + __pgprot(clear_prot | PTE_PRESENT_VALID_KERNEL)); if (ret) return ret; @@ -311,8 +317,8 @@ static int __set_memory_enc_dec(unsigned long addr, return ret; return __change_memory_common(addr, PAGE_SIZE * numpages, - __pgprot(PTE_VALID), - __pgprot(0)); + __pgprot(PTE_PRESENT_VALID_KERNEL), + __pgprot(PTE_PRESENT_INVALID)); } static int realm_set_memory_encrypted(unsigned long addr, int numpages) @@ -404,15 +410,15 @@ bool kernel_page_present(struct page *page) pud = READ_ONCE(*pudp); if (pud_none(pud)) return false; - if (pud_sect(pud)) - return true; + if (pud_leaf(pud)) + return pud_valid(pud); pmdp = pmd_offset(pudp, addr); pmd = READ_ONCE(*pmdp); if (pmd_none(pmd)) return false; - if (pmd_sect(pmd)) - return true; + if (pmd_leaf(pmd)) + return pmd_valid(pmd); ptep = pte_offset_kernel(pmdp, addr); return pte_valid(__ptep_get(ptep)); diff --git a/arch/arm64/mm/proc.S b/arch/arm64/mm/proc.S index 5d907ce3b6d3f0..22866b49be3720 100644 --- a/arch/arm64/mm/proc.S +++ b/arch/arm64/mm/proc.S @@ -48,14 +48,14 @@ #define TCR_KASAN_SW_FLAGS 0 #endif -#ifdef CONFIG_KASAN_HW_TAGS -#define TCR_MTE_FLAGS TCR_EL1_TCMA1 | TCR_EL1_TBI1 | TCR_EL1_TBID1 -#elif defined(CONFIG_ARM64_MTE) +#ifdef CONFIG_ARM64_MTE /* * The mte_zero_clear_page_tags() implementation uses DC GZVA, which relies on - * TBI being enabled at EL1. + * TBI being enabled at EL1. TCMA1 is needed to treat accesses with the + * match-all tag (0xF) as Tag Unchecked, irrespective of the SCTLR_EL1.TCF + * setting. */ -#define TCR_MTE_FLAGS TCR_EL1_TBI1 | TCR_EL1_TBID1 +#define TCR_MTE_FLAGS TCR_EL1_TCMA1 | TCR_EL1_TBI1 | TCR_EL1_TBID1 #else #define TCR_MTE_FLAGS 0 #endif diff --git a/arch/arm64/mm/trans_pgd.c b/arch/arm64/mm/trans_pgd.c index 18543b603c77bd..cca9706a875c30 100644 --- a/arch/arm64/mm/trans_pgd.c +++ b/arch/arm64/mm/trans_pgd.c @@ -31,36 +31,6 @@ static void *trans_alloc(struct trans_pgd_info *info) return info->trans_alloc_page(info->trans_alloc_arg); } -static void _copy_pte(pte_t *dst_ptep, pte_t *src_ptep, unsigned long addr) -{ - pte_t pte = __ptep_get(src_ptep); - - if (pte_valid(pte)) { - /* - * Resume will overwrite areas that may be marked - * read only (code, rodata). Clear the RDONLY bit from - * the temporary mappings we use during restore. - */ - __set_pte(dst_ptep, pte_mkwrite_novma(pte)); - } else if (!pte_none(pte)) { - /* - * debug_pagealloc will removed the PTE_VALID bit if - * the page isn't in use by the resume kernel. It may have - * been in use by the original kernel, in which case we need - * to put it back in our copy to do the restore. - * - * Other cases include kfence / vmalloc / memfd_secret which - * may call `set_direct_map_invalid_noflush()`. - * - * Before marking this entry valid, check the pfn should - * be mapped. - */ - BUG_ON(!pfn_valid(pte_pfn(pte))); - - __set_pte(dst_ptep, pte_mkvalid(pte_mkwrite_novma(pte))); - } -} - static int copy_pte(struct trans_pgd_info *info, pmd_t *dst_pmdp, pmd_t *src_pmdp, unsigned long start, unsigned long end) { @@ -76,7 +46,11 @@ static int copy_pte(struct trans_pgd_info *info, pmd_t *dst_pmdp, src_ptep = pte_offset_kernel(src_pmdp, start); do { - _copy_pte(dst_ptep, src_ptep, addr); + pte_t pte = __ptep_get(src_ptep); + + if (pte_none(pte)) + continue; + __set_pte(dst_ptep, pte_mkvalid_k(pte_mkwrite_novma(pte))); } while (dst_ptep++, src_ptep++, addr += PAGE_SIZE, addr != end); return 0; @@ -109,8 +83,7 @@ static int copy_pmd(struct trans_pgd_info *info, pud_t *dst_pudp, if (copy_pte(info, dst_pmdp, src_pmdp, addr, next)) return -ENOMEM; } else { - set_pmd(dst_pmdp, - __pmd(pmd_val(pmd) & ~PMD_SECT_RDONLY)); + set_pmd(dst_pmdp, pmd_mkvalid_k(pmd_mkwrite_novma(pmd))); } } while (dst_pmdp++, src_pmdp++, addr = next, addr != end); @@ -145,8 +118,7 @@ static int copy_pud(struct trans_pgd_info *info, p4d_t *dst_p4dp, if (copy_pmd(info, dst_pudp, src_pudp, addr, next)) return -ENOMEM; } else { - set_pud(dst_pudp, - __pud(pud_val(pud) & ~PUD_SECT_RDONLY)); + set_pud(dst_pudp, pud_mkvalid_k(pud_mkwrite_novma(pud))); } } while (dst_pudp++, src_pudp++, addr = next, addr != end); diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c index b6eb7a465ad248..f9fcd699f2e94c 100644 --- a/arch/arm64/net/bpf_jit_comp.c +++ b/arch/arm64/net/bpf_jit_comp.c @@ -2126,7 +2126,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) extable_offset = round_up(prog_size + PLT_TARGET_SIZE, extable_align); image_size = extable_offset + extable_size; ro_header = bpf_jit_binary_pack_alloc(image_size, &ro_image_ptr, - sizeof(u32), &header, &image_ptr, + sizeof(u64), &header, &image_ptr, jit_fill_hole); if (!ro_header) { prog = orig_prog; @@ -2951,7 +2951,7 @@ int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type old_t, u64 plt_target = 0ULL; bool poking_bpf_entry; - if (!__bpf_address_lookup((unsigned long)ip, &size, &offset, namebuf)) + if (!bpf_address_lookup((unsigned long)ip, &size, &offset, namebuf)) /* Only poking bpf text is supported. Since kernel function * entry is set up by ftrace, we reply on ftrace to poke kernel * functions. diff --git a/arch/arm64/tools/cpucaps b/arch/arm64/tools/cpucaps index 0fac75f0153439..ee29e7c1c50465 100644 --- a/arch/arm64/tools/cpucaps +++ b/arch/arm64/tools/cpucaps @@ -8,6 +8,8 @@ BTI # Unreliable: use system_supports_32bit_el0() instead. HAS_32BIT_EL0_DO_NOT_USE HAS_32BIT_EL1 +HAS_ACTLR_VIRT +HAS_ACTLR_VIRT_APPLE HAS_ADDRESS_AUTH HAS_ADDRESS_AUTH_ARCH_QARMA3 HAS_ADDRESS_AUTH_ARCH_QARMA5 @@ -62,6 +64,8 @@ HAS_STAGE2_FWB HAS_TCR2 HAS_TIDCP1 HAS_TLB_RANGE +HAS_TSO_APPLE +HAS_TSO_FIXED HAS_VA52 HAS_VIRT_HOST_EXTN HAS_WFXT diff --git a/arch/csky/kernel/vmlinux.lds.S b/arch/csky/kernel/vmlinux.lds.S index d718961786d246..81943981b3af4a 100644 --- a/arch/csky/kernel/vmlinux.lds.S +++ b/arch/csky/kernel/vmlinux.lds.S @@ -109,6 +109,7 @@ SECTIONS STABS_DEBUG DWARF_DEBUG + MODINFO ELF_DETAILS DISCARDS diff --git a/arch/hexagon/kernel/vmlinux.lds.S b/arch/hexagon/kernel/vmlinux.lds.S index 1150b77fa281ce..aae22283b5e003 100644 --- a/arch/hexagon/kernel/vmlinux.lds.S +++ b/arch/hexagon/kernel/vmlinux.lds.S @@ -62,6 +62,7 @@ SECTIONS STABS_DEBUG DWARF_DEBUG + MODINFO ELF_DETAILS .hexagon.attributes 0 : { *(.hexagon.attributes) } diff --git a/arch/loongarch/include/asm/linkage.h b/arch/loongarch/include/asm/linkage.h index e2eca1a25b4ef7..a1bd6a3ee03a19 100644 --- a/arch/loongarch/include/asm/linkage.h +++ b/arch/loongarch/include/asm/linkage.h @@ -41,4 +41,40 @@ .cfi_endproc; \ SYM_END(name, SYM_T_NONE) +/* + * This is for the signal handler trampoline, which is used as the return + * address of the signal handlers in userspace instead of called normally. + * The long standing libgcc bug https://gcc.gnu.org/PR124050 requires a + * nop between .cfi_startproc and the actual address of the trampoline, so + * we cannot simply use SYM_FUNC_START. + * + * This wrapper also contains all the .cfi_* directives for recovering + * the content of the GPRs and the "return address" (where the rt_sigreturn + * syscall will jump to), assuming there is a struct rt_sigframe (where + * a struct sigcontext containing those information we need to recover) at + * $sp. The "DWARF for the LoongArch(TM) Architecture" manual states + * column 0 is for $zero, but it does not make too much sense to + * save/restore the hardware zero register. Repurpose this column here + * for the return address (here it's not the content of $ra we cannot use + * the default column 3). + */ +#define SYM_SIGFUNC_START(name) \ + .cfi_startproc; \ + .cfi_signal_frame; \ + .cfi_def_cfa 3, RT_SIGFRAME_SC; \ + .cfi_return_column 0; \ + .cfi_offset 0, SC_PC; \ + \ + .irp num, 1, 2, 3, 4, 5, 6, 7, 8, \ + 9, 10, 11, 12, 13, 14, 15, 16, \ + 17, 18, 19, 20, 21, 22, 23, 24, \ + 25, 26, 27, 28, 29, 30, 31; \ + .cfi_offset \num, SC_REGS + \num * SZREG; \ + .endr; \ + \ + nop; \ + SYM_START(name, SYM_L_GLOBAL, SYM_A_ALIGN) + +#define SYM_SIGFUNC_END(name) SYM_FUNC_END(name) + #endif diff --git a/arch/loongarch/include/asm/setup.h b/arch/loongarch/include/asm/setup.h index 3c2fb16b11b649..f81375e5e89c0d 100644 --- a/arch/loongarch/include/asm/setup.h +++ b/arch/loongarch/include/asm/setup.h @@ -7,6 +7,7 @@ #define _LOONGARCH_SETUP_H #include +#include #include #include @@ -14,6 +15,8 @@ extern unsigned long eentry; extern unsigned long tlbrentry; +extern unsigned long pcpu_handlers[NR_CPUS]; +extern long exception_handlers[VECSIZE * 128 / sizeof(long)]; extern char init_command_line[COMMAND_LINE_SIZE]; extern void tlb_init(int cpu); extern void cpu_cache_init(void); diff --git a/arch/loongarch/include/asm/sigframe.h b/arch/loongarch/include/asm/sigframe.h new file mode 100644 index 00000000000000..109298b8d7e0b0 --- /dev/null +++ b/arch/loongarch/include/asm/sigframe.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ + +#include +#include + +struct rt_sigframe { + struct siginfo rs_info; + struct ucontext rs_uctx; +}; diff --git a/arch/loongarch/include/asm/topology.h b/arch/loongarch/include/asm/topology.h index f06e7ff25bb7cb..6b79d6183085a2 100644 --- a/arch/loongarch/include/asm/topology.h +++ b/arch/loongarch/include/asm/topology.h @@ -12,7 +12,7 @@ extern cpumask_t cpus_on_node[]; -#define cpumask_of_node(node) (&cpus_on_node[node]) +#define cpumask_of_node(node) ((node) == NUMA_NO_NODE ? cpu_all_mask : &cpus_on_node[node]) struct pci_bus; extern int pcibus_to_node(struct pci_bus *); diff --git a/arch/loongarch/include/asm/uaccess.h b/arch/loongarch/include/asm/uaccess.h index 4e259d490e4567..438269313e78c4 100644 --- a/arch/loongarch/include/asm/uaccess.h +++ b/arch/loongarch/include/asm/uaccess.h @@ -253,8 +253,13 @@ do { \ \ __get_kernel_common(*((type *)(dst)), sizeof(type), \ (__force type *)(src)); \ - if (unlikely(__gu_err)) \ + if (unlikely(__gu_err)) { \ + pr_info("%s: memory access failed, ecode 0x%x\n", \ + __func__, read_csr_excode()); \ + pr_info("%s: the caller is %pS\n", \ + __func__, __builtin_return_address(0)); \ goto err_label; \ + } \ } while (0) #define __put_kernel_nofault(dst, src, type, err_label) \ @@ -264,8 +269,13 @@ do { \ \ __pu_val = *(__force type *)(src); \ __put_kernel_common(((type *)(dst)), sizeof(type)); \ - if (unlikely(__pu_err)) \ + if (unlikely(__pu_err)) { \ + pr_info("%s: memory access failed, ecode 0x%x\n", \ + __func__, read_csr_excode()); \ + pr_info("%s: the caller is %pS\n", \ + __func__, __builtin_return_address(0)); \ goto err_label; \ + } \ } while (0) extern unsigned long __copy_user(void *to, const void *from, __kernel_size_t n); diff --git a/arch/loongarch/kernel/asm-offsets.c b/arch/loongarch/kernel/asm-offsets.c index 3017c715760099..2cc953f113ac04 100644 --- a/arch/loongarch/kernel/asm-offsets.c +++ b/arch/loongarch/kernel/asm-offsets.c @@ -16,6 +16,7 @@ #include #include #include +#include #include static void __used output_ptreg_defines(void) @@ -220,6 +221,7 @@ static void __used output_sc_defines(void) COMMENT("Linux sigcontext offsets."); OFFSET(SC_REGS, sigcontext, sc_regs); OFFSET(SC_PC, sigcontext, sc_pc); + OFFSET(RT_SIGFRAME_SC, rt_sigframe, rs_uctx.uc_mcontext); BLANK(); } diff --git a/arch/loongarch/kernel/env.c b/arch/loongarch/kernel/env.c index 841206fde3ab72..652456768b5516 100644 --- a/arch/loongarch/kernel/env.c +++ b/arch/loongarch/kernel/env.c @@ -42,16 +42,15 @@ static int __init init_cpu_fullname(void) int cpu, ret; char *cpuname; const char *model; - struct device_node *root; /* Parsing cpuname from DTS model property */ - root = of_find_node_by_path("/"); - ret = of_property_read_string(root, "model", &model); + ret = of_property_read_string(of_root, "model", &model); if (ret == 0) { cpuname = kstrdup(model, GFP_KERNEL); + if (!cpuname) + return -ENOMEM; loongson_sysconf.cpuname = strsep(&cpuname, " "); } - of_node_put(root); if (loongson_sysconf.cpuname && !strncmp(loongson_sysconf.cpuname, "Loongson", 8)) { for (cpu = 0; cpu < NR_CPUS; cpu++) diff --git a/arch/loongarch/kernel/inst.c b/arch/loongarch/kernel/inst.c index bf037f0c6b26c9..9a9c34ea24b60a 100644 --- a/arch/loongarch/kernel/inst.c +++ b/arch/loongarch/kernel/inst.c @@ -246,18 +246,21 @@ static int text_copy_cb(void *data) if (smp_processor_id() == copy->cpu) { ret = copy_to_kernel_nofault(copy->dst, copy->src, copy->len); - if (ret) + if (ret) { pr_err("%s: operation failed\n", __func__); + return ret; + } } flush_icache_range((unsigned long)copy->dst, (unsigned long)copy->dst + copy->len); - return ret; + return 0; } int larch_insn_text_copy(void *dst, void *src, size_t len) { int ret = 0; + int err = 0; size_t start, end; struct insn_copy copy = { .dst = dst, @@ -269,9 +272,19 @@ int larch_insn_text_copy(void *dst, void *src, size_t len) start = round_down((size_t)dst, PAGE_SIZE); end = round_up((size_t)dst + len, PAGE_SIZE); - set_memory_rw(start, (end - start) / PAGE_SIZE); + err = set_memory_rw(start, (end - start) / PAGE_SIZE); + if (err) { + pr_info("%s: set_memory_rw() failed\n", __func__); + return err; + } + ret = stop_machine(text_copy_cb, ©, cpu_online_mask); - set_memory_rox(start, (end - start) / PAGE_SIZE); + + err = set_memory_rox(start, (end - start) / PAGE_SIZE); + if (err) { + pr_info("%s: set_memory_rox() failed\n", __func__); + return err; + } return ret; } diff --git a/arch/loongarch/kernel/setup.c b/arch/loongarch/kernel/setup.c index 20cb6f30645683..2b260d15b2e251 100644 --- a/arch/loongarch/kernel/setup.c +++ b/arch/loongarch/kernel/setup.c @@ -421,6 +421,7 @@ static void __init arch_mem_init(char **cmdline_p) PFN_UP(__pa_symbol(&__nosave_end))); memblock_dump_all(); + memblock_set_bottom_up(false); early_memtest(PFN_PHYS(ARCH_PFN_OFFSET), PFN_PHYS(max_low_pfn)); } diff --git a/arch/loongarch/kernel/signal.c b/arch/loongarch/kernel/signal.c index c9f7ca778364ed..d4151d2fb82ee2 100644 --- a/arch/loongarch/kernel/signal.c +++ b/arch/loongarch/kernel/signal.c @@ -35,6 +35,7 @@ #include #include #include +#include #include #include @@ -51,11 +52,6 @@ #define lock_lbt_owner() ({ preempt_disable(); pagefault_disable(); }) #define unlock_lbt_owner() ({ pagefault_enable(); preempt_enable(); }) -struct rt_sigframe { - struct siginfo rs_info; - struct ucontext rs_uctx; -}; - struct _ctx_layout { struct sctx_info *addr; unsigned int size; diff --git a/arch/loongarch/kernel/unwind_orc.c b/arch/loongarch/kernel/unwind_orc.c index 8a6e3429a860e9..9cfb5bb1991f26 100644 --- a/arch/loongarch/kernel/unwind_orc.c +++ b/arch/loongarch/kernel/unwind_orc.c @@ -350,7 +350,21 @@ EXPORT_SYMBOL_GPL(unwind_start); static inline unsigned long bt_address(unsigned long ra) { - extern unsigned long eentry; +#if defined(CONFIG_NUMA) && !defined(CONFIG_PREEMPT_RT) + int cpu; + int vec_sz = sizeof(exception_handlers); + + for_each_possible_cpu(cpu) { + if (!pcpu_handlers[cpu]) + continue; + + if (ra >= pcpu_handlers[cpu] && + ra < pcpu_handlers[cpu] + vec_sz) { + ra = ra + eentry - pcpu_handlers[cpu]; + break; + } + } +#endif if (ra >= eentry && ra < eentry + EXCCODE_INT_END * VECSIZE) { unsigned long func; @@ -494,7 +508,7 @@ bool unwind_next_frame(struct unwind_state *state) state->pc = bt_address(pc); if (!state->pc) { - pr_err("cannot find unwind pc at %p\n", (void *)pc); + pr_err("cannot find unwind pc at %px\n", (void *)pc); goto err; } diff --git a/arch/loongarch/kernel/unwind_prologue.c b/arch/loongarch/kernel/unwind_prologue.c index 729e775bd40dde..da07acad7973aa 100644 --- a/arch/loongarch/kernel/unwind_prologue.c +++ b/arch/loongarch/kernel/unwind_prologue.c @@ -23,10 +23,6 @@ extern const int unwind_hint_lasx; extern const int unwind_hint_lbt; extern const int unwind_hint_ri; extern const int unwind_hint_watch; -extern unsigned long eentry; -#ifdef CONFIG_NUMA -extern unsigned long pcpu_handlers[NR_CPUS]; -#endif static inline bool scan_handlers(unsigned long entry_offset) { @@ -65,7 +61,7 @@ static inline bool scan_handlers(unsigned long entry_offset) static inline bool fix_exception(unsigned long pc) { -#ifdef CONFIG_NUMA +#if defined(CONFIG_NUMA) && !defined(CONFIG_PREEMPT_RT) int cpu; for_each_possible_cpu(cpu) { diff --git a/arch/loongarch/kernel/vmlinux.lds.S b/arch/loongarch/kernel/vmlinux.lds.S index 08ea921cdec16e..d0e1377a041d63 100644 --- a/arch/loongarch/kernel/vmlinux.lds.S +++ b/arch/loongarch/kernel/vmlinux.lds.S @@ -147,6 +147,7 @@ SECTIONS STABS_DEBUG DWARF_DEBUG + MODINFO ELF_DETAILS #ifdef CONFIG_EFI_STUB diff --git a/arch/loongarch/kvm/intc/eiointc.c b/arch/loongarch/kvm/intc/eiointc.c index dfaf6ccfdd8b31..8861d76f7e10fb 100644 --- a/arch/loongarch/kvm/intc/eiointc.c +++ b/arch/loongarch/kvm/intc/eiointc.c @@ -83,7 +83,7 @@ static inline void eiointc_update_sw_coremap(struct loongarch_eiointc *s, if (!(s->status & BIT(EIOINTC_ENABLE_CPU_ENCODE))) { cpuid = ffs(cpuid) - 1; - cpuid = (cpuid >= 4) ? 0 : cpuid; + cpuid = ((cpuid < 0) || (cpuid >= 4)) ? 0 : cpuid; } vcpu = kvm_get_vcpu_by_cpuid(s->kvm, cpuid); @@ -481,34 +481,34 @@ static int kvm_eiointc_regs_access(struct kvm_device *dev, switch (addr) { case EIOINTC_NODETYPE_START ... EIOINTC_NODETYPE_END: offset = (addr - EIOINTC_NODETYPE_START) / 4; - p = s->nodetype + offset * 4; + p = (void *)s->nodetype + offset * 4; break; case EIOINTC_IPMAP_START ... EIOINTC_IPMAP_END: offset = (addr - EIOINTC_IPMAP_START) / 4; - p = &s->ipmap + offset * 4; + p = (void *)&s->ipmap + offset * 4; break; case EIOINTC_ENABLE_START ... EIOINTC_ENABLE_END: offset = (addr - EIOINTC_ENABLE_START) / 4; - p = s->enable + offset * 4; + p = (void *)s->enable + offset * 4; break; case EIOINTC_BOUNCE_START ... EIOINTC_BOUNCE_END: offset = (addr - EIOINTC_BOUNCE_START) / 4; - p = s->bounce + offset * 4; + p = (void *)s->bounce + offset * 4; break; case EIOINTC_ISR_START ... EIOINTC_ISR_END: offset = (addr - EIOINTC_ISR_START) / 4; - p = s->isr + offset * 4; + p = (void *)s->isr + offset * 4; break; case EIOINTC_COREISR_START ... EIOINTC_COREISR_END: if (cpu >= s->num_cpu) return -EINVAL; offset = (addr - EIOINTC_COREISR_START) / 4; - p = s->coreisr[cpu] + offset * 4; + p = (void *)s->coreisr[cpu] + offset * 4; break; case EIOINTC_COREMAP_START ... EIOINTC_COREMAP_END: offset = (addr - EIOINTC_COREMAP_START) / 4; - p = s->coremap + offset * 4; + p = (void *)s->coremap + offset * 4; break; default: kvm_err("%s: unknown eiointc register, addr = %d\n", __func__, addr); diff --git a/arch/loongarch/kvm/vcpu.c b/arch/loongarch/kvm/vcpu.c index 656b954c1134b1..3a359d83b022ab 100644 --- a/arch/loongarch/kvm/vcpu.c +++ b/arch/loongarch/kvm/vcpu.c @@ -14,7 +14,7 @@ #define CREATE_TRACE_POINTS #include "trace.h" -const struct _kvm_stats_desc kvm_vcpu_stats_desc[] = { +const struct kvm_stats_desc kvm_vcpu_stats_desc[] = { KVM_GENERIC_VCPU_STATS(), STATS_DESC_COUNTER(VCPU, int_exits), STATS_DESC_COUNTER(VCPU, idle_exits), @@ -562,6 +562,9 @@ struct kvm_vcpu *kvm_get_vcpu_by_cpuid(struct kvm *kvm, int cpuid) { struct kvm_phyid_map *map; + if (cpuid < 0) + return NULL; + if (cpuid >= KVM_MAX_PHYID) return NULL; diff --git a/arch/loongarch/kvm/vm.c b/arch/loongarch/kvm/vm.c index 194ccbcdc3b383..7deff56e0e1ab9 100644 --- a/arch/loongarch/kvm/vm.c +++ b/arch/loongarch/kvm/vm.c @@ -10,7 +10,7 @@ #include #include -const struct _kvm_stats_desc kvm_vm_stats_desc[] = { +const struct kvm_stats_desc kvm_vm_stats_desc[] = { KVM_GENERIC_VM_STATS(), STATS_DESC_ICOUNTER(VM, pages), STATS_DESC_ICOUNTER(VM, hugepages), diff --git a/arch/loongarch/mm/kasan_init.c b/arch/loongarch/mm/kasan_init.c index 170da98ad4f551..0fc02ca0645738 100644 --- a/arch/loongarch/mm/kasan_init.c +++ b/arch/loongarch/mm/kasan_init.c @@ -40,39 +40,43 @@ static pgd_t kasan_pg_dir[PTRS_PER_PGD] __initdata __aligned(PAGE_SIZE); #define __pte_none(early, pte) (early ? pte_none(pte) : \ ((pte_val(pte) & _PFN_MASK) == (unsigned long)__pa(kasan_early_shadow_page))) -void *kasan_mem_to_shadow(const void *addr) +static void *mem_to_shadow(const void *addr) { - if (!kasan_enabled()) { + unsigned long offset = 0; + unsigned long maddr = (unsigned long)addr; + unsigned long xrange = (maddr >> XRANGE_SHIFT) & 0xffff; + + if (maddr >= FIXADDR_START) return (void *)(kasan_early_shadow_page); - } else { - unsigned long maddr = (unsigned long)addr; - unsigned long xrange = (maddr >> XRANGE_SHIFT) & 0xffff; - unsigned long offset = 0; - - if (maddr >= FIXADDR_START) - return (void *)(kasan_early_shadow_page); - - maddr &= XRANGE_SHADOW_MASK; - switch (xrange) { - case XKPRANGE_CC_SEG: - offset = XKPRANGE_CC_SHADOW_OFFSET; - break; - case XKPRANGE_UC_SEG: - offset = XKPRANGE_UC_SHADOW_OFFSET; - break; - case XKPRANGE_WC_SEG: - offset = XKPRANGE_WC_SHADOW_OFFSET; - break; - case XKVRANGE_VC_SEG: - offset = XKVRANGE_VC_SHADOW_OFFSET; - break; - default: - WARN_ON(1); - return NULL; - } - return (void *)((maddr >> KASAN_SHADOW_SCALE_SHIFT) + offset); + maddr &= XRANGE_SHADOW_MASK; + switch (xrange) { + case XKPRANGE_CC_SEG: + offset = XKPRANGE_CC_SHADOW_OFFSET; + break; + case XKPRANGE_UC_SEG: + offset = XKPRANGE_UC_SHADOW_OFFSET; + break; + case XKPRANGE_WC_SEG: + offset = XKPRANGE_WC_SHADOW_OFFSET; + break; + case XKVRANGE_VC_SEG: + offset = XKVRANGE_VC_SHADOW_OFFSET; + break; + default: + WARN_ON(1); + return NULL; } + + return (void *)((maddr >> KASAN_SHADOW_SCALE_SHIFT) + offset); +} + +void *kasan_mem_to_shadow(const void *addr) +{ + if (kasan_enabled()) + return mem_to_shadow(addr); + else + return (void *)(kasan_early_shadow_page); } const void *kasan_shadow_to_mem(const void *shadow_addr) @@ -293,11 +297,8 @@ void __init kasan_init(void) /* Maps everything to a single page of zeroes */ kasan_pgd_populate(KASAN_SHADOW_START, KASAN_SHADOW_END, NUMA_NO_NODE, true); - kasan_populate_early_shadow(kasan_mem_to_shadow((void *)VMALLOC_START), - kasan_mem_to_shadow((void *)KFENCE_AREA_END)); - - /* Enable KASAN here before kasan_mem_to_shadow(). */ - kasan_init_generic(); + kasan_populate_early_shadow(mem_to_shadow((void *)VMALLOC_START), + mem_to_shadow((void *)KFENCE_AREA_END)); /* Populate the linear mapping */ for_each_mem_range(i, &pa_start, &pa_end) { @@ -307,13 +308,13 @@ void __init kasan_init(void) if (start >= end) break; - kasan_map_populate((unsigned long)kasan_mem_to_shadow(start), - (unsigned long)kasan_mem_to_shadow(end), NUMA_NO_NODE); + kasan_map_populate((unsigned long)mem_to_shadow(start), + (unsigned long)mem_to_shadow(end), NUMA_NO_NODE); } /* Populate modules mapping */ - kasan_map_populate((unsigned long)kasan_mem_to_shadow((void *)MODULES_VADDR), - (unsigned long)kasan_mem_to_shadow((void *)MODULES_END), NUMA_NO_NODE); + kasan_map_populate((unsigned long)mem_to_shadow((void *)MODULES_VADDR), + (unsigned long)mem_to_shadow((void *)MODULES_END), NUMA_NO_NODE); /* * KAsan may reuse the contents of kasan_early_shadow_pte directly, so we * should make sure that it maps the zero page read-only. @@ -328,4 +329,5 @@ void __init kasan_init(void) /* At this point kasan is fully initialized. Enable error messages */ init_task.kasan_depth = 0; + kasan_init_generic(); } diff --git a/arch/loongarch/mm/tlb.c b/arch/loongarch/mm/tlb.c index 6a3c91b9cacdcf..aaf7d685cc2aaf 100644 --- a/arch/loongarch/mm/tlb.c +++ b/arch/loongarch/mm/tlb.c @@ -202,7 +202,7 @@ void __update_tlb(struct vm_area_struct *vma, unsigned long address, pte_t *ptep local_irq_restore(flags); } -static void setup_ptwalker(void) +static void __no_sanitize_address setup_ptwalker(void) { unsigned long pwctl0, pwctl1; unsigned long pgd_i = 0, pgd_w = 0; @@ -262,7 +262,6 @@ static void output_pgtable_bits_defines(void) #ifdef CONFIG_NUMA unsigned long pcpu_handlers[NR_CPUS]; #endif -extern long exception_handlers[VECSIZE * 128 / sizeof(long)]; static void setup_tlb_handler(int cpu) { diff --git a/arch/loongarch/net/bpf_jit.c b/arch/loongarch/net/bpf_jit.c index d1d5a65308b9eb..3b63bc5b99d9a4 100644 --- a/arch/loongarch/net/bpf_jit.c +++ b/arch/loongarch/net/bpf_jit.c @@ -1319,7 +1319,7 @@ int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type old_t, /* Only poking bpf text is supported. Since kernel function entry * is set up by ftrace, we rely on ftrace to poke kernel functions. */ - if (!__bpf_address_lookup((unsigned long)ip, &size, &offset, namebuf)) + if (!bpf_address_lookup((unsigned long)ip, &size, &offset, namebuf)) return -ENOTSUPP; image = ip - offset; diff --git a/arch/loongarch/pci/pci.c b/arch/loongarch/pci/pci.c index d923295ab8c665..d233ea2218fe0a 100644 --- a/arch/loongarch/pci/pci.c +++ b/arch/loongarch/pci/pci.c @@ -5,9 +5,11 @@ #include #include #include +#include #include #include #include +#include #include #include @@ -15,6 +17,9 @@ #define PCI_DEVICE_ID_LOONGSON_DC1 0x7a06 #define PCI_DEVICE_ID_LOONGSON_DC2 0x7a36 #define PCI_DEVICE_ID_LOONGSON_DC3 0x7a46 +#define PCI_DEVICE_ID_LOONGSON_GPU1 0x7a15 +#define PCI_DEVICE_ID_LOONGSON_GPU2 0x7a25 +#define PCI_DEVICE_ID_LOONGSON_GPU3 0x7a35 int raw_pci_read(unsigned int domain, unsigned int bus, unsigned int devfn, int reg, int len, u32 *val) @@ -99,3 +104,78 @@ static void pci_fixup_vgadev(struct pci_dev *pdev) DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_LOONGSON, PCI_DEVICE_ID_LOONGSON_DC1, pci_fixup_vgadev); DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_LOONGSON, PCI_DEVICE_ID_LOONGSON_DC2, pci_fixup_vgadev); DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_LOONGSON, PCI_DEVICE_ID_LOONGSON_DC3, pci_fixup_vgadev); + +#define CRTC_NUM_MAX 2 +#define CRTC_OUTPUT_ENABLE 0x100 + +static void loongson_gpu_fixup_dma_hang(struct pci_dev *pdev, bool on) +{ + u32 i, val, count, crtc_offset, device; + void __iomem *crtc_reg, *base, *regbase; + static u32 crtc_status[CRTC_NUM_MAX] = { 0 }; + + base = pdev->bus->ops->map_bus(pdev->bus, pdev->devfn + 1, 0); + device = readw(base + PCI_DEVICE_ID); + + regbase = ioremap(readq(base + PCI_BASE_ADDRESS_0) & ~0xffull, SZ_64K); + if (!regbase) { + pci_err(pdev, "Failed to ioremap()\n"); + return; + } + + switch (device) { + case PCI_DEVICE_ID_LOONGSON_DC2: + crtc_reg = regbase + 0x1240; + crtc_offset = 0x10; + break; + case PCI_DEVICE_ID_LOONGSON_DC3: + crtc_reg = regbase; + crtc_offset = 0x400; + break; + } + + for (i = 0; i < CRTC_NUM_MAX; i++, crtc_reg += crtc_offset) { + val = readl(crtc_reg); + + if (!on) + crtc_status[i] = val; + + /* No need to fixup if the status is off at startup. */ + if (!(crtc_status[i] & CRTC_OUTPUT_ENABLE)) + continue; + + if (on) + val |= CRTC_OUTPUT_ENABLE; + else + val &= ~CRTC_OUTPUT_ENABLE; + + mb(); + writel(val, crtc_reg); + + for (count = 0; count < 40; count++) { + val = readl(crtc_reg) & CRTC_OUTPUT_ENABLE; + if ((on && val) || (!on && !val)) + break; + udelay(1000); + } + + pci_info(pdev, "DMA hang fixup at reg[0x%lx]: 0x%x\n", + (unsigned long)crtc_reg & 0xffff, readl(crtc_reg)); + } + + iounmap(regbase); +} + +static void pci_fixup_dma_hang_early(struct pci_dev *pdev) +{ + loongson_gpu_fixup_dma_hang(pdev, false); +} +DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_LOONGSON, PCI_DEVICE_ID_LOONGSON_GPU2, pci_fixup_dma_hang_early); +DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_LOONGSON, PCI_DEVICE_ID_LOONGSON_GPU3, pci_fixup_dma_hang_early); + +static void pci_fixup_dma_hang_final(struct pci_dev *pdev) +{ + loongson_gpu_fixup_dma_hang(pdev, true); +} +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_LOONGSON, PCI_DEVICE_ID_LOONGSON_GPU2, pci_fixup_dma_hang_final); +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_LOONGSON, PCI_DEVICE_ID_LOONGSON_GPU3, pci_fixup_dma_hang_final); diff --git a/arch/loongarch/vdso/Makefile b/arch/loongarch/vdso/Makefile index 520f1513f07ddb..294c16b9517fd3 100644 --- a/arch/loongarch/vdso/Makefile +++ b/arch/loongarch/vdso/Makefile @@ -26,7 +26,7 @@ cflags-vdso := $(ccflags-vdso) \ $(filter -W%,$(filter-out -Wa$(comma)%,$(KBUILD_CFLAGS))) \ -std=gnu11 -fms-extensions -O2 -g -fno-strict-aliasing -fno-common -fno-builtin \ -fno-stack-protector -fno-jump-tables -DDISABLE_BRANCH_PROFILING \ - $(call cc-option, -fno-asynchronous-unwind-tables) \ + $(call cc-option, -fasynchronous-unwind-tables) \ $(call cc-option, -fno-stack-protector) aflags-vdso := $(ccflags-vdso) \ -D__ASSEMBLY__ -Wa,-gdwarf-2 @@ -41,7 +41,7 @@ endif # VDSO linker flags. ldflags-y := -Bsymbolic --no-undefined -soname=linux-vdso.so.1 \ - $(filter -E%,$(KBUILD_CFLAGS)) -shared --build-id -T + $(filter -E%,$(KBUILD_CFLAGS)) -shared --build-id --eh-frame-hdr -T # # Shared build commands. diff --git a/arch/loongarch/vdso/sigreturn.S b/arch/loongarch/vdso/sigreturn.S index 9cb3c58fad03bf..59f940d928de7b 100644 --- a/arch/loongarch/vdso/sigreturn.S +++ b/arch/loongarch/vdso/sigreturn.S @@ -12,13 +12,13 @@ #include #include +#include .section .text - .cfi_sections .debug_frame -SYM_FUNC_START(__vdso_rt_sigreturn) +SYM_SIGFUNC_START(__vdso_rt_sigreturn) li.w a7, __NR_rt_sigreturn syscall 0 -SYM_FUNC_END(__vdso_rt_sigreturn) +SYM_SIGFUNC_END(__vdso_rt_sigreturn) diff --git a/arch/m68k/kernel/vmlinux-nommu.lds b/arch/m68k/kernel/vmlinux-nommu.lds index 2624fc18c131f1..45d7f4b0177b49 100644 --- a/arch/m68k/kernel/vmlinux-nommu.lds +++ b/arch/m68k/kernel/vmlinux-nommu.lds @@ -85,6 +85,7 @@ SECTIONS { _end = .; STABS_DEBUG + MODINFO ELF_DETAILS /* Sections to be discarded */ diff --git a/arch/m68k/kernel/vmlinux-std.lds b/arch/m68k/kernel/vmlinux-std.lds index 1ccdd04ae46242..7326586afe15f5 100644 --- a/arch/m68k/kernel/vmlinux-std.lds +++ b/arch/m68k/kernel/vmlinux-std.lds @@ -58,6 +58,7 @@ SECTIONS _end = . ; STABS_DEBUG + MODINFO ELF_DETAILS /* Sections to be discarded */ diff --git a/arch/m68k/kernel/vmlinux-sun3.lds b/arch/m68k/kernel/vmlinux-sun3.lds index f13ddcc2af5c28..1b19fef201fba6 100644 --- a/arch/m68k/kernel/vmlinux-sun3.lds +++ b/arch/m68k/kernel/vmlinux-sun3.lds @@ -51,6 +51,7 @@ __init_begin = .; _end = . ; STABS_DEBUG + MODINFO ELF_DETAILS /* Sections to be discarded */ diff --git a/arch/m68k/lib/memmove.c b/arch/m68k/lib/memmove.c index 6519f7f349f665..e33f00b02e4c0f 100644 --- a/arch/m68k/lib/memmove.c +++ b/arch/m68k/lib/memmove.c @@ -24,6 +24,15 @@ void *memmove(void *dest, const void *src, size_t n) src = csrc; n--; } +#if defined(CONFIG_M68000) + if ((long)src & 1) { + char *cdest = dest; + const char *csrc = src; + for (; n; n--) + *cdest++ = *csrc++; + return xdest; + } +#endif if (n > 2 && (long)dest & 2) { short *sdest = dest; const short *ssrc = src; @@ -66,6 +75,15 @@ void *memmove(void *dest, const void *src, size_t n) src = csrc; n--; } +#if defined(CONFIG_M68000) + if ((long)src & 1) { + char *cdest = dest; + const char *csrc = src; + for (; n; n--) + *--cdest = *--csrc; + return xdest; + } +#endif if (n > 2 && (long)dest & 2) { short *sdest = dest; const short *ssrc = src; diff --git a/arch/mips/Kconfig b/arch/mips/Kconfig index b88b97139fa8e1..d87db7c535ea1d 100644 --- a/arch/mips/Kconfig +++ b/arch/mips/Kconfig @@ -1408,7 +1408,6 @@ config CPU_LOONGSON32 select CPU_MIPS32 select CPU_MIPSR2 select CPU_HAS_PREFETCH - select CPU_HAS_LOAD_STORE_LR select CPU_SUPPORTS_32BIT_KERNEL select CPU_SUPPORTS_HIGHMEM select CPU_SUPPORTS_CPUFREQ diff --git a/arch/mips/include/asm/cpu-features.h b/arch/mips/include/asm/cpu-features.h index 404390bb87eaf5..3f11e5218e6c65 100644 --- a/arch/mips/include/asm/cpu-features.h +++ b/arch/mips/include/asm/cpu-features.h @@ -484,7 +484,6 @@ # endif # ifndef cpu_vmbits # define cpu_vmbits cpu_data[0].vmbits -# define __NEED_VMBITS_PROBE # endif #endif diff --git a/arch/mips/include/asm/cpu-info.h b/arch/mips/include/asm/cpu-info.h index fd60837ce50bc0..211b578af6aa0a 100644 --- a/arch/mips/include/asm/cpu-info.h +++ b/arch/mips/include/asm/cpu-info.h @@ -80,9 +80,7 @@ struct cpuinfo_mips { int srsets; /* Shadow register sets */ int package;/* physical package number */ unsigned int globalnumber; -#ifdef CONFIG_64BIT int vmbits; /* Virtual memory size in bits */ -#endif void *data; /* Additional data */ unsigned int watch_reg_count; /* Number that exist */ unsigned int watch_reg_use_cnt; /* Usable by ptrace */ diff --git a/arch/mips/include/asm/mach-loongson2ef/loongson.h b/arch/mips/include/asm/mach-loongson2ef/loongson.h index 4a098fb1023251..0e586787eb87a7 100644 --- a/arch/mips/include/asm/mach-loongson2ef/loongson.h +++ b/arch/mips/include/asm/mach-loongson2ef/loongson.h @@ -324,4 +324,10 @@ extern unsigned long _loongson_addrwincfg_base; #endif /* ! CONFIG_CPU_SUPPORTS_ADDRWINCFG */ +#ifdef CONFIG_PCI +void loongson2ef_pcibios_init(void); +#else +static inline void loongson2ef_pcibios_init(void) { } +#endif + #endif /* __ASM_MACH_LOONGSON2EF_LOONGSON_H */ diff --git a/arch/mips/include/asm/mach-loongson64/topology.h b/arch/mips/include/asm/mach-loongson64/topology.h index 3414a1fd17835e..89bb4deab98a67 100644 --- a/arch/mips/include/asm/mach-loongson64/topology.h +++ b/arch/mips/include/asm/mach-loongson64/topology.h @@ -7,7 +7,7 @@ #define cpu_to_node(cpu) (cpu_logical_map(cpu) >> 2) extern cpumask_t __node_cpumask[]; -#define cpumask_of_node(node) (&__node_cpumask[node]) +#define cpumask_of_node(node) ((node) == NUMA_NO_NODE ? cpu_all_mask : &__node_cpumask[node]) struct pci_bus; extern int pcibus_to_node(struct pci_bus *); diff --git a/arch/mips/include/asm/mipsregs.h b/arch/mips/include/asm/mipsregs.h index f799c0d723dac2..12a095dbf9e2a9 100644 --- a/arch/mips/include/asm/mipsregs.h +++ b/arch/mips/include/asm/mipsregs.h @@ -1871,6 +1871,8 @@ do { \ #define read_c0_entryhi() __read_ulong_c0_register($10, 0) #define write_c0_entryhi(val) __write_ulong_c0_register($10, 0, val) +#define read_c0_entryhi_64() __read_64bit_c0_register($10, 0) +#define write_c0_entryhi_64(val) __write_64bit_c0_register($10, 0, val) #define read_c0_guestctl1() __read_32bit_c0_register($10, 4) #define write_c0_guestctl1(val) __write_32bit_c0_register($10, 4, val) diff --git a/arch/mips/kernel/cpu-probe.c b/arch/mips/kernel/cpu-probe.c index 1e49e05ac8b1c4..489612ed9d4987 100644 --- a/arch/mips/kernel/cpu-probe.c +++ b/arch/mips/kernel/cpu-probe.c @@ -210,11 +210,14 @@ static inline void set_elf_base_platform(const char *plat) static inline void cpu_probe_vmbits(struct cpuinfo_mips *c) { -#ifdef __NEED_VMBITS_PROBE - write_c0_entryhi(0x3fffffffffffe000ULL); - back_to_back_c0_hazard(); - c->vmbits = fls64(read_c0_entryhi() & 0x3fffffffffffe000ULL); -#endif + int vmbits = 31; + + if (cpu_has_64bits) { + write_c0_entryhi_64(0x3fffffffffffe000ULL); + back_to_back_c0_hazard(); + vmbits = fls64(read_c0_entryhi_64() & 0x3fffffffffffe000ULL); + } + c->vmbits = vmbits; } static void set_isa(struct cpuinfo_mips *c, unsigned int isa) diff --git a/arch/mips/kernel/cpu-r3k-probe.c b/arch/mips/kernel/cpu-r3k-probe.c index 0c826f729f7527..edcf04de0a6fbf 100644 --- a/arch/mips/kernel/cpu-r3k-probe.c +++ b/arch/mips/kernel/cpu-r3k-probe.c @@ -137,6 +137,8 @@ void cpu_probe(void) else cpu_set_nofpu_opts(c); + c->vmbits = 31; + reserve_exception_space(0, 0x400); } diff --git a/arch/mips/kernel/relocate.c b/arch/mips/kernel/relocate.c index 7f1c136ad85062..59833210542fff 100644 --- a/arch/mips/kernel/relocate.c +++ b/arch/mips/kernel/relocate.c @@ -420,7 +420,20 @@ void *__init relocate_kernel(void) goto out; /* The current thread is now within the relocated image */ +#ifndef CONFIG_CC_IS_CLANG __current_thread_info = RELOCATED(&init_thread_union); +#else + /* + * LLVM may wrongly restore $gp ($28) in epilog even if it's + * intentionally modified. Work around this by using inline + * assembly to assign $gp. $gp couldn't be listed as output or + * clobber, or LLVM will still restore its original value. + * See also LLVM upstream issue + * https://github.com/llvm/llvm-project/issues/176546 + */ + asm volatile("move $28, %0" : : + "r" (RELOCATED(&init_thread_union))); +#endif /* Return the new kernel's entry point */ kernel_entry = RELOCATED(start_kernel); diff --git a/arch/mips/kernel/vmlinux.lds.S b/arch/mips/kernel/vmlinux.lds.S index 2b708fac8d2c17..579b2cc1995aee 100644 --- a/arch/mips/kernel/vmlinux.lds.S +++ b/arch/mips/kernel/vmlinux.lds.S @@ -217,6 +217,7 @@ SECTIONS STABS_DEBUG DWARF_DEBUG + MODINFO ELF_DETAILS /* These must appear regardless of . */ diff --git a/arch/mips/kvm/mips.c b/arch/mips/kvm/mips.c index b0fb92fda4d423..23e69baad453cb 100644 --- a/arch/mips/kvm/mips.c +++ b/arch/mips/kvm/mips.c @@ -38,7 +38,7 @@ #define VECTORSPACING 0x100 /* for EI/VI mode */ #endif -const struct _kvm_stats_desc kvm_vm_stats_desc[] = { +const struct kvm_stats_desc kvm_vm_stats_desc[] = { KVM_GENERIC_VM_STATS() }; @@ -51,7 +51,7 @@ const struct kvm_stats_header kvm_vm_stats_header = { sizeof(kvm_vm_stats_desc), }; -const struct _kvm_stats_desc kvm_vcpu_stats_desc[] = { +const struct kvm_stats_desc kvm_vcpu_stats_desc[] = { KVM_GENERIC_VCPU_STATS(), STATS_DESC_COUNTER(VCPU, wait_exits), STATS_DESC_COUNTER(VCPU, cache_exits), diff --git a/arch/mips/lib/multi3.c b/arch/mips/lib/multi3.c index 4c2483f410c26b..92b3778bb56feb 100644 --- a/arch/mips/lib/multi3.c +++ b/arch/mips/lib/multi3.c @@ -4,12 +4,12 @@ #include "libgcc.h" /* - * GCC 7 & older can suboptimally generate __multi3 calls for mips64r6, so for + * GCC 9 & older can suboptimally generate __multi3 calls for mips64r6, so for * that specific case only we implement that intrinsic here. * * See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=82981 */ -#if defined(CONFIG_64BIT) && defined(CONFIG_CPU_MIPSR6) && (__GNUC__ < 8) +#if defined(CONFIG_64BIT) && defined(CONFIG_CPU_MIPSR6) && (__GNUC__ < 10) /* multiply 64-bit values, low 64-bits returned */ static inline long long notrace dmulu(long long a, long long b) @@ -51,4 +51,4 @@ ti_type notrace __multi3(ti_type a, ti_type b) } EXPORT_SYMBOL(__multi3); -#endif /* 64BIT && CPU_MIPSR6 && GCC7 */ +#endif /* 64BIT && CPU_MIPSR6 && GCC9 */ diff --git a/arch/mips/loongson2ef/common/pci.c b/arch/mips/loongson2ef/common/pci.c index 7d9ea51e8c01ec..0f11392104bfdf 100644 --- a/arch/mips/loongson2ef/common/pci.c +++ b/arch/mips/loongson2ef/common/pci.c @@ -17,7 +17,7 @@ static struct resource loongson_pci_mem_resource = { static struct resource loongson_pci_io_resource = { .name = "pci io space", - .start = LOONGSON_PCI_IO_START, + .start = 0x00000000UL, /* See loongson2ef_pcibios_init(). */ .end = IO_SPACE_LIMIT, .flags = IORESOURCE_IO, }; @@ -73,15 +73,19 @@ static void __init setup_pcimap(void) #endif } -static int __init pcibios_init(void) +void __init loongson2ef_pcibios_init(void) { setup_pcimap(); + /* + * ISA-mode only IDE controllers have a hard dependency on ISA IO ports. + * + * Claim them by setting PCI IO space to start at 0x00000000, and set + * PCIBIOS_MIN_IO to prevent non-legacy PCI devices from touching + * reserved regions. + */ + PCIBIOS_MIN_IO = LOONGSON_PCI_IO_START; + loongson_pci_controller.io_map_base = mips_io_port_base; register_pci_controller(&loongson_pci_controller); - - - return 0; } - -arch_initcall(pcibios_init); diff --git a/arch/mips/loongson2ef/common/setup.c b/arch/mips/loongson2ef/common/setup.c index 4fd27f4f90edb6..a639e35acce59b 100644 --- a/arch/mips/loongson2ef/common/setup.c +++ b/arch/mips/loongson2ef/common/setup.c @@ -27,4 +27,5 @@ EXPORT_SYMBOL(__wbflush); void __init plat_mem_setup(void) { + loongson2ef_pcibios_init(); } diff --git a/arch/mips/mm/cache.c b/arch/mips/mm/cache.c index e3b4224c9a4061..ad9b0430a28e2d 100644 --- a/arch/mips/mm/cache.c +++ b/arch/mips/mm/cache.c @@ -207,7 +207,8 @@ void cpu_cache_init(void) { if (IS_ENABLED(CONFIG_CPU_R3000) && cpu_has_3k_cache) r3k_cache_init(); - if (IS_ENABLED(CONFIG_CPU_R4K_CACHE_TLB) && cpu_has_4k_cache) + if ((IS_ENABLED(CONFIG_CPU_R4K_CACHE_TLB) || + IS_ENABLED(CONFIG_CPU_SB1)) && cpu_has_4k_cache) r4k_cache_init(); if (IS_ENABLED(CONFIG_CPU_CAVIUM_OCTEON) && cpu_has_octeon_cache) diff --git a/arch/mips/mm/tlb-r4k.c b/arch/mips/mm/tlb-r4k.c index 44a662536148e2..24fe85fa169d16 100644 --- a/arch/mips/mm/tlb-r4k.c +++ b/arch/mips/mm/tlb-r4k.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -24,6 +25,7 @@ #include #include #include +#include #include #include #include @@ -511,87 +513,259 @@ static int __init set_ntlb(char *str) __setup("ntlb=", set_ntlb); -/* Comparison function for EntryHi VPN fields. */ -static int r4k_vpn_cmp(const void *a, const void *b) +/* The start bit position of VPN2 and Mask in EntryHi/PageMask registers. */ +#define VPN2_SHIFT 13 + +/* Read full EntryHi even with CONFIG_32BIT. */ +static inline unsigned long long read_c0_entryhi_native(void) +{ + return cpu_has_64bits ? read_c0_entryhi_64() : read_c0_entryhi(); +} + +/* Write full EntryHi even with CONFIG_32BIT. */ +static inline void write_c0_entryhi_native(unsigned long long v) { - long v = *(unsigned long *)a - *(unsigned long *)b; - int s = sizeof(long) > sizeof(int) ? sizeof(long) * 8 - 1: 0; - return s ? (v != 0) | v >> s : v; + if (cpu_has_64bits) + write_c0_entryhi_64(v); + else + write_c0_entryhi(v); } +/* TLB entry state for uniquification. */ +struct tlbent { + unsigned long long wired:1; + unsigned long long global:1; + unsigned long long asid:10; + unsigned long long vpn:51; + unsigned long long pagesz:5; + unsigned long long index:14; +}; + /* - * Initialise all TLB entries with unique values that do not clash with - * what we have been handed over and what we'll be using ourselves. + * Comparison function for TLB entry sorting. Place wired entries first, + * then global entries, then order by the increasing VPN/ASID and the + * decreasing page size. This lets us avoid clashes with wired entries + * easily and get entries for larger pages out of the way first. + * + * We could group bits so as to reduce the number of comparisons, but this + * is seldom executed and not performance-critical, so prefer legibility. */ -static void __ref r4k_tlb_uniquify(void) +static int r4k_entry_cmp(const void *a, const void *b) { - int tlbsize = current_cpu_data.tlbsize; - bool use_slab = slab_is_available(); - int start = num_wired_entries(); - phys_addr_t tlb_vpn_size; - unsigned long *tlb_vpns; - unsigned long vpn_mask; - int cnt, ent, idx, i; - - vpn_mask = GENMASK(cpu_vmbits - 1, 13); - vpn_mask |= IS_ENABLED(CONFIG_64BIT) ? 3ULL << 62 : 1 << 31; + struct tlbent ea = *(struct tlbent *)a, eb = *(struct tlbent *)b; + + if (ea.wired > eb.wired) + return -1; + else if (ea.wired < eb.wired) + return 1; + else if (ea.global > eb.global) + return -1; + else if (ea.global < eb.global) + return 1; + else if (ea.vpn < eb.vpn) + return -1; + else if (ea.vpn > eb.vpn) + return 1; + else if (ea.asid < eb.asid) + return -1; + else if (ea.asid > eb.asid) + return 1; + else if (ea.pagesz > eb.pagesz) + return -1; + else if (ea.pagesz < eb.pagesz) + return 1; + else + return 0; +} - tlb_vpn_size = tlbsize * sizeof(*tlb_vpns); - tlb_vpns = (use_slab ? - kmalloc(tlb_vpn_size, GFP_KERNEL) : - memblock_alloc_raw(tlb_vpn_size, sizeof(*tlb_vpns))); - if (WARN_ON(!tlb_vpns)) - return; /* Pray local_flush_tlb_all() is good enough. */ +/* + * Fetch all the TLB entries. Mask individual VPN values retrieved with + * the corresponding page mask and ignoring any 1KiB extension as we'll + * be using 4KiB pages for uniquification. + */ +static void __ref r4k_tlb_uniquify_read(struct tlbent *tlb_vpns, int tlbsize) +{ + int start = num_wired_entries(); + unsigned long long vpn_mask; + bool global; + int i; - htw_stop(); + vpn_mask = GENMASK(current_cpu_data.vmbits - 1, VPN2_SHIFT); + vpn_mask |= cpu_has_64bits ? 3ULL << 62 : 1 << 31; - for (i = start, cnt = 0; i < tlbsize; i++, cnt++) { - unsigned long vpn; + for (i = 0; i < tlbsize; i++) { + unsigned long long entryhi, vpn, mask, asid; + unsigned int pagesz; write_c0_index(i); mtc0_tlbr_hazard(); tlb_read(); tlb_read_hazard(); - vpn = read_c0_entryhi(); - vpn &= vpn_mask & PAGE_MASK; - tlb_vpns[cnt] = vpn; - /* Prevent any large pages from overlapping regular ones. */ - write_c0_pagemask(read_c0_pagemask() & PM_DEFAULT_MASK); - mtc0_tlbw_hazard(); - tlb_write_indexed(); - tlbw_use_hazard(); + global = !!(read_c0_entrylo0() & ENTRYLO_G); + entryhi = read_c0_entryhi_native(); + mask = read_c0_pagemask(); + + asid = entryhi & cpu_asid_mask(¤t_cpu_data); + vpn = (entryhi & vpn_mask & ~mask) >> VPN2_SHIFT; + pagesz = ilog2((mask >> VPN2_SHIFT) + 1); + + tlb_vpns[i].global = global; + tlb_vpns[i].asid = global ? 0 : asid; + tlb_vpns[i].vpn = vpn; + tlb_vpns[i].pagesz = pagesz; + tlb_vpns[i].wired = i < start; + tlb_vpns[i].index = i; } +} + +/* + * Write unique values to all but the wired TLB entries each, using + * the 4KiB page size. This size might not be supported with R6, but + * EHINV is mandatory for R6, so we won't ever be called in that case. + * + * A sorted table is supplied with any wired entries at the beginning, + * followed by any global entries, and then finally regular entries. + * We start at the VPN and ASID values of zero and only assign user + * addresses, therefore guaranteeing no clash with addresses produced + * by UNIQUE_ENTRYHI. We avoid any VPN values used by wired or global + * entries, by increasing the VPN value beyond the span of such entry. + * + * When a VPN/ASID clash is found with a regular entry we increment the + * ASID instead until no VPN/ASID clash has been found or the ASID space + * has been exhausted, in which case we increase the VPN value beyond + * the span of the largest clashing entry. + * + * We do not need to be concerned about FTLB or MMID configurations as + * those are required to implement the EHINV feature. + */ +static void __ref r4k_tlb_uniquify_write(struct tlbent *tlb_vpns, int tlbsize) +{ + unsigned long long asid, vpn, vpn_size, pagesz; + int widx, gidx, idx, sidx, lidx, i; - sort(tlb_vpns, cnt, sizeof(tlb_vpns[0]), r4k_vpn_cmp, NULL); + vpn_size = 1ULL << (current_cpu_data.vmbits - VPN2_SHIFT); + pagesz = ilog2((PM_4K >> VPN2_SHIFT) + 1); - write_c0_pagemask(PM_DEFAULT_MASK); + write_c0_pagemask(PM_4K); write_c0_entrylo0(0); write_c0_entrylo1(0); - idx = 0; - ent = tlbsize; - for (i = start; i < tlbsize; i++) - while (1) { - unsigned long entryhi, vpn; + asid = 0; + vpn = 0; + widx = 0; + gidx = 0; + for (sidx = 0; sidx < tlbsize && tlb_vpns[sidx].wired; sidx++) + ; + for (lidx = sidx; lidx < tlbsize && tlb_vpns[lidx].global; lidx++) + ; + idx = gidx = sidx + 1; + for (i = sidx; i < tlbsize; i++) { + unsigned long long entryhi, vpn_pagesz = 0; - entryhi = UNIQUE_ENTRYHI(ent); - vpn = entryhi & vpn_mask & PAGE_MASK; + while (1) { + if (WARN_ON(vpn >= vpn_size)) { + dump_tlb_all(); + /* Pray local_flush_tlb_all() will cope. */ + return; + } - if (idx >= cnt || vpn < tlb_vpns[idx]) { - write_c0_entryhi(entryhi); - write_c0_index(i); - mtc0_tlbw_hazard(); - tlb_write_indexed(); - ent++; - break; - } else if (vpn == tlb_vpns[idx]) { - ent++; - } else { + /* VPN must be below the next wired entry. */ + if (widx < sidx && vpn >= tlb_vpns[widx].vpn) { + vpn = max(vpn, + (tlb_vpns[widx].vpn + + (1ULL << tlb_vpns[widx].pagesz))); + asid = 0; + widx++; + continue; + } + /* VPN must be below the next global entry. */ + if (gidx < lidx && vpn >= tlb_vpns[gidx].vpn) { + vpn = max(vpn, + (tlb_vpns[gidx].vpn + + (1ULL << tlb_vpns[gidx].pagesz))); + asid = 0; + gidx++; + continue; + } + /* Try to find a free ASID so as to conserve VPNs. */ + if (idx < tlbsize && vpn == tlb_vpns[idx].vpn && + asid == tlb_vpns[idx].asid) { + unsigned long long idx_pagesz; + + idx_pagesz = tlb_vpns[idx].pagesz; + vpn_pagesz = max(vpn_pagesz, idx_pagesz); + do + idx++; + while (idx < tlbsize && + vpn == tlb_vpns[idx].vpn && + asid == tlb_vpns[idx].asid); + asid++; + if (asid > cpu_asid_mask(¤t_cpu_data)) { + vpn += vpn_pagesz; + asid = 0; + vpn_pagesz = 0; + } + continue; + } + /* VPN mustn't be above the next regular entry. */ + if (idx < tlbsize && vpn > tlb_vpns[idx].vpn) { + vpn = max(vpn, + (tlb_vpns[idx].vpn + + (1ULL << tlb_vpns[idx].pagesz))); + asid = 0; idx++; + continue; } + break; } + entryhi = (vpn << VPN2_SHIFT) | asid; + write_c0_entryhi_native(entryhi); + write_c0_index(tlb_vpns[i].index); + mtc0_tlbw_hazard(); + tlb_write_indexed(); + + tlb_vpns[i].asid = asid; + tlb_vpns[i].vpn = vpn; + tlb_vpns[i].pagesz = pagesz; + + asid++; + if (asid > cpu_asid_mask(¤t_cpu_data)) { + vpn += 1ULL << pagesz; + asid = 0; + } + } +} + +/* + * Initialise all TLB entries with unique values that do not clash with + * what we have been handed over and what we'll be using ourselves. + */ +static void __ref r4k_tlb_uniquify(void) +{ + int tlbsize = current_cpu_data.tlbsize; + bool use_slab = slab_is_available(); + phys_addr_t tlb_vpn_size; + struct tlbent *tlb_vpns; + + tlb_vpn_size = tlbsize * sizeof(*tlb_vpns); + tlb_vpns = (use_slab ? + kmalloc(tlb_vpn_size, GFP_ATOMIC) : + memblock_alloc_raw(tlb_vpn_size, sizeof(*tlb_vpns))); + if (WARN_ON(!tlb_vpns)) + return; /* Pray local_flush_tlb_all() is good enough. */ + + htw_stop(); + + r4k_tlb_uniquify_read(tlb_vpns, tlbsize); + + sort(tlb_vpns, tlbsize, sizeof(*tlb_vpns), r4k_entry_cmp, NULL); + + r4k_tlb_uniquify_write(tlb_vpns, tlbsize); + + write_c0_pagemask(PM_DEFAULT_MASK); + tlbw_use_hazard(); htw_start(); flush_micro_tlb(); @@ -640,7 +814,8 @@ static void r4k_tlb_configure(void) temp_tlb_entry = current_cpu_data.tlbsize - 1; /* From this point on the ARC firmware is dead. */ - r4k_tlb_uniquify(); + if (!cpu_has_tlbinv) + r4k_tlb_uniquify(); local_flush_tlb_all(); /* Did I tell you that ARC SUCKS? */ diff --git a/arch/mips/ralink/clk.c b/arch/mips/ralink/clk.c index 9db73fcac522eb..5c1eb46ef5d07e 100644 --- a/arch/mips/ralink/clk.c +++ b/arch/mips/ralink/clk.c @@ -21,16 +21,16 @@ static const char *clk_cpu(int *idx) { switch (ralink_soc) { case RT2880_SOC: - *idx = 0; + *idx = 1; return "ralink,rt2880-sysc"; case RT3883_SOC: - *idx = 0; + *idx = 1; return "ralink,rt3883-sysc"; case RT305X_SOC_RT3050: - *idx = 0; + *idx = 1; return "ralink,rt3050-sysc"; case RT305X_SOC_RT3052: - *idx = 0; + *idx = 1; return "ralink,rt3052-sysc"; case RT305X_SOC_RT3350: *idx = 1; diff --git a/arch/mips/rb532/devices.c b/arch/mips/rb532/devices.c index b7f6f782d9a130..ffa4d38ca95df7 100644 --- a/arch/mips/rb532/devices.c +++ b/arch/mips/rb532/devices.c @@ -212,11 +212,12 @@ static struct platform_device rb532_wdt = { static struct plat_serial8250_port rb532_uart_res[] = { { .type = PORT_16550A, - .membase = (char *)KSEG1ADDR(REGBASE + UART0BASE), + .mapbase = REGBASE + UART0BASE, + .mapsize = 0x1000, .irq = UART0_IRQ, .regshift = 2, .iotype = UPIO_MEM, - .flags = UPF_BOOT_AUTOCONF, + .flags = UPF_BOOT_AUTOCONF | UPF_IOREMAP, }, { .flags = 0, diff --git a/arch/nios2/kernel/vmlinux.lds.S b/arch/nios2/kernel/vmlinux.lds.S index 37b95805506469..206f92445bfad8 100644 --- a/arch/nios2/kernel/vmlinux.lds.S +++ b/arch/nios2/kernel/vmlinux.lds.S @@ -57,6 +57,7 @@ SECTIONS STABS_DEBUG DWARF_DEBUG + MODINFO ELF_DETAILS DISCARDS diff --git a/arch/openrisc/include/asm/barrier.h b/arch/openrisc/include/asm/barrier.h index 7538294721bed7..8e592c99090235 100644 --- a/arch/openrisc/include/asm/barrier.h +++ b/arch/openrisc/include/asm/barrier.h @@ -4,6 +4,8 @@ #define mb() asm volatile ("l.msync" ::: "memory") +#define nop() asm volatile ("l.nop") + #include #endif /* __ASM_BARRIER_H */ diff --git a/arch/openrisc/kernel/vmlinux.lds.S b/arch/openrisc/kernel/vmlinux.lds.S index 049bff45f61265..9b29c3211774ca 100644 --- a/arch/openrisc/kernel/vmlinux.lds.S +++ b/arch/openrisc/kernel/vmlinux.lds.S @@ -101,6 +101,7 @@ SECTIONS /* Throw in the debugging sections */ STABS_DEBUG DWARF_DEBUG + MODINFO ELF_DETAILS /* Sections to be discarded -- must be last */ diff --git a/arch/parisc/boot/compressed/vmlinux.lds.S b/arch/parisc/boot/compressed/vmlinux.lds.S index ab7b439908578c..87d24cc824b668 100644 --- a/arch/parisc/boot/compressed/vmlinux.lds.S +++ b/arch/parisc/boot/compressed/vmlinux.lds.S @@ -90,6 +90,7 @@ SECTIONS /* Sections to be discarded */ DISCARDS /DISCARD/ : { + *(.modinfo) #ifdef CONFIG_64BIT /* temporary hack until binutils is fixed to not emit these * for static binaries diff --git a/arch/parisc/include/asm/pgtable.h b/arch/parisc/include/asm/pgtable.h index 2c139a4dbf4b86..17afe7a59edfde 100644 --- a/arch/parisc/include/asm/pgtable.h +++ b/arch/parisc/include/asm/pgtable.h @@ -85,7 +85,7 @@ extern void __update_cache(pte_t pte); printk("%s:%d: bad pgd %08lx.\n", __FILE__, __LINE__, (unsigned long)pgd_val(e)) /* This is the size of the initially mapped kernel memory */ -#if defined(CONFIG_64BIT) +#if defined(CONFIG_64BIT) || defined(CONFIG_KALLSYMS) #define KERNEL_INITIAL_ORDER 26 /* 1<<26 = 64MB */ #else #define KERNEL_INITIAL_ORDER 25 /* 1<<25 = 32MB */ diff --git a/arch/parisc/kernel/cache.c b/arch/parisc/kernel/cache.c index 4c5240d3a3c7cf..b189265785dc79 100644 --- a/arch/parisc/kernel/cache.c +++ b/arch/parisc/kernel/cache.c @@ -953,7 +953,7 @@ SYSCALL_DEFINE3(cacheflush, unsigned long, addr, unsigned long, bytes, #else "1: cmpb,<<,n %0,%2,1b\n" #endif - " fic,m %3(%4,%0)\n" + " fdc,m %3(%4,%0)\n" "2: sync\n" ASM_EXCEPTIONTABLE_ENTRY_EFAULT(1b, 2b, "%1") : "+r" (start), "+r" (error) @@ -968,7 +968,7 @@ SYSCALL_DEFINE3(cacheflush, unsigned long, addr, unsigned long, bytes, #else "1: cmpb,<<,n %0,%2,1b\n" #endif - " fdc,m %3(%4,%0)\n" + " fic,m %3(%4,%0)\n" "2: sync\n" ASM_EXCEPTIONTABLE_ENTRY_EFAULT(1b, 2b, "%1") : "+r" (start), "+r" (error) diff --git a/arch/parisc/kernel/drivers.c b/arch/parisc/kernel/drivers.c index 8d23fe42b0cee5..809e3c171ad541 100644 --- a/arch/parisc/kernel/drivers.c +++ b/arch/parisc/kernel/drivers.c @@ -435,7 +435,7 @@ static struct parisc_device * __init create_tree_node(char id, dev->dev.dma_mask = &dev->dma_mask; dev->dev.coherent_dma_mask = dev->dma_mask; if (device_register(&dev->dev)) { - kfree(dev); + put_device(&dev->dev); return NULL; } diff --git a/arch/parisc/kernel/head.S b/arch/parisc/kernel/head.S index 96e0264ac96163..9188c8d8743704 100644 --- a/arch/parisc/kernel/head.S +++ b/arch/parisc/kernel/head.S @@ -56,6 +56,7 @@ ENTRY(parisc_kernel_start) .import __bss_start,data .import __bss_stop,data + .import __end,data load32 PA(__bss_start),%r3 load32 PA(__bss_stop),%r4 @@ -149,7 +150,11 @@ $cpu_ok: * everything ... it will get remapped correctly later */ ldo 0+_PAGE_KERNEL_RWX(%r0),%r3 /* Hardwired 0 phys addr start */ load32 (1<<(KERNEL_INITIAL_ORDER-PAGE_SHIFT)),%r11 /* PFN count */ - load32 PA(pg0),%r1 + load32 PA(_end),%r1 + SHRREG %r1,PAGE_SHIFT,%r1 /* %r1 is PFN count for _end symbol */ + cmpb,<<,n %r11,%r1,1f + copy %r1,%r11 /* %r1 PFN count smaller than %r11 */ +1: load32 PA(pg0),%r1 $pgt_fill_loop: STREGM %r3,ASM_PTE_ENTRY_SIZE(%r1) diff --git a/arch/parisc/kernel/process.c b/arch/parisc/kernel/process.c index e64ab5d2a40d61..703644e5bfc4a1 100644 --- a/arch/parisc/kernel/process.c +++ b/arch/parisc/kernel/process.c @@ -85,6 +85,9 @@ void machine_restart(char *cmd) #endif /* set up a new led state on systems shipped with a LED State panel */ pdc_chassis_send_status(PDC_CHASSIS_DIRECT_SHUTDOWN); + + /* prevent interrupts during reboot */ + set_eiem(0); /* "Normal" system reset */ pdc_do_reset(); diff --git a/arch/parisc/kernel/setup.c b/arch/parisc/kernel/setup.c index ace483b6f19adc..d3e17a7a89016a 100644 --- a/arch/parisc/kernel/setup.c +++ b/arch/parisc/kernel/setup.c @@ -120,14 +120,6 @@ void __init setup_arch(char **cmdline_p) #endif printk(KERN_CONT ".\n"); - /* - * Check if initial kernel page mappings are sufficient. - * panic early if not, else we may access kernel functions - * and variables which can't be reached. - */ - if (__pa((unsigned long) &_end) >= KERNEL_INITIAL_SIZE) - panic("KERNEL_INITIAL_ORDER too small!"); - #ifdef CONFIG_64BIT if(parisc_narrow_firmware) { printk(KERN_INFO "Kernel is using PDC in 32-bit mode.\n"); @@ -279,6 +271,18 @@ void __init start_parisc(void) int ret, cpunum; struct pdc_coproc_cfg coproc_cfg; + /* + * Check if initial kernel page mapping is sufficient. + * Print warning if not, because we may access kernel functions and + * variables which can't be reached yet through the initial mappings. + * Note that the panic() and printk() functions are not functional + * yet, so we need to use direct iodc() firmware calls instead. + */ + const char warn1[] = "CRITICAL: Kernel may crash because " + "KERNEL_INITIAL_ORDER is too small.\n"; + if (__pa((unsigned long) &_end) >= KERNEL_INITIAL_SIZE) + pdc_iodc_print(warn1, sizeof(warn1) - 1); + /* check QEMU/SeaBIOS marker in PAGE0 */ running_on_qemu = (memcmp(&PAGE0->pad0, "SeaBIOS", 8) == 0); diff --git a/arch/parisc/kernel/vmlinux.lds.S b/arch/parisc/kernel/vmlinux.lds.S index b445e47903cfd0..0ca93d6d723543 100644 --- a/arch/parisc/kernel/vmlinux.lds.S +++ b/arch/parisc/kernel/vmlinux.lds.S @@ -165,6 +165,7 @@ SECTIONS _end = . ; STABS_DEBUG + MODINFO ELF_DETAILS .note 0 : { *(.note) } diff --git a/arch/powerpc/include/asm/eeh.h b/arch/powerpc/include/asm/eeh.h index 5e34611de9ef40..b7ebb4ac2c7103 100644 --- a/arch/powerpc/include/asm/eeh.h +++ b/arch/powerpc/include/asm/eeh.h @@ -289,6 +289,8 @@ void eeh_pe_dev_traverse(struct eeh_pe *root, void eeh_pe_restore_bars(struct eeh_pe *pe); const char *eeh_pe_loc_get(struct eeh_pe *pe); struct pci_bus *eeh_pe_bus_get(struct eeh_pe *pe); +const char *eeh_pe_loc_get_bus(struct pci_bus *bus); +struct pci_bus *eeh_pe_bus_get_nolock(struct eeh_pe *pe); void eeh_show_enabled(void); int __init eeh_init(struct eeh_ops *ops); diff --git a/arch/powerpc/include/asm/kup.h b/arch/powerpc/include/asm/kup.h index dab63b82a8d4f3..f2009d7c8cfa75 100644 --- a/arch/powerpc/include/asm/kup.h +++ b/arch/powerpc/include/asm/kup.h @@ -134,7 +134,6 @@ static __always_inline void kuap_assert_locked(void) static __always_inline void allow_read_from_user(const void __user *from, unsigned long size) { - barrier_nospec(); allow_user_access(NULL, from, size, KUAP_READ); } @@ -146,7 +145,6 @@ static __always_inline void allow_write_to_user(void __user *to, unsigned long s static __always_inline void allow_read_write_user(void __user *to, const void __user *from, unsigned long size) { - barrier_nospec(); allow_user_access(to, from, size, KUAP_READ_WRITE); } diff --git a/arch/powerpc/include/asm/uaccess.h b/arch/powerpc/include/asm/uaccess.h index 784a00e681fa37..ea4e685f908854 100644 --- a/arch/powerpc/include/asm/uaccess.h +++ b/arch/powerpc/include/asm/uaccess.h @@ -253,7 +253,7 @@ __gus_failed: \ ".section .fixup,\"ax\"\n" \ "4: li %0,%3\n" \ " li %1,0\n" \ - " li %1+1,0\n" \ + " li %L1,0\n" \ " b 3b\n" \ ".previous\n" \ EX_TABLE(1b, 4b) \ @@ -301,6 +301,7 @@ do { \ __typeof__(sizeof(*(ptr))) __gu_size = sizeof(*(ptr)); \ \ might_fault(); \ + barrier_nospec(); \ allow_read_from_user(__gu_addr, __gu_size); \ __get_user_size_allowed(__gu_val, __gu_addr, __gu_size, __gu_err); \ prevent_read_from_user(__gu_addr, __gu_size); \ @@ -329,6 +330,7 @@ raw_copy_in_user(void __user *to, const void __user *from, unsigned long n) { unsigned long ret; + barrier_nospec(); allow_read_write_user(to, from, n); ret = __copy_tofrom_user(to, from, n); prevent_read_write_user(to, from, n); @@ -405,8 +407,7 @@ copy_mc_to_user(void __user *to, const void *from, unsigned long n) } #endif -extern long __copy_from_user_flushcache(void *dst, const void __user *src, - unsigned size); +extern size_t copy_from_user_flushcache(void *dst, const void __user *src, size_t size); static __must_check __always_inline bool user_access_begin(const void __user *ptr, size_t len) { @@ -415,6 +416,7 @@ static __must_check __always_inline bool user_access_begin(const void __user *pt might_fault(); + barrier_nospec(); allow_read_write_user((void __user *)ptr, ptr, len); return true; } @@ -431,6 +433,7 @@ user_read_access_begin(const void __user *ptr, size_t len) might_fault(); + barrier_nospec(); allow_read_from_user(ptr, len); return true; } diff --git a/arch/powerpc/kernel/eeh_driver.c b/arch/powerpc/kernel/eeh_driver.c index ef78ff77cf8f21..028f6915853234 100644 --- a/arch/powerpc/kernel/eeh_driver.c +++ b/arch/powerpc/kernel/eeh_driver.c @@ -846,7 +846,7 @@ void eeh_handle_normal_event(struct eeh_pe *pe) pci_lock_rescan_remove(); - bus = eeh_pe_bus_get(pe); + bus = eeh_pe_bus_get_nolock(pe); if (!bus) { pr_err("%s: Cannot find PCI bus for PHB#%x-PE#%x\n", __func__, pe->phb->global_number, pe->addr); @@ -886,14 +886,15 @@ void eeh_handle_normal_event(struct eeh_pe *pe) /* Log the event */ if (pe->type & EEH_PE_PHB) { pr_err("EEH: Recovering PHB#%x, location: %s\n", - pe->phb->global_number, eeh_pe_loc_get(pe)); + pe->phb->global_number, eeh_pe_loc_get_bus(bus)); } else { struct eeh_pe *phb_pe = eeh_phb_pe_get(pe->phb); pr_err("EEH: Recovering PHB#%x-PE#%x\n", pe->phb->global_number, pe->addr); pr_err("EEH: PE location: %s, PHB location: %s\n", - eeh_pe_loc_get(pe), eeh_pe_loc_get(phb_pe)); + eeh_pe_loc_get_bus(bus), + eeh_pe_loc_get_bus(eeh_pe_bus_get_nolock(phb_pe))); } #ifdef CONFIG_STACKTRACE @@ -1098,7 +1099,7 @@ void eeh_handle_normal_event(struct eeh_pe *pe) eeh_pe_state_clear(pe, EEH_PE_PRI_BUS, true); eeh_pe_dev_mode_mark(pe, EEH_DEV_REMOVED); - bus = eeh_pe_bus_get(pe); + bus = eeh_pe_bus_get_nolock(pe); if (bus) pci_hp_remove_devices(bus); else @@ -1222,7 +1223,7 @@ void eeh_handle_special_event(void) (phb_pe->state & EEH_PE_RECOVERING)) continue; - bus = eeh_pe_bus_get(phb_pe); + bus = eeh_pe_bus_get_nolock(phb_pe); if (!bus) { pr_err("%s: Cannot find PCI bus for " "PHB#%x-PE#%x\n", diff --git a/arch/powerpc/kernel/eeh_pe.c b/arch/powerpc/kernel/eeh_pe.c index e740101fadf3b1..040e8f69a4aa86 100644 --- a/arch/powerpc/kernel/eeh_pe.c +++ b/arch/powerpc/kernel/eeh_pe.c @@ -812,6 +812,24 @@ void eeh_pe_restore_bars(struct eeh_pe *pe) const char *eeh_pe_loc_get(struct eeh_pe *pe) { struct pci_bus *bus = eeh_pe_bus_get(pe); + return eeh_pe_loc_get_bus(bus); +} + +/** + * eeh_pe_loc_get_bus - Retrieve location code binding to the given PCI bus + * @bus: PCI bus + * + * Retrieve the location code associated with the given PCI bus. If the bus + * is a root bus, the location code is fetched from the PHB device tree node + * or root port. Otherwise, the location code is obtained from the device + * tree node of the upstream bridge of the bus. The function walks up the + * bus hierarchy if necessary, checking each node for the appropriate + * location code property ("ibm,io-base-loc-code" for root buses, + * "ibm,slot-location-code" for others). If no location code is found, + * returns "N/A". + */ +const char *eeh_pe_loc_get_bus(struct pci_bus *bus) +{ struct device_node *dn; const char *loc = NULL; @@ -838,8 +856,9 @@ const char *eeh_pe_loc_get(struct eeh_pe *pe) } /** - * eeh_pe_bus_get - Retrieve PCI bus according to the given PE + * _eeh_pe_bus_get - Retrieve PCI bus according to the given PE * @pe: EEH PE + * @do_lock: Is the caller already held the pci_lock_rescan_remove? * * Retrieve the PCI bus according to the given PE. Basically, * there're 3 types of PEs: PHB/Bus/Device. For PHB PE, the @@ -847,7 +866,7 @@ const char *eeh_pe_loc_get(struct eeh_pe *pe) * returned for BUS PE. However, we don't have associated PCI * bus for DEVICE PE. */ -struct pci_bus *eeh_pe_bus_get(struct eeh_pe *pe) +static struct pci_bus *_eeh_pe_bus_get(struct eeh_pe *pe, bool do_lock) { struct eeh_dev *edev; struct pci_dev *pdev; @@ -862,11 +881,58 @@ struct pci_bus *eeh_pe_bus_get(struct eeh_pe *pe) /* Retrieve the parent PCI bus of first (top) PCI device */ edev = list_first_entry_or_null(&pe->edevs, struct eeh_dev, entry); - pci_lock_rescan_remove(); + if (do_lock) + pci_lock_rescan_remove(); pdev = eeh_dev_to_pci_dev(edev); if (pdev) bus = pdev->bus; - pci_unlock_rescan_remove(); + if (do_lock) + pci_unlock_rescan_remove(); return bus; } + +/** + * eeh_pe_bus_get - Retrieve PCI bus associated with the given EEH PE, locking + * if needed + * @pe: Pointer to the EEH PE + * + * This function is a wrapper around _eeh_pe_bus_get(), which retrieves the PCI + * bus associated with the provided EEH PE structure. It acquires the PCI + * rescans lock to ensure safe access to shared data during the retrieval + * process. This function should be used when the caller requires the PCI bus + * while holding the rescan/remove lock, typically during operations that modify + * or inspect PCIe device state in a safe manner. + * + * RETURNS: + * A pointer to the PCI bus associated with the EEH PE, or NULL if none found. + */ + +struct pci_bus *eeh_pe_bus_get(struct eeh_pe *pe) +{ + return _eeh_pe_bus_get(pe, true); +} + +/** + * eeh_pe_bus_get_nolock - Retrieve PCI bus associated with the given EEH PE + * without locking + * @pe: Pointer to the EEH PE + * + * This function is a variant of _eeh_pe_bus_get() that retrieves the PCI bus + * associated with the specified EEH PE without acquiring the + * pci_lock_rescan_remove lock. It should only be used when the caller can + * guarantee safe access to PE structures without the need for that lock, + * typically in contexts where the lock is already held locking is otherwise + * managed. + * + * RETURNS: + * pointer to the PCI bus associated with the EEH PE, or NULL if none is found. + * + * NOTE: + * Use this function carefully to avoid race conditions and data corruption. + */ + +struct pci_bus *eeh_pe_bus_get_nolock(struct eeh_pe *pe) +{ + return _eeh_pe_bus_get(pe, false); +} diff --git a/arch/powerpc/kernel/smp.c b/arch/powerpc/kernel/smp.c index 292fee8809bc8b..cad3358fa4c359 100644 --- a/arch/powerpc/kernel/smp.c +++ b/arch/powerpc/kernel/smp.c @@ -822,6 +822,8 @@ static int parse_thread_groups(struct device_node *dn, count = of_property_count_u32_elems(dn, "ibm,thread-groups"); thread_group_array = kcalloc(count, sizeof(u32), GFP_KERNEL); + if (!thread_group_array) + return -ENOMEM; ret = of_property_read_u32_array(dn, "ibm,thread-groups", thread_group_array, count); if (ret) diff --git a/arch/powerpc/kernel/vmlinux.lds.S b/arch/powerpc/kernel/vmlinux.lds.S index 15850296c0a9cc..8fc11d6565bfb4 100644 --- a/arch/powerpc/kernel/vmlinux.lds.S +++ b/arch/powerpc/kernel/vmlinux.lds.S @@ -397,6 +397,7 @@ SECTIONS _end = . ; DWARF_DEBUG + MODINFO ELF_DETAILS DISCARDS diff --git a/arch/powerpc/kexec/core.c b/arch/powerpc/kexec/core.c index 104c05520bf056..dc44f11be353e3 100644 --- a/arch/powerpc/kexec/core.c +++ b/arch/powerpc/kexec/core.c @@ -23,6 +23,7 @@ #include #define cpu_to_be_ulong __PASTE(cpu_to_be, BITS_PER_LONG) +#define __be_word __PASTE(__be, BITS_PER_LONG) #ifdef CONFIG_CRASH_DUMP void machine_crash_shutdown(struct pt_regs *regs) @@ -146,25 +147,25 @@ int __init overlaps_crashkernel(unsigned long start, unsigned long size) } /* Values we need to export to the second kernel via the device tree. */ -static phys_addr_t crashk_base; -static phys_addr_t crashk_size; -static unsigned long long mem_limit; +static __be_word crashk_base; +static __be_word crashk_size; +static __be_word mem_limit; static struct property crashk_base_prop = { .name = "linux,crashkernel-base", - .length = sizeof(phys_addr_t), + .length = sizeof(__be_word), .value = &crashk_base }; static struct property crashk_size_prop = { .name = "linux,crashkernel-size", - .length = sizeof(phys_addr_t), + .length = sizeof(__be_word), .value = &crashk_size, }; static struct property memory_limit_prop = { .name = "linux,memory-limit", - .length = sizeof(unsigned long long), + .length = sizeof(__be_word), .value = &mem_limit, }; @@ -193,11 +194,11 @@ static void __init export_crashk_values(struct device_node *node) } #endif /* CONFIG_CRASH_RESERVE */ -static phys_addr_t kernel_end; +static __be_word kernel_end; static struct property kernel_end_prop = { .name = "linux,kernel-end", - .length = sizeof(phys_addr_t), + .length = sizeof(__be_word), .value = &kernel_end, }; diff --git a/arch/powerpc/kexec/file_load_64.c b/arch/powerpc/kexec/file_load_64.c index e7ef8b2a25546b..5f6d50e4c3d450 100644 --- a/arch/powerpc/kexec/file_load_64.c +++ b/arch/powerpc/kexec/file_load_64.c @@ -450,6 +450,11 @@ static int load_elfcorehdr_segment(struct kimage *image, struct kexec_buf *kbuf) kbuf->buffer = headers; kbuf->mem = KEXEC_BUF_MEM_UNKNOWN; kbuf->bufsz = headers_sz; + + /* + * Account for extra space required to accommodate additional memory + * ranges in elfcorehdr due to memory hotplug events. + */ kbuf->memsz = headers_sz + kdump_extra_elfcorehdr_size(cmem); kbuf->top_down = false; @@ -460,7 +465,14 @@ static int load_elfcorehdr_segment(struct kimage *image, struct kexec_buf *kbuf) } image->elf_load_addr = kbuf->mem; - image->elf_headers_sz = headers_sz; + + /* + * If CONFIG_CRASH_HOTPLUG is enabled, the elfcorehdr kexec segment + * memsz can be larger than bufsz. Always initialize elf_headers_sz + * with memsz. This ensures the correct size is reserved for elfcorehdr + * memory in the FDT prepared for kdump. + */ + image->elf_headers_sz = kbuf->memsz; image->elf_headers = headers; out: kfree(cmem); diff --git a/arch/powerpc/kvm/book3s.c b/arch/powerpc/kvm/book3s.c index d79c5d1098c05c..2efbe05caed760 100644 --- a/arch/powerpc/kvm/book3s.c +++ b/arch/powerpc/kvm/book3s.c @@ -38,7 +38,7 @@ /* #define EXIT_DEBUG */ -const struct _kvm_stats_desc kvm_vm_stats_desc[] = { +const struct kvm_stats_desc kvm_vm_stats_desc[] = { KVM_GENERIC_VM_STATS(), STATS_DESC_ICOUNTER(VM, num_2M_pages), STATS_DESC_ICOUNTER(VM, num_1G_pages) @@ -53,7 +53,7 @@ const struct kvm_stats_header kvm_vm_stats_header = { sizeof(kvm_vm_stats_desc), }; -const struct _kvm_stats_desc kvm_vcpu_stats_desc[] = { +const struct kvm_stats_desc kvm_vcpu_stats_desc[] = { KVM_GENERIC_VCPU_STATS(), STATS_DESC_COUNTER(VCPU, sum_exits), STATS_DESC_COUNTER(VCPU, mmio_exits), diff --git a/arch/powerpc/kvm/booke.c b/arch/powerpc/kvm/booke.c index 3401b96be475ef..f3ddb24ece749f 100644 --- a/arch/powerpc/kvm/booke.c +++ b/arch/powerpc/kvm/booke.c @@ -36,7 +36,7 @@ unsigned long kvmppc_booke_handlers; -const struct _kvm_stats_desc kvm_vm_stats_desc[] = { +const struct kvm_stats_desc kvm_vm_stats_desc[] = { KVM_GENERIC_VM_STATS(), STATS_DESC_ICOUNTER(VM, num_2M_pages), STATS_DESC_ICOUNTER(VM, num_1G_pages) @@ -51,7 +51,7 @@ const struct kvm_stats_header kvm_vm_stats_header = { sizeof(kvm_vm_stats_desc), }; -const struct _kvm_stats_desc kvm_vcpu_stats_desc[] = { +const struct kvm_stats_desc kvm_vcpu_stats_desc[] = { KVM_GENERIC_VCPU_STATS(), STATS_DESC_COUNTER(VCPU, sum_exits), STATS_DESC_COUNTER(VCPU, mmio_exits), diff --git a/arch/powerpc/lib/pmem.c b/arch/powerpc/lib/pmem.c index 4e724c4c01add7..0f0f2d851ac670 100644 --- a/arch/powerpc/lib/pmem.c +++ b/arch/powerpc/lib/pmem.c @@ -66,15 +66,16 @@ EXPORT_SYMBOL_GPL(arch_invalidate_pmem); /* * CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE symbols */ -long __copy_from_user_flushcache(void *dest, const void __user *src, - unsigned size) +size_t copy_from_user_flushcache(void *dest, const void __user *src, + size_t size) { - unsigned long copied, start = (unsigned long) dest; + unsigned long not_copied, start = (unsigned long) dest; - copied = __copy_from_user(dest, src, size); + src = mask_user_address(src); + not_copied = __copy_from_user(dest, src, size); clean_pmem_range(start, start + size); - return copied; + return not_copied; } void memcpy_flushcache(void *dest, const void *src, size_t size) diff --git a/arch/powerpc/net/bpf_jit_comp.c b/arch/powerpc/net/bpf_jit_comp.c index 5e976730b2f5f5..0d2d2190af9a79 100644 --- a/arch/powerpc/net/bpf_jit_comp.c +++ b/arch/powerpc/net/bpf_jit_comp.c @@ -437,7 +437,7 @@ void bpf_jit_free(struct bpf_prog *fp) bool bpf_jit_supports_kfunc_call(void) { - return true; + return IS_ENABLED(CONFIG_PPC64); } bool bpf_jit_supports_arena(void) @@ -722,9 +722,9 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im * retval_off [ return value ] * [ reg argN ] * [ ... ] - * regs_off [ reg_arg1 ] prog ctx context - * nregs_off [ args count ] - * ip_off [ traced function ] + * regs_off [ reg_arg1 ] prog_ctx + * nregs_off [ args count ] ((u64 *)prog_ctx)[-1] + * ip_off [ traced function ] ((u64 *)prog_ctx)[-2] * [ ... ] * run_ctx_off [ bpf_tramp_run_ctx ] * [ reg argN ] @@ -824,7 +824,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im bpf_trampoline_save_args(image, ctx, func_frame_offset, nr_regs, regs_off); - /* Save our return address */ + /* Save our LR/return address */ EMIT(PPC_RAW_MFLR(_R3)); if (IS_ENABLED(CONFIG_PPC_FTRACE_OUT_OF_LINE)) EMIT(PPC_RAW_STL(_R3, _R1, alt_lr_off)); @@ -832,24 +832,34 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im EMIT(PPC_RAW_STL(_R3, _R1, bpf_frame_size + PPC_LR_STKOFF)); /* - * Save ip address of the traced function. - * We could recover this from LR, but we will need to address for OOL trampoline, - * and optional GEP area. + * Derive IP address of the traced function. + * In case of CONFIG_PPC_FTRACE_OUT_OF_LINE or BPF program, LR points to the instruction + * after the 'bl' instruction in the OOL stub. Refer to ftrace_init_ool_stub() and + * bpf_arch_text_poke() for OOL stub of kernel functions and bpf programs respectively. + * Relevant stub sequence: + * + * bl + * LR (R3) => mtlr r0 + * b + * + * Recover kernel function/bpf program address from the unconditional + * branch instruction at the end of OOL stub. */ if (IS_ENABLED(CONFIG_PPC_FTRACE_OUT_OF_LINE) || flags & BPF_TRAMP_F_IP_ARG) { EMIT(PPC_RAW_LWZ(_R4, _R3, 4)); EMIT(PPC_RAW_SLWI(_R4, _R4, 6)); EMIT(PPC_RAW_SRAWI(_R4, _R4, 6)); EMIT(PPC_RAW_ADD(_R3, _R3, _R4)); - EMIT(PPC_RAW_ADDI(_R3, _R3, 4)); } if (flags & BPF_TRAMP_F_IP_ARG) EMIT(PPC_RAW_STL(_R3, _R1, ip_off)); - if (IS_ENABLED(CONFIG_PPC_FTRACE_OUT_OF_LINE)) + if (IS_ENABLED(CONFIG_PPC_FTRACE_OUT_OF_LINE)) { /* Fake our LR for unwind */ + EMIT(PPC_RAW_ADDI(_R3, _R3, 4)); EMIT(PPC_RAW_STL(_R3, _R1, bpf_frame_size + PPC_LR_STKOFF)); + } /* Save function arg count -- see bpf_get_func_arg_cnt() */ EMIT(PPC_RAW_LI(_R3, nr_regs)); @@ -1122,7 +1132,7 @@ int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type old_t, bpf_func = (unsigned long)ip; /* We currently only support poking bpf programs */ - if (!__bpf_address_lookup(bpf_func, &size, &offset, name)) { + if (!bpf_address_lookup(bpf_func, &size, &offset, name)) { pr_err("%s (0x%lx): kernel/modules are not supported\n", __func__, bpf_func); return -EOPNOTSUPP; } diff --git a/arch/powerpc/net/bpf_jit_comp64.c b/arch/powerpc/net/bpf_jit_comp64.c index 1fe37128c87640..94f3f031d039b0 100644 --- a/arch/powerpc/net/bpf_jit_comp64.c +++ b/arch/powerpc/net/bpf_jit_comp64.c @@ -319,6 +319,83 @@ int bpf_jit_emit_func_call_rel(u32 *image, u32 *fimage, struct codegen_context * return 0; } +static int zero_extend(u32 *image, struct codegen_context *ctx, u32 src_reg, u32 dst_reg, u32 size) +{ + switch (size) { + case 1: + /* zero-extend 8 bits into 64 bits */ + EMIT(PPC_RAW_RLDICL(dst_reg, src_reg, 0, 56)); + return 0; + case 2: + /* zero-extend 16 bits into 64 bits */ + EMIT(PPC_RAW_RLDICL(dst_reg, src_reg, 0, 48)); + return 0; + case 4: + /* zero-extend 32 bits into 64 bits */ + EMIT(PPC_RAW_RLDICL(dst_reg, src_reg, 0, 32)); + fallthrough; + case 8: + /* Nothing to do */ + return 0; + default: + return -1; + } +} + +static int sign_extend(u32 *image, struct codegen_context *ctx, u32 src_reg, u32 dst_reg, u32 size) +{ + switch (size) { + case 1: + /* sign-extend 8 bits into 64 bits */ + EMIT(PPC_RAW_EXTSB(dst_reg, src_reg)); + return 0; + case 2: + /* sign-extend 16 bits into 64 bits */ + EMIT(PPC_RAW_EXTSH(dst_reg, src_reg)); + return 0; + case 4: + /* sign-extend 32 bits into 64 bits */ + EMIT(PPC_RAW_EXTSW(dst_reg, src_reg)); + fallthrough; + case 8: + /* Nothing to do */ + return 0; + default: + return -1; + } +} + +/* + * Handle powerpc ABI expectations from caller: + * - Unsigned arguments are zero-extended. + * - Signed arguments are sign-extended. + */ +static int prepare_for_kfunc_call(const struct bpf_prog *fp, u32 *image, + struct codegen_context *ctx, + const struct bpf_insn *insn) +{ + const struct btf_func_model *m = bpf_jit_find_kfunc_model(fp, insn); + int i; + + if (!m) + return -1; + + for (i = 0; i < m->nr_args; i++) { + /* Note that BPF ABI only allows up to 5 args for kfuncs */ + u32 reg = bpf_to_ppc(BPF_REG_1 + i), size = m->arg_size[i]; + + if (!(m->arg_flags[i] & BTF_FMODEL_SIGNED_ARG)) { + if (zero_extend(image, ctx, reg, reg, size)) + return -1; + } else { + if (sign_extend(image, ctx, reg, reg, size)) + return -1; + } + } + + return 0; +} + static int bpf_jit_emit_tail_call(u32 *image, struct codegen_context *ctx, u32 out) { /* @@ -353,27 +430,32 @@ static int bpf_jit_emit_tail_call(u32 *image, struct codegen_context *ctx, u32 o /* * tail_call_cnt++; + * Writeback this updated value only if tailcall succeeds. */ EMIT(PPC_RAW_ADDI(bpf_to_ppc(TMP_REG_1), bpf_to_ppc(TMP_REG_1), 1)); - EMIT(PPC_RAW_STD(bpf_to_ppc(TMP_REG_1), _R1, bpf_jit_stack_tailcallcnt(ctx))); /* prog = array->ptrs[index]; */ - EMIT(PPC_RAW_MULI(bpf_to_ppc(TMP_REG_1), b2p_index, 8)); - EMIT(PPC_RAW_ADD(bpf_to_ppc(TMP_REG_1), bpf_to_ppc(TMP_REG_1), b2p_bpf_array)); - EMIT(PPC_RAW_LD(bpf_to_ppc(TMP_REG_1), bpf_to_ppc(TMP_REG_1), offsetof(struct bpf_array, ptrs))); + EMIT(PPC_RAW_MULI(bpf_to_ppc(TMP_REG_2), b2p_index, 8)); + EMIT(PPC_RAW_ADD(bpf_to_ppc(TMP_REG_2), bpf_to_ppc(TMP_REG_2), b2p_bpf_array)); + EMIT(PPC_RAW_LD(bpf_to_ppc(TMP_REG_2), bpf_to_ppc(TMP_REG_2), + offsetof(struct bpf_array, ptrs))); /* * if (prog == NULL) * goto out; */ - EMIT(PPC_RAW_CMPLDI(bpf_to_ppc(TMP_REG_1), 0)); + EMIT(PPC_RAW_CMPLDI(bpf_to_ppc(TMP_REG_2), 0)); PPC_BCC_SHORT(COND_EQ, out); /* goto *(prog->bpf_func + prologue_size); */ - EMIT(PPC_RAW_LD(bpf_to_ppc(TMP_REG_1), bpf_to_ppc(TMP_REG_1), offsetof(struct bpf_prog, bpf_func))); - EMIT(PPC_RAW_ADDI(bpf_to_ppc(TMP_REG_1), bpf_to_ppc(TMP_REG_1), - FUNCTION_DESCR_SIZE + bpf_tailcall_prologue_size)); - EMIT(PPC_RAW_MTCTR(bpf_to_ppc(TMP_REG_1))); + EMIT(PPC_RAW_LD(bpf_to_ppc(TMP_REG_2), bpf_to_ppc(TMP_REG_2), + offsetof(struct bpf_prog, bpf_func))); + EMIT(PPC_RAW_ADDI(bpf_to_ppc(TMP_REG_2), bpf_to_ppc(TMP_REG_2), + FUNCTION_DESCR_SIZE + bpf_tailcall_prologue_size)); + EMIT(PPC_RAW_MTCTR(bpf_to_ppc(TMP_REG_2))); + + /* Writeback updated tailcall count */ + EMIT(PPC_RAW_STD(bpf_to_ppc(TMP_REG_1), _R1, bpf_jit_stack_tailcallcnt(ctx))); /* tear down stack, restore NVRs, ... */ bpf_jit_emit_common_epilogue(image, ctx); @@ -931,14 +1013,16 @@ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, u32 *fimage, struct code /* special mov32 for zext */ EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg, 0, 0, 31)); break; - } else if (off == 8) { - EMIT(PPC_RAW_EXTSB(dst_reg, src_reg)); - } else if (off == 16) { - EMIT(PPC_RAW_EXTSH(dst_reg, src_reg)); - } else if (off == 32) { - EMIT(PPC_RAW_EXTSW(dst_reg, src_reg)); - } else if (dst_reg != src_reg) - EMIT(PPC_RAW_MR(dst_reg, src_reg)); + } + if (off == 0) { + /* MOV */ + if (dst_reg != src_reg) + EMIT(PPC_RAW_MR(dst_reg, src_reg)); + } else { + /* MOVSX: dst = (s8,s16,s32)src (off = 8,16,32) */ + if (sign_extend(image, ctx, src_reg, dst_reg, off / 8)) + return -1; + } goto bpf_alu32_trunc; case BPF_ALU | BPF_MOV | BPF_K: /* (u32) dst = imm */ case BPF_ALU64 | BPF_MOV | BPF_K: /* dst = (s64) imm */ @@ -1395,6 +1479,12 @@ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, u32 *fimage, struct code if (ret < 0) return ret; + /* Take care of powerpc ABI requirements before kfunc call */ + if (insn[i].src_reg == BPF_PSEUDO_KFUNC_CALL) { + if (prepare_for_kfunc_call(fp, image, ctx, &insn[i])) + return -1; + } + ret = bpf_jit_emit_func_call_rel(image, fimage, ctx, func_addr); if (ret) return ret; diff --git a/arch/powerpc/perf/callchain.c b/arch/powerpc/perf/callchain.c index 26aa26482c9ac0..992cc5c9821444 100644 --- a/arch/powerpc/perf/callchain.c +++ b/arch/powerpc/perf/callchain.c @@ -103,6 +103,11 @@ perf_callchain_kernel(struct perf_callchain_entry_ctx *entry, struct pt_regs *re void perf_callchain_user(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs) { + perf_callchain_store(entry, perf_arch_instruction_pointer(regs)); + + if (!current->mm) + return; + if (!is_32bit_task()) perf_callchain_user_64(entry, regs); else diff --git a/arch/powerpc/perf/callchain_32.c b/arch/powerpc/perf/callchain_32.c index ddcc2d8aa64a55..0de21c5d272c26 100644 --- a/arch/powerpc/perf/callchain_32.c +++ b/arch/powerpc/perf/callchain_32.c @@ -142,7 +142,6 @@ void perf_callchain_user_32(struct perf_callchain_entry_ctx *entry, next_ip = perf_arch_instruction_pointer(regs); lr = regs->link; sp = regs->gpr[1]; - perf_callchain_store(entry, next_ip); while (entry->nr < entry->max_stack) { fp = (unsigned int __user *) (unsigned long) sp; diff --git a/arch/powerpc/perf/callchain_64.c b/arch/powerpc/perf/callchain_64.c index 115d1c105e8a84..30fb61c5f0cb04 100644 --- a/arch/powerpc/perf/callchain_64.c +++ b/arch/powerpc/perf/callchain_64.c @@ -77,7 +77,6 @@ void perf_callchain_user_64(struct perf_callchain_entry_ctx *entry, next_ip = perf_arch_instruction_pointer(regs); lr = regs->link; sp = regs->gpr[1]; - perf_callchain_store(entry, next_ip); while (entry->nr < entry->max_stack) { fp = (unsigned long __user *) sp; diff --git a/arch/powerpc/platforms/83xx/km83xx.c b/arch/powerpc/platforms/83xx/km83xx.c index 2b5d187d9b62d5..9ef8fb39dd1b18 100644 --- a/arch/powerpc/platforms/83xx/km83xx.c +++ b/arch/powerpc/platforms/83xx/km83xx.c @@ -155,8 +155,8 @@ machine_device_initcall(mpc83xx_km, mpc83xx_declare_of_platform_devices); /* list of the supported boards */ static char *board[] __initdata = { - "Keymile,KMETER1", - "Keymile,kmpbec8321", + "keymile,KMETER1", + "keymile,kmpbec8321", NULL }; diff --git a/arch/powerpc/platforms/pseries/msi.c b/arch/powerpc/platforms/pseries/msi.c index a82aaa786e9e02..56f17296545ac9 100644 --- a/arch/powerpc/platforms/pseries/msi.c +++ b/arch/powerpc/platforms/pseries/msi.c @@ -19,6 +19,11 @@ #include "pseries.h" +struct pseries_msi_device { + unsigned int msi_quota; + unsigned int msi_used; +}; + static int query_token, change_token; #define RTAS_QUERY_FN 0 @@ -433,8 +438,28 @@ static int pseries_msi_ops_prepare(struct irq_domain *domain, struct device *dev struct msi_domain_info *info = domain->host_data; struct pci_dev *pdev = to_pci_dev(dev); int type = (info->flags & MSI_FLAG_PCI_MSIX) ? PCI_CAP_ID_MSIX : PCI_CAP_ID_MSI; + int ret; + + struct pseries_msi_device *pseries_dev __free(kfree) + = kmalloc(sizeof(*pseries_dev), GFP_KERNEL); + if (!pseries_dev) + return -ENOMEM; + + while (1) { + ret = rtas_prepare_msi_irqs(pdev, nvec, type, arg); + if (!ret) + break; + else if (ret > 0) + nvec = ret; + else + return ret; + } - return rtas_prepare_msi_irqs(pdev, nvec, type, arg); + pseries_dev->msi_quota = nvec; + pseries_dev->msi_used = 0; + + arg->scratchpad[0].ptr = no_free_ptr(pseries_dev); + return 0; } /* @@ -443,9 +468,13 @@ static int pseries_msi_ops_prepare(struct irq_domain *domain, struct device *dev */ static void pseries_msi_ops_teardown(struct irq_domain *domain, msi_alloc_info_t *arg) { + struct pseries_msi_device *pseries_dev = arg->scratchpad[0].ptr; struct pci_dev *pdev = to_pci_dev(domain->dev); rtas_disable_msi(pdev); + + WARN_ON(pseries_dev->msi_used); + kfree(pseries_dev); } static void pseries_msi_shutdown(struct irq_data *d) @@ -546,12 +575,18 @@ static int pseries_irq_domain_alloc(struct irq_domain *domain, unsigned int virq unsigned int nr_irqs, void *arg) { struct pci_controller *phb = domain->host_data; + struct pseries_msi_device *pseries_dev; msi_alloc_info_t *info = arg; struct msi_desc *desc = info->desc; struct pci_dev *pdev = msi_desc_to_pci_dev(desc); int hwirq; int i, ret; + pseries_dev = info->scratchpad[0].ptr; + + if (pseries_dev->msi_used + nr_irqs > pseries_dev->msi_quota) + return -ENOSPC; + hwirq = rtas_query_irq_number(pci_get_pdn(pdev), desc->msi_index); if (hwirq < 0) { dev_err(&pdev->dev, "Failed to query HW IRQ: %d\n", hwirq); @@ -567,9 +602,10 @@ static int pseries_irq_domain_alloc(struct irq_domain *domain, unsigned int virq goto out; irq_domain_set_hwirq_and_chip(domain, virq + i, hwirq + i, - &pseries_msi_irq_chip, domain->host_data); + &pseries_msi_irq_chip, pseries_dev); } + pseries_dev->msi_used += nr_irqs; return 0; out: @@ -582,9 +618,11 @@ static void pseries_irq_domain_free(struct irq_domain *domain, unsigned int virq unsigned int nr_irqs) { struct irq_data *d = irq_domain_get_irq_data(domain, virq); - struct pci_controller *phb = irq_data_get_irq_chip_data(d); + struct pseries_msi_device *pseries_dev = irq_data_get_irq_chip_data(d); + struct pci_controller *phb = domain->host_data; pr_debug("%s bridge %pOF %d #%d\n", __func__, phb->dn, virq, nr_irqs); + pseries_dev->msi_used -= nr_irqs; irq_domain_free_irqs_parent(domain, virq, nr_irqs); } diff --git a/arch/powerpc/tools/ftrace-gen-ool-stubs.sh b/arch/powerpc/tools/ftrace-gen-ool-stubs.sh index bac186bdf64a7e..9218d43aeb5489 100755 --- a/arch/powerpc/tools/ftrace-gen-ool-stubs.sh +++ b/arch/powerpc/tools/ftrace-gen-ool-stubs.sh @@ -15,9 +15,9 @@ if [ -z "$is_64bit" ]; then RELOCATION=R_PPC_ADDR32 fi -num_ool_stubs_total=$($objdump -r -j __patchable_function_entries "$vmlinux_o" | +num_ool_stubs_total=$($objdump -r -j __patchable_function_entries -d "$vmlinux_o" | grep -c "$RELOCATION") -num_ool_stubs_inittext=$($objdump -r -j __patchable_function_entries "$vmlinux_o" | +num_ool_stubs_inittext=$($objdump -r -j __patchable_function_entries -d "$vmlinux_o" | grep -e ".init.text" -e ".text.startup" | grep -c "$RELOCATION") num_ool_stubs_text=$((num_ool_stubs_total - num_ool_stubs_inittext)) diff --git a/arch/riscv/boot/dts/sophgo/cv180x.dtsi b/arch/riscv/boot/dts/sophgo/cv180x.dtsi index 1b2b1969a6484e..06b0ce5a2db7af 100644 --- a/arch/riscv/boot/dts/sophgo/cv180x.dtsi +++ b/arch/riscv/boot/dts/sophgo/cv180x.dtsi @@ -438,8 +438,8 @@ clocks = <&clk CLK_AXI4_USB>, <&clk CLK_APB_USB>; clock-names = "otg", "utmi"; g-np-tx-fifo-size = <32>; - g-rx-fifo-size = <536>; - g-tx-fifo-size = <768 512 512 384 128 128>; + g-rx-fifo-size = <1536>; + g-tx-fifo-size = <128 128 64 64 64 64 32 32>; interrupts = ; phys = <&usbphy>; phy-names = "usb2-phy"; diff --git a/arch/riscv/kernel/kgdb.c b/arch/riscv/kernel/kgdb.c index 15fec5d1e6decf..0bf629204c76a4 100644 --- a/arch/riscv/kernel/kgdb.c +++ b/arch/riscv/kernel/kgdb.c @@ -175,7 +175,7 @@ struct dbg_reg_def_t dbg_reg_def[DBG_MAX_REG_NUM] = { {DBG_REG_T1, GDB_SIZEOF_REG, offsetof(struct pt_regs, t1)}, {DBG_REG_T2, GDB_SIZEOF_REG, offsetof(struct pt_regs, t2)}, {DBG_REG_FP, GDB_SIZEOF_REG, offsetof(struct pt_regs, s0)}, - {DBG_REG_S1, GDB_SIZEOF_REG, offsetof(struct pt_regs, a1)}, + {DBG_REG_S1, GDB_SIZEOF_REG, offsetof(struct pt_regs, s1)}, {DBG_REG_A0, GDB_SIZEOF_REG, offsetof(struct pt_regs, a0)}, {DBG_REG_A1, GDB_SIZEOF_REG, offsetof(struct pt_regs, a1)}, {DBG_REG_A2, GDB_SIZEOF_REG, offsetof(struct pt_regs, a2)}, @@ -244,8 +244,9 @@ sleeping_thread_to_gdb_regs(unsigned long *gdb_regs, struct task_struct *task) gdb_regs[DBG_REG_S6_OFF] = task->thread.s[6]; gdb_regs[DBG_REG_S7_OFF] = task->thread.s[7]; gdb_regs[DBG_REG_S8_OFF] = task->thread.s[8]; - gdb_regs[DBG_REG_S9_OFF] = task->thread.s[10]; - gdb_regs[DBG_REG_S10_OFF] = task->thread.s[11]; + gdb_regs[DBG_REG_S9_OFF] = task->thread.s[9]; + gdb_regs[DBG_REG_S10_OFF] = task->thread.s[10]; + gdb_regs[DBG_REG_S11_OFF] = task->thread.s[11]; gdb_regs[DBG_REG_EPC_OFF] = task->thread.ra; } diff --git a/arch/riscv/kernel/process.c b/arch/riscv/kernel/process.c index 31a392993cb452..b5188dc74727d1 100644 --- a/arch/riscv/kernel/process.c +++ b/arch/riscv/kernel/process.c @@ -324,8 +324,10 @@ long set_tagged_addr_ctrl(struct task_struct *task, unsigned long arg) if (arg & PR_TAGGED_ADDR_ENABLE && (tagged_addr_disabled || !pmlen)) return -EINVAL; - if (!(arg & PR_TAGGED_ADDR_ENABLE)) + if (!(arg & PR_TAGGED_ADDR_ENABLE)) { pmlen = PMLEN_0; + pmm = ENVCFG_PMM_PMLEN_0; + } if (mmap_write_lock_killable(mm)) return -EINTR; diff --git a/arch/riscv/kernel/vector.c b/arch/riscv/kernel/vector.c index 3ed071dab9d832..b112166d51e9f5 100644 --- a/arch/riscv/kernel/vector.c +++ b/arch/riscv/kernel/vector.c @@ -111,8 +111,8 @@ bool insn_is_vector(u32 insn_buf) return false; } -static int riscv_v_thread_zalloc(struct kmem_cache *cache, - struct __riscv_v_ext_state *ctx) +static int riscv_v_thread_ctx_alloc(struct kmem_cache *cache, + struct __riscv_v_ext_state *ctx) { void *datap; @@ -122,13 +122,15 @@ static int riscv_v_thread_zalloc(struct kmem_cache *cache, ctx->datap = datap; memset(ctx, 0, offsetof(struct __riscv_v_ext_state, datap)); + ctx->vlenb = riscv_v_vsize / 32; + return 0; } void riscv_v_thread_alloc(struct task_struct *tsk) { #ifdef CONFIG_RISCV_ISA_V_PREEMPTIVE - riscv_v_thread_zalloc(riscv_v_kernel_cachep, &tsk->thread.kernel_vstate); + riscv_v_thread_ctx_alloc(riscv_v_kernel_cachep, &tsk->thread.kernel_vstate); #endif } @@ -214,12 +216,14 @@ bool riscv_v_first_use_handler(struct pt_regs *regs) * context where VS has been off. So, try to allocate the user's V * context and resume execution. */ - if (riscv_v_thread_zalloc(riscv_v_user_cachep, ¤t->thread.vstate)) { + if (riscv_v_thread_ctx_alloc(riscv_v_user_cachep, ¤t->thread.vstate)) { force_sig(SIGBUS); return true; } + riscv_v_vstate_on(regs); riscv_v_vstate_set_restore(current, regs); + return true; } diff --git a/arch/riscv/kernel/vmlinux.lds.S b/arch/riscv/kernel/vmlinux.lds.S index 61bd5ba6680a78..997f9eb3b22b13 100644 --- a/arch/riscv/kernel/vmlinux.lds.S +++ b/arch/riscv/kernel/vmlinux.lds.S @@ -170,6 +170,7 @@ SECTIONS STABS_DEBUG DWARF_DEBUG + MODINFO ELF_DETAILS .riscv.attributes 0 : { *(.riscv.attributes) } diff --git a/arch/riscv/kvm/vcpu.c b/arch/riscv/kvm/vcpu.c index a55a95da54d0fa..fdd99ac1e71482 100644 --- a/arch/riscv/kvm/vcpu.c +++ b/arch/riscv/kvm/vcpu.c @@ -24,7 +24,7 @@ #define CREATE_TRACE_POINTS #include "trace.h" -const struct _kvm_stats_desc kvm_vcpu_stats_desc[] = { +const struct kvm_stats_desc kvm_vcpu_stats_desc[] = { KVM_GENERIC_VCPU_STATS(), STATS_DESC_COUNTER(VCPU, ecall_exit_stat), STATS_DESC_COUNTER(VCPU, wfi_exit_stat), diff --git a/arch/riscv/kvm/vm.c b/arch/riscv/kvm/vm.c index 66d91ae6e9b2ad..715a06ae8c1313 100644 --- a/arch/riscv/kvm/vm.c +++ b/arch/riscv/kvm/vm.c @@ -13,7 +13,7 @@ #include #include -const struct _kvm_stats_desc kvm_vm_stats_desc[] = { +const struct kvm_stats_desc kvm_vm_stats_desc[] = { KVM_GENERIC_VM_STATS() }; static_assert(ARRAY_SIZE(kvm_vm_stats_desc) == diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig index 0e5fad5f06ca11..783be50f38f2b3 100644 --- a/arch/s390/Kconfig +++ b/arch/s390/Kconfig @@ -275,6 +275,7 @@ config S390 select SPARSE_IRQ select SWIOTLB select SYSCTL_EXCEPTION_TRACE + select SYSTEM_DATA_VERIFICATION if KEXEC_SIG select THREAD_INFO_IN_TASK select TRACE_IRQFLAGS_SUPPORT select TTY @@ -301,7 +302,7 @@ config ARCH_SUPPORTS_KEXEC_FILE def_bool y config ARCH_SUPPORTS_KEXEC_SIG - def_bool MODULE_SIG_FORMAT + def_bool y config ARCH_SUPPORTS_KEXEC_PURGATORY def_bool y diff --git a/arch/s390/boot/Makefile b/arch/s390/boot/Makefile index 490167faba7a43..a1e719a79d38cf 100644 --- a/arch/s390/boot/Makefile +++ b/arch/s390/boot/Makefile @@ -21,6 +21,7 @@ KBUILD_AFLAGS := $(filter-out $(CC_FLAGS_MARCH),$(KBUILD_AFLAGS_DECOMPRESSOR)) KBUILD_CFLAGS := $(filter-out $(CC_FLAGS_MARCH),$(KBUILD_CFLAGS_DECOMPRESSOR)) KBUILD_AFLAGS += $(CC_FLAGS_MARCH_MINIMUM) -D__DISABLE_EXPORTS KBUILD_CFLAGS += $(CC_FLAGS_MARCH_MINIMUM) -D__DISABLE_EXPORTS +KBUILD_CFLAGS += $(call cc-option, -Wno-default-const-init-unsafe) CFLAGS_sclp_early_core.o += -I$(srctree)/drivers/s390/char diff --git a/arch/s390/include/asm/barrier.h b/arch/s390/include/asm/barrier.h index f3184073e754ff..dad02f5b3c8d30 100644 --- a/arch/s390/include/asm/barrier.h +++ b/arch/s390/include/asm/barrier.h @@ -62,8 +62,8 @@ do { \ * @size: number of elements in array */ #define array_index_mask_nospec array_index_mask_nospec -static inline unsigned long array_index_mask_nospec(unsigned long index, - unsigned long size) +static __always_inline unsigned long array_index_mask_nospec(unsigned long index, + unsigned long size) { unsigned long mask; diff --git a/arch/s390/include/asm/idle.h b/arch/s390/include/asm/idle.h index 09f763b9eb40aa..133059d9a949c7 100644 --- a/arch/s390/include/asm/idle.h +++ b/arch/s390/include/asm/idle.h @@ -23,5 +23,6 @@ extern struct device_attribute dev_attr_idle_count; extern struct device_attribute dev_attr_idle_time_us; void psw_idle(struct s390_idle_data *data, unsigned long psw_mask); +void update_timer_idle(void); #endif /* _S390_IDLE_H */ diff --git a/arch/s390/include/asm/processor.h b/arch/s390/include/asm/processor.h index 3affba95845bdd..86076f27c2d9b7 100644 --- a/arch/s390/include/asm/processor.h +++ b/arch/s390/include/asm/processor.h @@ -158,7 +158,7 @@ static __always_inline void __stackleak_poison(unsigned long erase_low, " j 4f\n" "3: mvc 8(1,%[addr]),0(%[addr])\n" "4:" - : [addr] "+&a" (erase_low), [count] "+&d" (count), [tmp] "=&a" (tmp) + : [addr] "+&a" (erase_low), [count] "+&a" (count), [tmp] "=&a" (tmp) : [poison] "d" (poison) : "memory", "cc" ); diff --git a/arch/s390/kernel/entry.S b/arch/s390/kernel/entry.S index b7f1553d9ee5ba..fc385e99f1f80d 100644 --- a/arch/s390/kernel/entry.S +++ b/arch/s390/kernel/entry.S @@ -260,6 +260,7 @@ SYM_CODE_START(system_call) xgr %r9,%r9 xgr %r10,%r10 xgr %r11,%r11 + xgr %r12,%r12 la %r2,STACK_FRAME_OVERHEAD(%r15) # pointer to pt_regs mvc __PT_R8(64,%r2),__LC_SAVE_AREA(%r13) MBEAR %r2,%r13 @@ -396,6 +397,7 @@ SYM_CODE_START(\name) xgr %r6,%r6 xgr %r7,%r7 xgr %r10,%r10 + xgr %r12,%r12 xc __PT_FLAGS(8,%r11),__PT_FLAGS(%r11) mvc __PT_R8(64,%r11),__LC_SAVE_AREA(%r13) MBEAR %r11,%r13 @@ -485,6 +487,7 @@ SYM_CODE_START(mcck_int_handler) xgr %r6,%r6 xgr %r7,%r7 xgr %r10,%r10 + xgr %r12,%r12 stmg %r8,%r9,__PT_PSW(%r11) xc __PT_FLAGS(8,%r11),__PT_FLAGS(%r11) xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15) diff --git a/arch/s390/kernel/idle.c b/arch/s390/kernel/idle.c index 39cb8d0ae34806..0f9e53f0a06861 100644 --- a/arch/s390/kernel/idle.c +++ b/arch/s390/kernel/idle.c @@ -21,11 +21,10 @@ static DEFINE_PER_CPU(struct s390_idle_data, s390_idle); -void account_idle_time_irq(void) +void update_timer_idle(void) { struct s390_idle_data *idle = this_cpu_ptr(&s390_idle); struct lowcore *lc = get_lowcore(); - unsigned long idle_time; u64 cycles_new[8]; int i; @@ -35,13 +34,19 @@ void account_idle_time_irq(void) this_cpu_add(mt_cycles[i], cycles_new[i] - idle->mt_cycles_enter[i]); } - idle_time = lc->int_clock - idle->clock_idle_enter; - lc->steal_timer += idle->clock_idle_enter - lc->last_update_clock; lc->last_update_clock = lc->int_clock; lc->system_timer += lc->last_update_timer - idle->timer_idle_enter; lc->last_update_timer = lc->sys_enter_timer; +} + +void account_idle_time_irq(void) +{ + struct s390_idle_data *idle = this_cpu_ptr(&s390_idle); + unsigned long idle_time; + + idle_time = get_lowcore()->int_clock - idle->clock_idle_enter; /* Account time spent with enabled wait psw loaded as idle time. */ WRITE_ONCE(idle->idle_time, READ_ONCE(idle->idle_time) + idle_time); diff --git a/arch/s390/kernel/ipl.c b/arch/s390/kernel/ipl.c index dcdc7e27484867..049c557c452ff9 100644 --- a/arch/s390/kernel/ipl.c +++ b/arch/s390/kernel/ipl.c @@ -2377,7 +2377,7 @@ void __init setup_ipl(void) atomic_notifier_chain_register(&panic_notifier_list, &on_panic_nb); } -void s390_reset_system(void) +void __no_stack_protector s390_reset_system(void) { /* Disable prefixing */ set_prefix(0); diff --git a/arch/s390/kernel/irq.c b/arch/s390/kernel/irq.c index bdf9c7cb5685b9..080e9285b33795 100644 --- a/arch/s390/kernel/irq.c +++ b/arch/s390/kernel/irq.c @@ -146,6 +146,10 @@ void noinstr do_io_irq(struct pt_regs *regs) struct pt_regs *old_regs = set_irq_regs(regs); bool from_idle; + from_idle = test_and_clear_cpu_flag(CIF_ENABLED_WAIT); + if (from_idle) + update_timer_idle(); + irq_enter_rcu(); if (user_mode(regs)) { @@ -154,7 +158,6 @@ void noinstr do_io_irq(struct pt_regs *regs) current->thread.last_break = regs->last_break; } - from_idle = test_and_clear_cpu_flag(CIF_ENABLED_WAIT); if (from_idle) account_idle_time_irq(); @@ -182,6 +185,10 @@ void noinstr do_ext_irq(struct pt_regs *regs) struct pt_regs *old_regs = set_irq_regs(regs); bool from_idle; + from_idle = test_and_clear_cpu_flag(CIF_ENABLED_WAIT); + if (from_idle) + update_timer_idle(); + irq_enter_rcu(); if (user_mode(regs)) { @@ -194,7 +201,6 @@ void noinstr do_ext_irq(struct pt_regs *regs) regs->int_parm = get_lowcore()->ext_params; regs->int_parm_long = get_lowcore()->ext_params2; - from_idle = test_and_clear_cpu_flag(CIF_ENABLED_WAIT); if (from_idle) account_idle_time_irq(); diff --git a/arch/s390/kernel/perf_cpum_sf.c b/arch/s390/kernel/perf_cpum_sf.c index 459af23a47a5e3..26926cb077ae5b 100644 --- a/arch/s390/kernel/perf_cpum_sf.c +++ b/arch/s390/kernel/perf_cpum_sf.c @@ -841,7 +841,7 @@ static bool is_callchain_event(struct perf_event *event) u64 sample_type = event->attr.sample_type; return sample_type & (PERF_SAMPLE_CALLCHAIN | PERF_SAMPLE_REGS_USER | - PERF_SAMPLE_STACK_USER); + PERF_SAMPLE_REGS_INTR | PERF_SAMPLE_STACK_USER); } static int cpumsf_pmu_event_init(struct perf_event *event) @@ -1168,6 +1168,7 @@ static void hw_collect_samples(struct perf_event *event, unsigned long *sdbt, static void hw_perf_event_update(struct perf_event *event, int flush_all) { unsigned long long event_overflow, sampl_overflow, num_sdb; + struct cpu_hw_sf *cpuhw = this_cpu_ptr(&cpu_hw_sf); struct hw_perf_event *hwc = &event->hw; union hws_trailer_header prev, new; struct hws_trailer_entry *te; @@ -1247,8 +1248,11 @@ static void hw_perf_event_update(struct perf_event *event, int flush_all) * are dropped. * Slightly increase the interval to avoid hitting this limit. */ - if (event_overflow) + if (event_overflow) { SAMPL_RATE(hwc) += DIV_ROUND_UP(SAMPL_RATE(hwc), 10); + if (SAMPL_RATE(hwc) > cpuhw->qsi.max_sampl_rate) + SAMPL_RATE(hwc) = cpuhw->qsi.max_sampl_rate; + } } static inline unsigned long aux_sdb_index(struct aux_buffer *aux, diff --git a/arch/s390/kernel/syscall.c b/arch/s390/kernel/syscall.c index 795b6cca74c9be..d103c853e12067 100644 --- a/arch/s390/kernel/syscall.c +++ b/arch/s390/kernel/syscall.c @@ -13,6 +13,7 @@ */ #include +#include #include #include #include @@ -131,8 +132,10 @@ void noinstr __do_syscall(struct pt_regs *regs, int per_trap) if (unlikely(test_and_clear_pt_regs_flag(regs, PIF_SYSCALL_RET_SET))) goto out; regs->gprs[2] = -ENOSYS; - if (likely(nr < NR_syscalls)) + if (likely(nr < NR_syscalls)) { + nr = array_index_nospec(nr, NR_syscalls); regs->gprs[2] = sys_call_table[nr](regs); + } out: syscall_exit_to_user_mode(regs); } diff --git a/arch/s390/kernel/vmlinux.lds.S b/arch/s390/kernel/vmlinux.lds.S index 53bcbb91bb9bd5..2b62395e35bfb1 100644 --- a/arch/s390/kernel/vmlinux.lds.S +++ b/arch/s390/kernel/vmlinux.lds.S @@ -221,6 +221,7 @@ SECTIONS /* Debugging sections. */ STABS_DEBUG DWARF_DEBUG + MODINFO ELF_DETAILS /* diff --git a/arch/s390/kernel/vtime.c b/arch/s390/kernel/vtime.c index 234a0ba3051082..122d30b1044017 100644 --- a/arch/s390/kernel/vtime.c +++ b/arch/s390/kernel/vtime.c @@ -225,10 +225,6 @@ static u64 vtime_delta(void) return timer - lc->last_update_timer; } -/* - * Update process times based on virtual cpu times stored by entry.S - * to the lowcore fields user_timer, system_timer & steal_clock. - */ void vtime_account_kernel(struct task_struct *tsk) { struct lowcore *lc = get_lowcore(); @@ -238,27 +234,17 @@ void vtime_account_kernel(struct task_struct *tsk) lc->guest_timer += delta; else lc->system_timer += delta; - - virt_timer_forward(delta); } EXPORT_SYMBOL_GPL(vtime_account_kernel); void vtime_account_softirq(struct task_struct *tsk) { - u64 delta = vtime_delta(); - - get_lowcore()->softirq_timer += delta; - - virt_timer_forward(delta); + get_lowcore()->softirq_timer += vtime_delta(); } void vtime_account_hardirq(struct task_struct *tsk) { - u64 delta = vtime_delta(); - - get_lowcore()->hardirq_timer += delta; - - virt_timer_forward(delta); + get_lowcore()->hardirq_timer += vtime_delta(); } /* diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c index 56a50524b3eee4..495141bf03989a 100644 --- a/arch/s390/kvm/kvm-s390.c +++ b/arch/s390/kvm/kvm-s390.c @@ -64,7 +64,7 @@ #define VCPU_IRQS_MAX_BUF (sizeof(struct kvm_s390_irq) * \ (KVM_MAX_VCPUS + LOCAL_IRQS)) -const struct _kvm_stats_desc kvm_vm_stats_desc[] = { +const struct kvm_stats_desc kvm_vm_stats_desc[] = { KVM_GENERIC_VM_STATS(), STATS_DESC_COUNTER(VM, inject_io), STATS_DESC_COUNTER(VM, inject_float_mchk), @@ -90,7 +90,7 @@ const struct kvm_stats_header kvm_vm_stats_header = { sizeof(kvm_vm_stats_desc), }; -const struct _kvm_stats_desc kvm_vcpu_stats_desc[] = { +const struct kvm_stats_desc kvm_vcpu_stats_desc[] = { KVM_GENERIC_VCPU_STATS(), STATS_DESC_COUNTER(VCPU, exit_userspace), STATS_DESC_COUNTER(VCPU, exit_null), diff --git a/arch/s390/lib/xor.c b/arch/s390/lib/xor.c index 1721b73b780369..81c0235c04666e 100644 --- a/arch/s390/lib/xor.c +++ b/arch/s390/lib/xor.c @@ -28,8 +28,8 @@ static void xor_xc_2(unsigned long bytes, unsigned long * __restrict p1, " j 3f\n" "2: xc 0(1,%1),0(%2)\n" "3:" - : : "d" (bytes), "a" (p1), "a" (p2) - : "0", "cc", "memory"); + : "+d" (bytes), "+a" (p1), "+a" (p2) + : : "0", "cc", "memory"); } static void xor_xc_3(unsigned long bytes, unsigned long * __restrict p1, @@ -96,7 +96,6 @@ static void xor_xc_5(unsigned long bytes, unsigned long * __restrict p1, const unsigned long * __restrict p5) { asm volatile( - " larl 1,2f\n" " aghi %0,-1\n" " jm 6f\n" " srlg 0,%0,8\n" diff --git a/arch/s390/mm/fault.c b/arch/s390/mm/fault.c index e2e13778c36a93..b977150443550a 100644 --- a/arch/s390/mm/fault.c +++ b/arch/s390/mm/fault.c @@ -441,10 +441,17 @@ void do_secure_storage_access(struct pt_regs *regs) folio = phys_to_folio(addr); if (unlikely(!folio_try_get(folio))) return; - rc = arch_make_folio_accessible(folio); + rc = uv_convert_from_secure(folio_to_phys(folio)); + if (!rc) + clear_bit(PG_arch_1, &folio->flags.f); folio_put(folio); + /* + * There are some valid fixup types for kernel + * accesses to donated secure memory. zeropad is one + * of them. + */ if (rc) - BUG(); + return handle_fault_error_nolock(regs, 0); } else { if (faulthandler_disabled()) return handle_fault_error_nolock(regs, 0); diff --git a/arch/s390/mm/pfault.c b/arch/s390/mm/pfault.c index 2f829448c719e3..6ecd6b0a22a8cb 100644 --- a/arch/s390/mm/pfault.c +++ b/arch/s390/mm/pfault.c @@ -62,7 +62,7 @@ int __pfault_init(void) "0: nopr %%r7\n" EX_TABLE(0b, 0b) : [rc] "+d" (rc) - : [refbk] "a" (&pfault_init_refbk), "m" (pfault_init_refbk) + : [refbk] "a" (virt_to_phys(&pfault_init_refbk)), "m" (pfault_init_refbk) : "cc"); return rc; } @@ -84,7 +84,7 @@ void __pfault_fini(void) "0: nopr %%r7\n" EX_TABLE(0b, 0b) : - : [refbk] "a" (&pfault_fini_refbk), "m" (pfault_fini_refbk) + : [refbk] "a" (virt_to_phys(&pfault_fini_refbk)), "m" (pfault_fini_refbk) : "cc"); } diff --git a/arch/s390/pci/pci.c b/arch/s390/pci/pci.c index 57f3980b98a927..7f44b0644a20eb 100644 --- a/arch/s390/pci/pci.c +++ b/arch/s390/pci/pci.c @@ -231,24 +231,33 @@ int zpci_fmb_disable_device(struct zpci_dev *zdev) static int zpci_cfg_load(struct zpci_dev *zdev, int offset, u32 *val, u8 len) { u64 req = ZPCI_CREATE_REQ(zdev->fh, ZPCI_PCIAS_CFGSPC, len); + int rc = -ENODEV; u64 data; - int rc; + + if (!zdev_enabled(zdev)) + goto out_err; rc = __zpci_load(&data, req, offset); - if (!rc) { - data = le64_to_cpu((__force __le64) data); - data >>= (8 - len) * 8; - *val = (u32) data; - } else - *val = 0xffffffff; + if (rc) + goto out_err; + data = le64_to_cpu((__force __le64)data); + data >>= (8 - len) * 8; + *val = (u32)data; + return 0; + +out_err: + PCI_SET_ERROR_RESPONSE(val); return rc; } static int zpci_cfg_store(struct zpci_dev *zdev, int offset, u32 val, u8 len) { u64 req = ZPCI_CREATE_REQ(zdev->fh, ZPCI_PCIAS_CFGSPC, len); + int rc = -ENODEV; u64 data = val; - int rc; + + if (!zdev_enabled(zdev)) + return rc; data <<= (8 - len) * 8; data = (__force u64) cpu_to_le64(data); diff --git a/arch/s390/purgatory/Makefile b/arch/s390/purgatory/Makefile index 0c196a5b194af0..61d240a37633d8 100644 --- a/arch/s390/purgatory/Makefile +++ b/arch/s390/purgatory/Makefile @@ -23,6 +23,7 @@ KBUILD_CFLAGS += -D__DISABLE_EXPORTS KBUILD_CFLAGS += $(CLANG_FLAGS) KBUILD_CFLAGS += $(if $(CONFIG_CC_IS_CLANG),-Wno-microsoft-anon-tag) KBUILD_CFLAGS += $(call cc-option,-fno-PIE) +KBUILD_CFLAGS += $(call cc-option, -Wno-default-const-init-unsafe) KBUILD_AFLAGS := $(filter-out -DCC_USING_EXPOLINE,$(KBUILD_AFLAGS)) KBUILD_AFLAGS += -D__DISABLE_EXPORTS diff --git a/arch/sh/drivers/platform_early.c b/arch/sh/drivers/platform_early.c index 143747c45206fe..48ddbc547bd9ac 100644 --- a/arch/sh/drivers/platform_early.c +++ b/arch/sh/drivers/platform_early.c @@ -26,10 +26,6 @@ static int platform_match(struct device *dev, struct device_driver *drv) struct platform_device *pdev = to_platform_device(dev); struct platform_driver *pdrv = to_platform_driver(drv); - /* When driver_override is set, only bind to the matching driver */ - if (pdev->driver_override) - return !strcmp(pdev->driver_override, drv->name); - /* Then try to match against the id table */ if (pdrv->id_table) return platform_match_id(pdrv->id_table, pdev) != NULL; diff --git a/arch/sh/kernel/vmlinux.lds.S b/arch/sh/kernel/vmlinux.lds.S index 008c30289eaa6d..169c63fb3c1dcb 100644 --- a/arch/sh/kernel/vmlinux.lds.S +++ b/arch/sh/kernel/vmlinux.lds.S @@ -89,6 +89,7 @@ SECTIONS STABS_DEBUG DWARF_DEBUG + MODINFO ELF_DETAILS DISCARDS diff --git a/arch/sparc/include/uapi/asm/ioctls.h b/arch/sparc/include/uapi/asm/ioctls.h index 7fd2f5873c9e7a..a8bbdf9877a41a 100644 --- a/arch/sparc/include/uapi/asm/ioctls.h +++ b/arch/sparc/include/uapi/asm/ioctls.h @@ -5,10 +5,10 @@ #include /* Big T */ -#define TCGETA _IOR('T', 1, struct termio) -#define TCSETA _IOW('T', 2, struct termio) -#define TCSETAW _IOW('T', 3, struct termio) -#define TCSETAF _IOW('T', 4, struct termio) +#define TCGETA 0x40125401 /* _IOR('T', 1, struct termio) */ +#define TCSETA 0x80125402 /* _IOW('T', 2, struct termio) */ +#define TCSETAW 0x80125403 /* _IOW('T', 3, struct termio) */ +#define TCSETAF 0x80125404 /* _IOW('T', 4, struct termio) */ #define TCSBRK _IO('T', 5) #define TCXONC _IO('T', 6) #define TCFLSH _IO('T', 7) diff --git a/arch/sparc/kernel/iommu.c b/arch/sparc/kernel/iommu.c index 46ef88bc9c26ef..7613ab0ffb89d0 100644 --- a/arch/sparc/kernel/iommu.c +++ b/arch/sparc/kernel/iommu.c @@ -312,6 +312,8 @@ static dma_addr_t dma_4u_map_phys(struct device *dev, phys_addr_t phys, if (direction != DMA_TO_DEVICE) iopte_protection |= IOPTE_WRITE; + phys &= IO_PAGE_MASK; + for (i = 0; i < npages; i++, base++, phys += IO_PAGE_SIZE) iopte_val(*base) = iopte_protection | phys; diff --git a/arch/sparc/kernel/pci_sun4v.c b/arch/sparc/kernel/pci_sun4v.c index 791f0a76665f62..58ca4148f86be5 100644 --- a/arch/sparc/kernel/pci_sun4v.c +++ b/arch/sparc/kernel/pci_sun4v.c @@ -410,6 +410,8 @@ static dma_addr_t dma_4v_map_phys(struct device *dev, phys_addr_t phys, iommu_batch_start(dev, prot, entry); + phys &= IO_PAGE_MASK; + for (i = 0; i < npages; i++, phys += IO_PAGE_SIZE) { long err = iommu_batch_add(phys, mask); if (unlikely(err < 0L)) diff --git a/arch/sparc/kernel/process.c b/arch/sparc/kernel/process.c index 0442ab00518d3c..7d69877511fac9 100644 --- a/arch/sparc/kernel/process.c +++ b/arch/sparc/kernel/process.c @@ -17,14 +17,18 @@ asmlinkage long sparc_fork(struct pt_regs *regs) { - unsigned long orig_i1 = regs->u_regs[UREG_I1]; + unsigned long orig_i1; long ret; struct kernel_clone_args args = { .exit_signal = SIGCHLD, - /* Reuse the parent's stack for the child. */ - .stack = regs->u_regs[UREG_FP], }; + synchronize_user_stack(); + + orig_i1 = regs->u_regs[UREG_I1]; + /* Reuse the parent's stack for the child. */ + args.stack = regs->u_regs[UREG_FP]; + ret = kernel_clone(&args); /* If we get an error and potentially restart the system @@ -40,16 +44,19 @@ asmlinkage long sparc_fork(struct pt_regs *regs) asmlinkage long sparc_vfork(struct pt_regs *regs) { - unsigned long orig_i1 = regs->u_regs[UREG_I1]; + unsigned long orig_i1; long ret; - struct kernel_clone_args args = { .flags = CLONE_VFORK | CLONE_VM, .exit_signal = SIGCHLD, - /* Reuse the parent's stack for the child. */ - .stack = regs->u_regs[UREG_FP], }; + synchronize_user_stack(); + + orig_i1 = regs->u_regs[UREG_I1]; + /* Reuse the parent's stack for the child. */ + args.stack = regs->u_regs[UREG_FP]; + ret = kernel_clone(&args); /* If we get an error and potentially restart the system @@ -65,15 +72,18 @@ asmlinkage long sparc_vfork(struct pt_regs *regs) asmlinkage long sparc_clone(struct pt_regs *regs) { - unsigned long orig_i1 = regs->u_regs[UREG_I1]; - unsigned int flags = lower_32_bits(regs->u_regs[UREG_I0]); + unsigned long orig_i1; + unsigned int flags; long ret; + struct kernel_clone_args args = {0}; - struct kernel_clone_args args = { - .flags = (flags & ~CSIGNAL), - .exit_signal = (flags & CSIGNAL), - .tls = regs->u_regs[UREG_I3], - }; + synchronize_user_stack(); + + orig_i1 = regs->u_regs[UREG_I1]; + flags = lower_32_bits(regs->u_regs[UREG_I0]); + args.flags = (flags & ~CSIGNAL); + args.exit_signal = (flags & CSIGNAL); + args.tls = regs->u_regs[UREG_I3]; #ifdef CONFIG_COMPAT if (test_thread_flag(TIF_32BIT)) { diff --git a/arch/sparc/kernel/vmlinux.lds.S b/arch/sparc/kernel/vmlinux.lds.S index f1b86eb3034043..7ea510d9b42f24 100644 --- a/arch/sparc/kernel/vmlinux.lds.S +++ b/arch/sparc/kernel/vmlinux.lds.S @@ -191,6 +191,7 @@ SECTIONS STABS_DEBUG DWARF_DEBUG + MODINFO ELF_DETAILS DISCARDS diff --git a/arch/um/kernel/dyn.lds.S b/arch/um/kernel/dyn.lds.S index a36b7918a011ac..ad3cefeff2acb1 100644 --- a/arch/um/kernel/dyn.lds.S +++ b/arch/um/kernel/dyn.lds.S @@ -172,6 +172,7 @@ SECTIONS STABS_DEBUG DWARF_DEBUG + MODINFO ELF_DETAILS DISCARDS diff --git a/arch/um/kernel/uml.lds.S b/arch/um/kernel/uml.lds.S index a409d4b66114f7..30aa24348d60cd 100644 --- a/arch/um/kernel/uml.lds.S +++ b/arch/um/kernel/uml.lds.S @@ -113,6 +113,7 @@ SECTIONS STABS_DEBUG DWARF_DEBUG + MODINFO ELF_DETAILS DISCARDS diff --git a/arch/um/os-Linux/signal.c b/arch/um/os-Linux/signal.c index 327fb3c52fc793..de372b936a8041 100644 --- a/arch/um/os-Linux/signal.c +++ b/arch/um/os-Linux/signal.c @@ -36,7 +36,6 @@ void (*sig_info[NSIG])(int, struct siginfo *, struct uml_pt_regs *, void *mc) = static void sig_handler_common(int sig, struct siginfo *si, mcontext_t *mc) { struct uml_pt_regs r; - int save_errno = errno; r.is_user = 0; if (sig == SIGSEGV) { @@ -50,8 +49,6 @@ static void sig_handler_common(int sig, struct siginfo *si, mcontext_t *mc) unblock_signals_trace(); (*sig_info[sig])(sig, si, &r, mc); - - errno = save_errno; } /* @@ -207,8 +204,11 @@ static void hard_handler(int sig, siginfo_t *si, void *p) { ucontext_t *uc = p; mcontext_t *mc = &uc->uc_mcontext; + int save_errno = errno; (*handlers[sig])(sig, (struct siginfo *)si, mc); + + errno = save_errno; } void set_handler(int sig) diff --git a/arch/x86/boot/compressed/Makefile b/arch/x86/boot/compressed/Makefile index 68f9d7a1683b57..b8b2b7bea1d31a 100644 --- a/arch/x86/boot/compressed/Makefile +++ b/arch/x86/boot/compressed/Makefile @@ -113,6 +113,7 @@ vmlinux-objs-$(CONFIG_EFI_SBAT) += $(obj)/sbat.o ifdef CONFIG_EFI_SBAT $(obj)/sbat.o: $(CONFIG_EFI_SBAT_FILE) +AFLAGS_sbat.o += -I $(srctree) endif $(obj)/vmlinux: $(vmlinux-objs-y) $(vmlinux-libs-y) FORCE diff --git a/arch/x86/boot/compressed/sev.c b/arch/x86/boot/compressed/sev.c index c8c1464b3a56e7..e468476e9e4a07 100644 --- a/arch/x86/boot/compressed/sev.c +++ b/arch/x86/boot/compressed/sev.c @@ -28,17 +28,17 @@ #include "sev.h" static struct ghcb boot_ghcb_page __aligned(PAGE_SIZE); -struct ghcb *boot_ghcb; +struct ghcb *boot_ghcb __section(".data"); #undef __init #define __init #define __BOOT_COMPRESSED -u8 snp_vmpl; -u16 ghcb_version; +u8 snp_vmpl __section(".data"); +u16 ghcb_version __section(".data"); -u64 boot_svsm_caa_pa; +u64 boot_svsm_caa_pa __section(".data"); /* Include code for early handlers */ #include "../../boot/startup/sev-shared.c" @@ -188,6 +188,7 @@ bool sev_es_check_ghcb_fault(unsigned long address) MSR_AMD64_SNP_RESERVED_BIT13 | \ MSR_AMD64_SNP_RESERVED_BIT15 | \ MSR_AMD64_SNP_SECURE_AVIC | \ + MSR_AMD64_SNP_RESERVED_BITS19_22 | \ MSR_AMD64_SNP_RESERVED_MASK) #ifdef CONFIG_AMD_SECURE_AVIC diff --git a/arch/x86/boot/compressed/vmlinux.lds.S b/arch/x86/boot/compressed/vmlinux.lds.S index 587ce3e7c5048a..e0b152715d9c6f 100644 --- a/arch/x86/boot/compressed/vmlinux.lds.S +++ b/arch/x86/boot/compressed/vmlinux.lds.S @@ -88,7 +88,7 @@ SECTIONS /DISCARD/ : { *(.dynamic) *(.dynsym) *(.dynstr) *(.dynbss) *(.hash) *(.gnu.hash) - *(.note.*) + *(.note.*) *(.modinfo) } .got.plt (INFO) : { diff --git a/arch/x86/boot/startup/sev-shared.c b/arch/x86/boot/startup/sev-shared.c index a0fa8bb2b9458e..d9ac3a929d335a 100644 --- a/arch/x86/boot/startup/sev-shared.c +++ b/arch/x86/boot/startup/sev-shared.c @@ -31,7 +31,7 @@ static u32 cpuid_std_range_max __ro_after_init; static u32 cpuid_hyp_range_max __ro_after_init; static u32 cpuid_ext_range_max __ro_after_init; -bool sev_snp_needs_sfw; +bool sev_snp_needs_sfw __section(".data"); void __noreturn sev_es_terminate(unsigned int set, unsigned int reason) diff --git a/arch/x86/coco/sev/core.c b/arch/x86/coco/sev/core.c index 9ae3b11754e655..d20e9cc065a87d 100644 --- a/arch/x86/coco/sev/core.c +++ b/arch/x86/coco/sev/core.c @@ -122,6 +122,7 @@ static const char * const sev_status_feat_names[] = { [MSR_AMD64_SNP_VMSA_REG_PROT_BIT] = "VMSARegProt", [MSR_AMD64_SNP_SMT_PROT_BIT] = "SMTProt", [MSR_AMD64_SNP_SECURE_AVIC_BIT] = "SecureAVIC", + [MSR_AMD64_SNP_IBPB_ON_ENTRY_BIT] = "IBPBOnEntry", }; /* @@ -2008,8 +2009,7 @@ void snp_msg_free(struct snp_msg_desc *mdesc) free_shared_pages(mdesc->request, sizeof(struct snp_guest_msg)); iounmap((__force void __iomem *)mdesc->secrets); - memset(mdesc, 0, sizeof(*mdesc)); - kfree(mdesc); + kfree_sensitive(mdesc); } EXPORT_SYMBOL_GPL(snp_msg_free); diff --git a/arch/x86/coco/sev/noinstr.c b/arch/x86/coco/sev/noinstr.c index b527eafb631235..422ce72f51d58b 100644 --- a/arch/x86/coco/sev/noinstr.c +++ b/arch/x86/coco/sev/noinstr.c @@ -120,6 +120,9 @@ noinstr struct ghcb *__sev_get_ghcb(struct ghcb_state *state) WARN_ON(!irqs_disabled()); + if (!sev_cfg.ghcbs_initialized) + return boot_ghcb; + data = this_cpu_read(runtime_data); ghcb = &data->ghcb_page; @@ -163,6 +166,9 @@ noinstr void __sev_put_ghcb(struct ghcb_state *state) WARN_ON(!irqs_disabled()); + if (!sev_cfg.ghcbs_initialized) + return; + data = this_cpu_read(runtime_data); ghcb = &data->ghcb_page; diff --git a/arch/x86/entry/entry_fred.c b/arch/x86/entry/entry_fred.c index 94e626cc6a0742..52c50e36bf8df6 100644 --- a/arch/x86/entry/entry_fred.c +++ b/arch/x86/entry/entry_fred.c @@ -159,8 +159,6 @@ void __init fred_complete_exception_setup(void) static noinstr void fred_extint(struct pt_regs *regs) { unsigned int vector = regs->fred_ss.vector; - unsigned int index = array_index_nospec(vector - FIRST_SYSTEM_VECTOR, - NR_SYSTEM_VECTORS); if (WARN_ON_ONCE(vector < FIRST_EXTERNAL_VECTOR)) return; @@ -169,7 +167,8 @@ static noinstr void fred_extint(struct pt_regs *regs) irqentry_state_t state = irqentry_enter(regs); instrumentation_begin(); - sysvec_table[index](regs); + sysvec_table[array_index_nospec(vector - FIRST_SYSTEM_VECTOR, + NR_SYSTEM_VECTORS)](regs); instrumentation_end(); irqentry_exit(regs, state); } else { @@ -177,6 +176,16 @@ static noinstr void fred_extint(struct pt_regs *regs) } } +#ifdef CONFIG_AMD_MEM_ENCRYPT +noinstr void exc_vmm_communication(struct pt_regs *regs, unsigned long error_code) +{ + if (user_mode(regs)) + return user_exc_vmm_communication(regs, error_code); + else + return kernel_exc_vmm_communication(regs, error_code); +} +#endif + static noinstr void fred_hwexc(struct pt_regs *regs, unsigned long error_code) { /* Optimize for #PF. That's the only exception which matters performance wise */ @@ -207,6 +216,10 @@ static noinstr void fred_hwexc(struct pt_regs *regs, unsigned long error_code) #ifdef CONFIG_X86_CET case X86_TRAP_CP: return exc_control_protection(regs, error_code); #endif +#ifdef CONFIG_AMD_MEM_ENCRYPT + case X86_TRAP_VC: return exc_vmm_communication(regs, error_code); +#endif + default: return fred_bad_type(regs, error_code); } diff --git a/arch/x86/events/core.c b/arch/x86/events/core.c index 576baa9a52c5bd..7a6b15b0f1c66a 100644 --- a/arch/x86/events/core.c +++ b/arch/x86/events/core.c @@ -1369,14 +1369,17 @@ static void x86_pmu_enable(struct pmu *pmu) else if (i < n_running) continue; - if (hwc->state & PERF_HES_ARCH) + cpuc->events[hwc->idx] = event; + + if (hwc->state & PERF_HES_ARCH) { + static_call(x86_pmu_set_period)(event); continue; + } /* * if cpuc->enabled = 0, then no wrmsr as * per x86_pmu_enable_event() */ - cpuc->events[hwc->idx] = event; x86_pmu_start(event, PERF_EF_RELOAD); } cpuc->n_added = 0; @@ -3073,8 +3076,8 @@ void perf_get_x86_pmu_capability(struct x86_pmu_capability *cap) cap->version = x86_pmu.version; cap->num_counters_gp = x86_pmu_num_counters(NULL); cap->num_counters_fixed = x86_pmu_num_counters_fixed(NULL); - cap->bit_width_gp = x86_pmu.cntval_bits; - cap->bit_width_fixed = x86_pmu.cntval_bits; + cap->bit_width_gp = cap->num_counters_gp ? x86_pmu.cntval_bits : 0; + cap->bit_width_fixed = cap->num_counters_fixed ? x86_pmu.cntval_bits : 0; cap->events_mask = (unsigned int)x86_pmu.events_maskl; cap->events_mask_len = x86_pmu.events_mask_len; cap->pebs_ept = x86_pmu.pebs_ept; diff --git a/arch/x86/events/intel/core.c b/arch/x86/events/intel/core.c index bdf3f0d0fe2167..bebaac1dbaeb35 100644 --- a/arch/x86/events/intel/core.c +++ b/arch/x86/events/intel/core.c @@ -4367,6 +4367,19 @@ static inline void intel_pmu_set_acr_caused_constr(struct perf_event *event, event->hw.dyn_constraint &= hybrid(event->pmu, acr_cause_mask64); } +static inline int intel_set_branch_counter_constr(struct perf_event *event, + int *num) +{ + if (branch_sample_call_stack(event)) + return -EINVAL; + if (branch_sample_counters(event)) { + (*num)++; + event->hw.dyn_constraint &= x86_pmu.lbr_counters; + } + + return 0; +} + static int intel_pmu_hw_config(struct perf_event *event) { int ret = x86_pmu_hw_config(event); @@ -4437,21 +4450,19 @@ static int intel_pmu_hw_config(struct perf_event *event) * group, which requires the extra space to store the counters. */ leader = event->group_leader; - if (branch_sample_call_stack(leader)) + if (intel_set_branch_counter_constr(leader, &num)) return -EINVAL; - if (branch_sample_counters(leader)) { - num++; - leader->hw.dyn_constraint &= x86_pmu.lbr_counters; - } leader->hw.flags |= PERF_X86_EVENT_BRANCH_COUNTERS; for_each_sibling_event(sibling, leader) { - if (branch_sample_call_stack(sibling)) + if (intel_set_branch_counter_constr(sibling, &num)) + return -EINVAL; + } + + /* event isn't installed as a sibling yet. */ + if (event != leader) { + if (intel_set_branch_counter_constr(event, &num)) return -EINVAL; - if (branch_sample_counters(sibling)) { - num++; - sibling->hw.dyn_constraint &= x86_pmu.lbr_counters; - } } if (num > fls(x86_pmu.lbr_counters)) @@ -4583,8 +4594,10 @@ static int intel_pmu_hw_config(struct perf_event *event) intel_pmu_set_acr_caused_constr(leader, idx++, cause_mask); if (leader->nr_siblings) { - for_each_sibling_event(sibling, leader) - intel_pmu_set_acr_caused_constr(sibling, idx++, cause_mask); + for_each_sibling_event(sibling, leader) { + if (is_x86_event(sibling)) + intel_pmu_set_acr_caused_constr(sibling, idx++, cause_mask); + } } if (leader != event) @@ -7405,6 +7418,7 @@ __init int intel_pmu_init(void) case INTEL_ATOM_SILVERMONT_D: case INTEL_ATOM_SILVERMONT_MID: case INTEL_ATOM_AIRMONT: + case INTEL_ATOM_AIRMONT_NP: case INTEL_ATOM_SILVERMONT_MID2: memcpy(hw_cache_event_ids, slm_hw_cache_event_ids, sizeof(hw_cache_event_ids)); diff --git a/arch/x86/events/intel/cstate.c b/arch/x86/events/intel/cstate.c index fa67fda6e45b4a..c1e318bdaa3975 100644 --- a/arch/x86/events/intel/cstate.c +++ b/arch/x86/events/intel/cstate.c @@ -599,6 +599,7 @@ static const struct x86_cpu_id intel_cstates_match[] __initconst = { X86_MATCH_VFM(INTEL_ATOM_SILVERMONT, &slm_cstates), X86_MATCH_VFM(INTEL_ATOM_SILVERMONT_D, &slm_cstates), X86_MATCH_VFM(INTEL_ATOM_AIRMONT, &slm_cstates), + X86_MATCH_VFM(INTEL_ATOM_AIRMONT_NP, &slm_cstates), X86_MATCH_VFM(INTEL_BROADWELL, &snb_cstates), X86_MATCH_VFM(INTEL_BROADWELL_D, &snb_cstates), diff --git a/arch/x86/events/intel/uncore.c b/arch/x86/events/intel/uncore.c index e228e564b15ea6..8301a589d9a610 100644 --- a/arch/x86/events/intel/uncore.c +++ b/arch/x86/events/intel/uncore.c @@ -67,6 +67,7 @@ int uncore_die_to_segment(int die) return bus ? pci_domain_nr(bus) : -EINVAL; } +/* Note: This API can only be used when NUMA information is available. */ int uncore_device_to_die(struct pci_dev *dev) { int node = pcibus_to_node(dev->bus); diff --git a/arch/x86/events/intel/uncore_discovery.c b/arch/x86/events/intel/uncore_discovery.c index 7d57ce706feb1b..c5adbe44090470 100644 --- a/arch/x86/events/intel/uncore_discovery.c +++ b/arch/x86/events/intel/uncore_discovery.c @@ -383,7 +383,7 @@ static bool intel_uncore_has_discovery_tables_pci(int *ignore) (val & UNCORE_DISCOVERY_DVSEC2_BIR_MASK) * UNCORE_DISCOVERY_BIR_STEP; die = get_device_die_id(dev); - if (die < 0) + if ((die < 0) || (die >= uncore_max_dies())) continue; parse_discovery_table(dev, die, bar_offset, &parsed, ignore); diff --git a/arch/x86/events/intel/uncore_snbep.c b/arch/x86/events/intel/uncore_snbep.c index e1f370b8d065f9..0182785cad1fe5 100644 --- a/arch/x86/events/intel/uncore_snbep.c +++ b/arch/x86/events/intel/uncore_snbep.c @@ -1475,13 +1475,7 @@ static int snbep_pci2phy_map_init(int devid, int nodeid_loc, int idmap_loc, bool } map->pbus_to_dieid[bus] = die_id = uncore_device_to_die(ubox_dev); - raw_spin_unlock(&pci2phy_map_lock); - - if (WARN_ON_ONCE(die_id == -1)) { - err = -EINVAL; - break; - } } } @@ -6533,7 +6527,7 @@ static void spr_update_device_location(int type_id) while ((dev = pci_get_device(PCI_VENDOR_ID_INTEL, device, dev)) != NULL) { - die = uncore_device_to_die(dev); + die = uncore_pcibus_to_dieid(dev->bus); if (die < 0) continue; @@ -6557,6 +6551,11 @@ static void spr_update_device_location(int type_id) int spr_uncore_pci_init(void) { + int ret = snbep_pci2phy_map_init(0x3250, SKX_CPUNODEID, SKX_GIDNIDMAP, true); + + if (ret) + return ret; + /* * The discovery table of UPI on some SPR variant is broken, * which impacts the detection of both UPI and M3UPI uncore PMON. @@ -6610,6 +6609,32 @@ static struct intel_uncore_type gnr_uncore_ubox = { .attr_update = uncore_alias_groups, }; +static struct uncore_event_desc gnr_uncore_imc_events[] = { + INTEL_UNCORE_EVENT_DESC(clockticks, "event=0x01,umask=0x00"), + INTEL_UNCORE_EVENT_DESC(cas_count_read_sch0, "event=0x05,umask=0xcf"), + INTEL_UNCORE_EVENT_DESC(cas_count_read_sch0.scale, "6.103515625e-5"), + INTEL_UNCORE_EVENT_DESC(cas_count_read_sch0.unit, "MiB"), + INTEL_UNCORE_EVENT_DESC(cas_count_read_sch1, "event=0x06,umask=0xcf"), + INTEL_UNCORE_EVENT_DESC(cas_count_read_sch1.scale, "6.103515625e-5"), + INTEL_UNCORE_EVENT_DESC(cas_count_read_sch1.unit, "MiB"), + INTEL_UNCORE_EVENT_DESC(cas_count_write_sch0, "event=0x05,umask=0xf0"), + INTEL_UNCORE_EVENT_DESC(cas_count_write_sch0.scale, "6.103515625e-5"), + INTEL_UNCORE_EVENT_DESC(cas_count_write_sch0.unit, "MiB"), + INTEL_UNCORE_EVENT_DESC(cas_count_write_sch1, "event=0x06,umask=0xf0"), + INTEL_UNCORE_EVENT_DESC(cas_count_write_sch1.scale, "6.103515625e-5"), + INTEL_UNCORE_EVENT_DESC(cas_count_write_sch1.unit, "MiB"), + { /* end: all zeroes */ }, +}; + +static struct intel_uncore_type gnr_uncore_imc = { + SPR_UNCORE_MMIO_COMMON_FORMAT(), + .name = "imc", + .fixed_ctr_bits = 48, + .fixed_ctr = SNR_IMC_MMIO_PMON_FIXED_CTR, + .fixed_ctl = SNR_IMC_MMIO_PMON_FIXED_CTL, + .event_descs = gnr_uncore_imc_events, +}; + static struct intel_uncore_type gnr_uncore_pciex8 = { SPR_UNCORE_PCI_COMMON_FORMAT(), .name = "pciex8", @@ -6657,7 +6682,7 @@ static struct intel_uncore_type *gnr_uncores[UNCORE_GNR_NUM_UNCORE_TYPES] = { NULL, &spr_uncore_pcu, &gnr_uncore_ubox, - &spr_uncore_imc, + &gnr_uncore_imc, NULL, &gnr_uncore_upi, NULL, diff --git a/arch/x86/events/msr.c b/arch/x86/events/msr.c index 7f5007a4752a17..8052596b850364 100644 --- a/arch/x86/events/msr.c +++ b/arch/x86/events/msr.c @@ -78,6 +78,7 @@ static bool test_intel(int idx, void *data) case INTEL_ATOM_SILVERMONT: case INTEL_ATOM_SILVERMONT_D: case INTEL_ATOM_AIRMONT: + case INTEL_ATOM_AIRMONT_NP: case INTEL_ATOM_GOLDMONT: case INTEL_ATOM_GOLDMONT_D: diff --git a/arch/x86/hyperv/hv_crash.c b/arch/x86/hyperv/hv_crash.c index c0e22921ace1a4..1d91051daa3de7 100644 --- a/arch/x86/hyperv/hv_crash.c +++ b/arch/x86/hyperv/hv_crash.c @@ -107,14 +107,12 @@ static void __noreturn hv_panic_timeout_reboot(void) cpu_relax(); } -/* This cannot be inlined as it needs stack */ -static noinline __noclone void hv_crash_restore_tss(void) +static void hv_crash_restore_tss(void) { load_TR_desc(); } -/* This cannot be inlined as it needs stack */ -static noinline void hv_crash_clear_kernpt(void) +static void hv_crash_clear_kernpt(void) { pgd_t *pgd; p4d_t *p4d; @@ -125,6 +123,25 @@ static noinline void hv_crash_clear_kernpt(void) native_p4d_clear(p4d); } + +static void __noreturn hv_crash_handle(void) +{ + hv_crash_restore_tss(); + hv_crash_clear_kernpt(); + + /* we are now fully in devirtualized normal kernel mode */ + __crash_kexec(NULL); + + hv_panic_timeout_reboot(); +} + +/* + * __naked functions do not permit function calls, not even to __always_inline + * functions that only contain asm() blocks themselves. So use a macro instead. + */ +#define hv_wrmsr(msr, val) \ + asm volatile("wrmsr" :: "c"(msr), "a"((u32)val), "d"((u32)(val >> 32)) : "memory") + /* * This is the C entry point from the asm glue code after the disable hypercall. * We enter here in IA32-e long mode, ie, full 64bit mode running on kernel @@ -133,51 +150,38 @@ static noinline void hv_crash_clear_kernpt(void) * available. We restore kernel GDT, and rest of the context, and continue * to kexec. */ -static asmlinkage void __noreturn hv_crash_c_entry(void) +static void __naked hv_crash_c_entry(void) { - struct hv_crash_ctxt *ctxt = &hv_crash_ctxt; - /* first thing, restore kernel gdt */ - native_load_gdt(&ctxt->gdtr); + asm volatile("lgdt %0" : : "m" (hv_crash_ctxt.gdtr)); - asm volatile("movw %%ax, %%ss" : : "a"(ctxt->ss)); - asm volatile("movq %0, %%rsp" : : "m"(ctxt->rsp)); + asm volatile("movw %0, %%ss\n\t" + "movq %1, %%rsp" + :: "m"(hv_crash_ctxt.ss), "m"(hv_crash_ctxt.rsp)); - asm volatile("movw %%ax, %%ds" : : "a"(ctxt->ds)); - asm volatile("movw %%ax, %%es" : : "a"(ctxt->es)); - asm volatile("movw %%ax, %%fs" : : "a"(ctxt->fs)); - asm volatile("movw %%ax, %%gs" : : "a"(ctxt->gs)); + asm volatile("movw %0, %%ds" : : "m"(hv_crash_ctxt.ds)); + asm volatile("movw %0, %%es" : : "m"(hv_crash_ctxt.es)); + asm volatile("movw %0, %%fs" : : "m"(hv_crash_ctxt.fs)); + asm volatile("movw %0, %%gs" : : "m"(hv_crash_ctxt.gs)); - native_wrmsrq(MSR_IA32_CR_PAT, ctxt->pat); - asm volatile("movq %0, %%cr0" : : "r"(ctxt->cr0)); + hv_wrmsr(MSR_IA32_CR_PAT, hv_crash_ctxt.pat); + asm volatile("movq %0, %%cr0" : : "r"(hv_crash_ctxt.cr0)); - asm volatile("movq %0, %%cr8" : : "r"(ctxt->cr8)); - asm volatile("movq %0, %%cr4" : : "r"(ctxt->cr4)); - asm volatile("movq %0, %%cr2" : : "r"(ctxt->cr4)); + asm volatile("movq %0, %%cr8" : : "r"(hv_crash_ctxt.cr8)); + asm volatile("movq %0, %%cr4" : : "r"(hv_crash_ctxt.cr4)); + asm volatile("movq %0, %%cr2" : : "r"(hv_crash_ctxt.cr2)); - native_load_idt(&ctxt->idtr); - native_wrmsrq(MSR_GS_BASE, ctxt->gsbase); - native_wrmsrq(MSR_EFER, ctxt->efer); + asm volatile("lidt %0" : : "m" (hv_crash_ctxt.idtr)); + hv_wrmsr(MSR_GS_BASE, hv_crash_ctxt.gsbase); + hv_wrmsr(MSR_EFER, hv_crash_ctxt.efer); /* restore the original kernel CS now via far return */ - asm volatile("movzwq %0, %%rax\n\t" - "pushq %%rax\n\t" - "pushq $1f\n\t" - "lretq\n\t" - "1:nop\n\t" : : "m"(ctxt->cs) : "rax"); - - /* We are in asmlinkage without stack frame, hence make C function - * calls which will buy stack frames. - */ - hv_crash_restore_tss(); - hv_crash_clear_kernpt(); - - /* we are now fully in devirtualized normal kernel mode */ - __crash_kexec(NULL); - - hv_panic_timeout_reboot(); + asm volatile("pushq %q0\n\t" + "pushq %q1\n\t" + "lretq" + :: "r"(hv_crash_ctxt.cs), "r"(hv_crash_handle)); } -/* Tell gcc we are using lretq long jump in the above function intentionally */ +/* Tell objtool we are using lretq long jump in the above function intentionally */ STACK_FRAME_NON_STANDARD(hv_crash_c_entry); static void hv_mark_tss_not_busy(void) @@ -628,7 +632,9 @@ void hv_root_crash_init(void) if (rc) goto err_out; +#ifdef CONFIG_SMP smp_ops.crash_stop_other_cpus = hv_crash_stop_other_cpus; +#endif crash_kexec_post_notifiers = true; hv_crash_enabled = true; diff --git a/arch/x86/hyperv/hv_init.c b/arch/x86/hyperv/hv_init.c index 14de43f4bc6c18..7f3301bd081ec5 100644 --- a/arch/x86/hyperv/hv_init.c +++ b/arch/x86/hyperv/hv_init.c @@ -558,7 +558,6 @@ void __init hyperv_init(void) memunmap(src); hv_remap_tsc_clocksource(); - hv_root_crash_init(); hv_sleep_notifiers_register(); } else { hypercall_msr.guest_physical_address = vmalloc_to_pfn(hv_hypercall_pg); @@ -567,6 +566,9 @@ void __init hyperv_init(void) hv_set_hypercall_pg(hv_hypercall_pg); + if (hv_root_partition()) /* after set hypercall pg */ + hv_root_crash_init(); + skip_hypercall_pg_init: /* * hyperv_init() is called before LAPIC is initialized: see diff --git a/arch/x86/hyperv/hv_vtl.c b/arch/x86/hyperv/hv_vtl.c index c0edaed0efb303..9b6a9bc4ab760e 100644 --- a/arch/x86/hyperv/hv_vtl.c +++ b/arch/x86/hyperv/hv_vtl.c @@ -110,7 +110,7 @@ static void hv_vtl_ap_entry(void) static int hv_vtl_bringup_vcpu(u32 target_vp_index, int cpu, u64 eip_ignored) { - u64 status; + u64 status, rsp, rip; int ret = 0; struct hv_enable_vp_vtl *input; unsigned long irq_flags; @@ -123,9 +123,11 @@ static int hv_vtl_bringup_vcpu(u32 target_vp_index, int cpu, u64 eip_ignored) struct desc_struct *gdt; struct task_struct *idle = idle_thread_get(cpu); - u64 rsp = (unsigned long)idle->thread.sp; + if (IS_ERR(idle)) + return PTR_ERR(idle); - u64 rip = (u64)&hv_vtl_ap_entry; + rsp = (unsigned long)idle->thread.sp; + rip = (u64)&hv_vtl_ap_entry; native_store_gdt(&gdt_ptr); store_idt(&idt_ptr); diff --git a/arch/x86/include/asm/cfi.h b/arch/x86/include/asm/cfi.h index c40b9ebc1fb40f..ab3fbbd947ed94 100644 --- a/arch/x86/include/asm/cfi.h +++ b/arch/x86/include/asm/cfi.h @@ -111,6 +111,12 @@ extern bhi_thunk __bhi_args_end[]; struct pt_regs; +#ifdef CONFIG_CALL_PADDING +#define CFI_OFFSET (CONFIG_FUNCTION_PADDING_CFI+5) +#else +#define CFI_OFFSET 5 +#endif + #ifdef CONFIG_CFI enum bug_trap_type handle_cfi_failure(struct pt_regs *regs); #define __bpfcall @@ -119,11 +125,9 @@ static inline int cfi_get_offset(void) { switch (cfi_mode) { case CFI_FINEIBT: - return 16; + return /* fineibt_prefix_size */ 16; case CFI_KCFI: - if (IS_ENABLED(CONFIG_CALL_PADDING)) - return 16; - return 5; + return CFI_OFFSET; default: return 0; } diff --git a/arch/x86/include/asm/efi.h b/arch/x86/include/asm/efi.h index f227a70ac91f04..51b4cdbea061ae 100644 --- a/arch/x86/include/asm/efi.h +++ b/arch/x86/include/asm/efi.h @@ -138,7 +138,7 @@ extern void __init efi_apply_memmap_quirks(void); extern int __init efi_reuse_config(u64 tables, int nr_tables); extern void efi_delete_dummy_variable(void); extern void efi_crash_gracefully_on_page_fault(unsigned long phys_addr); -extern void efi_free_boot_services(void); +extern void efi_unmap_boot_services(void); void arch_efi_call_virt_setup(void); void arch_efi_call_virt_teardown(void); diff --git a/arch/x86/include/asm/ftrace.h b/arch/x86/include/asm/ftrace.h index b08c95872eed9f..c56e1e63b89324 100644 --- a/arch/x86/include/asm/ftrace.h +++ b/arch/x86/include/asm/ftrace.h @@ -57,7 +57,7 @@ arch_ftrace_get_regs(struct ftrace_regs *fregs) } #define arch_ftrace_partial_regs(regs) do { \ - regs->flags &= ~X86_EFLAGS_FIXED; \ + regs->flags |= X86_EFLAGS_FIXED; \ regs->cs = __KERNEL_CS; \ } while (0) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 5a3bfa293e8b1a..8b8f7bb5cc82bc 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -1226,6 +1226,12 @@ enum kvm_irqchip_mode { KVM_IRQCHIP_SPLIT, /* created with KVM_CAP_SPLIT_IRQCHIP */ }; +enum kvm_suppress_eoi_broadcast_mode { + KVM_SUPPRESS_EOI_BROADCAST_QUIRKED, /* Legacy behavior */ + KVM_SUPPRESS_EOI_BROADCAST_ENABLED, /* Enable Suppress EOI broadcast */ + KVM_SUPPRESS_EOI_BROADCAST_DISABLED /* Disable Suppress EOI broadcast */ +}; + struct kvm_x86_msr_filter { u8 count; bool default_allow:1; @@ -1475,6 +1481,7 @@ struct kvm_arch { bool x2apic_format; bool x2apic_broadcast_quirk_disabled; + enum kvm_suppress_eoi_broadcast_mode suppress_eoi_broadcast_mode; bool has_mapped_host_mmio; bool guest_can_read_msr_platform_info; @@ -2463,7 +2470,8 @@ int memslot_rmap_alloc(struct kvm_memory_slot *slot, unsigned long npages); KVM_X86_QUIRK_MWAIT_NEVER_UD_FAULTS | \ KVM_X86_QUIRK_SLOT_ZAP_ALL | \ KVM_X86_QUIRK_STUFF_FEATURE_MSRS | \ - KVM_X86_QUIRK_IGNORE_GUEST_PAT) + KVM_X86_QUIRK_IGNORE_GUEST_PAT | \ + KVM_X86_QUIRK_VMCS12_ALLOW_FREEZE_IN_SMM) #define KVM_X86_CONDITIONAL_QUIRKS \ (KVM_X86_QUIRK_CD_NW_CLEARED | \ diff --git a/arch/x86/include/asm/linkage.h b/arch/x86/include/asm/linkage.h index 9d38ae744a2e4a..a7294656ad908a 100644 --- a/arch/x86/include/asm/linkage.h +++ b/arch/x86/include/asm/linkage.h @@ -68,7 +68,7 @@ * Depending on -fpatchable-function-entry=N,N usage (CONFIG_CALL_PADDING) the * CFI symbol layout changes. * - * Without CALL_THUNKS: + * Without CALL_PADDING: * * .align FUNCTION_ALIGNMENT * __cfi_##name: @@ -77,7 +77,7 @@ * .long __kcfi_typeid_##name * name: * - * With CALL_THUNKS: + * With CALL_PADDING: * * .align FUNCTION_ALIGNMENT * __cfi_##name: diff --git a/arch/x86/include/asm/msr-index.h b/arch/x86/include/asm/msr-index.h index 3d0a0950d20a16..75d2218cd0328c 100644 --- a/arch/x86/include/asm/msr-index.h +++ b/arch/x86/include/asm/msr-index.h @@ -669,6 +669,9 @@ #define MSR_AMD64_DC_CFG 0xc0011022 #define MSR_AMD64_TW_CFG 0xc0011023 +#define MSR_AMD64_FP_CFG 0xc0011028 +#define MSR_AMD64_FP_CFG_ZEN1_DENORM_FIX_BIT 9 + #define MSR_AMD64_DE_CFG 0xc0011029 #define MSR_AMD64_DE_CFG_LFENCE_SERIALIZE_BIT 1 #define MSR_AMD64_DE_CFG_LFENCE_SERIALIZE BIT_ULL(MSR_AMD64_DE_CFG_LFENCE_SERIALIZE_BIT) @@ -735,7 +738,10 @@ #define MSR_AMD64_SNP_SMT_PROT BIT_ULL(MSR_AMD64_SNP_SMT_PROT_BIT) #define MSR_AMD64_SNP_SECURE_AVIC_BIT 18 #define MSR_AMD64_SNP_SECURE_AVIC BIT_ULL(MSR_AMD64_SNP_SECURE_AVIC_BIT) -#define MSR_AMD64_SNP_RESV_BIT 19 +#define MSR_AMD64_SNP_RESERVED_BITS19_22 GENMASK_ULL(22, 19) +#define MSR_AMD64_SNP_IBPB_ON_ENTRY_BIT 23 +#define MSR_AMD64_SNP_IBPB_ON_ENTRY BIT_ULL(MSR_AMD64_SNP_IBPB_ON_ENTRY_BIT) +#define MSR_AMD64_SNP_RESV_BIT 24 #define MSR_AMD64_SNP_RESERVED_MASK GENMASK_ULL(63, MSR_AMD64_SNP_RESV_BIT) #define MSR_AMD64_SAVIC_CONTROL 0xc0010138 #define MSR_AMD64_SAVIC_EN_BIT 0 diff --git a/arch/x86/include/asm/numa.h b/arch/x86/include/asm/numa.h index 53ba39ce010cda..a9063f332fa6e5 100644 --- a/arch/x86/include/asm/numa.h +++ b/arch/x86/include/asm/numa.h @@ -22,6 +22,7 @@ extern int numa_off; */ extern s16 __apicid_to_node[MAX_LOCAL_APIC]; extern nodemask_t numa_nodes_parsed __initdata; +extern nodemask_t numa_phys_nodes_parsed __initdata; static inline void set_apicid_to_node(int apicid, s16 node) { @@ -48,6 +49,7 @@ extern void __init init_cpu_to_node(void); extern void numa_add_cpu(unsigned int cpu); extern void numa_remove_cpu(unsigned int cpu); extern void init_gi_nodes(void); +extern int num_phys_nodes(void); #else /* CONFIG_NUMA */ static inline void numa_set_node(int cpu, int node) { } static inline void numa_clear_node(int cpu) { } @@ -55,6 +57,10 @@ static inline void init_cpu_to_node(void) { } static inline void numa_add_cpu(unsigned int cpu) { } static inline void numa_remove_cpu(unsigned int cpu) { } static inline void init_gi_nodes(void) { } +static inline int num_phys_nodes(void) +{ + return 1; +} #endif /* CONFIG_NUMA */ #ifdef CONFIG_DEBUG_PER_CPU_MAPS diff --git a/arch/x86/include/asm/topology.h b/arch/x86/include/asm/topology.h index 1fadf0cf520c58..0ba9bdb9987176 100644 --- a/arch/x86/include/asm/topology.h +++ b/arch/x86/include/asm/topology.h @@ -155,6 +155,7 @@ extern unsigned int __max_logical_packages; extern unsigned int __max_threads_per_core; extern unsigned int __num_threads_per_package; extern unsigned int __num_cores_per_package; +extern unsigned int __num_nodes_per_package; const char *get_topology_cpu_type_name(struct cpuinfo_x86 *c); enum x86_topology_cpu_type get_topology_cpu_type(struct cpuinfo_x86 *c); @@ -179,6 +180,11 @@ static inline unsigned int topology_num_threads_per_package(void) return __num_threads_per_package; } +static inline unsigned int topology_num_nodes_per_package(void) +{ + return __num_nodes_per_package; +} + #ifdef CONFIG_X86_LOCAL_APIC int topology_get_logical_id(u32 apicid, enum x86_topology_domains at_level); #else diff --git a/arch/x86/include/asm/traps.h b/arch/x86/include/asm/traps.h index 869b8806180184..3f24cc472ce9be 100644 --- a/arch/x86/include/asm/traps.h +++ b/arch/x86/include/asm/traps.h @@ -25,6 +25,8 @@ extern int ibt_selftest_noendbr(void); void handle_invalid_op(struct pt_regs *regs); #endif +noinstr bool handle_bug(struct pt_regs *regs); + static inline int get_si_code(unsigned long condition) { if (condition & DR_STEP) diff --git a/arch/x86/include/asm/uaccess.h b/arch/x86/include/asm/uaccess.h index 367297b188c39c..3a0dd3c2b233e4 100644 --- a/arch/x86/include/asm/uaccess.h +++ b/arch/x86/include/asm/uaccess.h @@ -507,7 +507,7 @@ extern struct movsl_mask { } ____cacheline_aligned_in_smp movsl_mask; #endif -#define ARCH_HAS_NOCACHE_UACCESS 1 +#define ARCH_HAS_NONTEMPORAL_UACCESS 1 /* * The "unsafe" user accesses aren't really "unsafe", but the naming diff --git a/arch/x86/include/asm/uaccess_32.h b/arch/x86/include/asm/uaccess_32.h index 40379a1adbb846..fff19e73ccb333 100644 --- a/arch/x86/include/asm/uaccess_32.h +++ b/arch/x86/include/asm/uaccess_32.h @@ -26,13 +26,7 @@ raw_copy_from_user(void *to, const void __user *from, unsigned long n) return __copy_user_ll(to, (__force const void *)from, n); } -static __always_inline unsigned long -__copy_from_user_inatomic_nocache(void *to, const void __user *from, - unsigned long n) -{ - return __copy_from_user_ll_nocache_nozero(to, from, n); -} - +unsigned long __must_check copy_from_user_inatomic_nontemporal(void *, const void __user *, unsigned long n); unsigned long __must_check clear_user(void __user *mem, unsigned long len); unsigned long __must_check __clear_user(void __user *mem, unsigned long len); diff --git a/arch/x86/include/asm/uaccess_64.h b/arch/x86/include/asm/uaccess_64.h index 915124011c2764..20de34cc9aa6e7 100644 --- a/arch/x86/include/asm/uaccess_64.h +++ b/arch/x86/include/asm/uaccess_64.h @@ -147,26 +147,28 @@ raw_copy_to_user(void __user *dst, const void *src, unsigned long size) return copy_user_generic((__force void *)dst, src, size); } -extern long __copy_user_nocache(void *dst, const void __user *src, unsigned size); -extern long __copy_user_flushcache(void *dst, const void __user *src, unsigned size); +#define copy_to_nontemporal copy_to_nontemporal +extern size_t copy_to_nontemporal(void *dst, const void *src, size_t size); +extern size_t copy_user_flushcache(void *dst, const void __user *src, size_t size); static inline int -__copy_from_user_inatomic_nocache(void *dst, const void __user *src, +copy_from_user_inatomic_nontemporal(void *dst, const void __user *src, unsigned size) { long ret; kasan_check_write(dst, size); + src = mask_user_address(src); stac(); - ret = __copy_user_nocache(dst, src, size); + ret = copy_to_nontemporal(dst, (__force const void *)src, size); clac(); return ret; } -static inline int -__copy_from_user_flushcache(void *dst, const void __user *src, unsigned size) +static inline size_t +copy_from_user_flushcache(void *dst, const void __user *src, size_t size) { kasan_check_write(dst, size); - return __copy_user_flushcache(dst, src, size); + return copy_user_flushcache(dst, src, size); } /* diff --git a/arch/x86/include/uapi/asm/kvm.h b/arch/x86/include/uapi/asm/kvm.h index 7ceff65836525c..d94b2471aa216a 100644 --- a/arch/x86/include/uapi/asm/kvm.h +++ b/arch/x86/include/uapi/asm/kvm.h @@ -197,13 +197,13 @@ struct kvm_msrs { __u32 nmsrs; /* number of msrs in entries */ __u32 pad; - struct kvm_msr_entry entries[]; + __DECLARE_FLEX_ARRAY(struct kvm_msr_entry, entries); }; /* for KVM_GET_MSR_INDEX_LIST */ struct kvm_msr_list { __u32 nmsrs; /* number of msrs in entries */ - __u32 indices[]; + __DECLARE_FLEX_ARRAY(__u32, indices); }; /* Maximum size of any access bitmap in bytes */ @@ -245,7 +245,7 @@ struct kvm_cpuid_entry { struct kvm_cpuid { __u32 nent; __u32 padding; - struct kvm_cpuid_entry entries[]; + __DECLARE_FLEX_ARRAY(struct kvm_cpuid_entry, entries); }; struct kvm_cpuid_entry2 { @@ -267,7 +267,7 @@ struct kvm_cpuid_entry2 { struct kvm_cpuid2 { __u32 nent; __u32 padding; - struct kvm_cpuid_entry2 entries[]; + __DECLARE_FLEX_ARRAY(struct kvm_cpuid_entry2, entries); }; /* for KVM_GET_PIT and KVM_SET_PIT */ @@ -398,7 +398,7 @@ struct kvm_xsave { * the contents of CPUID leaf 0xD on the host. */ __u32 region[1024]; - __u32 extra[]; + __DECLARE_FLEX_ARRAY(__u32, extra); }; #define KVM_MAX_XCRS 16 @@ -476,6 +476,7 @@ struct kvm_sync_regs { #define KVM_X86_QUIRK_SLOT_ZAP_ALL (1 << 7) #define KVM_X86_QUIRK_STUFF_FEATURE_MSRS (1 << 8) #define KVM_X86_QUIRK_IGNORE_GUEST_PAT (1 << 9) +#define KVM_X86_QUIRK_VMCS12_ALLOW_FREEZE_IN_SMM (1 << 10) #define KVM_STATE_NESTED_FORMAT_VMX 0 #define KVM_STATE_NESTED_FORMAT_SVM 1 @@ -564,7 +565,7 @@ struct kvm_pmu_event_filter { __u32 fixed_counter_bitmap; __u32 flags; __u32 pad[4]; - __u64 events[]; + __DECLARE_FLEX_ARRAY(__u64, events); }; #define KVM_PMU_EVENT_ALLOW 0 @@ -914,8 +915,10 @@ struct kvm_sev_snp_launch_finish { __u64 pad1[4]; }; -#define KVM_X2APIC_API_USE_32BIT_IDS (1ULL << 0) -#define KVM_X2APIC_API_DISABLE_BROADCAST_QUIRK (1ULL << 1) +#define KVM_X2APIC_API_USE_32BIT_IDS _BITULL(0) +#define KVM_X2APIC_API_DISABLE_BROADCAST_QUIRK _BITULL(1) +#define KVM_X2APIC_ENABLE_SUPPRESS_EOI_BROADCAST _BITULL(2) +#define KVM_X2APIC_DISABLE_SUPPRESS_EOI_BROADCAST _BITULL(3) struct kvm_hyperv_eventfd { __u32 conn_id; diff --git a/arch/x86/kernel/Makefile b/arch/x86/kernel/Makefile index bc184dd38d993b..558b96d53e0024 100644 --- a/arch/x86/kernel/Makefile +++ b/arch/x86/kernel/Makefile @@ -44,6 +44,20 @@ KCOV_INSTRUMENT_unwind_orc.o := n KCOV_INSTRUMENT_unwind_frame.o := n KCOV_INSTRUMENT_unwind_guess.o := n +# Disable KCOV to prevent crashes during kexec: load_segments() invalidates +# the GS base, which KCOV relies on for per-CPU data. +# +# As KCOV and KEXEC compatibility should be preserved (e.g. syzkaller is +# using it to collect crash dumps during kernel fuzzing), disabling +# KCOV for KEXEC kernels is not an option. Selectively disabling KCOV +# instrumentation for individual affected functions can be fragile, while +# adding more checks to KCOV would slow it down. +# +# As a compromise solution, disable KCOV instrumentation for the whole +# source code file. If its coverage is ever needed, other approaches +# should be considered. +KCOV_INSTRUMENT_machine_kexec_64.o := n + CFLAGS_head32.o := -fno-stack-protector CFLAGS_head64.o := -fno-stack-protector CFLAGS_irq.o := -I $(src)/../include/asm/trace diff --git a/arch/x86/kernel/acpi/boot.c b/arch/x86/kernel/acpi/boot.c index 9fa321a95eb33f..d6138b2b633a31 100644 --- a/arch/x86/kernel/acpi/boot.c +++ b/arch/x86/kernel/acpi/boot.c @@ -35,6 +35,7 @@ #include #include #include +#include #include "sleep.h" /* To include x86_acpi_suspend_lowlevel */ static int __initdata acpi_force = 0; @@ -164,11 +165,14 @@ static bool __init acpi_is_processor_usable(u32 lapic_flags) if (lapic_flags & ACPI_MADT_ENABLED) return true; - if (!acpi_support_online_capable || - (lapic_flags & ACPI_MADT_ONLINE_CAPABLE)) - return true; + if (acpi_support_online_capable) + return lapic_flags & ACPI_MADT_ONLINE_CAPABLE; - return false; + /* + * QEMU expects legacy "Enabled=0" LAPIC entries to be counted as usable + * in order to support CPU hotplug in guests. + */ + return !hypervisor_is_type(X86_HYPER_NATIVE); } static int __init diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c index 28518371d8bf3c..a3f81cde2bb59e 100644 --- a/arch/x86/kernel/alternative.c +++ b/arch/x86/kernel/alternative.c @@ -1147,7 +1147,7 @@ void __init_or_module noinline apply_seal_endbr(s32 *start, s32 *end) poison_endbr(addr); if (IS_ENABLED(CONFIG_FINEIBT)) - poison_cfi(addr - 16); + poison_cfi(addr - CFI_OFFSET); } } @@ -1354,6 +1354,8 @@ extern u8 fineibt_preamble_end[]; #define fineibt_preamble_ud 0x13 #define fineibt_preamble_hash 5 +#define fineibt_prefix_size (fineibt_preamble_size - ENDBR_INSN_SIZE) + /* * : * 0: b8 78 56 34 12 mov $0x12345678, %eax @@ -1599,7 +1601,7 @@ static int cfi_rewrite_preamble(s32 *start, s32 *end) * have determined there are no indirect calls to it and we * don't need no CFI either. */ - if (!is_endbr(addr + 16)) + if (!is_endbr(addr + CFI_OFFSET)) continue; hash = decode_preamble_hash(addr, &arity); @@ -1607,6 +1609,15 @@ static int cfi_rewrite_preamble(s32 *start, s32 *end) addr, addr, 5, addr)) return -EINVAL; + /* + * FineIBT relies on being at func-16, so if the preamble is + * actually larger than that, place it the tail end. + * + * NOTE: this is possible with things like DEBUG_CALL_THUNKS + * and DEBUG_FORCE_FUNCTION_ALIGN_64B. + */ + addr += CFI_OFFSET - fineibt_prefix_size; + text_poke_early(addr, fineibt_preamble_start, fineibt_preamble_size); WARN_ON(*(u32 *)(addr + fineibt_preamble_hash) != 0x12345678); text_poke_early(addr + fineibt_preamble_hash, &hash, 4); @@ -1629,10 +1640,10 @@ static void cfi_rewrite_endbr(s32 *start, s32 *end) for (s = start; s < end; s++) { void *addr = (void *)s + *s; - if (!exact_endbr(addr + 16)) + if (!exact_endbr(addr + CFI_OFFSET)) continue; - poison_endbr(addr + 16); + poison_endbr(addr + CFI_OFFSET); } } @@ -1737,7 +1748,8 @@ static void __apply_fineibt(s32 *start_retpoline, s32 *end_retpoline, if (FINEIBT_WARN(fineibt_preamble_size, 20) || FINEIBT_WARN(fineibt_preamble_bhi + fineibt_bhi1_size, 20) || FINEIBT_WARN(fineibt_caller_size, 14) || - FINEIBT_WARN(fineibt_paranoid_size, 20)) + FINEIBT_WARN(fineibt_paranoid_size, 20) || + WARN_ON_ONCE(CFI_OFFSET < fineibt_prefix_size)) return; if (cfi_mode == CFI_AUTO) { @@ -1850,6 +1862,11 @@ static void poison_cfi(void *addr) */ switch (cfi_mode) { case CFI_FINEIBT: + /* + * FineIBT preamble is at func-16. + */ + addr += CFI_OFFSET - fineibt_prefix_size; + /* * FineIBT prefix should start with an ENDBR. */ @@ -1888,8 +1905,6 @@ static void poison_cfi(void *addr) } } -#define fineibt_prefix_size (fineibt_preamble_size - ENDBR_INSN_SIZE) - /* * When regs->ip points to a 0xD6 byte in the FineIBT preamble, * return true and fill out target and type. diff --git a/arch/x86/kernel/apic/apic.c b/arch/x86/kernel/apic/apic.c index d93f87f29d03b4..961714e6adae1c 100644 --- a/arch/x86/kernel/apic/apic.c +++ b/arch/x86/kernel/apic/apic.c @@ -1894,6 +1894,7 @@ void __init check_x2apic(void) static inline void try_to_enable_x2apic(int remap_mode) { } static inline void __x2apic_enable(void) { } +static inline void __x2apic_disable(void) { } #endif /* !CONFIG_X86_X2APIC */ void __init enable_IR_x2apic(void) @@ -2456,6 +2457,11 @@ static void lapic_resume(void *data) if (x2apic_mode) { __x2apic_enable(); } else { + if (x2apic_enabled()) { + pr_warn_once("x2apic: re-enabled by firmware during resume. Disabling\n"); + __x2apic_disable(); + } + /* * Make sure the APICBASE points to the right address * diff --git a/arch/x86/kernel/apic/x2apic_uv_x.c b/arch/x86/kernel/apic/x2apic_uv_x.c index 15209f220e1fda..42568ceec48162 100644 --- a/arch/x86/kernel/apic/x2apic_uv_x.c +++ b/arch/x86/kernel/apic/x2apic_uv_x.c @@ -1708,8 +1708,22 @@ static void __init uv_system_init_hub(void) struct uv_hub_info_s *new_hub; /* Allocate & fill new per hub info list */ - new_hub = (bid == 0) ? &uv_hub_info_node0 - : kzalloc_node(bytes, GFP_KERNEL, uv_blade_to_node(bid)); + if (bid == 0) { + new_hub = &uv_hub_info_node0; + } else { + int nid; + + /* + * Deconfigured sockets are mapped to SOCK_EMPTY. Use + * NUMA_NO_NODE to allocate on a valid node. + */ + nid = uv_blade_to_node(bid); + if (nid == SOCK_EMPTY) + nid = NUMA_NO_NODE; + + new_hub = kzalloc_node(bytes, GFP_KERNEL, nid); + } + if (WARN_ON_ONCE(!new_hub)) { /* do not kfree() bid 0, which is statically allocated */ while (--bid > 0) diff --git a/arch/x86/kernel/cpu/amd.c b/arch/x86/kernel/cpu/amd.c index bc94ff1e250ad2..b3499c84d89dd8 100644 --- a/arch/x86/kernel/cpu/amd.c +++ b/arch/x86/kernel/cpu/amd.c @@ -949,29 +949,20 @@ static void init_amd_zen1(struct cpuinfo_x86 *c) msr_clear_bit(MSR_K7_HWCR, MSR_K7_HWCR_IRPERF_EN_BIT); clear_cpu_cap(c, X86_FEATURE_IRPERF); } -} - -static bool cpu_has_zenbleed_microcode(void) -{ - u32 good_rev = 0; - switch (boot_cpu_data.x86_model) { - case 0x30 ... 0x3f: good_rev = 0x0830107b; break; - case 0x60 ... 0x67: good_rev = 0x0860010c; break; - case 0x68 ... 0x6f: good_rev = 0x08608107; break; - case 0x70 ... 0x7f: good_rev = 0x08701033; break; - case 0xa0 ... 0xaf: good_rev = 0x08a00009; break; - - default: - return false; - } - - if (boot_cpu_data.microcode < good_rev) - return false; - - return true; + pr_notice_once("AMD Zen1 FPDSS bug detected, enabling mitigation.\n"); + msr_set_bit(MSR_AMD64_FP_CFG, MSR_AMD64_FP_CFG_ZEN1_DENORM_FIX_BIT); } +static const struct x86_cpu_id amd_zenbleed_microcode[] = { + ZEN_MODEL_STEP_UCODE(0x17, 0x31, 0x0, 0x0830107b), + ZEN_MODEL_STEP_UCODE(0x17, 0x60, 0x1, 0x0860010c), + ZEN_MODEL_STEP_UCODE(0x17, 0x68, 0x1, 0x08608107), + ZEN_MODEL_STEP_UCODE(0x17, 0x71, 0x0, 0x08701033), + ZEN_MODEL_STEP_UCODE(0x17, 0xa0, 0x0, 0x08a00009), + {} +}; + static void zen2_zenbleed_check(struct cpuinfo_x86 *c) { if (cpu_has(c, X86_FEATURE_HYPERVISOR)) @@ -980,7 +971,7 @@ static void zen2_zenbleed_check(struct cpuinfo_x86 *c) if (!cpu_has(c, X86_FEATURE_AVX)) return; - if (!cpu_has_zenbleed_microcode()) { + if (!x86_match_min_microcode_rev(amd_zenbleed_microcode)) { pr_notice_once("Zenbleed: please update your microcode for the most optimal fix\n"); msr_set_bit(MSR_AMD64_DE_CFG, MSR_AMD64_DE_CFG_ZEN2_FP_BACKUP_FIX_BIT); } else { diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c index e7ab22fce3b57c..e7adc89013002f 100644 --- a/arch/x86/kernel/cpu/common.c +++ b/arch/x86/kernel/cpu/common.c @@ -95,6 +95,9 @@ EXPORT_SYMBOL(__max_dies_per_package); unsigned int __max_logical_packages __ro_after_init = 1; EXPORT_SYMBOL(__max_logical_packages); +unsigned int __num_nodes_per_package __ro_after_init = 1; +EXPORT_SYMBOL(__num_nodes_per_package); + unsigned int __num_cores_per_package __ro_after_init = 1; EXPORT_SYMBOL(__num_cores_per_package); @@ -430,7 +433,7 @@ static __always_inline void setup_lass(struct cpuinfo_x86 *c) /* These bits should not change their value after CPU init is finished. */ static const unsigned long cr4_pinned_mask = X86_CR4_SMEP | X86_CR4_SMAP | X86_CR4_UMIP | - X86_CR4_FSGSBASE | X86_CR4_CET | X86_CR4_FRED; + X86_CR4_FSGSBASE | X86_CR4_CET; static DEFINE_STATIC_KEY_FALSE_RO(cr_pinning); static unsigned long cr4_pinned_bits __ro_after_init; @@ -2039,12 +2042,6 @@ static void identify_cpu(struct cpuinfo_x86 *c) setup_umip(c); setup_lass(c); - /* Enable FSGSBASE instructions if available. */ - if (cpu_has(c, X86_FEATURE_FSGSBASE)) { - cr4_set_bits(X86_CR4_FSGSBASE); - elf_hwcap2 |= HWCAP2_FSGSBASE; - } - /* * The vendor-specific functions might have changed features. * Now we do "generic changes." @@ -2405,6 +2402,18 @@ void cpu_init_exception_handling(bool boot_cpu) /* GHCB needs to be setup to handle #VC. */ setup_ghcb(); + /* + * On CPUs with FSGSBASE support, paranoid_entry() uses + * ALTERNATIVE-patched RDGSBASE/WRGSBASE instructions. Secondary CPUs + * boot after alternatives are patched globally, so early exceptions + * execute patched code that depends on FSGSBASE. Enable the feature + * before any exceptions occur. + */ + if (cpu_feature_enabled(X86_FEATURE_FSGSBASE)) { + cr4_set_bits(X86_CR4_FSGSBASE); + elf_hwcap2 |= HWCAP2_FSGSBASE; + } + if (cpu_feature_enabled(X86_FEATURE_FRED)) { /* The boot CPU has enabled FRED during early boot */ if (!boot_cpu) diff --git a/arch/x86/kernel/cpu/mce/amd.c b/arch/x86/kernel/cpu/mce/amd.c index 3f1dda35530759..159f0becf8cca7 100644 --- a/arch/x86/kernel/cpu/mce/amd.c +++ b/arch/x86/kernel/cpu/mce/amd.c @@ -604,6 +604,14 @@ bool amd_filter_mce(struct mce *m) enum smca_bank_types bank_type = smca_get_bank_type(m->extcpu, m->bank); struct cpuinfo_x86 *c = &boot_cpu_data; + /* Bogus hw errors on Cezanne A0. */ + if (c->x86 == 0x19 && + c->x86_model == 0x50 && + c->x86_stepping == 0x0) { + if (!(m->status & MCI_STATUS_EN)) + return true; + } + /* See Family 17h Models 10h-2Fh Erratum #1114. */ if (c->x86 == 0x17 && c->x86_model >= 0x10 && c->x86_model <= 0x2F && @@ -875,13 +883,18 @@ void amd_clear_bank(struct mce *m) { amd_reset_thr_limit(m->bank); - /* Clear MCA_DESTAT for all deferred errors even those logged in MCA_STATUS. */ - if (m->status & MCI_STATUS_DEFERRED) - mce_wrmsrq(MSR_AMD64_SMCA_MCx_DESTAT(m->bank), 0); + if (mce_flags.smca) { + /* + * Clear MCA_DESTAT for all deferred errors even those + * logged in MCA_STATUS. + */ + if (m->status & MCI_STATUS_DEFERRED) + mce_wrmsrq(MSR_AMD64_SMCA_MCx_DESTAT(m->bank), 0); - /* Don't clear MCA_STATUS if MCA_DESTAT was used exclusively. */ - if (m->kflags & MCE_CHECK_DFR_REGS) - return; + /* Don't clear MCA_STATUS if MCA_DESTAT was used exclusively. */ + if (m->kflags & MCE_CHECK_DFR_REGS) + return; + } mce_wrmsrq(mca_msr_reg(m->bank, MCA_STATUS), 0); } diff --git a/arch/x86/kernel/cpu/topology.c b/arch/x86/kernel/cpu/topology.c index f55ea3cdbf88ef..eafcb1fc185ad6 100644 --- a/arch/x86/kernel/cpu/topology.c +++ b/arch/x86/kernel/cpu/topology.c @@ -27,11 +27,11 @@ #include #include -#include #include #include #include #include +#include #include "cpu.h" @@ -236,20 +236,6 @@ static __init void topo_register_apic(u32 apic_id, u32 acpi_id, bool present) cpuid_to_apicid[cpu] = apic_id; topo_set_cpuids(cpu, apic_id, acpi_id); } else { - u32 pkgid = topo_apicid(apic_id, TOPO_PKG_DOMAIN); - - /* - * Check for present APICs in the same package when running - * on bare metal. Allow the bogosity in a guest. - */ - if (hypervisor_is_type(X86_HYPER_NATIVE) && - topo_unit_count(pkgid, TOPO_PKG_DOMAIN, phys_cpu_present_map)) { - pr_info_once("Ignoring hot-pluggable APIC ID %x in present package.\n", - apic_id); - topo_info.nr_rejected_cpus++; - return; - } - topo_info.nr_disabled_cpus++; } @@ -507,11 +493,19 @@ void __init topology_init_possible_cpus(void) set_nr_cpu_ids(allowed); cnta = domain_weight(TOPO_PKG_DOMAIN); - cntb = domain_weight(TOPO_DIE_DOMAIN); __max_logical_packages = cnta; + + pr_info("Max. logical packages: %3u\n", __max_logical_packages); + + cntb = num_phys_nodes(); + __num_nodes_per_package = DIV_ROUND_UP(cntb, cnta); + + pr_info("Max. logical nodes: %3u\n", cntb); + pr_info("Num. nodes per package:%3u\n", __num_nodes_per_package); + + cntb = domain_weight(TOPO_DIE_DOMAIN); __max_dies_per_package = 1U << (get_count_order(cntb) - get_count_order(cnta)); - pr_info("Max. logical packages: %3u\n", cnta); pr_info("Max. logical dies: %3u\n", cntb); pr_info("Max. dies per package: %3u\n", __max_dies_per_package); diff --git a/arch/x86/kernel/ftrace_64.S b/arch/x86/kernel/ftrace_64.S index a132608265f6c0..62c1c93aa1c6a6 100644 --- a/arch/x86/kernel/ftrace_64.S +++ b/arch/x86/kernel/ftrace_64.S @@ -364,6 +364,9 @@ SYM_CODE_START(return_to_handler) UNWIND_HINT_UNDEFINED ANNOTATE_NOENDBR + /* Store original rsp for pt_regs.sp value. */ + movq %rsp, %rdi + /* Restore return_to_handler value that got eaten by previous ret instruction. */ subq $8, %rsp UNWIND_HINT_FUNC @@ -374,7 +377,7 @@ SYM_CODE_START(return_to_handler) movq %rax, RAX(%rsp) movq %rdx, RDX(%rsp) movq %rbp, RBP(%rsp) - movq %rsp, RSP(%rsp) + movq %rdi, RSP(%rsp) movq %rsp, %rdi call ftrace_return_to_handler diff --git a/arch/x86/kernel/kexec-bzimage64.c b/arch/x86/kernel/kexec-bzimage64.c index c3244ac680d148..f3b451eb49be16 100644 --- a/arch/x86/kernel/kexec-bzimage64.c +++ b/arch/x86/kernel/kexec-bzimage64.c @@ -192,6 +192,13 @@ setup_efi_state(struct boot_params *params, unsigned long params_load_addr, struct efi_info *current_ei = &boot_params.efi_info; struct efi_info *ei = ¶ms->efi_info; + if (!params->acpi_rsdp_addr) { + if (efi.acpi20 != EFI_INVALID_TABLE_ADDR) + params->acpi_rsdp_addr = efi.acpi20; + else if (efi.acpi != EFI_INVALID_TABLE_ADDR) + params->acpi_rsdp_addr = efi.acpi; + } + if (!efi_enabled(EFI_RUNTIME_SERVICES)) return 0; diff --git a/arch/x86/kernel/setup.c b/arch/x86/kernel/setup.c index 1b2edd07a3e176..383d4a4784f5b0 100644 --- a/arch/x86/kernel/setup.c +++ b/arch/x86/kernel/setup.c @@ -439,9 +439,15 @@ int __init ima_free_kexec_buffer(void) int __init ima_get_kexec_buffer(void **addr, size_t *size) { + int ret; + if (!ima_kexec_buffer_size) return -ENOENT; + ret = ima_validate_range(ima_kexec_buffer_phys, ima_kexec_buffer_size); + if (ret) + return ret; + *addr = __va(ima_kexec_buffer_phys); *size = ima_kexec_buffer_size; diff --git a/arch/x86/kernel/shstk.c b/arch/x86/kernel/shstk.c index 978232b6d48d76..ff8edea8511b48 100644 --- a/arch/x86/kernel/shstk.c +++ b/arch/x86/kernel/shstk.c @@ -351,7 +351,8 @@ static int shstk_pop_sigframe(unsigned long *ssp) need_to_check_vma = PAGE_ALIGN(*ssp) == *ssp; if (need_to_check_vma) - mmap_read_lock_killable(current->mm); + if (mmap_read_lock_killable(current->mm)) + return -EINTR; err = get_shstk_data(&token_addr, (unsigned long __user *)*ssp); if (unlikely(err)) diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c index 5cd6950ab672a0..294a8ea6029869 100644 --- a/arch/x86/kernel/smpboot.c +++ b/arch/x86/kernel/smpboot.c @@ -468,13 +468,6 @@ static int x86_cluster_flags(void) } #endif -/* - * Set if a package/die has multiple NUMA nodes inside. - * AMD Magny-Cours, Intel Cluster-on-Die, and Intel - * Sub-NUMA Clustering have this. - */ -static bool x86_has_numa_in_package; - static struct sched_domain_topology_level x86_topology[] = { SDTL_INIT(tl_smt_mask, cpu_smt_flags, SMT), #ifdef CONFIG_SCHED_CLUSTER @@ -496,7 +489,7 @@ static void __init build_sched_topology(void) * PKG domain since the NUMA domains will auto-magically create the * right spanning domains based on the SLIT. */ - if (x86_has_numa_in_package) { + if (topology_num_nodes_per_package() > 1) { unsigned int pkgdom = ARRAY_SIZE(x86_topology) - 2; memset(&x86_topology[pkgdom], 0, sizeof(x86_topology[pkgdom])); @@ -513,33 +506,149 @@ static void __init build_sched_topology(void) } #ifdef CONFIG_NUMA -static int sched_avg_remote_distance; -static int avg_remote_numa_distance(void) +/* + * Test if the on-trace cluster at (N,N) is symmetric. + * Uses upper triangle iteration to avoid obvious duplicates. + */ +static bool slit_cluster_symmetric(int N) { - int i, j; - int distance, nr_remote, total_distance; - - if (sched_avg_remote_distance > 0) - return sched_avg_remote_distance; - - nr_remote = 0; - total_distance = 0; - for_each_node_state(i, N_CPU) { - for_each_node_state(j, N_CPU) { - distance = node_distance(i, j); - - if (distance >= REMOTE_DISTANCE) { - nr_remote++; - total_distance += distance; - } + int u = topology_num_nodes_per_package(); + + for (int k = 0; k < u; k++) { + for (int l = k; l < u; l++) { + if (node_distance(N + k, N + l) != + node_distance(N + l, N + k)) + return false; } } - if (nr_remote) - sched_avg_remote_distance = total_distance / nr_remote; - else - sched_avg_remote_distance = REMOTE_DISTANCE; - return sched_avg_remote_distance; + return true; +} + +/* + * Return the package-id of the cluster, or ~0 if indeterminate. + * Each node in the on-trace cluster should have the same package-id. + */ +static u32 slit_cluster_package(int N) +{ + int u = topology_num_nodes_per_package(); + u32 pkg_id = ~0; + + for (int n = 0; n < u; n++) { + const struct cpumask *cpus = cpumask_of_node(N + n); + int cpu; + + for_each_cpu(cpu, cpus) { + u32 id = topology_logical_package_id(cpu); + + if (pkg_id == ~0) + pkg_id = id; + if (pkg_id != id) + return ~0; + } + } + + return pkg_id; +} + +/* + * Validate the SLIT table is of the form expected for SNC, specifically: + * + * - each on-trace cluster should be symmetric, + * - each on-trace cluster should have a unique package-id. + * + * If you NUMA_EMU on top of SNC, you get to keep the pieces. + */ +static bool slit_validate(void) +{ + int u = topology_num_nodes_per_package(); + u32 pkg_id, prev_pkg_id = ~0; + + for (int pkg = 0; pkg < topology_max_packages(); pkg++) { + int n = pkg * u; + + /* + * Ensure the on-trace cluster is symmetric and each cluster + * has a different package id. + */ + if (!slit_cluster_symmetric(n)) + return false; + pkg_id = slit_cluster_package(n); + if (pkg_id == ~0) + return false; + if (pkg && pkg_id == prev_pkg_id) + return false; + + prev_pkg_id = pkg_id; + } + + return true; +} + +/* + * Compute a sanitized SLIT table for SNC; notably SNC-3 can end up with + * asymmetric off-trace clusters, reflecting physical assymmetries. However + * this leads to 'unfortunate' sched_domain configurations. + * + * For example dual socket GNR with SNC-3: + * + * node distances: + * node 0 1 2 3 4 5 + * 0: 10 15 17 21 28 26 + * 1: 15 10 15 23 26 23 + * 2: 17 15 10 26 23 21 + * 3: 21 28 26 10 15 17 + * 4: 23 26 23 15 10 15 + * 5: 26 23 21 17 15 10 + * + * Fix things up by averaging out the off-trace clusters; resulting in: + * + * node 0 1 2 3 4 5 + * 0: 10 15 17 24 24 24 + * 1: 15 10 15 24 24 24 + * 2: 17 15 10 24 24 24 + * 3: 24 24 24 10 15 17 + * 4: 24 24 24 15 10 15 + * 5: 24 24 24 17 15 10 + */ +static int slit_cluster_distance(int i, int j) +{ + static int slit_valid = -1; + int u = topology_num_nodes_per_package(); + long d = 0; + int x, y; + + if (slit_valid < 0) { + slit_valid = slit_validate(); + if (!slit_valid) + pr_err(FW_BUG "SLIT table doesn't have the expected form for SNC -- fixup disabled!\n"); + else + pr_info("Fixing up SNC SLIT table.\n"); + } + + /* + * Is this a unit cluster on the trace? + */ + if ((i / u) == (j / u) || !slit_valid) + return node_distance(i, j); + + /* + * Off-trace cluster. + * + * Notably average out the symmetric pair of off-trace clusters to + * ensure the resulting SLIT table is symmetric. + */ + x = i - (i % u); + y = j - (j % u); + + for (i = x; i < x + u; i++) { + for (j = y; j < y + u; j++) { + d += node_distance(i, j); + d += node_distance(j, i); + } + } + + return d / (2*u*u); } int arch_sched_node_distance(int from, int to) @@ -549,34 +658,14 @@ int arch_sched_node_distance(int from, int to) switch (boot_cpu_data.x86_vfm) { case INTEL_GRANITERAPIDS_X: case INTEL_ATOM_DARKMONT_X: - - if (!x86_has_numa_in_package || topology_max_packages() == 1 || - d < REMOTE_DISTANCE) + if (topology_max_packages() == 1 || + topology_num_nodes_per_package() < 3) return d; /* - * With SNC enabled, there could be too many levels of remote - * NUMA node distances, creating NUMA domain levels - * including local nodes and partial remote nodes. - * - * Trim finer distance tuning for NUMA nodes in remote package - * for the purpose of building sched domains. Group NUMA nodes - * in the remote package in the same sched group. - * Simplify NUMA domains and avoid extra NUMA levels including - * different remote NUMA nodes and local nodes. - * - * GNR and CWF don't expect systems with more than 2 packages - * and more than 2 hops between packages. Single average remote - * distance won't be appropriate if there are more than 2 - * packages as average distance to different remote packages - * could be different. + * Handle SNC-3 asymmetries. */ - WARN_ONCE(topology_max_packages() > 2, - "sched: Expect only up to 2 packages for GNR or CWF, " - "but saw %d packages when building sched domains.", - topology_max_packages()); - - d = avg_remote_numa_distance(); + return slit_cluster_distance(from, to); } return d; } @@ -606,7 +695,7 @@ void set_cpu_sibling_map(int cpu) o = &cpu_data(i); if (match_pkg(c, o) && !topology_same_node(c, o)) - x86_has_numa_in_package = true; + WARN_ON_ONCE(topology_num_nodes_per_package() == 1); if ((i == cpu) || (has_smt && match_smt(c, o))) link_mask(topology_sibling_cpumask, cpu, i); diff --git a/arch/x86/kernel/traps.c b/arch/x86/kernel/traps.c index bcf1dedc1d008e..aca1eca5daffac 100644 --- a/arch/x86/kernel/traps.c +++ b/arch/x86/kernel/traps.c @@ -397,7 +397,7 @@ static inline void handle_invalid_op(struct pt_regs *regs) ILL_ILLOPN, error_get_trap_addr(regs)); } -static noinstr bool handle_bug(struct pt_regs *regs) +noinstr bool handle_bug(struct pt_regs *regs) { unsigned long addr = regs->ip; bool handled = false; diff --git a/arch/x86/kernel/uprobes.c b/arch/x86/kernel/uprobes.c index 7be8e361ca55b8..619dddf54424e9 100644 --- a/arch/x86/kernel/uprobes.c +++ b/arch/x86/kernel/uprobes.c @@ -1823,3 +1823,27 @@ bool is_uprobe_at_func_entry(struct pt_regs *regs) return false; } + +#ifdef CONFIG_IA32_EMULATION +unsigned long arch_uprobe_get_xol_area(void) +{ + struct thread_info *ti = current_thread_info(); + unsigned long vaddr; + + /* + * HACK: we are not in a syscall, but x86 get_unmapped_area() paths + * ignore TIF_ADDR32 and rely on in_32bit_syscall() to calculate + * vm_unmapped_area_info.high_limit. + * + * The #ifdef above doesn't cover the CONFIG_X86_X32_ABI=y case, + * but in this case in_32bit_syscall() -> in_x32_syscall() always + * (falsely) returns true because ->orig_ax == -1. + */ + if (test_thread_flag(TIF_ADDR32)) + ti->status |= TS_COMPAT; + vaddr = get_unmapped_area(NULL, TASK_SIZE - PAGE_SIZE, PAGE_SIZE, 0, 0); + ti->status &= ~TS_COMPAT; + + return vaddr; +} +#endif diff --git a/arch/x86/kernel/vmlinux.lds.S b/arch/x86/kernel/vmlinux.lds.S index d7af4a64c211b7..4ed82b1fe173b4 100644 --- a/arch/x86/kernel/vmlinux.lds.S +++ b/arch/x86/kernel/vmlinux.lds.S @@ -424,6 +424,7 @@ SECTIONS .llvm_bb_addr_map : { *(.llvm_bb_addr_map) } #endif + MODINFO ELF_DETAILS DISCARDS diff --git a/arch/x86/kvm/ioapic.c b/arch/x86/kvm/ioapic.c index 2c2783296aedb4..a26fa4222f292a 100644 --- a/arch/x86/kvm/ioapic.c +++ b/arch/x86/kvm/ioapic.c @@ -561,7 +561,7 @@ static void kvm_ioapic_update_eoi_one(struct kvm_vcpu *vcpu, spin_lock(&ioapic->lock); if (trigger_mode != IOAPIC_LEVEL_TRIG || - kvm_lapic_get_reg(apic, APIC_SPIV) & APIC_SPIV_DIRECTED_EOI) + kvm_lapic_suppress_eoi_broadcast(apic)) return; ASSERT(ent->fields.trig_mode == IOAPIC_LEVEL_TRIG); diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c index 1597dd0b0cc664..9ec577b10e0514 100644 --- a/arch/x86/kvm/lapic.c +++ b/arch/x86/kvm/lapic.c @@ -105,6 +105,63 @@ bool kvm_apic_pending_eoi(struct kvm_vcpu *vcpu, int vector) apic_test_vector(vector, apic->regs + APIC_IRR); } +static bool kvm_lapic_advertise_suppress_eoi_broadcast(struct kvm *kvm) +{ + switch (kvm->arch.suppress_eoi_broadcast_mode) { + case KVM_SUPPRESS_EOI_BROADCAST_ENABLED: + return true; + case KVM_SUPPRESS_EOI_BROADCAST_DISABLED: + return false; + case KVM_SUPPRESS_EOI_BROADCAST_QUIRKED: + /* + * The default in-kernel I/O APIC emulates the 82093AA and does not + * implement an EOI register. Some guests (e.g. Windows with the + * Hyper-V role enabled) disable LAPIC EOI broadcast without + * checking the I/O APIC version, which can cause level-triggered + * interrupts to never be EOI'd. + * + * To avoid this, KVM doesn't advertise Suppress EOI Broadcast + * support when using the default in-kernel I/O APIC. + * + * Historically, in split IRQCHIP mode, KVM always advertised + * Suppress EOI Broadcast support but did not actually suppress + * EOIs, resulting in quirky behavior. + */ + return !ioapic_in_kernel(kvm); + default: + WARN_ON_ONCE(1); + return false; + } +} + +bool kvm_lapic_suppress_eoi_broadcast(struct kvm_lapic *apic) +{ + struct kvm *kvm = apic->vcpu->kvm; + + if (!(kvm_lapic_get_reg(apic, APIC_SPIV) & APIC_SPIV_DIRECTED_EOI)) + return false; + + switch (kvm->arch.suppress_eoi_broadcast_mode) { + case KVM_SUPPRESS_EOI_BROADCAST_ENABLED: + return true; + case KVM_SUPPRESS_EOI_BROADCAST_DISABLED: + return false; + case KVM_SUPPRESS_EOI_BROADCAST_QUIRKED: + /* + * Historically, in split IRQCHIP mode, KVM ignored the suppress + * EOI broadcast bit set by the guest and broadcasts EOIs to the + * userspace I/O APIC. For In-kernel I/O APIC, the support itself + * is not advertised, can only be enabled via KVM_SET_APIC_STATE, + * and KVM's I/O APIC doesn't emulate Directed EOIs; but if the + * feature is enabled, it is respected (with odd behavior). + */ + return ioapic_in_kernel(kvm); + default: + WARN_ON_ONCE(1); + return false; + } +} + __read_mostly DEFINE_STATIC_KEY_FALSE(kvm_has_noapic_vcpu); EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_has_noapic_vcpu); @@ -554,15 +611,9 @@ void kvm_apic_set_version(struct kvm_vcpu *vcpu) v = APIC_VERSION | ((apic->nr_lvt_entries - 1) << 16); - /* - * KVM emulates 82093AA datasheet (with in-kernel IOAPIC implementation) - * which doesn't have EOI register; Some buggy OSes (e.g. Windows with - * Hyper-V role) disable EOI broadcast in lapic not checking for IOAPIC - * version first and level-triggered interrupts never get EOIed in - * IOAPIC. - */ + if (guest_cpu_cap_has(vcpu, X86_FEATURE_X2APIC) && - !ioapic_in_kernel(vcpu->kvm)) + kvm_lapic_advertise_suppress_eoi_broadcast(vcpu->kvm)) v |= APIC_LVR_DIRECTED_EOI; kvm_lapic_set_reg(apic, APIC_LVR, v); } @@ -1517,6 +1568,15 @@ static void kvm_ioapic_send_eoi(struct kvm_lapic *apic, int vector) /* Request a KVM exit to inform the userspace IOAPIC. */ if (irqchip_split(apic->vcpu->kvm)) { + /* + * Don't exit to userspace if the guest has enabled Directed + * EOI, a.k.a. Suppress EOI Broadcasts, in which case the local + * APIC doesn't broadcast EOIs (the guest must EOI the target + * I/O APIC(s) directly). + */ + if (kvm_lapic_suppress_eoi_broadcast(apic)) + return; + apic->vcpu->arch.pending_ioapic_eoi = vector; kvm_make_request(KVM_REQ_IOAPIC_EOI_EXIT, apic->vcpu); return; diff --git a/arch/x86/kvm/lapic.h b/arch/x86/kvm/lapic.h index 282b9b7da98cd1..e5f5a222eced0f 100644 --- a/arch/x86/kvm/lapic.h +++ b/arch/x86/kvm/lapic.h @@ -231,6 +231,8 @@ static inline int kvm_lapic_latched_init(struct kvm_vcpu *vcpu) bool kvm_apic_pending_eoi(struct kvm_vcpu *vcpu, int vector); +bool kvm_lapic_suppress_eoi_broadcast(struct kvm_lapic *apic); + void kvm_wait_lapic_expire(struct kvm_vcpu *vcpu); void kvm_bitmap_or_dest_vcpus(struct kvm *kvm, struct kvm_lapic_irq *irq, diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 02c450686b4a8a..440e3d9fc68965 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -3044,12 +3044,6 @@ static int mmu_set_spte(struct kvm_vcpu *vcpu, struct kvm_memory_slot *slot, bool prefetch = !fault || fault->prefetch; bool write_fault = fault && fault->write; - if (unlikely(is_noslot_pfn(pfn))) { - vcpu->stat.pf_mmio_spte_created++; - mark_mmio_spte(vcpu, sptep, gfn, pte_access); - return RET_PF_EMULATE; - } - if (is_shadow_present_pte(*sptep)) { if (prefetch && is_last_spte(*sptep, level) && pfn == spte_to_pfn(*sptep)) @@ -3066,13 +3060,22 @@ static int mmu_set_spte(struct kvm_vcpu *vcpu, struct kvm_memory_slot *slot, child = spte_to_child_sp(pte); drop_parent_pte(vcpu->kvm, child, sptep); flush = true; - } else if (WARN_ON_ONCE(pfn != spte_to_pfn(*sptep))) { + } else if (pfn != spte_to_pfn(*sptep)) { + WARN_ON_ONCE(vcpu->arch.mmu->root_role.direct); drop_spte(vcpu->kvm, sptep); flush = true; } else was_rmapped = 1; } + if (unlikely(is_noslot_pfn(pfn))) { + vcpu->stat.pf_mmio_spte_created++; + mark_mmio_spte(vcpu, sptep, gfn, pte_access); + if (flush) + kvm_flush_remote_tlbs_gfn(vcpu->kvm, gfn, level); + return RET_PF_EMULATE; + } + wrprot = make_spte(vcpu, sp, slot, pte_access, gfn, pfn, *sptep, prefetch, false, host_writable, &spte); diff --git a/arch/x86/kvm/svm/avic.c b/arch/x86/kvm/svm/avic.c index 0f6c8596719b83..b87398dff73fac 100644 --- a/arch/x86/kvm/svm/avic.c +++ b/arch/x86/kvm/svm/avic.c @@ -189,12 +189,12 @@ static void avic_activate_vmcb(struct vcpu_svm *svm) struct kvm_vcpu *vcpu = &svm->vcpu; vmcb->control.int_ctl &= ~(AVIC_ENABLE_MASK | X2APIC_MODE_MASK); - vmcb->control.avic_physical_id &= ~AVIC_PHYSICAL_MAX_INDEX_MASK; vmcb->control.avic_physical_id |= avic_get_max_physical_id(vcpu); - vmcb->control.int_ctl |= AVIC_ENABLE_MASK; + svm_clr_intercept(svm, INTERCEPT_CR8_WRITE); + /* * Note: KVM supports hybrid-AVIC mode, where KVM emulates x2APIC MSR * accesses, while interrupt injection to a running vCPU can be @@ -226,6 +226,9 @@ static void avic_deactivate_vmcb(struct vcpu_svm *svm) vmcb->control.int_ctl &= ~(AVIC_ENABLE_MASK | X2APIC_MODE_MASK); vmcb->control.avic_physical_id &= ~AVIC_PHYSICAL_MAX_INDEX_MASK; + if (!sev_es_guest(svm->vcpu.kvm)) + svm_set_intercept(svm, INTERCEPT_CR8_WRITE); + /* * If running nested and the guest uses its own MSR bitmap, there * is no need to update L0's msr bitmap @@ -368,7 +371,7 @@ void avic_init_vmcb(struct vcpu_svm *svm, struct vmcb *vmcb) vmcb->control.avic_physical_id = __sme_set(__pa(kvm_svm->avic_physical_id_table)); vmcb->control.avic_vapic_bar = APIC_DEFAULT_PHYS_BASE; - if (kvm_apicv_activated(svm->vcpu.kvm)) + if (kvm_vcpu_apicv_active(&svm->vcpu)) avic_activate_vmcb(svm); else avic_deactivate_vmcb(svm); diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c index ba0f11c68372b9..9be67040e94d96 100644 --- a/arch/x86/kvm/svm/nested.c +++ b/arch/x86/kvm/svm/nested.c @@ -1870,10 +1870,9 @@ static int svm_set_nested_state(struct kvm_vcpu *vcpu, * thus MMU might not be initialized correctly. * Set it again to fix this. */ - ret = nested_svm_load_cr3(&svm->vcpu, vcpu->arch.cr3, nested_npt_enabled(svm), false); - if (WARN_ON_ONCE(ret)) + if (ret) goto out_free; svm->nested.force_msr_bitmap_recalc = true; diff --git a/arch/x86/kvm/svm/sev.c b/arch/x86/kvm/svm/sev.c index f59c65abe3cfad..f2a57891e570f2 100644 --- a/arch/x86/kvm/svm/sev.c +++ b/arch/x86/kvm/svm/sev.c @@ -683,10 +683,16 @@ static struct page **sev_pin_memory(struct kvm *kvm, unsigned long uaddr, if (ulen == 0 || uaddr + ulen < uaddr) return ERR_PTR(-EINVAL); - /* Calculate number of pages. */ + /* + * Calculate the number of pages that need to be pinned to cover the + * entire range. Note! This isn't simply ulen >> PAGE_SHIFT, as KVM + * doesn't require the incoming address+size to be page aligned! + */ first = (uaddr & PAGE_MASK) >> PAGE_SHIFT; last = ((uaddr + ulen - 1) & PAGE_MASK) >> PAGE_SHIFT; npages = (last - first + 1); + if (npages > INT_MAX) + return ERR_PTR(-EINVAL); locked = sev->pages_locked + npages; lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; @@ -695,9 +701,6 @@ static struct page **sev_pin_memory(struct kvm *kvm, unsigned long uaddr, return ERR_PTR(-ENOMEM); } - if (WARN_ON_ONCE(npages > INT_MAX)) - return ERR_PTR(-EINVAL); - /* Avoid using vmalloc for smaller buffers. */ size = npages * sizeof(struct page *); if (size > PAGE_SIZE) @@ -875,6 +878,11 @@ static int sev_es_sync_vmsa(struct vcpu_svm *svm) u8 *d; int i; + lockdep_assert_held(&vcpu->mutex); + + if (vcpu->arch.guest_state_protected) + return -EINVAL; + /* Check some debug related fields before encrypting the VMSA */ if (svm->vcpu.guest_debug || (svm->vmcb->save.dr7 & ~DR7_FIXED_1)) return -EINVAL; @@ -1020,6 +1028,9 @@ static int sev_launch_update_vmsa(struct kvm *kvm, struct kvm_sev_cmd *argp) if (!sev_es_guest(kvm)) return -ENOTTY; + if (kvm_is_vcpu_creation_in_progress(kvm)) + return -EBUSY; + kvm_for_each_vcpu(i, vcpu, kvm) { ret = mutex_lock_killable(&vcpu->mutex); if (ret) @@ -2040,8 +2051,8 @@ static int sev_check_source_vcpus(struct kvm *dst, struct kvm *src) struct kvm_vcpu *src_vcpu; unsigned long i; - if (src->created_vcpus != atomic_read(&src->online_vcpus) || - dst->created_vcpus != atomic_read(&dst->online_vcpus)) + if (kvm_is_vcpu_creation_in_progress(src) || + kvm_is_vcpu_creation_in_progress(dst)) return -EBUSY; if (!sev_es_guest(src)) @@ -2451,6 +2462,13 @@ static int snp_launch_update_vmsa(struct kvm *kvm, struct kvm_sev_cmd *argp) unsigned long i; int ret; + if (kvm_is_vcpu_creation_in_progress(kvm)) + return -EBUSY; + + ret = kvm_lock_all_vcpus(kvm); + if (ret) + return ret; + data.gctx_paddr = __psp_pa(sev->snp_context); data.page_type = SNP_PAGE_TYPE_VMSA; @@ -2460,12 +2478,12 @@ static int snp_launch_update_vmsa(struct kvm *kvm, struct kvm_sev_cmd *argp) ret = sev_es_sync_vmsa(svm); if (ret) - return ret; + goto out; /* Transition the VMSA page to a firmware state. */ ret = rmp_make_private(pfn, INITIAL_VMSA_GPA, PG_LEVEL_4K, sev->asid, true); if (ret) - return ret; + goto out; /* Issue the SNP command to encrypt the VMSA */ data.address = __sme_pa(svm->sev_es.vmsa); @@ -2474,7 +2492,7 @@ static int snp_launch_update_vmsa(struct kvm *kvm, struct kvm_sev_cmd *argp) if (ret) { snp_page_reclaim(kvm, pfn); - return ret; + goto out; } svm->vcpu.arch.guest_state_protected = true; @@ -2488,7 +2506,9 @@ static int snp_launch_update_vmsa(struct kvm *kvm, struct kvm_sev_cmd *argp) svm_enable_lbrv(vcpu); } - return 0; +out: + kvm_unlock_all_vcpus(kvm); + return ret; } static int snp_launch_finish(struct kvm *kvm, struct kvm_sev_cmd *argp) @@ -2692,6 +2712,8 @@ int sev_mem_enc_register_region(struct kvm *kvm, struct enc_region *region; int ret = 0; + guard(mutex)(&kvm->lock); + if (!sev_guest(kvm)) return -ENOTTY; @@ -2706,12 +2728,10 @@ int sev_mem_enc_register_region(struct kvm *kvm, if (!region) return -ENOMEM; - mutex_lock(&kvm->lock); region->pages = sev_pin_memory(kvm, range->addr, range->size, ®ion->npages, FOLL_WRITE | FOLL_LONGTERM); if (IS_ERR(region->pages)) { ret = PTR_ERR(region->pages); - mutex_unlock(&kvm->lock); goto e_free; } @@ -2729,8 +2749,6 @@ int sev_mem_enc_register_region(struct kvm *kvm, region->size = range->size; list_add_tail(®ion->list, &sev->regions_list); - mutex_unlock(&kvm->lock); - return ret; e_free: diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index 4394be40fe78d7..1b07f665e0afbb 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -1032,8 +1032,7 @@ static void init_vmcb(struct kvm_vcpu *vcpu, bool init_event) svm_set_intercept(svm, INTERCEPT_CR0_WRITE); svm_set_intercept(svm, INTERCEPT_CR3_WRITE); svm_set_intercept(svm, INTERCEPT_CR4_WRITE); - if (!kvm_vcpu_apicv_active(vcpu)) - svm_set_intercept(svm, INTERCEPT_CR8_WRITE); + svm_set_intercept(svm, INTERCEPT_CR8_WRITE); set_dr_intercepts(svm); @@ -1141,7 +1140,7 @@ static void init_vmcb(struct kvm_vcpu *vcpu, bool init_event) svm_clr_intercept(svm, INTERCEPT_PAUSE); } - if (kvm_vcpu_apicv_active(vcpu)) + if (enable_apicv && irqchip_in_kernel(vcpu->kvm)) avic_init_vmcb(svm, vmcb); if (vnmi) @@ -2099,12 +2098,13 @@ static int vmload_vmsave_interception(struct kvm_vcpu *vcpu, bool vmload) ret = kvm_skip_emulated_instruction(vcpu); + /* KVM always performs VMLOAD/VMSAVE on VMCB01 (see __svm_vcpu_run()) */ if (vmload) { - svm_copy_vmloadsave_state(svm->vmcb, vmcb12); + svm_copy_vmloadsave_state(svm->vmcb01.ptr, vmcb12); svm->sysenter_eip_hi = 0; svm->sysenter_esp_hi = 0; } else { - svm_copy_vmloadsave_state(vmcb12, svm->vmcb); + svm_copy_vmloadsave_state(vmcb12, svm->vmcb01.ptr); } kvm_vcpu_unmap(vcpu, &map); @@ -2602,9 +2602,11 @@ static int dr_interception(struct kvm_vcpu *vcpu) static int cr8_write_interception(struct kvm_vcpu *vcpu) { + u8 cr8_prev = kvm_get_cr8(vcpu); int r; - u8 cr8_prev = kvm_get_cr8(vcpu); + WARN_ON_ONCE(kvm_vcpu_apicv_active(vcpu)); + /* instruction emulation calls kvm_set_cr8() */ r = cr_interception(vcpu); if (lapic_in_kernel(vcpu)) diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index 6137e5307d0f6c..89ef3551e35c0c 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -3292,10 +3292,24 @@ static int nested_vmx_check_guest_state(struct kvm_vcpu *vcpu, if (CC(vmcs12->guest_cr4 & X86_CR4_CET && !(vmcs12->guest_cr0 & X86_CR0_WP))) return -EINVAL; - if ((vmcs12->vm_entry_controls & VM_ENTRY_LOAD_DEBUG_CONTROLS) && - (CC(!kvm_dr7_valid(vmcs12->guest_dr7)) || - CC(!vmx_is_valid_debugctl(vcpu, vmcs12->guest_ia32_debugctl, false)))) - return -EINVAL; + if (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_DEBUG_CONTROLS) { + u64 debugctl = vmcs12->guest_ia32_debugctl; + + /* + * FREEZE_IN_SMM is not virtualized, but allow L1 to set it in + * vmcs12's DEBUGCTL under a quirk for backwards compatibility. + * Note that the quirk only relaxes the consistency check. The + * vmcc02 bit is still under the control of the host. In + * particular, if a host administrator decides to clear the bit, + * then L1 has no say in the matter. + */ + if (kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_VMCS12_ALLOW_FREEZE_IN_SMM)) + debugctl &= ~DEBUGCTLMSR_FREEZE_IN_SMM; + + if (CC(!kvm_dr7_valid(vmcs12->guest_dr7)) || + CC(!vmx_is_valid_debugctl(vcpu, debugctl, false))) + return -EINVAL; + } if ((vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_PAT) && CC(!kvm_pat_valid(vmcs12->guest_ia32_pat))) diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 72d37c8930ad78..e6ff8a2d7b54d0 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -121,8 +121,10 @@ static u64 __read_mostly efer_reserved_bits = ~((u64)EFER_SCE); #define KVM_CAP_PMU_VALID_MASK KVM_PMU_CAP_DISABLE -#define KVM_X2APIC_API_VALID_FLAGS (KVM_X2APIC_API_USE_32BIT_IDS | \ - KVM_X2APIC_API_DISABLE_BROADCAST_QUIRK) +#define KVM_X2APIC_API_VALID_FLAGS (KVM_X2APIC_API_USE_32BIT_IDS | \ + KVM_X2APIC_API_DISABLE_BROADCAST_QUIRK | \ + KVM_X2APIC_ENABLE_SUPPRESS_EOI_BROADCAST | \ + KVM_X2APIC_DISABLE_SUPPRESS_EOI_BROADCAST) static void update_cr8_intercept(struct kvm_vcpu *vcpu); static void process_nmi(struct kvm_vcpu *vcpu); @@ -237,7 +239,7 @@ EXPORT_SYMBOL_FOR_KVM_INTERNAL(enable_ipiv); bool __read_mostly enable_device_posted_irqs = true; EXPORT_SYMBOL_FOR_KVM_INTERNAL(enable_device_posted_irqs); -const struct _kvm_stats_desc kvm_vm_stats_desc[] = { +const struct kvm_stats_desc kvm_vm_stats_desc[] = { KVM_GENERIC_VM_STATS(), STATS_DESC_COUNTER(VM, mmu_shadow_zapped), STATS_DESC_COUNTER(VM, mmu_pte_write), @@ -263,7 +265,7 @@ const struct kvm_stats_header kvm_vm_stats_header = { sizeof(kvm_vm_stats_desc), }; -const struct _kvm_stats_desc kvm_vcpu_stats_desc[] = { +const struct kvm_stats_desc kvm_vcpu_stats_desc[] = { KVM_GENERIC_VCPU_STATS(), STATS_DESC_COUNTER(VCPU, pf_taken), STATS_DESC_COUNTER(VCPU, pf_fixed), @@ -4096,47 +4098,47 @@ int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info) break; case MSR_KVM_WALL_CLOCK_NEW: if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE2)) - return 1; + return KVM_MSR_RET_UNSUPPORTED; vcpu->kvm->arch.wall_clock = data; kvm_write_wall_clock(vcpu->kvm, data, 0); break; case MSR_KVM_WALL_CLOCK: if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE)) - return 1; + return KVM_MSR_RET_UNSUPPORTED; vcpu->kvm->arch.wall_clock = data; kvm_write_wall_clock(vcpu->kvm, data, 0); break; case MSR_KVM_SYSTEM_TIME_NEW: if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE2)) - return 1; + return KVM_MSR_RET_UNSUPPORTED; kvm_write_system_time(vcpu, data, false, msr_info->host_initiated); break; case MSR_KVM_SYSTEM_TIME: if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE)) - return 1; + return KVM_MSR_RET_UNSUPPORTED; kvm_write_system_time(vcpu, data, true, msr_info->host_initiated); break; case MSR_KVM_ASYNC_PF_EN: if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF)) - return 1; + return KVM_MSR_RET_UNSUPPORTED; if (kvm_pv_enable_async_pf(vcpu, data)) return 1; break; case MSR_KVM_ASYNC_PF_INT: if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT)) - return 1; + return KVM_MSR_RET_UNSUPPORTED; if (kvm_pv_enable_async_pf_int(vcpu, data)) return 1; break; case MSR_KVM_ASYNC_PF_ACK: if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT)) - return 1; + return KVM_MSR_RET_UNSUPPORTED; if (data & 0x1) { /* * Pairs with the smp_mb__after_atomic() in @@ -4149,7 +4151,7 @@ int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info) break; case MSR_KVM_STEAL_TIME: if (!guest_pv_has(vcpu, KVM_FEATURE_STEAL_TIME)) - return 1; + return KVM_MSR_RET_UNSUPPORTED; if (unlikely(!sched_info_on())) return 1; @@ -4167,7 +4169,7 @@ int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info) break; case MSR_KVM_PV_EOI_EN: if (!guest_pv_has(vcpu, KVM_FEATURE_PV_EOI)) - return 1; + return KVM_MSR_RET_UNSUPPORTED; if (kvm_lapic_set_pv_eoi(vcpu, data, sizeof(u8))) return 1; @@ -4175,7 +4177,7 @@ int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info) case MSR_KVM_POLL_CONTROL: if (!guest_pv_has(vcpu, KVM_FEATURE_POLL_CONTROL)) - return 1; + return KVM_MSR_RET_UNSUPPORTED; /* only enable bit supported */ if (data & (-1ULL << 1)) @@ -4476,61 +4478,61 @@ int kvm_get_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info) break; case MSR_KVM_WALL_CLOCK: if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE)) - return 1; + return KVM_MSR_RET_UNSUPPORTED; msr_info->data = vcpu->kvm->arch.wall_clock; break; case MSR_KVM_WALL_CLOCK_NEW: if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE2)) - return 1; + return KVM_MSR_RET_UNSUPPORTED; msr_info->data = vcpu->kvm->arch.wall_clock; break; case MSR_KVM_SYSTEM_TIME: if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE)) - return 1; + return KVM_MSR_RET_UNSUPPORTED; msr_info->data = vcpu->arch.time; break; case MSR_KVM_SYSTEM_TIME_NEW: if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE2)) - return 1; + return KVM_MSR_RET_UNSUPPORTED; msr_info->data = vcpu->arch.time; break; case MSR_KVM_ASYNC_PF_EN: if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF)) - return 1; + return KVM_MSR_RET_UNSUPPORTED; msr_info->data = vcpu->arch.apf.msr_en_val; break; case MSR_KVM_ASYNC_PF_INT: if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT)) - return 1; + return KVM_MSR_RET_UNSUPPORTED; msr_info->data = vcpu->arch.apf.msr_int_val; break; case MSR_KVM_ASYNC_PF_ACK: if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT)) - return 1; + return KVM_MSR_RET_UNSUPPORTED; msr_info->data = 0; break; case MSR_KVM_STEAL_TIME: if (!guest_pv_has(vcpu, KVM_FEATURE_STEAL_TIME)) - return 1; + return KVM_MSR_RET_UNSUPPORTED; msr_info->data = vcpu->arch.st.msr_val; break; case MSR_KVM_PV_EOI_EN: if (!guest_pv_has(vcpu, KVM_FEATURE_PV_EOI)) - return 1; + return KVM_MSR_RET_UNSUPPORTED; msr_info->data = vcpu->arch.pv_eoi.msr_val; break; case MSR_KVM_POLL_CONTROL: if (!guest_pv_has(vcpu, KVM_FEATURE_POLL_CONTROL)) - return 1; + return KVM_MSR_RET_UNSUPPORTED; msr_info->data = vcpu->arch.msr_kvm_poll_control; break; @@ -4931,6 +4933,8 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) break; case KVM_CAP_X2APIC_API: r = KVM_X2APIC_API_VALID_FLAGS; + if (kvm && !irqchip_split(kvm)) + r &= ~KVM_X2APIC_ENABLE_SUPPRESS_EOI_BROADCAST; break; case KVM_CAP_NESTED_STATE: r = kvm_x86_ops.nested_ops->get_state ? @@ -6748,11 +6752,24 @@ int kvm_vm_ioctl_enable_cap(struct kvm *kvm, if (cap->args[0] & ~KVM_X2APIC_API_VALID_FLAGS) break; + if ((cap->args[0] & KVM_X2APIC_ENABLE_SUPPRESS_EOI_BROADCAST) && + (cap->args[0] & KVM_X2APIC_DISABLE_SUPPRESS_EOI_BROADCAST)) + break; + + if ((cap->args[0] & KVM_X2APIC_ENABLE_SUPPRESS_EOI_BROADCAST) && + !irqchip_split(kvm)) + break; + if (cap->args[0] & KVM_X2APIC_API_USE_32BIT_IDS) kvm->arch.x2apic_format = true; if (cap->args[0] & KVM_X2APIC_API_DISABLE_BROADCAST_QUIRK) kvm->arch.x2apic_broadcast_quirk_disabled = true; + if (cap->args[0] & KVM_X2APIC_ENABLE_SUPPRESS_EOI_BROADCAST) + kvm->arch.suppress_eoi_broadcast_mode = KVM_SUPPRESS_EOI_BROADCAST_ENABLED; + if (cap->args[0] & KVM_X2APIC_DISABLE_SUPPRESS_EOI_BROADCAST) + kvm->arch.suppress_eoi_broadcast_mode = KVM_SUPPRESS_EOI_BROADCAST_DISABLED; + r = 0; break; case KVM_CAP_X86_DISABLE_EXITS: @@ -8197,7 +8214,13 @@ static int emulator_read_write_onepage(unsigned long addr, void *val, WARN_ON(vcpu->mmio_nr_fragments >= KVM_MAX_MMIO_FRAGMENTS); frag = &vcpu->mmio_fragments[vcpu->mmio_nr_fragments++]; frag->gpa = gpa; - frag->data = val; + if (write && bytes <= 8u) { + frag->val = 0; + frag->data = &frag->val; + memcpy(&frag->val, val, bytes); + } else { + frag->data = val; + } frag->len = bytes; return X86EMUL_CONTINUE; } @@ -8212,6 +8235,9 @@ static int emulator_read_write(struct x86_emulate_ctxt *ctxt, gpa_t gpa; int rc; + if (WARN_ON_ONCE((bytes > 8u || !ops->write) && object_is_on_stack(val))) + return X86EMUL_UNHANDLEABLE; + if (ops->read_write_prepare && ops->read_write_prepare(vcpu, val, bytes)) return X86EMUL_CONTINUE; @@ -11609,8 +11635,7 @@ static inline int vcpu_block(struct kvm_vcpu *vcpu) if (is_guest_mode(vcpu)) { int r = kvm_check_nested_events(vcpu); - WARN_ON_ONCE(r == -EBUSY); - if (r < 0) + if (r < 0 && r != -EBUSY) return 0; } @@ -11818,6 +11843,9 @@ static int complete_emulated_mmio(struct kvm_vcpu *vcpu) frag++; vcpu->mmio_cur_fragment++; } else { + if (WARN_ON_ONCE(frag->data == &frag->val)) + return -EIO; + /* Go forward to the next mmio piece. */ frag->data += len; frag->gpa += len; @@ -12158,9 +12186,11 @@ static void __get_sregs2(struct kvm_vcpu *vcpu, struct kvm_sregs2 *sregs2) return; if (is_pae_paging(vcpu)) { + kvm_vcpu_srcu_read_lock(vcpu); for (i = 0 ; i < 4 ; i++) sregs2->pdptrs[i] = kvm_pdptr_read(vcpu, i); sregs2->flags |= KVM_SREGS2_FLAGS_PDPTRS_VALID; + kvm_vcpu_srcu_read_unlock(vcpu); } } diff --git a/arch/x86/lib/copy_user_uncached_64.S b/arch/x86/lib/copy_user_uncached_64.S index 18350b343c2abb..8ed0ce3ad2271f 100644 --- a/arch/x86/lib/copy_user_uncached_64.S +++ b/arch/x86/lib/copy_user_uncached_64.S @@ -27,7 +27,7 @@ * Output: * rax uncopied bytes or 0 if successful. */ -SYM_FUNC_START(__copy_user_nocache) +SYM_FUNC_START(copy_to_nontemporal) ANNOTATE_NOENDBR /* If destination is not 7-byte aligned, we'll have to align it */ testb $7,%dil @@ -240,5 +240,5 @@ _ASM_EXTABLE_UA(95b, .Ldone) _ASM_EXTABLE_UA(52b, .Ldone0) _ASM_EXTABLE_UA(53b, .Ldone0) -SYM_FUNC_END(__copy_user_nocache) -EXPORT_SYMBOL(__copy_user_nocache) +SYM_FUNC_END(copy_to_nontemporal) +EXPORT_SYMBOL(copy_to_nontemporal) diff --git a/arch/x86/lib/usercopy_32.c b/arch/x86/lib/usercopy_32.c index f6f436f1d57375..ac27e39fc993d9 100644 --- a/arch/x86/lib/usercopy_32.c +++ b/arch/x86/lib/usercopy_32.c @@ -322,10 +322,11 @@ unsigned long __copy_user_ll(void *to, const void *from, unsigned long n) } EXPORT_SYMBOL(__copy_user_ll); -unsigned long __copy_from_user_ll_nocache_nozero(void *to, const void __user *from, +unsigned long copy_from_user_inatomic_nontemporal(void *to, const void __user *from, unsigned long n) { - __uaccess_begin_nospec(); + if (!user_access_begin(from, n)) + return n; #ifdef CONFIG_X86_INTEL_USERCOPY if (n > 64 && static_cpu_has(X86_FEATURE_XMM2)) n = __copy_user_intel_nocache(to, from, n); @@ -334,7 +335,7 @@ unsigned long __copy_from_user_ll_nocache_nozero(void *to, const void __user *fr #else __copy_user(to, from, n); #endif - __uaccess_end(); + user_access_end(); return n; } -EXPORT_SYMBOL(__copy_from_user_ll_nocache_nozero); +EXPORT_SYMBOL(copy_from_user_inatomic_nontemporal); diff --git a/arch/x86/lib/usercopy_64.c b/arch/x86/lib/usercopy_64.c index 654280aaa3e9e0..c47d8cd0e243b1 100644 --- a/arch/x86/lib/usercopy_64.c +++ b/arch/x86/lib/usercopy_64.c @@ -43,17 +43,17 @@ void arch_wb_cache_pmem(void *addr, size_t size) } EXPORT_SYMBOL_GPL(arch_wb_cache_pmem); -long __copy_user_flushcache(void *dst, const void __user *src, unsigned size) +size_t copy_user_flushcache(void *dst, const void __user *src, size_t size) { unsigned long flushed, dest = (unsigned long) dst; - long rc; + unsigned long rc; - stac(); - rc = __copy_user_nocache(dst, src, size); - clac(); + src = masked_user_access_begin(src); + rc = copy_to_nontemporal(dst, (__force const void *)src, size); + user_access_end(); /* - * __copy_user_nocache() uses non-temporal stores for the bulk + * copy_to_nontemporal() uses non-temporal stores for the bulk * of the transfer, but we need to manually flush if the * transfer is unaligned. A cached memory copy is used when * destination or size is not naturally aligned. That is: diff --git a/arch/x86/mm/Makefile b/arch/x86/mm/Makefile index 5b9908f13dcfd0..3a5364853eab87 100644 --- a/arch/x86/mm/Makefile +++ b/arch/x86/mm/Makefile @@ -4,6 +4,8 @@ KCOV_INSTRUMENT_tlb.o := n KCOV_INSTRUMENT_mem_encrypt.o := n KCOV_INSTRUMENT_mem_encrypt_amd.o := n KCOV_INSTRUMENT_pgprot.o := n +# See the "Disable KCOV" comment in arch/x86/kernel/Makefile. +KCOV_INSTRUMENT_physaddr.o := n KASAN_SANITIZE_mem_encrypt.o := n KASAN_SANITIZE_mem_encrypt_amd.o := n diff --git a/arch/x86/mm/extable.c b/arch/x86/mm/extable.c index 2fdc1f1f5adb95..6b9ff1c6cafa27 100644 --- a/arch/x86/mm/extable.c +++ b/arch/x86/mm/extable.c @@ -411,14 +411,11 @@ void __init early_fixup_exception(struct pt_regs *regs, int trapnr) return; if (trapnr == X86_TRAP_UD) { - if (report_bug(regs->ip, regs) == BUG_TRAP_TYPE_WARN) { - /* Skip the ud2. */ - regs->ip += LEN_UD2; + if (handle_bug(regs)) return; - } /* - * If this was a BUG and report_bug returns or if this + * If this was a BUG and handle_bug returns or if this * was just a normal #UD, we want to continue onward and * crash. */ diff --git a/arch/x86/mm/numa.c b/arch/x86/mm/numa.c index 7a97327140df89..99d0a9332c1459 100644 --- a/arch/x86/mm/numa.c +++ b/arch/x86/mm/numa.c @@ -48,6 +48,8 @@ s16 __apicid_to_node[MAX_LOCAL_APIC] = { [0 ... MAX_LOCAL_APIC-1] = NUMA_NO_NODE }; +nodemask_t numa_phys_nodes_parsed __initdata; + int numa_cpu_node(int cpu) { u32 apicid = early_per_cpu(x86_cpu_to_apicid, cpu); @@ -57,6 +59,11 @@ int numa_cpu_node(int cpu) return NUMA_NO_NODE; } +int __init num_phys_nodes(void) +{ + return bitmap_weight(numa_phys_nodes_parsed.bits, MAX_NUMNODES); +} + cpumask_var_t node_to_cpumask_map[MAX_NUMNODES]; EXPORT_SYMBOL(node_to_cpumask_map); @@ -210,6 +217,7 @@ static int __init dummy_numa_init(void) 0LLU, PFN_PHYS(max_pfn) - 1); node_set(0, numa_nodes_parsed); + node_set(0, numa_phys_nodes_parsed); numa_add_memblk(0, 0, PFN_PHYS(max_pfn)); return 0; diff --git a/arch/x86/mm/srat.c b/arch/x86/mm/srat.c index 6f8e0f21c7103d..44ca6665175617 100644 --- a/arch/x86/mm/srat.c +++ b/arch/x86/mm/srat.c @@ -57,6 +57,7 @@ acpi_numa_x2apic_affinity_init(struct acpi_srat_x2apic_cpu_affinity *pa) } set_apicid_to_node(apic_id, node); node_set(node, numa_nodes_parsed); + node_set(node, numa_phys_nodes_parsed); pr_debug("SRAT: PXM %u -> APIC 0x%04x -> Node %u\n", pxm, apic_id, node); } @@ -97,6 +98,7 @@ acpi_numa_processor_affinity_init(struct acpi_srat_cpu_affinity *pa) set_apicid_to_node(apic_id, node); node_set(node, numa_nodes_parsed); + node_set(node, numa_phys_nodes_parsed); pr_debug("SRAT: PXM %u -> APIC 0x%02x -> Node %u\n", pxm, apic_id, node); } diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index b0bac2a66eff3c..ea76949ddda5e4 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -438,17 +438,8 @@ static void emit_kcfi(u8 **pprog, u32 hash) EMIT1_off32(0xb8, hash); /* movl $hash, %eax */ #ifdef CONFIG_CALL_PADDING - EMIT1(0x90); - EMIT1(0x90); - EMIT1(0x90); - EMIT1(0x90); - EMIT1(0x90); - EMIT1(0x90); - EMIT1(0x90); - EMIT1(0x90); - EMIT1(0x90); - EMIT1(0x90); - EMIT1(0x90); + for (int i = 0; i < CONFIG_FUNCTION_PADDING_CFI; i++) + EMIT1(0x90); #endif EMIT_ENDBR(); diff --git a/arch/x86/platform/efi/efi.c b/arch/x86/platform/efi/efi.c index 463b784499a8f5..791c52c8393f46 100644 --- a/arch/x86/platform/efi/efi.c +++ b/arch/x86/platform/efi/efi.c @@ -837,7 +837,7 @@ static void __init __efi_enter_virtual_mode(void) } efi_check_for_embedded_firmwares(); - efi_free_boot_services(); + efi_unmap_boot_services(); if (!efi_is_mixed()) efi_native_runtime_setup(); diff --git a/arch/x86/platform/efi/quirks.c b/arch/x86/platform/efi/quirks.c index 553f330198f2f3..79f0818131e830 100644 --- a/arch/x86/platform/efi/quirks.c +++ b/arch/x86/platform/efi/quirks.c @@ -341,7 +341,7 @@ void __init efi_reserve_boot_services(void) /* * Because the following memblock_reserve() is paired - * with memblock_free_late() for this region in + * with free_reserved_area() for this region in * efi_free_boot_services(), we must be extremely * careful not to reserve, and subsequently free, * critical regions of memory (like the kernel image) or @@ -404,17 +404,33 @@ static void __init efi_unmap_pages(efi_memory_desc_t *md) pr_err("Failed to unmap VA mapping for 0x%llx\n", va); } -void __init efi_free_boot_services(void) +struct efi_freeable_range { + u64 start; + u64 end; +}; + +static struct efi_freeable_range *ranges_to_free; + +void __init efi_unmap_boot_services(void) { struct efi_memory_map_data data = { 0 }; efi_memory_desc_t *md; int num_entries = 0; + int idx = 0; + size_t sz; void *new, *new_md; /* Keep all regions for /sys/kernel/debug/efi */ if (efi_enabled(EFI_DBG)) return; + sz = sizeof(*ranges_to_free) * (efi.memmap.nr_map + 1); + ranges_to_free = kzalloc(sz, GFP_KERNEL); + if (!ranges_to_free) { + pr_err("Failed to allocate storage for freeable EFI regions\n"); + return; + } + for_each_efi_memory_desc(md) { unsigned long long start = md->phys_addr; unsigned long long size = md->num_pages << EFI_PAGE_SHIFT; @@ -471,7 +487,15 @@ void __init efi_free_boot_services(void) start = SZ_1M; } - memblock_free_late(start, size); + /* + * With CONFIG_DEFERRED_STRUCT_PAGE_INIT parts of the memory + * map are still not initialized and we can't reliably free + * memory here. + * Queue the ranges to free at a later point. + */ + ranges_to_free[idx].start = start; + ranges_to_free[idx].end = start + size; + idx++; } if (!num_entries) @@ -512,6 +536,31 @@ void __init efi_free_boot_services(void) } } +static int __init efi_free_boot_services(void) +{ + struct efi_freeable_range *range = ranges_to_free; + unsigned long freed = 0; + + if (!ranges_to_free) + return 0; + + while (range->start) { + void *start = phys_to_virt(range->start); + void *end = phys_to_virt(range->end); + + free_reserved_area(start, end, -1, NULL); + freed += (end - start); + range++; + } + kfree(ranges_to_free); + + if (freed) + pr_info("Freeing EFI boot services memory: %ldK\n", freed / SZ_1K); + + return 0; +} +arch_initcall(efi_free_boot_services); + /* * A number of config table entries get remapped to virtual addresses * after entering EFI virtual mode. However, the kexec kernel requires diff --git a/arch/x86/platform/pvh/head.S b/arch/x86/platform/pvh/head.S index 344030c1a81d46..53ee2d53fcf8e1 100644 --- a/arch/x86/platform/pvh/head.S +++ b/arch/x86/platform/pvh/head.S @@ -91,10 +91,12 @@ SYM_CODE_START(pvh_start_xen) leal rva(early_stack_end)(%ebp), %esp +#if defined(CONFIG_X86_64) || defined(CONFIG_X86_PAE) /* Enable PAE mode. */ mov %cr4, %eax orl $X86_CR4_PAE, %eax mov %eax, %cr4 +#endif #ifdef CONFIG_X86_64 /* Enable Long mode. */ diff --git a/arch/x86/xen/enlighten.c b/arch/x86/xen/enlighten.c index 53282dc7d5ac5b..23b91bf9b66303 100644 --- a/arch/x86/xen/enlighten.c +++ b/arch/x86/xen/enlighten.c @@ -470,7 +470,7 @@ int __init arch_xen_unpopulated_init(struct resource **res) * driver to know how much of the physmap is unpopulated and * set an accurate initial memory target. */ - xen_released_pages += xen_extra_mem[i].n_pfns; + xen_unpopulated_pages += xen_extra_mem[i].n_pfns; /* Zero so region is not also added to the balloon driver. */ xen_extra_mem[i].n_pfns = 0; } diff --git a/block/bfq-iosched.c b/block/bfq-iosched.c index 6e54b1d3d8bc2a..9e9d081e86bb27 100644 --- a/block/bfq-iosched.c +++ b/block/bfq-iosched.c @@ -697,7 +697,7 @@ static void bfq_limit_depth(blk_opf_t opf, struct blk_mq_alloc_data *data) unsigned int limit, act_idx; /* Sync reads have full depth available */ - if (op_is_sync(opf) && !op_is_write(opf)) + if (blk_mq_is_sync_read(opf)) limit = data->q->nr_requests; else limit = bfqd->async_depths[!!bfqd->wr_busy_queues][op_is_sync(opf)]; diff --git a/block/blk-merge.c b/block/blk-merge.c index d3115d7469df07..bf8faadb0bd461 100644 --- a/block/blk-merge.c +++ b/block/blk-merge.c @@ -158,8 +158,9 @@ static struct bio *bio_submit_split(struct bio *bio, int split_sectors) return bio; } -struct bio *bio_split_discard(struct bio *bio, const struct queue_limits *lim, - unsigned *nsegs) +static struct bio *__bio_split_discard(struct bio *bio, + const struct queue_limits *lim, unsigned *nsegs, + unsigned int max_sectors) { unsigned int max_discard_sectors, granularity; sector_t tmp; @@ -169,8 +170,7 @@ struct bio *bio_split_discard(struct bio *bio, const struct queue_limits *lim, granularity = max(lim->discard_granularity >> 9, 1U); - max_discard_sectors = - min(lim->max_discard_sectors, bio_allowed_max_sectors(lim)); + max_discard_sectors = min(max_sectors, bio_allowed_max_sectors(lim)); max_discard_sectors -= max_discard_sectors % granularity; if (unlikely(!max_discard_sectors)) return bio; @@ -194,6 +194,19 @@ struct bio *bio_split_discard(struct bio *bio, const struct queue_limits *lim, return bio_submit_split(bio, split_sectors); } +struct bio *bio_split_discard(struct bio *bio, const struct queue_limits *lim, + unsigned *nsegs) +{ + unsigned int max_sectors; + + if (bio_op(bio) == REQ_OP_SECURE_ERASE) + max_sectors = lim->max_secure_erase_sectors; + else + max_sectors = lim->max_discard_sectors; + + return __bio_split_discard(bio, lim, nsegs, max_sectors); +} + static inline unsigned int blk_boundary_sectors(const struct queue_limits *lim, bool is_atomic) { diff --git a/block/blk-mq-debugfs.c b/block/blk-mq-debugfs.c index 4896525b1c0541..553d93b88e194f 100644 --- a/block/blk-mq-debugfs.c +++ b/block/blk-mq-debugfs.c @@ -686,8 +686,10 @@ void blk_mq_debugfs_register_hctxs(struct request_queue *q) struct blk_mq_hw_ctx *hctx; unsigned long i; + mutex_lock(&q->debugfs_mutex); queue_for_each_hw_ctx(q, hctx, i) blk_mq_debugfs_register_hctx(q, hctx); + mutex_unlock(&q->debugfs_mutex); } void blk_mq_debugfs_unregister_hctxs(struct request_queue *q) diff --git a/block/blk-mq-dma.c b/block/blk-mq-dma.c index fb018fffffdcc5..feead1934301a8 100644 --- a/block/blk-mq-dma.c +++ b/block/blk-mq-dma.c @@ -126,17 +126,20 @@ static bool blk_rq_dma_map_iova(struct request *req, struct device *dma_dev, error = dma_iova_link(dma_dev, state, vec->paddr, mapped, vec->len, dir, attrs); if (error) - break; + goto out_unlink; mapped += vec->len; } while (blk_map_iter_next(req, &iter->iter, vec)); error = dma_iova_sync(dma_dev, state, 0, mapped); - if (error) { - iter->status = errno_to_blk_status(error); - return false; - } + if (error) + goto out_unlink; return true; + +out_unlink: + dma_iova_destroy(dma_dev, state, mapped, dir, attrs); + iter->status = errno_to_blk_status(error); + return false; } static inline void blk_rq_map_iter_init(struct request *rq, diff --git a/block/blk-mq-sched.h b/block/blk-mq-sched.h index 02c40a72e9598c..5678e15bd33c4b 100644 --- a/block/blk-mq-sched.h +++ b/block/blk-mq-sched.h @@ -137,4 +137,9 @@ static inline void blk_mq_set_min_shallow_depth(struct request_queue *q, depth); } +static inline bool blk_mq_is_sync_read(blk_opf_t opf) +{ + return op_is_sync(opf) && !op_is_write(opf); +} + #endif diff --git a/block/blk-mq.c b/block/blk-mq.c index 968699277c3d5a..3b58dd5876114f 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -4778,38 +4778,45 @@ static void blk_mq_update_queue_map(struct blk_mq_tag_set *set) } } -static int blk_mq_realloc_tag_set_tags(struct blk_mq_tag_set *set, - int new_nr_hw_queues) +static struct blk_mq_tags **blk_mq_prealloc_tag_set_tags( + struct blk_mq_tag_set *set, + int new_nr_hw_queues) { struct blk_mq_tags **new_tags; int i; if (set->nr_hw_queues >= new_nr_hw_queues) - goto done; + return NULL; new_tags = kcalloc_node(new_nr_hw_queues, sizeof(struct blk_mq_tags *), GFP_KERNEL, set->numa_node); if (!new_tags) - return -ENOMEM; + return ERR_PTR(-ENOMEM); if (set->tags) memcpy(new_tags, set->tags, set->nr_hw_queues * sizeof(*set->tags)); - kfree(set->tags); - set->tags = new_tags; for (i = set->nr_hw_queues; i < new_nr_hw_queues; i++) { - if (!__blk_mq_alloc_map_and_rqs(set, i)) { - while (--i >= set->nr_hw_queues) - __blk_mq_free_map_and_rqs(set, i); - return -ENOMEM; + if (blk_mq_is_shared_tags(set->flags)) { + new_tags[i] = set->shared_tags; + } else { + new_tags[i] = blk_mq_alloc_map_and_rqs(set, i, + set->queue_depth); + if (!new_tags[i]) + goto out_unwind; } cond_resched(); } -done: - set->nr_hw_queues = new_nr_hw_queues; - return 0; + return new_tags; +out_unwind: + while (--i >= set->nr_hw_queues) { + if (!blk_mq_is_shared_tags(set->flags)) + blk_mq_free_map_and_rqs(set, new_tags[i], i); + } + kfree(new_tags); + return ERR_PTR(-ENOMEM); } /* @@ -5093,6 +5100,7 @@ static void __blk_mq_update_nr_hw_queues(struct blk_mq_tag_set *set, unsigned int memflags; int i; struct xarray elv_tbl; + struct blk_mq_tags **new_tags; bool queues_frozen = false; lockdep_assert_held(&set->tag_list_lock); @@ -5127,11 +5135,18 @@ static void __blk_mq_update_nr_hw_queues(struct blk_mq_tag_set *set, if (blk_mq_elv_switch_none(q, &elv_tbl)) goto switch_back; + new_tags = blk_mq_prealloc_tag_set_tags(set, nr_hw_queues); + if (IS_ERR(new_tags)) + goto switch_back; + list_for_each_entry(q, &set->tag_list, tag_set_list) blk_mq_freeze_queue_nomemsave(q); queues_frozen = true; - if (blk_mq_realloc_tag_set_tags(set, nr_hw_queues) < 0) - goto switch_back; + if (new_tags) { + kfree(set->tags); + set->tags = new_tags; + } + set->nr_hw_queues = nr_hw_queues; fallback: blk_mq_update_queue_map(set); diff --git a/block/blk-sysfs.c b/block/blk-sysfs.c index e0a70d26972b3b..af12526d866a9c 100644 --- a/block/blk-sysfs.c +++ b/block/blk-sysfs.c @@ -78,8 +78,14 @@ queue_requests_store(struct gendisk *disk, const char *page, size_t count) /* * Serialize updating nr_requests with concurrent queue_requests_store() * and switching elevator. + * + * Use trylock to avoid circular lock dependency with kernfs active + * reference during concurrent disk deletion: + * update_nr_hwq_lock -> kn->active (via del_gendisk -> kobject_del) + * kn->active -> update_nr_hwq_lock (via this sysfs write path) */ - down_write(&set->update_nr_hwq_lock); + if (!down_write_trylock(&set->update_nr_hwq_lock)) + return -EBUSY; if (nr == q->nr_requests) goto unlock; diff --git a/block/blk.h b/block/blk.h index e4c433f62dfc7a..4cd5a91346d8a3 100644 --- a/block/blk.h +++ b/block/blk.h @@ -208,10 +208,14 @@ static inline unsigned int blk_queue_get_max_sectors(struct request *rq) struct request_queue *q = rq->q; enum req_op op = req_op(rq); - if (unlikely(op == REQ_OP_DISCARD || op == REQ_OP_SECURE_ERASE)) + if (unlikely(op == REQ_OP_DISCARD)) return min(q->limits.max_discard_sectors, UINT_MAX >> SECTOR_SHIFT); + if (unlikely(op == REQ_OP_SECURE_ERASE)) + return min(q->limits.max_secure_erase_sectors, + UINT_MAX >> SECTOR_SHIFT); + if (unlikely(op == REQ_OP_WRITE_ZEROES)) return q->limits.max_write_zeroes_sectors; diff --git a/block/elevator.c b/block/elevator.c index a2f8b2251dc6e6..7a97998cd8bd72 100644 --- a/block/elevator.c +++ b/block/elevator.c @@ -806,7 +806,16 @@ ssize_t elv_iosched_store(struct gendisk *disk, const char *buf, elv_iosched_load_module(ctx.name); ctx.type = elevator_find_get(ctx.name); - down_read(&set->update_nr_hwq_lock); + /* + * Use trylock to avoid circular lock dependency with kernfs active + * reference during concurrent disk deletion: + * update_nr_hwq_lock -> kn->active (via del_gendisk -> kobject_del) + * kn->active -> update_nr_hwq_lock (via this sysfs write path) + */ + if (!down_read_trylock(&set->update_nr_hwq_lock)) { + ret = -EBUSY; + goto out; + } if (!blk_queue_no_elv_switch(q)) { ret = elevator_change(q, &ctx); if (!ret) @@ -816,6 +825,7 @@ ssize_t elv_iosched_store(struct gendisk *disk, const char *buf, } up_read(&set->update_nr_hwq_lock); +out: if (ctx.type) elevator_put(ctx.type); return ret; diff --git a/block/ioctl.c b/block/ioctl.c index 344478348a54e3..337e4c3b65b2cc 100644 --- a/block/ioctl.c +++ b/block/ioctl.c @@ -318,7 +318,13 @@ int blkdev_compat_ptr_ioctl(struct block_device *bdev, blk_mode_t mode, EXPORT_SYMBOL(blkdev_compat_ptr_ioctl); #endif -static bool blkdev_pr_allowed(struct block_device *bdev, blk_mode_t mode) +enum pr_direction { + PR_IN, /* read from device */ + PR_OUT, /* write to device */ +}; + +static bool blkdev_pr_allowed(struct block_device *bdev, blk_mode_t mode, + enum pr_direction dir) { /* no sense to make reservations for partitions */ if (bdev_is_partition(bdev)) @@ -326,11 +332,17 @@ static bool blkdev_pr_allowed(struct block_device *bdev, blk_mode_t mode) if (capable(CAP_SYS_ADMIN)) return true; + /* - * Only allow unprivileged reservations if the file descriptor is open - * for writing. + * Only allow unprivileged reservation _out_ commands if the file + * descriptor is open for writing. Allow reservation _in_ commands if + * the file descriptor is open for reading since they do not modify the + * device. */ - return mode & BLK_OPEN_WRITE; + if (dir == PR_IN) + return mode & BLK_OPEN_READ; + else + return mode & BLK_OPEN_WRITE; } static int blkdev_pr_register(struct block_device *bdev, blk_mode_t mode, @@ -339,7 +351,7 @@ static int blkdev_pr_register(struct block_device *bdev, blk_mode_t mode, const struct pr_ops *ops = bdev->bd_disk->fops->pr_ops; struct pr_registration reg; - if (!blkdev_pr_allowed(bdev, mode)) + if (!blkdev_pr_allowed(bdev, mode, PR_OUT)) return -EPERM; if (!ops || !ops->pr_register) return -EOPNOTSUPP; @@ -357,7 +369,7 @@ static int blkdev_pr_reserve(struct block_device *bdev, blk_mode_t mode, const struct pr_ops *ops = bdev->bd_disk->fops->pr_ops; struct pr_reservation rsv; - if (!blkdev_pr_allowed(bdev, mode)) + if (!blkdev_pr_allowed(bdev, mode, PR_OUT)) return -EPERM; if (!ops || !ops->pr_reserve) return -EOPNOTSUPP; @@ -375,7 +387,7 @@ static int blkdev_pr_release(struct block_device *bdev, blk_mode_t mode, const struct pr_ops *ops = bdev->bd_disk->fops->pr_ops; struct pr_reservation rsv; - if (!blkdev_pr_allowed(bdev, mode)) + if (!blkdev_pr_allowed(bdev, mode, PR_OUT)) return -EPERM; if (!ops || !ops->pr_release) return -EOPNOTSUPP; @@ -393,7 +405,7 @@ static int blkdev_pr_preempt(struct block_device *bdev, blk_mode_t mode, const struct pr_ops *ops = bdev->bd_disk->fops->pr_ops; struct pr_preempt p; - if (!blkdev_pr_allowed(bdev, mode)) + if (!blkdev_pr_allowed(bdev, mode, PR_OUT)) return -EPERM; if (!ops || !ops->pr_preempt) return -EOPNOTSUPP; @@ -411,7 +423,7 @@ static int blkdev_pr_clear(struct block_device *bdev, blk_mode_t mode, const struct pr_ops *ops = bdev->bd_disk->fops->pr_ops; struct pr_clear c; - if (!blkdev_pr_allowed(bdev, mode)) + if (!blkdev_pr_allowed(bdev, mode, PR_OUT)) return -EPERM; if (!ops || !ops->pr_clear) return -EOPNOTSUPP; @@ -434,7 +446,7 @@ static int blkdev_pr_read_keys(struct block_device *bdev, blk_mode_t mode, size_t keys_copy_len; int ret; - if (!blkdev_pr_allowed(bdev, mode)) + if (!blkdev_pr_allowed(bdev, mode, PR_IN)) return -EPERM; if (!ops || !ops->pr_read_keys) return -EOPNOTSUPP; @@ -486,7 +498,7 @@ static int blkdev_pr_read_reservation(struct block_device *bdev, struct pr_read_reservation out = {}; int ret; - if (!blkdev_pr_allowed(bdev, mode)) + if (!blkdev_pr_allowed(bdev, mode, PR_IN)) return -EPERM; if (!ops || !ops->pr_read_reservation) return -EOPNOTSUPP; diff --git a/block/kyber-iosched.c b/block/kyber-iosched.c index c1b36ffd19ceb4..2b3f5b8959af03 100644 --- a/block/kyber-iosched.c +++ b/block/kyber-iosched.c @@ -556,7 +556,7 @@ static void kyber_limit_depth(blk_opf_t opf, struct blk_mq_alloc_data *data) * We use the scheduler tags as per-hardware queue queueing tokens. * Async requests can be limited at this stage. */ - if (!op_is_sync(opf)) { + if (!blk_mq_is_sync_read(opf)) { struct kyber_queue_data *kqd = data->q->elevator->elevator_data; data->shallow_depth = kqd->async_depth; diff --git a/block/mq-deadline.c b/block/mq-deadline.c index 3e3719093aec72..29d00221fbea6a 100644 --- a/block/mq-deadline.c +++ b/block/mq-deadline.c @@ -495,7 +495,7 @@ static void dd_limit_depth(blk_opf_t opf, struct blk_mq_alloc_data *data) struct deadline_data *dd = data->q->elevator->elevator_data; /* Do not throttle synchronous reads. */ - if (op_is_sync(opf) && !op_is_write(opf)) + if (blk_mq_is_sync_read(opf)) return; /* diff --git a/crypto/af_alg.c b/crypto/af_alg.c index e468714f539dfb..8953e2ffd55ceb 100644 --- a/crypto/af_alg.c +++ b/crypto/af_alg.c @@ -623,8 +623,10 @@ static int af_alg_alloc_tsgl(struct sock *sk) sg_init_table(sgl->sg, MAX_SGL_ENTS + 1); sgl->cur = 0; - if (sg) + if (sg) { + sg_unmark_end(sg + MAX_SGL_ENTS - 1); sg_chain(sg, MAX_SGL_ENTS + 1, sgl->sg); + } list_add_tail(&sgl->list, &ctx->tsgl_list); } @@ -635,15 +637,13 @@ static int af_alg_alloc_tsgl(struct sock *sk) /** * af_alg_count_tsgl - Count number of TX SG entries * - * The counting starts from the beginning of the SGL to @bytes. If - * an @offset is provided, the counting of the SG entries starts at the @offset. + * The counting starts from the beginning of the SGL to @bytes. * * @sk: socket of connection to user space * @bytes: Count the number of SG entries holding given number of bytes. - * @offset: Start the counting of SG entries from the given offset. * Return: Number of TX SG entries found given the constraints */ -unsigned int af_alg_count_tsgl(struct sock *sk, size_t bytes, size_t offset) +unsigned int af_alg_count_tsgl(struct sock *sk, size_t bytes) { const struct alg_sock *ask = alg_sk(sk); const struct af_alg_ctx *ctx = ask->private; @@ -658,25 +658,11 @@ unsigned int af_alg_count_tsgl(struct sock *sk, size_t bytes, size_t offset) const struct scatterlist *sg = sgl->sg; for (i = 0; i < sgl->cur; i++) { - size_t bytes_count; - - /* Skip offset */ - if (offset >= sg[i].length) { - offset -= sg[i].length; - bytes -= sg[i].length; - continue; - } - - bytes_count = sg[i].length - offset; - - offset = 0; sgl_count++; - - /* If we have seen requested number of bytes, stop */ - if (bytes_count >= bytes) + if (sg[i].length >= bytes) return sgl_count; - bytes -= bytes_count; + bytes -= sg[i].length; } } @@ -688,19 +674,14 @@ EXPORT_SYMBOL_GPL(af_alg_count_tsgl); * af_alg_pull_tsgl - Release the specified buffers from TX SGL * * If @dst is non-null, reassign the pages to @dst. The caller must release - * the pages. If @dst_offset is given only reassign the pages to @dst starting - * at the @dst_offset (byte). The caller must ensure that @dst is large - * enough (e.g. by using af_alg_count_tsgl with the same offset). + * the pages. * * @sk: socket of connection to user space * @used: Number of bytes to pull from TX SGL * @dst: If non-NULL, buffer is reassigned to dst SGL instead of releasing. The * caller must release the buffers in dst. - * @dst_offset: Reassign the TX SGL from given offset. All buffers before - * reaching the offset is released. */ -void af_alg_pull_tsgl(struct sock *sk, size_t used, struct scatterlist *dst, - size_t dst_offset) +void af_alg_pull_tsgl(struct sock *sk, size_t used, struct scatterlist *dst) { struct alg_sock *ask = alg_sk(sk); struct af_alg_ctx *ctx = ask->private; @@ -724,19 +705,11 @@ void af_alg_pull_tsgl(struct sock *sk, size_t used, struct scatterlist *dst, * Assumption: caller created af_alg_count_tsgl(len) * SG entries in dst. */ - if (dst) { - if (dst_offset >= plen) { - /* discard page before offset */ - dst_offset -= plen; - } else { - /* reassign page to dst after offset */ - get_page(page); - sg_set_page(dst + j, page, - plen - dst_offset, - sg[i].offset + dst_offset); - dst_offset = 0; - j++; - } + if (dst && plen) { + /* reassign page to dst */ + get_page(page); + sg_set_page(dst + j, page, plen, sg[i].offset); + j++; } sg[i].length -= plen; @@ -1256,6 +1229,8 @@ int af_alg_get_rsgl(struct sock *sk, struct msghdr *msg, int flags, seglen = min_t(size_t, (maxsize - len), msg_data_left(msg)); + /* Never pin more pages than the remaining RX accounting budget. */ + seglen = min_t(size_t, seglen, af_alg_rcvbuf(sk)); if (list_empty(&areq->rsgl_list)) { rsgl = &areq->first_rsgl; diff --git a/crypto/algif_aead.c b/crypto/algif_aead.c index 79b016a899a1ef..cb651ab58d6293 100644 --- a/crypto/algif_aead.c +++ b/crypto/algif_aead.c @@ -26,7 +26,6 @@ #include #include #include -#include #include #include #include @@ -72,10 +71,11 @@ static int _aead_recvmsg(struct socket *sock, struct msghdr *msg, struct alg_sock *pask = alg_sk(psk); struct af_alg_ctx *ctx = ask->private; struct crypto_aead *tfm = pask->private; - unsigned int i, as = crypto_aead_authsize(tfm); + unsigned int as = crypto_aead_authsize(tfm); + unsigned int ivsize = crypto_aead_ivsize(tfm); struct af_alg_async_req *areq; - struct af_alg_tsgl *tsgl, *tmp; struct scatterlist *rsgl_src, *tsgl_src = NULL; + void *iv; int err = 0; size_t used = 0; /* [in] TX bufs to be en/decrypted */ size_t outlen = 0; /* [out] RX bufs produced by kernel */ @@ -127,10 +127,14 @@ static int _aead_recvmsg(struct socket *sock, struct msghdr *msg, /* Allocate cipher request for current operation. */ areq = af_alg_alloc_areq(sk, sizeof(struct af_alg_async_req) + - crypto_aead_reqsize(tfm)); + crypto_aead_reqsize(tfm) + ivsize); if (IS_ERR(areq)) return PTR_ERR(areq); + iv = (u8 *)aead_request_ctx(&areq->cra_u.aead_req) + + crypto_aead_reqsize(tfm); + memcpy(iv, ctx->iv, ivsize); + /* convert iovecs of output buffers into RX SGL */ err = af_alg_get_rsgl(sk, msg, flags, areq, outlen, &usedpages); if (err) @@ -146,7 +150,7 @@ static int _aead_recvmsg(struct socket *sock, struct msghdr *msg, if (usedpages < outlen) { size_t less = outlen - usedpages; - if (used < less) { + if (used < less + (ctx->enc ? 0 : as)) { err = -EINVAL; goto free; } @@ -154,23 +158,24 @@ static int _aead_recvmsg(struct socket *sock, struct msghdr *msg, outlen -= less; } + /* + * Create a per request TX SGL for this request which tracks the + * SG entries from the global TX SGL. + */ processed = used + ctx->aead_assoclen; - list_for_each_entry_safe(tsgl, tmp, &ctx->tsgl_list, list) { - for (i = 0; i < tsgl->cur; i++) { - struct scatterlist *process_sg = tsgl->sg + i; - - if (!(process_sg->length) || !sg_page(process_sg)) - continue; - tsgl_src = process_sg; - break; - } - if (tsgl_src) - break; - } - if (processed && !tsgl_src) { - err = -EFAULT; + areq->tsgl_entries = af_alg_count_tsgl(sk, processed); + if (!areq->tsgl_entries) + areq->tsgl_entries = 1; + areq->tsgl = sock_kmalloc(sk, array_size(sizeof(*areq->tsgl), + areq->tsgl_entries), + GFP_KERNEL); + if (!areq->tsgl) { + err = -ENOMEM; goto free; } + sg_init_table(areq->tsgl, areq->tsgl_entries); + af_alg_pull_tsgl(sk, processed, areq->tsgl); + tsgl_src = areq->tsgl; /* * Copy of AAD from source to destination @@ -179,77 +184,16 @@ static int _aead_recvmsg(struct socket *sock, struct msghdr *msg, * when user space uses an in-place cipher operation, the kernel * will copy the data as it does not see whether such in-place operation * is initiated. - * - * To ensure efficiency, the following implementation ensure that the - * ciphers are invoked to perform a crypto operation in-place. This - * is achieved by memory management specified as follows. */ /* Use the RX SGL as source (and destination) for crypto op. */ rsgl_src = areq->first_rsgl.sgl.sgt.sgl; - if (ctx->enc) { - /* - * Encryption operation - The in-place cipher operation is - * achieved by the following operation: - * - * TX SGL: AAD || PT - * | | - * | copy | - * v v - * RX SGL: AAD || PT || Tag - */ - memcpy_sglist(areq->first_rsgl.sgl.sgt.sgl, tsgl_src, - processed); - af_alg_pull_tsgl(sk, processed, NULL, 0); - } else { - /* - * Decryption operation - To achieve an in-place cipher - * operation, the following SGL structure is used: - * - * TX SGL: AAD || CT || Tag - * | | ^ - * | copy | | Create SGL link. - * v v | - * RX SGL: AAD || CT ----+ - */ - - /* Copy AAD || CT to RX SGL buffer for in-place operation. */ - memcpy_sglist(areq->first_rsgl.sgl.sgt.sgl, tsgl_src, outlen); - - /* Create TX SGL for tag and chain it to RX SGL. */ - areq->tsgl_entries = af_alg_count_tsgl(sk, processed, - processed - as); - if (!areq->tsgl_entries) - areq->tsgl_entries = 1; - areq->tsgl = sock_kmalloc(sk, array_size(sizeof(*areq->tsgl), - areq->tsgl_entries), - GFP_KERNEL); - if (!areq->tsgl) { - err = -ENOMEM; - goto free; - } - sg_init_table(areq->tsgl, areq->tsgl_entries); - - /* Release TX SGL, except for tag data and reassign tag data. */ - af_alg_pull_tsgl(sk, processed, areq->tsgl, processed - as); - - /* chain the areq TX SGL holding the tag with RX SGL */ - if (usedpages) { - /* RX SGL present */ - struct af_alg_sgl *sgl_prev = &areq->last_rsgl->sgl; - struct scatterlist *sg = sgl_prev->sgt.sgl; - - sg_unmark_end(sg + sgl_prev->sgt.nents - 1); - sg_chain(sg, sgl_prev->sgt.nents + 1, areq->tsgl); - } else - /* no RX SGL present (e.g. authentication only) */ - rsgl_src = areq->tsgl; - } + memcpy_sglist(rsgl_src, tsgl_src, ctx->aead_assoclen); /* Initialize the crypto operation */ - aead_request_set_crypt(&areq->cra_u.aead_req, rsgl_src, - areq->first_rsgl.sgl.sgt.sgl, used, ctx->iv); + aead_request_set_crypt(&areq->cra_u.aead_req, tsgl_src, + areq->first_rsgl.sgl.sgt.sgl, used, iv); aead_request_set_ad(&areq->cra_u.aead_req, ctx->aead_assoclen); aead_request_set_tfm(&areq->cra_u.aead_req, tfm); @@ -450,7 +394,7 @@ static void aead_sock_destruct(struct sock *sk) struct crypto_aead *tfm = pask->private; unsigned int ivlen = crypto_aead_ivsize(tfm); - af_alg_pull_tsgl(sk, ctx->used, NULL, 0); + af_alg_pull_tsgl(sk, ctx->used, NULL); sock_kzfree_s(sk, ctx->iv, ivlen); sock_kfree_s(sk, ctx, ctx->len); af_alg_release_parent(sk); diff --git a/crypto/algif_skcipher.c b/crypto/algif_skcipher.c index 125d395c5e009e..ba0a17fd95aca2 100644 --- a/crypto/algif_skcipher.c +++ b/crypto/algif_skcipher.c @@ -130,6 +130,11 @@ static int _skcipher_recvmsg(struct socket *sock, struct msghdr *msg, * full block size buffers. */ if (ctx->more || len < ctx->used) { + if (len < bs) { + err = -EINVAL; + goto free; + } + len -= len % bs; cflags |= CRYPTO_SKCIPHER_REQ_NOTFINAL; } @@ -138,7 +143,7 @@ static int _skcipher_recvmsg(struct socket *sock, struct msghdr *msg, * Create a per request TX SGL for this request which tracks the * SG entries from the global TX SGL. */ - areq->tsgl_entries = af_alg_count_tsgl(sk, len, 0); + areq->tsgl_entries = af_alg_count_tsgl(sk, len); if (!areq->tsgl_entries) areq->tsgl_entries = 1; areq->tsgl = sock_kmalloc(sk, array_size(sizeof(*areq->tsgl), @@ -149,7 +154,7 @@ static int _skcipher_recvmsg(struct socket *sock, struct msghdr *msg, goto free; } sg_init_table(areq->tsgl, areq->tsgl_entries); - af_alg_pull_tsgl(sk, len, areq->tsgl, 0); + af_alg_pull_tsgl(sk, len, areq->tsgl); /* Initialize the crypto operation */ skcipher_request_set_tfm(&areq->cra_u.skcipher_req, tfm); @@ -363,7 +368,7 @@ static void skcipher_sock_destruct(struct sock *sk) struct alg_sock *pask = alg_sk(psk); struct crypto_skcipher *tfm = pask->private; - af_alg_pull_tsgl(sk, ctx->used, NULL, 0); + af_alg_pull_tsgl(sk, ctx->used, NULL); sock_kzfree_s(sk, ctx->iv, crypto_skcipher_ivsize(tfm)); if (ctx->state) sock_kzfree_s(sk, ctx->state, crypto_skcipher_statesize(tfm)); diff --git a/crypto/asymmetric_keys/x509_cert_parser.c b/crypto/asymmetric_keys/x509_cert_parser.c index b37cae914987b6..aac2d55345a9f8 100644 --- a/crypto/asymmetric_keys/x509_cert_parser.c +++ b/crypto/asymmetric_keys/x509_cert_parser.c @@ -584,10 +584,10 @@ int x509_process_extension(void *context, size_t hdrlen, * 0x04 is where keyCertSign lands in this bit string * 0x80 is where digitalSignature lands in this bit string */ - if (v[0] != ASN1_BTS) - return -EBADMSG; if (vlen < 4) return -EBADMSG; + if (v[0] != ASN1_BTS) + return -EBADMSG; if (v[2] >= 8) return -EBADMSG; if (v[3] & 0x80) @@ -620,10 +620,10 @@ int x509_process_extension(void *context, size_t hdrlen, * (Expect 0xFF if the CA is TRUE) * vlen should match the entire extension size */ - if (v[0] != (ASN1_CONS_BIT | ASN1_SEQ)) - return -EBADMSG; if (vlen < 2) return -EBADMSG; + if (v[0] != (ASN1_CONS_BIT | ASN1_SEQ)) + return -EBADMSG; if (v[1] != vlen - 2) return -EBADMSG; /* Empty SEQUENCE means CA:FALSE (default value omitted per DER) */ diff --git a/crypto/authencesn.c b/crypto/authencesn.c index 542a978663b9e7..af3d584e584fbc 100644 --- a/crypto/authencesn.c +++ b/crypto/authencesn.c @@ -207,6 +207,7 @@ static int crypto_authenc_esn_decrypt_tail(struct aead_request *req, u8 *ohash = areq_ctx->tail; unsigned int cryptlen = req->cryptlen - authsize; unsigned int assoclen = req->assoclen; + struct scatterlist *src = req->src; struct scatterlist *dst = req->dst; u8 *ihash = ohash + crypto_ahash_digestsize(auth); u32 tmp[2]; @@ -214,23 +215,29 @@ static int crypto_authenc_esn_decrypt_tail(struct aead_request *req, if (!authsize) goto decrypt; - /* Move high-order bits of sequence number back. */ - scatterwalk_map_and_copy(tmp, dst, 4, 4, 0); - scatterwalk_map_and_copy(tmp + 1, dst, assoclen + cryptlen, 4, 0); - scatterwalk_map_and_copy(tmp, dst, 0, 8, 1); + if (src == dst) { + /* Move high-order bits of sequence number back. */ + scatterwalk_map_and_copy(tmp, dst, 4, 4, 0); + scatterwalk_map_and_copy(tmp + 1, dst, assoclen + cryptlen, 4, 0); + scatterwalk_map_and_copy(tmp, dst, 0, 8, 1); + } else + memcpy_sglist(dst, src, assoclen); if (crypto_memneq(ihash, ohash, authsize)) return -EBADMSG; decrypt: - sg_init_table(areq_ctx->dst, 2); dst = scatterwalk_ffwd(areq_ctx->dst, dst, assoclen); + if (req->src == req->dst) + src = dst; + else + src = scatterwalk_ffwd(areq_ctx->src, src, assoclen); skcipher_request_set_tfm(skreq, ctx->enc); skcipher_request_set_callback(skreq, flags, req->base.complete, req->base.data); - skcipher_request_set_crypt(skreq, dst, dst, cryptlen, req->iv); + skcipher_request_set_crypt(skreq, src, dst, cryptlen, req->iv); return crypto_skcipher_decrypt(skreq); } @@ -255,6 +262,7 @@ static int crypto_authenc_esn_decrypt(struct aead_request *req) unsigned int assoclen = req->assoclen; unsigned int cryptlen = req->cryptlen; u8 *ihash = ohash + crypto_ahash_digestsize(auth); + struct scatterlist *src = req->src; struct scatterlist *dst = req->dst; u32 tmp[2]; int err; @@ -262,24 +270,28 @@ static int crypto_authenc_esn_decrypt(struct aead_request *req) if (assoclen < 8) return -EINVAL; - cryptlen -= authsize; - - if (req->src != dst) - memcpy_sglist(dst, req->src, assoclen + cryptlen); + if (!authsize) + goto tail; + cryptlen -= authsize; scatterwalk_map_and_copy(ihash, req->src, assoclen + cryptlen, authsize, 0); - if (!authsize) - goto tail; - /* Move high-order bits of sequence number to the end. */ - scatterwalk_map_and_copy(tmp, dst, 0, 8, 0); - scatterwalk_map_and_copy(tmp, dst, 4, 4, 1); - scatterwalk_map_and_copy(tmp + 1, dst, assoclen + cryptlen, 4, 1); - - sg_init_table(areq_ctx->dst, 2); - dst = scatterwalk_ffwd(areq_ctx->dst, dst, 4); + scatterwalk_map_and_copy(tmp, src, 0, 8, 0); + if (src == dst) { + scatterwalk_map_and_copy(tmp, dst, 4, 4, 1); + scatterwalk_map_and_copy(tmp + 1, dst, assoclen + cryptlen, 4, 1); + dst = scatterwalk_ffwd(areq_ctx->dst, dst, 4); + } else { + scatterwalk_map_and_copy(tmp, dst, 0, 4, 1); + scatterwalk_map_and_copy(tmp + 1, dst, assoclen + cryptlen - 4, 4, 1); + + src = scatterwalk_ffwd(areq_ctx->src, src, 8); + dst = scatterwalk_ffwd(areq_ctx->dst, dst, 4); + memcpy_sglist(dst, src, assoclen + cryptlen - 8); + dst = req->dst; + } ahash_request_set_tfm(ahreq, auth); ahash_request_set_crypt(ahreq, dst, ohash, assoclen + cryptlen); diff --git a/crypto/deflate.c b/crypto/deflate.c index a3e1fff55661b7..8df17e7880c9bf 100644 --- a/crypto/deflate.c +++ b/crypto/deflate.c @@ -164,18 +164,21 @@ static int deflate_decompress_one(struct acomp_req *req, do { unsigned int dcur; + unsigned long avail_in; dcur = acomp_walk_next_dst(&walk); - if (!dcur) { - out_of_space = true; - break; - } stream->avail_out = dcur; stream->next_out = walk.dst.virt.addr; + avail_in = stream->avail_in; ret = zlib_inflate(stream, Z_NO_FLUSH); + if (!dcur && avail_in == stream->avail_in) { + out_of_space = true; + break; + } + dcur -= stream->avail_out; acomp_walk_done_dst(&walk, dcur); } while (ret == Z_OK && stream->avail_in); diff --git a/drivers/accel/amdxdna/aie2_ctx.c b/drivers/accel/amdxdna/aie2_ctx.c index 42d876a427c595..9284c35aacfbf1 100644 --- a/drivers/accel/amdxdna/aie2_ctx.c +++ b/drivers/accel/amdxdna/aie2_ctx.c @@ -23,9 +23,9 @@ #include "amdxdna_pci_drv.h" #include "amdxdna_pm.h" -static bool force_cmdlist; +static bool force_cmdlist = true; module_param(force_cmdlist, bool, 0600); -MODULE_PARM_DESC(force_cmdlist, "Force use command list (Default false)"); +MODULE_PARM_DESC(force_cmdlist, "Force use command list (Default true)"); #define HWCTX_MAX_TIMEOUT 60000 /* milliseconds */ @@ -47,23 +47,13 @@ static void aie2_job_put(struct amdxdna_sched_job *job) kref_put(&job->refcnt, aie2_job_release); } -static void aie2_hwctx_status_shift_stop(struct amdxdna_hwctx *hwctx) -{ - hwctx->old_status = hwctx->status; - hwctx->status = HWCTX_STAT_STOP; -} - -static void aie2_hwctx_status_restore(struct amdxdna_hwctx *hwctx) -{ - hwctx->status = hwctx->old_status; -} - /* The bad_job is used in aie2_sched_job_timedout, otherwise, set it to NULL */ static void aie2_hwctx_stop(struct amdxdna_dev *xdna, struct amdxdna_hwctx *hwctx, struct drm_sched_job *bad_job) { drm_sched_stop(&hwctx->priv->sched, bad_job); aie2_destroy_context(xdna->dev_handle, hwctx); + drm_sched_start(&hwctx->priv->sched, 0); } static int aie2_hwctx_restart(struct amdxdna_dev *xdna, struct amdxdna_hwctx *hwctx) @@ -84,11 +74,6 @@ static int aie2_hwctx_restart(struct amdxdna_dev *xdna, struct amdxdna_hwctx *hw goto out; } - if (hwctx->status != HWCTX_STAT_READY) { - XDNA_DBG(xdna, "hwctx is not ready, status %d", hwctx->status); - goto out; - } - ret = aie2_config_cu(hwctx, NULL); if (ret) { XDNA_ERR(xdna, "Config cu failed, ret %d", ret); @@ -96,7 +81,6 @@ static int aie2_hwctx_restart(struct amdxdna_dev *xdna, struct amdxdna_hwctx *hw } out: - drm_sched_start(&hwctx->priv->sched, 0); XDNA_DBG(xdna, "%s restarted, ret %d", hwctx->name, ret); return ret; } @@ -140,7 +124,6 @@ static int aie2_hwctx_suspend_cb(struct amdxdna_hwctx *hwctx, void *arg) aie2_hwctx_wait_for_idle(hwctx); aie2_hwctx_stop(xdna, hwctx, NULL); - aie2_hwctx_status_shift_stop(hwctx); return 0; } @@ -162,7 +145,6 @@ static int aie2_hwctx_resume_cb(struct amdxdna_hwctx *hwctx, void *arg) { struct amdxdna_dev *xdna = hwctx->client->xdna; - aie2_hwctx_status_restore(hwctx); return aie2_hwctx_restart(xdna, hwctx); } @@ -183,7 +165,6 @@ aie2_sched_notify(struct amdxdna_sched_job *job) trace_xdna_job(&job->base, job->hwctx->name, "signaled fence", job->seq); - amdxdna_pm_suspend_put(job->hwctx->client->xdna); job->hwctx->priv->completed++; dma_fence_signal(fence); @@ -204,13 +185,13 @@ aie2_sched_resp_handler(void *handle, void __iomem *data, size_t size) cmd_abo = job->cmd_bo; if (unlikely(job->job_timeout)) { - amdxdna_cmd_set_state(cmd_abo, ERT_CMD_STATE_TIMEOUT); + amdxdna_cmd_set_error(cmd_abo, job, 0, ERT_CMD_STATE_TIMEOUT); ret = -EINVAL; goto out; } if (unlikely(!data) || unlikely(size != sizeof(u32))) { - amdxdna_cmd_set_state(cmd_abo, ERT_CMD_STATE_ABORT); + amdxdna_cmd_set_error(cmd_abo, job, 0, ERT_CMD_STATE_ABORT); ret = -EINVAL; goto out; } @@ -220,7 +201,7 @@ aie2_sched_resp_handler(void *handle, void __iomem *data, size_t size) if (status == AIE2_STATUS_SUCCESS) amdxdna_cmd_set_state(cmd_abo, ERT_CMD_STATE_COMPLETED); else - amdxdna_cmd_set_state(cmd_abo, ERT_CMD_STATE_ERROR); + amdxdna_cmd_set_error(cmd_abo, job, 0, ERT_CMD_STATE_ERROR); out: aie2_sched_notify(job); @@ -262,13 +243,13 @@ aie2_sched_cmdlist_resp_handler(void *handle, void __iomem *data, size_t size) cmd_abo = job->cmd_bo; if (unlikely(job->job_timeout)) { - amdxdna_cmd_set_state(cmd_abo, ERT_CMD_STATE_TIMEOUT); + amdxdna_cmd_set_error(cmd_abo, job, 0, ERT_CMD_STATE_TIMEOUT); ret = -EINVAL; goto out; } if (unlikely(!data) || unlikely(size != sizeof(u32) * 3)) { - amdxdna_cmd_set_state(cmd_abo, ERT_CMD_STATE_ABORT); + amdxdna_cmd_set_error(cmd_abo, job, 0, ERT_CMD_STATE_ABORT); ret = -EINVAL; goto out; } @@ -288,19 +269,12 @@ aie2_sched_cmdlist_resp_handler(void *handle, void __iomem *data, size_t size) fail_cmd_idx, fail_cmd_status); if (fail_cmd_status == AIE2_STATUS_SUCCESS) { - amdxdna_cmd_set_state(cmd_abo, ERT_CMD_STATE_ABORT); + amdxdna_cmd_set_error(cmd_abo, job, fail_cmd_idx, ERT_CMD_STATE_ABORT); ret = -EINVAL; - goto out; + } else { + amdxdna_cmd_set_error(cmd_abo, job, fail_cmd_idx, ERT_CMD_STATE_ERROR); } - amdxdna_cmd_set_state(cmd_abo, fail_cmd_status); - - if (amdxdna_cmd_get_op(cmd_abo) == ERT_CMD_CHAIN) { - struct amdxdna_cmd_chain *cc = amdxdna_cmd_get_payload(cmd_abo, NULL); - cc->error_index = fail_cmd_idx; - if (cc->error_index >= cc->command_count) - cc->error_index = 0; - } out: aie2_sched_notify(job); return ret; @@ -315,6 +289,9 @@ aie2_sched_job_run(struct drm_sched_job *sched_job) struct dma_fence *fence; int ret; + if (!hwctx->priv->mbox_chann) + return NULL; + if (!mmget_not_zero(job->mm)) return ERR_PTR(-ESRCH); @@ -468,6 +445,12 @@ static int aie2_alloc_resource(struct amdxdna_hwctx *hwctx) struct alloc_requests *xrs_req; int ret; + if (AIE2_FEATURE_ON(xdna->dev_handle, AIE2_TEMPORAL_ONLY)) { + hwctx->num_unused_col = xdna->dev_handle->total_col - hwctx->num_col; + hwctx->num_col = xdna->dev_handle->total_col; + return aie2_create_context(xdna->dev_handle, hwctx); + } + xrs_req = kzalloc(sizeof(*xrs_req), GFP_KERNEL); if (!xrs_req) return -ENOMEM; @@ -499,9 +482,15 @@ static void aie2_release_resource(struct amdxdna_hwctx *hwctx) struct amdxdna_dev *xdna = hwctx->client->xdna; int ret; - ret = xrs_release_resource(xdna->xrs_hdl, (uintptr_t)hwctx); - if (ret) - XDNA_ERR(xdna, "Release AIE resource failed, ret %d", ret); + if (AIE2_FEATURE_ON(xdna->dev_handle, AIE2_TEMPORAL_ONLY)) { + ret = aie2_destroy_context(xdna->dev_handle, hwctx); + if (ret && ret != -ENODEV) + XDNA_ERR(xdna, "Destroy temporal only context failed, ret %d", ret); + } else { + ret = xrs_release_resource(xdna->xrs_hdl, (uintptr_t)hwctx); + if (ret) + XDNA_ERR(xdna, "Release AIE resource failed, ret %d", ret); + } } static int aie2_ctx_syncobj_create(struct amdxdna_hwctx *hwctx) @@ -627,7 +616,7 @@ int aie2_hwctx_init(struct amdxdna_hwctx *hwctx) goto free_entity; } - ret = amdxdna_pm_resume_get(xdna); + ret = amdxdna_pm_resume_get_locked(xdna); if (ret) goto free_col_list; @@ -651,7 +640,6 @@ int aie2_hwctx_init(struct amdxdna_hwctx *hwctx) } amdxdna_pm_suspend_put(xdna); - hwctx->status = HWCTX_STAT_INIT; init_waitqueue_head(&priv->job_free_wq); XDNA_DBG(xdna, "hwctx %s init completed", hwctx->name); @@ -693,7 +681,9 @@ void aie2_hwctx_fini(struct amdxdna_hwctx *hwctx) aie2_hwctx_wait_for_idle(hwctx); /* Request fw to destroy hwctx and cancel the rest pending requests */ + drm_sched_stop(&hwctx->priv->sched, NULL); aie2_release_resource(hwctx); + drm_sched_start(&hwctx->priv->sched, 0); mutex_unlock(&xdna->dev_lock); drm_sched_entity_destroy(&hwctx->priv->entity); @@ -737,7 +727,7 @@ static int aie2_hwctx_cu_config(struct amdxdna_hwctx *hwctx, void *buf, u32 size if (XDNA_MBZ_DBG(xdna, config->pad, sizeof(config->pad))) return -EINVAL; - if (hwctx->status != HWCTX_STAT_INIT) { + if (hwctx->cus) { XDNA_ERR(xdna, "Not support re-config CU"); return -EINVAL; } @@ -757,7 +747,7 @@ static int aie2_hwctx_cu_config(struct amdxdna_hwctx *hwctx, void *buf, u32 size if (!hwctx->cus) return -ENOMEM; - ret = amdxdna_pm_resume_get(xdna); + ret = amdxdna_pm_resume_get_locked(xdna); if (ret) goto free_cus; @@ -768,7 +758,6 @@ static int aie2_hwctx_cu_config(struct amdxdna_hwctx *hwctx, void *buf, u32 size } wmb(); /* To avoid locking in command submit when check status */ - hwctx->status = HWCTX_STAT_READY; return 0; @@ -991,15 +980,11 @@ int aie2_cmd_submit(struct amdxdna_hwctx *hwctx, struct amdxdna_sched_job *job, goto free_chain; } - ret = amdxdna_pm_resume_get(xdna); - if (ret) - goto cleanup_job; - retry: ret = drm_gem_lock_reservations(job->bos, job->bo_cnt, &acquire_ctx); if (ret) { XDNA_WARN(xdna, "Failed to lock BOs, ret %d", ret); - goto suspend_put; + goto cleanup_job; } for (i = 0; i < job->bo_cnt; i++) { @@ -1007,7 +992,7 @@ int aie2_cmd_submit(struct amdxdna_hwctx *hwctx, struct amdxdna_sched_job *job, if (ret) { XDNA_WARN(xdna, "Failed to reserve fences %d", ret); drm_gem_unlock_reservations(job->bos, job->bo_cnt, &acquire_ctx); - goto suspend_put; + goto cleanup_job; } } @@ -1022,12 +1007,12 @@ int aie2_cmd_submit(struct amdxdna_hwctx *hwctx, struct amdxdna_sched_job *job, msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT); } else if (time_after(jiffies, timeout)) { ret = -ETIME; - goto suspend_put; + goto cleanup_job; } ret = aie2_populate_range(abo); if (ret) - goto suspend_put; + goto cleanup_job; goto retry; } } @@ -1053,8 +1038,6 @@ int aie2_cmd_submit(struct amdxdna_hwctx *hwctx, struct amdxdna_sched_job *job, return 0; -suspend_put: - amdxdna_pm_suspend_put(xdna); cleanup_job: drm_sched_job_cleanup(&job->base); free_chain: @@ -1074,6 +1057,8 @@ void aie2_hmm_invalidate(struct amdxdna_gem_obj *abo, ret = dma_resv_wait_timeout(gobj->resv, DMA_RESV_USAGE_BOOKKEEP, true, MAX_SCHEDULE_TIMEOUT); - if (!ret || ret == -ERESTARTSYS) + if (!ret) XDNA_ERR(xdna, "Failed to wait for bo, ret %ld", ret); + else if (ret == -ERESTARTSYS) + XDNA_DBG(xdna, "Wait for bo interrupted by signal"); } diff --git a/drivers/accel/amdxdna/aie2_message.c b/drivers/accel/amdxdna/aie2_message.c index d493bb1c33606f..f0fb98131068ca 100644 --- a/drivers/accel/amdxdna/aie2_message.c +++ b/drivers/accel/amdxdna/aie2_message.c @@ -39,13 +39,9 @@ static int aie2_send_mgmt_msg_wait(struct amdxdna_dev_hdl *ndev, if (!ndev->mgmt_chann) return -ENODEV; - drm_WARN_ON(&xdna->ddev, xdna->rpm_on && !mutex_is_locked(&xdna->dev_lock)); ret = xdna_send_msg_wait(xdna, ndev->mgmt_chann, msg); - if (ret == -ETIME) { - xdna_mailbox_stop_channel(ndev->mgmt_chann); - xdna_mailbox_destroy_channel(ndev->mgmt_chann); - ndev->mgmt_chann = NULL; - } + if (ret == -ETIME) + aie2_destroy_mgmt_chann(ndev); if (!ret && *hdl->status != AIE2_STATUS_SUCCESS) { XDNA_ERR(xdna, "command opcode 0x%x failed, status 0x%x", @@ -186,6 +182,21 @@ int aie2_query_firmware_version(struct amdxdna_dev_hdl *ndev, return 0; } +static int aie2_destroy_context_req(struct amdxdna_dev_hdl *ndev, u32 id) +{ + DECLARE_AIE2_MSG(destroy_ctx, MSG_OP_DESTROY_CONTEXT); + struct amdxdna_dev *xdna = ndev->xdna; + int ret; + + req.context_id = id; + ret = aie2_send_mgmt_msg_wait(ndev, &msg); + if (ret && ret != -ENODEV) + XDNA_WARN(xdna, "Destroy context failed, ret %d", ret); + else if (ret == -ENODEV) + XDNA_DBG(xdna, "Destroy context: device already stopped"); + + return ret; +} int aie2_create_context(struct amdxdna_dev_hdl *ndev, struct amdxdna_hwctx *hwctx) { DECLARE_AIE2_MSG(create_ctx, MSG_OP_CREATE_CONTEXT); @@ -199,6 +210,7 @@ int aie2_create_context(struct amdxdna_dev_hdl *ndev, struct amdxdna_hwctx *hwct req.aie_type = 1; req.start_col = hwctx->start_col; req.num_col = hwctx->num_col; + req.num_unused_col = hwctx->num_unused_col; req.num_cq_pairs_requested = 1; req.pasid = hwctx->client->pasid; req.context_priority = 2; @@ -208,13 +220,14 @@ int aie2_create_context(struct amdxdna_dev_hdl *ndev, struct amdxdna_hwctx *hwct return ret; hwctx->fw_ctx_id = resp.context_id; - WARN_ONCE(hwctx->fw_ctx_id == -1, "Unexpected context id"); + if (WARN_ON_ONCE(hwctx->fw_ctx_id == -1)) + return -EINVAL; if (ndev->force_preempt_enabled) { ret = aie2_runtime_cfg(ndev, AIE2_RT_CFG_FORCE_PREEMPT, &hwctx->fw_ctx_id); if (ret) { XDNA_ERR(xdna, "failed to enable force preempt %d", ret); - return ret; + goto del_ctx_req; } } @@ -231,51 +244,42 @@ int aie2_create_context(struct amdxdna_dev_hdl *ndev, struct amdxdna_hwctx *hwct ret = pci_irq_vector(to_pci_dev(xdna->ddev.dev), resp.msix_id); if (ret == -EINVAL) { - XDNA_ERR(xdna, "not able to create channel"); - goto out_destroy_context; + XDNA_ERR(xdna, "Alloc IRQ failed %d", ret); + goto del_ctx_req; } intr_reg = i2x.mb_head_ptr_reg + 4; hwctx->priv->mbox_chann = xdna_mailbox_create_channel(ndev->mbox, &x2i, &i2x, intr_reg, ret); if (!hwctx->priv->mbox_chann) { - XDNA_ERR(xdna, "not able to create channel"); + XDNA_ERR(xdna, "Not able to create channel"); ret = -EINVAL; - goto out_destroy_context; + goto del_ctx_req; } ndev->hwctx_num++; - XDNA_DBG(xdna, "%s mailbox channel irq: %d, msix_id: %d", - hwctx->name, ret, resp.msix_id); - XDNA_DBG(xdna, "%s created fw ctx %d pasid %d", hwctx->name, - hwctx->fw_ctx_id, hwctx->client->pasid); + XDNA_DBG(xdna, "Mailbox channel irq: %d, msix_id: %d", ret, resp.msix_id); + XDNA_DBG(xdna, "Created fw ctx %d pasid %d", hwctx->fw_ctx_id, hwctx->client->pasid); return 0; -out_destroy_context: - aie2_destroy_context(ndev, hwctx); +del_ctx_req: + aie2_destroy_context_req(ndev, hwctx->fw_ctx_id); return ret; } int aie2_destroy_context(struct amdxdna_dev_hdl *ndev, struct amdxdna_hwctx *hwctx) { - DECLARE_AIE2_MSG(destroy_ctx, MSG_OP_DESTROY_CONTEXT); struct amdxdna_dev *xdna = ndev->xdna; int ret; - if (hwctx->fw_ctx_id == -1) + if (!hwctx->priv->mbox_chann) return 0; xdna_mailbox_stop_channel(hwctx->priv->mbox_chann); - - req.context_id = hwctx->fw_ctx_id; - ret = aie2_send_mgmt_msg_wait(ndev, &msg); - if (ret) - XDNA_WARN(xdna, "%s destroy context failed, ret %d", hwctx->name, ret); - + ret = aie2_destroy_context_req(ndev, hwctx->fw_ctx_id); xdna_mailbox_destroy_channel(hwctx->priv->mbox_chann); - XDNA_DBG(xdna, "%s destroyed fw ctx %d", hwctx->name, - hwctx->fw_ctx_id); + XDNA_DBG(xdna, "Destroyed fw ctx %d", hwctx->fw_ctx_id); hwctx->priv->mbox_chann = NULL; hwctx->fw_ctx_id = -1; ndev->hwctx_num--; @@ -448,6 +452,9 @@ int aie2_config_cu(struct amdxdna_hwctx *hwctx, if (!chann) return -ENODEV; + if (!hwctx->cus) + return 0; + if (hwctx->cus->num_cus > MAX_NUM_CUS) { XDNA_DBG(xdna, "Exceed maximum CU %d", MAX_NUM_CUS); return -EINVAL; @@ -650,11 +657,11 @@ aie2_cmdlist_fill_npu_cf(struct amdxdna_gem_obj *cmd_bo, void *slot, size_t *siz if (*size < sizeof(*npu_slot) + cmd_len) return -EINVAL; + memset(npu_slot, 0, sizeof(*npu_slot)); npu_slot->cu_idx = amdxdna_cmd_get_cu_idx(cmd_bo); if (npu_slot->cu_idx == INVALID_CU_IDX) return -EINVAL; - memset(npu_slot, 0, sizeof(*npu_slot)); npu_slot->type = EXEC_NPU_TYPE_NON_ELF; npu_slot->arg_cnt = cmd_len / sizeof(u32); memcpy(npu_slot->args, cmd, cmd_len); @@ -679,11 +686,11 @@ aie2_cmdlist_fill_npu_dpu(struct amdxdna_gem_obj *cmd_bo, void *slot, size_t *si if (*size < sizeof(*npu_slot) + arg_sz) return -EINVAL; + memset(npu_slot, 0, sizeof(*npu_slot)); npu_slot->cu_idx = amdxdna_cmd_get_cu_idx(cmd_bo); if (npu_slot->cu_idx == INVALID_CU_IDX) return -EINVAL; - memset(npu_slot, 0, sizeof(*npu_slot)); npu_slot->type = EXEC_NPU_TYPE_PARTIAL_ELF; npu_slot->inst_buf_addr = sn->buffer; npu_slot->inst_size = sn->buffer_size; @@ -711,11 +718,11 @@ aie2_cmdlist_fill_npu_preempt(struct amdxdna_gem_obj *cmd_bo, void *slot, size_t if (*size < sizeof(*npu_slot) + arg_sz) return -EINVAL; + memset(npu_slot, 0, sizeof(*npu_slot)); npu_slot->cu_idx = amdxdna_cmd_get_cu_idx(cmd_bo); if (npu_slot->cu_idx == INVALID_CU_IDX) return -EINVAL; - memset(npu_slot, 0, sizeof(*npu_slot)); npu_slot->type = EXEC_NPU_TYPE_PREEMPT; npu_slot->inst_buf_addr = pd->inst_buf; npu_slot->save_buf_addr = pd->save_buf; @@ -861,6 +868,20 @@ void aie2_msg_init(struct amdxdna_dev_hdl *ndev) ndev->exec_msg_ops = &legacy_exec_message_ops; } +void aie2_destroy_mgmt_chann(struct amdxdna_dev_hdl *ndev) +{ + struct amdxdna_dev *xdna = ndev->xdna; + + drm_WARN_ON(&xdna->ddev, !mutex_is_locked(&xdna->dev_lock)); + + if (!ndev->mgmt_chann) + return; + + xdna_mailbox_stop_channel(ndev->mgmt_chann); + xdna_mailbox_destroy_channel(ndev->mgmt_chann); + ndev->mgmt_chann = NULL; +} + static inline struct amdxdna_gem_obj * aie2_cmdlist_get_cmd_buf(struct amdxdna_sched_job *job) { diff --git a/drivers/accel/amdxdna/aie2_msg_priv.h b/drivers/accel/amdxdna/aie2_msg_priv.h index 1c957a6298d39d..cc912b7899ce5a 100644 --- a/drivers/accel/amdxdna/aie2_msg_priv.h +++ b/drivers/accel/amdxdna/aie2_msg_priv.h @@ -112,7 +112,8 @@ struct create_ctx_req { __u32 aie_type; __u8 start_col; __u8 num_col; - __u16 reserved; + __u8 num_unused_col; + __u8 reserved; __u8 num_cq_pairs_requested; __u8 reserved1; __u16 pasid; diff --git a/drivers/accel/amdxdna/aie2_pci.c b/drivers/accel/amdxdna/aie2_pci.c index 8141d8e5163605..0a8e7a8710eeaa 100644 --- a/drivers/accel/amdxdna/aie2_pci.c +++ b/drivers/accel/amdxdna/aie2_pci.c @@ -322,7 +322,7 @@ static int aie2_xrs_set_dft_dpm_level(struct drm_device *ddev, u32 dpm_level) if (ndev->pw_mode != POWER_MODE_DEFAULT || ndev->dpm_level == dpm_level) return 0; - return ndev->priv->hw_ops.set_dpm(ndev, dpm_level); + return aie2_pm_set_dpm(ndev, dpm_level); } static struct xrs_action_ops aie2_xrs_actions = { @@ -341,10 +341,9 @@ static void aie2_hw_stop(struct amdxdna_dev *xdna) return; } + aie2_runtime_cfg(ndev, AIE2_RT_CFG_CLK_GATING, NULL); aie2_mgmt_fw_fini(ndev); - xdna_mailbox_stop_channel(ndev->mgmt_chann); - xdna_mailbox_destroy_channel(ndev->mgmt_chann); - ndev->mgmt_chann = NULL; + aie2_destroy_mgmt_chann(ndev); drmm_kfree(&xdna->ddev, ndev->mbox); ndev->mbox = NULL; aie2_psp_stop(ndev->psp_hdl); @@ -424,15 +423,15 @@ static int aie2_hw_start(struct amdxdna_dev *xdna) goto stop_psp; } - ret = aie2_pm_init(ndev); + ret = aie2_mgmt_fw_init(ndev); if (ret) { - XDNA_ERR(xdna, "failed to init pm, ret %d", ret); + XDNA_ERR(xdna, "initial mgmt firmware failed, ret %d", ret); goto destroy_mgmt_chann; } - ret = aie2_mgmt_fw_init(ndev); + ret = aie2_pm_init(ndev); if (ret) { - XDNA_ERR(xdna, "initial mgmt firmware failed, ret %d", ret); + XDNA_ERR(xdna, "failed to init pm, ret %d", ret); goto destroy_mgmt_chann; } @@ -453,8 +452,7 @@ static int aie2_hw_start(struct amdxdna_dev *xdna) return 0; destroy_mgmt_chann: - xdna_mailbox_stop_channel(ndev->mgmt_chann); - xdna_mailbox_destroy_channel(ndev->mgmt_chann); + aie2_destroy_mgmt_chann(ndev); stop_psp: aie2_psp_stop(ndev->psp_hdl); fini_smu: @@ -469,7 +467,6 @@ static int aie2_hw_suspend(struct amdxdna_dev *xdna) { struct amdxdna_client *client; - guard(mutex)(&xdna->dev_lock); list_for_each_entry(client, &xdna->client_list, node) aie2_hwctx_suspend(client); @@ -969,7 +966,7 @@ static int aie2_get_info(struct amdxdna_client *client, struct amdxdna_drm_get_i if (!drm_dev_enter(&xdna->ddev, &idx)) return -ENODEV; - ret = amdxdna_pm_resume_get(xdna); + ret = amdxdna_pm_resume_get_locked(xdna); if (ret) goto dev_exit; @@ -1062,7 +1059,7 @@ static int aie2_get_array(struct amdxdna_client *client, if (!drm_dev_enter(&xdna->ddev, &idx)) return -ENODEV; - ret = amdxdna_pm_resume_get(xdna); + ret = amdxdna_pm_resume_get_locked(xdna); if (ret) goto dev_exit; @@ -1152,7 +1149,7 @@ static int aie2_set_state(struct amdxdna_client *client, if (!drm_dev_enter(&xdna->ddev, &idx)) return -ENODEV; - ret = amdxdna_pm_resume_get(xdna); + ret = amdxdna_pm_resume_get_locked(xdna); if (ret) goto dev_exit; diff --git a/drivers/accel/amdxdna/aie2_pci.h b/drivers/accel/amdxdna/aie2_pci.h index a5f9c42155d178..482ee555f6c477 100644 --- a/drivers/accel/amdxdna/aie2_pci.h +++ b/drivers/accel/amdxdna/aie2_pci.h @@ -231,6 +231,7 @@ struct aie2_hw_ops { enum aie2_fw_feature { AIE2_NPU_COMMAND, AIE2_PREEMPT, + AIE2_TEMPORAL_ONLY, AIE2_FEATURE_MAX }; @@ -285,6 +286,7 @@ int npu4_set_dpm(struct amdxdna_dev_hdl *ndev, u32 dpm_level); /* aie2_pm.c */ int aie2_pm_init(struct amdxdna_dev_hdl *ndev); int aie2_pm_set_mode(struct amdxdna_dev_hdl *ndev, enum amdxdna_power_mode_type target); +int aie2_pm_set_dpm(struct amdxdna_dev_hdl *ndev, u32 dpm_level); /* aie2_psp.c */ struct psp_device *aie2m_psp_create(struct drm_device *ddev, struct psp_config *conf); @@ -300,6 +302,7 @@ int aie2_get_array_async_error(struct amdxdna_dev_hdl *ndev, /* aie2_message.c */ void aie2_msg_init(struct amdxdna_dev_hdl *ndev); +void aie2_destroy_mgmt_chann(struct amdxdna_dev_hdl *ndev); int aie2_suspend_fw(struct amdxdna_dev_hdl *ndev); int aie2_resume_fw(struct amdxdna_dev_hdl *ndev); int aie2_set_runtime_cfg(struct amdxdna_dev_hdl *ndev, u32 type, u64 value); diff --git a/drivers/accel/amdxdna/aie2_pm.c b/drivers/accel/amdxdna/aie2_pm.c index 426c38fce84829..29bd4403a94d44 100644 --- a/drivers/accel/amdxdna/aie2_pm.c +++ b/drivers/accel/amdxdna/aie2_pm.c @@ -10,6 +10,7 @@ #include "aie2_pci.h" #include "amdxdna_pci_drv.h" +#include "amdxdna_pm.h" #define AIE2_CLK_GATING_ENABLE 1 #define AIE2_CLK_GATING_DISABLE 0 @@ -26,6 +27,22 @@ static int aie2_pm_set_clk_gating(struct amdxdna_dev_hdl *ndev, u32 val) return 0; } +int aie2_pm_set_dpm(struct amdxdna_dev_hdl *ndev, u32 dpm_level) +{ + int ret; + + ret = amdxdna_pm_resume_get_locked(ndev->xdna); + if (ret) + return ret; + + ret = ndev->priv->hw_ops.set_dpm(ndev, dpm_level); + if (!ret) + ndev->dpm_level = dpm_level; + amdxdna_pm_suspend_put(ndev->xdna); + + return ret; +} + int aie2_pm_init(struct amdxdna_dev_hdl *ndev) { int ret; @@ -50,6 +67,7 @@ int aie2_pm_init(struct amdxdna_dev_hdl *ndev) ret = ndev->priv->hw_ops.set_dpm(ndev, ndev->max_dpm_level); if (ret) return ret; + ndev->dpm_level = ndev->max_dpm_level; ret = aie2_pm_set_clk_gating(ndev, AIE2_CLK_GATING_ENABLE); if (ret) @@ -94,7 +112,7 @@ int aie2_pm_set_mode(struct amdxdna_dev_hdl *ndev, enum amdxdna_power_mode_type return -EOPNOTSUPP; } - ret = ndev->priv->hw_ops.set_dpm(ndev, dpm_level); + ret = aie2_pm_set_dpm(ndev, dpm_level); if (ret) return ret; diff --git a/drivers/accel/amdxdna/aie2_smu.c b/drivers/accel/amdxdna/aie2_smu.c index bd94ee96c2bc06..d8c31924e501ba 100644 --- a/drivers/accel/amdxdna/aie2_smu.c +++ b/drivers/accel/amdxdna/aie2_smu.c @@ -11,7 +11,6 @@ #include "aie2_pci.h" #include "amdxdna_pci_drv.h" -#include "amdxdna_pm.h" #define SMU_RESULT_OK 1 @@ -67,16 +66,12 @@ int npu1_set_dpm(struct amdxdna_dev_hdl *ndev, u32 dpm_level) u32 freq; int ret; - ret = amdxdna_pm_resume_get(ndev->xdna); - if (ret) - return ret; - ret = aie2_smu_exec(ndev, AIE2_SMU_SET_MPNPUCLK_FREQ, ndev->priv->dpm_clk_tbl[dpm_level].npuclk, &freq); if (ret) { XDNA_ERR(ndev->xdna, "Set npu clock to %d failed, ret %d\n", ndev->priv->dpm_clk_tbl[dpm_level].npuclk, ret); - goto suspend_put; + return ret; } ndev->npuclk_freq = freq; @@ -85,12 +80,10 @@ int npu1_set_dpm(struct amdxdna_dev_hdl *ndev, u32 dpm_level) if (ret) { XDNA_ERR(ndev->xdna, "Set h clock to %d failed, ret %d\n", ndev->priv->dpm_clk_tbl[dpm_level].hclk, ret); - goto suspend_put; + return ret; } - amdxdna_pm_suspend_put(ndev->xdna); ndev->hclk_freq = freq; - ndev->dpm_level = dpm_level; ndev->max_tops = 2 * ndev->total_col; ndev->curr_tops = ndev->max_tops * freq / 1028; @@ -98,38 +91,28 @@ int npu1_set_dpm(struct amdxdna_dev_hdl *ndev, u32 dpm_level) ndev->npuclk_freq, ndev->hclk_freq); return 0; - -suspend_put: - amdxdna_pm_suspend_put(ndev->xdna); - return ret; } int npu4_set_dpm(struct amdxdna_dev_hdl *ndev, u32 dpm_level) { int ret; - ret = amdxdna_pm_resume_get(ndev->xdna); - if (ret) - return ret; - ret = aie2_smu_exec(ndev, AIE2_SMU_SET_HARD_DPMLEVEL, dpm_level, NULL); if (ret) { XDNA_ERR(ndev->xdna, "Set hard dpm level %d failed, ret %d ", dpm_level, ret); - goto suspend_put; + return ret; } ret = aie2_smu_exec(ndev, AIE2_SMU_SET_SOFT_DPMLEVEL, dpm_level, NULL); if (ret) { XDNA_ERR(ndev->xdna, "Set soft dpm level %d failed, ret %d", dpm_level, ret); - goto suspend_put; + return ret; } - amdxdna_pm_suspend_put(ndev->xdna); ndev->npuclk_freq = ndev->priv->dpm_clk_tbl[dpm_level].npuclk; ndev->hclk_freq = ndev->priv->dpm_clk_tbl[dpm_level].hclk; - ndev->dpm_level = dpm_level; ndev->max_tops = NPU4_DPM_TOPS(ndev, ndev->max_dpm_level); ndev->curr_tops = NPU4_DPM_TOPS(ndev, dpm_level); @@ -137,10 +120,6 @@ int npu4_set_dpm(struct amdxdna_dev_hdl *ndev, u32 dpm_level) ndev->npuclk_freq, ndev->hclk_freq); return 0; - -suspend_put: - amdxdna_pm_suspend_put(ndev->xdna); - return ret; } int aie2_smu_init(struct amdxdna_dev_hdl *ndev) diff --git a/drivers/accel/amdxdna/amdxdna_ctx.c b/drivers/accel/amdxdna/amdxdna_ctx.c index d17aef89a0addd..f678ae4c682d16 100644 --- a/drivers/accel/amdxdna/amdxdna_ctx.c +++ b/drivers/accel/amdxdna/amdxdna_ctx.c @@ -17,6 +17,7 @@ #include "amdxdna_ctx.h" #include "amdxdna_gem.h" #include "amdxdna_pci_drv.h" +#include "amdxdna_pm.h" #define MAX_HWCTX_ID 255 #define MAX_ARG_COUNT 4095 @@ -104,7 +105,10 @@ void *amdxdna_cmd_get_payload(struct amdxdna_gem_obj *abo, u32 *size) if (size) { count = FIELD_GET(AMDXDNA_CMD_COUNT, cmd->header); - if (unlikely(count <= num_masks)) { + if (unlikely(count <= num_masks || + count * sizeof(u32) + + offsetof(struct amdxdna_cmd, data[0]) > + abo->mem.size)) { *size = 0; return NULL; } @@ -132,6 +136,33 @@ u32 amdxdna_cmd_get_cu_idx(struct amdxdna_gem_obj *abo) return INVALID_CU_IDX; } +int amdxdna_cmd_set_error(struct amdxdna_gem_obj *abo, + struct amdxdna_sched_job *job, u32 cmd_idx, + enum ert_cmd_state error_state) +{ + struct amdxdna_client *client = job->hwctx->client; + struct amdxdna_cmd *cmd = abo->mem.kva; + struct amdxdna_cmd_chain *cc = NULL; + + cmd->header &= ~AMDXDNA_CMD_STATE; + cmd->header |= FIELD_PREP(AMDXDNA_CMD_STATE, error_state); + + if (amdxdna_cmd_get_op(abo) == ERT_CMD_CHAIN) { + cc = amdxdna_cmd_get_payload(abo, NULL); + cc->error_index = (cmd_idx < cc->command_count) ? cmd_idx : 0; + abo = amdxdna_gem_get_obj(client, cc->data[0], AMDXDNA_BO_CMD); + if (!abo) + return -EINVAL; + cmd = abo->mem.kva; + } + + memset(cmd->data, 0xff, abo->mem.size - sizeof(*cmd)); + if (cc) + amdxdna_gem_put_obj(abo); + + return 0; +} + /* * This should be called in close() and remove(). DO NOT call in other syscalls. * This guarantee that when hwctx and resources will be released, if user @@ -266,9 +297,9 @@ int amdxdna_drm_config_hwctx_ioctl(struct drm_device *dev, void *data, struct dr struct amdxdna_drm_config_hwctx *args = data; struct amdxdna_dev *xdna = to_xdna_dev(dev); struct amdxdna_hwctx *hwctx; - int ret, idx; u32 buf_size; void *buf; + int ret; u64 val; if (XDNA_MBZ_DBG(xdna, &args->pad, sizeof(args->pad))) @@ -310,20 +341,17 @@ int amdxdna_drm_config_hwctx_ioctl(struct drm_device *dev, void *data, struct dr return -EINVAL; } - mutex_lock(&xdna->dev_lock); - idx = srcu_read_lock(&client->hwctx_srcu); + guard(mutex)(&xdna->dev_lock); hwctx = xa_load(&client->hwctx_xa, args->handle); if (!hwctx) { XDNA_DBG(xdna, "PID %d failed to get hwctx %d", client->pid, args->handle); ret = -EINVAL; - goto unlock_srcu; + goto free_buf; } ret = xdna->dev_info->ops->hwctx_config(hwctx, args->param_type, val, buf, buf_size); -unlock_srcu: - srcu_read_unlock(&client->hwctx_srcu, idx); - mutex_unlock(&xdna->dev_lock); +free_buf: kfree(buf); return ret; } @@ -334,7 +362,7 @@ int amdxdna_hwctx_sync_debug_bo(struct amdxdna_client *client, u32 debug_bo_hdl) struct amdxdna_hwctx *hwctx; struct amdxdna_gem_obj *abo; struct drm_gem_object *gobj; - int ret, idx; + int ret; if (!xdna->dev_info->ops->hwctx_sync_debug_bo) return -EOPNOTSUPP; @@ -345,17 +373,15 @@ int amdxdna_hwctx_sync_debug_bo(struct amdxdna_client *client, u32 debug_bo_hdl) abo = to_xdna_obj(gobj); guard(mutex)(&xdna->dev_lock); - idx = srcu_read_lock(&client->hwctx_srcu); hwctx = xa_load(&client->hwctx_xa, abo->assigned_hwctx); if (!hwctx) { ret = -EINVAL; - goto unlock_srcu; + goto put_obj; } ret = xdna->dev_info->ops->hwctx_sync_debug_bo(hwctx, debug_bo_hdl); -unlock_srcu: - srcu_read_unlock(&client->hwctx_srcu, idx); +put_obj: drm_gem_object_put(gobj); return ret; } @@ -420,6 +446,7 @@ amdxdna_arg_bos_lookup(struct amdxdna_client *client, void amdxdna_sched_job_cleanup(struct amdxdna_sched_job *job) { trace_amdxdna_debug_point(job->hwctx->name, job->seq, "job release"); + amdxdna_pm_suspend_put(job->hwctx->client->xdna); amdxdna_arg_bos_put(job); amdxdna_gem_put_obj(job->cmd_bo); dma_fence_put(job->fence); @@ -457,6 +484,12 @@ int amdxdna_cmd_submit(struct amdxdna_client *client, goto cmd_put; } + ret = amdxdna_pm_resume_get(xdna); + if (ret) { + XDNA_ERR(xdna, "Resume failed, ret %d", ret); + goto put_bos; + } + idx = srcu_read_lock(&client->hwctx_srcu); hwctx = xa_load(&client->hwctx_xa, hwctx_hdl); if (!hwctx) { @@ -497,6 +530,8 @@ int amdxdna_cmd_submit(struct amdxdna_client *client, dma_fence_put(job->fence); unlock_srcu: srcu_read_unlock(&client->hwctx_srcu, idx); + amdxdna_pm_suspend_put(xdna); +put_bos: amdxdna_arg_bos_put(job); cmd_put: amdxdna_gem_put_obj(job->cmd_bo); diff --git a/drivers/accel/amdxdna/amdxdna_ctx.h b/drivers/accel/amdxdna/amdxdna_ctx.h index b6151244d64fe2..fbdf9d00087136 100644 --- a/drivers/accel/amdxdna/amdxdna_ctx.h +++ b/drivers/accel/amdxdna/amdxdna_ctx.h @@ -98,11 +98,7 @@ struct amdxdna_hwctx { u32 *col_list; u32 start_col; u32 num_col; -#define HWCTX_STAT_INIT 0 -#define HWCTX_STAT_READY 1 -#define HWCTX_STAT_STOP 2 - u32 status; - u32 old_status; + u32 num_unused_col; struct amdxdna_qos_info qos; struct amdxdna_hwctx_param_config_cu *cus; @@ -171,6 +167,9 @@ amdxdna_cmd_get_state(struct amdxdna_gem_obj *abo) void *amdxdna_cmd_get_payload(struct amdxdna_gem_obj *abo, u32 *size); u32 amdxdna_cmd_get_cu_idx(struct amdxdna_gem_obj *abo); +int amdxdna_cmd_set_error(struct amdxdna_gem_obj *abo, + struct amdxdna_sched_job *job, u32 cmd_idx, + enum ert_cmd_state error_state); void amdxdna_sched_job_cleanup(struct amdxdna_sched_job *job); void amdxdna_hwctx_remove_all(struct amdxdna_client *client); diff --git a/drivers/accel/amdxdna/amdxdna_gem.c b/drivers/accel/amdxdna/amdxdna_gem.c index dfa916eeb2d9c8..56341b7668b103 100644 --- a/drivers/accel/amdxdna/amdxdna_gem.c +++ b/drivers/accel/amdxdna/amdxdna_gem.c @@ -21,8 +21,6 @@ #include "amdxdna_pci_drv.h" #include "amdxdna_ubuf.h" -#define XDNA_MAX_CMD_BO_SIZE SZ_32K - MODULE_IMPORT_NS("DMA_BUF"); static int @@ -746,12 +744,6 @@ amdxdna_drm_create_cmd_bo(struct drm_device *dev, { struct amdxdna_dev *xdna = to_xdna_dev(dev); struct amdxdna_gem_obj *abo; - int ret; - - if (args->size > XDNA_MAX_CMD_BO_SIZE) { - XDNA_ERR(xdna, "Command bo size 0x%llx too large", args->size); - return ERR_PTR(-EINVAL); - } if (args->size < sizeof(struct amdxdna_cmd)) { XDNA_DBG(xdna, "Command BO size 0x%llx too small", args->size); @@ -765,17 +757,7 @@ amdxdna_drm_create_cmd_bo(struct drm_device *dev, abo->type = AMDXDNA_BO_CMD; abo->client = filp->driver_priv; - ret = amdxdna_gem_obj_vmap(abo, &abo->mem.kva); - if (ret) { - XDNA_ERR(xdna, "Vmap cmd bo failed, ret %d", ret); - goto release_obj; - } - return abo; - -release_obj: - drm_gem_object_put(to_gobj(abo)); - return ERR_PTR(ret); } int amdxdna_drm_create_bo_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) @@ -872,6 +854,7 @@ struct amdxdna_gem_obj *amdxdna_gem_get_obj(struct amdxdna_client *client, struct amdxdna_dev *xdna = client->xdna; struct amdxdna_gem_obj *abo; struct drm_gem_object *gobj; + int ret; gobj = drm_gem_object_lookup(client->filp, bo_hdl); if (!gobj) { @@ -880,9 +863,26 @@ struct amdxdna_gem_obj *amdxdna_gem_get_obj(struct amdxdna_client *client, } abo = to_xdna_obj(gobj); - if (bo_type == AMDXDNA_BO_INVALID || abo->type == bo_type) + if (bo_type != AMDXDNA_BO_INVALID && abo->type != bo_type) + goto put_obj; + + if (bo_type != AMDXDNA_BO_CMD || abo->mem.kva) return abo; + if (abo->mem.size > SZ_32K) { + XDNA_ERR(xdna, "Cmd bo is too big %ld", abo->mem.size); + goto put_obj; + } + + ret = amdxdna_gem_obj_vmap(abo, &abo->mem.kva); + if (ret) { + XDNA_ERR(xdna, "Vmap cmd bo failed, ret %d", ret); + goto put_obj; + } + + return abo; + +put_obj: drm_gem_object_put(gobj); return NULL; } diff --git a/drivers/accel/amdxdna/amdxdna_mailbox.c b/drivers/accel/amdxdna/amdxdna_mailbox.c index 858df97cd3fbd4..469242ed82246d 100644 --- a/drivers/accel/amdxdna/amdxdna_mailbox.c +++ b/drivers/accel/amdxdna/amdxdna_mailbox.c @@ -112,22 +112,6 @@ static u32 mailbox_reg_read(struct mailbox_channel *mb_chann, u32 mbox_reg) return readl(ringbuf_addr); } -static int mailbox_reg_read_non_zero(struct mailbox_channel *mb_chann, u32 mbox_reg, u32 *val) -{ - struct xdna_mailbox_res *mb_res = &mb_chann->mb->res; - void __iomem *ringbuf_addr = mb_res->mbox_base + mbox_reg; - int ret, value; - - /* Poll till value is not zero */ - ret = readx_poll_timeout(readl, ringbuf_addr, value, - value, 1 /* us */, 100); - if (ret < 0) - return ret; - - *val = value; - return 0; -} - static inline void mailbox_set_headptr(struct mailbox_channel *mb_chann, u32 headptr_val) { @@ -207,26 +191,34 @@ mailbox_send_msg(struct mailbox_channel *mb_chann, struct mailbox_msg *mb_msg) u32 head, tail; u32 start_addr; u32 tmp_tail; + int ret; head = mailbox_get_headptr(mb_chann, CHAN_RES_X2I); tail = mb_chann->x2i_tail; - ringbuf_size = mailbox_get_ringbuf_size(mb_chann, CHAN_RES_X2I); + ringbuf_size = mailbox_get_ringbuf_size(mb_chann, CHAN_RES_X2I) - sizeof(u32); start_addr = mb_chann->res[CHAN_RES_X2I].rb_start_addr; tmp_tail = tail + mb_msg->pkg_size; - if (tail < head && tmp_tail >= head) - goto no_space; - if (tail >= head && (tmp_tail > ringbuf_size - sizeof(u32) && - mb_msg->pkg_size >= head)) - goto no_space; - - if (tail >= head && tmp_tail > ringbuf_size - sizeof(u32)) { +check_again: + if (tail >= head && tmp_tail > ringbuf_size) { write_addr = mb_chann->mb->res.ringbuf_base + start_addr + tail; writel(TOMBSTONE, write_addr); /* tombstone is set. Write from the start of the ringbuf */ tail = 0; + tmp_tail = tail + mb_msg->pkg_size; + } + + if (tail < head && tmp_tail >= head) { + ret = read_poll_timeout(mailbox_get_headptr, head, + tmp_tail < head || tail >= head, + 1, 100, false, mb_chann, CHAN_RES_X2I); + if (ret) + return ret; + + if (tail >= head) + goto check_again; } write_addr = mb_chann->mb->res.ringbuf_base + start_addr + tail; @@ -238,9 +230,6 @@ mailbox_send_msg(struct mailbox_channel *mb_chann, struct mailbox_msg *mb_msg) mb_msg->pkg.header.id); return 0; - -no_space: - return -ENOSPC; } static int @@ -286,8 +275,7 @@ static int mailbox_get_msg(struct mailbox_channel *mb_chann) u32 start_addr; int ret; - if (mailbox_reg_read_non_zero(mb_chann, mb_chann->res[CHAN_RES_I2X].mb_tail_ptr_reg, &tail)) - return -EINVAL; + tail = mailbox_get_tailptr(mb_chann, CHAN_RES_I2X); head = mb_chann->i2x_head; ringbuf_size = mailbox_get_ringbuf_size(mb_chann, CHAN_RES_I2X); start_addr = mb_chann->res[CHAN_RES_I2X].rb_start_addr; diff --git a/drivers/accel/amdxdna/amdxdna_pci_drv.c b/drivers/accel/amdxdna/amdxdna_pci_drv.c index 1973ab67721be0..bcb0d77b63cbb5 100644 --- a/drivers/accel/amdxdna/amdxdna_pci_drv.c +++ b/drivers/accel/amdxdna/amdxdna_pci_drv.c @@ -83,6 +83,8 @@ static int amdxdna_drm_open(struct drm_device *ddev, struct drm_file *filp) ret = -ENODEV; goto unbind_sva; } + client->mm = current->mm; + mmgrab(client->mm); init_srcu_struct(&client->hwctx_srcu); xa_init_flags(&client->hwctx_xa, XA_FLAGS_ALLOC); mutex_init(&client->mm_lock); @@ -119,6 +121,7 @@ static void amdxdna_drm_close(struct drm_device *ddev, struct drm_file *filp) drm_gem_object_put(to_gobj(client->dev_heap)); iommu_sva_unbind_device(client->sva); + mmdrop(client->mm); XDNA_DBG(xdna, "pid %d closed", client->pid); kfree(client); @@ -282,7 +285,7 @@ static int amdxdna_probe(struct pci_dev *pdev, const struct pci_device_id *id) fs_reclaim_release(GFP_KERNEL); } - xdna->notifier_wq = alloc_ordered_workqueue("notifier_wq", 0); + xdna->notifier_wq = alloc_ordered_workqueue("notifier_wq", WQ_MEM_RECLAIM); if (!xdna->notifier_wq) return -ENOMEM; diff --git a/drivers/accel/amdxdna/amdxdna_pci_drv.h b/drivers/accel/amdxdna/amdxdna_pci_drv.h index c99477f5e454f0..ec21cb378a472a 100644 --- a/drivers/accel/amdxdna/amdxdna_pci_drv.h +++ b/drivers/accel/amdxdna/amdxdna_pci_drv.h @@ -101,7 +101,6 @@ struct amdxdna_dev { struct amdxdna_fw_ver fw_ver; struct rw_semaphore notifier_lock; /* for mmu notifier*/ struct workqueue_struct *notifier_wq; - bool rpm_on; }; /* @@ -131,6 +130,7 @@ struct amdxdna_client { struct iommu_sva *sva; int pasid; + struct mm_struct *mm; }; #define amdxdna_for_each_hwctx(client, hwctx_id, entry) \ diff --git a/drivers/accel/amdxdna/amdxdna_pm.c b/drivers/accel/amdxdna/amdxdna_pm.c index fa38e65d617c4b..b1fafddd7ad597 100644 --- a/drivers/accel/amdxdna/amdxdna_pm.c +++ b/drivers/accel/amdxdna/amdxdna_pm.c @@ -15,14 +15,10 @@ int amdxdna_pm_suspend(struct device *dev) { struct amdxdna_dev *xdna = to_xdna_dev(dev_get_drvdata(dev)); int ret = -EOPNOTSUPP; - bool rpm; - if (xdna->dev_info->ops->suspend) { - rpm = xdna->rpm_on; - xdna->rpm_on = false; + guard(mutex)(&xdna->dev_lock); + if (xdna->dev_info->ops->suspend) ret = xdna->dev_info->ops->suspend(xdna); - xdna->rpm_on = rpm; - } XDNA_DBG(xdna, "Suspend done ret %d", ret); return ret; @@ -32,14 +28,10 @@ int amdxdna_pm_resume(struct device *dev) { struct amdxdna_dev *xdna = to_xdna_dev(dev_get_drvdata(dev)); int ret = -EOPNOTSUPP; - bool rpm; - if (xdna->dev_info->ops->resume) { - rpm = xdna->rpm_on; - xdna->rpm_on = false; + guard(mutex)(&xdna->dev_lock); + if (xdna->dev_info->ops->resume) ret = xdna->dev_info->ops->resume(xdna); - xdna->rpm_on = rpm; - } XDNA_DBG(xdna, "Resume done ret %d", ret); return ret; @@ -50,9 +42,6 @@ int amdxdna_pm_resume_get(struct amdxdna_dev *xdna) struct device *dev = xdna->ddev.dev; int ret; - if (!xdna->rpm_on) - return 0; - ret = pm_runtime_resume_and_get(dev); if (ret) { XDNA_ERR(xdna, "Resume failed: %d", ret); @@ -66,9 +55,6 @@ void amdxdna_pm_suspend_put(struct amdxdna_dev *xdna) { struct device *dev = xdna->ddev.dev; - if (!xdna->rpm_on) - return; - pm_runtime_put_autosuspend(dev); } @@ -81,14 +67,12 @@ void amdxdna_pm_init(struct amdxdna_dev *xdna) pm_runtime_use_autosuspend(dev); pm_runtime_allow(dev); pm_runtime_put_autosuspend(dev); - xdna->rpm_on = true; } void amdxdna_pm_fini(struct amdxdna_dev *xdna) { struct device *dev = xdna->ddev.dev; - xdna->rpm_on = false; pm_runtime_get_noresume(dev); pm_runtime_forbid(dev); } diff --git a/drivers/accel/amdxdna/amdxdna_pm.h b/drivers/accel/amdxdna/amdxdna_pm.h index 77b2d6e4557001..3d26b973e0e36b 100644 --- a/drivers/accel/amdxdna/amdxdna_pm.h +++ b/drivers/accel/amdxdna/amdxdna_pm.h @@ -15,4 +15,15 @@ void amdxdna_pm_suspend_put(struct amdxdna_dev *xdna); void amdxdna_pm_init(struct amdxdna_dev *xdna); void amdxdna_pm_fini(struct amdxdna_dev *xdna); +static inline int amdxdna_pm_resume_get_locked(struct amdxdna_dev *xdna) +{ + int ret; + + mutex_unlock(&xdna->dev_lock); + ret = amdxdna_pm_resume_get(xdna); + mutex_lock(&xdna->dev_lock); + + return ret; +} + #endif /* _AMDXDNA_PM_H_ */ diff --git a/drivers/accel/amdxdna/amdxdna_ubuf.c b/drivers/accel/amdxdna/amdxdna_ubuf.c index 077b2261cf2a04..62a478f6b45fb2 100644 --- a/drivers/accel/amdxdna/amdxdna_ubuf.c +++ b/drivers/accel/amdxdna/amdxdna_ubuf.c @@ -7,6 +7,7 @@ #include #include #include +#include #include #include @@ -34,15 +35,21 @@ static struct sg_table *amdxdna_ubuf_map(struct dma_buf_attachment *attach, ret = sg_alloc_table_from_pages(sg, ubuf->pages, ubuf->nr_pages, 0, ubuf->nr_pages << PAGE_SHIFT, GFP_KERNEL); if (ret) - return ERR_PTR(ret); + goto err_free_sg; if (ubuf->flags & AMDXDNA_UBUF_FLAG_MAP_DMA) { ret = dma_map_sgtable(attach->dev, sg, direction, 0); if (ret) - return ERR_PTR(ret); + goto err_free_table; } return sg; + +err_free_table: + sg_free_table(sg); +err_free_sg: + kfree(sg); + return ERR_PTR(ret); } static void amdxdna_ubuf_unmap(struct dma_buf_attachment *attach, @@ -170,7 +177,10 @@ struct dma_buf *amdxdna_get_ubuf(struct drm_device *dev, goto free_ent; } - exp_info.size += va_ent[i].len; + if (check_add_overflow(exp_info.size, va_ent[i].len, &exp_info.size)) { + ret = -EINVAL; + goto free_ent; + } } ubuf->nr_pages = exp_info.size >> PAGE_SHIFT; diff --git a/drivers/accel/amdxdna/npu4_regs.c b/drivers/accel/amdxdna/npu4_regs.c index 986a5f28ba2452..2ceedfe583a8c9 100644 --- a/drivers/accel/amdxdna/npu4_regs.c +++ b/drivers/accel/amdxdna/npu4_regs.c @@ -89,6 +89,7 @@ const struct dpm_clk_freq npu4_dpm_clk_table[] = { const struct aie2_fw_feature_tbl npu4_fw_feature_table[] = { { .feature = AIE2_NPU_COMMAND, .min_minor = 15 }, { .feature = AIE2_PREEMPT, .min_minor = 12 }, + { .feature = AIE2_TEMPORAL_ONLY, .min_minor = 12 }, { 0 } }; diff --git a/drivers/accel/ethosu/ethosu_gem.c b/drivers/accel/ethosu/ethosu_gem.c index 473b5f5d751440..4e84481a29d2f3 100644 --- a/drivers/accel/ethosu/ethosu_gem.c +++ b/drivers/accel/ethosu/ethosu_gem.c @@ -154,7 +154,7 @@ static void cmd_state_init(struct cmd_state *st) static u64 cmd_to_addr(u32 *cmd) { - return ((u64)((cmd[0] & 0xff0000) << 16)) | cmd[1]; + return (((u64)cmd[0] & 0xff0000) << 16) | cmd[1]; } static u64 dma_length(struct ethosu_validated_cmdstream_info *info, @@ -417,7 +417,10 @@ static int ethosu_gem_cmdstream_copy_and_validate(struct drm_device *ddev, return ret; break; case NPU_OP_ELEMENTWISE: - use_ifm2 = !((st.ifm2.broadcast == 8) || (param == 5) || + use_scale = ethosu_is_u65(edev) ? + (st.ifm2.broadcast & 0x80) : + (st.ifm2.broadcast == 8); + use_ifm2 = !(use_scale || (param == 5) || (param == 6) || (param == 7) || (param == 0x24)); use_ifm = st.ifm.broadcast != 8; ret = calc_sizes_elemwise(ddev, info, cmd, &st, use_ifm, use_ifm2); diff --git a/drivers/accel/ethosu/ethosu_job.c b/drivers/accel/ethosu/ethosu_job.c index 26e7a2f64d71ae..70a144803b0966 100644 --- a/drivers/accel/ethosu/ethosu_job.c +++ b/drivers/accel/ethosu/ethosu_job.c @@ -143,23 +143,29 @@ static int ethosu_job_push(struct ethosu_job *job) return ret; } +static void ethosu_job_err_cleanup(struct ethosu_job *job) +{ + unsigned int i; + + for (i = 0; i < job->region_cnt; i++) + drm_gem_object_put(job->region_bo[i]); + + drm_gem_object_put(job->cmd_bo); + + kfree(job); +} + static void ethosu_job_cleanup(struct kref *ref) { struct ethosu_job *job = container_of(ref, struct ethosu_job, refcount); - unsigned int i; pm_runtime_put_autosuspend(job->dev->base.dev); dma_fence_put(job->done_fence); dma_fence_put(job->inference_done_fence); - for (i = 0; i < job->region_cnt; i++) - drm_gem_object_put(job->region_bo[i]); - - drm_gem_object_put(job->cmd_bo); - - kfree(job); + ethosu_job_err_cleanup(job); } static void ethosu_job_put(struct ethosu_job *job) @@ -454,12 +460,16 @@ static int ethosu_ioctl_submit_job(struct drm_device *dev, struct drm_file *file } } ret = ethosu_job_push(ejob); + if (!ret) { + ethosu_job_put(ejob); + return 0; + } out_cleanup_job: if (ret) drm_sched_job_cleanup(&ejob->base); out_put_job: - ethosu_job_put(ejob); + ethosu_job_err_cleanup(ejob); return ret; } diff --git a/drivers/accel/ivpu/ivpu_drv.h b/drivers/accel/ivpu/ivpu_drv.h index 5b34b6f50e696a..f1b6155065ff3a 100644 --- a/drivers/accel/ivpu/ivpu_drv.h +++ b/drivers/accel/ivpu/ivpu_drv.h @@ -35,6 +35,7 @@ #define IVPU_HW_IP_60XX 60 #define IVPU_HW_IP_REV_LNL_B0 4 +#define IVPU_HW_IP_REV_NVL_A0 0 #define IVPU_HW_BTRS_MTL 1 #define IVPU_HW_BTRS_LNL 2 diff --git a/drivers/accel/ivpu/ivpu_hw.c b/drivers/accel/ivpu/ivpu_hw.c index d69cd0d935694a..d4a9bcda4100fb 100644 --- a/drivers/accel/ivpu/ivpu_hw.c +++ b/drivers/accel/ivpu/ivpu_hw.c @@ -70,8 +70,10 @@ static void wa_init(struct ivpu_device *vdev) if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_MTL) vdev->wa.interrupt_clear_with_0 = ivpu_hw_btrs_irqs_clear_with_0_mtl(vdev); - if (ivpu_device_id(vdev) == PCI_DEVICE_ID_LNL && - ivpu_revision(vdev) < IVPU_HW_IP_REV_LNL_B0) + if ((ivpu_device_id(vdev) == PCI_DEVICE_ID_LNL && + ivpu_revision(vdev) < IVPU_HW_IP_REV_LNL_B0) || + (ivpu_device_id(vdev) == PCI_DEVICE_ID_NVL && + ivpu_revision(vdev) == IVPU_HW_IP_REV_NVL_A0)) vdev->wa.disable_clock_relinquish = true; if (ivpu_test_mode & IVPU_TEST_MODE_CLK_RELINQ_ENABLE) diff --git a/drivers/accel/qaic/mhi_controller.c b/drivers/accel/qaic/mhi_controller.c index 13a14c6c61689f..4d787f77ce419f 100644 --- a/drivers/accel/qaic/mhi_controller.c +++ b/drivers/accel/qaic/mhi_controller.c @@ -39,7 +39,6 @@ static const struct mhi_channel_config aic100_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -55,7 +54,6 @@ static const struct mhi_channel_config aic100_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -71,7 +69,6 @@ static const struct mhi_channel_config aic100_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -87,7 +84,6 @@ static const struct mhi_channel_config aic100_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -103,7 +99,6 @@ static const struct mhi_channel_config aic100_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -119,7 +114,6 @@ static const struct mhi_channel_config aic100_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -135,7 +129,6 @@ static const struct mhi_channel_config aic100_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -151,7 +144,6 @@ static const struct mhi_channel_config aic100_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -167,7 +159,6 @@ static const struct mhi_channel_config aic100_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -183,7 +174,6 @@ static const struct mhi_channel_config aic100_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -199,7 +189,6 @@ static const struct mhi_channel_config aic100_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -215,7 +204,6 @@ static const struct mhi_channel_config aic100_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -231,7 +219,6 @@ static const struct mhi_channel_config aic100_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -247,7 +234,6 @@ static const struct mhi_channel_config aic100_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -263,7 +249,6 @@ static const struct mhi_channel_config aic100_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -279,7 +264,6 @@ static const struct mhi_channel_config aic100_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -295,7 +279,6 @@ static const struct mhi_channel_config aic100_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -311,7 +294,6 @@ static const struct mhi_channel_config aic100_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -327,7 +309,6 @@ static const struct mhi_channel_config aic100_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -343,7 +324,6 @@ static const struct mhi_channel_config aic100_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -359,7 +339,6 @@ static const struct mhi_channel_config aic100_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -375,7 +354,6 @@ static const struct mhi_channel_config aic100_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -391,7 +369,6 @@ static const struct mhi_channel_config aic100_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -407,7 +384,6 @@ static const struct mhi_channel_config aic100_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -423,7 +399,6 @@ static const struct mhi_channel_config aic100_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -439,7 +414,6 @@ static const struct mhi_channel_config aic100_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = true, .wake_capable = false, }, }; @@ -458,7 +432,6 @@ static const struct mhi_channel_config aic200_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -474,7 +447,6 @@ static const struct mhi_channel_config aic200_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -490,7 +462,6 @@ static const struct mhi_channel_config aic200_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -506,7 +477,6 @@ static const struct mhi_channel_config aic200_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -522,7 +492,6 @@ static const struct mhi_channel_config aic200_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -538,7 +507,6 @@ static const struct mhi_channel_config aic200_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -554,7 +522,6 @@ static const struct mhi_channel_config aic200_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -570,7 +537,6 @@ static const struct mhi_channel_config aic200_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -586,7 +552,6 @@ static const struct mhi_channel_config aic200_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -602,7 +567,6 @@ static const struct mhi_channel_config aic200_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -618,7 +582,6 @@ static const struct mhi_channel_config aic200_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -634,7 +597,6 @@ static const struct mhi_channel_config aic200_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -650,7 +612,6 @@ static const struct mhi_channel_config aic200_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -666,7 +627,6 @@ static const struct mhi_channel_config aic200_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -682,7 +642,6 @@ static const struct mhi_channel_config aic200_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -698,7 +657,6 @@ static const struct mhi_channel_config aic200_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -714,7 +672,6 @@ static const struct mhi_channel_config aic200_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -730,7 +687,6 @@ static const struct mhi_channel_config aic200_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = true, .wake_capable = false, }, }; diff --git a/drivers/accel/qaic/qaic_control.c b/drivers/accel/qaic/qaic_control.c index 428d8f65bff363..3842e59291b932 100644 --- a/drivers/accel/qaic/qaic_control.c +++ b/drivers/accel/qaic/qaic_control.c @@ -913,7 +913,7 @@ static int decode_deactivate(struct qaic_device *qdev, void *trans, u32 *msg_len */ return -ENODEV; - if (status) { + if (usr && status) { /* * Releasing resources failed on the device side, which puts * us in a bind since they may still be in use, so enable the @@ -1108,6 +1108,9 @@ static void *msg_xfer(struct qaic_device *qdev, struct wrapper_list *wrappers, u mutex_lock(&qdev->cntl_mutex); if (!list_empty(&elem.list)) list_del(&elem.list); + /* resp_worker() processed the response but the wait was interrupted */ + else if (ret == -ERESTARTSYS) + ret = 0; if (!ret && !elem.buf) ret = -ETIMEDOUT; else if (ret > 0 && !elem.buf) @@ -1418,9 +1421,49 @@ static void resp_worker(struct work_struct *work) } mutex_unlock(&qdev->cntl_mutex); - if (!found) + if (!found) { + /* + * The user might have gone away at this point without waiting + * for QAIC_TRANS_DEACTIVATE_FROM_DEV transaction coming from + * the device. If this is not handled correctly, the host will + * not know that the DBC[n] has been freed on the device. + * Due to this failure in synchronization between the device and + * the host, if another user requests to activate a network, and + * the device assigns DBC[n] again, save_dbc_buf() will hang, + * waiting for dbc[n]->in_use to be set to false, which will not + * happen unless the qaic_dev_reset_clean_local_state() gets + * called by resetting the device (or re-inserting the module). + * + * As a solution, we look for QAIC_TRANS_DEACTIVATE_FROM_DEV + * transactions in the message before disposing of it, then + * handle releasing the DBC resources. + * + * Since the user has gone away, if the device could not + * deactivate the network (status != 0), there is no way to + * enable and reassign the DBC to the user. We can put trust in + * the device that it will release all the active DBCs in + * response to the QAIC_TRANS_TERMINATE_TO_DEV transaction, + * otherwise, the user can issue an soc_reset to the device. + */ + u32 msg_count = le32_to_cpu(msg->hdr.count); + u32 msg_len = le32_to_cpu(msg->hdr.len); + u32 len = 0; + int j; + + for (j = 0; j < msg_count && len < msg_len; ++j) { + struct wire_trans_hdr *trans_hdr; + + trans_hdr = (struct wire_trans_hdr *)(msg->data + len); + if (le32_to_cpu(trans_hdr->type) == QAIC_TRANS_DEACTIVATE_FROM_DEV) { + if (decode_deactivate(qdev, trans_hdr, &len, NULL)) + len += le32_to_cpu(trans_hdr->len); + } else { + len += le32_to_cpu(trans_hdr->len); + } + } /* request must have timed out, drop packet */ kfree(msg); + } kfree(resp); } diff --git a/drivers/accel/rocket/rocket_core.c b/drivers/accel/rocket/rocket_core.c index abe7719c1db468..b3b2fa9ba645a6 100644 --- a/drivers/accel/rocket/rocket_core.c +++ b/drivers/accel/rocket/rocket_core.c @@ -59,8 +59,11 @@ int rocket_core_init(struct rocket_core *core) core->iommu_group = iommu_group_get(dev); err = rocket_job_init(core); - if (err) + if (err) { + iommu_group_put(core->iommu_group); + core->iommu_group = NULL; return err; + } pm_runtime_use_autosuspend(dev); @@ -76,7 +79,7 @@ int rocket_core_init(struct rocket_core *core) err = pm_runtime_resume_and_get(dev); if (err) { - rocket_job_fini(core); + rocket_core_fini(core); return err; } diff --git a/drivers/accel/rocket/rocket_drv.c b/drivers/accel/rocket/rocket_drv.c index 5c0b63f0a8f00d..f6ef4c7aeef11d 100644 --- a/drivers/accel/rocket/rocket_drv.c +++ b/drivers/accel/rocket/rocket_drv.c @@ -13,6 +13,7 @@ #include #include +#include "rocket_device.h" #include "rocket_drv.h" #include "rocket_gem.h" #include "rocket_job.h" @@ -158,6 +159,8 @@ static const struct drm_driver rocket_drm_driver = { static int rocket_probe(struct platform_device *pdev) { + int ret; + if (rdev == NULL) { /* First core probing, initialize DRM device. */ rdev = rocket_device_init(drm_dev, &rocket_drm_driver); @@ -177,7 +180,17 @@ static int rocket_probe(struct platform_device *pdev) rdev->num_cores++; - return rocket_core_init(&rdev->cores[core]); + ret = rocket_core_init(&rdev->cores[core]); + if (ret) { + rdev->num_cores--; + + if (rdev->num_cores == 0) { + rocket_device_fini(rdev); + rdev = NULL; + } + } + + return ret; } static void rocket_remove(struct platform_device *pdev) diff --git a/drivers/acpi/acpi_processor.c b/drivers/acpi/acpi_processor.c index 7ec1dc04fd11b9..5a562e27d3a80c 100644 --- a/drivers/acpi/acpi_processor.c +++ b/drivers/acpi/acpi_processor.c @@ -50,6 +50,7 @@ static int acpi_processor_errata_piix4(struct pci_dev *dev) { u8 value1 = 0; u8 value2 = 0; + struct pci_dev *ide_dev = NULL, *isa_dev = NULL; if (!dev) @@ -107,12 +108,16 @@ static int acpi_processor_errata_piix4(struct pci_dev *dev) * each IDE controller's DMA status to make sure we catch all * DMA activity. */ - dev = pci_get_subsys(PCI_VENDOR_ID_INTEL, + ide_dev = pci_get_subsys(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82371AB, PCI_ANY_ID, PCI_ANY_ID, NULL); - if (dev) { - errata.piix4.bmisx = pci_resource_start(dev, 4); - pci_dev_put(dev); + if (ide_dev) { + errata.piix4.bmisx = pci_resource_start(ide_dev, 4); + if (errata.piix4.bmisx) + dev_dbg(&ide_dev->dev, + "Bus master activity detection (BM-IDE) erratum enabled\n"); + + pci_dev_put(ide_dev); } /* @@ -124,25 +129,23 @@ static int acpi_processor_errata_piix4(struct pci_dev *dev) * disable C3 support if this is enabled, as some legacy * devices won't operate well if fast DMA is disabled. */ - dev = pci_get_subsys(PCI_VENDOR_ID_INTEL, + isa_dev = pci_get_subsys(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82371AB_0, PCI_ANY_ID, PCI_ANY_ID, NULL); - if (dev) { - pci_read_config_byte(dev, 0x76, &value1); - pci_read_config_byte(dev, 0x77, &value2); - if ((value1 & 0x80) || (value2 & 0x80)) + if (isa_dev) { + pci_read_config_byte(isa_dev, 0x76, &value1); + pci_read_config_byte(isa_dev, 0x77, &value2); + if ((value1 & 0x80) || (value2 & 0x80)) { errata.piix4.fdma = 1; - pci_dev_put(dev); + dev_dbg(&isa_dev->dev, + "Type-F DMA livelock erratum (C3 disabled)\n"); + } + pci_dev_put(isa_dev); } break; } - if (errata.piix4.bmisx) - dev_dbg(&dev->dev, "Bus master activity detection (BM-IDE) erratum enabled\n"); - if (errata.piix4.fdma) - dev_dbg(&dev->dev, "Type-F DMA livelock erratum (C3 disabled)\n"); - return 0; } diff --git a/drivers/acpi/acpica/acpredef.h b/drivers/acpi/acpica/acpredef.h index da2c45880cc7e9..c9e65c6a206909 100644 --- a/drivers/acpi/acpica/acpredef.h +++ b/drivers/acpi/acpica/acpredef.h @@ -450,7 +450,7 @@ const union acpi_predefined_info acpi_gbl_predefined_methods[] = { {{"_DSM", METHOD_4ARGS(ACPI_TYPE_BUFFER, ACPI_TYPE_INTEGER, ACPI_TYPE_INTEGER, - ACPI_TYPE_ANY | ACPI_TYPE_PACKAGE) | + ACPI_TYPE_PACKAGE | ACPI_TYPE_ANY) | ARG_COUNT_IS_MINIMUM, METHOD_RETURNS(ACPI_RTYPE_ALL)}}, /* Must return a value, but it can be of any type */ diff --git a/drivers/acpi/acpica/evregion.c b/drivers/acpi/acpica/evregion.c index fa3475da7ea9b6..b6198f73c81dfa 100644 --- a/drivers/acpi/acpica/evregion.c +++ b/drivers/acpi/acpica/evregion.c @@ -163,7 +163,9 @@ acpi_ev_address_space_dispatch(union acpi_operand_object *region_obj, return_ACPI_STATUS(AE_NOT_EXIST); } - if (region_obj->region.space_id == ACPI_ADR_SPACE_PLATFORM_COMM) { + if (field_obj + && region_obj->region.space_id == + ACPI_ADR_SPACE_PLATFORM_COMM) { struct acpi_pcc_info *ctx = handler_desc->address_space.context; diff --git a/drivers/acpi/acpica/exoparg3.c b/drivers/acpi/acpica/exoparg3.c index bf08110ed6d257..c8c8c4e49563ef 100644 --- a/drivers/acpi/acpica/exoparg3.c +++ b/drivers/acpi/acpica/exoparg3.c @@ -10,6 +10,7 @@ #include #include "accommon.h" #include "acinterp.h" +#include #include "acparser.h" #include "amlcode.h" @@ -51,8 +52,7 @@ ACPI_MODULE_NAME("exoparg3") acpi_status acpi_ex_opcode_3A_0T_0R(struct acpi_walk_state *walk_state) { union acpi_operand_object **operand = &walk_state->operands[0]; - struct acpi_signal_fatal_info *fatal; - acpi_status status = AE_OK; + struct acpi_signal_fatal_info fatal; ACPI_FUNCTION_TRACE_STR(ex_opcode_3A_0T_0R, acpi_ps_get_opcode_name(walk_state->opcode)); @@ -60,28 +60,23 @@ acpi_status acpi_ex_opcode_3A_0T_0R(struct acpi_walk_state *walk_state) switch (walk_state->opcode) { case AML_FATAL_OP: /* Fatal (fatal_type fatal_code fatal_arg) */ - ACPI_DEBUG_PRINT((ACPI_DB_INFO, - "FatalOp: Type %X Code %X Arg %X " - "<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<\n", - (u32)operand[0]->integer.value, - (u32)operand[1]->integer.value, - (u32)operand[2]->integer.value)); - - fatal = ACPI_ALLOCATE(sizeof(struct acpi_signal_fatal_info)); - if (fatal) { - fatal->type = (u32) operand[0]->integer.value; - fatal->code = (u32) operand[1]->integer.value; - fatal->argument = (u32) operand[2]->integer.value; - } + fatal.type = (u32)operand[0]->integer.value; + fatal.code = (u32)operand[1]->integer.value; + fatal.argument = (u32)operand[2]->integer.value; - /* Always signal the OS! */ + ACPI_BIOS_ERROR((AE_INFO, + "Fatal ACPI BIOS error (Type 0x%X Code 0x%X Arg 0x%X)\n", + fatal.type, fatal.code, fatal.argument)); - status = acpi_os_signal(ACPI_SIGNAL_FATAL, fatal); + /* Always signal the OS! */ - /* Might return while OS is shutting down, just continue */ + acpi_os_signal(ACPI_SIGNAL_FATAL, &fatal); - ACPI_FREE(fatal); - goto cleanup; + /* + * Might return while OS is shutting down, so abort the AML execution + * by returning an error. + */ + return_ACPI_STATUS(AE_ERROR); case AML_EXTERNAL_OP: /* @@ -93,21 +88,16 @@ acpi_status acpi_ex_opcode_3A_0T_0R(struct acpi_walk_state *walk_state) * wrong if an external opcode ever gets here. */ ACPI_ERROR((AE_INFO, "Executed External Op")); - status = AE_OK; - goto cleanup; + + return_ACPI_STATUS(AE_OK); default: ACPI_ERROR((AE_INFO, "Unknown AML opcode 0x%X", walk_state->opcode)); - status = AE_AML_BAD_OPCODE; - goto cleanup; + return_ACPI_STATUS(AE_AML_BAD_OPCODE); } - -cleanup: - - return_ACPI_STATUS(status); } /******************************************************************************* diff --git a/drivers/acpi/apei/Makefile b/drivers/acpi/apei/Makefile index 2c474e6477e12a..1a0b85923cd425 100644 --- a/drivers/acpi/apei/Makefile +++ b/drivers/acpi/apei/Makefile @@ -1,6 +1,11 @@ # SPDX-License-Identifier: GPL-2.0 obj-$(CONFIG_ACPI_APEI) += apei.o obj-$(CONFIG_ACPI_APEI_GHES) += ghes.o +# clang versions prior to 18 may blow out the stack with KASAN +ifeq ($(CONFIG_COMPILE_TEST)_$(CONFIG_CC_IS_CLANG)_$(call clang-min-version, 180000),y_y_) +KASAN_SANITIZE_ghes.o := n +endif +obj-$(CONFIG_ACPI_APEI_PCIEAER) += ghes_helpers.o obj-$(CONFIG_ACPI_APEI_EINJ) += einj.o einj-y := einj-core.o einj-$(CONFIG_ACPI_APEI_EINJ_CXL) += einj-cxl.o diff --git a/drivers/acpi/apei/ghes.c b/drivers/acpi/apei/ghes.c index 0dc767392a6c6e..9919c31e42c074 100644 --- a/drivers/acpi/apei/ghes.c +++ b/drivers/acpi/apei/ghes.c @@ -29,6 +29,7 @@ #include #include #include +#include #include #include #include @@ -294,6 +295,7 @@ static struct ghes *ghes_new(struct acpi_hest_generic *generic) error_block_length = GHES_ESTATUS_MAX_SIZE; } ghes->estatus = kmalloc(error_block_length, GFP_KERNEL); + ghes->estatus_length = error_block_length; if (!ghes->estatus) { rc = -ENOMEM; goto err_unmap_status_addr; @@ -365,13 +367,15 @@ static int __ghes_check_estatus(struct ghes *ghes, struct acpi_hest_generic_status *estatus) { u32 len = cper_estatus_len(estatus); + u32 max_len = min(ghes->generic->error_block_length, + ghes->estatus_length); if (len < sizeof(*estatus)) { pr_warn_ratelimited(FW_WARN GHES_PFX "Truncated error status block!\n"); return -EIO; } - if (len > ghes->generic->error_block_length) { + if (!len || len > max_len) { pr_warn_ratelimited(FW_WARN GHES_PFX "Invalid error status block length!\n"); return -EIO; } @@ -552,21 +556,45 @@ static bool ghes_handle_arm_hw_error(struct acpi_hest_generic_data *gdata, { struct cper_sec_proc_arm *err = acpi_hest_get_payload(gdata); int flags = sync ? MF_ACTION_REQUIRED : 0; + int length = gdata->error_data_length; char error_type[120]; bool queued = false; int sec_sev, i; char *p; sec_sev = ghes_severity(gdata->error_severity); - log_arm_hw_error(err, sec_sev); + if (length >= sizeof(*err)) { + log_arm_hw_error(err, sec_sev); + } else { + pr_warn(FW_BUG "arm error length: %d\n", length); + pr_warn(FW_BUG "length is too small\n"); + pr_warn(FW_BUG "firmware-generated error record is incorrect\n"); + return false; + } + if (sev != GHES_SEV_RECOVERABLE || sec_sev != GHES_SEV_RECOVERABLE) return false; p = (char *)(err + 1); + length -= sizeof(err); + for (i = 0; i < err->err_info_num; i++) { - struct cper_arm_err_info *err_info = (struct cper_arm_err_info *)p; - bool is_cache = err_info->type & CPER_ARM_CACHE_ERROR; - bool has_pa = (err_info->validation_bits & CPER_ARM_INFO_VALID_PHYSICAL_ADDR); + struct cper_arm_err_info *err_info; + bool is_cache, has_pa; + + /* Ensure we have enough data for the error info header */ + if (length < sizeof(*err_info)) + break; + + err_info = (struct cper_arm_err_info *)p; + + /* Validate the claimed length before using it */ + length -= err_info->length; + if (length < 0) + break; + + is_cache = err_info->type & CPER_ARM_CACHE_ERROR; + has_pa = (err_info->validation_bits & CPER_ARM_INFO_VALID_PHYSICAL_ADDR); /* * The field (err_info->error_info & BIT(26)) is fixed to set to @@ -713,24 +741,8 @@ static void cxl_cper_post_prot_err(struct cxl_cper_sec_prot_err *prot_err, struct cxl_cper_prot_err_work_data wd; u8 *dvsec_start, *cap_start; - if (!(prot_err->valid_bits & PROT_ERR_VALID_AGENT_ADDRESS)) { - pr_err_ratelimited("CXL CPER invalid agent type\n"); + if (cxl_cper_sec_prot_err_valid(prot_err)) return; - } - - if (!(prot_err->valid_bits & PROT_ERR_VALID_ERROR_LOG)) { - pr_err_ratelimited("CXL CPER invalid protocol error log\n"); - return; - } - - if (prot_err->err_len != sizeof(struct cxl_ras_capability_regs)) { - pr_err_ratelimited("CXL CPER invalid RAS Cap size (%u)\n", - prot_err->err_len); - return; - } - - if (!(prot_err->valid_bits & PROT_ERR_VALID_SERIAL_NUMBER)) - pr_warn(FW_WARN "CXL CPER no device serial number\n"); guard(spinlock_irqsave)(&cxl_cper_prot_err_work_lock); diff --git a/drivers/acpi/apei/ghes_helpers.c b/drivers/acpi/apei/ghes_helpers.c new file mode 100644 index 00000000000000..f3d162139a974d --- /dev/null +++ b/drivers/acpi/apei/ghes_helpers.c @@ -0,0 +1,33 @@ +// SPDX-License-Identifier: GPL-2.0-only +// Copyright(c) 2025 Intel Corporation. All rights reserved + +#include +#include + +int cxl_cper_sec_prot_err_valid(struct cxl_cper_sec_prot_err *prot_err) +{ + if (!(prot_err->valid_bits & PROT_ERR_VALID_AGENT_ADDRESS)) { + pr_err_ratelimited("CXL CPER invalid agent type\n"); + return -EINVAL; + } + + if (!(prot_err->valid_bits & PROT_ERR_VALID_ERROR_LOG)) { + pr_err_ratelimited("CXL CPER invalid protocol error log\n"); + return -EINVAL; + } + + if (prot_err->err_len != sizeof(struct cxl_ras_capability_regs)) { + pr_err_ratelimited("CXL CPER invalid RAS Cap size (%u)\n", + prot_err->err_len); + return -EINVAL; + } + + if ((prot_err->agent_type == RCD || prot_err->agent_type == DEVICE || + prot_err->agent_type == LD || prot_err->agent_type == FMLD) && + !(prot_err->valid_bits & PROT_ERR_VALID_SERIAL_NUMBER)) + pr_warn_ratelimited(FW_WARN + "CXL CPER no device serial number\n"); + + return 0; +} +EXPORT_SYMBOL_GPL(cxl_cper_sec_prot_err_valid); diff --git a/drivers/acpi/battery.c b/drivers/acpi/battery.c index 34181fa52e937e..4b28ef79e6ac8a 100644 --- a/drivers/acpi/battery.c +++ b/drivers/acpi/battery.c @@ -211,7 +211,14 @@ static int acpi_battery_get_property(struct power_supply *psy, if (battery->state & ACPI_BATTERY_STATE_DISCHARGING) val->intval = acpi_battery_handle_discharging(battery); else if (battery->state & ACPI_BATTERY_STATE_CHARGING) - val->intval = POWER_SUPPLY_STATUS_CHARGING; + /* Validate the status by checking the current. */ + if (battery->rate_now != ACPI_BATTERY_VALUE_UNKNOWN && + battery->rate_now == 0) { + /* On charge but no current (0W/0mA). */ + val->intval = POWER_SUPPLY_STATUS_NOT_CHARGING; + } else { + val->intval = POWER_SUPPLY_STATUS_CHARGING; + } else if (battery->state & ACPI_BATTERY_STATE_CHARGE_LIMITING) val->intval = POWER_SUPPLY_STATUS_NOT_CHARGING; else if (acpi_battery_is_charged(battery)) diff --git a/drivers/acpi/cppc_acpi.c b/drivers/acpi/cppc_acpi.c index e66e20d1f31b76..b59b0100d03c51 100644 --- a/drivers/acpi/cppc_acpi.c +++ b/drivers/acpi/cppc_acpi.c @@ -362,7 +362,7 @@ static int send_pcc_cmd(int pcc_ss_id, u16 cmd) end: if (cmd == CMD_WRITE) { if (unlikely(ret)) { - for_each_possible_cpu(i) { + for_each_online_cpu(i) { struct cpc_desc *desc = per_cpu(cpc_desc_ptr, i); if (!desc) @@ -524,7 +524,7 @@ int acpi_get_psd_map(unsigned int cpu, struct cppc_cpudata *cpu_data) else if (pdomain->coord_type == DOMAIN_COORD_TYPE_SW_ANY) cpu_data->shared_type = CPUFREQ_SHARED_TYPE_ANY; - for_each_possible_cpu(i) { + for_each_online_cpu(i) { if (i == cpu) continue; diff --git a/drivers/acpi/ec.c b/drivers/acpi/ec.c index 59b3d50ff01ecf..c981a53434edf3 100644 --- a/drivers/acpi/ec.c +++ b/drivers/acpi/ec.c @@ -1655,6 +1655,8 @@ static int acpi_ec_setup(struct acpi_ec *ec, struct acpi_device *device, bool ca ret = ec_install_handlers(ec, device, call_reg); if (ret) { + ec_remove_handlers(ec); + if (ec == first_ec) first_ec = NULL; diff --git a/drivers/acpi/osi.c b/drivers/acpi/osi.c index f2c943b934be0a..9470f1830ff50f 100644 --- a/drivers/acpi/osi.c +++ b/drivers/acpi/osi.c @@ -389,6 +389,19 @@ static const struct dmi_system_id acpi_osi_dmi_table[] __initconst = { }, }, + /* + * The screen backlight turns off during udev device creation + * when returning true for _OSI("Windows 2009") + */ + { + .callback = dmi_disable_osi_win7, + .ident = "Acer Aspire One D255", + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "Acer"), + DMI_MATCH(DMI_PRODUCT_NAME, "AOD255"), + }, + }, + /* * The wireless hotkey does not work on those machines when * returning true for _OSI("Windows 2012") diff --git a/drivers/acpi/osl.c b/drivers/acpi/osl.c index 05393a7315fecc..2addb40961b60e 100644 --- a/drivers/acpi/osl.c +++ b/drivers/acpi/osl.c @@ -1681,7 +1681,7 @@ acpi_status __init acpi_os_initialize(void) * Use acpi_os_map_generic_address to pre-map the reset * register if it's in system memory. */ - void *rv; + void __iomem *rv; rv = acpi_os_map_generic_address(&acpi_gbl_FADT.reset_register); pr_debug("%s: Reset register mapping %s\n", __func__, diff --git a/drivers/acpi/power.c b/drivers/acpi/power.c index 361a7721a6a873..7da5ae5594a727 100644 --- a/drivers/acpi/power.c +++ b/drivers/acpi/power.c @@ -1113,6 +1113,19 @@ static const struct dmi_system_id dmi_leave_unused_power_resources_on[] = { DMI_MATCH(DMI_PRODUCT_NAME, "SATELLITE Click Mini L9W-B"), }, }, + { + /* + * THUNDEROBOT ZERO laptop: Due to its SSDT table bug, power + * resource 'PXP' will be shut down on initialization, making + * the NVMe #2 and the NVIDIA dGPU both unavailable (they're + * both controlled by 'PXP'). + */ + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "THUNDEROBOT"), + DMI_MATCH(DMI_PRODUCT_NAME, "ZERO"), + } + + }, {} }; diff --git a/drivers/acpi/resource.c b/drivers/acpi/resource.c index d16906f46484d8..bc8050d8a6f51c 100644 --- a/drivers/acpi/resource.c +++ b/drivers/acpi/resource.c @@ -532,6 +532,12 @@ static const struct dmi_system_id irq1_level_low_skip_override[] = { DMI_MATCH(DMI_BOARD_NAME, "16T90SP"), }, }, + { + /* JWIPC JVC9100 */ + .matches = { + DMI_MATCH(DMI_BOARD_NAME, "JVC9100"), + }, + }, { } }; @@ -706,6 +712,8 @@ struct irq_override_cmp { static const struct irq_override_cmp override_table[] = { { irq1_level_low_skip_override, 1, ACPI_LEVEL_SENSITIVE, ACPI_ACTIVE_LOW, 0, false }, + { irq1_level_low_skip_override, 10, ACPI_LEVEL_SENSITIVE, ACPI_ACTIVE_LOW, 1, false }, + { irq1_level_low_skip_override, 11, ACPI_LEVEL_SENSITIVE, ACPI_ACTIVE_LOW, 1, false }, { irq1_edge_low_force_override, 1, ACPI_EDGE_SENSITIVE, ACPI_ACTIVE_LOW, 1, true }, }; diff --git a/drivers/acpi/riscv/rimt.c b/drivers/acpi/riscv/rimt.c index 7f423405e5ef0e..8eaa8731bddd6c 100644 --- a/drivers/acpi/riscv/rimt.c +++ b/drivers/acpi/riscv/rimt.c @@ -263,6 +263,13 @@ static int rimt_iommu_xlate(struct device *dev, struct acpi_rimt_node *node, u32 if (!rimt_fwnode) return -EPROBE_DEFER; + /* + * EPROBE_DEFER ensures IOMMU is probed before the devices that + * depend on them. During shutdown, however, the IOMMU may be removed + * first, leading to issues. To avoid this, a device link is added + * which enforces the correct removal order. + */ + device_link_add(dev, rimt_fwnode->dev, DL_FLAG_AUTOREMOVE_CONSUMER); return acpi_iommu_fwspec_init(dev, deviceid, rimt_fwnode); } diff --git a/drivers/acpi/scan.c b/drivers/acpi/scan.c index 416d87f9bd107b..b78f6be2f94689 100644 --- a/drivers/acpi/scan.c +++ b/drivers/acpi/scan.c @@ -5,6 +5,7 @@ #define pr_fmt(fmt) "ACPI: " fmt +#include #include #include #include @@ -2360,46 +2361,34 @@ static int acpi_dev_get_next_consumer_dev_cb(struct acpi_dep_data *dep, void *da return 0; } -struct acpi_scan_clear_dep_work { - struct work_struct work; - struct acpi_device *adev; -}; - -static void acpi_scan_clear_dep_fn(struct work_struct *work) +static void acpi_scan_clear_dep_fn(void *dev, async_cookie_t cookie) { - struct acpi_scan_clear_dep_work *cdw; - - cdw = container_of(work, struct acpi_scan_clear_dep_work, work); + struct acpi_device *adev = to_acpi_device(dev); acpi_scan_lock_acquire(); - acpi_bus_attach(cdw->adev, (void *)true); + acpi_bus_attach(adev, (void *)true); acpi_scan_lock_release(); - acpi_dev_put(cdw->adev); - kfree(cdw); + acpi_dev_put(adev); } static bool acpi_scan_clear_dep_queue(struct acpi_device *adev) { - struct acpi_scan_clear_dep_work *cdw; - if (adev->dep_unmet) return false; - cdw = kmalloc(sizeof(*cdw), GFP_KERNEL); - if (!cdw) - return false; - - cdw->adev = adev; - INIT_WORK(&cdw->work, acpi_scan_clear_dep_fn); /* - * Since the work function may block on the lock until the entire - * initial enumeration of devices is complete, put it into the unbound - * workqueue. + * Async schedule the deferred acpi_scan_clear_dep_fn() since: + * - acpi_bus_attach() needs to hold acpi_scan_lock which cannot + * be acquired under acpi_dep_list_lock (held here) + * - the deferred work at boot stage is ensured to be finished + * before userspace init task by the async_synchronize_full() + * barrier + * + * Use _nocall variant since it'll return on failure instead of + * run the function synchronously. */ - queue_work(system_dfl_wq, &cdw->work); - - return true; + return async_schedule_dev_nocall(acpi_scan_clear_dep_fn, &adev->dev); } static void acpi_scan_delete_dep_data(struct acpi_dep_data *dep) diff --git a/drivers/acpi/sleep.c b/drivers/acpi/sleep.c index 66ec81e306d476..132a9df9847137 100644 --- a/drivers/acpi/sleep.c +++ b/drivers/acpi/sleep.c @@ -386,6 +386,14 @@ static const struct dmi_system_id acpisleep_dmi_table[] __initconst = { DMI_MATCH(DMI_PRODUCT_NAME, "80E1"), }, }, + { + .callback = init_nvs_save_s3, + .ident = "Lenovo G70-35", + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "LENOVO"), + DMI_MATCH(DMI_PRODUCT_NAME, "80Q5"), + }, + }, /* * ThinkPad X1 Tablet(2016) cannot do suspend-to-idle using * the Low Power S0 Idle firmware interface (see diff --git a/drivers/acpi/x86/s2idle.c b/drivers/acpi/x86/s2idle.c index cc3c83e4cc23b9..2189330ffc6d33 100644 --- a/drivers/acpi/x86/s2idle.c +++ b/drivers/acpi/x86/s2idle.c @@ -49,6 +49,7 @@ static const struct acpi_device_id lps0_device_ids[] = { #define ACPI_LPS0_EXIT 6 #define ACPI_LPS0_MS_ENTRY 7 #define ACPI_LPS0_MS_EXIT 8 +#define ACPI_MS_TURN_ON_DISPLAY 9 /* AMD */ #define ACPI_LPS0_DSM_UUID_AMD "e3f32452-febc-43ce-9039-932122d37721" @@ -356,6 +357,8 @@ static const char *acpi_sleep_dsm_state_to_str(unsigned int state) return "lps0 ms entry"; case ACPI_LPS0_MS_EXIT: return "lps0 ms exit"; + case ACPI_MS_TURN_ON_DISPLAY: + return "lps0 ms turn on display"; } } else { switch (state) { @@ -617,6 +620,9 @@ static void acpi_s2idle_restore_early_lps0(void) if (lps0_dsm_func_mask_microsoft > 0) { acpi_sleep_run_lps0_dsm(ACPI_LPS0_EXIT, lps0_dsm_func_mask_microsoft, lps0_dsm_guid_microsoft); + /* Intent to turn on display */ + acpi_sleep_run_lps0_dsm(ACPI_MS_TURN_ON_DISPLAY, + lps0_dsm_func_mask_microsoft, lps0_dsm_guid_microsoft); /* Modern Standby exit */ acpi_sleep_run_lps0_dsm(ACPI_LPS0_MS_EXIT, lps0_dsm_func_mask_microsoft, lps0_dsm_guid_microsoft); diff --git a/drivers/acpi/x86/utils.c b/drivers/acpi/x86/utils.c index 4ee30c2897a2b9..418951639f5116 100644 --- a/drivers/acpi/x86/utils.c +++ b/drivers/acpi/x86/utils.c @@ -81,6 +81,18 @@ static const struct override_status_id override_status_ids[] = { DMI_MATCH(DMI_PRODUCT_NAME, "Mipad2"), }), + /* + * Lenovo Yoga Book uses PWM2 for touch keyboard backlight control. + * It needs to be enabled only for the Android device version (YB1-X90* + * aka YETI-11); the Windows version (YB1-X91*) uses ACPI control + * methods. + */ + PRESENT_ENTRY_HID("80862289", "2", INTEL_ATOM_AIRMONT, { + DMI_EXACT_MATCH(DMI_SYS_VENDOR, "Intel Corporation"), + DMI_EXACT_MATCH(DMI_PRODUCT_NAME, "CHERRYVIEW D1 PLATFORM"), + DMI_EXACT_MATCH(DMI_PRODUCT_VERSION, "YETI-11"), + }), + /* * The INT0002 device is necessary to clear wakeup interrupt sources * on Cherry Trail devices, without it we get nobody cared IRQ msgs. diff --git a/drivers/android/binder.c b/drivers/android/binder.c index b356c9b882544f..33e4dad0915bb1 100644 --- a/drivers/android/binder.c +++ b/drivers/android/binder.c @@ -4523,7 +4523,7 @@ static int binder_thread_write(struct binder_proc *proc, } } binder_debug(BINDER_DEBUG_DEAD_BINDER, - "%d:%d BC_DEAD_BINDER_DONE %016llx found %pK\n", + "%d:%d BC_DEAD_BINDER_DONE %016llx found %p\n", proc->pid, thread->pid, (u64)cookie, death); if (death == NULL) { diff --git a/drivers/android/binder/page_range.rs b/drivers/android/binder/page_range.rs index fdd97112ef5c8b..3d5bfaeda4a35d 100644 --- a/drivers/android/binder/page_range.rs +++ b/drivers/android/binder/page_range.rs @@ -13,6 +13,8 @@ // // The shrinker will use trylock methods because it locks them in a different order. +use crate::AssertSync; + use core::{ marker::PhantomPinned, mem::{size_of, size_of_val, MaybeUninit}, @@ -142,6 +144,30 @@ pub(crate) struct ShrinkablePageRange { _pin: PhantomPinned, } +// We do not define any ops. For now, used only to check identity of vmas. +static BINDER_VM_OPS: AssertSync = AssertSync(pin_init::zeroed()); + +// To ensure that we do not accidentally install pages into or zap pages from the wrong vma, we +// check its vm_ops and private data before using it. +fn check_vma(vma: &virt::VmaRef, owner: *const ShrinkablePageRange) -> Option<&virt::VmaMixedMap> { + // SAFETY: Just reading the vm_ops pointer of any active vma is safe. + let vm_ops = unsafe { (*vma.as_ptr()).vm_ops }; + if !ptr::eq(vm_ops, &BINDER_VM_OPS.0) { + return None; + } + + // SAFETY: Reading the vm_private_data pointer of a binder-owned vma is safe. + let vm_private_data = unsafe { (*vma.as_ptr()).vm_private_data }; + // The ShrinkablePageRange is only dropped when the Process is dropped, which only happens once + // the file's ->release handler is invoked, which means the ShrinkablePageRange outlives any + // VMA associated with it, so there can't be any false positives due to pointer reuse here. + if !ptr::eq(vm_private_data, owner.cast()) { + return None; + } + + vma.as_mixedmap_vma() +} + struct Inner { /// Array of pages. /// @@ -308,6 +334,18 @@ impl ShrinkablePageRange { inner.size = num_pages; inner.vma_addr = vma.start(); + // This pointer is only used for comparison - it's not dereferenced. + // + // SAFETY: We own the vma, and we don't use any methods on VmaNew that rely on + // `vm_private_data`. + unsafe { + (*vma.as_ptr()).vm_private_data = ptr::from_ref(self).cast_mut().cast::() + }; + + // SAFETY: We own the vma, and we don't use any methods on VmaNew that rely on + // `vm_ops`. + unsafe { (*vma.as_ptr()).vm_ops = &BINDER_VM_OPS.0 }; + Ok(num_pages) } @@ -399,22 +437,24 @@ impl ShrinkablePageRange { // // Using `mmput_async` avoids this, because then the `mm` cleanup is instead queued to a // workqueue. - MmWithUser::into_mmput_async(self.mm.mmget_not_zero().ok_or(ESRCH)?) - .mmap_read_lock() - .vma_lookup(vma_addr) - .ok_or(ESRCH)? - .as_mixedmap_vma() - .ok_or(ESRCH)? - .vm_insert_page(user_page_addr, &new_page) - .inspect_err(|err| { - pr_warn!( - "Failed to vm_insert_page({}): vma_addr:{} i:{} err:{:?}", - user_page_addr, - vma_addr, - i, - err - ) - })?; + check_vma( + MmWithUser::into_mmput_async(self.mm.mmget_not_zero().ok_or(ESRCH)?) + .mmap_read_lock() + .vma_lookup(vma_addr) + .ok_or(ESRCH)?, + self, + ) + .ok_or(ESRCH)? + .vm_insert_page(user_page_addr, &new_page) + .inspect_err(|err| { + pr_warn!( + "Failed to vm_insert_page({}): vma_addr:{} i:{} err:{:?}", + user_page_addr, + vma_addr, + i, + err + ) + })?; let inner = self.lock.lock(); @@ -667,12 +707,15 @@ unsafe extern "C" fn rust_shrink_free_page( let mmap_read; let mm_mutex; let vma_addr; + let range_ptr; { // CAST: The `list_head` field is first in `PageInfo`. let info = item as *mut PageInfo; // SAFETY: The `range` field of `PageInfo` is immutable. - let range = unsafe { &*((*info).range) }; + range_ptr = unsafe { (*info).range }; + // SAFETY: The `range` outlives its `PageInfo` values. + let range = unsafe { &*range_ptr }; mm = match range.mm.mmget_not_zero() { Some(mm) => MmWithUser::into_mmput_async(mm), @@ -717,9 +760,11 @@ unsafe extern "C" fn rust_shrink_free_page( // SAFETY: The lru lock is locked when this method is called. unsafe { bindings::spin_unlock(&raw mut (*lru).lock) }; - if let Some(vma) = mmap_read.vma_lookup(vma_addr) { - let user_page_addr = vma_addr + (page_index << PAGE_SHIFT); - vma.zap_page_range_single(user_page_addr, PAGE_SIZE); + if let Some(unchecked_vma) = mmap_read.vma_lookup(vma_addr) { + if let Some(vma) = check_vma(unchecked_vma, range_ptr) { + let user_page_addr = vma_addr + (page_index << PAGE_SHIFT); + vma.zap_page_range_single(user_page_addr, PAGE_SIZE); + } } drop(mmap_read); diff --git a/drivers/android/binder/process.rs b/drivers/android/binder/process.rs index 132055b4790f0e..0b03ec6418bf44 100644 --- a/drivers/android/binder/process.rs +++ b/drivers/android/binder/process.rs @@ -1289,7 +1289,8 @@ impl Process { } pub(crate) fn dead_binder_done(&self, cookie: u64, thread: &Thread) { - if let Some(death) = self.inner.lock().pull_delivered_death(cookie) { + let death = self.inner.lock().pull_delivered_death(cookie); + if let Some(death) = death { death.set_notification_done(thread); } } diff --git a/drivers/android/binder/range_alloc/array.rs b/drivers/android/binder/range_alloc/array.rs index 07e1dec2ce630f..ada1d1b4302e53 100644 --- a/drivers/android/binder/range_alloc/array.rs +++ b/drivers/android/binder/range_alloc/array.rs @@ -118,7 +118,7 @@ impl ArrayRangeAllocator { size: usize, is_oneway: bool, pid: Pid, - ) -> Result { + ) -> Result<(usize, bool)> { // Compute new value of free_oneway_space, which is set only on success. let new_oneway_space = if is_oneway { match self.free_oneway_space.checked_sub(size) { @@ -146,7 +146,38 @@ impl ArrayRangeAllocator { .ok() .unwrap(); - Ok(insert_at_offset) + // Start detecting spammers once we have less than 20% + // of async space left (which is less than 10% of total + // buffer size). + // + // (This will short-circuit, so `low_oneway_space` is + // only called when necessary.) + let oneway_spam_detected = + is_oneway && new_oneway_space < self.size / 10 && self.low_oneway_space(pid); + + Ok((insert_at_offset, oneway_spam_detected)) + } + + /// Find the amount and size of buffers allocated by the current caller. + /// + /// The idea is that once we cross the threshold, whoever is responsible + /// for the low async space is likely to try to send another async transaction, + /// and at some point we'll catch them in the act. This is more efficient + /// than keeping a map per pid. + fn low_oneway_space(&self, calling_pid: Pid) -> bool { + let mut total_alloc_size = 0; + let mut num_buffers = 0; + + // Warn if this pid has more than 50 transactions, or more than 50% of + // async space (which is 25% of total buffer size). Oneway spam is only + // detected when the threshold is exceeded. + for range in &self.ranges { + if range.state.is_oneway() && range.state.pid() == calling_pid { + total_alloc_size += range.size; + num_buffers += 1; + } + } + num_buffers > 50 || total_alloc_size > self.size / 4 } pub(crate) fn reservation_abort(&mut self, offset: usize) -> Result { diff --git a/drivers/android/binder/range_alloc/mod.rs b/drivers/android/binder/range_alloc/mod.rs index 2301e2bc1a1fcd..1f4734468ff11f 100644 --- a/drivers/android/binder/range_alloc/mod.rs +++ b/drivers/android/binder/range_alloc/mod.rs @@ -188,11 +188,11 @@ impl RangeAllocator { self.reserve_new(args) } Impl::Array(array) => { - let offset = + let (offset, oneway_spam_detected) = array.reserve_new(args.debug_id, args.size, args.is_oneway, args.pid)?; Ok(ReserveNew::Success(ReserveNewSuccess { offset, - oneway_spam_detected: false, + oneway_spam_detected, _empty_array_alloc: args.empty_array_alloc, _new_tree_alloc: args.new_tree_alloc, _tree_alloc: args.tree_alloc, diff --git a/drivers/android/binder/range_alloc/tree.rs b/drivers/android/binder/range_alloc/tree.rs index 838fdd2b47ea78..48796fcdb36249 100644 --- a/drivers/android/binder/range_alloc/tree.rs +++ b/drivers/android/binder/range_alloc/tree.rs @@ -164,15 +164,6 @@ impl TreeRangeAllocator { self.free_oneway_space }; - // Start detecting spammers once we have less than 20% - // of async space left (which is less than 10% of total - // buffer size). - // - // (This will short-circut, so `low_oneway_space` is - // only called when necessary.) - let oneway_spam_detected = - is_oneway && new_oneway_space < self.size / 10 && self.low_oneway_space(pid); - let (found_size, found_off, tree_node, free_tree_node) = match self.find_best_match(size) { None => { pr_warn!("ENOSPC from range_alloc.reserve_new - size: {}", size); @@ -203,6 +194,15 @@ impl TreeRangeAllocator { self.free_tree.insert(free_tree_node); } + // Start detecting spammers once we have less than 20% + // of async space left (which is less than 10% of total + // buffer size). + // + // (This will short-circuit, so `low_oneway_space` is + // only called when necessary.) + let oneway_spam_detected = + is_oneway && new_oneway_space < self.size / 10 && self.low_oneway_space(pid); + Ok((found_off, oneway_spam_detected)) } diff --git a/drivers/android/binder/rust_binder_main.rs b/drivers/android/binder/rust_binder_main.rs index c79a9e7422401f..1488d82184efd0 100644 --- a/drivers/android/binder/rust_binder_main.rs +++ b/drivers/android/binder/rust_binder_main.rs @@ -300,7 +300,7 @@ impl kernel::Module for BinderModule { /// Makes the inner type Sync. #[repr(transparent)] pub struct AssertSync(T); -// SAFETY: Used only to insert `file_operations` into a global, which is safe. +// SAFETY: Used only to insert C bindings types into globals, which is safe. unsafe impl Sync for AssertSync {} /// File operations that rust_binderfs.c can use. @@ -314,6 +314,7 @@ pub static rust_binder_fops: AssertSync = { owner: THIS_MODULE.as_ptr(), poll: Some(rust_binder_poll), unlocked_ioctl: Some(rust_binder_ioctl), + #[cfg(CONFIG_COMPAT)] compat_ioctl: Some(bindings::compat_ptr_ioctl), mmap: Some(rust_binder_mmap), open: Some(rust_binder_open), diff --git a/drivers/android/binder/thread.rs b/drivers/android/binder/thread.rs index e0ea33ccfe58bd..637563d9b8134a 100644 --- a/drivers/android/binder/thread.rs +++ b/drivers/android/binder/thread.rs @@ -1018,12 +1018,9 @@ impl Thread { // Copy offsets if there are any. if offsets_size > 0 { - { - let mut reader = - UserSlice::new(UserPtr::from_addr(trd_data_ptr.offsets as _), offsets_size) - .reader(); - alloc.copy_into(&mut reader, aligned_data_size, offsets_size)?; - } + let mut offsets_reader = + UserSlice::new(UserPtr::from_addr(trd_data_ptr.offsets as _), offsets_size) + .reader(); let offsets_start = aligned_data_size; let offsets_end = aligned_data_size + offsets_size; @@ -1044,11 +1041,9 @@ impl Thread { .step_by(size_of::()) .enumerate() { - let offset: usize = view - .alloc - .read::(index_offset)? - .try_into() - .map_err(|_| EINVAL)?; + let offset = offsets_reader.read::()?; + view.alloc.write(index_offset, &offset)?; + let offset: usize = offset.try_into().map_err(|_| EINVAL)?; if offset < end_of_previous_object || !is_aligned(offset, size_of::()) { pr_warn!("Got transaction with invalid offset."); diff --git a/drivers/android/binder_alloc.c b/drivers/android/binder_alloc.c index 979c96b74cad36..d5ed64543bbf44 100644 --- a/drivers/android/binder_alloc.c +++ b/drivers/android/binder_alloc.c @@ -81,7 +81,7 @@ static void binder_insert_free_buffer(struct binder_alloc *alloc, new_buffer_size = binder_alloc_buffer_size(alloc, new_buffer); binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC, - "%d: add free buffer, size %zd, at %pK\n", + "%d: add free buffer, size %zd, at %p\n", alloc->pid, new_buffer_size, new_buffer); while (*p) { @@ -572,7 +572,7 @@ static struct binder_buffer *binder_alloc_new_buf_locked( } binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC, - "%d: binder_alloc_buf size %zd got buffer %pK size %zd\n", + "%d: binder_alloc_buf size %zd got buffer %p size %zd\n", alloc->pid, size, buffer, buffer_size); /* @@ -748,7 +748,7 @@ static void binder_free_buf_locked(struct binder_alloc *alloc, ALIGN(buffer->extra_buffers_size, sizeof(void *)); binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC, - "%d: binder_free_buf %pK size %zd buffer_size %zd\n", + "%d: binder_free_buf %p size %zd buffer_size %zd\n", alloc->pid, buffer, size, buffer_size); BUG_ON(buffer->free); diff --git a/drivers/ata/ahci.c b/drivers/ata/ahci.c index 931d0081169b9c..1d73a53370cf3e 100644 --- a/drivers/ata/ahci.c +++ b/drivers/ata/ahci.c @@ -68,6 +68,7 @@ enum board_ids { /* board IDs for specific chipsets in alphabetical order */ board_ahci_al, board_ahci_avn, + board_ahci_jmb585, board_ahci_mcp65, board_ahci_mcp77, board_ahci_mcp89, @@ -212,6 +213,15 @@ static const struct ata_port_info ahci_port_info[] = { .udma_mask = ATA_UDMA6, .port_ops = &ahci_avn_ops, }, + /* JMicron JMB582/585: 64-bit DMA is broken, force 32-bit */ + [board_ahci_jmb585] = { + AHCI_HFLAGS (AHCI_HFLAG_IGN_IRQ_IF_ERR | + AHCI_HFLAG_32BIT_ONLY), + .flags = AHCI_FLAG_COMMON, + .pio_mask = ATA_PIO4, + .udma_mask = ATA_UDMA6, + .port_ops = &ahci_ops, + }, [board_ahci_mcp65] = { AHCI_HFLAGS (AHCI_HFLAG_NO_FPDMA_AA | AHCI_HFLAG_NO_PMP | AHCI_HFLAG_YES_NCQ), @@ -439,6 +449,10 @@ static const struct pci_device_id ahci_pci_tbl[] = { /* Elkhart Lake IDs 0x4b60 & 0x4b62 https://sata-io.org/product/8803 not tested yet */ { PCI_VDEVICE(INTEL, 0x4b63), board_ahci_pcs_quirk }, /* Elkhart Lake AHCI */ + /* JMicron JMB582/585: force 32-bit DMA (broken 64-bit implementation) */ + { PCI_VDEVICE(JMICRON, 0x0582), board_ahci_jmb585 }, + { PCI_VDEVICE(JMICRON, 0x0585), board_ahci_jmb585 }, + /* JMicron 360/1/3/5/6, match class to avoid IDE function */ { PCI_VENDOR_ID_JMICRON, PCI_ANY_ID, PCI_ANY_ID, PCI_ANY_ID, PCI_CLASS_STORAGE_SATA_AHCI, 0xffffff, board_ahci_ign_iferr }, diff --git a/drivers/ata/libata-core.c b/drivers/ata/libata-core.c index ddf9a7b28a5943..a5332505372277 100644 --- a/drivers/ata/libata-core.c +++ b/drivers/ata/libata-core.c @@ -77,6 +77,7 @@ static unsigned int ata_dev_init_params(struct ata_device *dev, static unsigned int ata_dev_set_xfermode(struct ata_device *dev); static void ata_dev_xfermask(struct ata_device *dev); static unsigned int ata_dev_quirks(const struct ata_device *dev); +static u64 ata_dev_get_quirk_value(struct ata_device *dev, unsigned int quirk); static DEFINE_IDA(ata_ida); @@ -2358,6 +2359,24 @@ static bool ata_dev_check_adapter(struct ata_device *dev, return false; } +bool ata_adapter_is_online(struct ata_port *ap) +{ + struct device *dev; + + if (!ap || !ap->host) + return false; + + dev = ap->host->dev; + if (!dev) + return false; + + if (dev_is_pci(dev) && + pci_channel_offline(to_pci_dev(dev))) + return false; + + return true; +} + static int ata_dev_config_ncq(struct ata_device *dev, char *desc, size_t desc_sz) { @@ -3152,9 +3171,10 @@ int ata_dev_configure(struct ata_device *dev) dev->max_sectors = min_t(unsigned int, ATA_MAX_SECTORS_1024, dev->max_sectors); - if (dev->quirks & ATA_QUIRK_MAX_SEC_8191) - dev->max_sectors = min_t(unsigned int, ATA_MAX_SECTORS_8191, - dev->max_sectors); + if (dev->quirks & ATA_QUIRK_MAX_SEC) + dev->max_sectors = min_t(unsigned int, dev->max_sectors, + ata_dev_get_quirk_value(dev, + ATA_QUIRK_MAX_SEC)); if (dev->quirks & ATA_QUIRK_MAX_SEC_LBA48) dev->max_sectors = ATA_MAX_SECTORS_LBA48; @@ -4008,7 +4028,7 @@ static const char * const ata_quirk_names[] = { [__ATA_QUIRK_NO_DMA_LOG] = "nodmalog", [__ATA_QUIRK_NOTRIM] = "notrim", [__ATA_QUIRK_MAX_SEC_1024] = "maxsec1024", - [__ATA_QUIRK_MAX_SEC_8191] = "maxsec8191", + [__ATA_QUIRK_MAX_SEC] = "maxsec", [__ATA_QUIRK_MAX_TRIM_128M] = "maxtrim128m", [__ATA_QUIRK_NO_NCQ_ON_ATI] = "noncqonati", [__ATA_QUIRK_NO_LPM_ON_ATI] = "nolpmonati", @@ -4053,6 +4073,22 @@ static void ata_dev_print_quirks(const struct ata_device *dev, kfree(str); } +struct ata_dev_quirk_value { + const char *model_num; + const char *model_rev; + u64 val; +}; + +static const struct ata_dev_quirk_value __ata_dev_max_sec_quirks[] = { + { "TORiSAN DVD-ROM DRD-N216", NULL, 128 }, + { "ST380013AS", "3.20", 1024 }, + { "LITEON CX1-JB*-HP", NULL, 1024 }, + { "LITEON EP1-*", NULL, 1024 }, + { "DELLBOSS VD", "MV.R00-0", 8191 }, + { "INTEL SSDSC2KG480G8", "XCV10120", 8191 }, + { }, +}; + struct ata_dev_quirks_entry { const char *model_num; const char *model_rev; @@ -4097,7 +4133,7 @@ static const struct ata_dev_quirks_entry __ata_dev_quirks[] = { { "ASMT109x- Config", NULL, ATA_QUIRK_DISABLE }, /* Weird ATAPI devices */ - { "TORiSAN DVD-ROM DRD-N216", NULL, ATA_QUIRK_MAX_SEC_128 }, + { "TORiSAN DVD-ROM DRD-N216", NULL, ATA_QUIRK_MAX_SEC }, { "QUANTUM DAT DAT72-000", NULL, ATA_QUIRK_ATAPI_MOD16_DMA }, { "Slimtype DVD A DS8A8SH", NULL, ATA_QUIRK_MAX_SEC_LBA48 }, { "Slimtype DVD A DS8A9SH", NULL, ATA_QUIRK_MAX_SEC_LBA48 }, @@ -4106,20 +4142,20 @@ static const struct ata_dev_quirks_entry __ata_dev_quirks[] = { * Causes silent data corruption with higher max sects. * http://lkml.kernel.org/g/x49wpy40ysk.fsf@segfault.boston.devel.redhat.com */ - { "ST380013AS", "3.20", ATA_QUIRK_MAX_SEC_1024 }, + { "ST380013AS", "3.20", ATA_QUIRK_MAX_SEC }, /* * These devices time out with higher max sects. * https://bugzilla.kernel.org/show_bug.cgi?id=121671 */ - { "LITEON CX1-JB*-HP", NULL, ATA_QUIRK_MAX_SEC_1024 }, - { "LITEON EP1-*", NULL, ATA_QUIRK_MAX_SEC_1024 }, + { "LITEON CX1-JB*-HP", NULL, ATA_QUIRK_MAX_SEC }, + { "LITEON EP1-*", NULL, ATA_QUIRK_MAX_SEC }, /* * These devices time out with higher max sects. * https://bugzilla.kernel.org/show_bug.cgi?id=220693 */ - { "DELLBOSS VD", "MV.R00-0", ATA_QUIRK_MAX_SEC_8191 }, + { "DELLBOSS VD", "MV.R00-0", ATA_QUIRK_MAX_SEC }, /* Devices we expect to fail diagnostics */ @@ -4149,7 +4185,11 @@ static const struct ata_dev_quirks_entry __ata_dev_quirks[] = { { "ST3320[68]13AS", "SD1[5-9]", ATA_QUIRK_NONCQ | ATA_QUIRK_FIRMWARE_WARN }, + /* ADATA devices with LPM issues. */ + { "ADATA SU680", NULL, ATA_QUIRK_NOLPM }, + /* Seagate disks with LPM issues */ + { "ST1000DM010-2EP102", NULL, ATA_QUIRK_NOLPM }, { "ST2000DM008-2FR102", NULL, ATA_QUIRK_NOLPM }, /* drives which fail FPDMA_AA activation (some may freeze afterwards) @@ -4192,6 +4232,7 @@ static const struct ata_dev_quirks_entry __ata_dev_quirks[] = { /* Devices that do not need bridging limits applied */ { "MTRON MSP-SATA*", NULL, ATA_QUIRK_BRIDGE_OK }, { "BUFFALO HD-QSU2/R5", NULL, ATA_QUIRK_BRIDGE_OK }, + { "QEMU HARDDISK", "2.5+", ATA_QUIRK_BRIDGE_OK }, /* Devices which aren't very happy with higher link speeds */ { "WD My Book", NULL, ATA_QUIRK_1_5_GBPS }, @@ -4307,6 +4348,8 @@ static const struct ata_dev_quirks_entry __ata_dev_quirks[] = { { "Micron*", NULL, ATA_QUIRK_ZERO_AFTER_TRIM }, { "Crucial*", NULL, ATA_QUIRK_ZERO_AFTER_TRIM }, + { "INTEL SSDSC2KG480G8", "XCV10120", ATA_QUIRK_ZERO_AFTER_TRIM | + ATA_QUIRK_MAX_SEC }, { "INTEL*SSD*", NULL, ATA_QUIRK_ZERO_AFTER_TRIM }, { "SSD*INTEL*", NULL, ATA_QUIRK_ZERO_AFTER_TRIM }, { "Samsung*SSD*", NULL, ATA_QUIRK_ZERO_AFTER_TRIM }, @@ -4372,6 +4415,39 @@ static unsigned int ata_dev_quirks(const struct ata_device *dev) return 0; } +static u64 ata_dev_get_max_sec_quirk_value(struct ata_device *dev) +{ + unsigned char model_num[ATA_ID_PROD_LEN + 1]; + unsigned char model_rev[ATA_ID_FW_REV_LEN + 1]; + const struct ata_dev_quirk_value *ad = __ata_dev_max_sec_quirks; + u64 val = 0; + + ata_id_c_string(dev->id, model_num, ATA_ID_PROD, sizeof(model_num)); + ata_id_c_string(dev->id, model_rev, ATA_ID_FW_REV, sizeof(model_rev)); + + while (ad->model_num) { + if (glob_match(ad->model_num, model_num) && + (!ad->model_rev || glob_match(ad->model_rev, model_rev))) { + val = ad->val; + break; + } + ad++; + } + + ata_dev_warn(dev, "%s quirk is using value: %llu\n", + ata_quirk_names[__ATA_QUIRK_MAX_SEC], val); + + return val; +} + +static u64 ata_dev_get_quirk_value(struct ata_device *dev, unsigned int quirk) +{ + if (quirk == ATA_QUIRK_MAX_SEC) + return ata_dev_get_max_sec_quirk_value(dev); + + return 0; +} + static bool ata_dev_nodma(const struct ata_device *dev) { /* @@ -5082,6 +5158,12 @@ void ata_qc_issue(struct ata_queued_cmd *qc) qc->flags |= ATA_QCFLAG_ACTIVE; ap->qc_active |= 1ULL << qc->tag; + /* Make sure the device is still accessible. */ + if (!ata_adapter_is_online(ap)) { + qc->err_mask |= AC_ERR_HOST_BUS; + goto sys_err; + } + /* * We guarantee to LLDs that they will have at least one * non-zero sg if the command is a data command. @@ -5567,6 +5649,7 @@ struct ata_port *ata_port_alloc(struct ata_host *host) mutex_init(&ap->scsi_scan_mutex); INIT_DELAYED_WORK(&ap->hotplug_task, ata_scsi_hotplug); INIT_DELAYED_WORK(&ap->scsi_rescan_task, ata_scsi_dev_rescan); + INIT_WORK(&ap->deferred_qc_work, ata_scsi_deferred_qc_work); INIT_LIST_HEAD(&ap->eh_done_q); init_waitqueue_head(&ap->eh_wait_q); init_completion(&ap->park_req_pending); @@ -6189,9 +6272,11 @@ static void ata_port_detach(struct ata_port *ap) /* wait till EH commits suicide */ ata_port_wait_eh(ap); - /* it better be dead now */ + /* It better be dead now and not have any remaining deferred qc. */ WARN_ON(!(ap->pflags & ATA_PFLAG_UNLOADED)); + WARN_ON(ap->deferred_qc); + cancel_work_sync(&ap->deferred_qc_work); cancel_delayed_work_sync(&ap->hotplug_task); cancel_delayed_work_sync(&ap->scsi_rescan_task); diff --git a/drivers/ata/libata-eh.c b/drivers/ata/libata-eh.c index 2586e77ebf45d0..23be85418b3b1d 100644 --- a/drivers/ata/libata-eh.c +++ b/drivers/ata/libata-eh.c @@ -640,12 +640,29 @@ void ata_scsi_cmd_error_handler(struct Scsi_Host *host, struct ata_port *ap, set_host_byte(scmd, DID_OK); ata_qc_for_each_raw(ap, qc, i) { - if (qc->flags & ATA_QCFLAG_ACTIVE && - qc->scsicmd == scmd) + if (qc->scsicmd != scmd) + continue; + if ((qc->flags & ATA_QCFLAG_ACTIVE) || + qc == ap->deferred_qc) break; } - if (i < ATA_MAX_QUEUE) { + if (i < ATA_MAX_QUEUE && qc == ap->deferred_qc) { + /* + * This is a deferred command that timed out while + * waiting for the command queue to drain. Since the qc + * is not active yet (deferred_qc is still set, so the + * deferred qc work has not issued the command yet), + * simply signal the timeout by finishing the SCSI + * command and clear the deferred qc to prevent the + * deferred qc work from issuing this qc. + */ + WARN_ON_ONCE(qc->flags & ATA_QCFLAG_ACTIVE); + ap->deferred_qc = NULL; + cancel_work(&ap->deferred_qc_work); + set_host_byte(scmd, DID_TIME_OUT); + scsi_eh_finish_cmd(scmd, &ap->eh_done_q); + } else if (i < ATA_MAX_QUEUE) { /* the scmd has an associated qc */ if (!(qc->flags & ATA_QCFLAG_EH)) { /* which hasn't failed yet, timeout */ @@ -736,7 +753,8 @@ void ata_scsi_port_error_handler(struct Scsi_Host *host, struct ata_port *ap) spin_unlock_irqrestore(ap->lock, flags); /* invoke EH, skip if unloading or suspended */ - if (!(ap->pflags & (ATA_PFLAG_UNLOADING | ATA_PFLAG_SUSPENDED))) + if (!(ap->pflags & (ATA_PFLAG_UNLOADING | ATA_PFLAG_SUSPENDED)) && + ata_adapter_is_online(ap)) ap->ops->error_handler(ap); else { /* if unloading, commence suicide */ @@ -917,6 +935,12 @@ static void ata_eh_set_pending(struct ata_port *ap, bool fastdrain) ap->pflags |= ATA_PFLAG_EH_PENDING; + /* + * If we have a deferred qc, requeue it so that it is retried once EH + * completes. + */ + ata_scsi_requeue_deferred_qc(ap); + if (!fastdrain) return; diff --git a/drivers/ata/libata-scsi.c b/drivers/ata/libata-scsi.c index 721d3f270c8ec6..d93c4a1b0de54e 100644 --- a/drivers/ata/libata-scsi.c +++ b/drivers/ata/libata-scsi.c @@ -1658,8 +1658,78 @@ static void ata_qc_done(struct ata_queued_cmd *qc) done(cmd); } +void ata_scsi_deferred_qc_work(struct work_struct *work) +{ + struct ata_port *ap = + container_of(work, struct ata_port, deferred_qc_work); + struct ata_queued_cmd *qc; + unsigned long flags; + + spin_lock_irqsave(ap->lock, flags); + + /* + * If we still have a deferred qc and we are not in EH, issue it. In + * such case, we should not need any more deferring the qc, so warn if + * qc_defer() says otherwise. + */ + qc = ap->deferred_qc; + if (qc && !ata_port_eh_scheduled(ap)) { + WARN_ON_ONCE(ap->ops->qc_defer(qc)); + ap->deferred_qc = NULL; + ata_qc_issue(qc); + } + + spin_unlock_irqrestore(ap->lock, flags); +} + +void ata_scsi_requeue_deferred_qc(struct ata_port *ap) +{ + struct ata_queued_cmd *qc = ap->deferred_qc; + struct scsi_cmnd *scmd; + + lockdep_assert_held(ap->lock); + + /* + * If we have a deferred qc when a reset occurs or NCQ commands fail, + * do not try to be smart about what to do with this deferred command + * and simply retry it by completing it with DID_SOFT_ERROR. + */ + if (!qc) + return; + + scmd = qc->scsicmd; + ap->deferred_qc = NULL; + cancel_work(&ap->deferred_qc_work); + ata_qc_free(qc); + scmd->result = (DID_SOFT_ERROR << 16); + scsi_done(scmd); +} + +static void ata_scsi_schedule_deferred_qc(struct ata_port *ap) +{ + struct ata_queued_cmd *qc = ap->deferred_qc; + + lockdep_assert_held(ap->lock); + + /* + * If we have a deferred qc, then qc_defer() is defined and we can use + * this callback to determine if this qc is good to go, unless EH has + * been scheduled. + */ + if (!qc) + return; + + if (ata_port_eh_scheduled(ap)) { + ata_scsi_requeue_deferred_qc(ap); + return; + } + if (!ap->ops->qc_defer(qc)) + queue_work(system_highpri_wq, &ap->deferred_qc_work); +} + static void ata_scsi_qc_complete(struct ata_queued_cmd *qc) { + struct ata_port *ap = qc->ap; struct scsi_cmnd *cmd = qc->scsicmd; u8 *cdb = cmd->cmnd; bool have_sense = qc->flags & ATA_QCFLAG_SENSE_VALID; @@ -1689,6 +1759,66 @@ static void ata_scsi_qc_complete(struct ata_queued_cmd *qc) } ata_qc_done(qc); + + ata_scsi_schedule_deferred_qc(ap); +} + +static int ata_scsi_qc_issue(struct ata_port *ap, struct ata_queued_cmd *qc) +{ + int ret; + + if (!ap->ops->qc_defer) + goto issue; + + /* + * If we already have a deferred qc, then rely on the SCSI layer to + * requeue and defer all incoming commands until the deferred qc is + * processed, once all on-going commands complete. + */ + if (ap->deferred_qc) { + ata_qc_free(qc); + return SCSI_MLQUEUE_DEVICE_BUSY; + } + + /* Check if the command needs to be deferred. */ + ret = ap->ops->qc_defer(qc); + switch (ret) { + case 0: + break; + case ATA_DEFER_LINK: + ret = SCSI_MLQUEUE_DEVICE_BUSY; + break; + case ATA_DEFER_PORT: + ret = SCSI_MLQUEUE_HOST_BUSY; + break; + default: + WARN_ON_ONCE(1); + ret = SCSI_MLQUEUE_HOST_BUSY; + break; + } + + if (ret) { + /* + * We must defer this qc: if this is not an NCQ command, keep + * this qc as a deferred one and report to the SCSI layer that + * we issued it so that it is not requeued. The deferred qc will + * be issued with the port deferred_qc_work once all on-going + * commands complete. + */ + if (!ata_is_ncq(qc->tf.protocol)) { + ap->deferred_qc = qc; + return 0; + } + + /* Force a requeue of the command to defer its execution. */ + ata_qc_free(qc); + return ret; + } + +issue: + ata_qc_issue(qc); + + return 0; } /** @@ -1714,66 +1844,49 @@ static void ata_scsi_qc_complete(struct ata_queued_cmd *qc) * spin_lock_irqsave(host lock) * * RETURNS: - * 0 on success, SCSI_ML_QUEUE_DEVICE_BUSY if the command - * needs to be deferred. + * 0 on success, SCSI_ML_QUEUE_DEVICE_BUSY or SCSI_MLQUEUE_HOST_BUSY if the + * command needs to be deferred. */ static int ata_scsi_translate(struct ata_device *dev, struct scsi_cmnd *cmd, ata_xlat_func_t xlat_func) { struct ata_port *ap = dev->link->ap; struct ata_queued_cmd *qc; - int rc; + lockdep_assert_held(ap->lock); + + /* + * ata_scsi_qc_new() calls scsi_done(cmd) in case of failure. So we + * have nothing further to do when allocating a qc fails. + */ qc = ata_scsi_qc_new(dev, cmd); if (!qc) - goto err_mem; + return 0; /* data is present; dma-map it */ if (cmd->sc_data_direction == DMA_FROM_DEVICE || cmd->sc_data_direction == DMA_TO_DEVICE) { if (unlikely(scsi_bufflen(cmd) < 1)) { ata_dev_warn(dev, "WARNING: zero len r/w req\n"); - goto err_did; + cmd->result = (DID_ERROR << 16); + goto done; } ata_sg_init(qc, scsi_sglist(cmd), scsi_sg_count(cmd)); - qc->dma_dir = cmd->sc_data_direction; } qc->complete_fn = ata_scsi_qc_complete; if (xlat_func(qc)) - goto early_finish; - - if (ap->ops->qc_defer) { - if ((rc = ap->ops->qc_defer(qc))) - goto defer; - } - - /* select device, send command to hardware */ - ata_qc_issue(qc); - - return 0; + goto done; -early_finish: - ata_qc_free(qc); - scsi_done(cmd); - return 0; + return ata_scsi_qc_issue(ap, qc); -err_did: +done: ata_qc_free(qc); - cmd->result = (DID_ERROR << 16); scsi_done(cmd); -err_mem: return 0; - -defer: - ata_qc_free(qc); - if (rc == ATA_DEFER_LINK) - return SCSI_MLQUEUE_DEVICE_BUSY; - else - return SCSI_MLQUEUE_HOST_BUSY; } /** @@ -2982,6 +3095,9 @@ ata_scsi_find_dev(struct ata_port *ap, const struct scsi_device *scsidev) { struct ata_device *dev = __ata_scsi_find_dev(ap, scsidev); + if (!ata_adapter_is_online(ap)) + return NULL; + if (unlikely(!dev || !ata_dev_enabled(dev))) return NULL; @@ -3484,7 +3600,7 @@ static unsigned int ata_scsiop_maint_in(struct ata_device *dev, if (cdb[2] != 1 && cdb[2] != 3) { ata_dev_warn(dev, "invalid command format %d\n", cdb[2]); - ata_scsi_set_invalid_field(dev, cmd, 1, 0xff); + ata_scsi_set_invalid_field(dev, cmd, 2, 0xff); return 0; } diff --git a/drivers/ata/libata.h b/drivers/ata/libata.h index 0e7ecac7368098..9b4e578ad07ec5 100644 --- a/drivers/ata/libata.h +++ b/drivers/ata/libata.h @@ -94,6 +94,7 @@ extern int atapi_check_dma(struct ata_queued_cmd *qc); extern void swap_buf_le16(u16 *buf, unsigned int buf_words); extern bool ata_phys_link_online(struct ata_link *link); extern bool ata_phys_link_offline(struct ata_link *link); +bool ata_adapter_is_online(struct ata_port *ap); extern void ata_dev_init(struct ata_device *dev); extern void ata_link_init(struct ata_port *ap, struct ata_link *link, int pmp); extern int sata_link_init_spd(struct ata_link *link); @@ -165,6 +166,8 @@ void ata_scsi_sdev_config(struct scsi_device *sdev); int ata_scsi_dev_config(struct scsi_device *sdev, struct queue_limits *lim, struct ata_device *dev); int __ata_scsi_queuecmd(struct scsi_cmnd *scmd, struct ata_device *dev); +void ata_scsi_deferred_qc_work(struct work_struct *work); +void ata_scsi_requeue_deferred_qc(struct ata_port *ap); /* libata-eh.c */ extern unsigned int ata_internal_cmd_timeout(struct ata_device *dev, u8 cmd); diff --git a/drivers/ata/pata_ftide010.c b/drivers/ata/pata_ftide010.c index c3a8384c3e04d4..c41da296eb389d 100644 --- a/drivers/ata/pata_ftide010.c +++ b/drivers/ata/pata_ftide010.c @@ -122,10 +122,10 @@ static const u8 mwdma_50_active_time[3] = {6, 2, 2}; static const u8 mwdma_50_recovery_time[3] = {6, 2, 1}; static const u8 mwdma_66_active_time[3] = {8, 3, 3}; static const u8 mwdma_66_recovery_time[3] = {8, 2, 1}; -static const u8 udma_50_setup_time[6] = {3, 3, 2, 2, 1, 1}; +static const u8 udma_50_setup_time[6] = {3, 3, 2, 2, 1, 9}; static const u8 udma_50_hold_time[6] = {3, 1, 1, 1, 1, 1}; -static const u8 udma_66_setup_time[7] = {4, 4, 3, 2, }; -static const u8 udma_66_hold_time[7] = {}; +static const u8 udma_66_setup_time[7] = {4, 4, 3, 2, 1, 9, 9}; +static const u8 udma_66_hold_time[7] = {4, 2, 1, 1, 1, 1, 1}; /* * We set 66 MHz for all MWDMA modes diff --git a/drivers/atm/fore200e.c b/drivers/atm/fore200e.c index f62e3857144031..fec081db36dc4b 100644 --- a/drivers/atm/fore200e.c +++ b/drivers/atm/fore200e.c @@ -373,6 +373,10 @@ fore200e_shutdown(struct fore200e* fore200e) fallthrough; case FORE200E_STATE_IRQ: free_irq(fore200e->irq, fore200e->atm_dev); +#ifdef FORE200E_USE_TASKLET + tasklet_kill(&fore200e->tx_tasklet); + tasklet_kill(&fore200e->rx_tasklet); +#endif fallthrough; case FORE200E_STATE_ALLOC_BUF: diff --git a/drivers/auxdisplay/arm-charlcd.c b/drivers/auxdisplay/arm-charlcd.c index a7eae99a48f77d..4e22882f57c9c2 100644 --- a/drivers/auxdisplay/arm-charlcd.c +++ b/drivers/auxdisplay/arm-charlcd.c @@ -323,7 +323,7 @@ static int __init charlcd_probe(struct platform_device *pdev) out_no_irq: iounmap(lcd->virtbase); out_no_memregion: - release_mem_region(lcd->phybase, SZ_4K); + release_mem_region(lcd->phybase, lcd->physize); out_no_resource: kfree(lcd); return ret; diff --git a/drivers/auxdisplay/line-display.c b/drivers/auxdisplay/line-display.c index 4e22373fcc1a97..e80e9426283055 100644 --- a/drivers/auxdisplay/line-display.c +++ b/drivers/auxdisplay/line-display.c @@ -365,7 +365,7 @@ static DEFINE_IDA(linedisp_id); static void linedisp_release(struct device *dev) { - struct linedisp *linedisp = to_linedisp(dev); + struct linedisp *linedisp = container_of(dev, struct linedisp, dev); kfree(linedisp->map); kfree(linedisp->message); diff --git a/drivers/base/bus.c b/drivers/base/bus.c index 9eb7771706f018..7c7d8d97215be8 100644 --- a/drivers/base/bus.c +++ b/drivers/base/bus.c @@ -504,6 +504,36 @@ int bus_for_each_drv(const struct bus_type *bus, struct device_driver *start, } EXPORT_SYMBOL_GPL(bus_for_each_drv); +static ssize_t driver_override_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + int ret; + + ret = __device_set_driver_override(dev, buf, count); + if (ret) + return ret; + + return count; +} + +static ssize_t driver_override_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + guard(spinlock)(&dev->driver_override.lock); + return sysfs_emit(buf, "%s\n", dev->driver_override.name); +} +static DEVICE_ATTR_RW(driver_override); + +static struct attribute *driver_override_dev_attrs[] = { + &dev_attr_driver_override.attr, + NULL, +}; + +static const struct attribute_group driver_override_dev_group = { + .attrs = driver_override_dev_attrs, +}; + /** * bus_add_device - add device to bus * @dev: device being added @@ -537,9 +567,15 @@ int bus_add_device(struct device *dev) if (error) goto out_put; + if (dev->bus->driver_override) { + error = device_add_group(dev, &driver_override_dev_group); + if (error) + goto out_groups; + } + error = sysfs_create_link(&sp->devices_kset->kobj, &dev->kobj, dev_name(dev)); if (error) - goto out_groups; + goto out_override; error = sysfs_create_link(&dev->kobj, &sp->subsys.kobj, "subsystem"); if (error) @@ -550,6 +586,9 @@ int bus_add_device(struct device *dev) out_subsys: sysfs_remove_link(&sp->devices_kset->kobj, dev_name(dev)); +out_override: + if (dev->bus->driver_override) + device_remove_group(dev, &driver_override_dev_group); out_groups: device_remove_groups(dev, sp->bus->dev_groups); out_put: @@ -607,6 +646,8 @@ void bus_remove_device(struct device *dev) sysfs_remove_link(&dev->kobj, "subsystem"); sysfs_remove_link(&sp->devices_kset->kobj, dev_name(dev)); + if (dev->bus->driver_override) + device_remove_group(dev, &driver_override_dev_group); device_remove_groups(dev, dev->bus->dev_groups); if (klist_node_attached(&dev->p->knode_bus)) klist_del(&dev->p->knode_bus); diff --git a/drivers/base/core.c b/drivers/base/core.c index 40de2f51a1b1ab..3f9759f60bcaa5 100644 --- a/drivers/base/core.c +++ b/drivers/base/core.c @@ -2328,6 +2328,32 @@ static void fw_devlink_link_device(struct device *dev) __fw_devlink_link_to_suppliers(dev, fwnode); } +/** + * fw_devlink_count_absent_consumers - Return how many consumers have + * either not been created yet, or do not yet have a driver attached. + * @fwnode: fwnode of the supplier + */ +int fw_devlink_count_absent_consumers(struct fwnode_handle *fwnode) +{ + struct fwnode_link *link, *tmp; + struct device_link *dlink, *dtmp; + struct device *sup_dev = get_dev_from_fwnode(fwnode); + int count = 0; + + list_for_each_entry_safe(link, tmp, &fwnode->consumers, s_hook) + count++; + + if (!sup_dev) + return count; + + list_for_each_entry_safe(dlink, dtmp, &sup_dev->links.consumers, s_node) + if (dlink->consumer->links.status != DL_DEV_DRIVER_BOUND) + count++; + + return count; +} +EXPORT_SYMBOL_GPL(fw_devlink_count_absent_consumers); + /* Device links support end. */ static struct kobject *dev_kobj; @@ -2556,6 +2582,7 @@ static void device_release(struct kobject *kobj) devres_release_all(dev); kfree(dev->dma_range_map); + kfree(dev->driver_override.name); if (dev->release) dev->release(dev); @@ -3159,6 +3186,7 @@ void device_initialize(struct device *dev) kobject_init(&dev->kobj, &device_ktype); INIT_LIST_HEAD(&dev->dma_pools); mutex_init(&dev->mutex); + spin_lock_init(&dev->driver_override.lock); lockdep_set_novalidate_class(&dev->mutex); spin_lock_init(&dev->devres_lock); INIT_LIST_HEAD(&dev->devres_head); diff --git a/drivers/base/dd.c b/drivers/base/dd.c index bea8da5f8a3a92..37c7e54e0e4c74 100644 --- a/drivers/base/dd.c +++ b/drivers/base/dd.c @@ -381,6 +381,66 @@ static void __exit deferred_probe_exit(void) } __exitcall(deferred_probe_exit); +int __device_set_driver_override(struct device *dev, const char *s, size_t len) +{ + const char *new, *old; + char *cp; + + if (!s) + return -EINVAL; + + /* + * The stored value will be used in sysfs show callback (sysfs_emit()), + * which has a length limit of PAGE_SIZE and adds a trailing newline. + * Thus we can store one character less to avoid truncation during sysfs + * show. + */ + if (len >= (PAGE_SIZE - 1)) + return -EINVAL; + + /* + * Compute the real length of the string in case userspace sends us a + * bunch of \0 characters like python likes to do. + */ + len = strlen(s); + + if (!len) { + /* Empty string passed - clear override */ + spin_lock(&dev->driver_override.lock); + old = dev->driver_override.name; + dev->driver_override.name = NULL; + spin_unlock(&dev->driver_override.lock); + kfree(old); + + return 0; + } + + cp = strnchr(s, len, '\n'); + if (cp) + len = cp - s; + + new = kstrndup(s, len, GFP_KERNEL); + if (!new) + return -ENOMEM; + + spin_lock(&dev->driver_override.lock); + old = dev->driver_override.name; + if (cp != s) { + dev->driver_override.name = new; + spin_unlock(&dev->driver_override.lock); + } else { + /* "\n" passed - clear override */ + dev->driver_override.name = NULL; + spin_unlock(&dev->driver_override.lock); + + kfree(new); + } + kfree(old); + + return 0; +} +EXPORT_SYMBOL_GPL(__device_set_driver_override); + /** * device_is_bound() - Check if device is bound to a driver * @dev: device to check diff --git a/drivers/base/faux.c b/drivers/base/faux.c index 21dd02124231a9..23d72581723256 100644 --- a/drivers/base/faux.c +++ b/drivers/base/faux.c @@ -29,9 +29,7 @@ struct faux_object { }; #define to_faux_object(dev) container_of_const(dev, struct faux_object, faux_dev.dev) -static struct device faux_bus_root = { - .init_name = "faux", -}; +static struct device *faux_bus_root; static int faux_match(struct device *dev, const struct device_driver *drv) { @@ -152,7 +150,7 @@ struct faux_device *faux_device_create_with_groups(const char *name, if (parent) dev->parent = parent; else - dev->parent = &faux_bus_root; + dev->parent = faux_bus_root; dev->bus = &faux_bus_type; dev_set_name(dev, "%s", name); device_set_pm_not_required(dev); @@ -236,9 +234,15 @@ int __init faux_bus_init(void) { int ret; - ret = device_register(&faux_bus_root); + faux_bus_root = kzalloc(sizeof(*faux_bus_root), GFP_KERNEL); + if (!faux_bus_root) + return -ENOMEM; + + dev_set_name(faux_bus_root, "faux"); + + ret = device_register(faux_bus_root); if (ret) { - put_device(&faux_bus_root); + put_device(faux_bus_root); return ret; } @@ -256,6 +260,6 @@ int __init faux_bus_init(void) bus_unregister(&faux_bus_type); error_bus: - device_unregister(&faux_bus_root); + device_unregister(faux_bus_root); return ret; } diff --git a/drivers/base/firmware_loader/main.c b/drivers/base/firmware_loader/main.c index 4ebdca9e4da47f..b6aabfc70732de 100644 --- a/drivers/base/firmware_loader/main.c +++ b/drivers/base/firmware_loader/main.c @@ -471,6 +471,8 @@ static int fw_decompress_xz(struct device *dev, struct fw_priv *fw_priv, static char fw_path_para[256]; static const char * const fw_path[] = { fw_path_para, + "/lib/firmware/vendor/" UTS_RELEASE, + "/lib/firmware/vendor", "/lib/firmware/updates/" UTS_RELEASE, "/lib/firmware/updates", "/lib/firmware/" UTS_RELEASE, diff --git a/drivers/base/platform.c b/drivers/base/platform.c index b45d41b018ca6d..d44591d52e3633 100644 --- a/drivers/base/platform.c +++ b/drivers/base/platform.c @@ -603,7 +603,6 @@ static void platform_device_release(struct device *dev) kfree(pa->pdev.dev.platform_data); kfree(pa->pdev.mfd_cell); kfree(pa->pdev.resource); - kfree(pa->pdev.driver_override); kfree(pa); } @@ -1306,38 +1305,9 @@ static ssize_t numa_node_show(struct device *dev, } static DEVICE_ATTR_RO(numa_node); -static ssize_t driver_override_show(struct device *dev, - struct device_attribute *attr, char *buf) -{ - struct platform_device *pdev = to_platform_device(dev); - ssize_t len; - - device_lock(dev); - len = sysfs_emit(buf, "%s\n", pdev->driver_override); - device_unlock(dev); - - return len; -} - -static ssize_t driver_override_store(struct device *dev, - struct device_attribute *attr, - const char *buf, size_t count) -{ - struct platform_device *pdev = to_platform_device(dev); - int ret; - - ret = driver_set_override(dev, &pdev->driver_override, buf, count); - if (ret) - return ret; - - return count; -} -static DEVICE_ATTR_RW(driver_override); - static struct attribute *platform_dev_attrs[] = { &dev_attr_modalias.attr, &dev_attr_numa_node.attr, - &dev_attr_driver_override.attr, NULL, }; @@ -1377,10 +1347,12 @@ static int platform_match(struct device *dev, const struct device_driver *drv) { struct platform_device *pdev = to_platform_device(dev); struct platform_driver *pdrv = to_platform_driver(drv); + int ret; /* When driver_override is set, only bind to the matching driver */ - if (pdev->driver_override) - return !strcmp(pdev->driver_override, drv->name); + ret = device_match_driver_override(dev, drv); + if (ret >= 0) + return ret; /* Attempt an OF style match first */ if (of_driver_match_device(dev, drv)) @@ -1516,6 +1488,7 @@ static const struct dev_pm_ops platform_dev_pm_ops = { const struct bus_type platform_bus_type = { .name = "platform", .dev_groups = platform_dev_groups, + .driver_override = true, .match = platform_match, .uevent = platform_uevent, .probe = platform_probe, diff --git a/drivers/base/power/runtime.c b/drivers/base/power/runtime.c index 0ee8ea971aa468..335288e8b5b312 100644 --- a/drivers/base/power/runtime.c +++ b/drivers/base/power/runtime.c @@ -1895,6 +1895,7 @@ void pm_runtime_reinit(struct device *dev) void pm_runtime_remove(struct device *dev) { __pm_runtime_disable(dev, false); + flush_work(&dev->power.work); pm_runtime_reinit(dev); } diff --git a/drivers/base/power/wakeirq.c b/drivers/base/power/wakeirq.c index 8aa28c08b2891f..c0809d18fc5406 100644 --- a/drivers/base/power/wakeirq.c +++ b/drivers/base/power/wakeirq.c @@ -83,13 +83,16 @@ EXPORT_SYMBOL_GPL(dev_pm_set_wake_irq); */ void dev_pm_clear_wake_irq(struct device *dev) { - struct wake_irq *wirq = dev->power.wakeirq; + struct wake_irq *wirq; unsigned long flags; - if (!wirq) + spin_lock_irqsave(&dev->power.lock, flags); + wirq = dev->power.wakeirq; + if (!wirq) { + spin_unlock_irqrestore(&dev->power.lock, flags); return; + } - spin_lock_irqsave(&dev->power.lock, flags); device_wakeup_detach_irq(dev); dev->power.wakeirq = NULL; spin_unlock_irqrestore(&dev->power.lock, flags); diff --git a/drivers/base/power/wakeup.c b/drivers/base/power/wakeup.c index 1e1a0e7eeac5fa..e69033d16fba04 100644 --- a/drivers/base/power/wakeup.c +++ b/drivers/base/power/wakeup.c @@ -275,9 +275,7 @@ EXPORT_SYMBOL_GPL(wakeup_sources_read_unlock); */ struct wakeup_source *wakeup_sources_walk_start(void) { - struct list_head *ws_head = &wakeup_sources; - - return list_entry_rcu(ws_head->next, struct wakeup_source, entry); + return list_first_or_null_rcu(&wakeup_sources, struct wakeup_source, entry); } EXPORT_SYMBOL_GPL(wakeup_sources_walk_start); diff --git a/drivers/base/property.c b/drivers/base/property.c index 6a63860579dd31..8d9a34be57fbf1 100644 --- a/drivers/base/property.c +++ b/drivers/base/property.c @@ -797,7 +797,18 @@ struct fwnode_handle * fwnode_get_next_child_node(const struct fwnode_handle *fwnode, struct fwnode_handle *child) { - return fwnode_call_ptr_op(fwnode, get_next_child_node, child); + struct fwnode_handle *next; + + if (IS_ERR_OR_NULL(fwnode)) + return NULL; + + /* Try to find a child in primary fwnode */ + next = fwnode_call_ptr_op(fwnode, get_next_child_node, child); + if (next) + return next; + + /* When no more children in primary, continue with secondary */ + return fwnode_call_ptr_op(fwnode->secondary, get_next_child_node, child); } EXPORT_SYMBOL_GPL(fwnode_get_next_child_node); @@ -841,19 +852,7 @@ EXPORT_SYMBOL_GPL(fwnode_get_next_available_child_node); struct fwnode_handle *device_get_next_child_node(const struct device *dev, struct fwnode_handle *child) { - const struct fwnode_handle *fwnode = dev_fwnode(dev); - struct fwnode_handle *next; - - if (IS_ERR_OR_NULL(fwnode)) - return NULL; - - /* Try to find a child in primary fwnode */ - next = fwnode_get_next_child_node(fwnode, child); - if (next) - return next; - - /* When no more children in primary, continue with secondary */ - return fwnode_get_next_child_node(fwnode->secondary, child); + return fwnode_get_next_child_node(dev_fwnode(dev), child); } EXPORT_SYMBOL_GPL(device_get_next_child_node); diff --git a/drivers/base/regmap/regmap.c b/drivers/base/regmap/regmap.c index ae2215d4e61c3e..a6482185072364 100644 --- a/drivers/base/regmap/regmap.c +++ b/drivers/base/regmap/regmap.c @@ -1543,6 +1543,7 @@ static int _regmap_select_page(struct regmap *map, unsigned int *reg, unsigned int val_num) { void *orig_work_buf; + unsigned int selector_reg; unsigned int win_offset; unsigned int win_page; bool page_chg; @@ -1561,10 +1562,31 @@ static int _regmap_select_page(struct regmap *map, unsigned int *reg, return -EINVAL; } - /* It is possible to have selector register inside data window. - In that case, selector register is located on every page and - it needs no page switching, when accessed alone. */ + /* + * Calculate the address of the selector register in the corresponding + * data window if it is located on every page. + */ + page_chg = in_range(range->selector_reg, range->window_start, range->window_len); + if (page_chg) + selector_reg = range->range_min + win_page * range->window_len + + range->selector_reg - range->window_start; + + /* + * It is possible to have selector register inside data window. + * In that case, selector register is located on every page and it + * needs no page switching, when accessed alone. + * + * Nevertheless we should synchronize the cache values for it. + * This can't be properly achieved if the selector register is + * the first and the only one to be read inside the data window. + * That's why we update it in that case as well. + * + * However, we specifically avoid updating it for the default page, + * when it's overlapped with the real data window, to prevent from + * infinite looping. + */ if (val_num > 1 || + (page_chg && selector_reg != range->selector_reg) || range->window_start + win_offset != range->selector_reg) { /* Use separate work_buf during page switching */ orig_work_buf = map->work_buf; @@ -1573,7 +1595,7 @@ static int _regmap_select_page(struct regmap *map, unsigned int *reg, ret = _regmap_update_bits(map, range->selector_reg, range->selector_mask, win_page << range->selector_shift, - &page_chg, false); + NULL, false); map->work_buf = orig_work_buf; diff --git a/drivers/block/drbd/drbd_actlog.c b/drivers/block/drbd/drbd_actlog.c index 742b2908ff686a..b3dbf6c76e98f4 100644 --- a/drivers/block/drbd/drbd_actlog.c +++ b/drivers/block/drbd/drbd_actlog.c @@ -483,38 +483,20 @@ void drbd_al_begin_io(struct drbd_device *device, struct drbd_interval *i) int drbd_al_begin_io_nonblock(struct drbd_device *device, struct drbd_interval *i) { - struct lru_cache *al = device->act_log; /* for bios crossing activity log extent boundaries, * we may need to activate two extents in one go */ unsigned first = i->sector >> (AL_EXTENT_SHIFT-9); unsigned last = i->size == 0 ? first : (i->sector + (i->size >> 9) - 1) >> (AL_EXTENT_SHIFT-9); - unsigned nr_al_extents; - unsigned available_update_slots; unsigned enr; - D_ASSERT(device, first <= last); - - nr_al_extents = 1 + last - first; /* worst case: all touched extends are cold. */ - available_update_slots = min(al->nr_elements - al->used, - al->max_pending_changes - al->pending_changes); - - /* We want all necessary updates for a given request within the same transaction - * We could first check how many updates are *actually* needed, - * and use that instead of the worst-case nr_al_extents */ - if (available_update_slots < nr_al_extents) { - /* Too many activity log extents are currently "hot". - * - * If we have accumulated pending changes already, - * we made progress. - * - * If we cannot get even a single pending change through, - * stop the fast path until we made some progress, - * or requests to "cold" extents could be starved. */ - if (!al->pending_changes) - __set_bit(__LC_STARVING, &device->act_log->flags); - return -ENOBUFS; + if (i->partially_in_al_next_enr) { + D_ASSERT(device, first < i->partially_in_al_next_enr); + D_ASSERT(device, last >= i->partially_in_al_next_enr); + first = i->partially_in_al_next_enr; } + D_ASSERT(device, first <= last); + /* Is resync active in this area? */ for (enr = first; enr <= last; enr++) { struct lc_element *tmp; @@ -529,14 +511,21 @@ int drbd_al_begin_io_nonblock(struct drbd_device *device, struct drbd_interval * } } - /* Checkout the refcounts. - * Given that we checked for available elements and update slots above, - * this has to be successful. */ + /* Try to checkout the refcounts. */ for (enr = first; enr <= last; enr++) { struct lc_element *al_ext; al_ext = lc_get_cumulative(device->act_log, enr); - if (!al_ext) - drbd_info(device, "LOGIC BUG for enr=%u\n", enr); + + if (!al_ext) { + /* Did not work. We may have exhausted the possible + * changes per transaction. Or raced with someone + * "locking" it against changes. + * Remember where to continue from. + */ + if (enr > first) + i->partially_in_al_next_enr = enr; + return -ENOBUFS; + } } return 0; } @@ -556,7 +545,11 @@ void drbd_al_complete_io(struct drbd_device *device, struct drbd_interval *i) for (enr = first; enr <= last; enr++) { extent = lc_find(device->act_log, enr); - if (!extent) { + /* Yes, this masks a bug elsewhere. However, during normal + * operation this is harmless, so no need to crash the kernel + * by the BUG_ON(refcount == 0) in lc_put(). + */ + if (!extent || extent->refcnt == 0) { drbd_err(device, "al_complete_io() called on inactive extent %u\n", enr); continue; } diff --git a/drivers/block/drbd/drbd_interval.h b/drivers/block/drbd/drbd_interval.h index 366489b72fe971..5d3213b81eede7 100644 --- a/drivers/block/drbd/drbd_interval.h +++ b/drivers/block/drbd/drbd_interval.h @@ -8,12 +8,15 @@ struct drbd_interval { struct rb_node rb; sector_t sector; /* start sector of the interval */ - unsigned int size; /* size in bytes */ sector_t end; /* highest interval end in subtree */ + unsigned int size; /* size in bytes */ unsigned int local:1 /* local or remote request? */; unsigned int waiting:1; /* someone is waiting for completion */ unsigned int completed:1; /* this has been completed already; * ignore for conflict detection */ + + /* to resume a partially successful drbd_al_begin_io_nonblock(); */ + unsigned int partially_in_al_next_enr; }; static inline void drbd_clear_interval(struct drbd_interval *i) diff --git a/drivers/block/drbd/drbd_main.c b/drivers/block/drbd/drbd_main.c index c73376886e7a51..1f6ac9202b66a1 100644 --- a/drivers/block/drbd/drbd_main.c +++ b/drivers/block/drbd/drbd_main.c @@ -2659,9 +2659,6 @@ enum drbd_ret_code drbd_create_device(struct drbd_config_context *adm_ctx, unsig * connect. */ .max_hw_sectors = DRBD_MAX_BIO_SIZE_SAFE >> 8, - .features = BLK_FEAT_WRITE_CACHE | BLK_FEAT_FUA | - BLK_FEAT_ROTATIONAL | - BLK_FEAT_STABLE_WRITES, }; device = minor_to_device(minor); diff --git a/drivers/block/drbd/drbd_nl.c b/drivers/block/drbd/drbd_nl.c index 91f3b8afb63ce6..b502038be0a923 100644 --- a/drivers/block/drbd/drbd_nl.c +++ b/drivers/block/drbd/drbd_nl.c @@ -1296,6 +1296,8 @@ void drbd_reconsider_queue_parameters(struct drbd_device *device, lim.max_segments = drbd_backing_dev_max_segments(device); } else { lim.max_segments = BLK_MAX_SEGMENTS; + lim.features = BLK_FEAT_WRITE_CACHE | BLK_FEAT_FUA | + BLK_FEAT_ROTATIONAL | BLK_FEAT_STABLE_WRITES; } lim.max_hw_sectors = new >> SECTOR_SHIFT; @@ -1318,8 +1320,24 @@ void drbd_reconsider_queue_parameters(struct drbd_device *device, lim.max_hw_discard_sectors = 0; } - if (bdev) + if (bdev) { blk_stack_limits(&lim, &b->limits, 0); + /* + * blk_set_stacking_limits() cleared the features, and + * blk_stack_limits() may or may not have inherited + * BLK_FEAT_STABLE_WRITES from the backing device. + * + * DRBD always requires stable writes because: + * 1. The same bio data is read for both local disk I/O and + * network transmission. If the page changes mid-flight, + * the local and remote copies could diverge. + * 2. When data integrity is enabled, DRBD calculates a + * checksum before sending the data. If the page changes + * between checksum calculation and transmission, the + * receiver will detect a checksum mismatch. + */ + lim.features |= BLK_FEAT_STABLE_WRITES; + } /* * If we can handle "zeroes" efficiently on the protocol, we want to do diff --git a/drivers/block/drbd/drbd_req.c b/drivers/block/drbd/drbd_req.c index d15826f6ee81d4..70f75ef079457d 100644 --- a/drivers/block/drbd/drbd_req.c +++ b/drivers/block/drbd/drbd_req.c @@ -621,7 +621,8 @@ int __req_mod(struct drbd_request *req, enum drbd_req_event what, break; case READ_COMPLETED_WITH_ERROR: - drbd_set_out_of_sync(peer_device, req->i.sector, req->i.size); + drbd_set_out_of_sync(first_peer_device(device), + req->i.sector, req->i.size); drbd_report_io_error(device, req); __drbd_chk_io_error(device, DRBD_READ_ERROR); fallthrough; diff --git a/drivers/block/rnbd/rnbd-srv.c b/drivers/block/rnbd/rnbd-srv.c index 2df8941a6b1463..7eeb321d61402f 100644 --- a/drivers/block/rnbd/rnbd-srv.c +++ b/drivers/block/rnbd/rnbd-srv.c @@ -145,18 +145,30 @@ static int process_rdma(struct rnbd_srv_session *srv_sess, priv->sess_dev = sess_dev; priv->id = id; - bio = bio_alloc(file_bdev(sess_dev->bdev_file), 1, + bio = bio_alloc(file_bdev(sess_dev->bdev_file), !!datalen, rnbd_to_bio_flags(le32_to_cpu(msg->rw)), GFP_KERNEL); - bio_add_virt_nofail(bio, data, datalen); - - bio->bi_opf = rnbd_to_bio_flags(le32_to_cpu(msg->rw)); - if (bio_has_data(bio) && - bio->bi_iter.bi_size != le32_to_cpu(msg->bi_size)) { - rnbd_srv_err_rl(sess_dev, "Datalen mismatch: bio bi_size (%u), bi_size (%u)\n", - bio->bi_iter.bi_size, msg->bi_size); - err = -EINVAL; - goto bio_put; + if (unlikely(!bio)) { + err = -ENOMEM; + goto put_sess_dev; } + + if (!datalen) { + /* + * For special requests like DISCARD and WRITE_ZEROES, the datalen is zero. + */ + bio->bi_iter.bi_size = le32_to_cpu(msg->bi_size); + } else { + bio_add_virt_nofail(bio, data, datalen); + bio->bi_opf = rnbd_to_bio_flags(le32_to_cpu(msg->rw)); + if (bio->bi_iter.bi_size != le32_to_cpu(msg->bi_size)) { + rnbd_srv_err_rl(sess_dev, + "Datalen mismatch: bio bi_size (%u), bi_size (%u)\n", + bio->bi_iter.bi_size, msg->bi_size); + err = -EINVAL; + goto bio_put; + } + } + bio->bi_end_io = rnbd_dev_bi_end_io; bio->bi_private = priv; bio->bi_iter.bi_sector = le64_to_cpu(msg->sector); @@ -170,6 +182,7 @@ static int process_rdma(struct rnbd_srv_session *srv_sess, bio_put: bio_put(bio); +put_sess_dev: rnbd_put_sess_dev(sess_dev); err: kfree(priv); @@ -538,6 +551,8 @@ static void rnbd_srv_fill_msg_open_rsp(struct rnbd_msg_open_rsp *rsp, { struct block_device *bdev = file_bdev(sess_dev->bdev_file); + memset(rsp, 0, sizeof(*rsp)); + rsp->hdr.type = cpu_to_le16(RNBD_MSG_OPEN_RSP); rsp->device_id = cpu_to_le32(sess_dev->device_id); rsp->nsectors = cpu_to_le64(bdev_nr_sectors(bdev)); @@ -644,6 +659,7 @@ static void process_msg_sess_info(struct rnbd_srv_session *srv_sess, trace_process_msg_sess_info(srv_sess, sess_info_msg); + memset(rsp, 0, sizeof(*rsp)); rsp->hdr.type = cpu_to_le16(RNBD_MSG_SESS_INFO_RSP); rsp->ver = srv_sess->ver; } diff --git a/drivers/block/ublk_drv.c b/drivers/block/ublk_drv.c index cd1e84653002d5..60053af9242ea7 100644 --- a/drivers/block/ublk_drv.c +++ b/drivers/block/ublk_drv.c @@ -2462,11 +2462,11 @@ static int ublk_ch_uring_cmd_local(struct io_uring_cmd *cmd, io->res = result; req = ublk_fill_io_cmd(io, cmd); ret = ublk_config_io_buf(ub, io, cmd, addr, &buf_idx); + if (buf_idx != UBLK_INVALID_BUF_IDX) + io_buffer_unregister_bvec(cmd, buf_idx, issue_flags); compl = ublk_need_complete_req(ub, io); /* can't touch 'ublk_io' any more */ - if (buf_idx != UBLK_INVALID_BUF_IDX) - io_buffer_unregister_bvec(cmd, buf_idx, issue_flags); if (req_op(req) == REQ_OP_ZONE_APPEND) req->__sector = addr; if (compl) @@ -3311,12 +3311,11 @@ static int ublk_ctrl_del_dev(struct ublk_device **p_ub, bool wait) return 0; } -static inline void ublk_ctrl_cmd_dump(struct io_uring_cmd *cmd) +static inline void ublk_ctrl_cmd_dump(u32 cmd_op, + const struct ublksrv_ctrl_cmd *header) { - const struct ublksrv_ctrl_cmd *header = io_uring_sqe_cmd(cmd->sqe); - pr_devel("%s: cmd_op %x, dev id %d qid %d data %llx buf %llx len %u\n", - __func__, cmd->cmd_op, header->dev_id, header->queue_id, + __func__, cmd_op, header->dev_id, header->queue_id, header->data[0], header->addr, header->len); } @@ -3535,15 +3534,22 @@ static int ublk_ctrl_get_features(const struct ublksrv_ctrl_cmd *header) return 0; } -static void ublk_ctrl_set_size(struct ublk_device *ub, const struct ublksrv_ctrl_cmd *header) +static int ublk_ctrl_set_size(struct ublk_device *ub, const struct ublksrv_ctrl_cmd *header) { struct ublk_param_basic *p = &ub->params.basic; u64 new_size = header->data[0]; + int ret = 0; mutex_lock(&ub->mutex); + if (!ub->ub_disk) { + ret = -ENODEV; + goto out; + } p->dev_sectors = new_size; set_capacity_and_notify(ub->ub_disk, p->dev_sectors); +out: mutex_unlock(&ub->mutex); + return ret; } struct count_busy { @@ -3685,9 +3691,8 @@ static int ublk_char_dev_permission(struct ublk_device *ub, } static int ublk_ctrl_uring_cmd_permission(struct ublk_device *ub, - struct io_uring_cmd *cmd) + u32 cmd_op, struct ublksrv_ctrl_cmd *header) { - struct ublksrv_ctrl_cmd *header = (struct ublksrv_ctrl_cmd *)io_uring_sqe_cmd(cmd->sqe); bool unprivileged = ub->dev_info.flags & UBLK_F_UNPRIVILEGED_DEV; void __user *argp = (void __user *)(unsigned long)header->addr; char *dev_path = NULL; @@ -3703,7 +3708,7 @@ static int ublk_ctrl_uring_cmd_permission(struct ublk_device *ub, * know if the specified device is created as unprivileged * mode. */ - if (_IOC_NR(cmd->cmd_op) != UBLK_CMD_GET_DEV_INFO2) + if (_IOC_NR(cmd_op) != UBLK_CMD_GET_DEV_INFO2) return 0; } @@ -3724,7 +3729,7 @@ static int ublk_ctrl_uring_cmd_permission(struct ublk_device *ub, return PTR_ERR(dev_path); ret = -EINVAL; - switch (_IOC_NR(cmd->cmd_op)) { + switch (_IOC_NR(cmd_op)) { case UBLK_CMD_GET_DEV_INFO: case UBLK_CMD_GET_DEV_INFO2: case UBLK_CMD_GET_QUEUE_AFFINITY: @@ -3753,7 +3758,7 @@ static int ublk_ctrl_uring_cmd_permission(struct ublk_device *ub, header->addr += header->dev_path_len; } pr_devel("%s: dev id %d cmd_op %x uid %d gid %d path %s ret %d\n", - __func__, ub->ub_number, cmd->cmd_op, + __func__, ub->ub_number, cmd_op, ub->dev_info.owner_uid, ub->dev_info.owner_gid, dev_path, ret); exit: @@ -3777,7 +3782,9 @@ static bool ublk_ctrl_uring_cmd_may_sleep(u32 cmd_op) static int ublk_ctrl_uring_cmd(struct io_uring_cmd *cmd, unsigned int issue_flags) { - const struct ublksrv_ctrl_cmd *header = io_uring_sqe_cmd(cmd->sqe); + /* May point to userspace-mapped memory */ + const struct ublksrv_ctrl_cmd *ub_src = io_uring_sqe_cmd(cmd->sqe); + struct ublksrv_ctrl_cmd header; struct ublk_device *ub = NULL; u32 cmd_op = cmd->cmd_op; int ret = -EINVAL; @@ -3786,44 +3793,50 @@ static int ublk_ctrl_uring_cmd(struct io_uring_cmd *cmd, issue_flags & IO_URING_F_NONBLOCK) return -EAGAIN; - ublk_ctrl_cmd_dump(cmd); - if (!(issue_flags & IO_URING_F_SQE128)) - goto out; + return -EINVAL; + + header.dev_id = READ_ONCE(ub_src->dev_id); + header.queue_id = READ_ONCE(ub_src->queue_id); + header.len = READ_ONCE(ub_src->len); + header.addr = READ_ONCE(ub_src->addr); + header.data[0] = READ_ONCE(ub_src->data[0]); + header.dev_path_len = READ_ONCE(ub_src->dev_path_len); + ublk_ctrl_cmd_dump(cmd_op, &header); ret = ublk_check_cmd_op(cmd_op); if (ret) goto out; if (cmd_op == UBLK_U_CMD_GET_FEATURES) { - ret = ublk_ctrl_get_features(header); + ret = ublk_ctrl_get_features(&header); goto out; } if (_IOC_NR(cmd_op) != UBLK_CMD_ADD_DEV) { ret = -ENODEV; - ub = ublk_get_device_from_id(header->dev_id); + ub = ublk_get_device_from_id(header.dev_id); if (!ub) goto out; - ret = ublk_ctrl_uring_cmd_permission(ub, cmd); + ret = ublk_ctrl_uring_cmd_permission(ub, cmd_op, &header); if (ret) goto put_dev; } switch (_IOC_NR(cmd_op)) { case UBLK_CMD_START_DEV: - ret = ublk_ctrl_start_dev(ub, header); + ret = ublk_ctrl_start_dev(ub, &header); break; case UBLK_CMD_STOP_DEV: ret = ublk_ctrl_stop_dev(ub); break; case UBLK_CMD_GET_DEV_INFO: case UBLK_CMD_GET_DEV_INFO2: - ret = ublk_ctrl_get_dev_info(ub, header); + ret = ublk_ctrl_get_dev_info(ub, &header); break; case UBLK_CMD_ADD_DEV: - ret = ublk_ctrl_add_dev(header); + ret = ublk_ctrl_add_dev(&header); break; case UBLK_CMD_DEL_DEV: ret = ublk_ctrl_del_dev(&ub, true); @@ -3832,26 +3845,25 @@ static int ublk_ctrl_uring_cmd(struct io_uring_cmd *cmd, ret = ublk_ctrl_del_dev(&ub, false); break; case UBLK_CMD_GET_QUEUE_AFFINITY: - ret = ublk_ctrl_get_queue_affinity(ub, header); + ret = ublk_ctrl_get_queue_affinity(ub, &header); break; case UBLK_CMD_GET_PARAMS: - ret = ublk_ctrl_get_params(ub, header); + ret = ublk_ctrl_get_params(ub, &header); break; case UBLK_CMD_SET_PARAMS: - ret = ublk_ctrl_set_params(ub, header); + ret = ublk_ctrl_set_params(ub, &header); break; case UBLK_CMD_START_USER_RECOVERY: - ret = ublk_ctrl_start_recovery(ub, header); + ret = ublk_ctrl_start_recovery(ub, &header); break; case UBLK_CMD_END_USER_RECOVERY: - ret = ublk_ctrl_end_recovery(ub, header); + ret = ublk_ctrl_end_recovery(ub, &header); break; case UBLK_CMD_UPDATE_SIZE: - ublk_ctrl_set_size(ub, header); - ret = 0; + ret = ublk_ctrl_set_size(ub, &header); break; case UBLK_CMD_QUIESCE_DEV: - ret = ublk_ctrl_quiesce_dev(ub, header); + ret = ublk_ctrl_quiesce_dev(ub, &header); break; default: ret = -EOPNOTSUPP; @@ -3863,7 +3875,7 @@ static int ublk_ctrl_uring_cmd(struct io_uring_cmd *cmd, ublk_put_device(ub); out: pr_devel("%s: cmd done ret %d cmd_op %x, dev id %d qid %d\n", - __func__, ret, cmd->cmd_op, header->dev_id, header->queue_id); + __func__, ret, cmd_op, header.dev_id, header.queue_id); return ret; } diff --git a/drivers/block/zloop.c b/drivers/block/zloop.c index 8e334f5025fc0a..9e3bb538d5fcfc 100644 --- a/drivers/block/zloop.c +++ b/drivers/block/zloop.c @@ -542,6 +542,21 @@ static void zloop_rw(struct zloop_cmd *cmd) zloop_put_cmd(cmd); } +/* + * Sync the entire FS containing the zone files instead of walking all files. + */ +static int zloop_flush(struct zloop_device *zlo) +{ + struct super_block *sb = file_inode(zlo->data_dir)->i_sb; + int ret; + + down_read(&sb->s_umount); + ret = sync_filesystem(sb); + up_read(&sb->s_umount); + + return ret; +} + static void zloop_handle_cmd(struct zloop_cmd *cmd) { struct request *rq = blk_mq_rq_from_pdu(cmd); @@ -562,11 +577,7 @@ static void zloop_handle_cmd(struct zloop_cmd *cmd) zloop_rw(cmd); return; case REQ_OP_FLUSH: - /* - * Sync the entire FS containing the zone files instead of - * walking all files - */ - cmd->ret = sync_filesystem(file_inode(zlo->data_dir)->i_sb); + cmd->ret = zloop_flush(zlo); break; case REQ_OP_ZONE_RESET: cmd->ret = zloop_reset_zone(zlo, rq_zone_no(rq)); @@ -981,7 +992,8 @@ static int zloop_ctl_add(struct zloop_options *opts) struct queue_limits lim = { .max_hw_sectors = SZ_1M >> SECTOR_SHIFT, .chunk_sectors = opts->zone_size, - .features = BLK_FEAT_ZONED, + .features = BLK_FEAT_ZONED | BLK_FEAT_WRITE_CACHE, + }; unsigned int nr_zones, i, j; struct zloop_device *zlo; @@ -1162,7 +1174,12 @@ static int zloop_ctl_remove(struct zloop_options *opts) int ret; if (!(opts->mask & ZLOOP_OPT_ID)) { - pr_err("No ID specified\n"); + pr_err("No ID specified for remove\n"); + return -EINVAL; + } + + if (opts->mask & ~ZLOOP_OPT_ID) { + pr_err("Invalid option specified for remove\n"); return -EINVAL; } diff --git a/drivers/bluetooth/btintel.c b/drivers/bluetooth/btintel.c index 9d29ab811f802a..5e0a05edcbfd10 100644 --- a/drivers/bluetooth/btintel.c +++ b/drivers/bluetooth/btintel.c @@ -251,11 +251,13 @@ void btintel_hw_error(struct hci_dev *hdev, u8 code) bt_dev_err(hdev, "Hardware error 0x%2.2x", code); + hci_req_sync_lock(hdev); + skb = __hci_cmd_sync(hdev, HCI_OP_RESET, 0, NULL, HCI_INIT_TIMEOUT); if (IS_ERR(skb)) { bt_dev_err(hdev, "Reset after hardware error failed (%ld)", PTR_ERR(skb)); - return; + goto unlock; } kfree_skb(skb); @@ -263,18 +265,21 @@ void btintel_hw_error(struct hci_dev *hdev, u8 code) if (IS_ERR(skb)) { bt_dev_err(hdev, "Retrieving Intel exception info failed (%ld)", PTR_ERR(skb)); - return; + goto unlock; } if (skb->len != 13) { bt_dev_err(hdev, "Exception info size mismatch"); kfree_skb(skb); - return; + goto unlock; } bt_dev_err(hdev, "Exception info %s", (char *)(skb->data + 1)); kfree_skb(skb); + +unlock: + hci_req_sync_unlock(hdev); } EXPORT_SYMBOL_GPL(btintel_hw_error); diff --git a/drivers/bluetooth/btintel_pcie.c b/drivers/bluetooth/btintel_pcie.c index 2936b535479f26..704767b334b98b 100644 --- a/drivers/bluetooth/btintel_pcie.c +++ b/drivers/bluetooth/btintel_pcie.c @@ -1431,11 +1431,6 @@ static void btintel_pcie_msix_rx_handle(struct btintel_pcie_data *data) } } -static irqreturn_t btintel_pcie_msix_isr(int irq, void *data) -{ - return IRQ_WAKE_THREAD; -} - static inline bool btintel_pcie_is_rxq_empty(struct btintel_pcie_data *data) { return data->ia.cr_hia[BTINTEL_PCIE_RXQ_NUM] == data->ia.cr_tia[BTINTEL_PCIE_RXQ_NUM]; @@ -1537,9 +1532,9 @@ static int btintel_pcie_setup_irq(struct btintel_pcie_data *data) err = devm_request_threaded_irq(&data->pdev->dev, msix_entry->vector, - btintel_pcie_msix_isr, + NULL, btintel_pcie_irq_msix_handler, - IRQF_SHARED, + IRQF_ONESHOT | IRQF_SHARED, KBUILD_MODNAME, msix_entry); if (err) { diff --git a/drivers/bluetooth/btqca.c b/drivers/bluetooth/btqca.c index 7c958d6065bec1..86a48d009d1ba2 100644 --- a/drivers/bluetooth/btqca.c +++ b/drivers/bluetooth/btqca.c @@ -804,6 +804,8 @@ int qca_uart_setup(struct hci_dev *hdev, uint8_t baudrate, */ if (soc_type == QCA_WCN3988) rom_ver = ((soc_ver & 0x00000f00) >> 0x05) | (soc_ver & 0x0000000f); + else if (soc_type == QCA_WCN3998) + rom_ver = ((soc_ver & 0x0000f000) >> 0x07) | (soc_ver & 0x0000000f); else rom_ver = ((soc_ver & 0x00000f00) >> 0x04) | (soc_ver & 0x0000000f); diff --git a/drivers/bluetooth/btusb.c b/drivers/bluetooth/btusb.c index ded09e94d296df..4e161dcca00d8e 100644 --- a/drivers/bluetooth/btusb.c +++ b/drivers/bluetooth/btusb.c @@ -521,12 +521,16 @@ static const struct usb_device_id quirks_table[] = { { USB_DEVICE(0x0bda, 0xb850), .driver_info = BTUSB_REALTEK }, { USB_DEVICE(0x13d3, 0x3600), .driver_info = BTUSB_REALTEK }, { USB_DEVICE(0x13d3, 0x3601), .driver_info = BTUSB_REALTEK }, + { USB_DEVICE(0x0489, 0xe112), .driver_info = BTUSB_REALTEK | + BTUSB_WIDEBAND_SPEECH }, /* Realtek 8851BU Bluetooth devices */ { USB_DEVICE(0x3625, 0x010b), .driver_info = BTUSB_REALTEK | BTUSB_WIDEBAND_SPEECH }, { USB_DEVICE(0x2001, 0x332a), .driver_info = BTUSB_REALTEK | BTUSB_WIDEBAND_SPEECH }, + { USB_DEVICE(0x7392, 0xe611), .driver_info = BTUSB_REALTEK | + BTUSB_WIDEBAND_SPEECH }, /* Realtek 8852AE Bluetooth devices */ { USB_DEVICE(0x0bda, 0x2852), .driver_info = BTUSB_REALTEK | @@ -557,6 +561,8 @@ static const struct usb_device_id quirks_table[] = { BTUSB_WIDEBAND_SPEECH }, { USB_DEVICE(0x13d3, 0x3592), .driver_info = BTUSB_REALTEK | BTUSB_WIDEBAND_SPEECH }, + { USB_DEVICE(0x13d3, 0x3612), .driver_info = BTUSB_REALTEK | + BTUSB_WIDEBAND_SPEECH }, { USB_DEVICE(0x0489, 0xe122), .driver_info = BTUSB_REALTEK | BTUSB_WIDEBAND_SPEECH }, @@ -637,6 +643,8 @@ static const struct usb_device_id quirks_table[] = { BTUSB_WIDEBAND_SPEECH }, { USB_DEVICE(0x13d3, 0x3622), .driver_info = BTUSB_MEDIATEK | BTUSB_WIDEBAND_SPEECH }, + { USB_DEVICE(0x0489, 0xe158), .driver_info = BTUSB_MEDIATEK | + BTUSB_WIDEBAND_SPEECH }, /* Additional MediaTek MT7921 Bluetooth devices */ { USB_DEVICE(0x0489, 0xe0c8), .driver_info = BTUSB_MEDIATEK | @@ -773,6 +781,7 @@ static const struct usb_device_id quirks_table[] = { /* Additional Realtek 8723BU Bluetooth devices */ { USB_DEVICE(0x7392, 0xa611), .driver_info = BTUSB_REALTEK }, + { USB_DEVICE(0x2c0a, 0x8761), .driver_info = BTUSB_REALTEK }, /* Additional Realtek 8723DE Bluetooth devices */ { USB_DEVICE(0x0bda, 0xb009), .driver_info = BTUSB_REALTEK }, @@ -2369,8 +2378,11 @@ static void btusb_work(struct work_struct *work) if (data->air_mode == HCI_NOTIFY_ENABLE_SCO_CVSD) { if (hdev->voice_setting & 0x0020) { static const int alts[3] = { 2, 4, 5 }; + unsigned int sco_idx; - new_alts = alts[data->sco_num - 1]; + sco_idx = min_t(unsigned int, data->sco_num - 1, + ARRAY_SIZE(alts) - 1); + new_alts = alts[sco_idx]; } else { new_alts = data->sco_num; } diff --git a/drivers/bluetooth/hci_bcm4377.c b/drivers/bluetooth/hci_bcm4377.c index 45e6d84224ee3f..8f58c4e17e5e45 100644 --- a/drivers/bluetooth/hci_bcm4377.c +++ b/drivers/bluetooth/hci_bcm4377.c @@ -2397,6 +2397,8 @@ static int bcm4377_probe(struct pci_dev *pdev, const struct pci_device_id *id) if (bcm4377->hw->broken_le_ext_adv_report_phy) hci_set_quirk(hdev, HCI_QUIRK_FIXUP_LE_EXT_ADV_REPORT_PHY); + hci_set_brcm_capable(hdev); + pci_set_drvdata(pdev, bcm4377); hci_set_drvdata(hdev, bcm4377); SET_HCIDEV_DEV(hdev, &pdev->dev); diff --git a/drivers/bluetooth/hci_h4.c b/drivers/bluetooth/hci_h4.c index ec017df8572c83..1e9e2cad9ddf64 100644 --- a/drivers/bluetooth/hci_h4.c +++ b/drivers/bluetooth/hci_h4.c @@ -109,9 +109,6 @@ static int h4_recv(struct hci_uart *hu, const void *data, int count) { struct h4_struct *h4 = hu->priv; - if (!test_bit(HCI_UART_REGISTERED, &hu->flags)) - return -EUNATCH; - h4->rx_skb = h4_recv_buf(hu, h4->rx_skb, data, count, h4_recv_pkts, ARRAY_SIZE(h4_recv_pkts)); if (IS_ERR(h4->rx_skb)) { diff --git a/drivers/bluetooth/hci_ll.c b/drivers/bluetooth/hci_ll.c index 6f4e25917b8639..c4584f40857664 100644 --- a/drivers/bluetooth/hci_ll.c +++ b/drivers/bluetooth/hci_ll.c @@ -541,6 +541,8 @@ static int download_firmware(struct ll_device *lldev) if (err || !fw->data || !fw->size) { bt_dev_err(lldev->hu.hdev, "request_firmware failed(errno %d) for %s", err, bts_scr_name); + if (!err) + release_firmware(fw); return -EINVAL; } ptr = (void *)fw->data; diff --git a/drivers/bluetooth/hci_qca.c b/drivers/bluetooth/hci_qca.c index 888176b0faa906..c0cc04995fc2fa 100644 --- a/drivers/bluetooth/hci_qca.c +++ b/drivers/bluetooth/hci_qca.c @@ -1653,6 +1653,39 @@ static void qca_hw_error(struct hci_dev *hdev, u8 code) skb_queue_purge(&qca->rx_memdump_q); } + /* + * If the BT chip's bt_en pin is connected to a 3.3V power supply via + * hardware and always stays high, driver cannot control the bt_en pin. + * As a result, during SSR (SubSystem Restart), QCA_SSR_TRIGGERED and + * QCA_IBS_DISABLED flags cannot be cleared, which leads to a reset + * command timeout. + * Add an msleep delay to ensure controller completes the SSR process. + * + * Host will not download the firmware after SSR, controller to remain + * in the IBS_WAKE state, and the host needs to synchronize with it + * + * Since the bluetooth chip has been reset, clear the memdump state. + */ + if (!hci_test_quirk(hu->hdev, HCI_QUIRK_NON_PERSISTENT_SETUP)) { + /* + * When the SSR (SubSystem Restart) duration exceeds 2 seconds, + * it triggers host tx_idle_delay, which sets host TX state + * to sleep. Reset tx_idle_timer after SSR to prevent + * host enter TX IBS_Sleep mode. + */ + mod_timer(&qca->tx_idle_timer, jiffies + + msecs_to_jiffies(qca->tx_idle_delay)); + + /* Controller reset completion time is 50ms */ + msleep(50); + + clear_bit(QCA_SSR_TRIGGERED, &qca->flags); + clear_bit(QCA_IBS_DISABLED, &qca->flags); + + qca->tx_ibs_state = HCI_IBS_TX_AWAKE; + qca->memdump_state = QCA_MEMDUMP_IDLE; + } + clear_bit(QCA_HW_ERROR_EVENT, &qca->flags); } @@ -2012,19 +2045,23 @@ static int qca_setup(struct hci_uart *hu) } out: - if (ret && retries < MAX_INIT_RETRIES) { - bt_dev_warn(hdev, "Retry BT power ON:%d", retries); + if (ret) { qca_power_shutdown(hu); - if (hu->serdev) { - serdev_device_close(hu->serdev); - ret = serdev_device_open(hu->serdev); - if (ret) { - bt_dev_err(hdev, "failed to open port"); - return ret; + + if (retries < MAX_INIT_RETRIES) { + bt_dev_warn(hdev, "Retry BT power ON:%d", retries); + if (hu->serdev) { + serdev_device_close(hu->serdev); + ret = serdev_device_open(hu->serdev); + if (ret) { + bt_dev_err(hdev, "failed to open port"); + return ret; + } } + retries++; + goto retry; } - retries++; - goto retry; + return ret; } /* Setup bdaddr */ diff --git a/drivers/bus/fsl-mc/fsl-mc-bus.c b/drivers/bus/fsl-mc/fsl-mc-bus.c index 25845c04e5620a..eb7b6c0ba9e7c6 100644 --- a/drivers/bus/fsl-mc/fsl-mc-bus.c +++ b/drivers/bus/fsl-mc/fsl-mc-bus.c @@ -202,8 +202,12 @@ static ssize_t driver_override_show(struct device *dev, struct device_attribute *attr, char *buf) { struct fsl_mc_device *mc_dev = to_fsl_mc_device(dev); + ssize_t len; - return sysfs_emit(buf, "%s\n", mc_dev->driver_override); + device_lock(dev); + len = sysfs_emit(buf, "%s\n", mc_dev->driver_override); + device_unlock(dev); + return len; } static DEVICE_ATTR_RW(driver_override); @@ -905,11 +909,7 @@ int fsl_mc_device_add(struct fsl_mc_obj_desc *obj_desc, return 0; error_cleanup_dev: - kfree(mc_dev->regions); - if (mc_bus) - kfree(mc_bus); - else - kfree(mc_dev); + put_device(&mc_dev->dev); return error; } diff --git a/drivers/bus/mhi/host/pci_generic.c b/drivers/bus/mhi/host/pci_generic.c index e3bc737313a2f0..0884a384b77fc3 100644 --- a/drivers/bus/mhi/host/pci_generic.c +++ b/drivers/bus/mhi/host/pci_generic.c @@ -94,22 +94,6 @@ struct mhi_pci_dev_info { .doorbell_mode_switch = false, \ } -#define MHI_CHANNEL_CONFIG_DL_AUTOQUEUE(ch_num, ch_name, el_count, ev_ring) \ - { \ - .num = ch_num, \ - .name = ch_name, \ - .num_elements = el_count, \ - .event_ring = ev_ring, \ - .dir = DMA_FROM_DEVICE, \ - .ee_mask = BIT(MHI_EE_AMSS), \ - .pollcfg = 0, \ - .doorbell = MHI_DB_BRST_DISABLE, \ - .lpm_notify = false, \ - .offload_channel = false, \ - .doorbell_mode_switch = false, \ - .auto_queue = true, \ - } - #define MHI_EVENT_CONFIG_CTRL(ev_ring, el_count) \ { \ .num_elements = el_count, \ @@ -329,7 +313,7 @@ static const struct mhi_channel_config modem_qcom_v1_mhi_channels[] = { MHI_CHANNEL_CONFIG_UL(14, "QMI", 4, 0), MHI_CHANNEL_CONFIG_DL(15, "QMI", 4, 0), MHI_CHANNEL_CONFIG_UL(20, "IPCR", 8, 0), - MHI_CHANNEL_CONFIG_DL_AUTOQUEUE(21, "IPCR", 8, 0), + MHI_CHANNEL_CONFIG_DL(21, "IPCR", 8, 0), MHI_CHANNEL_CONFIG_UL_FP(34, "FIREHOSE", 32, 0), MHI_CHANNEL_CONFIG_DL_FP(35, "FIREHOSE", 32, 0), MHI_CHANNEL_CONFIG_UL(46, "IP_SW0", 64, 2), @@ -762,7 +746,7 @@ static const struct mhi_channel_config mhi_telit_fn980_hw_v1_channels[] = { MHI_CHANNEL_CONFIG_UL(14, "QMI", 32, 0), MHI_CHANNEL_CONFIG_DL(15, "QMI", 32, 0), MHI_CHANNEL_CONFIG_UL(20, "IPCR", 16, 0), - MHI_CHANNEL_CONFIG_DL_AUTOQUEUE(21, "IPCR", 16, 0), + MHI_CHANNEL_CONFIG_DL(21, "IPCR", 16, 0), MHI_CHANNEL_CONFIG_HW_UL(100, "IP_HW0", 128, 1), MHI_CHANNEL_CONFIG_HW_DL(101, "IP_HW0", 128, 2), }; diff --git a/drivers/bus/omap-ocp2scp.c b/drivers/bus/omap-ocp2scp.c index e4dfda7b3b1027..eee5ad191ea9c7 100644 --- a/drivers/bus/omap-ocp2scp.c +++ b/drivers/bus/omap-ocp2scp.c @@ -17,15 +17,6 @@ #define OCP2SCP_TIMING 0x18 #define SYNC2_MASK 0xf -static int ocp2scp_remove_devices(struct device *dev, void *c) -{ - struct platform_device *pdev = to_platform_device(dev); - - platform_device_unregister(pdev); - - return 0; -} - static int omap_ocp2scp_probe(struct platform_device *pdev) { int ret; @@ -79,7 +70,7 @@ static int omap_ocp2scp_probe(struct platform_device *pdev) pm_runtime_disable(&pdev->dev); err0: - device_for_each_child(&pdev->dev, NULL, ocp2scp_remove_devices); + of_platform_depopulate(&pdev->dev); return ret; } @@ -87,7 +78,7 @@ static int omap_ocp2scp_probe(struct platform_device *pdev) static void omap_ocp2scp_remove(struct platform_device *pdev) { pm_runtime_disable(&pdev->dev); - device_for_each_child(&pdev->dev, NULL, ocp2scp_remove_devices); + of_platform_depopulate(&pdev->dev); } #ifdef CONFIG_OF diff --git a/drivers/bus/simple-pm-bus.c b/drivers/bus/simple-pm-bus.c index 3f00d953fb9a0e..6d351ca51aa4e6 100644 --- a/drivers/bus/simple-pm-bus.c +++ b/drivers/bus/simple-pm-bus.c @@ -36,7 +36,7 @@ static int simple_pm_bus_probe(struct platform_device *pdev) * that's not listed in simple_pm_bus_of_match. We don't want to do any * of the simple-pm-bus tasks for these devices, so return early. */ - if (pdev->driver_override) + if (device_has_driver_override(&pdev->dev)) return 0; match = of_match_device(dev->driver->of_match_table, dev); @@ -78,7 +78,7 @@ static void simple_pm_bus_remove(struct platform_device *pdev) { const void *data = of_device_get_match_data(&pdev->dev); - if (pdev->driver_override || data) + if (device_has_driver_override(&pdev->dev) || data) return; dev_dbg(&pdev->dev, "%s\n", __func__); @@ -142,6 +142,15 @@ static const struct of_device_id simple_pm_bus_of_match[] = { { .compatible = "simple-mfd", .data = ONLY_BUS }, { .compatible = "isa", .data = ONLY_BUS }, { .compatible = "arm,amba-bus", .data = ONLY_BUS }, + { .compatible = "apple,s5l8960x-pmgr", .data = ONLY_BUS }, + { .compatible = "apple,t7000-pmgr", .data = ONLY_BUS }, + { .compatible = "apple,s8000-pmgr", .data = ONLY_BUS }, + { .compatible = "apple,t8010-pmgr", .data = ONLY_BUS }, + { .compatible = "apple,t8015-pmgr", .data = ONLY_BUS }, + { .compatible = "apple,t8103-pmgr", .data = ONLY_BUS }, + { .compatible = "apple,t8112-pmgr", .data = ONLY_BUS }, + { .compatible = "apple,t6000-pmgr", .data = ONLY_BUS }, + { .compatible = "apple,t6020-pmgr", .data = ONLY_BUS },\ { .compatible = "fsl,ls1021a-scfg", }, { .compatible = "fsl,ls1043a-scfg", }, { .compatible = "fsl,ls1046a-scfg", }, diff --git a/drivers/cache/ax45mp_cache.c b/drivers/cache/ax45mp_cache.c index 1d7dd3d2c101cd..934c5087ec2bda 100644 --- a/drivers/cache/ax45mp_cache.c +++ b/drivers/cache/ax45mp_cache.c @@ -178,11 +178,11 @@ static const struct of_device_id ax45mp_cache_ids[] = { static int __init ax45mp_cache_init(void) { - struct device_node *np; struct resource res; int ret; - np = of_find_matching_node(NULL, ax45mp_cache_ids); + struct device_node *np __free(device_node) = + of_find_matching_node(NULL, ax45mp_cache_ids); if (!of_device_is_available(np)) return -ENODEV; diff --git a/drivers/cache/starfive_starlink_cache.c b/drivers/cache/starfive_starlink_cache.c index 24c7d078ca2272..3a25d2d7c70ca3 100644 --- a/drivers/cache/starfive_starlink_cache.c +++ b/drivers/cache/starfive_starlink_cache.c @@ -102,11 +102,11 @@ static const struct of_device_id starlink_cache_ids[] = { static int __init starlink_cache_init(void) { - struct device_node *np; u32 block_size; int ret; - np = of_find_matching_node(NULL, starlink_cache_ids); + struct device_node *np __free(device_node) = + of_find_matching_node(NULL, starlink_cache_ids); if (!of_device_is_available(np)) return -ENODEV; diff --git a/drivers/char/hw_random/airoha-trng.c b/drivers/char/hw_random/airoha-trng.c index 1dbfa9505c214f..9a648f6d9fd409 100644 --- a/drivers/char/hw_random/airoha-trng.c +++ b/drivers/char/hw_random/airoha-trng.c @@ -212,6 +212,7 @@ static int airoha_trng_probe(struct platform_device *pdev) trng->rng.init = airoha_trng_init; trng->rng.cleanup = airoha_trng_cleanup; trng->rng.read = airoha_trng_read; + trng->rng.quality = 900; ret = devm_hwrng_register(dev, &trng->rng); if (ret) { diff --git a/drivers/char/hw_random/core.c b/drivers/char/hw_random/core.c index 96d7fe41b373d5..aba92d777f7260 100644 --- a/drivers/char/hw_random/core.c +++ b/drivers/char/hw_random/core.c @@ -20,23 +20,25 @@ #include #include #include +#include #include #include #include #include #include +#include #define RNG_MODULE_NAME "hw_random" #define RNG_BUFFER_SIZE (SMP_CACHE_BYTES < 32 ? 32 : SMP_CACHE_BYTES) -static struct hwrng *current_rng; +static struct hwrng __rcu *current_rng; /* the current rng has been explicitly chosen by user via sysfs */ static int cur_rng_set_by_user; static struct task_struct *hwrng_fill; /* list of registered rngs */ static LIST_HEAD(rng_list); -/* Protects rng_list and current_rng */ +/* Protects rng_list, hwrng_fill and updating on current_rng */ static DEFINE_MUTEX(rng_mutex); /* Protects rng read functions, data_avail, rng_buffer and rng_fillbuf */ static DEFINE_MUTEX(reading_mutex); @@ -64,18 +66,39 @@ static size_t rng_buffer_size(void) return RNG_BUFFER_SIZE; } -static inline void cleanup_rng(struct kref *kref) +static void cleanup_rng_work(struct work_struct *work) { - struct hwrng *rng = container_of(kref, struct hwrng, ref); + struct hwrng *rng = container_of(work, struct hwrng, cleanup_work); + + /* + * Hold rng_mutex here so we serialize in case they set_current_rng + * on rng again immediately. + */ + mutex_lock(&rng_mutex); + + /* Skip if rng has been reinitialized. */ + if (kref_read(&rng->ref)) { + mutex_unlock(&rng_mutex); + return; + } if (rng->cleanup) rng->cleanup(rng); complete(&rng->cleanup_done); + mutex_unlock(&rng_mutex); +} + +static inline void cleanup_rng(struct kref *kref) +{ + struct hwrng *rng = container_of(kref, struct hwrng, ref); + + schedule_work(&rng->cleanup_work); } static int set_current_rng(struct hwrng *rng) { + struct hwrng *old_rng; int err; BUG_ON(!mutex_is_locked(&rng_mutex)); @@ -84,8 +107,14 @@ static int set_current_rng(struct hwrng *rng) if (err) return err; - drop_current_rng(); - current_rng = rng; + old_rng = rcu_dereference_protected(current_rng, + lockdep_is_held(&rng_mutex)); + rcu_assign_pointer(current_rng, rng); + + if (old_rng) { + synchronize_rcu(); + kref_put(&old_rng->ref, cleanup_rng); + } /* if necessary, start hwrng thread */ if (!hwrng_fill) { @@ -101,47 +130,56 @@ static int set_current_rng(struct hwrng *rng) static void drop_current_rng(void) { - BUG_ON(!mutex_is_locked(&rng_mutex)); - if (!current_rng) + struct hwrng *rng; + + rng = rcu_dereference_protected(current_rng, + lockdep_is_held(&rng_mutex)); + if (!rng) return; + RCU_INIT_POINTER(current_rng, NULL); + synchronize_rcu(); + + if (hwrng_fill) { + kthread_stop(hwrng_fill); + hwrng_fill = NULL; + } + /* decrease last reference for triggering the cleanup */ - kref_put(¤t_rng->ref, cleanup_rng); - current_rng = NULL; + kref_put(&rng->ref, cleanup_rng); } -/* Returns ERR_PTR(), NULL or refcounted hwrng */ +/* Returns NULL or refcounted hwrng */ static struct hwrng *get_current_rng_nolock(void) { - if (current_rng) - kref_get(¤t_rng->ref); + struct hwrng *rng; + + rng = rcu_dereference_protected(current_rng, + lockdep_is_held(&rng_mutex)); + if (rng) + kref_get(&rng->ref); - return current_rng; + return rng; } static struct hwrng *get_current_rng(void) { struct hwrng *rng; - if (mutex_lock_interruptible(&rng_mutex)) - return ERR_PTR(-ERESTARTSYS); + rcu_read_lock(); + rng = rcu_dereference(current_rng); + if (rng) + kref_get(&rng->ref); - rng = get_current_rng_nolock(); + rcu_read_unlock(); - mutex_unlock(&rng_mutex); return rng; } static void put_rng(struct hwrng *rng) { - /* - * Hold rng_mutex here so we serialize in case they set_current_rng - * on rng again immediately. - */ - mutex_lock(&rng_mutex); if (rng) kref_put(&rng->ref, cleanup_rng); - mutex_unlock(&rng_mutex); } static int hwrng_init(struct hwrng *rng) @@ -213,10 +251,6 @@ static ssize_t rng_dev_read(struct file *filp, char __user *buf, while (size) { rng = get_current_rng(); - if (IS_ERR(rng)) { - err = PTR_ERR(rng); - goto out; - } if (!rng) { err = -ENODEV; goto out; @@ -303,7 +337,7 @@ static struct miscdevice rng_miscdev = { static int enable_best_rng(void) { - struct hwrng *rng, *new_rng = NULL; + struct hwrng *rng, *cur_rng, *new_rng = NULL; int ret = -ENODEV; BUG_ON(!mutex_is_locked(&rng_mutex)); @@ -321,7 +355,9 @@ static int enable_best_rng(void) new_rng = rng; } - ret = ((new_rng == current_rng) ? 0 : set_current_rng(new_rng)); + cur_rng = rcu_dereference_protected(current_rng, + lockdep_is_held(&rng_mutex)); + ret = ((new_rng == cur_rng) ? 0 : set_current_rng(new_rng)); if (!ret) cur_rng_set_by_user = 0; @@ -371,8 +407,6 @@ static ssize_t rng_current_show(struct device *dev, struct hwrng *rng; rng = get_current_rng(); - if (IS_ERR(rng)) - return PTR_ERR(rng); ret = sysfs_emit(buf, "%s\n", rng ? rng->name : "none"); put_rng(rng); @@ -416,8 +450,6 @@ static ssize_t rng_quality_show(struct device *dev, struct hwrng *rng; rng = get_current_rng(); - if (IS_ERR(rng)) - return PTR_ERR(rng); if (!rng) /* no need to put_rng */ return -ENODEV; @@ -432,6 +464,7 @@ static ssize_t rng_quality_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t len) { + struct hwrng *rng; u16 quality; int ret = -EINVAL; @@ -448,12 +481,13 @@ static ssize_t rng_quality_store(struct device *dev, goto out; } - if (!current_rng) { + rng = rcu_dereference_protected(current_rng, lockdep_is_held(&rng_mutex)); + if (!rng) { ret = -ENODEV; goto out; } - current_rng->quality = quality; + rng->quality = quality; current_quality = quality; /* obsolete */ /* the best available RNG may have changed */ @@ -489,8 +523,20 @@ static int hwrng_fillfn(void *unused) struct hwrng *rng; rng = get_current_rng(); - if (IS_ERR(rng) || !rng) + if (!rng) { + /* + * Keep the task_struct alive until kthread_stop() + * is called to avoid UAF in drop_current_rng(). + */ + while (!kthread_should_stop()) { + set_current_state(TASK_INTERRUPTIBLE); + if (!kthread_should_stop()) + schedule(); + } + set_current_state(TASK_RUNNING); break; + } + mutex_lock(&reading_mutex); rc = rng_get_data(rng, rng_fillbuf, rng_buffer_size(), 1); @@ -518,14 +564,13 @@ static int hwrng_fillfn(void *unused) add_hwgenerator_randomness((void *)rng_fillbuf, rc, entropy >> 10, true); } - hwrng_fill = NULL; return 0; } int hwrng_register(struct hwrng *rng) { int err = -EINVAL; - struct hwrng *tmp; + struct hwrng *cur_rng, *tmp; if (!rng->name || (!rng->data_read && !rng->read)) goto out; @@ -540,6 +585,7 @@ int hwrng_register(struct hwrng *rng) } list_add_tail(&rng->list, &rng_list); + INIT_WORK(&rng->cleanup_work, cleanup_rng_work); init_completion(&rng->cleanup_done); complete(&rng->cleanup_done); init_completion(&rng->dying); @@ -547,16 +593,19 @@ int hwrng_register(struct hwrng *rng) /* Adjust quality field to always have a proper value */ rng->quality = min3(default_quality, 1024, rng->quality ?: 1024); - if (!cur_rng_set_by_user && - (!current_rng || rng->quality > current_rng->quality)) { - /* - * Set new rng as current as the new rng source - * provides better entropy quality and was not - * chosen by userspace. - */ - err = set_current_rng(rng); - if (err) - goto out_unlock; + if (!cur_rng_set_by_user) { + cur_rng = rcu_dereference_protected(current_rng, + lockdep_is_held(&rng_mutex)); + if (!cur_rng || rng->quality > cur_rng->quality) { + /* + * Set new rng as current as the new rng source + * provides better entropy quality and was not + * chosen by userspace. + */ + err = set_current_rng(rng); + if (err) + goto out_unlock; + } } mutex_unlock(&rng_mutex); return 0; @@ -569,14 +618,17 @@ EXPORT_SYMBOL_GPL(hwrng_register); void hwrng_unregister(struct hwrng *rng) { - struct hwrng *new_rng; + struct hwrng *cur_rng; int err; mutex_lock(&rng_mutex); list_del(&rng->list); complete_all(&rng->dying); - if (current_rng == rng) { + + cur_rng = rcu_dereference_protected(current_rng, + lockdep_is_held(&rng_mutex)); + if (cur_rng == rng) { err = enable_best_rng(); if (err) { drop_current_rng(); @@ -584,17 +636,7 @@ void hwrng_unregister(struct hwrng *rng) } } - new_rng = get_current_rng_nolock(); - if (list_empty(&rng_list)) { - mutex_unlock(&rng_mutex); - if (hwrng_fill) - kthread_stop(hwrng_fill); - } else - mutex_unlock(&rng_mutex); - - if (new_rng) - put_rng(new_rng); - + mutex_unlock(&rng_mutex); wait_for_completion(&rng->cleanup_done); } EXPORT_SYMBOL_GPL(hwrng_unregister); @@ -682,7 +724,7 @@ static int __init hwrng_modinit(void) static void __exit hwrng_modexit(void) { mutex_lock(&rng_mutex); - BUG_ON(current_rng); + WARN_ON(rcu_access_pointer(current_rng)); kfree(rng_buffer); kfree(rng_fillbuf); mutex_unlock(&rng_mutex); diff --git a/drivers/char/ipmi/ipmi_ipmb.c b/drivers/char/ipmi/ipmi_ipmb.c index 3a51e58b248754..28818952a7a4bf 100644 --- a/drivers/char/ipmi/ipmi_ipmb.c +++ b/drivers/char/ipmi/ipmi_ipmb.c @@ -202,11 +202,16 @@ static int ipmi_ipmb_slave_cb(struct i2c_client *client, break; case I2C_SLAVE_READ_REQUESTED: + *val = 0xff; + ipmi_ipmb_check_msg_done(iidev); + break; + case I2C_SLAVE_STOP: ipmi_ipmb_check_msg_done(iidev); break; case I2C_SLAVE_READ_PROCESSED: + *val = 0xff; break; } diff --git a/drivers/char/ipmi/ipmi_msghandler.c b/drivers/char/ipmi/ipmi_msghandler.c index 3f48fc6ab596d3..f8c3c1e445200f 100644 --- a/drivers/char/ipmi/ipmi_msghandler.c +++ b/drivers/char/ipmi/ipmi_msghandler.c @@ -602,6 +602,22 @@ static int __ipmi_bmc_register(struct ipmi_smi *intf, static int __scan_channels(struct ipmi_smi *intf, struct ipmi_device_id *id, bool rescan); +static void ipmi_lock_xmit_msgs(struct ipmi_smi *intf, int run_to_completion, + unsigned long *flags) +{ + if (run_to_completion) + return; + spin_lock_irqsave(&intf->xmit_msgs_lock, *flags); +} + +static void ipmi_unlock_xmit_msgs(struct ipmi_smi *intf, int run_to_completion, + unsigned long *flags) +{ + if (run_to_completion) + return; + spin_unlock_irqrestore(&intf->xmit_msgs_lock, *flags); +} + static void free_ipmi_user(struct kref *ref) { struct ipmi_user *user = container_of(ref, struct ipmi_user, refcount); @@ -1871,21 +1887,32 @@ static struct ipmi_smi_msg *smi_add_send_msg(struct ipmi_smi *intf, return smi_msg; } -static void smi_send(struct ipmi_smi *intf, +static int smi_send(struct ipmi_smi *intf, const struct ipmi_smi_handlers *handlers, struct ipmi_smi_msg *smi_msg, int priority) { int run_to_completion = READ_ONCE(intf->run_to_completion); unsigned long flags = 0; + int rv = 0; - if (!run_to_completion) - spin_lock_irqsave(&intf->xmit_msgs_lock, flags); + ipmi_lock_xmit_msgs(intf, run_to_completion, &flags); smi_msg = smi_add_send_msg(intf, smi_msg, priority); - if (!run_to_completion) - spin_unlock_irqrestore(&intf->xmit_msgs_lock, flags); + ipmi_unlock_xmit_msgs(intf, run_to_completion, &flags); - if (smi_msg) - handlers->sender(intf->send_info, smi_msg); + if (smi_msg) { + rv = handlers->sender(intf->send_info, smi_msg); + if (rv) { + ipmi_lock_xmit_msgs(intf, run_to_completion, &flags); + intf->curr_msg = NULL; + ipmi_unlock_xmit_msgs(intf, run_to_completion, &flags); + /* + * Something may have been added to the transmit + * queue, so schedule a check for that. + */ + queue_work(system_wq, &intf->smi_work); + } + } + return rv; } static bool is_maintenance_mode_cmd(struct kernel_ipmi_msg *msg) @@ -2298,6 +2325,7 @@ static int i_ipmi_request(struct ipmi_user *user, struct ipmi_recv_msg *recv_msg; int run_to_completion = READ_ONCE(intf->run_to_completion); int rv = 0; + bool in_seq_table = false; if (supplied_recv) { recv_msg = supplied_recv; @@ -2351,33 +2379,50 @@ static int i_ipmi_request(struct ipmi_user *user, rv = i_ipmi_req_ipmb(intf, addr, msgid, msg, smi_msg, recv_msg, source_address, source_lun, retries, retry_time_ms); + in_seq_table = true; } else if (is_ipmb_direct_addr(addr)) { rv = i_ipmi_req_ipmb_direct(intf, addr, msgid, msg, smi_msg, recv_msg, source_lun); } else if (is_lan_addr(addr)) { rv = i_ipmi_req_lan(intf, addr, msgid, msg, smi_msg, recv_msg, source_lun, retries, retry_time_ms); + in_seq_table = true; } else { - /* Unknown address type. */ + /* Unknown address type. */ ipmi_inc_stat(intf, sent_invalid_commands); rv = -EINVAL; } - if (rv) { -out_err: - if (!supplied_smi) - ipmi_free_smi_msg(smi_msg); - if (!supplied_recv) - ipmi_free_recv_msg(recv_msg); - } else { + if (!rv) { dev_dbg(intf->si_dev, "Send: %*ph\n", smi_msg->data_size, smi_msg->data); - smi_send(intf, intf->handlers, smi_msg, priority); + rv = smi_send(intf, intf->handlers, smi_msg, priority); + if (rv != IPMI_CC_NO_ERROR) + /* smi_send() returns an IPMI err, return a Linux one. */ + rv = -EIO; + if (rv && in_seq_table) { + /* + * If it's in the sequence table, it will be + * retried later, so ignore errors. + */ + rv = 0; + /* But we need to fix the timeout. */ + intf_start_seq_timer(intf, smi_msg->msgid); + ipmi_free_smi_msg(smi_msg); + smi_msg = NULL; + } } +out_err: if (!run_to_completion) mutex_unlock(&intf->users_mutex); + if (rv) { + if (!supplied_smi) + ipmi_free_smi_msg(smi_msg); + if (!supplied_recv) + ipmi_free_recv_msg(recv_msg); + } return rv; } @@ -3951,12 +3996,12 @@ static int handle_ipmb_get_msg_cmd(struct ipmi_smi *intf, dev_dbg(intf->si_dev, "Invalid command: %*ph\n", msg->data_size, msg->data); - smi_send(intf, intf->handlers, msg, 0); - /* - * We used the message, so return the value that - * causes it to not be freed or queued. - */ - rv = -1; + if (smi_send(intf, intf->handlers, msg, 0) == IPMI_CC_NO_ERROR) + /* + * We used the message, so return the value that + * causes it to not be freed or queued. + */ + rv = -1; } else if (!IS_ERR(recv_msg)) { /* Extract the source address from the data. */ ipmb_addr = (struct ipmi_ipmb_addr *) &recv_msg->addr; @@ -4030,12 +4075,12 @@ static int handle_ipmb_direct_rcv_cmd(struct ipmi_smi *intf, msg->data[4] = IPMI_INVALID_CMD_COMPLETION_CODE; msg->data_size = 5; - smi_send(intf, intf->handlers, msg, 0); - /* - * We used the message, so return the value that - * causes it to not be freed or queued. - */ - rv = -1; + if (smi_send(intf, intf->handlers, msg, 0) == IPMI_CC_NO_ERROR) + /* + * We used the message, so return the value that + * causes it to not be freed or queued. + */ + rv = -1; } else if (!IS_ERR(recv_msg)) { /* Extract the source address from the data. */ daddr = (struct ipmi_ipmb_direct_addr *)&recv_msg->addr; @@ -4175,7 +4220,7 @@ static int handle_lan_get_msg_cmd(struct ipmi_smi *intf, struct ipmi_smi_msg *msg) { struct cmd_rcvr *rcvr; - int rv = 0; + int rv = 0; /* Free by default */ unsigned char netfn; unsigned char cmd; unsigned char chan; @@ -4228,12 +4273,12 @@ static int handle_lan_get_msg_cmd(struct ipmi_smi *intf, dev_dbg(intf->si_dev, "Invalid command: %*ph\n", msg->data_size, msg->data); - smi_send(intf, intf->handlers, msg, 0); - /* - * We used the message, so return the value that - * causes it to not be freed or queued. - */ - rv = -1; + if (smi_send(intf, intf->handlers, msg, 0) == IPMI_CC_NO_ERROR) + /* + * We used the message, so return the value that + * causes it to not be freed or queued. + */ + rv = -1; } else if (!IS_ERR(recv_msg)) { /* Extract the source address from the data. */ lan_addr = (struct ipmi_lan_addr *) &recv_msg->addr; @@ -4826,8 +4871,7 @@ static void smi_work(struct work_struct *t) * message delivery. */ restart: - if (!run_to_completion) - spin_lock_irqsave(&intf->xmit_msgs_lock, flags); + ipmi_lock_xmit_msgs(intf, run_to_completion, &flags); if (intf->curr_msg == NULL && !intf->in_shutdown) { struct list_head *entry = NULL; @@ -4843,8 +4887,7 @@ static void smi_work(struct work_struct *t) intf->curr_msg = newmsg; } } - if (!run_to_completion) - spin_unlock_irqrestore(&intf->xmit_msgs_lock, flags); + ipmi_unlock_xmit_msgs(intf, run_to_completion, &flags); if (newmsg) { cc = intf->handlers->sender(intf->send_info, newmsg); @@ -4852,8 +4895,11 @@ static void smi_work(struct work_struct *t) if (newmsg->recv_msg) deliver_err_response(intf, newmsg->recv_msg, cc); - else - ipmi_free_smi_msg(newmsg); + ipmi_lock_xmit_msgs(intf, run_to_completion, &flags); + intf->curr_msg = NULL; + ipmi_unlock_xmit_msgs(intf, run_to_completion, &flags); + ipmi_free_smi_msg(newmsg); + newmsg = NULL; goto restart; } } @@ -4921,16 +4967,14 @@ void ipmi_smi_msg_received(struct ipmi_smi *intf, spin_unlock_irqrestore(&intf->waiting_rcv_msgs_lock, flags); - if (!run_to_completion) - spin_lock_irqsave(&intf->xmit_msgs_lock, flags); + ipmi_lock_xmit_msgs(intf, run_to_completion, &flags); /* * We can get an asynchronous event or receive message in addition * to commands we send. */ if (msg == intf->curr_msg) intf->curr_msg = NULL; - if (!run_to_completion) - spin_unlock_irqrestore(&intf->xmit_msgs_lock, flags); + ipmi_unlock_xmit_msgs(intf, run_to_completion, &flags); if (run_to_completion) smi_work(&intf->smi_work); @@ -5043,7 +5087,12 @@ static void check_msg_timeout(struct ipmi_smi *intf, struct seq_table *ent, ipmi_inc_stat(intf, retransmitted_ipmb_commands); - smi_send(intf, intf->handlers, smi_msg, 0); + /* If this fails we'll retry later or timeout. */ + if (smi_send(intf, intf->handlers, smi_msg, 0) != IPMI_CC_NO_ERROR) { + /* But fix the timeout. */ + intf_start_seq_timer(intf, smi_msg->msgid); + ipmi_free_smi_msg(smi_msg); + } } else ipmi_free_smi_msg(smi_msg); diff --git a/drivers/char/ipmi/ipmi_si_intf.c b/drivers/char/ipmi/ipmi_si_intf.c index 5459ffdde8dc6e..6eda61664aaa89 100644 --- a/drivers/char/ipmi/ipmi_si_intf.c +++ b/drivers/char/ipmi/ipmi_si_intf.c @@ -809,6 +809,12 @@ static enum si_sm_result smi_event_handler(struct smi_info *smi_info, */ return_hosed_msg(smi_info, IPMI_BUS_ERR); } + if (smi_info->waiting_msg != NULL) { + /* Also handle if there was a message waiting. */ + smi_info->curr_msg = smi_info->waiting_msg; + smi_info->waiting_msg = NULL; + return_hosed_msg(smi_info, IPMI_BUS_ERR); + } smi_mod_timer(smi_info, jiffies + SI_TIMEOUT_HOSED); goto out; } @@ -918,9 +924,14 @@ static int sender(void *send_info, struct ipmi_smi_msg *msg) { struct smi_info *smi_info = send_info; unsigned long flags; + int rv = IPMI_CC_NO_ERROR; debug_timestamp(smi_info, "Enqueue"); + /* + * Check here for run to completion mode. A check under lock is + * later. + */ if (smi_info->si_state == SI_HOSED) return IPMI_BUS_ERR; @@ -934,18 +945,15 @@ static int sender(void *send_info, struct ipmi_smi_msg *msg) } spin_lock_irqsave(&smi_info->si_lock, flags); - /* - * The following two lines don't need to be under the lock for - * the lock's sake, but they do need SMP memory barriers to - * avoid getting things out of order. We are already claiming - * the lock, anyway, so just do it under the lock to avoid the - * ordering problem. - */ - BUG_ON(smi_info->waiting_msg); - smi_info->waiting_msg = msg; - check_start_timer_thread(smi_info); + if (smi_info->si_state == SI_HOSED) { + rv = IPMI_BUS_ERR; + } else { + BUG_ON(smi_info->waiting_msg); + smi_info->waiting_msg = msg; + check_start_timer_thread(smi_info); + } spin_unlock_irqrestore(&smi_info->si_lock, flags); - return IPMI_CC_NO_ERROR; + return rv; } static void set_run_to_completion(void *send_info, bool i_run_to_completion) @@ -1113,7 +1121,9 @@ static void smi_timeout(struct timer_list *t) * SI_USEC_PER_JIFFY); smi_result = smi_event_handler(smi_info, time_diff); - if ((smi_info->io.irq) && (!smi_info->interrupt_disabled)) { + if (smi_info->si_state == SI_HOSED) { + timeout = jiffies + SI_TIMEOUT_HOSED; + } else if ((smi_info->io.irq) && (!smi_info->interrupt_disabled)) { /* Running with interrupts, only do long timeouts. */ timeout = jiffies + SI_TIMEOUT_JIFFIES; smi_inc_stat(smi_info, long_timeouts); @@ -2226,7 +2236,8 @@ static void wait_msg_processed(struct smi_info *smi_info) unsigned long jiffies_now; long time_diff; - while (smi_info->curr_msg || (smi_info->si_state != SI_NORMAL)) { + while (smi_info->si_state != SI_HOSED && + (smi_info->curr_msg || (smi_info->si_state != SI_NORMAL))) { jiffies_now = jiffies; time_diff = (((long)jiffies_now - (long)smi_info->last_timeout_jiffies) * SI_USEC_PER_JIFFY); diff --git a/drivers/char/misc_minor_kunit.c b/drivers/char/misc_minor_kunit.c index 6fc8b05169c575..e930c78e1ef97e 100644 --- a/drivers/char/misc_minor_kunit.c +++ b/drivers/char/misc_minor_kunit.c @@ -166,7 +166,7 @@ static void __init miscdev_test_can_open(struct kunit *test, struct miscdevice * KUNIT_FAIL(test, "failed to create node\n"); filp = filp_open(devname, O_RDONLY, 0); - if (IS_ERR_OR_NULL(filp)) + if (IS_ERR(filp)) KUNIT_FAIL(test, "failed to open misc device: %ld\n", PTR_ERR(filp)); else fput(filp); diff --git a/drivers/char/random.c b/drivers/char/random.c index bab03c7c4194a7..c36c76c2e88e04 100644 --- a/drivers/char/random.c +++ b/drivers/char/random.c @@ -96,8 +96,7 @@ static ATOMIC_NOTIFIER_HEAD(random_ready_notifier); /* Control how we warn userspace. */ static struct ratelimit_state urandom_warning = RATELIMIT_STATE_INIT_FLAGS("urandom_warning", HZ, 3, RATELIMIT_MSG_ON_RELEASE); -static int ratelimit_disable __read_mostly = - IS_ENABLED(CONFIG_WARN_ALL_UNSEEDED_RANDOM); +static int ratelimit_disable __read_mostly = 0; module_param_named(ratelimit_disable, ratelimit_disable, int, 0644); MODULE_PARM_DESC(ratelimit_disable, "Disable random ratelimit suppression"); @@ -168,12 +167,6 @@ int __cold execute_with_initialized_rng(struct notifier_block *nb) return ret; } -#define warn_unseeded_randomness() \ - if (IS_ENABLED(CONFIG_WARN_ALL_UNSEEDED_RANDOM) && !crng_ready()) \ - printk_deferred(KERN_NOTICE "random: %s called from %pS with crng_init=%d\n", \ - __func__, (void *)_RET_IP_, crng_init) - - /********************************************************************* * * Fast key erasure RNG, the "crng". @@ -434,7 +427,6 @@ static void _get_random_bytes(void *buf, size_t len) */ void get_random_bytes(void *buf, size_t len) { - warn_unseeded_randomness(); _get_random_bytes(buf, len); } EXPORT_SYMBOL(get_random_bytes); @@ -523,8 +515,6 @@ type get_random_ ##type(void) \ struct batch_ ##type *batch; \ unsigned long next_gen; \ \ - warn_unseeded_randomness(); \ - \ if (!crng_ready()) { \ _get_random_bytes(&ret, sizeof(ret)); \ return ret; \ diff --git a/drivers/char/tpm/st33zp24/st33zp24.c b/drivers/char/tpm/st33zp24/st33zp24.c index 2ed7815e4899b7..e2b7451ea7ccd3 100644 --- a/drivers/char/tpm/st33zp24/st33zp24.c +++ b/drivers/char/tpm/st33zp24/st33zp24.c @@ -328,8 +328,10 @@ static int st33zp24_send(struct tpm_chip *chip, unsigned char *buf, for (i = 0; i < len - 1;) { burstcnt = get_burstcount(chip); - if (burstcnt < 0) - return burstcnt; + if (burstcnt < 0) { + ret = burstcnt; + goto out_err; + } size = min_t(int, len - i - 1, burstcnt); ret = tpm_dev->ops->send(tpm_dev->phy_id, TPM_DATA_FIFO, buf + i, size); diff --git a/drivers/char/tpm/tpm_i2c_infineon.c b/drivers/char/tpm/tpm_i2c_infineon.c index bdf1f329a67946..8b7d32de0b2ef9 100644 --- a/drivers/char/tpm/tpm_i2c_infineon.c +++ b/drivers/char/tpm/tpm_i2c_infineon.c @@ -544,8 +544,10 @@ static int tpm_tis_i2c_send(struct tpm_chip *chip, u8 *buf, size_t bufsiz, burstcnt = get_burstcount(chip); /* burstcnt < 0 = TPM is busy */ - if (burstcnt < 0) - return burstcnt; + if (burstcnt < 0) { + rc = burstcnt; + goto out_err; + } if (burstcnt > (len - 1 - count)) burstcnt = len - 1 - count; diff --git a/drivers/char/tpm/tpm_tis_i2c_cr50.c b/drivers/char/tpm/tpm_tis_i2c_cr50.c index fc6891a0b6936d..b48cacacc0664a 100644 --- a/drivers/char/tpm/tpm_tis_i2c_cr50.c +++ b/drivers/char/tpm/tpm_tis_i2c_cr50.c @@ -749,8 +749,7 @@ static int tpm_cr50_i2c_probe(struct i2c_client *client) if (client->irq > 0) { rc = devm_request_irq(dev, client->irq, tpm_cr50_i2c_int_handler, - IRQF_TRIGGER_FALLING | IRQF_ONESHOT | - IRQF_NO_AUTOEN, + IRQF_TRIGGER_FALLING | IRQF_NO_AUTOEN, dev->driver->name, chip); if (rc < 0) { dev_err(dev, "Failed to probe IRQ %d\n", client->irq); diff --git a/drivers/char/tpm/tpm_tis_spi_cr50.c b/drivers/char/tpm/tpm_tis_spi_cr50.c index f4937280e94061..32920b4cecfb44 100644 --- a/drivers/char/tpm/tpm_tis_spi_cr50.c +++ b/drivers/char/tpm/tpm_tis_spi_cr50.c @@ -287,7 +287,7 @@ int cr50_spi_probe(struct spi_device *spi) if (spi->irq > 0) { ret = devm_request_irq(&spi->dev, spi->irq, cr50_spi_irq_handler, - IRQF_TRIGGER_RISING | IRQF_ONESHOT, + IRQF_TRIGGER_RISING, "cr50_spi", cr50_phy); if (ret < 0) { if (ret == -EPROBE_DEFER) diff --git a/drivers/clk/actions/owl-composite.c b/drivers/clk/actions/owl-composite.c index 00b74f8bc4375a..9540444307d6c9 100644 --- a/drivers/clk/actions/owl-composite.c +++ b/drivers/clk/actions/owl-composite.c @@ -57,15 +57,10 @@ static int owl_comp_div_determine_rate(struct clk_hw *hw, struct clk_rate_request *req) { struct owl_composite *comp = hw_to_owl_comp(hw); - long rate; - - rate = owl_divider_helper_round_rate(&comp->common, &comp->rate.div_hw, - req->rate, &req->best_parent_rate); - if (rate < 0) - return rate; + struct owl_divider_hw *div = &comp->rate.div_hw; - req->rate = rate; - return 0; + return divider_determine_rate(&comp->common.hw, req, div->table, + div->width, div->div_flags); } static unsigned long owl_comp_div_recalc_rate(struct clk_hw *hw, diff --git a/drivers/clk/actions/owl-divider.c b/drivers/clk/actions/owl-divider.c index 118f1393c6780f..316ace80e87e3b 100644 --- a/drivers/clk/actions/owl-divider.c +++ b/drivers/clk/actions/owl-divider.c @@ -13,26 +13,13 @@ #include "owl-divider.h" -long owl_divider_helper_round_rate(struct owl_clk_common *common, - const struct owl_divider_hw *div_hw, - unsigned long rate, - unsigned long *parent_rate) -{ - return divider_round_rate(&common->hw, rate, parent_rate, - div_hw->table, div_hw->width, - div_hw->div_flags); -} - static int owl_divider_determine_rate(struct clk_hw *hw, struct clk_rate_request *req) { struct owl_divider *div = hw_to_owl_divider(hw); - req->rate = owl_divider_helper_round_rate(&div->common, &div->div_hw, - req->rate, - &req->best_parent_rate); - - return 0; + return divider_determine_rate(hw, req, div->div_hw.table, + div->div_hw.width, div->div_hw.div_flags); } unsigned long owl_divider_helper_recalc_rate(struct owl_clk_common *common, diff --git a/drivers/clk/actions/owl-divider.h b/drivers/clk/actions/owl-divider.h index d76f58782c5288..1d3bb4e5898a3c 100644 --- a/drivers/clk/actions/owl-divider.h +++ b/drivers/clk/actions/owl-divider.h @@ -56,11 +56,6 @@ static inline struct owl_divider *hw_to_owl_divider(struct clk_hw *hw) return container_of(common, struct owl_divider, common); } -long owl_divider_helper_round_rate(struct owl_clk_common *common, - const struct owl_divider_hw *div_hw, - unsigned long rate, - unsigned long *parent_rate); - unsigned long owl_divider_helper_recalc_rate(struct owl_clk_common *common, const struct owl_divider_hw *div_hw, unsigned long parent_rate); diff --git a/drivers/clk/clk-apple-nco.c b/drivers/clk/clk-apple-nco.c index d3ced4a0f029ec..434c067968bbc1 100644 --- a/drivers/clk/clk-apple-nco.c +++ b/drivers/clk/clk-apple-nco.c @@ -320,6 +320,7 @@ static int applnco_probe(struct platform_device *pdev) } static const struct of_device_id applnco_ids[] = { + { .compatible = "apple,t8103-nco" }, { .compatible = "apple,nco" }, { } }; diff --git a/drivers/clk/clk-bm1880.c b/drivers/clk/clk-bm1880.c index dac190bc6e19a6..d2617fe16d2e42 100644 --- a/drivers/clk/clk-bm1880.c +++ b/drivers/clk/clk-bm1880.c @@ -629,10 +629,7 @@ static int bm1880_clk_div_determine_rate(struct clk_hw *hw, return 0; } - req->rate = divider_round_rate(hw, req->rate, &req->best_parent_rate, - div->table, div->width, div->flags); - - return 0; + return divider_determine_rate(hw, req, div->table, div->width, div->flags); } static int bm1880_clk_div_set_rate(struct clk_hw *hw, unsigned long rate, diff --git a/drivers/clk/clk-loongson1.c b/drivers/clk/clk-loongson1.c index f9f060d08a5fae..1674181a1107dc 100644 --- a/drivers/clk/clk-loongson1.c +++ b/drivers/clk/clk-loongson1.c @@ -99,10 +99,7 @@ static int ls1x_divider_determine_rate(struct clk_hw *hw, struct ls1x_clk *ls1x_clk = to_ls1x_clk(hw); const struct ls1x_clk_div_data *d = ls1x_clk->data; - req->rate = divider_round_rate(hw, req->rate, &req->best_parent_rate, - d->table, d->width, d->flags); - - return 0; + return divider_determine_rate(hw, req, d->table, d->width, d->flags); } static int ls1x_divider_set_rate(struct clk_hw *hw, unsigned long rate, diff --git a/drivers/clk/clk-milbeaut.c b/drivers/clk/clk-milbeaut.c index b4f9b7143eaa65..bb94d02a76cf10 100644 --- a/drivers/clk/clk-milbeaut.c +++ b/drivers/clk/clk-milbeaut.c @@ -407,10 +407,7 @@ static int m10v_clk_divider_determine_rate(struct clk_hw *hw, return 0; } - req->rate = divider_round_rate(hw, req->rate, &req->best_parent_rate, - divider->table, divider->width, divider->flags); - - return 0; + return divider_determine_rate(hw, req, divider->table, divider->width, divider->flags); } static int m10v_clk_divider_set_rate(struct clk_hw *hw, unsigned long rate, diff --git a/drivers/clk/clk-renesas-pcie.c b/drivers/clk/clk-renesas-pcie.c index 4c3a5e4eb77ac6..f94a9c4d0b6700 100644 --- a/drivers/clk/clk-renesas-pcie.c +++ b/drivers/clk/clk-renesas-pcie.c @@ -64,7 +64,7 @@ struct rs9_driver_data { struct i2c_client *client; struct regmap *regmap; const struct rs9_chip_info *chip_info; - struct clk_hw *clk_dif[4]; + struct clk_hw *clk_dif[8]; u8 pll_amplitude; u8 pll_ssc; u8 clk_dif_sr; diff --git a/drivers/clk/clk-versaclock3.c b/drivers/clk/clk-versaclock3.c index 1849863dbd673f..27b6cf70f3ae1b 100644 --- a/drivers/clk/clk-versaclock3.c +++ b/drivers/clk/clk-versaclock3.c @@ -523,11 +523,8 @@ static int vc3_div_determine_rate(struct clk_hw *hw, return 0; } - req->rate = divider_round_rate(hw, req->rate, &req->best_parent_rate, - div_data->table, - div_data->width, div_data->flags); - - return 0; + return divider_determine_rate(hw, req, div_data->table, div_data->width, + div_data->flags); } static int vc3_div_set_rate(struct clk_hw *hw, unsigned long rate, diff --git a/drivers/clk/hisilicon/clkdivider-hi6220.c b/drivers/clk/hisilicon/clkdivider-hi6220.c index 6bae18a84cb6c0..fd7ceb92d65157 100644 --- a/drivers/clk/hisilicon/clkdivider-hi6220.c +++ b/drivers/clk/hisilicon/clkdivider-hi6220.c @@ -60,10 +60,8 @@ static int hi6220_clkdiv_determine_rate(struct clk_hw *hw, { struct hi6220_clk_divider *dclk = to_hi6220_clk_divider(hw); - req->rate = divider_round_rate(hw, req->rate, &req->best_parent_rate, dclk->table, - dclk->width, CLK_DIVIDER_ROUND_CLOSEST); - - return 0; + return divider_determine_rate(hw, req, dclk->table, dclk->width, + CLK_DIVIDER_ROUND_CLOSEST); } static int hi6220_clkdiv_set_rate(struct clk_hw *hw, unsigned long rate, diff --git a/drivers/clk/imx/clk-imx8qxp.c b/drivers/clk/imx/clk-imx8qxp.c index 3ae162625bb1a4..c781425a005ef7 100644 --- a/drivers/clk/imx/clk-imx8qxp.c +++ b/drivers/clk/imx/clk-imx8qxp.c @@ -346,7 +346,29 @@ static struct platform_driver imx8qxp_clk_driver = { }, .probe = imx8qxp_clk_probe, }; -module_platform_driver(imx8qxp_clk_driver); + +static int __init imx8qxp_clk_init(void) +{ + int ret; + + ret = platform_driver_register(&imx8qxp_clk_driver); + if (ret) + return ret; + + ret = imx_clk_scu_module_init(); + if (ret) + platform_driver_unregister(&imx8qxp_clk_driver); + + return ret; +} +module_init(imx8qxp_clk_init); + +static void __exit imx8qxp_clk_exit(void) +{ + imx_clk_scu_module_exit(); + platform_driver_unregister(&imx8qxp_clk_driver); +} +module_exit(imx8qxp_clk_exit); MODULE_AUTHOR("Aisheng Dong "); MODULE_DESCRIPTION("NXP i.MX8QXP clock driver"); diff --git a/drivers/clk/imx/clk-scu.c b/drivers/clk/imx/clk-scu.c index 34c9dc1fb20e55..e6b273d8a09ae7 100644 --- a/drivers/clk/imx/clk-scu.c +++ b/drivers/clk/imx/clk-scu.c @@ -191,6 +191,16 @@ static bool imx_scu_clk_is_valid(u32 rsrc_id) return p != NULL; } +int __init imx_clk_scu_module_init(void) +{ + return platform_driver_register(&imx_clk_scu_driver); +} + +void __exit imx_clk_scu_module_exit(void) +{ + return platform_driver_unregister(&imx_clk_scu_driver); +} + int imx_clk_scu_init(struct device_node *np, const struct imx_clk_scu_rsrc_table *data) { @@ -215,7 +225,7 @@ int imx_clk_scu_init(struct device_node *np, rsrc_table = data; } - return platform_driver_register(&imx_clk_scu_driver); + return 0; } /* @@ -696,8 +706,7 @@ struct clk_hw *imx_clk_scu_alloc_dev(const char *name, if (ret) goto put_device; - ret = driver_set_override(&pdev->dev, &pdev->driver_override, - "imx-scu-clk", strlen("imx-scu-clk")); + ret = device_set_driver_override(&pdev->dev, "imx-scu-clk"); if (ret) goto put_device; diff --git a/drivers/clk/imx/clk-scu.h b/drivers/clk/imx/clk-scu.h index af7b697f51cae3..ca82f2cce89740 100644 --- a/drivers/clk/imx/clk-scu.h +++ b/drivers/clk/imx/clk-scu.h @@ -25,6 +25,8 @@ extern const struct imx_clk_scu_rsrc_table imx_clk_scu_rsrc_imx8dxl; extern const struct imx_clk_scu_rsrc_table imx_clk_scu_rsrc_imx8qxp; extern const struct imx_clk_scu_rsrc_table imx_clk_scu_rsrc_imx8qm; +int __init imx_clk_scu_module_init(void); +void __exit imx_clk_scu_module_exit(void); int imx_clk_scu_init(struct device_node *np, const struct imx_clk_scu_rsrc_table *data); struct clk_hw *imx_scu_of_clk_src_get(struct of_phandle_args *clkspec, diff --git a/drivers/clk/mediatek/clk-mt7981-eth.c b/drivers/clk/mediatek/clk-mt7981-eth.c index 906aec9ddff54b..0655ebb6c561f6 100644 --- a/drivers/clk/mediatek/clk-mt7981-eth.c +++ b/drivers/clk/mediatek/clk-mt7981-eth.c @@ -31,7 +31,7 @@ static const struct mtk_gate_regs sgmii0_cg_regs = { .ops = &mtk_clk_gate_ops_no_setclr_inv, \ } -static const struct mtk_gate sgmii0_clks[] __initconst = { +static const struct mtk_gate sgmii0_clks[] = { GATE_SGMII0(CLK_SGM0_TX_EN, "sgm0_tx_en", "usb_tx250m", 2), GATE_SGMII0(CLK_SGM0_RX_EN, "sgm0_rx_en", "usb_eq_rx250m", 3), GATE_SGMII0(CLK_SGM0_CK0_EN, "sgm0_ck0_en", "usb_ln0", 4), @@ -53,7 +53,7 @@ static const struct mtk_gate_regs sgmii1_cg_regs = { .ops = &mtk_clk_gate_ops_no_setclr_inv, \ } -static const struct mtk_gate sgmii1_clks[] __initconst = { +static const struct mtk_gate sgmii1_clks[] = { GATE_SGMII1(CLK_SGM1_TX_EN, "sgm1_tx_en", "usb_tx250m", 2), GATE_SGMII1(CLK_SGM1_RX_EN, "sgm1_rx_en", "usb_eq_rx250m", 3), GATE_SGMII1(CLK_SGM1_CK1_EN, "sgm1_ck1_en", "usb_ln0", 4), @@ -75,7 +75,7 @@ static const struct mtk_gate_regs eth_cg_regs = { .ops = &mtk_clk_gate_ops_no_setclr_inv, \ } -static const struct mtk_gate eth_clks[] __initconst = { +static const struct mtk_gate eth_clks[] = { GATE_ETH(CLK_ETH_FE_EN, "eth_fe_en", "netsys_2x", 6), GATE_ETH(CLK_ETH_GP2_EN, "eth_gp2_en", "sgm_325m", 7), GATE_ETH(CLK_ETH_GP1_EN, "eth_gp1_en", "sgm_325m", 8), diff --git a/drivers/clk/mediatek/clk-mt8196-mfg.c b/drivers/clk/mediatek/clk-mt8196-mfg.c index ae1eb9de79ae29..f40795b47ff1fd 100644 --- a/drivers/clk/mediatek/clk-mt8196-mfg.c +++ b/drivers/clk/mediatek/clk-mt8196-mfg.c @@ -58,24 +58,25 @@ .pcw_shift = _pcw_shift, \ .pcwbits = _pcwbits, \ .pcwibits = MT8196_INTEGER_BITS, \ + .parent_name = "mfg_eb", \ } static const struct mtk_pll_data mfg_ao_plls[] = { - PLL(CLK_MFG_AO_MFGPLL, "mfgpll", MFGPLL_CON0, MFGPLL_CON0, 0, 0, 0, - BIT(0), MFGPLL_CON1, 24, 0, 0, 0, + PLL(CLK_MFG_AO_MFGPLL, "mfgpll", MFGPLL_CON0, MFGPLL_CON0, 0, 0, + PLL_PARENT_EN, BIT(0), MFGPLL_CON1, 24, 0, 0, 0, MFGPLL_CON1, 0, 22), }; static const struct mtk_pll_data mfgsc0_ao_plls[] = { PLL(CLK_MFGSC0_AO_MFGPLL_SC0, "mfgpll-sc0", MFGPLL_SC0_CON0, - MFGPLL_SC0_CON0, 0, 0, 0, BIT(0), MFGPLL_SC0_CON1, 24, 0, 0, 0, - MFGPLL_SC0_CON1, 0, 22), + MFGPLL_SC0_CON0, 0, 0, PLL_PARENT_EN, BIT(0), MFGPLL_SC0_CON1, 24, + 0, 0, 0, MFGPLL_SC0_CON1, 0, 22), }; static const struct mtk_pll_data mfgsc1_ao_plls[] = { PLL(CLK_MFGSC1_AO_MFGPLL_SC1, "mfgpll-sc1", MFGPLL_SC1_CON0, - MFGPLL_SC1_CON0, 0, 0, 0, BIT(0), MFGPLL_SC1_CON1, 24, 0, 0, 0, - MFGPLL_SC1_CON1, 0, 22), + MFGPLL_SC1_CON0, 0, 0, PLL_PARENT_EN, BIT(0), MFGPLL_SC1_CON1, 24, + 0, 0, 0, MFGPLL_SC1_CON1, 0, 22), }; static const struct of_device_id of_match_clk_mt8196_mfg[] = { diff --git a/drivers/clk/mediatek/clk-mt8516.c b/drivers/clk/mediatek/clk-mt8516.c index 21eb052b0a539c..342a59019fea9b 100644 --- a/drivers/clk/mediatek/clk-mt8516.c +++ b/drivers/clk/mediatek/clk-mt8516.c @@ -544,7 +544,7 @@ static const struct mtk_gate_regs top5_cg_regs = { #define GATE_TOP5(_id, _name, _parent, _shift) \ GATE_MTK(_id, _name, _parent, &top5_cg_regs, _shift, &mtk_clk_gate_ops_setclr) -static const struct mtk_gate top_clks[] __initconst = { +static const struct mtk_gate top_clks[] = { /* TOP1 */ GATE_TOP1(CLK_TOP_THEM, "them", "ahb_infra_sel", 1), GATE_TOP1(CLK_TOP_APDMA, "apdma", "ahb_infra_sel", 2), diff --git a/drivers/clk/mediatek/clk-mtk.c b/drivers/clk/mediatek/clk-mtk.c index 19cd27941747aa..deafe55a96cb1d 100644 --- a/drivers/clk/mediatek/clk-mtk.c +++ b/drivers/clk/mediatek/clk-mtk.c @@ -497,14 +497,16 @@ static int __mtk_clk_simple_probe(struct platform_device *pdev, if (mcd->need_runtime_pm) { - devm_pm_runtime_enable(&pdev->dev); + r = devm_pm_runtime_enable(&pdev->dev); + if (r) + goto unmap_io; /* * Do a pm_runtime_resume_and_get() to workaround a possible * deadlock between clk_register() and the genpd framework. */ r = pm_runtime_resume_and_get(&pdev->dev); if (r) - return r; + goto unmap_io; } /* Calculate how many clk_hw_onecell_data entries to allocate */ @@ -618,11 +620,11 @@ static int __mtk_clk_simple_probe(struct platform_device *pdev, free_data: mtk_free_clk_data(clk_data); free_base: - if (mcd->shared_io && base) - iounmap(base); - if (mcd->need_runtime_pm) pm_runtime_put(&pdev->dev); +unmap_io: + if (mcd->shared_io && base) + iounmap(base); return r; } diff --git a/drivers/clk/mediatek/clk-pll.c b/drivers/clk/mediatek/clk-pll.c index cd2b6ce551c6b0..de3eb02670554f 100644 --- a/drivers/clk/mediatek/clk-pll.c +++ b/drivers/clk/mediatek/clk-pll.c @@ -358,6 +358,9 @@ struct clk_hw *mtk_clk_register_pll_ops(struct mtk_clk_pll *pll, init.name = data->name; init.flags = (data->flags & PLL_AO) ? CLK_IS_CRITICAL : 0; + if (data->flags & PLL_PARENT_EN) + init.flags |= CLK_OPS_PARENT_ENABLE; + init.ops = pll_ops; if (data->parent_name) init.parent_names = &data->parent_name; diff --git a/drivers/clk/mediatek/clk-pll.h b/drivers/clk/mediatek/clk-pll.h index d71c150ce83e4b..de5a8fb7cbcfe9 100644 --- a/drivers/clk/mediatek/clk-pll.h +++ b/drivers/clk/mediatek/clk-pll.h @@ -21,6 +21,7 @@ struct mtk_pll_div_table { #define HAVE_RST_BAR BIT(0) #define PLL_AO BIT(1) +#define PLL_PARENT_EN BIT(2) #define POSTDIV_MASK GENMASK(2, 0) struct mtk_pll_data { diff --git a/drivers/clk/meson/g12a.c b/drivers/clk/meson/g12a.c index 185b6348251dbd..d0d4c7b6dc8277 100644 --- a/drivers/clk/meson/g12a.c +++ b/drivers/clk/meson/g12a.c @@ -777,12 +777,23 @@ static struct clk_regmap g12a_hdmi_pll_dco = { }, }; +/* + * G12/SM1 hdmi OD dividers are POWER_OF_TWO dividers but limited to /4. + * A divider value of 3 should map to /8 but instead map /4 so ignore it. + */ +static const struct clk_div_table g12a_hdmi_pll_od_div_table[] = { + { .val = 0, .div = 1 }, + { .val = 1, .div = 2 }, + { .val = 2, .div = 4 }, + { /* sentinel */ } +}; + static struct clk_regmap g12a_hdmi_pll_od = { .data = &(struct clk_regmap_div_data){ .offset = HHI_HDMI_PLL_CNTL0, .shift = 16, .width = 2, - .flags = CLK_DIVIDER_POWER_OF_TWO, + .table = g12a_hdmi_pll_od_div_table, }, .hw.init = &(struct clk_init_data){ .name = "hdmi_pll_od", @@ -800,7 +811,7 @@ static struct clk_regmap g12a_hdmi_pll_od2 = { .offset = HHI_HDMI_PLL_CNTL0, .shift = 18, .width = 2, - .flags = CLK_DIVIDER_POWER_OF_TWO, + .table = g12a_hdmi_pll_od_div_table, }, .hw.init = &(struct clk_init_data){ .name = "hdmi_pll_od2", @@ -818,7 +829,7 @@ static struct clk_regmap g12a_hdmi_pll = { .offset = HHI_HDMI_PLL_CNTL0, .shift = 20, .width = 2, - .flags = CLK_DIVIDER_POWER_OF_TWO, + .table = g12a_hdmi_pll_od_div_table, }, .hw.init = &(struct clk_init_data){ .name = "hdmi_pll", diff --git a/drivers/clk/meson/gxbb.c b/drivers/clk/meson/gxbb.c index 5a229c4ffae105..ec9a3414875ac3 100644 --- a/drivers/clk/meson/gxbb.c +++ b/drivers/clk/meson/gxbb.c @@ -349,12 +349,23 @@ static struct clk_regmap gxbb_hdmi_pll = { }, }; +/* + * GXL hdmi OD dividers are POWER_OF_TWO dividers but limited to /4. + * A divider value of 3 should map to /8 but instead map /4 so ignore it. + */ +static const struct clk_div_table gxl_hdmi_pll_od_div_table[] = { + { .val = 0, .div = 1 }, + { .val = 1, .div = 2 }, + { .val = 2, .div = 4 }, + { /* sentinel */ } +}; + static struct clk_regmap gxl_hdmi_pll_od = { .data = &(struct clk_regmap_div_data){ .offset = HHI_HDMI_PLL_CNTL + 8, .shift = 21, .width = 2, - .flags = CLK_DIVIDER_POWER_OF_TWO, + .table = gxl_hdmi_pll_od_div_table, }, .hw.init = &(struct clk_init_data){ .name = "hdmi_pll_od", @@ -372,7 +383,7 @@ static struct clk_regmap gxl_hdmi_pll_od2 = { .offset = HHI_HDMI_PLL_CNTL + 8, .shift = 23, .width = 2, - .flags = CLK_DIVIDER_POWER_OF_TWO, + .table = gxl_hdmi_pll_od_div_table, }, .hw.init = &(struct clk_init_data){ .name = "hdmi_pll_od2", @@ -390,7 +401,7 @@ static struct clk_regmap gxl_hdmi_pll = { .offset = HHI_HDMI_PLL_CNTL + 8, .shift = 19, .width = 2, - .flags = CLK_DIVIDER_POWER_OF_TWO, + .table = gxl_hdmi_pll_od_div_table, }, .hw.init = &(struct clk_init_data){ .name = "hdmi_pll", diff --git a/drivers/clk/meson/s4-peripherals.c b/drivers/clk/meson/s4-peripherals.c index 6d69b132d1e1f5..bab4f5700de473 100644 --- a/drivers/clk/meson/s4-peripherals.c +++ b/drivers/clk/meson/s4-peripherals.c @@ -1106,7 +1106,6 @@ static struct clk_regmap s4_cts_enci_sel = { .ops = &clk_regmap_mux_ops, .parent_hws = s4_cts_parents, .num_parents = ARRAY_SIZE(s4_cts_parents), - .flags = CLK_SET_RATE_PARENT, }, }; @@ -1122,7 +1121,6 @@ static struct clk_regmap s4_cts_encp_sel = { .ops = &clk_regmap_mux_ops, .parent_hws = s4_cts_parents, .num_parents = ARRAY_SIZE(s4_cts_parents), - .flags = CLK_SET_RATE_PARENT, }, }; @@ -1138,7 +1136,6 @@ static struct clk_regmap s4_cts_vdac_sel = { .ops = &clk_regmap_mux_ops, .parent_hws = s4_cts_parents, .num_parents = ARRAY_SIZE(s4_cts_parents), - .flags = CLK_SET_RATE_PARENT, }, }; @@ -1169,7 +1166,6 @@ static struct clk_regmap s4_hdmi_tx_sel = { .ops = &clk_regmap_mux_ops, .parent_hws = s4_hdmi_tx_parents, .num_parents = ARRAY_SIZE(s4_hdmi_tx_parents), - .flags = CLK_SET_RATE_PARENT, }, }; diff --git a/drivers/clk/microchip/clk-core.c b/drivers/clk/microchip/clk-core.c index b34348d491f3e1..82f62731fc0ed5 100644 --- a/drivers/clk/microchip/clk-core.c +++ b/drivers/clk/microchip/clk-core.c @@ -283,14 +283,13 @@ static u8 roclk_get_parent(struct clk_hw *hw) v = (readl(refo->ctrl_reg) >> REFO_SEL_SHIFT) & REFO_SEL_MASK; - if (!refo->parent_map) - return v; - - for (i = 0; i < clk_hw_get_num_parents(hw); i++) - if (refo->parent_map[i] == v) - return i; + if (refo->parent_map) { + for (i = 0; i < clk_hw_get_num_parents(hw); i++) + if (refo->parent_map[i] == v) + return i; + } - return -EINVAL; + return v; } static unsigned long roclk_calc_rate(unsigned long parent_rate, @@ -780,15 +779,6 @@ static unsigned long sclk_get_rate(struct clk_hw *hw, unsigned long parent_rate) return parent_rate / div; } -static int sclk_determine_rate(struct clk_hw *hw, - struct clk_rate_request *req) -{ - req->rate = calc_best_divided_rate(req->rate, req->best_parent_rate, - SLEW_SYSDIV, 1); - - return 0; -} - static int sclk_set_rate(struct clk_hw *hw, unsigned long rate, unsigned long parent_rate) { @@ -826,13 +816,13 @@ static u8 sclk_get_parent(struct clk_hw *hw) v = (readl(sclk->mux_reg) >> OSC_CUR_SHIFT) & OSC_CUR_MASK; - if (!sclk->parent_map) - return v; + if (sclk->parent_map) { + for (i = 0; i < clk_hw_get_num_parents(hw); i++) + if (sclk->parent_map[i] == v) + return i; + } - for (i = 0; i < clk_hw_get_num_parents(hw); i++) - if (sclk->parent_map[i] == v) - return i; - return -EINVAL; + return v; } static int sclk_set_parent(struct clk_hw *hw, u8 index) @@ -912,7 +902,6 @@ static int sclk_init(struct clk_hw *hw) const struct clk_ops pic32_sclk_ops = { .get_parent = sclk_get_parent, .set_parent = sclk_set_parent, - .determine_rate = sclk_determine_rate, .set_rate = sclk_set_rate, .recalc_rate = sclk_get_rate, .init = sclk_init, diff --git a/drivers/clk/nuvoton/clk-ma35d1-divider.c b/drivers/clk/nuvoton/clk-ma35d1-divider.c index e39f53d5bf4578..e992e7c3034197 100644 --- a/drivers/clk/nuvoton/clk-ma35d1-divider.c +++ b/drivers/clk/nuvoton/clk-ma35d1-divider.c @@ -44,11 +44,8 @@ static int ma35d1_clkdiv_determine_rate(struct clk_hw *hw, { struct ma35d1_adc_clk_div *dclk = to_ma35d1_adc_clk_div(hw); - req->rate = divider_round_rate(hw, req->rate, &req->best_parent_rate, - dclk->table, dclk->width, - CLK_DIVIDER_ROUND_CLOSEST); - - return 0; + return divider_determine_rate(hw, req, dclk->table, dclk->width, + CLK_DIVIDER_ROUND_CLOSEST); } static int ma35d1_clkdiv_set_rate(struct clk_hw *hw, unsigned long rate, unsigned long parent_rate) diff --git a/drivers/clk/nxp/clk-lpc32xx.c b/drivers/clk/nxp/clk-lpc32xx.c index 23f980cf6a2b59..ae2fa5341a2e4f 100644 --- a/drivers/clk/nxp/clk-lpc32xx.c +++ b/drivers/clk/nxp/clk-lpc32xx.c @@ -975,10 +975,8 @@ static int clk_divider_determine_rate(struct clk_hw *hw, return 0; } - req->rate = divider_round_rate(hw, req->rate, &req->best_parent_rate, - divider->table, divider->width, divider->flags); - - return 0; + return divider_determine_rate(hw, req, divider->table, divider->width, + divider->flags); } static int clk_divider_set_rate(struct clk_hw *hw, unsigned long rate, diff --git a/drivers/clk/qcom/clk-alpha-pll.c b/drivers/clk/qcom/clk-alpha-pll.c index 6aeba40358c11e..a84e8bee653462 100644 --- a/drivers/clk/qcom/clk-alpha-pll.c +++ b/drivers/clk/qcom/clk-alpha-pll.c @@ -1257,11 +1257,8 @@ static int clk_alpha_pll_postdiv_determine_rate(struct clk_hw *hw, else table = clk_alpha_div_table; - req->rate = divider_round_rate(hw, req->rate, &req->best_parent_rate, - table, pll->width, - CLK_DIVIDER_POWER_OF_TWO); - - return 0; + return divider_determine_rate(hw, req, table, pll->width, + CLK_DIVIDER_POWER_OF_TWO); } static int clk_alpha_pll_postdiv_ro_determine_rate(struct clk_hw *hw, @@ -1617,11 +1614,8 @@ static int clk_trion_pll_postdiv_determine_rate(struct clk_hw *hw, { struct clk_alpha_pll_postdiv *pll = to_clk_alpha_pll_postdiv(hw); - req->rate = divider_round_rate(hw, req->rate, &req->best_parent_rate, - pll->post_div_table, - pll->width, CLK_DIVIDER_ROUND_CLOSEST); - - return 0; + return divider_determine_rate(hw, req, pll->post_div_table, pll->width, + CLK_DIVIDER_ROUND_CLOSEST); }; static int @@ -1657,11 +1651,8 @@ static int clk_alpha_pll_postdiv_fabia_determine_rate(struct clk_hw *hw, { struct clk_alpha_pll_postdiv *pll = to_clk_alpha_pll_postdiv(hw); - req->rate = divider_round_rate(hw, req->rate, &req->best_parent_rate, - pll->post_div_table, - pll->width, CLK_DIVIDER_ROUND_CLOSEST); - - return 0; + return divider_determine_rate(hw, req, pll->post_div_table, pll->width, + CLK_DIVIDER_ROUND_CLOSEST); } static int clk_alpha_pll_postdiv_fabia_set_rate(struct clk_hw *hw, diff --git a/drivers/clk/qcom/clk-rcg2.c b/drivers/clk/qcom/clk-rcg2.c index e18cb8807d7353..d0a5847f911144 100644 --- a/drivers/clk/qcom/clk-rcg2.c +++ b/drivers/clk/qcom/clk-rcg2.c @@ -755,7 +755,7 @@ static int clk_rcg2_get_duty_cycle(struct clk_hw *hw, struct clk_duty *duty) static int clk_rcg2_set_duty_cycle(struct clk_hw *hw, struct clk_duty *duty) { struct clk_rcg2 *rcg = to_clk_rcg2(hw); - u32 notn_m, n, m, d, not2d, mask, duty_per, cfg; + u32 notn_m, n, m, d, not2d, mask, cfg; int ret; /* Duty-cycle cannot be modified for non-MND RCGs */ @@ -774,10 +774,8 @@ static int clk_rcg2_set_duty_cycle(struct clk_hw *hw, struct clk_duty *duty) n = (~(notn_m) + m) & mask; - duty_per = (duty->num * 100) / duty->den; - /* Calculate 2d value */ - d = DIV_ROUND_CLOSEST(n * duty_per * 2, 100); + d = DIV_ROUND_CLOSEST(n * duty->num * 2, duty->den); /* * Check bit widths of 2d. If D is too big reduce duty cycle. @@ -1266,6 +1264,7 @@ static int clk_gfx3d_determine_rate(struct clk_hw *hw, if (req->max_rate < parent_req.max_rate) parent_req.max_rate = req->max_rate; + parent_req.best_parent_hw = req->best_parent_hw; ret = __clk_determine_rate(req->best_parent_hw, &parent_req); if (ret) return ret; diff --git a/drivers/clk/qcom/clk-regmap-divider.c b/drivers/clk/qcom/clk-regmap-divider.c index 4f5395f0ab6d0e..672e82caf20504 100644 --- a/drivers/clk/qcom/clk-regmap-divider.c +++ b/drivers/clk/qcom/clk-regmap-divider.c @@ -26,24 +26,16 @@ static int div_ro_determine_rate(struct clk_hw *hw, val >>= divider->shift; val &= BIT(divider->width) - 1; - req->rate = divider_ro_round_rate(hw, req->rate, - &req->best_parent_rate, NULL, - divider->width, - CLK_DIVIDER_ROUND_CLOSEST, val); - - return 0; + return divider_ro_determine_rate(hw, req, NULL, divider->width, + CLK_DIVIDER_ROUND_CLOSEST, val); } static int div_determine_rate(struct clk_hw *hw, struct clk_rate_request *req) { struct clk_regmap_div *divider = to_clk_regmap_div(hw); - req->rate = divider_round_rate(hw, req->rate, &req->best_parent_rate, - NULL, - divider->width, - CLK_DIVIDER_ROUND_CLOSEST); - - return 0; + return divider_determine_rate(hw, req, NULL, divider->width, + CLK_DIVIDER_ROUND_CLOSEST); } static int div_set_rate(struct clk_hw *hw, unsigned long rate, diff --git a/drivers/clk/qcom/common.c b/drivers/clk/qcom/common.c index 12159188677418..eec369d2173b5c 100644 --- a/drivers/clk/qcom/common.c +++ b/drivers/clk/qcom/common.c @@ -454,7 +454,7 @@ int qcom_cc_probe_by_index(struct platform_device *pdev, int index, base = devm_platform_ioremap_resource(pdev, index); if (IS_ERR(base)) - return -ENOMEM; + return PTR_ERR(base); regmap = devm_regmap_init_mmio(&pdev->dev, base, desc->config); if (IS_ERR(regmap)) diff --git a/drivers/clk/qcom/dispcc-sdm845.c b/drivers/clk/qcom/dispcc-sdm845.c index 2f9e9665d7e93f..78e43f6d750260 100644 --- a/drivers/clk/qcom/dispcc-sdm845.c +++ b/drivers/clk/qcom/dispcc-sdm845.c @@ -280,7 +280,7 @@ static struct clk_rcg2 disp_cc_mdss_pclk0_clk_src = { .name = "disp_cc_mdss_pclk0_clk_src", .parent_data = disp_cc_parent_data_4, .num_parents = ARRAY_SIZE(disp_cc_parent_data_4), - .flags = CLK_SET_RATE_PARENT, + .flags = CLK_SET_RATE_PARENT | CLK_OPS_PARENT_ENABLE, .ops = &clk_pixel_ops, }, }; @@ -295,7 +295,7 @@ static struct clk_rcg2 disp_cc_mdss_pclk1_clk_src = { .name = "disp_cc_mdss_pclk1_clk_src", .parent_data = disp_cc_parent_data_4, .num_parents = ARRAY_SIZE(disp_cc_parent_data_4), - .flags = CLK_SET_RATE_PARENT, + .flags = CLK_SET_RATE_PARENT | CLK_OPS_PARENT_ENABLE, .ops = &clk_pixel_ops, }, }; diff --git a/drivers/clk/qcom/dispcc-sm7150.c b/drivers/clk/qcom/dispcc-sm7150.c index 811d380a8e9f9b..ed8e34ffd69b05 100644 --- a/drivers/clk/qcom/dispcc-sm7150.c +++ b/drivers/clk/qcom/dispcc-sm7150.c @@ -371,7 +371,7 @@ static struct clk_rcg2 dispcc_mdss_pclk1_clk_src = { .name = "dispcc_mdss_pclk1_clk_src", .parent_data = dispcc_parent_data_4, .num_parents = ARRAY_SIZE(dispcc_parent_data_4), - .flags = CLK_SET_RATE_PARENT, + .flags = CLK_SET_RATE_PARENT | CLK_OPS_PARENT_ENABLE, .ops = &clk_pixel_ops, }, }; diff --git a/drivers/clk/qcom/gcc-glymur.c b/drivers/clk/qcom/gcc-glymur.c index deab819576d0e1..238e205735ed59 100644 --- a/drivers/clk/qcom/gcc-glymur.c +++ b/drivers/clk/qcom/gcc-glymur.c @@ -2317,7 +2317,7 @@ static struct clk_rcg2 gcc_sdcc2_apps_clk_src = { .parent_data = gcc_parent_data_17, .num_parents = ARRAY_SIZE(gcc_parent_data_17), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_floor_ops, + .ops = &clk_rcg2_shared_floor_ops, }, }; @@ -2339,7 +2339,7 @@ static struct clk_rcg2 gcc_sdcc4_apps_clk_src = { .parent_data = gcc_parent_data_3, .num_parents = ARRAY_SIZE(gcc_parent_data_3), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_floor_ops, + .ops = &clk_rcg2_shared_floor_ops, }, }; diff --git a/drivers/clk/qcom/gcc-ipq5018.c b/drivers/clk/qcom/gcc-ipq5018.c index dcda2be8c1a519..64792cda062021 100644 --- a/drivers/clk/qcom/gcc-ipq5018.c +++ b/drivers/clk/qcom/gcc-ipq5018.c @@ -1340,6 +1340,7 @@ static struct clk_branch gcc_sleep_clk_src = { .name = "gcc_sleep_clk_src", .parent_data = gcc_sleep_clk_data, .num_parents = ARRAY_SIZE(gcc_sleep_clk_data), + .flags = CLK_IS_CRITICAL, .ops = &clk_branch2_ops, }, }, diff --git a/drivers/clk/qcom/gcc-milos.c b/drivers/clk/qcom/gcc-milos.c index c9d61b05bafa16..81fa09ec55d7f6 100644 --- a/drivers/clk/qcom/gcc-milos.c +++ b/drivers/clk/qcom/gcc-milos.c @@ -917,7 +917,7 @@ static struct clk_rcg2 gcc_sdcc1_apps_clk_src = { .name = "gcc_sdcc1_apps_clk_src", .parent_data = gcc_parent_data_9, .num_parents = ARRAY_SIZE(gcc_parent_data_9), - .ops = &clk_rcg2_shared_ops, + .ops = &clk_rcg2_shared_floor_ops, }, }; @@ -938,7 +938,7 @@ static struct clk_rcg2 gcc_sdcc1_ice_core_clk_src = { .name = "gcc_sdcc1_ice_core_clk_src", .parent_data = gcc_parent_data_10, .num_parents = ARRAY_SIZE(gcc_parent_data_10), - .ops = &clk_rcg2_shared_ops, + .ops = &clk_rcg2_shared_floor_ops, }, }; @@ -962,7 +962,7 @@ static struct clk_rcg2 gcc_sdcc2_apps_clk_src = { .name = "gcc_sdcc2_apps_clk_src", .parent_data = gcc_parent_data_11, .num_parents = ARRAY_SIZE(gcc_parent_data_11), - .ops = &clk_rcg2_shared_ops, + .ops = &clk_rcg2_shared_floor_ops, }, }; diff --git a/drivers/clk/qcom/gcc-msm8917.c b/drivers/clk/qcom/gcc-msm8917.c index 0a1aa623cd49af..9d1c5a9953e2c6 100644 --- a/drivers/clk/qcom/gcc-msm8917.c +++ b/drivers/clk/qcom/gcc-msm8917.c @@ -3409,7 +3409,6 @@ static struct gdsc cpp_gdsc = { .pd = { .name = "cpp_gdsc", }, - .flags = ALWAYS_ON, .pwrsts = PWRSTS_OFF_ON, }; diff --git a/drivers/clk/qcom/gcc-msm8953.c b/drivers/clk/qcom/gcc-msm8953.c index 8f29ecc74c50bf..8fe1d3e421440c 100644 --- a/drivers/clk/qcom/gcc-msm8953.c +++ b/drivers/clk/qcom/gcc-msm8953.c @@ -3946,7 +3946,6 @@ static struct gdsc cpp_gdsc = { .pd = { .name = "cpp_gdsc", }, - .flags = ALWAYS_ON, .pwrsts = PWRSTS_OFF_ON, }; diff --git a/drivers/clk/qcom/gcc-qdu1000.c b/drivers/clk/qcom/gcc-qdu1000.c index dbe9e9437939af..915bb9b4ff8130 100644 --- a/drivers/clk/qcom/gcc-qdu1000.c +++ b/drivers/clk/qcom/gcc-qdu1000.c @@ -904,7 +904,7 @@ static struct clk_rcg2 gcc_sdcc5_apps_clk_src = { .name = "gcc_sdcc5_apps_clk_src", .parent_data = gcc_parent_data_8, .num_parents = ARRAY_SIZE(gcc_parent_data_8), - .ops = &clk_rcg2_floor_ops, + .ops = &clk_rcg2_shared_floor_ops, }, }; @@ -923,7 +923,7 @@ static struct clk_rcg2 gcc_sdcc5_ice_core_clk_src = { .name = "gcc_sdcc5_ice_core_clk_src", .parent_data = gcc_parent_data_2, .num_parents = ARRAY_SIZE(gcc_parent_data_2), - .ops = &clk_rcg2_floor_ops, + .ops = &clk_rcg2_shared_floor_ops, }, }; diff --git a/drivers/clk/qcom/gcc-sdx75.c b/drivers/clk/qcom/gcc-sdx75.c index 453a6bf8e87863..1f3cd58483a2d6 100644 --- a/drivers/clk/qcom/gcc-sdx75.c +++ b/drivers/clk/qcom/gcc-sdx75.c @@ -1033,7 +1033,7 @@ static struct clk_rcg2 gcc_sdcc1_apps_clk_src = { .name = "gcc_sdcc1_apps_clk_src", .parent_data = gcc_parent_data_17, .num_parents = ARRAY_SIZE(gcc_parent_data_17), - .ops = &clk_rcg2_floor_ops, + .ops = &clk_rcg2_shared_floor_ops, }, }; @@ -1057,7 +1057,7 @@ static struct clk_rcg2 gcc_sdcc2_apps_clk_src = { .name = "gcc_sdcc2_apps_clk_src", .parent_data = gcc_parent_data_18, .num_parents = ARRAY_SIZE(gcc_parent_data_18), - .ops = &clk_rcg2_floor_ops, + .ops = &clk_rcg2_shared_floor_ops, }, }; diff --git a/drivers/clk/qcom/gcc-sm4450.c b/drivers/clk/qcom/gcc-sm4450.c index e2d9e4691c5b71..023d840e9f4ef0 100644 --- a/drivers/clk/qcom/gcc-sm4450.c +++ b/drivers/clk/qcom/gcc-sm4450.c @@ -769,7 +769,7 @@ static struct clk_rcg2 gcc_sdcc1_apps_clk_src = { .parent_data = gcc_parent_data_4, .num_parents = ARRAY_SIZE(gcc_parent_data_4), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_floor_ops, + .ops = &clk_rcg2_shared_floor_ops, }, }; @@ -791,7 +791,7 @@ static struct clk_rcg2 gcc_sdcc1_ice_core_clk_src = { .parent_data = gcc_parent_data_4, .num_parents = ARRAY_SIZE(gcc_parent_data_4), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_floor_ops, + .ops = &clk_rcg2_shared_floor_ops, }, }; @@ -815,7 +815,7 @@ static struct clk_rcg2 gcc_sdcc2_apps_clk_src = { .parent_data = gcc_parent_data_6, .num_parents = ARRAY_SIZE(gcc_parent_data_6), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_floor_ops, + .ops = &clk_rcg2_shared_floor_ops, }, }; diff --git a/drivers/clk/qcom/gcc-sm8450.c b/drivers/clk/qcom/gcc-sm8450.c index 65d7d52bce0343..b18bb34889ab28 100644 --- a/drivers/clk/qcom/gcc-sm8450.c +++ b/drivers/clk/qcom/gcc-sm8450.c @@ -1034,7 +1034,7 @@ static struct clk_rcg2 gcc_sdcc2_apps_clk_src = { .parent_data = gcc_parent_data_7, .num_parents = ARRAY_SIZE(gcc_parent_data_7), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_floor_ops, + .ops = &clk_rcg2_shared_floor_ops, }, }; @@ -1057,7 +1057,7 @@ static struct clk_rcg2 gcc_sdcc4_apps_clk_src = { .parent_data = gcc_parent_data_0, .num_parents = ARRAY_SIZE(gcc_parent_data_0), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_floor_ops, + .ops = &clk_rcg2_shared_floor_ops, }, }; diff --git a/drivers/clk/qcom/gcc-sm8550.c b/drivers/clk/qcom/gcc-sm8550.c index 862a9bf73bcb5d..36a5b7de5b55d2 100644 --- a/drivers/clk/qcom/gcc-sm8550.c +++ b/drivers/clk/qcom/gcc-sm8550.c @@ -1025,7 +1025,7 @@ static struct clk_rcg2 gcc_sdcc2_apps_clk_src = { .parent_data = gcc_parent_data_9, .num_parents = ARRAY_SIZE(gcc_parent_data_9), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_shared_ops, + .ops = &clk_rcg2_shared_floor_ops, }, }; @@ -1048,7 +1048,7 @@ static struct clk_rcg2 gcc_sdcc4_apps_clk_src = { .parent_data = gcc_parent_data_0, .num_parents = ARRAY_SIZE(gcc_parent_data_0), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_shared_ops, + .ops = &clk_rcg2_shared_floor_ops, }, }; diff --git a/drivers/clk/qcom/gcc-sm8650.c b/drivers/clk/qcom/gcc-sm8650.c index 24f98062b9dd50..2dd6444ce0365f 100644 --- a/drivers/clk/qcom/gcc-sm8650.c +++ b/drivers/clk/qcom/gcc-sm8650.c @@ -1257,7 +1257,7 @@ static struct clk_rcg2 gcc_sdcc2_apps_clk_src = { .parent_data = gcc_parent_data_11, .num_parents = ARRAY_SIZE(gcc_parent_data_11), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_shared_ops, + .ops = &clk_rcg2_shared_floor_ops, }, }; @@ -1279,7 +1279,7 @@ static struct clk_rcg2 gcc_sdcc4_apps_clk_src = { .parent_data = gcc_parent_data_0, .num_parents = ARRAY_SIZE(gcc_parent_data_0), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_shared_ops, + .ops = &clk_rcg2_shared_floor_ops, }, }; diff --git a/drivers/clk/qcom/gcc-sm8750.c b/drivers/clk/qcom/gcc-sm8750.c index def86b71a3da53..db81569dd4b17d 100644 --- a/drivers/clk/qcom/gcc-sm8750.c +++ b/drivers/clk/qcom/gcc-sm8750.c @@ -1030,7 +1030,7 @@ static struct clk_rcg2 gcc_sdcc2_apps_clk_src = { .parent_data = gcc_parent_data_8, .num_parents = ARRAY_SIZE(gcc_parent_data_8), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_floor_ops, + .ops = &clk_rcg2_shared_floor_ops, }, }; @@ -1052,7 +1052,7 @@ static struct clk_rcg2 gcc_sdcc4_apps_clk_src = { .parent_data = gcc_parent_data_0, .num_parents = ARRAY_SIZE(gcc_parent_data_0), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_floor_ops, + .ops = &clk_rcg2_shared_floor_ops, }, }; diff --git a/drivers/clk/qcom/gcc-x1e80100.c b/drivers/clk/qcom/gcc-x1e80100.c index b63c8abdd2fc24..e46e65e631513e 100644 --- a/drivers/clk/qcom/gcc-x1e80100.c +++ b/drivers/clk/qcom/gcc-x1e80100.c @@ -1516,7 +1516,7 @@ static struct clk_rcg2 gcc_sdcc2_apps_clk_src = { .parent_data = gcc_parent_data_9, .num_parents = ARRAY_SIZE(gcc_parent_data_9), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_floor_ops, + .ops = &clk_rcg2_shared_floor_ops, }, }; @@ -1538,7 +1538,7 @@ static struct clk_rcg2 gcc_sdcc4_apps_clk_src = { .parent_data = gcc_parent_data_0, .num_parents = ARRAY_SIZE(gcc_parent_data_0), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_floor_ops, + .ops = &clk_rcg2_shared_floor_ops, }, }; diff --git a/drivers/clk/renesas/rzg2l-cpg.c b/drivers/clk/renesas/rzg2l-cpg.c index 64d1ef6e4c943c..f670c6408ea153 100644 --- a/drivers/clk/renesas/rzg2l-cpg.c +++ b/drivers/clk/renesas/rzg2l-cpg.c @@ -122,8 +122,8 @@ struct div_hw_data { struct rzg2l_pll5_param { u32 pl5_fracin; + u16 pl5_intin; u8 pl5_refdiv; - u8 pl5_intin; u8 pl5_postdiv1; u8 pl5_postdiv2; u8 pl5_spread; @@ -572,8 +572,8 @@ rzg2l_cpg_get_foutpostdiv_rate(struct rzg2l_pll5_param *params, foutvco_rate = div_u64(mul_u32_u32(EXTAL_FREQ_IN_MEGA_HZ * MEGA, (params->pl5_intin << 24) + params->pl5_fracin), params->pl5_refdiv) >> 24; - foutpostdiv_rate = DIV_ROUND_CLOSEST_ULL(foutvco_rate, - params->pl5_postdiv1 * params->pl5_postdiv2); + foutpostdiv_rate = DIV_ROUND_CLOSEST(foutvco_rate, + params->pl5_postdiv1 * params->pl5_postdiv2); return foutpostdiv_rate; } @@ -1647,6 +1647,7 @@ static int __rzg2l_cpg_assert(struct reset_controller_dev *rcdev, u32 mask = BIT(info->resets[id].bit); s8 monbit = info->resets[id].monbit; u32 value = mask << 16; + u32 mon; int ret; dev_dbg(rcdev->dev, "%s id:%ld offset:0x%x\n", @@ -1667,10 +1668,10 @@ static int __rzg2l_cpg_assert(struct reset_controller_dev *rcdev, return 0; } - ret = readl_poll_timeout_atomic(priv->base + reg, value, - assert == !!(value & mask), 10, 200); - if (ret && !assert) { - value = mask << 16; + ret = readl_poll_timeout_atomic(priv->base + reg, mon, + assert == !!(mon & mask), 10, 200); + if (ret) { + value ^= mask; writel(value, priv->base + CLK_RST_R(info->resets[id].off)); } diff --git a/drivers/clk/rockchip/clk.c b/drivers/clk/rockchip/clk.c index 2601df3b1066b1..9ac9d13e87de0f 100644 --- a/drivers/clk/rockchip/clk.c +++ b/drivers/clk/rockchip/clk.c @@ -693,7 +693,7 @@ void rockchip_clk_register_late_branches(struct device *dev, break; } - if (!pdev) + if (IS_ERR_OR_NULL(pdev)) dev_err(dev, "failed to register device for clock %s\n", list->name); } } diff --git a/drivers/clk/sophgo/clk-sg2042-clkgen.c b/drivers/clk/sophgo/clk-sg2042-clkgen.c index 683661b71787c9..9725ac4e050a4e 100644 --- a/drivers/clk/sophgo/clk-sg2042-clkgen.c +++ b/drivers/clk/sophgo/clk-sg2042-clkgen.c @@ -180,7 +180,6 @@ static int sg2042_clk_divider_determine_rate(struct clk_hw *hw, struct clk_rate_request *req) { struct sg2042_divider_clock *divider = to_sg2042_clk_divider(hw); - unsigned long ret_rate; u32 bestdiv; /* if read only, just return current value */ @@ -191,17 +190,13 @@ static int sg2042_clk_divider_determine_rate(struct clk_hw *hw, bestdiv = readl(divider->reg) >> divider->shift; bestdiv &= clk_div_mask(divider->width); } - ret_rate = DIV_ROUND_UP_ULL((u64)req->best_parent_rate, bestdiv); - } else { - ret_rate = divider_round_rate(hw, req->rate, &req->best_parent_rate, NULL, - divider->width, divider->div_flags); - } + req->rate = DIV_ROUND_UP_ULL((u64)req->best_parent_rate, bestdiv); - pr_debug("--> %s: divider_round_rate: val = %ld\n", - clk_hw_get_name(hw), ret_rate); - req->rate = ret_rate; + return 0; + } - return 0; + return divider_determine_rate(hw, req, NULL, divider->width, + divider->div_flags); } static int sg2042_clk_divider_set_rate(struct clk_hw *hw, diff --git a/drivers/clk/spacemit/Makefile b/drivers/clk/spacemit/Makefile index 5ec6da61db98e0..ad2bf315109b80 100644 --- a/drivers/clk/spacemit/Makefile +++ b/drivers/clk/spacemit/Makefile @@ -1,5 +1,10 @@ # SPDX-License-Identifier: GPL-2.0 -obj-$(CONFIG_SPACEMIT_K1_CCU) = spacemit-ccu-k1.o -spacemit-ccu-k1-y = ccu_pll.o ccu_mix.o ccu_ddn.o +obj-$(CONFIG_SPACEMIT_CCU) += spacemit-ccu.o +spacemit-ccu-y += ccu_common.o +spacemit-ccu-y += ccu_pll.o +spacemit-ccu-y += ccu_mix.o +spacemit-ccu-y += ccu_ddn.o + +obj-$(CONFIG_SPACEMIT_K1_CCU) += spacemit-ccu-k1.o spacemit-ccu-k1-y += ccu-k1.o diff --git a/drivers/clk/spacemit/ccu-k1.c b/drivers/clk/spacemit/ccu-k1.c index 4761bc1e3b6e60..01d9485b615d32 100644 --- a/drivers/clk/spacemit/ccu-k1.c +++ b/drivers/clk/spacemit/ccu-k1.c @@ -1204,6 +1204,7 @@ static struct platform_driver k1_ccu_driver = { }; module_platform_driver(k1_ccu_driver); +MODULE_IMPORT_NS("CLK_SPACEMIT"); MODULE_DESCRIPTION("SpacemiT K1 CCU driver"); MODULE_AUTHOR("Haylen Chu "); MODULE_LICENSE("GPL"); diff --git a/drivers/clk/spacemit/ccu_common.c b/drivers/clk/spacemit/ccu_common.c new file mode 100644 index 00000000000000..4412c4104dabb3 --- /dev/null +++ b/drivers/clk/spacemit/ccu_common.c @@ -0,0 +1,6 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#include + +MODULE_DESCRIPTION("SpacemiT CCU common clock driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/spacemit/ccu_ddn.c b/drivers/clk/spacemit/ccu_ddn.c index 5b16e273bee5b1..b5540e0781ffa3 100644 --- a/drivers/clk/spacemit/ccu_ddn.c +++ b/drivers/clk/spacemit/ccu_ddn.c @@ -84,3 +84,4 @@ const struct clk_ops spacemit_ccu_ddn_ops = { .determine_rate = ccu_ddn_determine_rate, .set_rate = ccu_ddn_set_rate, }; +EXPORT_SYMBOL_NS_GPL(spacemit_ccu_ddn_ops, "CLK_SPACEMIT"); diff --git a/drivers/clk/spacemit/ccu_mix.c b/drivers/clk/spacemit/ccu_mix.c index 7b799087537231..67f8b12b4f5b7d 100644 --- a/drivers/clk/spacemit/ccu_mix.c +++ b/drivers/clk/spacemit/ccu_mix.c @@ -198,24 +198,28 @@ const struct clk_ops spacemit_ccu_gate_ops = { .enable = ccu_gate_enable, .is_enabled = ccu_gate_is_enabled, }; +EXPORT_SYMBOL_NS_GPL(spacemit_ccu_gate_ops, "CLK_SPACEMIT"); const struct clk_ops spacemit_ccu_factor_ops = { .determine_rate = ccu_factor_determine_rate, .recalc_rate = ccu_factor_recalc_rate, .set_rate = ccu_factor_set_rate, }; +EXPORT_SYMBOL_NS_GPL(spacemit_ccu_factor_ops, "CLK_SPACEMIT"); const struct clk_ops spacemit_ccu_mux_ops = { .determine_rate = ccu_mix_determine_rate, .get_parent = ccu_mux_get_parent, .set_parent = ccu_mux_set_parent, }; +EXPORT_SYMBOL_NS_GPL(spacemit_ccu_mux_ops, "CLK_SPACEMIT"); const struct clk_ops spacemit_ccu_div_ops = { .determine_rate = ccu_mix_determine_rate, .recalc_rate = ccu_div_recalc_rate, .set_rate = ccu_mix_set_rate, }; +EXPORT_SYMBOL_NS_GPL(spacemit_ccu_div_ops, "CLK_SPACEMIT"); const struct clk_ops spacemit_ccu_factor_gate_ops = { .disable = ccu_gate_disable, @@ -226,6 +230,7 @@ const struct clk_ops spacemit_ccu_factor_gate_ops = { .recalc_rate = ccu_factor_recalc_rate, .set_rate = ccu_factor_set_rate, }; +EXPORT_SYMBOL_NS_GPL(spacemit_ccu_factor_gate_ops, "CLK_SPACEMIT"); const struct clk_ops spacemit_ccu_mux_gate_ops = { .disable = ccu_gate_disable, @@ -236,6 +241,7 @@ const struct clk_ops spacemit_ccu_mux_gate_ops = { .get_parent = ccu_mux_get_parent, .set_parent = ccu_mux_set_parent, }; +EXPORT_SYMBOL_NS_GPL(spacemit_ccu_mux_gate_ops, "CLK_SPACEMIT"); const struct clk_ops spacemit_ccu_div_gate_ops = { .disable = ccu_gate_disable, @@ -246,6 +252,7 @@ const struct clk_ops spacemit_ccu_div_gate_ops = { .recalc_rate = ccu_div_recalc_rate, .set_rate = ccu_mix_set_rate, }; +EXPORT_SYMBOL_NS_GPL(spacemit_ccu_div_gate_ops, "CLK_SPACEMIT"); const struct clk_ops spacemit_ccu_mux_div_gate_ops = { .disable = ccu_gate_disable, @@ -259,6 +266,7 @@ const struct clk_ops spacemit_ccu_mux_div_gate_ops = { .recalc_rate = ccu_div_recalc_rate, .set_rate = ccu_mix_set_rate, }; +EXPORT_SYMBOL_NS_GPL(spacemit_ccu_mux_div_gate_ops, "CLK_SPACEMIT"); const struct clk_ops spacemit_ccu_mux_div_ops = { .get_parent = ccu_mux_get_parent, @@ -268,3 +276,4 @@ const struct clk_ops spacemit_ccu_mux_div_ops = { .recalc_rate = ccu_div_recalc_rate, .set_rate = ccu_mix_set_rate, }; +EXPORT_SYMBOL_NS_GPL(spacemit_ccu_mux_div_ops, "CLK_SPACEMIT"); diff --git a/drivers/clk/spacemit/ccu_pll.c b/drivers/clk/spacemit/ccu_pll.c index d92f0dae65a490..76d0244873d87c 100644 --- a/drivers/clk/spacemit/ccu_pll.c +++ b/drivers/clk/spacemit/ccu_pll.c @@ -157,3 +157,4 @@ const struct clk_ops spacemit_ccu_pll_ops = { .determine_rate = ccu_pll_determine_rate, .is_enabled = ccu_pll_is_enabled, }; +EXPORT_SYMBOL_NS_GPL(spacemit_ccu_pll_ops, "CLK_SPACEMIT"); diff --git a/drivers/clk/sprd/div.c b/drivers/clk/sprd/div.c index 01342388196800..cd57163a7204c0 100644 --- a/drivers/clk/sprd/div.c +++ b/drivers/clk/sprd/div.c @@ -14,11 +14,7 @@ static int sprd_div_determine_rate(struct clk_hw *hw, { struct sprd_div *cd = hw_to_sprd_div(hw); - req->rate = divider_round_rate(&cd->common.hw, req->rate, - &req->best_parent_rate, - NULL, cd->div.width, 0); - - return 0; + return divider_determine_rate(&cd->common.hw, req, NULL, cd->div.width, 0); } unsigned long sprd_div_helper_recalc_rate(struct sprd_clk_common *common, diff --git a/drivers/clk/stm32/clk-stm32-core.c b/drivers/clk/stm32/clk-stm32-core.c index 72825b9c36a4d3..e921c25a929c30 100644 --- a/drivers/clk/stm32/clk-stm32-core.c +++ b/drivers/clk/stm32/clk-stm32-core.c @@ -369,22 +369,14 @@ static int clk_stm32_divider_determine_rate(struct clk_hw *hw, val = readl(div->base + divider->offset) >> divider->shift; val &= clk_div_mask(divider->width); - req->rate = divider_ro_round_rate(hw, req->rate, - &req->best_parent_rate, - divider->table, - divider->width, - divider->flags, val); - - return 0; + return divider_ro_determine_rate(hw, req, + divider->table, + divider->width, + divider->flags, val); } - req->rate = divider_round_rate_parent(hw, clk_hw_get_parent(hw), - req->rate, - &req->best_parent_rate, - divider->table, - divider->width, divider->flags); - - return 0; + return divider_determine_rate(hw, req, divider->table, divider->width, + divider->flags); } static unsigned long clk_stm32_divider_recalc_rate(struct clk_hw *hw, @@ -441,7 +433,6 @@ static int clk_stm32_composite_determine_rate(struct clk_hw *hw, { struct clk_stm32_composite *composite = to_clk_stm32_composite(hw); const struct stm32_div_cfg *divider; - long rate; if (composite->div_id == NO_STM32_DIV) return 0; @@ -455,24 +446,13 @@ static int clk_stm32_composite_determine_rate(struct clk_hw *hw, val = readl(composite->base + divider->offset) >> divider->shift; val &= clk_div_mask(divider->width); - rate = divider_ro_round_rate(hw, req->rate, &req->best_parent_rate, - divider->table, divider->width, divider->flags, - val); - if (rate < 0) - return rate; - - req->rate = rate; - return 0; + return divider_ro_determine_rate(hw, req, divider->table, + divider->width, divider->flags, + val); } - rate = divider_round_rate_parent(hw, clk_hw_get_parent(hw), - req->rate, &req->best_parent_rate, - divider->table, divider->width, divider->flags); - if (rate < 0) - return rate; - - req->rate = rate; - return 0; + return divider_determine_rate(hw, req, divider->table, divider->width, + divider->flags); } static u8 clk_stm32_composite_get_parent(struct clk_hw *hw) diff --git a/drivers/clk/tegra/clk-tegra124-emc.c b/drivers/clk/tegra/clk-tegra124-emc.c index 2a6db043428159..5f1af6dfe71546 100644 --- a/drivers/clk/tegra/clk-tegra124-emc.c +++ b/drivers/clk/tegra/clk-tegra124-emc.c @@ -197,8 +197,8 @@ static struct tegra_emc *emc_ensure_emc_driver(struct tegra_clk_emc *tegra) tegra->emc_node = NULL; tegra->emc = platform_get_drvdata(pdev); + put_device(&pdev->dev); if (!tegra->emc) { - put_device(&pdev->dev); pr_err("%s: cannot find EMC driver\n", __func__); return NULL; } @@ -538,8 +538,10 @@ struct clk *tegra124_clk_register_emc(void __iomem *base, struct device_node *np tegra->hw.init = &init; clk = clk_register(NULL, &tegra->hw); - if (IS_ERR(clk)) + if (IS_ERR(clk)) { + kfree(tegra); return clk; + } tegra->prev_parent = clk_hw_get_parent_by_index( &tegra->hw, emc_get_parent(&tegra->hw))->clk; diff --git a/drivers/clk/thead/clk-th1520-ap.c b/drivers/clk/thead/clk-th1520-ap.c index 71ad03a998e8e1..d870f0c665f8a4 100644 --- a/drivers/clk/thead/clk-th1520-ap.c +++ b/drivers/clk/thead/clk-th1520-ap.c @@ -8,11 +8,14 @@ #include #include #include +#include #include #include #include #include +#define TH1520_PLL_STS 0x80 + #define TH1520_PLL_POSTDIV2 GENMASK(26, 24) #define TH1520_PLL_POSTDIV1 GENMASK(22, 20) #define TH1520_PLL_FBDIV GENMASK(19, 8) @@ -23,6 +26,13 @@ #define TH1520_PLL_FRAC GENMASK(23, 0) #define TH1520_PLL_FRAC_BITS 24 +/* + * All PLLs in TH1520 take 21250ns at maximum to lock, let's take its double + * for safety. + */ +#define TH1520_PLL_LOCK_TIMEOUT_US 44 +#define TH1520_PLL_STABLE_DELAY_US 30 + struct ccu_internal { u8 shift; u8 width; @@ -64,6 +74,7 @@ struct ccu_div { struct ccu_pll { struct ccu_common common; + u32 lock_sts_mask; }; #define TH_CCU_ARG(_shift, _width) \ @@ -299,9 +310,21 @@ static void ccu_pll_disable(struct clk_hw *hw) static int ccu_pll_enable(struct clk_hw *hw) { struct ccu_pll *pll = hw_to_ccu_pll(hw); + u32 reg; + int ret; - return regmap_clear_bits(pll->common.map, pll->common.cfg1, - TH1520_PLL_VCO_RST); + regmap_clear_bits(pll->common.map, pll->common.cfg1, + TH1520_PLL_VCO_RST); + + ret = regmap_read_poll_timeout_atomic(pll->common.map, TH1520_PLL_STS, + reg, reg & pll->lock_sts_mask, + 5, TH1520_PLL_LOCK_TIMEOUT_US); + if (ret) + return ret; + + udelay(TH1520_PLL_STABLE_DELAY_US); + + return 0; } static int ccu_pll_is_enabled(struct clk_hw *hw) @@ -389,6 +412,7 @@ static struct ccu_pll cpu_pll0_clk = { &clk_pll_ops, CLK_IS_CRITICAL), }, + .lock_sts_mask = BIT(1), }; static struct ccu_pll cpu_pll1_clk = { @@ -401,6 +425,7 @@ static struct ccu_pll cpu_pll1_clk = { &clk_pll_ops, CLK_IS_CRITICAL), }, + .lock_sts_mask = BIT(4), }; static struct ccu_pll gmac_pll_clk = { @@ -413,6 +438,7 @@ static struct ccu_pll gmac_pll_clk = { &clk_pll_ops, CLK_IS_CRITICAL), }, + .lock_sts_mask = BIT(3), }; static const struct clk_hw *gmac_pll_clk_parent[] = { @@ -433,6 +459,7 @@ static struct ccu_pll video_pll_clk = { &clk_pll_ops, CLK_IS_CRITICAL), }, + .lock_sts_mask = BIT(7), }; static const struct clk_hw *video_pll_clk_parent[] = { @@ -453,6 +480,7 @@ static struct ccu_pll dpu0_pll_clk = { &clk_pll_ops, 0), }, + .lock_sts_mask = BIT(8), }; static const struct clk_hw *dpu0_pll_clk_parent[] = { @@ -469,6 +497,7 @@ static struct ccu_pll dpu1_pll_clk = { &clk_pll_ops, 0), }, + .lock_sts_mask = BIT(9), }; static const struct clk_hw *dpu1_pll_clk_parent[] = { @@ -485,6 +514,7 @@ static struct ccu_pll tee_pll_clk = { &clk_pll_ops, CLK_IS_CRITICAL), }, + .lock_sts_mask = BIT(10), }; static const struct clk_parent_data c910_i0_parents[] = { diff --git a/drivers/clk/x86/clk-cgu.c b/drivers/clk/x86/clk-cgu.c index d099667355f8d8..92ee05d75af2b2 100644 --- a/drivers/clk/x86/clk-cgu.c +++ b/drivers/clk/x86/clk-cgu.c @@ -137,10 +137,8 @@ static int lgm_clk_divider_determine_rate(struct clk_hw *hw, { struct lgm_clk_divider *divider = to_lgm_clk_divider(hw); - req->rate = divider_round_rate(hw, req->rate, &req->best_parent_rate, divider->table, - divider->width, divider->flags); - - return 0; + return divider_determine_rate(hw, req, divider->table, divider->width, + divider->flags); } static int diff --git a/drivers/clk/zynqmp/divider.c b/drivers/clk/zynqmp/divider.c index c824eeacd8ebd4..984e577ea67115 100644 --- a/drivers/clk/zynqmp/divider.c +++ b/drivers/clk/zynqmp/divider.c @@ -111,10 +111,9 @@ static unsigned long zynqmp_clk_divider_recalc_rate(struct clk_hw *hw, } /** - * zynqmp_clk_divider_round_rate() - Round rate of divider clock + * zynqmp_clk_divider_determine_rate() - Determine rate of divider clock * @hw: handle between common and hardware-specific interfaces - * @rate: rate of clock to be set - * @prate: rate of parent clock + * @req: rate of clock to be set * * Return: 0 on success else error+reason */ @@ -151,8 +150,9 @@ static int zynqmp_clk_divider_determine_rate(struct clk_hw *hw, width = fls(divider->max_div); - req->rate = divider_round_rate(hw, req->rate, &req->best_parent_rate, - NULL, width, divider->flags); + ret = divider_determine_rate(hw, req, NULL, width, divider->flags); + if (ret != 0) + return ret; if (divider->is_frac && (clk_hw_get_flags(hw) & CLK_SET_RATE_PARENT) && (req->rate % req->best_parent_rate)) diff --git a/drivers/clk/zynqmp/pll.c b/drivers/clk/zynqmp/pll.c index 630a3936c97c34..6bc2c3934f564f 100644 --- a/drivers/clk/zynqmp/pll.c +++ b/drivers/clk/zynqmp/pll.c @@ -91,10 +91,9 @@ static inline void zynqmp_pll_set_mode(struct clk_hw *hw, bool on) } /** - * zynqmp_pll_round_rate() - Round a clock frequency + * zynqmp_pll_determine_rate() - Round a clock frequency * @hw: Handle between common and hardware-specific interfaces - * @rate: Desired clock frequency - * @prate: Clock frequency of parent clock + * @req: Desired clock frequency * * Return: Frequency closest to @rate the hardware can generate */ diff --git a/drivers/clocksource/Kconfig b/drivers/clocksource/Kconfig index aa59e5b133510f..fd911270654542 100644 --- a/drivers/clocksource/Kconfig +++ b/drivers/clocksource/Kconfig @@ -254,6 +254,7 @@ config KEYSTONE_TIMER config INTEGRATOR_AP_TIMER bool "Integrator-AP timer driver" if COMPILE_TEST + depends on OF select CLKSRC_MMIO help Enables support for the Integrator-AP timer. diff --git a/drivers/clocksource/sh_tmu.c b/drivers/clocksource/sh_tmu.c index beffff81c00f3d..3fc6ed9b56300d 100644 --- a/drivers/clocksource/sh_tmu.c +++ b/drivers/clocksource/sh_tmu.c @@ -143,16 +143,6 @@ static void sh_tmu_start_stop_ch(struct sh_tmu_channel *ch, int start) static int __sh_tmu_enable(struct sh_tmu_channel *ch) { - int ret; - - /* enable clock */ - ret = clk_enable(ch->tmu->clk); - if (ret) { - dev_err(&ch->tmu->pdev->dev, "ch%u: cannot enable clock\n", - ch->index); - return ret; - } - /* make sure channel is disabled */ sh_tmu_start_stop_ch(ch, 0); @@ -174,7 +164,6 @@ static int sh_tmu_enable(struct sh_tmu_channel *ch) if (ch->enable_count++ > 0) return 0; - pm_runtime_get_sync(&ch->tmu->pdev->dev); dev_pm_syscore_device(&ch->tmu->pdev->dev, true); return __sh_tmu_enable(ch); @@ -187,9 +176,6 @@ static void __sh_tmu_disable(struct sh_tmu_channel *ch) /* disable interrupts in TMU block */ sh_tmu_write(ch, TCR, TCR_TPSC_CLK4); - - /* stop clock */ - clk_disable(ch->tmu->clk); } static void sh_tmu_disable(struct sh_tmu_channel *ch) @@ -203,7 +189,6 @@ static void sh_tmu_disable(struct sh_tmu_channel *ch) __sh_tmu_disable(ch); dev_pm_syscore_device(&ch->tmu->pdev->dev, false); - pm_runtime_put(&ch->tmu->pdev->dev); } static void sh_tmu_set_next(struct sh_tmu_channel *ch, unsigned long delta, @@ -552,7 +537,6 @@ static int sh_tmu_setup(struct sh_tmu_device *tmu, struct platform_device *pdev) goto err_clk_unprepare; tmu->rate = clk_get_rate(tmu->clk) / 4; - clk_disable(tmu->clk); /* Map the memory resource. */ ret = sh_tmu_map_memory(tmu); @@ -626,8 +610,6 @@ static int sh_tmu_probe(struct platform_device *pdev) out: if (tmu->has_clockevent || tmu->has_clocksource) pm_runtime_irq_safe(&pdev->dev); - else - pm_runtime_idle(&pdev->dev); return 0; } diff --git a/drivers/clocksource/timer-sp804.c b/drivers/clocksource/timer-sp804.c index e82a95ea472478..d698584273596b 100644 --- a/drivers/clocksource/timer-sp804.c +++ b/drivers/clocksource/timer-sp804.c @@ -106,21 +106,25 @@ static u64 notrace sp804_read(void) return ~readl_relaxed(sched_clkevt->value); } +/* Register delay timer backed by the hardware counter */ #ifdef CONFIG_ARM static struct delay_timer delay; +static struct sp804_clkevt *delay_clkevt; + static unsigned long sp804_read_delay_timer_read(void) { - return sp804_read(); + return ~readl_relaxed(delay_clkevt->value); } -static void sp804_register_delay_timer(int freq) +static void sp804_register_delay_timer(struct sp804_clkevt *clk, int freq) { + delay_clkevt = clk; delay.freq = freq; delay.read_current_timer = sp804_read_delay_timer_read; register_current_timer_delay(&delay); } #else -static inline void sp804_register_delay_timer(int freq) {} +static inline void sp804_register_delay_timer(struct sp804_clkevt *clk, int freq) {} #endif static int __init sp804_clocksource_and_sched_clock_init(void __iomem *base, @@ -135,8 +139,6 @@ static int __init sp804_clocksource_and_sched_clock_init(void __iomem *base, if (rate < 0) return -EINVAL; - sp804_register_delay_timer(rate); - clkevt = sp804_clkevt_get(base); writel(0, clkevt->ctrl); @@ -152,6 +154,8 @@ static int __init sp804_clocksource_and_sched_clock_init(void __iomem *base, clocksource_mmio_init(clkevt->value, name, rate, 200, 32, clocksource_mmio_readl_down); + sp804_register_delay_timer(clkevt, rate); + if (use_sched_clock) { sched_clkevt = clkevt; sched_clock_register(sp804_read, 32, rate); diff --git a/drivers/comedi/comedi_fops.c b/drivers/comedi/comedi_fops.c index 2c3eb9e89571a1..67bd71efcfa864 100644 --- a/drivers/comedi/comedi_fops.c +++ b/drivers/comedi/comedi_fops.c @@ -793,13 +793,15 @@ static void do_become_nonbusy(struct comedi_device *dev, __comedi_clear_subdevice_runflags(s, COMEDI_SRF_RUNNING | COMEDI_SRF_BUSY); spin_unlock_irqrestore(&s->spin_lock, flags); - if (comedi_is_runflags_busy(runflags)) { + if (async) { /* * "Run active" counter was set to 1 when setting up the * command. Decrement it and wait for it to become 0. */ - comedi_put_is_subdevice_running(s); - wait_for_completion(&async->run_complete); + if (comedi_is_runflags_busy(runflags)) { + comedi_put_is_subdevice_running(s); + wait_for_completion(&async->run_complete); + } comedi_buf_reset(s); async->inttrig = NULL; kfree(async->cmd.chanlist); diff --git a/drivers/comedi/drivers.c b/drivers/comedi/drivers.c index 69cd2a253c663a..7453f778f2c3f8 100644 --- a/drivers/comedi/drivers.c +++ b/drivers/comedi/drivers.c @@ -1063,6 +1063,14 @@ int comedi_device_attach(struct comedi_device *dev, struct comedi_devconfig *it) ret = -EIO; goto out; } + if (IS_ENABLED(CONFIG_LOCKDEP)) { + /* + * dev->spinlock is for private use by the attached low-level + * driver. Reinitialize it to stop lock-dependency tracking + * between attachments to different low-level drivers. + */ + spin_lock_init(&dev->spinlock); + } dev->driver = driv; dev->board_name = dev->board_ptr ? *(const char **)dev->board_ptr : dev->driver->driver_name; diff --git a/drivers/comedi/drivers/dt2815.c b/drivers/comedi/drivers/dt2815.c index 03ba2fd18a2169..d066dc303520b9 100644 --- a/drivers/comedi/drivers/dt2815.c +++ b/drivers/comedi/drivers/dt2815.c @@ -175,6 +175,18 @@ static int dt2815_attach(struct comedi_device *dev, struct comedi_devconfig *it) ? current_range_type : voltage_range_type; } + /* + * Check if hardware is present before attempting any I/O operations. + * Reading 0xff from status register typically indicates no hardware + * on the bus (floating bus reads as all 1s). + */ + if (inb(dev->iobase + DT2815_STATUS) == 0xff) { + dev_err(dev->class_dev, + "No hardware detected at I/O base 0x%lx\n", + dev->iobase); + return -ENODEV; + } + /* Init the 2815 */ outb(0x00, dev->iobase + DT2815_STATUS); for (i = 0; i < 100; i++) { diff --git a/drivers/comedi/drivers/me4000.c b/drivers/comedi/drivers/me4000.c index 7dd3a007186300..effe9fdbbafe3c 100644 --- a/drivers/comedi/drivers/me4000.c +++ b/drivers/comedi/drivers/me4000.c @@ -315,6 +315,18 @@ static int me4000_xilinx_download(struct comedi_device *dev, unsigned int val; unsigned int i; + /* Get data stream length from header. */ + if (size >= 4) { + file_length = (((unsigned int)data[0] & 0xff) << 24) + + (((unsigned int)data[1] & 0xff) << 16) + + (((unsigned int)data[2] & 0xff) << 8) + + ((unsigned int)data[3] & 0xff); + } + if (size < 16 || file_length > size - 16) { + dev_err(dev->class_dev, "Firmware length inconsistency\n"); + return -EINVAL; + } + if (!xilinx_iobase) return -ENODEV; @@ -346,10 +358,6 @@ static int me4000_xilinx_download(struct comedi_device *dev, outl(val, devpriv->plx_regbase + PLX9052_CNTRL); /* Download Xilinx firmware */ - file_length = (((unsigned int)data[0] & 0xff) << 24) + - (((unsigned int)data[1] & 0xff) << 16) + - (((unsigned int)data[2] & 0xff) << 8) + - ((unsigned int)data[3] & 0xff); usleep_range(10, 1000); for (i = 0; i < file_length; i++) { diff --git a/drivers/comedi/drivers/me_daq.c b/drivers/comedi/drivers/me_daq.c index 076b15097afd9f..2f2ea029cffc2a 100644 --- a/drivers/comedi/drivers/me_daq.c +++ b/drivers/comedi/drivers/me_daq.c @@ -344,6 +344,25 @@ static int me2600_xilinx_download(struct comedi_device *dev, unsigned int file_length; unsigned int i; + /* + * Format of the firmware + * Build longs from the byte-wise coded header + * Byte 1-3: length of the array + * Byte 4-7: version + * Byte 8-11: date + * Byte 12-15: reserved + */ + if (size >= 4) { + file_length = (((unsigned int)data[0] & 0xff) << 24) + + (((unsigned int)data[1] & 0xff) << 16) + + (((unsigned int)data[2] & 0xff) << 8) + + ((unsigned int)data[3] & 0xff); + } + if (size < 16 || file_length > size - 16) { + dev_err(dev->class_dev, "Firmware length inconsistency\n"); + return -EINVAL; + } + /* disable irq's on PLX */ writel(0x00, devpriv->plx_regbase + PLX9052_INTCSR); @@ -357,22 +376,6 @@ static int me2600_xilinx_download(struct comedi_device *dev, writeb(0x00, dev->mmio + 0x0); sleep(1); - /* - * Format of the firmware - * Build longs from the byte-wise coded header - * Byte 1-3: length of the array - * Byte 4-7: version - * Byte 8-11: date - * Byte 12-15: reserved - */ - if (size < 16) - return -EINVAL; - - file_length = (((unsigned int)data[0] & 0xff) << 24) + - (((unsigned int)data[1] & 0xff) << 16) + - (((unsigned int)data[2] & 0xff) << 8) + - ((unsigned int)data[3] & 0xff); - /* * Loop for writing firmware byte by byte to xilinx * Firmware data start at offset 16 diff --git a/drivers/comedi/drivers/ni_atmio16d.c b/drivers/comedi/drivers/ni_atmio16d.c index e5e7cc423c8796..b057b3b3582e17 100644 --- a/drivers/comedi/drivers/ni_atmio16d.c +++ b/drivers/comedi/drivers/ni_atmio16d.c @@ -698,7 +698,8 @@ static int atmio16d_attach(struct comedi_device *dev, static void atmio16d_detach(struct comedi_device *dev) { - reset_atmio16d(dev); + if (dev->private) + reset_atmio16d(dev); comedi_legacy_detach(dev); } diff --git a/drivers/counter/rz-mtu3-cnt.c b/drivers/counter/rz-mtu3-cnt.c index e755d54dfece94..7bfb6979193ce6 100644 --- a/drivers/counter/rz-mtu3-cnt.c +++ b/drivers/counter/rz-mtu3-cnt.c @@ -107,9 +107,9 @@ static bool rz_mtu3_is_counter_invalid(struct counter_device *counter, int id) struct rz_mtu3_cnt *const priv = counter_priv(counter); unsigned long tmdr; - pm_runtime_get_sync(priv->ch->dev); + pm_runtime_get_sync(counter->parent); tmdr = rz_mtu3_shared_reg_read(priv->ch, RZ_MTU3_TMDR3); - pm_runtime_put(priv->ch->dev); + pm_runtime_put(counter->parent); if (id == RZ_MTU3_32_BIT_CH && test_bit(RZ_MTU3_TMDR3_LWA, &tmdr)) return false; @@ -165,12 +165,12 @@ static int rz_mtu3_count_read(struct counter_device *counter, if (ret) return ret; - pm_runtime_get_sync(ch->dev); + pm_runtime_get_sync(counter->parent); if (count->id == RZ_MTU3_32_BIT_CH) *val = rz_mtu3_32bit_ch_read(ch, RZ_MTU3_TCNTLW); else *val = rz_mtu3_16bit_ch_read(ch, RZ_MTU3_TCNT); - pm_runtime_put(ch->dev); + pm_runtime_put(counter->parent); mutex_unlock(&priv->lock); return 0; @@ -187,26 +187,26 @@ static int rz_mtu3_count_write(struct counter_device *counter, if (ret) return ret; - pm_runtime_get_sync(ch->dev); + pm_runtime_get_sync(counter->parent); if (count->id == RZ_MTU3_32_BIT_CH) rz_mtu3_32bit_ch_write(ch, RZ_MTU3_TCNTLW, val); else rz_mtu3_16bit_ch_write(ch, RZ_MTU3_TCNT, val); - pm_runtime_put(ch->dev); + pm_runtime_put(counter->parent); mutex_unlock(&priv->lock); return 0; } static int rz_mtu3_count_function_read_helper(struct rz_mtu3_channel *const ch, - struct rz_mtu3_cnt *const priv, + struct counter_device *const counter, enum counter_function *function) { u8 timer_mode; - pm_runtime_get_sync(ch->dev); + pm_runtime_get_sync(counter->parent); timer_mode = rz_mtu3_8bit_ch_read(ch, RZ_MTU3_TMDR1); - pm_runtime_put(ch->dev); + pm_runtime_put(counter->parent); switch (timer_mode & RZ_MTU3_TMDR1_PH_CNT_MODE_MASK) { case RZ_MTU3_TMDR1_PH_CNT_MODE_1: @@ -240,7 +240,7 @@ static int rz_mtu3_count_function_read(struct counter_device *counter, if (ret) return ret; - ret = rz_mtu3_count_function_read_helper(ch, priv, function); + ret = rz_mtu3_count_function_read_helper(ch, counter, function); mutex_unlock(&priv->lock); return ret; @@ -279,9 +279,9 @@ static int rz_mtu3_count_function_write(struct counter_device *counter, return -EINVAL; } - pm_runtime_get_sync(ch->dev); + pm_runtime_get_sync(counter->parent); rz_mtu3_8bit_ch_write(ch, RZ_MTU3_TMDR1, timer_mode); - pm_runtime_put(ch->dev); + pm_runtime_put(counter->parent); mutex_unlock(&priv->lock); return 0; @@ -300,9 +300,9 @@ static int rz_mtu3_count_direction_read(struct counter_device *counter, if (ret) return ret; - pm_runtime_get_sync(ch->dev); + pm_runtime_get_sync(counter->parent); tsr = rz_mtu3_8bit_ch_read(ch, RZ_MTU3_TSR); - pm_runtime_put(ch->dev); + pm_runtime_put(counter->parent); *direction = (tsr & RZ_MTU3_TSR_TCFD) ? COUNTER_COUNT_DIRECTION_FORWARD : COUNTER_COUNT_DIRECTION_BACKWARD; @@ -377,14 +377,14 @@ static int rz_mtu3_count_ceiling_write(struct counter_device *counter, return -EINVAL; } - pm_runtime_get_sync(ch->dev); + pm_runtime_get_sync(counter->parent); if (count->id == RZ_MTU3_32_BIT_CH) rz_mtu3_32bit_ch_write(ch, RZ_MTU3_TGRALW, ceiling); else rz_mtu3_16bit_ch_write(ch, RZ_MTU3_TGRA, ceiling); rz_mtu3_8bit_ch_write(ch, RZ_MTU3_TCR, RZ_MTU3_TCR_CCLR_TGRA); - pm_runtime_put(ch->dev); + pm_runtime_put(counter->parent); mutex_unlock(&priv->lock); return 0; @@ -495,25 +495,28 @@ static int rz_mtu3_count_enable_read(struct counter_device *counter, static int rz_mtu3_count_enable_write(struct counter_device *counter, struct counter_count *count, u8 enable) { - struct rz_mtu3_channel *const ch = rz_mtu3_get_ch(counter, count->id); struct rz_mtu3_cnt *const priv = counter_priv(counter); int ret = 0; + mutex_lock(&priv->lock); + + if (priv->count_is_enabled[count->id] == enable) + goto exit; + if (enable) { - mutex_lock(&priv->lock); - pm_runtime_get_sync(ch->dev); + pm_runtime_get_sync(counter->parent); ret = rz_mtu3_initialize_counter(counter, count->id); if (ret == 0) priv->count_is_enabled[count->id] = true; - mutex_unlock(&priv->lock); } else { - mutex_lock(&priv->lock); rz_mtu3_terminate_counter(counter, count->id); priv->count_is_enabled[count->id] = false; - pm_runtime_put(ch->dev); - mutex_unlock(&priv->lock); + pm_runtime_put(counter->parent); } +exit: + mutex_unlock(&priv->lock); + return ret; } @@ -540,9 +543,9 @@ static int rz_mtu3_cascade_counts_enable_get(struct counter_device *counter, if (ret) return ret; - pm_runtime_get_sync(priv->ch->dev); + pm_runtime_get_sync(counter->parent); tmdr = rz_mtu3_shared_reg_read(priv->ch, RZ_MTU3_TMDR3); - pm_runtime_put(priv->ch->dev); + pm_runtime_put(counter->parent); *cascade_enable = test_bit(RZ_MTU3_TMDR3_LWA, &tmdr); mutex_unlock(&priv->lock); @@ -559,10 +562,10 @@ static int rz_mtu3_cascade_counts_enable_set(struct counter_device *counter, if (ret) return ret; - pm_runtime_get_sync(priv->ch->dev); + pm_runtime_get_sync(counter->parent); rz_mtu3_shared_reg_update_bit(priv->ch, RZ_MTU3_TMDR3, RZ_MTU3_TMDR3_LWA, cascade_enable); - pm_runtime_put(priv->ch->dev); + pm_runtime_put(counter->parent); mutex_unlock(&priv->lock); return 0; @@ -579,9 +582,9 @@ static int rz_mtu3_ext_input_phase_clock_select_get(struct counter_device *count if (ret) return ret; - pm_runtime_get_sync(priv->ch->dev); + pm_runtime_get_sync(counter->parent); tmdr = rz_mtu3_shared_reg_read(priv->ch, RZ_MTU3_TMDR3); - pm_runtime_put(priv->ch->dev); + pm_runtime_put(counter->parent); *ext_input_phase_clock_select = test_bit(RZ_MTU3_TMDR3_PHCKSEL, &tmdr); mutex_unlock(&priv->lock); @@ -598,11 +601,11 @@ static int rz_mtu3_ext_input_phase_clock_select_set(struct counter_device *count if (ret) return ret; - pm_runtime_get_sync(priv->ch->dev); + pm_runtime_get_sync(counter->parent); rz_mtu3_shared_reg_update_bit(priv->ch, RZ_MTU3_TMDR3, RZ_MTU3_TMDR3_PHCKSEL, ext_input_phase_clock_select); - pm_runtime_put(priv->ch->dev); + pm_runtime_put(counter->parent); mutex_unlock(&priv->lock); return 0; @@ -640,7 +643,7 @@ static int rz_mtu3_action_read(struct counter_device *counter, if (ret) return ret; - ret = rz_mtu3_count_function_read_helper(ch, priv, &function); + ret = rz_mtu3_count_function_read_helper(ch, counter, &function); if (ret) { mutex_unlock(&priv->lock); return ret; diff --git a/drivers/cpufreq/cpufreq-dt-platdev.c b/drivers/cpufreq/cpufreq-dt-platdev.c index b06a43143d23c6..2fecab989dacc8 100644 --- a/drivers/cpufreq/cpufreq-dt-platdev.c +++ b/drivers/cpufreq/cpufreq-dt-platdev.c @@ -169,8 +169,11 @@ static const struct of_device_id blocklist[] __initconst = { { .compatible = "qcom,sdm845", }, { .compatible = "qcom,sdx75", }, { .compatible = "qcom,sm6115", }, + { .compatible = "qcom,sm6125", }, + { .compatible = "qcom,sm6150", }, { .compatible = "qcom,sm6350", }, { .compatible = "qcom,sm6375", }, + { .compatible = "qcom,sm7125", }, { .compatible = "qcom,sm7225", }, { .compatible = "qcom,sm7325", }, { .compatible = "qcom,sm8150", }, diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c index 4472bb1ec83c72..0d3e77cf96a7de 100644 --- a/drivers/cpufreq/cpufreq.c +++ b/drivers/cpufreq/cpufreq.c @@ -1421,12 +1421,9 @@ static int cpufreq_policy_online(struct cpufreq_policy *policy, * If there is a problem with its frequency table, take it * offline and drop it. */ - if (policy->freq_table_sorted != CPUFREQ_TABLE_SORTED_ASCENDING && - policy->freq_table_sorted != CPUFREQ_TABLE_SORTED_DESCENDING) { - ret = cpufreq_table_validate_and_sort(policy); - if (ret) - goto out_offline_policy; - } + ret = cpufreq_table_validate_and_sort(policy); + if (ret) + goto out_offline_policy; /* related_cpus should at least include policy->cpus. */ cpumask_copy(policy->related_cpus, policy->cpus); diff --git a/drivers/cpufreq/cpufreq_conservative.c b/drivers/cpufreq/cpufreq_conservative.c index cce6a8d113e12f..305a32b6b302e7 100644 --- a/drivers/cpufreq/cpufreq_conservative.c +++ b/drivers/cpufreq/cpufreq_conservative.c @@ -313,6 +313,17 @@ static void cs_start(struct cpufreq_policy *policy) dbs_info->requested_freq = policy->cur; } +static void cs_limits(struct cpufreq_policy *policy) +{ + struct cs_policy_dbs_info *dbs_info = to_dbs_info(policy->governor_data); + + /* + * The limits have changed, so may have the current frequency. Reset + * requested_freq to avoid any unintended outcomes due to the mismatch. + */ + dbs_info->requested_freq = policy->cur; +} + static struct dbs_governor cs_governor = { .gov = CPUFREQ_DBS_GOVERNOR_INITIALIZER("conservative"), .kobj_type = { .default_groups = cs_groups }, @@ -322,6 +333,7 @@ static struct dbs_governor cs_governor = { .init = cs_init, .exit = cs_exit, .start = cs_start, + .limits = cs_limits, }; #define CPU_FREQ_GOV_CONSERVATIVE (cs_governor.gov) diff --git a/drivers/cpufreq/cpufreq_governor.c b/drivers/cpufreq/cpufreq_governor.c index 1a7fcaf39cc9b5..00a26fa6292c52 100644 --- a/drivers/cpufreq/cpufreq_governor.c +++ b/drivers/cpufreq/cpufreq_governor.c @@ -468,13 +468,13 @@ int cpufreq_dbs_governor_init(struct cpufreq_policy *policy) /* Failure, so roll back. */ pr_err("initialization failed (dbs_data kobject init error %d)\n", ret); - kobject_put(&dbs_data->attr_set.kobj); - policy->governor_data = NULL; if (!have_governor_per_policy()) gov->gdbs_data = NULL; - gov->exit(dbs_data); + + kobject_put(&dbs_data->attr_set.kobj); + goto free_policy_dbs_info; free_dbs_data: kfree(dbs_data); @@ -563,6 +563,7 @@ EXPORT_SYMBOL_GPL(cpufreq_dbs_governor_stop); void cpufreq_dbs_governor_limits(struct cpufreq_policy *policy) { + struct dbs_governor *gov = dbs_governor_of(policy); struct policy_dbs_info *policy_dbs; /* Protect gov->gdbs_data against cpufreq_dbs_governor_exit() */ @@ -574,6 +575,8 @@ void cpufreq_dbs_governor_limits(struct cpufreq_policy *policy) mutex_lock(&policy_dbs->update_mutex); cpufreq_policy_apply_limits(policy); gov_update_sample_delay(policy_dbs, 0); + if (gov->limits) + gov->limits(policy); mutex_unlock(&policy_dbs->update_mutex); out: diff --git a/drivers/cpufreq/cpufreq_governor.h b/drivers/cpufreq/cpufreq_governor.h index 168c23fd7fcac7..1462d59277bd12 100644 --- a/drivers/cpufreq/cpufreq_governor.h +++ b/drivers/cpufreq/cpufreq_governor.h @@ -138,6 +138,7 @@ struct dbs_governor { int (*init)(struct dbs_data *dbs_data); void (*exit)(struct dbs_data *dbs_data); void (*start)(struct cpufreq_policy *policy); + void (*limits)(struct cpufreq_policy *policy); }; static inline struct dbs_governor *dbs_governor_of(struct cpufreq_policy *policy) diff --git a/drivers/cpufreq/freq_table.c b/drivers/cpufreq/freq_table.c index 7f251daf03ce32..5b364d8da4f927 100644 --- a/drivers/cpufreq/freq_table.c +++ b/drivers/cpufreq/freq_table.c @@ -360,6 +360,10 @@ int cpufreq_table_validate_and_sort(struct cpufreq_policy *policy) if (policy_has_boost_freq(policy)) policy->boost_supported = true; + if (policy->freq_table_sorted == CPUFREQ_TABLE_SORTED_ASCENDING || + policy->freq_table_sorted == CPUFREQ_TABLE_SORTED_DESCENDING) + return 0; + return set_freq_table_sorted(policy); } diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c index ec4abe3745736b..31df8d85816a22 100644 --- a/drivers/cpufreq/intel_pstate.c +++ b/drivers/cpufreq/intel_pstate.c @@ -1161,7 +1161,7 @@ static void hybrid_init_cpu_capacity_scaling(bool refresh) * the capacity of SMT threads is not deterministic even approximately, * do not do that when SMT is in use. */ - if (hwp_is_hybrid && !sched_smt_active() && arch_enable_hybrid_capacity_scale()) { + if (hwp_is_hybrid && !cpu_smt_possible() && arch_enable_hybrid_capacity_scale()) { hybrid_refresh_cpu_capacity_scaling(); /* * Disabling ITMT causes sched domains to be rebuilt to disable asym @@ -1476,13 +1476,13 @@ static void __intel_pstate_update_max_freq(struct cpufreq_policy *policy, refresh_frequency_limits(policy); } -static bool intel_pstate_update_max_freq(struct cpudata *cpudata) +static bool intel_pstate_update_max_freq(int cpu) { - struct cpufreq_policy *policy __free(put_cpufreq_policy) = cpufreq_cpu_get(cpudata->cpu); + struct cpufreq_policy *policy __free(put_cpufreq_policy) = cpufreq_cpu_get(cpu); if (!policy) return false; - __intel_pstate_update_max_freq(policy, cpudata); + __intel_pstate_update_max_freq(policy, all_cpu_data[cpu]); return true; } @@ -1501,7 +1501,7 @@ static void intel_pstate_update_limits_for_all(void) int cpu; for_each_possible_cpu(cpu) - intel_pstate_update_max_freq(all_cpu_data[cpu]); + intel_pstate_update_max_freq(cpu); mutex_lock(&hybrid_capacity_lock); @@ -1647,8 +1647,8 @@ static ssize_t store_no_turbo(struct kobject *a, struct kobj_attribute *b, static void update_cpu_qos_request(int cpu, enum freq_qos_req_type type) { struct cpudata *cpudata = all_cpu_data[cpu]; - unsigned int freq = cpudata->pstate.turbo_freq; struct freq_qos_request *req; + unsigned int freq; struct cpufreq_policy *policy __free(put_cpufreq_policy) = cpufreq_cpu_get(cpu); if (!policy) @@ -1661,6 +1661,8 @@ static void update_cpu_qos_request(int cpu, enum freq_qos_req_type type) if (hwp_active) intel_pstate_get_hwp_cap(cpudata); + freq = cpudata->pstate.turbo_freq; + if (type == FREQ_QOS_MIN) { freq = DIV_ROUND_UP(freq * global.min_perf_pct, 100); } else { @@ -1908,7 +1910,7 @@ static void intel_pstate_notify_work(struct work_struct *work) struct cpudata *cpudata = container_of(to_delayed_work(work), struct cpudata, hwp_notify_work); - if (intel_pstate_update_max_freq(cpudata)) { + if (intel_pstate_update_max_freq(cpudata->cpu)) { /* * The driver will not be unregistered while this function is * running, so update the capacity without acquiring the driver diff --git a/drivers/cpufreq/scmi-cpufreq.c b/drivers/cpufreq/scmi-cpufreq.c index d2a110079f5fd5..c450cf9c881d49 100644 --- a/drivers/cpufreq/scmi-cpufreq.c +++ b/drivers/cpufreq/scmi-cpufreq.c @@ -101,6 +101,7 @@ static int scmi_cpu_domain_id(struct device *cpu_dev) return -EINVAL; } + of_node_put(domain_id.np); return domain_id.args[0]; } diff --git a/drivers/cpuidle/Kconfig.arm b/drivers/cpuidle/Kconfig.arm index a1ee475d180dac..c6870f08457632 100644 --- a/drivers/cpuidle/Kconfig.arm +++ b/drivers/cpuidle/Kconfig.arm @@ -130,3 +130,11 @@ config ARM_QCOM_SPM_CPUIDLE The Subsystem Power Manager (SPM) controls low power modes for the CPU and L2 cores. It interface with various system drivers to put the cores in low power modes. + +config ARM_APPLE_CPUIDLE + bool "Apple SoC CPU idle driver" + depends on ARM64 + default ARCH_APPLE + select CPU_IDLE_MULTIPLE_DRIVERS + help + Select this to enable cpuidle on Apple SoCs. diff --git a/drivers/cpuidle/Makefile b/drivers/cpuidle/Makefile index 1de9e92c5b0fc9..f9e7a71d52c13f 100644 --- a/drivers/cpuidle/Makefile +++ b/drivers/cpuidle/Makefile @@ -29,6 +29,7 @@ obj-$(CONFIG_ARM_PSCI_CPUIDLE) += cpuidle-psci.o obj-$(CONFIG_ARM_PSCI_CPUIDLE_DOMAIN) += cpuidle-psci-domain.o obj-$(CONFIG_ARM_TEGRA_CPUIDLE) += cpuidle-tegra.o obj-$(CONFIG_ARM_QCOM_SPM_CPUIDLE) += cpuidle-qcom-spm.o +obj-$(CONFIG_ARM_APPLE_CPUIDLE) += cpuidle-apple.o ############################################################################### # MIPS drivers diff --git a/drivers/cpuidle/cpuidle-apple.c b/drivers/cpuidle/cpuidle-apple.c new file mode 100644 index 00000000000000..b7504066d75aa8 --- /dev/null +++ b/drivers/cpuidle/cpuidle-apple.c @@ -0,0 +1,167 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Copyright The Asahi Linux Contributors + * + * CPU idle support for Apple SoCs + */ + +#include +#include +#include +#include +#include +#include + +enum idle_state { + STATE_WFI, + STATE_PWRDOWN, + STATE_COUNT +}; + +asm( + ".pushsection .cpuidle.text, \"ax\"\n" + ".type apple_cpu_deep_wfi, @function\n" + "apple_cpu_deep_wfi:\n" + "str x30, [sp, #-16]!\n" + "stp x28, x29, [sp, #-16]!\n" + "stp x26, x27, [sp, #-16]!\n" + "stp x24, x25, [sp, #-16]!\n" + "stp x22, x23, [sp, #-16]!\n" + "stp x20, x21, [sp, #-16]!\n" + "stp x18, x19, [sp, #-16]!\n" + + "mrs x0, s3_5_c15_c5_0\n" + "orr x0, x0, #(3L << 24)\n" + "msr s3_5_c15_c5_0, x0\n" + + "1:\n" + "dsb sy\n" + "wfi\n" + + "mrs x0, ISR_EL1\n" + "cbz x0, 1b\n" + + "mrs x0, s3_5_c15_c5_0\n" + "bic x0, x0, #(1L << 24)\n" + "msr s3_5_c15_c5_0, x0\n" + + "ldp x18, x19, [sp], #16\n" + "ldp x20, x21, [sp], #16\n" + "ldp x22, x23, [sp], #16\n" + "ldp x24, x25, [sp], #16\n" + "ldp x26, x27, [sp], #16\n" + "ldp x28, x29, [sp], #16\n" + "ldr x30, [sp], #16\n" + + "ret\n" + ".popsection\n" +); + +void apple_cpu_deep_wfi(void); + +static __cpuidle int apple_enter_wfi(struct cpuidle_device *dev, struct cpuidle_driver *drv, int index) +{ + cpu_do_idle(); + return index; +} + +static __cpuidle int apple_enter_idle(struct cpuidle_device *dev, struct cpuidle_driver *drv, int index) +{ + /* + * Deep WFI will clobber FP state, among other things. + * The CPU PM notifier will take care of saving that and anything else + * that needs to be notified of the CPU powering down. + */ + if (cpu_pm_enter()) + return -1; + + ct_cpuidle_enter(); + + switch(index) { + case STATE_PWRDOWN: + apple_cpu_deep_wfi(); + break; + default: + WARN_ON(1); + break; + } + + ct_cpuidle_exit(); + + cpu_pm_exit(); + + return index; +} + +static struct cpuidle_driver apple_idle_driver = { + .name = "apple_idle", + .owner = THIS_MODULE, + .states = { + [STATE_WFI] = { + .enter = apple_enter_wfi, + .enter_s2idle = apple_enter_wfi, + .exit_latency = 1, + .target_residency = 1, + .power_usage = UINT_MAX, + .name = "WFI", + .desc = "CPU clock-gated", + .flags = 0, + }, + [STATE_PWRDOWN] = { + .enter = apple_enter_idle, + .enter_s2idle = apple_enter_idle, + .exit_latency = 10, + .target_residency = 10000, + .power_usage = 0, + .name = "CPU PD", + .desc = "CPU/cluster powered down", + .flags = CPUIDLE_FLAG_RCU_IDLE, + }, + }, + .safe_state_index = STATE_WFI, + .state_count = STATE_COUNT, +}; + +static int apple_cpuidle_probe(struct platform_device *pdev) +{ + return cpuidle_register(&apple_idle_driver, NULL); +} + +static struct platform_driver apple_cpuidle_driver = { + .driver = { + .name = "cpuidle-apple", + }, + .probe = apple_cpuidle_probe, +}; + +static int __init apple_cpuidle_init(void) +{ + struct platform_device *pdev; + int ret; + + ret = platform_driver_register(&apple_cpuidle_driver); + if (ret) + return ret; + + if (!of_machine_is_compatible("apple,arm-platform")) + return 0; + + if (!(of_machine_is_compatible("apple,t8103") || + of_machine_is_compatible("apple,t8112") || + of_machine_is_compatible("apple,t6000") || + of_machine_is_compatible("apple,t6001") || + of_machine_is_compatible("apple,t6002") || + of_machine_is_compatible("apple,t6020") || + of_machine_is_compatible("apple,t6021") || + of_machine_is_compatible("apple,t6022"))) + return 0; + + pdev = platform_device_register_simple("cpuidle-apple", -1, NULL, 0); + if (IS_ERR(pdev)) { + platform_driver_unregister(&apple_cpuidle_driver); + return PTR_ERR(pdev); + } + + return 0; +} +device_initcall(apple_cpuidle_init); diff --git a/drivers/cpuidle/governors/menu.c b/drivers/cpuidle/governors/menu.c index 64d6f7a1c77663..ca863ba03d4544 100644 --- a/drivers/cpuidle/governors/menu.c +++ b/drivers/cpuidle/governors/menu.c @@ -239,7 +239,7 @@ static int menu_select(struct cpuidle_driver *drv, struct cpuidle_device *dev, /* Find the shortest expected idle interval. */ predicted_ns = get_typical_interval(data) * NSEC_PER_USEC; - if (predicted_ns > RESIDENCY_THRESHOLD_NS) { + if (predicted_ns > RESIDENCY_THRESHOLD_NS || tick_nohz_tick_stopped()) { unsigned int timer_us; /* Determine the time till the closest timer. */ @@ -259,6 +259,16 @@ static int menu_select(struct cpuidle_driver *drv, struct cpuidle_device *dev, RESOLUTION * DECAY * NSEC_PER_USEC); /* Use the lowest expected idle interval to pick the idle state. */ predicted_ns = min((u64)timer_us * NSEC_PER_USEC, predicted_ns); + /* + * If the tick is already stopped, the cost of possible short + * idle duration misprediction is much higher, because the CPU + * may be stuck in a shallow idle state for a long time as a + * result of it. In that case, say we might mispredict and use + * the known time till the closest timer event for the idle + * state selection. + */ + if (tick_nohz_tick_stopped() && predicted_ns < TICK_NSEC) + predicted_ns = data->next_timer_ns; } else { /* * Because the next timer event is not going to be determined @@ -284,16 +294,6 @@ static int menu_select(struct cpuidle_driver *drv, struct cpuidle_device *dev, return 0; } - /* - * If the tick is already stopped, the cost of possible short idle - * duration misprediction is much higher, because the CPU may be stuck - * in a shallow idle state for a long time as a result of it. In that - * case, say we might mispredict and use the known time till the closest - * timer event for the idle state selection. - */ - if (tick_nohz_tick_stopped() && predicted_ns < TICK_NSEC) - predicted_ns = data->next_timer_ns; - /* * Find the idle state with the lowest power while satisfying * our constraints. diff --git a/drivers/crypto/atmel-sha204a.c b/drivers/crypto/atmel-sha204a.c index 0fcf4a39de279d..a12653a658699a 100644 --- a/drivers/crypto/atmel-sha204a.c +++ b/drivers/crypto/atmel-sha204a.c @@ -52,9 +52,10 @@ static int atmel_sha204a_rng_read_nonblocking(struct hwrng *rng, void *data, rng->priv = 0; } else { work_data = kmalloc(sizeof(*work_data), GFP_ATOMIC); - if (!work_data) + if (!work_data) { + atomic_dec(&i2c_priv->tfm_count); return -ENOMEM; - + } work_data->ctx = i2c_priv; work_data->client = i2c_priv->client; diff --git a/drivers/crypto/caam/caamalg_qi2.c b/drivers/crypto/caam/caamalg_qi2.c index 107ccb2ade4205..07665494c87589 100644 --- a/drivers/crypto/caam/caamalg_qi2.c +++ b/drivers/crypto/caam/caamalg_qi2.c @@ -3326,9 +3326,10 @@ static int ahash_setkey(struct crypto_ahash *ahash, const u8 *key, if (aligned_len < keylen) return -EOVERFLOW; - hashed_key = kmemdup(key, aligned_len, GFP_KERNEL); + hashed_key = kmalloc(aligned_len, GFP_KERNEL); if (!hashed_key) return -ENOMEM; + memcpy(hashed_key, key, keylen); ret = hash_digest_key(ctx, &keylen, hashed_key, digestsize); if (ret) goto bad_free_key; @@ -4814,7 +4815,8 @@ static void dpaa2_dpseci_free(struct dpaa2_caam_priv *priv) { struct device *dev = priv->dev; struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev); - int err; + struct dpaa2_caam_priv_per_cpu *ppriv; + int i, err; if (DPSECI_VER(priv->major_ver, priv->minor_ver) > DPSECI_VER(5, 3)) { err = dpseci_reset(priv->mc_io, 0, ls_dev->mc_handle); @@ -4822,6 +4824,12 @@ static void dpaa2_dpseci_free(struct dpaa2_caam_priv *priv) dev_err(dev, "dpseci_reset() failed\n"); } + for_each_cpu(i, priv->clean_mask) { + ppriv = per_cpu_ptr(priv->ppriv, i); + free_netdev(ppriv->net_dev); + } + free_cpumask_var(priv->clean_mask); + dpaa2_dpseci_congestion_free(priv); dpseci_close(priv->mc_io, 0, ls_dev->mc_handle); } @@ -5007,16 +5015,15 @@ static int __cold dpaa2_dpseci_setup(struct fsl_mc_device *ls_dev) struct device *dev = &ls_dev->dev; struct dpaa2_caam_priv *priv; struct dpaa2_caam_priv_per_cpu *ppriv; - cpumask_var_t clean_mask; int err, cpu; u8 i; err = -ENOMEM; - if (!zalloc_cpumask_var(&clean_mask, GFP_KERNEL)) - goto err_cpumask; - priv = dev_get_drvdata(dev); + if (!zalloc_cpumask_var(&priv->clean_mask, GFP_KERNEL)) + goto err_cpumask; + priv->dev = dev; priv->dpsec_id = ls_dev->obj_desc.id; @@ -5118,7 +5125,7 @@ static int __cold dpaa2_dpseci_setup(struct fsl_mc_device *ls_dev) err = -ENOMEM; goto err_alloc_netdev; } - cpumask_set_cpu(cpu, clean_mask); + cpumask_set_cpu(cpu, priv->clean_mask); ppriv->net_dev->dev = *dev; netif_napi_add_tx_weight(ppriv->net_dev, &ppriv->napi, @@ -5126,18 +5133,16 @@ static int __cold dpaa2_dpseci_setup(struct fsl_mc_device *ls_dev) DPAA2_CAAM_NAPI_WEIGHT); } - err = 0; - goto free_cpumask; + return 0; err_alloc_netdev: - free_dpaa2_pcpu_netdev(priv, clean_mask); + free_dpaa2_pcpu_netdev(priv, priv->clean_mask); err_get_rx_queue: dpaa2_dpseci_congestion_free(priv); err_get_vers: dpseci_close(priv->mc_io, 0, ls_dev->mc_handle); err_open: -free_cpumask: - free_cpumask_var(clean_mask); + free_cpumask_var(priv->clean_mask); err_cpumask: return err; } @@ -5182,7 +5187,6 @@ static int __cold dpaa2_dpseci_disable(struct dpaa2_caam_priv *priv) ppriv = per_cpu_ptr(priv->ppriv, i); napi_disable(&ppriv->napi); netif_napi_del(&ppriv->napi); - free_netdev(ppriv->net_dev); } return 0; diff --git a/drivers/crypto/caam/caamalg_qi2.h b/drivers/crypto/caam/caamalg_qi2.h index 61d1219a202fcb..8e65b4b28c7bab 100644 --- a/drivers/crypto/caam/caamalg_qi2.h +++ b/drivers/crypto/caam/caamalg_qi2.h @@ -42,6 +42,7 @@ * @mc_io: pointer to MC portal's I/O object * @domain: IOMMU domain * @ppriv: per CPU pointers to privata data + * @clean_mask: CPU mask of CPUs that have allocated netdevs */ struct dpaa2_caam_priv { int dpsec_id; @@ -65,6 +66,7 @@ struct dpaa2_caam_priv { struct dpaa2_caam_priv_per_cpu __percpu *ppriv; struct dentry *dfs_root; + cpumask_var_t clean_mask; }; /** diff --git a/drivers/crypto/caam/caamhash.c b/drivers/crypto/caam/caamhash.c index 25c02e26725858..053af748be86d7 100644 --- a/drivers/crypto/caam/caamhash.c +++ b/drivers/crypto/caam/caamhash.c @@ -441,9 +441,10 @@ static int ahash_setkey(struct crypto_ahash *ahash, if (aligned_len < keylen) return -EOVERFLOW; - hashed_key = kmemdup(key, keylen, GFP_KERNEL); + hashed_key = kmalloc(aligned_len, GFP_KERNEL); if (!hashed_key) return -ENOMEM; + memcpy(hashed_key, key, keylen); ret = hash_digest_key(ctx, &keylen, hashed_key, digestsize); if (ret) goto bad_free_key; diff --git a/drivers/crypto/cavium/cpt/cptvf_main.c b/drivers/crypto/cavium/cpt/cptvf_main.c index c246920e6f540c..bccd680c7f7ee4 100644 --- a/drivers/crypto/cavium/cpt/cptvf_main.c +++ b/drivers/crypto/cavium/cpt/cptvf_main.c @@ -180,7 +180,8 @@ static void free_command_queues(struct cpt_vf *cptvf, hlist_for_each_entry_safe(chunk, node, &cqinfo->queue[i].chead, nextchunk) { - dma_free_coherent(&pdev->dev, chunk->size, + dma_free_coherent(&pdev->dev, + chunk->size + CPT_NEXT_CHUNK_PTR_SIZE, chunk->head, chunk->dma_addr); chunk->head = NULL; diff --git a/drivers/crypto/ccp/ccp-ops.c b/drivers/crypto/ccp/ccp-ops.c index d78865d9d5f09c..d0412e58476253 100644 --- a/drivers/crypto/ccp/ccp-ops.c +++ b/drivers/crypto/ccp/ccp-ops.c @@ -642,7 +642,7 @@ ccp_run_aes_gcm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) struct ccp_data dst; struct ccp_data aad; struct ccp_op op; - } *wa __cleanup(kfree) = kzalloc(sizeof *wa, GFP_KERNEL); + } *wa __free(kfree) = kzalloc(sizeof(*wa), GFP_KERNEL); unsigned int dm_offset; unsigned int authsize; unsigned int jobid; diff --git a/drivers/crypto/ccp/psp-dev.c b/drivers/crypto/ccp/psp-dev.c index 9e21da0e298ad7..5c7f7e02a7d8ab 100644 --- a/drivers/crypto/ccp/psp-dev.c +++ b/drivers/crypto/ccp/psp-dev.c @@ -351,6 +351,17 @@ struct psp_device *psp_get_master_device(void) return sp ? sp->psp_data : NULL; } +int psp_restore(struct sp_device *sp) +{ + struct psp_device *psp = sp->psp_data; + int ret = 0; + + if (psp->tee_data) + ret = tee_restore(psp); + + return ret; +} + void psp_pci_init(void) { psp_master = psp_get_master_device(); diff --git a/drivers/crypto/ccp/sev-dev-tsm.c b/drivers/crypto/ccp/sev-dev-tsm.c index 40d02adaf3f6da..7ad7e7a413c0f6 100644 --- a/drivers/crypto/ccp/sev-dev-tsm.c +++ b/drivers/crypto/ccp/sev-dev-tsm.c @@ -378,9 +378,9 @@ void sev_tsm_init_locked(struct sev_device *sev, void *tio_status_page) return; error_exit: - kfree(t); pr_err("Failed to enable SEV-TIO: ret=%d en=%d initdone=%d SEV=%d\n", ret, t->tio_en, t->tio_init_done, boot_cpu_has(X86_FEATURE_SEV)); + kfree(t); } void sev_tsm_uninit(struct sev_device *sev) diff --git a/drivers/crypto/ccp/sev-dev.c b/drivers/crypto/ccp/sev-dev.c index 956ea609d0cce1..a554fe3de3fd2c 100644 --- a/drivers/crypto/ccp/sev-dev.c +++ b/drivers/crypto/ccp/sev-dev.c @@ -127,13 +127,6 @@ static size_t sev_es_tmr_size = SEV_TMR_SIZE; #define NV_LENGTH (32 * 1024) static void *sev_init_ex_buffer; -/* - * SEV_DATA_RANGE_LIST: - * Array containing range of pages that firmware transitions to HV-fixed - * page state. - */ -static struct sev_data_range_list *snp_range_list; - static void __sev_firmware_shutdown(struct sev_device *sev, bool panic); static int snp_shutdown_on_panic(struct notifier_block *nb, @@ -1112,15 +1105,12 @@ struct page *snp_alloc_hv_fixed_pages(unsigned int num_2mb_pages) { struct psp_device *psp_master = psp_get_master_device(); struct snp_hv_fixed_pages_entry *entry; - struct sev_device *sev; unsigned int order; struct page *page; - if (!psp_master || !psp_master->sev_data) + if (!psp_master) return NULL; - sev = psp_master->sev_data; - order = get_order(PMD_SIZE * num_2mb_pages); /* @@ -1133,7 +1123,8 @@ struct page *snp_alloc_hv_fixed_pages(unsigned int num_2mb_pages) * This API uses SNP_INIT_EX to transition allocated pages to HV_Fixed * page state, fail if SNP is already initialized. */ - if (sev->snp_initialized) + if (psp_master->sev_data && + ((struct sev_device *)psp_master->sev_data)->snp_initialized) return NULL; /* Re-use freed pages that match the request */ @@ -1169,7 +1160,7 @@ void snp_free_hv_fixed_pages(struct page *page) struct psp_device *psp_master = psp_get_master_device(); struct snp_hv_fixed_pages_entry *entry, *nentry; - if (!psp_master || !psp_master->sev_data) + if (!psp_master) return; /* @@ -1361,6 +1352,7 @@ static int snp_filter_reserved_mem_regions(struct resource *rs, void *arg) static int __sev_snp_init_locked(int *error, unsigned int max_snp_asid) { + struct sev_data_range_list *snp_range_list __free(kfree) = NULL; struct psp_device *psp = psp_master; struct sev_data_snp_init_ex data; struct sev_device *sev; @@ -2378,11 +2370,10 @@ static int sev_ioctl_do_pdh_export(struct sev_issue_cmd *argp, bool writable) static int sev_ioctl_do_snp_platform_status(struct sev_issue_cmd *argp) { struct sev_device *sev = psp_master->sev_data; - bool shutdown_required = false; struct sev_data_snp_addr buf; struct page *status_page; - int ret, error; void *data; + int ret; if (!argp->data) return -EINVAL; @@ -2393,31 +2384,33 @@ static int sev_ioctl_do_snp_platform_status(struct sev_issue_cmd *argp) data = page_address(status_page); - if (!sev->snp_initialized) { - ret = snp_move_to_init_state(argp, &shutdown_required); - if (ret) - goto cleanup; - } - /* - * Firmware expects status page to be in firmware-owned state, otherwise - * it will report firmware error code INVALID_PAGE_STATE (0x1A). + * SNP_PLATFORM_STATUS can be executed in any SNP state. But if executed + * when SNP has been initialized, the status page must be firmware-owned. */ - if (rmp_mark_pages_firmware(__pa(data), 1, true)) { - ret = -EFAULT; - goto cleanup; + if (sev->snp_initialized) { + /* + * Firmware expects the status page to be in Firmware state, + * otherwise it will report an error INVALID_PAGE_STATE. + */ + if (rmp_mark_pages_firmware(__pa(data), 1, true)) { + ret = -EFAULT; + goto cleanup; + } } buf.address = __psp_pa(data); ret = __sev_do_cmd_locked(SEV_CMD_SNP_PLATFORM_STATUS, &buf, &argp->error); - /* - * Status page will be transitioned to Reclaim state upon success, or - * left in Firmware state in failure. Use snp_reclaim_pages() to - * transition either case back to Hypervisor-owned state. - */ - if (snp_reclaim_pages(__pa(data), 1, true)) - return -EFAULT; + if (sev->snp_initialized) { + /* + * The status page will be in Reclaim state on success, or left + * in Firmware state on failure. Use snp_reclaim_pages() to + * transition either case back to Hypervisor-owned state. + */ + if (snp_reclaim_pages(__pa(data), 1, true)) + return -EFAULT; + } if (ret) goto cleanup; @@ -2427,9 +2420,6 @@ static int sev_ioctl_do_snp_platform_status(struct sev_issue_cmd *argp) ret = -EFAULT; cleanup: - if (shutdown_required) - __sev_snp_shutdown_locked(&error, false); - __free_pages(status_page, 0); return ret; } @@ -2780,11 +2770,6 @@ static void __sev_firmware_shutdown(struct sev_device *sev, bool panic) sev_init_ex_buffer = NULL; } - if (snp_range_list) { - kfree(snp_range_list); - snp_range_list = NULL; - } - __sev_snp_shutdown_locked(&error, panic); } diff --git a/drivers/crypto/ccp/sp-dev.c b/drivers/crypto/ccp/sp-dev.c index 3467f6db4f505a..f204aa5df96e24 100644 --- a/drivers/crypto/ccp/sp-dev.c +++ b/drivers/crypto/ccp/sp-dev.c @@ -230,6 +230,18 @@ int sp_resume(struct sp_device *sp) return 0; } +int sp_restore(struct sp_device *sp) +{ + if (sp->psp_data) { + int ret = psp_restore(sp); + + if (ret) + return ret; + } + + return sp_resume(sp); +} + struct sp_device *sp_get_psp_master_device(void) { struct sp_device *i, *ret = NULL; diff --git a/drivers/crypto/ccp/sp-dev.h b/drivers/crypto/ccp/sp-dev.h index 1335a83fe052ea..a83751cfd00603 100644 --- a/drivers/crypto/ccp/sp-dev.h +++ b/drivers/crypto/ccp/sp-dev.h @@ -141,6 +141,7 @@ void sp_destroy(struct sp_device *sp); int sp_suspend(struct sp_device *sp); int sp_resume(struct sp_device *sp); +int sp_restore(struct sp_device *sp); int sp_request_ccp_irq(struct sp_device *sp, irq_handler_t handler, const char *name, void *data); void sp_free_ccp_irq(struct sp_device *sp, void *data); @@ -174,6 +175,7 @@ int psp_dev_init(struct sp_device *sp); void psp_pci_init(void); void psp_dev_destroy(struct sp_device *sp); void psp_pci_exit(void); +int psp_restore(struct sp_device *sp); #else /* !CONFIG_CRYPTO_DEV_SP_PSP */ @@ -181,6 +183,7 @@ static inline int psp_dev_init(struct sp_device *sp) { return 0; } static inline void psp_pci_init(void) { } static inline void psp_dev_destroy(struct sp_device *sp) { } static inline void psp_pci_exit(void) { } +static inline int psp_restore(struct sp_device *sp) { return 0; } #endif /* CONFIG_CRYPTO_DEV_SP_PSP */ diff --git a/drivers/crypto/ccp/sp-pci.c b/drivers/crypto/ccp/sp-pci.c index 8891ceee1d7d05..6ac805d99ccb30 100644 --- a/drivers/crypto/ccp/sp-pci.c +++ b/drivers/crypto/ccp/sp-pci.c @@ -353,6 +353,13 @@ static int __maybe_unused sp_pci_resume(struct device *dev) return sp_resume(sp); } +static int __maybe_unused sp_pci_restore(struct device *dev) +{ + struct sp_device *sp = dev_get_drvdata(dev); + + return sp_restore(sp); +} + #ifdef CONFIG_CRYPTO_DEV_SP_PSP static const struct sev_vdata sevv1 = { .cmdresp_reg = 0x10580, /* C2PMSG_32 */ @@ -563,7 +570,14 @@ static const struct pci_device_id sp_pci_table[] = { }; MODULE_DEVICE_TABLE(pci, sp_pci_table); -static SIMPLE_DEV_PM_OPS(sp_pci_pm_ops, sp_pci_suspend, sp_pci_resume); +static const struct dev_pm_ops sp_pci_pm_ops = { + .suspend = pm_sleep_ptr(sp_pci_suspend), + .resume = pm_sleep_ptr(sp_pci_resume), + .freeze = pm_sleep_ptr(sp_pci_suspend), + .thaw = pm_sleep_ptr(sp_pci_resume), + .poweroff = pm_sleep_ptr(sp_pci_suspend), + .restore_early = pm_sleep_ptr(sp_pci_restore), +}; static struct pci_driver sp_pci_driver = { .name = "ccp", diff --git a/drivers/crypto/ccp/tee-dev.c b/drivers/crypto/ccp/tee-dev.c index 5e1d80724678d0..92ffa412622a24 100644 --- a/drivers/crypto/ccp/tee-dev.c +++ b/drivers/crypto/ccp/tee-dev.c @@ -86,10 +86,34 @@ static inline void tee_free_cmd_buffer(struct tee_init_ring_cmd *cmd) kfree(cmd); } +static bool tee_send_destroy_cmd(struct psp_tee_device *tee) +{ + unsigned int reg; + int ret; + + ret = psp_mailbox_command(tee->psp, PSP_CMD_TEE_RING_DESTROY, NULL, + TEE_DEFAULT_CMD_TIMEOUT, ®); + if (ret) { + dev_err(tee->dev, "tee: ring destroy command timed out, disabling TEE support\n"); + psp_dead = true; + return false; + } + + if (FIELD_GET(PSP_CMDRESP_STS, reg)) { + dev_err(tee->dev, "tee: ring destroy command failed (%#010lx)\n", + FIELD_GET(PSP_CMDRESP_STS, reg)); + psp_dead = true; + return false; + } + + return true; +} + static int tee_init_ring(struct psp_tee_device *tee) { int ring_size = MAX_RING_BUFFER_ENTRIES * sizeof(struct tee_ring_cmd); struct tee_init_ring_cmd *cmd; + bool retry = false; unsigned int reg; int ret; @@ -112,6 +136,7 @@ static int tee_init_ring(struct psp_tee_device *tee) /* Send command buffer details to Trusted OS by writing to * CPU-PSP message registers */ +retry_init: ret = psp_mailbox_command(tee->psp, PSP_CMD_TEE_RING_INIT, cmd, TEE_DEFAULT_CMD_TIMEOUT, ®); if (ret) { @@ -122,9 +147,22 @@ static int tee_init_ring(struct psp_tee_device *tee) } if (FIELD_GET(PSP_CMDRESP_STS, reg)) { + /* + * During the hibernate resume sequence driver may have gotten loaded + * but the ring not properly destroyed. If the ring doesn't work, try + * to destroy and re-init once. + */ + if (!retry && FIELD_GET(PSP_CMDRESP_STS, reg) == PSP_TEE_STS_RING_BUSY) { + dev_info(tee->dev, "tee: ring init command failed with busy status, retrying\n"); + if (tee_send_destroy_cmd(tee)) { + retry = true; + goto retry_init; + } + } dev_err(tee->dev, "tee: ring init command failed (%#010lx)\n", FIELD_GET(PSP_CMDRESP_STS, reg)); tee_free_ring(tee); + psp_dead = true; ret = -EIO; } @@ -136,24 +174,13 @@ static int tee_init_ring(struct psp_tee_device *tee) static void tee_destroy_ring(struct psp_tee_device *tee) { - unsigned int reg; - int ret; - if (!tee->rb_mgr.ring_start) return; if (psp_dead) goto free_ring; - ret = psp_mailbox_command(tee->psp, PSP_CMD_TEE_RING_DESTROY, NULL, - TEE_DEFAULT_CMD_TIMEOUT, ®); - if (ret) { - dev_err(tee->dev, "tee: ring destroy command timed out, disabling TEE support\n"); - psp_dead = true; - } else if (FIELD_GET(PSP_CMDRESP_STS, reg)) { - dev_err(tee->dev, "tee: ring destroy command failed (%#010lx)\n", - FIELD_GET(PSP_CMDRESP_STS, reg)); - } + tee_send_destroy_cmd(tee); free_ring: tee_free_ring(tee); @@ -365,3 +392,8 @@ int psp_check_tee_status(void) return 0; } EXPORT_SYMBOL(psp_check_tee_status); + +int tee_restore(struct psp_device *psp) +{ + return tee_init_ring(psp->tee_data); +} diff --git a/drivers/crypto/ccp/tee-dev.h b/drivers/crypto/ccp/tee-dev.h index ea9a2b7c05f577..c23416cb7bb37f 100644 --- a/drivers/crypto/ccp/tee-dev.h +++ b/drivers/crypto/ccp/tee-dev.h @@ -111,5 +111,6 @@ struct tee_ring_cmd { int tee_dev_init(struct psp_device *psp); void tee_dev_destroy(struct psp_device *psp); +int tee_restore(struct psp_device *psp); #endif /* __TEE_DEV_H__ */ diff --git a/drivers/crypto/hisilicon/Kconfig b/drivers/crypto/hisilicon/Kconfig index 4835bdebdbb381..a0cb1a8186ac9d 100644 --- a/drivers/crypto/hisilicon/Kconfig +++ b/drivers/crypto/hisilicon/Kconfig @@ -57,6 +57,7 @@ config CRYPTO_DEV_HISI_ZIP depends on UACCE || UACCE=n depends on ACPI select CRYPTO_DEV_HISI_QM + select CRYPTO_DEFLATE help Support for HiSilicon ZIP Driver diff --git a/drivers/crypto/hisilicon/hpre/hpre.h b/drivers/crypto/hisilicon/hpre/hpre.h index 0f3ddbadbcf991..021dbd9a1d48fb 100644 --- a/drivers/crypto/hisilicon/hpre/hpre.h +++ b/drivers/crypto/hisilicon/hpre/hpre.h @@ -94,9 +94,8 @@ struct hpre_sqe { __le64 key; __le64 in; __le64 out; - __le16 tag; - __le16 resv2; -#define _HPRE_SQE_ALIGN_EXT 7 + __le64 tag; +#define _HPRE_SQE_ALIGN_EXT 6 __le32 rsvd1[_HPRE_SQE_ALIGN_EXT]; }; diff --git a/drivers/crypto/hisilicon/hpre/hpre_crypto.c b/drivers/crypto/hisilicon/hpre/hpre_crypto.c index 21ccf879f70c55..839c1f67714369 100644 --- a/drivers/crypto/hisilicon/hpre/hpre_crypto.c +++ b/drivers/crypto/hisilicon/hpre/hpre_crypto.c @@ -93,6 +93,7 @@ struct hpre_dh_ctx { char *g; /* m */ dma_addr_t dma_g; + struct crypto_kpp *soft_tfm; }; struct hpre_ecdh_ctx { @@ -103,17 +104,15 @@ struct hpre_ecdh_ctx { /* low address: x->y */ unsigned char *g; dma_addr_t dma_g; + struct crypto_kpp *soft_tfm; }; struct hpre_ctx { struct hisi_qp *qp; struct device *dev; - struct hpre_asym_request **req_list; struct hpre *hpre; - spinlock_t req_lock; unsigned int key_sz; bool crt_g2_mode; - struct idr req_idr; union { struct hpre_rsa_ctx rsa; struct hpre_dh_ctx dh; @@ -123,6 +122,7 @@ struct hpre_ctx { unsigned int curve_id; /* for high performance core */ u8 enable_hpcore; + bool fallback; }; struct hpre_asym_request { @@ -136,7 +136,6 @@ struct hpre_asym_request { struct kpp_request *ecdh; } areq; int err; - int req_id; hpre_cb cb; struct timespec64 req_time; }; @@ -151,79 +150,13 @@ static inline unsigned int hpre_align_pd(void) return (hpre_align_sz() - 1) & ~(crypto_tfm_ctx_alignment() - 1); } -static int hpre_alloc_req_id(struct hpre_ctx *ctx) +static void hpre_dfx_add_req_time(struct hpre_asym_request *hpre_req) { - unsigned long flags; - int id; - - spin_lock_irqsave(&ctx->req_lock, flags); - id = idr_alloc(&ctx->req_idr, NULL, 0, ctx->qp->sq_depth, GFP_ATOMIC); - spin_unlock_irqrestore(&ctx->req_lock, flags); - - return id; -} - -static void hpre_free_req_id(struct hpre_ctx *ctx, int req_id) -{ - unsigned long flags; - - spin_lock_irqsave(&ctx->req_lock, flags); - idr_remove(&ctx->req_idr, req_id); - spin_unlock_irqrestore(&ctx->req_lock, flags); -} - -static int hpre_add_req_to_ctx(struct hpre_asym_request *hpre_req) -{ - struct hpre_ctx *ctx; - struct hpre_dfx *dfx; - int id; - - ctx = hpre_req->ctx; - id = hpre_alloc_req_id(ctx); - if (unlikely(id < 0)) - return -EINVAL; - - ctx->req_list[id] = hpre_req; - hpre_req->req_id = id; + struct hpre_ctx *ctx = hpre_req->ctx; + struct hpre_dfx *dfx = ctx->hpre->debug.dfx; - dfx = ctx->hpre->debug.dfx; if (atomic64_read(&dfx[HPRE_OVERTIME_THRHLD].value)) ktime_get_ts64(&hpre_req->req_time); - - return id; -} - -static void hpre_rm_req_from_ctx(struct hpre_asym_request *hpre_req) -{ - struct hpre_ctx *ctx = hpre_req->ctx; - int id = hpre_req->req_id; - - if (hpre_req->req_id >= 0) { - hpre_req->req_id = HPRE_INVLD_REQ_ID; - ctx->req_list[id] = NULL; - hpre_free_req_id(ctx, id); - } -} - -static struct hisi_qp *hpre_get_qp_and_start(u8 type) -{ - struct hisi_qp *qp; - int ret; - - qp = hpre_create_qp(type); - if (!qp) { - pr_err("Can not create hpre qp!\n"); - return ERR_PTR(-ENODEV); - } - - ret = hisi_qm_start_qp(qp, 0); - if (ret < 0) { - hisi_qm_free_qps(&qp, 1); - pci_err(qp->qm->pdev, "Can not start qp!\n"); - return ERR_PTR(-EINVAL); - } - - return qp; } static int hpre_get_data_dma_addr(struct hpre_asym_request *hpre_req, @@ -340,26 +273,19 @@ static void hpre_hw_data_clr_all(struct hpre_ctx *ctx, static int hpre_alg_res_post_hf(struct hpre_ctx *ctx, struct hpre_sqe *sqe, void **kreq) { - struct hpre_asym_request *req; unsigned int err, done, alg; - int id; #define HPRE_NO_HW_ERR 0 #define HPRE_HW_TASK_DONE 3 #define HREE_HW_ERR_MASK GENMASK(10, 0) #define HREE_SQE_DONE_MASK GENMASK(1, 0) #define HREE_ALG_TYPE_MASK GENMASK(4, 0) - id = (int)le16_to_cpu(sqe->tag); - req = ctx->req_list[id]; - hpre_rm_req_from_ctx(req); - *kreq = req; + *kreq = (void *)le64_to_cpu(sqe->tag); err = (le32_to_cpu(sqe->dw0) >> HPRE_SQE_ALG_BITS) & HREE_HW_ERR_MASK; - done = (le32_to_cpu(sqe->dw0) >> HPRE_SQE_DONE_SHIFT) & HREE_SQE_DONE_MASK; - if (likely(err == HPRE_NO_HW_ERR && done == HPRE_HW_TASK_DONE)) return 0; @@ -370,36 +296,10 @@ static int hpre_alg_res_post_hf(struct hpre_ctx *ctx, struct hpre_sqe *sqe, return -EINVAL; } -static int hpre_ctx_set(struct hpre_ctx *ctx, struct hisi_qp *qp, int qlen) -{ - struct hpre *hpre; - - if (!ctx || !qp || qlen < 0) - return -EINVAL; - - spin_lock_init(&ctx->req_lock); - ctx->qp = qp; - ctx->dev = &qp->qm->pdev->dev; - - hpre = container_of(ctx->qp->qm, struct hpre, qm); - ctx->hpre = hpre; - ctx->req_list = kcalloc(qlen, sizeof(void *), GFP_KERNEL); - if (!ctx->req_list) - return -ENOMEM; - ctx->key_sz = 0; - ctx->crt_g2_mode = false; - idr_init(&ctx->req_idr); - - return 0; -} - static void hpre_ctx_clear(struct hpre_ctx *ctx, bool is_clear_all) { - if (is_clear_all) { - idr_destroy(&ctx->req_idr); - kfree(ctx->req_list); + if (is_clear_all) hisi_qm_free_qps(&ctx->qp, 1); - } ctx->crt_g2_mode = false; ctx->key_sz = 0; @@ -467,49 +367,44 @@ static void hpre_rsa_cb(struct hpre_ctx *ctx, void *resp) static void hpre_alg_cb(struct hisi_qp *qp, void *resp) { - struct hpre_ctx *ctx = qp->qp_ctx; - struct hpre_dfx *dfx = ctx->hpre->debug.dfx; + struct hpre_asym_request *h_req; struct hpre_sqe *sqe = resp; - struct hpre_asym_request *req = ctx->req_list[le16_to_cpu(sqe->tag)]; - if (unlikely(!req)) { - atomic64_inc(&dfx[HPRE_INVALID_REQ_CNT].value); + h_req = (struct hpre_asym_request *)le64_to_cpu(sqe->tag); + if (unlikely(!h_req)) { + pr_err("Failed to get request, and qp_id is %u\n", qp->qp_id); return; } - req->cb(ctx, resp); -} - -static void hpre_stop_qp_and_put(struct hisi_qp *qp) -{ - hisi_qm_stop_qp(qp); - hisi_qm_free_qps(&qp, 1); + h_req->cb(h_req->ctx, resp); } static int hpre_ctx_init(struct hpre_ctx *ctx, u8 type) { struct hisi_qp *qp; - int ret; + struct hpre *hpre; - qp = hpre_get_qp_and_start(type); - if (IS_ERR(qp)) - return PTR_ERR(qp); + qp = hpre_create_qp(type); + if (!qp) { + ctx->qp = NULL; + return -ENODEV; + } - qp->qp_ctx = ctx; qp->req_cb = hpre_alg_cb; + ctx->qp = qp; + ctx->dev = &qp->qm->pdev->dev; + hpre = container_of(ctx->qp->qm, struct hpre, qm); + ctx->hpre = hpre; + ctx->key_sz = 0; + ctx->crt_g2_mode = false; - ret = hpre_ctx_set(ctx, qp, qp->sq_depth); - if (ret) - hpre_stop_qp_and_put(qp); - - return ret; + return 0; } static int hpre_msg_request_set(struct hpre_ctx *ctx, void *req, bool is_rsa) { struct hpre_asym_request *h_req; struct hpre_sqe *msg; - int req_id; void *tmp; if (is_rsa) { @@ -549,11 +444,8 @@ static int hpre_msg_request_set(struct hpre_ctx *ctx, void *req, bool is_rsa) msg->task_len1 = (ctx->key_sz >> HPRE_BITS_2_BYTES_SHIFT) - 1; h_req->ctx = ctx; - req_id = hpre_add_req_to_ctx(h_req); - if (req_id < 0) - return -EBUSY; - - msg->tag = cpu_to_le16((u16)req_id); + hpre_dfx_add_req_time(h_req); + msg->tag = cpu_to_le64((uintptr_t)h_req); return 0; } @@ -566,9 +458,7 @@ static int hpre_send(struct hpre_ctx *ctx, struct hpre_sqe *msg) do { atomic64_inc(&dfx[HPRE_SEND_CNT].value); - spin_lock_bh(&ctx->req_lock); ret = hisi_qp_send(ctx->qp, msg); - spin_unlock_bh(&ctx->req_lock); if (ret != -EBUSY) break; atomic64_inc(&dfx[HPRE_SEND_BUSY_CNT].value); @@ -619,12 +509,53 @@ static int hpre_dh_compute_value(struct kpp_request *req) return -EINPROGRESS; clear_all: - hpre_rm_req_from_ctx(hpre_req); hpre_hw_data_clr_all(ctx, hpre_req, req->dst, req->src); return ret; } +static struct kpp_request *hpre_dh_prepare_fb_req(struct kpp_request *req) +{ + struct kpp_request *fb_req = kpp_request_ctx(req); + struct crypto_kpp *tfm = crypto_kpp_reqtfm(req); + struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); + + kpp_request_set_tfm(fb_req, ctx->dh.soft_tfm); + kpp_request_set_callback(fb_req, req->base.flags, req->base.complete, req->base.data); + kpp_request_set_input(fb_req, req->src, req->src_len); + kpp_request_set_output(fb_req, req->dst, req->dst_len); + + return fb_req; +} + +static int hpre_dh_generate_public_key(struct kpp_request *req) +{ + struct crypto_kpp *tfm = crypto_kpp_reqtfm(req); + struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); + struct kpp_request *fb_req; + + if (ctx->fallback) { + fb_req = hpre_dh_prepare_fb_req(req); + return crypto_kpp_generate_public_key(fb_req); + } + + return hpre_dh_compute_value(req); +} + +static int hpre_dh_compute_shared_secret(struct kpp_request *req) +{ + struct crypto_kpp *tfm = crypto_kpp_reqtfm(req); + struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); + struct kpp_request *fb_req; + + if (ctx->fallback) { + fb_req = hpre_dh_prepare_fb_req(req); + return crypto_kpp_compute_shared_secret(fb_req); + } + + return hpre_dh_compute_value(req); +} + static int hpre_is_dh_params_length_valid(unsigned int key_sz) { #define _HPRE_DH_GRP1 768 @@ -651,13 +582,6 @@ static int hpre_dh_set_params(struct hpre_ctx *ctx, struct dh *params) struct device *dev = ctx->dev; unsigned int sz; - if (params->p_size > HPRE_DH_MAX_P_SZ) - return -EINVAL; - - if (hpre_is_dh_params_length_valid(params->p_size << - HPRE_BITS_2_BYTES_SHIFT)) - return -EINVAL; - sz = ctx->key_sz = params->p_size; ctx->dh.xa_p = dma_alloc_coherent(dev, sz << 1, &ctx->dh.dma_xa_p, GFP_KERNEL); @@ -690,8 +614,8 @@ static void hpre_dh_clear_ctx(struct hpre_ctx *ctx, bool is_clear_all) struct device *dev = ctx->dev; unsigned int sz = ctx->key_sz; - if (is_clear_all) - hisi_qm_stop_qp(ctx->qp); + if (!ctx->qp) + return; if (ctx->dh.g) { dma_free_coherent(dev, sz, ctx->dh.g, ctx->dh.dma_g); @@ -718,6 +642,13 @@ static int hpre_dh_set_secret(struct crypto_kpp *tfm, const void *buf, if (crypto_dh_decode_key(buf, len, ¶ms) < 0) return -EINVAL; + if (!ctx->qp) + goto set_soft_secret; + + if (hpre_is_dh_params_length_valid(params.p_size << + HPRE_BITS_2_BYTES_SHIFT)) + goto set_soft_secret; + /* Free old secret if any */ hpre_dh_clear_ctx(ctx, false); @@ -728,27 +659,55 @@ static int hpre_dh_set_secret(struct crypto_kpp *tfm, const void *buf, memcpy(ctx->dh.xa_p + (ctx->key_sz - params.key_size), params.key, params.key_size); + ctx->fallback = false; return 0; err_clear_ctx: hpre_dh_clear_ctx(ctx, false); return ret; +set_soft_secret: + ctx->fallback = true; + return crypto_kpp_set_secret(ctx->dh.soft_tfm, buf, len); } static unsigned int hpre_dh_max_size(struct crypto_kpp *tfm) { struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); + if (ctx->fallback) + return crypto_kpp_maxsize(ctx->dh.soft_tfm); + return ctx->key_sz; } static int hpre_dh_init_tfm(struct crypto_kpp *tfm) { struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); + const char *alg = kpp_alg_name(tfm); + unsigned int reqsize; + int ret; + + ctx->dh.soft_tfm = crypto_alloc_kpp(alg, 0, CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(ctx->dh.soft_tfm)) { + pr_err("Failed to alloc dh tfm!\n"); + return PTR_ERR(ctx->dh.soft_tfm); + } - kpp_set_reqsize(tfm, sizeof(struct hpre_asym_request) + hpre_align_pd()); + crypto_kpp_set_flags(ctx->dh.soft_tfm, crypto_kpp_get_flags(tfm)); - return hpre_ctx_init(ctx, HPRE_V2_ALG_TYPE); + reqsize = max(sizeof(struct hpre_asym_request) + hpre_align_pd(), + sizeof(struct kpp_request) + crypto_kpp_reqsize(ctx->dh.soft_tfm)); + kpp_set_reqsize(tfm, reqsize); + + ret = hpre_ctx_init(ctx, HPRE_V2_ALG_TYPE); + if (ret && ret != -ENODEV) { + crypto_free_kpp(ctx->dh.soft_tfm); + return ret; + } else if (ret == -ENODEV) { + ctx->fallback = true; + } + + return 0; } static void hpre_dh_exit_tfm(struct crypto_kpp *tfm) @@ -756,6 +715,7 @@ static void hpre_dh_exit_tfm(struct crypto_kpp *tfm) struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); hpre_dh_clear_ctx(ctx, true); + crypto_free_kpp(ctx->dh.soft_tfm); } static void hpre_rsa_drop_leading_zeros(const char **ptr, size_t *len) @@ -795,9 +755,8 @@ static int hpre_rsa_enc(struct akcipher_request *req) struct hpre_sqe *msg = &hpre_req->req; int ret; - /* For 512 and 1536 bits key size, use soft tfm instead */ - if (ctx->key_sz == HPRE_RSA_512BITS_KSZ || - ctx->key_sz == HPRE_RSA_1536BITS_KSZ) { + /* For unsupported key size and unavailable devices, use soft tfm instead */ + if (ctx->fallback) { akcipher_request_set_tfm(req, ctx->rsa.soft_tfm); ret = crypto_akcipher_encrypt(req); akcipher_request_set_tfm(req, tfm); @@ -828,7 +787,6 @@ static int hpre_rsa_enc(struct akcipher_request *req) return -EINPROGRESS; clear_all: - hpre_rm_req_from_ctx(hpre_req); hpre_hw_data_clr_all(ctx, hpre_req, req->dst, req->src); return ret; @@ -843,9 +801,8 @@ static int hpre_rsa_dec(struct akcipher_request *req) struct hpre_sqe *msg = &hpre_req->req; int ret; - /* For 512 and 1536 bits key size, use soft tfm instead */ - if (ctx->key_sz == HPRE_RSA_512BITS_KSZ || - ctx->key_sz == HPRE_RSA_1536BITS_KSZ) { + /* For unsupported key size and unavailable devices, use soft tfm instead */ + if (ctx->fallback) { akcipher_request_set_tfm(req, ctx->rsa.soft_tfm); ret = crypto_akcipher_decrypt(req); akcipher_request_set_tfm(req, tfm); @@ -883,7 +840,6 @@ static int hpre_rsa_dec(struct akcipher_request *req) return -EINPROGRESS; clear_all: - hpre_rm_req_from_ctx(hpre_req); hpre_hw_data_clr_all(ctx, hpre_req, req->dst, req->src); return ret; @@ -899,8 +855,10 @@ static int hpre_rsa_set_n(struct hpre_ctx *ctx, const char *value, ctx->key_sz = vlen; /* if invalid key size provided, we use software tfm */ - if (!hpre_rsa_key_size_is_support(ctx->key_sz)) + if (!hpre_rsa_key_size_is_support(ctx->key_sz)) { + ctx->fallback = true; return 0; + } ctx->rsa.pubkey = dma_alloc_coherent(ctx->dev, vlen << 1, &ctx->rsa.dma_pubkey, @@ -1035,8 +993,8 @@ static void hpre_rsa_clear_ctx(struct hpre_ctx *ctx, bool is_clear_all) unsigned int half_key_sz = ctx->key_sz >> 1; struct device *dev = ctx->dev; - if (is_clear_all) - hisi_qm_stop_qp(ctx->qp); + if (!ctx->qp) + return; if (ctx->rsa.pubkey) { dma_free_coherent(dev, ctx->key_sz << 1, @@ -1117,6 +1075,7 @@ static int hpre_rsa_setkey(struct hpre_ctx *ctx, const void *key, goto free; } + ctx->fallback = false; return 0; free: @@ -1134,6 +1093,9 @@ static int hpre_rsa_setpubkey(struct crypto_akcipher *tfm, const void *key, if (ret) return ret; + if (!ctx->qp) + return 0; + return hpre_rsa_setkey(ctx, key, keylen, false); } @@ -1147,6 +1109,9 @@ static int hpre_rsa_setprivkey(struct crypto_akcipher *tfm, const void *key, if (ret) return ret; + if (!ctx->qp) + return 0; + return hpre_rsa_setkey(ctx, key, keylen, true); } @@ -1154,9 +1119,8 @@ static unsigned int hpre_rsa_max_size(struct crypto_akcipher *tfm) { struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm); - /* For 512 and 1536 bits key size, use soft tfm instead */ - if (ctx->key_sz == HPRE_RSA_512BITS_KSZ || - ctx->key_sz == HPRE_RSA_1536BITS_KSZ) + /* For unsupported key size and unavailable devices, use soft tfm instead */ + if (ctx->fallback) return crypto_akcipher_maxsize(ctx->rsa.soft_tfm); return ctx->key_sz; @@ -1177,10 +1141,14 @@ static int hpre_rsa_init_tfm(struct crypto_akcipher *tfm) hpre_align_pd()); ret = hpre_ctx_init(ctx, HPRE_V2_ALG_TYPE); - if (ret) + if (ret && ret != -ENODEV) { crypto_free_akcipher(ctx->rsa.soft_tfm); + return ret; + } else if (ret == -ENODEV) { + ctx->fallback = true; + } - return ret; + return 0; } static void hpre_rsa_exit_tfm(struct crypto_akcipher *tfm) @@ -1207,9 +1175,6 @@ static void hpre_ecc_clear_ctx(struct hpre_ctx *ctx, bool is_clear_all) unsigned int sz = ctx->key_sz; unsigned int shift = sz << 1; - if (is_clear_all) - hisi_qm_stop_qp(ctx->qp); - if (ctx->ecdh.p) { /* ecdh: p->a->k->b */ memzero_explicit(ctx->ecdh.p + shift, sz); @@ -1346,7 +1311,7 @@ static int hpre_ecdh_set_param(struct hpre_ctx *ctx, struct ecdh *params) return 0; } -static bool hpre_key_is_zero(char *key, unsigned short key_sz) +static bool hpre_key_is_zero(const char *key, unsigned short key_sz) { int i; @@ -1387,6 +1352,9 @@ static int hpre_ecdh_set_secret(struct crypto_kpp *tfm, const void *buf, struct ecdh params; int ret; + if (ctx->fallback) + return crypto_kpp_set_secret(ctx->ecdh.soft_tfm, buf, len); + if (crypto_ecdh_decode_key(buf, len, ¶ms) < 0) { dev_err(dev, "failed to decode ecdh key!\n"); return -EINVAL; @@ -1488,7 +1456,6 @@ static int hpre_ecdh_msg_request_set(struct hpre_ctx *ctx, { struct hpre_asym_request *h_req; struct hpre_sqe *msg; - int req_id; void *tmp; if (req->dst_len < ctx->key_sz << 1) { @@ -1510,11 +1477,8 @@ static int hpre_ecdh_msg_request_set(struct hpre_ctx *ctx, msg->task_len1 = (ctx->key_sz >> HPRE_BITS_2_BYTES_SHIFT) - 1; h_req->ctx = ctx; - req_id = hpre_add_req_to_ctx(h_req); - if (req_id < 0) - return -EBUSY; - - msg->tag = cpu_to_le16((u16)req_id); + hpre_dfx_add_req_time(h_req); + msg->tag = cpu_to_le64((uintptr_t)h_req); return 0; } @@ -1612,28 +1576,86 @@ static int hpre_ecdh_compute_value(struct kpp_request *req) return -EINPROGRESS; clear_all: - hpre_rm_req_from_ctx(hpre_req); hpre_ecdh_hw_data_clr_all(ctx, hpre_req, req->dst, req->src); return ret; } +static int hpre_ecdh_generate_public_key(struct kpp_request *req) +{ + struct crypto_kpp *tfm = crypto_kpp_reqtfm(req); + struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); + int ret; + + if (ctx->fallback) { + kpp_request_set_tfm(req, ctx->ecdh.soft_tfm); + ret = crypto_kpp_generate_public_key(req); + kpp_request_set_tfm(req, tfm); + return ret; + } + + return hpre_ecdh_compute_value(req); +} + +static int hpre_ecdh_compute_shared_secret(struct kpp_request *req) +{ + struct crypto_kpp *tfm = crypto_kpp_reqtfm(req); + struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); + int ret; + + if (ctx->fallback) { + kpp_request_set_tfm(req, ctx->ecdh.soft_tfm); + ret = crypto_kpp_compute_shared_secret(req); + kpp_request_set_tfm(req, tfm); + return ret; + } + + return hpre_ecdh_compute_value(req); +} + static unsigned int hpre_ecdh_max_size(struct crypto_kpp *tfm) { struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); + if (ctx->fallback) + return crypto_kpp_maxsize(ctx->ecdh.soft_tfm); + /* max size is the pub_key_size, include x and y */ return ctx->key_sz << 1; } +static int hpre_ecdh_init_tfm(struct crypto_kpp *tfm) +{ + struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); + const char *alg = kpp_alg_name(tfm); + int ret; + + ret = hpre_ctx_init(ctx, HPRE_V3_ECC_ALG_TYPE); + if (!ret) { + kpp_set_reqsize(tfm, sizeof(struct hpre_asym_request) + hpre_align_pd()); + return 0; + } else if (ret && ret != -ENODEV) { + return ret; + } + + ctx->ecdh.soft_tfm = crypto_alloc_kpp(alg, 0, CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(ctx->ecdh.soft_tfm)) { + pr_err("Failed to alloc %s tfm!\n", alg); + return PTR_ERR(ctx->ecdh.soft_tfm); + } + + crypto_kpp_set_flags(ctx->ecdh.soft_tfm, crypto_kpp_get_flags(tfm)); + ctx->fallback = true; + + return 0; +} + static int hpre_ecdh_nist_p192_init_tfm(struct crypto_kpp *tfm) { struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); ctx->curve_id = ECC_CURVE_NIST_P192; - kpp_set_reqsize(tfm, sizeof(struct hpre_asym_request) + hpre_align_pd()); - - return hpre_ctx_init(ctx, HPRE_V3_ECC_ALG_TYPE); + return hpre_ecdh_init_tfm(tfm); } static int hpre_ecdh_nist_p256_init_tfm(struct crypto_kpp *tfm) @@ -1643,9 +1665,7 @@ static int hpre_ecdh_nist_p256_init_tfm(struct crypto_kpp *tfm) ctx->curve_id = ECC_CURVE_NIST_P256; ctx->enable_hpcore = 1; - kpp_set_reqsize(tfm, sizeof(struct hpre_asym_request) + hpre_align_pd()); - - return hpre_ctx_init(ctx, HPRE_V3_ECC_ALG_TYPE); + return hpre_ecdh_init_tfm(tfm); } static int hpre_ecdh_nist_p384_init_tfm(struct crypto_kpp *tfm) @@ -1654,15 +1674,18 @@ static int hpre_ecdh_nist_p384_init_tfm(struct crypto_kpp *tfm) ctx->curve_id = ECC_CURVE_NIST_P384; - kpp_set_reqsize(tfm, sizeof(struct hpre_asym_request) + hpre_align_pd()); - - return hpre_ctx_init(ctx, HPRE_V3_ECC_ALG_TYPE); + return hpre_ecdh_init_tfm(tfm); } static void hpre_ecdh_exit_tfm(struct crypto_kpp *tfm) { struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); + if (ctx->fallback) { + crypto_free_kpp(ctx->ecdh.soft_tfm); + return; + } + hpre_ecc_clear_ctx(ctx, true); } @@ -1680,13 +1703,14 @@ static struct akcipher_alg rsa = { .cra_name = "rsa", .cra_driver_name = "hpre-rsa", .cra_module = THIS_MODULE, + .cra_flags = CRYPTO_ALG_NEED_FALLBACK, }, }; static struct kpp_alg dh = { .set_secret = hpre_dh_set_secret, - .generate_public_key = hpre_dh_compute_value, - .compute_shared_secret = hpre_dh_compute_value, + .generate_public_key = hpre_dh_generate_public_key, + .compute_shared_secret = hpre_dh_compute_shared_secret, .max_size = hpre_dh_max_size, .init = hpre_dh_init_tfm, .exit = hpre_dh_exit_tfm, @@ -1696,14 +1720,15 @@ static struct kpp_alg dh = { .cra_name = "dh", .cra_driver_name = "hpre-dh", .cra_module = THIS_MODULE, + .cra_flags = CRYPTO_ALG_NEED_FALLBACK, }, }; static struct kpp_alg ecdh_curves[] = { { .set_secret = hpre_ecdh_set_secret, - .generate_public_key = hpre_ecdh_compute_value, - .compute_shared_secret = hpre_ecdh_compute_value, + .generate_public_key = hpre_ecdh_generate_public_key, + .compute_shared_secret = hpre_ecdh_compute_shared_secret, .max_size = hpre_ecdh_max_size, .init = hpre_ecdh_nist_p192_init_tfm, .exit = hpre_ecdh_exit_tfm, @@ -1713,11 +1738,12 @@ static struct kpp_alg ecdh_curves[] = { .cra_name = "ecdh-nist-p192", .cra_driver_name = "hpre-ecdh-nist-p192", .cra_module = THIS_MODULE, + .cra_flags = CRYPTO_ALG_NEED_FALLBACK, }, }, { .set_secret = hpre_ecdh_set_secret, - .generate_public_key = hpre_ecdh_compute_value, - .compute_shared_secret = hpre_ecdh_compute_value, + .generate_public_key = hpre_ecdh_generate_public_key, + .compute_shared_secret = hpre_ecdh_compute_shared_secret, .max_size = hpre_ecdh_max_size, .init = hpre_ecdh_nist_p256_init_tfm, .exit = hpre_ecdh_exit_tfm, @@ -1727,11 +1753,12 @@ static struct kpp_alg ecdh_curves[] = { .cra_name = "ecdh-nist-p256", .cra_driver_name = "hpre-ecdh-nist-p256", .cra_module = THIS_MODULE, + .cra_flags = CRYPTO_ALG_NEED_FALLBACK, }, }, { .set_secret = hpre_ecdh_set_secret, - .generate_public_key = hpre_ecdh_compute_value, - .compute_shared_secret = hpre_ecdh_compute_value, + .generate_public_key = hpre_ecdh_generate_public_key, + .compute_shared_secret = hpre_ecdh_compute_shared_secret, .max_size = hpre_ecdh_max_size, .init = hpre_ecdh_nist_p384_init_tfm, .exit = hpre_ecdh_exit_tfm, @@ -1741,6 +1768,7 @@ static struct kpp_alg ecdh_curves[] = { .cra_name = "ecdh-nist-p384", .cra_driver_name = "hpre-ecdh-nist-p384", .cra_module = THIS_MODULE, + .cra_flags = CRYPTO_ALG_NEED_FALLBACK, }, } }; diff --git a/drivers/crypto/hisilicon/hpre/hpre_main.c b/drivers/crypto/hisilicon/hpre/hpre_main.c index b94fecd765eebd..884d5d0afaf41f 100644 --- a/drivers/crypto/hisilicon/hpre/hpre_main.c +++ b/drivers/crypto/hisilicon/hpre/hpre_main.c @@ -465,7 +465,7 @@ struct hisi_qp *hpre_create_qp(u8 type) * type: 0 - RSA/DH. algorithm supported in V2, * 1 - ECC algorithm in V3. */ - ret = hisi_qm_alloc_qps_node(&hpre_devices, 1, type, node, &qp); + ret = hisi_qm_alloc_qps_node(&hpre_devices, 1, &type, node, &qp); if (!ret) return qp; diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c index d47bf06a90f7d3..cf58d0d01b1995 100644 --- a/drivers/crypto/hisilicon/qm.c +++ b/drivers/crypto/hisilicon/qm.c @@ -609,9 +609,13 @@ static void qm_mb_write(struct hisi_qm *qm, const void *src) } #if IS_ENABLED(CONFIG_ARM64) + /* + * The dmb oshst instruction ensures that the data in the + * mailbox is written before it is sent to the hardware. + */ asm volatile("ldp %0, %1, %3\n" - "stp %0, %1, %2\n" "dmb oshst\n" + "stp %0, %1, %2\n" : "=&r" (tmp0), "=&r" (tmp1), "+Q" (*((char __iomem *)fun_base)) @@ -2219,6 +2223,7 @@ static void qp_stop_fail_cb(struct hisi_qp *qp) for (i = 0; i < qp_used; i++) { pos = (i + cur_head) % sq_depth; qp->req_cb(qp, qp->sqe + (u32)(qm->sqe_size * pos)); + qm_cq_head_update(qp); atomic_dec(&qp->qp_status.used); } } @@ -2368,25 +2373,33 @@ EXPORT_SYMBOL_GPL(hisi_qm_stop_qp); int hisi_qp_send(struct hisi_qp *qp, const void *msg) { struct hisi_qp_status *qp_status = &qp->qp_status; - u16 sq_tail = qp_status->sq_tail; - u16 sq_tail_next = (sq_tail + 1) % qp->sq_depth; - void *sqe = qm_get_avail_sqe(qp); + u16 sq_tail, sq_tail_next; + void *sqe; + spin_lock_bh(&qp->qp_lock); if (unlikely(atomic_read(&qp->qp_status.flags) == QP_STOP || atomic_read(&qp->qm->status.flags) == QM_STOP || qp->is_resetting)) { + spin_unlock_bh(&qp->qp_lock); dev_info_ratelimited(&qp->qm->pdev->dev, "QP is stopped or resetting\n"); return -EAGAIN; } - if (!sqe) + sqe = qm_get_avail_sqe(qp); + if (!sqe) { + spin_unlock_bh(&qp->qp_lock); return -EBUSY; + } + sq_tail = qp_status->sq_tail; + sq_tail_next = (sq_tail + 1) % qp->sq_depth; memcpy(sqe, msg, qp->qm->sqe_size); + qp->msg[sq_tail] = msg; qm_db(qp->qm, qp->qp_id, QM_DOORBELL_CMD_SQ, sq_tail_next, 0); atomic_inc(&qp->qp_status.used); qp_status->sq_tail = sq_tail_next; + spin_unlock_bh(&qp->qp_lock); return 0; } @@ -2919,12 +2932,13 @@ EXPORT_SYMBOL_GPL(hisi_qm_wait_task_finish); static void hisi_qp_memory_uninit(struct hisi_qm *qm, int num) { struct device *dev = &qm->pdev->dev; - struct qm_dma *qdma; + struct hisi_qp *qp; int i; for (i = num - 1; i >= 0; i--) { - qdma = &qm->qp_array[i].qdma; - dma_free_coherent(dev, qdma->size, qdma->va, qdma->dma); + qp = &qm->qp_array[i]; + dma_free_coherent(dev, qp->qdma.size, qp->qdma.va, qp->qdma.dma); + kfree(qp->msg); kfree(qm->poll_data[i].qp_finish_id); } @@ -2946,10 +2960,14 @@ static int hisi_qp_memory_init(struct hisi_qm *qm, size_t dma_size, int id, return -ENOMEM; qp = &qm->qp_array[id]; + qp->msg = kmalloc_array(sq_depth, sizeof(void *), GFP_KERNEL); + if (!qp->msg) + goto err_free_qp_finish_id; + qp->qdma.va = dma_alloc_coherent(dev, dma_size, &qp->qdma.dma, GFP_KERNEL); if (!qp->qdma.va) - goto err_free_qp_finish_id; + goto err_free_qp_msg; qp->sqe = qp->qdma.va; qp->sqe_dma = qp->qdma.dma; @@ -2961,8 +2979,14 @@ static int hisi_qp_memory_init(struct hisi_qm *qm, size_t dma_size, int id, qp->qm = qm; qp->qp_id = id; + spin_lock_init(&qp->qp_lock); + spin_lock_init(&qp->backlog.lock); + INIT_LIST_HEAD(&qp->backlog.list); + return 0; +err_free_qp_msg: + kfree(qp->msg); err_free_qp_finish_id: kfree(qm->poll_data[id].qp_finish_id); return ret; @@ -3533,6 +3557,14 @@ void hisi_qm_dev_err_uninit(struct hisi_qm *qm) } EXPORT_SYMBOL_GPL(hisi_qm_dev_err_uninit); +static void qm_release_qp_nolock(struct hisi_qp *qp) +{ + struct hisi_qm *qm = qp->qm; + + qm->qp_in_used--; + idr_remove(&qm->qp_idr, qp->qp_id); +} + /** * hisi_qm_free_qps() - free multiple queue pairs. * @qps: The queue pairs need to be freed. @@ -3545,8 +3577,15 @@ void hisi_qm_free_qps(struct hisi_qp **qps, int qp_num) if (!qps || qp_num <= 0) return; - for (i = qp_num - 1; i >= 0; i--) - hisi_qm_release_qp(qps[i]); + down_write(&qps[0]->qm->qps_lock); + + for (i = qp_num - 1; i >= 0; i--) { + qm_stop_qp_nolock(qps[i]); + qm_release_qp_nolock(qps[i]); + } + + up_write(&qps[0]->qm->qps_lock); + qm_pm_put_sync(qps[0]->qm); } EXPORT_SYMBOL_GPL(hisi_qm_free_qps); @@ -3560,6 +3599,43 @@ static void free_list(struct list_head *head) } } +static int qm_get_and_start_qp(struct hisi_qm *qm, int qp_num, struct hisi_qp **qps, u8 *alg_type) +{ + int i, ret; + + ret = qm_pm_get_sync(qm); + if (ret) + return ret; + + down_write(&qm->qps_lock); + for (i = 0; i < qp_num; i++) { + qps[i] = qm_create_qp_nolock(qm, alg_type[i]); + if (IS_ERR(qps[i])) { + ret = -ENODEV; + goto stop_and_free; + } + + ret = qm_start_qp_nolock(qps[i], 0); + if (ret) { + qm_release_qp_nolock(qps[i]); + goto stop_and_free; + } + } + up_write(&qm->qps_lock); + + return 0; + +stop_and_free: + for (i--; i >= 0; i--) { + qm_stop_qp_nolock(qps[i]); + qm_release_qp_nolock(qps[i]); + } + up_write(&qm->qps_lock); + qm_pm_put_sync(qm); + + return ret; +} + static int hisi_qm_sort_devices(int node, struct list_head *head, struct hisi_qm_list *qm_list) { @@ -3608,12 +3684,11 @@ static int hisi_qm_sort_devices(int node, struct list_head *head, * not meet the requirements will return error. */ int hisi_qm_alloc_qps_node(struct hisi_qm_list *qm_list, int qp_num, - u8 alg_type, int node, struct hisi_qp **qps) + u8 *alg_type, int node, struct hisi_qp **qps) { struct hisi_qm_resource *tmp; int ret = -ENODEV; LIST_HEAD(head); - int i; if (!qps || !qm_list || qp_num <= 0) return -EINVAL; @@ -3625,24 +3700,15 @@ int hisi_qm_alloc_qps_node(struct hisi_qm_list *qm_list, int qp_num, } list_for_each_entry(tmp, &head, list) { - for (i = 0; i < qp_num; i++) { - qps[i] = hisi_qm_create_qp(tmp->qm, alg_type); - if (IS_ERR(qps[i])) { - hisi_qm_free_qps(qps, i); - break; - } - } - - if (i == qp_num) { - ret = 0; + ret = qm_get_and_start_qp(tmp->qm, qp_num, qps, alg_type); + if (!ret) break; - } } mutex_unlock(&qm_list->lock); if (ret) - pr_info("Failed to create qps, node[%d], alg[%u], qp[%d]!\n", - node, alg_type, qp_num); + pr_info("Failed to create qps, node[%d], qp[%d]!\n", + node, qp_num); err: free_list(&head); diff --git a/drivers/crypto/hisilicon/sec2/sec.h b/drivers/crypto/hisilicon/sec2/sec.h index 81d0beda93b2bb..0710977861f323 100644 --- a/drivers/crypto/hisilicon/sec2/sec.h +++ b/drivers/crypto/hisilicon/sec2/sec.h @@ -82,11 +82,6 @@ struct sec_aead_req { __u8 out_mac_buf[SEC_MAX_MAC_LEN]; }; -struct sec_instance_backlog { - struct list_head list; - spinlock_t lock; -}; - /* SEC request of Crypto */ struct sec_req { union { @@ -112,7 +107,6 @@ struct sec_req { bool use_pbuf; struct list_head list; - struct sec_instance_backlog *backlog; struct sec_request_buf buf; }; @@ -172,7 +166,6 @@ struct sec_qp_ctx { spinlock_t id_lock; struct hisi_acc_sgl_pool *c_in_pool; struct hisi_acc_sgl_pool *c_out_pool; - struct sec_instance_backlog backlog; u16 send_head; }; diff --git a/drivers/crypto/hisilicon/sec2/sec_crypto.c b/drivers/crypto/hisilicon/sec2/sec_crypto.c index 31590d01139a37..c462b58d303435 100644 --- a/drivers/crypto/hisilicon/sec2/sec_crypto.c +++ b/drivers/crypto/hisilicon/sec2/sec_crypto.c @@ -54,7 +54,6 @@ #define SEC_AUTH_CIPHER_V3 0x40 #define SEC_FLAG_OFFSET 7 #define SEC_FLAG_MASK 0x0780 -#define SEC_TYPE_MASK 0x0F #define SEC_DONE_MASK 0x0001 #define SEC_ICV_MASK 0x000E @@ -148,7 +147,7 @@ static void sec_free_req_id(struct sec_req *req) spin_unlock_bh(&qp_ctx->id_lock); } -static u8 pre_parse_finished_bd(struct bd_status *status, void *resp) +static void pre_parse_finished_bd(struct bd_status *status, void *resp) { struct sec_sqe *bd = resp; @@ -158,11 +157,9 @@ static u8 pre_parse_finished_bd(struct bd_status *status, void *resp) SEC_FLAG_MASK) >> SEC_FLAG_OFFSET; status->tag = le16_to_cpu(bd->type2.tag); status->err_type = bd->type2.error_type; - - return bd->type_cipher_auth & SEC_TYPE_MASK; } -static u8 pre_parse_finished_bd3(struct bd_status *status, void *resp) +static void pre_parse_finished_bd3(struct bd_status *status, void *resp) { struct sec_sqe3 *bd3 = resp; @@ -172,8 +169,6 @@ static u8 pre_parse_finished_bd3(struct bd_status *status, void *resp) SEC_FLAG_MASK) >> SEC_FLAG_OFFSET; status->tag = le64_to_cpu(bd3->tag); status->err_type = bd3->error_type; - - return le32_to_cpu(bd3->bd_param) & SEC_TYPE_MASK; } static int sec_cb_status_check(struct sec_req *req, @@ -244,7 +239,7 @@ static void sec_alg_send_backlog_soft(struct sec_ctx *ctx, struct sec_qp_ctx *qp struct sec_req *req, *tmp; int ret; - list_for_each_entry_safe(req, tmp, &qp_ctx->backlog.list, list) { + list_for_each_entry_safe(req, tmp, &qp_ctx->qp->backlog.list, list) { list_del(&req->list); ctx->req_op->buf_unmap(ctx, req); if (req->req_id >= 0) @@ -265,11 +260,12 @@ static void sec_alg_send_backlog_soft(struct sec_ctx *ctx, struct sec_qp_ctx *qp static void sec_alg_send_backlog(struct sec_ctx *ctx, struct sec_qp_ctx *qp_ctx) { + struct hisi_qp *qp = qp_ctx->qp; struct sec_req *req, *tmp; int ret; - spin_lock_bh(&qp_ctx->backlog.lock); - list_for_each_entry_safe(req, tmp, &qp_ctx->backlog.list, list) { + spin_lock_bh(&qp->backlog.lock); + list_for_each_entry_safe(req, tmp, &qp->backlog.list, list) { ret = qp_send_message(req); switch (ret) { case -EINPROGRESS: @@ -287,42 +283,46 @@ static void sec_alg_send_backlog(struct sec_ctx *ctx, struct sec_qp_ctx *qp_ctx) } unlock: - spin_unlock_bh(&qp_ctx->backlog.lock); + spin_unlock_bh(&qp->backlog.lock); } static void sec_req_cb(struct hisi_qp *qp, void *resp) { - struct sec_qp_ctx *qp_ctx = qp->qp_ctx; - struct sec_dfx *dfx = &qp_ctx->ctx->sec->debug.dfx; - u8 type_supported = qp_ctx->ctx->type_supported; + const struct sec_sqe *sqe = qp->msg[qp->qp_status.cq_head]; + struct sec_req *req = container_of(sqe, struct sec_req, sec_sqe); + struct sec_ctx *ctx = req->ctx; + struct sec_dfx *dfx = &ctx->sec->debug.dfx; struct bd_status status; - struct sec_ctx *ctx; - struct sec_req *req; int err; - u8 type; - if (type_supported == SEC_BD_TYPE2) { - type = pre_parse_finished_bd(&status, resp); - req = qp_ctx->req_list[status.tag]; - } else { - type = pre_parse_finished_bd3(&status, resp); - req = (void *)(uintptr_t)status.tag; - } + pre_parse_finished_bd(&status, resp); - if (unlikely(type != type_supported)) { - atomic64_inc(&dfx->err_bd_cnt); - pr_err("err bd type [%u]\n", type); - return; - } + req->err_type = status.err_type; + err = sec_cb_status_check(req, &status); + if (err) + atomic64_inc(&dfx->done_flag_cnt); - if (unlikely(!req)) { - atomic64_inc(&dfx->invalid_req_cnt); - atomic_inc(&qp->qp_status.used); - return; - } + atomic64_inc(&dfx->recv_cnt); + ctx->req_op->buf_unmap(ctx, req); + ctx->req_op->callback(ctx, req, err); +} + +static void sec_req_cb3(struct hisi_qp *qp, void *resp) +{ + struct bd_status status; + struct sec_ctx *ctx; + struct sec_dfx *dfx; + struct sec_req *req; + int err; + + pre_parse_finished_bd3(&status, resp); + + req = (void *)(uintptr_t)status.tag; req->err_type = status.err_type; ctx = req->ctx; + dfx = &ctx->sec->debug.dfx; + err = sec_cb_status_check(req, &status); if (err) atomic64_inc(&dfx->done_flag_cnt); @@ -330,7 +330,6 @@ static void sec_req_cb(struct hisi_qp *qp, void *resp) atomic64_inc(&dfx->recv_cnt); ctx->req_op->buf_unmap(ctx, req); - ctx->req_op->callback(ctx, req, err); } @@ -348,8 +347,10 @@ static int sec_alg_send_message_retry(struct sec_req *req) static int sec_alg_try_enqueue(struct sec_req *req) { + struct hisi_qp *qp = req->qp_ctx->qp; + /* Check if any request is already backlogged */ - if (!list_empty(&req->backlog->list)) + if (!list_empty(&qp->backlog.list)) return -EBUSY; /* Try to enqueue to HW ring */ @@ -359,17 +360,18 @@ static int sec_alg_try_enqueue(struct sec_req *req) static int sec_alg_send_message_maybacklog(struct sec_req *req) { + struct hisi_qp *qp = req->qp_ctx->qp; int ret; ret = sec_alg_try_enqueue(req); if (ret != -EBUSY) return ret; - spin_lock_bh(&req->backlog->lock); + spin_lock_bh(&qp->backlog.lock); ret = sec_alg_try_enqueue(req); if (ret == -EBUSY) - list_add_tail(&req->list, &req->backlog->list); - spin_unlock_bh(&req->backlog->lock); + list_add_tail(&req->list, &qp->backlog.list); + spin_unlock_bh(&qp->backlog.lock); return ret; } @@ -624,32 +626,25 @@ static int sec_create_qp_ctx(struct sec_ctx *ctx, int qp_ctx_id) qp_ctx = &ctx->qp_ctx[qp_ctx_id]; qp = ctx->qps[qp_ctx_id]; - qp->req_type = 0; - qp->qp_ctx = qp_ctx; qp_ctx->qp = qp; qp_ctx->ctx = ctx; - qp->req_cb = sec_req_cb; + if (ctx->type_supported == SEC_BD_TYPE3) + qp->req_cb = sec_req_cb3; + else + qp->req_cb = sec_req_cb; spin_lock_init(&qp_ctx->req_lock); idr_init(&qp_ctx->req_idr); - spin_lock_init(&qp_ctx->backlog.lock); spin_lock_init(&qp_ctx->id_lock); - INIT_LIST_HEAD(&qp_ctx->backlog.list); qp_ctx->send_head = 0; ret = sec_alloc_qp_ctx_resource(ctx, qp_ctx); if (ret) goto err_destroy_idr; - ret = hisi_qm_start_qp(qp, 0); - if (ret < 0) - goto err_resource_free; - return 0; -err_resource_free: - sec_free_qp_ctx_resource(ctx, qp_ctx); err_destroy_idr: idr_destroy(&qp_ctx->req_idr); return ret; @@ -658,7 +653,6 @@ static int sec_create_qp_ctx(struct sec_ctx *ctx, int qp_ctx_id) static void sec_release_qp_ctx(struct sec_ctx *ctx, struct sec_qp_ctx *qp_ctx) { - hisi_qm_stop_qp(qp_ctx->qp); sec_free_qp_ctx_resource(ctx, qp_ctx); idr_destroy(&qp_ctx->req_idr); } @@ -669,10 +663,8 @@ static int sec_ctx_base_init(struct sec_ctx *ctx) int i, ret; ctx->qps = sec_create_qps(); - if (!ctx->qps) { - pr_err("Can not create sec qps!\n"); + if (!ctx->qps) return -ENODEV; - } sec = container_of(ctx->qps[0]->qm, struct sec_dev, qm); ctx->sec = sec; @@ -708,6 +700,9 @@ static void sec_ctx_base_uninit(struct sec_ctx *ctx) { int i; + if (!ctx->qps) + return; + for (i = 0; i < ctx->sec->ctx_q_num; i++) sec_release_qp_ctx(ctx, &ctx->qp_ctx[i]); @@ -719,6 +714,9 @@ static int sec_cipher_init(struct sec_ctx *ctx) { struct sec_cipher_ctx *c_ctx = &ctx->c_ctx; + if (!ctx->qps) + return 0; + c_ctx->c_key = dma_alloc_coherent(ctx->dev, SEC_MAX_KEY_SIZE, &c_ctx->c_key_dma, GFP_KERNEL); if (!c_ctx->c_key) @@ -731,6 +729,9 @@ static void sec_cipher_uninit(struct sec_ctx *ctx) { struct sec_cipher_ctx *c_ctx = &ctx->c_ctx; + if (!ctx->qps) + return; + memzero_explicit(c_ctx->c_key, SEC_MAX_KEY_SIZE); dma_free_coherent(ctx->dev, SEC_MAX_KEY_SIZE, c_ctx->c_key, c_ctx->c_key_dma); @@ -752,6 +753,9 @@ static void sec_auth_uninit(struct sec_ctx *ctx) { struct sec_auth_ctx *a_ctx = &ctx->a_ctx; + if (!ctx->qps) + return; + memzero_explicit(a_ctx->a_key, SEC_MAX_AKEY_SIZE); dma_free_coherent(ctx->dev, SEC_MAX_AKEY_SIZE, a_ctx->a_key, a_ctx->a_key_dma); @@ -789,7 +793,7 @@ static int sec_skcipher_init(struct crypto_skcipher *tfm) } ret = sec_ctx_base_init(ctx); - if (ret) + if (ret && ret != -ENODEV) return ret; ret = sec_cipher_init(ctx); @@ -898,6 +902,9 @@ static int sec_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key, struct device *dev = ctx->dev; int ret; + if (!ctx->qps) + goto set_soft_key; + if (c_mode == SEC_CMODE_XTS) { ret = xts_verify_key(tfm, key, keylen); if (ret) { @@ -928,13 +935,14 @@ static int sec_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key, } memcpy(c_ctx->c_key, key, keylen); - if (c_ctx->fbtfm) { - ret = crypto_sync_skcipher_setkey(c_ctx->fbtfm, key, keylen); - if (ret) { - dev_err(dev, "failed to set fallback skcipher key!\n"); - return ret; - } + +set_soft_key: + ret = crypto_sync_skcipher_setkey(c_ctx->fbtfm, key, keylen); + if (ret) { + dev_err(dev, "failed to set fallback skcipher key!\n"); + return ret; } + return 0; } @@ -1398,6 +1406,9 @@ static int sec_aead_setkey(struct crypto_aead *tfm, const u8 *key, struct crypto_authenc_keys keys; int ret; + if (!ctx->qps) + return sec_aead_fallback_setkey(a_ctx, tfm, key, keylen); + ctx->a_ctx.a_alg = a_alg; ctx->c_ctx.c_alg = c_alg; c_ctx->c_mode = c_mode; @@ -1952,7 +1963,6 @@ static int sec_request_init(struct sec_ctx *ctx, struct sec_req *req) } while (req->req_id < 0 && ++i < ctx->sec->ctx_q_num); req->qp_ctx = qp_ctx; - req->backlog = &qp_ctx->backlog; return 0; } @@ -2055,6 +2065,9 @@ static int sec_skcipher_ctx_init(struct crypto_skcipher *tfm) if (ret) return ret; + if (!ctx->qps) + return 0; + if (ctx->sec->qm.ver < QM_HW_V3) { ctx->type_supported = SEC_BD_TYPE2; ctx->req_op = &sec_skcipher_req_ops; @@ -2063,7 +2076,7 @@ static int sec_skcipher_ctx_init(struct crypto_skcipher *tfm) ctx->req_op = &sec_skcipher_req_ops_v3; } - return ret; + return 0; } static void sec_skcipher_ctx_exit(struct crypto_skcipher *tfm) @@ -2131,7 +2144,7 @@ static int sec_aead_ctx_init(struct crypto_aead *tfm, const char *hash_name) int ret; ret = sec_aead_init(tfm); - if (ret) { + if (ret && ret != -ENODEV) { pr_err("hisi_sec2: aead init error!\n"); return ret; } @@ -2173,7 +2186,7 @@ static int sec_aead_xcm_ctx_init(struct crypto_aead *tfm) int ret; ret = sec_aead_init(tfm); - if (ret) { + if (ret && ret != -ENODEV) { dev_err(ctx->dev, "hisi_sec2: aead xcm init error!\n"); return ret; } @@ -2318,6 +2331,9 @@ static int sec_skcipher_crypto(struct skcipher_request *sk_req, bool encrypt) bool need_fallback = false; int ret; + if (!ctx->qps) + goto soft_crypto; + if (!sk_req->cryptlen) { if (ctx->c_ctx.c_mode == SEC_CMODE_XTS) return -EINVAL; @@ -2335,9 +2351,12 @@ static int sec_skcipher_crypto(struct skcipher_request *sk_req, bool encrypt) return -EINVAL; if (unlikely(ctx->c_ctx.fallback || need_fallback)) - return sec_skcipher_soft_crypto(ctx, sk_req, encrypt); + goto soft_crypto; return ctx->req_op->process(ctx, req); + +soft_crypto: + return sec_skcipher_soft_crypto(ctx, sk_req, encrypt); } static int sec_skcipher_encrypt(struct skcipher_request *sk_req) @@ -2545,6 +2564,9 @@ static int sec_aead_crypto(struct aead_request *a_req, bool encrypt) bool need_fallback = false; int ret; + if (!ctx->qps) + goto soft_crypto; + req->flag = a_req->base.flags; req->aead_req.aead_req = a_req; req->c_req.encrypt = encrypt; @@ -2555,11 +2577,14 @@ static int sec_aead_crypto(struct aead_request *a_req, bool encrypt) ret = sec_aead_param_check(ctx, req, &need_fallback); if (unlikely(ret)) { if (need_fallback) - return sec_aead_soft_crypto(ctx, a_req, encrypt); + goto soft_crypto; return -EINVAL; } return ctx->req_op->process(ctx, req); + +soft_crypto: + return sec_aead_soft_crypto(ctx, a_req, encrypt); } static int sec_aead_encrypt(struct aead_request *a_req) diff --git a/drivers/crypto/hisilicon/sec2/sec_main.c b/drivers/crypto/hisilicon/sec2/sec_main.c index 5eb2d68207426e..7dd125f5f511fa 100644 --- a/drivers/crypto/hisilicon/sec2/sec_main.c +++ b/drivers/crypto/hisilicon/sec2/sec_main.c @@ -417,18 +417,29 @@ struct hisi_qp **sec_create_qps(void) int node = cpu_to_node(raw_smp_processor_id()); u32 ctx_num = ctx_q_num; struct hisi_qp **qps; + u8 *type; int ret; qps = kcalloc(ctx_num, sizeof(struct hisi_qp *), GFP_KERNEL); if (!qps) return NULL; - ret = hisi_qm_alloc_qps_node(&sec_devices, ctx_num, 0, node, qps); - if (!ret) - return qps; + /* The type of SEC is all 0, so just allocated by kcalloc */ + type = kcalloc(ctx_num, sizeof(u8), GFP_KERNEL); + if (!type) { + kfree(qps); + return NULL; + } - kfree(qps); - return NULL; + ret = hisi_qm_alloc_qps_node(&sec_devices, ctx_num, type, node, qps); + if (ret) { + kfree(type); + kfree(qps); + return NULL; + } + + kfree(type); + return qps; } u64 sec_get_alg_bitmap(struct hisi_qm *qm, u32 high, u32 low) diff --git a/drivers/crypto/hisilicon/sgl.c b/drivers/crypto/hisilicon/sgl.c index 24c7b6ab285ba6..d41b34405c21c4 100644 --- a/drivers/crypto/hisilicon/sgl.c +++ b/drivers/crypto/hisilicon/sgl.c @@ -260,7 +260,7 @@ hisi_acc_sg_buf_map_to_hw_sgl(struct device *dev, struct scatterlist *sgl, return curr_hw_sgl; err_unmap: - dma_unmap_sg(dev, sgl, sg_n, DMA_BIDIRECTIONAL); + dma_unmap_sg(dev, sgl, sg_n, dir); return ERR_PTR(ret); } diff --git a/drivers/crypto/hisilicon/trng/trng.c b/drivers/crypto/hisilicon/trng/trng.c index ac74df4a947126..5ca0b90859a81e 100644 --- a/drivers/crypto/hisilicon/trng/trng.c +++ b/drivers/crypto/hisilicon/trng/trng.c @@ -40,6 +40,7 @@ #define SEED_SHIFT_24 24 #define SEED_SHIFT_16 16 #define SEED_SHIFT_8 8 +#define SW_MAX_RANDOM_BYTES 65520 struct hisi_trng_list { struct mutex lock; @@ -53,8 +54,10 @@ struct hisi_trng { struct list_head list; struct hwrng rng; u32 ver; - bool is_used; - struct mutex mutex; + u32 ctx_num; + /* The bytes of the random number generated since the last seeding. */ + u32 random_bytes; + struct mutex lock; }; struct hisi_trng_ctx { @@ -63,10 +66,14 @@ struct hisi_trng_ctx { static atomic_t trng_active_devs; static struct hisi_trng_list trng_devices; +static int hisi_trng_read(struct hwrng *rng, void *buf, size_t max, bool wait); -static void hisi_trng_set_seed(struct hisi_trng *trng, const u8 *seed) +static int hisi_trng_set_seed(struct hisi_trng *trng, const u8 *seed) { u32 val, seed_reg, i; + int ret; + + writel(0x0, trng->base + SW_DRBG_BLOCKS); for (i = 0; i < SW_DRBG_SEED_SIZE; i += SW_DRBG_SEED_SIZE / SW_DRBG_SEED_REGS_NUM) { @@ -78,6 +85,20 @@ static void hisi_trng_set_seed(struct hisi_trng *trng, const u8 *seed) seed_reg = (i >> SW_DRBG_NUM_SHIFT) % SW_DRBG_SEED_REGS_NUM; writel(val, trng->base + SW_DRBG_SEED(seed_reg)); } + + writel(SW_DRBG_BLOCKS_NUM | (0x1 << SW_DRBG_ENABLE_SHIFT), + trng->base + SW_DRBG_BLOCKS); + writel(0x1, trng->base + SW_DRBG_INIT); + ret = readl_relaxed_poll_timeout(trng->base + SW_DRBG_STATUS, + val, val & BIT(0), SLEEP_US, TIMEOUT_US); + if (ret) { + pr_err("failed to init trng(%d)\n", ret); + return -EIO; + } + + trng->random_bytes = 0; + + return 0; } static int hisi_trng_seed(struct crypto_rng *tfm, const u8 *seed, @@ -85,8 +106,7 @@ static int hisi_trng_seed(struct crypto_rng *tfm, const u8 *seed, { struct hisi_trng_ctx *ctx = crypto_rng_ctx(tfm); struct hisi_trng *trng = ctx->trng; - u32 val = 0; - int ret = 0; + int ret; if (slen < SW_DRBG_SEED_SIZE) { pr_err("slen(%u) is not matched with trng(%d)\n", slen, @@ -94,43 +114,45 @@ static int hisi_trng_seed(struct crypto_rng *tfm, const u8 *seed, return -EINVAL; } - writel(0x0, trng->base + SW_DRBG_BLOCKS); - hisi_trng_set_seed(trng, seed); + mutex_lock(&trng->lock); + ret = hisi_trng_set_seed(trng, seed); + mutex_unlock(&trng->lock); - writel(SW_DRBG_BLOCKS_NUM | (0x1 << SW_DRBG_ENABLE_SHIFT), - trng->base + SW_DRBG_BLOCKS); - writel(0x1, trng->base + SW_DRBG_INIT); + return ret; +} - ret = readl_relaxed_poll_timeout(trng->base + SW_DRBG_STATUS, - val, val & BIT(0), SLEEP_US, TIMEOUT_US); - if (ret) - pr_err("fail to init trng(%d)\n", ret); +static int hisi_trng_reseed(struct hisi_trng *trng) +{ + u8 seed[SW_DRBG_SEED_SIZE]; + int size; - return ret; + if (!trng->random_bytes) + return 0; + + size = hisi_trng_read(&trng->rng, seed, SW_DRBG_SEED_SIZE, false); + if (size != SW_DRBG_SEED_SIZE) + return -EIO; + + return hisi_trng_set_seed(trng, seed); } -static int hisi_trng_generate(struct crypto_rng *tfm, const u8 *src, - unsigned int slen, u8 *dstn, unsigned int dlen) +static int hisi_trng_get_bytes(struct hisi_trng *trng, u8 *dstn, unsigned int dlen) { - struct hisi_trng_ctx *ctx = crypto_rng_ctx(tfm); - struct hisi_trng *trng = ctx->trng; u32 data[SW_DRBG_DATA_NUM]; u32 currsize = 0; u32 val = 0; int ret; u32 i; - if (dlen > SW_DRBG_BLOCKS_NUM * SW_DRBG_BYTES || dlen == 0) { - pr_err("dlen(%u) exceeds limit(%d)!\n", dlen, - SW_DRBG_BLOCKS_NUM * SW_DRBG_BYTES); - return -EINVAL; - } + ret = hisi_trng_reseed(trng); + if (ret) + return ret; do { ret = readl_relaxed_poll_timeout(trng->base + SW_DRBG_STATUS, - val, val & BIT(1), SLEEP_US, TIMEOUT_US); + val, val & BIT(1), SLEEP_US, TIMEOUT_US); if (ret) { - pr_err("fail to generate random number(%d)!\n", ret); + pr_err("failed to generate random number(%d)!\n", ret); break; } @@ -145,30 +167,57 @@ static int hisi_trng_generate(struct crypto_rng *tfm, const u8 *src, currsize = dlen; } + trng->random_bytes += SW_DRBG_BYTES; writel(0x1, trng->base + SW_DRBG_GEN); } while (currsize < dlen); return ret; } +static int hisi_trng_generate(struct crypto_rng *tfm, const u8 *src, + unsigned int slen, u8 *dstn, unsigned int dlen) +{ + struct hisi_trng_ctx *ctx = crypto_rng_ctx(tfm); + struct hisi_trng *trng = ctx->trng; + unsigned int currsize = 0; + unsigned int block_size; + int ret; + + if (!dstn || !dlen) { + pr_err("output is error, dlen %u!\n", dlen); + return -EINVAL; + } + + do { + block_size = min_t(unsigned int, dlen - currsize, SW_MAX_RANDOM_BYTES); + mutex_lock(&trng->lock); + ret = hisi_trng_get_bytes(trng, dstn + currsize, block_size); + mutex_unlock(&trng->lock); + if (ret) + return ret; + currsize += block_size; + } while (currsize < dlen); + + return 0; +} + static int hisi_trng_init(struct crypto_tfm *tfm) { struct hisi_trng_ctx *ctx = crypto_tfm_ctx(tfm); struct hisi_trng *trng; - int ret = -EBUSY; + u32 ctx_num = ~0; mutex_lock(&trng_devices.lock); list_for_each_entry(trng, &trng_devices.list, list) { - if (!trng->is_used) { - trng->is_used = true; + if (trng->ctx_num < ctx_num) { + ctx_num = trng->ctx_num; ctx->trng = trng; - ret = 0; - break; } } + ctx->trng->ctx_num++; mutex_unlock(&trng_devices.lock); - return ret; + return 0; } static void hisi_trng_exit(struct crypto_tfm *tfm) @@ -176,7 +225,7 @@ static void hisi_trng_exit(struct crypto_tfm *tfm) struct hisi_trng_ctx *ctx = crypto_tfm_ctx(tfm); mutex_lock(&trng_devices.lock); - ctx->trng->is_used = false; + ctx->trng->ctx_num--; mutex_unlock(&trng_devices.lock); } @@ -238,7 +287,7 @@ static int hisi_trng_del_from_list(struct hisi_trng *trng) int ret = -EBUSY; mutex_lock(&trng_devices.lock); - if (!trng->is_used) { + if (!trng->ctx_num) { list_del(&trng->list); ret = 0; } @@ -262,7 +311,9 @@ static int hisi_trng_probe(struct platform_device *pdev) if (IS_ERR(trng->base)) return PTR_ERR(trng->base); - trng->is_used = false; + trng->ctx_num = 0; + trng->random_bytes = SW_MAX_RANDOM_BYTES; + mutex_init(&trng->lock); trng->ver = readl(trng->base + HISI_TRNG_VERSION); if (!trng_devices.is_init) { INIT_LIST_HEAD(&trng_devices.list); diff --git a/drivers/crypto/hisilicon/zip/zip.h b/drivers/crypto/hisilicon/zip/zip.h index 9fb2a9c01132b4..b83f228281ab1e 100644 --- a/drivers/crypto/hisilicon/zip/zip.h +++ b/drivers/crypto/hisilicon/zip/zip.h @@ -99,7 +99,7 @@ enum zip_cap_table_type { ZIP_CORE5_BITMAP, }; -int zip_create_qps(struct hisi_qp **qps, int qp_num, int node); +int zip_create_qps(struct hisi_qp **qps, int qp_num, int node, u8 *alg_type); int hisi_zip_register_to_crypto(struct hisi_qm *qm); void hisi_zip_unregister_from_crypto(struct hisi_qm *qm); bool hisi_zip_alg_support(struct hisi_qm *qm, u32 alg); diff --git a/drivers/crypto/hisilicon/zip/zip_crypto.c b/drivers/crypto/hisilicon/zip/zip_crypto.c index b97513981a3b79..e140d4f8afe0e8 100644 --- a/drivers/crypto/hisilicon/zip/zip_crypto.c +++ b/drivers/crypto/hisilicon/zip/zip_crypto.c @@ -39,6 +39,7 @@ enum { HZIP_CTX_Q_NUM }; +#define GET_REQ_FROM_SQE(sqe) ((u64)(sqe)->dw26 | (u64)(sqe)->dw27 << 32) #define COMP_NAME_TO_TYPE(alg_name) \ (!strcmp((alg_name), "deflate") ? HZIP_ALG_TYPE_DEFLATE : 0) @@ -48,6 +49,7 @@ struct hisi_zip_req { struct hisi_acc_hw_sgl *hw_dst; dma_addr_t dma_src; dma_addr_t dma_dst; + struct hisi_zip_qp_ctx *qp_ctx; u16 req_id; }; @@ -64,6 +66,7 @@ struct hisi_zip_qp_ctx { struct hisi_acc_sgl_pool *sgl_pool; struct hisi_zip *zip_dev; struct hisi_zip_ctx *ctx; + u8 req_type; }; struct hisi_zip_sqe_ops { @@ -74,7 +77,6 @@ struct hisi_zip_sqe_ops { void (*fill_req_type)(struct hisi_zip_sqe *sqe, u8 req_type); void (*fill_tag)(struct hisi_zip_sqe *sqe, struct hisi_zip_req *req); void (*fill_sqe_type)(struct hisi_zip_sqe *sqe, u8 sqe_type); - u32 (*get_tag)(struct hisi_zip_sqe *sqe); u32 (*get_status)(struct hisi_zip_sqe *sqe); u32 (*get_dstlen)(struct hisi_zip_sqe *sqe); }; @@ -82,6 +84,7 @@ struct hisi_zip_sqe_ops { struct hisi_zip_ctx { struct hisi_zip_qp_ctx qp_ctx[HZIP_CTX_Q_NUM]; const struct hisi_zip_sqe_ops *ops; + bool fallback; }; static int sgl_sge_nr_set(const char *val, const struct kernel_param *kp) @@ -108,6 +111,24 @@ static u16 sgl_sge_nr = HZIP_SGL_SGE_NR; module_param_cb(sgl_sge_nr, &sgl_sge_nr_ops, &sgl_sge_nr, 0444); MODULE_PARM_DESC(sgl_sge_nr, "Number of sge in sgl(1-255)"); +static int hisi_zip_fallback_do_work(struct acomp_req *acomp_req, bool is_decompress) +{ + ACOMP_FBREQ_ON_STACK(fbreq, acomp_req); + int ret; + + if (!is_decompress) + ret = crypto_acomp_compress(fbreq); + else + ret = crypto_acomp_decompress(fbreq); + if (ret) { + pr_err("failed to do fallback work, ret=%d\n", ret); + return ret; + } + + acomp_req->dlen = fbreq->dlen; + return ret; +} + static struct hisi_zip_req *hisi_zip_create_req(struct hisi_zip_qp_ctx *qp_ctx, struct acomp_req *req) { @@ -131,6 +152,7 @@ static struct hisi_zip_req *hisi_zip_create_req(struct hisi_zip_qp_ctx *qp_ctx, req_cache = q + req_id; req_cache->req_id = req_id; req_cache->req = req; + req_cache->qp_ctx = qp_ctx; return req_cache; } @@ -181,7 +203,8 @@ static void hisi_zip_fill_req_type(struct hisi_zip_sqe *sqe, u8 req_type) static void hisi_zip_fill_tag(struct hisi_zip_sqe *sqe, struct hisi_zip_req *req) { - sqe->dw26 = req->req_id; + sqe->dw26 = lower_32_bits((u64)req); + sqe->dw27 = upper_32_bits((u64)req); } static void hisi_zip_fill_sqe_type(struct hisi_zip_sqe *sqe, u8 sqe_type) @@ -213,7 +236,6 @@ static int hisi_zip_do_work(struct hisi_zip_qp_ctx *qp_ctx, { struct hisi_acc_sgl_pool *pool = qp_ctx->sgl_pool; struct hisi_zip_dfx *dfx = &qp_ctx->zip_dev->dfx; - struct hisi_zip_req_q *req_q = &qp_ctx->req_q; struct acomp_req *a_req = req->req; struct hisi_qp *qp = qp_ctx->qp; struct device *dev = &qp->qm->pdev->dev; @@ -237,18 +259,16 @@ static int hisi_zip_do_work(struct hisi_zip_qp_ctx *qp_ctx, &req->dma_dst, DMA_FROM_DEVICE); if (IS_ERR(req->hw_dst)) { ret = PTR_ERR(req->hw_dst); - dev_err(dev, "failed to map the dst buffer to hw slg (%d)!\n", + dev_err(dev, "failed to map the dst buffer to hw sgl (%d)!\n", ret); goto err_unmap_input; } - hisi_zip_fill_sqe(qp_ctx->ctx, &zip_sqe, qp->req_type, req); + hisi_zip_fill_sqe(qp_ctx->ctx, &zip_sqe, qp_ctx->req_type, req); /* send command to start a task */ atomic64_inc(&dfx->send_cnt); - spin_lock_bh(&req_q->req_lock); ret = hisi_qp_send(qp, &zip_sqe); - spin_unlock_bh(&req_q->req_lock); if (unlikely(ret < 0)) { atomic64_inc(&dfx->send_busy_cnt); ret = -EAGAIN; @@ -265,11 +285,6 @@ static int hisi_zip_do_work(struct hisi_zip_qp_ctx *qp_ctx, return ret; } -static u32 hisi_zip_get_tag(struct hisi_zip_sqe *sqe) -{ - return sqe->dw26; -} - static u32 hisi_zip_get_status(struct hisi_zip_sqe *sqe) { return sqe->dw3 & HZIP_BD_STATUS_M; @@ -282,14 +297,12 @@ static u32 hisi_zip_get_dstlen(struct hisi_zip_sqe *sqe) static void hisi_zip_acomp_cb(struct hisi_qp *qp, void *data) { - struct hisi_zip_qp_ctx *qp_ctx = qp->qp_ctx; + struct hisi_zip_sqe *sqe = data; + struct hisi_zip_req *req = (struct hisi_zip_req *)GET_REQ_FROM_SQE(sqe); + struct hisi_zip_qp_ctx *qp_ctx = req->qp_ctx; const struct hisi_zip_sqe_ops *ops = qp_ctx->ctx->ops; struct hisi_zip_dfx *dfx = &qp_ctx->zip_dev->dfx; - struct hisi_zip_req_q *req_q = &qp_ctx->req_q; struct device *dev = &qp->qm->pdev->dev; - struct hisi_zip_sqe *sqe = data; - u32 tag = ops->get_tag(sqe); - struct hisi_zip_req *req = req_q->q + tag; struct acomp_req *acomp_req = req->req; int err = 0; u32 status; @@ -319,10 +332,15 @@ static int hisi_zip_acompress(struct acomp_req *acomp_req) { struct hisi_zip_ctx *ctx = crypto_tfm_ctx(acomp_req->base.tfm); struct hisi_zip_qp_ctx *qp_ctx = &ctx->qp_ctx[HZIP_QPC_COMP]; - struct device *dev = &qp_ctx->qp->qm->pdev->dev; struct hisi_zip_req *req; + struct device *dev; int ret; + if (ctx->fallback) + return hisi_zip_fallback_do_work(acomp_req, 0); + + dev = &qp_ctx->qp->qm->pdev->dev; + req = hisi_zip_create_req(qp_ctx, acomp_req); if (IS_ERR(req)) return PTR_ERR(req); @@ -340,10 +358,15 @@ static int hisi_zip_adecompress(struct acomp_req *acomp_req) { struct hisi_zip_ctx *ctx = crypto_tfm_ctx(acomp_req->base.tfm); struct hisi_zip_qp_ctx *qp_ctx = &ctx->qp_ctx[HZIP_QPC_DECOMP]; - struct device *dev = &qp_ctx->qp->qm->pdev->dev; struct hisi_zip_req *req; + struct device *dev; int ret; + if (ctx->fallback) + return hisi_zip_fallback_do_work(acomp_req, 1); + + dev = &qp_ctx->qp->qm->pdev->dev; + req = hisi_zip_create_req(qp_ctx, acomp_req); if (IS_ERR(req)) return PTR_ERR(req); @@ -358,33 +381,6 @@ static int hisi_zip_adecompress(struct acomp_req *acomp_req) return ret; } -static int hisi_zip_start_qp(struct hisi_qp *qp, struct hisi_zip_qp_ctx *qp_ctx, - int alg_type, int req_type) -{ - struct device *dev = &qp->qm->pdev->dev; - int ret; - - qp->req_type = req_type; - qp->alg_type = alg_type; - qp->qp_ctx = qp_ctx; - - ret = hisi_qm_start_qp(qp, 0); - if (ret < 0) { - dev_err(dev, "failed to start qp (%d)!\n", ret); - return ret; - } - - qp_ctx->qp = qp; - - return 0; -} - -static void hisi_zip_release_qp(struct hisi_zip_qp_ctx *qp_ctx) -{ - hisi_qm_stop_qp(qp_ctx->qp); - hisi_qm_free_qps(&qp_ctx->qp, 1); -} - static const struct hisi_zip_sqe_ops hisi_zip_ops = { .sqe_type = 0x3, .fill_addr = hisi_zip_fill_addr, @@ -393,7 +389,6 @@ static const struct hisi_zip_sqe_ops hisi_zip_ops = { .fill_req_type = hisi_zip_fill_req_type, .fill_tag = hisi_zip_fill_tag, .fill_sqe_type = hisi_zip_fill_sqe_type, - .get_tag = hisi_zip_get_tag, .get_status = hisi_zip_get_status, .get_dstlen = hisi_zip_get_dstlen, }; @@ -402,10 +397,15 @@ static int hisi_zip_ctx_init(struct hisi_zip_ctx *hisi_zip_ctx, u8 req_type, int { struct hisi_qp *qps[HZIP_CTX_Q_NUM] = { NULL }; struct hisi_zip_qp_ctx *qp_ctx; + u8 alg_type[HZIP_CTX_Q_NUM]; struct hisi_zip *hisi_zip; - int ret, i, j; + int ret, i; - ret = zip_create_qps(qps, HZIP_CTX_Q_NUM, node); + /* alg_type = 0 for compress, 1 for decompress in hw sqe */ + for (i = 0; i < HZIP_CTX_Q_NUM; i++) + alg_type[i] = i; + + ret = zip_create_qps(qps, HZIP_CTX_Q_NUM, node, alg_type); if (ret) { pr_err("failed to create zip qps (%d)!\n", ret); return -ENODEV; @@ -414,19 +414,11 @@ static int hisi_zip_ctx_init(struct hisi_zip_ctx *hisi_zip_ctx, u8 req_type, int hisi_zip = container_of(qps[0]->qm, struct hisi_zip, qm); for (i = 0; i < HZIP_CTX_Q_NUM; i++) { - /* alg_type = 0 for compress, 1 for decompress in hw sqe */ qp_ctx = &hisi_zip_ctx->qp_ctx[i]; qp_ctx->ctx = hisi_zip_ctx; - ret = hisi_zip_start_qp(qps[i], qp_ctx, i, req_type); - if (ret) { - for (j = i - 1; j >= 0; j--) - hisi_qm_stop_qp(hisi_zip_ctx->qp_ctx[j].qp); - - hisi_qm_free_qps(qps, HZIP_CTX_Q_NUM); - return ret; - } - qp_ctx->zip_dev = hisi_zip; + qp_ctx->req_type = req_type; + qp_ctx->qp = qps[i]; } hisi_zip_ctx->ops = &hisi_zip_ops; @@ -436,10 +428,13 @@ static int hisi_zip_ctx_init(struct hisi_zip_ctx *hisi_zip_ctx, u8 req_type, int static void hisi_zip_ctx_exit(struct hisi_zip_ctx *hisi_zip_ctx) { + struct hisi_qp *qps[HZIP_CTX_Q_NUM] = { NULL }; int i; for (i = 0; i < HZIP_CTX_Q_NUM; i++) - hisi_zip_release_qp(&hisi_zip_ctx->qp_ctx[i]); + qps[i] = hisi_zip_ctx->qp_ctx[i].qp; + + hisi_qm_free_qps(qps, HZIP_CTX_Q_NUM); } static int hisi_zip_create_req_q(struct hisi_zip_ctx *ctx) @@ -549,7 +544,7 @@ static int hisi_zip_acomp_init(struct crypto_acomp *tfm) ret = hisi_zip_ctx_init(ctx, COMP_NAME_TO_TYPE(alg_name), tfm->base.node); if (ret) { pr_err("failed to init ctx (%d)!\n", ret); - return ret; + goto switch_to_soft; } dev = &ctx->qp_ctx[0].qp->qm->pdev->dev; @@ -574,17 +569,20 @@ static int hisi_zip_acomp_init(struct crypto_acomp *tfm) hisi_zip_release_req_q(ctx); err_ctx_exit: hisi_zip_ctx_exit(ctx); - return ret; +switch_to_soft: + ctx->fallback = true; + return 0; } static void hisi_zip_acomp_exit(struct crypto_acomp *tfm) { struct hisi_zip_ctx *ctx = crypto_tfm_ctx(&tfm->base); - hisi_zip_set_acomp_cb(ctx, NULL); - hisi_zip_release_sgl_pool(ctx); - hisi_zip_release_req_q(ctx); - hisi_zip_ctx_exit(ctx); + if (!ctx->fallback) { + hisi_zip_release_sgl_pool(ctx); + hisi_zip_release_req_q(ctx); + hisi_zip_ctx_exit(ctx); + } } static struct acomp_alg hisi_zip_acomp_deflate = { @@ -595,7 +593,8 @@ static struct acomp_alg hisi_zip_acomp_deflate = { .base = { .cra_name = "deflate", .cra_driver_name = "hisi-deflate-acomp", - .cra_flags = CRYPTO_ALG_ASYNC, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, .cra_module = THIS_MODULE, .cra_priority = HZIP_ALG_PRIORITY, .cra_ctxsize = sizeof(struct hisi_zip_ctx), diff --git a/drivers/crypto/hisilicon/zip/zip_main.c b/drivers/crypto/hisilicon/zip/zip_main.c index 4fcbe6bada0663..85b26ef1754855 100644 --- a/drivers/crypto/hisilicon/zip/zip_main.c +++ b/drivers/crypto/hisilicon/zip/zip_main.c @@ -446,12 +446,12 @@ static const struct pci_device_id hisi_zip_dev_ids[] = { }; MODULE_DEVICE_TABLE(pci, hisi_zip_dev_ids); -int zip_create_qps(struct hisi_qp **qps, int qp_num, int node) +int zip_create_qps(struct hisi_qp **qps, int qp_num, int node, u8 *alg_type) { if (node == NUMA_NO_NODE) node = cpu_to_node(raw_smp_processor_id()); - return hisi_qm_alloc_qps_node(&zip_devices, qp_num, 0, node, qps); + return hisi_qm_alloc_qps_node(&zip_devices, qp_num, alg_type, node, qps); } bool hisi_zip_alg_support(struct hisi_qm *qm, u32 alg) diff --git a/drivers/crypto/inside-secure/eip93/eip93-main.c b/drivers/crypto/inside-secure/eip93/eip93-main.c index 0b38a567da0e01..b7fd9795062d4c 100644 --- a/drivers/crypto/inside-secure/eip93/eip93-main.c +++ b/drivers/crypto/inside-secure/eip93/eip93-main.c @@ -77,11 +77,44 @@ inline void eip93_irq_clear(struct eip93_device *eip93, u32 mask) __raw_writel(mask, eip93->base + EIP93_REG_INT_CLR); } -static void eip93_unregister_algs(unsigned int i) +static int eip93_algo_is_supported(u32 alg_flags, u32 supported_algo_flags) +{ + if ((IS_DES(alg_flags) || IS_3DES(alg_flags)) && + !(supported_algo_flags & EIP93_PE_OPTION_TDES)) + return 0; + + if (IS_AES(alg_flags) && + !(supported_algo_flags & EIP93_PE_OPTION_AES)) + return 0; + + if (IS_HASH_MD5(alg_flags) && + !(supported_algo_flags & EIP93_PE_OPTION_MD5)) + return 0; + + if (IS_HASH_SHA1(alg_flags) && + !(supported_algo_flags & EIP93_PE_OPTION_SHA_1)) + return 0; + + if (IS_HASH_SHA224(alg_flags) && + !(supported_algo_flags & EIP93_PE_OPTION_SHA_224)) + return 0; + + if (IS_HASH_SHA256(alg_flags) && + !(supported_algo_flags & EIP93_PE_OPTION_SHA_256)) + return 0; + + return 1; +} + +static void eip93_unregister_algs(u32 supported_algo_flags, unsigned int i) { unsigned int j; for (j = 0; j < i; j++) { + if (!eip93_algo_is_supported(eip93_algs[j]->flags, + supported_algo_flags)) + continue; + switch (eip93_algs[j]->type) { case EIP93_ALG_TYPE_SKCIPHER: crypto_unregister_skcipher(&eip93_algs[j]->alg.skcipher); @@ -90,7 +123,7 @@ static void eip93_unregister_algs(unsigned int i) crypto_unregister_aead(&eip93_algs[j]->alg.aead); break; case EIP93_ALG_TYPE_HASH: - crypto_unregister_ahash(&eip93_algs[i]->alg.ahash); + crypto_unregister_ahash(&eip93_algs[j]->alg.ahash); break; } } @@ -106,49 +139,27 @@ static int eip93_register_algs(struct eip93_device *eip93, u32 supported_algo_fl eip93_algs[i]->eip93 = eip93; - if ((IS_DES(alg_flags) || IS_3DES(alg_flags)) && - !(supported_algo_flags & EIP93_PE_OPTION_TDES)) + if (!eip93_algo_is_supported(alg_flags, supported_algo_flags)) continue; - if (IS_AES(alg_flags)) { - if (!(supported_algo_flags & EIP93_PE_OPTION_AES)) - continue; + if (IS_AES(alg_flags) && !IS_HMAC(alg_flags)) { + if (supported_algo_flags & EIP93_PE_OPTION_AES_KEY128) + eip93_algs[i]->alg.skcipher.max_keysize = + AES_KEYSIZE_128; - if (!IS_HMAC(alg_flags)) { - if (supported_algo_flags & EIP93_PE_OPTION_AES_KEY128) - eip93_algs[i]->alg.skcipher.max_keysize = - AES_KEYSIZE_128; + if (supported_algo_flags & EIP93_PE_OPTION_AES_KEY192) + eip93_algs[i]->alg.skcipher.max_keysize = + AES_KEYSIZE_192; - if (supported_algo_flags & EIP93_PE_OPTION_AES_KEY192) - eip93_algs[i]->alg.skcipher.max_keysize = - AES_KEYSIZE_192; + if (supported_algo_flags & EIP93_PE_OPTION_AES_KEY256) + eip93_algs[i]->alg.skcipher.max_keysize = + AES_KEYSIZE_256; - if (supported_algo_flags & EIP93_PE_OPTION_AES_KEY256) - eip93_algs[i]->alg.skcipher.max_keysize = - AES_KEYSIZE_256; - - if (IS_RFC3686(alg_flags)) - eip93_algs[i]->alg.skcipher.max_keysize += - CTR_RFC3686_NONCE_SIZE; - } + if (IS_RFC3686(alg_flags)) + eip93_algs[i]->alg.skcipher.max_keysize += + CTR_RFC3686_NONCE_SIZE; } - if (IS_HASH_MD5(alg_flags) && - !(supported_algo_flags & EIP93_PE_OPTION_MD5)) - continue; - - if (IS_HASH_SHA1(alg_flags) && - !(supported_algo_flags & EIP93_PE_OPTION_SHA_1)) - continue; - - if (IS_HASH_SHA224(alg_flags) && - !(supported_algo_flags & EIP93_PE_OPTION_SHA_224)) - continue; - - if (IS_HASH_SHA256(alg_flags) && - !(supported_algo_flags & EIP93_PE_OPTION_SHA_256)) - continue; - switch (eip93_algs[i]->type) { case EIP93_ALG_TYPE_SKCIPHER: ret = crypto_register_skcipher(&eip93_algs[i]->alg.skcipher); @@ -167,7 +178,7 @@ static int eip93_register_algs(struct eip93_device *eip93, u32 supported_algo_fl return 0; fail: - eip93_unregister_algs(i); + eip93_unregister_algs(supported_algo_flags, i); return ret; } @@ -469,8 +480,11 @@ static int eip93_crypto_probe(struct platform_device *pdev) static void eip93_crypto_remove(struct platform_device *pdev) { struct eip93_device *eip93 = platform_get_drvdata(pdev); + u32 algo_flags; + + algo_flags = readl(eip93->base + EIP93_REG_PE_OPTION_1); - eip93_unregister_algs(ARRAY_SIZE(eip93_algs)); + eip93_unregister_algs(algo_flags, ARRAY_SIZE(eip93_algs)); eip93_cleanup(eip93); } diff --git a/drivers/crypto/intel/iaa/iaa_crypto_main.c b/drivers/crypto/intel/iaa/iaa_crypto_main.c index d0058757b0000d..da9b2bc515194f 100644 --- a/drivers/crypto/intel/iaa/iaa_crypto_main.c +++ b/drivers/crypto/intel/iaa/iaa_crypto_main.c @@ -221,15 +221,13 @@ static struct iaa_compression_mode *iaa_compression_modes[IAA_COMP_MODES_MAX]; static int find_empty_iaa_compression_mode(void) { - int i = -EINVAL; + int i; - for (i = 0; i < IAA_COMP_MODES_MAX; i++) { - if (iaa_compression_modes[i]) - continue; - break; - } + for (i = 0; i < IAA_COMP_MODES_MAX; i++) + if (!iaa_compression_modes[i]) + return i; - return i; + return -EINVAL; } static struct iaa_compression_mode *find_iaa_compression_mode(const char *name, int *idx) diff --git a/drivers/crypto/intel/qat/qat_common/adf_pfvf_pf_proto.c b/drivers/crypto/intel/qat/qat_common/adf_pfvf_pf_proto.c index b9b5e744a3f163..af8dbc7517cf82 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_pfvf_pf_proto.c +++ b/drivers/crypto/intel/qat/qat_common/adf_pfvf_pf_proto.c @@ -148,6 +148,16 @@ static struct pfvf_message handle_blkmsg_req(struct adf_accel_vf_info *vf_info, blk_byte = FIELD_GET(ADF_VF2PF_SMALL_BLOCK_BYTE_MASK, req.data); byte_max = ADF_VF2PF_SMALL_BLOCK_BYTE_MAX; break; + default: + dev_err(&GET_DEV(vf_info->accel_dev), + "Invalid BlockMsg type 0x%.4x received from VF%u\n", + req.type, vf_info->vf_nr); + resp.type = ADF_PF2VF_MSGTYPE_BLKMSG_RESP; + resp.data = FIELD_PREP(ADF_PF2VF_BLKMSG_RESP_TYPE_MASK, + ADF_PF2VF_BLKMSG_RESP_TYPE_ERROR) | + FIELD_PREP(ADF_PF2VF_BLKMSG_RESP_DATA_MASK, + ADF_PF2VF_UNSPECIFIED_ERROR); + return resp; } /* Is this a request for CRC or data? */ diff --git a/drivers/crypto/intel/qat/qat_common/qat_asym_algs.c b/drivers/crypto/intel/qat/qat_common/qat_asym_algs.c index 85c682e248fb91..e09b9edfce423d 100644 --- a/drivers/crypto/intel/qat/qat_common/qat_asym_algs.c +++ b/drivers/crypto/intel/qat/qat_common/qat_asym_algs.c @@ -255,8 +255,8 @@ static int qat_dh_compute_value(struct kpp_request *req) qat_req->areq.dh = req; msg->pke_hdr.service_type = ICP_QAT_FW_COMN_REQ_CPM_FW_PKE; msg->pke_hdr.comn_req_flags = - ICP_QAT_FW_COMN_FLAGS_BUILD(QAT_COMN_PTR_TYPE_FLAT, - QAT_COMN_CD_FLD_TYPE_64BIT_ADR); + ICP_QAT_FW_COMN_FLAGS_BUILD(QAT_COMN_CD_FLD_TYPE_64BIT_ADR, + QAT_COMN_PTR_TYPE_FLAT); /* * If no source is provided use g as base @@ -731,8 +731,8 @@ static int qat_rsa_enc(struct akcipher_request *req) qat_req->areq.rsa = req; msg->pke_hdr.service_type = ICP_QAT_FW_COMN_REQ_CPM_FW_PKE; msg->pke_hdr.comn_req_flags = - ICP_QAT_FW_COMN_FLAGS_BUILD(QAT_COMN_PTR_TYPE_FLAT, - QAT_COMN_CD_FLD_TYPE_64BIT_ADR); + ICP_QAT_FW_COMN_FLAGS_BUILD(QAT_COMN_CD_FLD_TYPE_64BIT_ADR, + QAT_COMN_PTR_TYPE_FLAT); qat_req->in.rsa.enc.e = ctx->dma_e; qat_req->in.rsa.enc.n = ctx->dma_n; @@ -867,8 +867,8 @@ static int qat_rsa_dec(struct akcipher_request *req) qat_req->areq.rsa = req; msg->pke_hdr.service_type = ICP_QAT_FW_COMN_REQ_CPM_FW_PKE; msg->pke_hdr.comn_req_flags = - ICP_QAT_FW_COMN_FLAGS_BUILD(QAT_COMN_PTR_TYPE_FLAT, - QAT_COMN_CD_FLD_TYPE_64BIT_ADR); + ICP_QAT_FW_COMN_FLAGS_BUILD(QAT_COMN_CD_FLD_TYPE_64BIT_ADR, + QAT_COMN_PTR_TYPE_FLAT); if (ctx->crt_mode) { qat_req->in.rsa.dec_crt.p = ctx->dma_p; diff --git a/drivers/crypto/marvell/octeontx/otx_cptpf_ucode.c b/drivers/crypto/marvell/octeontx/otx_cptpf_ucode.c index 9f5601c0280bf1..417a48f4135053 100644 --- a/drivers/crypto/marvell/octeontx/otx_cptpf_ucode.c +++ b/drivers/crypto/marvell/octeontx/otx_cptpf_ucode.c @@ -1326,7 +1326,7 @@ static ssize_t ucode_load_store(struct device *dev, int del_grp_idx = -1; int ucode_idx = 0; - if (strlen(buf) > OTX_CPT_UCODE_NAME_LENGTH) + if (count >= OTX_CPT_UCODE_NAME_LENGTH) return -EINVAL; eng_grps = container_of(attr, struct otx_cpt_eng_grps, ucode_load_attr); diff --git a/drivers/crypto/marvell/octeontx/otx_cptvf_main.c b/drivers/crypto/marvell/octeontx/otx_cptvf_main.c index 88a41d1ca5f644..6c0bfb3ea1c9f2 100644 --- a/drivers/crypto/marvell/octeontx/otx_cptvf_main.c +++ b/drivers/crypto/marvell/octeontx/otx_cptvf_main.c @@ -168,7 +168,8 @@ static void free_command_queues(struct otx_cptvf *cptvf, chunk = list_first_entry(&cqinfo->queue[i].chead, struct otx_cpt_cmd_chunk, nextchunk); - dma_free_coherent(&pdev->dev, chunk->size, + dma_free_coherent(&pdev->dev, + chunk->size + OTX_CPT_NEXT_CHUNK_PTR_SIZE, chunk->head, chunk->dma_addr); chunk->head = NULL; diff --git a/drivers/crypto/omap-crypto.c b/drivers/crypto/omap-crypto.c index a4cc6bf146ec09..0345c9383d5097 100644 --- a/drivers/crypto/omap-crypto.c +++ b/drivers/crypto/omap-crypto.c @@ -21,7 +21,7 @@ static int omap_crypto_copy_sg_lists(int total, int bs, struct scatterlist *tmp; if (!(flags & OMAP_CRYPTO_FORCE_SINGLE_ENTRY)) { - new_sg = kmalloc_array(n, sizeof(*sg), GFP_KERNEL); + new_sg = kmalloc_array(n, sizeof(*new_sg), GFP_KERNEL); if (!new_sg) return -ENOMEM; diff --git a/drivers/crypto/padlock-sha.c b/drivers/crypto/padlock-sha.c index 329f60ad422e6f..9214bbfc868f5d 100644 --- a/drivers/crypto/padlock-sha.c +++ b/drivers/crypto/padlock-sha.c @@ -332,6 +332,13 @@ static int __init padlock_init(void) if (!x86_match_cpu(padlock_sha_ids) || !boot_cpu_has(X86_FEATURE_PHE_EN)) return -ENODEV; + /* + * Skip family 0x07 and newer used by Zhaoxin processors, + * as the driver's self-tests fail on these CPUs. + */ + if (c->x86 >= 0x07) + return -ENODEV; + /* Register the newly added algorithm module if on * * VIA Nano processor, or else just do as before */ if (c->x86_model < 0x0f) { diff --git a/drivers/crypto/starfive/jh7110-aes.c b/drivers/crypto/starfive/jh7110-aes.c index 426b24889af853..01195664cc7cd9 100644 --- a/drivers/crypto/starfive/jh7110-aes.c +++ b/drivers/crypto/starfive/jh7110-aes.c @@ -669,8 +669,10 @@ static int starfive_aes_aead_do_one_req(struct crypto_engine *engine, void *areq return -ENOMEM; if (sg_copy_to_buffer(req->src, sg_nents_for_len(req->src, cryp->assoclen), - rctx->adata, cryp->assoclen) != cryp->assoclen) + rctx->adata, cryp->assoclen) != cryp->assoclen) { + kfree(rctx->adata); return -EINVAL; + } } if (cryp->total_in) @@ -681,8 +683,11 @@ static int starfive_aes_aead_do_one_req(struct crypto_engine *engine, void *areq ctx->rctx = rctx; ret = starfive_aes_hw_init(ctx); - if (ret) + if (ret) { + if (cryp->assoclen) + kfree(rctx->adata); return ret; + } if (!cryp->assoclen) goto write_text; diff --git a/drivers/crypto/tegra/tegra-se-aes.c b/drivers/crypto/tegra/tegra-se-aes.c index 0e07d0523291a5..9210cceb4b7b2c 100644 --- a/drivers/crypto/tegra/tegra-se-aes.c +++ b/drivers/crypto/tegra/tegra-se-aes.c @@ -529,7 +529,7 @@ static struct tegra_se_alg tegra_aes_algs[] = { .cra_name = "cbc(aes)", .cra_driver_name = "cbc-aes-tegra", .cra_priority = 500, - .cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | CRYPTO_ALG_ASYNC, + .cra_flags = CRYPTO_ALG_ASYNC, .cra_blocksize = AES_BLOCK_SIZE, .cra_ctxsize = sizeof(struct tegra_aes_ctx), .cra_alignmask = 0xf, @@ -550,7 +550,7 @@ static struct tegra_se_alg tegra_aes_algs[] = { .cra_name = "ecb(aes)", .cra_driver_name = "ecb-aes-tegra", .cra_priority = 500, - .cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | CRYPTO_ALG_ASYNC, + .cra_flags = CRYPTO_ALG_ASYNC, .cra_blocksize = AES_BLOCK_SIZE, .cra_ctxsize = sizeof(struct tegra_aes_ctx), .cra_alignmask = 0xf, @@ -572,7 +572,7 @@ static struct tegra_se_alg tegra_aes_algs[] = { .cra_name = "ctr(aes)", .cra_driver_name = "ctr-aes-tegra", .cra_priority = 500, - .cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | CRYPTO_ALG_ASYNC, + .cra_flags = CRYPTO_ALG_ASYNC, .cra_blocksize = 1, .cra_ctxsize = sizeof(struct tegra_aes_ctx), .cra_alignmask = 0xf, @@ -594,6 +594,7 @@ static struct tegra_se_alg tegra_aes_algs[] = { .cra_name = "xts(aes)", .cra_driver_name = "xts-aes-tegra", .cra_priority = 500, + .cra_flags = CRYPTO_ALG_ASYNC, .cra_blocksize = AES_BLOCK_SIZE, .cra_ctxsize = sizeof(struct tegra_aes_ctx), .cra_alignmask = (__alignof__(u64) - 1), @@ -1922,6 +1923,7 @@ static struct tegra_se_alg tegra_aead_algs[] = { .cra_name = "gcm(aes)", .cra_driver_name = "gcm-aes-tegra", .cra_priority = 500, + .cra_flags = CRYPTO_ALG_ASYNC, .cra_blocksize = 1, .cra_ctxsize = sizeof(struct tegra_aead_ctx), .cra_alignmask = 0xf, @@ -1944,6 +1946,7 @@ static struct tegra_se_alg tegra_aead_algs[] = { .cra_name = "ccm(aes)", .cra_driver_name = "ccm-aes-tegra", .cra_priority = 500, + .cra_flags = CRYPTO_ALG_ASYNC, .cra_blocksize = 1, .cra_ctxsize = sizeof(struct tegra_aead_ctx), .cra_alignmask = 0xf, @@ -1971,7 +1974,7 @@ static struct tegra_se_alg tegra_cmac_algs[] = { .cra_name = "cmac(aes)", .cra_driver_name = "tegra-se-cmac", .cra_priority = 300, - .cra_flags = CRYPTO_ALG_TYPE_AHASH, + .cra_flags = CRYPTO_ALG_ASYNC, .cra_blocksize = AES_BLOCK_SIZE, .cra_ctxsize = sizeof(struct tegra_cmac_ctx), .cra_alignmask = 0, diff --git a/drivers/crypto/tegra/tegra-se-hash.c b/drivers/crypto/tegra/tegra-se-hash.c index 4a298ace6e9f79..06bb5bf0fa335c 100644 --- a/drivers/crypto/tegra/tegra-se-hash.c +++ b/drivers/crypto/tegra/tegra-se-hash.c @@ -761,7 +761,7 @@ static struct tegra_se_alg tegra_hash_algs[] = { .cra_name = "sha1", .cra_driver_name = "tegra-se-sha1", .cra_priority = 300, - .cra_flags = CRYPTO_ALG_TYPE_AHASH, + .cra_flags = CRYPTO_ALG_ASYNC, .cra_blocksize = SHA1_BLOCK_SIZE, .cra_ctxsize = sizeof(struct tegra_sha_ctx), .cra_alignmask = 0, @@ -786,7 +786,7 @@ static struct tegra_se_alg tegra_hash_algs[] = { .cra_name = "sha224", .cra_driver_name = "tegra-se-sha224", .cra_priority = 300, - .cra_flags = CRYPTO_ALG_TYPE_AHASH, + .cra_flags = CRYPTO_ALG_ASYNC, .cra_blocksize = SHA224_BLOCK_SIZE, .cra_ctxsize = sizeof(struct tegra_sha_ctx), .cra_alignmask = 0, @@ -811,7 +811,7 @@ static struct tegra_se_alg tegra_hash_algs[] = { .cra_name = "sha256", .cra_driver_name = "tegra-se-sha256", .cra_priority = 300, - .cra_flags = CRYPTO_ALG_TYPE_AHASH, + .cra_flags = CRYPTO_ALG_ASYNC, .cra_blocksize = SHA256_BLOCK_SIZE, .cra_ctxsize = sizeof(struct tegra_sha_ctx), .cra_alignmask = 0, @@ -836,7 +836,7 @@ static struct tegra_se_alg tegra_hash_algs[] = { .cra_name = "sha384", .cra_driver_name = "tegra-se-sha384", .cra_priority = 300, - .cra_flags = CRYPTO_ALG_TYPE_AHASH, + .cra_flags = CRYPTO_ALG_ASYNC, .cra_blocksize = SHA384_BLOCK_SIZE, .cra_ctxsize = sizeof(struct tegra_sha_ctx), .cra_alignmask = 0, @@ -861,7 +861,7 @@ static struct tegra_se_alg tegra_hash_algs[] = { .cra_name = "sha512", .cra_driver_name = "tegra-se-sha512", .cra_priority = 300, - .cra_flags = CRYPTO_ALG_TYPE_AHASH, + .cra_flags = CRYPTO_ALG_ASYNC, .cra_blocksize = SHA512_BLOCK_SIZE, .cra_ctxsize = sizeof(struct tegra_sha_ctx), .cra_alignmask = 0, @@ -886,7 +886,7 @@ static struct tegra_se_alg tegra_hash_algs[] = { .cra_name = "sha3-224", .cra_driver_name = "tegra-se-sha3-224", .cra_priority = 300, - .cra_flags = CRYPTO_ALG_TYPE_AHASH, + .cra_flags = CRYPTO_ALG_ASYNC, .cra_blocksize = SHA3_224_BLOCK_SIZE, .cra_ctxsize = sizeof(struct tegra_sha_ctx), .cra_alignmask = 0, @@ -911,7 +911,7 @@ static struct tegra_se_alg tegra_hash_algs[] = { .cra_name = "sha3-256", .cra_driver_name = "tegra-se-sha3-256", .cra_priority = 300, - .cra_flags = CRYPTO_ALG_TYPE_AHASH, + .cra_flags = CRYPTO_ALG_ASYNC, .cra_blocksize = SHA3_256_BLOCK_SIZE, .cra_ctxsize = sizeof(struct tegra_sha_ctx), .cra_alignmask = 0, @@ -936,7 +936,7 @@ static struct tegra_se_alg tegra_hash_algs[] = { .cra_name = "sha3-384", .cra_driver_name = "tegra-se-sha3-384", .cra_priority = 300, - .cra_flags = CRYPTO_ALG_TYPE_AHASH, + .cra_flags = CRYPTO_ALG_ASYNC, .cra_blocksize = SHA3_384_BLOCK_SIZE, .cra_ctxsize = sizeof(struct tegra_sha_ctx), .cra_alignmask = 0, @@ -961,7 +961,7 @@ static struct tegra_se_alg tegra_hash_algs[] = { .cra_name = "sha3-512", .cra_driver_name = "tegra-se-sha3-512", .cra_priority = 300, - .cra_flags = CRYPTO_ALG_TYPE_AHASH, + .cra_flags = CRYPTO_ALG_ASYNC, .cra_blocksize = SHA3_512_BLOCK_SIZE, .cra_ctxsize = sizeof(struct tegra_sha_ctx), .cra_alignmask = 0, @@ -988,7 +988,8 @@ static struct tegra_se_alg tegra_hash_algs[] = { .cra_name = "hmac(sha224)", .cra_driver_name = "tegra-se-hmac-sha224", .cra_priority = 300, - .cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_NEED_FALLBACK, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, .cra_blocksize = SHA224_BLOCK_SIZE, .cra_ctxsize = sizeof(struct tegra_sha_ctx), .cra_alignmask = 0, @@ -1015,7 +1016,8 @@ static struct tegra_se_alg tegra_hash_algs[] = { .cra_name = "hmac(sha256)", .cra_driver_name = "tegra-se-hmac-sha256", .cra_priority = 300, - .cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_NEED_FALLBACK, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, .cra_blocksize = SHA256_BLOCK_SIZE, .cra_ctxsize = sizeof(struct tegra_sha_ctx), .cra_alignmask = 0, @@ -1042,7 +1044,8 @@ static struct tegra_se_alg tegra_hash_algs[] = { .cra_name = "hmac(sha384)", .cra_driver_name = "tegra-se-hmac-sha384", .cra_priority = 300, - .cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_NEED_FALLBACK, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, .cra_blocksize = SHA384_BLOCK_SIZE, .cra_ctxsize = sizeof(struct tegra_sha_ctx), .cra_alignmask = 0, @@ -1069,7 +1072,8 @@ static struct tegra_se_alg tegra_hash_algs[] = { .cra_name = "hmac(sha512)", .cra_driver_name = "tegra-se-hmac-sha512", .cra_priority = 300, - .cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_NEED_FALLBACK, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, .cra_blocksize = SHA512_BLOCK_SIZE, .cra_ctxsize = sizeof(struct tegra_sha_ctx), .cra_alignmask = 0, diff --git a/drivers/crypto/virtio/virtio_crypto_core.c b/drivers/crypto/virtio/virtio_crypto_core.c index 3d241446099cc9..ccc6b5c1b24b37 100644 --- a/drivers/crypto/virtio/virtio_crypto_core.c +++ b/drivers/crypto/virtio/virtio_crypto_core.c @@ -75,15 +75,20 @@ static void virtcrypto_done_task(unsigned long data) struct data_queue *data_vq = (struct data_queue *)data; struct virtqueue *vq = data_vq->vq; struct virtio_crypto_request *vc_req; + unsigned long flags; unsigned int len; + spin_lock_irqsave(&data_vq->lock, flags); do { virtqueue_disable_cb(vq); while ((vc_req = virtqueue_get_buf(vq, &len)) != NULL) { + spin_unlock_irqrestore(&data_vq->lock, flags); if (vc_req->alg_cb) vc_req->alg_cb(vc_req, len); + spin_lock_irqsave(&data_vq->lock, flags); } } while (!virtqueue_enable_cb(vq)); + spin_unlock_irqrestore(&data_vq->lock, flags); } static void virtcrypto_dataq_callback(struct virtqueue *vq) diff --git a/drivers/crypto/virtio/virtio_crypto_skcipher_algs.c b/drivers/crypto/virtio/virtio_crypto_skcipher_algs.c index 1b3fb21a2a7de2..11053d1786d4d2 100644 --- a/drivers/crypto/virtio/virtio_crypto_skcipher_algs.c +++ b/drivers/crypto/virtio/virtio_crypto_skcipher_algs.c @@ -541,8 +541,6 @@ int virtio_crypto_skcipher_crypt_req( if (ret < 0) return ret; - virtqueue_kick(data_vq->vq); - return 0; } diff --git a/drivers/cxl/Kconfig b/drivers/cxl/Kconfig index 48b7314afdb88c..599e126a18eb72 100644 --- a/drivers/cxl/Kconfig +++ b/drivers/cxl/Kconfig @@ -58,6 +58,7 @@ config CXL_ACPI tristate "CXL ACPI: Platform Support" depends on ACPI depends on ACPI_NUMA + depends on CXL_PMEM || !CXL_PMEM default CXL_BUS select ACPI_TABLE_LIB select ACPI_HMAT diff --git a/drivers/cxl/core/edac.c b/drivers/cxl/core/edac.c index 79994ca9bc9f37..81160260e26b7e 100644 --- a/drivers/cxl/core/edac.c +++ b/drivers/cxl/core/edac.c @@ -1988,6 +1988,40 @@ static int cxl_memdev_soft_ppr_init(struct cxl_memdev *cxlmd, return 0; } +static void err_rec_free(void *_cxlmd) +{ + struct cxl_memdev *cxlmd = _cxlmd; + struct cxl_mem_err_rec *array_rec = cxlmd->err_rec_array; + struct cxl_event_gen_media *rec_gen_media; + struct cxl_event_dram *rec_dram; + unsigned long index; + + cxlmd->err_rec_array = NULL; + xa_for_each(&array_rec->rec_dram, index, rec_dram) + kfree(rec_dram); + xa_destroy(&array_rec->rec_dram); + + xa_for_each(&array_rec->rec_gen_media, index, rec_gen_media) + kfree(rec_gen_media); + xa_destroy(&array_rec->rec_gen_media); + kfree(array_rec); +} + +static int devm_cxl_memdev_setup_err_rec(struct cxl_memdev *cxlmd) +{ + struct cxl_mem_err_rec *array_rec = + kzalloc(sizeof(*array_rec), GFP_KERNEL); + + if (!array_rec) + return -ENOMEM; + + xa_init(&array_rec->rec_gen_media); + xa_init(&array_rec->rec_dram); + cxlmd->err_rec_array = array_rec; + + return devm_add_action_or_reset(&cxlmd->dev, err_rec_free, cxlmd); +} + int devm_cxl_memdev_edac_register(struct cxl_memdev *cxlmd) { struct edac_dev_feature ras_features[CXL_NR_EDAC_DEV_FEATURES]; @@ -2038,15 +2072,9 @@ int devm_cxl_memdev_edac_register(struct cxl_memdev *cxlmd) } if (repair_inst) { - struct cxl_mem_err_rec *array_rec = - devm_kzalloc(&cxlmd->dev, sizeof(*array_rec), - GFP_KERNEL); - if (!array_rec) - return -ENOMEM; - - xa_init(&array_rec->rec_gen_media); - xa_init(&array_rec->rec_dram); - cxlmd->err_rec_array = array_rec; + rc = devm_cxl_memdev_setup_err_rec(cxlmd); + if (rc) + return rc; } } @@ -2088,22 +2116,4 @@ int devm_cxl_region_edac_register(struct cxl_region *cxlr) } EXPORT_SYMBOL_NS_GPL(devm_cxl_region_edac_register, "CXL"); -void devm_cxl_memdev_edac_release(struct cxl_memdev *cxlmd) -{ - struct cxl_mem_err_rec *array_rec = cxlmd->err_rec_array; - struct cxl_event_gen_media *rec_gen_media; - struct cxl_event_dram *rec_dram; - unsigned long index; - - if (!IS_ENABLED(CONFIG_CXL_EDAC_MEM_REPAIR) || !array_rec) - return; - - xa_for_each(&array_rec->rec_dram, index, rec_dram) - kfree(rec_dram); - xa_destroy(&array_rec->rec_dram); - xa_for_each(&array_rec->rec_gen_media, index, rec_gen_media) - kfree(rec_gen_media); - xa_destroy(&array_rec->rec_gen_media); -} -EXPORT_SYMBOL_NS_GPL(devm_cxl_memdev_edac_release, "CXL"); diff --git a/drivers/cxl/core/hdm.c b/drivers/cxl/core/hdm.c index eb5a3a7640c601..ce27074bb5c7d0 100644 --- a/drivers/cxl/core/hdm.c +++ b/drivers/cxl/core/hdm.c @@ -94,7 +94,6 @@ static bool should_emulate_decoders(struct cxl_endpoint_dvsec_info *info) struct cxl_hdm *cxlhdm; void __iomem *hdm; u32 ctrl; - int i; if (!info) return false; @@ -113,22 +112,16 @@ static bool should_emulate_decoders(struct cxl_endpoint_dvsec_info *info) return false; /* - * If any decoders are committed already, there should not be any - * emulated DVSEC decoders. + * If HDM decoders are globally enabled, do not fall back to DVSEC + * range emulation. Zeroed decoder registers after region teardown + * do not imply absence of HDM capability. + * + * Falling back to DVSEC here would treat the decoder as AUTO and + * may incorrectly latch default interleave settings. */ - for (i = 0; i < cxlhdm->decoder_count; i++) { - ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(i)); - dev_dbg(&info->port->dev, - "decoder%d.%d: committed: %ld base: %#x_%.8x size: %#x_%.8x\n", - info->port->id, i, - FIELD_GET(CXL_HDM_DECODER0_CTRL_COMMITTED, ctrl), - readl(hdm + CXL_HDM_DECODER0_BASE_HIGH_OFFSET(i)), - readl(hdm + CXL_HDM_DECODER0_BASE_LOW_OFFSET(i)), - readl(hdm + CXL_HDM_DECODER0_SIZE_HIGH_OFFSET(i)), - readl(hdm + CXL_HDM_DECODER0_SIZE_LOW_OFFSET(i))); - if (FIELD_GET(CXL_HDM_DECODER0_CTRL_COMMITTED, ctrl)) - return false; - } + ctrl = readl(hdm + CXL_HDM_DECODER_CTRL_OFFSET); + if (ctrl & CXL_HDM_DECODER_ENABLE) + return false; return true; } @@ -844,14 +837,13 @@ static int cxl_decoder_commit(struct cxl_decoder *cxld) scoped_guard(rwsem_read, &cxl_rwsem.dpa) setup_hw_decoder(cxld, hdm); - port->commit_end++; rc = cxld_await_commit(hdm, cxld->id); if (rc) { dev_dbg(&port->dev, "%s: error %d committing decoder\n", dev_name(&cxld->dev), rc); - cxld->reset(cxld); return rc; } + port->commit_end++; cxld->flags |= CXL_DECODER_F_ENABLE; return 0; @@ -966,7 +958,7 @@ static int cxl_setup_hdm_decoder_from_dvsec( rc = devm_cxl_dpa_reserve(cxled, *dpa_base, len, 0); if (rc) { dev_err(&port->dev, - "decoder%d.%d: Failed to reserve DPA range %#llx - %#llx\n (%d)", + "decoder%d.%d: Failed to reserve DPA range %#llx - %#llx: %d\n", port->id, cxld->id, *dpa_base, *dpa_base + len - 1, rc); return rc; } @@ -1117,7 +1109,7 @@ static int init_hdm_decoder(struct cxl_port *port, struct cxl_decoder *cxld, rc = devm_cxl_dpa_reserve(cxled, *dpa_base + skip, dpa_size, skip); if (rc) { dev_err(&port->dev, - "decoder%d.%d: Failed to reserve DPA range %#llx - %#llx\n (%d)", + "decoder%d.%d: Failed to reserve DPA range %#llx - %#llx: %d\n", port->id, cxld->id, *dpa_base, *dpa_base + dpa_size + skip - 1, rc); return rc; diff --git a/drivers/cxl/core/mbox.c b/drivers/cxl/core/mbox.c index fa6dd0c94656f7..e7a6452bf5445b 100644 --- a/drivers/cxl/core/mbox.c +++ b/drivers/cxl/core/mbox.c @@ -311,6 +311,7 @@ static bool cxl_mem_raw_command_allowed(u16 opcode) * cxl_payload_from_user_allowed() - Check contents of in_payload. * @opcode: The mailbox command opcode. * @payload_in: Pointer to the input payload passed in from user space. + * @in_size: Size of @payload_in in bytes. * * Return: * * true - payload_in passes check for @opcode. @@ -325,12 +326,15 @@ static bool cxl_mem_raw_command_allowed(u16 opcode) * * The specific checks are determined by the opcode. */ -static bool cxl_payload_from_user_allowed(u16 opcode, void *payload_in) +static bool cxl_payload_from_user_allowed(u16 opcode, void *payload_in, + size_t in_size) { switch (opcode) { case CXL_MBOX_OP_SET_PARTITION_INFO: { struct cxl_mbox_set_partition_info *pi = payload_in; + if (in_size < sizeof(*pi)) + return false; if (pi->flags & CXL_SET_PARTITION_IMMEDIATE_FLAG) return false; break; @@ -338,6 +342,8 @@ static bool cxl_payload_from_user_allowed(u16 opcode, void *payload_in) case CXL_MBOX_OP_CLEAR_LOG: { const uuid_t *uuid = (uuid_t *)payload_in; + if (in_size < sizeof(uuid_t)) + return false; /* * Restrict the ‘Clear log’ action to only apply to * Vendor debug logs. @@ -365,7 +371,8 @@ static int cxl_mbox_cmd_ctor(struct cxl_mbox_cmd *mbox_cmd, if (IS_ERR(mbox_cmd->payload_in)) return PTR_ERR(mbox_cmd->payload_in); - if (!cxl_payload_from_user_allowed(opcode, mbox_cmd->payload_in)) { + if (!cxl_payload_from_user_allowed(opcode, mbox_cmd->payload_in, + in_size)) { dev_dbg(cxl_mbox->host, "%s: input payload not allowed\n", cxl_mem_opcode_to_name(opcode)); kvfree(mbox_cmd->payload_in); diff --git a/drivers/cxl/core/memdev.c b/drivers/cxl/core/memdev.c index e370d733e44001..4dff7f44d908e0 100644 --- a/drivers/cxl/core/memdev.c +++ b/drivers/cxl/core/memdev.c @@ -27,7 +27,6 @@ static void cxl_memdev_release(struct device *dev) struct cxl_memdev *cxlmd = to_cxl_memdev(dev); ida_free(&cxl_memdev_ida, cxlmd->id); - devm_cxl_memdev_edac_release(cxlmd); kfree(cxlmd); } diff --git a/drivers/cxl/core/pmem.c b/drivers/cxl/core/pmem.c index 8853415c106a98..e3a8b8d8133335 100644 --- a/drivers/cxl/core/pmem.c +++ b/drivers/cxl/core/pmem.c @@ -115,15 +115,17 @@ static void unregister_nvb(void *_cxl_nvb) device_unregister(&cxl_nvb->dev); } -/** - * devm_cxl_add_nvdimm_bridge() - add the root of a LIBNVDIMM topology - * @host: platform firmware root device - * @port: CXL port at the root of a CXL topology - * - * Return: bridge device that can host cxl_nvdimm objects - */ -struct cxl_nvdimm_bridge *devm_cxl_add_nvdimm_bridge(struct device *host, - struct cxl_port *port) +static bool cxl_nvdimm_bridge_failed_attach(struct cxl_nvdimm_bridge *cxl_nvb) +{ + struct device *dev = &cxl_nvb->dev; + + guard(device)(dev); + /* If the device has no driver, then it failed to attach. */ + return dev->driver == NULL; +} + +struct cxl_nvdimm_bridge *__devm_cxl_add_nvdimm_bridge(struct device *host, + struct cxl_port *port) { struct cxl_nvdimm_bridge *cxl_nvb; struct device *dev; @@ -145,6 +147,11 @@ struct cxl_nvdimm_bridge *devm_cxl_add_nvdimm_bridge(struct device *host, if (rc) goto err; + if (cxl_nvdimm_bridge_failed_attach(cxl_nvb)) { + unregister_nvb(cxl_nvb); + return ERR_PTR(-ENODEV); + } + rc = devm_add_action_or_reset(host, unregister_nvb, cxl_nvb); if (rc) return ERR_PTR(rc); @@ -155,7 +162,7 @@ struct cxl_nvdimm_bridge *devm_cxl_add_nvdimm_bridge(struct device *host, put_device(dev); return ERR_PTR(rc); } -EXPORT_SYMBOL_NS_GPL(devm_cxl_add_nvdimm_bridge, "CXL"); +EXPORT_SYMBOL_FOR_MODULES(__devm_cxl_add_nvdimm_bridge, "cxl_pmem"); static void cxl_nvdimm_release(struct device *dev) { @@ -254,6 +261,21 @@ int devm_cxl_add_nvdimm(struct cxl_port *parent_port, if (!cxl_nvb) return -ENODEV; + /* + * Take the uport_dev lock to guard against race of nvdimm_bus object. + * cxl_acpi_probe() registers the nvdimm_bus and is done under the + * root port uport_dev lock. + * + * Take the cxl_nvb device lock to ensure that cxl_nvb driver is in a + * consistent state. And the driver registers nvdimm_bus. + */ + guard(device)(cxl_nvb->port->uport_dev); + guard(device)(&cxl_nvb->dev); + if (!cxl_nvb->nvdimm_bus) { + rc = -ENODEV; + goto err_alloc; + } + cxl_nvd = cxl_nvdimm_alloc(cxl_nvb, cxlmd); if (IS_ERR(cxl_nvd)) { rc = PTR_ERR(cxl_nvd); diff --git a/drivers/cxl/core/port.c b/drivers/cxl/core/port.c index 3310dbfae9d663..aa8b47c50c9622 100644 --- a/drivers/cxl/core/port.c +++ b/drivers/cxl/core/port.c @@ -552,10 +552,13 @@ static void cxl_port_release(struct device *dev) xa_destroy(&port->dports); xa_destroy(&port->regions); ida_free(&cxl_port_ida, port->id); - if (is_cxl_root(port)) + + if (is_cxl_root(port)) { kfree(to_cxl_root(port)); - else + } else { + put_device(dev->parent); kfree(port); + } } static ssize_t decoders_committed_show(struct device *dev, @@ -721,6 +724,7 @@ static struct cxl_port *cxl_port_alloc(struct device *uport_dev, struct cxl_port *iter; dev->parent = &parent_port->dev; + get_device(dev->parent); port->depth = parent_port->depth + 1; port->parent_dport = parent_dport; @@ -822,16 +826,18 @@ DEFINE_DEBUGFS_ATTRIBUTE(cxl_einj_inject_fops, NULL, cxl_einj_inject, static void cxl_debugfs_create_dport_dir(struct cxl_dport *dport) { + struct cxl_port *parent = parent_port_of(dport->port); struct dentry *dir; if (!einj_cxl_is_initialized()) return; /* - * dport_dev needs to be a PCIe port for CXL 2.0+ ports because - * EINJ expects a dport SBDF to be specified for 2.0 error injection. + * Protocol error injection is only available for CXL 2.0+ root ports + * and CXL 1.1 downstream ports */ - if (!dport->rch && !dev_is_pci(dport->dport_dev)) + if (!dport->rch && + !(dev_is_pci(dport->dport_dev) && parent && is_cxl_root(parent))) return; dir = cxl_debugfs_create_dir(dev_name(dport->dport_dev)); diff --git a/drivers/cxl/core/region.c b/drivers/cxl/core/region.c index 5bd1213737fa2f..a3d06b852d05e5 100644 --- a/drivers/cxl/core/region.c +++ b/drivers/cxl/core/region.c @@ -3616,8 +3616,10 @@ static int __construct_region(struct cxl_region *cxlr, } rc = sysfs_update_group(&cxlr->dev.kobj, &cxl_region_group); - if (rc) + if (rc) { + kfree(res); return rc; + } rc = insert_resource(cxlrd->res, res); if (rc) { diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h index ba17fa86d249eb..e477cd72d3000b 100644 --- a/drivers/cxl/cxl.h +++ b/drivers/cxl/cxl.h @@ -563,11 +563,16 @@ struct cxl_nvdimm_bridge { #define CXL_DEV_ID_LEN 19 +enum { + CXL_NVD_F_INVALIDATED = 0, +}; + struct cxl_nvdimm { struct device dev; struct cxl_memdev *cxlmd; u8 dev_id[CXL_DEV_ID_LEN]; /* for nvdimm, string of 'serial' */ u64 dirty_shutdowns; + unsigned long flags; }; struct cxl_pmem_region_mapping { @@ -893,6 +898,8 @@ void cxl_driver_unregister(struct cxl_driver *cxl_drv); struct cxl_nvdimm_bridge *to_cxl_nvdimm_bridge(struct device *dev); struct cxl_nvdimm_bridge *devm_cxl_add_nvdimm_bridge(struct device *host, struct cxl_port *port); +struct cxl_nvdimm_bridge *__devm_cxl_add_nvdimm_bridge(struct device *host, + struct cxl_port *port); struct cxl_nvdimm *to_cxl_nvdimm(struct device *dev); bool is_cxl_nvdimm(struct device *dev); int devm_cxl_add_nvdimm(struct cxl_port *parent_port, struct cxl_memdev *cxlmd); diff --git a/drivers/cxl/cxlmem.h b/drivers/cxl/cxlmem.h index 434031a0c1f74a..c12ab4fc951238 100644 --- a/drivers/cxl/cxlmem.h +++ b/drivers/cxl/cxlmem.h @@ -63,7 +63,7 @@ struct cxl_memdev { int depth; u8 scrub_cycle; int scrub_region_id; - void *err_rec_array; + struct cxl_mem_err_rec *err_rec_array; }; static inline struct cxl_memdev *to_cxl_memdev(struct device *dev) @@ -877,7 +877,6 @@ int devm_cxl_memdev_edac_register(struct cxl_memdev *cxlmd); int devm_cxl_region_edac_register(struct cxl_region *cxlr); int cxl_store_rec_gen_media(struct cxl_memdev *cxlmd, union cxl_event *evt); int cxl_store_rec_dram(struct cxl_memdev *cxlmd, union cxl_event *evt); -void devm_cxl_memdev_edac_release(struct cxl_memdev *cxlmd); #else static inline int devm_cxl_memdev_edac_register(struct cxl_memdev *cxlmd) { return 0; } @@ -889,8 +888,6 @@ static inline int cxl_store_rec_gen_media(struct cxl_memdev *cxlmd, static inline int cxl_store_rec_dram(struct cxl_memdev *cxlmd, union cxl_event *evt) { return 0; } -static inline void devm_cxl_memdev_edac_release(struct cxl_memdev *cxlmd) -{ return; } #endif #ifdef CONFIG_CXL_SUSPEND diff --git a/drivers/cxl/pmem.c b/drivers/cxl/pmem.c index e197883690efc1..3432fd83b1e2a7 100644 --- a/drivers/cxl/pmem.c +++ b/drivers/cxl/pmem.c @@ -13,6 +13,20 @@ static __read_mostly DECLARE_BITMAP(exclusive_cmds, CXL_MEM_COMMAND_ID_MAX); +/** + * devm_cxl_add_nvdimm_bridge() - add the root of a LIBNVDIMM topology + * @host: platform firmware root device + * @port: CXL port at the root of a CXL topology + * + * Return: bridge device that can host cxl_nvdimm objects + */ +struct cxl_nvdimm_bridge *devm_cxl_add_nvdimm_bridge(struct device *host, + struct cxl_port *port) +{ + return __devm_cxl_add_nvdimm_bridge(host, port); +} +EXPORT_SYMBOL_NS_GPL(devm_cxl_add_nvdimm_bridge, "CXL"); + static void clear_exclusive(void *mds) { clear_exclusive_cxl_commands(mds, exclusive_cmds); @@ -129,6 +143,9 @@ static int cxl_nvdimm_probe(struct device *dev) struct nvdimm *nvdimm; int rc; + if (test_bit(CXL_NVD_F_INVALIDATED, &cxl_nvd->flags)) + return -EBUSY; + set_exclusive_cxl_commands(mds, exclusive_cmds); rc = devm_add_action_or_reset(dev, clear_exclusive, mds); if (rc) @@ -309,8 +326,10 @@ static int detach_nvdimm(struct device *dev, void *data) scoped_guard(device, dev) { if (dev->driver) { cxl_nvd = to_cxl_nvdimm(dev); - if (cxl_nvd->cxlmd && cxl_nvd->cxlmd->cxl_nvb == data) + if (cxl_nvd->cxlmd && cxl_nvd->cxlmd->cxl_nvb == data) { release = true; + set_bit(CXL_NVD_F_INVALIDATED, &cxl_nvd->flags); + } } } if (release) @@ -353,6 +372,7 @@ static struct cxl_driver cxl_nvdimm_bridge_driver = { .probe = cxl_nvdimm_bridge_probe, .id = CXL_DEVICE_NVDIMM_BRIDGE, .drv = { + .probe_type = PROBE_FORCE_SYNCHRONOUS, .suppress_bind_attrs = true, }, }; @@ -534,7 +554,7 @@ static __exit void cxl_pmem_exit(void) MODULE_DESCRIPTION("CXL PMEM: Persistent Memory Support"); MODULE_LICENSE("GPL v2"); -module_init(cxl_pmem_init); +subsys_initcall(cxl_pmem_init); module_exit(cxl_pmem_exit); MODULE_IMPORT_NS("CXL"); MODULE_ALIAS_CXL(CXL_DEVICE_NVDIMM_BRIDGE); diff --git a/drivers/dma/Kconfig b/drivers/dma/Kconfig index 8bb0a119ecd48a..3ba2aedaae79d2 100644 --- a/drivers/dma/Kconfig +++ b/drivers/dma/Kconfig @@ -89,9 +89,20 @@ config APPLE_ADMAC tristate "Apple ADMAC support" depends on ARCH_APPLE || COMPILE_TEST select DMA_ENGINE + select DMA_VIRTUAL_CHANNELS help Enable support for Audio DMA Controller found on Apple Silicon SoCs. +config APPLE_SIO + tristate "Apple SIO support" + depends on ARCH_APPLE || COMPILE_TEST + depends on APPLE_RTKIT + depends on OF_ADDRESS + select DMA_ENGINE + help + Enable support for the SIO coprocessor found on Apple Silicon SoCs + where it provides DMA services. + config ARM_DMA350 tristate "Arm DMA-350 support" depends on ARM || ARM64 || COMPILE_TEST diff --git a/drivers/dma/Makefile b/drivers/dma/Makefile index a54d7688392b1a..1c11fdc02692cc 100644 --- a/drivers/dma/Makefile +++ b/drivers/dma/Makefile @@ -17,6 +17,7 @@ obj-$(CONFIG_ALTERA_MSGDMA) += altera-msgdma.o obj-$(CONFIG_AMBA_PL08X) += amba-pl08x.o obj-$(CONFIG_AMCC_PPC440SPE_ADMA) += ppc4xx/ obj-$(CONFIG_APPLE_ADMAC) += apple-admac.o +obj-$(CONFIG_APPLE_SIO) += apple-sio.o obj-$(CONFIG_ARM_DMA350) += arm-dma350.o obj-$(CONFIG_AT_HDMAC) += at_hdmac.o obj-$(CONFIG_AT_XDMAC) += at_xdmac.o diff --git a/drivers/dma/apple-sio.c b/drivers/dma/apple-sio.c new file mode 100644 index 00000000000000..511f91999ed3de --- /dev/null +++ b/drivers/dma/apple-sio.c @@ -0,0 +1,942 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Driver for SIO coprocessor on t8103 (M1) and other Apple SoCs + * + * Copyright (C) The Asahi Linux Contributors + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "dmaengine.h" +#include "virt-dma.h" + +#define NCHANNELS_MAX 0x80 + +#define REG_CPU_CONTROL 0x44 +#define CPU_CONTROL_RUN BIT(4) + +#define SIOMSG_DATA GENMASK(63, 32) +#define SIOMSG_TYPE GENMASK(23, 16) +#define SIOMSG_PARAM GENMASK(31, 24) +#define SIOMSG_TAG GENMASK(13, 8) +#define SIOMSG_EP GENMASK(7, 0) + +#define EP_SIO 0x20 + +#define MSG_START 0x2 +#define MSG_SETUP 0x3 +#define MSG_CONFIGURE 0x5 +#define MSG_ISSUE 0x6 +#define MSG_TERMINATE 0x8 +#define MSG_ACK 0x65 +#define MSG_NACK 0x66 +#define MSG_STARTED 0x67 +#define MSG_REPORT 0x68 + +#define SIO_CALL_TIMEOUT_MS 100 +#define SIO_SHMEM_SIZE 0x1000 +#define SIO_NO_DESC_SLOTS 64 + +/* + * There are two kinds of 'transaction descriptors' in play here. + * + * There's the struct sio_tx, and the struct dma_async_tx_descriptor embedded + * inside, which jointly represent a transaction to the dmaengine subsystem. + * At this time we only support those transactions to be cyclic. + * + * Then there are the coprocessor descriptors, which is what the coprocessor + * knows and understands. These don't seem to have a cyclic regime, so we can't + * map the dmaengine transaction on an exact coprocessor counterpart. Instead + * we continually queue up many coprocessor descriptors to implement a cyclic + * transaction. + * + * The number below is the maximum of how far ahead (how many) coprocessor + * descriptors we should be queuing up, per channel, for a cyclic transaction. + * Basically it's a made-up number. + */ +#define SIO_MAX_NINFLIGHT 4 + +struct sio_coproc_desc { + u32 pad1; + u32 flag; + u64 unk; + u64 iova; + u64 size; + u64 pad2; + u64 pad3; +} __packed; +static_assert(sizeof(struct sio_coproc_desc) == 48); + +struct sio_shmem_chan_config { + u32 datashape; + u32 timeout; + u32 fifo; + u32 threshold; + u32 limit; +} __packed; + +struct sio_data; +struct sio_tx; + +struct sio_chan { + unsigned int no; + struct sio_data *host; + struct virt_dma_chan vc; + struct work_struct terminate_wq; + + bool configured; + struct sio_shmem_chan_config cfg; + + struct sio_tx *current_tx; +}; + +#define SIO_NTAGS 16 + +typedef void (*sio_ack_callback)(struct sio_chan *, void *, bool); + +struct sio_data { + void __iomem *base; + struct dma_device dma; + struct device *dev; + struct apple_rtkit *rtk; + void *shmem; + struct sio_coproc_desc *shmem_desc_base; + unsigned long *desc_allocated; + + struct sio_tagdata { + DECLARE_BITMAP(allocated, SIO_NTAGS); + int last_tag; + + struct completion completions[SIO_NTAGS]; + bool atomic[SIO_NTAGS]; + bool acked[SIO_NTAGS]; + + sio_ack_callback ack_callback[SIO_NTAGS]; + void *cookie[SIO_NTAGS]; + } tags; + + int nchannels; + struct sio_chan channels[]; +}; + +struct sio_tx { + struct virt_dma_desc vd; + struct completion done; + + bool terminated; + size_t period_len; + int nperiods; + int ninflight; + int next; + + struct sio_coproc_desc *siodesc[]; +}; + +static int sio_send_siomsg(struct sio_data *sio, u64 msg); +static int sio_send_siomsg_atomic(struct sio_data *sio, u64 msg, + sio_ack_callback ack_callback, + void *cookie); +static int sio_call(struct sio_data *sio, u64 msg); + +static struct sio_chan *to_sio_chan(struct dma_chan *chan) +{ + return container_of(chan, struct sio_chan, vc.chan); +} + +static struct sio_tx *to_sio_tx(struct dma_async_tx_descriptor *tx) +{ + return container_of(tx, struct sio_tx, vd.tx); +} + +static int sio_alloc_tag(struct sio_data *sio) +{ + struct sio_tagdata *tags = &sio->tags; + int tag, i; + + /* + * Because tag number 0 is special, the usable tag range + * is 1...(SIO_NTAGS - 1). So, to pick the next usable tag, + * we do modulo (SIO_NTAGS - 1) *then* plus one. + */ + +#define SIO_USABLE_TAGS (SIO_NTAGS - 1) + tag = (READ_ONCE(tags->last_tag) % SIO_USABLE_TAGS) + 1; + + for (i = 0; i < SIO_USABLE_TAGS; i++) { + if (!test_and_set_bit(tag, tags->allocated)) + break; + + tag = (tag % SIO_USABLE_TAGS) + 1; + } + + WRITE_ONCE(tags->last_tag, tag); + + if (i < SIO_USABLE_TAGS) + return tag; + else + return -EBUSY; +#undef SIO_USABLE_TAGS +} + +static void sio_free_tag(struct sio_data *sio, int tag) +{ + struct sio_tagdata *tags = &sio->tags; + + if (WARN_ON(tag >= SIO_NTAGS)) + return; + + tags->atomic[tag] = false; + tags->ack_callback[tag] = NULL; + + WARN_ON(!test_and_clear_bit(tag, tags->allocated)); +} + +static void sio_set_tag_atomic(struct sio_data *sio, int tag, + sio_ack_callback ack_callback, + void *cookie) +{ + struct sio_tagdata *tags = &sio->tags; + + tags->atomic[tag] = true; + tags->ack_callback[tag] = ack_callback; + tags->cookie[tag] = cookie; +} + +static struct sio_coproc_desc *sio_alloc_desc(struct sio_data *sio) +{ + int i; + + for (i = 0; i < SIO_NO_DESC_SLOTS; i++) + if (!test_and_set_bit(i, sio->desc_allocated)) + return sio->shmem_desc_base + i; + + return NULL; +} + +static void sio_free_desc(struct sio_data *sio, struct sio_coproc_desc *desc) +{ + clear_bit(desc - sio->shmem_desc_base, sio->desc_allocated); +} + +static int sio_coproc_desc_slot(struct sio_data *sio, struct sio_coproc_desc *desc) +{ + return (desc - sio->shmem_desc_base) * 4; +} + +static enum dma_transfer_direction sio_chan_direction(int channo) +{ + /* Channel directions are fixed based on channel number */ + return (channo & 1) ? DMA_DEV_TO_MEM : DMA_MEM_TO_DEV; +} + +static void sio_tx_free(struct virt_dma_desc *vd) +{ + struct sio_data *sio = to_sio_chan(vd->tx.chan)->host; + struct sio_tx *siotx = to_sio_tx(&vd->tx); + int i; + + for (i = 0; i < siotx->nperiods; i++) + if (siotx->siodesc[i]) + sio_free_desc(sio, siotx->siodesc[i]); + kfree(siotx); +} + +static struct dma_async_tx_descriptor *sio_prep_dma_cyclic( + struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len, + size_t period_len, enum dma_transfer_direction direction, + unsigned long flags) +{ + struct sio_chan *siochan = to_sio_chan(chan); + struct sio_tx *siotx = NULL; + int i, nperiods = buf_len / period_len; + + if (direction != sio_chan_direction(siochan->no)) + return NULL; + + siotx = kzalloc(struct_size(siotx, siodesc, nperiods), GFP_NOWAIT); + if (!siotx) + return NULL; + + init_completion(&siotx->done); + siotx->period_len = period_len; + siotx->nperiods = nperiods; + + for (i = 0; i < nperiods; i++) { + struct sio_coproc_desc *d; + + siotx->siodesc[i] = d = sio_alloc_desc(siochan->host); + if (!d) { + siotx->vd.tx.chan = &siochan->vc.chan; + sio_tx_free(&siotx->vd); + return NULL; + } + + d->flag = 1; /* not sure what's up with this */ + d->iova = buf_addr + period_len * i; + d->size = period_len; + } + dma_wmb(); + + return vchan_tx_prep(&siochan->vc, &siotx->vd, flags); +} + +static enum dma_status sio_tx_status(struct dma_chan *chan, dma_cookie_t cookie, + struct dma_tx_state *txstate) +{ + struct sio_chan *siochan = to_sio_chan(chan); + struct virt_dma_desc *vd; + struct sio_tx *siotx; + enum dma_status ret; + unsigned long flags; + int periods_residue; + size_t residue; + + ret = dma_cookie_status(chan, cookie, txstate); + if (ret == DMA_COMPLETE || !txstate) + return ret; + + spin_lock_irqsave(&siochan->vc.lock, flags); + siotx = siochan->current_tx; + + if (siotx && siotx->vd.tx.cookie == cookie) { + ret = DMA_IN_PROGRESS; + periods_residue = siotx->next - siotx->ninflight; + while (periods_residue < 0) + periods_residue += siotx->nperiods; + residue = (siotx->nperiods - periods_residue) * siotx->period_len; + } else { + ret = DMA_IN_PROGRESS; + residue = 0; + vd = vchan_find_desc(&siochan->vc, cookie); + if (vd) { + siotx = to_sio_tx(&vd->tx); + residue = siotx->period_len * siotx->nperiods; + } + } + spin_unlock_irqrestore(&siochan->vc.lock, flags); + dma_set_residue(txstate, residue); + + return ret; +} + +static bool sio_fill_in_locked(struct sio_chan *siochan); + +static void sio_handle_issue_ack(struct sio_chan *siochan, void *cookie, bool ok) +{ + dma_cookie_t tx_cookie = (unsigned long) cookie; + unsigned long flags; + struct sio_tx *tx; + + if (!ok) { + dev_err(siochan->host->dev, "nacked issue on chan %d\n", siochan->no); + return; + } + + spin_lock_irqsave(&siochan->vc.lock, flags); + if (!siochan->current_tx || tx_cookie != siochan->current_tx->vd.tx.cookie || + siochan->current_tx->terminated) + goto out; + + tx = siochan->current_tx; + tx->next = (tx->next + 1) % tx->nperiods; + tx->ninflight++; + sio_fill_in_locked(siochan); + +out: + spin_unlock_irqrestore(&siochan->vc.lock, flags); +} + +static bool sio_fill_in_locked(struct sio_chan *siochan) +{ + struct sio_data *sio = siochan->host; + struct sio_tx *tx = siochan->current_tx; + struct sio_coproc_desc *d = tx->siodesc[tx->next]; + int ret; + + if (tx->ninflight >= SIO_MAX_NINFLIGHT || tx->terminated) + return false; + + static_assert(sizeof(dma_cookie_t) <= sizeof(void *)); + ret = sio_send_siomsg_atomic(sio, FIELD_PREP(SIOMSG_EP, siochan->no) | + FIELD_PREP(SIOMSG_TYPE, MSG_ISSUE) | + FIELD_PREP(SIOMSG_DATA, sio_coproc_desc_slot(sio, d)), + sio_handle_issue_ack, (void *) (uintptr_t) tx->vd.tx.cookie); + if (ret < 0) + dev_err_ratelimited(sio->dev, "can't issue on chan %d ninflight %d: %d\n", + siochan->no, tx->ninflight, ret); + return true; +} + +static void sio_update_current_tx_locked(struct sio_chan *siochan) +{ + struct virt_dma_desc *vd = vchan_next_desc(&siochan->vc); + + if (vd && !siochan->current_tx) { + list_del(&vd->node); + siochan->current_tx = to_sio_tx(&vd->tx); + sio_fill_in_locked(siochan); + } +} + +static void sio_issue_pending(struct dma_chan *chan) +{ + struct sio_chan *siochan = to_sio_chan(chan); + unsigned long flags; + + spin_lock_irqsave(&siochan->vc.lock, flags); + vchan_issue_pending(&siochan->vc); + sio_update_current_tx_locked(siochan); + spin_unlock_irqrestore(&siochan->vc.lock, flags); +} + +static int sio_terminate_all(struct dma_chan *chan) +{ + struct sio_chan *siochan = to_sio_chan(chan); + unsigned long flags; + LIST_HEAD(to_free); + + spin_lock_irqsave(&siochan->vc.lock, flags); + if (siochan->current_tx && !siochan->current_tx->terminated) { + dma_cookie_complete(&siochan->current_tx->vd.tx); + siochan->current_tx->terminated = true; + schedule_work(&siochan->terminate_wq); + } + vchan_get_all_descriptors(&siochan->vc, &to_free); + spin_unlock_irqrestore(&siochan->vc.lock, flags); + + vchan_dma_desc_free_list(&siochan->vc, &to_free); + + return 0; +} + +static void sio_terminate_work(struct work_struct *wq) +{ + struct sio_chan *siochan = container_of(wq, struct sio_chan, terminate_wq); + struct sio_tx *tx; + unsigned long flags; + int ret; + + spin_lock_irqsave(&siochan->vc.lock, flags); + tx = siochan->current_tx; + spin_unlock_irqrestore(&siochan->vc.lock, flags); + + if (WARN_ON(!tx)) + return; + + ret = sio_call(siochan->host, FIELD_PREP(SIOMSG_EP, siochan->no) | + FIELD_PREP(SIOMSG_TYPE, MSG_TERMINATE)); + if (ret < 0) + dev_err(siochan->host->dev, "terminate call on chan %d failed: %d\n", + siochan->no, ret); + + ret = wait_for_completion_timeout(&tx->done, msecs_to_jiffies(500)); + if (!ret) + dev_err(siochan->host->dev, "terminate descriptor wait timed out\n"); + + tasklet_kill(&siochan->vc.task); + + spin_lock_irqsave(&siochan->vc.lock, flags); + WARN_ON(siochan->current_tx != tx); + siochan->current_tx = NULL; + sio_update_current_tx_locked(siochan); + spin_unlock_irqrestore(&siochan->vc.lock, flags); + + sio_tx_free(&tx->vd); +} + +static void sio_synchronize(struct dma_chan *chan) +{ + struct sio_chan *siochan = to_sio_chan(chan); + + flush_work(&siochan->terminate_wq); +} + +static void sio_free_chan_resources(struct dma_chan *chan) +{ + sio_terminate_all(chan); + sio_synchronize(chan); + vchan_free_chan_resources(&to_sio_chan(chan)->vc); +} + +static struct dma_chan *sio_dma_of_xlate(struct of_phandle_args *dma_spec, + struct of_dma *ofdma) +{ + struct sio_data *sio = (struct sio_data *) ofdma->of_dma_data; + unsigned int index = dma_spec->args[0]; + + if (dma_spec->args_count != 1 || index >= sio->nchannels) + return ERR_PTR(-EINVAL); + + return dma_get_slave_channel(&sio->channels[index].vc.chan); +} + +static void sio_rtk_crashed(void *cookie, const void *crashlog, size_t crashlog_size) +{ + struct sio_data *sio = cookie; + + dev_err(sio->dev, "SIO down (crashed)"); +} + +static void sio_process_report(struct sio_chan *siochan) +{ + unsigned long flags; + + spin_lock_irqsave(&siochan->vc.lock, flags); + if (siochan->current_tx) { + struct sio_tx *tx = siochan->current_tx; + + if (tx->ninflight) + tx->ninflight--; + vchan_cyclic_callback(&tx->vd); + if (!sio_fill_in_locked(siochan) && !tx->ninflight) + complete(&tx->done); + } + spin_unlock_irqrestore(&siochan->vc.lock, flags); +} + +static void sio_recv_msg(void *cookie, u8 ep, u64 msg) +{ + struct sio_data *sio = cookie; + struct sio_tagdata *tags = &sio->tags; + u32 data; + u8 type, tag, sioep; + + if (ep != EP_SIO) + goto unknown; + + data = FIELD_GET(SIOMSG_DATA, msg); + // param = FIELD_GET(SIOMSG_PARAM, msg); + type = FIELD_GET(SIOMSG_TYPE, msg); + tag = FIELD_GET(SIOMSG_TAG, msg); + sioep = FIELD_GET(SIOMSG_EP, msg); + + switch (type) { + case MSG_STARTED: + dev_info(sio->dev, "SIO protocol v%u\n", data); + type = MSG_ACK; /* Pretend this is an ACK */ + fallthrough; + case MSG_ACK: + case MSG_NACK: + if (WARN_ON(tag >= SIO_NTAGS)) + break; + + if (tags->atomic[tag]) { + sio_ack_callback callback = tags->ack_callback[tag]; + + if (callback && !WARN_ON(sioep >= sio->nchannels)) + callback(&sio->channels[sioep], + tags->cookie[tag], type == MSG_ACK); + if (type == MSG_NACK) + dev_err(sio->dev, "got a NACK on channel %d\n", sioep); + sio_free_tag(sio, tag); + } else { + tags->acked[tag] = (type == MSG_ACK); + complete(&tags->completions[tag]); + } + break; + + case MSG_REPORT: + if (WARN_ON(sioep >= sio->nchannels)) + break; + + sio_process_report(&sio->channels[sioep]); + break; + + default: + goto unknown; + } + return; + +unknown: + dev_warn(sio->dev, "received unknown message: ep %x data %016llx\n", + ep, msg); +} + +static int _sio_send_siomsg(struct sio_data *sio, u64 msg, bool atomic, + sio_ack_callback ack_callback, void *cookie) +{ + int tag, ret; + + tag = sio_alloc_tag(sio); + if (tag < 0) + return tag; + + if (atomic) + sio_set_tag_atomic(sio, tag, ack_callback, cookie); + else + reinit_completion(&sio->tags.completions[tag]); + + msg &= ~SIOMSG_TAG; + msg |= FIELD_PREP(SIOMSG_TAG, tag); + ret = apple_rtkit_send_message(sio->rtk, EP_SIO, msg, NULL, + atomic); + if (ret < 0) { + sio_free_tag(sio, tag); + return ret; + } + + return tag; +} + +static int sio_send_siomsg(struct sio_data *sio, u64 msg) +{ + return _sio_send_siomsg(sio, msg, false, NULL, NULL); +} + +static int sio_send_siomsg_atomic(struct sio_data *sio, u64 msg, + sio_ack_callback ack_callback, + void *cookie) +{ + return _sio_send_siomsg(sio, msg, true, ack_callback, cookie); +} + +static int sio_call(struct sio_data *sio, u64 msg) +{ + int tag, ret; + + tag = sio_send_siomsg(sio, msg); + if (tag < 0) + return tag; + + ret = wait_for_completion_timeout(&sio->tags.completions[tag], + msecs_to_jiffies(SIO_CALL_TIMEOUT_MS)); + if (!ret) { + dev_warn(sio->dev, "call %8llx timed out\n", msg); + sio_free_tag(sio, tag); + return -ETIME; + } + + ret = sio->tags.acked[tag]; + sio_free_tag(sio, tag); + + return ret; +} + +static const struct apple_rtkit_ops sio_rtkit_ops = { + .crashed = sio_rtk_crashed, + .recv_message = sio_recv_msg, +}; + +static int sio_device_config(struct dma_chan *chan, + struct dma_slave_config *config) +{ + struct sio_chan *siochan = to_sio_chan(chan); + struct sio_data *sio = siochan->host; + bool is_tx = sio_chan_direction(siochan->no) == DMA_MEM_TO_DEV; + struct sio_shmem_chan_config *cfg_shmem = sio->shmem; + struct sio_shmem_chan_config cfg; + int ret; + + switch (is_tx ? config->dst_addr_width : config->src_addr_width) { + case DMA_SLAVE_BUSWIDTH_1_BYTE: + cfg.datashape = 0; + break; + case DMA_SLAVE_BUSWIDTH_2_BYTES: + cfg.datashape = 1; + break; + case DMA_SLAVE_BUSWIDTH_4_BYTES: + cfg.datashape = 2; + break; + default: + return -EINVAL; + } + + cfg.timeout = 0; + cfg.fifo = 0x800; + cfg.limit = 0x800; + cfg.threshold = 0x800; + + /* + * Dmaengine prescribes we ought to apply the new configuration only + * to newly-queued descriptors. + * + * To comply with dmaengine's interface we take the lazy path here: + * we apply the configuration right away, we only allow the channel + * to be configured once, which means subsequent calls to `device_config` + * either return -EBUSY if the configuration differs, or they are + * a no-op if the configuration is the same as the starting one. + * + * This is the reasonable thing to do given that these sio channels + * are tied to fixed peripherals, and what's more given that the + * only planned consumer of this dmaengine driver in the kernel is + * diplayport audio support, where the DMA configuration is fixed, + * and no more than a single descriptor (a cyclic one) gets ever issued + * at the same time. + * + * The code complexity cost of tracking to which descriptor + * the configuration relates would be significant here, especially + * since we need to do a non-atomic operation to apply it (a call to + * the coprocessor) and dmaengine has its bunch of atomicity + * restrictions. And this complexity would be for naught since it + * doesn't even get exercised by the only planned consumer. + */ + if (siochan->configured && memcmp(&siochan->cfg, &cfg, sizeof(cfg))) + return -EBUSY; + + *cfg_shmem = cfg; + dma_wmb(); + + ret = sio_call(sio, FIELD_PREP(SIOMSG_TYPE, MSG_CONFIGURE) | + FIELD_PREP(SIOMSG_EP, siochan->no)); + + if (ret == 1) + ret = 0; + else if (ret == 0) + ret = -EINVAL; + + if (ret == 0) { + siochan->configured = true; + siochan->cfg = cfg; + } + + return ret; +} + +static int sio_alloc_shmem(struct sio_data *sio) +{ + dma_addr_t iova; + int err; + + sio->shmem = dma_alloc_coherent(sio->dev, SIO_SHMEM_SIZE, + &iova, GFP_KERNEL | __GFP_ZERO); + if (!sio->shmem) + return -ENOMEM; + + sio->shmem_desc_base = (struct sio_coproc_desc *) (sio->shmem + 56); + sio->desc_allocated = devm_kzalloc(sio->dev, SIO_NO_DESC_SLOTS / 32, + GFP_KERNEL); + if (!sio->desc_allocated) + return -ENOMEM; + + err = sio_call(sio, FIELD_PREP(SIOMSG_TYPE, MSG_SETUP) | + FIELD_PREP(SIOMSG_PARAM, 1) | + FIELD_PREP(SIOMSG_DATA, iova >> 12)); + if (err != 1) { + if (err == 0) + err = -EINVAL; + return err; + } + + err = sio_call(sio, FIELD_PREP(SIOMSG_TYPE, MSG_SETUP) | + FIELD_PREP(SIOMSG_PARAM, 2) | + FIELD_PREP(SIOMSG_DATA, SIO_SHMEM_SIZE)); + if (err != 1) { + if (err == 0) + err = -EINVAL; + return err; + } + + return 0; +} + +static int sio_send_dt_params(struct sio_data *sio) +{ + struct device_node *np = sio->dev->of_node; + const char *propname = "apple,sio-firmware-params"; + int nparams, err, i; + + nparams = of_property_count_u32_elems(np, propname); + if (nparams < 0) { + err = nparams; + goto badprop; + } + + for (i = 0; i < nparams / 2; i++) { + u32 key, val; + + err = of_property_read_u32_index(np, propname, 2 * i, &key); + if (err) + goto badprop; + err = of_property_read_u32_index(np, propname, 2 * i + 1, &val); + if (err) + goto badprop; + + err = sio_call(sio, FIELD_PREP(SIOMSG_TYPE, MSG_SETUP) | + FIELD_PREP(SIOMSG_PARAM, key & 0xff) | + FIELD_PREP(SIOMSG_EP, key >> 8) | + FIELD_PREP(SIOMSG_DATA, val)); + if (err < 1) { + if (err == 0) + err = -ENXIO; + return dev_err_probe(sio->dev, err, "sending SIO parameter %#x value %#x\n", + key, val); + } + } + + return 0; + +badprop: + return dev_err_probe(sio->dev, err, "failed to read '%s'\n", propname); +} + +static int sio_probe(struct platform_device *pdev) +{ + struct device_node *np = pdev->dev.of_node; + struct sio_data *sio; + struct dma_device *dma; + int nchannels; + int err, i; + + err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(42)); + if (err) + return dev_err_probe(&pdev->dev, err, "Failed to set DMA mask\n"); + + err = of_property_read_u32(np, "dma-channels", &nchannels); + if (err || nchannels > NCHANNELS_MAX) + return dev_err_probe(&pdev->dev, -EINVAL, + "missing or invalid dma-channels property\n"); + + sio = devm_kzalloc(&pdev->dev, struct_size(sio, channels, nchannels), GFP_KERNEL); + if (!sio) + return -ENOMEM; + + platform_set_drvdata(pdev, sio); + sio->dev = &pdev->dev; + sio->nchannels = nchannels; + + sio->base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(sio->base)) + return PTR_ERR(sio->base); + + pm_runtime_get_noresume(&pdev->dev); + pm_runtime_set_active(&pdev->dev); + err = devm_pm_runtime_enable(&pdev->dev); + if (err < 0) + return dev_err_probe(&pdev->dev, err, + "pm_runtime_enable failed: %d\n", err); + + sio->rtk = devm_apple_rtkit_init(&pdev->dev, sio, NULL, 0, &sio_rtkit_ops); + if (IS_ERR(sio->rtk)) { + err = PTR_ERR(sio->rtk); + dev_err(&pdev->dev, "couldn't initialize rtkit\n"); + goto rpm_put; + } + for (i = 1; i < SIO_NTAGS; i++) + init_completion(&sio->tags.completions[i]); + + dma = &sio->dma; + dma_cap_set(DMA_PRIVATE, dma->cap_mask); + dma_cap_set(DMA_CYCLIC, dma->cap_mask); + + dma->dev = &pdev->dev; + dma->device_free_chan_resources = sio_free_chan_resources; + dma->device_tx_status = sio_tx_status; + dma->device_issue_pending = sio_issue_pending; + dma->device_terminate_all = sio_terminate_all; + dma->device_synchronize = sio_synchronize; + dma->device_prep_dma_cyclic = sio_prep_dma_cyclic; + dma->device_config = sio_device_config; + + dma->directions = BIT(DMA_MEM_TO_DEV); + dma->residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT; + dma->dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | + BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | + BIT(DMA_SLAVE_BUSWIDTH_4_BYTES); + + INIT_LIST_HEAD(&dma->channels); + for (i = 0; i < nchannels; i++) { + struct sio_chan *siochan = &sio->channels[i]; + + siochan->host = sio; + siochan->no = i; + siochan->vc.desc_free = sio_tx_free; + INIT_WORK(&siochan->terminate_wq, sio_terminate_work); + vchan_init(&siochan->vc, dma); + } + + writel(CPU_CONTROL_RUN, sio->base + REG_CPU_CONTROL); + + err = apple_rtkit_boot(sio->rtk); + if (err) + return dev_err_probe(&pdev->dev, err, "SIO did not boot\n"); + + err = apple_rtkit_start_ep(sio->rtk, EP_SIO); + if (err) + return dev_err_probe(&pdev->dev, err, "starting SIO endpoint\n"); + + err = sio_call(sio, FIELD_PREP(SIOMSG_TYPE, MSG_START)); + if (err < 1) { + if (err == 0) + err = -ENXIO; + return dev_err_probe(&pdev->dev, err, "starting SIO service\n"); + } + + err = sio_send_dt_params(sio); + if (err < 0) + return dev_err_probe(&pdev->dev, err, "failed to send boot-up parameters\n"); + + err = sio_alloc_shmem(sio); + if (err < 0) + return err; + + err = dma_async_device_register(&sio->dma); + if (err) + return dev_err_probe(&pdev->dev, err, "failed to register DMA device\n"); + + err = of_dma_controller_register(pdev->dev.of_node, sio_dma_of_xlate, sio); + if (err) { + dma_async_device_unregister(&sio->dma); + return dev_err_probe(&pdev->dev, err, "failed to register with OF\n"); + } + +rpm_put: + pm_runtime_put(&pdev->dev); + + return err; +} + +static void sio_remove(struct platform_device *pdev) +{ + struct sio_data *sio = platform_get_drvdata(pdev); + + of_dma_controller_free(pdev->dev.of_node); + dma_async_device_unregister(&sio->dma); +} + +static const struct of_device_id sio_of_match[] = { + { .compatible = "apple,sio", }, + { } +}; +MODULE_DEVICE_TABLE(of, sio_of_match); + +static __maybe_unused int sio_suspend(struct device *dev) +{ + /* + * TODO: SIO coproc sleep state + */ + return 0; +} + +static __maybe_unused int sio_resume(struct device *dev) +{ + return 0; +} + +static DEFINE_RUNTIME_DEV_PM_OPS(sio_pm_ops, sio_suspend, sio_resume, NULL); + +static struct platform_driver apple_sio_driver = { + .driver = { + .name = "apple-sio", + .of_match_table = sio_of_match, + .pm = pm_ptr(&sio_pm_ops), + }, + .probe = sio_probe, + .remove = sio_remove, +}; +module_platform_driver(apple_sio_driver); + +MODULE_AUTHOR("Martin Povišer "); +MODULE_DESCRIPTION("Driver for SIO coprocessor on Apple SoCs"); +MODULE_LICENSE("Dual MIT/GPL"); diff --git a/drivers/dma/dma-axi-dmac.c b/drivers/dma/dma-axi-dmac.c index 5b06b0dc67ee12..0f25f6d8ae71fa 100644 --- a/drivers/dma/dma-axi-dmac.c +++ b/drivers/dma/dma-axi-dmac.c @@ -233,11 +233,9 @@ static void axi_dmac_start_transfer(struct axi_dmac_chan *chan) unsigned int flags = 0; unsigned int val; - if (!chan->hw_sg) { - val = axi_dmac_read(dmac, AXI_DMAC_REG_START_TRANSFER); - if (val) /* Queue is full, wait for the next SOT IRQ */ - return; - } + val = axi_dmac_read(dmac, AXI_DMAC_REG_START_TRANSFER); + if (val) /* Queue is full, wait for the next SOT IRQ */ + return; desc = chan->next_desc; @@ -247,6 +245,7 @@ static void axi_dmac_start_transfer(struct axi_dmac_chan *chan) return; list_move_tail(&vdesc->node, &chan->active_descs); desc = to_axi_dmac_desc(vdesc); + chan->next_desc = desc; } sg = &desc->sg[desc->num_submitted]; @@ -265,8 +264,6 @@ static void axi_dmac_start_transfer(struct axi_dmac_chan *chan) else chan->next_desc = NULL; flags |= AXI_DMAC_FLAG_LAST; - } else { - chan->next_desc = desc; } sg->hw->id = axi_dmac_read(dmac, AXI_DMAC_REG_TRANSFER_ID); diff --git a/drivers/dma/dw-edma/dw-hdma-v0-core.c b/drivers/dma/dw-edma/dw-hdma-v0-core.c index e3f8db4fe909a1..ce8f7254bab21b 100644 --- a/drivers/dma/dw-edma/dw-hdma-v0-core.c +++ b/drivers/dma/dw-edma/dw-hdma-v0-core.c @@ -252,10 +252,10 @@ static void dw_hdma_v0_core_start(struct dw_edma_chunk *chunk, bool first) lower_32_bits(chunk->ll_region.paddr)); SET_CH_32(dw, chan->dir, chan->id, llp.msb, upper_32_bits(chunk->ll_region.paddr)); + /* Set consumer cycle */ + SET_CH_32(dw, chan->dir, chan->id, cycle_sync, + HDMA_V0_CONSUMER_CYCLE_STAT | HDMA_V0_CONSUMER_CYCLE_BIT); } - /* Set consumer cycle */ - SET_CH_32(dw, chan->dir, chan->id, cycle_sync, - HDMA_V0_CONSUMER_CYCLE_STAT | HDMA_V0_CONSUMER_CYCLE_BIT); dw_hdma_v0_sync_ll_data(chunk); diff --git a/drivers/dma/fsl-edma-main.c b/drivers/dma/fsl-edma-main.c index a753b7cbfa7a33..b596baa0a182e3 100644 --- a/drivers/dma/fsl-edma-main.c +++ b/drivers/dma/fsl-edma-main.c @@ -317,10 +317,8 @@ static struct dma_chan *fsl_edma3_xlate(struct of_phandle_args *dma_spec, return NULL; i = fsl_chan - fsl_edma->chans; - fsl_chan->priority = dma_spec->args[1]; - fsl_chan->is_rxchan = dma_spec->args[2] & FSL_EDMA_RX; - fsl_chan->is_remote = dma_spec->args[2] & FSL_EDMA_REMOTE; - fsl_chan->is_multi_fifo = dma_spec->args[2] & FSL_EDMA_MULTI_FIFO; + if (!b_chmux && i != dma_spec->args[0]) + continue; if ((dma_spec->args[2] & FSL_EDMA_EVEN_CH) && (i & 0x1)) continue; @@ -328,17 +326,15 @@ static struct dma_chan *fsl_edma3_xlate(struct of_phandle_args *dma_spec, if ((dma_spec->args[2] & FSL_EDMA_ODD_CH) && !(i & 0x1)) continue; - if (!b_chmux && i == dma_spec->args[0]) { - chan = dma_get_slave_channel(chan); - chan->device->privatecnt++; - return chan; - } else if (b_chmux && !fsl_chan->srcid) { - /* if controller support channel mux, choose a free channel */ - chan = dma_get_slave_channel(chan); - chan->device->privatecnt++; - fsl_chan->srcid = dma_spec->args[0]; - return chan; - } + fsl_chan->srcid = dma_spec->args[0]; + fsl_chan->priority = dma_spec->args[1]; + fsl_chan->is_rxchan = dma_spec->args[2] & FSL_EDMA_RX; + fsl_chan->is_remote = dma_spec->args[2] & FSL_EDMA_REMOTE; + fsl_chan->is_multi_fifo = dma_spec->args[2] & FSL_EDMA_MULTI_FIFO; + + chan = dma_get_slave_channel(chan); + chan->device->privatecnt++; + return chan; } return NULL; } @@ -915,7 +911,6 @@ static void fsl_edma_remove(struct platform_device *pdev) of_dma_controller_free(np); dma_async_device_unregister(&fsl_edma->dma_dev); fsl_edma_cleanup_vchan(&fsl_edma->dma_dev); - fsl_disable_clocks(fsl_edma, fsl_edma->drvdata->dmamuxs); } static int fsl_edma_suspend_late(struct device *dev) diff --git a/drivers/dma/idxd/cdev.c b/drivers/dma/idxd/cdev.c index 7e4715f9277327..4105688cf3f060 100644 --- a/drivers/dma/idxd/cdev.c +++ b/drivers/dma/idxd/cdev.c @@ -158,11 +158,7 @@ static const struct device_type idxd_cdev_file_type = { static void idxd_cdev_dev_release(struct device *dev) { struct idxd_cdev *idxd_cdev = dev_to_cdev(dev); - struct idxd_cdev_context *cdev_ctx; - struct idxd_wq *wq = idxd_cdev->wq; - cdev_ctx = &ictx[wq->idxd->data->type]; - ida_free(&cdev_ctx->minor_ida, idxd_cdev->minor); kfree(idxd_cdev); } @@ -582,11 +578,15 @@ int idxd_wq_add_cdev(struct idxd_wq *wq) void idxd_wq_del_cdev(struct idxd_wq *wq) { + struct idxd_cdev_context *cdev_ctx; struct idxd_cdev *idxd_cdev; idxd_cdev = wq->idxd_cdev; wq->idxd_cdev = NULL; cdev_device_del(&idxd_cdev->cdev, cdev_dev(idxd_cdev)); + + cdev_ctx = &ictx[wq->idxd->data->type]; + ida_free(&cdev_ctx->minor_ida, idxd_cdev->minor); put_device(cdev_dev(idxd_cdev)); } diff --git a/drivers/dma/idxd/device.c b/drivers/dma/idxd/device.c index c2cdf41b6e5764..f4b134c8651635 100644 --- a/drivers/dma/idxd/device.c +++ b/drivers/dma/idxd/device.c @@ -175,6 +175,7 @@ void idxd_wq_free_resources(struct idxd_wq *wq) free_descs(wq); dma_free_coherent(dev, wq->compls_size, wq->compls, wq->compls_addr); sbitmap_queue_free(&wq->sbq); + wq->type = IDXD_WQT_NONE; } EXPORT_SYMBOL_NS_GPL(idxd_wq_free_resources, "IDXD"); @@ -382,7 +383,6 @@ static void idxd_wq_disable_cleanup(struct idxd_wq *wq) lockdep_assert_held(&wq->wq_lock); wq->state = IDXD_WQ_DISABLED; memset(wq->wqcfg, 0, idxd->wqcfg_size); - wq->type = IDXD_WQT_NONE; wq->threshold = 0; wq->priority = 0; wq->enqcmds_retries = IDXD_ENQCMDS_RETRIES; @@ -830,8 +830,7 @@ static void idxd_device_evl_free(struct idxd_device *idxd) struct device *dev = &idxd->pdev->dev; struct idxd_evl *evl = idxd->evl; - gencfg.bits = ioread32(idxd->reg_base + IDXD_GENCFG_OFFSET); - if (!gencfg.evl_en) + if (!evl) return; mutex_lock(&evl->lock); @@ -1122,7 +1121,11 @@ int idxd_device_config(struct idxd_device *idxd) { int rc; - lockdep_assert_held(&idxd->dev_lock); + guard(spinlock)(&idxd->dev_lock); + + if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) + return 0; + rc = idxd_wqs_setup(idxd); if (rc < 0) return rc; @@ -1449,11 +1452,7 @@ int idxd_drv_enable_wq(struct idxd_wq *wq) } } - rc = 0; - spin_lock(&idxd->dev_lock); - if (test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) - rc = idxd_device_config(idxd); - spin_unlock(&idxd->dev_lock); + rc = idxd_device_config(idxd); if (rc < 0) { dev_dbg(dev, "Writing wq %d config failed: %d\n", wq->id, rc); goto err; @@ -1528,7 +1527,6 @@ void idxd_drv_disable_wq(struct idxd_wq *wq) idxd_wq_reset(wq); idxd_wq_free_resources(wq); percpu_ref_exit(&wq->wq_active); - wq->type = IDXD_WQT_NONE; wq->client_count = 0; } EXPORT_SYMBOL_NS_GPL(idxd_drv_disable_wq, "IDXD"); @@ -1549,10 +1547,7 @@ int idxd_device_drv_probe(struct idxd_dev *idxd_dev) } /* Device configuration */ - spin_lock(&idxd->dev_lock); - if (test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) - rc = idxd_device_config(idxd); - spin_unlock(&idxd->dev_lock); + rc = idxd_device_config(idxd); if (rc < 0) return -ENXIO; diff --git a/drivers/dma/idxd/init.c b/drivers/dma/idxd/init.c index 2acc34b3daff8e..afba88f9c3e437 100644 --- a/drivers/dma/idxd/init.c +++ b/drivers/dma/idxd/init.c @@ -962,7 +962,8 @@ static void idxd_device_config_restore(struct idxd_device *idxd, idxd->rdbuf_limit = idxd_saved->saved_idxd.rdbuf_limit; - idxd->evl->size = saved_evl->size; + if (idxd->evl) + idxd->evl->size = saved_evl->size; for (i = 0; i < idxd->max_groups; i++) { struct idxd_group *saved_group, *group; @@ -1093,12 +1094,10 @@ static void idxd_reset_done(struct pci_dev *pdev) idxd_device_config_restore(idxd, idxd->idxd_saved); /* Re-configure IDXD device if allowed. */ - if (test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) { - rc = idxd_device_config(idxd); - if (rc < 0) { - dev_err(dev, "HALT: %s config fails\n", idxd_name); - goto out; - } + rc = idxd_device_config(idxd); + if (rc < 0) { + dev_err(dev, "HALT: %s config fails\n", idxd_name); + goto out; } /* Bind IDXD device to driver. */ @@ -1136,6 +1135,7 @@ static void idxd_reset_done(struct pci_dev *pdev) } out: kfree(idxd->idxd_saved); + idxd->idxd_saved = NULL; } static const struct pci_error_handlers idxd_error_handler = { diff --git a/drivers/dma/idxd/submit.c b/drivers/dma/idxd/submit.c index 6db1c5fcedc580..03217041b8b3eb 100644 --- a/drivers/dma/idxd/submit.c +++ b/drivers/dma/idxd/submit.c @@ -138,7 +138,7 @@ static void llist_abort_desc(struct idxd_wq *wq, struct idxd_irq_entry *ie, */ list_for_each_entry_safe(d, t, &flist, list) { list_del_init(&d->list); - idxd_dma_complete_txd(found, IDXD_COMPLETE_ABORT, true, + idxd_dma_complete_txd(d, IDXD_COMPLETE_ABORT, true, NULL, NULL); } } diff --git a/drivers/dma/idxd/sysfs.c b/drivers/dma/idxd/sysfs.c index 9f0701021af0e6..cdd7a59140d90c 100644 --- a/drivers/dma/idxd/sysfs.c +++ b/drivers/dma/idxd/sysfs.c @@ -1812,6 +1812,7 @@ static void idxd_conf_device_release(struct device *dev) { struct idxd_device *idxd = confdev_to_idxd(dev); + destroy_workqueue(idxd->wq); kfree(idxd->groups); bitmap_free(idxd->wq_enable_map); kfree(idxd->wqs); diff --git a/drivers/dma/mediatek/mtk-uart-apdma.c b/drivers/dma/mediatek/mtk-uart-apdma.c index 08e15177427b94..96c18c815f1df6 100644 --- a/drivers/dma/mediatek/mtk-uart-apdma.c +++ b/drivers/dma/mediatek/mtk-uart-apdma.c @@ -41,7 +41,7 @@ #define VFF_STOP_CLR_B 0 #define VFF_EN_CLR_B 0 #define VFF_INT_EN_CLR_B 0 -#define VFF_4G_SUPPORT_CLR_B 0 +#define VFF_ADDR2_CLR_B 0 /* * interrupt trigger level for tx @@ -72,7 +72,7 @@ /* TX: the buffer size SW can write. RX: the buffer size HW can write. */ #define VFF_LEFT_SIZE 0x40 #define VFF_DEBUG_STATUS 0x50 -#define VFF_4G_SUPPORT 0x54 +#define VFF_ADDR2 0x54 struct mtk_uart_apdmadev { struct dma_device ddev; @@ -149,7 +149,7 @@ static void mtk_uart_apdma_start_tx(struct mtk_chan *c) mtk_uart_apdma_write(c, VFF_INT_FLAG, VFF_TX_INT_CLR_B); if (mtkd->support_33bits) - mtk_uart_apdma_write(c, VFF_4G_SUPPORT, VFF_4G_EN_B); + mtk_uart_apdma_write(c, VFF_ADDR2, upper_32_bits(d->addr)); } mtk_uart_apdma_write(c, VFF_EN, VFF_EN_B); @@ -192,7 +192,7 @@ static void mtk_uart_apdma_start_rx(struct mtk_chan *c) mtk_uart_apdma_write(c, VFF_INT_FLAG, VFF_RX_INT_CLR_B); if (mtkd->support_33bits) - mtk_uart_apdma_write(c, VFF_4G_SUPPORT, VFF_4G_EN_B); + mtk_uart_apdma_write(c, VFF_ADDR2, upper_32_bits(d->addr)); } mtk_uart_apdma_write(c, VFF_INT_EN, VFF_RX_INT_EN_B); @@ -298,7 +298,7 @@ static int mtk_uart_apdma_alloc_chan_resources(struct dma_chan *chan) } if (mtkd->support_33bits) - mtk_uart_apdma_write(c, VFF_4G_SUPPORT, VFF_4G_SUPPORT_CLR_B); + mtk_uart_apdma_write(c, VFF_ADDR2, VFF_ADDR2_CLR_B); err_pm: pm_runtime_put_noidle(mtkd->ddev.dev); diff --git a/drivers/dma/sh/rz-dmac.c b/drivers/dma/sh/rz-dmac.c index 9e5f088355e226..818d1ef6f0bf94 100644 --- a/drivers/dma/sh/rz-dmac.c +++ b/drivers/dma/sh/rz-dmac.c @@ -10,6 +10,7 @@ */ #include +#include #include #include #include @@ -297,13 +298,10 @@ static void rz_dmac_disable_hw(struct rz_dmac_chan *channel) { struct dma_chan *chan = &channel->vc.chan; struct rz_dmac *dmac = to_rz_dmac(chan->device); - unsigned long flags; dev_dbg(dmac->dev, "%s channel %d\n", __func__, channel->index); - local_irq_save(flags); rz_dmac_ch_writel(channel, CHCTRL_DEFAULT, CHCTRL, 1); - local_irq_restore(flags); } static void rz_dmac_set_dmars_register(struct rz_dmac *dmac, int nr, u32 dmars) @@ -448,6 +446,7 @@ static int rz_dmac_alloc_chan_resources(struct dma_chan *chan) if (!desc) break; + /* No need to lock. This is called only for the 1st client. */ list_add_tail(&desc->node, &channel->ld_free); channel->descs_allocated++; } @@ -503,18 +502,21 @@ rz_dmac_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, dev_dbg(dmac->dev, "%s channel: %d src=0x%pad dst=0x%pad len=%zu\n", __func__, channel->index, &src, &dest, len); - if (list_empty(&channel->ld_free)) - return NULL; + scoped_guard(spinlock_irqsave, &channel->vc.lock) { + if (list_empty(&channel->ld_free)) + return NULL; - desc = list_first_entry(&channel->ld_free, struct rz_dmac_desc, node); + desc = list_first_entry(&channel->ld_free, struct rz_dmac_desc, node); - desc->type = RZ_DMAC_DESC_MEMCPY; - desc->src = src; - desc->dest = dest; - desc->len = len; - desc->direction = DMA_MEM_TO_MEM; + desc->type = RZ_DMAC_DESC_MEMCPY; + desc->src = src; + desc->dest = dest; + desc->len = len; + desc->direction = DMA_MEM_TO_MEM; + + list_move_tail(channel->ld_free.next, &channel->ld_queue); + } - list_move_tail(channel->ld_free.next, &channel->ld_queue); return vchan_tx_prep(&channel->vc, &desc->vd, flags); } @@ -530,27 +532,29 @@ rz_dmac_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, int dma_length = 0; int i = 0; - if (list_empty(&channel->ld_free)) - return NULL; + scoped_guard(spinlock_irqsave, &channel->vc.lock) { + if (list_empty(&channel->ld_free)) + return NULL; - desc = list_first_entry(&channel->ld_free, struct rz_dmac_desc, node); + desc = list_first_entry(&channel->ld_free, struct rz_dmac_desc, node); - for_each_sg(sgl, sg, sg_len, i) { - dma_length += sg_dma_len(sg); - } + for_each_sg(sgl, sg, sg_len, i) + dma_length += sg_dma_len(sg); - desc->type = RZ_DMAC_DESC_SLAVE_SG; - desc->sg = sgl; - desc->sgcount = sg_len; - desc->len = dma_length; - desc->direction = direction; + desc->type = RZ_DMAC_DESC_SLAVE_SG; + desc->sg = sgl; + desc->sgcount = sg_len; + desc->len = dma_length; + desc->direction = direction; - if (direction == DMA_DEV_TO_MEM) - desc->src = channel->src_per_address; - else - desc->dest = channel->dst_per_address; + if (direction == DMA_DEV_TO_MEM) + desc->src = channel->src_per_address; + else + desc->dest = channel->dst_per_address; + + list_move_tail(channel->ld_free.next, &channel->ld_queue); + } - list_move_tail(channel->ld_free.next, &channel->ld_queue); return vchan_tx_prep(&channel->vc, &desc->vd, flags); } @@ -562,8 +566,8 @@ static int rz_dmac_terminate_all(struct dma_chan *chan) unsigned int i; LIST_HEAD(head); - rz_dmac_disable_hw(channel); spin_lock_irqsave(&channel->vc.lock, flags); + rz_dmac_disable_hw(channel); for (i = 0; i < DMAC_NR_LMDESC; i++) lmdesc[i].header = 0; @@ -700,7 +704,9 @@ static void rz_dmac_irq_handle_channel(struct rz_dmac_chan *channel) if (chstat & CHSTAT_ER) { dev_err(dmac->dev, "DMAC err CHSTAT_%d = %08X\n", channel->index, chstat); - rz_dmac_ch_writel(channel, CHCTRL_DEFAULT, CHCTRL, 1); + + scoped_guard(spinlock_irqsave, &channel->vc.lock) + rz_dmac_ch_writel(channel, CHCTRL_DEFAULT, CHCTRL, 1); goto done; } diff --git a/drivers/dma/stm32/stm32-dma3.c b/drivers/dma/stm32/stm32-dma3.c index 50e7106c5cb733..9500164c8f6888 100644 --- a/drivers/dma/stm32/stm32-dma3.c +++ b/drivers/dma/stm32/stm32-dma3.c @@ -1914,12 +1914,7 @@ static struct platform_driver stm32_dma3_driver = { }, }; -static int __init stm32_dma3_init(void) -{ - return platform_driver_register(&stm32_dma3_driver); -} - -subsys_initcall(stm32_dma3_init); +module_platform_driver(stm32_dma3_driver); MODULE_DESCRIPTION("STM32 DMA3 controller driver"); MODULE_AUTHOR("Amelie Delaunay "); diff --git a/drivers/dma/stm32/stm32-mdma.c b/drivers/dma/stm32/stm32-mdma.c index 080c1c725216cb..b87d41b234df11 100644 --- a/drivers/dma/stm32/stm32-mdma.c +++ b/drivers/dma/stm32/stm32-mdma.c @@ -731,7 +731,7 @@ static int stm32_mdma_setup_xfer(struct stm32_mdma_chan *chan, struct stm32_mdma_chan_config *chan_config = &chan->chan_config; struct scatterlist *sg; dma_addr_t src_addr, dst_addr; - u32 m2m_hw_period, ccr, ctcr, ctbr; + u32 m2m_hw_period = 0, ccr = 0, ctcr, ctbr; int i, ret = 0; if (chan_config->m2m_hw) diff --git a/drivers/dma/sun6i-dma.c b/drivers/dma/sun6i-dma.c index 2215ff877bf7d0..f9d876deb1f05c 100644 --- a/drivers/dma/sun6i-dma.c +++ b/drivers/dma/sun6i-dma.c @@ -583,6 +583,22 @@ static irqreturn_t sun6i_dma_interrupt(int irq, void *dev_id) return ret; } +static u32 find_burst_size(const u32 burst_lengths, u32 maxburst) +{ + if (!maxburst) + return 1; + + if (BIT(maxburst) & burst_lengths) + return maxburst; + + /* Hardware only does power-of-two bursts. */ + for (u32 burst = rounddown_pow_of_two(maxburst); burst > 0; burst /= 2) + if (BIT(burst) & burst_lengths) + return burst; + + return 1; +} + static int set_config(struct sun6i_dma_dev *sdev, struct dma_slave_config *sconfig, enum dma_transfer_direction direction, @@ -616,15 +632,13 @@ static int set_config(struct sun6i_dma_dev *sdev, return -EINVAL; if (!(BIT(dst_addr_width) & sdev->slave.dst_addr_widths)) return -EINVAL; - if (!(BIT(src_maxburst) & sdev->cfg->src_burst_lengths)) - return -EINVAL; - if (!(BIT(dst_maxburst) & sdev->cfg->dst_burst_lengths)) - return -EINVAL; src_width = convert_buswidth(src_addr_width); dst_width = convert_buswidth(dst_addr_width); - dst_burst = convert_burst(dst_maxburst); - src_burst = convert_burst(src_maxburst); + src_burst = find_burst_size(sdev->cfg->src_burst_lengths, src_maxburst); + dst_burst = find_burst_size(sdev->cfg->dst_burst_lengths, dst_maxburst); + dst_burst = convert_burst(dst_burst); + src_burst = convert_burst(src_burst); *p_cfg = DMA_CHAN_CFG_SRC_WIDTH(src_width) | DMA_CHAN_CFG_DST_WIDTH(dst_width); diff --git a/drivers/dma/xilinx/xdma.c b/drivers/dma/xilinx/xdma.c index 5ecf8223c112e4..58e01e22b9765b 100644 --- a/drivers/dma/xilinx/xdma.c +++ b/drivers/dma/xilinx/xdma.c @@ -1236,8 +1236,8 @@ static int xdma_probe(struct platform_device *pdev) xdev->rmap = devm_regmap_init_mmio(&pdev->dev, reg_base, &xdma_regmap_config); - if (!xdev->rmap) { - xdma_err(xdev, "config regmap failed: %d", ret); + if (IS_ERR(xdev->rmap)) { + xdma_err(xdev, "config regmap failed: %pe", xdev->rmap); goto failed; } INIT_LIST_HEAD(&xdev->dma_dev.channels); diff --git a/drivers/dma/xilinx/xilinx_dma.c b/drivers/dma/xilinx/xilinx_dma.c index 89a8254d9cdc62..7dec5e6babe144 100644 --- a/drivers/dma/xilinx/xilinx_dma.c +++ b/drivers/dma/xilinx/xilinx_dma.c @@ -997,16 +997,16 @@ static u32 xilinx_dma_get_residue(struct xilinx_dma_chan *chan, struct xilinx_cdma_tx_segment, node); cdma_hw = &cdma_seg->hw; - residue += (cdma_hw->control - cdma_hw->status) & - chan->xdev->max_buffer_len; + residue += (cdma_hw->control & chan->xdev->max_buffer_len) - + (cdma_hw->status & chan->xdev->max_buffer_len); } else if (chan->xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA) { axidma_seg = list_entry(entry, struct xilinx_axidma_tx_segment, node); axidma_hw = &axidma_seg->hw; - residue += (axidma_hw->control - axidma_hw->status) & - chan->xdev->max_buffer_len; + residue += (axidma_hw->control & chan->xdev->max_buffer_len) - + (axidma_hw->status & chan->xdev->max_buffer_len); } else { aximcdma_seg = list_entry(entry, @@ -1014,8 +1014,8 @@ static u32 xilinx_dma_get_residue(struct xilinx_dma_chan *chan, node); aximcdma_hw = &aximcdma_seg->hw; residue += - (aximcdma_hw->control - aximcdma_hw->status) & - chan->xdev->max_buffer_len; + (aximcdma_hw->control & chan->xdev->max_buffer_len) - + (aximcdma_hw->status & chan->xdev->max_buffer_len); } } @@ -1217,14 +1217,6 @@ static int xilinx_dma_alloc_chan_resources(struct dma_chan *dchan) dma_cookie_init(dchan); - if (chan->xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA) { - /* For AXI DMA resetting once channel will reset the - * other channel as well so enable the interrupts here. - */ - dma_ctrl_set(chan, XILINX_DMA_REG_DMACR, - XILINX_DMA_DMAXR_ALL_IRQ_MASK); - } - if ((chan->xdev->dma_config->dmatype == XDMA_TYPE_CDMA) && chan->has_sg) dma_ctrl_set(chan, XILINX_DMA_REG_DMACR, XILINX_CDMA_CR_SGMODE); @@ -1546,8 +1538,29 @@ static void xilinx_dma_start_transfer(struct xilinx_dma_chan *chan) if (chan->err) return; - if (list_empty(&chan->pending_list)) + if (list_empty(&chan->pending_list)) { + if (chan->cyclic) { + struct xilinx_dma_tx_descriptor *desc; + struct list_head *entry; + + desc = list_last_entry(&chan->done_list, + struct xilinx_dma_tx_descriptor, node); + list_for_each(entry, &desc->segments) { + struct xilinx_axidma_tx_segment *axidma_seg; + struct xilinx_axidma_desc_hw *axidma_hw; + axidma_seg = list_entry(entry, + struct xilinx_axidma_tx_segment, + node); + axidma_hw = &axidma_seg->hw; + axidma_hw->status = 0; + } + + list_splice_tail_init(&chan->done_list, &chan->active_list); + chan->desc_pendingcount = 0; + chan->idle = false; + } return; + } if (!chan->idle) return; @@ -1573,6 +1586,7 @@ static void xilinx_dma_start_transfer(struct xilinx_dma_chan *chan) head_desc->async_tx.phys); reg &= ~XILINX_DMA_CR_DELAY_MAX; reg |= chan->irq_delay << XILINX_DMA_CR_DELAY_SHIFT; + reg |= XILINX_DMA_DMAXR_ALL_IRQ_MASK; dma_ctrl_write(chan, XILINX_DMA_REG_DMACR, reg); xilinx_dma_start(chan); @@ -3003,7 +3017,7 @@ static int xilinx_dma_chan_probe(struct xilinx_dma_device *xdev, return -EINVAL; } - xdev->common.directions |= chan->direction; + xdev->common.directions |= BIT(chan->direction); /* Request the interrupt */ chan->irq = of_irq_get(node, chan->tdest); diff --git a/drivers/dpll/zl3073x/core.c b/drivers/dpll/zl3073x/core.c index 383e2397dd033e..b9b7c751b7602b 100644 --- a/drivers/dpll/zl3073x/core.c +++ b/drivers/dpll/zl3073x/core.c @@ -978,11 +978,7 @@ zl3073x_devm_dpll_init(struct zl3073x_dev *zldev, u8 num_dplls) } /* Add devres action to release DPLL related resources */ - rc = devm_add_action_or_reset(zldev->dev, zl3073x_dev_dpll_fini, zldev); - if (rc) - goto error; - - return 0; + return devm_add_action_or_reset(zldev->dev, zl3073x_dev_dpll_fini, zldev); error: zl3073x_dev_dpll_fini(zldev); @@ -1023,6 +1019,7 @@ int zl3073x_dev_probe(struct zl3073x_dev *zldev, "Unknown or non-match chip ID: 0x%0x\n", id); } + zldev->chip_id = id; /* Read revision, firmware version and custom config version */ rc = zl3073x_read_u16(zldev, ZL_REG_REVISION, &revision); diff --git a/drivers/dpll/zl3073x/core.h b/drivers/dpll/zl3073x/core.h index 09bca2d0926d50..f6a792557ccd63 100644 --- a/drivers/dpll/zl3073x/core.h +++ b/drivers/dpll/zl3073x/core.h @@ -35,6 +35,7 @@ struct zl3073x_dpll; * @dev: pointer to device * @regmap: regmap to access device registers * @multiop_lock: to serialize multiple register operations + * @chip_id: chip ID read from hardware * @ref: array of input references' invariants * @out: array of outs' invariants * @synth: array of synths' invariants @@ -48,6 +49,7 @@ struct zl3073x_dev { struct device *dev; struct regmap *regmap; struct mutex multiop_lock; + u16 chip_id; /* Invariants */ struct zl3073x_ref ref[ZL3073X_NUM_REFS]; @@ -144,6 +146,32 @@ int zl3073x_write_hwreg_seq(struct zl3073x_dev *zldev, int zl3073x_ref_phase_offsets_update(struct zl3073x_dev *zldev, int channel); +/** + * zl3073x_dev_is_ref_phase_comp_32bit - check ref phase comp register size + * @zldev: pointer to zl3073x device + * + * Some chip IDs have a 32-bit wide ref_phase_offset_comp register instead + * of the default 48-bit. + * + * Return: true if the register is 32-bit, false if 48-bit + */ +static inline bool +zl3073x_dev_is_ref_phase_comp_32bit(struct zl3073x_dev *zldev) +{ + switch (zldev->chip_id) { + case 0x0E30: + case 0x0E93: + case 0x0E94: + case 0x0E95: + case 0x0E96: + case 0x0E97: + case 0x1F60: + return true; + default: + return false; + } +} + static inline bool zl3073x_is_n_pin(u8 id) { diff --git a/drivers/dpll/zl3073x/dpll.c b/drivers/dpll/zl3073x/dpll.c index 9879d85d29af0f..d7194418d15684 100644 --- a/drivers/dpll/zl3073x/dpll.c +++ b/drivers/dpll/zl3073x/dpll.c @@ -459,8 +459,11 @@ zl3073x_dpll_input_pin_phase_adjust_get(const struct dpll_pin *dpll_pin, ref_id = zl3073x_input_pin_ref_get(pin->id); ref = zl3073x_ref_state_get(zldev, ref_id); - /* Perform sign extension for 48bit signed value */ - phase_comp = sign_extend64(ref->phase_comp, 47); + /* Perform sign extension based on register width */ + if (zl3073x_dev_is_ref_phase_comp_32bit(zldev)) + phase_comp = sign_extend64(ref->phase_comp, 31); + else + phase_comp = sign_extend64(ref->phase_comp, 47); /* Reverse two's complement negation applied during set and convert * to 32bit signed int @@ -1039,10 +1042,8 @@ zl3073x_dpll_output_pin_phase_adjust_get(const struct dpll_pin *dpll_pin, out_id = zl3073x_output_pin_out_get(pin->id); out = zl3073x_out_state_get(zldev, out_id); - /* Convert value to ps and reverse two's complement negation applied - * during 'set' - */ - *phase_adjust = -out->phase_comp * pin->phase_gran; + /* The value in the register is expressed in half synth clock cycles. */ + *phase_adjust = out->phase_comp * pin->phase_gran; return 0; } @@ -1064,10 +1065,8 @@ zl3073x_dpll_output_pin_phase_adjust_set(const struct dpll_pin *dpll_pin, out_id = zl3073x_output_pin_out_get(pin->id); out = *zl3073x_out_state_get(zldev, out_id); - /* The value in the register is stored as two's complement negation - * of requested value and expressed in half synth clock cycles. - */ - out.phase_comp = -phase_adjust / pin->phase_gran; + /* The value in the register is expressed in half synth clock cycles. */ + out.phase_comp = phase_adjust / pin->phase_gran; /* Update output configuration from mailbox */ return zl3073x_out_state_set(zldev, out_id, &out); diff --git a/drivers/dpll/zl3073x/ref.c b/drivers/dpll/zl3073x/ref.c index aa2de13effa878..6b65e61039999b 100644 --- a/drivers/dpll/zl3073x/ref.c +++ b/drivers/dpll/zl3073x/ref.c @@ -121,8 +121,16 @@ int zl3073x_ref_state_fetch(struct zl3073x_dev *zldev, u8 index) return rc; /* Read phase compensation register */ - rc = zl3073x_read_u48(zldev, ZL_REG_REF_PHASE_OFFSET_COMP, - &ref->phase_comp); + if (zl3073x_dev_is_ref_phase_comp_32bit(zldev)) { + u32 val; + + rc = zl3073x_read_u32(zldev, ZL_REG_REF_PHASE_OFFSET_COMP_32, + &val); + ref->phase_comp = val; + } else { + rc = zl3073x_read_u48(zldev, ZL_REG_REF_PHASE_OFFSET_COMP, + &ref->phase_comp); + } if (rc) return rc; @@ -179,9 +187,16 @@ int zl3073x_ref_state_set(struct zl3073x_dev *zldev, u8 index, if (!rc && dref->sync_ctrl != ref->sync_ctrl) rc = zl3073x_write_u8(zldev, ZL_REG_REF_SYNC_CTRL, ref->sync_ctrl); - if (!rc && dref->phase_comp != ref->phase_comp) - rc = zl3073x_write_u48(zldev, ZL_REG_REF_PHASE_OFFSET_COMP, - ref->phase_comp); + if (!rc && dref->phase_comp != ref->phase_comp) { + if (zl3073x_dev_is_ref_phase_comp_32bit(zldev)) + rc = zl3073x_write_u32(zldev, + ZL_REG_REF_PHASE_OFFSET_COMP_32, + ref->phase_comp); + else + rc = zl3073x_write_u48(zldev, + ZL_REG_REF_PHASE_OFFSET_COMP, + ref->phase_comp); + } if (rc) return rc; diff --git a/drivers/dpll/zl3073x/ref.h b/drivers/dpll/zl3073x/ref.h index efc7f59cd9f9c6..0d8618f5ce8df5 100644 --- a/drivers/dpll/zl3073x/ref.h +++ b/drivers/dpll/zl3073x/ref.h @@ -91,6 +91,8 @@ zl3073x_ref_freq_set(struct zl3073x_ref *ref, u32 freq) ref->freq_base = base; ref->freq_mult = mult; + ref->freq_ratio_m = 1; + ref->freq_ratio_n = 1; return 0; } diff --git a/drivers/dpll/zl3073x/regs.h b/drivers/dpll/zl3073x/regs.h index d837bee72b1780..5573d7188406bb 100644 --- a/drivers/dpll/zl3073x/regs.h +++ b/drivers/dpll/zl3073x/regs.h @@ -194,6 +194,7 @@ #define ZL_REF_CONFIG_DIFF_EN BIT(2) #define ZL_REG_REF_PHASE_OFFSET_COMP ZL_REG(10, 0x28, 6) +#define ZL_REG_REF_PHASE_OFFSET_COMP_32 ZL_REG(10, 0x28, 4) #define ZL_REG_REF_SYNC_CTRL ZL_REG(10, 0x2e, 1) #define ZL_REF_SYNC_CTRL_MODE GENMASK(2, 0) diff --git a/drivers/edac/altera_edac.c b/drivers/edac/altera_edac.c index 0c5b94e64ea157..4edd2088c2db6f 100644 --- a/drivers/edac/altera_edac.c +++ b/drivers/edac/altera_edac.c @@ -1563,8 +1563,7 @@ static int altr_portb_setup(struct altr_edac_device_dev *device) goto err_release_group_1; } rc = devm_request_irq(&altdev->ddev, altdev->sb_irq, - prv->ecc_irq_handler, - IRQF_ONESHOT | IRQF_TRIGGER_HIGH, + prv->ecc_irq_handler, IRQF_TRIGGER_HIGH, ecc_name, altdev); if (rc) { edac_printk(KERN_ERR, EDAC_DEVICE, "PortB SBERR IRQ error\n"); @@ -1587,8 +1586,7 @@ static int altr_portb_setup(struct altr_edac_device_dev *device) goto err_release_group_1; } rc = devm_request_irq(&altdev->ddev, altdev->db_irq, - prv->ecc_irq_handler, - IRQF_ONESHOT | IRQF_TRIGGER_HIGH, + prv->ecc_irq_handler, IRQF_TRIGGER_HIGH, ecc_name, altdev); if (rc) { edac_printk(KERN_ERR, EDAC_DEVICE, "PortB DBERR IRQ error\n"); @@ -1970,8 +1968,7 @@ static int altr_edac_a10_device_add(struct altr_arria10_edac *edac, goto err_release_group1; } rc = devm_request_irq(edac->dev, altdev->sb_irq, prv->ecc_irq_handler, - IRQF_ONESHOT | IRQF_TRIGGER_HIGH, - ecc_name, altdev); + IRQF_TRIGGER_HIGH, ecc_name, altdev); if (rc) { edac_printk(KERN_ERR, EDAC_DEVICE, "No SBERR IRQ resource\n"); goto err_release_group1; @@ -1993,7 +1990,7 @@ static int altr_edac_a10_device_add(struct altr_arria10_edac *edac, goto err_release_group1; } rc = devm_request_irq(edac->dev, altdev->db_irq, prv->ecc_irq_handler, - IRQF_ONESHOT | IRQF_TRIGGER_HIGH, + IRQF_TRIGGER_HIGH, ecc_name, altdev); if (rc) { edac_printk(KERN_ERR, EDAC_DEVICE, "No DBERR IRQ resource\n"); diff --git a/drivers/edac/amd64_edac.c b/drivers/edac/amd64_edac.c index 2391f3469961ab..63fca0ee2c23bc 100644 --- a/drivers/edac/amd64_edac.c +++ b/drivers/edac/amd64_edac.c @@ -3911,7 +3911,7 @@ static int per_family_init(struct amd64_pvt *pvt) } if (tmp_name) - scnprintf(pvt->ctl_name, sizeof(pvt->ctl_name), tmp_name); + scnprintf(pvt->ctl_name, sizeof(pvt->ctl_name), "%s", tmp_name); else scnprintf(pvt->ctl_name, sizeof(pvt->ctl_name), "F%02Xh_M%02Xh", pvt->fam, pvt->model); diff --git a/drivers/edac/edac_mc.c b/drivers/edac/edac_mc.c index 0959320fe51c67..b510919e49c0ee 100644 --- a/drivers/edac/edac_mc.c +++ b/drivers/edac/edac_mc.c @@ -370,13 +370,13 @@ struct mem_ctl_info *edac_mc_alloc(unsigned int mc_num, if (!mci->layers) goto error; + mci->dev.release = mci_release; + device_initialize(&mci->dev); + mci->pvt_info = kzalloc(sz_pvt, GFP_KERNEL); if (!mci->pvt_info) goto error; - mci->dev.release = mci_release; - device_initialize(&mci->dev); - /* setup index and various internal pointers */ mci->mc_idx = mc_num; mci->tot_dimms = tot_dimms; diff --git a/drivers/edac/i5000_edac.c b/drivers/edac/i5000_edac.c index 4a1bebc1ff14d4..471b8540d18b0f 100644 --- a/drivers/edac/i5000_edac.c +++ b/drivers/edac/i5000_edac.c @@ -1111,6 +1111,7 @@ static void calculate_dimm_size(struct i5000_pvt *pvt) n = snprintf(p, space, " "); p += n; + space -= n; for (branch = 0; branch < MAX_BRANCHES; branch++) { n = snprintf(p, space, " branch %d | ", branch); p += n; diff --git a/drivers/edac/i5400_edac.c b/drivers/edac/i5400_edac.c index b5cf25905b059d..fb49a1d1df1120 100644 --- a/drivers/edac/i5400_edac.c +++ b/drivers/edac/i5400_edac.c @@ -1026,13 +1026,13 @@ static void calculate_dimm_size(struct i5400_pvt *pvt) space -= n; } - space -= n; edac_dbg(2, "%s\n", mem_buffer); p = mem_buffer; space = PAGE_SIZE; n = snprintf(p, space, " "); p += n; + space -= n; for (branch = 0; branch < MAX_BRANCHES; branch++) { n = snprintf(p, space, " branch %d | ", branch); p += n; diff --git a/drivers/edac/igen6_edac.c b/drivers/edac/igen6_edac.c index 553c31a2d9226f..f2c9270c1893c8 100644 --- a/drivers/edac/igen6_edac.c +++ b/drivers/edac/igen6_edac.c @@ -246,6 +246,8 @@ static struct work_struct ecclog_work; /* Compute did IDs for Amston Lake with IBECC */ #define DID_ASL_SKU1 0x464a +#define DID_ASL_SKU2 0x4646 +#define DID_ASL_SKU3 0x4652 /* Compute die IDs for Raptor Lake-P with IBECC */ #define DID_RPL_P_SKU1 0xa706 @@ -274,6 +276,16 @@ static struct work_struct ecclog_work; #define DID_PTL_H_SKU1 0xb000 #define DID_PTL_H_SKU2 0xb001 #define DID_PTL_H_SKU3 0xb002 +#define DID_PTL_H_SKU4 0xb003 +#define DID_PTL_H_SKU5 0xb004 +#define DID_PTL_H_SKU6 0xb005 +#define DID_PTL_H_SKU7 0xb008 +#define DID_PTL_H_SKU8 0xb011 +#define DID_PTL_H_SKU9 0xb014 +#define DID_PTL_H_SKU10 0xb015 +#define DID_PTL_H_SKU11 0xb028 +#define DID_PTL_H_SKU12 0xb029 +#define DID_PTL_H_SKU13 0xb02a /* Compute die IDs for Wildcat Lake with IBECC */ #define DID_WCL_SKU1 0xfd00 @@ -618,6 +630,8 @@ static struct pci_device_id igen6_pci_tbl[] = { { PCI_VDEVICE(INTEL, DID_ADL_N_SKU12), (kernel_ulong_t)&adl_n_cfg }, { PCI_VDEVICE(INTEL, DID_AZB_SKU1), (kernel_ulong_t)&adl_n_cfg }, { PCI_VDEVICE(INTEL, DID_ASL_SKU1), (kernel_ulong_t)&adl_n_cfg }, + { PCI_VDEVICE(INTEL, DID_ASL_SKU2), (kernel_ulong_t)&adl_n_cfg }, + { PCI_VDEVICE(INTEL, DID_ASL_SKU3), (kernel_ulong_t)&adl_n_cfg }, { PCI_VDEVICE(INTEL, DID_RPL_P_SKU1), (kernel_ulong_t)&rpl_p_cfg }, { PCI_VDEVICE(INTEL, DID_RPL_P_SKU2), (kernel_ulong_t)&rpl_p_cfg }, { PCI_VDEVICE(INTEL, DID_RPL_P_SKU3), (kernel_ulong_t)&rpl_p_cfg }, @@ -636,6 +650,16 @@ static struct pci_device_id igen6_pci_tbl[] = { { PCI_VDEVICE(INTEL, DID_PTL_H_SKU1), (kernel_ulong_t)&mtl_p_cfg }, { PCI_VDEVICE(INTEL, DID_PTL_H_SKU2), (kernel_ulong_t)&mtl_p_cfg }, { PCI_VDEVICE(INTEL, DID_PTL_H_SKU3), (kernel_ulong_t)&mtl_p_cfg }, + { PCI_VDEVICE(INTEL, DID_PTL_H_SKU4), (kernel_ulong_t)&mtl_p_cfg }, + { PCI_VDEVICE(INTEL, DID_PTL_H_SKU5), (kernel_ulong_t)&mtl_p_cfg }, + { PCI_VDEVICE(INTEL, DID_PTL_H_SKU6), (kernel_ulong_t)&mtl_p_cfg }, + { PCI_VDEVICE(INTEL, DID_PTL_H_SKU7), (kernel_ulong_t)&mtl_p_cfg }, + { PCI_VDEVICE(INTEL, DID_PTL_H_SKU8), (kernel_ulong_t)&mtl_p_cfg }, + { PCI_VDEVICE(INTEL, DID_PTL_H_SKU9), (kernel_ulong_t)&mtl_p_cfg }, + { PCI_VDEVICE(INTEL, DID_PTL_H_SKU10), (kernel_ulong_t)&mtl_p_cfg }, + { PCI_VDEVICE(INTEL, DID_PTL_H_SKU11), (kernel_ulong_t)&mtl_p_cfg }, + { PCI_VDEVICE(INTEL, DID_PTL_H_SKU12), (kernel_ulong_t)&mtl_p_cfg }, + { PCI_VDEVICE(INTEL, DID_PTL_H_SKU13), (kernel_ulong_t)&mtl_p_cfg }, { PCI_VDEVICE(INTEL, DID_WCL_SKU1), (kernel_ulong_t)&wcl_cfg }, { }, }; diff --git a/drivers/firewire/net.c b/drivers/firewire/net.c index 6d644671353959..e8294540895507 100644 --- a/drivers/firewire/net.c +++ b/drivers/firewire/net.c @@ -257,9 +257,10 @@ static void fwnet_header_cache_update(struct hh_cache *hh, memcpy((u8 *)hh->hh_data + HH_DATA_OFF(FWNET_HLEN), haddr, net->addr_len); } -static int fwnet_header_parse(const struct sk_buff *skb, unsigned char *haddr) +static int fwnet_header_parse(const struct sk_buff *skb, const struct net_device *dev, + unsigned char *haddr) { - memcpy(haddr, skb->dev->dev_addr, FWNET_ALEN); + memcpy(haddr, dev->dev_addr, FWNET_ALEN); return FWNET_ALEN; } diff --git a/drivers/firmware/arm_ffa/driver.c b/drivers/firmware/arm_ffa/driver.c index c72ee475658569..f6ceae987acbce 100644 --- a/drivers/firmware/arm_ffa/driver.c +++ b/drivers/firmware/arm_ffa/driver.c @@ -205,12 +205,12 @@ static int ffa_rxtx_map(phys_addr_t tx_buf, phys_addr_t rx_buf, u32 pg_cnt) return 0; } -static int ffa_rxtx_unmap(u16 vm_id) +static int ffa_rxtx_unmap(void) { ffa_value_t ret; invoke_ffa_fn((ffa_value_t){ - .a0 = FFA_RXTX_UNMAP, .a1 = PACK_TARGET_INFO(vm_id, 0), + .a0 = FFA_RXTX_UNMAP, }, &ret); if (ret.a0 == FFA_ERROR) @@ -981,10 +981,27 @@ static void __do_sched_recv_cb(u16 part_id, u16 vcpu, bool is_per_vcpu) } } +/* + * Map logical ID index to the u16 index within the packed ID list. + * + * For native responses (FF-A width == kernel word size), IDs are + * tightly packed: idx -> idx. + * + * For 32-bit responses on a 64-bit kernel, each 64-bit register + * contributes 4 x u16 values but only the lower 2 are defined; the + * upper 2 are garbage. This mapping skips those upper halves: + * 0,1,2,3,4,5,... -> 0,1,4,5,8,9,... + */ +static int list_idx_to_u16_idx(int idx, bool is_native_resp) +{ + return is_native_resp ? idx : idx + 2 * (idx >> 1); +} + static void ffa_notification_info_get(void) { - int idx, list, max_ids, lists_cnt, ids_processed, ids_count[MAX_IDS_64]; - bool is_64b_resp; + int ids_processed, ids_count[MAX_IDS_64]; + int idx, list, max_ids, lists_cnt; + bool is_64b_resp, is_native_resp; ffa_value_t ret; u64 id_list; @@ -1001,6 +1018,7 @@ static void ffa_notification_info_get(void) } is_64b_resp = (ret.a0 == FFA_FN64_SUCCESS); + is_native_resp = (ret.a0 == FFA_FN_NATIVE(SUCCESS)); ids_processed = 0; lists_cnt = FIELD_GET(NOTIFICATION_INFO_GET_ID_COUNT, ret.a2); @@ -1017,12 +1035,16 @@ static void ffa_notification_info_get(void) /* Process IDs */ for (list = 0; list < lists_cnt; list++) { + int u16_idx; u16 vcpu_id, part_id, *packed_id_list = (u16 *)&ret.a3; if (ids_processed >= max_ids - 1) break; - part_id = packed_id_list[ids_processed++]; + u16_idx = list_idx_to_u16_idx(ids_processed, + is_native_resp); + part_id = packed_id_list[u16_idx]; + ids_processed++; if (ids_count[list] == 1) { /* Global Notification */ __do_sched_recv_cb(part_id, 0, false); @@ -1034,7 +1056,10 @@ static void ffa_notification_info_get(void) if (ids_processed >= max_ids - 1) break; - vcpu_id = packed_id_list[ids_processed++]; + u16_idx = list_idx_to_u16_idx(ids_processed, + is_native_resp); + vcpu_id = packed_id_list[u16_idx]; + ids_processed++; __do_sched_recv_cb(part_id, vcpu_id, true); } @@ -2068,6 +2093,7 @@ static int __init ffa_init(void) pr_err("failed to setup partitions\n"); ffa_notifications_cleanup(); + ffa_rxtx_unmap(); free_pages: if (drv_info->tx_buffer) free_pages_exact(drv_info->tx_buffer, rxtx_bufsz); @@ -2082,7 +2108,7 @@ static void __exit ffa_exit(void) { ffa_notifications_cleanup(); ffa_partitions_cleanup(); - ffa_rxtx_unmap(drv_info->vm_id); + ffa_rxtx_unmap(); free_pages_exact(drv_info->tx_buffer, drv_info->rxtx_bufsz); free_pages_exact(drv_info->rx_buffer, drv_info->rxtx_bufsz); kfree(drv_info); diff --git a/drivers/firmware/arm_scmi/notify.c b/drivers/firmware/arm_scmi/notify.c index dee9f238f6fdd9..2047edbdc5f6b1 100644 --- a/drivers/firmware/arm_scmi/notify.c +++ b/drivers/firmware/arm_scmi/notify.c @@ -1066,7 +1066,7 @@ static int scmi_register_event_handler(struct scmi_notify_instance *ni, * since at creation time we usually want to have all setup and ready before * events really start flowing. * - * Return: A properly refcounted handler on Success, NULL on Failure + * Return: A properly refcounted handler on Success, ERR_PTR on Failure */ static inline struct scmi_event_handler * __scmi_event_handler_get_ops(struct scmi_notify_instance *ni, @@ -1113,7 +1113,7 @@ __scmi_event_handler_get_ops(struct scmi_notify_instance *ni, } mutex_unlock(&ni->pending_mtx); - return hndl; + return hndl ?: ERR_PTR(-ENODEV); } static struct scmi_event_handler * diff --git a/drivers/firmware/arm_scpi.c b/drivers/firmware/arm_scpi.c index 87c323de17b901..398642cc25d90d 100644 --- a/drivers/firmware/arm_scpi.c +++ b/drivers/firmware/arm_scpi.c @@ -18,6 +18,7 @@ #include #include +#include #include #include #include @@ -940,13 +941,13 @@ static int scpi_probe(struct platform_device *pdev) int idx = scpi_drvinfo->num_chans; struct scpi_chan *pchan = scpi_drvinfo->channels + idx; struct mbox_client *cl = &pchan->cl; - struct device_node *shmem = of_parse_phandle(np, "shmem", idx); + struct device_node *shmem __free(device_node) = + of_parse_phandle(np, "shmem", idx); if (!of_match_node(shmem_of_match, shmem)) return -ENXIO; ret = of_address_to_resource(shmem, 0, &res); - of_node_put(shmem); if (ret) { dev_err(dev, "failed to get SCPI payload mem resource\n"); return ret; diff --git a/drivers/firmware/cirrus/cs_dsp.c b/drivers/firmware/cirrus/cs_dsp.c index 73d201e7d9927a..a34633b8757580 100644 --- a/drivers/firmware/cirrus/cs_dsp.c +++ b/drivers/firmware/cirrus/cs_dsp.c @@ -412,18 +412,23 @@ static ssize_t cs_dsp_debugfs_string_read(struct cs_dsp *dsp, size_t count, loff_t *ppos, const char **pstr) { - const char *str __free(kfree) = NULL; + const char *str; + ssize_t ret = 0; scoped_guard(mutex, &dsp->pwr_lock) { - if (!*pstr) - return 0; - - str = kasprintf(GFP_KERNEL, "%s\n", *pstr); - if (!str) - return -ENOMEM; - - return simple_read_from_buffer(user_buf, count, ppos, str, strlen(str)); + if (*pstr) { + str = kasprintf(GFP_KERNEL, "%s\n", *pstr); + if (str) { + ret = simple_read_from_buffer(user_buf, count, + ppos, str, strlen(str)); + kfree(str); + } else { + ret = -ENOMEM; + } + } } + + return ret; } static ssize_t cs_dsp_debugfs_wmfw_read(struct file *file, @@ -1483,7 +1488,7 @@ static int cs_dsp_load(struct cs_dsp *dsp, const struct firmware *firmware, const struct wmfw_region *region; const struct cs_dsp_region *mem; const char *region_name; - u8 *buf __free(kfree) = NULL; + u8 *buf = NULL; size_t buf_len = 0; size_t region_len; unsigned int reg; @@ -1605,11 +1610,17 @@ static int cs_dsp_load(struct cs_dsp *dsp, const struct firmware *firmware, region_name); if (reg) { + /* + * Although we expect the underlying bus does not require + * physically-contiguous buffers, we pessimistically use + * a temporary buffer instead of trusting that the + * alignment of region->data is ok. + */ region_len = le32_to_cpu(region->len); if (region_len > buf_len) { buf_len = round_up(region_len, PAGE_SIZE); - kfree(buf); - buf = kmalloc(buf_len, GFP_KERNEL | GFP_DMA); + vfree(buf); + buf = vmalloc(buf_len); if (!buf) { ret = -ENOMEM; goto out_fw; @@ -1638,6 +1649,8 @@ static int cs_dsp_load(struct cs_dsp *dsp, const struct firmware *firmware, ret = 0; out_fw: + vfree(buf); + if (ret == -EOVERFLOW) cs_dsp_err(dsp, "%s: file content overflows file data\n", file); @@ -2169,7 +2182,7 @@ static int cs_dsp_load_coeff(struct cs_dsp *dsp, const struct firmware *firmware struct cs_dsp_alg_region *alg_region; const char *region_name; int ret, pos, blocks, type, offset, reg, version; - u8 *buf __free(kfree) = NULL; + u8 *buf = NULL; size_t buf_len = 0; size_t region_len; @@ -2313,11 +2326,17 @@ static int cs_dsp_load_coeff(struct cs_dsp *dsp, const struct firmware *firmware } if (reg) { + /* + * Although we expect the underlying bus does not require + * physically-contiguous buffers, we pessimistically use + * a temporary buffer instead of trusting that the + * alignment of blk->data is ok. + */ region_len = le32_to_cpu(blk->len); if (region_len > buf_len) { buf_len = round_up(region_len, PAGE_SIZE); - kfree(buf); - buf = kmalloc(buf_len, GFP_KERNEL | GFP_DMA); + vfree(buf); + buf = vmalloc(buf_len); if (!buf) { ret = -ENOMEM; goto out_fw; @@ -2348,6 +2367,8 @@ static int cs_dsp_load_coeff(struct cs_dsp *dsp, const struct firmware *firmware ret = 0; out_fw: + vfree(buf); + if (ret == -EOVERFLOW) cs_dsp_err(dsp, "%s: file content overflows file data\n", file); diff --git a/drivers/firmware/efi/cper-arm.c b/drivers/firmware/efi/cper-arm.c index 76542a53e20275..b21cb1232d8207 100644 --- a/drivers/firmware/efi/cper-arm.c +++ b/drivers/firmware/efi/cper-arm.c @@ -226,7 +226,8 @@ static void cper_print_arm_err_info(const char *pfx, u32 type, } void cper_print_proc_arm(const char *pfx, - const struct cper_sec_proc_arm *proc) + const struct cper_sec_proc_arm *proc, + u32 length) { int i, len, max_ctx_type; struct cper_arm_err_info *err_info; @@ -238,9 +239,12 @@ void cper_print_proc_arm(const char *pfx, len = proc->section_length - (sizeof(*proc) + proc->err_info_num * (sizeof(*err_info))); - if (len < 0) { - printk("%ssection length: %d\n", pfx, proc->section_length); - printk("%ssection length is too small\n", pfx); + + if (len < 0 || proc->section_length > length) { + printk("%ssection length: %d, CPER size: %d\n", + pfx, proc->section_length, length); + printk("%ssection length is too %s\n", pfx, + (len < 0) ? "small" : "big"); printk("%sfirmware-generated error record is incorrect\n", pfx); printk("%sERR_INFO_NUM is %d\n", pfx, proc->err_info_num); return; diff --git a/drivers/firmware/efi/cper.c b/drivers/firmware/efi/cper.c index bd99802cb0cad6..06b4fdb59917a1 100644 --- a/drivers/firmware/efi/cper.c +++ b/drivers/firmware/efi/cper.c @@ -560,6 +560,11 @@ static void cper_print_fw_err(const char *pfx, } else { offset = sizeof(*fw_err); } + if (offset > length) { + printk("%s""error section length is too small: offset=%d, length=%d\n", + pfx, offset, length); + return; + } buf += offset; length -= offset; @@ -659,7 +664,8 @@ cper_estatus_print_section(const char *pfx, struct acpi_hest_generic_data *gdata printk("%ssection_type: ARM processor error\n", newpfx); if (gdata->error_data_length >= sizeof(*arm_err)) - cper_print_proc_arm(newpfx, arm_err); + cper_print_proc_arm(newpfx, arm_err, + gdata->error_data_length); else goto err_section_too_small; #endif diff --git a/drivers/firmware/efi/efi.c b/drivers/firmware/efi/efi.c index 17b5f3415465ed..92e91c3eb4690e 100644 --- a/drivers/firmware/efi/efi.c +++ b/drivers/firmware/efi/efi.c @@ -692,13 +692,13 @@ static __init int match_config_table(const efi_guid_t *guid, static __init void reserve_unaccepted(struct efi_unaccepted_memory *unaccepted) { - phys_addr_t start, size; + phys_addr_t start, end; start = PAGE_ALIGN_DOWN(efi.unaccepted); - size = PAGE_ALIGN(sizeof(*unaccepted) + unaccepted->size); + end = PAGE_ALIGN(efi.unaccepted + sizeof(*unaccepted) + unaccepted->size); - memblock_add(start, size); - memblock_reserve(start, size); + memblock_add(start, end - start); + memblock_reserve(start, end - start); } int __init efi_config_parse_tables(const efi_config_table_t *config_tables, diff --git a/drivers/firmware/efi/mokvar-table.c b/drivers/firmware/efi/mokvar-table.c index aedbbd627706a1..741674a0a70c55 100644 --- a/drivers/firmware/efi/mokvar-table.c +++ b/drivers/firmware/efi/mokvar-table.c @@ -85,7 +85,7 @@ static struct kobject *mokvar_kobj; * as an alternative to ordinary EFI variables, due to platform-dependent * limitations. The memory occupied by this table is marked as reserved. * - * This routine must be called before efi_free_boot_services() in order + * This routine must be called before efi_unmap_boot_services() in order * to guarantee that it can mark the table as reserved. * * Implicit inputs: diff --git a/drivers/firmware/microchip/mpfs-auto-update.c b/drivers/firmware/microchip/mpfs-auto-update.c index e194f7acb2a9b0..8fc3749d4a7098 100644 --- a/drivers/firmware/microchip/mpfs-auto-update.c +++ b/drivers/firmware/microchip/mpfs-auto-update.c @@ -113,10 +113,6 @@ static enum fw_upload_err mpfs_auto_update_prepare(struct fw_upload *fw_uploader * be added here. */ - priv->flash = mpfs_sys_controller_get_flash(priv->sys_controller); - if (!priv->flash) - return FW_UPLOAD_ERR_HW_ERROR; - erase_size = round_up(erase_size, (u64)priv->flash->erasesize); /* @@ -427,6 +423,12 @@ static int mpfs_auto_update_probe(struct platform_device *pdev) return dev_err_probe(dev, PTR_ERR(priv->sys_controller), "Could not register as a sub device of the system controller\n"); + priv->flash = mpfs_sys_controller_get_flash(priv->sys_controller); + if (IS_ERR_OR_NULL(priv->flash)) { + dev_dbg(dev, "No flash connected to the system controller, auto-update not supported\n"); + return -ENODEV; + } + priv->dev = dev; platform_set_drvdata(pdev, priv); diff --git a/drivers/firmware/stratix10-rsu.c b/drivers/firmware/stratix10-rsu.c index 41da07c445a6f6..e1912108a0feec 100644 --- a/drivers/firmware/stratix10-rsu.c +++ b/drivers/firmware/stratix10-rsu.c @@ -768,7 +768,9 @@ static int stratix10_rsu_probe(struct platform_device *pdev) rsu_async_status_callback); if (ret) { dev_err(dev, "Error, getting RSU status %i\n", ret); + stratix10_svc_remove_async_client(priv->chan); stratix10_svc_free_channel(priv->chan); + return ret; } /* get DCMF version from firmware */ diff --git a/drivers/firmware/stratix10-svc.c b/drivers/firmware/stratix10-svc.c index 515b948ff320e7..5c9d55a94a1b45 100644 --- a/drivers/firmware/stratix10-svc.c +++ b/drivers/firmware/stratix10-svc.c @@ -37,15 +37,14 @@ * service layer will return error to FPGA manager when timeout occurs, * timeout is set to 30 seconds (30 * 1000) at Intel Stratix10 SoC. */ -#define SVC_NUM_DATA_IN_FIFO 32 +#define SVC_NUM_DATA_IN_FIFO 8 #define SVC_NUM_CHANNEL 4 -#define FPGA_CONFIG_DATA_CLAIM_TIMEOUT_MS 200 +#define FPGA_CONFIG_DATA_CLAIM_TIMEOUT_MS 2000 #define FPGA_CONFIG_STATUS_TIMEOUT_SEC 30 #define BYTE_TO_WORD_SIZE 4 /* stratix10 service layer clients */ #define STRATIX10_RSU "stratix10-rsu" -#define INTEL_FCS "intel-fcs" /* Maximum number of SDM client IDs. */ #define MAX_SDM_CLIENT_IDS 16 @@ -105,11 +104,9 @@ struct stratix10_svc_chan; /** * struct stratix10_svc - svc private data * @stratix10_svc_rsu: pointer to stratix10 RSU device - * @intel_svc_fcs: pointer to the FCS device */ struct stratix10_svc { struct platform_device *stratix10_svc_rsu; - struct platform_device *intel_svc_fcs; }; /** @@ -251,12 +248,10 @@ struct stratix10_async_ctrl { * @num_active_client: number of active service client * @node: list management * @genpool: memory pool pointing to the memory region - * @task: pointer to the thread task which handles SMC or HVC call - * @svc_fifo: a queue for storing service message data * @complete_status: state for completion - * @svc_fifo_lock: protect access to service message data queue * @invoke_fn: function to issue secure monitor call or hypervisor call * @svc: manages the list of client svc drivers + * @sdm_lock: only allows a single command single response to SDM * @actrl: async control structure * * This struct is used to create communication channels for service clients, to @@ -269,12 +264,10 @@ struct stratix10_svc_controller { int num_active_client; struct list_head node; struct gen_pool *genpool; - struct task_struct *task; - struct kfifo svc_fifo; struct completion complete_status; - spinlock_t svc_fifo_lock; svc_invoke_fn *invoke_fn; struct stratix10_svc *svc; + struct mutex sdm_lock; struct stratix10_async_ctrl actrl; }; @@ -283,6 +276,9 @@ struct stratix10_svc_controller { * @ctrl: pointer to service controller which is the provider of this channel * @scl: pointer to service client which owns the channel * @name: service client name associated with the channel + * @task: pointer to the thread task which handles SMC or HVC call + * @svc_fifo: a queue for storing service message data (separate fifo for every channel) + * @svc_fifo_lock: protect access to service message data queue (locking pending fifo) * @lock: protect access to the channel * @async_chan: reference to asynchronous channel object for this channel * @@ -293,6 +289,9 @@ struct stratix10_svc_chan { struct stratix10_svc_controller *ctrl; struct stratix10_svc_client *scl; char *name; + struct task_struct *task; + struct kfifo svc_fifo; + spinlock_t svc_fifo_lock; spinlock_t lock; struct stratix10_async_chan *async_chan; }; @@ -527,10 +526,10 @@ static void svc_thread_recv_status_ok(struct stratix10_svc_data *p_data, */ static int svc_normal_to_secure_thread(void *data) { - struct stratix10_svc_controller - *ctrl = (struct stratix10_svc_controller *)data; - struct stratix10_svc_data *pdata; - struct stratix10_svc_cb_data *cbdata; + struct stratix10_svc_chan *chan = (struct stratix10_svc_chan *)data; + struct stratix10_svc_controller *ctrl = chan->ctrl; + struct stratix10_svc_data *pdata = NULL; + struct stratix10_svc_cb_data *cbdata = NULL; struct arm_smccc_res res; unsigned long a0, a1, a2, a3, a4, a5, a6, a7; int ret_fifo = 0; @@ -555,12 +554,12 @@ static int svc_normal_to_secure_thread(void *data) a6 = 0; a7 = 0; - pr_debug("smc_hvc_shm_thread is running\n"); + pr_debug("%s: %s: Thread is running!\n", __func__, chan->name); while (!kthread_should_stop()) { - ret_fifo = kfifo_out_spinlocked(&ctrl->svc_fifo, + ret_fifo = kfifo_out_spinlocked(&chan->svc_fifo, pdata, sizeof(*pdata), - &ctrl->svc_fifo_lock); + &chan->svc_fifo_lock); if (!ret_fifo) continue; @@ -569,9 +568,25 @@ static int svc_normal_to_secure_thread(void *data) (unsigned int)pdata->paddr, pdata->command, (unsigned int)pdata->size); + /* SDM can only process one command at a time */ + pr_debug("%s: %s: Thread is waiting for mutex!\n", + __func__, chan->name); + if (mutex_lock_interruptible(&ctrl->sdm_lock)) { + /* item already dequeued; notify client to unblock it */ + cbdata->status = BIT(SVC_STATUS_ERROR); + cbdata->kaddr1 = NULL; + cbdata->kaddr2 = NULL; + cbdata->kaddr3 = NULL; + if (pdata->chan->scl) + pdata->chan->scl->receive_cb(pdata->chan->scl, + cbdata); + break; + } + switch (pdata->command) { case COMMAND_RECONFIG_DATA_CLAIM: svc_thread_cmd_data_claim(ctrl, pdata, cbdata); + mutex_unlock(&ctrl->sdm_lock); continue; case COMMAND_RECONFIG: a0 = INTEL_SIP_SMC_FPGA_CONFIG_START; @@ -700,10 +715,11 @@ static int svc_normal_to_secure_thread(void *data) break; default: pr_warn("it shouldn't happen\n"); - break; + mutex_unlock(&ctrl->sdm_lock); + continue; } - pr_debug("%s: before SMC call -- a0=0x%016x a1=0x%016x", - __func__, + pr_debug("%s: %s: before SMC call -- a0=0x%016x a1=0x%016x", + __func__, chan->name, (unsigned int)a0, (unsigned int)a1); pr_debug(" a2=0x%016x\n", (unsigned int)a2); @@ -712,8 +728,8 @@ static int svc_normal_to_secure_thread(void *data) pr_debug(" a5=0x%016x\n", (unsigned int)a5); ctrl->invoke_fn(a0, a1, a2, a3, a4, a5, a6, a7, &res); - pr_debug("%s: after SMC call -- res.a0=0x%016x", - __func__, (unsigned int)res.a0); + pr_debug("%s: %s: after SMC call -- res.a0=0x%016x", + __func__, chan->name, (unsigned int)res.a0); pr_debug(" res.a1=0x%016x, res.a2=0x%016x", (unsigned int)res.a1, (unsigned int)res.a2); pr_debug(" res.a3=0x%016x\n", (unsigned int)res.a3); @@ -728,6 +744,7 @@ static int svc_normal_to_secure_thread(void *data) cbdata->kaddr2 = NULL; cbdata->kaddr3 = NULL; pdata->chan->scl->receive_cb(pdata->chan->scl, cbdata); + mutex_unlock(&ctrl->sdm_lock); continue; } @@ -801,6 +818,8 @@ static int svc_normal_to_secure_thread(void *data) break; } + + mutex_unlock(&ctrl->sdm_lock); } kfree(cbdata); @@ -1317,7 +1336,7 @@ int stratix10_svc_async_send(struct stratix10_svc_chan *chan, void *msg, dev_dbg(ctrl->dev, "Async message sent with transaction_id 0x%02x\n", handle->transaction_id); - *handler = handle; + *handler = handle; return 0; case INTEL_SIP_SMC_STATUS_BUSY: dev_warn(ctrl->dev, "Mailbox is busy, try after some time\n"); @@ -1696,22 +1715,33 @@ int stratix10_svc_send(struct stratix10_svc_chan *chan, void *msg) if (!p_data) return -ENOMEM; - /* first client will create kernel thread */ - if (!chan->ctrl->task) { - chan->ctrl->task = - kthread_run_on_cpu(svc_normal_to_secure_thread, - (void *)chan->ctrl, - cpu, "svc_smc_hvc_thread"); - if (IS_ERR(chan->ctrl->task)) { - dev_err(chan->ctrl->dev, - "failed to create svc_smc_hvc_thread\n"); - kfree(p_data); - return -EINVAL; - } + /* first caller creates the per-channel kthread */ + if (!chan->task) { + struct task_struct *task; + + task = kthread_run_on_cpu(svc_normal_to_secure_thread, + (void *)chan, + cpu, "svc_smc_hvc_thread"); + if (IS_ERR(task)) { + dev_err(chan->ctrl->dev, + "failed to create svc_smc_hvc_thread\n"); + kfree(p_data); + return -EINVAL; + } + + spin_lock(&chan->lock); + if (chan->task) { + /* another caller won the race; discard our thread */ + spin_unlock(&chan->lock); + kthread_stop(task); + } else { + chan->task = task; + spin_unlock(&chan->lock); + } } - pr_debug("%s: sent P-va=%p, P-com=%x, P-size=%u\n", __func__, - p_msg->payload, p_msg->command, + pr_debug("%s: %s: sent P-va=%p, P-com=%x, P-size=%u\n", __func__, + chan->name, p_msg->payload, p_msg->command, (unsigned int)p_msg->payload_length); if (list_empty(&svc_data_mem)) { @@ -1747,12 +1777,16 @@ int stratix10_svc_send(struct stratix10_svc_chan *chan, void *msg) p_data->arg[2] = p_msg->arg[2]; p_data->size = p_msg->payload_length; p_data->chan = chan; - pr_debug("%s: put to FIFO pa=0x%016x, cmd=%x, size=%u\n", __func__, - (unsigned int)p_data->paddr, p_data->command, - (unsigned int)p_data->size); - ret = kfifo_in_spinlocked(&chan->ctrl->svc_fifo, p_data, + pr_debug("%s: %s: put to FIFO pa=0x%016x, cmd=%x, size=%u\n", + __func__, + chan->name, + (unsigned int)p_data->paddr, + p_data->command, + (unsigned int)p_data->size); + + ret = kfifo_in_spinlocked(&chan->svc_fifo, p_data, sizeof(*p_data), - &chan->ctrl->svc_fifo_lock); + &chan->svc_fifo_lock); kfree(p_data); @@ -1773,11 +1807,12 @@ EXPORT_SYMBOL_GPL(stratix10_svc_send); */ void stratix10_svc_done(struct stratix10_svc_chan *chan) { - /* stop thread when thread is running AND only one active client */ - if (chan->ctrl->task && chan->ctrl->num_active_client <= 1) { - pr_debug("svc_smc_hvc_shm_thread is stopped\n"); - kthread_stop(chan->ctrl->task); - chan->ctrl->task = NULL; + /* stop thread when thread is running */ + if (chan->task) { + pr_debug("%s: %s: svc_smc_hvc_shm_thread is stopping\n", + __func__, chan->name); + kthread_stop(chan->task); + chan->task = NULL; } } EXPORT_SYMBOL_GPL(stratix10_svc_done); @@ -1817,8 +1852,8 @@ void *stratix10_svc_allocate_memory(struct stratix10_svc_chan *chan, pmem->paddr = pa; pmem->size = s; list_add_tail(&pmem->node, &svc_data_mem); - pr_debug("%s: va=%p, pa=0x%016x\n", __func__, - pmem->vaddr, (unsigned int)pmem->paddr); + pr_debug("%s: %s: va=%p, pa=0x%016x\n", __func__, + chan->name, pmem->vaddr, (unsigned int)pmem->paddr); return (void *)va; } @@ -1855,6 +1890,13 @@ static const struct of_device_id stratix10_svc_drv_match[] = { {}, }; +static const char * const chan_names[SVC_NUM_CHANNEL] = { + SVC_CLIENT_FPGA, + SVC_CLIENT_RSU, + SVC_CLIENT_FCS, + SVC_CLIENT_HWMON +}; + static int stratix10_svc_drv_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; @@ -1862,11 +1904,11 @@ static int stratix10_svc_drv_probe(struct platform_device *pdev) struct stratix10_svc_chan *chans; struct gen_pool *genpool; struct stratix10_svc_sh_memory *sh_memory; - struct stratix10_svc *svc; + struct stratix10_svc *svc = NULL; svc_invoke_fn *invoke_fn; size_t fifo_size; - int ret; + int ret, i = 0; /* get SMC or HVC function */ invoke_fn = get_invoke_func(dev); @@ -1905,8 +1947,8 @@ static int stratix10_svc_drv_probe(struct platform_device *pdev) controller->num_active_client = 0; controller->chans = chans; controller->genpool = genpool; - controller->task = NULL; controller->invoke_fn = invoke_fn; + INIT_LIST_HEAD(&controller->node); init_completion(&controller->complete_status); ret = stratix10_svc_async_init(controller); @@ -1917,32 +1959,20 @@ static int stratix10_svc_drv_probe(struct platform_device *pdev) } fifo_size = sizeof(struct stratix10_svc_data) * SVC_NUM_DATA_IN_FIFO; - ret = kfifo_alloc(&controller->svc_fifo, fifo_size, GFP_KERNEL); - if (ret) { - dev_err(dev, "failed to allocate FIFO\n"); - goto err_async_exit; - } - spin_lock_init(&controller->svc_fifo_lock); - - chans[0].scl = NULL; - chans[0].ctrl = controller; - chans[0].name = SVC_CLIENT_FPGA; - spin_lock_init(&chans[0].lock); - - chans[1].scl = NULL; - chans[1].ctrl = controller; - chans[1].name = SVC_CLIENT_RSU; - spin_lock_init(&chans[1].lock); - - chans[2].scl = NULL; - chans[2].ctrl = controller; - chans[2].name = SVC_CLIENT_FCS; - spin_lock_init(&chans[2].lock); + mutex_init(&controller->sdm_lock); - chans[3].scl = NULL; - chans[3].ctrl = controller; - chans[3].name = SVC_CLIENT_HWMON; - spin_lock_init(&chans[3].lock); + for (i = 0; i < SVC_NUM_CHANNEL; i++) { + chans[i].scl = NULL; + chans[i].ctrl = controller; + chans[i].name = (char *)chan_names[i]; + spin_lock_init(&chans[i].lock); + ret = kfifo_alloc(&chans[i].svc_fifo, fifo_size, GFP_KERNEL); + if (ret) { + dev_err(dev, "failed to allocate FIFO %d\n", i); + goto err_free_fifos; + } + spin_lock_init(&chans[i].svc_fifo_lock); + } list_add_tail(&controller->node, &svc_ctrl); platform_set_drvdata(pdev, controller); @@ -1951,7 +1981,7 @@ static int stratix10_svc_drv_probe(struct platform_device *pdev) svc = devm_kzalloc(dev, sizeof(*svc), GFP_KERNEL); if (!svc) { ret = -ENOMEM; - goto err_free_kfifo; + goto err_free_fifos; } controller->svc = svc; @@ -1959,51 +1989,43 @@ static int stratix10_svc_drv_probe(struct platform_device *pdev) if (!svc->stratix10_svc_rsu) { dev_err(dev, "failed to allocate %s device\n", STRATIX10_RSU); ret = -ENOMEM; - goto err_free_kfifo; + goto err_free_fifos; } ret = platform_device_add(svc->stratix10_svc_rsu); - if (ret) { - platform_device_put(svc->stratix10_svc_rsu); - goto err_free_kfifo; - } - - svc->intel_svc_fcs = platform_device_alloc(INTEL_FCS, 1); - if (!svc->intel_svc_fcs) { - dev_err(dev, "failed to allocate %s device\n", INTEL_FCS); - ret = -ENOMEM; - goto err_unregister_rsu_dev; - } - - ret = platform_device_add(svc->intel_svc_fcs); - if (ret) { - platform_device_put(svc->intel_svc_fcs); - goto err_unregister_rsu_dev; - } + if (ret) + goto err_put_device; ret = of_platform_default_populate(dev_of_node(dev), NULL, dev); if (ret) - goto err_unregister_fcs_dev; + goto err_unregister_rsu_dev; pr_info("Intel Service Layer Driver Initialized\n"); return 0; -err_unregister_fcs_dev: - platform_device_unregister(svc->intel_svc_fcs); err_unregister_rsu_dev: platform_device_unregister(svc->stratix10_svc_rsu); -err_free_kfifo: - kfifo_free(&controller->svc_fifo); -err_async_exit: + goto err_free_fifos; +err_put_device: + platform_device_put(svc->stratix10_svc_rsu); +err_free_fifos: + /* only remove from list if list_add_tail() was reached */ + if (!list_empty(&controller->node)) + list_del(&controller->node); + /* free only the FIFOs that were successfully allocated */ + while (i--) + kfifo_free(&chans[i].svc_fifo); stratix10_svc_async_exit(controller); err_destroy_pool: gen_pool_destroy(genpool); + return ret; } static void stratix10_svc_drv_remove(struct platform_device *pdev) { + int i; struct stratix10_svc_controller *ctrl = platform_get_drvdata(pdev); struct stratix10_svc *svc = ctrl->svc; @@ -2011,14 +2033,16 @@ static void stratix10_svc_drv_remove(struct platform_device *pdev) of_platform_depopulate(ctrl->dev); - platform_device_unregister(svc->intel_svc_fcs); platform_device_unregister(svc->stratix10_svc_rsu); - kfifo_free(&ctrl->svc_fifo); - if (ctrl->task) { - kthread_stop(ctrl->task); - ctrl->task = NULL; + for (i = 0; i < SVC_NUM_CHANNEL; i++) { + if (ctrl->chans[i].task) { + kthread_stop(ctrl->chans[i].task); + ctrl->chans[i].task = NULL; + } + kfifo_free(&ctrl->chans[i].svc_fifo); } + if (ctrl->genpool) gen_pool_destroy(ctrl->genpool); list_del(&ctrl->node); diff --git a/drivers/firmware/thead,th1520-aon.c b/drivers/firmware/thead,th1520-aon.c index 38f812ac9920e3..b87d4e8235b125 100644 --- a/drivers/firmware/thead,th1520-aon.c +++ b/drivers/firmware/thead,th1520-aon.c @@ -170,10 +170,9 @@ int th1520_aon_power_update(struct th1520_aon_chan *aon_chan, u16 rsrc, hdr->func = TH1520_AON_PM_FUNC_SET_RESOURCE_POWER_MODE; hdr->size = TH1520_AON_RPC_MSG_NUM; - RPC_SET_BE16(&msg.resource, 0, rsrc); - RPC_SET_BE16(&msg.resource, 2, - (power_on ? TH1520_AON_PM_PW_MODE_ON : - TH1520_AON_PM_PW_MODE_OFF)); + msg.resource = cpu_to_be16(rsrc); + msg.mode = cpu_to_be16(power_on ? TH1520_AON_PM_PW_MODE_ON : + TH1520_AON_PM_PW_MODE_OFF); ret = th1520_aon_call_rpc(aon_chan, &msg); if (ret) diff --git a/drivers/fpga/dfl.c b/drivers/fpga/dfl.c index 7022657243c0a7..449c3a082e232b 100644 --- a/drivers/fpga/dfl.c +++ b/drivers/fpga/dfl.c @@ -2018,7 +2018,7 @@ static void __exit dfl_fpga_exit(void) bus_unregister(&dfl_bus_type); } -module_init(dfl_fpga_init); +subsys_initcall(dfl_fpga_init); module_exit(dfl_fpga_exit); MODULE_DESCRIPTION("FPGA Device Feature List (DFL) Support"); diff --git a/drivers/fpga/of-fpga-region.c b/drivers/fpga/of-fpga-region.c index 43db4bb77138a8..caa091224dc540 100644 --- a/drivers/fpga/of-fpga-region.c +++ b/drivers/fpga/of-fpga-region.c @@ -83,7 +83,7 @@ static struct fpga_manager *of_fpga_region_get_mgr(struct device_node *np) * done with the bridges. * * Return: 0 for success (even if there are no bridges specified) - * or -EBUSY if any of the bridges are in use. + * or an error code if any of the bridges are not available. */ static int of_fpga_region_get_bridges(struct fpga_region *region) { @@ -130,10 +130,10 @@ static int of_fpga_region_get_bridges(struct fpga_region *region) ®ion->bridge_list); of_node_put(br); - /* If any of the bridges are in use, give up */ - if (ret == -EBUSY) { + /* If any of the bridges are not available, give up */ + if (ret) { fpga_bridges_put(®ion->bridge_list); - return -EBUSY; + return ret; } } diff --git a/drivers/gpib/Kconfig b/drivers/gpib/Kconfig index eeb50956ce85c6..d43a28c62ed7b4 100644 --- a/drivers/gpib/Kconfig +++ b/drivers/gpib/Kconfig @@ -122,6 +122,7 @@ config GPIB_FLUKE depends on OF select GPIB_COMMON select GPIB_NEC7210 + depends on HAS_IOMEM help GPIB driver for Fluke based cda devices. diff --git a/drivers/gpib/common/gpib_os.c b/drivers/gpib/common/gpib_os.c index 9dbbac8b84360e..baa2fea5ebf7e1 100644 --- a/drivers/gpib/common/gpib_os.c +++ b/drivers/gpib/common/gpib_os.c @@ -888,10 +888,6 @@ static int read_ioctl(struct gpib_file_private *file_priv, struct gpib_board *bo if (read_cmd.completed_transfer_count > read_cmd.requested_transfer_count) return -EINVAL; - desc = handle_to_descriptor(file_priv, read_cmd.handle); - if (!desc) - return -EINVAL; - if (WARN_ON_ONCE(sizeof(userbuf) > sizeof(read_cmd.buffer_ptr))) return -EFAULT; @@ -904,6 +900,17 @@ static int read_ioctl(struct gpib_file_private *file_priv, struct gpib_board *bo if (!access_ok(userbuf, remain)) return -EFAULT; + /* Lock descriptors to prevent concurrent close from freeing descriptor */ + if (mutex_lock_interruptible(&file_priv->descriptors_mutex)) + return -ERESTARTSYS; + desc = handle_to_descriptor(file_priv, read_cmd.handle); + if (!desc) { + mutex_unlock(&file_priv->descriptors_mutex); + return -EINVAL; + } + atomic_inc(&desc->descriptor_busy); + mutex_unlock(&file_priv->descriptors_mutex); + atomic_set(&desc->io_in_progress, 1); /* Read buffer loads till we fill the user supplied buffer */ @@ -937,6 +944,7 @@ static int read_ioctl(struct gpib_file_private *file_priv, struct gpib_board *bo retval = copy_to_user((void __user *)arg, &read_cmd, sizeof(read_cmd)); atomic_set(&desc->io_in_progress, 0); + atomic_dec(&desc->descriptor_busy); wake_up_interruptible(&board->wait); if (retval) @@ -964,10 +972,6 @@ static int command_ioctl(struct gpib_file_private *file_priv, if (cmd.completed_transfer_count > cmd.requested_transfer_count) return -EINVAL; - desc = handle_to_descriptor(file_priv, cmd.handle); - if (!desc) - return -EINVAL; - userbuf = (u8 __user *)(unsigned long)cmd.buffer_ptr; userbuf += cmd.completed_transfer_count; @@ -980,6 +984,17 @@ static int command_ioctl(struct gpib_file_private *file_priv, if (!access_ok(userbuf, remain)) return -EFAULT; + /* Lock descriptors to prevent concurrent close from freeing descriptor */ + if (mutex_lock_interruptible(&file_priv->descriptors_mutex)) + return -ERESTARTSYS; + desc = handle_to_descriptor(file_priv, cmd.handle); + if (!desc) { + mutex_unlock(&file_priv->descriptors_mutex); + return -EINVAL; + } + atomic_inc(&desc->descriptor_busy); + mutex_unlock(&file_priv->descriptors_mutex); + /* * Write buffer loads till we empty the user supplied buffer. * Call drivers at least once, even if remain is zero, in @@ -1003,6 +1018,7 @@ static int command_ioctl(struct gpib_file_private *file_priv, userbuf += bytes_written; if (retval < 0) { atomic_set(&desc->io_in_progress, 0); + atomic_dec(&desc->descriptor_busy); wake_up_interruptible(&board->wait); break; @@ -1022,6 +1038,7 @@ static int command_ioctl(struct gpib_file_private *file_priv, */ if (!no_clear_io_in_prog || fault) atomic_set(&desc->io_in_progress, 0); + atomic_dec(&desc->descriptor_busy); wake_up_interruptible(&board->wait); if (fault) @@ -1047,10 +1064,6 @@ static int write_ioctl(struct gpib_file_private *file_priv, struct gpib_board *b if (write_cmd.completed_transfer_count > write_cmd.requested_transfer_count) return -EINVAL; - desc = handle_to_descriptor(file_priv, write_cmd.handle); - if (!desc) - return -EINVAL; - userbuf = (u8 __user *)(unsigned long)write_cmd.buffer_ptr; userbuf += write_cmd.completed_transfer_count; @@ -1060,6 +1073,17 @@ static int write_ioctl(struct gpib_file_private *file_priv, struct gpib_board *b if (!access_ok(userbuf, remain)) return -EFAULT; + /* Lock descriptors to prevent concurrent close from freeing descriptor */ + if (mutex_lock_interruptible(&file_priv->descriptors_mutex)) + return -ERESTARTSYS; + desc = handle_to_descriptor(file_priv, write_cmd.handle); + if (!desc) { + mutex_unlock(&file_priv->descriptors_mutex); + return -EINVAL; + } + atomic_inc(&desc->descriptor_busy); + mutex_unlock(&file_priv->descriptors_mutex); + atomic_set(&desc->io_in_progress, 1); /* Write buffer loads till we empty the user supplied buffer */ @@ -1094,6 +1118,7 @@ static int write_ioctl(struct gpib_file_private *file_priv, struct gpib_board *b fault = copy_to_user((void __user *)arg, &write_cmd, sizeof(write_cmd)); atomic_set(&desc->io_in_progress, 0); + atomic_dec(&desc->descriptor_busy); wake_up_interruptible(&board->wait); if (fault) @@ -1276,6 +1301,9 @@ static int close_dev_ioctl(struct file *filep, struct gpib_board *board, unsigne { struct gpib_close_dev_ioctl cmd; struct gpib_file_private *file_priv = filep->private_data; + struct gpib_descriptor *desc; + unsigned int pad; + int sad; int retval; retval = copy_from_user(&cmd, (void __user *)arg, sizeof(cmd)); @@ -1284,19 +1312,27 @@ static int close_dev_ioctl(struct file *filep, struct gpib_board *board, unsigne if (cmd.handle >= GPIB_MAX_NUM_DESCRIPTORS) return -EINVAL; - if (!file_priv->descriptors[cmd.handle]) - return -EINVAL; - retval = decrement_open_device_count(board, &board->device_list, - file_priv->descriptors[cmd.handle]->pad, - file_priv->descriptors[cmd.handle]->sad); - if (retval < 0) - return retval; - - kfree(file_priv->descriptors[cmd.handle]); + mutex_lock(&file_priv->descriptors_mutex); + desc = file_priv->descriptors[cmd.handle]; + if (!desc) { + mutex_unlock(&file_priv->descriptors_mutex); + return -EINVAL; + } + if (atomic_read(&desc->descriptor_busy)) { + mutex_unlock(&file_priv->descriptors_mutex); + return -EBUSY; + } + /* Remove from table while holding lock to prevent new IO from starting */ file_priv->descriptors[cmd.handle] = NULL; + pad = desc->pad; + sad = desc->sad; + mutex_unlock(&file_priv->descriptors_mutex); - return 0; + retval = decrement_open_device_count(board, &board->device_list, pad, sad); + + kfree(desc); + return retval; } static int serial_poll_ioctl(struct gpib_board *board, unsigned long arg) @@ -1331,12 +1367,25 @@ static int wait_ioctl(struct gpib_file_private *file_priv, struct gpib_board *bo if (retval) return -EFAULT; + /* + * Lock descriptors to prevent concurrent close from freeing + * descriptor. ibwait() releases big_gpib_mutex when wait_mask + * is non-zero, so desc must be pinned with descriptor_busy. + */ + mutex_lock(&file_priv->descriptors_mutex); desc = handle_to_descriptor(file_priv, wait_cmd.handle); - if (!desc) + if (!desc) { + mutex_unlock(&file_priv->descriptors_mutex); return -EINVAL; + } + atomic_inc(&desc->descriptor_busy); + mutex_unlock(&file_priv->descriptors_mutex); retval = ibwait(board, wait_cmd.wait_mask, wait_cmd.clear_mask, wait_cmd.set_mask, &wait_cmd.ibsta, wait_cmd.usec_timeout, desc); + + atomic_dec(&desc->descriptor_busy); + if (retval < 0) return retval; @@ -2035,6 +2084,7 @@ void init_gpib_descriptor(struct gpib_descriptor *desc) desc->is_board = 0; desc->autopoll_enabled = 0; atomic_set(&desc->io_in_progress, 0); + atomic_set(&desc->descriptor_busy, 0); } int gpib_register_driver(struct gpib_interface *interface, struct module *provider_module) diff --git a/drivers/gpib/common/iblib.c b/drivers/gpib/common/iblib.c index 7cbb6a467177da..b672dd6aad25f2 100644 --- a/drivers/gpib/common/iblib.c +++ b/drivers/gpib/common/iblib.c @@ -227,11 +227,10 @@ int ibonline(struct gpib_board *board) #ifndef CONFIG_NIOS2 board->autospoll_task = kthread_run(&autospoll_thread, board, "gpib%d_autospoll_kthread", board->minor); - retval = IS_ERR(board->autospoll_task); - if (retval) { + if (IS_ERR(board->autospoll_task)) { dev_err(board->gpib_dev, "failed to create autospoll thread\n"); board->interface->detach(board); - return retval; + return PTR_ERR(board->autospoll_task); } #endif board->online = 1; diff --git a/drivers/gpib/include/gpib_types.h b/drivers/gpib/include/gpib_types.h index 5a0978ae27e791..28b73157ffb7e8 100644 --- a/drivers/gpib/include/gpib_types.h +++ b/drivers/gpib/include/gpib_types.h @@ -364,6 +364,14 @@ struct gpib_descriptor { unsigned int pad; /* primary gpib address */ int sad; /* secondary gpib address (negative means disabled) */ atomic_t io_in_progress; + /* + * Kernel-only reference count to prevent descriptor from being + * freed while IO handlers hold a pointer to it. Incremented + * before each IO operation, decremented when done. Unlike + * io_in_progress, this cannot be modified from userspace via + * general_ibstatus(). + */ + atomic_t descriptor_busy; unsigned is_board : 1; unsigned autopoll_enabled : 1; }; diff --git a/drivers/gpib/lpvo_usb_gpib/lpvo_usb_gpib.c b/drivers/gpib/lpvo_usb_gpib/lpvo_usb_gpib.c index dd68c4843490a9..482369f3f502e7 100644 --- a/drivers/gpib/lpvo_usb_gpib/lpvo_usb_gpib.c +++ b/drivers/gpib/lpvo_usb_gpib/lpvo_usb_gpib.c @@ -38,8 +38,10 @@ MODULE_DESCRIPTION("GPIB driver for LPVO usb devices"); /* * Table of devices that work with this driver. * - * Currently, only one device is known to be used in the - * lpvo_usb_gpib adapter (FTDI 0403:6001). + * Currently, only one device is known to be used in the lpvo_usb_gpib + * adapter (FTDI 0403:6001) but as this device id is already handled by the + * ftdi_sio USB serial driver the LPVO driver must not bind to it by default. + * * If your adapter uses a different chip, insert a line * in the following table with proper , . * @@ -50,7 +52,6 @@ MODULE_DESCRIPTION("GPIB driver for LPVO usb devices"); */ static const struct usb_device_id skel_table[] = { - { USB_DEVICE(0x0403, 0x6001) }, { } /* Terminating entry */ }; MODULE_DEVICE_TABLE(usb, skel_table); @@ -405,7 +406,7 @@ static int usb_gpib_attach(struct gpib_board *board, const struct gpib_board_con for (j = 0 ; j < MAX_DEV ; j++) { if ((assigned_usb_minors & 1 << j) == 0) continue; - udev = usb_get_dev(interface_to_usbdev(lpvo_usb_interfaces[j])); + udev = interface_to_usbdev(lpvo_usb_interfaces[j]); device_path = kobject_get_path(&udev->dev.kobj, GFP_KERNEL); match = gpib_match_device_path(&lpvo_usb_interfaces[j]->dev, config->device_path); @@ -420,7 +421,7 @@ static int usb_gpib_attach(struct gpib_board *board, const struct gpib_board_con for (j = 0 ; j < MAX_DEV ; j++) { if ((assigned_usb_minors & 1 << j) == 0) continue; - udev = usb_get_dev(interface_to_usbdev(lpvo_usb_interfaces[j])); + udev = interface_to_usbdev(lpvo_usb_interfaces[j]); DIA_LOG(1, "dev. %d: bus %d -> %d dev: %d -> %d\n", j, udev->bus->busnum, config->pci_bus, udev->devnum, config->pci_slot); if (config->pci_bus == udev->bus->busnum && diff --git a/drivers/gpib/ni_usb/ni_usb_gpib.c b/drivers/gpib/ni_usb/ni_usb_gpib.c index 1f8412de9fa329..b6fddb437f5524 100644 --- a/drivers/gpib/ni_usb/ni_usb_gpib.c +++ b/drivers/gpib/ni_usb/ni_usb_gpib.c @@ -566,7 +566,7 @@ static int ni_usb_write_registers(struct ni_usb_priv *ni_priv, retval, bytes_read); ni_usb_dump_raw_block(in_data, bytes_read); kfree(in_data); - return retval; + return retval ?: -EINVAL; } mutex_unlock(&ni_priv->addressed_transfer_lock); @@ -1780,7 +1780,7 @@ static int ni_usb_setup_init(struct gpib_board *board, struct ni_usb_register *w i++; if (i > NUM_INIT_WRITES) { dev_err(&usb_dev->dev, "bug!, buffer overrun, i=%i\n", i); - return 0; + return -EINVAL; } return i; } @@ -1799,10 +1799,12 @@ static int ni_usb_init(struct gpib_board *board) return -ENOMEM; writes_len = ni_usb_setup_init(board, writes); - if (writes_len) - retval = ni_usb_write_registers(ni_priv, writes, writes_len, &ibsta); - else - return -EFAULT; + if (writes_len < 0) { + kfree(writes); + return writes_len; + } + + retval = ni_usb_write_registers(ni_priv, writes, writes_len, &ibsta); kfree(writes); if (retval) { dev_err(&usb_dev->dev, "register write failed, retval=%i\n", retval); diff --git a/drivers/gpio/Kconfig b/drivers/gpio/Kconfig index bd185482a7fdf1..3439e025ba1c64 100644 --- a/drivers/gpio/Kconfig +++ b/drivers/gpio/Kconfig @@ -1193,11 +1193,11 @@ config GPIO_PCA953X 8 bits: max7310, max7315, pca6107, pca9534, pca9538, pca9554, pca9556, pca9557, pca9574, tca6408, tca9554, xra1202, - pcal6408, pcal9554b, tca9538 + pcal6408, pcal9554b, tca9538, tcal6408 16 bits: max7312, max7313, pca9535, pca9539, pca9555, pca9575, tca6416, pca6416, pcal6416, pcal9535, pcal9555a, max7318, - tca9539 + tca9539, tcal6416 18 bits: tca6418 diff --git a/drivers/gpio/gpio-amd-fch.c b/drivers/gpio/gpio-amd-fch.c index e6c6c3ec7656e9..9f329938202bff 100644 --- a/drivers/gpio/gpio-amd-fch.c +++ b/drivers/gpio/gpio-amd-fch.c @@ -8,6 +8,7 @@ * */ +#include #include #include #include @@ -120,15 +121,15 @@ static int amd_fch_gpio_get(struct gpio_chip *gc, unsigned int offset) { unsigned long flags; - int ret; + u32 val; struct amd_fch_gpio_priv *priv = gpiochip_get_data(gc); void __iomem *ptr = amd_fch_gpio_addr(priv, offset); spin_lock_irqsave(&priv->lock, flags); - ret = (readl_relaxed(ptr) & AMD_FCH_GPIO_FLAG_READ); + val = readl_relaxed(ptr); spin_unlock_irqrestore(&priv->lock, flags); - return ret; + return FIELD_GET(AMD_FCH_GPIO_FLAG_READ, val); } static int amd_fch_gpio_request(struct gpio_chip *chip, diff --git a/drivers/gpio/gpio-aspeed-sgpio.c b/drivers/gpio/gpio-aspeed-sgpio.c index 7622f9e9f54af5..318cd0e3974168 100644 --- a/drivers/gpio/gpio-aspeed-sgpio.c +++ b/drivers/gpio/gpio-aspeed-sgpio.c @@ -516,7 +516,7 @@ static const struct of_device_id aspeed_sgpio_of_table[] = { MODULE_DEVICE_TABLE(of, aspeed_sgpio_of_table); -static int __init aspeed_sgpio_probe(struct platform_device *pdev) +static int aspeed_sgpio_probe(struct platform_device *pdev) { u32 nr_gpios, sgpio_freq, sgpio_clk_div, gpio_cnt_regval, pin_mask; const struct aspeed_sgpio_pdata *pdata; @@ -611,11 +611,12 @@ static int __init aspeed_sgpio_probe(struct platform_device *pdev) } static struct platform_driver aspeed_sgpio_driver = { + .probe = aspeed_sgpio_probe, .driver = { .name = KBUILD_MODNAME, .of_match_table = aspeed_sgpio_of_table, }, }; -module_platform_driver_probe(aspeed_sgpio_driver, aspeed_sgpio_probe); +module_platform_driver(aspeed_sgpio_driver); MODULE_DESCRIPTION("Aspeed Serial GPIO Driver"); diff --git a/drivers/gpio/gpio-mxc.c b/drivers/gpio/gpio-mxc.c index d7666fe9dbf86e..647b6f4861b744 100644 --- a/drivers/gpio/gpio-mxc.c +++ b/drivers/gpio/gpio-mxc.c @@ -584,12 +584,13 @@ static bool mxc_gpio_set_pad_wakeup(struct mxc_gpio_port *port, bool enable) unsigned long config; bool ret = false; int i, type; + bool is_imx8qm = of_device_is_compatible(port->dev->of_node, "fsl,imx8qm-gpio"); static const u32 pad_type_map[] = { IMX_SCU_WAKEUP_OFF, /* 0 */ IMX_SCU_WAKEUP_RISE_EDGE, /* IRQ_TYPE_EDGE_RISING */ IMX_SCU_WAKEUP_FALL_EDGE, /* IRQ_TYPE_EDGE_FALLING */ - IMX_SCU_WAKEUP_FALL_EDGE, /* IRQ_TYPE_EDGE_BOTH */ + IMX_SCU_WAKEUP_RISE_EDGE, /* IRQ_TYPE_EDGE_BOTH */ IMX_SCU_WAKEUP_HIGH_LVL, /* IRQ_TYPE_LEVEL_HIGH */ IMX_SCU_WAKEUP_OFF, /* 5 */ IMX_SCU_WAKEUP_OFF, /* 6 */ @@ -604,6 +605,13 @@ static bool mxc_gpio_set_pad_wakeup(struct mxc_gpio_port *port, bool enable) config = pad_type_map[type]; else config = IMX_SCU_WAKEUP_OFF; + + if (is_imx8qm && config == IMX_SCU_WAKEUP_FALL_EDGE) { + dev_warn_once(port->dev, + "No falling-edge support for wakeup on i.MX8QM\n"); + config = IMX_SCU_WAKEUP_OFF; + } + ret |= mxc_gpio_generic_config(port, i, config); } } diff --git a/drivers/gpio/gpio-nomadik.c b/drivers/gpio/gpio-nomadik.c index 97c5cd33279d56..e22b713166d71d 100644 --- a/drivers/gpio/gpio-nomadik.c +++ b/drivers/gpio/gpio-nomadik.c @@ -430,6 +430,9 @@ void nmk_gpio_dbg_show_one(struct seq_file *s, struct pinctrl_dev *pctldev, #ifdef CONFIG_PINCTRL_NOMADIK if (mode == NMK_GPIO_ALT_C && pctldev) { desc = gpio_device_get_desc(chip->gpiodev, offset); + if (IS_ERR(desc)) + return; + mode = nmk_prcm_gpiocr_get_mode(pctldev, desc_to_gpio(desc)); } #endif diff --git a/drivers/gpio/gpio-pca953x.c b/drivers/gpio/gpio-pca953x.c index f93a3dbb2daaf3..52e96cc5f67bb6 100644 --- a/drivers/gpio/gpio-pca953x.c +++ b/drivers/gpio/gpio-pca953x.c @@ -126,6 +126,9 @@ static const struct i2c_device_id pca953x_id[] = { { "tca9539", 16 | PCA953X_TYPE | PCA_INT, }, { "tca9554", 8 | PCA953X_TYPE | PCA_INT, }, { "xra1202", 8 | PCA953X_TYPE }, + + { "tcal6408", 8 | PCA953X_TYPE | PCA_LATCH_INT, }, + { "tcal6416", 16 | PCA953X_TYPE | PCA_LATCH_INT, }, { } }; MODULE_DEVICE_TABLE(i2c, pca953x_id); @@ -1469,6 +1472,9 @@ static const struct of_device_id pca953x_dt_ids[] = { { .compatible = "ti,tca9538", .data = OF_953X( 8, PCA_INT), }, { .compatible = "ti,tca9539", .data = OF_953X(16, PCA_INT), }, + { .compatible = "ti,tcal6408", .data = OF_953X( 8, PCA_LATCH_INT), }, + { .compatible = "ti,tcal6416", .data = OF_953X(16, PCA_LATCH_INT), }, + { .compatible = "onnn,cat9554", .data = OF_953X( 8, PCA_INT), }, { .compatible = "onnn,pca9654", .data = OF_953X( 8, PCA_INT), }, { .compatible = "onnn,pca9655", .data = OF_953X(16, PCA_INT), }, diff --git a/drivers/gpio/gpio-qixis-fpga.c b/drivers/gpio/gpio-qixis-fpga.c index 6e67f43ac0bdd4..3ced47db1521c8 100644 --- a/drivers/gpio/gpio-qixis-fpga.c +++ b/drivers/gpio/gpio-qixis-fpga.c @@ -60,8 +60,8 @@ static int qixis_cpld_gpio_probe(struct platform_device *pdev) return PTR_ERR(reg); regmap = devm_regmap_init_mmio(&pdev->dev, reg, ®map_config_8r_8v); - if (!regmap) - return -ENODEV; + if (IS_ERR(regmap)) + return PTR_ERR(regmap); /* In this case, the offset of our register is 0 inside the * regmap area that we just created. diff --git a/drivers/gpio/gpio-tegra.c b/drivers/gpio/gpio-tegra.c index 15a5762a82c25b..b14052fe64ac69 100644 --- a/drivers/gpio/gpio-tegra.c +++ b/drivers/gpio/gpio-tegra.c @@ -595,7 +595,7 @@ static void tegra_gpio_irq_release_resources(struct irq_data *d) struct tegra_gpio_info *tgi = gpiochip_get_data(chip); gpiochip_relres_irq(chip, d->hwirq); - tegra_gpio_enable(tgi, d->hwirq); + tegra_gpio_disable(tgi, d->hwirq); } static void tegra_gpio_irq_print_chip(struct irq_data *d, struct seq_file *s) diff --git a/drivers/gpio/gpiolib-cdev.c b/drivers/gpio/gpiolib-cdev.c index 2adc3c07090822..189127721e3832 100644 --- a/drivers/gpio/gpiolib-cdev.c +++ b/drivers/gpio/gpiolib-cdev.c @@ -388,7 +388,7 @@ static int linehandle_create(struct gpio_device *gdev, void __user *ip) fd_publish(fdf); dev_dbg(&gdev->dev, "registered chardev handle for %d lines\n", - lh->num_descs); + handlereq.lines); return 0; } diff --git a/drivers/gpio/gpiolib-shared.c b/drivers/gpio/gpiolib-shared.c index 9e654420343932..692f568ffe7a4b 100644 --- a/drivers/gpio/gpiolib-shared.c +++ b/drivers/gpio/gpiolib-shared.c @@ -443,8 +443,8 @@ static bool gpio_shared_dev_is_reset_gpio(struct device *consumer, } #endif /* CONFIG_RESET_GPIO */ -int gpio_shared_add_proxy_lookup(struct device *consumer, const char *con_id, - unsigned long lflags) +int gpio_shared_add_proxy_lookup(struct device *consumer, struct fwnode_handle *fwnode, + const char *con_id, unsigned long lflags) { const char *dev_id = dev_name(consumer); struct gpiod_lookup_table *lookup; @@ -463,7 +463,7 @@ int gpio_shared_add_proxy_lookup(struct device *consumer, const char *con_id, if (!ref->fwnode && strstarts(dev_name(consumer), "reset.gpio.")) { if (!gpio_shared_dev_is_reset_gpio(consumer, entry, ref)) continue; - } else if (!device_match_fwnode(consumer, ref->fwnode)) { + } else if (fwnode != ref->fwnode) { continue; } @@ -511,8 +511,9 @@ static void gpio_shared_remove_adev(struct auxiliary_device *adev) auxiliary_device_uninit(adev); } -int gpio_device_setup_shared(struct gpio_device *gdev) +int gpiochip_setup_shared(struct gpio_chip *gc) { + struct gpio_device *gdev = gc->gpiodev; struct gpio_shared_entry *entry; struct gpio_shared_ref *ref; struct gpio_desc *desc; @@ -543,20 +544,42 @@ int gpio_device_setup_shared(struct gpio_device *gdev) if (list_count_nodes(&entry->refs) <= 1) continue; - desc = &gdev->descs[entry->offset]; + scoped_guard(mutex, &entry->lock) { +#if IS_ENABLED(CONFIG_OF) + if (is_of_node(entry->fwnode) && gc->of_xlate) { + /* + * This is the earliest that we can tranlate the + * devicetree offset to the chip offset. + */ + struct of_phandle_args gpiospec = { }; - __set_bit(GPIOD_FLAG_SHARED, &desc->flags); - /* - * Shared GPIOs are not requested via the normal path. Make - * them inaccessible to anyone even before we register the - * chip. - */ - ret = gpiod_request_commit(desc, "shared"); - if (ret) - return ret; + gpiospec.np = to_of_node(entry->fwnode); + gpiospec.args_count = 2; + gpiospec.args[0] = entry->offset; + + ret = gc->of_xlate(gc, &gpiospec, NULL); + if (ret < 0) + return ret; + + entry->offset = ret; + } +#endif /* CONFIG_OF */ - pr_debug("GPIO %u owned by %s is shared by multiple consumers\n", - entry->offset, gpio_device_get_label(gdev)); + desc = &gdev->descs[entry->offset]; + + __set_bit(GPIOD_FLAG_SHARED, &desc->flags); + /* + * Shared GPIOs are not requested via the normal path. Make + * them inaccessible to anyone even before we register the + * chip. + */ + ret = gpiod_request_commit(desc, "shared"); + if (ret) + return ret; + + pr_debug("GPIO %u owned by %s is shared by multiple consumers\n", + entry->offset, gpio_device_get_label(gdev)); + } list_for_each_entry(ref, &entry->refs, list) { pr_debug("Setting up a shared GPIO entry for %s (con_id: '%s')\n", @@ -580,6 +603,8 @@ void gpio_device_teardown_shared(struct gpio_device *gdev) struct gpio_shared_ref *ref; list_for_each_entry(entry, &gpio_shared_list, list) { + guard(mutex)(&entry->lock); + if (!device_match_fwnode(&gdev->dev, entry->fwnode)) continue; @@ -753,14 +778,14 @@ static bool gpio_shared_entry_is_really_shared(struct gpio_shared_entry *entry) static void gpio_shared_free_exclusive(void) { struct gpio_shared_entry *entry, *epos; + struct gpio_shared_ref *ref, *rpos; list_for_each_entry_safe(entry, epos, &gpio_shared_list, list) { if (gpio_shared_entry_is_really_shared(entry)) continue; - gpio_shared_drop_ref(list_first_entry(&entry->refs, - struct gpio_shared_ref, - list)); + list_for_each_entry_safe(ref, rpos, &entry->refs, list) + gpio_shared_drop_ref(ref); gpio_shared_drop_entry(entry); } } diff --git a/drivers/gpio/gpiolib-shared.h b/drivers/gpio/gpiolib-shared.h index 40568ef7364ccb..15e72a8dcdb138 100644 --- a/drivers/gpio/gpiolib-shared.h +++ b/drivers/gpio/gpiolib-shared.h @@ -11,17 +11,19 @@ struct gpio_device; struct gpio_desc; struct device; +struct fwnode_handle; #if IS_ENABLED(CONFIG_GPIO_SHARED) -int gpio_device_setup_shared(struct gpio_device *gdev); +int gpiochip_setup_shared(struct gpio_chip *gc); void gpio_device_teardown_shared(struct gpio_device *gdev); -int gpio_shared_add_proxy_lookup(struct device *consumer, const char *con_id, - unsigned long lflags); +int gpio_shared_add_proxy_lookup(struct device *consumer, + struct fwnode_handle *fwnode, + const char *con_id, unsigned long lflags); #else -static inline int gpio_device_setup_shared(struct gpio_device *gdev) +static inline int gpiochip_setup_shared(struct gpio_chip *gc) { return 0; } @@ -29,6 +31,7 @@ static inline int gpio_device_setup_shared(struct gpio_device *gdev) static inline void gpio_device_teardown_shared(struct gpio_device *gdev) { } static inline int gpio_shared_add_proxy_lookup(struct device *consumer, + struct fwnode_handle *fwnode, const char *con_id, unsigned long lflags) { diff --git a/drivers/gpio/gpiolib-swnode.c b/drivers/gpio/gpiolib-swnode.c index b44f35d6845909..f02f5a61ddb83f 100644 --- a/drivers/gpio/gpiolib-swnode.c +++ b/drivers/gpio/gpiolib-swnode.c @@ -43,6 +43,25 @@ static struct gpio_device *swnode_get_gpio_device(struct fwnode_handle *fwnode) fwnode_lookup: gdev = gpio_device_find_by_fwnode(fwnode); + if (!gdev && gdev_node && gdev_node->name) + /* + * FIXME: We shouldn't need to compare the GPIO controller's + * label against the software node that is supposedly attached + * to it. However there are currently GPIO users that - knowing + * the expected label of the GPIO chip whose pins they want to + * control - set up dummy software nodes named after those GPIO + * controllers, which aren't actually attached to them. In this + * case gpio_device_find_by_fwnode() will fail as no device on + * the GPIO bus is actually associated with the fwnode we're + * looking for. + * + * As a fallback: continue checking the label if we have no + * match. However, the situation described above is an abuse + * of the software node API and should be phased out and the + * following line - eventually removed. + */ + gdev = gpio_device_find_by_label(gdev_node->name); + return gdev ?: ERR_PTR(-EPROBE_DEFER); } diff --git a/drivers/gpio/gpiolib-sysfs.c b/drivers/gpio/gpiolib-sysfs.c index cd553acf3055ed..d4a46a0a37d8f0 100644 --- a/drivers/gpio/gpiolib-sysfs.c +++ b/drivers/gpio/gpiolib-sysfs.c @@ -919,63 +919,68 @@ int gpiod_export_link(struct device *dev, const char *name, } EXPORT_SYMBOL_GPL(gpiod_export_link); -/** - * gpiod_unexport - reverse effect of gpiod_export() - * @desc: GPIO to make unavailable - * - * This is implicit on gpiod_free(). - */ -void gpiod_unexport(struct gpio_desc *desc) +static void gpiod_unexport_unlocked(struct gpio_desc *desc) { struct gpiod_data *tmp, *desc_data = NULL; struct gpiodev_data *gdev_data; struct gpio_device *gdev; - if (!desc) { - pr_warn("%s: invalid GPIO\n", __func__); + if (!test_bit(GPIOD_FLAG_EXPORT, &desc->flags)) return; - } - scoped_guard(mutex, &sysfs_lock) { - if (!test_bit(GPIOD_FLAG_EXPORT, &desc->flags)) - return; - - gdev = gpiod_to_gpio_device(desc); - gdev_data = gdev_get_data(gdev); - if (!gdev_data) - return; + gdev = gpiod_to_gpio_device(desc); + gdev_data = gdev_get_data(gdev); + if (!gdev_data) + return; - list_for_each_entry(tmp, &gdev_data->exported_lines, list) { - if (gpiod_is_equal(desc, tmp->desc)) { - desc_data = tmp; - break; - } + list_for_each_entry(tmp, &gdev_data->exported_lines, list) { + if (gpiod_is_equal(desc, tmp->desc)) { + desc_data = tmp; + break; } + } - if (!desc_data) - return; + if (!desc_data) + return; - list_del(&desc_data->list); - clear_bit(GPIOD_FLAG_EXPORT, &desc->flags); + list_del(&desc_data->list); + clear_bit(GPIOD_FLAG_EXPORT, &desc->flags); #if IS_ENABLED(CONFIG_GPIO_SYSFS_LEGACY) - sysfs_put(desc_data->value_kn); - device_unregister(desc_data->dev); - - /* - * Release irq after deregistration to prevent race with - * edge_store. - */ - if (desc_data->irq_flags) - gpio_sysfs_free_irq(desc_data); + sysfs_put(desc_data->value_kn); + device_unregister(desc_data->dev); + + /* + * Release irq after deregistration to prevent race with + * edge_store. + */ + if (desc_data->irq_flags) + gpio_sysfs_free_irq(desc_data); #endif /* CONFIG_GPIO_SYSFS_LEGACY */ - sysfs_remove_groups(desc_data->parent, - desc_data->chip_attr_groups); - } + sysfs_remove_groups(desc_data->parent, + desc_data->chip_attr_groups); mutex_destroy(&desc_data->mutex); kfree(desc_data); } + +/** + * gpiod_unexport - reverse effect of gpiod_export() + * @desc: GPIO to make unavailable + * + * This is implicit on gpiod_free(). + */ +void gpiod_unexport(struct gpio_desc *desc) +{ + if (!desc) { + pr_warn("%s: invalid GPIO\n", __func__); + return; + } + + guard(mutex)(&sysfs_lock); + + gpiod_unexport_unlocked(desc); +} EXPORT_SYMBOL_GPL(gpiod_unexport); int gpiochip_sysfs_register(struct gpio_device *gdev) @@ -1054,29 +1059,28 @@ void gpiochip_sysfs_unregister(struct gpio_device *gdev) struct gpio_desc *desc; struct gpio_chip *chip; - scoped_guard(mutex, &sysfs_lock) { - data = gdev_get_data(gdev); - if (!data) - return; + guard(mutex)(&sysfs_lock); -#if IS_ENABLED(CONFIG_GPIO_SYSFS_LEGACY) - device_unregister(data->cdev_base); -#endif /* CONFIG_GPIO_SYSFS_LEGACY */ - device_unregister(data->cdev_id); - kfree(data); - } + data = gdev_get_data(gdev); + if (!data) + return; guard(srcu)(&gdev->srcu); - chip = srcu_dereference(gdev->chip, &gdev->srcu); if (!chip) return; /* unregister gpiod class devices owned by sysfs */ for_each_gpio_desc_with_flag(chip, desc, GPIOD_FLAG_SYSFS) { - gpiod_unexport(desc); + gpiod_unexport_unlocked(desc); gpiod_free(desc); } + +#if IS_ENABLED(CONFIG_GPIO_SYSFS_LEGACY) + device_unregister(data->cdev_base); +#endif /* CONFIG_GPIO_SYSFS_LEGACY */ + device_unregister(data->cdev_id); + kfree(data); } /* diff --git a/drivers/gpio/gpiolib.c b/drivers/gpio/gpiolib.c index 1578cf3a8c74e8..ef3d66857ad6ad 100644 --- a/drivers/gpio/gpiolib.c +++ b/drivers/gpio/gpiolib.c @@ -893,13 +893,15 @@ static const struct device_type gpio_dev_type = { #define gcdev_unregister(gdev) device_del(&(gdev)->dev) #endif +/* + * An initial reference count has been held in gpiochip_add_data_with_key(). + * The caller should drop the reference via gpio_device_put() on errors. + */ static int gpiochip_setup_dev(struct gpio_device *gdev) { struct fwnode_handle *fwnode = dev_fwnode(&gdev->dev); int ret; - device_initialize(&gdev->dev); - /* * If fwnode doesn't belong to another device, it's safe to clear its * initialized flag. @@ -965,9 +967,11 @@ static void gpiochip_setup_devs(void) list_for_each_entry_srcu(gdev, &gpio_devices, list, srcu_read_lock_held(&gpio_devices_srcu)) { ret = gpiochip_setup_dev(gdev); - if (ret) + if (ret) { + gpio_device_put(gdev); dev_err(&gdev->dev, "Failed to initialize gpio device (%d)\n", ret); + } } } @@ -1048,71 +1052,72 @@ int gpiochip_add_data_with_key(struct gpio_chip *gc, void *data, int base = 0; int ret; - /* - * First: allocate and populate the internal stat container, and - * set up the struct device. - */ gdev = kzalloc(sizeof(*gdev), GFP_KERNEL); if (!gdev) return -ENOMEM; - - gdev->dev.type = &gpio_dev_type; - gdev->dev.bus = &gpio_bus_type; - gdev->dev.parent = gc->parent; - rcu_assign_pointer(gdev->chip, gc); - gc->gpiodev = gdev; gpiochip_set_data(gc, data); - device_set_node(&gdev->dev, gpiochip_choose_fwnode(gc)); - ret = ida_alloc(&gpio_ida, GFP_KERNEL); if (ret < 0) goto err_free_gdev; gdev->id = ret; - ret = dev_set_name(&gdev->dev, GPIOCHIP_NAME "%d", gdev->id); + ret = init_srcu_struct(&gdev->srcu); if (ret) goto err_free_ida; + rcu_assign_pointer(gdev->chip, gc); - if (gc->parent && gc->parent->driver) - gdev->owner = gc->parent->driver->owner; - else if (gc->owner) - /* TODO: remove chip->owner */ - gdev->owner = gc->owner; - else - gdev->owner = THIS_MODULE; + ret = init_srcu_struct(&gdev->desc_srcu); + if (ret) + goto err_cleanup_gdev_srcu; + + ret = dev_set_name(&gdev->dev, GPIOCHIP_NAME "%d", gdev->id); + if (ret) + goto err_cleanup_desc_srcu; + + device_initialize(&gdev->dev); + /* + * After this point any allocated resources to `gdev` will be + * free():ed by gpiodev_release(). If you add new resources + * then make sure they get free():ed there. + */ + gdev->dev.type = &gpio_dev_type; + gdev->dev.bus = &gpio_bus_type; + gdev->dev.parent = gc->parent; + device_set_node(&gdev->dev, gpiochip_choose_fwnode(gc)); ret = gpiochip_get_ngpios(gc, &gdev->dev); if (ret) - goto err_free_dev_name; + goto err_put_device; + gdev->ngpio = gc->ngpio; gdev->descs = kcalloc(gc->ngpio, sizeof(*gdev->descs), GFP_KERNEL); if (!gdev->descs) { ret = -ENOMEM; - goto err_free_dev_name; + goto err_put_device; } gdev->label = kstrdup_const(gc->label ?: "unknown", GFP_KERNEL); if (!gdev->label) { ret = -ENOMEM; - goto err_free_descs; + goto err_put_device; } - gdev->ngpio = gc->ngpio; gdev->can_sleep = gc->can_sleep; - rwlock_init(&gdev->line_state_lock); RAW_INIT_NOTIFIER_HEAD(&gdev->line_state_notifier); BLOCKING_INIT_NOTIFIER_HEAD(&gdev->device_notifier); - - ret = init_srcu_struct(&gdev->srcu); - if (ret) - goto err_free_label; - - ret = init_srcu_struct(&gdev->desc_srcu); - if (ret) - goto err_cleanup_gdev_srcu; +#ifdef CONFIG_PINCTRL + INIT_LIST_HEAD(&gdev->pin_ranges); +#endif + if (gc->parent && gc->parent->driver) + gdev->owner = gc->parent->driver->owner; + else if (gc->owner) + /* TODO: remove chip->owner */ + gdev->owner = gc->owner; + else + gdev->owner = THIS_MODULE; scoped_guard(mutex, &gpio_devices_lock) { /* @@ -1128,7 +1133,7 @@ int gpiochip_add_data_with_key(struct gpio_chip *gc, void *data, if (base < 0) { ret = base; base = 0; - goto err_cleanup_desc_srcu; + goto err_put_device; } /* @@ -1148,14 +1153,10 @@ int gpiochip_add_data_with_key(struct gpio_chip *gc, void *data, ret = gpiodev_add_to_list_unlocked(gdev); if (ret) { gpiochip_err(gc, "GPIO integer space overlap, cannot add chip\n"); - goto err_cleanup_desc_srcu; + goto err_put_device; } } -#ifdef CONFIG_PINCTRL - INIT_LIST_HEAD(&gdev->pin_ranges); -#endif - if (gc->names) gpiochip_set_desc_names(gc); @@ -1211,7 +1212,7 @@ int gpiochip_add_data_with_key(struct gpio_chip *gc, void *data, if (ret) goto err_remove_irqchip_mask; - ret = gpio_device_setup_shared(gdev); + ret = gpiochip_setup_shared(gc); if (ret) goto err_remove_irqchip; @@ -1249,25 +1250,19 @@ int gpiochip_add_data_with_key(struct gpio_chip *gc, void *data, scoped_guard(mutex, &gpio_devices_lock) list_del_rcu(&gdev->list); synchronize_srcu(&gpio_devices_srcu); - if (gdev->dev.release) { - /* release() has been registered by gpiochip_setup_dev() */ - gpio_device_put(gdev); - goto err_print_message; - } +err_put_device: + gpio_device_put(gdev); + goto err_print_message; + err_cleanup_desc_srcu: cleanup_srcu_struct(&gdev->desc_srcu); err_cleanup_gdev_srcu: cleanup_srcu_struct(&gdev->srcu); -err_free_label: - kfree_const(gdev->label); -err_free_descs: - kfree(gdev->descs); -err_free_dev_name: - kfree(dev_name(&gdev->dev)); err_free_ida: ida_free(&gpio_ida, gdev->id); err_free_gdev: kfree(gdev); + err_print_message: /* failures here can mean systems won't boot... */ if (ret != -EPROBE_DEFER) { @@ -2466,8 +2461,10 @@ int gpiod_request_commit(struct gpio_desc *desc, const char *label) return -EBUSY; offset = gpiod_hwgpio(desc); - if (!gpiochip_line_is_valid(guard.gc, offset)) - return -EINVAL; + if (!gpiochip_line_is_valid(guard.gc, offset)) { + ret = -EINVAL; + goto out_clear_bit; + } /* NOTE: gpio_request() can be called in early boot, * before IRQs are enabled, for non-sleeping (SOC) GPIOs. @@ -3268,8 +3265,12 @@ static int gpiochip_get(struct gpio_chip *gc, unsigned int offset) /* Make sure this is called after checking for gc->get(). */ ret = gc->get(gc, offset); - if (ret > 1) - ret = -EBADE; + if (ret > 1) { + gpiochip_warn(gc, + "invalid return value from gc->get(): %d, consider fixing the driver\n", + ret); + ret = !!ret; + } return ret; } @@ -4714,8 +4715,8 @@ struct gpio_desc *gpiod_find_and_request(struct device *consumer, * lookup table for the proxy device as previously * we only knew the consumer's fwnode. */ - ret = gpio_shared_add_proxy_lookup(consumer, con_id, - lookupflags); + ret = gpio_shared_add_proxy_lookup(consumer, fwnode, + con_id, lookupflags); if (ret) return ERR_PTR(ret); diff --git a/drivers/gpu/drm/Kconfig b/drivers/gpu/drm/Kconfig index ed85d0ceee3ba5..77190fc4fbd690 100644 --- a/drivers/gpu/drm/Kconfig +++ b/drivers/gpu/drm/Kconfig @@ -294,6 +294,8 @@ config DRM_VGEM source "drivers/gpu/drm/vkms/Kconfig" +source "drivers/gpu/drm/asahi/Kconfig" + source "drivers/gpu/drm/exynos/Kconfig" source "drivers/gpu/drm/rockchip/Kconfig" @@ -394,6 +396,8 @@ source "drivers/gpu/drm/solomon/Kconfig" source "drivers/gpu/drm/sprd/Kconfig" +source "drivers/gpu/drm/apple/Kconfig" + source "drivers/gpu/drm/imagination/Kconfig" source "drivers/gpu/drm/tyr/Kconfig" diff --git a/drivers/gpu/drm/Makefile b/drivers/gpu/drm/Makefile index d261917174280b..ed5c544198a3e2 100644 --- a/drivers/gpu/drm/Makefile +++ b/drivers/gpu/drm/Makefile @@ -220,12 +220,14 @@ obj-y += tiny/ obj-$(CONFIG_DRM_PL111) += pl111/ obj-$(CONFIG_DRM_TVE200) += tve200/ obj-$(CONFIG_DRM_ADP) += adp/ +obj-$(CONFIG_DRM_ASAHI) += asahi/ obj-$(CONFIG_DRM_XEN) += xen/ obj-$(CONFIG_DRM_VBOXVIDEO) += vboxvideo/ obj-$(CONFIG_DRM_LIMA) += lima/ obj-$(CONFIG_DRM_PANFROST) += panfrost/ obj-$(CONFIG_DRM_PANTHOR) += panthor/ obj-$(CONFIG_DRM_TYR) += tyr/ +obj-$(CONFIG_DRM_APPLE) += apple/ obj-$(CONFIG_DRM_ASPEED_GFX) += aspeed/ obj-$(CONFIG_DRM_MCDE) += mcde/ obj-$(CONFIG_DRM_TIDSS) += tidss/ diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_aca.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_aca.c index 9b318044915006..3f9b094e93a294 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_aca.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_aca.c @@ -641,6 +641,7 @@ static void aca_error_fini(struct aca_error *aerr) aca_bank_error_remove(aerr, bank_error); out_unlock: + mutex_unlock(&aerr->lock); mutex_destroy(&aerr->lock); } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_acpi.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_acpi.c index d31460a9e95829..7c9d8a6d0bfdb9 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_acpi.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_acpi.c @@ -1135,8 +1135,10 @@ static int amdgpu_acpi_enumerate_xcc(void) if (!dev_info) ret = amdgpu_acpi_dev_init(&dev_info, xcc_info, sbdf); - if (ret == -ENOMEM) + if (ret == -ENOMEM) { + kfree(xcc_info); return ret; + } if (!dev_info) { kfree(xcc_info); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c index a2879d2b7c8ec1..1ec26be82f30e1 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c @@ -687,9 +687,9 @@ int amdgpu_amdkfd_submit_ib(struct amdgpu_device *adev, goto err_ib_sched; } - /* Drop the initial kref_init count (see drm_sched_main as example) */ - dma_fence_put(f); ret = dma_fence_wait(f, false); + /* Drop the returned fence reference after the wait completes */ + dma_fence_put(f); err_ib_sched: amdgpu_job_free(job); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c index a51e76623bada0..51614de32e83c2 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c @@ -1428,7 +1428,10 @@ static int init_kfd_vm(struct amdgpu_vm *vm, void **process_info, *process_info = info; } - vm->process_info = *process_info; + if (cmpxchg(&vm->process_info, NULL, *process_info) != NULL) { + ret = -EINVAL; + goto already_acquired; + } /* Validate page directory and attach eviction fence */ ret = amdgpu_bo_reserve(vm->root.bo, true); @@ -1468,6 +1471,7 @@ static int init_kfd_vm(struct amdgpu_vm *vm, void **process_info, amdgpu_bo_unreserve(vm->root.bo); reserve_pd_fail: vm->process_info = NULL; +already_acquired: if (info) { dma_fence_put(&info->eviction_fence->base); *process_info = NULL; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.c index 66fb37b643882c..ded22f244adabf 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.c @@ -36,6 +36,7 @@ #define AMDGPU_BO_LIST_MAX_PRIORITY 32u #define AMDGPU_BO_LIST_NUM_BUCKETS (AMDGPU_BO_LIST_MAX_PRIORITY + 1) +#define AMDGPU_BO_LIST_MAX_ENTRIES (128 * 1024) static void amdgpu_bo_list_free_rcu(struct rcu_head *rcu) { @@ -190,6 +191,9 @@ int amdgpu_bo_create_list_entry_array(struct drm_amdgpu_bo_list_in *in, const uint32_t bo_number = in->bo_number; struct drm_amdgpu_bo_list_entry *info; + if (bo_number > AMDGPU_BO_LIST_MAX_ENTRIES) + return -EINVAL; + /* copy the handle array from userspace to a kernel buffer */ if (likely(info_size == bo_info_size)) { info = vmemdup_array_user(uptr, bo_number, info_size); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c index ecdfe6cb36ccdc..dac0b15823f2a6 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c @@ -892,8 +892,10 @@ static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p, struct amdgpu_bo *bo = e->bo; e->range = amdgpu_hmm_range_alloc(NULL); - if (unlikely(!e->range)) - return -ENOMEM; + if (unlikely(!e->range)) { + r = -ENOMEM; + goto out_free_user_pages; + } r = amdgpu_ttm_tt_get_user_pages(bo, e->range); if (r) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c index d2c3885de711f0..cd8a49e32e77c0 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c @@ -2843,8 +2843,10 @@ static int amdgpu_device_ip_early_init(struct amdgpu_device *adev) break; default: r = amdgpu_discovery_set_ip_blocks(adev); - if (r) + if (r) { + adev->num_ip_blocks = 0; return r; + } break; } @@ -3309,7 +3311,8 @@ static int amdgpu_device_ip_init(struct amdgpu_device *adev) if (r) goto init_failed; - if (adev->mman.buffer_funcs_ring->sched.ready) + if (adev->mman.buffer_funcs_ring && + adev->mman.buffer_funcs_ring->sched.ready) amdgpu_ttm_set_buffer_funcs_status(adev, true); /* Don't init kfd if whole hive need to be reset during init */ @@ -3401,6 +3404,8 @@ int amdgpu_device_set_cg_state(struct amdgpu_device *adev, i = state == AMD_CG_STATE_GATE ? j : adev->num_ip_blocks - j - 1; if (!adev->ip_blocks[i].status.late_initialized) continue; + if (!adev->ip_blocks[i].version) + continue; /* skip CG for GFX, SDMA on S0ix */ if (adev->in_s0ix && (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX || @@ -3440,6 +3445,8 @@ int amdgpu_device_set_pg_state(struct amdgpu_device *adev, i = state == AMD_PG_STATE_GATE ? j : adev->num_ip_blocks - j - 1; if (!adev->ip_blocks[i].status.late_initialized) continue; + if (!adev->ip_blocks[i].version) + continue; /* skip PG for GFX, SDMA on S0ix */ if (adev->in_s0ix && (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX || @@ -3647,6 +3654,8 @@ static int amdgpu_device_ip_fini_early(struct amdgpu_device *adev) int i, r; for (i = 0; i < adev->num_ip_blocks; i++) { + if (!adev->ip_blocks[i].version) + continue; if (!adev->ip_blocks[i].version->funcs->early_fini) continue; @@ -3658,9 +3667,6 @@ static int amdgpu_device_ip_fini_early(struct amdgpu_device *adev) } } - amdgpu_device_set_pg_state(adev, AMD_PG_STATE_UNGATE); - amdgpu_device_set_cg_state(adev, AMD_CG_STATE_UNGATE); - amdgpu_amdkfd_suspend(adev, true); amdgpu_userq_suspend(adev); @@ -3726,6 +3732,8 @@ static int amdgpu_device_ip_fini(struct amdgpu_device *adev) if (!adev->ip_blocks[i].status.sw) continue; + if (!adev->ip_blocks[i].version) + continue; if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) { amdgpu_ucode_free_bo(adev); amdgpu_free_static_csa(&adev->virt.csa_obj); @@ -3752,6 +3760,8 @@ static int amdgpu_device_ip_fini(struct amdgpu_device *adev) for (i = adev->num_ip_blocks - 1; i >= 0; i--) { if (!adev->ip_blocks[i].status.late_initialized) continue; + if (!adev->ip_blocks[i].version) + continue; if (adev->ip_blocks[i].version->funcs->late_fini) adev->ip_blocks[i].version->funcs->late_fini(&adev->ip_blocks[i]); adev->ip_blocks[i].status.late_initialized = false; @@ -4351,7 +4361,8 @@ static void amdgpu_device_xgmi_reset_func(struct work_struct *__work) static int amdgpu_device_get_job_timeout_settings(struct amdgpu_device *adev) { - char *input = amdgpu_lockup_timeout; + char buf[AMDGPU_MAX_TIMEOUT_PARAM_LENGTH]; + char *input = buf; char *timeout_setting = NULL; int index = 0; long timeout; @@ -4361,9 +4372,17 @@ static int amdgpu_device_get_job_timeout_settings(struct amdgpu_device *adev) adev->gfx_timeout = adev->compute_timeout = adev->sdma_timeout = adev->video_timeout = msecs_to_jiffies(2000); - if (!strnlen(input, AMDGPU_MAX_TIMEOUT_PARAM_LENGTH)) + if (!strnlen(amdgpu_lockup_timeout, AMDGPU_MAX_TIMEOUT_PARAM_LENGTH)) return 0; + /* + * strsep() destructively modifies its input by replacing delimiters + * with '\0'. Use a stack copy so the global module parameter buffer + * remains intact for multi-GPU systems where this function is called + * once per device. + */ + strscpy(buf, amdgpu_lockup_timeout, sizeof(buf)); + while ((timeout_setting = strsep(&input, ",")) && strnlen(timeout_setting, AMDGPU_MAX_TIMEOUT_PARAM_LENGTH)) { ret = kstrtol(timeout_setting, 0, &timeout); @@ -5046,6 +5065,9 @@ void amdgpu_device_fini_hw(struct amdgpu_device *adev) amdgpu_virt_fini_data_exchange(adev); } + amdgpu_device_set_pg_state(adev, AMD_PG_STATE_UNGATE); + amdgpu_device_set_cg_state(adev, AMD_CG_STATE_UNGATE); + /* disable all interrupts */ amdgpu_irq_disable_all(adev); if (adev->mode_info.mode_config_initialized) { @@ -5068,7 +5090,7 @@ void amdgpu_device_fini_hw(struct amdgpu_device *adev) * before ip_fini_early to prevent kfd locking refcount issues by calling * amdgpu_amdkfd_suspend() */ - if (drm_dev_is_unplugged(adev_to_drm(adev))) + if (pci_dev_is_disconnected(adev->pdev)) amdgpu_amdkfd_device_fini_sw(adev); amdgpu_device_ip_fini_early(adev); @@ -5080,7 +5102,7 @@ void amdgpu_device_fini_hw(struct amdgpu_device *adev) amdgpu_gart_dummy_page_fini(adev); - if (drm_dev_is_unplugged(adev_to_drm(adev))) + if (pci_dev_is_disconnected(adev->pdev)) amdgpu_device_unmap_mmio(adev); } @@ -5877,6 +5899,9 @@ int amdgpu_device_mode1_reset(struct amdgpu_device *adev) /* enable mmio access after mode 1 reset completed */ adev->no_hw_access = false; + /* ensure no_hw_access is updated before we access hw */ + smp_mb(); + amdgpu_device_load_pci_state(adev->pdev); ret = amdgpu_psp_wait_for_bootloader(adev); if (ret) @@ -7202,6 +7227,15 @@ pci_ers_result_t amdgpu_pci_slot_reset(struct pci_dev *pdev) dev_info(adev->dev, "PCI error: slot reset callback!!\n"); memset(&reset_context, 0, sizeof(reset_context)); + INIT_LIST_HEAD(&device_list); + hive = amdgpu_get_xgmi_hive(adev); + if (hive) { + mutex_lock(&hive->hive_lock); + list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) + list_add_tail(&tmp_adev->reset_list, &device_list); + } else { + list_add_tail(&adev->reset_list, &device_list); + } if (adev->pcie_reset_ctx.swus) link_dev = adev->pcie_reset_ctx.swus; @@ -7242,19 +7276,13 @@ pci_ers_result_t amdgpu_pci_slot_reset(struct pci_dev *pdev) reset_context.reset_req_dev = adev; set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags); set_bit(AMDGPU_SKIP_COREDUMP, &reset_context.flags); - INIT_LIST_HEAD(&device_list); - hive = amdgpu_get_xgmi_hive(adev); if (hive) { - mutex_lock(&hive->hive_lock); reset_context.hive = hive; - list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) { + list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) tmp_adev->pcie_reset_ctx.in_link_reset = true; - list_add_tail(&tmp_adev->reset_list, &device_list); - } } else { set_bit(AMDGPU_SKIP_HW_RESET, &reset_context.flags); - list_add_tail(&adev->reset_list, &device_list); } r = amdgpu_device_asic_reset(adev, &device_list, &reset_context); @@ -7501,6 +7529,9 @@ void amdgpu_device_halt(struct amdgpu_device *adev) amdgpu_xcp_dev_unplug(adev); drm_dev_unplug(ddev); + amdgpu_device_set_pg_state(adev, AMD_PG_STATE_UNGATE); + amdgpu_device_set_cg_state(adev, AMD_CG_STATE_UNGATE); + amdgpu_irq_disable_all(adev); amdgpu_fence_driver_hw_fini(adev); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c index fa2a22dfa04877..f9e0e80c4c1862 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c @@ -3059,6 +3059,7 @@ int amdgpu_discovery_set_ip_blocks(struct amdgpu_device *adev) case IP_VERSION(6, 0, 0): case IP_VERSION(6, 0, 1): case IP_VERSION(6, 1, 0): + case IP_VERSION(6, 1, 1): adev->hdp.funcs = &hdp_v6_0_funcs; break; case IP_VERSION(7, 0, 0): diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c index c1461317eb2987..83fed04436ad73 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c @@ -496,8 +496,15 @@ amdgpu_dma_buf_move_notify(struct dma_buf_attachment *attach) r = dma_resv_reserve_fences(resv, 2); if (!r) r = amdgpu_vm_clear_freed(adev, vm, NULL); + + /* Don't pass 'ticket' to amdgpu_vm_handle_moved: we want the clear=true + * path to be used otherwise we might update the PT of another process + * while it's using the BO. + * With clear=true, amdgpu_vm_bo_update will sync to command submission + * from the same VM. + */ if (!r) - r = amdgpu_vm_handle_moved(adev, vm, ticket); + r = amdgpu_vm_handle_moved(adev, vm, NULL); if (r && r != -EBUSY) DRM_ERROR("Failed to invalidate VM page tables (%d))\n", diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c index 39387da8586b4f..b56b6759e1c3fd 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c @@ -2703,8 +2703,12 @@ static int amdgpu_pmops_freeze(struct device *dev) if (r) return r; - if (amdgpu_acpi_should_gpu_reset(adev)) - return amdgpu_asic_reset(adev); + if (amdgpu_acpi_should_gpu_reset(adev)) { + amdgpu_device_lock_reset_domain(adev->reset_domain); + r = amdgpu_asic_reset(adev); + amdgpu_device_unlock_reset_domain(adev->reset_domain); + return r; + } return 0; } @@ -3172,7 +3176,6 @@ static int __init amdgpu_init(void) if (r) goto error_fence; - DRM_INFO("amdgpu kernel modesetting enabled.\n"); amdgpu_register_atpx_handler(); amdgpu_acpi_detect(); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c index 3e38c5db298712..b39862256b7694 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c @@ -112,47 +112,6 @@ amdgpu_gem_update_timeline_node(struct drm_file *filp, return 0; } -static void -amdgpu_gem_update_bo_mapping(struct drm_file *filp, - struct amdgpu_bo_va *bo_va, - uint32_t operation, - uint64_t point, - struct dma_fence *fence, - struct drm_syncobj *syncobj, - struct dma_fence_chain *chain) -{ - struct amdgpu_bo *bo = bo_va ? bo_va->base.bo : NULL; - struct amdgpu_fpriv *fpriv = filp->driver_priv; - struct amdgpu_vm *vm = &fpriv->vm; - struct dma_fence *last_update; - - if (!syncobj) - return; - - /* Find the last update fence */ - switch (operation) { - case AMDGPU_VA_OP_MAP: - case AMDGPU_VA_OP_REPLACE: - if (bo && (bo->tbo.base.resv == vm->root.bo->tbo.base.resv)) - last_update = vm->last_update; - else - last_update = bo_va->last_pt_update; - break; - case AMDGPU_VA_OP_UNMAP: - case AMDGPU_VA_OP_CLEAR: - last_update = fence; - break; - default: - return; - } - - /* Add fence to timeline */ - if (!point) - drm_syncobj_replace_fence(syncobj, last_update); - else - drm_syncobj_add_point(syncobj, chain, last_update, point); -} - static vm_fault_t amdgpu_gem_fault(struct vm_fault *vmf) { struct ttm_buffer_object *bo = vmf->vma->vm_private_data; @@ -458,9 +417,6 @@ int amdgpu_gem_create_ioctl(struct drm_device *dev, void *data, /* always clear VRAM */ flags |= AMDGPU_GEM_CREATE_VRAM_CLEARED; - if (args->in.domains & AMDGPU_GEM_DOMAIN_MMIO_REMAP) - return -EINVAL; - /* create a gem object to contain this object in */ if (args->in.domains & (AMDGPU_GEM_DOMAIN_GDS | AMDGPU_GEM_DOMAIN_GWS | AMDGPU_GEM_DOMAIN_OA)) { @@ -764,16 +720,27 @@ amdgpu_gem_va_update_vm(struct amdgpu_device *adev, struct amdgpu_bo_va *bo_va, uint32_t operation) { - struct dma_fence *fence = dma_fence_get_stub(); - int r; + struct dma_fence *fence; + int r = 0; + + /* Always start from the VM's existing last update fence. */ + fence = dma_fence_get(vm->last_update); if (!amdgpu_vm_ready(vm)) return fence; + /* + * First clean up any freed mappings in the VM. + * + * amdgpu_vm_clear_freed() may replace @fence with a new fence if it + * schedules GPU work. If nothing needs clearing, @fence can remain as + * the original vm->last_update. + */ r = amdgpu_vm_clear_freed(adev, vm, &fence); if (r) goto error; + /* For MAP/REPLACE we also need to update the BO mappings. */ if (operation == AMDGPU_VA_OP_MAP || operation == AMDGPU_VA_OP_REPLACE) { r = amdgpu_vm_bo_update(adev, bo_va, false); @@ -781,7 +748,46 @@ amdgpu_gem_va_update_vm(struct amdgpu_device *adev, goto error; } + /* Always update PDEs after we touched the mappings. */ r = amdgpu_vm_update_pdes(adev, vm, false); + if (r) + goto error; + + /* + * Decide which fence best represents the last update: + * + * MAP/REPLACE: + * - For always-valid mappings, use vm->last_update. + * - Otherwise, export bo_va->last_pt_update. + * + * UNMAP/CLEAR: + * Keep the fence returned by amdgpu_vm_clear_freed(). If no work was + * needed, it can remain as vm->last_pt_update. + * + * The VM and BO update fences are always initialized to a valid value. + * vm->last_update and bo_va->last_pt_update always start as valid fences. + * and are never expected to be NULL. + */ + switch (operation) { + case AMDGPU_VA_OP_MAP: + case AMDGPU_VA_OP_REPLACE: + /* + * For MAP/REPLACE, return the page table update fence for the + * mapping we just modified. bo_va is expected to be valid here. + */ + dma_fence_put(fence); + + if (amdgpu_vm_is_bo_always_valid(vm, bo_va->base.bo)) + fence = dma_fence_get(vm->last_update); + else + fence = dma_fence_get(bo_va->last_pt_update); + break; + case AMDGPU_VA_OP_UNMAP: + case AMDGPU_VA_OP_CLEAR: + default: + /* keep @fence as returned by amdgpu_vm_clear_freed() */ + break; + } error: if (r && r != -ERESTARTSYS) @@ -813,6 +819,7 @@ int amdgpu_gem_va_ioctl(struct drm_device *dev, void *data, uint64_t vm_size; int r = 0; + /* Validate virtual address range against reserved regions. */ if (args->va_address < AMDGPU_VA_RESERVED_BOTTOM) { dev_dbg(dev->dev, "va_address 0x%llx is in reserved area 0x%llx\n", @@ -846,6 +853,7 @@ int amdgpu_gem_va_ioctl(struct drm_device *dev, void *data, return -EINVAL; } + /* Validate operation type. */ switch (args->operation) { case AMDGPU_VA_OP_MAP: case AMDGPU_VA_OP_UNMAP: @@ -869,6 +877,7 @@ int amdgpu_gem_va_ioctl(struct drm_device *dev, void *data, abo = NULL; } + /* Add input syncobj fences (if any) for synchronization. */ r = amdgpu_gem_add_input_fence(filp, args->input_fence_syncobj_handles, args->num_syncobj_handles); @@ -891,6 +900,7 @@ int amdgpu_gem_va_ioctl(struct drm_device *dev, void *data, goto error; } + /* Resolve the BO-VA mapping for this VM/BO combination. */ if (abo) { bo_va = amdgpu_vm_bo_find(&fpriv->vm, abo); if (!bo_va) { @@ -903,6 +913,11 @@ int amdgpu_gem_va_ioctl(struct drm_device *dev, void *data, bo_va = NULL; } + /* + * Prepare the timeline syncobj node if the user requested a VM + * timeline update. This only allocates/looks up the syncobj and + * chain node; the actual fence is attached later. + */ r = amdgpu_gem_update_timeline_node(filp, args->vm_timeline_syncobj_out, args->vm_timeline_point, @@ -934,18 +949,30 @@ int amdgpu_gem_va_ioctl(struct drm_device *dev, void *data, default: break; } + + /* + * Once the VA operation is done, update the VM and obtain the fence + * that represents the last relevant update for this mapping. This + * fence can then be exported to the user-visible VM timeline. + */ if (!r && !(args->flags & AMDGPU_VM_DELAY_UPDATE) && !adev->debug_vm) { fence = amdgpu_gem_va_update_vm(adev, &fpriv->vm, bo_va, args->operation); - if (timeline_syncobj) - amdgpu_gem_update_bo_mapping(filp, bo_va, - args->operation, - args->vm_timeline_point, - fence, timeline_syncobj, - timeline_chain); - else - dma_fence_put(fence); + if (timeline_syncobj && fence) { + if (!args->vm_timeline_point) { + /* Replace the existing fence when no point is given. */ + drm_syncobj_replace_fence(timeline_syncobj, + fence); + } else { + /* Attach the last-update fence at a specific point. */ + drm_syncobj_add_point(timeline_syncobj, + timeline_chain, + fence, + args->vm_timeline_point); + } + } + dma_fence_put(fence); } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c index d9c7ad297293bd..b8613888c5c33c 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c @@ -1019,6 +1019,16 @@ void amdgpu_gmc_get_vbios_allocations(struct amdgpu_device *adev) case CHIP_RENOIR: adev->mman.keep_stolen_vga_memory = true; break; + case CHIP_POLARIS10: + case CHIP_POLARIS11: + case CHIP_POLARIS12: + /* MacBookPros with switchable graphics put VRAM at 0 when + * the iGPU is enabled which results in cursor issues if + * the cursor ends up at 0. Reserve vram at 0 in that case. + */ + if (adev->gmc.vram_start == 0) + adev->mman.keep_stolen_vga_memory = true; + break; default: adev->mman.keep_stolen_vga_memory = false; break; @@ -1387,7 +1397,7 @@ int amdgpu_gmc_get_nps_memranges(struct amdgpu_device *adev, if (!*exp_ranges) *exp_ranges = range_cnt; err: - kfree(ranges); + kvfree(ranges); return ret; } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c index 44f230d67da242..bfa64cd7a62d43 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c @@ -229,7 +229,7 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned int num_ibs, r = amdgpu_vm_flush(ring, job, need_pipe_sync); if (r) { amdgpu_ring_undo(ring); - return r; + goto free_fence; } } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ids.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ids.c index 9cab36322c1658..e495a8fa13fdba 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ids.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ids.c @@ -35,10 +35,13 @@ * PASIDs are global address space identifiers that can be shared * between the GPU, an IOMMU and the driver. VMs on different devices * may use the same PASID if they share the same address - * space. Therefore PASIDs are allocated using a global IDA. VMs are - * looked up from the PASID per amdgpu_device. + * space. Therefore PASIDs are allocated using IDR cyclic allocator + * (similar to kernel PID allocation) which naturally delays reuse. + * VMs are looked up from the PASID per amdgpu_device. */ -static DEFINE_IDA(amdgpu_pasid_ida); + +static DEFINE_IDR(amdgpu_pasid_idr); +static DEFINE_SPINLOCK(amdgpu_pasid_idr_lock); /* Helper to free pasid from a fence callback */ struct amdgpu_pasid_cb { @@ -50,8 +53,8 @@ struct amdgpu_pasid_cb { * amdgpu_pasid_alloc - Allocate a PASID * @bits: Maximum width of the PASID in bits, must be at least 1 * - * Allocates a PASID of the given width while keeping smaller PASIDs - * available if possible. + * Uses kernel's IDR cyclic allocator (same as PID allocation). + * Allocates sequentially with automatic wrap-around. * * Returns a positive integer on success. Returns %-EINVAL if bits==0. * Returns %-ENOSPC if no PASID was available. Returns %-ENOMEM on @@ -59,14 +62,18 @@ struct amdgpu_pasid_cb { */ int amdgpu_pasid_alloc(unsigned int bits) { - int pasid = -EINVAL; + int pasid; - for (bits = min(bits, 31U); bits > 0; bits--) { - pasid = ida_alloc_range(&amdgpu_pasid_ida, 1U << (bits - 1), - (1U << bits) - 1, GFP_KERNEL); - if (pasid != -ENOSPC) - break; - } + if (bits == 0) + return -EINVAL; + + spin_lock(&amdgpu_pasid_idr_lock); + /* TODO: Need to replace the idr with an xarry, and then + * handle the internal locking with ATOMIC safe paths. + */ + pasid = idr_alloc_cyclic(&amdgpu_pasid_idr, NULL, 1, + 1U << bits, GFP_ATOMIC); + spin_unlock(&amdgpu_pasid_idr_lock); if (pasid >= 0) trace_amdgpu_pasid_allocated(pasid); @@ -81,7 +88,10 @@ int amdgpu_pasid_alloc(unsigned int bits) void amdgpu_pasid_free(u32 pasid) { trace_amdgpu_pasid_freed(pasid); - ida_free(&amdgpu_pasid_ida, pasid); + + spin_lock(&amdgpu_pasid_idr_lock); + idr_remove(&amdgpu_pasid_idr, pasid); + spin_unlock(&amdgpu_pasid_idr_lock); } static void amdgpu_pasid_free_cb(struct dma_fence *fence, @@ -616,3 +626,15 @@ void amdgpu_vmid_mgr_fini(struct amdgpu_device *adev) } } } + +/** + * amdgpu_pasid_mgr_cleanup - cleanup PASID manager + * + * Cleanup the IDR allocator. + */ +void amdgpu_pasid_mgr_cleanup(void) +{ + spin_lock(&amdgpu_pasid_idr_lock); + idr_destroy(&amdgpu_pasid_idr); + spin_unlock(&amdgpu_pasid_idr_lock); +} diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ids.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ids.h index b3649cd3af5699..a57919478d3bd7 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ids.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ids.h @@ -74,6 +74,7 @@ int amdgpu_pasid_alloc(unsigned int bits); void amdgpu_pasid_free(u32 pasid); void amdgpu_pasid_free_delayed(struct dma_resv *resv, u32 pasid); +void amdgpu_pasid_mgr_cleanup(void); bool amdgpu_vmid_had_gpu_reset(struct amdgpu_device *adev, struct amdgpu_vmid *id); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c index 7ccb724b2488d7..aaf5477fcd7ac6 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c @@ -147,7 +147,8 @@ static enum drm_gpu_sched_stat amdgpu_job_timedout(struct drm_sched_job *s_job) dev_err(adev->dev, "Ring %s reset failed\n", ring->sched.name); } - dma_fence_set_error(&s_job->s_fence->finished, -ETIME); + if (dma_fence_get_status(&s_job->s_fence->finished) == 0) + dma_fence_set_error(&s_job->s_fence->finished, -ETIME); if (amdgpu_device_should_recover_gpu(ring->adev)) { struct amdgpu_reset_context reset_context; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c index f65edd80cabfa1..380e4576f88beb 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c @@ -83,7 +83,7 @@ void amdgpu_driver_unload_kms(struct drm_device *dev) { struct amdgpu_device *adev = drm_to_adev(dev); - if (adev == NULL) + if (adev == NULL || !adev->num_ip_blocks) return; amdgpu_unregister_gpu_instance(adev); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mode.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_mode.h index dc8d2f52c7d615..e244c12ceb2389 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mode.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mode.h @@ -368,15 +368,15 @@ struct amdgpu_mode_info { struct drm_property *plane_ctm_property; /** - * @shaper_lut_property: Plane property to set pre-blending shaper LUT - * that converts color content before 3D LUT. If - * plane_shaper_tf_property != Identity TF, AMD color module will + * @plane_shaper_lut_property: Plane property to set pre-blending + * shaper LUT that converts color content before 3D LUT. + * If plane_shaper_tf_property != Identity TF, AMD color module will * combine the user LUT values with pre-defined TF into the LUT * parameters to be programmed. */ struct drm_property *plane_shaper_lut_property; /** - * @shaper_lut_size_property: Plane property for the size of + * @plane_shaper_lut_size_property: Plane property for the size of * pre-blending shaper LUT as supported by the driver (read-only). */ struct drm_property *plane_shaper_lut_size_property; @@ -400,10 +400,10 @@ struct amdgpu_mode_info { */ struct drm_property *plane_lut3d_property; /** - * @plane_degamma_lut_size_property: Plane property to define the max - * size of 3D LUT as supported by the driver (read-only). The max size - * is the max size of one dimension and, therefore, the max number of - * entries for 3D LUT array is the 3D LUT size cubed; + * @plane_lut3d_size_property: Plane property to define the max size + * of 3D LUT as supported by the driver (read-only). The max size is + * the max size of one dimension and, therefore, the max number of + * entries for 3D LUT array is the 3D LUT size cubed. */ struct drm_property *plane_lut3d_size_property; /** diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c index e08f58de4b17f2..f51ad28d0cbd5c 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c @@ -153,14 +153,6 @@ void amdgpu_bo_placement_from_domain(struct amdgpu_bo *abo, u32 domain) c++; } - if (domain & AMDGPU_GEM_DOMAIN_MMIO_REMAP) { - places[c].fpfn = 0; - places[c].lpfn = 0; - places[c].mem_type = AMDGPU_PL_MMIO_REMAP; - places[c].flags = 0; - c++; - } - if (domain & AMDGPU_GEM_DOMAIN_GTT) { places[c].fpfn = 0; places[c].lpfn = 0; @@ -1534,8 +1526,17 @@ u64 amdgpu_bo_gpu_offset_no_check(struct amdgpu_bo *bo) */ uint32_t amdgpu_bo_mem_stats_placement(struct amdgpu_bo *bo) { - uint32_t domain = bo->preferred_domains & AMDGPU_GEM_DOMAIN_MASK; + u32 domain; + /* + * MMIO_REMAP is internal now, so it no longer maps from a userspace + * domain bit. Keep fdinfo/mem-stats visibility by checking the actual + * TTM placement. + */ + if (bo->tbo.resource && bo->tbo.resource->mem_type == AMDGPU_PL_MMIO_REMAP) + return AMDGPU_PL_MMIO_REMAP; + + domain = bo->preferred_domains & AMDGPU_GEM_DOMAIN_MASK; if (!domain) return TTM_PL_SYSTEM; @@ -1554,8 +1555,6 @@ uint32_t amdgpu_bo_mem_stats_placement(struct amdgpu_bo *bo) return AMDGPU_PL_OA; case AMDGPU_GEM_DOMAIN_DOORBELL: return AMDGPU_PL_DOORBELL; - case AMDGPU_GEM_DOMAIN_MMIO_REMAP: - return AMDGPU_PL_MMIO_REMAP; default: return TTM_PL_SYSTEM; } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h index 52c2d1731aabf6..912c9afaf9e11e 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h @@ -168,8 +168,6 @@ static inline unsigned amdgpu_mem_type_to_domain(u32 mem_type) return AMDGPU_GEM_DOMAIN_OA; case AMDGPU_PL_DOORBELL: return AMDGPU_GEM_DOMAIN_DOORBELL; - case AMDGPU_PL_MMIO_REMAP: - return AMDGPU_GEM_DOMAIN_MMIO_REMAP; default: break; } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c index 0b10497d487c31..81bdd6aaad2a10 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c @@ -726,7 +726,7 @@ psp_cmd_submit_buf(struct psp_context *psp, ras_intr = amdgpu_ras_intr_triggered(); if (ras_intr) break; - usleep_range(10, 100); + usleep_range(60, 100); amdgpu_device_invalidate_hdp(psp->adev, NULL); } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp_ta.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp_ta.c index 6e8aad91bcd305..0d3c18f04ac360 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp_ta.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp_ta.c @@ -332,13 +332,13 @@ static ssize_t ta_if_invoke_debugfs_write(struct file *fp, const char *buf, size if (!context || !context->initialized) { dev_err(adev->dev, "TA is not initialized\n"); ret = -EINVAL; - goto err_free_shared_buf; + goto free_shared_buf; } if (!psp->ta_funcs || !psp->ta_funcs->fn_ta_invoke) { dev_err(adev->dev, "Unsupported function to invoke TA\n"); ret = -EOPNOTSUPP; - goto err_free_shared_buf; + goto free_shared_buf; } context->session_id = ta_id; @@ -346,7 +346,7 @@ static ssize_t ta_if_invoke_debugfs_write(struct file *fp, const char *buf, size mutex_lock(&psp->ras_context.mutex); ret = prep_ta_mem_context(&context->mem_context, shared_buf, shared_buf_len); if (ret) - goto err_free_shared_buf; + goto unlock; ret = psp_fn_ta_invoke(psp, cmd_id); if (ret || context->resp_status) { @@ -354,15 +354,17 @@ static ssize_t ta_if_invoke_debugfs_write(struct file *fp, const char *buf, size ret, context->resp_status); if (!ret) { ret = -EINVAL; - goto err_free_shared_buf; + goto unlock; } } if (copy_to_user((char *)&buf[copy_pos], context->mem_context.shared_buf, shared_buf_len)) ret = -EFAULT; -err_free_shared_buf: +unlock: mutex_unlock(&psp->ras_context.mutex); + +free_shared_buf: kfree(shared_buf); return ret; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c index 2a6cf7963dde22..ee4d08b0988d30 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c @@ -2777,6 +2777,10 @@ static int amdgpu_ras_badpages_read(struct amdgpu_device *adev, if (!data->bps[i].ts) continue; + /* U64_MAX is used to mark the record as invalid */ + if (data->bps[i].retired_page == U64_MAX) + continue; + bps[r].bp = data->bps[i].retired_page; r++; if (r >= count) @@ -3076,18 +3080,20 @@ static int __amdgpu_ras_restore_bad_pages(struct amdgpu_device *adev, struct ras_err_handler_data *data = con->eh_data; for (j = 0; j < count; j++) { + if (!data->space_left && + amdgpu_ras_realloc_eh_data_space(adev, data, 256)) { + return -ENOMEM; + } + if (amdgpu_ras_check_bad_page_unlock(con, bps[j].retired_page << AMDGPU_GPU_PAGE_SHIFT)) { + /* set to U64_MAX to mark it as invalid */ + data->bps[data->count].retired_page = U64_MAX; data->count++; data->space_left--; continue; } - if (!data->space_left && - amdgpu_ras_realloc_eh_data_space(adev, data, 256)) { - return -ENOMEM; - } - amdgpu_ras_reserve_page(adev, bps[j].retired_page); memcpy(&data->bps[data->count], &(bps[j]), @@ -3249,8 +3255,6 @@ int amdgpu_ras_add_bad_pages(struct amdgpu_device *adev, /* deal with retire_unit records a time */ ret = __amdgpu_ras_convert_rec_array_from_rom(adev, &bps[i], &err_data, nps); - if (ret) - con->bad_page_num -= adev->umc.retire_unit; i += (adev->umc.retire_unit - 1); } else { break; @@ -3263,8 +3267,6 @@ int amdgpu_ras_add_bad_pages(struct amdgpu_device *adev, for (; i < pages; i++) { ret = __amdgpu_ras_convert_rec_from_rom(adev, &bps[i], &err_data, nps); - if (ret) - con->bad_page_num -= adev->umc.retire_unit; } con->eh_data->count_saved = con->eh_data->count; @@ -4343,7 +4345,7 @@ int amdgpu_ras_init(struct amdgpu_device *adev) * to handle fatal error */ r = amdgpu_nbio_ras_sw_init(adev); if (r) - return r; + goto release_con; if (adev->nbio.ras && adev->nbio.ras->init_ras_controller_interrupt) { diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras_eeprom.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras_eeprom.c index 64dd7a81bff5fa..710a8fe79fccd6 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras_eeprom.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras_eeprom.c @@ -1701,10 +1701,12 @@ int amdgpu_ras_eeprom_check(struct amdgpu_ras_eeprom_control *control) } res = __verify_ras_table_checksum(control); - if (res) + if (res) { dev_err(adev->dev, "RAS table incorrect checksum or error:%d\n", res); + return -EINVAL; + } /* Warn if we are at 90% of the threshold or above */ diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.c index 8b8a04138711cf..321310ba2c08e8 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.c @@ -558,6 +558,9 @@ int amdgpu_sdma_reset_engine(struct amdgpu_device *adev, uint32_t instance_id, struct amdgpu_ring *gfx_ring = &sdma_instance->ring; struct amdgpu_ring *page_ring = &sdma_instance->page; + if (amdgpu_sriov_vf(adev)) + return -EOPNOTSUPP; + mutex_lock(&sdma_instance->engine_reset_mutex); if (!caller_handles_kernel_queues) { diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c index 2b931e855abd9d..c200d5dc7b6bd4 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c @@ -1812,10 +1812,10 @@ static void amdgpu_ttm_pools_fini(struct amdgpu_device *adev) } /** - * amdgpu_ttm_mmio_remap_bo_init - Allocate the singleton 4K MMIO_REMAP BO + * amdgpu_ttm_mmio_remap_bo_init - Allocate the singleton MMIO_REMAP BO * @adev: amdgpu device * - * Allocates a one-page (4K) GEM BO in AMDGPU_GEM_DOMAIN_MMIO_REMAP when the + * Allocates a global BO with backing AMDGPU_PL_MMIO_REMAP when the * hardware exposes a remap base (adev->rmmio_remap.bus_addr) and the host * PAGE_SIZE is <= AMDGPU_GPU_PAGE_SIZE (4K). The BO is created as a regular * GEM object (amdgpu_bo_create). @@ -1824,42 +1824,95 @@ static void amdgpu_ttm_pools_fini(struct amdgpu_device *adev) * * 0 on success or intentional skip (feature not present/unsupported) * * negative errno on allocation failure */ -static int amdgpu_ttm_mmio_remap_bo_init(struct amdgpu_device *adev) +static int amdgpu_ttm_alloc_mmio_remap_bo(struct amdgpu_device *adev) { + struct ttm_operation_ctx ctx = { false, false }; + struct ttm_placement placement; + struct ttm_buffer_object *tbo; + struct ttm_place placements; struct amdgpu_bo_param bp; + struct ttm_resource *tmp; int r; /* Skip if HW doesn't expose remap, or if PAGE_SIZE > AMDGPU_GPU_PAGE_SIZE (4K). */ if (!adev->rmmio_remap.bus_addr || PAGE_SIZE > AMDGPU_GPU_PAGE_SIZE) return 0; + /* + * Allocate a BO first and then move it to AMDGPU_PL_MMIO_REMAP. + * The initial TTM resource assigned by amdgpu_bo_create() is + * replaced below with a fixed MMIO_REMAP placement. + */ memset(&bp, 0, sizeof(bp)); - - /* Create exactly one GEM BO in the MMIO_REMAP domain. */ - bp.type = ttm_bo_type_device; /* userspace-mappable GEM */ - bp.size = AMDGPU_GPU_PAGE_SIZE; /* 4K */ + bp.type = ttm_bo_type_device; + bp.size = AMDGPU_GPU_PAGE_SIZE; bp.byte_align = AMDGPU_GPU_PAGE_SIZE; - bp.domain = AMDGPU_GEM_DOMAIN_MMIO_REMAP; + bp.domain = 0; bp.flags = 0; bp.resv = NULL; bp.bo_ptr_size = sizeof(struct amdgpu_bo); - r = amdgpu_bo_create(adev, &bp, &adev->rmmio_remap.bo); if (r) return r; + r = amdgpu_bo_reserve(adev->rmmio_remap.bo, true); + if (r) + goto err_unref; + + tbo = &adev->rmmio_remap.bo->tbo; + + /* + * MMIO_REMAP is a fixed I/O placement (AMDGPU_PL_MMIO_REMAP). + */ + placement.num_placement = 1; + placement.placement = &placements; + placements.fpfn = 0; + placements.lpfn = 0; + placements.mem_type = AMDGPU_PL_MMIO_REMAP; + placements.flags = 0; + /* Force the BO into the fixed MMIO_REMAP placement */ + r = ttm_bo_mem_space(tbo, &placement, &tmp, &ctx); + if (unlikely(r)) + goto err_unlock; + + ttm_resource_free(tbo, &tbo->resource); + ttm_bo_assign_mem(tbo, tmp); + ttm_bo_pin(tbo); + + amdgpu_bo_unreserve(adev->rmmio_remap.bo); return 0; + +err_unlock: + amdgpu_bo_unreserve(adev->rmmio_remap.bo); + +err_unref: + amdgpu_bo_unref(&adev->rmmio_remap.bo); + adev->rmmio_remap.bo = NULL; + return r; } /** - * amdgpu_ttm_mmio_remap_bo_fini - Free the singleton MMIO_REMAP BO + * amdgpu_ttm_free_mmio_remap_bo - Free the singleton MMIO_REMAP BO * @adev: amdgpu device * * Frees the kernel-owned MMIO_REMAP BO if it was allocated by * amdgpu_ttm_mmio_remap_bo_init(). */ -static void amdgpu_ttm_mmio_remap_bo_fini(struct amdgpu_device *adev) +static void amdgpu_ttm_free_mmio_remap_bo(struct amdgpu_device *adev) { + if (!adev->rmmio_remap.bo) + return; + + if (!amdgpu_bo_reserve(adev->rmmio_remap.bo, true)) { + ttm_bo_unpin(&adev->rmmio_remap.bo->tbo); + amdgpu_bo_unreserve(adev->rmmio_remap.bo); + } + + /* + * At this point we rely on normal DRM teardown ordering: + * no new user ioctls can access the global MMIO_REMAP BO + * once TTM teardown begins. + */ amdgpu_bo_unref(&adev->rmmio_remap.bo); adev->rmmio_remap.bo = NULL; } @@ -2043,8 +2096,8 @@ int amdgpu_ttm_init(struct amdgpu_device *adev) return r; } - /* Allocate the singleton MMIO_REMAP BO (4K) if supported */ - r = amdgpu_ttm_mmio_remap_bo_init(adev); + /* Allocate the singleton MMIO_REMAP BO if supported */ + r = amdgpu_ttm_alloc_mmio_remap_bo(adev); if (r) return r; @@ -2111,7 +2164,7 @@ void amdgpu_ttm_fini(struct amdgpu_device *adev) amdgpu_bo_free_kernel(&adev->mman.sdma_access_bo, NULL, &adev->mman.sdma_access_ptr); - amdgpu_ttm_mmio_remap_bo_fini(adev); + amdgpu_ttm_free_mmio_remap_bo(adev); amdgpu_ttm_fw_reserve_vram_fini(adev); amdgpu_ttm_drv_reserve_vram_fini(adev); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c index 58b26c78b64258..0d41f6f5cc0b45 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c @@ -550,6 +550,13 @@ amdgpu_userq_get_doorbell_index(struct amdgpu_userq_mgr *uq_mgr, goto unpin_bo; } + /* Validate doorbell_offset is within the doorbell BO */ + if ((u64)db_info->doorbell_offset * db_size + db_size > + amdgpu_bo_size(db_obj->obj)) { + r = -EINVAL; + goto unpin_bo; + } + index = amdgpu_doorbell_index_on_bar(uq_mgr->adev, db_obj->obj, db_info->doorbell_offset, db_size); drm_dbg_driver(adev_to_drm(uq_mgr->adev), @@ -860,6 +867,17 @@ static int amdgpu_userq_input_args_validate(struct drm_device *dev, drm_file_err(filp, "invalidate userq queue va or size\n"); return -EINVAL; } + + if (!is_power_of_2(args->in.queue_size)) { + drm_file_err(filp, "Queue size must be a power of 2\n"); + return -EINVAL; + } + + if (args->in.queue_size < AMDGPU_GPU_PAGE_SIZE) { + drm_file_err(filp, "Queue size smaller than AMDGPU_GPU_PAGE_SIZE\n"); + return -EINVAL; + } + if (!args->in.wptr_va || !args->in.rptr_va) { drm_file_err(filp, "invalidate userq queue rptr or wptr\n"); return -EINVAL; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c index 85e9edc1cb6ffd..35d261db14740a 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c @@ -35,6 +35,8 @@ static const struct dma_fence_ops amdgpu_userq_fence_ops; static struct kmem_cache *amdgpu_userq_fence_slab; +#define AMDGPU_USERQ_MAX_HANDLES (1U << 16) + int amdgpu_userq_fence_slab_init(void) { amdgpu_userq_fence_slab = kmem_cache_create("amdgpu_userq_fence", @@ -476,6 +478,11 @@ int amdgpu_userq_signal_ioctl(struct drm_device *dev, void *data, if (!amdgpu_userq_enabled(dev)) return -ENOTSUPP; + if (args->num_syncobj_handles > AMDGPU_USERQ_MAX_HANDLES || + args->num_bo_write_handles > AMDGPU_USERQ_MAX_HANDLES || + args->num_bo_read_handles > AMDGPU_USERQ_MAX_HANDLES) + return -EINVAL; + num_syncobj_handles = args->num_syncobj_handles; syncobj_handles = memdup_user(u64_to_user_ptr(args->syncobj_handles), size_mul(sizeof(u32), num_syncobj_handles)); @@ -661,6 +668,11 @@ int amdgpu_userq_wait_ioctl(struct drm_device *dev, void *data, if (!amdgpu_userq_enabled(dev)) return -ENOTSUPP; + if (wait_info->num_syncobj_handles > AMDGPU_USERQ_MAX_HANDLES || + wait_info->num_bo_write_handles > AMDGPU_USERQ_MAX_HANDLES || + wait_info->num_bo_read_handles > AMDGPU_USERQ_MAX_HANDLES) + return -EINVAL; + num_read_bo_handles = wait_info->num_bo_read_handles; bo_handles_read = memdup_user(u64_to_user_ptr(wait_info->bo_read_handles), size_mul(sizeof(u32), num_read_bo_handles)); @@ -830,7 +842,7 @@ int amdgpu_userq_wait_ioctl(struct drm_device *dev, void *data, dma_resv_for_each_fence(&resv_cursor, gobj_read[i]->resv, DMA_RESV_USAGE_READ, fence) { - if (WARN_ON_ONCE(num_fences >= wait_info->num_fences)) { + if (num_fences >= wait_info->num_fences) { r = -EINVAL; goto free_fences; } @@ -847,7 +859,7 @@ int amdgpu_userq_wait_ioctl(struct drm_device *dev, void *data, dma_resv_for_each_fence(&resv_cursor, gobj_write[i]->resv, DMA_RESV_USAGE_WRITE, fence) { - if (WARN_ON_ONCE(num_fences >= wait_info->num_fences)) { + if (num_fences >= wait_info->num_fences) { r = -EINVAL; goto free_fences; } @@ -871,8 +883,9 @@ int amdgpu_userq_wait_ioctl(struct drm_device *dev, void *data, goto free_fences; dma_fence_unwrap_for_each(f, &iter, fence) { - if (WARN_ON_ONCE(num_fences >= wait_info->num_fences)) { + if (num_fences >= wait_info->num_fences) { r = -EINVAL; + dma_fence_put(fence); goto free_fences; } @@ -895,8 +908,9 @@ int amdgpu_userq_wait_ioctl(struct drm_device *dev, void *data, if (r) goto free_fences; - if (WARN_ON_ONCE(num_fences >= wait_info->num_fences)) { + if (num_fences >= wait_info->num_fences) { r = -EINVAL; + dma_fence_put(fence); goto free_fences; } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c index 47a6ce4fdc7444..292e2706286a16 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c @@ -1261,6 +1261,7 @@ bool amdgpu_virt_fw_load_skip_check(struct amdgpu_device *adev, uint32_t ucode_i || ucode_id == AMDGPU_UCODE_ID_SDMA5 || ucode_id == AMDGPU_UCODE_ID_SDMA6 || ucode_id == AMDGPU_UCODE_ID_SDMA7 + || ucode_id == AMDGPU_UCODE_ID_SDMA_RS64 || ucode_id == AMDGPU_UCODE_ID_RLC_G || ucode_id == AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL || ucode_id == AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c index a67285118c37ba..636a0cbbb1447f 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c @@ -1069,7 +1069,10 @@ amdgpu_vm_tlb_flush(struct amdgpu_vm_update_params *params, } /* Prepare a TLB flush fence to be attached to PTs */ - if (!params->unlocked) { + /* The check for need_tlb_fence should be dropped once we + * sort out the issues with KIQ/MES TLB invalidation timeouts. + */ + if (!params->unlocked && vm->need_tlb_fence) { amdgpu_vm_tlb_fence_create(params->adev, vm, fence); /* Makes sure no PD/PT is freed before the flush */ @@ -2405,6 +2408,9 @@ void amdgpu_vm_adjust_size(struct amdgpu_device *adev, uint32_t min_vm_size, tmp = DIV_ROUND_UP(fls64(tmp) - 1, 9) - 1; adev->vm_manager.num_level = min_t(unsigned int, max_level, tmp); switch (adev->vm_manager.num_level) { + case 4: + adev->vm_manager.root_level = AMDGPU_VM_PDB3; + break; case 3: adev->vm_manager.root_level = AMDGPU_VM_PDB2; break; @@ -2599,6 +2605,7 @@ int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm, ttm_lru_bulk_move_init(&vm->lru_bulk_move); vm->is_compute_context = false; + vm->need_tlb_fence = amdgpu_userq_enabled(&adev->ddev); vm->use_cpu_for_update = !!(adev->vm_manager.vm_update_mode & AMDGPU_VM_USE_CPU_FOR_GFX); @@ -2736,6 +2743,7 @@ int amdgpu_vm_make_compute(struct amdgpu_device *adev, struct amdgpu_vm *vm) dma_fence_put(vm->last_update); vm->last_update = dma_fence_get_stub(); vm->is_compute_context = true; + vm->need_tlb_fence = true; unreserve_bo: amdgpu_bo_unreserve(vm->root.bo); @@ -2890,6 +2898,7 @@ void amdgpu_vm_manager_fini(struct amdgpu_device *adev) xa_destroy(&adev->vm_manager.pasids); amdgpu_vmid_mgr_fini(adev); + amdgpu_pasid_mgr_cleanup(); } /** @@ -2965,14 +2974,14 @@ bool amdgpu_vm_handle_fault(struct amdgpu_device *adev, u32 pasid, if (!root) return false; - addr /= AMDGPU_GPU_PAGE_SIZE; - if (is_compute_context && !svm_range_restore_pages(adev, pasid, vmid, - node_id, addr, ts, write_fault)) { + node_id, addr >> PAGE_SHIFT, ts, write_fault)) { amdgpu_bo_unref(&root); return true; } + addr /= AMDGPU_GPU_PAGE_SIZE; + r = amdgpu_bo_reserve(root, true); if (r) goto error_unref; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h index 15d757c016cbbc..42b4f246cddc8e 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h @@ -172,7 +172,7 @@ struct amdgpu_bo_vm; #define AMDGPU_VA_RESERVED_SEQ64_SIZE (2ULL << 20) #define AMDGPU_VA_RESERVED_SEQ64_START(adev) (AMDGPU_VA_RESERVED_CSA_START(adev) \ - AMDGPU_VA_RESERVED_SEQ64_SIZE) -#define AMDGPU_VA_RESERVED_TRAP_SIZE (2ULL << 12) +#define AMDGPU_VA_RESERVED_TRAP_SIZE (1ULL << 16) #define AMDGPU_VA_RESERVED_TRAP_START(adev) (AMDGPU_VA_RESERVED_SEQ64_START(adev) \ - AMDGPU_VA_RESERVED_TRAP_SIZE) #define AMDGPU_VA_RESERVED_BOTTOM (1ULL << 16) @@ -185,9 +185,10 @@ struct amdgpu_bo_vm; #define AMDGPU_VM_USE_CPU_FOR_COMPUTE (1 << 1) /* VMPT level enumerate, and the hiberachy is: - * PDB2->PDB1->PDB0->PTB + * PDB3->PDB2->PDB1->PDB0->PTB */ enum amdgpu_vm_level { + AMDGPU_VM_PDB3, AMDGPU_VM_PDB2, AMDGPU_VM_PDB1, AMDGPU_VM_PDB0, @@ -439,6 +440,8 @@ struct amdgpu_vm { struct ttm_lru_bulk_move lru_bulk_move; /* Flag to indicate if VM is used for compute */ bool is_compute_context; + /* Flag to indicate if VM needs a TLB fence (KFD or KGD) */ + bool need_tlb_fence; /* Memory partition number, -1 means any partition */ int8_t mem_id; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c index f794fb1cc06e66..c7a7d51080a876 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c @@ -51,6 +51,7 @@ static unsigned int amdgpu_vm_pt_level_shift(struct amdgpu_device *adev, unsigned int level) { switch (level) { + case AMDGPU_VM_PDB3: case AMDGPU_VM_PDB2: case AMDGPU_VM_PDB1: case AMDGPU_VM_PDB0: diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c index f1ee3921d970c4..015d10658345cd 100644 --- a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c @@ -693,28 +693,35 @@ static int gmc_v9_0_process_interrupt(struct amdgpu_device *adev, } else { switch (amdgpu_ip_version(adev, MMHUB_HWIP, 0)) { case IP_VERSION(9, 0, 0): - mmhub_cid = mmhub_client_ids_vega10[cid][rw]; + mmhub_cid = cid < ARRAY_SIZE(mmhub_client_ids_vega10) ? + mmhub_client_ids_vega10[cid][rw] : NULL; break; case IP_VERSION(9, 3, 0): - mmhub_cid = mmhub_client_ids_vega12[cid][rw]; + mmhub_cid = cid < ARRAY_SIZE(mmhub_client_ids_vega12) ? + mmhub_client_ids_vega12[cid][rw] : NULL; break; case IP_VERSION(9, 4, 0): - mmhub_cid = mmhub_client_ids_vega20[cid][rw]; + mmhub_cid = cid < ARRAY_SIZE(mmhub_client_ids_vega20) ? + mmhub_client_ids_vega20[cid][rw] : NULL; break; case IP_VERSION(9, 4, 1): - mmhub_cid = mmhub_client_ids_arcturus[cid][rw]; + mmhub_cid = cid < ARRAY_SIZE(mmhub_client_ids_arcturus) ? + mmhub_client_ids_arcturus[cid][rw] : NULL; break; case IP_VERSION(9, 1, 0): case IP_VERSION(9, 2, 0): - mmhub_cid = mmhub_client_ids_raven[cid][rw]; + mmhub_cid = cid < ARRAY_SIZE(mmhub_client_ids_raven) ? + mmhub_client_ids_raven[cid][rw] : NULL; break; case IP_VERSION(1, 5, 0): case IP_VERSION(2, 4, 0): - mmhub_cid = mmhub_client_ids_renoir[cid][rw]; + mmhub_cid = cid < ARRAY_SIZE(mmhub_client_ids_renoir) ? + mmhub_client_ids_renoir[cid][rw] : NULL; break; case IP_VERSION(1, 8, 0): case IP_VERSION(9, 4, 2): - mmhub_cid = mmhub_client_ids_aldebaran[cid][rw]; + mmhub_cid = cid < ARRAY_SIZE(mmhub_client_ids_aldebaran) ? + mmhub_client_ids_aldebaran[cid][rw] : NULL; break; default: mmhub_cid = NULL; diff --git a/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c b/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c index ceddc953785fe1..896400b1073d06 100644 --- a/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c +++ b/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c @@ -718,11 +718,6 @@ static int mes_v11_0_set_hw_resources(struct amdgpu_mes *mes) mes_set_hw_res_pkt.enable_reg_active_poll = 1; mes_set_hw_res_pkt.enable_level_process_quantum_check = 1; mes_set_hw_res_pkt.oversubscription_timer = 50; - if ((mes->adev->mes.sched_version & AMDGPU_MES_VERSION_MASK) >= 0x7f) - mes_set_hw_res_pkt.enable_lr_compute_wa = 1; - else - dev_info_once(mes->adev->dev, - "MES FW version must be >= 0x7f to enable LR compute workaround.\n"); if (amdgpu_mes_log_enable) { mes_set_hw_res_pkt.enable_mes_event_int_logging = 1; diff --git a/drivers/gpu/drm/amd/amdgpu/mes_v12_0.c b/drivers/gpu/drm/amd/amdgpu/mes_v12_0.c index 744e95d3984ad1..dcafbd7066c407 100644 --- a/drivers/gpu/drm/amd/amdgpu/mes_v12_0.c +++ b/drivers/gpu/drm/amd/amdgpu/mes_v12_0.c @@ -731,6 +731,9 @@ static int mes_v12_0_set_hw_resources(struct amdgpu_mes *mes, int pipe) int i; struct amdgpu_device *adev = mes->adev; union MESAPI_SET_HW_RESOURCES mes_set_hw_res_pkt; + uint32_t mes_rev = (pipe == AMDGPU_MES_SCHED_PIPE) ? + (mes->sched_version & AMDGPU_MES_VERSION_MASK) : + (mes->kiq_version & AMDGPU_MES_VERSION_MASK); memset(&mes_set_hw_res_pkt, 0, sizeof(mes_set_hw_res_pkt)); @@ -779,18 +782,13 @@ static int mes_v12_0_set_hw_resources(struct amdgpu_mes *mes, int pipe) mes_set_hw_res_pkt.use_different_vmid_compute = 1; mes_set_hw_res_pkt.enable_reg_active_poll = 1; mes_set_hw_res_pkt.enable_level_process_quantum_check = 1; - if ((mes->adev->mes.sched_version & AMDGPU_MES_VERSION_MASK) >= 0x82) - mes_set_hw_res_pkt.enable_lr_compute_wa = 1; - else - dev_info_once(adev->dev, - "MES FW version must be >= 0x82 to enable LR compute workaround.\n"); /* * Keep oversubscribe timer for sdma . When we have unmapped doorbell * handling support, other queue will not use the oversubscribe timer. * handling mode - 0: disabled; 1: basic version; 2: basic+ version */ - mes_set_hw_res_pkt.oversubscription_timer = 50; + mes_set_hw_res_pkt.oversubscription_timer = mes_rev < 0x8b ? 0 : 50; mes_set_hw_res_pkt.unmapped_doorbell_handling = 1; if (amdgpu_mes_log_enable) { diff --git a/drivers/gpu/drm/amd/amdgpu/mmhub_v2_0.c b/drivers/gpu/drm/amd/amdgpu/mmhub_v2_0.c index a0cc8e218ca1ea..534cb4c544dc45 100644 --- a/drivers/gpu/drm/amd/amdgpu/mmhub_v2_0.c +++ b/drivers/gpu/drm/amd/amdgpu/mmhub_v2_0.c @@ -154,14 +154,17 @@ mmhub_v2_0_print_l2_protection_fault_status(struct amdgpu_device *adev, switch (amdgpu_ip_version(adev, MMHUB_HWIP, 0)) { case IP_VERSION(2, 0, 0): case IP_VERSION(2, 0, 2): - mmhub_cid = mmhub_client_ids_navi1x[cid][rw]; + mmhub_cid = cid < ARRAY_SIZE(mmhub_client_ids_navi1x) ? + mmhub_client_ids_navi1x[cid][rw] : NULL; break; case IP_VERSION(2, 1, 0): case IP_VERSION(2, 1, 1): - mmhub_cid = mmhub_client_ids_sienna_cichlid[cid][rw]; + mmhub_cid = cid < ARRAY_SIZE(mmhub_client_ids_sienna_cichlid) ? + mmhub_client_ids_sienna_cichlid[cid][rw] : NULL; break; case IP_VERSION(2, 1, 2): - mmhub_cid = mmhub_client_ids_beige_goby[cid][rw]; + mmhub_cid = cid < ARRAY_SIZE(mmhub_client_ids_beige_goby) ? + mmhub_client_ids_beige_goby[cid][rw] : NULL; break; default: mmhub_cid = NULL; diff --git a/drivers/gpu/drm/amd/amdgpu/mmhub_v2_3.c b/drivers/gpu/drm/amd/amdgpu/mmhub_v2_3.c index 5eb8122e27469c..ceb2f6b46de521 100644 --- a/drivers/gpu/drm/amd/amdgpu/mmhub_v2_3.c +++ b/drivers/gpu/drm/amd/amdgpu/mmhub_v2_3.c @@ -94,7 +94,8 @@ mmhub_v2_3_print_l2_protection_fault_status(struct amdgpu_device *adev, case IP_VERSION(2, 3, 0): case IP_VERSION(2, 4, 0): case IP_VERSION(2, 4, 1): - mmhub_cid = mmhub_client_ids_vangogh[cid][rw]; + mmhub_cid = cid < ARRAY_SIZE(mmhub_client_ids_vangogh) ? + mmhub_client_ids_vangogh[cid][rw] : NULL; break; default: mmhub_cid = NULL; diff --git a/drivers/gpu/drm/amd/amdgpu/mmhub_v3_0.c b/drivers/gpu/drm/amd/amdgpu/mmhub_v3_0.c index 7d5242df58a511..ab966e69a342a3 100644 --- a/drivers/gpu/drm/amd/amdgpu/mmhub_v3_0.c +++ b/drivers/gpu/drm/amd/amdgpu/mmhub_v3_0.c @@ -110,7 +110,8 @@ mmhub_v3_0_print_l2_protection_fault_status(struct amdgpu_device *adev, switch (amdgpu_ip_version(adev, MMHUB_HWIP, 0)) { case IP_VERSION(3, 0, 0): case IP_VERSION(3, 0, 1): - mmhub_cid = mmhub_client_ids_v3_0_0[cid][rw]; + mmhub_cid = cid < ARRAY_SIZE(mmhub_client_ids_v3_0_0) ? + mmhub_client_ids_v3_0_0[cid][rw] : NULL; break; default: mmhub_cid = NULL; diff --git a/drivers/gpu/drm/amd/amdgpu/mmhub_v3_0_1.c b/drivers/gpu/drm/amd/amdgpu/mmhub_v3_0_1.c index 910337dc28d105..14a742d3a99d78 100644 --- a/drivers/gpu/drm/amd/amdgpu/mmhub_v3_0_1.c +++ b/drivers/gpu/drm/amd/amdgpu/mmhub_v3_0_1.c @@ -117,7 +117,8 @@ mmhub_v3_0_1_print_l2_protection_fault_status(struct amdgpu_device *adev, switch (amdgpu_ip_version(adev, MMHUB_HWIP, 0)) { case IP_VERSION(3, 0, 1): - mmhub_cid = mmhub_client_ids_v3_0_1[cid][rw]; + mmhub_cid = cid < ARRAY_SIZE(mmhub_client_ids_v3_0_1) ? + mmhub_client_ids_v3_0_1[cid][rw] : NULL; break; default: mmhub_cid = NULL; diff --git a/drivers/gpu/drm/amd/amdgpu/mmhub_v3_0_2.c b/drivers/gpu/drm/amd/amdgpu/mmhub_v3_0_2.c index f0f182f033b988..e1f07f2a185272 100644 --- a/drivers/gpu/drm/amd/amdgpu/mmhub_v3_0_2.c +++ b/drivers/gpu/drm/amd/amdgpu/mmhub_v3_0_2.c @@ -108,7 +108,8 @@ mmhub_v3_0_2_print_l2_protection_fault_status(struct amdgpu_device *adev, "MMVM_L2_PROTECTION_FAULT_STATUS:0x%08X\n", status); - mmhub_cid = mmhub_client_ids_v3_0_2[cid][rw]; + mmhub_cid = cid < ARRAY_SIZE(mmhub_client_ids_v3_0_2) ? + mmhub_client_ids_v3_0_2[cid][rw] : NULL; dev_err(adev->dev, "\t Faulty UTCL2 client ID: %s (0x%x)\n", mmhub_cid ? mmhub_cid : "unknown", cid); dev_err(adev->dev, "\t MORE_FAULTS: 0x%lx\n", diff --git a/drivers/gpu/drm/amd/amdgpu/mmhub_v4_1_0.c b/drivers/gpu/drm/amd/amdgpu/mmhub_v4_1_0.c index 951998454b2572..88bfe321f83aaa 100644 --- a/drivers/gpu/drm/amd/amdgpu/mmhub_v4_1_0.c +++ b/drivers/gpu/drm/amd/amdgpu/mmhub_v4_1_0.c @@ -102,7 +102,8 @@ mmhub_v4_1_0_print_l2_protection_fault_status(struct amdgpu_device *adev, status); switch (amdgpu_ip_version(adev, MMHUB_HWIP, 0)) { case IP_VERSION(4, 1, 0): - mmhub_cid = mmhub_client_ids_v4_1_0[cid][rw]; + mmhub_cid = cid < ARRAY_SIZE(mmhub_client_ids_v4_1_0) ? + mmhub_client_ids_v4_1_0[cid][rw] : NULL; break; default: mmhub_cid = NULL; diff --git a/drivers/gpu/drm/amd/amdgpu/psp_v11_0.c b/drivers/gpu/drm/amd/amdgpu/psp_v11_0.c index a9be7a50502683..27d883fda5fa9a 100644 --- a/drivers/gpu/drm/amd/amdgpu/psp_v11_0.c +++ b/drivers/gpu/drm/amd/amdgpu/psp_v11_0.c @@ -170,7 +170,8 @@ static int psp_v11_0_wait_for_bootloader(struct psp_context *psp) int retry_loop; /* For a reset done at the end of S3, only wait for TOS to be unloaded */ - if (adev->in_s3 && !(adev->flags & AMD_IS_APU) && amdgpu_in_reset(adev)) + if ((adev->in_s4 || adev->in_s3) && !(adev->flags & AMD_IS_APU) && + amdgpu_in_reset(adev)) return psp_v11_wait_for_tos_unload(psp); for (retry_loop = 0; retry_loop < 20; retry_loop++) { diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v5_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v5_0.c index 8ddc4df06a1fde..45e2933214a802 100644 --- a/drivers/gpu/drm/amd/amdgpu/sdma_v5_0.c +++ b/drivers/gpu/drm/amd/amdgpu/sdma_v5_0.c @@ -1424,18 +1424,9 @@ static int sdma_v5_0_sw_init(struct amdgpu_ip_block *ip_block) adev->sdma.supported_reset = amdgpu_get_soft_full_reset_mask(&adev->sdma.instance[0].ring); - switch (amdgpu_ip_version(adev, SDMA0_HWIP, 0)) { - case IP_VERSION(5, 0, 0): - case IP_VERSION(5, 0, 2): - case IP_VERSION(5, 0, 5): - if ((adev->sdma.instance[0].fw_version >= 35) && - !amdgpu_sriov_vf(adev) && - !adev->debug_disable_gpu_ring_reset) - adev->sdma.supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE; - break; - default: - break; - } + if (!amdgpu_sriov_vf(adev) && + !adev->debug_disable_gpu_ring_reset) + adev->sdma.supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE; /* Allocate memory for SDMA IP Dump buffer */ ptr = kcalloc(adev->sdma.num_instances * reg_count, sizeof(uint32_t), GFP_KERNEL); diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v5_2.c b/drivers/gpu/drm/amd/amdgpu/sdma_v5_2.c index 51101b0aa2fabf..82b1d34a6533ef 100644 --- a/drivers/gpu/drm/amd/amdgpu/sdma_v5_2.c +++ b/drivers/gpu/drm/amd/amdgpu/sdma_v5_2.c @@ -1342,25 +1342,9 @@ static int sdma_v5_2_sw_init(struct amdgpu_ip_block *ip_block) adev->sdma.supported_reset = amdgpu_get_soft_full_reset_mask(&adev->sdma.instance[0].ring); - switch (amdgpu_ip_version(adev, SDMA0_HWIP, 0)) { - case IP_VERSION(5, 2, 0): - case IP_VERSION(5, 2, 2): - case IP_VERSION(5, 2, 3): - case IP_VERSION(5, 2, 4): - if ((adev->sdma.instance[0].fw_version >= 76) && - !amdgpu_sriov_vf(adev) && - !adev->debug_disable_gpu_ring_reset) - adev->sdma.supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE; - break; - case IP_VERSION(5, 2, 5): - if ((adev->sdma.instance[0].fw_version >= 34) && - !amdgpu_sriov_vf(adev) && - !adev->debug_disable_gpu_ring_reset) - adev->sdma.supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE; - break; - default: - break; - } + if (!amdgpu_sriov_vf(adev) && + !adev->debug_disable_gpu_ring_reset) + adev->sdma.supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE; /* Allocate memory for SDMA IP Dump buffer */ ptr = kcalloc(adev->sdma.num_instances * reg_count, sizeof(uint32_t), GFP_KERNEL); diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c index 21704004498728..6809c6d4be5b1b 100644 --- a/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c +++ b/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c @@ -1351,18 +1351,9 @@ static int sdma_v6_0_sw_init(struct amdgpu_ip_block *ip_block) adev->sdma.supported_reset = amdgpu_get_soft_full_reset_mask(&adev->sdma.instance[0].ring); - switch (amdgpu_ip_version(adev, SDMA0_HWIP, 0)) { - case IP_VERSION(6, 0, 0): - case IP_VERSION(6, 0, 2): - case IP_VERSION(6, 0, 3): - if ((adev->sdma.instance[0].fw_version >= 21) && - !amdgpu_sriov_vf(adev) && - !adev->debug_disable_gpu_ring_reset) - adev->sdma.supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE; - break; - default: - break; - } + if (!amdgpu_sriov_vf(adev) && + !adev->debug_disable_gpu_ring_reset) + adev->sdma.supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE; if (amdgpu_sdma_ras_sw_init(adev)) { dev_err(adev->dev, "Failed to initialize sdma ras block!\n"); diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v2_0.c b/drivers/gpu/drm/amd/amdgpu/vcn_v2_0.c index 8897dcc9c1a0a9..e35fae9cdaf66e 100644 --- a/drivers/gpu/drm/amd/amdgpu/vcn_v2_0.c +++ b/drivers/gpu/drm/amd/amdgpu/vcn_v2_0.c @@ -1964,7 +1964,8 @@ static int vcn_v2_0_start_sriov(struct amdgpu_device *adev) struct mmsch_v2_0_cmd_end end = { {0} }; struct mmsch_v2_0_init_header *header; uint32_t *init_table = adev->virt.mm_table.cpu_addr; - uint8_t i = 0; + + /* This path only programs VCN instance 0. */ header = (struct mmsch_v2_0_init_header *)init_table; direct_wt.cmd_header.command_type = MMSCH_COMMAND__DIRECT_REG_WRITE; @@ -1983,93 +1984,93 @@ static int vcn_v2_0_start_sriov(struct amdgpu_device *adev) size = AMDGPU_GPU_PAGE_ALIGN(adev->vcn.inst[0].fw->size + 4); MMSCH_V2_0_INSERT_DIRECT_RD_MOD_WT( - SOC15_REG_OFFSET(UVD, i, mmUVD_STATUS), + SOC15_REG_OFFSET(UVD, 0, mmUVD_STATUS), 0xFFFFFFFF, 0x00000004); /* mc resume*/ if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { MMSCH_V2_0_INSERT_DIRECT_WT( - SOC15_REG_OFFSET(UVD, i, + SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW), adev->firmware.ucode[AMDGPU_UCODE_ID_VCN].tmr_mc_addr_lo); MMSCH_V2_0_INSERT_DIRECT_WT( - SOC15_REG_OFFSET(UVD, i, + SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH), adev->firmware.ucode[AMDGPU_UCODE_ID_VCN].tmr_mc_addr_hi); offset = 0; } else { MMSCH_V2_0_INSERT_DIRECT_WT( - SOC15_REG_OFFSET(UVD, i, + SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW), lower_32_bits(adev->vcn.inst->gpu_addr)); MMSCH_V2_0_INSERT_DIRECT_WT( - SOC15_REG_OFFSET(UVD, i, + SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH), upper_32_bits(adev->vcn.inst->gpu_addr)); offset = size; } MMSCH_V2_0_INSERT_DIRECT_WT( - SOC15_REG_OFFSET(UVD, i, mmUVD_VCPU_CACHE_OFFSET0), + SOC15_REG_OFFSET(UVD, 0, mmUVD_VCPU_CACHE_OFFSET0), 0); MMSCH_V2_0_INSERT_DIRECT_WT( - SOC15_REG_OFFSET(UVD, i, mmUVD_VCPU_CACHE_SIZE0), + SOC15_REG_OFFSET(UVD, 0, mmUVD_VCPU_CACHE_SIZE0), size); MMSCH_V2_0_INSERT_DIRECT_WT( - SOC15_REG_OFFSET(UVD, i, + SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW), lower_32_bits(adev->vcn.inst->gpu_addr + offset)); MMSCH_V2_0_INSERT_DIRECT_WT( - SOC15_REG_OFFSET(UVD, i, + SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH), upper_32_bits(adev->vcn.inst->gpu_addr + offset)); MMSCH_V2_0_INSERT_DIRECT_WT( - SOC15_REG_OFFSET(UVD, i, mmUVD_VCPU_CACHE_OFFSET1), + SOC15_REG_OFFSET(UVD, 0, mmUVD_VCPU_CACHE_OFFSET1), 0); MMSCH_V2_0_INSERT_DIRECT_WT( - SOC15_REG_OFFSET(UVD, i, mmUVD_VCPU_CACHE_SIZE1), + SOC15_REG_OFFSET(UVD, 0, mmUVD_VCPU_CACHE_SIZE1), AMDGPU_VCN_STACK_SIZE); MMSCH_V2_0_INSERT_DIRECT_WT( - SOC15_REG_OFFSET(UVD, i, + SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW), lower_32_bits(adev->vcn.inst->gpu_addr + offset + AMDGPU_VCN_STACK_SIZE)); MMSCH_V2_0_INSERT_DIRECT_WT( - SOC15_REG_OFFSET(UVD, i, + SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH), upper_32_bits(adev->vcn.inst->gpu_addr + offset + AMDGPU_VCN_STACK_SIZE)); MMSCH_V2_0_INSERT_DIRECT_WT( - SOC15_REG_OFFSET(UVD, i, mmUVD_VCPU_CACHE_OFFSET2), + SOC15_REG_OFFSET(UVD, 0, mmUVD_VCPU_CACHE_OFFSET2), 0); MMSCH_V2_0_INSERT_DIRECT_WT( - SOC15_REG_OFFSET(UVD, i, mmUVD_VCPU_CACHE_SIZE2), + SOC15_REG_OFFSET(UVD, 0, mmUVD_VCPU_CACHE_SIZE2), AMDGPU_VCN_CONTEXT_SIZE); for (r = 0; r < adev->vcn.inst[0].num_enc_rings; ++r) { ring = &adev->vcn.inst->ring_enc[r]; ring->wptr = 0; MMSCH_V2_0_INSERT_DIRECT_WT( - SOC15_REG_OFFSET(UVD, i, mmUVD_RB_BASE_LO), + SOC15_REG_OFFSET(UVD, 0, mmUVD_RB_BASE_LO), lower_32_bits(ring->gpu_addr)); MMSCH_V2_0_INSERT_DIRECT_WT( - SOC15_REG_OFFSET(UVD, i, mmUVD_RB_BASE_HI), + SOC15_REG_OFFSET(UVD, 0, mmUVD_RB_BASE_HI), upper_32_bits(ring->gpu_addr)); MMSCH_V2_0_INSERT_DIRECT_WT( - SOC15_REG_OFFSET(UVD, i, mmUVD_RB_SIZE), + SOC15_REG_OFFSET(UVD, 0, mmUVD_RB_SIZE), ring->ring_size / 4); } ring = &adev->vcn.inst->ring_dec; ring->wptr = 0; MMSCH_V2_0_INSERT_DIRECT_WT( - SOC15_REG_OFFSET(UVD, i, + SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_RBC_RB_64BIT_BAR_LOW), lower_32_bits(ring->gpu_addr)); MMSCH_V2_0_INSERT_DIRECT_WT( - SOC15_REG_OFFSET(UVD, i, + SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_RBC_RB_64BIT_BAR_HIGH), upper_32_bits(ring->gpu_addr)); /* force RBC into idle state */ @@ -2080,7 +2081,7 @@ static int vcn_v2_0_start_sriov(struct amdgpu_device *adev) tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_UPDATE, 1); tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_RPTR_WR_EN, 1); MMSCH_V2_0_INSERT_DIRECT_WT( - SOC15_REG_OFFSET(UVD, i, mmUVD_RBC_RB_CNTL), tmp); + SOC15_REG_OFFSET(UVD, 0, mmUVD_RBC_RB_CNTL), tmp); /* add end packet */ tmp = sizeof(struct mmsch_v2_0_cmd_end); diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v2_5.c b/drivers/gpu/drm/amd/amdgpu/vcn_v2_5.c index cebee453871c1f..006a1545119717 100644 --- a/drivers/gpu/drm/amd/amdgpu/vcn_v2_5.c +++ b/drivers/gpu/drm/amd/amdgpu/vcn_v2_5.c @@ -521,7 +521,9 @@ static int vcn_v2_5_hw_fini(struct amdgpu_ip_block *ip_block) RREG32_SOC15(VCN, i, mmUVD_STATUS))) vinst->set_pg_state(vinst, AMD_PG_STATE_GATE); - if (amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__VCN)) + /* VF doesn't enable interrupt operations for RAS */ + if (!amdgpu_sriov_vf(adev) && + amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__VCN)) amdgpu_irq_put(adev, &vinst->ras_poison_irq, 0); } diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v5_0_0.c b/drivers/gpu/drm/amd/amdgpu/vcn_v5_0_0.c index 0202df5db1e128..6109124f852e52 100644 --- a/drivers/gpu/drm/amd/amdgpu/vcn_v5_0_0.c +++ b/drivers/gpu/drm/amd/amdgpu/vcn_v5_0_0.c @@ -174,6 +174,10 @@ static int vcn_v5_0_0_sw_init(struct amdgpu_ip_block *ip_block) fw_shared->present_flag_0 = cpu_to_le32(AMDGPU_FW_SHARED_FLAG_0_UNIFIED_QUEUE); fw_shared->sq.is_enabled = 1; + fw_shared->present_flag_0 |= cpu_to_le32(AMDGPU_VCN_SMU_DPM_INTERFACE_FLAG); + fw_shared->smu_dpm_interface.smu_interface_type = (adev->flags & AMD_IS_APU) ? + AMDGPU_VCN_SMU_DPM_INTERFACE_APU : AMDGPU_VCN_SMU_DPM_INTERFACE_DGPU; + if (amdgpu_vcnfw_log) amdgpu_vcn_fwlog_init(&adev->vcn.inst[i]); diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_debug.c b/drivers/gpu/drm/amd/amdkfd/kfd_debug.c index ba99e0f258aee2..986cb297de8f83 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_debug.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_debug.c @@ -401,27 +401,25 @@ static int kfd_dbg_get_dev_watch_id(struct kfd_process_device *pdd, int *watch_i return -ENOMEM; } -static void kfd_dbg_clear_dev_watch_id(struct kfd_process_device *pdd, int watch_id) +static void kfd_dbg_clear_dev_watch_id(struct kfd_process_device *pdd, u32 watch_id) { spin_lock(&pdd->dev->watch_points_lock); /* process owns device watch point so safe to clear */ - if ((pdd->alloc_watch_ids >> watch_id) & 0x1) { - pdd->alloc_watch_ids &= ~(0x1 << watch_id); - pdd->dev->alloc_watch_ids &= ~(0x1 << watch_id); + if (pdd->alloc_watch_ids & BIT(watch_id)) { + pdd->alloc_watch_ids &= ~BIT(watch_id); + pdd->dev->alloc_watch_ids &= ~BIT(watch_id); } spin_unlock(&pdd->dev->watch_points_lock); } -static bool kfd_dbg_owns_dev_watch_id(struct kfd_process_device *pdd, int watch_id) +static bool kfd_dbg_owns_dev_watch_id(struct kfd_process_device *pdd, u32 watch_id) { bool owns_watch_id = false; spin_lock(&pdd->dev->watch_points_lock); - owns_watch_id = watch_id < MAX_WATCH_ADDRESSES && - ((pdd->alloc_watch_ids >> watch_id) & 0x1); - + owns_watch_id = pdd->alloc_watch_ids & BIT(watch_id); spin_unlock(&pdd->dev->watch_points_lock); return owns_watch_id; @@ -432,6 +430,9 @@ int kfd_dbg_trap_clear_dev_address_watch(struct kfd_process_device *pdd, { int r; + if (watch_id >= MAX_WATCH_ADDRESSES) + return -EINVAL; + if (!kfd_dbg_owns_dev_watch_id(pdd, watch_id)) return -EINVAL; @@ -469,6 +470,9 @@ int kfd_dbg_trap_set_dev_address_watch(struct kfd_process_device *pdd, if (r) return r; + if (*watch_id >= MAX_WATCH_ADDRESSES) + return -EINVAL; + if (!pdd->dev->kfd->shared_resources.enable_mes) { r = debug_lock_and_unmap(pdd->dev->dqm); if (r) { diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_events.c b/drivers/gpu/drm/amd/amdkfd/kfd_events.c index 5a190dd6be4e27..844a6a28a64081 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_events.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_events.c @@ -331,6 +331,12 @@ static int kfd_event_page_set(struct kfd_process *p, void *kernel_address, if (p->signal_page) return -EBUSY; + if (size < KFD_SIGNAL_EVENT_LIMIT * 8) { + pr_err("Event page size %llu is too small, need at least %lu bytes\n", + size, (unsigned long)(KFD_SIGNAL_EVENT_LIMIT * 8)); + return -EINVAL; + } + page = kzalloc(sizeof(*page), GFP_KERNEL); if (!page) return -ENOMEM; diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c index 6ada7b4af7c685..5086caac3fd066 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c @@ -61,7 +61,7 @@ svm_migrate_gart_map(struct amdgpu_ring *ring, u64 npages, *gart_addr = adev->gmc.gart_start; num_dw = ALIGN(adev->mman.buffer_funcs->copy_num_dw, 8); - num_bytes = npages * 8; + num_bytes = npages * 8 * AMDGPU_GPU_PAGES_IN_CPU_PAGE; r = amdgpu_job_alloc_with_ib(adev, &adev->mman.high_pr, AMDGPU_FENCE_OWNER_UNDEFINED, diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h index 70ef051511bb1e..4f4eb0791138a4 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h +++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h @@ -102,8 +102,8 @@ * The first chunk is the TBA used for the CWSR ISA code. The second * chunk is used as TMA for user-mode trap handler setup in daisy-chain mode. */ -#define KFD_CWSR_TBA_TMA_SIZE (PAGE_SIZE * 2) -#define KFD_CWSR_TMA_OFFSET (PAGE_SIZE + 2048) +#define KFD_CWSR_TBA_TMA_SIZE (AMDGPU_GPU_PAGE_SIZE * 2) +#define KFD_CWSR_TMA_OFFSET (AMDGPU_GPU_PAGE_SIZE + 2048) #define KFD_MAX_NUM_OF_QUEUES_PER_DEVICE \ (KFD_MAX_NUM_OF_PROCESSES * \ diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process.c b/drivers/gpu/drm/amd/amdkfd/kfd_process.c index a085faac9fe1a4..5c15168f4faf6b 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_process.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_process.c @@ -1996,7 +1996,7 @@ static int signal_eviction_fence(struct kfd_process *p) ef = dma_fence_get_rcu_safe(&p->ef); rcu_read_unlock(); if (!ef) - return -EINVAL; + return true; ret = dma_fence_signal(ef); dma_fence_put(ef); diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c index 7fbb5c274ccc42..7bf712032c52c0 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c @@ -606,6 +606,7 @@ int pqm_update_queue_properties(struct process_queue_manager *pqm, p->queue_size)) { pr_debug("ring buf 0x%llx size 0x%llx not mapped on GPU\n", p->queue_address, p->queue_size); + amdgpu_bo_unreserve(vm->root.bo); return -EFAULT; } diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_queue.c b/drivers/gpu/drm/amd/amdkfd/kfd_queue.c index 80c4fa2b0975dc..b97f4a51db6e37 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_queue.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_queue.c @@ -275,8 +275,8 @@ int kfd_queue_acquire_buffers(struct kfd_process_device *pdd, struct queue_prope /* EOP buffer is not required for all ASICs */ if (properties->eop_ring_buffer_address) { - if (properties->eop_ring_buffer_size != topo_dev->node_props.eop_buffer_size) { - pr_debug("queue eop bo size 0x%x not equal to node eop buf size 0x%x\n", + if (properties->eop_ring_buffer_size < topo_dev->node_props.eop_buffer_size) { + pr_debug("queue eop bo size 0x%x is less than node eop buf size 0x%x\n", properties->eop_ring_buffer_size, topo_dev->node_props.eop_buffer_size); err = -EINVAL; @@ -284,7 +284,7 @@ int kfd_queue_acquire_buffers(struct kfd_process_device *pdd, struct queue_prope } err = kfd_queue_buffer_get(vm, (void *)properties->eop_ring_buffer_address, &properties->eop_buf_bo, - properties->eop_ring_buffer_size); + ALIGN(properties->eop_ring_buffer_size, PAGE_SIZE)); if (err) goto out_err_unreserve; } @@ -444,10 +444,11 @@ void kfd_queue_ctx_save_restore_size(struct kfd_topology_device *dev) min(cu_num * 40, props->array_count / props->simd_arrays_per_engine * 512) : cu_num * 32; - wg_data_size = ALIGN(cu_num * WG_CONTEXT_DATA_SIZE_PER_CU(gfxv, props), PAGE_SIZE); + wg_data_size = ALIGN(cu_num * WG_CONTEXT_DATA_SIZE_PER_CU(gfxv, props), + AMDGPU_GPU_PAGE_SIZE); ctl_stack_size = wave_num * CNTL_STACK_BYTES_PER_WAVE(gfxv) + 8; ctl_stack_size = ALIGN(SIZEOF_HSA_USER_CONTEXT_SAVE_AREA_HEADER + ctl_stack_size, - PAGE_SIZE); + AMDGPU_GPU_PAGE_SIZE); if ((gfxv / 10000 * 10000) == 100000) { /* HW design limits control stack size to 0x7000. @@ -459,7 +460,7 @@ void kfd_queue_ctx_save_restore_size(struct kfd_topology_device *dev) props->ctl_stack_size = ctl_stack_size; props->debug_memory_size = ALIGN(wave_num * DEBUGGER_BYTES_PER_WAVE, DEBUGGER_BYTES_ALIGN); - props->cwsr_size = ctl_stack_size + wg_data_size; + props->cwsr_size = ALIGN(ctl_stack_size + wg_data_size, PAGE_SIZE); if (gfxv == 80002) /* GFX_VERSION_TONGA */ props->eop_buffer_size = 0x8000; diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c index 79ea138897fcf0..a10cf8650c92b8 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c @@ -33,6 +33,7 @@ #include "amdgpu_hmm.h" #include "amdgpu.h" #include "amdgpu_xgmi.h" +#include "amdgpu_reset.h" #include "kfd_priv.h" #include "kfd_svm.h" #include "kfd_migrate.h" @@ -2349,6 +2350,9 @@ static void svm_range_drain_retry_fault(struct svm_range_list *svms) pr_debug("drain retry fault gpu %d svms %p\n", i, svms); + if (!down_read_trylock(&pdd->dev->adev->reset_domain->sem)) + continue; + amdgpu_ih_wait_on_checkpoint_process_ts(pdd->dev->adev, pdd->dev->adev->irq.retry_cam_enabled ? &pdd->dev->adev->irq.ih : @@ -2358,6 +2362,7 @@ static void svm_range_drain_retry_fault(struct svm_range_list *svms) amdgpu_ih_wait_on_checkpoint_process_ts(pdd->dev->adev, &pdd->dev->adev->irq.ih_soft); + up_read(&pdd->dev->adev->reset_domain->sem); pr_debug("drain retry fault gpu %d svms 0x%p done\n", i, svms); } @@ -2541,7 +2546,7 @@ svm_range_unmap_from_cpu(struct mm_struct *mm, struct svm_range *prange, adev = pdd->dev->adev; /* Check and drain ih1 ring if cam not available */ - if (adev->irq.ih1.ring_size) { + if (!adev->irq.retry_cam_enabled && adev->irq.ih1.ring_size) { ih = &adev->irq.ih1; checkpoint_wptr = amdgpu_ih_get_wptr(adev, ih); if (ih->rptr != checkpoint_wptr) { diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c index a8a59126b2d2b0..6c5a1ba4cb004b 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c @@ -3468,7 +3468,17 @@ static int dm_resume(struct amdgpu_ip_block *ip_block) struct dc_commit_streams_params commit_params = {}; if (dm->dc->caps.ips_support) { + if (!amdgpu_in_reset(adev)) + mutex_lock(&dm->dc_lock); + + /* Need to set POWER_STATE_D0 first or it will not execute + * idle_power_optimizations command to DMUB. + */ + dc_dmub_srv_set_power_state(dm->dc->ctx->dmub_srv, DC_ACPI_CM_POWER_STATE_D0); dc_dmub_srv_apply_idle_power_optimizations(dm->dc, false); + + if (!amdgpu_in_reset(adev)) + mutex_unlock(&dm->dc_lock); } if (amdgpu_in_reset(adev)) { @@ -3881,8 +3891,9 @@ void amdgpu_dm_update_connector_after_detect( aconnector->dc_sink = sink; dc_sink_retain(aconnector->dc_sink); + drm_edid_free(aconnector->drm_edid); + aconnector->drm_edid = NULL; if (sink->dc_edid.length == 0) { - aconnector->drm_edid = NULL; hdmi_cec_unset_edid(aconnector); if (aconnector->dc_link->aux_mode) { drm_dp_cec_unset_edid(&aconnector->dm_dp_aux.aux); @@ -5395,7 +5406,7 @@ static void setup_backlight_device(struct amdgpu_display_manager *dm, caps = &dm->backlight_caps[aconnector->bl_idx]; /* Only offer ABM property when non-OLED and user didn't turn off by module parameter */ - if (!caps->ext_caps->bits.oled && amdgpu_dm_abm_level < 0) + if (caps->ext_caps && !caps->ext_caps->bits.oled && amdgpu_dm_abm_level < 0) drm_object_attach_property(&aconnector->base.base, dm->adev->mode_info.abm_level_property, ABM_SYSFS_CONTROL); @@ -7434,7 +7445,7 @@ amdgpu_dm_connector_poll(struct amdgpu_dm_connector *aconnector, bool force) * * Only allow to poll such a connector again when forcing. */ - if (!force && link->local_sink && link->type == dc_connection_dac_load) + if (!force && link->local_sink && link->type == dc_connection_analog_load) return connector->status; mutex_lock(&aconnector->hpd_lock); @@ -10648,10 +10659,10 @@ static void dm_set_writeback(struct amdgpu_display_manager *dm, wb_info->dwb_params.capture_rate = dwb_capture_rate_0; - wb_info->dwb_params.scaler_taps.h_taps = 4; - wb_info->dwb_params.scaler_taps.v_taps = 4; - wb_info->dwb_params.scaler_taps.h_taps_c = 2; - wb_info->dwb_params.scaler_taps.v_taps_c = 2; + wb_info->dwb_params.scaler_taps.h_taps = 1; + wb_info->dwb_params.scaler_taps.v_taps = 1; + wb_info->dwb_params.scaler_taps.h_taps_c = 1; + wb_info->dwb_params.scaler_taps.v_taps_c = 1; wb_info->dwb_params.subsample_position = DWB_INTERSTITIAL_SUBSAMPLING; wb_info->mcif_buf_params.luma_pitch = afb->base.pitches[0]; @@ -10951,7 +10962,7 @@ static void amdgpu_dm_atomic_commit_tail(struct drm_atomic_state *state) continue; } for (j = 0; j < status->plane_count; j++) - dummy_updates[j].surface = status->plane_states[0]; + dummy_updates[j].surface = status->plane_states[j]; sort(dummy_updates, status->plane_count, sizeof(*dummy_updates), dm_plane_layer_index_cmp, NULL); @@ -11667,6 +11678,8 @@ static bool should_reset_plane(struct drm_atomic_state *state, struct drm_crtc_state *old_crtc_state, *new_crtc_state; struct dm_crtc_state *old_dm_crtc_state, *new_dm_crtc_state; struct amdgpu_device *adev = drm_to_adev(plane->dev); + struct drm_connector_state *new_con_state; + struct drm_connector *connector; int i; /* @@ -11677,6 +11690,15 @@ static bool should_reset_plane(struct drm_atomic_state *state, state->allow_modeset) return true; + /* Check for writeback commit */ + for_each_new_connector_in_state(state, connector, new_con_state, i) { + if (connector->connector_type != DRM_MODE_CONNECTOR_WRITEBACK) + continue; + + if (new_con_state->writeback_job) + return true; + } + if (amdgpu_in_reset(adev) && state->allow_modeset) return true; @@ -11853,7 +11875,7 @@ static int dm_check_cursor_fb(struct amdgpu_crtc *new_acrtc, * check tiling flags when the FB doesn't have a modifier. */ if (!(fb->flags & DRM_MODE_FB_MODIFIERS)) { - if (adev->family >= AMDGPU_FAMILY_GC_12_0_0) { + if (adev->family == AMDGPU_FAMILY_GC_12_0_0) { linear = AMDGPU_TILING_GET(afb->tiling_flags, GFX12_SWIZZLE_MODE) == 0; } else if (adev->family >= AMDGPU_FAMILY_AI) { linear = AMDGPU_TILING_GET(afb->tiling_flags, SWIZZLE_MODE) == 0; @@ -12275,10 +12297,9 @@ static int dm_crtc_get_cursor_mode(struct amdgpu_device *adev, /* Overlay cursor not supported on HW before DCN * DCN401 does not have the cursor-on-scaled-plane or cursor-on-yuv-plane restrictions - * as previous DCN generations, so enable native mode on DCN401 in addition to DCE + * as previous DCN generations, so enable native mode on DCN401 */ - if (amdgpu_ip_version(adev, DCE_HWIP, 0) == 0 || - amdgpu_ip_version(adev, DCE_HWIP, 0) == IP_VERSION(4, 0, 1)) { + if (amdgpu_ip_version(adev, DCE_HWIP, 0) == IP_VERSION(4, 0, 1)) { *cursor_mode = DM_CURSOR_NATIVE_MODE; return 0; } @@ -12485,6 +12506,11 @@ static int amdgpu_dm_atomic_check(struct drm_device *dev, } if (dc_resource_is_dsc_encoding_supported(dc)) { + for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) { + dm_new_crtc_state = to_dm_crtc_state(new_crtc_state); + dm_new_crtc_state->mode_changed_independent_from_dsc = new_crtc_state->mode_changed; + } + for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) { if (drm_atomic_crtc_needs_modeset(new_crtc_state)) { ret = add_affected_mst_dsc_crtcs(state, crtc); @@ -12598,6 +12624,12 @@ static int amdgpu_dm_atomic_check(struct drm_device *dev, * need to be added for DC to not disable a plane by mistake */ if (dm_new_crtc_state->cursor_mode == DM_CURSOR_OVERLAY_MODE) { + if (amdgpu_ip_version(adev, DCE_HWIP, 0) == 0) { + drm_dbg(dev, "Overlay cursor not supported on DCE\n"); + ret = -EINVAL; + goto fail; + } + ret = drm_atomic_add_affected_planes(state, crtc); if (ret) goto fail; @@ -13075,7 +13107,7 @@ static void parse_edid_displayid_vrr(struct drm_connector *connector, u16 min_vfreq; u16 max_vfreq; - if (edid == NULL || edid->extensions == 0) + if (!edid || !edid->extensions) return; /* Find DisplayID extension */ @@ -13085,7 +13117,7 @@ static void parse_edid_displayid_vrr(struct drm_connector *connector, break; } - if (edid_ext == NULL) + if (i == edid->extensions) return; while (j < EDID_LENGTH) { diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h index beb0d04d3e6827..dbc3db0d68292b 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h @@ -965,6 +965,7 @@ struct dm_crtc_state { bool freesync_vrr_info_changed; + bool mode_changed_independent_from_dsc; bool dsc_force_changed; bool vrr_supported; struct mod_freesync_config freesync_config; diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_color.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_color.c index 1dcc79b35225f5..12c52bffe9964d 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_color.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_color.c @@ -23,6 +23,9 @@ * Authors: AMD * */ + +#include + #include "amdgpu.h" #include "amdgpu_mode.h" #include "amdgpu_dm.h" @@ -1703,6 +1706,7 @@ __set_dm_plane_colorop_3dlut(struct drm_plane_state *plane_state, struct dc_transfer_func *tf = &dc_plane_state->in_shaper_func; struct drm_atomic_state *state = plane_state->state; const struct amdgpu_device *adev = drm_to_adev(colorop->dev); + bool has_3dlut = adev->dm.dc->caps.color.dpp.hw_3d_lut || adev->dm.dc->caps.color.mpc.preblend; const struct drm_device *dev = colorop->dev; const struct drm_color_lut32 *lut3d; uint32_t lut3d_size; @@ -1719,7 +1723,7 @@ __set_dm_plane_colorop_3dlut(struct drm_plane_state *plane_state, } if (colorop_state && !colorop_state->bypass && colorop->type == DRM_COLOROP_3D_LUT) { - if (!adev->dm.dc->caps.color.dpp.hw_3d_lut) { + if (!has_3dlut) { drm_dbg(dev, "3D LUT is not supported by hardware\n"); return -EINVAL; } @@ -1872,6 +1876,7 @@ amdgpu_dm_plane_set_colorop_properties(struct drm_plane_state *plane_state, struct drm_colorop *colorop = plane_state->color_pipeline; struct drm_device *dev = plane_state->plane->dev; struct amdgpu_device *adev = drm_to_adev(dev); + bool has_3dlut = adev->dm.dc->caps.color.dpp.hw_3d_lut || adev->dm.dc->caps.color.mpc.preblend; int ret; /* 1D Curve - DEGAM TF */ @@ -1904,7 +1909,7 @@ amdgpu_dm_plane_set_colorop_properties(struct drm_plane_state *plane_state, if (ret) return ret; - if (adev->dm.dc->caps.color.dpp.hw_3d_lut) { + if (has_3dlut) { /* 1D Curve & LUT - SHAPER TF & LUT */ colorop = colorop->next; if (!colorop) { diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_colorop.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_colorop.c index a2de3bba834646..212c13b745d0c9 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_colorop.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_colorop.c @@ -37,19 +37,19 @@ const u64 amdgpu_dm_supported_degam_tfs = BIT(DRM_COLOROP_1D_CURVE_SRGB_EOTF) | BIT(DRM_COLOROP_1D_CURVE_PQ_125_EOTF) | BIT(DRM_COLOROP_1D_CURVE_BT2020_INV_OETF) | - BIT(DRM_COLOROP_1D_CURVE_GAMMA22_INV); + BIT(DRM_COLOROP_1D_CURVE_GAMMA22); const u64 amdgpu_dm_supported_shaper_tfs = BIT(DRM_COLOROP_1D_CURVE_SRGB_INV_EOTF) | BIT(DRM_COLOROP_1D_CURVE_PQ_125_INV_EOTF) | BIT(DRM_COLOROP_1D_CURVE_BT2020_OETF) | - BIT(DRM_COLOROP_1D_CURVE_GAMMA22); + BIT(DRM_COLOROP_1D_CURVE_GAMMA22_INV); const u64 amdgpu_dm_supported_blnd_tfs = BIT(DRM_COLOROP_1D_CURVE_SRGB_EOTF) | BIT(DRM_COLOROP_1D_CURVE_PQ_125_EOTF) | BIT(DRM_COLOROP_1D_CURVE_BT2020_INV_OETF) | - BIT(DRM_COLOROP_1D_CURVE_GAMMA22_INV); + BIT(DRM_COLOROP_1D_CURVE_GAMMA22); #define MAX_COLOR_PIPELINE_OPS 10 @@ -60,6 +60,7 @@ int amdgpu_dm_initialize_default_pipeline(struct drm_plane *plane, struct drm_pr struct drm_colorop *ops[MAX_COLOR_PIPELINE_OPS]; struct drm_device *dev = plane->dev; struct amdgpu_device *adev = drm_to_adev(dev); + bool has_3dlut = adev->dm.dc->caps.color.dpp.hw_3d_lut || adev->dm.dc->caps.color.mpc.preblend; int ret; int i = 0; @@ -112,7 +113,7 @@ int amdgpu_dm_initialize_default_pipeline(struct drm_plane *plane, struct drm_pr i++; - if (adev->dm.dc->caps.color.dpp.hw_3d_lut) { + if (has_3dlut) { /* 1D curve - SHAPER TF */ ops[i] = kzalloc(sizeof(*ops[0]), GFP_KERNEL); if (!ops[i]) { diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crtc.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crtc.c index 9fcd72d87d25b1..39fcbc3e702dcc 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crtc.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crtc.c @@ -765,15 +765,15 @@ int amdgpu_dm_crtc_init(struct amdgpu_display_manager *dm, dm->adev->mode_info.crtcs[crtc_index] = acrtc; /* Don't enable DRM CRTC degamma property for - * 1. Degamma is replaced by color pipeline. - * 2. DCE since it doesn't support programmable degamma anywhere. - * 3. DCN401 since pre-blending degamma LUT doesn't apply to cursor. + * 1. DCE since it doesn't support programmable degamma anywhere. + * 2. DCN401 since pre-blending degamma LUT doesn't apply to cursor. + * Note: DEGAMMA properties are created even if the primary plane has the + * COLOR_PIPELINE property. User space can use either the DEGAMMA properties + * or the COLOR_PIPELINE property. An atomic commit which attempts to enable + * both is rejected. */ - if (plane->color_pipeline_property) - has_degamma = false; - else - has_degamma = dm->adev->dm.dc->caps.color.dpp.dcn_arch && - dm->adev->dm.dc->ctx->dce_version != DCN_VERSION_4_01; + has_degamma = dm->adev->dm.dc->caps.color.dpp.dcn_arch && + dm->adev->dm.dc->ctx->dce_version != DCN_VERSION_4_01; drm_crtc_enable_color_mgmt(&acrtc->base, has_degamma ? MAX_COLOR_LUT_ENTRIES : 0, true, MAX_COLOR_LUT_ENTRIES); diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_irq.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_irq.c index e7b0928bd3db79..5948e2a6219e37 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_irq.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_irq.c @@ -919,16 +919,15 @@ void amdgpu_dm_hpd_init(struct amdgpu_device *adev) continue; amdgpu_dm_connector = to_amdgpu_dm_connector(connector); + dc_link = amdgpu_dm_connector->dc_link; + if (!dc_link) + continue; /* * Analog connectors may be hot-plugged unlike other connector * types that don't support HPD. Only poll analog connectors. */ - use_polling |= - amdgpu_dm_connector->dc_link && - dc_connector_supports_analog(amdgpu_dm_connector->dc_link->link_id.id); - - dc_link = amdgpu_dm_connector->dc_link; + use_polling |= dc_connector_supports_analog(dc_link->link_id.id); /* * Get a base driver irq reference for hpd ints for the lifetime diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c index 5e92eaa67aa33d..2e0895f4f9b10f 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c @@ -1744,9 +1744,11 @@ int pre_validate_dsc(struct drm_atomic_state *state, int ind = find_crtc_index_in_state_by_stream(state, stream); if (ind >= 0) { + struct dm_crtc_state *dm_new_crtc_state = to_dm_crtc_state(state->crtcs[ind].new_state); + DRM_INFO_ONCE("%s:%d MST_DSC no mode changed for stream 0x%p\n", __func__, __LINE__, stream); - state->crtcs[ind].new_state->mode_changed = 0; + dm_new_crtc_state->base.mode_changed = dm_new_crtc_state->mode_changed_independent_from_dsc; } } } diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_plane.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_plane.c index 7c4496fb4b9d48..44b9c8ca6d7195 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_plane.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_plane.c @@ -278,7 +278,7 @@ static int amdgpu_dm_plane_validate_dcc(struct amdgpu_device *adev, if (!dcc->enable) return 0; - if (adev->family < AMDGPU_FAMILY_GC_12_0_0 && + if (adev->family != AMDGPU_FAMILY_GC_12_0_0 && format >= SURFACE_PIXEL_FORMAT_VIDEO_BEGIN) return -EINVAL; @@ -901,7 +901,7 @@ int amdgpu_dm_plane_fill_plane_buffer_attributes(struct amdgpu_device *adev, upper_32_bits(chroma_addr); } - if (adev->family >= AMDGPU_FAMILY_GC_12_0_0) { + if (adev->family == AMDGPU_FAMILY_GC_12_0_0) { ret = amdgpu_dm_plane_fill_gfx12_plane_attributes_from_modifiers(adev, afb, format, rotation, plane_size, tiling_info, dcc, @@ -1060,10 +1060,15 @@ static void amdgpu_dm_plane_get_min_max_dc_plane_scaling(struct drm_device *dev, *min_downscale = plane_cap->max_downscale_factor.nv12; break; + /* All 64 bpp formats have the same fp16 scaling limits */ case DRM_FORMAT_XRGB16161616F: case DRM_FORMAT_ARGB16161616F: case DRM_FORMAT_XBGR16161616F: case DRM_FORMAT_ABGR16161616F: + case DRM_FORMAT_XRGB16161616: + case DRM_FORMAT_ARGB16161616: + case DRM_FORMAT_XBGR16161616: + case DRM_FORMAT_ABGR16161616: *max_upscale = plane_cap->max_upscale_factor.fp16; *min_downscale = plane_cap->max_downscale_factor.fp16; break; @@ -1251,6 +1256,14 @@ static int amdgpu_dm_plane_atomic_check(struct drm_plane *plane, if (ret) return ret; + /* Reject commits that attempt to use both COLOR_PIPELINE and CRTC DEGAMMA_LUT */ + if (new_plane_state->color_pipeline && new_crtc_state->degamma_lut) { + drm_dbg_atomic(plane->dev, + "[PLANE:%d:%s] COLOR_PIPELINE and CRTC DEGAMMA_LUT cannot be enabled simultaneously\n", + plane->base.id, plane->name); + return -EINVAL; + } + ret = amdgpu_dm_plane_fill_dc_scaling_info(adev, new_plane_state, &scaling_info); if (ret) return ret; diff --git a/drivers/gpu/drm/amd/display/dc/bios/command_table.c b/drivers/gpu/drm/amd/display/dc/bios/command_table.c index 76a3559f0ddc15..b692fa37402d9f 100644 --- a/drivers/gpu/drm/amd/display/dc/bios/command_table.c +++ b/drivers/gpu/drm/amd/display/dc/bios/command_table.c @@ -1874,8 +1874,7 @@ static void dac_encoder_control_prepare_params( uint8_t dac_standard) { params->ucDacStandard = dac_standard; - if (action == ENCODER_CONTROL_SETUP || - action == ENCODER_CONTROL_INIT) + if (action == ENCODER_CONTROL_INIT) params->ucAction = ATOM_ENCODER_INIT; else if (action == ENCODER_CONTROL_ENABLE) params->ucAction = ATOM_ENABLE; diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/clk_mgr.c index 15cf13ec53026f..c450feae5fa5b1 100644 --- a/drivers/gpu/drm/amd/display/dc/clk_mgr/clk_mgr.c +++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/clk_mgr.c @@ -255,6 +255,10 @@ struct clk_mgr *dc_clk_mgr_create(struct dc_context *ctx, struct pp_smu_funcs *p BREAK_TO_DEBUGGER(); return NULL; } + if (ctx->dce_version == DCN_VERSION_2_01) { + dcn201_clk_mgr_construct(ctx, clk_mgr, pp_smu, dccg); + return &clk_mgr->base; + } if (ASICREV_IS_SIENNA_CICHLID_P(asic_id.hw_internal_rev)) { dcn3_clk_mgr_construct(ctx, clk_mgr, pp_smu, dccg); return &clk_mgr->base; @@ -267,10 +271,6 @@ struct clk_mgr *dc_clk_mgr_create(struct dc_context *ctx, struct pp_smu_funcs *p dcn3_clk_mgr_construct(ctx, clk_mgr, pp_smu, dccg); return &clk_mgr->base; } - if (ctx->dce_version == DCN_VERSION_2_01) { - dcn201_clk_mgr_construct(ctx, clk_mgr, pp_smu, dccg); - return &clk_mgr->base; - } dcn20_clk_mgr_construct(ctx, clk_mgr, pp_smu, dccg); return &clk_mgr->base; } diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn314/dcn314_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn314/dcn314_clk_mgr.c index db687a13174d5b..0cb37827a62b6e 100644 --- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn314/dcn314_clk_mgr.c +++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn314/dcn314_clk_mgr.c @@ -77,7 +77,6 @@ static const struct IP_BASE CLK_BASE = { { { { 0x00016C00, 0x02401800, 0, 0, 0, #undef DC_LOGGER #define DC_LOGGER \ clk_mgr->base.base.ctx->logger - #define regCLK1_CLK_PLL_REQ 0x0237 #define regCLK1_CLK_PLL_REQ_BASE_IDX 0 @@ -88,70 +87,8 @@ static const struct IP_BASE CLK_BASE = { { { { 0x00016C00, 0x02401800, 0, 0, 0, #define CLK1_CLK_PLL_REQ__PllSpineDiv_MASK 0x0000F000L #define CLK1_CLK_PLL_REQ__FbMult_frac_MASK 0xFFFF0000L -#define regCLK1_CLK0_DFS_CNTL 0x0269 -#define regCLK1_CLK0_DFS_CNTL_BASE_IDX 0 -#define regCLK1_CLK1_DFS_CNTL 0x026c -#define regCLK1_CLK1_DFS_CNTL_BASE_IDX 0 -#define regCLK1_CLK2_DFS_CNTL 0x026f -#define regCLK1_CLK2_DFS_CNTL_BASE_IDX 0 -#define regCLK1_CLK3_DFS_CNTL 0x0272 -#define regCLK1_CLK3_DFS_CNTL_BASE_IDX 0 -#define regCLK1_CLK4_DFS_CNTL 0x0275 -#define regCLK1_CLK4_DFS_CNTL_BASE_IDX 0 -#define regCLK1_CLK5_DFS_CNTL 0x0278 -#define regCLK1_CLK5_DFS_CNTL_BASE_IDX 0 - -#define regCLK1_CLK0_CURRENT_CNT 0x02fb -#define regCLK1_CLK0_CURRENT_CNT_BASE_IDX 0 -#define regCLK1_CLK1_CURRENT_CNT 0x02fc -#define regCLK1_CLK1_CURRENT_CNT_BASE_IDX 0 -#define regCLK1_CLK2_CURRENT_CNT 0x02fd -#define regCLK1_CLK2_CURRENT_CNT_BASE_IDX 0 -#define regCLK1_CLK3_CURRENT_CNT 0x02fe -#define regCLK1_CLK3_CURRENT_CNT_BASE_IDX 0 -#define regCLK1_CLK4_CURRENT_CNT 0x02ff -#define regCLK1_CLK4_CURRENT_CNT_BASE_IDX 0 -#define regCLK1_CLK5_CURRENT_CNT 0x0300 -#define regCLK1_CLK5_CURRENT_CNT_BASE_IDX 0 - -#define regCLK1_CLK0_BYPASS_CNTL 0x028a -#define regCLK1_CLK0_BYPASS_CNTL_BASE_IDX 0 -#define regCLK1_CLK1_BYPASS_CNTL 0x0293 -#define regCLK1_CLK1_BYPASS_CNTL_BASE_IDX 0 #define regCLK1_CLK2_BYPASS_CNTL 0x029c #define regCLK1_CLK2_BYPASS_CNTL_BASE_IDX 0 -#define regCLK1_CLK3_BYPASS_CNTL 0x02a5 -#define regCLK1_CLK3_BYPASS_CNTL_BASE_IDX 0 -#define regCLK1_CLK4_BYPASS_CNTL 0x02ae -#define regCLK1_CLK4_BYPASS_CNTL_BASE_IDX 0 -#define regCLK1_CLK5_BYPASS_CNTL 0x02b7 -#define regCLK1_CLK5_BYPASS_CNTL_BASE_IDX 0 - -#define regCLK1_CLK0_DS_CNTL 0x0283 -#define regCLK1_CLK0_DS_CNTL_BASE_IDX 0 -#define regCLK1_CLK1_DS_CNTL 0x028c -#define regCLK1_CLK1_DS_CNTL_BASE_IDX 0 -#define regCLK1_CLK2_DS_CNTL 0x0295 -#define regCLK1_CLK2_DS_CNTL_BASE_IDX 0 -#define regCLK1_CLK3_DS_CNTL 0x029e -#define regCLK1_CLK3_DS_CNTL_BASE_IDX 0 -#define regCLK1_CLK4_DS_CNTL 0x02a7 -#define regCLK1_CLK4_DS_CNTL_BASE_IDX 0 -#define regCLK1_CLK5_DS_CNTL 0x02b0 -#define regCLK1_CLK5_DS_CNTL_BASE_IDX 0 - -#define regCLK1_CLK0_ALLOW_DS 0x0284 -#define regCLK1_CLK0_ALLOW_DS_BASE_IDX 0 -#define regCLK1_CLK1_ALLOW_DS 0x028d -#define regCLK1_CLK1_ALLOW_DS_BASE_IDX 0 -#define regCLK1_CLK2_ALLOW_DS 0x0296 -#define regCLK1_CLK2_ALLOW_DS_BASE_IDX 0 -#define regCLK1_CLK3_ALLOW_DS 0x029f -#define regCLK1_CLK3_ALLOW_DS_BASE_IDX 0 -#define regCLK1_CLK4_ALLOW_DS 0x02a8 -#define regCLK1_CLK4_ALLOW_DS_BASE_IDX 0 -#define regCLK1_CLK5_ALLOW_DS 0x02b1 -#define regCLK1_CLK5_ALLOW_DS_BASE_IDX 0 #define CLK1_CLK2_BYPASS_CNTL__CLK2_BYPASS_SEL__SHIFT 0x0 #define CLK1_CLK2_BYPASS_CNTL__CLK2_BYPASS_DIV__SHIFT 0x10 @@ -248,8 +185,6 @@ void dcn314_init_clocks(struct clk_mgr *clk_mgr) { struct clk_mgr_internal *clk_mgr_int = TO_CLK_MGR_INTERNAL(clk_mgr); uint32_t ref_dtbclk = clk_mgr->clks.ref_dtbclk_khz; - struct clk_mgr_dcn314 *clk_mgr_dcn314 = TO_CLK_MGR_DCN314(clk_mgr_int); - struct clk_log_info log_info = {0}; memset(&(clk_mgr->clks), 0, sizeof(struct dc_clocks)); // Assumption is that boot state always supports pstate @@ -265,9 +200,6 @@ void dcn314_init_clocks(struct clk_mgr *clk_mgr) dce_adjust_dp_ref_freq_for_ss(clk_mgr_int, clk_mgr->dprefclk_khz); else clk_mgr->dp_dto_source_clock_in_khz = clk_mgr->dprefclk_khz; - - dcn314_dump_clk_registers(&clk_mgr->boot_snapshot, &clk_mgr_dcn314->base.base, &log_info); - clk_mgr->clks.dispclk_khz = clk_mgr->boot_snapshot.dispclk * 1000; } void dcn314_update_clocks(struct clk_mgr *clk_mgr_base, @@ -278,7 +210,7 @@ void dcn314_update_clocks(struct clk_mgr *clk_mgr_base, struct clk_mgr_internal *clk_mgr = TO_CLK_MGR_INTERNAL(clk_mgr_base); struct dc_clocks *new_clocks = &context->bw_ctx.bw.dcn.clk; struct dc *dc = clk_mgr_base->ctx->dc; - int display_count; + int display_count = 0; bool update_dppclk = false; bool update_dispclk = false; bool dpp_clock_lowered = false; @@ -287,7 +219,6 @@ void dcn314_update_clocks(struct clk_mgr *clk_mgr_base, return; display_count = dcn314_get_active_display_cnt_wa(dc, context); - /* * if it is safe to lower, but we are already in the lower state, we don't have to do anything * also if safe to lower is false, we just go in the higher state @@ -363,7 +294,7 @@ void dcn314_update_clocks(struct clk_mgr *clk_mgr_base, } if (should_set_clock(safe_to_lower, new_clocks->dispclk_khz, clk_mgr_base->clks.dispclk_khz) && - (new_clocks->dispclk_khz > 0 || (safe_to_lower && display_count == 0))) { + (new_clocks->dispclk_khz > 0 || (safe_to_lower && display_count == 0))) { int requested_dispclk_khz = new_clocks->dispclk_khz; dcn314_disable_otg_wa(clk_mgr_base, context, safe_to_lower, true); @@ -374,7 +305,6 @@ void dcn314_update_clocks(struct clk_mgr *clk_mgr_base, dcn314_smu_set_dispclk(clk_mgr, requested_dispclk_khz); clk_mgr_base->clks.dispclk_khz = new_clocks->dispclk_khz; - dcn314_disable_otg_wa(clk_mgr_base, context, safe_to_lower, false); update_dispclk = true; @@ -462,65 +392,10 @@ bool dcn314_are_clock_states_equal(struct dc_clocks *a, return true; } - -static void dcn314_dump_clk_registers_internal(struct dcn35_clk_internal *internal, struct clk_mgr *clk_mgr_base) -{ - struct clk_mgr_internal *clk_mgr = TO_CLK_MGR_INTERNAL(clk_mgr_base); - - // read dtbclk - internal->CLK1_CLK4_CURRENT_CNT = REG_READ(CLK1_CLK4_CURRENT_CNT); - internal->CLK1_CLK4_BYPASS_CNTL = REG_READ(CLK1_CLK4_BYPASS_CNTL); - - // read dcfclk - internal->CLK1_CLK3_CURRENT_CNT = REG_READ(CLK1_CLK3_CURRENT_CNT); - internal->CLK1_CLK3_BYPASS_CNTL = REG_READ(CLK1_CLK3_BYPASS_CNTL); - - // read dcf deep sleep divider - internal->CLK1_CLK3_DS_CNTL = REG_READ(CLK1_CLK3_DS_CNTL); - internal->CLK1_CLK3_ALLOW_DS = REG_READ(CLK1_CLK3_ALLOW_DS); - - // read dppclk - internal->CLK1_CLK1_CURRENT_CNT = REG_READ(CLK1_CLK1_CURRENT_CNT); - internal->CLK1_CLK1_BYPASS_CNTL = REG_READ(CLK1_CLK1_BYPASS_CNTL); - - // read dprefclk - internal->CLK1_CLK2_CURRENT_CNT = REG_READ(CLK1_CLK2_CURRENT_CNT); - internal->CLK1_CLK2_BYPASS_CNTL = REG_READ(CLK1_CLK2_BYPASS_CNTL); - - // read dispclk - internal->CLK1_CLK0_CURRENT_CNT = REG_READ(CLK1_CLK0_CURRENT_CNT); - internal->CLK1_CLK0_BYPASS_CNTL = REG_READ(CLK1_CLK0_BYPASS_CNTL); -} - -void dcn314_dump_clk_registers(struct clk_state_registers_and_bypass *regs_and_bypass, +static void dcn314_dump_clk_registers(struct clk_state_registers_and_bypass *regs_and_bypass, struct clk_mgr *clk_mgr_base, struct clk_log_info *log_info) { - - struct dcn35_clk_internal internal = {0}; - - dcn314_dump_clk_registers_internal(&internal, clk_mgr_base); - - regs_and_bypass->dcfclk = internal.CLK1_CLK3_CURRENT_CNT / 10; - regs_and_bypass->dcf_deep_sleep_divider = internal.CLK1_CLK3_DS_CNTL / 10; - regs_and_bypass->dcf_deep_sleep_allow = internal.CLK1_CLK3_ALLOW_DS; - regs_and_bypass->dprefclk = internal.CLK1_CLK2_CURRENT_CNT / 10; - regs_and_bypass->dispclk = internal.CLK1_CLK0_CURRENT_CNT / 10; - regs_and_bypass->dppclk = internal.CLK1_CLK1_CURRENT_CNT / 10; - regs_and_bypass->dtbclk = internal.CLK1_CLK4_CURRENT_CNT / 10; - - regs_and_bypass->dppclk_bypass = internal.CLK1_CLK1_BYPASS_CNTL & 0x0007; - if (regs_and_bypass->dppclk_bypass > 4) - regs_and_bypass->dppclk_bypass = 0; - regs_and_bypass->dcfclk_bypass = internal.CLK1_CLK3_BYPASS_CNTL & 0x0007; - if (regs_and_bypass->dcfclk_bypass > 4) - regs_and_bypass->dcfclk_bypass = 0; - regs_and_bypass->dispclk_bypass = internal.CLK1_CLK0_BYPASS_CNTL & 0x0007; - if (regs_and_bypass->dispclk_bypass > 4) - regs_and_bypass->dispclk_bypass = 0; - regs_and_bypass->dprefclk_bypass = internal.CLK1_CLK2_BYPASS_CNTL & 0x0007; - if (regs_and_bypass->dprefclk_bypass > 4) - regs_and_bypass->dprefclk_bypass = 0; - + return; } static struct clk_bw_params dcn314_bw_params = { diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn314/dcn314_clk_mgr.h b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn314/dcn314_clk_mgr.h index 0577eb527bc36e..002c28e807208e 100644 --- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn314/dcn314_clk_mgr.h +++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn314/dcn314_clk_mgr.h @@ -65,9 +65,4 @@ void dcn314_clk_mgr_construct(struct dc_context *ctx, void dcn314_clk_mgr_destroy(struct clk_mgr_internal *clk_mgr_int); - -void dcn314_dump_clk_registers(struct clk_state_registers_and_bypass *regs_and_bypass, - struct clk_mgr *clk_mgr_base, struct clk_log_info *log_info); - - #endif //__DCN314_CLK_MGR_H__ diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn315/dcn315_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn315/dcn315_clk_mgr.c index 3a881451e9da4f..c49268db85f68d 100644 --- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn315/dcn315_clk_mgr.c +++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn315/dcn315_clk_mgr.c @@ -40,7 +40,7 @@ #include "dm_helpers.h" #include "dc_dmub_srv.h" -#include "reg_helper.h" + #include "logger_types.h" #undef DC_LOGGER #define DC_LOGGER \ @@ -48,43 +48,9 @@ #include "link_service.h" -#define MAX_INSTANCE 7 -#define MAX_SEGMENT 8 - -struct IP_BASE_INSTANCE { - unsigned int segment[MAX_SEGMENT]; -}; - -struct IP_BASE { - struct IP_BASE_INSTANCE instance[MAX_INSTANCE]; -}; - -static const struct IP_BASE CLK_BASE = { { { { 0x00016C00, 0x02401800, 0, 0, 0, 0, 0, 0 } }, - { { 0x00016E00, 0x02401C00, 0, 0, 0, 0, 0, 0 } }, - { { 0x00017000, 0x02402000, 0, 0, 0, 0, 0, 0 } }, - { { 0x00017200, 0x02402400, 0, 0, 0, 0, 0, 0 } }, - { { 0x0001B000, 0x0242D800, 0, 0, 0, 0, 0, 0 } }, - { { 0x0001B200, 0x0242DC00, 0, 0, 0, 0, 0, 0 } } } }; - -#define regCLK1_CLK0_CURRENT_CNT 0x0314 -#define regCLK1_CLK0_CURRENT_CNT_BASE_IDX 0 -#define regCLK1_CLK1_CURRENT_CNT 0x0315 -#define regCLK1_CLK1_CURRENT_CNT_BASE_IDX 0 -#define regCLK1_CLK2_CURRENT_CNT 0x0316 -#define regCLK1_CLK2_CURRENT_CNT_BASE_IDX 0 -#define regCLK1_CLK3_CURRENT_CNT 0x0317 -#define regCLK1_CLK3_CURRENT_CNT_BASE_IDX 0 -#define regCLK1_CLK4_CURRENT_CNT 0x0318 -#define regCLK1_CLK4_CURRENT_CNT_BASE_IDX 0 -#define regCLK1_CLK5_CURRENT_CNT 0x0319 -#define regCLK1_CLK5_CURRENT_CNT_BASE_IDX 0 - #define TO_CLK_MGR_DCN315(clk_mgr)\ container_of(clk_mgr, struct clk_mgr_dcn315, base) -#define REG(reg_name) \ - (CLK_BASE.instance[0].segment[reg ## reg_name ## _BASE_IDX] + reg ## reg_name) - #define UNSUPPORTED_DCFCLK 10000000 #define MIN_DPP_DISP_CLK 100000 @@ -172,7 +138,7 @@ static void dcn315_update_clocks(struct clk_mgr *clk_mgr_base, if (dc->work_arounds.skip_clock_update) return; - clk_mgr_base->clks.zstate_support = new_clocks->zstate_support; + display_count = dcn315_get_active_display_cnt_wa(dc, context); /* * if it is safe to lower, but we are already in the lower state, we don't have to do anything * also if safe to lower is false, we just go in the higher state @@ -185,7 +151,6 @@ static void dcn315_update_clocks(struct clk_mgr *clk_mgr_base, } /* check that we're not already in lower */ if (clk_mgr_base->clks.pwr_state != DCN_PWR_STATE_LOW_POWER) { - display_count = dcn315_get_active_display_cnt_wa(dc, context); /* if we can go lower, go lower */ if (display_count == 0) { union display_idle_optimization_u idle_info = { 0 }; @@ -279,38 +244,9 @@ static void dcn315_update_clocks(struct clk_mgr *clk_mgr_base, dc_wake_and_execute_dmub_cmd(dc->ctx, &cmd, DM_DMUB_WAIT_TYPE_WAIT); } -static void dcn315_dump_clk_registers_internal(struct dcn35_clk_internal *internal, struct clk_mgr *clk_mgr_base) -{ - struct clk_mgr_internal *clk_mgr = TO_CLK_MGR_INTERNAL(clk_mgr_base); - - // read dtbclk - internal->CLK1_CLK4_CURRENT_CNT = REG_READ(CLK1_CLK4_CURRENT_CNT); - - // read dcfclk - internal->CLK1_CLK3_CURRENT_CNT = REG_READ(CLK1_CLK3_CURRENT_CNT); - - // read dppclk - internal->CLK1_CLK1_CURRENT_CNT = REG_READ(CLK1_CLK1_CURRENT_CNT); - - // read dprefclk - internal->CLK1_CLK2_CURRENT_CNT = REG_READ(CLK1_CLK2_CURRENT_CNT); - - // read dispclk - internal->CLK1_CLK0_CURRENT_CNT = REG_READ(CLK1_CLK0_CURRENT_CNT); -} - static void dcn315_dump_clk_registers(struct clk_state_registers_and_bypass *regs_and_bypass, struct clk_mgr *clk_mgr_base, struct clk_log_info *log_info) { - struct dcn35_clk_internal internal = {0}; - - dcn315_dump_clk_registers_internal(&internal, clk_mgr_base); - - regs_and_bypass->dcfclk = internal.CLK1_CLK3_CURRENT_CNT / 10; - regs_and_bypass->dprefclk = internal.CLK1_CLK2_CURRENT_CNT / 10; - regs_and_bypass->dispclk = internal.CLK1_CLK0_CURRENT_CNT / 10; - regs_and_bypass->dppclk = internal.CLK1_CLK1_CURRENT_CNT / 10; - regs_and_bypass->dtbclk = internal.CLK1_CLK4_CURRENT_CNT / 10; return; } @@ -657,32 +593,13 @@ static struct clk_mgr_funcs dcn315_funcs = { .get_dp_ref_clk_frequency = dce12_get_dp_ref_freq_khz, .get_dtb_ref_clk_frequency = dcn31_get_dtb_ref_freq_khz, .update_clocks = dcn315_update_clocks, - .init_clocks = dcn315_init_clocks, + .init_clocks = dcn31_init_clocks, .enable_pme_wa = dcn315_enable_pme_wa, .are_clock_states_equal = dcn31_are_clock_states_equal, .notify_wm_ranges = dcn315_notify_wm_ranges }; extern struct clk_mgr_funcs dcn3_fpga_funcs; -void dcn315_init_clocks(struct clk_mgr *clk_mgr) -{ - struct clk_mgr_internal *clk_mgr_int = TO_CLK_MGR_INTERNAL(clk_mgr); - uint32_t ref_dtbclk = clk_mgr->clks.ref_dtbclk_khz; - struct clk_mgr_dcn315 *clk_mgr_dcn315 = TO_CLK_MGR_DCN315(clk_mgr_int); - struct clk_log_info log_info = {0}; - - memset(&(clk_mgr->clks), 0, sizeof(struct dc_clocks)); - // Assumption is that boot state always supports pstate - clk_mgr->clks.ref_dtbclk_khz = ref_dtbclk; // restore ref_dtbclk - clk_mgr->clks.p_state_change_support = true; - clk_mgr->clks.prev_p_state_change_support = true; - clk_mgr->clks.pwr_state = DCN_PWR_STATE_UNKNOWN; - clk_mgr->clks.zstate_support = DCN_ZSTATE_SUPPORT_UNKNOWN; - - dcn315_dump_clk_registers(&clk_mgr->boot_snapshot, &clk_mgr_dcn315->base.base, &log_info); - clk_mgr->clks.dispclk_khz = clk_mgr->boot_snapshot.dispclk * 1000; -} - void dcn315_clk_mgr_construct( struct dc_context *ctx, struct clk_mgr_dcn315 *clk_mgr, @@ -743,7 +660,6 @@ void dcn315_clk_mgr_construct( /* Saved clocks configured at boot for debug purposes */ dcn315_dump_clk_registers(&clk_mgr->base.base.boot_snapshot, &clk_mgr->base.base, &log_info); - clk_mgr->base.base.clks.dispclk_khz = clk_mgr->base.base.boot_snapshot.dispclk * 1000; clk_mgr->base.base.dprefclk_khz = 600000; clk_mgr->base.base.dprefclk_khz = dcn315_smu_get_dpref_clk(&clk_mgr->base); diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn315/dcn315_clk_mgr.h b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn315/dcn315_clk_mgr.h index 642ae3d4a7909c..ac36ddf5dd1af8 100644 --- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn315/dcn315_clk_mgr.h +++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn315/dcn315_clk_mgr.h @@ -44,7 +44,6 @@ void dcn315_clk_mgr_construct(struct dc_context *ctx, struct pp_smu_funcs *pp_smu, struct dccg *dccg); -void dcn315_init_clocks(struct clk_mgr *clk_mgr); void dcn315_clk_mgr_destroy(struct clk_mgr_internal *clk_mgr_int); #endif //__DCN315_CLK_MGR_H__ diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn35/dcn35_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn35/dcn35_clk_mgr.c index dfd0c9505af096..0a21910c87e103 100644 --- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn35/dcn35_clk_mgr.c +++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn35/dcn35_clk_mgr.c @@ -768,32 +768,32 @@ static struct wm_table ddr5_wm_table = { .wm_inst = WM_A, .wm_type = WM_TYPE_PSTATE_CHG, .pstate_latency_us = 11.72, - .sr_exit_time_us = 28.0, - .sr_enter_plus_exit_time_us = 30.0, + .sr_exit_time_us = 31.0, + .sr_enter_plus_exit_time_us = 33.0, .valid = true, }, { .wm_inst = WM_B, .wm_type = WM_TYPE_PSTATE_CHG, .pstate_latency_us = 11.72, - .sr_exit_time_us = 28.0, - .sr_enter_plus_exit_time_us = 30.0, + .sr_exit_time_us = 31.0, + .sr_enter_plus_exit_time_us = 33.0, .valid = true, }, { .wm_inst = WM_C, .wm_type = WM_TYPE_PSTATE_CHG, .pstate_latency_us = 11.72, - .sr_exit_time_us = 28.0, - .sr_enter_plus_exit_time_us = 30.0, + .sr_exit_time_us = 31.0, + .sr_enter_plus_exit_time_us = 33.0, .valid = true, }, { .wm_inst = WM_D, .wm_type = WM_TYPE_PSTATE_CHG, .pstate_latency_us = 11.72, - .sr_exit_time_us = 28.0, - .sr_enter_plus_exit_time_us = 30.0, + .sr_exit_time_us = 31.0, + .sr_enter_plus_exit_time_us = 33.0, .valid = true, }, } diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_hw_sequencer.c b/drivers/gpu/drm/amd/display/dc/core/dc_hw_sequencer.c index e2763b60482a09..052d573408c3eb 100644 --- a/drivers/gpu/drm/amd/display/dc/core/dc_hw_sequencer.c +++ b/drivers/gpu/drm/amd/display/dc/core/dc_hw_sequencer.c @@ -741,6 +741,7 @@ void hwss_build_fast_sequence(struct dc *dc, struct dce_hwseq *hws = dc->hwseq; struct pipe_ctx *current_pipe = NULL; struct pipe_ctx *current_mpc_pipe = NULL; + bool is_dmub_lock_required = false; unsigned int i = 0; *num_steps = 0; // Initialize to 0 @@ -763,11 +764,12 @@ void hwss_build_fast_sequence(struct dc *dc, (*num_steps)++; } if (dc->hwss.dmub_hw_control_lock_fast) { + is_dmub_lock_required = dc_state_is_fams2_in_use(dc, context) || + dmub_hw_lock_mgr_does_link_require_lock(dc, stream->link); + block_sequence[*num_steps].params.dmub_hw_control_lock_fast_params.dc = dc; block_sequence[*num_steps].params.dmub_hw_control_lock_fast_params.lock = true; - block_sequence[*num_steps].params.dmub_hw_control_lock_fast_params.is_required = - dc_state_is_fams2_in_use(dc, context) || - dmub_hw_lock_mgr_does_link_require_lock(dc, stream->link); + block_sequence[*num_steps].params.dmub_hw_control_lock_fast_params.is_required = is_dmub_lock_required; block_sequence[*num_steps].func = DMUB_HW_CONTROL_LOCK_FAST; (*num_steps)++; } @@ -906,7 +908,7 @@ void hwss_build_fast_sequence(struct dc *dc, if (dc->hwss.dmub_hw_control_lock_fast) { block_sequence[*num_steps].params.dmub_hw_control_lock_fast_params.dc = dc; block_sequence[*num_steps].params.dmub_hw_control_lock_fast_params.lock = false; - block_sequence[*num_steps].params.dmub_hw_control_lock_fast_params.is_required = dc_state_is_fams2_in_use(dc, context); + block_sequence[*num_steps].params.dmub_hw_control_lock_fast_params.is_required = is_dmub_lock_required; block_sequence[*num_steps].func = DMUB_HW_CONTROL_LOCK_FAST; (*num_steps)++; } diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_stream.c b/drivers/gpu/drm/amd/display/dc/core/dc_stream.c index 129cd5f8498377..da44c1f01bef1e 100644 --- a/drivers/gpu/drm/amd/display/dc/core/dc_stream.c +++ b/drivers/gpu/drm/amd/display/dc/core/dc_stream.c @@ -169,7 +169,7 @@ struct dc_stream_state *dc_create_stream_for_sink( if (sink == NULL) return NULL; - stream = kzalloc(sizeof(struct dc_stream_state), GFP_KERNEL); + stream = kzalloc(sizeof(struct dc_stream_state), GFP_ATOMIC); if (stream == NULL) goto alloc_fail; diff --git a/drivers/gpu/drm/amd/display/dc/dc.h b/drivers/gpu/drm/amd/display/dc/dc.h index 29edfa51ea2cc0..0a9758a042586f 100644 --- a/drivers/gpu/drm/amd/display/dc/dc.h +++ b/drivers/gpu/drm/amd/display/dc/dc.h @@ -2793,7 +2793,7 @@ void dc_get_underflow_debug_data_for_otg(struct dc *dc, int primary_otg_inst, st void dc_get_power_feature_status(struct dc *dc, int primary_otg_inst, struct power_features *out_data); -/** +/* * Software state variables used to program register fields across the display pipeline */ struct dc_register_software_state { diff --git a/drivers/gpu/drm/amd/display/dc/dc_types.h b/drivers/gpu/drm/amd/display/dc/dc_types.h index f46039f642034f..3e63d7bda1661c 100644 --- a/drivers/gpu/drm/amd/display/dc/dc_types.h +++ b/drivers/gpu/drm/amd/display/dc/dc_types.h @@ -354,7 +354,7 @@ enum dc_connection_type { dc_connection_single, dc_connection_mst_branch, dc_connection_sst_branch, - dc_connection_dac_load + dc_connection_analog_load }; struct dc_csc_adjustments { diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_link_encoder.c b/drivers/gpu/drm/amd/display/dc/dce/dce_link_encoder.c index 87dbb8d7ed27d8..bec8dab156eecb 100644 --- a/drivers/gpu/drm/amd/display/dc/dce/dce_link_encoder.c +++ b/drivers/gpu/drm/amd/display/dc/dce/dce_link_encoder.c @@ -102,6 +102,7 @@ static const struct link_encoder_funcs dce110_lnk_enc_funcs = { .enable_dp_output = dce110_link_encoder_enable_dp_output, .enable_dp_mst_output = dce110_link_encoder_enable_dp_mst_output, .enable_lvds_output = dce110_link_encoder_enable_lvds_output, + .enable_analog_output = dce110_link_encoder_enable_analog_output, .disable_output = dce110_link_encoder_disable_output, .dp_set_lane_settings = dce110_link_encoder_dp_set_lane_settings, .dp_set_phy_pattern = dce110_link_encoder_dp_set_phy_pattern, @@ -131,6 +132,21 @@ static enum bp_result link_transmitter_control( return result; } +static enum bp_result link_dac_encoder_control( + struct dce110_link_encoder *link_enc, + enum bp_encoder_control_action action, + uint32_t pix_clk_100hz) +{ + struct dc_bios *bios = link_enc->base.ctx->dc_bios; + struct bp_encoder_control encoder_control = {0}; + + encoder_control.action = action; + encoder_control.engine_id = link_enc->base.analog_engine; + encoder_control.pixel_clock = pix_clk_100hz / 10; + + return bios->funcs->encoder_control(bios, &encoder_control); +} + static void enable_phy_bypass_mode( struct dce110_link_encoder *enc110, bool enable) @@ -850,6 +866,7 @@ void dce110_link_encoder_construct( enc110->base.funcs = &dce110_lnk_enc_funcs; enc110->base.ctx = init_data->ctx; enc110->base.id = init_data->encoder; + enc110->base.analog_id = init_data->analog_encoder; enc110->base.hpd_source = init_data->hpd_source; enc110->base.connector = init_data->connector; @@ -1017,6 +1034,16 @@ void dce110_link_encoder_hw_init( cntl.coherent = false; cntl.hpd_sel = enc110->base.hpd_source; + if (enc110->base.analog_engine != ENGINE_ID_UNKNOWN) { + result = link_dac_encoder_control(enc110, ENCODER_CONTROL_INIT, 0); + if (result != BP_RESULT_OK) { + DC_LOG_ERROR("%s: Failed to execute VBIOS command table for DAC!\n", + __func__); + BREAK_TO_DEBUGGER(); + return; + } + } + /* The code below is only applicable to encoders with a digital transmitter. */ if (enc110->base.transmitter == TRANSMITTER_UNKNOWN) return; @@ -1166,6 +1193,22 @@ void dce110_link_encoder_enable_lvds_output( } } +void dce110_link_encoder_enable_analog_output( + struct link_encoder *enc, + uint32_t pixel_clock) +{ + struct dce110_link_encoder *enc110 = TO_DCE110_LINK_ENC(enc); + enum bp_result result; + + result = link_dac_encoder_control(enc110, ENCODER_CONTROL_ENABLE, pixel_clock); + + if (result != BP_RESULT_OK) { + DC_LOG_ERROR("%s: Failed to execute VBIOS command table!\n", + __func__); + BREAK_TO_DEBUGGER(); + } +} + /* enables DP PHY output */ void dce110_link_encoder_enable_dp_output( struct link_encoder *enc, @@ -1336,19 +1379,8 @@ void dce110_link_encoder_disable_output( struct bp_transmitter_control cntl = { 0 }; enum bp_result result; - switch (enc->analog_engine) { - case ENGINE_ID_DACA: - REG_UPDATE(DAC_ENABLE, DAC_ENABLE, 0); - break; - case ENGINE_ID_DACB: - /* DACB doesn't seem to be present on DCE6+, - * although there are references to it in the register file. - */ - DC_LOG_ERROR("%s DACB is unsupported\n", __func__); - break; - default: - break; - } + if (enc->analog_engine != ENGINE_ID_UNKNOWN) + link_dac_encoder_control(enc110, ENCODER_CONTROL_DISABLE, 0); /* The code below only applies to connectors that support digital signals. */ if (enc->transmitter == TRANSMITTER_UNKNOWN) @@ -1761,6 +1793,7 @@ static const struct link_encoder_funcs dce60_lnk_enc_funcs = { .enable_dp_output = dce60_link_encoder_enable_dp_output, .enable_dp_mst_output = dce60_link_encoder_enable_dp_mst_output, .enable_lvds_output = dce110_link_encoder_enable_lvds_output, + .enable_analog_output = dce110_link_encoder_enable_analog_output, .disable_output = dce110_link_encoder_disable_output, .dp_set_lane_settings = dce110_link_encoder_dp_set_lane_settings, .dp_set_phy_pattern = dce60_link_encoder_dp_set_phy_pattern, @@ -1793,6 +1826,7 @@ void dce60_link_encoder_construct( enc110->base.funcs = &dce60_lnk_enc_funcs; enc110->base.ctx = init_data->ctx; enc110->base.id = init_data->encoder; + enc110->base.analog_id = init_data->analog_encoder; enc110->base.hpd_source = init_data->hpd_source; enc110->base.connector = init_data->connector; diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_link_encoder.h b/drivers/gpu/drm/amd/display/dc/dce/dce_link_encoder.h index c58b69bc319b73..6870cb619d208e 100644 --- a/drivers/gpu/drm/amd/display/dc/dce/dce_link_encoder.h +++ b/drivers/gpu/drm/amd/display/dc/dce/dce_link_encoder.h @@ -273,6 +273,11 @@ void dce110_link_encoder_enable_lvds_output( enum clock_source_id clock_source, uint32_t pixel_clock); +/* enables analog output from the DAC */ +void dce110_link_encoder_enable_analog_output( + struct link_encoder *enc, + uint32_t pixel_clock); + /* disable PHY output */ void dce110_link_encoder_disable_output( struct link_encoder *enc, diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_stream_encoder.c b/drivers/gpu/drm/amd/display/dc/dce/dce_stream_encoder.c index 574618d5d4a4e0..87c19f17c799fc 100644 --- a/drivers/gpu/drm/amd/display/dc/dce/dce_stream_encoder.c +++ b/drivers/gpu/drm/amd/display/dc/dce/dce_stream_encoder.c @@ -1498,7 +1498,10 @@ static void dig_connect_to_otg( { struct dce110_stream_encoder *enc110 = DCE110STRENC_FROM_STRENC(enc); - REG_UPDATE(DIG_FE_CNTL, DIG_SOURCE_SELECT, tg_inst); + if (enc->id == ENGINE_ID_DACA || enc->id == ENGINE_ID_DACB) + REG_UPDATE(DAC_SOURCE_SELECT, DAC_SOURCE_SELECT, tg_inst); + else + REG_UPDATE(DIG_FE_CNTL, DIG_SOURCE_SELECT, tg_inst); } static unsigned int dig_source_otg( @@ -1507,7 +1510,10 @@ static unsigned int dig_source_otg( uint32_t tg_inst = 0; struct dce110_stream_encoder *enc110 = DCE110STRENC_FROM_STRENC(enc); - REG_GET(DIG_FE_CNTL, DIG_SOURCE_SELECT, &tg_inst); + if (enc->id == ENGINE_ID_DACA || enc->id == ENGINE_ID_DACB) + REG_GET(DAC_SOURCE_SELECT, DAC_SOURCE_SELECT, &tg_inst); + else + REG_GET(DIG_FE_CNTL, DIG_SOURCE_SELECT, &tg_inst); return tg_inst; } @@ -1568,16 +1574,25 @@ void dce110_stream_encoder_construct( enc110->se_mask = se_mask; } -static const struct stream_encoder_funcs dce110_an_str_enc_funcs = {}; +static const struct stream_encoder_funcs dce110_an_str_enc_funcs = { + .dig_connect_to_otg = dig_connect_to_otg, + .dig_source_otg = dig_source_otg, +}; void dce110_analog_stream_encoder_construct( struct dce110_stream_encoder *enc110, struct dc_context *ctx, struct dc_bios *bp, - enum engine_id eng_id) + enum engine_id eng_id, + const struct dce110_stream_enc_registers *regs, + const struct dce_stream_encoder_shift *se_shift, + const struct dce_stream_encoder_mask *se_mask) { enc110->base.funcs = &dce110_an_str_enc_funcs; enc110->base.ctx = ctx; enc110->base.id = eng_id; enc110->base.bp = bp; + enc110->regs = regs; + enc110->se_shift = se_shift; + enc110->se_mask = se_mask; } diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_stream_encoder.h b/drivers/gpu/drm/amd/display/dc/dce/dce_stream_encoder.h index 068de1392121ed..342c0afe6a9494 100644 --- a/drivers/gpu/drm/amd/display/dc/dce/dce_stream_encoder.h +++ b/drivers/gpu/drm/amd/display/dc/dce/dce_stream_encoder.h @@ -65,6 +65,7 @@ SRI(AFMT_60958_1, DIG, id), \ SRI(AFMT_60958_2, DIG, id), \ SRI(DIG_FE_CNTL, DIG, id), \ + SR(DAC_SOURCE_SELECT), \ SRI(HDMI_CONTROL, DIG, id), \ SRI(HDMI_GC, DIG, id), \ SRI(HDMI_GENERIC_PACKET_CONTROL0, DIG, id), \ @@ -290,7 +291,8 @@ #define SE_COMMON_MASK_SH_LIST_DCE80_100(mask_sh)\ SE_COMMON_MASK_SH_LIST_DCE_COMMON(mask_sh),\ SE_SF(TMDS_CNTL, TMDS_PIXEL_ENCODING, mask_sh),\ - SE_SF(TMDS_CNTL, TMDS_COLOR_FORMAT, mask_sh) + SE_SF(TMDS_CNTL, TMDS_COLOR_FORMAT, mask_sh),\ + SE_SF(DAC_SOURCE_SELECT, DAC_SOURCE_SELECT, mask_sh) #define SE_COMMON_MASK_SH_LIST_DCE110(mask_sh)\ SE_COMMON_MASK_SH_LIST_DCE_COMMON(mask_sh),\ @@ -494,6 +496,7 @@ struct dce_stream_encoder_shift { uint8_t DP_VID_N_MUL; uint8_t DP_VID_M_DOUBLE_VALUE_EN; uint8_t DIG_SOURCE_SELECT; + uint8_t DAC_SOURCE_SELECT; }; struct dce_stream_encoder_mask { @@ -626,6 +629,7 @@ struct dce_stream_encoder_mask { uint32_t DP_VID_N_MUL; uint32_t DP_VID_M_DOUBLE_VALUE_EN; uint32_t DIG_SOURCE_SELECT; + uint32_t DAC_SOURCE_SELECT; }; struct dce110_stream_enc_registers { @@ -653,6 +657,7 @@ struct dce110_stream_enc_registers { uint32_t AFMT_60958_1; uint32_t AFMT_60958_2; uint32_t DIG_FE_CNTL; + uint32_t DAC_SOURCE_SELECT; uint32_t DP_MSE_RATE_CNTL; uint32_t DP_MSE_RATE_UPDATE; uint32_t DP_PIXEL_FORMAT; @@ -712,7 +717,10 @@ void dce110_analog_stream_encoder_construct( struct dce110_stream_encoder *enc110, struct dc_context *ctx, struct dc_bios *bp, - enum engine_id eng_id); + enum engine_id eng_id, + const struct dce110_stream_enc_registers *regs, + const struct dce_stream_encoder_shift *se_shift, + const struct dce_stream_encoder_mask *se_mask); void dce110_se_audio_mute_control( struct stream_encoder *enc, bool mute); diff --git a/drivers/gpu/drm/amd/display/dc/dio/dcn32/dcn32_dio_link_encoder.c b/drivers/gpu/drm/amd/display/dc/dio/dcn32/dcn32_dio_link_encoder.c index 06907e8a4eda12..ddc736af776c95 100644 --- a/drivers/gpu/drm/amd/display/dc/dio/dcn32/dcn32_dio_link_encoder.c +++ b/drivers/gpu/drm/amd/display/dc/dio/dcn32/dcn32_dio_link_encoder.c @@ -188,9 +188,18 @@ void dcn32_link_encoder_get_max_link_cap(struct link_encoder *enc, if (!query_dp_alt_from_dmub(enc, &cmd)) return; - if (cmd.query_dp_alt.data.is_usb && - cmd.query_dp_alt.data.is_dp4 == 0) - link_settings->lane_count = MIN(LANE_COUNT_TWO, link_settings->lane_count); + /* + * USB-C DisplayPort Alt Mode lane count limitation logic: + * When USB and DP share the same USB-C connector, hardware must allocate + * some lanes for USB data, limiting DP to maximum 2 lanes instead of 4. + * This ensures USB functionality remains available while DP is active. + */ + if (cmd.query_dp_alt.data.is_dp_alt_disable == 0 && + cmd.query_dp_alt.data.is_usb && + cmd.query_dp_alt.data.is_dp4 == 0) { + link_settings->lane_count = + MIN(LANE_COUNT_TWO, link_settings->lane_count); + } } diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn35/dcn35_fpu.c b/drivers/gpu/drm/amd/display/dc/dml/dcn35/dcn35_fpu.c index 817a370e80a770..8a177d5ae213e7 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dcn35/dcn35_fpu.c +++ b/drivers/gpu/drm/amd/display/dc/dml/dcn35/dcn35_fpu.c @@ -164,8 +164,8 @@ struct _vcs_dpi_soc_bounding_box_st dcn3_5_soc = { }, }, .num_states = 5, - .sr_exit_time_us = 28.0, - .sr_enter_plus_exit_time_us = 30.0, + .sr_exit_time_us = 31.0, + .sr_enter_plus_exit_time_us = 33.0, .sr_exit_z8_time_us = 250.0, .sr_enter_plus_exit_z8_time_us = 350.0, .fclk_change_latency_us = 24.0, diff --git a/drivers/gpu/drm/amd/display/dc/dpp/dcn30/dcn30_dpp.c b/drivers/gpu/drm/amd/display/dc/dpp/dcn30/dcn30_dpp.c index ef4a161171814f..c7923531da83da 100644 --- a/drivers/gpu/drm/amd/display/dc/dpp/dcn30/dcn30_dpp.c +++ b/drivers/gpu/drm/amd/display/dc/dpp/dcn30/dcn30_dpp.c @@ -376,10 +376,10 @@ void dpp3_cnv_setup ( tbl_entry.color_space = input_color_space; - if (color_space >= COLOR_SPACE_YCBCR601) - select = INPUT_CSC_SELECT_ICSC; - else + if (dpp3_should_bypass_post_csc_for_colorspace(color_space)) select = INPUT_CSC_SELECT_BYPASS; + else + select = INPUT_CSC_SELECT_ICSC; dpp3_program_post_csc(dpp_base, color_space, select, &tbl_entry); @@ -1541,3 +1541,18 @@ bool dpp3_construct( return true; } +bool dpp3_should_bypass_post_csc_for_colorspace(enum dc_color_space dc_color_space) +{ + switch (dc_color_space) { + case COLOR_SPACE_UNKNOWN: + case COLOR_SPACE_SRGB: + case COLOR_SPACE_XR_RGB: + case COLOR_SPACE_SRGB_LIMITED: + case COLOR_SPACE_MSREF_SCRGB: + case COLOR_SPACE_2020_RGB_FULLRANGE: + case COLOR_SPACE_2020_RGB_LIMITEDRANGE: + return true; + default: + return false; + } +} diff --git a/drivers/gpu/drm/amd/display/dc/dpp/dcn30/dcn30_dpp.h b/drivers/gpu/drm/amd/display/dc/dpp/dcn30/dcn30_dpp.h index d4a70b4379eafe..6a61b99d6a7986 100644 --- a/drivers/gpu/drm/amd/display/dc/dpp/dcn30/dcn30_dpp.h +++ b/drivers/gpu/drm/amd/display/dc/dpp/dcn30/dcn30_dpp.h @@ -644,4 +644,8 @@ void dpp3_program_cm_dealpha( void dpp3_cm_get_gamut_remap(struct dpp *dpp_base, struct dpp_grph_csc_adjustment *adjust); + +bool dpp3_should_bypass_post_csc_for_colorspace( + enum dc_color_space dc_color_space); + #endif /* __DC_HWSS_DCN30_H__ */ diff --git a/drivers/gpu/drm/amd/display/dc/dpp/dcn401/dcn401_dpp.c b/drivers/gpu/drm/amd/display/dc/dpp/dcn401/dcn401_dpp.c index 96c2c853de42c3..2d6a646462e217 100644 --- a/drivers/gpu/drm/amd/display/dc/dpp/dcn401/dcn401_dpp.c +++ b/drivers/gpu/drm/amd/display/dc/dpp/dcn401/dcn401_dpp.c @@ -206,10 +206,10 @@ void dpp401_dpp_setup( tbl_entry.color_space = input_color_space; - if (color_space >= COLOR_SPACE_YCBCR601) - select = INPUT_CSC_SELECT_ICSC; - else + if (dpp3_should_bypass_post_csc_for_colorspace(color_space)) select = INPUT_CSC_SELECT_BYPASS; + else + select = INPUT_CSC_SELECT_ICSC; dpp3_program_post_csc(dpp_base, color_space, select, &tbl_entry); diff --git a/drivers/gpu/drm/amd/display/dc/hubp/dcn401/dcn401_hubp.c b/drivers/gpu/drm/amd/display/dc/hubp/dcn401/dcn401_hubp.c index f01eae50d02f78..c205500290ecd7 100644 --- a/drivers/gpu/drm/amd/display/dc/hubp/dcn401/dcn401_hubp.c +++ b/drivers/gpu/drm/amd/display/dc/hubp/dcn401/dcn401_hubp.c @@ -733,10 +733,8 @@ void hubp401_cursor_set_position( const struct dc_cursor_mi_param *param) { struct dcn20_hubp *hubp2 = TO_DCN20_HUBP(hubp); - int x_pos = pos->x - param->recout.x; - int y_pos = pos->y - param->recout.y; - int rec_x_offset = x_pos - pos->x_hotspot; - int rec_y_offset = y_pos - pos->y_hotspot; + int rec_x_offset = pos->x - pos->x_hotspot; + int rec_y_offset = pos->y - pos->y_hotspot; int dst_x_offset; int x_pos_viewport = 0; int x_hot_viewport = 0; @@ -748,10 +746,10 @@ void hubp401_cursor_set_position( * within preceeding ODM slices. */ if (param->recout.width) { - x_pos_viewport = x_pos * param->viewport.width / param->recout.width; + x_pos_viewport = pos->x * param->viewport.width / param->recout.width; x_hot_viewport = pos->x_hotspot * param->viewport.width / param->recout.width; } else { - ASSERT(!cur_en || x_pos == 0); + ASSERT(!cur_en || pos->x == 0); ASSERT(!cur_en || pos->x_hotspot == 0); } @@ -790,8 +788,8 @@ void hubp401_cursor_set_position( if (!hubp->cursor_offload) { REG_SET_2(CURSOR_POSITION, 0, - CURSOR_X_POSITION, x_pos, - CURSOR_Y_POSITION, y_pos); + CURSOR_X_POSITION, pos->x, + CURSOR_Y_POSITION, pos->y); REG_SET_2(CURSOR_HOT_SPOT, 0, CURSOR_HOT_SPOT_X, pos->x_hotspot, diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dce110/dce110_hwseq.c b/drivers/gpu/drm/amd/display/dc/hwss/dce110/dce110_hwseq.c index ebd74b43e935ee..8b8410ef3e47ab 100644 --- a/drivers/gpu/drm/amd/display/dc/hwss/dce110/dce110_hwseq.c +++ b/drivers/gpu/drm/amd/display/dc/hwss/dce110/dce110_hwseq.c @@ -59,6 +59,7 @@ #include "dc_state_priv.h" #include "dpcd_defs.h" #include "dsc.h" +#include "dc_dp_types.h" /* include DCE11 register header files */ #include "dce/dce_11_0_d.h" #include "dce/dce_11_0_sh_mask.h" @@ -659,20 +660,6 @@ void dce110_update_info_frame(struct pipe_ctx *pipe_ctx) } } -static void -dce110_dac_encoder_control(struct pipe_ctx *pipe_ctx, bool enable) -{ - struct dc_link *link = pipe_ctx->stream->link; - struct dc_bios *bios = link->ctx->dc_bios; - struct bp_encoder_control encoder_control = {0}; - - encoder_control.action = enable ? ENCODER_CONTROL_ENABLE : ENCODER_CONTROL_DISABLE; - encoder_control.engine_id = link->link_enc->analog_engine; - encoder_control.pixel_clock = pipe_ctx->stream->timing.pix_clk_100hz / 10; - - bios->funcs->encoder_control(bios, &encoder_control); -} - void dce110_enable_stream(struct pipe_ctx *pipe_ctx) { enum dc_lane_count lane_count = @@ -703,8 +690,6 @@ void dce110_enable_stream(struct pipe_ctx *pipe_ctx) tg->funcs->set_early_control(tg, early_control); - if (dc_is_rgb_signal(pipe_ctx->stream->signal)) - dce110_dac_encoder_control(pipe_ctx, true); } static enum bp_result link_transmitter_control( @@ -1218,9 +1203,6 @@ void dce110_disable_stream(struct pipe_ctx *pipe_ctx) dccg->funcs->disable_symclk_se(dccg, stream_enc->stream_enc_inst, link_enc->transmitter - TRANSMITTER_UNIPHY_A); } - - if (dc_is_rgb_signal(pipe_ctx->stream->signal)) - dce110_dac_encoder_control(pipe_ctx, false); } void dce110_unblank_stream(struct pipe_ctx *pipe_ctx, @@ -1603,25 +1585,6 @@ static enum dc_status dce110_enable_stream_timing( return DC_OK; } -static void -dce110_select_crtc_source(struct pipe_ctx *pipe_ctx) -{ - struct dc_link *link = pipe_ctx->stream->link; - struct dc_bios *bios = link->ctx->dc_bios; - struct bp_crtc_source_select crtc_source_select = {0}; - enum engine_id engine_id = link->link_enc->preferred_engine; - - if (dc_is_rgb_signal(pipe_ctx->stream->signal)) - engine_id = link->link_enc->analog_engine; - - crtc_source_select.controller_id = CONTROLLER_ID_D0 + pipe_ctx->stream_res.tg->inst; - crtc_source_select.color_depth = pipe_ctx->stream->timing.display_color_depth; - crtc_source_select.engine_id = engine_id; - crtc_source_select.sink_signal = pipe_ctx->stream->signal; - - bios->funcs->select_crtc_source(bios, &crtc_source_select); -} - enum dc_status dce110_apply_single_controller_ctx_to_hw( struct pipe_ctx *pipe_ctx, struct dc_state *context, @@ -1641,10 +1604,6 @@ enum dc_status dce110_apply_single_controller_ctx_to_hw( hws->funcs.disable_stream_gating(dc, pipe_ctx); } - if (pipe_ctx->stream->signal == SIGNAL_TYPE_RGB) { - dce110_select_crtc_source(pipe_ctx); - } - if (pipe_ctx->stream_res.audio != NULL) { struct audio_output audio_output = {0}; @@ -1724,8 +1683,7 @@ enum dc_status dce110_apply_single_controller_ctx_to_hw( pipe_ctx->stream_res.tg->funcs->set_static_screen_control( pipe_ctx->stream_res.tg, event_triggers, 2); - if (!dc_is_virtual_signal(pipe_ctx->stream->signal) && - !dc_is_rgb_signal(pipe_ctx->stream->signal)) + if (!dc_is_virtual_signal(pipe_ctx->stream->signal)) pipe_ctx->stream_res.stream_enc->funcs->dig_connect_to_otg( pipe_ctx->stream_res.stream_enc, pipe_ctx->stream_res.tg->inst); @@ -1779,20 +1737,25 @@ static void power_down_encoders(struct dc *dc) int i; for (i = 0; i < dc->link_count; i++) { - enum signal_type signal = dc->links[i]->connector_signal; - - dc->link_srv->blank_dp_stream(dc->links[i], false); + struct dc_link *link = dc->links[i]; + struct link_encoder *link_enc = link->link_enc; + enum signal_type signal = link->connector_signal; + dc->link_srv->blank_dp_stream(link, false); if (signal != SIGNAL_TYPE_EDP) signal = SIGNAL_TYPE_NONE; - if (dc->links[i]->ep_type == DISPLAY_ENDPOINT_PHY) - dc->links[i]->link_enc->funcs->disable_output( - dc->links[i]->link_enc, signal); + if (link->ep_type == DISPLAY_ENDPOINT_PHY) + link_enc->funcs->disable_output(link_enc, signal); + + if (link->fec_state == dc_link_fec_enabled) { + link_enc->funcs->fec_set_enable(link_enc, false); + link_enc->funcs->fec_set_ready(link_enc, false); + link->fec_state = dc_link_fec_not_ready; + } - dc->links[i]->link_status.link_active = false; - memset(&dc->links[i]->cur_link_settings, 0, - sizeof(dc->links[i]->cur_link_settings)); + link->link_status.link_active = false; + memset(&link->cur_link_settings, 0, sizeof(link->cur_link_settings)); } } @@ -1840,6 +1803,9 @@ static void disable_vga_and_power_gate_all_controllers( struct timing_generator *tg; struct dc_context *ctx = dc->ctx; + if (dc->caps.ips_support) + return; + for (i = 0; i < dc->res_pool->timing_generator_count; i++) { tg = dc->res_pool->timing_generators[i]; @@ -1916,13 +1882,16 @@ static void clean_up_dsc_blocks(struct dc *dc) /* disable DSC in OPTC */ if (i < dc->res_pool->timing_generator_count) { tg = dc->res_pool->timing_generators[i]; - tg->funcs->set_dsc_config(tg, OPTC_DSC_DISABLED, 0, 0); + if (tg->funcs->set_dsc_config) + tg->funcs->set_dsc_config(tg, OPTC_DSC_DISABLED, 0, 0); } /* disable DSC in stream encoder */ if (i < dc->res_pool->stream_enc_count) { se = dc->res_pool->stream_enc[i]; - se->funcs->dp_set_dsc_config(se, OPTC_DSC_DISABLED, 0, 0); - se->funcs->dp_set_dsc_pps_info_packet(se, false, NULL, true); + if (se->funcs->dp_set_dsc_config) + se->funcs->dp_set_dsc_config(se, OPTC_DSC_DISABLED, 0, 0); + if (se->funcs->dp_set_dsc_pps_info_packet) + se->funcs->dp_set_dsc_pps_info_packet(se, false, NULL, true); } /* disable DSC block */ if (dccg->funcs->set_ref_dscclk) @@ -1972,8 +1941,8 @@ void dce110_enable_accelerated_mode(struct dc *dc, struct dc_state *context) get_edp_streams(context, edp_streams, &edp_stream_num); - /* Check fastboot support, disable on DCE 6-8 because of blank screens */ - if (edp_num && edp_stream_num && dc->ctx->dce_version < DCE_VERSION_10_0) { + /* Check fastboot support, disable on DCE 6-8-10 because of blank screens */ + if (edp_num && edp_stream_num && dc->ctx->dce_version > DCE_VERSION_10_0) { for (i = 0; i < edp_num; i++) { edp_link = edp_links[i]; if (edp_link != edp_streams[0]->link) @@ -3312,6 +3281,15 @@ void dce110_enable_tmds_link_output(struct dc_link *link, link->phy_state.symclk_state = SYMCLK_ON_TX_ON; } +static void dce110_enable_analog_link_output( + struct dc_link *link, + uint32_t pix_clk_100hz) +{ + link->link_enc->funcs->enable_analog_output( + link->link_enc, + pix_clk_100hz); +} + void dce110_enable_dp_link_output( struct dc_link *link, const struct link_resource *link_res, @@ -3449,6 +3427,7 @@ static const struct hw_sequencer_funcs dce110_funcs = { .enable_lvds_link_output = dce110_enable_lvds_link_output, .enable_tmds_link_output = dce110_enable_tmds_link_output, .enable_dp_link_output = dce110_enable_dp_link_output, + .enable_analog_link_output = dce110_enable_analog_link_output, .disable_link_output = dce110_disable_link_output, }; diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn20/dcn20_hwseq.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn20/dcn20_hwseq.c index c8ff8ae85a0306..517d4c08d34c44 100644 --- a/drivers/gpu/drm/amd/display/dc/hwss/dcn20/dcn20_hwseq.c +++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn20/dcn20_hwseq.c @@ -3058,9 +3058,17 @@ void dcn20_enable_stream(struct pipe_ctx *pipe_ctx) dccg->funcs->enable_symclk32_se(dccg, dp_hpo_inst, phyd32clk); } } else { - if (dccg->funcs->enable_symclk_se) - dccg->funcs->enable_symclk_se(dccg, stream_enc->stream_enc_inst, + if (dccg->funcs->enable_symclk_se && link_enc) { + if (link->ep_type == DISPLAY_ENDPOINT_USB4_DPIA + && link->cur_link_settings.link_rate == LINK_RATE_UNKNOWN + && !link->link_status.link_active) { + if (dccg->funcs->disable_symclk_se) + dccg->funcs->disable_symclk_se(dccg, stream_enc->stream_enc_inst, link_enc->transmitter - TRANSMITTER_UNIPHY_A); + } else + dccg->funcs->enable_symclk_se(dccg, stream_enc->stream_enc_inst, + link_enc->transmitter - TRANSMITTER_UNIPHY_A); + } } if (dc->res_pool->dccg->funcs->set_pixel_rate_div) diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn31/dcn31_hwseq.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn31/dcn31_hwseq.c index d1ecdb92b072b1..20f700b59847cf 100644 --- a/drivers/gpu/drm/amd/display/dc/hwss/dcn31/dcn31_hwseq.c +++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn31/dcn31_hwseq.c @@ -546,8 +546,22 @@ static void dcn31_reset_back_end_for_pipe( if (pipe_ctx->stream_res.tg->funcs->set_odm_bypass) pipe_ctx->stream_res.tg->funcs->set_odm_bypass( pipe_ctx->stream_res.tg, &pipe_ctx->stream->timing); + /* + * TODO - convert symclk_ref_cnts for otg to a bit map to solve + * the case where the same symclk is shared across multiple otg + * instances + */ if (dc_is_hdmi_tmds_signal(pipe_ctx->stream->signal)) - pipe_ctx->stream->link->phy_state.symclk_ref_cnts.otg = 0; + link->phy_state.symclk_ref_cnts.otg = 0; + + if (pipe_ctx->top_pipe == NULL) { + if (link->phy_state.symclk_state == SYMCLK_ON_TX_OFF) { + const struct link_hwss *link_hwss = get_link_hwss(link, &pipe_ctx->link_res); + + link_hwss->disable_link_output(link, &pipe_ctx->link_res, pipe_ctx->stream->signal); + link->phy_state.symclk_state = SYMCLK_OFF_TX_OFF; + } + } set_drr_and_clear_adjust_pending(pipe_ctx, pipe_ctx->stream, NULL); diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn314/dcn314_hwseq.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn314/dcn314_hwseq.c index 4ee6ed610de0b9..3e239124c17d8b 100644 --- a/drivers/gpu/drm/amd/display/dc/hwss/dcn314/dcn314_hwseq.c +++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn314/dcn314_hwseq.c @@ -108,7 +108,7 @@ static void update_dsc_on_stream(struct pipe_ctx *pipe_ctx, bool enable) dsc_cfg.dc_dsc_cfg = stream->timing.dsc_cfg; ASSERT(dsc_cfg.dc_dsc_cfg.num_slices_h % opp_cnt == 0); dsc_cfg.dc_dsc_cfg.num_slices_h /= opp_cnt; - dsc_cfg.dsc_padding = pipe_ctx->dsc_padding_params.dsc_hactive_padding; + dsc_cfg.dsc_padding = 0; dsc->funcs->dsc_set_config(dsc, &dsc_cfg, &dsc_optc_cfg); dsc->funcs->dsc_enable(dsc, pipe_ctx->stream_res.opp->inst); diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn32/dcn32_hwseq.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn32/dcn32_hwseq.c index be1f3caf4096fe..24af5e94c7fce2 100644 --- a/drivers/gpu/drm/amd/display/dc/hwss/dcn32/dcn32_hwseq.c +++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn32/dcn32_hwseq.c @@ -1063,7 +1063,7 @@ void dcn32_update_dsc_on_stream(struct pipe_ctx *pipe_ctx, bool enable) dsc_cfg.dc_dsc_cfg = stream->timing.dsc_cfg; ASSERT(dsc_cfg.dc_dsc_cfg.num_slices_h % opp_cnt == 0); dsc_cfg.dc_dsc_cfg.num_slices_h /= opp_cnt; - dsc_cfg.dsc_padding = pipe_ctx->dsc_padding_params.dsc_hactive_padding; + dsc_cfg.dsc_padding = 0; if (should_use_dto_dscclk) dccg->funcs->set_dto_dscclk(dccg, dsc->inst, dsc_cfg.dc_dsc_cfg.num_slices_h); diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn35/dcn35_hwseq.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn35/dcn35_hwseq.c index 7aa0f452e8f7a3..88542ca7155730 100644 --- a/drivers/gpu/drm/amd/display/dc/hwss/dcn35/dcn35_hwseq.c +++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn35/dcn35_hwseq.c @@ -364,7 +364,7 @@ static void update_dsc_on_stream(struct pipe_ctx *pipe_ctx, bool enable) dsc_cfg.dc_dsc_cfg = stream->timing.dsc_cfg; ASSERT(dsc_cfg.dc_dsc_cfg.num_slices_h % opp_cnt == 0); dsc_cfg.dc_dsc_cfg.num_slices_h /= opp_cnt; - dsc_cfg.dsc_padding = pipe_ctx->dsc_padding_params.dsc_hactive_padding; + dsc_cfg.dsc_padding = 0; dsc->funcs->dsc_set_config(dsc, &dsc_cfg, &dsc_optc_cfg); dsc->funcs->dsc_enable(dsc, pipe_ctx->stream_res.opp->inst); @@ -1726,3 +1726,55 @@ void dcn35_program_cursor_offload_now(struct dc *dc, const struct pipe_ctx *pipe { dc_dmub_srv_program_cursor_now(dc, pipe); } + +static void disable_link_output_symclk_on_tx_off(struct dc_link *link, enum dp_link_encoding link_encoding) +{ + struct dc *dc = link->ctx->dc; + struct pipe_ctx *pipe_ctx = NULL; + uint8_t i; + + for (i = 0; i < MAX_PIPES; i++) { + pipe_ctx = &dc->current_state->res_ctx.pipe_ctx[i]; + if (pipe_ctx->stream && pipe_ctx->stream->link == link && pipe_ctx->top_pipe == NULL) { + pipe_ctx->clock_source->funcs->program_pix_clk( + pipe_ctx->clock_source, + &pipe_ctx->stream_res.pix_clk_params, + link_encoding, + &pipe_ctx->pll_settings); + break; + } + } +} + +void dcn35_disable_link_output(struct dc_link *link, + const struct link_resource *link_res, + enum signal_type signal) +{ + struct dc *dc = link->ctx->dc; + const struct link_hwss *link_hwss = get_link_hwss(link, link_res); + struct dmcu *dmcu = dc->res_pool->dmcu; + + if (signal == SIGNAL_TYPE_EDP && + link->dc->hwss.edp_backlight_control && + !link->skip_implict_edp_power_control) + link->dc->hwss.edp_backlight_control(link, false); + else if (dmcu != NULL && dmcu->funcs->lock_phy) + dmcu->funcs->lock_phy(dmcu); + + if (dc_is_tmds_signal(signal) && link->phy_state.symclk_ref_cnts.otg > 0) { + disable_link_output_symclk_on_tx_off(link, DP_UNKNOWN_ENCODING); + link->phy_state.symclk_state = SYMCLK_ON_TX_OFF; + } else { + link_hwss->disable_link_output(link, link_res, signal); + link->phy_state.symclk_state = SYMCLK_OFF_TX_OFF; + } + /* + * Add the logic to extract BOTH power up and power down sequences + * from enable/disable link output and only call edp panel control + * in enable_link_dp and disable_link_dp once. + */ + if (dmcu != NULL && dmcu->funcs->unlock_phy) + dmcu->funcs->unlock_phy(dmcu); + + dc->link_srv->dp_trace_source_sequence(link, DPCD_SOURCE_SEQ_AFTER_DISABLE_LINK_PHY); +} diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn35/dcn35_hwseq.h b/drivers/gpu/drm/amd/display/dc/hwss/dcn35/dcn35_hwseq.h index 1ff41dba556c03..e3459546a908a1 100644 --- a/drivers/gpu/drm/amd/display/dc/hwss/dcn35/dcn35_hwseq.h +++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn35/dcn35_hwseq.h @@ -108,5 +108,8 @@ void dcn35_update_cursor_offload_pipe(struct dc *dc, const struct pipe_ctx *pipe void dcn35_notify_cursor_offload_drr_update(struct dc *dc, struct dc_state *context, const struct dc_stream_state *stream); void dcn35_program_cursor_offload_now(struct dc *dc, const struct pipe_ctx *pipe); +void dcn35_disable_link_output(struct dc_link *link, + const struct link_resource *link_res, + enum signal_type signal); #endif /* __DC_HWSS_DCN35_H__ */ diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn35/dcn35_init.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn35/dcn35_init.c index 5a66c9db267094..81bd36f3381dbd 100644 --- a/drivers/gpu/drm/amd/display/dc/hwss/dcn35/dcn35_init.c +++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn35/dcn35_init.c @@ -113,7 +113,7 @@ static const struct hw_sequencer_funcs dcn35_funcs = { .enable_lvds_link_output = dce110_enable_lvds_link_output, .enable_tmds_link_output = dce110_enable_tmds_link_output, .enable_dp_link_output = dce110_enable_dp_link_output, - .disable_link_output = dcn32_disable_link_output, + .disable_link_output = dcn35_disable_link_output, .z10_restore = dcn35_z10_restore, .z10_save_init = dcn31_z10_save_init, .set_disp_pattern_generator = dcn30_set_disp_pattern_generator, diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn401/dcn401_hwseq.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn401/dcn401_hwseq.c index 5eda7648d0d2ba..d0ed620a26a910 100644 --- a/drivers/gpu/drm/amd/display/dc/hwss/dcn401/dcn401_hwseq.c +++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn401/dcn401_hwseq.c @@ -71,7 +71,11 @@ void dcn401_initialize_min_clocks(struct dc *dc) * audio corruption. Read current DISPCLK from DENTIST and request the same * freq to ensure that the timing is valid and unchanged. */ - clocks->dispclk_khz = dc->clk_mgr->funcs->get_dispclk_from_dentist(dc->clk_mgr); + if (dc->clk_mgr->funcs->get_dispclk_from_dentist) { + clocks->dispclk_khz = dc->clk_mgr->funcs->get_dispclk_from_dentist(dc->clk_mgr); + } else { + clocks->dispclk_khz = dc->clk_mgr->boot_snapshot.dispclk * 1000; + } } clocks->ref_dtbclk_khz = dc->clk_mgr->bw_params->clk_table.entries[0].dtbclk_mhz * 1000; clocks->fclk_p_state_change_support = true; @@ -142,6 +146,7 @@ void dcn401_init_hw(struct dc *dc) int edp_num; uint32_t backlight = MAX_BACKLIGHT_LEVEL; uint32_t user_level = MAX_BACKLIGHT_LEVEL; + bool dchub_ref_freq_changed; int current_dchub_ref_freq = 0; if (dc->clk_mgr && dc->clk_mgr->funcs && dc->clk_mgr->funcs->init_clocks) { @@ -287,6 +292,8 @@ void dcn401_init_hw(struct dc *dc) for (i = 0; i < dc->link_count; i++) { struct dc_link *link = dc->links[i]; + if (link->ep_type != DISPLAY_ENDPOINT_PHY) + continue; if (link->link_enc->funcs->is_dig_enabled && link->link_enc->funcs->is_dig_enabled(link->link_enc) && hws->funcs.power_down) { @@ -297,7 +304,6 @@ void dcn401_init_hw(struct dc *dc) } } } - for (i = 0; i < res_pool->audio_count; i++) { struct audio *audio = res_pool->audios[i]; @@ -354,14 +360,18 @@ void dcn401_init_hw(struct dc *dc) dc->caps.dmub_caps.psr = dc->ctx->dmub_srv->dmub->feature_caps.psr; dc->caps.dmub_caps.mclk_sw = dc->ctx->dmub_srv->dmub->feature_caps.fw_assisted_mclk_switch_ver > 0; dc->caps.dmub_caps.fams_ver = dc->ctx->dmub_srv->dmub->feature_caps.fw_assisted_mclk_switch_ver; + + /* sw and fw FAMS versions must match for support */ dc->debug.fams2_config.bits.enable &= - dc->caps.dmub_caps.fams_ver == dc->debug.fams_version.ver; // sw & fw fams versions must match for support - if ((!dc->debug.fams2_config.bits.enable && dc->res_pool->funcs->update_bw_bounding_box) - || res_pool->ref_clocks.dchub_ref_clock_inKhz / 1000 != current_dchub_ref_freq) { + dc->caps.dmub_caps.fams_ver == dc->debug.fams_version.ver; + dchub_ref_freq_changed = + res_pool->ref_clocks.dchub_ref_clock_inKhz / 1000 != current_dchub_ref_freq; + if ((!dc->debug.fams2_config.bits.enable || dchub_ref_freq_changed) && + dc->res_pool->funcs->update_bw_bounding_box && + dc->clk_mgr && dc->clk_mgr->bw_params) { /* update bounding box if FAMS2 disabled, or if dchub clk has changed */ - if (dc->clk_mgr) - dc->res_pool->funcs->update_bw_bounding_box(dc, - dc->clk_mgr->bw_params); + dc->res_pool->funcs->update_bw_bounding_box(dc, + dc->clk_mgr->bw_params); } } } @@ -917,10 +927,10 @@ static void dcn401_enable_stream_calc( pipe_ctx->stream->link->cur_link_settings.lane_count; uint32_t active_total_with_borders; - if (dc->link_srv->dp_is_128b_132b_signal(pipe_ctx)) + if (dc->link_srv->dp_is_128b_132b_signal(pipe_ctx)) { *dp_hpo_inst = pipe_ctx->stream_res.hpo_dp_stream_enc->inst; - - *phyd32clk = get_phyd32clk_src(pipe_ctx->stream->link); + *phyd32clk = get_phyd32clk_src(pipe_ctx->stream->link); + } if (dc_is_tmds_signal(pipe_ctx->stream->signal)) dcn401_calculate_dccg_tmds_div_value(pipe_ctx, tmds_div); @@ -1215,6 +1225,9 @@ void dcn401_set_cursor_position(struct pipe_ctx *pipe_ctx) if (recout_y_pos + (int)hubp->curs_attr.height <= 0) pos_cpy.enable = false; /* not visible beyond top edge*/ + pos_cpy.x = x_pos; + pos_cpy.y = y_pos; + hubp->funcs->set_cursor_position(hubp, &pos_cpy, ¶m); dpp->funcs->set_cursor_position(dpp, &pos_cpy, ¶m, hubp->curs_attr.width, hubp->curs_attr.height); } @@ -1771,7 +1784,8 @@ void dcn401_unblank_stream(struct pipe_ctx *pipe_ctx, void dcn401_hardware_release(struct dc *dc) { if (!dc->debug.disable_force_pstate_allow_on_hw_release) { - dc_dmub_srv_fams2_update_config(dc, dc->current_state, false); + if (dc->ctx->dmub_srv && dc->debug.fams2_config.bits.enable) + dc_dmub_srv_fams2_update_config(dc, dc->current_state, false); /* If pstate unsupported, or still supported * by firmware, force it supported by dcn @@ -1791,7 +1805,9 @@ void dcn401_hardware_release(struct dc *dc) dc->clk_mgr->clks.p_state_change_support = false; dc->clk_mgr->funcs->update_clocks(dc->clk_mgr, dc->current_state, true); } - dc_dmub_srv_fams2_update_config(dc, dc->current_state, false); + + if (dc->ctx->dmub_srv && dc->debug.fams2_config.bits.enable) + dc_dmub_srv_fams2_update_config(dc, dc->current_state, false); } } diff --git a/drivers/gpu/drm/amd/display/dc/hwss/hw_sequencer.h b/drivers/gpu/drm/amd/display/dc/hwss/hw_sequencer.h index 8ed9eea40c5646..4f6bd365e055a6 100644 --- a/drivers/gpu/drm/amd/display/dc/hwss/hw_sequencer.h +++ b/drivers/gpu/drm/amd/display/dc/hwss/hw_sequencer.h @@ -1184,6 +1184,8 @@ struct hw_sequencer_funcs { const struct link_resource *link_res, enum clock_source_id clock_source, uint32_t pixel_clock); + void (*enable_analog_link_output)(struct dc_link *link, + uint32_t pixel_clock); void (*disable_link_output)(struct dc_link *link, const struct link_resource *link_res, enum signal_type signal); diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/link_encoder.h b/drivers/gpu/drm/amd/display/dc/inc/hw/link_encoder.h index df512920a9fabe..b1a88618c5bf85 100644 --- a/drivers/gpu/drm/amd/display/dc/inc/hw/link_encoder.h +++ b/drivers/gpu/drm/amd/display/dc/inc/hw/link_encoder.h @@ -47,6 +47,7 @@ struct encoder_init_data { enum hpd_source_id hpd_source; /* TODO: in DAL2, here was pointer to EventManagerInterface */ struct graphics_object_id encoder; + struct graphics_object_id analog_encoder; enum engine_id analog_engine; struct dc_context *ctx; enum transmitter transmitter; @@ -81,6 +82,7 @@ struct link_encoder { int32_t aux_channel_offset; struct dc_context *ctx; struct graphics_object_id id; + struct graphics_object_id analog_id; struct graphics_object_id connector; uint32_t output_signals; enum engine_id preferred_engine; @@ -128,6 +130,8 @@ struct link_encoder_funcs { void (*enable_lvds_output)(struct link_encoder *enc, enum clock_source_id clock_source, uint32_t pixel_clock); + void (*enable_analog_output)(struct link_encoder *enc, + uint32_t pixel_clock); void (*disable_output)(struct link_encoder *link_enc, enum signal_type signal); void (*dp_set_lane_settings)(struct link_encoder *enc, diff --git a/drivers/gpu/drm/amd/display/dc/link/link_detection.c b/drivers/gpu/drm/amd/display/dc/link/link_detection.c index 7fa6bc97a91935..f986b57381bab8 100644 --- a/drivers/gpu/drm/amd/display/dc/link/link_detection.c +++ b/drivers/gpu/drm/amd/display/dc/link/link_detection.c @@ -1173,11 +1173,10 @@ static bool detect_link_and_local_sink(struct dc_link *link, * - cheap DVI-A cable or adapter that doesn't connect DDC */ if (dc_connector_supports_analog(link->link_id.id)) { - /* If we didn't do DAC load detection yet, do it now - * to verify there really is a display connected. + /* If we didn't already detect a display using + * DAC load detection, we know it isn't connected. */ - if (link->type != dc_connection_dac_load && - !link_detect_dac_load_detect(link)) { + if (link->type != dc_connection_analog_load) { if (prev_sink) dc_sink_release(prev_sink); link_disconnect_sink(link); @@ -1185,7 +1184,7 @@ static bool detect_link_and_local_sink(struct dc_link *link, } DC_LOG_INFO("%s detected analog display without EDID\n", __func__); - link->type = dc_connection_dac_load; + link->type = dc_connection_analog_load; sink->edid_caps.analog = true; break; } @@ -1372,7 +1371,7 @@ static bool detect_link_and_local_sink(struct dc_link *link, * @link: DC link to evaluate (must support analog signalling). * @type: Updated with the detected connection type: * dc_connection_single (analog via DDC), - * dc_connection_dac_load (via load-detect), + * dc_connection_analog_load (via load-detect), * or dc_connection_none. * * Return: true if detection completed. @@ -1388,7 +1387,7 @@ static bool link_detect_analog(struct dc_link *link, enum dc_connection_type *ty } if (link_detect_dac_load_detect(link)) { - *type = dc_connection_dac_load; + *type = dc_connection_analog_load; return true; } diff --git a/drivers/gpu/drm/amd/display/dc/link/link_dpms.c b/drivers/gpu/drm/amd/display/dc/link/link_dpms.c index 6ae13414761716..a36762915943b3 100644 --- a/drivers/gpu/drm/amd/display/dc/link/link_dpms.c +++ b/drivers/gpu/drm/amd/display/dc/link/link_dpms.c @@ -841,7 +841,7 @@ void link_set_dsc_on_stream(struct pipe_ctx *pipe_ctx, bool enable) dsc_cfg.dc_dsc_cfg = stream->timing.dsc_cfg; ASSERT(dsc_cfg.dc_dsc_cfg.num_slices_h % opp_cnt == 0); dsc_cfg.dc_dsc_cfg.num_slices_h /= opp_cnt; - dsc_cfg.dsc_padding = pipe_ctx->dsc_padding_params.dsc_hactive_padding; + dsc_cfg.dsc_padding = 0; if (should_use_dto_dscclk) dccg->funcs->set_dto_dscclk(dccg, dsc->inst, dsc_cfg.dc_dsc_cfg.num_slices_h); @@ -857,6 +857,7 @@ void link_set_dsc_on_stream(struct pipe_ctx *pipe_ctx, bool enable) } dsc_cfg.dc_dsc_cfg.num_slices_h *= opp_cnt; dsc_cfg.pic_width *= opp_cnt; + dsc_cfg.dsc_padding = pipe_ctx->dsc_padding_params.dsc_hactive_padding; optc_dsc_mode = dsc_optc_cfg.is_pixel_format_444 ? OPTC_DSC_ENABLED_444 : OPTC_DSC_ENABLED_NATIVE_SUBSAMPLED; @@ -1930,7 +1931,7 @@ static void disable_link_dp(struct dc_link *link, link->dc->hwss.edp_power_control(link, false); } - if (signal == SIGNAL_TYPE_DISPLAY_PORT_MST) + if (signal == SIGNAL_TYPE_DISPLAY_PORT_MST && link->sink_count == 0) /* set the sink to SST mode after disabling the link */ enable_mst_on_sink(link, false); @@ -2081,7 +2082,12 @@ static enum dc_status enable_link_dp(struct dc_state *state, pipe_ctx->stream->signal == SIGNAL_TYPE_DISPLAY_PORT && link->dc->debug.set_mst_en_for_sst) { enable_mst_on_sink(link, true); + } else if (link->dpcd_caps.is_mst_capable && + pipe_ctx->stream->signal == SIGNAL_TYPE_DISPLAY_PORT) { + /* disable mst on sink */ + enable_mst_on_sink(link, false); } + if (pipe_ctx->stream->signal == SIGNAL_TYPE_EDP) { /*in case it is not on*/ if (!link->dc->config.edp_no_power_sequencing) @@ -2208,6 +2214,18 @@ static enum dc_status enable_link_dp_mst( return enable_link_dp(state, pipe_ctx); } +static enum dc_status enable_link_analog( + struct dc_state *state, + struct pipe_ctx *pipe_ctx) +{ + struct dc_link *link = pipe_ctx->stream->link; + + link->dc->hwss.enable_analog_link_output( + link, pipe_ctx->stream->timing.pix_clk_100hz); + + return DC_OK; +} + static enum dc_status enable_link_virtual(struct pipe_ctx *pipe_ctx) { struct dc_link *link = pipe_ctx->stream->link; @@ -2263,7 +2281,7 @@ static enum dc_status enable_link( status = DC_OK; break; case SIGNAL_TYPE_RGB: - status = DC_OK; + status = enable_link_analog(state, pipe_ctx); break; case SIGNAL_TYPE_VIRTUAL: status = enable_link_virtual(pipe_ctx); @@ -2367,9 +2385,9 @@ void link_set_dpms_off(struct pipe_ctx *pipe_ctx) if (pipe_ctx->stream->sink) { if (pipe_ctx->stream->sink->sink_signal != SIGNAL_TYPE_VIRTUAL && pipe_ctx->stream->sink->sink_signal != SIGNAL_TYPE_NONE) { - DC_LOG_DC("%s pipe_ctx dispname=%s signal=%x link=%d\n", __func__, + DC_LOG_DC("%s pipe_ctx dispname=%s signal=%x link=%d sink_count=%d\n", __func__, pipe_ctx->stream->sink->edid_caps.display_name, - pipe_ctx->stream->signal, link->link_index); + pipe_ctx->stream->signal, link->link_index, link->sink_count); } } @@ -2483,10 +2501,11 @@ void link_set_dpms_on( if (pipe_ctx->stream->sink) { if (pipe_ctx->stream->sink->sink_signal != SIGNAL_TYPE_VIRTUAL && pipe_ctx->stream->sink->sink_signal != SIGNAL_TYPE_NONE) { - DC_LOG_DC("%s pipe_ctx dispname=%s signal=%x link=%d\n", __func__, + DC_LOG_DC("%s pipe_ctx dispname=%s signal=%x link=%d sink_count=%d\n", __func__, pipe_ctx->stream->sink->edid_caps.display_name, pipe_ctx->stream->signal, - link->link_index); + link->link_index, + link->sink_count); } } diff --git a/drivers/gpu/drm/amd/display/dc/link/link_factory.c b/drivers/gpu/drm/amd/display/dc/link/link_factory.c index a6e2b0821969b9..9003e0d314e00b 100644 --- a/drivers/gpu/drm/amd/display/dc/link/link_factory.c +++ b/drivers/gpu/drm/amd/display/dc/link/link_factory.c @@ -350,24 +350,6 @@ static enum transmitter translate_encoder_to_transmitter( return TRANSMITTER_UNKNOWN; } break; - case ENCODER_ID_EXTERNAL_NUTMEG: - switch (encoder.enum_id) { - case ENUM_ID_1: - return TRANSMITTER_NUTMEG_CRT; - default: - return TRANSMITTER_UNKNOWN; - } - break; - case ENCODER_ID_EXTERNAL_TRAVIS: - switch (encoder.enum_id) { - case ENUM_ID_1: - return TRANSMITTER_TRAVIS_CRT; - case ENUM_ID_2: - return TRANSMITTER_TRAVIS_LCD; - default: - return TRANSMITTER_UNKNOWN; - } - break; default: return TRANSMITTER_UNKNOWN; } @@ -451,20 +433,19 @@ static enum channel_id get_ddc_line(struct dc_link *link) return channel; } -static enum engine_id find_analog_engine(struct dc_link *link) +static enum engine_id find_analog_engine(struct dc_link *link, struct graphics_object_id *enc) { struct dc_bios *bp = link->ctx->dc_bios; - struct graphics_object_id encoder = {0}; enum bp_result bp_result = BP_RESULT_OK; int i; for (i = 0; i < 3; i++) { - bp_result = bp->funcs->get_src_obj(bp, link->link_id, i, &encoder); + bp_result = bp->funcs->get_src_obj(bp, link->link_id, i, enc); if (bp_result != BP_RESULT_OK) return ENGINE_ID_UNKNOWN; - switch (encoder.id) { + switch (enc->id) { case ENCODER_ID_INTERNAL_DAC1: case ENCODER_ID_INTERNAL_KLDSCP_DAC1: return ENGINE_ID_DACA; @@ -474,17 +455,10 @@ static enum engine_id find_analog_engine(struct dc_link *link) } } + memset(enc, 0, sizeof(*enc)); return ENGINE_ID_UNKNOWN; } -static bool transmitter_supported(const enum transmitter transmitter) -{ - return transmitter != TRANSMITTER_UNKNOWN && - transmitter != TRANSMITTER_NUTMEG_CRT && - transmitter != TRANSMITTER_TRAVIS_CRT && - transmitter != TRANSMITTER_TRAVIS_LCD; -} - static bool analog_engine_supported(const enum engine_id engine_id) { return engine_id == ENGINE_ID_DACA || @@ -502,6 +476,9 @@ static bool construct_phy(struct dc_link *link, struct dc_bios *bios = init_params->dc->ctx->dc_bios; const struct dc_vbios_funcs *bp_funcs = bios->funcs; struct bp_disp_connector_caps_info disp_connect_caps_info = { 0 }; + struct graphics_object_id link_encoder = { 0 }; + enum transmitter transmitter_from_encoder; + enum engine_id link_analog_engine; DC_LOGGER_INIT(dc_ctx->logger); @@ -522,21 +499,21 @@ static bool construct_phy(struct dc_link *link, link->link_id = bios->funcs->get_connector_id(bios, init_params->connector_index); - /* Determine early if the link has any supported encoders, - * so that we avoid initializing DDC and HPD, etc. - */ - bp_funcs->get_src_obj(bios, link->link_id, 0, &enc_init_data.encoder); - enc_init_data.transmitter = translate_encoder_to_transmitter(enc_init_data.encoder); - enc_init_data.analog_engine = find_analog_engine(link); - link->ep_type = DISPLAY_ENDPOINT_PHY; DC_LOG_DC("BIOS object table - link_id: %d", link->link_id.id); - if (!transmitter_supported(enc_init_data.transmitter) && - !analog_engine_supported(enc_init_data.analog_engine)) { + /* Determine early if the link has any supported encoders, + * so that we avoid initializing DDC and HPD, etc. + */ + bp_funcs->get_src_obj(bios, link->link_id, 0, &link_encoder); + transmitter_from_encoder = translate_encoder_to_transmitter(link_encoder); + link_analog_engine = find_analog_engine(link, &enc_init_data.analog_encoder); + + if (transmitter_from_encoder == TRANSMITTER_UNKNOWN && + !analog_engine_supported(link_analog_engine)) { DC_LOG_WARNING("link_id %d has unsupported encoder\n", link->link_id.id); - goto unsupported_fail; + goto create_fail; } if (bios->funcs->get_disp_connector_caps_info) { @@ -670,6 +647,9 @@ static bool construct_phy(struct dc_link *link, enc_init_data.connector = link->link_id; enc_init_data.channel = get_ddc_line(link); enc_init_data.hpd_source = get_hpd_line(link); + enc_init_data.transmitter = transmitter_from_encoder; + enc_init_data.encoder = link_encoder; + enc_init_data.analog_engine = link_analog_engine; link->hpd_src = enc_init_data.hpd_source; @@ -806,7 +786,6 @@ static bool construct_phy(struct dc_link *link, link->hpd_gpio = NULL; } -unsupported_fail: DC_LOG_DC("BIOS object table - %s failed.\n", __func__); return false; } diff --git a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_training_fixed_vs_pe_retimer.c b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_training_fixed_vs_pe_retimer.c index ce174ce5579c07..6a7c4a59ff4c7b 100644 --- a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_training_fixed_vs_pe_retimer.c +++ b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_training_fixed_vs_pe_retimer.c @@ -271,7 +271,7 @@ enum link_training_result dp_perform_fixed_vs_pe_training_sequence( rate = get_dpcd_link_rate(<_settings->link_settings); // Only perform toggle if FIXED_VS LTTPR reports no IEEE OUI - if (memcmp("\x0,\x0,\x0", &link->dpcd_caps.lttpr_caps.lttpr_ieee_oui[0], 3) == 0) { + if (memcmp("\x00\x00\x00", &link->dpcd_caps.lttpr_caps.lttpr_ieee_oui[0], 3) == 0) { /* Vendor specific: Toggle link rate */ toggle_rate = (rate == 0x6) ? 0xA : 0x6; diff --git a/drivers/gpu/drm/amd/display/dc/mpc/dcn32/dcn32_mpc.c b/drivers/gpu/drm/amd/display/dc/mpc/dcn32/dcn32_mpc.c index 83bbbf34bcac7c..badcef027b8468 100644 --- a/drivers/gpu/drm/amd/display/dc/mpc/dcn32/dcn32_mpc.c +++ b/drivers/gpu/drm/amd/display/dc/mpc/dcn32/dcn32_mpc.c @@ -724,8 +724,7 @@ bool mpc32_program_shaper( return false; } - if (mpc->ctx->dc->debug.enable_mem_low_power.bits.mpc) - mpc32_power_on_shaper_3dlut(mpc, mpcc_id, true); + mpc32_power_on_shaper_3dlut(mpc, mpcc_id, true); current_mode = mpc32_get_shaper_current(mpc, mpcc_id); diff --git a/drivers/gpu/drm/amd/display/dc/resource/dce100/dce100_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dce100/dce100_resource.c index d40d91ec2035ff..5fb96490627c50 100644 --- a/drivers/gpu/drm/amd/display/dc/resource/dce100/dce100_resource.c +++ b/drivers/gpu/drm/amd/display/dc/resource/dce100/dce100_resource.c @@ -242,7 +242,8 @@ static const struct dce110_stream_enc_registers stream_enc_regs[] = { stream_enc_regs(3), stream_enc_regs(4), stream_enc_regs(5), - stream_enc_regs(6) + stream_enc_regs(6), + {SR(DAC_SOURCE_SELECT),} /* DACA */ }; static const struct dce_stream_encoder_shift se_shift = { @@ -491,7 +492,8 @@ static struct stream_encoder *dce100_stream_encoder_create( return NULL; if (eng_id == ENGINE_ID_DACA || eng_id == ENGINE_ID_DACB) { - dce110_analog_stream_encoder_construct(enc110, ctx, ctx->dc_bios, eng_id); + dce110_analog_stream_encoder_construct(enc110, ctx, ctx->dc_bios, eng_id, + &stream_enc_regs[eng_id], &se_shift, &se_mask); return &enc110->base; } @@ -638,7 +640,8 @@ static struct link_encoder *dce100_link_encoder_create( if (!enc110) return NULL; - if (enc_init_data->connector.id == CONNECTOR_ID_VGA) { + if (enc_init_data->connector.id == CONNECTOR_ID_VGA && + enc_init_data->analog_engine != ENGINE_ID_UNKNOWN) { dce110_link_encoder_construct(enc110, enc_init_data, &link_enc_feature, @@ -648,9 +651,6 @@ static struct link_encoder *dce100_link_encoder_create( return &enc110->base; } - if (enc_init_data->hpd_source >= ARRAY_SIZE(link_enc_hpd_regs)) - return NULL; - link_regs_id = map_transmitter_id_to_phy_instance(enc_init_data->transmitter); @@ -659,7 +659,8 @@ static struct link_encoder *dce100_link_encoder_create( &link_enc_feature, &link_enc_regs[link_regs_id], &link_enc_aux_regs[enc_init_data->channel - 1], - &link_enc_hpd_regs[enc_init_data->hpd_source]); + enc_init_data->hpd_source >= ARRAY_SIZE(link_enc_hpd_regs) ? + NULL : &link_enc_hpd_regs[enc_init_data->hpd_source]); return &enc110->base; } @@ -978,7 +979,10 @@ struct stream_encoder *dce100_find_first_free_match_stream_enc_for_link( struct dc_link *link = stream->link; enum engine_id preferred_engine = link->link_enc->preferred_engine; - if (dc_is_rgb_signal(stream->signal)) + /* Prefer analog engine if the link encoder has one. + * Otherwise, it's an external encoder. + */ + if (dc_is_rgb_signal(stream->signal) && link->link_enc->analog_engine != ENGINE_ID_UNKNOWN) preferred_engine = link->link_enc->analog_engine; for (i = 0; i < pool->stream_enc_count; i++) { diff --git a/drivers/gpu/drm/amd/display/dc/resource/dce110/dce110_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dce110/dce110_resource.c index cd54382c0af3ee..d957598911cbcd 100644 --- a/drivers/gpu/drm/amd/display/dc/resource/dce110/dce110_resource.c +++ b/drivers/gpu/drm/amd/display/dc/resource/dce110/dce110_resource.c @@ -672,7 +672,7 @@ static struct link_encoder *dce110_link_encoder_create( kzalloc(sizeof(struct dce110_link_encoder), GFP_KERNEL); int link_regs_id; - if (!enc110 || enc_init_data->hpd_source >= ARRAY_SIZE(link_enc_hpd_regs)) + if (!enc110) return NULL; link_regs_id = @@ -683,7 +683,8 @@ static struct link_encoder *dce110_link_encoder_create( &link_enc_feature, &link_enc_regs[link_regs_id], &link_enc_aux_regs[enc_init_data->channel - 1], - &link_enc_hpd_regs[enc_init_data->hpd_source]); + enc_init_data->hpd_source >= ARRAY_SIZE(link_enc_hpd_regs) ? + NULL : &link_enc_hpd_regs[enc_init_data->hpd_source]); return &enc110->base; } @@ -895,6 +896,8 @@ static void get_pixel_clock_parameters( */ pixel_clk_params->requested_pix_clk_100hz = stream->timing.pix_clk_100hz; pixel_clk_params->encoder_object_id = stream->link->link_enc->id; + if (dc_is_rgb_signal(pipe_ctx->stream->signal)) + pixel_clk_params->encoder_object_id = stream->link->link_enc->analog_id; pixel_clk_params->signal_type = pipe_ctx->stream->signal; pixel_clk_params->controller_id = pipe_ctx->stream_res.tg->inst + 1; /* TODO: un-hardcode*/ diff --git a/drivers/gpu/drm/amd/display/dc/resource/dce112/dce112_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dce112/dce112_resource.c index 3f0a6bc4dcc233..1cce903d477aa8 100644 --- a/drivers/gpu/drm/amd/display/dc/resource/dce112/dce112_resource.c +++ b/drivers/gpu/drm/amd/display/dc/resource/dce112/dce112_resource.c @@ -633,7 +633,7 @@ static struct link_encoder *dce112_link_encoder_create( kzalloc(sizeof(struct dce110_link_encoder), GFP_KERNEL); int link_regs_id; - if (!enc110 || enc_init_data->hpd_source >= ARRAY_SIZE(link_enc_hpd_regs)) + if (!enc110) return NULL; link_regs_id = @@ -644,7 +644,8 @@ static struct link_encoder *dce112_link_encoder_create( &link_enc_feature, &link_enc_regs[link_regs_id], &link_enc_aux_regs[enc_init_data->channel - 1], - &link_enc_hpd_regs[enc_init_data->hpd_source]); + enc_init_data->hpd_source >= ARRAY_SIZE(link_enc_hpd_regs) ? + NULL : &link_enc_hpd_regs[enc_init_data->hpd_source]); return &enc110->base; } diff --git a/drivers/gpu/drm/amd/display/dc/resource/dce120/dce120_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dce120/dce120_resource.c index b1570b6b1af306..9f72ae739d8831 100644 --- a/drivers/gpu/drm/amd/display/dc/resource/dce120/dce120_resource.c +++ b/drivers/gpu/drm/amd/display/dc/resource/dce120/dce120_resource.c @@ -717,7 +717,7 @@ static struct link_encoder *dce120_link_encoder_create( kzalloc(sizeof(struct dce110_link_encoder), GFP_KERNEL); int link_regs_id; - if (!enc110 || enc_init_data->hpd_source >= ARRAY_SIZE(link_enc_hpd_regs)) + if (!enc110) return NULL; link_regs_id = @@ -728,7 +728,8 @@ static struct link_encoder *dce120_link_encoder_create( &link_enc_feature, &link_enc_regs[link_regs_id], &link_enc_aux_regs[enc_init_data->channel - 1], - &link_enc_hpd_regs[enc_init_data->hpd_source]); + enc_init_data->hpd_source >= ARRAY_SIZE(link_enc_hpd_regs) ? + NULL : &link_enc_hpd_regs[enc_init_data->hpd_source]); return &enc110->base; } diff --git a/drivers/gpu/drm/amd/display/dc/resource/dce60/dce60_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dce60/dce60_resource.c index f0152933bee2c0..8332413ccc5f96 100644 --- a/drivers/gpu/drm/amd/display/dc/resource/dce60/dce60_resource.c +++ b/drivers/gpu/drm/amd/display/dc/resource/dce60/dce60_resource.c @@ -258,7 +258,9 @@ static const struct dce110_stream_enc_registers stream_enc_regs[] = { stream_enc_regs(2), stream_enc_regs(3), stream_enc_regs(4), - stream_enc_regs(5) + stream_enc_regs(5), + {0}, + {SR(DAC_SOURCE_SELECT),} /* DACA */ }; static const struct dce_stream_encoder_shift se_shift = { @@ -607,7 +609,8 @@ static struct stream_encoder *dce60_stream_encoder_create( return NULL; if (eng_id == ENGINE_ID_DACA || eng_id == ENGINE_ID_DACB) { - dce110_analog_stream_encoder_construct(enc110, ctx, ctx->dc_bios, eng_id); + dce110_analog_stream_encoder_construct(enc110, ctx, ctx->dc_bios, eng_id, + &stream_enc_regs[eng_id], &se_shift, &se_mask); return &enc110->base; } @@ -733,8 +736,9 @@ static struct link_encoder *dce60_link_encoder_create( if (!enc110) return NULL; - if (enc_init_data->connector.id == CONNECTOR_ID_VGA) { - dce110_link_encoder_construct(enc110, + if (enc_init_data->connector.id == CONNECTOR_ID_VGA && + enc_init_data->analog_engine != ENGINE_ID_UNKNOWN) { + dce60_link_encoder_construct(enc110, enc_init_data, &link_enc_feature, &link_enc_regs[ENGINE_ID_DACA], @@ -743,18 +747,16 @@ static struct link_encoder *dce60_link_encoder_create( return &enc110->base; } - if (enc_init_data->hpd_source >= ARRAY_SIZE(link_enc_hpd_regs)) - return NULL; - link_regs_id = map_transmitter_id_to_phy_instance(enc_init_data->transmitter); dce60_link_encoder_construct(enc110, - enc_init_data, - &link_enc_feature, - &link_enc_regs[link_regs_id], - &link_enc_aux_regs[enc_init_data->channel - 1], - &link_enc_hpd_regs[enc_init_data->hpd_source]); + enc_init_data, + &link_enc_feature, + &link_enc_regs[link_regs_id], + &link_enc_aux_regs[enc_init_data->channel - 1], + enc_init_data->hpd_source >= ARRAY_SIZE(link_enc_hpd_regs) ? + NULL : &link_enc_hpd_regs[enc_init_data->hpd_source]); return &enc110->base; } diff --git a/drivers/gpu/drm/amd/display/dc/resource/dce80/dce80_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dce80/dce80_resource.c index 8687104cabb723..5a6d14f657f64c 100644 --- a/drivers/gpu/drm/amd/display/dc/resource/dce80/dce80_resource.c +++ b/drivers/gpu/drm/amd/display/dc/resource/dce80/dce80_resource.c @@ -258,7 +258,8 @@ static const struct dce110_stream_enc_registers stream_enc_regs[] = { stream_enc_regs(3), stream_enc_regs(4), stream_enc_regs(5), - stream_enc_regs(6) + stream_enc_regs(6), + {SR(DAC_SOURCE_SELECT),} /* DACA */ }; static const struct dce_stream_encoder_shift se_shift = { @@ -614,7 +615,8 @@ static struct stream_encoder *dce80_stream_encoder_create( return NULL; if (eng_id == ENGINE_ID_DACA || eng_id == ENGINE_ID_DACB) { - dce110_analog_stream_encoder_construct(enc110, ctx, ctx->dc_bios, eng_id); + dce110_analog_stream_encoder_construct(enc110, ctx, ctx->dc_bios, eng_id, + &stream_enc_regs[eng_id], &se_shift, &se_mask); return &enc110->base; } @@ -740,7 +742,8 @@ static struct link_encoder *dce80_link_encoder_create( if (!enc110) return NULL; - if (enc_init_data->connector.id == CONNECTOR_ID_VGA) { + if (enc_init_data->connector.id == CONNECTOR_ID_VGA && + enc_init_data->analog_engine != ENGINE_ID_UNKNOWN) { dce110_link_encoder_construct(enc110, enc_init_data, &link_enc_feature, @@ -750,9 +753,6 @@ static struct link_encoder *dce80_link_encoder_create( return &enc110->base; } - if (enc_init_data->hpd_source >= ARRAY_SIZE(link_enc_hpd_regs)) - return NULL; - link_regs_id = map_transmitter_id_to_phy_instance(enc_init_data->transmitter); @@ -761,7 +761,8 @@ static struct link_encoder *dce80_link_encoder_create( &link_enc_feature, &link_enc_regs[link_regs_id], &link_enc_aux_regs[enc_init_data->channel - 1], - &link_enc_hpd_regs[enc_init_data->hpd_source]); + enc_init_data->hpd_source >= ARRAY_SIZE(link_enc_hpd_regs) ? + NULL : &link_enc_hpd_regs[enc_init_data->hpd_source]); return &enc110->base; } diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn20/dcn20_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn20/dcn20_resource.c index 6679c1a14f2fec..8d10aac9c510c3 100644 --- a/drivers/gpu/drm/amd/display/dc/resource/dcn20/dcn20_resource.c +++ b/drivers/gpu/drm/amd/display/dc/resource/dcn20/dcn20_resource.c @@ -1660,8 +1660,8 @@ bool dcn20_validate_dsc(struct dc *dc, struct dc_state *new_ctx) if (pipe_ctx->top_pipe || pipe_ctx->prev_odm_pipe || !stream || !stream->timing.flags.DSC) continue; - dsc_cfg.pic_width = (stream->timing.h_addressable + stream->timing.h_border_left - + stream->timing.h_border_right) / opp_cnt; + dsc_cfg.pic_width = (stream->timing.h_addressable + pipe_ctx->dsc_padding_params.dsc_hactive_padding + + stream->timing.h_border_left + stream->timing.h_border_right) / opp_cnt; dsc_cfg.pic_height = stream->timing.v_addressable + stream->timing.v_border_top + stream->timing.v_border_bottom; dsc_cfg.pixel_encoding = stream->timing.pixel_encoding; @@ -1669,7 +1669,7 @@ bool dcn20_validate_dsc(struct dc *dc, struct dc_state *new_ctx) dsc_cfg.is_odm = pipe_ctx->next_odm_pipe ? true : false; dsc_cfg.dc_dsc_cfg = stream->timing.dsc_cfg; dsc_cfg.dc_dsc_cfg.num_slices_h /= opp_cnt; - dsc_cfg.dsc_padding = pipe_ctx->dsc_padding_params.dsc_hactive_padding; + dsc_cfg.dsc_padding = 0; if (!pipe_ctx->stream_res.dsc->funcs->dsc_validate_stream(pipe_ctx->stream_res.dsc, &dsc_cfg)) return false; diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn315/dcn315_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn315/dcn315_resource.c index 4e962f522f1bee..228ae665c7893f 100644 --- a/drivers/gpu/drm/amd/display/dc/resource/dcn315/dcn315_resource.c +++ b/drivers/gpu/drm/amd/display/dc/resource/dcn315/dcn315_resource.c @@ -1230,12 +1230,12 @@ static struct stream_encoder *dcn315_stream_encoder_create( /*PHYB is wired off in HW, allow front end to remapping, otherwise needs more changes*/ /* Mapping of VPG, AFMT, DME register blocks to DIO block instance */ - if (eng_id <= ENGINE_ID_DIGF) { - vpg_inst = eng_id; - afmt_inst = eng_id; - } else + if (eng_id < 0 || eng_id >= ARRAY_SIZE(stream_enc_regs)) return NULL; + vpg_inst = eng_id; + afmt_inst = eng_id; + enc1 = kzalloc(sizeof(struct dcn10_stream_encoder), GFP_KERNEL); vpg = dcn31_vpg_create(ctx, vpg_inst); afmt = dcn31_afmt_create(ctx, afmt_inst); diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn316/dcn316_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn316/dcn316_resource.c index 5a95dd54cb4298..45abf3b2eb2c48 100644 --- a/drivers/gpu/drm/amd/display/dc/resource/dcn316/dcn316_resource.c +++ b/drivers/gpu/drm/amd/display/dc/resource/dcn316/dcn316_resource.c @@ -1223,12 +1223,12 @@ static struct stream_encoder *dcn316_stream_encoder_create( int afmt_inst; /* Mapping of VPG, AFMT, DME register blocks to DIO block instance */ - if (eng_id <= ENGINE_ID_DIGF) { - vpg_inst = eng_id; - afmt_inst = eng_id; - } else + if (eng_id < 0 || eng_id >= ARRAY_SIZE(stream_enc_regs)) return NULL; + vpg_inst = eng_id; + afmt_inst = eng_id; + enc1 = kzalloc(sizeof(struct dcn10_stream_encoder), GFP_KERNEL); vpg = dcn31_vpg_create(ctx, vpg_inst); afmt = dcn31_afmt_create(ctx, afmt_inst); diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn32/dcn32_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn32/dcn32_resource.c index b276fec3e479a2..31eabdf198a191 100644 --- a/drivers/gpu/drm/amd/display/dc/resource/dcn32/dcn32_resource.c +++ b/drivers/gpu/drm/amd/display/dc/resource/dcn32/dcn32_resource.c @@ -1211,12 +1211,12 @@ static struct stream_encoder *dcn32_stream_encoder_create( int afmt_inst; /* Mapping of VPG, AFMT, DME register blocks to DIO block instance */ - if (eng_id <= ENGINE_ID_DIGF) { - vpg_inst = eng_id; - afmt_inst = eng_id; - } else + if (eng_id < 0 || eng_id >= ARRAY_SIZE(stream_enc_regs)) return NULL; + vpg_inst = eng_id; + afmt_inst = eng_id; + enc1 = kzalloc(sizeof(struct dcn10_stream_encoder), GFP_KERNEL); vpg = dcn32_vpg_create(ctx, vpg_inst); afmt = dcn32_afmt_create(ctx, afmt_inst); @@ -1789,7 +1789,10 @@ static bool dml1_validate(struct dc *dc, struct dc_state *context, enum dc_valid dc->res_pool->funcs->calculate_wm_and_dlg(dc, context, pipes, pipe_cnt, vlevel); + DC_FP_START(); dcn32_override_min_req_memclk(dc, context); + DC_FP_END(); + dcn32_override_min_req_dcfclk(dc, context); BW_VAL_TRACE_END_WATERMARKS(); diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn321/dcn321_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn321/dcn321_resource.c index 3466ca34c93fed..c72c6dbc0cb4d7 100644 --- a/drivers/gpu/drm/amd/display/dc/resource/dcn321/dcn321_resource.c +++ b/drivers/gpu/drm/amd/display/dc/resource/dcn321/dcn321_resource.c @@ -1192,12 +1192,12 @@ static struct stream_encoder *dcn321_stream_encoder_create( int afmt_inst; /* Mapping of VPG, AFMT, DME register blocks to DIO block instance */ - if (eng_id <= ENGINE_ID_DIGF) { - vpg_inst = eng_id; - afmt_inst = eng_id; - } else + if (eng_id < 0 || eng_id >= ARRAY_SIZE(stream_enc_regs)) return NULL; + vpg_inst = eng_id; + afmt_inst = eng_id; + enc1 = kzalloc(sizeof(struct dcn10_stream_encoder), GFP_KERNEL); vpg = dcn321_vpg_create(ctx, vpg_inst); afmt = dcn321_afmt_create(ctx, afmt_inst); diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn35/dcn35_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn35/dcn35_resource.c index d056e5fd545871..9edabd8ceca9bf 100644 --- a/drivers/gpu/drm/amd/display/dc/resource/dcn35/dcn35_resource.c +++ b/drivers/gpu/drm/amd/display/dc/resource/dcn35/dcn35_resource.c @@ -1274,12 +1274,12 @@ static struct stream_encoder *dcn35_stream_encoder_create( int afmt_inst; /* Mapping of VPG, AFMT, DME register blocks to DIO block instance */ - if (eng_id <= ENGINE_ID_DIGF) { - vpg_inst = eng_id; - afmt_inst = eng_id; - } else + if (eng_id < 0 || eng_id >= ARRAY_SIZE(stream_enc_regs)) return NULL; + vpg_inst = eng_id; + afmt_inst = eng_id; + enc1 = kzalloc(sizeof(struct dcn10_stream_encoder), GFP_KERNEL); vpg = dcn31_vpg_create(ctx, vpg_inst); afmt = dcn31_afmt_create(ctx, afmt_inst); diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn351/dcn351_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn351/dcn351_resource.c index 9fab3169069c40..43ece2bbcd64f6 100644 --- a/drivers/gpu/drm/amd/display/dc/resource/dcn351/dcn351_resource.c +++ b/drivers/gpu/drm/amd/display/dc/resource/dcn351/dcn351_resource.c @@ -1254,12 +1254,12 @@ static struct stream_encoder *dcn35_stream_encoder_create( int afmt_inst; /* Mapping of VPG, AFMT, DME register blocks to DIO block instance */ - if (eng_id <= ENGINE_ID_DIGF) { - vpg_inst = eng_id; - afmt_inst = eng_id; - } else + if (eng_id < 0 || eng_id >= ARRAY_SIZE(stream_enc_regs)) return NULL; + vpg_inst = eng_id; + afmt_inst = eng_id; + enc1 = kzalloc(sizeof(struct dcn10_stream_encoder), GFP_KERNEL); vpg = dcn31_vpg_create(ctx, vpg_inst); afmt = dcn31_afmt_create(ctx, afmt_inst); diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn36/dcn36_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn36/dcn36_resource.c index 6469d5fe2e6d46..a1132102afde43 100644 --- a/drivers/gpu/drm/amd/display/dc/resource/dcn36/dcn36_resource.c +++ b/drivers/gpu/drm/amd/display/dc/resource/dcn36/dcn36_resource.c @@ -769,7 +769,7 @@ static const struct dc_debug_options debug_defaults_drv = { }; static const struct dc_check_config config_defaults = { - .enable_legacy_fast_update = true, + .enable_legacy_fast_update = false, }; static const struct dc_panel_config panel_config_defaults = { diff --git a/drivers/gpu/drm/amd/include/amd_shared.h b/drivers/gpu/drm/amd/include/amd_shared.h index 17945094a13834..ac2d3701e2bdcc 100644 --- a/drivers/gpu/drm/amd/include/amd_shared.h +++ b/drivers/gpu/drm/amd/include/amd_shared.h @@ -89,6 +89,7 @@ enum amd_apu_flags { * @AMD_IP_BLOCK_TYPE_VPE: Video Processing Engine * @AMD_IP_BLOCK_TYPE_UMSCH_MM: User Mode Scheduler for Multimedia * @AMD_IP_BLOCK_TYPE_ISP: Image Signal Processor +* @AMD_IP_BLOCK_TYPE_RAS: Reliability, Availability, Serviceability * @AMD_IP_BLOCK_TYPE_NUM: Total number of IP block types */ enum amd_ip_block_type { diff --git a/drivers/gpu/drm/amd/pm/amdgpu_dpm.c b/drivers/gpu/drm/amd/pm/amdgpu_dpm.c index 302af1fb6901e7..9c9e96655c4be2 100644 --- a/drivers/gpu/drm/amd/pm/amdgpu_dpm.c +++ b/drivers/gpu/drm/amd/pm/amdgpu_dpm.c @@ -1589,7 +1589,7 @@ int amdgpu_dpm_get_power_limit(struct amdgpu_device *adev, int ret = 0; if (!pp_funcs->get_power_limit) - return -ENODATA; + return -EOPNOTSUPP; mutex_lock(&adev->pm.mutex); ret = pp_funcs->get_power_limit(adev->powerplay.pp_handle, diff --git a/drivers/gpu/drm/amd/pm/legacy-dpm/si_dpm.c b/drivers/gpu/drm/amd/pm/legacy-dpm/si_dpm.c index 695432d3045ffa..c8a23ece8654db 100644 --- a/drivers/gpu/drm/amd/pm/legacy-dpm/si_dpm.c +++ b/drivers/gpu/drm/amd/pm/legacy-dpm/si_dpm.c @@ -3454,9 +3454,11 @@ static void si_apply_state_adjust_rules(struct amdgpu_device *adev, if (adev->asic_type == CHIP_HAINAN) { if ((adev->pdev->revision == 0x81) || (adev->pdev->revision == 0xC3) || + (adev->pdev->device == 0x6660) || (adev->pdev->device == 0x6664) || (adev->pdev->device == 0x6665) || - (adev->pdev->device == 0x6667)) { + (adev->pdev->device == 0x6667) || + (adev->pdev->device == 0x666F)) { max_sclk = 75000; } if ((adev->pdev->revision == 0xC3) || @@ -3464,6 +3466,11 @@ static void si_apply_state_adjust_rules(struct amdgpu_device *adev, max_sclk = 60000; max_mclk = 80000; } + if ((adev->pdev->device == 0x666f) && + (adev->pdev->revision == 0x00)) { + max_sclk = 80000; + max_mclk = 95000; + } } else if (adev->asic_type == CHIP_OLAND) { if ((adev->pdev->revision == 0xC7) || (adev->pdev->revision == 0x80) || diff --git a/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c b/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c index f51fa265230b37..2a0e826d0317d9 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c +++ b/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c @@ -618,6 +618,9 @@ int amdgpu_smu_ras_send_msg(struct amdgpu_device *adev, enum smu_message_type ms struct smu_context *smu = adev->powerplay.pp_handle; int ret = -EOPNOTSUPP; + if (!smu) + return ret; + if (smu->ppt_funcs && smu->ppt_funcs->ras_send_msg) ret = smu->ppt_funcs->ras_send_msg(smu, msg, param, read_arg); diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu11/smu_v11_0.c b/drivers/gpu/drm/amd/pm/swsmu/smu11/smu_v11_0.c index b0d6487171d70d..48bf000f12e5c7 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu11/smu_v11_0.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu11/smu_v11_0.c @@ -262,7 +262,6 @@ int smu_v11_0_check_fw_version(struct smu_context *smu) "smu fw program = %d, version = 0x%08x (%d.%d.%d)\n", smu->smc_driver_if_version, if_version, smu_program, smu_version, smu_major, smu_minor, smu_debug); - dev_info(smu->adev->dev, "SMU driver if version not matched\n"); } return ret; diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu12/smu_v12_0.c b/drivers/gpu/drm/amd/pm/swsmu/smu12/smu_v12_0.c index 3d3cd546f0adc4..63098ed9fcfe5f 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu12/smu_v12_0.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu12/smu_v12_0.c @@ -101,7 +101,6 @@ int smu_v12_0_check_fw_version(struct smu_context *smu) "smu fw program = %d, smu fw version = 0x%08x (%d.%d.%d)\n", smu->smc_driver_if_version, if_version, smu_program, smu_version, smu_major, smu_minor, smu_debug); - dev_info(smu->adev->dev, "SMU driver if version not matched\n"); } return ret; diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_0_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_0_ppt.c index eaeff6a9bc50fc..cd7db433795f81 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_0_ppt.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_0_ppt.c @@ -59,6 +59,10 @@ #define to_amdgpu_device(x) (container_of(x, struct amdgpu_device, pm.smu_i2c)) +static void smu_v13_0_0_get_od_setting_limits(struct smu_context *smu, + int od_feature_bit, + int32_t *min, int32_t *max); + #define FEATURE_MASK(feature) (1ULL << feature) #define SMC_DPM_FEATURE ( \ FEATURE_MASK(FEATURE_DPM_GFXCLK_BIT) | \ @@ -1061,8 +1065,35 @@ static bool smu_v13_0_0_is_od_feature_supported(struct smu_context *smu, PPTable_t *pptable = smu->smu_table.driver_pptable; const OverDriveLimits_t * const overdrive_upperlimits = &pptable->SkuTable.OverDriveLimitsBasicMax; + int32_t min_value, max_value; + bool feature_enabled; - return overdrive_upperlimits->FeatureCtrlMask & (1U << od_feature_bit); + switch (od_feature_bit) { + case PP_OD_FEATURE_FAN_CURVE_BIT: + feature_enabled = !!(overdrive_upperlimits->FeatureCtrlMask & (1U << od_feature_bit)); + if (feature_enabled) { + smu_v13_0_0_get_od_setting_limits(smu, PP_OD_FEATURE_FAN_CURVE_TEMP, + &min_value, &max_value); + if (!min_value && !max_value) { + feature_enabled = false; + goto out; + } + + smu_v13_0_0_get_od_setting_limits(smu, PP_OD_FEATURE_FAN_CURVE_PWM, + &min_value, &max_value); + if (!min_value && !max_value) { + feature_enabled = false; + goto out; + } + } + break; + default: + feature_enabled = !!(overdrive_upperlimits->FeatureCtrlMask & (1U << od_feature_bit)); + break; + } + +out: + return feature_enabled; } static void smu_v13_0_0_get_od_setting_limits(struct smu_context *smu, @@ -2110,6 +2141,7 @@ static ssize_t smu_v13_0_0_get_gpu_metrics(struct smu_context *smu, (struct gpu_metrics_v1_3 *)smu_table->gpu_metrics_table; SmuMetricsExternal_t metrics_ext; SmuMetrics_t *metrics = &metrics_ext.SmuMetrics; + uint32_t mp1_ver = amdgpu_ip_version(smu->adev, MP1_HWIP, 0); int ret = 0; ret = smu_cmn_get_metrics_table(smu, @@ -2134,7 +2166,12 @@ static ssize_t smu_v13_0_0_get_gpu_metrics(struct smu_context *smu, metrics->Vcn1ActivityPercentage); gpu_metrics->average_socket_power = metrics->AverageSocketPower; - gpu_metrics->energy_accumulator = metrics->EnergyAccumulator; + + if ((mp1_ver == IP_VERSION(13, 0, 0) && smu->smc_fw_version <= 0x004e1e00) || + (mp1_ver == IP_VERSION(13, 0, 10) && smu->smc_fw_version <= 0x00500800)) + gpu_metrics->energy_accumulator = metrics->EnergyAccumulator; + else + gpu_metrics->energy_accumulator = UINT_MAX; if (metrics->AverageGfxActivity <= SMU_13_0_0_BUSY_THRESHOLD) gpu_metrics->average_gfxclk_frequency = metrics->AverageGfxclkFrequencyPostDs; @@ -2290,7 +2327,8 @@ static int smu_v13_0_0_restore_user_od_settings(struct smu_context *smu) user_od_table->OverDriveTable.FeatureCtrlMask = BIT(PP_OD_FEATURE_GFXCLK_BIT) | BIT(PP_OD_FEATURE_UCLK_BIT) | BIT(PP_OD_FEATURE_GFX_VF_CURVE_BIT) | - BIT(PP_OD_FEATURE_FAN_CURVE_BIT); + BIT(PP_OD_FEATURE_FAN_CURVE_BIT) | + BIT(PP_OD_FEATURE_ZERO_FAN_BIT); res = smu_v13_0_0_upload_overdrive_table(smu, user_od_table); user_od_table->OverDriveTable.FeatureCtrlMask = 0; if (res == 0) diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_12_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_12_ppt.c index 9e635f733fbfdf..370326ec65d948 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_12_ppt.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_12_ppt.c @@ -922,7 +922,7 @@ void smu_v13_0_12_get_gpu_metrics(struct smu_context *smu, void **table, gpu_metrics->gfx_below_host_limit_total_acc [i] = SMUQ10_ROUND( metrics->GfxclkBelowHostLimitTotalAcc[inst]); - }; + } } gpu_metrics->xgmi_link_width = metrics->XgmiWidth; diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_6_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_6_ppt.c index 44e1cd821eec9a..f81679dbb3d7be 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_6_ppt.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_6_ppt.c @@ -1520,7 +1520,7 @@ static int smu_v13_0_6_print_clk_levels(struct smu_context *smu, case SMU_OD_MCLK: if (!smu_v13_0_6_cap_supported(smu, SMU_CAP(SET_UCLK_MAX))) - return 0; + return -EOPNOTSUPP; size += sysfs_emit_at(buf, size, "%s:\n", "OD_MCLK"); size += sysfs_emit_at(buf, size, "0: %uMhz\n1: %uMhz\n", diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_7_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_7_ppt.c index a3fc35b9011e44..da31a6504ac054 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_7_ppt.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_7_ppt.c @@ -59,6 +59,10 @@ #define to_amdgpu_device(x) (container_of(x, struct amdgpu_device, pm.smu_i2c)) +static void smu_v13_0_7_get_od_setting_limits(struct smu_context *smu, + int od_feature_bit, + int32_t *min, int32_t *max); + #define FEATURE_MASK(feature) (1ULL << feature) #define SMC_DPM_FEATURE ( \ FEATURE_MASK(FEATURE_DPM_GFXCLK_BIT) | \ @@ -1050,8 +1054,35 @@ static bool smu_v13_0_7_is_od_feature_supported(struct smu_context *smu, PPTable_t *pptable = smu->smu_table.driver_pptable; const OverDriveLimits_t * const overdrive_upperlimits = &pptable->SkuTable.OverDriveLimitsBasicMax; + int32_t min_value, max_value; + bool feature_enabled; - return overdrive_upperlimits->FeatureCtrlMask & (1U << od_feature_bit); + switch (od_feature_bit) { + case PP_OD_FEATURE_FAN_CURVE_BIT: + feature_enabled = !!(overdrive_upperlimits->FeatureCtrlMask & (1U << od_feature_bit)); + if (feature_enabled) { + smu_v13_0_7_get_od_setting_limits(smu, PP_OD_FEATURE_FAN_CURVE_TEMP, + &min_value, &max_value); + if (!min_value && !max_value) { + feature_enabled = false; + goto out; + } + + smu_v13_0_7_get_od_setting_limits(smu, PP_OD_FEATURE_FAN_CURVE_PWM, + &min_value, &max_value); + if (!min_value && !max_value) { + feature_enabled = false; + goto out; + } + } + break; + default: + feature_enabled = !!(overdrive_upperlimits->FeatureCtrlMask & (1U << od_feature_bit)); + break; + } + +out: + return feature_enabled; } static void smu_v13_0_7_get_od_setting_limits(struct smu_context *smu, @@ -2120,7 +2151,8 @@ static ssize_t smu_v13_0_7_get_gpu_metrics(struct smu_context *smu, metrics->Vcn1ActivityPercentage); gpu_metrics->average_socket_power = metrics->AverageSocketPower; - gpu_metrics->energy_accumulator = metrics->EnergyAccumulator; + gpu_metrics->energy_accumulator = smu->smc_fw_version <= 0x00521400 ? + metrics->EnergyAccumulator : UINT_MAX; if (metrics->AverageGfxActivity <= SMU_13_0_7_BUSY_THRESHOLD) gpu_metrics->average_gfxclk_frequency = metrics->AverageGfxclkFrequencyPostDs; @@ -2276,7 +2308,8 @@ static int smu_v13_0_7_restore_user_od_settings(struct smu_context *smu) user_od_table->OverDriveTable.FeatureCtrlMask = BIT(PP_OD_FEATURE_GFXCLK_BIT) | BIT(PP_OD_FEATURE_UCLK_BIT) | BIT(PP_OD_FEATURE_GFX_VF_CURVE_BIT) | - BIT(PP_OD_FEATURE_FAN_CURVE_BIT); + BIT(PP_OD_FEATURE_FAN_CURVE_BIT) | + BIT(PP_OD_FEATURE_ZERO_FAN_BIT); res = smu_v13_0_7_upload_overdrive_table(smu, user_od_table); user_od_table->OverDriveTable.FeatureCtrlMask = 0; if (res == 0) diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu14/smu_v14_0.c b/drivers/gpu/drm/amd/pm/swsmu/smu14/smu_v14_0.c index 06a81533759cda..802b3fbbeb8e7e 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu14/smu_v14_0.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu14/smu_v14_0.c @@ -284,7 +284,6 @@ int smu_v14_0_check_fw_version(struct smu_context *smu) "smu fw program = %d, smu fw version = 0x%08x (%d.%d.%d)\n", smu->smc_driver_if_version, if_version, smu_program, smu_version, smu_major, smu_minor, smu_debug); - dev_info(adev->dev, "SMU driver if version not matched\n"); } return ret; diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu14/smu_v14_0_2_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu14/smu_v14_0_2_ppt.c index d7642d388bc386..fa535f43876b57 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu14/smu_v14_0_2_ppt.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu14/smu_v14_0_2_ppt.c @@ -2413,7 +2413,8 @@ static int smu_v14_0_2_restore_user_od_settings(struct smu_context *smu) user_od_table->OverDriveTable.FeatureCtrlMask = BIT(PP_OD_FEATURE_GFXCLK_BIT) | BIT(PP_OD_FEATURE_UCLK_BIT) | BIT(PP_OD_FEATURE_GFX_VF_CURVE_BIT) | - BIT(PP_OD_FEATURE_FAN_CURVE_BIT); + BIT(PP_OD_FEATURE_FAN_CURVE_BIT) | + BIT(PP_OD_FEATURE_ZERO_FAN_BIT); res = smu_v14_0_2_upload_overdrive_table(smu, user_od_table); user_od_table->OverDriveTable.FeatureCtrlMask = 0; if (res == 0) diff --git a/drivers/gpu/drm/apple/Kconfig b/drivers/gpu/drm/apple/Kconfig new file mode 100644 index 00000000000000..a1d4498c5d788e --- /dev/null +++ b/drivers/gpu/drm/apple/Kconfig @@ -0,0 +1,30 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +config DRM_APPLE + tristate "DRM Support for Apple display controllers" + depends on DRM && OF && ARM64 + depends on ARCH_APPLE || COMPILE_TEST + depends on APPLE_RTKIT + depends on OF_ADDRESS + select APPLE_PMP_REPORT + select DRM_CLIENT_SELECTION + select DRM_KMS_HELPER + select DRM_KMS_DMA_HELPER + select DRM_GEM_DMA_HELPER + select VIDEOMODE_HELPERS + select BACKLIGHT_CLASS_DEVICE + select MULTIPLEXER + help + Say Y if you have an Apple Silicon chipset. + +config DRM_APPLE_AUDIO + bool "DisplayPort/HDMI Audio support" + default y + depends on DRM_APPLE + depends on SND + select SND_PCM + select SND_DMAENGINE_PCM + +config DRM_APPLE_DEBUG + bool "Enable additional driver debugging" + depends on DRM_APPLE + depends on EXPERT # only for developers diff --git a/drivers/gpu/drm/apple/Makefile b/drivers/gpu/drm/apple/Makefile new file mode 100644 index 00000000000000..b22839bdd611a7 --- /dev/null +++ b/drivers/gpu/drm/apple/Makefile @@ -0,0 +1,19 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT + +CFLAGS_trace.o = -I$(src) + +appledrm-y := apple_drv.o +appledrm-y += afk.o dcp.o dcp_backlight.o dptxep.o iomfb.o parser.o systemep.o +appledrm-$(CONFIG_DRM_APPLE_AUDIO) += audio.o +appledrm-$(CONFIG_DRM_APPLE_AUDIO) += av.o +appledrm-y += connector.o +appledrm-y += ibootep.o +appledrm-y += iomfb_v12_3.o +appledrm-y += iomfb_v13_3.o +appledrm-y += epic/dpavservep.o +appledrm-y += plane.o + + +appledrm-$(CONFIG_TRACING) += trace.o + +obj-$(CONFIG_DRM_APPLE) += appledrm.o diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c new file mode 100644 index 00000000000000..d0de72072877b8 --- /dev/null +++ b/drivers/gpu/drm/apple/afk.c @@ -0,0 +1,1181 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright 2022 Sven Peter */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "afk.h" +#include "trace.h" + +struct afk_receive_message_work { + struct apple_dcp_afkep *ep; + u64 message; + struct work_struct work; +}; + +#define RBEP_TYPE GENMASK(63, 48) + +enum rbep_msg_type { + RBEP_INIT = 0x80, + RBEP_INIT_ACK = 0xa0, + RBEP_GETBUF = 0x89, + RBEP_GETBUF_ACK = 0xa1, + RBEP_INIT_TX = 0x8a, + RBEP_INIT_RX = 0x8b, + RBEP_START = 0xa3, + RBEP_START_ACK = 0x86, + RBEP_SEND = 0xa2, + RBEP_RECV = 0x85, + RBEP_SHUTDOWN = 0xc0, + RBEP_SHUTDOWN_ACK = 0xc1, +}; + +#define BLOCK_SHIFT 6 + +#define GETBUF_SIZE GENMASK(31, 16) +#define GETBUF_TAG GENMASK(15, 0) +#define GETBUF_ACK_DVA GENMASK(47, 0) + +#define INITRB_OFFSET GENMASK(47, 32) +#define INITRB_SIZE GENMASK(31, 16) +#define INITRB_TAG GENMASK(15, 0) + +#define SEND_WPTR GENMASK(31, 0) + +static void afk_send(struct apple_dcp_afkep *ep, u64 message) +{ + dcp_send_message(ep->dcp, ep->endpoint, message); +} + +struct apple_dcp_afkep *afk_init(struct apple_dcp *dcp, u32 endpoint, + const struct apple_epic_service_ops *ops) +{ + struct apple_dcp_afkep *afkep; + int ret; + + afkep = devm_kzalloc(dcp->dev, sizeof(*afkep), GFP_KERNEL); + if (!afkep) + return ERR_PTR(-ENOMEM); + + afkep->ops = ops; + afkep->dcp = dcp; + afkep->endpoint = endpoint; + afkep->wq = alloc_ordered_workqueue("apple-dcp-afkep%02x", + WQ_MEM_RECLAIM, endpoint); + if (!afkep->wq) { + ret = -ENOMEM; + goto out_free_afkep; + } + + // TODO: devm_ for wq + + init_completion(&afkep->started); + init_completion(&afkep->stopped); + spin_lock_init(&afkep->lock); + + return afkep; + +out_free_afkep: + devm_kfree(dcp->dev, afkep); + return ERR_PTR(ret); +} + +void afk_shutdown(struct apple_dcp_afkep *afkep) +{ + afk_send(afkep, FIELD_PREP(RBEP_TYPE, RBEP_SHUTDOWN)); + int ret; + + ret = wait_for_completion_timeout(&afkep->stopped, msecs_to_jiffies(1000)); + if (ret <= 0) { + dev_err(afkep->dcp->dev, "Timed out shutting down AFK endpoint %02x", afkep->endpoint); + } + + destroy_workqueue(afkep->wq); +} + +int afk_start(struct apple_dcp_afkep *ep) +{ + int ret; + + reinit_completion(&ep->started); + apple_rtkit_start_ep(ep->dcp->rtk, ep->endpoint); + afk_send(ep, FIELD_PREP(RBEP_TYPE, RBEP_INIT)); + + ret = wait_for_completion_timeout(&ep->started, msecs_to_jiffies(1000)); + if (ret <= 0) + return -ETIMEDOUT; + else + return 0; +} + +static void afk_getbuf(struct apple_dcp_afkep *ep, u64 message) +{ + u16 size = FIELD_GET(GETBUF_SIZE, message) << BLOCK_SHIFT; + u16 tag = FIELD_GET(GETBUF_TAG, message); + u64 reply; + + trace_afk_getbuf(ep, size, tag); + + if (ep->bfr) { + dev_err(ep->dcp->dev, + "Got GETBUF message but buffer already exists\n"); + return; + } + + ep->bfr = dmam_alloc_coherent(ep->dcp->dev, size, &ep->bfr_dma, + GFP_KERNEL); + if (!ep->bfr) { + dev_err(ep->dcp->dev, "Failed to allocate %d bytes buffer\n", + size); + return; + } + + ep->bfr_size = size; + ep->bfr_tag = tag; + + reply = FIELD_PREP(RBEP_TYPE, RBEP_GETBUF_ACK); + reply |= FIELD_PREP(GETBUF_ACK_DVA, ep->bfr_dma); + afk_send(ep, reply); +} + +static void afk_init_rxtx(struct apple_dcp_afkep *ep, u64 message, + struct afk_ringbuffer *bfr) +{ + u16 base = FIELD_GET(INITRB_OFFSET, message) << BLOCK_SHIFT; + u16 size = FIELD_GET(INITRB_SIZE, message) << BLOCK_SHIFT; + u16 tag = FIELD_GET(INITRB_TAG, message); + u32 bufsz, end; + + if (tag != ep->bfr_tag) { + dev_err(ep->dcp->dev, "AFK[ep:%02x]: expected tag 0x%x but got 0x%x\n", + ep->endpoint, ep->bfr_tag, tag); + return; + } + + if (bfr->ready) { + dev_err(ep->dcp->dev, "AFK[ep:%02x]: buffer is already initialized\n", + ep->endpoint); + return; + } + + if (base >= ep->bfr_size) { + dev_err(ep->dcp->dev, + "AFK[ep:%02x]: requested base 0x%x >= max size 0x%lx\n", + ep->endpoint, base, ep->bfr_size); + return; + } + + end = base + size; + if (end > ep->bfr_size) { + dev_err(ep->dcp->dev, + "AFK[ep:%02x]: requested end 0x%x > max size 0x%lx\n", + ep->endpoint, end, ep->bfr_size); + return; + } + + bfr->hdr = ep->bfr + base; + bufsz = le32_to_cpu(bfr->hdr->bufsz); + if (bufsz + sizeof(*bfr->hdr) != size) { + dev_err(ep->dcp->dev, + "AFK[ep:%02x]: ring buffer size 0x%x != expected 0x%lx\n", + ep->endpoint, bufsz, sizeof(*bfr->hdr)); + return; + } + + bfr->buf = bfr->hdr + 1; + bfr->bufsz = bufsz; + bfr->ready = true; + + if (ep->rxbfr.ready && ep->txbfr.ready) + afk_send(ep, FIELD_PREP(RBEP_TYPE, RBEP_START)); +} + +#if IS_ENABLED(CONFIG_DRM_APPLE_DEBUG) +static void afk_populate_service_debugfs(struct apple_epic_service *srv); +static void afk_remove_service_debugfs(struct apple_epic_service *srv); +#else +static void afk_populate_service_debugfs(struct apple_epic_service *srv) +{ +} +static void afk_remove_service_debugfs(struct apple_epic_service *srv) +{ +} +#endif + +static const struct apple_epic_service_ops * +afk_match_service(struct apple_dcp_afkep *ep, const char *name) +{ + const struct apple_epic_service_ops *ops; + + if (!name[0]) + return NULL; + if (!ep->ops) + return NULL; + + for (ops = ep->ops; ops->name[0]; ops++) { + if (strcmp(ops->name, name)) + continue; + + return ops; + } + + return NULL; +} + +static struct apple_epic_service *afk_epic_find_service(struct apple_dcp_afkep *ep, + u32 channel) +{ + for (u32 i = 0; i < ep->num_channels; i++) + if (ep->services[i].enabled && ep->services[i].channel == channel) + return &ep->services[i]; + + return NULL; +} + +static void afk_recv_handle_init(struct apple_dcp_afkep *ep, u32 channel, + u8 *payload, size_t payload_size) +{ + char name[32]; + s64 epic_unit = -1; + u32 ch_idx; + const char *service_name = name; + const char *epic_name = NULL, *epic_class = NULL; + const struct apple_epic_service_ops *ops; + struct dcp_parse_ctx ctx; + u8 *props = payload + sizeof(name); + size_t props_size = payload_size - sizeof(name); + + WARN_ON(afk_epic_find_service(ep, channel)); + + if (payload_size < sizeof(name)) { + dev_err(ep->dcp->dev, "AFK[ep:%02x]: payload too small: %lx\n", + ep->endpoint, payload_size); + return; + } + + if (ep->num_channels >= AFK_MAX_CHANNEL) { + dev_err(ep->dcp->dev, "AFK[ep:%02x]: too many enabled services!\n", + ep->endpoint); + return; + } + + strscpy(name, payload, sizeof(name)); + + /* + * in DCP firmware 13.2 DCP reports interface-name as name which starts + * with "dispext%d" using -1 s ID for "dcp". In the 12.3 firmware + * EPICProviderClass was used. If the init call has props parse them and + * use EPICProviderClass to match the service. + */ + if (props_size > 36) { + int ret = parse(props, props_size, &ctx); + if (ret) { + dev_err(ep->dcp->dev, + "AFK[ep:%02x]: Failed to parse service init props for %s\n", + ep->endpoint, name); + return; + } + ret = parse_epic_service_init(&ctx, &epic_name, &epic_class, &epic_unit); + if (ret) { + dev_err(ep->dcp->dev, + "AFK[ep:%02x]: failed to extract init props: %d\n", + ep->endpoint, ret); + return; + } + service_name = epic_class; + } else { + service_name = name; + } + + if (ep->match_epic_name) + ops = afk_match_service(ep, epic_name); + else + ops = afk_match_service(ep, service_name); + + if (!ops) { + dev_err(ep->dcp->dev, + "AFK[ep:%02x]: unable to match service %s on channel %d\n", + ep->endpoint, service_name, channel); + goto free; + } + + ch_idx = ep->num_channels++; + spin_lock_init(&ep->services[ch_idx].lock); + ep->services[ch_idx].enabled = true; + ep->services[ch_idx].torndown = false; + ep->services[ch_idx].ops = ops; + ep->services[ch_idx].ep = ep; + ep->services[ch_idx].channel = channel; + ep->services[ch_idx].cmd_tag = 0; + ops->init(&ep->services[ch_idx], epic_name, epic_class, epic_unit); + dev_info(ep->dcp->dev, "AFK[ep:%02x]: new service %s on channel %d\n", + ep->endpoint, service_name, channel); + + afk_populate_service_debugfs(&ep->services[ch_idx]); + +free: + kfree(epic_name); + kfree(epic_class); +} + +static void afk_recv_handle_teardown(struct apple_dcp_afkep *ep, u32 channel) +{ + struct apple_epic_service *service; + const struct apple_epic_service_ops *ops; + unsigned long flags; + + service = afk_epic_find_service(ep, channel); + if (!service) { + dev_warn(ep->dcp->dev, "AFK[ep:%02x]: teardown for disabled channel %u\n", + ep->endpoint, channel); + return; + } + + afk_remove_service_debugfs(service); + + // TODO: think through what locking is necessary + spin_lock_irqsave(&service->lock, flags); + /* + * teardown must not disable the service since since it may be sent as + * side effect of a COMMAND which for which a reply is expected. + * Seen with DCP's "av" endpoint during the close afk_service_call. + */ + service->torndown = true; + ops = service->ops; + spin_unlock_irqrestore(&service->lock, flags); + + if (ops->teardown) + ops->teardown(service); +} + +static void afk_recv_handle_reply(struct apple_dcp_afkep *ep, u32 channel, + u16 tag, void *payload, size_t payload_size) +{ + struct epic_cmd *cmd = payload; + struct apple_epic_service *service; + unsigned long flags; + u8 idx = tag & 0xff; + void *rxbuf, *txbuf; + dma_addr_t rxbuf_dma, txbuf_dma; + size_t rxlen, txlen; + + service = afk_epic_find_service(ep, channel); + if (!service) { + dev_warn(ep->dcp->dev, "AFK[ep:%02x]: command reply on disabled channel %u\n", + ep->endpoint, channel); + return; + } + + if (payload_size < sizeof(*cmd)) { + dev_err(ep->dcp->dev, + "AFK[ep:%02x]: command reply on channel %d too small: %ld\n", + ep->endpoint, channel, payload_size); + return; + } + + if (idx >= MAX_PENDING_CMDS) { + dev_err(ep->dcp->dev, + "AFK[ep:%02x]: command reply on channel %d out of range: %d\n", + ep->endpoint, channel, idx); + return; + } + + spin_lock_irqsave(&service->lock, flags); + if (service->cmds[idx].done) { + dev_err(ep->dcp->dev, + "AFK[ep:%02x]: command reply on channel %d already handled\n", + ep->endpoint, channel); + spin_unlock_irqrestore(&service->lock, flags); + return; + } + + if (tag != service->cmds[idx].tag) { + dev_err(ep->dcp->dev, + "AFK[ep:%02x]: command reply on channel %d has invalid tag: expected 0x%04x != 0x%04x\n", + ep->endpoint, channel, tag, service->cmds[idx].tag); + spin_unlock_irqrestore(&service->lock, flags); + return; + } + + service->cmds[idx].done = true; + service->cmds[idx].retcode = le32_to_cpu(cmd->retcode); + if (service->cmds[idx].free_on_ack) { + /* defer freeing until we're no longer in atomic context */ + rxbuf = service->cmds[idx].rxbuf; + txbuf = service->cmds[idx].txbuf; + rxlen = service->cmds[idx].rxlen; + txlen = service->cmds[idx].txlen; + rxbuf_dma = service->cmds[idx].rxbuf_dma; + txbuf_dma = service->cmds[idx].txbuf_dma; + bitmap_release_region(service->cmd_map, idx, 0); + } else { + rxbuf = txbuf = NULL; + rxlen = txlen = 0; + } + if (service->cmds[idx].completion) + complete(service->cmds[idx].completion); + + spin_unlock_irqrestore(&service->lock, flags); + + if (rxbuf && rxlen) + dma_free_coherent(ep->dcp->dev, rxlen, rxbuf, rxbuf_dma); + if (txbuf && txlen) + dma_free_coherent(ep->dcp->dev, txlen, txbuf, txbuf_dma); +} + +struct epic_std_service_ap_call { + __le32 unk0; + __le32 unk1; + __le32 type; + __le32 len; + __le32 magic; + u8 _unk[48]; +} __attribute__((packed)); + +static void afk_recv_handle_std_service(struct apple_dcp_afkep *ep, u32 channel, + u32 type, struct epic_hdr *ehdr, + struct epic_sub_hdr *eshdr, + void *payload, size_t payload_size) +{ + struct apple_epic_service *service = afk_epic_find_service(ep, channel); + + if (!service) { + dev_warn(ep->dcp->dev, + "AFK[ep:%02x]: std service notify on disabled channel %u\n", + ep->endpoint, channel); + return; + } + if (service->torndown) { + dev_warn(ep->dcp->dev, + "AFK[ep:%02x]: std service notify on torn down service " + "(chan:%u)\n", ep->endpoint, channel); + return; + } + + if (type == EPIC_TYPE_NOTIFY && eshdr->category == EPIC_CAT_NOTIFY) { + struct epic_std_service_ap_call *call = payload; + size_t call_size; + void *reply; + int ret; + + if (payload_size < sizeof(*call)) + return; + + call_size = le32_to_cpu(call->len); + if (payload_size < sizeof(*call) + call_size) + return; + + if (!service->ops->call) + return; + reply = kzalloc(payload_size, GFP_KERNEL); + if (!reply) + return; + + ret = service->ops->call(service, le32_to_cpu(call->type), + payload + sizeof(*call), call_size, + reply + sizeof(*call), call_size); + if (ret) { + kfree(reply); + return; + } + + memcpy(reply, call, sizeof(*call)); + afk_send_epic(ep, channel, le16_to_cpu(eshdr->tag), + EPIC_TYPE_NOTIFY_ACK, EPIC_CAT_REPLY, + EPIC_SUBTYPE_STD_SERVICE, reply, payload_size); + kfree(reply); + + return; + } + + if (type == EPIC_TYPE_NOTIFY && eshdr->category == EPIC_CAT_REPORT) { + if (service->ops->report) + service->ops->report(service, le16_to_cpu(eshdr->type), + payload, payload_size); + return; + } + + dev_err(ep->dcp->dev, + "AFK[ep:%02x]: channel %d received unhandled standard service message: %x / %x\n", + ep->endpoint, channel, type, eshdr->category); + print_hex_dump(KERN_INFO, "AFK: ", DUMP_PREFIX_NONE, 16, 1, payload, + payload_size, true); +} + +static void afk_recv_handle(struct apple_dcp_afkep *ep, u32 channel, u32 type, + u8 *data, size_t data_size) +{ + struct apple_epic_service *service; + struct epic_hdr *ehdr = (struct epic_hdr *)data; + struct epic_sub_hdr *eshdr = + (struct epic_sub_hdr *)(data + sizeof(*ehdr)); + u16 subtype = le16_to_cpu(eshdr->type); + u8 *payload = data + sizeof(*ehdr) + sizeof(*eshdr); + size_t payload_size; + + if (data_size < sizeof(*ehdr) + sizeof(*eshdr)) { + dev_err(ep->dcp->dev, "AFK[ep:%02x]: payload too small: %lx\n", + ep->endpoint, data_size); + return; + } + payload_size = data_size - sizeof(*ehdr) - sizeof(*eshdr); + + trace_afk_recv_handle(ep, channel, type, data_size, ehdr, eshdr); + + service = afk_epic_find_service(ep, channel); + + if (!service) { + if (type != EPIC_TYPE_NOTIFY && type != EPIC_TYPE_REPLY) { + dev_err(ep->dcp->dev, + "AFK[ep:%02x]: expected notify but got 0x%x on channel %d\n", + ep->endpoint, type, channel); + return; + } + if (eshdr->category != EPIC_CAT_REPORT) { + dev_err(ep->dcp->dev, + "AFK[ep:%02x]: expected report but got 0x%x on channel %d\n", + ep->endpoint, eshdr->category, channel); + return; + } + if (subtype == EPIC_SUBTYPE_TEARDOWN) { + dev_dbg(ep->dcp->dev, + "AFK[ep:%02x]: teardown without service on channel %d\n", + ep->endpoint, channel); + return; + } + if (subtype != EPIC_SUBTYPE_ANNOUNCE) { + dev_err(ep->dcp->dev, + "AFK[ep:%02x]: expected announce but got 0x%x on channel %d\n", + ep->endpoint, subtype, channel); + return; + } + + return afk_recv_handle_init(ep, channel, payload, payload_size); + } + + if (!service) { + dev_err(ep->dcp->dev, "AFK[ep:%02x]: channel %d has no service\n", + ep->endpoint, channel); + return; + } + + if (type == EPIC_TYPE_NOTIFY && eshdr->category == EPIC_CAT_REPORT && + subtype == EPIC_SUBTYPE_TEARDOWN) + return afk_recv_handle_teardown(ep, channel); + + if (type == EPIC_TYPE_REPLY && eshdr->category == EPIC_CAT_REPLY) + return afk_recv_handle_reply(ep, channel, + le16_to_cpu(eshdr->tag), payload, + payload_size); + + if (subtype == EPIC_SUBTYPE_STD_SERVICE) + return afk_recv_handle_std_service( + ep, channel, type, ehdr, eshdr, payload, payload_size); + + dev_err(ep->dcp->dev, "AFK[ep:%02x]: channel %d received unhandled message " + "(type %x subtype %x)\n", ep->endpoint, channel, type, subtype); + print_hex_dump(KERN_INFO, "AFK: ", DUMP_PREFIX_NONE, 16, 1, payload, + payload_size, true); +} + +static bool afk_recv(struct apple_dcp_afkep *ep) +{ + struct afk_qe *hdr; + u32 rptr, wptr; + u32 magic, size, channel, type; + + if (!ep->rxbfr.ready) { + dev_err(ep->dcp->dev, "AFK[ep:%02x]: got RECV but not ready\n", + ep->endpoint); + return false; + } + + rptr = le32_to_cpu(ep->rxbfr.hdr->rptr); + wptr = le32_to_cpu(ep->rxbfr.hdr->wptr); + trace_afk_recv_rwptr_pre(ep, rptr, wptr); + + if (rptr == wptr) + return false; + + if (rptr > (ep->rxbfr.bufsz - sizeof(*hdr))) { + dev_warn(ep->dcp->dev, + "AFK[ep:%02x]: rptr out of bounds: 0x%x > 0x%lx\n", + ep->endpoint, rptr, ep->rxbfr.bufsz - sizeof(*hdr)); + return false; + } + + dma_rmb(); + + hdr = ep->rxbfr.buf + rptr; + magic = le32_to_cpu(hdr->magic); + size = le32_to_cpu(hdr->size); + trace_afk_recv_qe(ep, rptr, magic, size); + + if (magic != QE_MAGIC) { + dev_warn(ep->dcp->dev, "AFK[ep:%02x]: invalid queue entry magic: 0x%x\n", + ep->endpoint, magic); + return false; + } + + /* + * If there's not enough space for the payload the co-processor inserted + * the current dummy queue entry and we have to advance to the next one + * which will contain the real data. + */ + if (rptr + size + sizeof(*hdr) > ep->rxbfr.bufsz) { + rptr = 0; + hdr = ep->rxbfr.buf + rptr; + magic = le32_to_cpu(hdr->magic); + size = le32_to_cpu(hdr->size); + trace_afk_recv_qe(ep, rptr, magic, size); + + if (magic != QE_MAGIC) { + dev_warn(ep->dcp->dev, + "AFK[ep:%02x]: invalid next queue entry magic: 0x%x\n", + ep->endpoint, magic); + return false; + } + + ep->rxbfr.hdr->rptr = cpu_to_le32(rptr); + } + + if (rptr + size + sizeof(*hdr) > ep->rxbfr.bufsz) { + dev_warn(ep->dcp->dev, + "AFK[ep:%02x]: queue entry out of bounds: 0x%lx > 0x%lx\n", + ep->endpoint, rptr + size + sizeof(*hdr), ep->rxbfr.bufsz); + return false; + } + + channel = le32_to_cpu(hdr->channel); + type = le32_to_cpu(hdr->type); + + rptr = ALIGN(rptr + sizeof(*hdr) + size, 1 << BLOCK_SHIFT); + if (WARN_ON(rptr > ep->rxbfr.bufsz)) + rptr = 0; + if (rptr == ep->rxbfr.bufsz) + rptr = 0; + + dma_mb(); + + ep->rxbfr.hdr->rptr = cpu_to_le32(rptr); + trace_afk_recv_rwptr_post(ep, rptr, wptr); + + /* + * TODO: this is theoretically unsafe since DCP could overwrite data + * after the read pointer was updated above. Do it anyway since + * it avoids 2 problems in the DCP tracer: + * 1. the tracer sees replies before the notifies from dcp + * 2. the tracer tries to read buffers after they are unmapped. + */ + afk_recv_handle(ep, channel, type, hdr->data, size); + + return true; +} + +static void afk_receive_message_worker(struct work_struct *work_) +{ + struct afk_receive_message_work *work; + u16 type; + + work = container_of(work_, struct afk_receive_message_work, work); + + type = FIELD_GET(RBEP_TYPE, work->message); + switch (type) { + case RBEP_INIT_ACK: + break; + + case RBEP_START_ACK: + complete_all(&work->ep->started); + break; + + case RBEP_SHUTDOWN_ACK: + complete_all(&work->ep->stopped); + break; + + case RBEP_GETBUF: + afk_getbuf(work->ep, work->message); + break; + + case RBEP_INIT_TX: + afk_init_rxtx(work->ep, work->message, &work->ep->txbfr); + break; + + case RBEP_INIT_RX: + afk_init_rxtx(work->ep, work->message, &work->ep->rxbfr); + break; + + case RBEP_RECV: + while (afk_recv(work->ep)) + ; + break; + + default: + dev_err(work->ep->dcp->dev, + "Received unknown AFK message type: 0x%x\n", type); + } + + kfree(work); +} + +int afk_receive_message(struct apple_dcp_afkep *ep, u64 message) +{ + struct afk_receive_message_work *work; + + // TODO: comment why decoupling from rtkit thread is required here + work = kzalloc(sizeof(*work), GFP_KERNEL); + if (!work) + return -ENOMEM; + + work->ep = ep; + work->message = message; + INIT_WORK(&work->work, afk_receive_message_worker); + queue_work(ep->wq, &work->work); + + return 0; +} + +int afk_send_epic(struct apple_dcp_afkep *ep, u32 channel, u16 tag, + enum epic_type etype, enum epic_category ecat, u8 stype, + const void *payload, size_t payload_len) +{ + u32 rptr, wptr; + struct afk_qe *hdr, *hdr2; + struct epic_hdr *ehdr; + struct epic_sub_hdr *eshdr; + unsigned long flags; + size_t total_epic_size, total_size; + int ret; + + spin_lock_irqsave(&ep->lock, flags); + + dma_rmb(); + rptr = le32_to_cpu(ep->txbfr.hdr->rptr); + wptr = le32_to_cpu(ep->txbfr.hdr->wptr); + trace_afk_send_rwptr_pre(ep, rptr, wptr); + total_epic_size = sizeof(*ehdr) + sizeof(*eshdr) + payload_len; + total_size = sizeof(*hdr) + total_epic_size; + + hdr = hdr2 = NULL; + + /* + * We need to figure out how to place the entire headers and payload + * into the ring buffer: + * - If the write pointer is in front of the read pointer we just need + * enough space inbetween to store everything. + * - If the read pointer has already wrapper around the end of the + * buffer we can + * a) either store the entire payload at the writer pointer if + * there's enough space until the end, + * b) or just store the queue entry at the write pointer to indicate + * that we need to wrap to the start and then store the headers + * and the payload at the beginning of the buffer. The queue + * header has to be store twice in this case. + * In either case we have to ensure that there's always enough space + * so that we don't accidentally overwrite other buffers. + */ + if (wptr < rptr) { + /* + * If wptr < rptr we can't wrap around and only have to make + * sure that there's enough space for the entire payload. + */ + if (wptr + total_size > rptr) { + ret = -ENOMEM; + goto out; + } + + hdr = ep->txbfr.buf + wptr; + wptr += sizeof(*hdr); + } else { + /* We need enough space to place at least a queue entry */ + if (wptr + sizeof(*hdr) > ep->txbfr.bufsz) { + ret = -ENOMEM; + goto out; + } + + /* + * If we can place a single queue entry but not the full payload + * we need to place one queue entry at the end of the ring + * buffer and then another one together with the entire + * payload at the beginning. + */ + if (wptr + total_size > ep->txbfr.bufsz) { + /* + * Ensure there's space for the queue entry at the + * beginning + */ + if (sizeof(*hdr) > rptr) { + ret = -ENOMEM; + goto out; + } + + /* + * Place two queue entries to indicate we want to wrap + * around to the firmware. + */ + hdr = ep->txbfr.buf + wptr; + hdr2 = ep->txbfr.buf; + wptr = sizeof(*hdr); + + /* Ensure there's enough space for the entire payload */ + if (wptr + total_epic_size > rptr) { + ret = -ENOMEM; + goto out; + } + } else { + /* We have enough space to place the entire payload */ + hdr = ep->txbfr.buf + wptr; + wptr += sizeof(*hdr); + } + } + /* + * At this point we're guaranteed that hdr (and possibly hdr2) point + * to a buffer large enough to fit the queue entry and that we have + * enough space at wptr to store the payload. + */ + + hdr->magic = cpu_to_le32(QE_MAGIC); + hdr->size = cpu_to_le32(total_epic_size); + hdr->channel = cpu_to_le32(channel); + hdr->type = cpu_to_le32(etype); + if (hdr2) + memcpy(hdr2, hdr, sizeof(*hdr)); + + ehdr = ep->txbfr.buf + wptr; + memset(ehdr, 0, sizeof(*ehdr)); + ehdr->version = 2; + ehdr->seq = cpu_to_le16(ep->qe_seq++); + ehdr->timestamp = cpu_to_le64(0); + wptr += sizeof(*ehdr); + + eshdr = ep->txbfr.buf + wptr; + memset(eshdr, 0, sizeof(*eshdr)); + eshdr->length = cpu_to_le32(payload_len); + eshdr->version = 4; + eshdr->category = ecat; + eshdr->type = cpu_to_le16(stype); + eshdr->timestamp = cpu_to_le64(0); + eshdr->tag = cpu_to_le16(tag); + if (ecat == EPIC_CAT_REPLY) + eshdr->inline_len = cpu_to_le16(payload_len - 4); + else + eshdr->inline_len = cpu_to_le16(0); + wptr += sizeof(*eshdr); + + memcpy(ep->txbfr.buf + wptr, payload, payload_len); + wptr += payload_len; + wptr = ALIGN(wptr, 1 << BLOCK_SHIFT); + if (wptr == ep->txbfr.bufsz) + wptr = 0; + trace_afk_send_rwptr_post(ep, rptr, wptr); + + ep->txbfr.hdr->wptr = cpu_to_le32(wptr); + afk_send(ep, FIELD_PREP(RBEP_TYPE, RBEP_SEND) | + FIELD_PREP(SEND_WPTR, wptr)); + ret = 0; + +out: + spin_unlock_irqrestore(&ep->lock, flags); + return ret; +} + +int afk_send_command(struct apple_epic_service *service, u8 type, + const void *payload, size_t payload_len, void *output, + size_t output_len, u32 *retcode) +{ + struct epic_cmd cmd; + void *rxbuf, *txbuf; + dma_addr_t rxbuf_dma, txbuf_dma; + unsigned long flags; + int ret, idx; + u16 tag; + struct apple_dcp_afkep *ep = service->ep; + DECLARE_COMPLETION_ONSTACK(completion); + + rxbuf = dma_alloc_coherent(ep->dcp->dev, output_len, &rxbuf_dma, + GFP_KERNEL); + if (!rxbuf) + return -ENOMEM; + txbuf = dma_alloc_coherent(ep->dcp->dev, payload_len, &txbuf_dma, + GFP_KERNEL); + if (!txbuf) { + ret = -ENOMEM; + goto err_free_rxbuf; + } + + memcpy(txbuf, payload, payload_len); + + memset(&cmd, 0, sizeof(cmd)); + cmd.retcode = cpu_to_le32(0); + cmd.rxbuf = cpu_to_le64(rxbuf_dma); + cmd.rxlen = cpu_to_le32(output_len); + cmd.txbuf = cpu_to_le64(txbuf_dma); + cmd.txlen = cpu_to_le32(payload_len); + + spin_lock_irqsave(&service->lock, flags); + idx = bitmap_find_free_region(service->cmd_map, MAX_PENDING_CMDS, 0); + if (idx < 0) { + ret = -ENOSPC; + goto err_unlock; + } + + tag = (service->cmd_tag & 0xff) << 8; + tag |= idx & 0xff; + service->cmd_tag++; + + service->cmds[idx].tag = tag; + service->cmds[idx].rxbuf = rxbuf; + service->cmds[idx].txbuf = txbuf; + service->cmds[idx].rxbuf_dma = rxbuf_dma; + service->cmds[idx].txbuf_dma = txbuf_dma; + service->cmds[idx].rxlen = output_len; + service->cmds[idx].txlen = payload_len; + service->cmds[idx].free_on_ack = false; + service->cmds[idx].done = false; + service->cmds[idx].completion = &completion; + init_completion(&completion); + + spin_unlock_irqrestore(&service->lock, flags); + + ret = afk_send_epic(service->ep, service->channel, tag, + EPIC_TYPE_COMMAND, EPIC_CAT_COMMAND, type, &cmd, + sizeof(cmd)); + if (ret) + goto err_free_cmd; + + ret = wait_for_completion_timeout(&completion, + msecs_to_jiffies(MSEC_PER_SEC)); + + if (ret <= 0) { + spin_lock_irqsave(&service->lock, flags); + /* + * Check again while we're inside the lock to make sure + * the command wasn't completed just after + * wait_for_completion_timeout returned. + */ + if (!service->cmds[idx].done) { + service->cmds[idx].completion = NULL; + service->cmds[idx].free_on_ack = true; + spin_unlock_irqrestore(&service->lock, flags); + return -ETIMEDOUT; + } + spin_unlock_irqrestore(&service->lock, flags); + } + + ret = 0; + if (retcode) + *retcode = service->cmds[idx].retcode; + if (output && output_len) + memcpy(output, rxbuf, output_len); + +err_free_cmd: + spin_lock_irqsave(&service->lock, flags); + bitmap_release_region(service->cmd_map, idx, 0); +err_unlock: + spin_unlock_irqrestore(&service->lock, flags); + dma_free_coherent(ep->dcp->dev, payload_len, txbuf, txbuf_dma); +err_free_rxbuf: + dma_free_coherent(ep->dcp->dev, output_len, rxbuf, rxbuf_dma); + return ret; +} + +int afk_service_call(struct apple_epic_service *service, u16 group, u32 command, + const void *data, size_t data_len, size_t data_pad, + void *output, size_t output_len, size_t output_pad) +{ + struct epic_service_call *call; + void *bfr; + size_t bfr_len = max(data_len + data_pad, output_len + output_pad) + + sizeof(*call); + int ret; + u32 retcode; + u32 retlen; + + bfr = kzalloc(bfr_len, GFP_KERNEL); + if (!bfr) + return -ENOMEM; + + call = bfr; + + memset(call, 0, sizeof(*call)); + call->group = cpu_to_le16(group); + call->command = cpu_to_le32(command); + call->data_len = cpu_to_le32(data_len + data_pad); + call->magic = cpu_to_le32(EPIC_SERVICE_CALL_MAGIC); + + memcpy(bfr + sizeof(*call), data, data_len); + + ret = afk_send_command(service, EPIC_SUBTYPE_STD_SERVICE, bfr, bfr_len, + bfr, bfr_len, &retcode); + if (ret) + goto out; + if (retcode) { + ret = -EINVAL; + goto out; + } + if (le32_to_cpu(call->magic) != EPIC_SERVICE_CALL_MAGIC || + le16_to_cpu(call->group) != group || + le32_to_cpu(call->command) != command) { + ret = -EINVAL; + goto out; + } + + retlen = le32_to_cpu(call->data_len); + if (output_len < retlen) + retlen = output_len; + if (output && output_len) { + memset(output, 0, output_len); + memcpy(output, bfr + sizeof(*call), retlen); + } + +out: + kfree(bfr); + return ret; +} + +#if IS_ENABLED(CONFIG_DRM_APPLE_DEBUG) + +#define AFK_DEBUGFS_MAX_REPLY 8192 + +static ssize_t service_call_write_file(struct file *file, const char __user *user_buf, + size_t count, loff_t *ppos) +{ + struct apple_epic_service *srv = file->private_data; + void *buf; + int ret; + struct { + u32 group; + u32 command; + } call_info; + + if (count < sizeof(call_info)) + return -EINVAL; + if (!srv->debugfs.scratch) { + srv->debugfs.scratch = \ + devm_kzalloc(srv->ep->dcp->dev, AFK_DEBUGFS_MAX_REPLY, GFP_KERNEL); + if (!srv->debugfs.scratch) + return -ENOMEM; + } + + ret = copy_from_user(&call_info, user_buf, sizeof(call_info)); + if (ret == sizeof(call_info)) + return -EFAULT; + user_buf += sizeof(call_info); + count -= sizeof(call_info); + + buf = kmalloc(count, GFP_KERNEL); + if (!buf) + return -ENOMEM; + ret = copy_from_user(buf, user_buf, count); + if (ret == count) { + kfree(buf); + return -EFAULT; + } + + memset(srv->debugfs.scratch, 0, AFK_DEBUGFS_MAX_REPLY); + dma_mb(); + + ret = afk_service_call(srv, call_info.group, call_info.command, buf, count, 0, + srv->debugfs.scratch, AFK_DEBUGFS_MAX_REPLY, 0); + kfree(buf); + + if (ret < 0) + return ret; + + return count + sizeof(call_info); +} + +static ssize_t service_call_read_file(struct file *file, char __user *user_buf, + size_t count, loff_t *ppos) +{ + struct apple_epic_service *srv = file->private_data; + + if (!srv->debugfs.scratch) + return -EINVAL; + + return simple_read_from_buffer(user_buf, count, ppos, + srv->debugfs.scratch, AFK_DEBUGFS_MAX_REPLY); +} + +static const struct file_operations service_call_fops = { + .open = simple_open, + .write = service_call_write_file, + .read = service_call_read_file, +}; + +static ssize_t service_raw_call_write_file(struct file *file, const char __user *user_buf, + size_t count, loff_t *ppos) +{ + struct apple_epic_service *srv = file->private_data; + u32 retcode; + int ret; + + if (!srv->debugfs.scratch) { + srv->debugfs.scratch = \ + devm_kzalloc(srv->ep->dcp->dev, AFK_DEBUGFS_MAX_REPLY, GFP_KERNEL); + if (!srv->debugfs.scratch) + return -ENOMEM; + } + + memset(srv->debugfs.scratch, 0, AFK_DEBUGFS_MAX_REPLY); + ret = copy_from_user(srv->debugfs.scratch, user_buf, count); + if (ret == count) + return -EFAULT; + + ret = afk_send_command(srv, EPIC_SUBTYPE_STD_SERVICE, srv->debugfs.scratch, count, + srv->debugfs.scratch, AFK_DEBUGFS_MAX_REPLY, &retcode); + if (ret < 0) + return ret; + if (retcode) + return -EINVAL; + + return count; +} + +static ssize_t service_raw_call_read_file(struct file *file, char __user *user_buf, + size_t count, loff_t *ppos) +{ + struct apple_epic_service *srv = file->private_data; + + if (!srv->debugfs.scratch) + return -EINVAL; + + return simple_read_from_buffer(user_buf, count, ppos, + srv->debugfs.scratch, AFK_DEBUGFS_MAX_REPLY); +} + +static const struct file_operations service_raw_call_fops = { + .open = simple_open, + .write = service_raw_call_write_file, + .read = service_raw_call_read_file, +}; + +static void afk_populate_service_debugfs(struct apple_epic_service *srv) +{ + if (!srv->ep->debugfs_entry || !srv->ops) + return; + + if (strcmp(srv->ops->name, "DCPAVAudioInterface") == 0) { + srv->debugfs.entry = debugfs_create_dir(srv->ops->name, + srv->ep->debugfs_entry); + debugfs_create_file("call", 0600, srv->debugfs.entry, srv, + &service_call_fops); + debugfs_create_file("raw_call", 0600, srv->debugfs.entry, srv, + &service_raw_call_fops); + } +} + +static void afk_remove_service_debugfs(struct apple_epic_service *srv) +{ + if (srv->debugfs.entry) { + debugfs_remove_recursive(srv->debugfs.entry); + srv->debugfs.entry = NULL; + } +} + +#endif diff --git a/drivers/gpu/drm/apple/afk.h b/drivers/gpu/drm/apple/afk.h new file mode 100644 index 00000000000000..a339c00a2a0138 --- /dev/null +++ b/drivers/gpu/drm/apple/afk.h @@ -0,0 +1,204 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * AFK (Apple Firmware Kit) EPIC (EndPoint Interface Client) support + */ +/* Copyright 2022 Sven Peter */ + +#ifndef _DRM_APPLE_DCP_AFK_H +#define _DRM_APPLE_DCP_AFK_H + +#include +#include +#include + +#include "dcp.h" + +#define AFK_MAX_CHANNEL 16 +#define MAX_PENDING_CMDS 16 + +struct apple_epic_service_ops; +struct apple_dcp_afkep; + +struct epic_cmd_info { + u16 tag; + + void *rxbuf; + void *txbuf; + dma_addr_t rxbuf_dma; + dma_addr_t txbuf_dma; + size_t rxlen; + size_t txlen; + + u32 retcode; + bool done; + bool free_on_ack; + struct completion *completion; +}; + +struct apple_epic_service { + const struct apple_epic_service_ops *ops; + struct apple_dcp_afkep *ep; + + struct epic_cmd_info cmds[MAX_PENDING_CMDS]; + DECLARE_BITMAP(cmd_map, MAX_PENDING_CMDS); + u8 cmd_tag; + spinlock_t lock; + + u32 channel; + bool enabled; + bool torndown; + + void *cookie; + + struct { + struct dentry *entry; + u8 *scratch; + } debugfs; +}; + +enum epic_subtype; + +struct apple_epic_service_ops { + const char name[32]; + + void (*init)(struct apple_epic_service *service, const char *name, + const char *class, s64 unit); + int (*call)(struct apple_epic_service *service, u32 idx, + const void *data, size_t data_size, void *reply, + size_t reply_size); + int (*report)(struct apple_epic_service *service, enum epic_subtype type, + const void *data, size_t data_size); + void (*teardown)(struct apple_epic_service *service); +}; + +struct afk_ringbuffer_header { + __le32 bufsz; + u32 unk; + u32 _pad1[14]; + __le32 rptr; + u32 _pad2[15]; + __le32 wptr; + u32 _pad3[15]; +}; + +struct afk_qe { +#define QE_MAGIC 0x20504f49 // ' POI' + __le32 magic; + __le32 size; + __le32 channel; + __le32 type; + u8 data[]; +}; + +struct epic_hdr { + u8 version; + __le16 seq; + u8 _pad; + __le32 unk; + __le64 timestamp; +} __attribute__((packed)); + +struct epic_sub_hdr { + __le32 length; + u8 version; + u8 category; + __le16 type; + __le64 timestamp; + __le16 tag; + __le16 unk; + __le32 inline_len; +} __attribute__((packed)); + +struct epic_cmd { + __le32 retcode; + __le64 rxbuf; + __le64 txbuf; + __le32 rxlen; + __le32 txlen; + u8 rxcookie; + u8 txcookie; +} __attribute__((packed)); + +struct epic_service_call { + u8 _pad0[2]; + __le16 group; + __le32 command; + __le32 data_len; +#define EPIC_SERVICE_CALL_MAGIC 0x69706378 + __le32 magic; + u8 _pad1[48]; +} __attribute__((packed)); +static_assert(sizeof(struct epic_service_call) == 64); + +enum epic_type { + EPIC_TYPE_NOTIFY = 0, + EPIC_TYPE_COMMAND = 3, + EPIC_TYPE_REPLY = 4, + EPIC_TYPE_NOTIFY_ACK = 8, +}; + +enum epic_category { + EPIC_CAT_REPORT = 0x00, + EPIC_CAT_NOTIFY = 0x10, + EPIC_CAT_REPLY = 0x20, + EPIC_CAT_COMMAND = 0x30, +}; + +enum epic_subtype { + EPIC_SUBTYPE_ANNOUNCE = 0x30, + EPIC_SUBTYPE_TEARDOWN = 0x32, + EPIC_SUBTYPE_STD_SERVICE = 0xc0, +}; + +struct afk_ringbuffer { + bool ready; + struct afk_ringbuffer_header *hdr; + u32 rptr; + void *buf; + size_t bufsz; +}; + +struct apple_dcp_afkep { + struct apple_dcp *dcp; + + u32 endpoint; + struct workqueue_struct *wq; + + struct completion started; + struct completion stopped; + + void *bfr; + u16 bfr_tag; + size_t bfr_size; + dma_addr_t bfr_dma; + + struct afk_ringbuffer txbfr; + struct afk_ringbuffer rxbfr; + + spinlock_t lock; + u16 qe_seq; + + const struct apple_epic_service_ops *ops; + struct apple_epic_service services[AFK_MAX_CHANNEL]; + u32 num_channels; + + struct dentry *debugfs_entry; + + bool match_epic_name; +}; + +struct apple_dcp_afkep *afk_init(struct apple_dcp *dcp, u32 endpoint, + const struct apple_epic_service_ops *ops); +int afk_start(struct apple_dcp_afkep *ep); +void afk_shutdown(struct apple_dcp_afkep *ep); +int afk_receive_message(struct apple_dcp_afkep *ep, u64 message); +int afk_send_epic(struct apple_dcp_afkep *ep, u32 channel, u16 tag, + enum epic_type etype, enum epic_category ecat, u8 stype, + const void *payload, size_t payload_len); +int afk_send_command(struct apple_epic_service *service, u8 type, + const void *payload, size_t payload_len, void *output, + size_t output_len, u32 *retcode); +int afk_service_call(struct apple_epic_service *service, u16 group, u32 command, + const void *data, size_t data_len, size_t data_pad, + void *output, size_t output_len, size_t output_pad); +#endif diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c new file mode 100644 index 00000000000000..31b86e909014ad --- /dev/null +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -0,0 +1,699 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright 2021 Alyssa Rosenzweig */ +/* Based on meson driver which is + * Copyright (C) 2016 BayLibre, SAS + * Author: Neil Armstrong + * Copyright (C) 2015 Amlogic, Inc. All rights reserved. + * Copyright (C) 2014 Endless Mobile + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "dcp.h" +#include "plane.h" + +#define DRIVER_NAME "apple" +#define DRIVER_DESC "Apple display controller DRM driver" + +#define MAX_COPROCESSORS 3 + +struct apple_drm_private { + struct drm_device drm; +}; + +DEFINE_DRM_GEM_DMA_FOPS(apple_fops); + +#define DART_PAGE_SIZE 16384 + +static int apple_drm_gem_dumb_create(struct drm_file *file_priv, + struct drm_device *drm, + struct drm_mode_create_dumb *args) +{ + args->pitch = ALIGN(DIV_ROUND_UP(args->width * args->bpp, 8), 64); + args->size = round_up(args->pitch * args->height, DART_PAGE_SIZE); + + return drm_gem_dma_dumb_create_internal(file_priv, drm, args); +} + +static const struct drm_driver apple_drm_driver = { + DRM_GEM_DMA_DRIVER_OPS_WITH_DUMB_CREATE(apple_drm_gem_dumb_create), + DRM_FBDEV_DMA_DRIVER_OPS, + .name = DRIVER_NAME, + .desc = DRIVER_DESC, + .major = 1, + .minor = 0, + .driver_features = DRIVER_MODESET | DRIVER_GEM | DRIVER_ATOMIC | DRIVER_SYNCOBJ | DRIVER_SYNCOBJ_TIMELINE, + .fops = &apple_fops, +}; + +static enum drm_connector_status +apple_connector_detect(struct drm_connector *connector, bool force) +{ + struct apple_connector *apple_connector = to_apple_connector(connector); + + return apple_connector->connected ? connector_status_connected : + connector_status_disconnected; +} + +static void apple_connector_oob_hotplug(struct drm_connector *connector, + enum drm_connector_status status) +{ + struct apple_connector *apple_connector = to_apple_connector(connector); + + printk("#### oob_hotplug status:0x%x ####\n", (u32)status); + + if (status == connector_status_connected) + dcp_dptx_connect_oob(apple_connector->dcp, 0); + else if (status == connector_status_disconnected) + dcp_dptx_disconnect_oob(apple_connector->dcp, 0); + else + dev_err(&apple_connector->dcp->dev, "unexpected connector status" + ":0x%x in oob_hotplug event\n", (u32)status); +} + +static void apple_crtc_atomic_enable(struct drm_crtc *crtc, + struct drm_atomic_state *state) +{ + struct drm_crtc_state *crtc_state; + crtc_state = drm_atomic_get_new_crtc_state(state, crtc); + + if (crtc_state->active_changed && crtc_state->active) { + struct apple_crtc *apple_crtc = to_apple_crtc(crtc); + dcp_poweron(apple_crtc->dcp); + /* Force the CTM to be set on first swap */ + crtc_state->color_mgmt_changed = true; + } + + if (crtc_state->active) + dcp_crtc_atomic_modeset(crtc, state); +} + +static void apple_crtc_atomic_disable(struct drm_crtc *crtc, + struct drm_atomic_state *state) +{ + struct drm_crtc_state *crtc_state; + crtc_state = drm_atomic_get_new_crtc_state(state, crtc); + + if (crtc_state->active_changed && !crtc_state->active) { + struct apple_crtc *apple_crtc = to_apple_crtc(crtc); + dcp_poweroff(apple_crtc->dcp); + } + + if (crtc->state->event && !crtc->state->active) { + spin_lock_irq(&crtc->dev->event_lock); + drm_crtc_send_vblank_event(crtc, crtc->state->event); + spin_unlock_irq(&crtc->dev->event_lock); + + crtc->state->event = NULL; + } +} + +static void apple_crtc_atomic_begin(struct drm_crtc *crtc, + struct drm_atomic_state *state) +{ + struct apple_crtc *apple_crtc = to_apple_crtc(crtc); + unsigned long flags; + + if (crtc->state->event) { + spin_lock_irqsave(&crtc->dev->event_lock, flags); + apple_crtc->event = crtc->state->event; + spin_unlock_irqrestore(&crtc->dev->event_lock, flags); + crtc->state->event = NULL; + } +} + +static void apple_crtc_cleanup(struct drm_crtc *crtc) +{ + drm_crtc_cleanup(crtc); + kfree(to_apple_crtc(crtc)); +} + +static int apple_crtc_parse_crc_source(const char *source, bool *enabled) +{ + int ret = 0; + + if (!source) { + *enabled = false; + } else if (strcmp(source, "auto") == 0) { + *enabled = true; + } else { + *enabled = false; + ret = -EINVAL; + } + + return ret; +} + +static int apple_crtc_set_crc_source(struct drm_crtc *crtc, const char *source) +{ + bool enabled = false; + + int ret = apple_crtc_parse_crc_source(source, &enabled); + + if (!ret) + dcp_set_crc(crtc, enabled); + + return ret; +} + +static int apple_crtc_verify_crc_source(struct drm_crtc *crtc, + const char *source, + size_t *values_cnt) +{ + bool enabled; + + if (apple_crtc_parse_crc_source(source, &enabled) < 0) { + pr_warn("dcp: Invalid CRC source name %s\n", source); + return -EINVAL; + } + + *values_cnt = 1; + + return 0; +} + +static const char * const apple_crtc_crc_sources[] = {"auto"}; + +static const char *const * apple_crtc_get_crc_sources(struct drm_crtc *crtc, + size_t *count) +{ + *count = ARRAY_SIZE(apple_crtc_crc_sources); + return apple_crtc_crc_sources; +} + +static const struct drm_crtc_funcs apple_crtc_funcs = { + .atomic_destroy_state = drm_atomic_helper_crtc_destroy_state, + .atomic_duplicate_state = drm_atomic_helper_crtc_duplicate_state, + .destroy = apple_crtc_cleanup, + .page_flip = drm_atomic_helper_page_flip, + .reset = drm_atomic_helper_crtc_reset, + .set_config = drm_atomic_helper_set_config, + .set_crc_source = apple_crtc_set_crc_source, + .verify_crc_source = apple_crtc_verify_crc_source, + .get_crc_sources = apple_crtc_get_crc_sources, + +}; + +static const struct drm_mode_config_funcs apple_mode_config_funcs = { + .atomic_check = drm_atomic_helper_check, + .atomic_commit = drm_atomic_helper_commit, + .fb_create = drm_gem_fb_create, +}; + +static const struct drm_mode_config_helper_funcs apple_mode_config_helpers = { + .atomic_commit_tail = drm_atomic_helper_commit_tail_rpm, +}; + +static void appledrm_connector_cleanup(struct drm_connector *connector) +{ + drm_connector_cleanup(connector); + kfree(to_apple_connector(connector)); +} + +static const struct drm_connector_funcs apple_connector_funcs = { + .fill_modes = drm_helper_probe_single_connector_modes, + .destroy = appledrm_connector_cleanup, + .reset = drm_atomic_helper_connector_reset, + .atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state, + .atomic_destroy_state = drm_atomic_helper_connector_destroy_state, + .detect = apple_connector_detect, + .debugfs_init = apple_connector_debugfs_init, + .oob_hotplug_event = apple_connector_oob_hotplug, +}; + +static const struct drm_connector_helper_funcs apple_connector_helper_funcs = { + .get_modes = dcp_get_modes, + .mode_valid = dcp_mode_valid, +}; + +static const struct drm_crtc_helper_funcs apple_crtc_helper_funcs = { + .atomic_begin = apple_crtc_atomic_begin, + .atomic_check = dcp_crtc_atomic_check, + .atomic_flush = dcp_flush, + .atomic_enable = apple_crtc_atomic_enable, + .atomic_disable = apple_crtc_atomic_disable, + .mode_fixup = dcp_crtc_mode_fixup, +}; + +static int apple_probe_per_dcp(struct device *dev, + struct drm_device *drm, + struct platform_device *dcp, + int num, bool dcp_ext) +{ + struct apple_crtc *crtc; + struct apple_connector *connector; + struct apple_encoder *enc; + struct drm_plane *planes[DCP_MAX_PLANES]; + int ret, i; + int immutable_zpos = 0; + bool supports_l10r = !dcp_fw_compat_is_12_x(dcp); + + planes[0] = apple_plane_init(drm, 1U << num, supports_l10r, + DRM_PLANE_TYPE_PRIMARY); + if (IS_ERR(planes[0])) + return PTR_ERR(planes[0]); + ret = drm_plane_create_zpos_immutable_property(planes[0], immutable_zpos); + if (ret) { + return ret; + } + + + /* Set up our other planes */ + for (i = 1; i < DCP_MAX_PLANES; i++) { + planes[i] = apple_plane_init(drm, 1U << num, supports_l10r, + DRM_PLANE_TYPE_OVERLAY); + if (IS_ERR(planes[i])) + return PTR_ERR(planes[i]); + immutable_zpos++; + ret = drm_plane_create_zpos_immutable_property(planes[i], immutable_zpos); + if (ret) { + return ret; + } + } + + /* + * Even though we have an overlay plane, we cannot expose it to legacy + * userspace for cursors as we cannot make the same guarantees as ye olde + * hardware cursor planes such userspace would expect us to. Modern userspace + * knows what to do with overlays. + */ + crtc = kzalloc(sizeof(*crtc), GFP_KERNEL); + ret = drm_crtc_init_with_planes(drm, &crtc->base, planes[0], NULL, + &apple_crtc_funcs, NULL); + if (ret) + return ret; + + drm_crtc_helper_add(&crtc->base, &apple_crtc_helper_funcs); + drm_crtc_enable_color_mgmt(&crtc->base, 0, true, 0); + + enc = drmm_simple_encoder_alloc(drm, struct apple_encoder, base, + DRM_MODE_ENCODER_TMDS); + if (IS_ERR(enc)) + return PTR_ERR(enc); + enc->base.possible_crtcs = drm_crtc_mask(&crtc->base); + + connector = kzalloc(sizeof(*connector), GFP_KERNEL); + mutex_init(&connector->chunk_lock); + drm_connector_helper_add(&connector->base, + &apple_connector_helper_funcs); + + // HACK: + if (dcp_ext) + connector->base.fwnode = fwnode_handle_get(dcp->dev.fwnode); + + ret = drm_connector_init(drm, &connector->base, &apple_connector_funcs, + dcp_get_connector_type(dcp)); + if (ret) + return ret; + + ret = drm_connector_attach_vrr_capable_property(&connector->base); + if (ret) + return ret; + + connector->base.polled = DRM_CONNECTOR_POLL_HPD; + connector->connected = false; + connector->dcp = dcp; + + INIT_WORK(&connector->hotplug_wq, dcp_hotplug); + + crtc->dcp = dcp; + dcp_link(dcp, crtc, connector); + + return drm_connector_attach_encoder(&connector->base, &enc->base); +} + +static int apple_get_fb_resource(struct device *dev, const char *name, + struct resource *fb_r) +{ + int idx, ret = -ENODEV; + struct device_node *node; + + idx = of_property_match_string(dev->of_node, "memory-region-names", name); + + node = of_parse_phandle(dev->of_node, "memory-region", idx); + if (!node) { + dev_err(dev, "reserved-memory node '%s' not found\n", name); + return -ENODEV; + } + + if (!of_device_is_available(node)) { + dev_err(dev, "reserved-memory node '%s' is unavailable\n", name); + goto err; + } + + if (!of_device_is_compatible(node, "framebuffer")) { + dev_err(dev, "reserved-memory node '%s' is incompatible\n", + node->full_name); + goto err; + } + + ret = of_address_to_resource(node, 0, fb_r); + +err: + of_node_put(node); + return ret; +} + +static const struct of_device_id apple_dcp_id_tbl[] = { + { .compatible = "apple,dcp" }, + { .compatible = "apple,dcpext" }, + {}, +}; + +static int apple_drm_init_dcp(struct device *dev) +{ + struct apple_drm_private *apple = dev_get_drvdata(dev); + struct platform_device *dcp[MAX_COPROCESSORS]; + struct device_node *np; + u64 timeout; + int i, ret, num_dcp = 0; + + for_each_matching_node(np, apple_dcp_id_tbl) { + bool dcp_ext; + if (!of_device_is_available(np)) { + of_node_put(np); + continue; + } + dcp_ext = of_device_is_compatible(np, "apple,dcpext") || + of_property_present(np, "phys"); + + dcp[num_dcp] = of_find_device_by_node(np); + of_node_put(np); + if (!dcp[num_dcp]) + continue; + + device_link_add(dev, &dcp[num_dcp]->dev, DL_FLAG_AUTOREMOVE_SUPPLIER); + + ret = apple_probe_per_dcp(dev, &apple->drm, dcp[num_dcp], + num_dcp, dcp_ext); + if (ret) + continue; + + ret = dcp_start(dcp[num_dcp]); + if (ret) + continue; + + num_dcp++; + } + + if (num_dcp < 1) + return -ENODEV; + + /* + * Starting DPTX might take some time. + */ + timeout = get_jiffies_64() + msecs_to_jiffies(3000); + + for (i = 0; i < num_dcp; ++i) { + u64 jiffies = get_jiffies_64(); + u64 wait = time_after_eq64(jiffies, timeout) ? + 0 : + timeout - jiffies; + ret = dcp_wait_ready(dcp[i], wait); + /* There is nothing we can do if a dcp/dcpext does not boot + * (successfully). Ignoring it should not do any harm now. + * Needs to reevaluated when adding dcpext support. + */ + if (ret) + dev_warn(dev, "DCP[%d] not ready: %d\n", i, ret); + } + /* HACK: Wait for dcp* to settle before a modeset */ + msleep(100); + + return 0; +} + +static int apple_drm_init(struct device *dev) +{ + struct apple_drm_private *apple; + struct resource fb_r; + resource_size_t fb_size; + int ret; + + ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(42)); + if (ret) + return ret; + + ret = apple_get_fb_resource(dev, "framebuffer", &fb_r); + if (ret) + return ret; + + apple = devm_drm_dev_alloc(dev, &apple_drm_driver, + struct apple_drm_private, drm); + if (IS_ERR(apple)) + return PTR_ERR(apple); + + dev_set_drvdata(dev, apple); + + ret = component_bind_all(dev, apple); + if (ret) + return ret; + + ret = drmm_mode_config_init(&apple->drm); + if (ret) + goto err_unbind; + + /* + * IOMFB::UPPipeDCP_H13P::verify_surfaces produces the error "plane + * requires a minimum of 32x32 for the source buffer" if smaller + */ + apple->drm.mode_config.min_width = 32; + apple->drm.mode_config.min_height = 32; + + /* + * TODO: this is the max framebuffer size not the maximal supported + * output resolution. DCP reports the maximal framebuffer size take it + * from there. + * Hardcode it for now to the M1 Max DCP reported 'MaxSrcBufferWidth' + * and 'MaxSrcBufferHeight' of 16384. + */ + apple->drm.mode_config.max_width = 16384; + apple->drm.mode_config.max_height = 16384; + + apple->drm.mode_config.funcs = &apple_mode_config_funcs; + apple->drm.mode_config.helper_private = &apple_mode_config_helpers; + + ret = apple_drm_init_dcp(dev); + if (ret) + goto err_unbind; + + drm_mode_config_reset(&apple->drm); + + fb_size = fb_r.end - fb_r.start + 1; + ret = aperture_remove_conflicting_devices(fb_r.start, fb_size, + apple_drm_driver.name); + if (ret) { + dev_err(dev, "Failed remove fb: %d\n", ret); + goto err_unbind; + } + + ret = drm_dev_register(&apple->drm, 0); + if (ret) + goto err_unbind; + + drm_client_setup_with_fourcc(&apple->drm, DRM_FORMAT_XRGB8888); + + return 0; + +err_unbind: + component_unbind_all(dev, NULL); + return ret; +} + +static void apple_drm_uninit(struct device *dev) +{ + struct apple_drm_private *apple = dev_get_drvdata(dev); + + drm_dev_unregister(&apple->drm); + drm_atomic_helper_shutdown(&apple->drm); + + component_unbind_all(dev, NULL); + + dev_set_drvdata(dev, NULL); +} + +static int apple_drm_bind(struct device *dev) +{ + return apple_drm_init(dev); +} + +static void apple_drm_unbind(struct device *dev) +{ + apple_drm_uninit(dev); +} + +const struct component_master_ops apple_drm_ops = { + .bind = apple_drm_bind, + .unbind = apple_drm_unbind, +}; + +static int add_dcp_components(struct device *dev, + struct component_match **matchptr) +{ + struct device_node *np, *endpoint, *port; + int num = 0; + + for_each_matching_node(np, apple_dcp_id_tbl) { + if (of_device_is_available(np)) { + drm_of_component_match_add(dev, matchptr, + component_compare_of, np); + num++; + for_each_endpoint_of_node(np, endpoint) { + port = of_graph_get_remote_port_parent(endpoint); + if (!port) + continue; + +#if !IS_ENABLED(CONFIG_DRM_APPLE_AUDIO) + if (of_device_is_compatible(port, "apple,dpaudio")) { + of_node_put(port); + continue; + } +#endif + + /* + * The ATC phy driver is not part of the component + * collection for the Apple display-subsystem so + * ignore it here. + */ + if (of_device_is_compatible(port, "apple,t8103-atcphy")) { + of_node_put(port); + continue; + } + + if (of_device_is_available(port)) + drm_of_component_match_add(dev, matchptr, + component_compare_of, + port); + of_node_put(port); + } + } + of_node_put(np); + } + + return num; +} + +static int apple_platform_probe(struct platform_device *pdev) +{ + struct device *mdev = &pdev->dev; + struct component_match *match = NULL; + int num_dcp; + + /* add DCP components, handle less than 1 as probe error */ + num_dcp = add_dcp_components(mdev, &match); + if (num_dcp < 1) + return -ENODEV; + + return component_master_add_with_match(mdev, &apple_drm_ops, match); +} + +static void apple_platform_remove(struct platform_device *pdev) +{ + component_master_del(&pdev->dev, &apple_drm_ops); +} + +static const struct of_device_id of_match[] = { + { .compatible = "apple,display-subsystem" }, + {} +}; +MODULE_DEVICE_TABLE(of, of_match); + +#ifdef CONFIG_PM_SLEEP +static int apple_platform_suspend(struct device *dev) +{ + struct apple_drm_private *apple = dev_get_drvdata(dev); + + if (apple) + return drm_mode_config_helper_suspend(&apple->drm); + + return 0; +} + +static int apple_platform_resume(struct device *dev) +{ + struct apple_drm_private *apple = dev_get_drvdata(dev); + + if (apple) + drm_mode_config_helper_resume(&apple->drm); + + return 0; +} + +static const struct dev_pm_ops apple_platform_pm_ops = { + .suspend = apple_platform_suspend, + .resume = apple_platform_resume, +}; +#endif + +static struct platform_driver apple_platform_driver = { + .driver = { + .name = "apple-drm", + .of_match_table = of_match, +#ifdef CONFIG_PM_SLEEP + .pm = &apple_platform_pm_ops, +#endif + }, + .probe = apple_platform_probe, + .remove = apple_platform_remove, +}; + + + +static int __init appledrm_register(void) +{ + if (drm_firmware_drivers_only()) + return -ENODEV; + +#if IS_ENABLED(CONFIG_DRM_APPLE_AUDIO) + dcp_audio_register(); +#endif + dcp_register(); + platform_driver_register(&apple_platform_driver); + + return 0; +} + +static void __exit appledrm_unregister(void) +{ +#if IS_ENABLED(CONFIG_DRM_APPLE_AUDIO) + dcp_audio_unregister(); +#endif + dcp_unregister(); + platform_driver_unregister(&apple_platform_driver); +} + +module_init(appledrm_register); +module_exit(appledrm_unregister); + +MODULE_AUTHOR("Asahi Linux contributors"); +MODULE_DESCRIPTION(DRIVER_DESC); +MODULE_LICENSE("Dual MIT/GPL"); diff --git a/drivers/gpu/drm/apple/audio.c b/drivers/gpu/drm/apple/audio.c new file mode 100644 index 00000000000000..38718e2f56117b --- /dev/null +++ b/drivers/gpu/drm/apple/audio.c @@ -0,0 +1,776 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * DCP Audio Bits + * + * Copyright (C) The Asahi Linux Contributors + * + * TODO: + * - figure some nice identification of the sound card (in case + * there's many DCP instances) + */ + +#define DEBUG + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "av.h" +#include "dcp.h" +#include "audio.h" +#include "parser.h" + +#define DCPAUD_ELEMENTS_MAXSIZE 16384 +#define DCPAUD_PRODUCTATTRS_MAXSIZE 1024 + +struct dcp_audio { + struct device *dev; + struct device *dcp_dev; + struct device *dma_dev; + struct device_link *dma_link; + struct dma_chan *chan; + struct snd_card *card; + struct snd_jack *jack; + struct snd_pcm_substream *substream; + unsigned int open_cookie; + + struct mutex data_lock; + bool dcp_connected; /// dcp status keep for delayed initialization + bool connected; + unsigned int connection_cookie; + + struct snd_pcm_chmap_elem selected_chmap; + struct dcp_sound_cookie selected_cookie; + void *elements; + void *productattrs; + + struct snd_pcm_chmap *chmap_info; +}; + +static const struct snd_pcm_hardware dcp_pcm_hw = { + .info = SNDRV_PCM_INFO_MMAP | SNDRV_PCM_INFO_MMAP_VALID | + SNDRV_PCM_INFO_INTERLEAVED, + .formats = SNDRV_PCM_FMTBIT_S16_LE | SNDRV_PCM_FMTBIT_S20_LE | + SNDRV_PCM_FMTBIT_S24_LE | SNDRV_PCM_FMTBIT_S32_LE, + .rates = SNDRV_PCM_RATE_CONTINUOUS, + .rate_min = 0, + .rate_max = UINT_MAX, + .channels_min = 1, + .channels_max = 16, + .buffer_bytes_max = SIZE_MAX, + .period_bytes_min = 4096, /* TODO */ + .period_bytes_max = SIZE_MAX, + .periods_min = 2, + .periods_max = UINT_MAX, +}; + +static int dcpaud_read_remote_info(struct dcp_audio *dcpaud) +{ + int ret; + + ret = dcp_audiosrv_get_elements(dcpaud->dcp_dev, dcpaud->elements, + DCPAUD_ELEMENTS_MAXSIZE); + if (ret < 0) + return ret; + + ret = dcp_audiosrv_get_product_attrs(dcpaud->dcp_dev, dcpaud->productattrs, + DCPAUD_PRODUCTATTRS_MAXSIZE); + if (ret < 0) + return ret; + + return 0; +} + +static int dcpaud_interval_bitmask(struct snd_interval *i, + unsigned int mask) +{ + struct snd_interval range; + if (!mask) + return -EINVAL; + + snd_interval_any(&range); + range.min = __ffs(mask); + range.max = __fls(mask); + return snd_interval_refine(i, &range); +} + +extern const struct snd_pcm_hw_constraint_list snd_pcm_known_rates; + +static void dcpaud_fill_fmt_sieve(struct snd_pcm_hw_params *params, + struct dcp_sound_format_mask *sieve) +{ + struct snd_interval *c = hw_param_interval(params, + SNDRV_PCM_HW_PARAM_CHANNELS); + struct snd_interval *r = hw_param_interval(params, + SNDRV_PCM_HW_PARAM_RATE); + struct snd_mask *f = hw_param_mask(params, + SNDRV_PCM_HW_PARAM_FORMAT); + int i; + + sieve->nchans = GENMASK(c->max, c->min); + sieve->formats = f->bits[0] | ((u64) f->bits[1]) << 32; /* TODO: don't open-code */ + + for (i = 0; i < snd_pcm_known_rates.count; i++) { + unsigned int rate = snd_pcm_known_rates.list[i]; + + if (snd_interval_test(r, rate)) + sieve->rates |= 1u << i; + } +} + +static void dcpaud_consult_elements(struct dcp_audio *dcpaud, + struct snd_pcm_hw_params *params, + struct dcp_sound_format_mask *hits) +{ + struct dcp_sound_format_mask sieve; + struct dcp_parse_ctx elements = { + .dcp = dev_get_drvdata(dcpaud->dcp_dev), + .blob = dcpaud->elements + 4, + .len = DCPAUD_ELEMENTS_MAXSIZE - 4, + .pos = 0, + }; + + dcpaud_fill_fmt_sieve(params, &sieve); + dev_dbg(dcpaud->dev, "elements in: %llx %x %x\n", sieve.formats, sieve.nchans, sieve.rates); + parse_sound_constraints(&elements, &sieve, hits); + dev_dbg(dcpaud->dev, "elements out: %llx %x %x\n", hits->formats, hits->nchans, hits->rates); +} + +static int dcpaud_select_cookie(struct dcp_audio *dcpaud, + struct snd_pcm_hw_params *params) +{ + struct dcp_sound_format_mask sieve; + struct dcp_parse_ctx elements = { + .dcp = dev_get_drvdata(dcpaud->dcp_dev), + .blob = dcpaud->elements + 4, + .len = DCPAUD_ELEMENTS_MAXSIZE - 4, + .pos = 0, + }; + + dcpaud_fill_fmt_sieve(params, &sieve); + return parse_sound_mode(&elements, &sieve, &dcpaud->selected_chmap, + &dcpaud->selected_cookie); +} + +static int dcpaud_rule_channels(struct snd_pcm_hw_params *params, + struct snd_pcm_hw_rule *rule) +{ + struct dcp_audio *dcpaud = rule->private; + struct snd_interval *c = hw_param_interval(params, + SNDRV_PCM_HW_PARAM_CHANNELS); + struct dcp_sound_format_mask hits = {0, 0, 0}; + + dcpaud_consult_elements(dcpaud, params, &hits); + + return dcpaud_interval_bitmask(c, hits.nchans); +} + +static int dcpaud_refine_fmt_mask(struct snd_mask *m, u64 mask) +{ + struct snd_mask mask_mask; + + if (!mask) + return -EINVAL; + mask_mask.bits[0] = mask; + mask_mask.bits[1] = mask >> 32; + + return snd_mask_refine(m, &mask_mask); +} + +static int dcpaud_rule_format(struct snd_pcm_hw_params *params, + struct snd_pcm_hw_rule *rule) +{ + struct dcp_audio *dcpaud = rule->private; + struct snd_mask *f = hw_param_mask(params, + SNDRV_PCM_HW_PARAM_FORMAT); + struct dcp_sound_format_mask hits; + + dcpaud_consult_elements(dcpaud, params, &hits); + + return dcpaud_refine_fmt_mask(f, hits.formats); +} + +static int dcpaud_rule_rate(struct snd_pcm_hw_params *params, + struct snd_pcm_hw_rule *rule) +{ + struct dcp_audio *dcpaud = rule->private; + struct snd_interval *r = hw_param_interval(params, + SNDRV_PCM_HW_PARAM_RATE); + struct dcp_sound_format_mask hits; + + dcpaud_consult_elements(dcpaud, params, &hits); + + return snd_interval_rate_bits(r, hits.rates); +} + +static int dcpaud_init_dma(struct dcp_audio *dcpaud) +{ + struct dma_chan *chan; + if (dcpaud->chan) + return 0; + + chan = of_dma_request_slave_channel(dcpaud->dev->of_node, "tx"); + /* squelch dma channel request errors, the driver will try again alter */ + if (!chan) { + dev_warn(dcpaud->dev, "audio TX DMA channel request failed\n"); + return -ENXIO; + } else if (chan == ERR_PTR(-EPROBE_DEFER)) { + dev_info(dcpaud->dev, "audio TX DMA channel is not ready yet\n"); + return -ENXIO; + } else if (IS_ERR(chan)) { + dev_warn(dcpaud->dev, "audio TX DMA channel request failed: %ld\n", PTR_ERR(chan)); + return PTR_ERR(chan); + } + dcpaud->chan = chan; + + snd_pcm_set_managed_buffer(dcpaud->substream, SNDRV_DMA_TYPE_DEV_IRAM, + dcpaud->chan->device->dev, 1024 * 1024, + SIZE_MAX); + + return 0; +} + +static int dcp_pcm_open(struct snd_pcm_substream *substream) +{ + struct dcp_audio *dcpaud = substream->pcm->private_data; + struct snd_dmaengine_dai_dma_data dma_data = { + .flags = SND_DMAENGINE_PCM_DAI_FLAG_PACK, + }; + struct snd_pcm_hardware hw; + int ret; + + mutex_lock(&dcpaud->data_lock); + ret = dcpaud_init_dma(dcpaud); + if (ret < 0) + return ret; + + if (!dcpaud->connected) { + mutex_unlock(&dcpaud->data_lock); + return -ENXIO; + } + dcpaud->open_cookie = dcpaud->connection_cookie; + mutex_unlock(&dcpaud->data_lock); + + ret = dcpaud_read_remote_info(dcpaud); + if (ret < 0) + return ret; + + snd_pcm_hw_rule_add(substream->runtime, 0, SNDRV_PCM_HW_PARAM_FORMAT, + dcpaud_rule_format, dcpaud, + SNDRV_PCM_HW_PARAM_CHANNELS, SNDRV_PCM_HW_PARAM_RATE, -1); + snd_pcm_hw_rule_add(substream->runtime, 0, SNDRV_PCM_HW_PARAM_CHANNELS, + dcpaud_rule_channels, dcpaud, + SNDRV_PCM_HW_PARAM_FORMAT, SNDRV_PCM_HW_PARAM_RATE, -1); + snd_pcm_hw_rule_add(substream->runtime, 0, SNDRV_PCM_HW_PARAM_RATE, + dcpaud_rule_rate, dcpaud, + SNDRV_PCM_HW_PARAM_FORMAT, SNDRV_PCM_HW_PARAM_CHANNELS, -1); + + hw = dcp_pcm_hw; + hw.info = SNDRV_PCM_INFO_MMAP | SNDRV_PCM_INFO_MMAP_VALID | + SNDRV_PCM_INFO_INTERLEAVED; + hw.periods_min = 2; + hw.periods_max = UINT_MAX; + hw.period_bytes_min = 256; + hw.period_bytes_max = SIZE_MAX; // TODO dma_get_max_seg_size(dma_dev); + hw.buffer_bytes_max = SIZE_MAX; + hw.fifo_size = 16; + ret = snd_dmaengine_pcm_refine_runtime_hwparams(substream, &dma_data, + &hw, dcpaud->chan); + if (ret) + return ret; + substream->runtime->hw = hw; + + return snd_dmaengine_pcm_open(substream, dcpaud->chan); +} + +static int dcp_pcm_close(struct snd_pcm_substream *substream) +{ + struct dcp_audio *dcpaud = substream->pcm->private_data; + dcpaud->selected_chmap.channels = 0; + + return snd_dmaengine_pcm_close(substream); +} + +static int dcpaud_connection_up(struct dcp_audio *dcpaud) +{ + bool ret; + mutex_lock(&dcpaud->data_lock); + ret = dcpaud->connected && + dcpaud->open_cookie == dcpaud->connection_cookie; + mutex_unlock(&dcpaud->data_lock); + return ret; +} + +static int dcp_pcm_hw_params(struct snd_pcm_substream *substream, + struct snd_pcm_hw_params *params) +{ + struct dcp_audio *dcpaud = substream->pcm->private_data; + struct dma_slave_config slave_config; + struct dma_chan *chan = snd_dmaengine_pcm_get_chan(substream); + int ret; + + if (!dcpaud_connection_up(dcpaud)) + return -ENXIO; + + ret = dcpaud_select_cookie(dcpaud, params); + if (ret < 0) + return ret; + if (!ret) + return -EINVAL; + + memset(&slave_config, 0, sizeof(slave_config)); + ret = snd_hwparams_to_dma_slave_config(substream, params, &slave_config); + dev_info(dcpaud->dev, "snd_hwparams_to_dma_slave_config: %d\n", ret); + if (ret < 0) + return ret; + + slave_config.direction = DMA_MEM_TO_DEV; + /* + * The data entry from the DMA controller to the DPA peripheral + * is 32-bit wide no matter the actual sample size. + */ + slave_config.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; + + ret = dmaengine_slave_config(chan, &slave_config); + dev_info(dcpaud->dev, "dmaengine_slave_config: %d\n", ret); + return ret; +} + +static int dcp_pcm_hw_free(struct snd_pcm_substream *substream) +{ + struct dcp_audio *dcpaud = substream->pcm->private_data; + + if (!dcpaud_connection_up(dcpaud)) + return 0; + + return dcp_audiosrv_unprepare(dcpaud->dcp_dev); +} + +static int dcp_pcm_prepare(struct snd_pcm_substream *substream) +{ + struct dcp_audio *dcpaud = substream->pcm->private_data; + + if (!dcpaud_connection_up(dcpaud)) + return -ENXIO; + + return dcp_audiosrv_prepare(dcpaud->dcp_dev, + &dcpaud->selected_cookie); +} + +static int dcp_pcm_trigger(struct snd_pcm_substream *substream, int cmd) +{ + struct dcp_audio *dcpaud = substream->pcm->private_data; + int ret; + + switch (cmd) { + case SNDRV_PCM_TRIGGER_START: + case SNDRV_PCM_TRIGGER_RESUME: + if (!dcpaud_connection_up(dcpaud)) + return -ENXIO; + + WARN_ON(pm_runtime_get_sync(dcpaud->dev) < 0); + ret = dcp_audiosrv_startlink(dcpaud->dcp_dev, + &dcpaud->selected_cookie); + if (ret < 0) + return ret; + break; + + case SNDRV_PCM_TRIGGER_STOP: + case SNDRV_PCM_TRIGGER_SUSPEND: + break; + + default: + return -EINVAL; + } + + ret = snd_dmaengine_pcm_trigger(substream, cmd); + if (ret < 0) + return ret; + + switch (cmd) { + case SNDRV_PCM_TRIGGER_START: + case SNDRV_PCM_TRIGGER_RESUME: + break; + + case SNDRV_PCM_TRIGGER_STOP: + case SNDRV_PCM_TRIGGER_SUSPEND: + ret = dcp_audiosrv_stoplink(dcpaud->dcp_dev); + pm_runtime_mark_last_busy(dcpaud->dev); + __pm_runtime_put_autosuspend(dcpaud->dev); + if (ret < 0) + return ret; + break; + } + + return 0; +} + +struct snd_pcm_ops dcp_playback_ops = { + .open = dcp_pcm_open, + .close = dcp_pcm_close, + .hw_params = dcp_pcm_hw_params, + .hw_free = dcp_pcm_hw_free, + .prepare = dcp_pcm_prepare, + .trigger = dcp_pcm_trigger, + .pointer = snd_dmaengine_pcm_pointer, +}; + +// Transitional workaround: for the chmap control TLV, advertise options +// copied from hdmi-codec.c +#include "hdmi-codec-chmap.h" + +static int dcpaud_chmap_ctl_get(struct snd_kcontrol *kcontrol, + struct snd_ctl_elem_value *ucontrol) +{ + struct snd_pcm_chmap *info = snd_kcontrol_chip(kcontrol); + struct dcp_audio *dcpaud = info->private_data; + unsigned int i; + + for (i = 0; i < info->max_channels; i++) + ucontrol->value.integer.value[i] = \ + (i < dcpaud->selected_chmap.channels) ? + dcpaud->selected_chmap.map[i] : SNDRV_CHMAP_UNKNOWN; + + return 0; +} + + +static int dcpaud_create_chmap_ctl(struct dcp_audio *dcpaud) +{ + struct snd_pcm *pcm = dcpaud->substream->pcm; + struct snd_pcm_chmap *chmap_info; + int ret; + + ret = snd_pcm_add_chmap_ctls(pcm, SNDRV_PCM_STREAM_PLAYBACK, NULL, + dcp_pcm_hw.channels_max, 0, &chmap_info); + if (ret < 0) + return ret; + + chmap_info->kctl->get = dcpaud_chmap_ctl_get; + chmap_info->chmap = hdmi_codec_8ch_chmaps; + chmap_info->private_data = dcpaud; + + return 0; +} + +static int dcpaud_create_pcm(struct dcp_audio *dcpaud) +{ + struct snd_card *card = dcpaud->card; + struct snd_pcm *pcm; + int ret; + +#define NUM_PLAYBACK 1 +#define NUM_CAPTURE 0 + + ret = snd_pcm_new(card, card->shortname, 0, NUM_PLAYBACK, NUM_CAPTURE, &pcm); + if (ret) + return ret; + + snd_pcm_set_ops(pcm, SNDRV_PCM_STREAM_PLAYBACK, &dcp_playback_ops); + dcpaud->substream = pcm->streams[SNDRV_PCM_STREAM_PLAYBACK].substream; + pcm->nonatomic = true; + pcm->private_data = dcpaud; + strscpy(pcm->name, card->shortname, sizeof(pcm->name)); + + return 0; +} + +/* expects to be called with data_lock locked and unlocks it */ +static void dcpaud_report_hotplug(struct dcp_audio *dcpaud, bool connected) +{ + struct snd_pcm_substream *substream = dcpaud->substream; + + if (!dcpaud->card || dcpaud->connected == connected) { + mutex_unlock(&dcpaud->data_lock); + return; + } + + dcpaud->connected = connected; + if (connected) + dcpaud->connection_cookie++; + mutex_unlock(&dcpaud->data_lock); + + snd_jack_report(dcpaud->jack, connected ? SND_JACK_AVOUT : 0); + + if (!connected) { + snd_pcm_stream_lock(substream); + if (substream->runtime) + snd_pcm_stop(substream, SNDRV_PCM_STATE_DISCONNECTED); + snd_pcm_stream_unlock(substream); + } +} + +static int dcpaud_create_jack(struct dcp_audio *dcpaud) +{ + struct snd_card *card = dcpaud->card; + + return snd_jack_new(card, "HDMI/DP", SND_JACK_AVOUT, + &dcpaud->jack, true, false); +} + +static void dcpaud_set_card_names(struct dcp_audio *dcpaud) +{ + struct snd_card *card = dcpaud->card; + + strscpy(card->driver, "apple_dcp", sizeof(card->driver)); + strscpy(card->longname, "Apple DisplayPort", sizeof(card->longname)); + strscpy(card->shortname, "Apple DisplayPort", sizeof(card->shortname)); +} + +#ifdef CONFIG_SND_DEBUG +static void dcpaud_expose_debugfs_blob(struct dcp_audio *dcpaud, const char *name, void *base, size_t size) +{ + struct debugfs_blob_wrapper *wrapper; + wrapper = devm_kzalloc(dcpaud->dev, sizeof(*wrapper), GFP_KERNEL); + if (!wrapper) + return; + wrapper->data = base; + wrapper->size = size; + debugfs_create_blob(name, 0600, dcpaud->card->debugfs_root, wrapper); +} +#else +static void dcpaud_expose_debugfs_blob(struct dcp_audio *dcpaud, const char *name, void *base, size_t size) {} +#endif + +extern bool hdmi_audio; + +static int dcpaud_init_snd_card(struct dcp_audio *dcpaud) +{ + int ret; + if (!hdmi_audio) + return -ENODEV; + + + ret = snd_card_new(dcpaud->dev, SNDRV_DEFAULT_IDX1, SNDRV_DEFAULT_STR1, + THIS_MODULE, 0, &dcpaud->card); + if (ret) + return ret; + + dcpaud_set_card_names(dcpaud); + + ret = dcpaud_create_pcm(dcpaud); + if (ret) + goto err_free_card; + + ret = dcpaud_create_chmap_ctl(dcpaud); + if (ret) + goto err_free_card; + + ret = dcpaud_create_jack(dcpaud); + if (ret) + goto err_free_card; + + ret = snd_card_register(dcpaud->card); + if (ret) + goto err_free_card; + + return 0; +err_free_card: + dev_warn(dcpaud->dev, "Failed to initialize sound card: %d\n", ret); + snd_card_free(dcpaud->card); + dcpaud->card = NULL; + return ret; +} + +void dcpaud_connect(struct platform_device *pdev, bool connected) +{ + struct dcp_audio *dcpaud = platform_get_drvdata(pdev); + + mutex_lock(&dcpaud->data_lock); + + dcpaud_report_hotplug(dcpaud, connected); +} + +void dcpaud_disconnect(struct platform_device *pdev) +{ + struct dcp_audio *dcpaud = platform_get_drvdata(pdev); + + mutex_lock(&dcpaud->data_lock); + + dcpaud_report_hotplug(dcpaud, false); +} + +static int dcpaud_comp_bind(struct device *dev, struct device *main, void *data) +{ + struct dcp_audio *dcpaud = dev_get_drvdata(dev); + struct device_node *endpoint, *dcp_node = NULL; + struct platform_device *dcp_pdev, *dma_pdev; + struct of_phandle_args dma_spec; + int index; + int ret; + + pm_runtime_get_noresume(dev); + pm_runtime_set_active(dev); + + ret = devm_pm_runtime_enable(dev); + if (ret) + return dev_err_probe(dev, ret, "Failed to enable runtime PM: %d\n", ret); + + /* find linked DCP instance */ + endpoint = of_graph_get_endpoint_by_regs(dev->of_node, 0, 0); + if (endpoint) { + dcp_node = of_graph_get_remote_port_parent(endpoint); + of_node_put(endpoint); + } + if (!dcp_node || !of_device_is_available(dcp_node)) { + of_node_put(dcp_node); + dev_info(dev, "No audio support\n"); + goto rpm_put; + } + + index = of_property_match_string(dev->of_node, "dma-names", "tx"); + if (index < 0) { + dev_err(dev, "No dma-names property\n"); + goto rpm_put; + } + + if (of_parse_phandle_with_args(dev->of_node, "dmas", "#dma-cells", index, + &dma_spec) || !dma_spec.np) { + dev_err(dev, "Failed to parse dmas property\n"); + goto rpm_put; + } + + dcp_pdev = of_find_device_by_node(dcp_node); + of_node_put(dcp_node); + if (!dcp_pdev) { + dev_info(dev, "No DP/HDMI audio device, dcp not ready\n"); + goto rpm_put; + } + dcpaud->dcp_dev = &dcp_pdev->dev; + + dma_pdev = of_find_device_by_node(dma_spec.np); + of_node_put(dma_spec.np); + if (!dma_pdev) { + dev_info(dev, "No DMA device\n"); + goto rpm_put; + } + dcpaud->dma_dev = &dma_pdev->dev; + + dcpaud->dma_link = device_link_add(dev, dcpaud->dma_dev, + DL_FLAG_PM_RUNTIME | + DL_FLAG_RPM_ACTIVE | + DL_FLAG_STATELESS); + + /* ignore errors to prevent audio issues affecting the display side */ + ret = dcpaud_init_snd_card(dcpaud); + + if (!ret) { + dcpaud_expose_debugfs_blob(dcpaud, "selected_cookie", &dcpaud->selected_cookie, + sizeof(dcpaud->selected_cookie)); + dcpaud_expose_debugfs_blob(dcpaud, "elements", dcpaud->elements, + DCPAUD_ELEMENTS_MAXSIZE); + dcpaud_expose_debugfs_blob(dcpaud, "product_attrs", dcpaud->productattrs, + DCPAUD_PRODUCTATTRS_MAXSIZE); + } + +rpm_put: + pm_runtime_put(dev); + + return 0; +} + +static void dcpaud_comp_unbind(struct device *dev, struct device *main, + void *data) +{ + struct dcp_audio *dcpaud = dev_get_drvdata(dev); + + /* snd_card_free_when_closed() checks for NULL */ + snd_card_free_when_closed(dcpaud->card); + + if (dcpaud->dma_link) + device_link_del(dcpaud->dma_link); +} + +static const struct component_ops dcpaud_comp_ops = { + .bind = dcpaud_comp_bind, + .unbind = dcpaud_comp_unbind, +}; + +static int dcpaud_probe(struct platform_device *pdev) +{ + struct dcp_audio *dcpaud; + + dcpaud = devm_kzalloc(&pdev->dev, sizeof(*dcpaud), GFP_KERNEL); + if (!dcpaud) + return -ENOMEM; + + dcpaud->elements = devm_kzalloc(&pdev->dev, DCPAUD_ELEMENTS_MAXSIZE, + GFP_KERNEL); + if (!dcpaud->elements) + return -ENOMEM; + + dcpaud->productattrs = devm_kzalloc(&pdev->dev, DCPAUD_PRODUCTATTRS_MAXSIZE, + GFP_KERNEL); + if (!dcpaud->productattrs) + return -ENOMEM; + + dcpaud->dev = &pdev->dev; + mutex_init(&dcpaud->data_lock); + platform_set_drvdata(pdev, dcpaud); + + return component_add(&pdev->dev, &dcpaud_comp_ops); +} + +static void dcpaud_remove(struct platform_device *pdev) +{ + component_del(&pdev->dev, &dcpaud_comp_ops); +} + +static void dcpaud_shutdown(struct platform_device *pdev) +{ + component_del(&pdev->dev, &dcpaud_comp_ops); +} + +static __maybe_unused int dcpaud_suspend(struct device *dev) +{ + /* + * Using snd_power_change_state() does not work since the sound card + * is what resumes runtime PM. + */ + + return 0; +} + +static __maybe_unused int dcpaud_resume(struct device *dev) +{ + return 0; +} + +static DEFINE_RUNTIME_DEV_PM_OPS(dcpaud_pm_ops, dcpaud_suspend, dcpaud_resume, NULL); + +static const struct of_device_id dcpaud_of_match[] = { + { .compatible = "apple,dpaudio" }, + {} +}; + +static struct platform_driver dcpaud_driver = { + .driver = { + .name = "dcp-dp-audio", + .of_match_table = dcpaud_of_match, + .pm = pm_ptr(&dcpaud_pm_ops), + }, + .probe = dcpaud_probe, + .remove = dcpaud_remove, + .shutdown = dcpaud_shutdown, +}; + +void __init dcp_audio_register(void) +{ + platform_driver_register(&dcpaud_driver); +} + +void __exit dcp_audio_unregister(void) +{ + platform_driver_unregister(&dcpaud_driver); +} + diff --git a/drivers/gpu/drm/apple/audio.h b/drivers/gpu/drm/apple/audio.h new file mode 100644 index 00000000000000..83b990dc6c343f --- /dev/null +++ b/drivers/gpu/drm/apple/audio.h @@ -0,0 +1,20 @@ +#ifndef __AUDIO_H__ +#define __AUDIO_H__ + +#include + +struct device; +struct platform_device; +struct dcp_sound_cookie; + +int dcp_audiosrv_prepare(struct device *dev, struct dcp_sound_cookie *cookie); +int dcp_audiosrv_startlink(struct device *dev, struct dcp_sound_cookie *cookie); +int dcp_audiosrv_stoplink(struct device *dev); +int dcp_audiosrv_unprepare(struct device *dev); +int dcp_audiosrv_get_elements(struct device *dev, void *elements, size_t maxsize); +int dcp_audiosrv_get_product_attrs(struct device *dev, void *attrs, size_t maxsize); + +void dcpaud_connect(struct platform_device *pdev, bool connected); +void dcpaud_disconnect(struct platform_device *pdev); + +#endif /* __AUDIO_H__ */ diff --git a/drivers/gpu/drm/apple/av.c b/drivers/gpu/drm/apple/av.c new file mode 100644 index 00000000000000..0d3c752f62d5f5 --- /dev/null +++ b/drivers/gpu/drm/apple/av.c @@ -0,0 +1,433 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright 2023 Martin Povišer */ + +// #define DEBUG + +#include +#include +#include +#include +#include +#include +#include + +#include "audio.h" +#include "afk.h" +#include "av.h" +#include "dcp.h" +#include "dcp-internal.h" + +struct dcp_av_audio_cmds { + /* commands in group 0*/ + u32 open; + u32 close; + u32 prepare; + u32 start_link; + u32 stop_link; + u32 unprepare; + /* commands in group 1*/ + u32 get_elements; + u32 get_product_attrs; +}; + +static const struct dcp_av_audio_cmds dcp_av_audio_cmds_v12_3 = { + .open = 6, + .close = 7, + .prepare = 8, + .start_link = 9, + .stop_link = 12, + .unprepare = 13, + .get_elements = 18, + .get_product_attrs = 20, +}; + +static const struct dcp_av_audio_cmds dcp_av_audio_cmds_v13_5 = { + .open = 4, + .close = 5, + .prepare = 6, + .start_link = 7, + .stop_link = 10, + .unprepare = 11, + .get_elements = 16, + .get_product_attrs = 18, +}; + +struct audiosrv_data { + struct platform_device *audio_dev; + bool plugged; + struct mutex plug_lock; + + struct apple_epic_service *srv; + struct rw_semaphore srv_rwsem; + /* Workqueue for starting the audio service */ + struct work_struct start_av_service_wq; + + struct dcp_av_audio_cmds cmds; + + bool warned_get_elements; + bool warned_get_product_attrs; + bool is_open; +}; + +static void av_interface_init(struct apple_epic_service *service, const char *name, + const char *class, s64 unit) +{ +} + +static void av_interface_teardown(struct apple_epic_service *service) +{ + struct apple_dcp *dcp = service->ep->dcp; + struct audiosrv_data *asrv = dcp->audiosrv; + + service->enabled = false; + + mutex_lock(&asrv->plug_lock); + + asrv->plugged = false; + if (asrv->audio_dev) + dcpaud_disconnect(asrv->audio_dev); + + mutex_unlock(&asrv->plug_lock); +} + +static void av_audiosrv_init(struct apple_epic_service *service, const char *name, + const char *class, s64 unit) +{ + struct apple_dcp *dcp = service->ep->dcp; + struct audiosrv_data *asrv = dcp->audiosrv; + + mutex_lock(&asrv->plug_lock); + + down_write(&asrv->srv_rwsem); + asrv->srv = service; + up_write(&asrv->srv_rwsem); + + asrv->plugged = true; + mutex_unlock(&asrv->plug_lock); + schedule_work(&asrv->start_av_service_wq); +} + +static void av_audiosrv_teardown(struct apple_epic_service *service) +{ + struct apple_dcp *dcp = service->ep->dcp; + struct audiosrv_data *asrv = dcp->audiosrv; + + mutex_lock(&asrv->plug_lock); + + down_write(&asrv->srv_rwsem); + asrv->srv = NULL; + up_write(&asrv->srv_rwsem); + + asrv->plugged = false; + if (asrv->audio_dev) + dcpaud_disconnect(asrv->audio_dev); + + mutex_unlock(&asrv->plug_lock); +} + +int dcp_audiosrv_prepare(struct device *dev, struct dcp_sound_cookie *cookie) +{ + struct apple_dcp *dcp = dev_get_drvdata(dev); + struct audiosrv_data *asrv = dcp->audiosrv; + int ret; + + down_write(&asrv->srv_rwsem); + ret = afk_service_call(asrv->srv, 0, asrv->cmds.prepare, cookie, + sizeof(*cookie), 64 - sizeof(*cookie), NULL, 0, + 64); + up_write(&asrv->srv_rwsem); + + return ret; +} + +int dcp_audiosrv_startlink(struct device *dev, struct dcp_sound_cookie *cookie) +{ + struct apple_dcp *dcp = dev_get_drvdata(dev); + struct audiosrv_data *asrv = dcp->audiosrv; + int ret; + + down_write(&asrv->srv_rwsem); + ret = afk_service_call(asrv->srv, 0, asrv->cmds.start_link, cookie, + sizeof(*cookie), 64 - sizeof(*cookie), NULL, 0, + 64); + up_write(&asrv->srv_rwsem); + + return ret; +} + +int dcp_audiosrv_stoplink(struct device *dev) +{ + struct apple_dcp *dcp = dev_get_drvdata(dev); + struct audiosrv_data *asrv = dcp->audiosrv; + int ret; + + down_write(&asrv->srv_rwsem); + ret = afk_service_call(asrv->srv, 0, asrv->cmds.stop_link, NULL, 0, 64, + NULL, 0, 64); + up_write(&asrv->srv_rwsem); + + return ret; +} + +int dcp_audiosrv_unprepare(struct device *dev) +{ + struct apple_dcp *dcp = dev_get_drvdata(dev); + struct audiosrv_data *asrv = dcp->audiosrv; + int ret; + + down_write(&asrv->srv_rwsem); + ret = afk_service_call(asrv->srv, 0, asrv->cmds.unprepare, NULL, 0, 64, + NULL, 0, 64); + up_write(&asrv->srv_rwsem); + + return ret; +} + +static int +dcp_audiosrv_osobject_call(struct apple_epic_service *service, u16 group, + u32 command, void *output, size_t output_maxsize, + size_t *output_size) +{ + struct { + __le64 max_size; + u8 _pad1[24]; + __le64 used_size; + u8 _pad2[8]; + } __attribute__((packed)) *hdr; + static_assert(sizeof(*hdr) == 48); + size_t bfr_len = output_maxsize + sizeof(*hdr); + void *bfr; + int ret; + + bfr = kzalloc(bfr_len, GFP_KERNEL); + if (!bfr) + return -ENOMEM; + + hdr = bfr; + hdr->max_size = cpu_to_le64(output_maxsize); + ret = afk_service_call(service, group, command, hdr, sizeof(*hdr), output_maxsize, + bfr, sizeof(*hdr) + output_maxsize, 0); + if (ret) + return ret; + + if (output) + memcpy(output, bfr + sizeof(*hdr), output_maxsize); + + if (output_size) + *output_size = le64_to_cpu(hdr->used_size); + + return 0; +} + +int dcp_audiosrv_get_elements(struct device *dev, void *elements, size_t maxsize) +{ + struct apple_dcp *dcp = dev_get_drvdata(dev); + struct audiosrv_data *asrv = dcp->audiosrv; + size_t size; + int ret; + + down_write(&asrv->srv_rwsem); + ret = dcp_audiosrv_osobject_call(asrv->srv, 1, asrv->cmds.get_elements, + elements, maxsize, &size); + up_write(&asrv->srv_rwsem); + + if (ret && asrv->warned_get_elements) { + dev_err(dev, "audiosrv: error getting elements: %d\n", ret); + asrv->warned_get_elements = true; + } else { + dev_dbg(dev, "audiosrv: got %zd bytes worth of elements\n", size); + } + + return ret; +} + +int dcp_audiosrv_get_product_attrs(struct device *dev, void *attrs, size_t maxsize) +{ + struct apple_dcp *dcp = dev_get_drvdata(dev); + struct audiosrv_data *asrv = dcp->audiosrv; + size_t size; + int ret; + + down_write(&asrv->srv_rwsem); + ret = dcp_audiosrv_osobject_call(asrv->srv, 1, + asrv->cmds.get_product_attrs, attrs, + maxsize, &size); + up_write(&asrv->srv_rwsem); + + if (ret && asrv->warned_get_product_attrs) { + dev_err(dev, "audiosrv: error getting product attributes: %d\n", ret); + asrv->warned_get_product_attrs = true; + } else { + dev_dbg(dev, "audiosrv: got %zd bytes worth of product attributes\n", size); + } + + return ret; +} + +static int av_audiosrv_report(struct apple_epic_service *service, u32 idx, + const void *data, size_t data_size) +{ + dev_dbg(service->ep->dcp->dev, "got audio report %d size %zx\n", idx, data_size); +#ifdef DEBUG + print_hex_dump(KERN_DEBUG, "audio report: ", DUMP_PREFIX_NONE, 16, 1, data, data_size, true); +#endif + + return 0; +} + +static const struct apple_epic_service_ops avep_ops[] = { + { + .name = "DCPAVSimpleVideoInterface", + .init = av_interface_init, + .teardown = av_interface_teardown, + }, + { + .name = "DCPAVAudioInterface", + .init = av_audiosrv_init, + .report = av_audiosrv_report, + .teardown = av_audiosrv_teardown, + }, + {} +}; + +void av_service_connect(struct apple_dcp *dcp) +{ + struct apple_epic_service *service; + struct audiosrv_data *asrv = dcp->audiosrv; + int ret; + + scoped_guard(rwsem_write, &asrv->srv_rwsem) { + if (!asrv->srv) + return; + service = asrv->srv; + } + + /* open AV audio service */ + dev_info(dcp->dev, "%s: starting audio service, plugged:%d\n", __func__, asrv->plugged); + if (asrv->is_open) + return; + + ret = afk_service_call(service, 0, asrv->cmds.open, NULL, 0, 32, + NULL, 0, 32); + if (ret) { + dev_err(dcp->dev, "error opening audio service: %d\n", ret); + return; + } + mutex_lock(&asrv->plug_lock); + asrv->is_open = true; + + if (asrv->audio_dev) + dcpaud_connect(asrv->audio_dev, asrv->plugged); + mutex_unlock(&asrv->plug_lock); +} + +void av_service_disconnect(struct apple_dcp *dcp) +{ + struct apple_epic_service *service; + struct audiosrv_data *asrv = dcp->audiosrv; + int ret; + + scoped_guard(rwsem_write, &asrv->srv_rwsem) { + if (!asrv->srv) + return; + service = asrv->srv; + } + + /* close AV audio service */ + dev_info(dcp->dev, "%s: stopping audio service\n", __func__); + if (!asrv->is_open) + return; + + mutex_lock(&asrv->plug_lock); + + if (asrv->audio_dev) + dcpaud_disconnect(asrv->audio_dev); + + mutex_unlock(&asrv->plug_lock); + + ret = afk_service_call(service, 0, asrv->cmds.close, NULL, 0, 16, + NULL, 0, 16); + if (ret) { + dev_err(dcp->dev, "error closing audio service: %d\n", ret); + } + if (service->torndown) + service->enabled = false; + asrv->is_open = false; +} + +static void av_work_service_start(struct work_struct *work) +{ + struct audiosrv_data *audiosrv_data; + struct apple_dcp *dcp; + + audiosrv_data = container_of(work, struct audiosrv_data, start_av_service_wq); + + scoped_guard(rwsem_read, &audiosrv_data->srv_rwsem) { + if (!audiosrv_data->srv || + !audiosrv_data->srv->ep || + !audiosrv_data->srv->ep->dcp) { + pr_err("%s: dcp: av: NULL ptr during startup\n", __func__); + return; + } + dcp = audiosrv_data->srv->ep->dcp; + } + + av_service_connect(dcp); +} + +int avep_init(struct apple_dcp *dcp) +{ + struct audiosrv_data *audiosrv_data; + struct platform_device *audio_pdev; + struct device *dev = dcp->dev; + struct device_node *endpoint, *audio_node = NULL; + + audiosrv_data = devm_kzalloc(dcp->dev, sizeof(*audiosrv_data), GFP_KERNEL); + if (!audiosrv_data) + return -ENOMEM; + init_rwsem(&audiosrv_data->srv_rwsem); + mutex_init(&audiosrv_data->plug_lock); + + switch (dcp->fw_compat) { + case DCP_FIRMWARE_V_12_3: + audiosrv_data->cmds = dcp_av_audio_cmds_v12_3; + break; + case DCP_FIRMWARE_V_13_5: + audiosrv_data->cmds = dcp_av_audio_cmds_v13_5; + break; + default: + dev_err(dcp->dev, "Audio not supported for firmware\n"); + return -ENODEV; + } + + dcp->audiosrv = audiosrv_data; + INIT_WORK(&audiosrv_data->start_av_service_wq, av_work_service_start); + + endpoint = of_graph_get_endpoint_by_regs(dev->of_node, 0, 0); + if (endpoint) { + audio_node = of_graph_get_remote_port_parent(endpoint); + of_node_put(endpoint); + } + if (!audio_node || !of_device_is_available(audio_node)) { + of_node_put(audio_node); + dev_info(dev, "No audio support\n"); + return 0; + } + + audio_pdev = of_find_device_by_node(audio_node); + of_node_put(audio_node); + if (!audio_pdev) { + dev_info(dev, "No DP/HDMI audio device not ready\n"); + return 0; + } + dcp->audiosrv->audio_dev = audio_pdev; + + device_link_add(&audio_pdev->dev, dev, + DL_FLAG_STATELESS | DL_FLAG_PM_RUNTIME); + + dcp->avep = afk_init(dcp, AV_ENDPOINT, avep_ops); + if (IS_ERR(dcp->avep)) + return PTR_ERR(dcp->avep); + dcp->avep->debugfs_entry = dcp->ep_debugfs[AV_ENDPOINT - 0x20]; + return afk_start(dcp->avep); +} diff --git a/drivers/gpu/drm/apple/av.h b/drivers/gpu/drm/apple/av.h new file mode 100644 index 00000000000000..c00cbef549fd2e --- /dev/null +++ b/drivers/gpu/drm/apple/av.h @@ -0,0 +1,17 @@ +#ifndef __AV_H__ +#define __AV_H__ + +#include "parser.h" + +//int avep_audiosrv_startlink(struct apple_dcp *dcp, struct dcp_sound_cookie *cookie); +//int avep_audiosrv_stoplink(struct apple_dcp *dcp); + +#if IS_ENABLED(CONFIG_DRM_APPLE_AUDIO) +void av_service_connect(struct apple_dcp *dcp); +void av_service_disconnect(struct apple_dcp *dcp); +#else +static inline void av_service_connect(struct apple_dcp *dcp) { } +static inline void av_service_disconnect(struct apple_dcp *dcp) { } +#endif + +#endif /* __AV_H__ */ diff --git a/drivers/gpu/drm/apple/connector.c b/drivers/gpu/drm/apple/connector.c new file mode 100644 index 00000000000000..15b3664d85631e --- /dev/null +++ b/drivers/gpu/drm/apple/connector.c @@ -0,0 +1,151 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * Copyright (C) The Asahi Linux Contributors + */ + +#include "connector.h" + +#include "linux/err.h" +#include +#include +#include +#include +#include + +#include + +#include "dcp-internal.h" + +enum dcp_chunk_type { + DCP_CHUNK_COLOR_ELEMENTS, + DCP_CHUNK_TIMING_ELELMENTS, + DCP_CHUNK_DISPLAY_ATTRIBUTES, + DCP_CHUNK_TRANSPORT, + DCP_CHUNK_NUM_TYPES, +}; + +static int chunk_show(struct seq_file *m, + enum dcp_chunk_type chunk_type) +{ + struct apple_connector *apple_con = m->private; + struct dcp_chunks *chunk = NULL; + + mutex_lock(&apple_con->chunk_lock); + + switch (chunk_type) { + case DCP_CHUNK_COLOR_ELEMENTS: + chunk = &apple_con->color_elements; + break; + case DCP_CHUNK_TIMING_ELELMENTS: + chunk = &apple_con->timing_elements; + break; + case DCP_CHUNK_DISPLAY_ATTRIBUTES: + chunk = &apple_con->display_attributes; + break; + case DCP_CHUNK_TRANSPORT: + chunk = &apple_con->transport; + break; + default: + break; + } + + if (chunk) + seq_write(m, chunk->data, chunk->length); + + mutex_unlock(&apple_con->chunk_lock); + + return 0; +} + +#define CONNECTOR_DEBUGFS_ENTRY(name, type) \ +static int chunk_ ## name ## _show(struct seq_file *m, void *data) \ +{ \ + return chunk_show(m, type); \ +} \ +static int chunk_ ## name ## _open(struct inode *inode, struct file *file) \ +{ \ + return single_open(file, chunk_ ## name ## _show, inode->i_private); \ +} \ +static const struct file_operations chunk_ ## name ## _fops = { \ + .owner = THIS_MODULE, \ + .open = chunk_ ## name ## _open, \ + .read = seq_read, \ + .llseek = seq_lseek, \ + .release = single_release, \ +} + +CONNECTOR_DEBUGFS_ENTRY(color, DCP_CHUNK_COLOR_ELEMENTS); +CONNECTOR_DEBUGFS_ENTRY(timing, DCP_CHUNK_TIMING_ELELMENTS); +CONNECTOR_DEBUGFS_ENTRY(display_attribs, DCP_CHUNK_DISPLAY_ATTRIBUTES); +CONNECTOR_DEBUGFS_ENTRY(transport, DCP_CHUNK_TRANSPORT); + +static void dcp_afk_debugfs_root(struct platform_device *pdev, int ep, struct dentry *root) +{ +#if IS_ENABLED(CONFIG_DRM_APPLE_DEBUG) + struct dentry *entry = NULL; + struct apple_dcp *dcp = platform_get_drvdata(pdev); + + switch (ep) { + case AV_ENDPOINT: + entry = debugfs_create_dir("avep", root); + break; + default: + break; + } + + if (!IS_ERR_OR_NULL(entry)) + dcp->ep_debugfs[ep - 0x20] = entry; +#endif +} + +void apple_connector_debugfs_init(struct drm_connector *connector, struct dentry *root) +{ + struct apple_connector *apple_con = to_apple_connector(connector); + + debugfs_create_file("ColorElements", 0444, root, apple_con, + &chunk_color_fops); + debugfs_create_file("TimingElements", 0444, root, apple_con, + &chunk_timing_fops); + debugfs_create_file("DisplayAttributes", 0444, root, apple_con, + &chunk_display_attribs_fops); + debugfs_create_file("Transport", 0444, root, apple_con, + &chunk_transport_fops); + + switch (connector->connector_type) { + case DRM_MODE_CONNECTOR_DisplayPort: + case DRM_MODE_CONNECTOR_HDMIA: + dcp_afk_debugfs_root(apple_con->dcp, AV_ENDPOINT, root); + break; + default: + break; + } +} + +static void dcp_connector_set_dict(struct apple_connector *connector, + struct dcp_chunks *dict, + struct dcp_chunks *chunks) +{ + if (dict->data) + devm_kfree(&connector->dcp->dev, dict->data); + + *dict = *chunks; +} + +void dcp_connector_update_dict(struct apple_connector *connector, const char *key, + struct dcp_chunks *chunks) +{ + mutex_lock(&connector->chunk_lock); + if (!strcmp(key, "ColorElements")) + dcp_connector_set_dict(connector, &connector->color_elements, chunks); + else if (!strcmp(key, "TimingElements")) + dcp_connector_set_dict(connector, &connector->timing_elements, chunks); + else if (!strcmp(key, "DisplayAttributes")) + dcp_connector_set_dict(connector, &connector->display_attributes, chunks); + else if (!strcmp(key, "Transport")) + dcp_connector_set_dict(connector, &connector->transport, chunks); + + chunks->data = NULL; + chunks->length = 0; + + mutex_unlock(&connector->chunk_lock); +} diff --git a/drivers/gpu/drm/apple/connector.h b/drivers/gpu/drm/apple/connector.h new file mode 100644 index 00000000000000..ef2c23737aac64 --- /dev/null +++ b/drivers/gpu/drm/apple/connector.h @@ -0,0 +1,44 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* "Copyright" 2021 Alyssa Rosenzweig */ + +#ifndef __APPLE_CONNECTOR_H__ +#define __APPLE_CONNECTOR_H__ + +#include + +#include +#include "drm/drm_connector.h" +#include "drm/drm_edid.h" + +struct apple_connector; + +#include "dcp-internal.h" + +void dcp_hotplug(struct work_struct *work); + +struct apple_connector { + struct drm_connector base; + bool connected; + + struct platform_device *dcp; + + const struct drm_edid *drm_edid; + + /* Workqueue for sending hotplug events to the associated device */ + struct work_struct hotplug_wq; + + struct mutex chunk_lock; + + struct dcp_chunks color_elements; + struct dcp_chunks timing_elements; + struct dcp_chunks display_attributes; + struct dcp_chunks transport; +}; + +#define to_apple_connector(x) container_of(x, struct apple_connector, base) + +void apple_connector_debugfs_init(struct drm_connector *connector, struct dentry *root); + +void dcp_connector_update_dict(struct apple_connector *connector, const char *key, + struct dcp_chunks *chunks); +#endif diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h new file mode 100644 index 00000000000000..0a6859448e19eb --- /dev/null +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -0,0 +1,278 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright 2021 Alyssa Rosenzweig */ + +#ifndef __APPLE_DCP_INTERNAL_H__ +#define __APPLE_DCP_INTERNAL_H__ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "dptxep.h" +#include "iomfb.h" +#include "iomfb_v12_3.h" +#include "iomfb_v13_3.h" +#include "epic/dpavservep.h" + +#define DCP_MAX_PLANES 2 + +struct apple_dcp; +struct apple_dcp_afkep; + +struct dcpav_service_epic; + +enum dcp_firmware_version { + DCP_FIRMWARE_UNKNOWN, + DCP_FIRMWARE_V_12_3, + DCP_FIRMWARE_V_13_5, +}; + +enum { + SYSTEM_ENDPOINT = 0x20, + TEST_ENDPOINT = 0x21, + DCP_EXPERT_ENDPOINT = 0x22, + DISP0_ENDPOINT = 0x23, + DPAVSERV_ENDPOINT = 0x28, + AV_ENDPOINT = 0x29, + DPTX_ENDPOINT = 0x2a, + HDCP_ENDPOINT = 0x2b, + REMOTE_ALLOC_ENDPOINT = 0x2d, + IOMFB_ENDPOINT = 0x37, +}; + +/* Temporary backing for a chunked transfer via setDCPAVPropStart/Chunk/End */ +struct dcp_chunks { + size_t length; + void *data; +}; + +#define DCP_MAX_MAPPINGS (128) /* should be enough */ +#define MAX_DISP_REGISTERS (7) + +struct dcp_mem_descriptor { + size_t size; + void *buf; + dma_addr_t dva; + struct sg_table map; + u64 reg; +}; + +/* Limit on call stack depth (arbitrary). Some nesting is required */ +#define DCP_MAX_CALL_DEPTH 8 + +typedef void (*dcp_callback_t)(struct apple_dcp *, void *, void *); + +struct dcp_channel { + dcp_callback_t callbacks[DCP_MAX_CALL_DEPTH]; + void *cookies[DCP_MAX_CALL_DEPTH]; + void *output[DCP_MAX_CALL_DEPTH]; + u16 end[DCP_MAX_CALL_DEPTH]; + + /* Current depth of the call stack. Less than DCP_MAX_CALL_DEPTH */ + u8 depth; + /* Already warned about busy channel */ + bool warned_busy; +}; + +struct dcp_fb_reference { + struct list_head head; + struct drm_framebuffer *fb; + u32 swap_id; +}; + +#define MAX_NOTCH_HEIGHT 160 + +struct dcp_brightness { + struct backlight_device *bl_dev; + u32 maximum; + u32 dac; + int nits; + int scale; + bool update; +}; + +struct audiosrv_data; + +/** laptop/AiO integrated panel parameters from DT */ +struct dcp_panel { + /// panel width in millimeter + int width_mm; + /// panel height in millimeter + int height_mm; + /// panel has a mini-LED backlight + bool has_mini_led; +}; + +struct apple_dcp_hw_data { + u32 num_dptx_ports; +}; + +/* TODO: move IOMFB members to its own struct */ +struct apple_dcp { + struct device *dev; + struct platform_device *piodma; + struct iommu_domain *iommu_dom; + struct apple_rtkit *rtk; + struct apple_crtc *crtc; + struct apple_connector *connector; + + struct apple_dcp_hw_data hw; + + /* firmware version and compatible firmware version */ + enum dcp_firmware_version fw_compat; + + /* Coprocessor control register */ + void __iomem *coproc_reg; + + /* DCP has crashed */ + bool crashed; + + /************* IOMFB ************************************************** + * everything below is mostly used inside IOMFB but it could make * + * sense to keep some of the members in apple_dcp. * + **********************************************************************/ + + /* clock rate request by dcp in */ + struct clk *clk; + + /* DCP shared memory */ + void *shmem; + + /* Display registers mappable to the DCP */ + struct resource *disp_registers[MAX_DISP_REGISTERS]; + unsigned int nr_disp_registers; + + struct resource disp_bw_scratch_res; + struct resource disp_bw_doorbell_res; + u32 disp_bw_scratch_index; + u32 disp_bw_scratch_offset; + u32 disp_bw_doorbell_index; + u32 disp_bw_doorbell_offset; + + u32 index; + + /* Bitmap of memory descriptors used for mappings made by the DCP */ + DECLARE_BITMAP(memdesc_map, DCP_MAX_MAPPINGS); + + /* Indexed table of memory descriptors */ + struct dcp_mem_descriptor memdesc[DCP_MAX_MAPPINGS]; + + struct dcp_channel ch_cmd, ch_oobcmd; + struct dcp_channel ch_cb, ch_oobcb, ch_async, ch_oobasync; + + /* iomfb EP callback handlers */ + const iomfb_cb_handler *cb_handlers; + + /* Active chunked transfer. There can only be one at a time. */ + struct dcp_chunks chunks; + + /* Queued swap. Owned by the DCP to avoid per-swap memory allocation */ + union { + struct dcp_swap_submit_req_v12_3 v12_3; + struct dcp_swap_submit_req_v13_3 v13_3; + } swap; + + /* swap id of the last completed swap */ + u32 last_swap_id; + ktime_t swap_start; + + /* Current display mode */ + bool during_modeset; + bool valid_mode; + bool use_timestamps; + bool vrr_enabled; + struct dcp_set_digital_out_mode_req mode; + + /* completion for active turning true */ + struct completion start_done; + + /* Is the DCP booted? */ + bool active; + + /* eDP display without DP-HDMI conversion */ + bool main_display; + + /* clear all surfaces on init */ + bool surfaces_cleared; + + /* enable CRC calculation */ + bool crc_enabled; + + /* Modes valid for the connected display */ + struct dcp_display_mode *modes; + unsigned int nr_modes; + + /* Attributes of the connector */ + int connector_type; + + /* Attributes of the connected display */ + int width_mm, height_mm; + + unsigned notch_height; + + /* Workqueue for sending vblank events when a dcp swap is not possible */ + struct work_struct vblank_wq; + + /* List of referenced drm_framebuffers which can be unreferenced + * on the next successfully completed swap. + */ + struct list_head swapped_out_fbs; + + struct dcp_brightness brightness; + /* Workqueue for updating the initial brightness */ + struct work_struct bl_register_wq; + struct mutex bl_register_mutex; + /* Workqueue for updating the brightness */ + struct work_struct bl_update_wq; + + /* integrated panel if present */ + struct dcp_panel panel; + + struct apple_dcp_afkep *systemep; + struct completion systemep_done; + + struct apple_dcp_afkep *ibootep; + struct apple_dcp_afkep *dcpavservep; + struct dcpavserv dcpavserv; + + struct apple_dcp_afkep *avep; + struct audiosrv_data *audiosrv; + + struct apple_dcp_afkep *dptxep; + + struct dptx_port dptxport[2]; + + /* debugfs entries */ + struct dentry *ep_debugfs[0x20]; + + /* these fields are output port specific */ + struct phy *phy; + struct mux_control *xbar; + struct typec_mux *typec_mux; + + struct gpio_desc *hdmi_hpd; + struct gpio_desc *hdmi_pwren; + struct gpio_desc *dp2hdmi_pwren; + + struct mutex hpd_mutex; + + u32 dptx_phy; + u32 dptx_die; + int hdmi_hpd_irq; +}; + +void dcp_drm_crtc_page_flip(struct apple_dcp *dcp, ktime_t now); + +int dcp_backlight_register(struct apple_dcp *dcp); +int dcp_backlight_update(struct apple_dcp *dcp); +bool dcp_has_panel(struct apple_dcp *dcp); + +#define DCP_AUDIO_MAX_CHANS 15 + +#endif /* __APPLE_DCP_INTERNAL_H__ */ diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c new file mode 100644 index 00000000000000..83ba20f7f02568 --- /dev/null +++ b/drivers/gpu/drm/apple/dcp.c @@ -0,0 +1,1378 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright 2021 Alyssa Rosenzweig */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "afk.h" +#include "av.h" +#include "dcp.h" +#include "dcp-internal.h" +#include "iomfb.h" +#include "parser.h" +#include "trace.h" + +#define APPLE_DCP_COPROC_CPU_CONTROL 0x44 +#define APPLE_DCP_COPROC_CPU_CONTROL_RUN BIT(4) + +#define DCP_BOOT_TIMEOUT msecs_to_jiffies(1000) + +static bool show_notch; +module_param(show_notch, bool, 0644); +MODULE_PARM_DESC(show_notch, "Use the full display height and shows the notch"); + +bool hdmi_audio; +module_param(hdmi_audio, bool, 0644); +MODULE_PARM_DESC(hdmi_audio, "Enable unstable HDMI audio support"); + +static bool unstable_edid = true; +module_param(unstable_edid, bool, 0644); +MODULE_PARM_DESC(unstable_edid, "Enable unstable EDID retrival support"); + +bool force_vrr; +module_param(force_vrr, bool, 0644); +MODULE_PARM_DESC(force_vrr, "Always enable Adaptive Sync/ProMotion on supported displays"); + +/* copied and simplified from drm_vblank.c */ +static void send_vblank_event(struct drm_device *dev, + struct drm_pending_vblank_event *e, + u64 seq, ktime_t now) +{ + struct timespec64 tv; + + if (e->event.base.type != DRM_EVENT_FLIP_COMPLETE) + return; + + tv = ktime_to_timespec64(now); + e->event.vbl.sequence = seq; + /* + * e->event is a user space structure, with hardcoded unsigned + * 32-bit seconds/microseconds. This is safe as we always use + * monotonic timestamps since linux-4.15 + */ + e->event.vbl.tv_sec = tv.tv_sec; + e->event.vbl.tv_usec = tv.tv_nsec / 1000; + + /* + * Use the same timestamp for any associated fence signal to avoid + * mismatch in timestamps for vsync & fence events triggered by the + * same HW event. Frameworks like SurfaceFlinger in Android expects the + * retire-fence timestamp to match exactly with HW vsync as it uses it + * for its software vsync modeling. + */ + drm_send_event_timestamp_locked(dev, &e->base, now); +} + +/** + * dcp_crtc_send_page_flip_event - helper to send vblank event after pageflip + * + * Compensate for unknown slack between page flip and arrival of the + * swap_complete callback. Minimal observed duration on DCP with HDMI output + * was around 2.3 ms. If the fb swap was submitted closer to the expected + * swap_complete it gets a penalty of one frame duration. This is on the border + * of unreasonable considering that Apple advertises support for 240 Hz (frame + * duration of 4.167 ms). + * It is unreasonable considering kwin's kms commit scheduling. Kwin commits + * 1.5 ms + the mode's vblank time before the expected next page flip + * completion. This results in presenting at half the display's rate for HDMI + * outputs. + * This might be a difference between dcp and dcpext. + */ +static void dcp_crtc_send_page_flip_event(struct apple_crtc *crtc, + struct drm_pending_vblank_event *e, + ktime_t now, ktime_t start) +{ + struct drm_device *dev = crtc->base.dev; + u64 seq; + unsigned int pipe = drm_crtc_index(&crtc->base); + ktime_t flip; + + seq = 0; + if (start != KTIME_MIN) { + s64 delta = ktime_us_delta(now, start); + if (delta <= 500) + flip = now; + else if (delta >= 2500) + flip = ktime_sub_us(now, 1000); + else + flip = ktime_sub_us(now, (delta - 500) / 2); + } else { + flip = now; + } + e->pipe = pipe; + send_vblank_event(dev, e, seq, flip); +} + +/* HACK: moved here to avoid circular dependency between apple_drv and dcp */ +void dcp_drm_crtc_vblank(struct apple_crtc *crtc) +{ + unsigned long flags; + + spin_lock_irqsave(&crtc->base.dev->event_lock, flags); + if (crtc->event) { + drm_crtc_send_vblank_event(&crtc->base, crtc->event); + crtc->event = NULL; + } + spin_unlock_irqrestore(&crtc->base.dev->event_lock, flags); +} + +void dcp_drm_crtc_page_flip(struct apple_dcp *dcp, ktime_t now) +{ + unsigned long flags; + struct apple_crtc *crtc = dcp->crtc; + + spin_lock_irqsave(&crtc->base.dev->event_lock, flags); + if (crtc->event) { + if (crtc->event->event.base.type == DRM_EVENT_FLIP_COMPLETE) + dcp_crtc_send_page_flip_event(crtc, crtc->event, now, dcp->swap_start); + else + drm_crtc_send_vblank_event(&crtc->base, crtc->event); + crtc->event = NULL; + dcp->swap_start = KTIME_MIN; + } + spin_unlock_irqrestore(&crtc->base.dev->event_lock, flags); +} + +void dcp_set_dimensions(struct apple_dcp *dcp) +{ + int i; + int width_mm = dcp->width_mm; + int height_mm = dcp->height_mm; + + if (width_mm == 0 || height_mm == 0) { + width_mm = dcp->panel.width_mm; + height_mm = dcp->panel.height_mm; + } + + /* Set the connector info */ + if (dcp->connector) { + struct drm_connector *connector = &dcp->connector->base; + + mutex_lock(&connector->dev->mode_config.mutex); + connector->display_info.width_mm = width_mm; + connector->display_info.height_mm = height_mm; + mutex_unlock(&connector->dev->mode_config.mutex); + } + + /* + * Fix up any probed modes. Modes are created when parsing + * TimingElements, dimensions are calculated when parsing + * DisplayAttributes, and TimingElements may be sent first + */ + for (i = 0; i < dcp->nr_modes; ++i) { + dcp->modes[i].mode.width_mm = width_mm; + dcp->modes[i].mode.height_mm = height_mm; + } +} + +bool dcp_has_panel(struct apple_dcp *dcp) +{ + return dcp->panel.width_mm > 0; +} + +int dcp_set_crc(struct drm_crtc *crtc, bool enabled) +{ + struct apple_crtc *ac = to_apple_crtc(crtc); + struct apple_dcp *dcp = platform_get_drvdata(ac->dcp); + + dcp->crc_enabled = enabled; + + return 0; +} + +/* + * Helper to send a DRM vblank event. We do not know how call swap_submit_dcp + * without surfaces. To avoid timeouts in drm_atomic_helper_wait_for_vblanks + * send a vblank event via a workqueue. + */ +static void dcp_delayed_vblank(struct work_struct *work) +{ + struct apple_dcp *dcp; + + dcp = container_of(work, struct apple_dcp, vblank_wq); + mdelay(5); + dcp_drm_crtc_vblank(dcp->crtc); +} + +static void dcp_recv_msg(void *cookie, u8 endpoint, u64 message) +{ + struct apple_dcp *dcp = cookie; + + trace_dcp_recv_msg(dcp, endpoint, message); + + switch (endpoint) { + case IOMFB_ENDPOINT: + return iomfb_recv_msg(dcp, message); + case AV_ENDPOINT: + afk_receive_message(dcp->avep, message); + return; + case SYSTEM_ENDPOINT: + afk_receive_message(dcp->systemep, message); + return; + case DISP0_ENDPOINT: + afk_receive_message(dcp->ibootep, message); + return; + case DPAVSERV_ENDPOINT: + afk_receive_message(dcp->dcpavservep, message); + return; + case DPTX_ENDPOINT: + afk_receive_message(dcp->dptxep, message); + return; + default: + WARN(endpoint, "unknown DCP endpoint %hhu\n", endpoint); + } +} + +static void dcp_rtk_crashed(void *cookie, const void *crashlog, size_t crashlog_size) +{ + struct apple_dcp *dcp = cookie; + + dcp->crashed = true; + dev_err(dcp->dev, "DCP has crashed\n"); + if (dcp->connector) { + dcp->connector->connected = 0; + drm_edid_free(dcp->connector->drm_edid); + dcp->connector->drm_edid = NULL; + schedule_work(&dcp->connector->hotplug_wq); + } + complete(&dcp->start_done); +} + +static int dcp_rtk_shmem_setup(void *cookie, struct apple_rtkit_shmem *bfr) +{ + struct apple_dcp *dcp = cookie; + + if (bfr->iova) { + struct iommu_domain *domain = + iommu_get_domain_for_dev(dcp->dev); + phys_addr_t phy_addr; + + if (!domain) + return -ENOMEM; + + // TODO: get map from device-tree + phy_addr = iommu_iova_to_phys(domain, bfr->iova); + if (!phy_addr) + return -ENOMEM; + + // TODO: verify phy_addr, cache attribute + bfr->buffer = memremap(phy_addr, bfr->size, MEMREMAP_WB); + if (!bfr->buffer) + return -ENOMEM; + + bfr->is_mapped = true; + dev_info(dcp->dev, + "shmem_setup: iova: %lx -> pa: %lx -> iomem: %lx\n", + (uintptr_t)bfr->iova, (uintptr_t)phy_addr, + (uintptr_t)bfr->buffer); + } else { + bfr->buffer = dma_alloc_coherent(dcp->dev, bfr->size, + &bfr->iova, GFP_KERNEL); + if (!bfr->buffer) + return -ENOMEM; + + dev_info(dcp->dev, "shmem_setup: iova: %lx, buffer: %lx\n", + (uintptr_t)bfr->iova, (uintptr_t)bfr->buffer); + } + + return 0; +} + +static void dcp_rtk_shmem_destroy(void *cookie, struct apple_rtkit_shmem *bfr) +{ + struct apple_dcp *dcp = cookie; + + if (bfr->is_mapped) + memunmap(bfr->buffer); + else + dma_free_coherent(dcp->dev, bfr->size, bfr->buffer, bfr->iova); +} + +static struct apple_rtkit_ops rtkit_ops = { + .crashed = dcp_rtk_crashed, + .recv_message = dcp_recv_msg, + .shmem_setup = dcp_rtk_shmem_setup, + .shmem_destroy = dcp_rtk_shmem_destroy, +}; + +void dcp_send_message(struct apple_dcp *dcp, u8 endpoint, u64 message) +{ + trace_dcp_send_msg(dcp, endpoint, message); + apple_rtkit_send_message(dcp->rtk, endpoint, message, NULL, + true); +} + +int dcp_crtc_atomic_check(struct drm_crtc *crtc, struct drm_atomic_state *state) +{ + struct platform_device *pdev = to_apple_crtc(crtc)->dcp; + struct apple_dcp *dcp = platform_get_drvdata(pdev); + struct drm_plane_state *new_state; + struct drm_plane *plane; + struct drm_crtc_state *crtc_state; + int plane_idx, plane_count = 0; + bool needs_modeset; + + if (dcp->crashed) + return -EINVAL; + + crtc_state = drm_atomic_get_new_crtc_state(state, crtc); + + needs_modeset = drm_atomic_crtc_needs_modeset(crtc_state) || !dcp->valid_mode; + if (!needs_modeset && !dcp->connector->connected) { + dev_err(dcp->dev, "crtc_atomic_check: disconnected but no modeset\n"); + return -EINVAL; + } + + for_each_new_plane_in_state(state, plane, new_state, plane_idx) { + /* skip planes not for this crtc */ + if (new_state->crtc != crtc) + continue; + + plane_count += 1; + } + + if (plane_count > DCP_MAX_PLANES) { + dev_err(dcp->dev, "crtc_atomic_check: Blend supports only 2 layers!\n"); + return -EINVAL; + } + + // if (dcp->vrr_enabled != crtc_state->vrr_enabled) { + // crtc_state->mode_changed = true; + // } + + return 0; +} + +int dcp_get_connector_type(struct platform_device *pdev) +{ + struct apple_dcp *dcp = platform_get_drvdata(pdev); + + return (dcp->connector_type); +} + +#define DPTX_CONNECT_TIMEOUT msecs_to_jiffies(2000) + +static int dcp_dptx_connect(struct apple_dcp *dcp, u32 port) +{ + int ret = 0; + + if (!dcp->phy) { + dev_warn(dcp->dev, "dcp_dptx_connect: missing phy\n"); + return -ENODEV; + } + dev_info(dcp->dev, "%s(port=%d)\n", __func__, port); + + mutex_lock(&dcp->hpd_mutex); + if (!dcp->dptxport[port].enabled) { + dev_warn(dcp->dev, "dcp_dptx_connect: dptx service for port %d not enabled\n", port); + ret = -ENODEV; + goto out_unlock; + } + + if (dcp->dptxport[port].connected) + goto out_unlock; + + reinit_completion(&dcp->dptxport[port].linkcfg_completion); + dcp->dptxport[port].atcphy = dcp->phy; + dptxport_connect(dcp->dptxport[port].service, 0, dcp->dptx_phy, dcp->dptx_die); + dptxport_request_display(dcp->dptxport[port].service); + dcp->dptxport[port].connected = true; + + mutex_unlock(&dcp->hpd_mutex); + ret = wait_for_completion_timeout(&dcp->dptxport[port].linkcfg_completion, + DPTX_CONNECT_TIMEOUT); + if (ret < 0) + dev_warn(dcp->dev, "dcp_dptx_connect: port %d link complete failed:%d\n", + port, ret); + else + dev_dbg(dcp->dev, "dcp_dptx_connect: waited %d ms for link\n", + jiffies_to_msecs(DPTX_CONNECT_TIMEOUT - ret)); + + usleep_range(5, 10); + + if (dcp->connector_type == DRM_MODE_CONNECTOR_DisplayPort) + dptxport_set_hpd(dcp->dptxport[port].service, true); + + if (dcp->avep) + av_service_connect(dcp); + + return 0; + +out_unlock: + mutex_unlock(&dcp->hpd_mutex); + return ret; +} + +static void disconnected_hpd_event(struct apple_connector *con) +{ + if (con && con->connected) { + con->connected = 0; + drm_kms_helper_connector_hotplug_event(&con->base); + } +} + +static int dcp_dptx_disconnect(struct apple_dcp *dcp, u32 port) +{ + dev_info(dcp->dev, "%s(port=%d)\n", __func__, port); + + mutex_lock(&dcp->hpd_mutex); + if (dcp->dptxport[port].enabled && dcp->dptxport[port].connected) { + dptxport_release_display(dcp->dptxport[port].service); + dcp->dptxport[port].connected = false; + } + mutex_unlock(&dcp->hpd_mutex); + + return 0; +} + +int dcp_dptx_connect_oob(struct platform_device *pdev, u32 port) +{ + struct apple_dcp *dcp = platform_get_drvdata(pdev); + return dcp_dptx_connect(dcp, port); +} + +int dcp_dptx_disconnect_oob(struct platform_device *pdev, u32 port) +{ + struct apple_dcp *dcp = platform_get_drvdata(pdev); + + disconnected_hpd_event(dcp->connector); + + if (dcp->avep) + av_service_disconnect(dcp); + + if (dcp->dptxport[port].enabled) + dptxport_set_hpd(dcp->dptxport[port].service, false); + + return dcp_dptx_disconnect(dcp, port); +} + +static irqreturn_t dcp_dp2hdmi_hpd(int irq, void *data) +{ + struct apple_dcp *dcp = data; + bool connected = gpiod_get_value_cansleep(dcp->hdmi_hpd); + + /* do nothing on disconnect and trust that dcp detects it itself. + * Parallel disconnect HPDs result drm disabling the CRTC even when it + * should not. + * The interrupt should be changed to rising but for now the disconnect + * IRQs might be helpful for debugging. + */ + dev_info(dcp->dev, "DP2HDMI HPD irq, connected:%d\n", connected); + + if (connected) { + msleep(500); + connected = gpiod_get_value_cansleep(dcp->hdmi_hpd); + dev_info(dcp->dev, "DP2HDMI HPD irq, 500ms debounce: connected:%d\n", connected); + } + + if (connected) + dcp_dptx_connect(dcp, 0); + + return IRQ_HANDLED; +} + +void dcp_link(struct platform_device *pdev, struct apple_crtc *crtc, + struct apple_connector *connector) +{ + struct apple_dcp *dcp = platform_get_drvdata(pdev); + + dcp->crtc = crtc; + dcp->connector = connector; +} + + +bool dcp_fw_compat_is_12_x(struct platform_device *pdev) +{ + struct apple_dcp *dcp = platform_get_drvdata(pdev); + + return dcp->fw_compat == DCP_FIRMWARE_V_12_3; +} + +int dcp_start(struct platform_device *pdev) +{ + struct apple_dcp *dcp = platform_get_drvdata(pdev); + int ret; + + init_completion(&dcp->start_done); + + /* start RTKit endpoints */ + ret = systemep_init(dcp); + if (ret) + dev_warn(dcp->dev, "Failed to start system endpoint: %d\n", ret); + + if (unstable_edid && !dcp_has_panel(dcp)) { + ret = dpavservep_init(dcp); + if (ret) + dev_warn(dcp->dev, "Failed to start DPAVSERV endpoint: %d", + ret); + } + + if (dcp->phy && dcp->fw_compat >= DCP_FIRMWARE_V_13_5) { + ret = ibootep_init(dcp); + if (ret) + dev_warn(dcp->dev, "Failed to start IBOOT endpoint: %d\n", + ret); + + ret = dptxep_init(dcp); + if (ret) { + dev_warn(dcp->dev, "Failed to start DPTX endpoint: %d\n", + ret); +#ifdef DCP_DPTX_DISCONNECT_ON_INIT + /* + * This disconnect / connect cycle on init is only necessary + * when using dcp0 on j473, j474s and presumedly j475c. + * Since dcp0 is not used at the moment let's avoid this + * since it is possibly the cause for startup issues. + */ + } else if (dcp->dptxport[0].enabled) { + bool connected; + /* force disconnect on start - necessary if the display + * is already up from m1n1 + */ + dptxport_set_hpd(dcp->dptxport[0].service, false); + dptxport_release_display(dcp->dptxport[0].service); + usleep_range(10 * USEC_PER_MSEC, 25 * USEC_PER_MSEC); + + connected = gpiod_get_value_cansleep(dcp->hdmi_hpd); + dev_info(dcp->dev, "%s: DP2HDMI HPD connected:%d\n", __func__, connected); + + // necessary on j473/j474 but not on j314c + if (connected) + dcp_dptx_connect(dcp, 0); +#endif + } + } else if (dcp->phy) { + dev_warn(dcp->dev, "OS firmware incompatible with dptxport EP\n"); + } + ret = iomfb_start_rtkit(dcp); + if (ret) + dev_err(dcp->dev, "Failed to start IOMFB endpoint: %d\n", ret); + +#if IS_ENABLED(CONFIG_DRM_APPLE_AUDIO) + if (hdmi_audio) { + ret = avep_init(dcp); + if (ret) + dev_warn(dcp->dev, "Failed to start AV endpoint: %d", ret); + ret = 0; + } +#endif + + return ret; +} + +static void _dcp_poweroff(struct apple_dcp *dcp) +{ + switch (dcp->fw_compat) { + case DCP_FIRMWARE_V_12_3: + iomfb_poweroff_v12_3(dcp); + break; + case DCP_FIRMWARE_V_13_5: + iomfb_poweroff_v13_3(dcp); + break; + default: + WARN_ONCE(true, "Unexpected firmware version: %u\n", dcp->fw_compat); + break; + } +} + +static int dcp_enable_dp2hdmi_hpd(struct apple_dcp *dcp) +{ + // check HPD state before enabling the edge triggered IRQ + if (dcp->hdmi_hpd) { + bool connected = gpiod_get_value_cansleep(dcp->hdmi_hpd); + dev_info(dcp->dev, "%s: DP2HDMI HPD connected:%d\n", __func__, connected); + + if (connected) + dcp_dptx_connect(dcp, 0); + else + _dcp_poweroff(dcp); + } + + if (dcp->hdmi_hpd_irq) + enable_irq(dcp->hdmi_hpd_irq); + + return 0; +} + +int dcp_wait_ready(struct platform_device *pdev, u64 timeout) +{ + struct apple_dcp *dcp = platform_get_drvdata(pdev); + int ret; + + if (dcp->crashed) + return -ENODEV; + if (dcp->active) + return dcp_enable_dp2hdmi_hpd(dcp); + if (timeout <= 0) + return -ETIMEDOUT; + + ret = wait_for_completion_timeout(&dcp->start_done, timeout); + if (ret < 0) + return ret; + + if (dcp->crashed) + return -ENODEV; + + if (dcp->active) + dcp_enable_dp2hdmi_hpd(dcp); + + return dcp->active ? 0 : -ETIMEDOUT; +} + +static void __maybe_unused dcp_sleep(struct apple_dcp *dcp) +{ + switch (dcp->fw_compat) { + case DCP_FIRMWARE_V_12_3: + iomfb_sleep_v12_3(dcp); + break; + case DCP_FIRMWARE_V_13_5: + iomfb_sleep_v13_3(dcp); + break; + default: + WARN_ONCE(true, "Unexpected firmware version: %u\n", dcp->fw_compat); + break; + } +} + +void dcp_poweron(struct platform_device *pdev) +{ + struct apple_dcp *dcp = platform_get_drvdata(pdev); + + if (dcp->hdmi_hpd) { + bool connected = gpiod_get_value_cansleep(dcp->hdmi_hpd); + dev_info(dcp->dev, "%s: DP2HDMI HPD connected:%d\n", __func__, connected); + + if (connected) + dcp_dptx_connect(dcp, 0); + } + + switch (dcp->fw_compat) { + case DCP_FIRMWARE_V_12_3: + iomfb_poweron_v12_3(dcp); + break; + case DCP_FIRMWARE_V_13_5: + iomfb_poweron_v13_3(dcp); + break; + default: + WARN_ONCE(true, "Unexpected firmware version: %u\n", dcp->fw_compat); + break; + } + + if (dcp->avep) + av_service_connect(dcp); +} + +void dcp_poweroff(struct platform_device *pdev) +{ + struct apple_dcp *dcp = platform_get_drvdata(pdev); + + if (dcp->avep) + av_service_disconnect(dcp); + + _dcp_poweroff(dcp); + + if (dcp->hdmi_hpd) { + bool connected = gpiod_get_value_cansleep(dcp->hdmi_hpd); + if (!connected) { + disconnected_hpd_event(dcp->connector); + dcp_dptx_disconnect(dcp, 0); + } + } +} + +static void dcp_work_register_backlight(struct work_struct *work) +{ + int ret; + struct apple_dcp *dcp; + + dcp = container_of(work, struct apple_dcp, bl_register_wq); + + mutex_lock(&dcp->bl_register_mutex); + if (dcp->brightness.bl_dev) + goto out_unlock; + + /* try to register backlight device, */ + ret = dcp_backlight_register(dcp); + if (ret) { + dev_err(dcp->dev, "Unable to register backlight device\n"); + dcp->brightness.maximum = 0; + } + +out_unlock: + mutex_unlock(&dcp->bl_register_mutex); +} + +static void dcp_work_update_backlight(struct work_struct *work) +{ + struct apple_dcp *dcp; + + dcp = container_of(work, struct apple_dcp, bl_update_wq); + + dcp_backlight_update(dcp); +} + +static int dcp_create_piodma_iommu_dev(struct apple_dcp *dcp) +{ + int ret; + struct device_node *node __free(device_node) = of_get_child_by_name(dcp->dev->of_node, "piodma"); + + if (!node) + return dev_err_probe(dcp->dev, -ENODEV, + "Failed to get piodma child DT node\n"); + + dcp->piodma = of_platform_device_create(node, NULL, dcp->dev); + if (!dcp->piodma) + return dev_err_probe(dcp->dev, -ENODEV, "Failed to create piodma pdev for %pOF\n", node); + + ret = dma_set_mask_and_coherent(&dcp->piodma->dev, DMA_BIT_MASK(42)); + if (ret) + goto err_destroy_pdev; + + ret = of_dma_configure(&dcp->piodma->dev, node, true); + if (ret) { + ret = dev_err_probe(dcp->dev, ret, + "Failed to configure IOMMU child DMA\n"); + goto err_destroy_pdev; + } + + dcp->iommu_dom = iommu_get_domain_for_dev(&dcp->piodma->dev); + if (IS_ERR(dcp->iommu_dom)) { + ret = dev_err_probe(dcp->dev, PTR_ERR(dcp->iommu_dom), + "Failed to get default iommu domain for " + "piodma device\n"); + dcp->iommu_dom = NULL; + goto err_destroy_pdev; + } + + return 0; +err_destroy_pdev: + of_platform_device_destroy(&dcp->piodma->dev, NULL); + return ret; +} + +static int dcp_get_bw_scratch_reg(struct apple_dcp *dcp, u32 expected) +{ + struct of_phandle_args ph_args; + u32 addr_idx, disp_idx, offset; + int ret; + + ret = of_parse_phandle_with_args(dcp->dev->of_node, "apple,bw-scratch", + "#apple,bw-scratch-cells", 0, &ph_args); + if (ret < 0) { + dev_err(dcp->dev, "Failed to read 'apple,bw-scratch': %d\n", ret); + return ret; + } + + if (ph_args.args_count != 3) { + dev_err(dcp->dev, "Unexpected 'apple,bw-scratch' arg count %d\n", + ph_args.args_count); + ret = -EINVAL; + goto err_of_node_put; + } + + addr_idx = ph_args.args[0]; + disp_idx = ph_args.args[1]; + offset = ph_args.args[2]; + + if (disp_idx != expected || disp_idx >= MAX_DISP_REGISTERS) { + dev_err(dcp->dev, "Unexpected disp_reg value in 'apple,bw-scratch': %d\n", + disp_idx); + ret = -EINVAL; + goto err_of_node_put; + } + + ret = of_address_to_resource(ph_args.np, addr_idx, &dcp->disp_bw_scratch_res); + if (ret < 0) { + dev_err(dcp->dev, "Failed to get 'apple,bw-scratch' resource %d from %pOF\n", + addr_idx, ph_args.np); + goto err_of_node_put; + } + if (offset > resource_size(&dcp->disp_bw_scratch_res) - 4) { + ret = -EINVAL; + goto err_of_node_put; + } + + dcp->disp_registers[disp_idx] = &dcp->disp_bw_scratch_res; + dcp->disp_bw_scratch_index = disp_idx; + dcp->disp_bw_scratch_offset = offset; + ret = 0; + +err_of_node_put: + of_node_put(ph_args.np); + return ret; +} + +static int dcp_get_bw_doorbell_reg(struct apple_dcp *dcp, u32 expected) +{ + struct of_phandle_args ph_args; + u32 addr_idx, disp_idx; + int ret; + + ret = of_parse_phandle_with_args(dcp->dev->of_node, "apple,bw-doorbell", + "#apple,bw-doorbell-cells", 0, &ph_args); + if (ret < 0) { + dev_err(dcp->dev, "Failed to read 'apple,bw-doorbell': %d\n", ret); + return ret; + } + + if (ph_args.args_count != 2) { + dev_err(dcp->dev, "Unexpected 'apple,bw-doorbell' arg count %d\n", + ph_args.args_count); + ret = -EINVAL; + goto err_of_node_put; + } + + addr_idx = ph_args.args[0]; + disp_idx = ph_args.args[1]; + + if (disp_idx != expected || disp_idx >= MAX_DISP_REGISTERS) { + dev_err(dcp->dev, "Unexpected disp_reg value in 'apple,bw-doorbell': %d\n", + disp_idx); + ret = -EINVAL; + goto err_of_node_put; + } + + ret = of_address_to_resource(ph_args.np, addr_idx, &dcp->disp_bw_doorbell_res); + if (ret < 0) { + dev_err(dcp->dev, "Failed to get 'apple,bw-doorbell' resource %d from %pOF\n", + addr_idx, ph_args.np); + goto err_of_node_put; + } + dcp->disp_bw_doorbell_index = disp_idx; + dcp->disp_registers[disp_idx] = &dcp->disp_bw_doorbell_res; + ret = 0; + +err_of_node_put: + of_node_put(ph_args.np); + return ret; +} + +static int dcp_get_disp_regs(struct apple_dcp *dcp) +{ + struct platform_device *pdev = to_platform_device(dcp->dev); + int count = pdev->num_resources - 1; + int i, ret; + + if (count <= 0 || count > MAX_DISP_REGISTERS) + return -EINVAL; + + for (i = 0; i < count; ++i) { + dcp->disp_registers[i] = + platform_get_resource(pdev, IORESOURCE_MEM, 1 + i); + } + + /* load pmgr bandwidth scratch resource and offset */ + ret = dcp_get_bw_scratch_reg(dcp, count); + if (ret < 0) + return ret; + count += 1; + + /* load pmgr bandwidth doorbell resource if present (only on t8103) */ + if (of_property_present(dcp->dev->of_node, "apple,bw-doorbell")) { + ret = dcp_get_bw_doorbell_reg(dcp, count); + if (ret < 0) + return ret; + count += 1; + } + + dcp->nr_disp_registers = count; + return 0; +} + +#define DCP_FW_VERSION_MIN_LEN 3 +#define DCP_FW_VERSION_MAX_LEN 5 +#define DCP_FW_VERSION_STR_LEN (DCP_FW_VERSION_MAX_LEN * 4) + +static int dcp_read_fw_version(struct device *dev, const char *name, + char *version_str) +{ + u32 ver[DCP_FW_VERSION_MAX_LEN]; + int len_str; + int len; + + len = of_property_read_variable_u32_array(dev->of_node, name, ver, + DCP_FW_VERSION_MIN_LEN, + DCP_FW_VERSION_MAX_LEN); + + switch (len) { + case 3: + len_str = scnprintf(version_str, DCP_FW_VERSION_STR_LEN, + "%d.%d.%d", ver[0], ver[1], ver[2]); + break; + case 4: + len_str = scnprintf(version_str, DCP_FW_VERSION_STR_LEN, + "%d.%d.%d.%d", ver[0], ver[1], ver[2], + ver[3]); + break; + case 5: + len_str = scnprintf(version_str, DCP_FW_VERSION_STR_LEN, + "%d.%d.%d.%d.%d", ver[0], ver[1], ver[2], + ver[3], ver[4]); + break; + default: + len_str = strscpy(version_str, "UNKNOWN", + DCP_FW_VERSION_STR_LEN); + if (len >= 0) + len = -EOVERFLOW; + break; + } + + if (len_str >= DCP_FW_VERSION_STR_LEN) + dev_warn(dev, "'%s' truncated: '%s'\n", name, version_str); + + return len; +} + +static enum dcp_firmware_version dcp_check_firmware_version(struct device *dev) +{ + char compat_str[DCP_FW_VERSION_STR_LEN]; + char fw_str[DCP_FW_VERSION_STR_LEN]; + int ret; + + /* firmware version is just informative */ + dcp_read_fw_version(dev, "apple,firmware-version", fw_str); + + ret = dcp_read_fw_version(dev, "apple,firmware-compat", compat_str); + if (ret < 0) { + dev_err(dev, "Could not read 'apple,firmware-compat': %d\n", ret); + return DCP_FIRMWARE_UNKNOWN; + } + + if (strncmp(compat_str, "12.3.0", sizeof(compat_str)) == 0) + return DCP_FIRMWARE_V_12_3; + /* + * m1n1 reports firmware version 13.5 as compatible with 13.3. This is + * only true for the iomfb endpoint. The interface for the dptx-port + * endpoint changed between 13.3 and 13.5. The driver will only support + * firmware 13.5. Check the actual firmware version for compat version + * 13.3 until m1n1 reports 13.5 as "firmware-compat". + */ + else if ((strncmp(compat_str, "13.3.0", sizeof(compat_str)) == 0) && + (strncmp(fw_str, "13.5.0", sizeof(compat_str)) == 0)) + return DCP_FIRMWARE_V_13_5; + else if (strncmp(compat_str, "13.5.0", sizeof(compat_str)) == 0) + return DCP_FIRMWARE_V_13_5; + + dev_err(dev, "DCP firmware-compat %s (FW: %s) is not supported\n", + compat_str, fw_str); + + return DCP_FIRMWARE_UNKNOWN; +} + +static int dcp_comp_bind(struct device *dev, struct device *main, void *data) +{ + struct device_node *panel_np; + struct apple_dcp *dcp = dev_get_drvdata(dev); + u32 cpu_ctrl; + int ret; + + ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(42)); + if (ret) + return ret; + + dcp->coproc_reg = devm_platform_ioremap_resource_byname(to_platform_device(dev), "coproc"); + if (IS_ERR(dcp->coproc_reg)) + return PTR_ERR(dcp->coproc_reg); + + of_property_read_u32(dev->of_node, "apple,dcp-index", + &dcp->index); + of_property_read_u32(dev->of_node, "apple,dptx-phy", + &dcp->dptx_phy); + of_property_read_u32(dev->of_node, "apple,dptx-die", + &dcp->dptx_die); + if (dcp->index || dcp->dptx_phy || dcp->dptx_die) + dev_info(dev, "DCP index:%u dptx target phy: %u dptx die: %u\n", + dcp->index, dcp->dptx_phy, dcp->dptx_die); + mutex_init(&dcp->hpd_mutex); + + if (!show_notch) + ret = of_property_read_u32(dev->of_node, "apple,notch-height", + &dcp->notch_height); + + if (dcp->notch_height > MAX_NOTCH_HEIGHT) + dcp->notch_height = MAX_NOTCH_HEIGHT; + if (dcp->notch_height > 0) + dev_info(dev, "Detected display with notch of %u pixel\n", dcp->notch_height); + + /* initialize brightness scale to a sensible default to avoid divide by 0*/ + dcp->brightness.scale = 65536; + panel_np = of_get_compatible_child(dev->of_node, "apple,panel-mini-led"); + if (panel_np) + dcp->panel.has_mini_led = true; + else + panel_np = of_get_compatible_child(dev->of_node, "apple,panel"); + + if (panel_np) { + const char height_prop[2][16] = { "adj-height-mm", "height-mm" }; + + if (of_device_is_available(panel_np)) { + ret = of_property_read_u32(panel_np, "apple,max-brightness", + &dcp->brightness.maximum); + if (ret) + dev_err(dev, "Missing property 'apple,max-brightness'\n"); + } + + of_property_read_u32(panel_np, "width-mm", &dcp->panel.width_mm); + /* use adjusted height as long as the notch is hidden */ + of_property_read_u32(panel_np, height_prop[!dcp->notch_height], + &dcp->panel.height_mm); + + of_node_put(panel_np); + dcp->connector_type = DRM_MODE_CONNECTOR_eDP; + INIT_WORK(&dcp->bl_register_wq, dcp_work_register_backlight); + mutex_init(&dcp->bl_register_mutex); + INIT_WORK(&dcp->bl_update_wq, dcp_work_update_backlight); + } else if (of_property_match_string(dev->of_node, "apple,connector-type", "HDMI-A") >= 0) + dcp->connector_type = DRM_MODE_CONNECTOR_HDMIA; + else if (of_property_match_string(dev->of_node, "apple,connector-type", "DP") >= 0) + dcp->connector_type = DRM_MODE_CONNECTOR_DisplayPort; + else if (of_property_match_string(dev->of_node, "apple,connector-type", "USB-C") >= 0) + dcp->connector_type = DRM_MODE_CONNECTOR_USB; + else + dcp->connector_type = DRM_MODE_CONNECTOR_Unknown; + + ret = dcp_create_piodma_iommu_dev(dcp); + if (ret || !dcp->iommu_dom) + return dev_err_probe(dev, ret, + "Failed to created PIODMA iommu child device"); + + ret = dcp_get_disp_regs(dcp); + if (ret) { + dev_err(dev, "failed to find display registers\n"); + return ret; + } + + dcp->clk = devm_clk_get(dev, NULL); + if (IS_ERR(dcp->clk)) + return dev_err_probe(dev, PTR_ERR(dcp->clk), + "Unable to find clock\n"); + + bitmap_zero(dcp->memdesc_map, DCP_MAX_MAPPINGS); + // TDOD: mem_desc IDs start at 1, for simplicity just skip '0' entry + set_bit(0, dcp->memdesc_map); + + INIT_WORK(&dcp->vblank_wq, dcp_delayed_vblank); + + dcp->swapped_out_fbs = + (struct list_head)LIST_HEAD_INIT(dcp->swapped_out_fbs); + + cpu_ctrl = + readl_relaxed(dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL); + writel_relaxed(cpu_ctrl | APPLE_DCP_COPROC_CPU_CONTROL_RUN, + dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL); + + dcp->rtk = devm_apple_rtkit_init(dev, dcp, "mbox", 0, &rtkit_ops); + if (IS_ERR(dcp->rtk)) + return dev_err_probe(dev, PTR_ERR(dcp->rtk), + "Failed to initialize RTKit\n"); + + ret = apple_rtkit_wake(dcp->rtk); + if (ret) + return dev_err_probe(dev, ret, + "Failed to boot RTKit: %d\n", ret); + return ret; +} + +/* + * We need to shutdown DCP before tearing down the display subsystem. Otherwise + * the DCP will crash and briefly flash a green screen of death. + */ +static void dcp_comp_unbind(struct device *dev, struct device *main, void *data) +{ + struct apple_dcp *dcp = dev_get_drvdata(dev); + + if (!dcp) + return; + + if (dcp->hdmi_hpd_irq) + disable_irq(dcp->hdmi_hpd_irq); + + typec_mux_put(dcp->typec_mux); + + if (dcp->avep) { + av_service_disconnect(dcp); + afk_shutdown(dcp->avep); + dcp->avep = NULL; + } + + if (dcp->dptxep) { + afk_shutdown(dcp->dptxep); + dcp->dptxep = NULL; + } + + if (dcp->ibootep) { + afk_shutdown(dcp->ibootep); + dcp->ibootep = NULL; + } + + if (dcp->systemep) { + afk_shutdown(dcp->systemep); + dcp->systemep = NULL; + } + + if (dcp->dcpavservep) { + afk_shutdown(dcp->dcpavservep); + dcp->dcpavservep = NULL; + } + + if (dcp->shmem) + iomfb_shutdown(dcp); + + if (dcp->piodma) { + dcp->iommu_dom = NULL; + of_platform_device_destroy(&dcp->piodma->dev, NULL); + dcp->piodma = NULL; + } + + if (dcp->connector_type == DRM_MODE_CONNECTOR_eDP) { + cancel_work_sync(&dcp->bl_register_wq); + cancel_work_sync(&dcp->bl_update_wq); + } + cancel_work_sync(&dcp->vblank_wq); + + devm_clk_put(dev, dcp->clk); + dcp->clk = NULL; +} + +static const struct component_ops dcp_comp_ops = { + .bind = dcp_comp_bind, + .unbind = dcp_comp_unbind, +}; + +static int dcp_platform_probe(struct platform_device *pdev) +{ + enum dcp_firmware_version fw_compat; + struct device *dev = &pdev->dev; + struct apple_dcp *dcp; + u32 mux_index; + + fw_compat = dcp_check_firmware_version(dev); + if (fw_compat == DCP_FIRMWARE_UNKNOWN) + return -ENODEV; + + /* Check for "apple,bw-scratch" to avoid probing appledrm with outdated + * device trees. This prevents replacing simpledrm and ending up without + * display. + */ + if (!of_property_present(dev->of_node, "apple,bw-scratch")) + return dev_err_probe(dev, -ENODEV, "Incompatible devicetree! " + "Use devicetree matching this kernel.\n"); + + dcp = devm_kzalloc(dev, sizeof(*dcp), GFP_KERNEL); + if (!dcp) + return -ENOMEM; + + dcp->fw_compat = fw_compat; + dcp->dev = dev; + dcp->hw = *(struct apple_dcp_hw_data *)of_device_get_match_data(dev); + + platform_set_drvdata(pdev, dcp); + + dcp->phy = devm_phy_optional_get(dev, "dp-phy"); + if (IS_ERR(dcp->phy)) { + dev_err(dev, "Failed to get dp-phy: %ld\n", PTR_ERR(dcp->phy)); + return PTR_ERR(dcp->phy); + } + if (dcp->phy) { + int ret; + /* + * Request DP2HDMI related GPIOs as optional for DP-altmode + * compatibility. J180D misses a dp2hdmi-pwren GPIO in the + * template ADT. TODO: check device ADT + */ + dcp->hdmi_hpd = devm_gpiod_get_optional(dev, "hdmi-hpd", GPIOD_IN); + if (IS_ERR(dcp->hdmi_hpd)) + return PTR_ERR(dcp->hdmi_hpd); + if (dcp->hdmi_hpd) { + int irq = gpiod_to_irq(dcp->hdmi_hpd); + if (irq < 0) { + dev_err(dev, "failed to translate HDMI hpd GPIO to IRQ\n"); + return irq; + } + dcp->hdmi_hpd_irq = irq; + + ret = devm_request_threaded_irq(dev, dcp->hdmi_hpd_irq, + NULL, dcp_dp2hdmi_hpd, + IRQF_ONESHOT | IRQF_NO_AUTOEN | + IRQF_TRIGGER_RISING | IRQF_TRIGGER_FALLING, + "dp2hdmi-hpd-irq", dcp); + if (ret < 0) { + dev_err(dev, "failed to request HDMI hpd irq %d: %d\n", + irq, ret); + return ret; + } + } + + /* + * Power DP2HDMI on as it is required for the HPD irq. + * TODO: check if one is sufficient for the hpd to save power + * on battery powered Macbooks. + */ + dcp->hdmi_pwren = devm_gpiod_get_optional(dev, "hdmi-pwren", GPIOD_OUT_HIGH); + if (IS_ERR(dcp->hdmi_pwren)) + return PTR_ERR(dcp->hdmi_pwren); + + dcp->dp2hdmi_pwren = devm_gpiod_get_optional(dev, "dp2hdmi-pwren", GPIOD_OUT_HIGH); + if (IS_ERR(dcp->dp2hdmi_pwren)) + return PTR_ERR(dcp->dp2hdmi_pwren); + + ret = of_property_read_u32(dev->of_node, "mux-index", &mux_index); + if (!ret) { + dcp->xbar = devm_mux_control_get(dev, "dp-xbar"); + if (IS_ERR(dcp->xbar)) { + dev_err(dev, "Failed to get dp-xbar: %ld\n", PTR_ERR(dcp->xbar)); + return PTR_ERR(dcp->xbar); + } + ret = mux_control_select(dcp->xbar, mux_index); + if (ret) + dev_warn(dev, "mux_control_select failed: %d\n", ret); + + /* + * Switch atcphy to DP-only. should move to a Macbook Pro + * 14-/16-inch specific DP-to-HDMI drm_bridge. + */ + dcp->typec_mux = fwnode_typec_mux_get(dev_fwnode(dcp->dev)); + if (!IS_ERR_OR_NULL(dcp->typec_mux)) { + struct typec_altmode alt = { + .svid = USB_TYPEC_DP_SID, + }; + struct typec_mux_state state = { + .alt = &alt, + .mode = TYPEC_DP_STATE_C, + }; + int ret = typec_mux_set(dcp->typec_mux, &state); + dev_info(dev, "typec_mux_set() returned: %d\n", ret); + } else { + dev_info(dev, "fwnode_typec_mux_get() returned: %ld\n", + IS_ERR(dcp->typec_mux) ? PTR_ERR(dcp->typec_mux) : 0); + dcp->typec_mux = NULL; + } + } + } + + return component_add(&pdev->dev, &dcp_comp_ops); +} + +static void dcp_platform_remove(struct platform_device *pdev) +{ + component_del(&pdev->dev, &dcp_comp_ops); +} + +static void dcp_platform_shutdown(struct platform_device *pdev) +{ + component_del(&pdev->dev, &dcp_comp_ops); +} + +static int dcp_platform_suspend(struct device *dev) +{ + struct apple_dcp *dcp = dev_get_drvdata(dev); + + if (dcp->avep) + av_service_disconnect(dcp); + + if (dcp->hdmi_hpd_irq) { + disable_irq(dcp->hdmi_hpd_irq); + disconnected_hpd_event(dcp->connector); + dcp_dptx_disconnect(dcp, 0); + } + /* + * Set the device as a wakeup device, which forces its power + * domains to stay on. We need this as we do not support full + * shutdown properly yet. + */ + device_set_wakeup_path(dev); + + return 0; +} + +static int dcp_platform_resume(struct device *dev) +{ + struct apple_dcp *dcp = dev_get_drvdata(dev); + + if (dcp->hdmi_hpd_irq) + enable_irq(dcp->hdmi_hpd_irq); + + if (dcp->avep) + av_service_connect(dcp); + + return 0; +} + +static DEFINE_SIMPLE_DEV_PM_OPS(dcp_platform_pm_ops, + dcp_platform_suspend, dcp_platform_resume); + + +static const struct apple_dcp_hw_data apple_dcp_hw_t6020 = { + .num_dptx_ports = 1, +}; + +static const struct apple_dcp_hw_data apple_dcp_hw_t8112 = { + .num_dptx_ports = 2, +}; + +static const struct apple_dcp_hw_data apple_dcp_hw_dcp = { + .num_dptx_ports = 0, +}; + +static const struct apple_dcp_hw_data apple_dcp_hw_dcpext = { + .num_dptx_ports = 2, +}; + +static const struct of_device_id of_match[] = { + { .compatible = "apple,t6020-dcp", .data = &apple_dcp_hw_t6020, }, + { .compatible = "apple,t8112-dcp", .data = &apple_dcp_hw_t8112, }, + { .compatible = "apple,dcp", .data = &apple_dcp_hw_dcp, }, + { .compatible = "apple,dcpext", .data = &apple_dcp_hw_dcpext, }, + {} +}; +MODULE_DEVICE_TABLE(of, of_match); + +static struct platform_driver apple_platform_driver = { + .probe = dcp_platform_probe, + .remove = dcp_platform_remove, + .shutdown = dcp_platform_shutdown, + .driver = { + .name = "apple-dcp", + .of_match_table = of_match, + .pm = pm_sleep_ptr(&dcp_platform_pm_ops), + }, +}; + +void __init dcp_register(void) +{ + platform_driver_register(&apple_platform_driver); +} + +void __exit dcp_unregister(void) +{ + platform_driver_unregister(&apple_platform_driver); +} diff --git a/drivers/gpu/drm/apple/dcp.h b/drivers/gpu/drm/apple/dcp.h new file mode 100644 index 00000000000000..bd20876847e0c3 --- /dev/null +++ b/drivers/gpu/drm/apple/dcp.h @@ -0,0 +1,78 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright 2021 Alyssa Rosenzweig */ + +#ifndef __APPLE_DCP_H__ +#define __APPLE_DCP_H__ + +#include +#include +#include + +#include "connector.h" +#include "dcp-internal.h" +#include "parser.h" + +struct apple_crtc { + struct drm_crtc base; + struct drm_pending_vblank_event *event; + bool vsync_disabled; + + /* Reference to the DCP device owning this CRTC */ + struct platform_device *dcp; +}; + +#define to_apple_crtc(x) container_of(x, struct apple_crtc, base) + +struct apple_encoder { + struct drm_encoder base; +}; + +#define to_apple_encoder(x) container_of(x, struct apple_encoder, base) + +void dcp_poweroff(struct platform_device *pdev); +void dcp_poweron(struct platform_device *pdev); +int dcp_set_crc(struct drm_crtc *crtc, bool enabled); +int dcp_crtc_atomic_check(struct drm_crtc *crtc, struct drm_atomic_state *state); +int dcp_get_connector_type(struct platform_device *pdev); +bool dcp_fw_compat_is_12_x(struct platform_device *pdev); +void dcp_link(struct platform_device *pdev, struct apple_crtc *apple, + struct apple_connector *connector); +int dcp_start(struct platform_device *pdev); +int dcp_wait_ready(struct platform_device *pdev, u64 timeout); +void dcp_flush(struct drm_crtc *crtc, struct drm_atomic_state *state); +bool dcp_is_initialized(struct platform_device *pdev); +void apple_crtc_vblank(struct apple_crtc *apple); +void dcp_drm_crtc_vblank(struct apple_crtc *crtc); +int dcp_get_modes(struct drm_connector *connector); +enum drm_mode_status dcp_mode_valid(struct drm_connector *connector, + const struct drm_display_mode *mode); +int dcp_crtc_atomic_modeset(struct drm_crtc *crtc, + struct drm_atomic_state *state); +bool dcp_crtc_mode_fixup(struct drm_crtc *crtc, + const struct drm_display_mode *mode, + struct drm_display_mode *adjusted_mode); +void dcp_set_dimensions(struct apple_dcp *dcp); +void dcp_send_message(struct apple_dcp *dcp, u8 endpoint, u64 message); + +int dcp_dptx_connect_oob(struct platform_device *pdev, u32 port); +int dcp_dptx_disconnect_oob(struct platform_device *pdev, u32 port); + +int iomfb_start_rtkit(struct apple_dcp *dcp); +void iomfb_shutdown(struct apple_dcp *dcp); +/* rtkit message handler for IOMFB messages */ +void iomfb_recv_msg(struct apple_dcp *dcp, u64 message); + +int systemep_init(struct apple_dcp *dcp); +int dptxep_init(struct apple_dcp *dcp); +int ibootep_init(struct apple_dcp *dcp); +int dpavservep_init(struct apple_dcp *dcp); +int avep_init(struct apple_dcp *dcp); + + +void __init dcp_register(void); +void __exit dcp_unregister(void); + +void __init dcp_audio_register(void); +void __exit dcp_audio_unregister(void); + +#endif diff --git a/drivers/gpu/drm/apple/dcp_backlight.c b/drivers/gpu/drm/apple/dcp_backlight.c new file mode 100644 index 00000000000000..9eb0c7d4eb5345 --- /dev/null +++ b/drivers/gpu/drm/apple/dcp_backlight.c @@ -0,0 +1,252 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright (C) The Asahi Linux Contributors */ + +#include +#include +#include +#include + +#include +#include +#include +#include "linux/jiffies.h" + +#include "dcp.h" +#include "dcp-internal.h" + +#define MIN_BRIGHTNESS_PART1 2U +#define MAX_BRIGHTNESS_PART1 99U +#define MIN_BRIGHTNESS_PART2 103U +#define MAX_BRIGHTNESS_PART2 510U + +/* + * lookup for display brightness 2 to 99 nits + * */ +static u32 brightness_part1[] = { + 0x0000000, 0x0810038, 0x0f000bd, 0x143011c, + 0x1850165, 0x1bc01a1, 0x1eb01d4, 0x2140200, + 0x2380227, 0x2590249, 0x2770269, 0x2930285, + 0x2ac02a0, 0x2c402b8, 0x2d902cf, 0x2ee02e4, + 0x30102f8, 0x314030b, 0x325031c, 0x335032d, + 0x345033d, 0x354034d, 0x362035b, 0x3700369, + 0x37d0377, 0x38a0384, 0x3960390, 0x3a2039c, + 0x3ad03a7, 0x3b803b3, 0x3c303bd, 0x3cd03c8, + 0x3d703d2, 0x3e103dc, 0x3ea03e5, 0x3f303ef, + 0x3fc03f8, 0x4050400, 0x40d0409, 0x4150411, + 0x41d0419, 0x4250421, 0x42d0429, 0x4340431, + 0x43c0438, 0x443043f, 0x44a0446, 0x451044d, + 0x4570454, 0x45e045b, 0x4640461, 0x46b0468, + 0x471046e, 0x4770474, 0x47d047a, 0x4830480, + 0x4890486, 0x48e048b, 0x4940491, 0x4990497, + 0x49f049c, 0x4a404a1, 0x4a904a7, 0x4ae04ac, + 0x4b304b1, 0x4b804b6, 0x4bd04bb, 0x4c204c0, + 0x4c704c5, 0x4cc04c9, 0x4d004ce, 0x4d504d3, + 0x4d904d7, 0x4de04dc, 0x4e204e0, 0x4e704e4, + 0x4eb04e9, 0x4ef04ed, 0x4f304f1, 0x4f704f5, + 0x4fb04f9, 0x4ff04fd, 0x5030501, 0x5070505, + 0x50b0509, 0x50f050d, 0x5130511, 0x5160515, + 0x51a0518, 0x51e051c, 0x5210520, 0x5250523, + 0x5290527, 0x52c052a, 0x52f052e, 0x5330531, + 0x5360535, 0x53a0538, 0x53d053b, 0x540053f, + 0x5440542, 0x5470545, 0x54a0548, 0x54d054c, + 0x550054f, 0x5530552, 0x5560555, 0x5590558, + 0x55c055b, 0x55f055e, 0x5620561, 0x5650564, + 0x5680567, 0x56b056a, 0x56e056d, 0x571056f, + 0x5740572, 0x5760575, 0x5790578, 0x57c057b, + 0x57f057d, 0x5810580, 0x5840583, 0x5870585, + 0x5890588, 0x58c058b, 0x58f058d +}; + +static u32 brightness_part12[] = { 0x58f058d, 0x59d058f }; + +/* + * lookup table for display brightness 103.3 to 510 nits + * */ +static u32 brightness_part2[] = { + 0x59d058f, 0x5b805ab, 0x5d105c5, 0x5e805dd, + 0x5fe05f3, 0x6120608, 0x625061c, 0x637062e, + 0x6480640, 0x6580650, 0x6680660, 0x677066f, + 0x685067e, 0x693068c, 0x6a00699, 0x6ac06a6, + 0x6b806b2, 0x6c406be, 0x6cf06ca, 0x6da06d5, + 0x6e506df, 0x6ef06ea, 0x6f906f4, 0x70206fe, + 0x70c0707, 0x7150710, 0x71e0719, 0x7260722, + 0x72f072a, 0x7370733, 0x73f073b, 0x7470743, + 0x74e074a, 0x7560752, 0x75d0759, 0x7640760, + 0x76b0768, 0x772076e, 0x7780775, 0x77f077c, + 0x7850782, 0x78c0789, 0x792078f, 0x7980795, + 0x79e079b, 0x7a407a1, 0x7aa07a7, 0x7af07ac, + 0x7b507b2, 0x7ba07b8, 0x7c007bd, 0x7c507c2, + 0x7ca07c8, 0x7cf07cd, 0x7d407d2, 0x7d907d7, + 0x7de07dc, 0x7e307e1, 0x7e807e5, 0x7ec07ea, + 0x7f107ef, 0x7f607f3, 0x7fa07f8, 0x7fe07fc +}; + + +static int dcp_get_brightness(struct backlight_device *bd) +{ + struct apple_dcp *dcp = bl_get_data(bd); + + return dcp->brightness.nits; +} + +#define SCALE_FACTOR (1 << 10) + +static u32 interpolate(int val, int min, int max, u32 *tbl, size_t tbl_size) +{ + u32 frac; + u64 low, high; + u32 interpolated = (tbl_size - 1) * ((val - min) * SCALE_FACTOR) / (max - min); + + size_t index = interpolated / SCALE_FACTOR; + + if (WARN(index + 1 >= tbl_size, "invalid index %zu for brightness %u\n", index, val)) + return tbl[tbl_size / 2]; + + frac = interpolated & (SCALE_FACTOR - 1); + low = tbl[index]; + high = tbl[index + 1]; + + return ((frac * high) + ((SCALE_FACTOR - frac) * low)) / SCALE_FACTOR; +} + +static u32 calculate_dac(struct apple_dcp *dcp, int val) +{ + u32 dac; + + if (val <= MIN_BRIGHTNESS_PART1) + return 16 * brightness_part1[0]; + else if (val == MAX_BRIGHTNESS_PART1) + return 16 * brightness_part1[ARRAY_SIZE(brightness_part1) - 1]; + else if (val == MIN_BRIGHTNESS_PART2) + return 16 * brightness_part2[0]; + else if (val >= MAX_BRIGHTNESS_PART2) + return brightness_part2[ARRAY_SIZE(brightness_part2) - 1]; + + if (val < MAX_BRIGHTNESS_PART1) { + dac = interpolate(val, MIN_BRIGHTNESS_PART1, MAX_BRIGHTNESS_PART1, + brightness_part1, ARRAY_SIZE(brightness_part1)); + } else if (val > MIN_BRIGHTNESS_PART2) { + dac = interpolate(val, MIN_BRIGHTNESS_PART2, MAX_BRIGHTNESS_PART2, + brightness_part2, ARRAY_SIZE(brightness_part2)); + } else { + dac = interpolate(val, MAX_BRIGHTNESS_PART1, MIN_BRIGHTNESS_PART2, + brightness_part12, ARRAY_SIZE(brightness_part12)); + } + + return 16 * dac; +} + +static int drm_crtc_set_brightness(struct apple_dcp *dcp) +{ + struct drm_atomic_state *state; + struct drm_crtc_state *crtc_state; + struct drm_modeset_acquire_ctx ctx; + struct drm_crtc *crtc = &dcp->crtc->base; + int ret = 0; + + drm_modeset_acquire_init(&ctx, DRM_MODESET_ACQUIRE_INTERRUPTIBLE); + ret = drm_modeset_lock(&crtc->mutex, &ctx); + if (ret == -EDEADLK) { + drm_modeset_backoff(&ctx); + return -EDEADLK; + } else if (ret == -ERESTARTSYS) { + return -ERESTARTSYS; + } + + if (!dcp->brightness.update) + goto done; + + state = drm_atomic_state_alloc(crtc->dev); + if (!state) { + ret = -ENOMEM; + goto done; + } + + state->acquire_ctx = &ctx; + crtc_state = drm_atomic_get_crtc_state(state, crtc); + if (IS_ERR(crtc_state)) { + ret = PTR_ERR(crtc_state); + goto fail; + } + + crtc_state->color_mgmt_changed |= true; + + ret = drm_atomic_commit(state); + +fail: + drm_atomic_state_put(state); +done: + drm_modeset_drop_locks(&ctx); + + return ret; +} + +int dcp_backlight_update(struct apple_dcp *dcp) +{ + /* + * Do not actively try to change brightness if no mode is set. + * TODO: should this be reflected the in backlight's power property? + * defer this hopefully until it becomes irrelevant due to proper + * drm integrated backlight handling + */ + if (!dcp->valid_mode) + return 0; + + /* Wait 1 vblank cycle in the hope an atomic swap has already updated + * the brightness */ + msleep((1001 + 23) / 24); // 42ms for 23.976 fps + + return drm_crtc_set_brightness(dcp); +} + +static int dcp_set_brightness(struct backlight_device *bd) +{ + int ret = 0; + struct apple_dcp *dcp = bl_get_data(bd); + struct drm_modeset_acquire_ctx ctx; + int brightness = backlight_get_brightness(bd); + + drm_modeset_acquire_init(&ctx, DRM_MODESET_ACQUIRE_INTERRUPTIBLE); + ret = drm_modeset_lock(&dcp->crtc->base.mutex, &ctx); + if (ret == -EDEADLK) { + drm_modeset_backoff(&ctx); + return -EDEADLK; + } else if (ret == -ERESTARTSYS) { + return -ERESTARTSYS; + } + + dcp->brightness.dac = calculate_dac(dcp, brightness); + dcp->brightness.update = true; + + drm_modeset_drop_locks(&ctx); + + return dcp_backlight_update(dcp); +} + +static const struct backlight_ops dcp_backlight_ops = { + .options = BL_CORE_SUSPENDRESUME, + .get_brightness = dcp_get_brightness, + .update_status = dcp_set_brightness, +}; + +int dcp_backlight_register(struct apple_dcp *dcp) +{ + struct device *dev = dcp->dev; + struct backlight_device *bl_dev; + struct backlight_properties props = { + .type = BACKLIGHT_PLATFORM, + .brightness = dcp->brightness.nits, + .scale = BACKLIGHT_SCALE_LINEAR, + }; + props.max_brightness = min(dcp->brightness.maximum, MAX_BRIGHTNESS_PART2 - 1); + + bl_dev = devm_backlight_device_register(dev, "apple-panel-bl", dev, dcp, + &dcp_backlight_ops, &props); + if (IS_ERR(bl_dev)) + return PTR_ERR(bl_dev); + + dcp->brightness.bl_dev = bl_dev; + dcp->brightness.dac = calculate_dac(dcp, dcp->brightness.nits); + + return 0; +} diff --git a/drivers/gpu/drm/apple/dcp_trace.c b/drivers/gpu/drm/apple/dcp_trace.c new file mode 100644 index 00000000000000..d18e71af73a74d --- /dev/null +++ b/drivers/gpu/drm/apple/dcp_trace.c @@ -0,0 +1,3 @@ +// SPDX-License-Identifier: GPL-2.0 +#define CREATE_TRACE_POINTS +#include "dcp_trace.h" \ No newline at end of file diff --git a/drivers/gpu/drm/apple/dptxep.c b/drivers/gpu/drm/apple/dptxep.c new file mode 100644 index 00000000000000..e21299e0124035 --- /dev/null +++ b/drivers/gpu/drm/apple/dptxep.c @@ -0,0 +1,640 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright 2022 Sven Peter */ + +#include +#include +#include +#include + +#include "afk.h" +#include "dcp.h" +#include "dptxep.h" +#include "parser.h" +#include "trace.h" + +struct dcpdptx_connection_cmd { + __le32 unk; + __le32 target; +} __attribute__((packed)); + +struct dcpdptx_hotplug_cmd { + u8 _pad0[16]; + __le32 unk; +} __attribute__((packed)); + +struct dptxport_apcall_link_rate { + __le32 retcode; + u8 _unk0[12]; + __le32 link_rate; + u8 _unk1[12]; +} __attribute__((packed)); + +struct dptxport_apcall_lane_count { + __le32 retcode; + u8 _unk0[12]; + __le64 lane_count; + u8 _unk1[8]; +} __attribute__((packed)); + +struct dptxport_apcall_set_active_lane_count { + __le32 retcode; + u8 _unk0[12]; + __le64 lane_count; + u8 _unk1[8]; +} __packed; + +struct dptxport_apcall_get_support { + __le32 retcode; + u8 _unk0[12]; + __le32 supported; + u8 _unk1[12]; +} __attribute__((packed)); + +struct dptxport_apcall_max_drive_settings { + __le32 retcode; + u8 _unk0[12]; + __le32 max_drive_settings[2]; + u8 _unk1[8]; +}; + +struct dptxport_apcall_drive_settings { + __le32 retcode; + u8 _unk0[12]; + __le32 unk1; + __le32 unk2; + __le32 unk3; + __le32 unk4; + __le32 unk5; + __le32 unk6; + __le32 unk7; +}; + +int dptxport_validate_connection(struct apple_epic_service *service, u8 core, + u8 atc, u8 die) +{ + struct dptx_port *dptx = service->cookie; + struct dcpdptx_connection_cmd cmd, resp; + int ret; + u32 target = FIELD_PREP(DCPDPTX_REMOTE_PORT_CORE, core) | + FIELD_PREP(DCPDPTX_REMOTE_PORT_ATC, atc) | + FIELD_PREP(DCPDPTX_REMOTE_PORT_DIE, die) | + DCPDPTX_REMOTE_PORT_CONNECTED; + + trace_dptxport_validate_connection(dptx, core, atc, die); + + cmd.target = cpu_to_le32(target); + cmd.unk = cpu_to_le32(0x100); + ret = afk_service_call(service, 0, 12, &cmd, sizeof(cmd), 40, &resp, + sizeof(resp), 40); + if (ret) + return ret; + + if (le32_to_cpu(resp.target) != target) + return -EINVAL; + if (le32_to_cpu(resp.unk) != 0x100) + return -EINVAL; + + return 0; +} + +int dptxport_connect(struct apple_epic_service *service, u8 core, u8 atc, + u8 die) +{ + struct dptx_port *dptx = service->cookie; + struct dcpdptx_connection_cmd cmd, resp; + u32 unk_field = 0x0; // seen as 0x100 under some conditions + int ret; + u32 target = FIELD_PREP(DCPDPTX_REMOTE_PORT_CORE, core) | + FIELD_PREP(DCPDPTX_REMOTE_PORT_ATC, atc) | + FIELD_PREP(DCPDPTX_REMOTE_PORT_DIE, die) | + DCPDPTX_REMOTE_PORT_CONNECTED; + + trace_dptxport_connect(dptx, core, atc, die); + + cmd.target = cpu_to_le32(target); + cmd.unk = cpu_to_le32(unk_field); + ret = afk_service_call(service, 0, 11, &cmd, sizeof(cmd), 24, &resp, + sizeof(resp), 24); + if (ret) + return ret; + + if (le32_to_cpu(resp.target) != target) + return -EINVAL; + if (le32_to_cpu(resp.unk) != unk_field) + dev_notice(service->ep->dcp->dev, "unexpected unk field in reply: 0x%x (0x%x)\n", + le32_to_cpu(resp.unk), unk_field); + + return 0; +} + +int dptxport_request_display(struct apple_epic_service *service) +{ + return afk_service_call(service, 0, 6, NULL, 0, 16, NULL, 0, 16); +} + +int dptxport_release_display(struct apple_epic_service *service) +{ + return afk_service_call(service, 0, 7, NULL, 0, 16, NULL, 0, 16); +} + +int dptxport_set_hpd(struct apple_epic_service *service, bool hpd) +{ + struct dcpdptx_hotplug_cmd cmd, resp; + int ret; + + memset(&cmd, 0, sizeof(cmd)); + + if (hpd) + cmd.unk = cpu_to_le32(1); + + ret = afk_service_call(service, 8, 8, &cmd, sizeof(cmd), 12, &resp, + sizeof(resp), 12); + if (ret) + return ret; + if (le32_to_cpu(resp.unk) != 1) + return -EINVAL; + return 0; +} + +static int +dptxport_call_get_max_drive_settings(struct apple_epic_service *service, + void *reply_, size_t reply_size) +{ + struct dptxport_apcall_max_drive_settings *reply = reply_; + + if (reply_size < sizeof(*reply)) + return -EINVAL; + + reply->retcode = cpu_to_le32(0); + reply->max_drive_settings[0] = cpu_to_le32(0x3); + reply->max_drive_settings[1] = cpu_to_le32(0x3); + + return 0; +} + +static int +dptxport_call_get_drive_settings(struct apple_epic_service *service, + const void *request_, size_t request_size, + void *reply_, size_t reply_size) +{ + struct dptx_port *dptx = service->cookie; + const struct dptxport_apcall_drive_settings *request = request_; + struct dptxport_apcall_drive_settings *reply = reply_; + + if (reply_size < sizeof(*reply) || request_size < sizeof(*request)) + return -EINVAL; + + *reply = *request; + + /* Clear the rest of the buffer */ + memset(reply_ + sizeof(*reply), 0, reply_size - sizeof(*reply)); + + /* + * retcode appears to be lane count, seeing 2 for USB-C dp alt mode + * with lanes splitted for DP/USB3. + */ + if (reply->retcode != dptx->lane_count) + dev_err(service->ep->dcp->dev, + "get_drive_settings: unexpected retcode %d\n", + reply->retcode); + + reply->retcode = dptx->lane_count; + reply->unk5 = dptx->drive_settings[0]; + reply->unk6 = 0; + reply->unk7 = dptx->drive_settings[1]; + + return 0; +} + +static int +dptxport_call_set_drive_settings(struct apple_epic_service *service, + const void *request_, size_t request_size, + void *reply_, size_t reply_size) +{ + struct dptx_port *dptx = service->cookie; + const struct dptxport_apcall_drive_settings *request = request_; + struct dptxport_apcall_drive_settings *reply = reply_; + + if (reply_size < sizeof(*reply) || request_size < sizeof(*request)) + return -EINVAL; + + *reply = *request; + reply->retcode = cpu_to_le32(0); + + dev_info(service->ep->dcp->dev, "set_drive_settings: %d:%d:%d:%d:%d:%d:%d\n", + request->unk1, request->unk2, request->unk3, request->unk4, + request->unk5, request->unk6, request->unk7); + + dptx->drive_settings[0] = reply->unk5; + dptx->drive_settings[1] = reply->unk7; + + return 0; +} + +static int dptxport_call_get_max_link_rate(struct apple_epic_service *service, + void *reply_, size_t reply_size) +{ + struct dptxport_apcall_link_rate *reply = reply_; + + if (reply_size < sizeof(*reply)) + return -EINVAL; + + reply->retcode = cpu_to_le32(0); + reply->link_rate = cpu_to_le32(LINK_RATE_HBR3); + + return 0; +} + +static int dptxport_call_get_max_lane_count(struct apple_epic_service *service, + void *reply_, size_t reply_size) +{ + struct dptxport_apcall_lane_count *reply = reply_; + struct dptx_port *dptx = service->cookie; + struct apple_dcp *dcp = service->ep->dcp; + union phy_configure_opts phy_ops; + int ret; + + if (reply_size < sizeof(*reply)) + return -EINVAL; + + ret = phy_validate(dptx->atcphy, PHY_MODE_DP, 0, &phy_ops); + if (ret < 0) { + dev_err(dcp->dev, "phy_validate failed: %d\n", ret); + reply->retcode = cpu_to_le32(1); + reply->lane_count = cpu_to_le64(0); + } else { + if (phy_ops.dp.lanes < 2) { + // phy_validate might return 0 lanes if atc phy is not + // yet switched to DP mode + dev_dbg(dcp->dev, "get_max_lane_count: phy lanes: %d\n", + phy_ops.dp.lanes); + // default to 4 lanes + dptx->lane_count = 4; + } else { + dptx->lane_count = phy_ops.dp.lanes; + } + reply->retcode = cpu_to_le32(0); + reply->lane_count = cpu_to_le64(dptx->lane_count); + } + + return 0; +} + +static int dptxport_call_set_active_lane_count(struct apple_epic_service *service, + const void *data, size_t data_size, + void *reply_, size_t reply_size) +{ + struct dptx_port *dptx = service->cookie; + struct apple_dcp *dcp = service->ep->dcp; + const struct dptxport_apcall_set_active_lane_count *request = data; + struct dptxport_apcall_set_active_lane_count *reply = reply_; + int ret = 0; + int retcode = 0; + + if (reply_size < sizeof(*reply)) + return -1; + if (data_size < sizeof(*request)) + return -1; + + u64 lane_count = cpu_to_le64(request->lane_count); + + if (dptx->lane_count < lane_count) + dev_err(dcp->dev, "set_active_lane_count: unexpected lane " + "count:%llu phy: %d\n", lane_count, dptx->lane_count); + + switch (lane_count) { + case 0 ... 2: + case 4: + dptx->phy_ops.dp.lanes = lane_count; + // Use dptx phy index > 3 as indication for dptx-phy or + // lpdptx-phy and configure the number of lanes for those + dptx->phy_ops.dp.set_lanes = (dcp->dptx_phy > 3); + break; + default: + dev_err(dcp->dev, "set_active_lane_count: invalid lane count:%llu\n", lane_count); + retcode = 1; + lane_count = 0; + break; + } + + if (dptx->phy_ops.dp.set_lanes) { + if (dptx->atcphy) { + ret = phy_configure(dptx->atcphy, &dptx->phy_ops); + if (ret) + return ret; + } + dptx->phy_ops.dp.set_lanes = 0; + dptx->lane_count = lane_count; + } + + reply->retcode = cpu_to_le32(retcode); + reply->lane_count = cpu_to_le64(lane_count); + + if (lane_count > 0) + complete(&dptx->linkcfg_completion); + + return ret; +} + +static int dptxport_call_get_link_rate(struct apple_epic_service *service, + void *reply_, size_t reply_size) +{ + struct dptx_port *dptx = service->cookie; + struct dptxport_apcall_link_rate *reply = reply_; + + if (reply_size < sizeof(*reply)) + return -EINVAL; + + reply->retcode = cpu_to_le32(0); + reply->link_rate = cpu_to_le32(dptx->link_rate); + + return 0; +} + +static int +dptxport_call_will_change_link_config(struct apple_epic_service *service) +{ + struct dptx_port *dptx = service->cookie; + + dptx->phy_ops.dp.set_lanes = 0; + dptx->phy_ops.dp.set_rate = 0; + dptx->phy_ops.dp.set_voltages = 0; + + return 0; +} + +static int +dptxport_call_did_change_link_config(struct apple_epic_service *service) +{ + /* assume the link config did change and wait a little bit */ + mdelay(10); + + return 0; +} + +static int dptxport_call_set_link_rate(struct apple_epic_service *service, + const void *data, size_t data_size, + void *reply_, size_t reply_size) +{ + struct dptx_port *dptx = service->cookie; + const struct dptxport_apcall_link_rate *request = data; + struct dptxport_apcall_link_rate *reply = reply_; + u32 link_rate, phy_link_rate; + bool phy_set_rate = false; + int ret; + + if (reply_size < sizeof(*reply)) + return -EINVAL; + if (data_size < sizeof(*request)) + return -EINVAL; + + link_rate = le32_to_cpu(request->link_rate); + trace_dptxport_call_set_link_rate(dptx, link_rate); + + switch (link_rate) { + case LINK_RATE_RBR: + phy_link_rate = 1620; + phy_set_rate = true; + break; + case LINK_RATE_HBR: + phy_link_rate = 2700; + phy_set_rate = true; + break; + case LINK_RATE_HBR2: + phy_link_rate = 5400; + phy_set_rate = true; + break; + case LINK_RATE_HBR3: + phy_link_rate = 8100; + phy_set_rate = true; + break; + case 0: + phy_link_rate = 0; + phy_set_rate = true; + break; + default: + dev_err(service->ep->dcp->dev, + "DPTXPort: Unsupported link rate 0x%x requested\n", + link_rate); + link_rate = 0; + phy_set_rate = false; + break; + } + + if (phy_set_rate) { + dptx->phy_ops.dp.link_rate = phy_link_rate; + dptx->phy_ops.dp.set_rate = 1; + + if (dptx->atcphy) { + ret = phy_configure(dptx->atcphy, &dptx->phy_ops); + if (ret) + return ret; + } + + //if (dptx->phy_ops.dp.set_rate) + dptx->link_rate = dptx->pending_link_rate = link_rate; + + } + + //dptx->pending_link_rate = link_rate; + reply->retcode = cpu_to_le32(0); + reply->link_rate = cpu_to_le32(link_rate); + + return 0; +} + +static int dptxport_call_get_supports_hpd(struct apple_epic_service *service, + void *reply_, size_t reply_size) +{ + struct dptxport_apcall_get_support *reply = reply_; + + if (reply_size < sizeof(*reply)) + return -EINVAL; + + reply->retcode = cpu_to_le32(0); + reply->supported = cpu_to_le32(0); + return 0; +} + +static int +dptxport_call_get_supports_downspread(struct apple_epic_service *service, + void *reply_, size_t reply_size) +{ + struct dptxport_apcall_get_support *reply = reply_; + + if (reply_size < sizeof(*reply)) + return -EINVAL; + + reply->retcode = cpu_to_le32(0); + reply->supported = cpu_to_le32(0); + return 0; +} + +static int +dptxport_call_activate(struct apple_epic_service *service, + const void *data, size_t data_size, + void *reply, size_t reply_size) +{ + struct dptx_port *dptx = service->cookie; + const struct apple_dcp *dcp = service->ep->dcp; + + // TODO: hack, use phy_set_mode to select the correct DCP(EXT) input + // for standalone phy (i.e. not atc phy). + if (!dcp->typec_mux) + phy_set_mode_ext(dptx->atcphy, PHY_MODE_DP, dcp->index); + + memcpy(reply, data, min(reply_size, data_size)); + if (reply_size >= 4) + memset(reply, 0, 4); + + return 0; +} + +static int +dptxport_call_deactivate(struct apple_epic_service *service, + const void *data, size_t data_size, + void *reply, size_t reply_size) +{ + struct dptx_port *dptx = service->cookie; + + /* deactivate phy */ + phy_set_mode_ext(dptx->atcphy, PHY_MODE_INVALID, 0); + + memcpy(reply, data, min(reply_size, data_size)); + if (reply_size >= 4) + memset(reply, 0, 4); + + return 0; +} + +static int dptxport_call(struct apple_epic_service *service, u32 idx, + const void *data, size_t data_size, void *reply, + size_t reply_size) +{ + struct dptx_port *dptx = service->cookie; + trace_dptxport_apcall(dptx, idx, data_size); + + switch (idx) { + case DPTX_APCALL_WILL_CHANGE_LINKG_CONFIG: + return dptxport_call_will_change_link_config(service); + case DPTX_APCALL_DID_CHANGE_LINK_CONFIG: + return dptxport_call_did_change_link_config(service); + case DPTX_APCALL_GET_MAX_LINK_RATE: + return dptxport_call_get_max_link_rate(service, reply, + reply_size); + case DPTX_APCALL_GET_LINK_RATE: + return dptxport_call_get_link_rate(service, reply, reply_size); + case DPTX_APCALL_SET_LINK_RATE: + return dptxport_call_set_link_rate(service, data, data_size, + reply, reply_size); + case DPTX_APCALL_GET_MAX_LANE_COUNT: + return dptxport_call_get_max_lane_count(service, reply, reply_size); + case DPTX_APCALL_SET_ACTIVE_LANE_COUNT: + return dptxport_call_set_active_lane_count(service, data, data_size, + reply, reply_size); + case DPTX_APCALL_GET_SUPPORTS_HPD: + return dptxport_call_get_supports_hpd(service, reply, + reply_size); + case DPTX_APCALL_GET_SUPPORTS_DOWN_SPREAD: + return dptxport_call_get_supports_downspread(service, reply, + reply_size); + case DPTX_APCALL_GET_MAX_DRIVE_SETTINGS: + return dptxport_call_get_max_drive_settings(service, reply, + reply_size); + case DPTX_APCALL_GET_DRIVE_SETTINGS: + return dptxport_call_get_drive_settings(service, data, data_size, + reply, reply_size); + case DPTX_APCALL_SET_DRIVE_SETTINGS: + return dptxport_call_set_drive_settings(service, data, data_size, + reply, reply_size); + case DPTX_APCALL_ACTIVATE: + return dptxport_call_activate(service, data, data_size, + reply, reply_size); + case DPTX_APCALL_DEACTIVATE: + return dptxport_call_deactivate(service, data, data_size, + reply, reply_size); + default: + /* just try to ACK and hope for the best... */ + dev_info(service->ep->dcp->dev, "DPTXPort: acking unhandled call %u\n", + idx); + fallthrough; + case DPTX_APCALL_GET_DOWN_SPREAD: + case DPTX_APCALL_SET_DOWN_SPREAD: + memcpy(reply, data, min(reply_size, data_size)); + if (reply_size >= 4) + memset(reply, 0, 4); + return 0; + } +} + +static void dptxport_init(struct apple_epic_service *service, const char *name, + const char *class, s64 unit) +{ + + if (strcmp(name, "dcpdptx-port-epic")) + return; + if (strcmp(class, "AppleDCPDPTXRemotePort")) + return; + + trace_dptxport_init(service->ep->dcp, unit); + + switch (unit) { + case 0: + case 1: + if (service->ep->dcp->dptxport[unit].enabled) { + dev_err(service->ep->dcp->dev, + "DPTXPort: unit %lld already exists\n", unit); + return; + } + service->ep->dcp->dptxport[unit].unit = unit; + service->ep->dcp->dptxport[unit].service = service; + service->ep->dcp->dptxport[unit].enabled = true; + service->cookie = (void *)&service->ep->dcp->dptxport[unit]; + complete(&service->ep->dcp->dptxport[unit].enable_completion); + break; + default: + dev_err(service->ep->dcp->dev, "DPTXPort: invalid unit %lld\n", + unit); + } +} + +static const struct apple_epic_service_ops dptxep_ops[] = { + { + .name = "AppleDCPDPTXRemotePort", + .init = dptxport_init, + .call = dptxport_call, + }, + {} +}; + +int dptxep_init(struct apple_dcp *dcp) +{ + int ret; + u32 port; + unsigned long timeout = msecs_to_jiffies(1000); + + init_completion(&dcp->dptxport[0].enable_completion); + init_completion(&dcp->dptxport[1].enable_completion); + init_completion(&dcp->dptxport[0].linkcfg_completion); + init_completion(&dcp->dptxport[1].linkcfg_completion); + + dcp->dptxep = afk_init(dcp, DPTX_ENDPOINT, dptxep_ops); + if (IS_ERR(dcp->dptxep)) + return PTR_ERR(dcp->dptxep); + + ret = afk_start(dcp->dptxep); + if (ret) + return ret; + + for (port = 0; port < dcp->hw.num_dptx_ports; port++) { + ret = wait_for_completion_timeout(&dcp->dptxport[port].enable_completion, + timeout); + if (!ret) + return -ETIMEDOUT; + else if (ret < 0) + return ret; + timeout = ret; + } + + return 0; +} diff --git a/drivers/gpu/drm/apple/dptxep.h b/drivers/gpu/drm/apple/dptxep.h new file mode 100644 index 00000000000000..0bf2534054fd7b --- /dev/null +++ b/drivers/gpu/drm/apple/dptxep.h @@ -0,0 +1,70 @@ +#ifndef __APPLE_DCP_DPTXEP_H__ +#define __APPLE_DCP_DPTXEP_H__ + +#include +#include + +enum dptx_apcall { + DPTX_APCALL_ACTIVATE = 0, + DPTX_APCALL_DEACTIVATE = 1, + DPTX_APCALL_GET_MAX_DRIVE_SETTINGS = 2, + DPTX_APCALL_SET_DRIVE_SETTINGS = 3, + DPTX_APCALL_GET_DRIVE_SETTINGS = 4, + DPTX_APCALL_WILL_CHANGE_LINKG_CONFIG = 5, + DPTX_APCALL_DID_CHANGE_LINK_CONFIG = 6, + DPTX_APCALL_GET_MAX_LINK_RATE = 7, + DPTX_APCALL_GET_LINK_RATE = 8, + DPTX_APCALL_SET_LINK_RATE = 9, + DPTX_APCALL_GET_MAX_LANE_COUNT = 10, + DPTX_APCALL_GET_ACTIVE_LANE_COUNT = 11, + DPTX_APCALL_SET_ACTIVE_LANE_COUNT = 12, + DPTX_APCALL_GET_SUPPORTS_DOWN_SPREAD = 13, + DPTX_APCALL_GET_DOWN_SPREAD = 14, + DPTX_APCALL_SET_DOWN_SPREAD = 15, + DPTX_APCALL_GET_SUPPORTS_LANE_MAPPING = 16, + DPTX_APCALL_SET_LANE_MAP = 17, + DPTX_APCALL_GET_SUPPORTS_HPD = 18, + DPTX_APCALL_FORCE_HOTPLUG_DETECT = 19, + DPTX_APCALL_INACTIVE_SINK_DETECTED = 20, + DPTX_APCALL_SET_TILED_DISPLAY_HINTS = 21, + DPTX_APCALL_DEVICE_NOT_RESPONDING = 22, + DPTX_APCALL_DEVICE_BUSY_TIMEOUT = 23, + DPTX_APCALL_DEVICE_NOT_STARTED = 24, +}; + +#define DCPDPTX_REMOTE_PORT_CORE GENMASK(3, 0) +#define DCPDPTX_REMOTE_PORT_ATC GENMASK(7, 4) +#define DCPDPTX_REMOTE_PORT_DIE GENMASK(11, 8) +#define DCPDPTX_REMOTE_PORT_CONNECTED BIT(15) + +enum dptx_link_rate { + LINK_RATE_RBR = 0x06, + LINK_RATE_HBR = 0x0a, + LINK_RATE_HBR2 = 0x14, + LINK_RATE_HBR3 = 0x1e, +}; + +struct apple_epic_service; + +struct dptx_port { + bool enabled, connected; + struct completion enable_completion; + struct completion linkcfg_completion; + u32 unit; + struct apple_epic_service *service; + union phy_configure_opts phy_ops; + struct phy *atcphy; + struct mux_control *mux; + u32 lane_count; + u32 link_rate, pending_link_rate; + u32 drive_settings[2]; +}; + +int dptxport_validate_connection(struct apple_epic_service *service, u8 core, + u8 atc, u8 die); +int dptxport_connect(struct apple_epic_service *service, u8 core, u8 atc, + u8 die); +int dptxport_request_display(struct apple_epic_service *service); +int dptxport_release_display(struct apple_epic_service *service); +int dptxport_set_hpd(struct apple_epic_service *service, bool hpd); +#endif diff --git a/drivers/gpu/drm/apple/epic/dpavservep.c b/drivers/gpu/drm/apple/epic/dpavservep.c new file mode 100644 index 00000000000000..2de9d2fe4c24a3 --- /dev/null +++ b/drivers/gpu/drm/apple/epic/dpavservep.c @@ -0,0 +1,233 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright The Asahi Linux Contributors */ + +#include "dpavservep.h" + +#include + +#include +#include +#include + +#include "../afk.h" +#include "../dcp.h" +#include "../dcp-internal.h" +#include "../trace.h" + +static void dcpavserv_init(struct apple_epic_service *service, const char *name, + const char *class, s64 unit) +{ + struct apple_dcp *dcp = service->ep->dcp; + trace_dcpavserv_init(dcp, unit); + + if (unit == 0 && name && !strcmp(name, "dcpav-service-epic")) { + if (dcp->dcpavserv.enabled) { + dev_err(dcp->dev, + "DCPAVSERV: unit %lld already exists\n", unit); + return; + } + dcp->dcpavserv.service = service; + dcp->dcpavserv.enabled = true; + service->cookie = &dcp->dcpavserv; + complete(&dcp->dcpavserv.enable_completion); + } +} + +static void dcpavserv_teardown(struct apple_epic_service *service) +{ + struct apple_dcp *dcp = service->ep->dcp; + service->enabled = false; + + if (dcp->dcpavserv.enabled) { + dcp->dcpavserv.enabled = false; + dcp->dcpavserv.service = NULL; + service->cookie = NULL; + reinit_completion(&dcp->dcpavserv.enable_completion); + } +} + +static void dcpdpserv_init(struct apple_epic_service *service, const char *name, + const char *class, s64 unit) +{ +} + +static void dcpdpserv_teardown(struct apple_epic_service *service) +{ + service->enabled = false; +} + +struct dcpavserv_status_report { + u32 unk00[4]; + u8 flag0; + u8 flag1; + u8 flag2; + u8 flag3; + u32 unk14[3]; + u32 status; + u32 unk24[3]; +} __packed; + +struct dpavserv_copy_edid_cmd { + __le64 max_size; + u8 _pad1[24]; + __le64 used_size; + u8 _pad2[8]; +} __packed; + +#define EDID_LEADING_DATA_SIZE 8 +#define EDID_BLOCK_SIZE 128 +#define EDID_EXT_BLOCK_COUNT_OFFSET 0x7E +#define EDID_MAX_SIZE SZ_32K +#define EDID_BUF_SIZE (EDID_LEADING_DATA_SIZE + EDID_MAX_SIZE) + +struct dpavserv_copy_edid_resp { + __le64 max_size; + u8 _pad1[24]; + __le64 used_size; + u8 _pad2[8]; + u8 data[]; +} __packed; + +static int parse_report(struct apple_epic_service *service, enum epic_subtype type, + const void *data, size_t data_size) +{ +#if defined(DEBUG) + struct apple_dcp *dcp = service->ep->dcp; + const struct epic_service_call *call; + const void *payload; + size_t payload_size; + + dev_dbg(dcp->dev, "dcpavserv[ch:%u]: report type:%02x len:%zu\n", + service->channel, type, data_size); + + if (type != EPIC_SUBTYPE_STD_SERVICE) + return 0; + + if (data_size < sizeof(*call)) + return 0; + + call = data; + + if (le32_to_cpu(call->magic) != EPIC_SERVICE_CALL_MAGIC) { + dev_warn(dcp->dev, "dcpavserv[ch:%u]: report magic 0x%08x != 0x%08x\n", + service->channel, le32_to_cpu(call->magic), EPIC_SERVICE_CALL_MAGIC); + return 0; + } + + payload_size = data_size - sizeof(*call); + if (payload_size < le32_to_cpu(call->data_len)) { + dev_warn(dcp->dev, "dcpavserv[ch:%u]: report payload size %zu call len %u\n", + service->channel, payload_size, le32_to_cpu(call->data_len)); + return 0; + } + payload_size = le32_to_cpu(call->data_len); + payload = data + sizeof(*call); + + if (le16_to_cpu(call->group) == 2 && le16_to_cpu(call->command) == 0) { + if (payload_size == sizeof(struct dcpavserv_status_report)) { + const struct dcpavserv_status_report *stat = payload; + dev_info(dcp->dev, "dcpavserv[ch:%u]: flags: 0x%02x,0x%02x,0x%02x,0x%02x status:%u\n", + service->channel, stat->flag0, stat->flag1, + stat->flag2, stat->flag3, stat->status); + } else { + dev_dbg(dcp->dev, "dcpavserv[ch:%u]: report payload size %zu\n", service->channel, payload_size); + } + } else { + print_hex_dump(KERN_DEBUG, "dcpavserv report: ", DUMP_PREFIX_NONE, + 16, 1, payload, payload_size, true); + } +#endif + + return 0; +} + +static int dcpavserv_report(struct apple_epic_service *service, + enum epic_subtype type, const void *data, + size_t data_size) +{ + return parse_report(service, type, data, data_size); +} + +static int dcpdpserv_report(struct apple_epic_service *service, + enum epic_subtype type, const void *data, + size_t data_size) +{ + return parse_report(service, type, data, data_size); +} + +const struct drm_edid *dcpavserv_copy_edid(struct apple_epic_service *service) +{ + struct dpavserv_copy_edid_cmd cmd; + struct dpavserv_copy_edid_resp *resp __free(kfree) = NULL; + int num_blocks; + u64 data_size; + int ret; + + memset(&cmd, 0, sizeof(cmd)); + cmd.max_size = cpu_to_le64(EDID_BUF_SIZE); + resp = kzalloc(sizeof(*resp) + EDID_BUF_SIZE, GFP_KERNEL); + if (!resp) + return ERR_PTR(-ENOMEM); + + ret = afk_service_call(service, 1, 7, &cmd, sizeof(cmd), EDID_BUF_SIZE, resp, + sizeof(resp) + EDID_BUF_SIZE, 0); + if (ret < 0) + return ERR_PTR(ret); + + if (le64_to_cpu(resp->max_size) != EDID_BUF_SIZE) + return ERR_PTR(-EIO); + + // print_hex_dump(KERN_DEBUG, "dpavserv EDID cmd: ", DUMP_PREFIX_NONE, + // 16, 1, resp, 192, true); + + data_size = le64_to_cpu(resp->used_size); + if (data_size < EDID_LEADING_DATA_SIZE + EDID_BLOCK_SIZE) + return ERR_PTR(-EIO); + + num_blocks = resp->data[EDID_LEADING_DATA_SIZE + EDID_EXT_BLOCK_COUNT_OFFSET]; + if ((1 + num_blocks) * EDID_BLOCK_SIZE != data_size - EDID_LEADING_DATA_SIZE) + return ERR_PTR(-EIO); + + return drm_edid_alloc(resp->data + EDID_LEADING_DATA_SIZE, + data_size - EDID_LEADING_DATA_SIZE); +} + +static const struct apple_epic_service_ops dpavservep_ops[] = { + { + .name = "dcpav-service-epic", + .init = dcpavserv_init, + .teardown = dcpavserv_teardown, + .report = dcpavserv_report, + }, + { + .name = "dcpdp-service-epic", + .init = dcpdpserv_init, + .teardown = dcpdpserv_teardown, + .report = dcpdpserv_report, + }, + {}, +}; + +int dpavservep_init(struct apple_dcp *dcp) +{ + int ret; + + init_completion(&dcp->dcpavserv.enable_completion); + + dcp->dcpavservep = afk_init(dcp, DPAVSERV_ENDPOINT, dpavservep_ops); + if (IS_ERR(dcp->dcpavservep)) + return PTR_ERR(dcp->dcpavservep); + + dcp->dcpavservep->match_epic_name = true; + + ret = afk_start(dcp->dcpavservep); + if (ret) + return ret; + + ret = wait_for_completion_timeout(&dcp->dcpavserv.enable_completion, + msecs_to_jiffies(1000)); + if (ret >= 0) + return 0; + + return ret; +} diff --git a/drivers/gpu/drm/apple/epic/dpavservep.h b/drivers/gpu/drm/apple/epic/dpavservep.h new file mode 100644 index 00000000000000..858ff14b0bd7be --- /dev/null +++ b/drivers/gpu/drm/apple/epic/dpavservep.h @@ -0,0 +1,22 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright The Asahi Linux Contributors */ + +#ifndef _DRM_APPLE_EPIC_DPAVSERV_H +#define _DRM_APPLE_EPIC_DPAVSERV_H + +#include +#include + +struct drm_edid; +struct apple_epic_service; + +struct dcpavserv { + bool enabled; + struct completion enable_completion; + u32 unit; + struct apple_epic_service *service; +}; + +const struct drm_edid *dcpavserv_copy_edid(struct apple_epic_service *service); + +#endif /* _DRM_APPLE_EPIC_DPAVSERV_H */ diff --git a/drivers/gpu/drm/apple/hdmi-codec-chmap.h b/drivers/gpu/drm/apple/hdmi-codec-chmap.h new file mode 100644 index 00000000000000..f98e1e86b89602 --- /dev/null +++ b/drivers/gpu/drm/apple/hdmi-codec-chmap.h @@ -0,0 +1,123 @@ +// copied from sound/soc/codecs/hdmi-codec.c + +#include + +/* Channel maps for multi-channel playbacks, up to 8 n_ch */ +static const struct snd_pcm_chmap_elem hdmi_codec_8ch_chmaps[] = { + { .channels = 2, /* CA_ID 0x00 */ + .map = { SNDRV_CHMAP_FL, SNDRV_CHMAP_FR } }, + { .channels = 4, /* CA_ID 0x01 */ + .map = { SNDRV_CHMAP_FL, SNDRV_CHMAP_FR, SNDRV_CHMAP_LFE, + SNDRV_CHMAP_NA } }, + { .channels = 4, /* CA_ID 0x02 */ + .map = { SNDRV_CHMAP_FL, SNDRV_CHMAP_FR, SNDRV_CHMAP_NA, + SNDRV_CHMAP_FC } }, + { .channels = 4, /* CA_ID 0x03 */ + .map = { SNDRV_CHMAP_FL, SNDRV_CHMAP_FR, SNDRV_CHMAP_LFE, + SNDRV_CHMAP_FC } }, + { .channels = 6, /* CA_ID 0x04 */ + .map = { SNDRV_CHMAP_FL, SNDRV_CHMAP_FR, SNDRV_CHMAP_NA, + SNDRV_CHMAP_NA, SNDRV_CHMAP_RC, SNDRV_CHMAP_NA } }, + { .channels = 6, /* CA_ID 0x05 */ + .map = { SNDRV_CHMAP_FL, SNDRV_CHMAP_FR, SNDRV_CHMAP_LFE, + SNDRV_CHMAP_NA, SNDRV_CHMAP_RC, SNDRV_CHMAP_NA } }, + { .channels = 6, /* CA_ID 0x06 */ + .map = { SNDRV_CHMAP_FL, SNDRV_CHMAP_FR, SNDRV_CHMAP_NA, + SNDRV_CHMAP_FC, SNDRV_CHMAP_RC, SNDRV_CHMAP_NA } }, + { .channels = 6, /* CA_ID 0x07 */ + .map = { SNDRV_CHMAP_FL, SNDRV_CHMAP_FR, SNDRV_CHMAP_LFE, + SNDRV_CHMAP_FC, SNDRV_CHMAP_RC, SNDRV_CHMAP_NA } }, + { .channels = 6, /* CA_ID 0x08 */ + .map = { SNDRV_CHMAP_FL, SNDRV_CHMAP_FR, SNDRV_CHMAP_NA, + SNDRV_CHMAP_NA, SNDRV_CHMAP_RL, SNDRV_CHMAP_RR } }, + { .channels = 6, /* CA_ID 0x09 */ + .map = { SNDRV_CHMAP_FL, SNDRV_CHMAP_FR, SNDRV_CHMAP_LFE, + SNDRV_CHMAP_NA, SNDRV_CHMAP_RL, SNDRV_CHMAP_RR } }, + { .channels = 6, /* CA_ID 0x0A */ + .map = { SNDRV_CHMAP_FL, SNDRV_CHMAP_FR, SNDRV_CHMAP_NA, + SNDRV_CHMAP_FC, SNDRV_CHMAP_RL, SNDRV_CHMAP_RR } }, + { .channels = 6, /* CA_ID 0x0B */ + .map = { SNDRV_CHMAP_FL, SNDRV_CHMAP_FR, SNDRV_CHMAP_LFE, + SNDRV_CHMAP_FC, SNDRV_CHMAP_RL, SNDRV_CHMAP_RR } }, + { .channels = 8, /* CA_ID 0x0C */ + .map = { SNDRV_CHMAP_FL, SNDRV_CHMAP_FR, SNDRV_CHMAP_NA, + SNDRV_CHMAP_NA, SNDRV_CHMAP_RL, SNDRV_CHMAP_RR, + SNDRV_CHMAP_RC, SNDRV_CHMAP_NA } }, + { .channels = 8, /* CA_ID 0x0D */ + .map = { SNDRV_CHMAP_FL, SNDRV_CHMAP_FR, SNDRV_CHMAP_LFE, + SNDRV_CHMAP_NA, SNDRV_CHMAP_RL, SNDRV_CHMAP_RR, + SNDRV_CHMAP_RC, SNDRV_CHMAP_NA } }, + { .channels = 8, /* CA_ID 0x0E */ + .map = { SNDRV_CHMAP_FL, SNDRV_CHMAP_FR, SNDRV_CHMAP_NA, + SNDRV_CHMAP_FC, SNDRV_CHMAP_RL, SNDRV_CHMAP_RR, + SNDRV_CHMAP_RC, SNDRV_CHMAP_NA } }, + { .channels = 8, /* CA_ID 0x0F */ + .map = { SNDRV_CHMAP_FL, SNDRV_CHMAP_FR, SNDRV_CHMAP_LFE, + SNDRV_CHMAP_FC, SNDRV_CHMAP_RL, SNDRV_CHMAP_RR, + SNDRV_CHMAP_RC, SNDRV_CHMAP_NA } }, + { .channels = 8, /* CA_ID 0x10 */ + .map = { SNDRV_CHMAP_FL, SNDRV_CHMAP_FR, SNDRV_CHMAP_NA, + SNDRV_CHMAP_NA, SNDRV_CHMAP_RL, SNDRV_CHMAP_RR, + SNDRV_CHMAP_RLC, SNDRV_CHMAP_RRC } }, + { .channels = 8, /* CA_ID 0x11 */ + .map = { SNDRV_CHMAP_FL, SNDRV_CHMAP_FR, SNDRV_CHMAP_LFE, + SNDRV_CHMAP_NA, SNDRV_CHMAP_RL, SNDRV_CHMAP_RR, + SNDRV_CHMAP_RLC, SNDRV_CHMAP_RRC } }, + { .channels = 8, /* CA_ID 0x12 */ + .map = { SNDRV_CHMAP_FL, SNDRV_CHMAP_FR, SNDRV_CHMAP_NA, + SNDRV_CHMAP_FC, SNDRV_CHMAP_RL, SNDRV_CHMAP_RR, + SNDRV_CHMAP_RLC, SNDRV_CHMAP_RRC } }, + { .channels = 8, /* CA_ID 0x13 */ + .map = { SNDRV_CHMAP_FL, SNDRV_CHMAP_FR, SNDRV_CHMAP_LFE, + SNDRV_CHMAP_FC, SNDRV_CHMAP_RL, SNDRV_CHMAP_RR, + SNDRV_CHMAP_RLC, SNDRV_CHMAP_RRC } }, + { .channels = 8, /* CA_ID 0x14 */ + .map = { SNDRV_CHMAP_FL, SNDRV_CHMAP_FR, SNDRV_CHMAP_NA, + SNDRV_CHMAP_NA, SNDRV_CHMAP_NA, SNDRV_CHMAP_NA, + SNDRV_CHMAP_FLC, SNDRV_CHMAP_FRC } }, + { .channels = 8, /* CA_ID 0x15 */ + .map = { SNDRV_CHMAP_FL, SNDRV_CHMAP_FR, SNDRV_CHMAP_LFE, + SNDRV_CHMAP_NA, SNDRV_CHMAP_NA, SNDRV_CHMAP_NA, + SNDRV_CHMAP_FLC, SNDRV_CHMAP_FRC } }, + { .channels = 8, /* CA_ID 0x16 */ + .map = { SNDRV_CHMAP_FL, SNDRV_CHMAP_FR, SNDRV_CHMAP_NA, + SNDRV_CHMAP_FC, SNDRV_CHMAP_NA, SNDRV_CHMAP_NA, + SNDRV_CHMAP_FLC, SNDRV_CHMAP_FRC } }, + { .channels = 8, /* CA_ID 0x17 */ + .map = { SNDRV_CHMAP_FL, SNDRV_CHMAP_FR, SNDRV_CHMAP_LFE, + SNDRV_CHMAP_FC, SNDRV_CHMAP_NA, SNDRV_CHMAP_NA, + SNDRV_CHMAP_FLC, SNDRV_CHMAP_FRC } }, + { .channels = 8, /* CA_ID 0x18 */ + .map = { SNDRV_CHMAP_FL, SNDRV_CHMAP_FR, SNDRV_CHMAP_NA, + SNDRV_CHMAP_NA, SNDRV_CHMAP_NA, SNDRV_CHMAP_NA, + SNDRV_CHMAP_FLC, SNDRV_CHMAP_FRC } }, + { .channels = 8, /* CA_ID 0x19 */ + .map = { SNDRV_CHMAP_FL, SNDRV_CHMAP_FR, SNDRV_CHMAP_LFE, + SNDRV_CHMAP_NA, SNDRV_CHMAP_NA, SNDRV_CHMAP_NA, + SNDRV_CHMAP_FLC, SNDRV_CHMAP_FRC } }, + { .channels = 8, /* CA_ID 0x1A */ + .map = { SNDRV_CHMAP_FL, SNDRV_CHMAP_FR, SNDRV_CHMAP_NA, + SNDRV_CHMAP_FC, SNDRV_CHMAP_NA, SNDRV_CHMAP_NA, + SNDRV_CHMAP_FLC, SNDRV_CHMAP_FRC } }, + { .channels = 8, /* CA_ID 0x1B */ + .map = { SNDRV_CHMAP_FL, SNDRV_CHMAP_FR, SNDRV_CHMAP_LFE, + SNDRV_CHMAP_FC, SNDRV_CHMAP_NA, SNDRV_CHMAP_NA, + SNDRV_CHMAP_FLC, SNDRV_CHMAP_FRC } }, + { .channels = 8, /* CA_ID 0x1C */ + .map = { SNDRV_CHMAP_FL, SNDRV_CHMAP_FR, SNDRV_CHMAP_NA, + SNDRV_CHMAP_NA, SNDRV_CHMAP_NA, SNDRV_CHMAP_NA, + SNDRV_CHMAP_FLC, SNDRV_CHMAP_FRC } }, + { .channels = 8, /* CA_ID 0x1D */ + .map = { SNDRV_CHMAP_FL, SNDRV_CHMAP_FR, SNDRV_CHMAP_LFE, + SNDRV_CHMAP_NA, SNDRV_CHMAP_NA, SNDRV_CHMAP_NA, + SNDRV_CHMAP_FLC, SNDRV_CHMAP_FRC } }, + { .channels = 8, /* CA_ID 0x1E */ + .map = { SNDRV_CHMAP_FL, SNDRV_CHMAP_FR, SNDRV_CHMAP_NA, + SNDRV_CHMAP_FC, SNDRV_CHMAP_NA, SNDRV_CHMAP_NA, + SNDRV_CHMAP_FLC, SNDRV_CHMAP_FRC } }, + { .channels = 8, /* CA_ID 0x1F */ + .map = { SNDRV_CHMAP_FL, SNDRV_CHMAP_FR, SNDRV_CHMAP_LFE, + SNDRV_CHMAP_FC, SNDRV_CHMAP_NA, SNDRV_CHMAP_NA, + SNDRV_CHMAP_FLC, SNDRV_CHMAP_FRC } }, + { } +}; diff --git a/drivers/gpu/drm/apple/ibootep.c b/drivers/gpu/drm/apple/ibootep.c new file mode 100644 index 00000000000000..ae4bc8a69f2a8d --- /dev/null +++ b/drivers/gpu/drm/apple/ibootep.c @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright 2023 */ + +#include + +#include "afk.h" +#include "dcp.h" + +static void disp_service_init(struct apple_epic_service *service, const char *name, + const char *class, s64 unit) +{ +} + + +static const struct apple_epic_service_ops ibootep_ops[] = { + { + .name = "disp0-service", + .init = disp_service_init, + }, + {} +}; + +int ibootep_init(struct apple_dcp *dcp) +{ + dcp->ibootep = afk_init(dcp, DISP0_ENDPOINT, ibootep_ops); + afk_start(dcp->ibootep); + + return 0; +} diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c new file mode 100644 index 00000000000000..1d90e4a2597303 --- /dev/null +++ b/drivers/gpu/drm/apple/iomfb.c @@ -0,0 +1,559 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright 2021 Alyssa Rosenzweig */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "dcp.h" +#include "dcp-internal.h" +#include "iomfb.h" +#include "iomfb_internal.h" +#include "parser.h" +#include "trace.h" + +static int dcp_tx_offset(enum dcp_context_id id) +{ + switch (id) { + case DCP_CONTEXT_CB: + case DCP_CONTEXT_CMD: + return 0x00000; + case DCP_CONTEXT_OOBCB: + case DCP_CONTEXT_OOBCMD: + return 0x08000; + default: + return -EINVAL; + } +} + +static int dcp_channel_offset(enum dcp_context_id id) +{ + switch (id) { + case DCP_CONTEXT_ASYNC: + return 0x40000; + case DCP_CONTEXT_OOBASYNC: + return 0x48000; + case DCP_CONTEXT_CB: + return 0x60000; + case DCP_CONTEXT_OOBCB: + return 0x68000; + default: + return dcp_tx_offset(id); + } +} + +static inline u64 dcpep_set_shmem(u64 dart_va) +{ + return FIELD_PREP(IOMFB_MESSAGE_TYPE, IOMFB_MESSAGE_TYPE_SET_SHMEM) | + FIELD_PREP(IOMFB_SHMEM_FLAG, IOMFB_SHMEM_FLAG_VALUE) | + FIELD_PREP(IOMFB_SHMEM_DVA, dart_va); +} + +static inline u64 dcpep_msg(enum dcp_context_id id, u32 length, u16 offset) +{ + return FIELD_PREP(IOMFB_MESSAGE_TYPE, IOMFB_MESSAGE_TYPE_MSG) | + FIELD_PREP(IOMFB_MSG_CONTEXT, id) | + FIELD_PREP(IOMFB_MSG_OFFSET, offset) | + FIELD_PREP(IOMFB_MSG_LENGTH, length); +} + +static inline u64 dcpep_ack(enum dcp_context_id id) +{ + return dcpep_msg(id, 0, 0) | IOMFB_MSG_ACK; +} + +/* + * A channel is busy if we have sent a message that has yet to be + * acked. The driver must not sent a message to a busy channel. + */ +static bool dcp_channel_busy(struct dcp_channel *ch) +{ + return (ch->depth != 0); +} + +/* + * Get the context ID passed to the DCP for a command we push. The rule is + * simple: callback contexts are used when replying to the DCP, command + * contexts are used otherwise. That corresponds to a non/zero call stack + * depth. This rule frees the caller from tracking the call context manually. + */ +static enum dcp_context_id dcp_call_context(struct apple_dcp *dcp, bool oob) +{ + u8 depth = oob ? dcp->ch_oobcmd.depth : dcp->ch_cmd.depth; + + if (depth) + return oob ? DCP_CONTEXT_OOBCB : DCP_CONTEXT_CB; + else + return oob ? DCP_CONTEXT_OOBCMD : DCP_CONTEXT_CMD; +} + +/* Get a channel for a context */ +static struct dcp_channel *dcp_get_channel(struct apple_dcp *dcp, + enum dcp_context_id context) +{ + switch (context) { + case DCP_CONTEXT_CB: + return &dcp->ch_cb; + case DCP_CONTEXT_CMD: + return &dcp->ch_cmd; + case DCP_CONTEXT_OOBCB: + return &dcp->ch_oobcb; + case DCP_CONTEXT_OOBCMD: + return &dcp->ch_oobcmd; + case DCP_CONTEXT_ASYNC: + return &dcp->ch_async; + case DCP_CONTEXT_OOBASYNC: + return &dcp->ch_oobasync; + default: + return NULL; + } +} + +/* Get the start of a packet: after the end of the previous packet */ +static u16 dcp_packet_start(struct dcp_channel *ch, u8 depth) +{ + if (depth > 0) + return ch->end[depth - 1]; + else + return 0; +} + +/* Pushes and pops the depth of the call stack with safety checks */ +static u8 dcp_push_depth(u8 *depth) +{ + u8 ret = (*depth)++; + + WARN_ON(ret >= DCP_MAX_CALL_DEPTH); + return ret; +} + +static u8 dcp_pop_depth(u8 *depth) +{ + WARN_ON((*depth) == 0); + + return --(*depth); +} + +/* Call a DCP function given by a tag */ +void dcp_push(struct apple_dcp *dcp, bool oob, const struct dcp_method_entry *call, + u32 in_len, u32 out_len, void *data, dcp_callback_t cb, + void *cookie) +{ + enum dcp_context_id context = dcp_call_context(dcp, oob); + struct dcp_channel *ch = dcp_get_channel(dcp, context); + + struct dcp_packet_header header = { + .in_len = in_len, + .out_len = out_len, + + /* Tag is reversed due to endianness of the fourcc */ + .tag[0] = call->tag[3], + .tag[1] = call->tag[2], + .tag[2] = call->tag[1], + .tag[3] = call->tag[0], + }; + + u8 depth = dcp_push_depth(&ch->depth); + u16 offset = dcp_packet_start(ch, depth); + + void *out = dcp->shmem + dcp_tx_offset(context) + offset; + void *out_data = out + sizeof(header); + size_t data_len = sizeof(header) + in_len + out_len; + + memcpy(out, &header, sizeof(header)); + + if (in_len > 0) + memcpy(out_data, data, in_len); + + trace_iomfb_push(dcp, call, context, offset, depth); + + ch->callbacks[depth] = cb; + ch->cookies[depth] = cookie; + ch->output[depth] = out + sizeof(header) + in_len; + ch->end[depth] = offset + ALIGN(data_len, DCP_PACKET_ALIGNMENT); + + dcp_send_message(dcp, IOMFB_ENDPOINT, + dcpep_msg(context, data_len, offset)); +} + +/* Parse a callback tag "D123" into the ID 123. Returns -EINVAL on failure. */ +int dcp_parse_tag(char tag[4]) +{ + u32 d[3]; + int i; + + if (tag[3] != 'D') + return -EINVAL; + + for (i = 0; i < 3; ++i) { + d[i] = (u32)(tag[i] - '0'); + + if (d[i] > 9) + return -EINVAL; + } + + return d[0] + (d[1] * 10) + (d[2] * 100); +} + +/* Ack a callback from the DCP */ +void dcp_ack(struct apple_dcp *dcp, enum dcp_context_id context) +{ + struct dcp_channel *ch = dcp_get_channel(dcp, context); + + dcp_pop_depth(&ch->depth); + dcp_send_message(dcp, IOMFB_ENDPOINT, + dcpep_ack(context)); +} + +/* + * Helper to send a DRM hotplug event. The DCP is accessed from a single + * (RTKit) thread. To handle hotplug callbacks, we need to call + * drm_kms_helper_hotplug_event, which does an atomic commit (via DCP) and + * waits for vblank (a DCP callback). That means we deadlock if we call from + * the RTKit thread! Instead, move the call to another thread via a workqueue. + */ +void dcp_hotplug(struct work_struct *work) +{ + struct apple_connector *connector; + struct apple_dcp *dcp; + + connector = container_of(work, struct apple_connector, hotplug_wq); + + dcp = platform_get_drvdata(connector->dcp); + dev_info(dcp->dev, "%s() connected:%d valid_mode:%d nr_modes:%u\n", __func__, + connector->connected, dcp->valid_mode, dcp->nr_modes); + + if (!connector->connected) { + drm_edid_free(connector->drm_edid); + drm_connector_set_vrr_capable_property(&connector->base, false); + connector->drm_edid = NULL; + } + + /* + * DCP defers link training until we set a display mode. But we set + * display modes from atomic_flush, so userspace needs to trigger a + * flush, or the CRTC gets no signal. + */ + if (connector->base.state && !dcp->valid_mode && connector->connected) + drm_connector_set_link_status_property(&connector->base, + DRM_MODE_LINK_STATUS_BAD); + + drm_kms_helper_connector_hotplug_event(&connector->base); +} + +static void dcpep_handle_cb(struct apple_dcp *dcp, enum dcp_context_id context, + void *data, u32 length, u16 offset) +{ + struct device *dev = dcp->dev; + struct dcp_packet_header *hdr = data; + void *in, *out; + int tag = dcp_parse_tag(hdr->tag); + struct dcp_channel *ch = dcp_get_channel(dcp, context); + u8 depth; + + if (tag < 0 || tag >= IOMFB_MAX_CB || !dcp->cb_handlers || !dcp->cb_handlers[tag]) { + dev_warn(dev, "received unknown callback %c%c%c%c\n", + hdr->tag[3], hdr->tag[2], hdr->tag[1], hdr->tag[0]); + return; + } + + in = data + sizeof(*hdr); + out = in + hdr->in_len; + + // TODO: verify that in_len and out_len match our prototypes + // for now just clear the out data to have at least consistent results + if (hdr->out_len) + memset(out, 0, hdr->out_len); + + depth = dcp_push_depth(&ch->depth); + ch->output[depth] = out; + ch->end[depth] = offset + ALIGN(length, DCP_PACKET_ALIGNMENT); + + if (dcp->cb_handlers[tag](dcp, tag, out, in)) + dcp_ack(dcp, context); +} + +static void dcpep_handle_ack(struct apple_dcp *dcp, enum dcp_context_id context, + void *data, u32 length) +{ + struct dcp_packet_header *header = data; + struct dcp_channel *ch = dcp_get_channel(dcp, context); + void *cookie; + dcp_callback_t cb; + + if (!ch) { + dev_warn(dcp->dev, "ignoring ack on context %X\n", context); + return; + } + + dcp_pop_depth(&ch->depth); + + cb = ch->callbacks[ch->depth]; + cookie = ch->cookies[ch->depth]; + + ch->callbacks[ch->depth] = NULL; + ch->cookies[ch->depth] = NULL; + + if (cb) + cb(dcp, data + sizeof(*header) + header->in_len, cookie); +} + +static void dcpep_got_msg(struct apple_dcp *dcp, u64 message) +{ + enum dcp_context_id ctx_id; + u16 offset; + u32 length; + int channel_offset; + void *data; + + ctx_id = FIELD_GET(IOMFB_MSG_CONTEXT, message); + offset = FIELD_GET(IOMFB_MSG_OFFSET, message); + length = FIELD_GET(IOMFB_MSG_LENGTH, message); + + channel_offset = dcp_channel_offset(ctx_id); + + if (channel_offset < 0) { + dev_warn(dcp->dev, "invalid context received %u\n", ctx_id); + return; + } + + data = dcp->shmem + channel_offset + offset; + + if (FIELD_GET(IOMFB_MSG_ACK, message)) + dcpep_handle_ack(dcp, ctx_id, data, length); + else + dcpep_handle_cb(dcp, ctx_id, data, length, offset); +} + +int dcp_get_modes(struct drm_connector *connector) +{ + struct apple_connector *apple_connector = to_apple_connector(connector); + struct platform_device *pdev = apple_connector->dcp; + struct apple_dcp *dcp = platform_get_drvdata(pdev); + + struct drm_device *dev = connector->dev; + struct drm_display_mode *mode; + int i; + + for (i = 0; i < dcp->nr_modes; ++i) { + mode = drm_mode_duplicate(dev, &dcp->modes[i].mode); + + if (!mode) { + dev_err(dev->dev, "Failed to duplicate display mode\n"); + return 0; + } + + drm_mode_probed_add(connector, mode); + } + + if (dcp->nr_modes && dcp->dcpavserv.enabled && + !apple_connector->drm_edid) { + const struct drm_edid *edid; + edid = dcpavserv_copy_edid(dcp->dcpavserv.service); + if (IS_ERR_OR_NULL(edid)) { + dev_info(dcp->dev, "copy_edid failed: %pe\n", edid); + } else { + drm_edid_free(apple_connector->drm_edid); + apple_connector->drm_edid = edid; + } + } + if (dcp->nr_modes && apple_connector->drm_edid) + drm_edid_connector_update(connector, apple_connector->drm_edid); + + return dcp->nr_modes; +} + +/* The user may own drm_display_mode, so we need to search for our copy */ +struct dcp_display_mode *lookup_mode(struct apple_dcp *dcp, + const struct drm_display_mode *mode) +{ + int i; + + for (i = 0; i < dcp->nr_modes; ++i) { + if (drm_mode_match(mode, &dcp->modes[i].mode, + DRM_MODE_MATCH_TIMINGS | + DRM_MODE_MATCH_CLOCK)) + return &dcp->modes[i]; + } + + return NULL; +} + +enum drm_mode_status dcp_mode_valid(struct drm_connector *connector, + const struct drm_display_mode *mode) +{ + struct apple_connector *apple_connector = to_apple_connector(connector); + struct platform_device *pdev = apple_connector->dcp; + struct apple_dcp *dcp = platform_get_drvdata(pdev); + + return lookup_mode(dcp, mode) ? MODE_OK : MODE_BAD; +} + +int dcp_crtc_atomic_modeset(struct drm_crtc *crtc, + struct drm_atomic_state *state) +{ + struct apple_crtc *apple_crtc = to_apple_crtc(crtc); + struct apple_dcp *dcp = platform_get_drvdata(apple_crtc->dcp); + struct drm_crtc_state *crtc_state; + int ret = -EIO; + bool modeset; + + crtc_state = drm_atomic_get_new_crtc_state(state, crtc); + if (!crtc_state) + return 0; + + modeset = drm_atomic_crtc_needs_modeset(crtc_state) || !dcp->valid_mode; + + if (!modeset) + return 0; + + /* ignore no mode, poweroff is handled elsewhere */ + if (crtc_state->mode.hdisplay == 0 && crtc_state->mode.vdisplay == 0) + return 0; + + switch (dcp->fw_compat) { + case DCP_FIRMWARE_V_12_3: + ret = iomfb_modeset_v12_3(dcp, crtc_state); + break; + case DCP_FIRMWARE_V_13_5: + ret = iomfb_modeset_v13_3(dcp, crtc_state); + break; + default: + WARN_ONCE(true, "Unexpected firmware version: %u\n", + dcp->fw_compat); + break; + } + + return ret; +} + +bool dcp_crtc_mode_fixup(struct drm_crtc *crtc, + const struct drm_display_mode *mode, + struct drm_display_mode *adjusted_mode) +{ + struct apple_crtc *apple_crtc = to_apple_crtc(crtc); + struct platform_device *pdev = apple_crtc->dcp; + struct apple_dcp *dcp = platform_get_drvdata(pdev); + + /* TODO: support synthesized modes through scaling */ + return lookup_mode(dcp, mode) != NULL; +} + + +void dcp_flush(struct drm_crtc *crtc, struct drm_atomic_state *state) +{ + struct platform_device *pdev = to_apple_crtc(crtc)->dcp; + struct apple_dcp *dcp = platform_get_drvdata(pdev); + + if (dcp_channel_busy(&dcp->ch_cmd)) + { + if (!dcp->ch_cmd.warned_busy) { + dev_err(dcp->dev, "unexpected busy command channel\n"); + dcp->ch_cmd.warned_busy = true; + } + /* HACK: issue a delayed vblank event to avoid timeouts in + * drm_atomic_helper_wait_for_vblanks(). + */ + schedule_work(&dcp->vblank_wq); + return; + } else if (dcp->ch_cmd.warned_busy) { + dcp->ch_cmd.warned_busy = false; + } + + switch (dcp->fw_compat) { + case DCP_FIRMWARE_V_12_3: + iomfb_flush_v12_3(dcp, crtc, state); + break; + case DCP_FIRMWARE_V_13_5: + iomfb_flush_v13_3(dcp, crtc, state); + break; + default: + WARN_ONCE(true, "Unexpected firmware version: %u\n", dcp->fw_compat); + break; + } +} + +static void iomfb_start(struct apple_dcp *dcp) +{ + switch (dcp->fw_compat) { + case DCP_FIRMWARE_V_12_3: + iomfb_start_v12_3(dcp); + break; + case DCP_FIRMWARE_V_13_5: + iomfb_start_v13_3(dcp); + break; + default: + WARN_ONCE(true, "Unexpected firmware version: %u\n", dcp->fw_compat); + break; + } +} + +bool dcp_is_initialized(struct platform_device *pdev) +{ + struct apple_dcp *dcp = platform_get_drvdata(pdev); + + return dcp->active; +} + +void iomfb_recv_msg(struct apple_dcp *dcp, u64 message) +{ + enum dcpep_type type = FIELD_GET(IOMFB_MESSAGE_TYPE, message); + + if (type == IOMFB_MESSAGE_TYPE_INITIALIZED) + iomfb_start(dcp); + else if (type == IOMFB_MESSAGE_TYPE_MSG) + dcpep_got_msg(dcp, message); + else + dev_warn(dcp->dev, "Ignoring unknown message %llx\n", message); +} + +int iomfb_start_rtkit(struct apple_dcp *dcp) +{ + dma_addr_t shmem_iova; + apple_rtkit_start_ep(dcp->rtk, IOMFB_ENDPOINT); + + dcp->shmem = dma_alloc_coherent(dcp->dev, DCP_SHMEM_SIZE, &shmem_iova, + GFP_KERNEL); + + dcp_send_message(dcp, IOMFB_ENDPOINT, dcpep_set_shmem(shmem_iova)); + + return 0; +} + +void iomfb_shutdown(struct apple_dcp *dcp) +{ + /* We're going down */ + dcp->active = false; + dcp->valid_mode = false; + + switch (dcp->fw_compat) { + case DCP_FIRMWARE_V_12_3: + iomfb_shutdown_v12_3(dcp); + break; + case DCP_FIRMWARE_V_13_5: + iomfb_shutdown_v13_3(dcp); + break; + default: + WARN_ONCE(true, "Unexpected firmware version: %u\n", dcp->fw_compat); + break; + } +} diff --git a/drivers/gpu/drm/apple/iomfb.h b/drivers/gpu/drm/apple/iomfb.h new file mode 100644 index 00000000000000..8a871db0b94a70 --- /dev/null +++ b/drivers/gpu/drm/apple/iomfb.h @@ -0,0 +1,403 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright 2021 Alyssa Rosenzweig */ + +#ifndef __APPLE_DCPEP_H__ +#define __APPLE_DCPEP_H__ + +#include + +#include "version_utils.h" + +/* Fixed size of shared memory between DCP and AP */ +#define DCP_SHMEM_SIZE 0x100000 + +/* DCP message contexts */ +enum dcp_context_id { + /* Callback */ + DCP_CONTEXT_CB = 0, + + /* Command */ + DCP_CONTEXT_CMD = 2, + + /* Asynchronous */ + DCP_CONTEXT_ASYNC = 3, + + /* Out-of-band callback */ + DCP_CONTEXT_OOBCB = 4, + + /* Out-of-band command */ + DCP_CONTEXT_OOBCMD = 6, + + /* Out-of-band Asynchronous */ + DCP_CONTEXT_OOBASYNC = 7, + + DCP_NUM_CONTEXTS +}; + +/* RTKit endpoint message types */ +enum dcpep_type { + /* Set shared memory */ + IOMFB_MESSAGE_TYPE_SET_SHMEM = 0, + + /* DCP is initialized */ + IOMFB_MESSAGE_TYPE_INITIALIZED = 1, + + /* Remote procedure call */ + IOMFB_MESSAGE_TYPE_MSG = 2, +}; + +/* + * IOMFB supports the setting of a number of parameters + * that alter various aspects of the connected sink's + * behaviour at runtime. + */ +enum iomfb_parameter { + IOMFBPARAM_ADAPTIVE_SYNC = 14, +}; + +#define IOMFB_MESSAGE_TYPE GENMASK_ULL( 3, 0) + +/* Message */ +#define IOMFB_MSG_LENGTH GENMASK_ULL(63, 32) +#define IOMFB_MSG_OFFSET GENMASK_ULL(31, 16) +#define IOMFB_MSG_CONTEXT GENMASK_ULL(11, 8) +#define IOMFB_MSG_ACK BIT_ULL(6) + +/* Set shmem */ +#define IOMFB_SHMEM_DVA GENMASK_ULL(63, 16) +#define IOMFB_SHMEM_FLAG GENMASK_ULL( 7, 4) +#define IOMFB_SHMEM_FLAG_VALUE 4 + +struct dcp_packet_header { + char tag[4]; + u32 in_len; + u32 out_len; +} __packed; + +#define DCP_IS_NULL(ptr) ((ptr) ? 1 : 0) +#define DCP_PACKET_ALIGNMENT (0x40) + +enum iomfb_property_id { + IOMFB_PROPERTY_NITS = 15, // divide by Brightness_Scale +}; + +#define IOMFB_BRIGHTNESS_MIN 0x10000000 + +/* Structures used in v12.0 firmware */ + +#define SWAP_SURFACES 4 +/* We have 4 surfaces, but we can only ever blend two */ +#define MAX_BLEND_SURFACES 2 + +struct dcp_iouserclient { + /* Handle for the IOUserClient. macOS sets this to a kernel VA. */ + u64 handle; + u32 unk; + u8 flag1; + u8 flag2; + u8 padding[2]; +} __packed; + +/* + * Update background color to struct dcp_swap.bg_color + */ +#define IOMFB_SET_BACKGROUND BIT(31) + +struct dcp_allocate_bandwidth_req { + u64 unk1; + u64 unk2; + u64 unk3; + u8 unk1_null; + u8 unk2_null; + u8 padding[8]; +} __packed; + +struct dcp_allocate_bandwidth_resp { + u64 unk1; + u64 unk2; + u32 ret; +} __packed; + +struct dcp_rt_bandwidth { + u64 unk1; + u64 reg_scratch; + u64 reg_doorbell; + u32 unk2; + u32 doorbell_bit; + u32 padding[7]; +} __packed; + +struct frame_sync_props { + u8 unk[28]; +}; + +struct dcp_set_frame_sync_props_req { + struct frame_sync_props props; + u8 frame_sync_props_null; + u8 padding[3]; +} __packed; + +struct dcp_set_frame_sync_props_resp { + struct frame_sync_props props; +} __packed; + +/* Method calls */ + +enum dcpep_method { + dcpep_late_init_signal, + dcpep_setup_video_limits, + dcpep_set_create_dfb, + dcpep_start_signal, + dcpep_swap_start, + dcpep_swap_submit, + dcpep_set_display_device, + dcpep_set_digital_out_mode, + dcpep_create_default_fb, + dcpep_set_display_refresh_properties, + dcpep_flush_supports_power, + dcpep_set_power_state, + dcpep_first_client_open, + dcpep_set_parameter_dcp, + dcpep_enable_disable_video_power_savings, + dcpep_is_main_display, + iomfbep_a131_pmu_service_matched, + iomfbep_a132_backlight_service_matched, + iomfbep_a358_vi_set_temperature_hint, + iomfbep_get_color_remap_mode, + iomfbep_last_client_close, + iomfbep_abort_swaps_dcp, + iomfbep_set_matrix, + dcpep_num_methods +}; + +#define IOMFB_METHOD(tag, name) [name] = { #name, { tag[0], tag[1], tag[2], tag[3] } } + +struct dcp_method_entry { + const char *name; + char tag[4]; +}; + +#define IOMFB_MAX_CB (1000) +struct apple_dcp; + +typedef bool (*iomfb_cb_handler)(struct apple_dcp *, int, void *, void *); + +/* Prototypes */ + +struct dcp_set_digital_out_mode_req { + u32 color_mode_id; + u32 timing_mode_id; +} __packed; + +struct dcp_map_buf_req { + u64 buffer; + u8 unk; + u8 buf_null; + u8 vaddr_null; + u8 dva_null; +} __packed; + +struct dcp_map_buf_resp { + u64 vaddr; + u64 dva; + u32 ret; +} __packed; + +struct dcp_unmap_buf_resp { + u64 buffer; + u64 vaddr; + u64 dva; + u8 unk; + u8 buf_null; +} __packed; + +struct dcp_allocate_buffer_req { + u32 unk0; + u64 size; + u32 unk2; + u8 paddr_null; + u8 dva_null; + u8 dva_size_null; + u8 padding; +} __packed; + +struct dcp_allocate_buffer_resp { + u64 paddr; + u64 dva; + u64 dva_size; + u32 mem_desc_id; +} __packed; + +struct dcp_map_physical_req { + u64 paddr; + u64 size; + u32 flags; + u8 dva_null; + u8 dva_size_null; + u8 padding[2]; +} __packed; + +struct dcp_map_physical_resp { + u64 dva; + u64 dva_size; + u32 mem_desc_id; +} __packed; + +struct dcp_swap_start_req { + u32 swap_id; + struct dcp_iouserclient client; + u8 swap_id_null; + u8 client_null; + u8 padding[2]; +} __packed; + +struct dcp_swap_start_resp { + u32 swap_id; + struct dcp_iouserclient client; + u32 ret; +} __packed; + +struct dcp_get_uint_prop_req { + char obj[4]; + char key[0x40]; + u64 value; + u8 value_null; + u8 padding[3]; +} __packed; + +struct dcp_get_uint_prop_resp { + u64 value; + u8 ret; + u8 padding[3]; +} __packed; + +struct iomfb_sr_set_property_int_req { + char obj[4]; + char key[0x40]; + u64 value; + u8 value_null; + u8 padding[3]; +} __packed; + +struct iomfb_set_fx_prop_req { + char obj[4]; + char key[0x40]; + u32 value; +} __packed; + +struct dcp_set_power_state_req { + u64 unklong; + u8 unkbool; + u8 unkint_null; + u8 padding[2]; +} __packed; + +struct dcp_set_power_state_resp { + u32 unkint; + u32 ret; +} __packed; + +struct dcp_set_dcpav_prop_chunk_req { + char data[0x1000]; + u32 offset; + u32 length; +} __packed; + +struct dcp_set_dcpav_prop_end_req { + char key[0x40]; +} __packed; + +struct dcp_set_parameter_dcp { + u32 param; + u32 value[8]; + u32 count; +} __packed; + +struct dcp_swap_complete_intent_gated { + u32 swap_id; + u8 unkBool; + u32 unkInt; + u32 width; + u32 height; +} __packed; + +struct dcp_read_edt_data_req { + char key[0x40]; + u32 count; + u32 value[8]; +} __packed; + +struct dcp_read_edt_data_resp { + u32 value[8]; + u8 ret; +} __packed; + +struct iomfb_property { + u32 id; + u32 value; +} __packed; + +struct iomfb_get_color_remap_mode_req { + u32 mode; + u8 mode_null; + u8 padding[3]; +} __packed; + +struct iomfb_get_color_remap_mode_resp { + u32 mode; + u32 ret; +} __packed; + +struct iomfb_last_client_close_req { + u8 unkint_null; + u8 padding[3]; +} __packed; + +struct iomfb_last_client_close_resp { + u32 unkint; +} __packed; + +struct io_user_client { + u64 addr; + u32 unk; + u8 flag1; + u8 flag2; + u8 pad[2]; +} __packed; + +struct iomfb_abort_swaps_dcp_req { + struct io_user_client client; + u8 client_null; + u8 pad[3]; +} __packed; + +struct iomfb_abort_swaps_dcp_resp { + struct io_user_client client; + u32 ret; +} __packed; + +struct iomfb_set_matrix_req { + u32 location; + u64 matrix[9]; + u8 matrix_null; + u8 padding[3]; +} __packed; + +struct iomfb_set_matrix_resp { + u32 ret; +} __packed; + +struct dcpep_get_tiling_state_req { + u32 event; + u32 param; + u32 value; + u8 value_null; + u8 padding[3]; +} __packed; + +struct dcpep_get_tiling_state_resp { + u32 value; + u32 ret; +} __packed; + +#endif diff --git a/drivers/gpu/drm/apple/iomfb_internal.h b/drivers/gpu/drm/apple/iomfb_internal.h new file mode 100644 index 00000000000000..75e9d7b0e8cc84 --- /dev/null +++ b/drivers/gpu/drm/apple/iomfb_internal.h @@ -0,0 +1,114 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright The Asahi Linux Contributors */ + +#include +#include + +#include "dcp-internal.h" + +struct apple_dcp; + +typedef void (*dcp_callback_t)(struct apple_dcp *, void *, void *); + + +#define DCP_THUNK_VOID(func, handle) \ + static void func(struct apple_dcp *dcp, bool oob, dcp_callback_t cb, \ + void *cookie) \ + { \ + dcp_push(dcp, oob, &dcp_methods[handle], 0, 0, NULL, cb, cookie); \ + } + +#define DCP_THUNK_OUT(func, handle, T) \ + static void func(struct apple_dcp *dcp, bool oob, dcp_callback_t cb, \ + void *cookie) \ + { \ + dcp_push(dcp, oob, &dcp_methods[handle], 0, sizeof(T), NULL, cb, cookie); \ + } + +#define DCP_THUNK_IN(func, handle, T) \ + static void func(struct apple_dcp *dcp, bool oob, T *data, \ + dcp_callback_t cb, void *cookie) \ + { \ + dcp_push(dcp, oob, &dcp_methods[handle], sizeof(T), 0, data, cb, cookie); \ + } + +#define DCP_THUNK_INOUT(func, handle, T_in, T_out) \ + static void func(struct apple_dcp *dcp, bool oob, T_in *data, \ + dcp_callback_t cb, void *cookie) \ + { \ + dcp_push(dcp, oob, &dcp_methods[handle], sizeof(T_in), sizeof(T_out), data, \ + cb, cookie); \ + } + +#define IOMFB_THUNK_INOUT(name) \ + static void iomfb_ ## name(struct apple_dcp *dcp, bool oob, \ + struct iomfb_ ## name ## _req *data, \ + dcp_callback_t cb, void *cookie) \ + { \ + dcp_push(dcp, oob, &dcp_methods[iomfbep_ ## name], \ + sizeof(struct iomfb_ ## name ## _req), \ + sizeof(struct iomfb_ ## name ## _resp), \ + data, cb, cookie); \ + } + +/* + * Define type-safe trampolines. Define typedefs to enforce type-safety on the + * input data (so if the types don't match, gcc errors out). + */ + +#define TRAMPOLINE_VOID(func, handler) \ + static bool __maybe_unused func(struct apple_dcp *dcp, int tag, void *out, void *in) \ + { \ + trace_iomfb_callback(dcp, tag, #handler); \ + handler(dcp); \ + return true; \ + } + +#define TRAMPOLINE_IN(func, handler, T_in) \ + typedef void (*callback_##handler)(struct apple_dcp *, T_in *); \ + \ + static bool __maybe_unused func(struct apple_dcp *dcp, int tag, void *out, void *in) \ + { \ + callback_##handler cb = handler; \ + \ + trace_iomfb_callback(dcp, tag, #handler); \ + cb(dcp, in); \ + return true; \ + } + +#define TRAMPOLINE_INOUT(func, handler, T_in, T_out) \ + typedef T_out (*callback_##handler)(struct apple_dcp *, T_in *); \ + \ + static bool __maybe_unused func(struct apple_dcp *dcp, int tag, void *out, void *in) \ + { \ + T_out *typed_out = out; \ + callback_##handler cb = handler; \ + \ + trace_iomfb_callback(dcp, tag, #handler); \ + *typed_out = cb(dcp, in); \ + return true; \ + } + +#define TRAMPOLINE_OUT(func, handler, T_out) \ + static bool __maybe_unused func(struct apple_dcp *dcp, int tag, void *out, void *in) \ + { \ + T_out *typed_out = out; \ + \ + trace_iomfb_callback(dcp, tag, #handler); \ + *typed_out = handler(dcp); \ + return true; \ + } + +/* Call a DCP function given by a tag */ +void dcp_push(struct apple_dcp *dcp, bool oob, const struct dcp_method_entry *call, + u32 in_len, u32 out_len, void *data, dcp_callback_t cb, + void *cookie); + +/* Parse a callback tag "D123" into the ID 123. Returns -EINVAL on failure. */ +int dcp_parse_tag(char tag[4]); + +void dcp_ack(struct apple_dcp *dcp, enum dcp_context_id context); + +/* The user may own drm_display_mode, so we need to search for our copy */ +struct dcp_display_mode *lookup_mode(struct apple_dcp *dcp, + const struct drm_display_mode *mode); diff --git a/drivers/gpu/drm/apple/iomfb_plane.h b/drivers/gpu/drm/apple/iomfb_plane.h new file mode 100644 index 00000000000000..0701978200311a --- /dev/null +++ b/drivers/gpu/drm/apple/iomfb_plane.h @@ -0,0 +1,109 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Copyright (C) The Asahi Linux Contributors + */ + + +#ifndef __APPLE_IOMFB_PLANE_H__ +#define __APPLE_IOMFB_PLANE_H__ + +#include + +#include + +#define DCP_SURF_MAX_PLANES 3 + +#define DCP_FORMAT_BGRA fourcc_code('A', 'R', 'G', 'B') +#define DCP_FORMAT_RGBA fourcc_code('A', 'B', 'G', 'R') + +#define DCP_FORMAT_W30R fourcc_code('r', '0', '3', 'w') // wide gamut packed 10-bit RGB without alpha +#define DCP_FORMAT_L10R fourcc_code('r', '0', '1', 'l') // full range packed 10-bit RGB with alpha + +#define DCP_FORMAT_420V fourcc_code('v', '0', '2', '4') // NV12 video range 2 plane 8-bit YCbCr +#define DCP_FORMAT_420F fourcc_code('f', '0', '2', '4') // NV12 full range 2 plane 8-bit YCbCr +#define DCP_FORMAT_422V fourcc_code('v', '2', '2', '4') // NV16 video range 2 plane 8-bit YCbCr +#define DCP_FORMAT_422F fourcc_code('f', '2', '2', '4') // NV16 full range 2 plane 8-bit YCbCr +#define DCP_FORMAT_444V fourcc_code('v', '4', '4', '4') // NV24 video range 2 plane 8-bit YCbCr +#define DCP_FORMAT_444F fourcc_code('f', '4', '4', '4') // NV24 full range 2 plane 8-bit YCbCr + +#define DCP_FORMAT_X420 fourcc_code('0', '2', '4', 'x') // P010 video range 2 plane 10-bit YCbCR +#define DCP_FORMAT_X422 fourcc_code('2', '2', '4', 'x') // P210 video range 2 plane 10-bit YCbCR +#define DCP_FORMAT_X444 fourcc_code('4', '4', '4', 'x') // P410 video range 2 plane 10-bit YCbCR + +#define DCP_FORMAT_XF20 fourcc_code('0', '2', 'f', 'x') // P010 full range 2 plane 10-bit YCbCR +#define DCP_FORMAT_XF22 fourcc_code('2', '2', 'f', 'x') // P210 full range 2 plane 10-bit YCbCR +#define DCP_FORMAT_XF44 fourcc_code('4', '4', 'f', 'x') // P410 full range 2 plane 10-bit YCbCR + +enum dcp_colorspace { + DCP_COLORSPACE_BG_SRGB = 0, + DCP_COLORSPACE_BT601 = 1, + DCP_COLORSPACE_BT709 = 2, + DCP_COLORSPACE_BG_BT2020 = 9, + DCP_COLORSPACE_NATIVE = 12, +}; + +enum dcp_xfer_func { + DCP_XFER_FUNC_BT601 = 1, + DCP_XFER_FUNC_BT1886 = 2, + DCP_XFER_FUNC_SDR = 13, + DCP_XFER_FUNC_HDR = 16, +}; + +struct dcp_rect { + u32 x; + u32 y; + u32 w; + u32 h; +} __packed; + +/* Information describing a plane of a planar compressed surface */ +struct dcp_plane_info { + u32 width; + u32 height; + u32 base; + u32 offset; + u32 stride; + u32 size; + u16 tile_size; + u8 tile_w; + u8 tile_h; + u32 unk[13]; +} __packed; + +struct dcp_component_types { + u8 count; + u8 types[7]; +} __packed; + +/* Information describing a surface */ +struct dcp_surface { + u8 is_tiled; + u8 is_tearing_allowed; + u8 is_premultiplied; + u32 plane_cnt; + u32 plane_cnt2; + u32 format; /* DCP fourcc */ + u32 ycbcr_matrix; + u8 xfer_func; + u8 colorspace; + u32 stride; + u16 pix_size; + u8 pel_w; + u8 pel_h; + u32 offset; + u32 width; + u32 height; + u32 buf_size; + u64 protection_opts; + u32 surface_id; + struct dcp_component_types comp_types[DCP_SURF_MAX_PLANES]; + u64 has_comp; + struct dcp_plane_info planes[DCP_SURF_MAX_PLANES]; + u64 has_planes; + u32 compression_info[DCP_SURF_MAX_PLANES][13]; + u64 has_compr_info; + u32 unk_num; + u32 unk_denom; +} __packed; + +#endif /* __APPLE_IOMFB_PLANE_H__ */ diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c new file mode 100644 index 00000000000000..0e5d5908a3c9b6 --- /dev/null +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -0,0 +1,1521 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Copyright 2021 Alyssa Rosenzweig + * Copyright The Asahi Linux Contributors + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "dcp.h" +#include "dcp-internal.h" +#include "iomfb.h" +#include "iomfb_internal.h" +#include "parser.h" +#include "trace.h" +#include "version_utils.h" + +/* Register defines used in bandwidth setup structure */ +#define REG_DOORBELL_BIT(idx) (2 + (idx)) + +extern bool force_vrr; + +struct dcp_wait_cookie { + struct kref refcount; + struct completion done; +}; + +static void release_wait_cookie(struct kref *ref) +{ + struct dcp_wait_cookie *cookie; + cookie = container_of(ref, struct dcp_wait_cookie, refcount); + + kfree(cookie); +} + +DCP_THUNK_OUT(iomfb_a131_pmu_service_matched, iomfbep_a131_pmu_service_matched, u32); +DCP_THUNK_OUT(iomfb_a132_backlight_service_matched, iomfbep_a132_backlight_service_matched, u32); +DCP_THUNK_OUT(iomfb_a358_vi_set_temperature_hint, iomfbep_a358_vi_set_temperature_hint, u32); + +IOMFB_THUNK_INOUT(set_matrix); +IOMFB_THUNK_INOUT(get_color_remap_mode); +IOMFB_THUNK_INOUT(last_client_close); +IOMFB_THUNK_INOUT(abort_swaps_dcp); + +DCP_THUNK_INOUT(dcp_swap_submit, dcpep_swap_submit, + struct DCP_FW_NAME(dcp_swap_submit_req), + struct DCP_FW_NAME(dcp_swap_submit_resp)); + +DCP_THUNK_INOUT(dcp_swap_start, dcpep_swap_start, struct dcp_swap_start_req, + struct dcp_swap_start_resp); + +DCP_THUNK_INOUT(dcp_set_power_state, dcpep_set_power_state, + struct dcp_set_power_state_req, + struct dcp_set_power_state_resp); + +DCP_THUNK_INOUT(dcp_set_digital_out_mode, dcpep_set_digital_out_mode, + struct dcp_set_digital_out_mode_req, u32); + +DCP_THUNK_INOUT(dcp_set_display_device, dcpep_set_display_device, u32, u32); + +DCP_THUNK_OUT(dcp_set_display_refresh_properties, + dcpep_set_display_refresh_properties, u32); + +#if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) +DCP_THUNK_INOUT(dcp_late_init_signal, dcpep_late_init_signal, u32, u32); +#else +DCP_THUNK_OUT(dcp_late_init_signal, dcpep_late_init_signal, u32); +#endif +DCP_THUNK_IN(dcp_flush_supports_power, dcpep_flush_supports_power, u32); +DCP_THUNK_OUT(dcp_create_default_fb, dcpep_create_default_fb, u32); +DCP_THUNK_OUT(dcp_start_signal, dcpep_start_signal, u32); +DCP_THUNK_VOID(dcp_setup_video_limits, dcpep_setup_video_limits); +DCP_THUNK_VOID(dcp_set_create_dfb, dcpep_set_create_dfb); +DCP_THUNK_VOID(dcp_first_client_open, dcpep_first_client_open); + +DCP_THUNK_INOUT(dcp_set_parameter_dcp, dcpep_set_parameter_dcp, + struct dcp_set_parameter_dcp, u32); + +DCP_THUNK_INOUT(dcp_enable_disable_video_power_savings, + dcpep_enable_disable_video_power_savings, u32, int); + +DCP_THUNK_OUT(dcp_is_main_display, dcpep_is_main_display, u32); + +/* DCP callback handlers */ +static void dcpep_cb_nop(struct apple_dcp *dcp) +{ + /* No operation */ +} + +static u8 dcpep_cb_true(struct apple_dcp *dcp) +{ + return true; +} + +static u8 dcpep_cb_false(struct apple_dcp *dcp) +{ + return false; +} + +static u32 dcpep_cb_zero(struct apple_dcp *dcp) +{ + return 0; +} + +static void dcpep_cb_swap_complete(struct apple_dcp *dcp, + struct DCP_FW_NAME(dc_swap_complete_resp) *resp) +{ + ktime_t now = ktime_get(); + trace_iomfb_swap_complete(dcp, resp->swap_id); + dcp->last_swap_id = resp->swap_id; + + dcp_drm_crtc_page_flip(dcp, now); + if (dcp->crc_enabled) { + u32 crc32 = 0; + drm_crtc_add_crc_entry(&dcp->crtc->base, true, resp->swap_id, &crc32); + } +} + +/* special */ +static void complete_vi_set_temperature_hint(struct apple_dcp *dcp, void *out, void *cookie) +{ + // ack D100 cb_match_pmu_service + dcp_ack(dcp, DCP_CONTEXT_CB); +} + +static bool iomfbep_cb_match_pmu_service(struct apple_dcp *dcp, int tag, void *out, void *in) +{ + trace_iomfb_callback(dcp, tag, __func__); + iomfb_a358_vi_set_temperature_hint(dcp, false, + complete_vi_set_temperature_hint, + NULL); + + // return false for deferred ACK + return false; +} + +static void complete_pmu_service_matched(struct apple_dcp *dcp, void *out, void *cookie) +{ + struct dcp_channel *ch = &dcp->ch_cb; + u8 *succ = ch->output[ch->depth - 1]; + + *succ = true; + + // ack D206 cb_match_pmu_service_2 + dcp_ack(dcp, DCP_CONTEXT_CB); +} + +static bool iomfbep_cb_match_pmu_service_2(struct apple_dcp *dcp, int tag, void *out, void *in) +{ + trace_iomfb_callback(dcp, tag, __func__); + + iomfb_a131_pmu_service_matched(dcp, false, complete_pmu_service_matched, + out); + + // return false for deferred ACK + return false; +} + +static void complete_backlight_service_matched(struct apple_dcp *dcp, void *out, void *cookie) +{ + struct dcp_channel *ch = &dcp->ch_cb; + u8 *succ = ch->output[ch->depth - 1]; + + *succ = true; + + // ack D206 cb_match_backlight_service + dcp_ack(dcp, DCP_CONTEXT_CB); +} + +static bool iomfbep_cb_match_backlight_service(struct apple_dcp *dcp, int tag, void *out, void *in) +{ + trace_iomfb_callback(dcp, tag, __func__); + + if (!dcp_has_panel(dcp)) { + u8 *succ = out; + *succ = true; + return true; + } + + iomfb_a132_backlight_service_matched(dcp, false, complete_backlight_service_matched, out); + + // return false for deferred ACK + return false; +} + +static void iomfb_cb_pr_publish(struct apple_dcp *dcp, struct iomfb_property *prop) +{ + switch (prop->id) { + case IOMFB_PROPERTY_NITS: + { + if (dcp_has_panel(dcp)) { + dcp->brightness.nits = prop->value / dcp->brightness.scale; + /* notify backlight device of the initial brightness */ + if (!dcp->brightness.bl_dev && dcp->brightness.maximum > 0) + schedule_work(&dcp->bl_register_wq); + trace_iomfb_brightness(dcp, prop->value); + } + break; + } + default: + dev_dbg(dcp->dev, "pr_publish: id: %d = %u\n", prop->id, prop->value); + } +} + +static struct dcp_get_uint_prop_resp +dcpep_cb_get_uint_prop(struct apple_dcp *dcp, struct dcp_get_uint_prop_req *req) +{ + struct dcp_get_uint_prop_resp resp = (struct dcp_get_uint_prop_resp){ + .value = 0 + }; + + if (dcp->panel.has_mini_led && + memcmp(req->obj, "SUMP", sizeof(req->obj)) == 0) { /* "PMUS */ + if (strncmp(req->key, "Temperature", sizeof(req->key)) == 0) { + /* + * TODO: value from j314c, find out if it is temperature in + * centigrade C and which temperature sensor reports it + */ + resp.value = 3029; + resp.ret = true; + } + } + + return resp; +} + +static u8 iomfbep_cb_sr_set_property_int(struct apple_dcp *dcp, + struct iomfb_sr_set_property_int_req *req) +{ + if (memcmp(req->obj, "FMOI", sizeof(req->obj)) == 0) { /* "IOMF */ + if (strncmp(req->key, "Brightness_Scale", sizeof(req->key)) == 0) { + if (!req->value_null) + dcp->brightness.scale = req->value; + } + } + + return 1; +} + +static void iomfbep_cb_set_fx_prop(struct apple_dcp *dcp, struct iomfb_set_fx_prop_req *req) +{ + // TODO: trace this, see if there properties which needs to used later +} + +/* + * Callback to map a buffer allocated with allocate_buf for PIODMA usage. + * PIODMA is separate from the main DCP and uses own IOVA space on a dedicated + * stream of the display DART, rather than the expected DCP DART. + */ +static struct dcp_map_buf_resp dcpep_cb_map_piodma(struct apple_dcp *dcp, + struct dcp_map_buf_req *req) +{ + struct dcp_mem_descriptor *memdesc; + struct sg_table *map; + ssize_t ret; + + if (req->buffer >= ARRAY_SIZE(dcp->memdesc)) + goto reject; + + memdesc = &dcp->memdesc[req->buffer]; + map = &memdesc->map; + + if (!map->sgl) + goto reject; + + /* use the piodma iommu domain to map against the right IOMMU */ + ret = iommu_map_sgtable(dcp->iommu_dom, memdesc->dva, map, + IOMMU_READ | IOMMU_WRITE); + + /* HACK: expect size to be 16K aligned since the iommu API only maps + * full pages + */ + if (ret < 0 || ret != ALIGN(memdesc->size, SZ_16K)) { + dev_err(dcp->dev, "iommu_map_sgtable() returned %zd instead of expected buffer size of %zu\n", ret, memdesc->size); + goto reject; + } + + return (struct dcp_map_buf_resp){ .dva = memdesc->dva }; + +reject: + dev_err(dcp->dev, "denying map of invalid buffer %llx for piodma\n", + req->buffer); + return (struct dcp_map_buf_resp){ .ret = EINVAL }; +} + +static void dcpep_cb_unmap_piodma(struct apple_dcp *dcp, + struct dcp_unmap_buf_resp *resp) +{ + struct dcp_mem_descriptor *memdesc; + + if (resp->buffer >= ARRAY_SIZE(dcp->memdesc)) { + dev_warn(dcp->dev, "unmap request for out of range buffer %llu\n", + resp->buffer); + return; + } + + memdesc = &dcp->memdesc[resp->buffer]; + + if (!memdesc->buf) { + dev_warn(dcp->dev, + "unmap for non-mapped buffer %llu iova:0x%08llx\n", + resp->buffer, resp->dva); + return; + } + + if (memdesc->dva != resp->dva) { + dev_warn(dcp->dev, "unmap buffer %llu address mismatch " + "memdesc.dva:%llx dva:%llx\n", resp->buffer, + memdesc->dva, resp->dva); + return; + } + + /* use the piodma iommu domain to unmap from the right IOMMU */ + /* HACK: expect size to be 16K aligned since the iommu API only maps + * full pages + */ + iommu_unmap(dcp->iommu_dom, memdesc->dva, ALIGN(memdesc->size, SZ_16K)); +} + +/* + * Allocate an IOVA contiguous buffer mapped to the DCP. The buffer need not be + * physically contiguous, however we should save the sgtable in case the + * buffer needs to be later mapped for PIODMA. + */ +static struct dcp_allocate_buffer_resp +dcpep_cb_allocate_buffer(struct apple_dcp *dcp, + struct dcp_allocate_buffer_req *req) +{ + struct dcp_allocate_buffer_resp resp = { 0 }; + struct dcp_mem_descriptor *memdesc; + size_t size; + u32 id; + + resp.dva_size = ALIGN(req->size, 4096); + resp.mem_desc_id = + find_first_zero_bit(dcp->memdesc_map, DCP_MAX_MAPPINGS); + + if (resp.mem_desc_id >= DCP_MAX_MAPPINGS) { + dev_warn(dcp->dev, "DCP overflowed mapping table, ignoring\n"); + resp.dva_size = 0; + resp.mem_desc_id = 0; + return resp; + } + id = resp.mem_desc_id; + set_bit(id, dcp->memdesc_map); + + memdesc = &dcp->memdesc[id]; + + memdesc->size = resp.dva_size; + /* HACK: align size to 16K since the iommu API only maps full pages */ + size = ALIGN(resp.dva_size, SZ_16K); + memdesc->buf = dma_alloc_coherent(dcp->dev, size, + &memdesc->dva, GFP_KERNEL); + + dma_get_sgtable(dcp->dev, &memdesc->map, memdesc->buf, memdesc->dva, + size); + resp.dva = memdesc->dva; + + return resp; +} + +static u8 dcpep_cb_release_mem_desc(struct apple_dcp *dcp, u32 *mem_desc_id) +{ + struct dcp_mem_descriptor *memdesc; + size_t size; + u32 id = *mem_desc_id; + + if (id >= DCP_MAX_MAPPINGS) { + dev_warn(dcp->dev, + "unmap request for out of range mem_desc_id %u", id); + return 0; + } + + if (!test_and_clear_bit(id, dcp->memdesc_map)) { + dev_warn(dcp->dev, "unmap request for unused mem_desc_id %u\n", + id); + return 0; + } + + memdesc = &dcp->memdesc[id]; + size = ALIGN(memdesc->size, SZ_16K); + if (memdesc->buf) { + dma_free_coherent(dcp->dev, size, memdesc->buf, memdesc->dva); + memdesc->buf = NULL; + memset(&memdesc->map, 0, sizeof(memdesc->map)); + } else { + memdesc->reg = 0; + } + + memdesc->size = 0; + + return 1; +} + +/* Validate that the specified region is a display register */ +static bool is_disp_register(struct apple_dcp *dcp, u64 start, u64 end) +{ + int i; + + for (i = 0; i < dcp->nr_disp_registers; ++i) { + struct resource *r = dcp->disp_registers[i]; + + if ((start >= r->start) && (end <= r->end)) + return true; + } + + return false; +} + +/* + * Map contiguous physical memory into the DCP's address space. The firmware + * uses this to map the display registers we advertise in + * sr_map_device_memory_with_index, so we bounds check against that to guard + * safe against malicious coprocessors. + */ +static struct dcp_map_physical_resp +dcpep_cb_map_physical(struct apple_dcp *dcp, struct dcp_map_physical_req *req) +{ + int size = ALIGN(req->size, 4096); + dma_addr_t dva; + u32 id; + + if (!is_disp_register(dcp, req->paddr, req->paddr + size - 1)) { + dev_err(dcp->dev, "refusing to map phys address %llx size %llx\n", + req->paddr, req->size); + return (struct dcp_map_physical_resp){}; + } + + id = find_first_zero_bit(dcp->memdesc_map, DCP_MAX_MAPPINGS); + set_bit(id, dcp->memdesc_map); + dcp->memdesc[id].size = size; + dcp->memdesc[id].reg = req->paddr; + + dva = dma_map_resource(dcp->dev, req->paddr, size, DMA_BIDIRECTIONAL, 0); + WARN_ON(dva == DMA_MAPPING_ERROR); + + return (struct dcp_map_physical_resp){ + .dva_size = size, + .mem_desc_id = id, + .dva = dva, + }; +} + +static u64 dcpep_cb_get_frequency(struct apple_dcp *dcp) +{ + return clk_get_rate(dcp->clk); +} + +static struct DCP_FW_NAME(dcp_map_reg_resp) dcpep_cb_map_reg(struct apple_dcp *dcp, + struct DCP_FW_NAME(dcp_map_reg_req) *req) +{ + if (req->index >= dcp->nr_disp_registers) { + dev_warn(dcp->dev, "attempted to read invalid reg index %u\n", + req->index); + + return (struct DCP_FW_NAME(dcp_map_reg_resp)){ .ret = 1 }; + } else { + struct resource *rsrc = dcp->disp_registers[req->index]; +#if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) + dma_addr_t dva = dma_map_resource(dcp->dev, rsrc->start, resource_size(rsrc), + DMA_BIDIRECTIONAL, 0); + WARN_ON(dva == DMA_MAPPING_ERROR); +#endif + + return (struct DCP_FW_NAME(dcp_map_reg_resp)){ + .addr = rsrc->start, + .length = resource_size(rsrc), +#if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) + .dva = dva, +#endif + }; + } +} + +static struct dcp_read_edt_data_resp +dcpep_cb_read_edt_data(struct apple_dcp *dcp, struct dcp_read_edt_data_req *req) +{ + return (struct dcp_read_edt_data_resp){ + .value[0] = req->value[0], + .ret = 0, + }; +} + +static void iomfbep_cb_enable_backlight_message_ap_gated(struct apple_dcp *dcp, + u8 *enabled) +{ + /* + * update backlight brightness on next swap, on non mini-LED displays + * DCP seems to set an invalid iDAC value after coming out of DPMS. + * syslog: "[BrightnessLCD.cpp:743][AFK]nitsToDBV: iDAC out of range" + */ + dcp->brightness.update = true; + schedule_work(&dcp->bl_update_wq); +} + +/* Chunked data transfer for property dictionaries */ +static u8 dcpep_cb_prop_start(struct apple_dcp *dcp, u32 *length) +{ + if (dcp->chunks.data != NULL) { + dev_warn(dcp->dev, "ignoring spurious transfer start\n"); + return false; + } + + dcp->chunks.length = *length; + dcp->chunks.data = devm_kzalloc(dcp->dev, *length, GFP_KERNEL); + + if (!dcp->chunks.data) { + dev_warn(dcp->dev, "failed to allocate chunks\n"); + return false; + } + + return true; +} + +static u8 dcpep_cb_prop_chunk(struct apple_dcp *dcp, + struct dcp_set_dcpav_prop_chunk_req *req) +{ + if (!dcp->chunks.data) { + dev_warn(dcp->dev, "ignoring spurious chunk\n"); + return false; + } + + if (req->offset + req->length > dcp->chunks.length) { + dev_warn(dcp->dev, "ignoring overflowing chunk\n"); + return false; + } + + memcpy(dcp->chunks.data + req->offset, req->data, req->length); + return true; +} + +static bool dcpep_process_chunks(struct apple_dcp *dcp, + struct dcp_set_dcpav_prop_end_req *req) +{ + // struct apple_connector *connector = dcp->connector; + struct dcp_parse_ctx ctx; + int ret; //, i; + + if (!dcp->chunks.data) { + dev_warn(dcp->dev, "ignoring spurious end\n"); + return false; + } + + /* used just as opaque pointer for tracing */ + ctx.dcp = dcp; + + ret = parse(dcp->chunks.data, dcp->chunks.length, &ctx); + + if (ret) { + dev_warn(dcp->dev, "bad header on dcpav props\n"); + return false; + } + + if (!strcmp(req->key, "TimingElements")) { + dcp->modes = enumerate_modes(&ctx, &dcp->nr_modes, + dcp->width_mm, dcp->height_mm, + dcp->notch_height); + + if (IS_ERR(dcp->modes)) { + dev_warn(dcp->dev, "failed to parse modes\n"); + dcp->modes = NULL; + dcp->nr_modes = 0; + return false; + } + if (dcp->nr_modes == 0) + dev_warn(dcp->dev, "TimingElements without valid modes!\n"); + } else if (!strcmp(req->key, "DisplayAttributes")) { + ret = parse_display_attributes(&ctx, &dcp->width_mm, + &dcp->height_mm); + + if (ret) { + dev_warn(dcp->dev, "failed to parse display attribs\n"); + return false; + } + + dcp_set_dimensions(dcp); + } + + // if (connector) { + // for (i = 0; i < dcp->nr_modes; i++) { + // if (dcp->modes[i].vrr) { + // drm_connector_set_vrr_capable_property(&connector->base, true); + // break; + // } + // } + // } + + return true; +} + +static u8 dcpep_cb_prop_end(struct apple_dcp *dcp, + struct dcp_set_dcpav_prop_end_req *req) +{ + u8 resp = dcpep_process_chunks(dcp, req); + + /* move chunked data to connector to provide it via debugfs */ + dcp_connector_update_dict(dcp->connector, req->key, &dcp->chunks); + dcp->chunks.data = NULL; + dcp->chunks.length = 0; + + return resp; +} + +/* Boot sequence */ +static void boot_done(struct apple_dcp *dcp, void *out, void *cookie) +{ + struct dcp_channel *ch = &dcp->ch_cb; + u8 *succ = ch->output[ch->depth - 1]; + dev_dbg(dcp->dev, "boot done\n"); + + *succ = true; + dcp_ack(dcp, DCP_CONTEXT_CB); +} + +static void boot_5(struct apple_dcp *dcp, void *out, void *cookie) +{ + dcp_set_display_refresh_properties(dcp, false, boot_done, NULL); +} + +static void boot_4(struct apple_dcp *dcp, void *out, void *cookie) +{ +#if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) + u32 v_true = 1; + dcp_late_init_signal(dcp, false, &v_true, boot_5, NULL); +#else + dcp_late_init_signal(dcp, false, boot_5, NULL); +#endif +} + +static void boot_3(struct apple_dcp *dcp, void *out, void *cookie) +{ + u32 v_true = true; + + dcp_flush_supports_power(dcp, false, &v_true, boot_4, NULL); +} + +static void boot_2(struct apple_dcp *dcp, void *out, void *cookie) +{ + dcp_setup_video_limits(dcp, false, boot_3, NULL); +} + +static void boot_1_5(struct apple_dcp *dcp, void *out, void *cookie) +{ + dcp_create_default_fb(dcp, false, boot_2, NULL); +} + +/* Use special function signature to defer the ACK */ +static bool dcpep_cb_boot_1(struct apple_dcp *dcp, int tag, void *out, void *in) +{ + trace_iomfb_callback(dcp, tag, __func__); + dcp_set_create_dfb(dcp, false, boot_1_5, NULL); + return false; +} + +static struct dcp_allocate_bandwidth_resp dcpep_cb_allocate_bandwidth(struct apple_dcp *dcp, + struct dcp_allocate_bandwidth_req *req) +{ + return (struct dcp_allocate_bandwidth_resp){ + .unk1 = req->unk1, + .unk2 = req->unk2, + .ret = 1, + }; +} + +static struct dcp_rt_bandwidth dcpep_cb_rt_bandwidth(struct apple_dcp *dcp) +{ + struct dcp_rt_bandwidth rt_bw = (struct dcp_rt_bandwidth){ + .reg_scratch = 0, + .reg_doorbell = 0, + .doorbell_bit = 0, + }; + + if (dcp->disp_bw_scratch_index) { + u32 offset = dcp->disp_bw_scratch_offset; + u32 index = dcp->disp_bw_scratch_index; + rt_bw.reg_scratch = dcp->disp_registers[index]->start + offset; + } + + if (dcp->disp_bw_doorbell_index) { + u32 index = dcp->disp_bw_doorbell_index; + rt_bw.reg_doorbell = dcp->disp_registers[index]->start; + rt_bw.doorbell_bit = REG_DOORBELL_BIT(dcp->index); + /* + * This is most certainly not padding. t8103-dcp crashes without + * setting this immediately during modeset on 12.3 and 13.5 + * firmware. + */ + rt_bw.padding[3] = 0x4; + } + + return rt_bw; +} + +static struct dcp_set_frame_sync_props_resp +dcpep_cb_set_frame_sync_props(struct apple_dcp *dcp, + struct dcp_set_frame_sync_props_req *req) +{ + return (struct dcp_set_frame_sync_props_resp){}; +} + +/* Callback to get the current time as milliseconds since the UNIX epoch */ +static u64 dcpep_cb_get_time(struct apple_dcp *dcp) +{ + return ktime_to_ms(ktime_get_real()); +} + +struct dcp_swap_cookie { + struct kref refcount; + struct completion done; + u32 swap_id; +}; + +static void release_swap_cookie(struct kref *ref) +{ + struct dcp_swap_cookie *cookie; + cookie = container_of(ref, struct dcp_swap_cookie, refcount); + + kfree(cookie); +} + +static void dcp_swap_cleared(struct apple_dcp *dcp, void *data, void *cookie) +{ + struct DCP_FW_NAME(dcp_swap_submit_resp) *resp = data; + + if (cookie) { + struct dcp_swap_cookie *info = cookie; + complete(&info->done); + kref_put(&info->refcount, release_swap_cookie); + } + + if (resp->ret) { + dev_err(dcp->dev, "swap_clear failed! status %u\n", resp->ret); + dcp_drm_crtc_vblank(dcp->crtc); + return; + } + + while (!list_empty(&dcp->swapped_out_fbs)) { + struct dcp_fb_reference *entry; + entry = list_first_entry(&dcp->swapped_out_fbs, + struct dcp_fb_reference, head); + if (entry->swap_id == dcp->last_swap_id) + break; + if (entry->fb) + drm_framebuffer_put(entry->fb); + list_del(&entry->head); + kfree(entry); + } +} + +static void dcp_swap_clear_started(struct apple_dcp *dcp, void *data, + void *cookie) +{ + struct dcp_swap_start_resp *resp = data; + DCP_FW_UNION(dcp->swap).swap.swap_id = resp->swap_id; + + if (cookie) { + struct dcp_swap_cookie *info = cookie; + info->swap_id = resp->swap_id; + } + + dcp_swap_submit(dcp, false, &DCP_FW_UNION(dcp->swap), dcp_swap_cleared, cookie); +} + +static void dcp_on_final(struct apple_dcp *dcp, void *out, void *cookie) +{ + struct dcp_wait_cookie *wait = cookie; + + if (wait) { + complete(&wait->done); + kref_put(&wait->refcount, release_wait_cookie); + } +} + +static void dcp_on_set_power_state(struct apple_dcp *dcp, void *out, void *cookie) +{ + struct dcp_set_power_state_req req = { + .unklong = 1, + }; + + dcp_set_power_state(dcp, false, &req, dcp_on_final, cookie); +} + +void DCP_FW_NAME(iomfb_poweron)(struct apple_dcp *dcp) +{ + struct dcp_wait_cookie *cookie; + int ret; + u32 handle; + dev_info(dcp->dev, "dcp_poweron() starting\n"); + + cookie = kzalloc(sizeof(*cookie), GFP_KERNEL); + if (!cookie) + return; + + init_completion(&cookie->done); + kref_init(&cookie->refcount); + /* increase refcount to ensure the receiver has a reference */ + kref_get(&cookie->refcount); + + handle = dcp->main_display ? 0 : 2; + + dcp_set_display_device(dcp, false, &handle, dcp_on_set_power_state, + cookie); + + ret = wait_for_completion_timeout(&cookie->done, msecs_to_jiffies(10000)); + + if (ret == 0) { + dev_warn(dcp->dev, "wait for power timed out, connector will be broken\n"); + } else if (ret > 0) { + int msecs = jiffies_to_msecs(ret); + if (msecs > 6000) + dev_info(dcp->dev, "dcp_set_power_state_req returned, %d ms remaining\n", msecs); + else + dev_warn(dcp->dev, "dcp_set_power_state_req returned, %d ms remaining\n", msecs); + } else { + drm_connector_set_link_status_property(&dcp->connector->base, + DRM_MODE_LINK_STATUS_BAD); + dev_warn(dcp->dev, "wait for completion error: %d\n", ret); + } + + kref_put(&cookie->refcount, release_wait_cookie);; + + /* Force a brightness update after poweron, to restore the brightness */ + dcp->brightness.update = true; +} + +static void complete_set_powerstate(struct apple_dcp *dcp, void *out, + void *cookie) +{ + struct dcp_wait_cookie *wait = cookie; + + if (wait) { + complete(&wait->done); + kref_put(&wait->refcount, release_wait_cookie); + } +} + +static void last_client_closed_poff(struct apple_dcp *dcp, void *out, void *cookie) +{ + struct dcp_set_power_state_req power_req = { + .unklong = 0, + }; + dcp_set_power_state(dcp, false, &power_req, complete_set_powerstate, + cookie); +} + +static void aborted_swaps_dcp_poff(struct apple_dcp *dcp, void *out, void *cookie) +{ + struct iomfb_last_client_close_req last_client_req = {}; + iomfb_last_client_close(dcp, false, &last_client_req, + last_client_closed_poff, cookie); +} + +void DCP_FW_NAME(iomfb_poweroff)(struct apple_dcp *dcp) +{ + int ret, swap_id; + struct iomfb_abort_swaps_dcp_req abort_req = { + .client = { + .flag2 = 1, + }, + }; + struct dcp_swap_cookie *cookie; + struct dcp_wait_cookie *poff_cookie; + struct dcp_swap_start_req swap_req = { 0 }; + struct DCP_FW_NAME(dcp_swap_submit_req) *swap = &DCP_FW_UNION(dcp->swap); + + cookie = kzalloc(sizeof(*cookie), GFP_KERNEL); + if (!cookie) + return; + init_completion(&cookie->done); + kref_init(&cookie->refcount); + /* increase refcount to ensure the receiver has a reference */ + kref_get(&cookie->refcount); + + // clear surfaces + memset(swap, 0, sizeof(*swap)); + + swap->swap.swap_enabled = + swap->swap.swap_completed = IOMFB_SET_BACKGROUND | 0x7; + swap->swap.bg_color = 0xFF000000; + + /* + * Turn off the backlight. This matters because the DCP's idea of + * backlight brightness gets desynced after a power change, and it + * needs to be told it's going to turn off so it will consider the + * subsequent update on poweron an actual change and restore the + * brightness. + */ + if (dcp_has_panel(dcp)) { + swap->swap.bl_unk = 1; + swap->swap.bl_value = 0; + swap->swap.bl_power = 0; + } + + /* Null all surfaces */ + for (int l = 0; l < SWAP_SURFACES; l++) + swap->surf_null[l] = true; +#if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) + for (int l = 0; l < 5; l++) + swap->surf2_null[l] = true; + swap->unkU32Ptr_null = true; + swap->unkU32out_null = true; +#endif + + dcp_swap_start(dcp, false, &swap_req, dcp_swap_clear_started, cookie); + + ret = wait_for_completion_timeout(&cookie->done, msecs_to_jiffies(50)); + swap_id = cookie->swap_id; + kref_put(&cookie->refcount, release_swap_cookie); + if (ret <= 0) { + dcp->crashed = true; + return; + } + + dev_dbg(dcp->dev, "%s: clear swap submitted: %u\n", __func__, swap_id); + + poff_cookie = kzalloc(sizeof(*poff_cookie), GFP_KERNEL); + if (!poff_cookie) + return; + init_completion(&poff_cookie->done); + kref_init(&poff_cookie->refcount); + /* increase refcount to ensure the receiver has a reference */ + kref_get(&poff_cookie->refcount); + + iomfb_abort_swaps_dcp(dcp, false, &abort_req, + aborted_swaps_dcp_poff, poff_cookie); + ret = wait_for_completion_timeout(&poff_cookie->done, + msecs_to_jiffies(1000)); + + if (ret == 0) + dev_warn(dcp->dev, "setPowerState(0) timeout %u ms\n", 1000); + else if (ret > 0) + dev_dbg(dcp->dev, + "setPowerState(0) finished with %d ms to spare", + jiffies_to_msecs(ret)); + + kref_put(&poff_cookie->refcount, release_wait_cookie); + + dev_info(dcp->dev, "dcp_poweroff() done\n"); +} + +static void last_client_closed_sleep(struct apple_dcp *dcp, void *out, void *cookie) +{ + struct dcp_set_power_state_req power_req = { + .unklong = 0, + }; + dcp_set_power_state(dcp, false, &power_req, complete_set_powerstate, cookie); +} + +static void aborted_swaps_dcp_sleep(struct apple_dcp *dcp, void *out, void *cookie) +{ + struct iomfb_last_client_close_req req = { 0 }; + iomfb_last_client_close(dcp, false, &req, last_client_closed_sleep, cookie); +} + +void DCP_FW_NAME(iomfb_sleep)(struct apple_dcp *dcp) +{ + int ret; + struct iomfb_abort_swaps_dcp_req req = { + .client = { + .flag2 = 1, + }, + }; + + struct dcp_wait_cookie *cookie; + + cookie = kzalloc(sizeof(*cookie), GFP_KERNEL); + if (!cookie) + return; + init_completion(&cookie->done); + kref_init(&cookie->refcount); + /* increase refcount to ensure the receiver has a reference */ + kref_get(&cookie->refcount); + + iomfb_abort_swaps_dcp(dcp, false, &req, aborted_swaps_dcp_sleep, + cookie); + ret = wait_for_completion_timeout(&cookie->done, + msecs_to_jiffies(1000)); + + if (ret == 0) + dev_warn(dcp->dev, "setDCPPower(0) timeout %u ms\n", 1000); + + kref_put(&cookie->refcount, release_wait_cookie); + dev_info(dcp->dev, "dcp_sleep() done\n"); +} + +static void dcpep_cb_hotplug(struct apple_dcp *dcp, u64 *connected) +{ + struct apple_connector *connector = dcp->connector; + + /* DCP issues hotplug_gated callbacks after SetPowerState() calls on + * devices with display (macbooks, imacs). This must not result in + * connector state changes on DRM side. Some applications won't enable + * a CRTC with a connector in disconnected state. Weston after DPMS off + * is one example. dcp_is_main_display() returns true on devices with + * integrated display. Ignore the hotplug_gated() callbacks there. + */ + if (dcp->main_display) + return; + + if (dcp->during_modeset) { + dev_info(dcp->dev, + "cb_hotplug() ignored during modeset connected:%llu\n", + *connected); + return; + } + + dev_info(dcp->dev, "cb_hotplug() connected:%llu, valid_mode:%d\n", + *connected, dcp->valid_mode); + + /* Hotplug invalidates mode. DRM doesn't always handle this. */ + if (!(*connected)) { + dcp->valid_mode = false; + /* after unplug swap will not complete until the next + * set_digital_out_mode */ + schedule_work(&dcp->vblank_wq); + } + + if (connector && connector->connected != !!(*connected)) { + connector->connected = !!(*connected); + dcp->valid_mode = false; + schedule_work(&connector->hotplug_wq); + } +} + +static void +dcpep_cb_swap_complete_intent_gated(struct apple_dcp *dcp, + struct dcp_swap_complete_intent_gated *info) +{ + trace_iomfb_swap_complete_intent_gated(dcp, info->swap_id, + info->width, info->height); +} + +static void +dcpep_cb_abort_swap_ap_gated(struct apple_dcp *dcp, u32 *swap_id) +{ + trace_iomfb_abort_swap_ap_gated(dcp, *swap_id); +} + +static struct dcpep_get_tiling_state_resp +dcpep_cb_get_tiling_state(struct apple_dcp *dcp, + struct dcpep_get_tiling_state_req *req) +{ + return (struct dcpep_get_tiling_state_resp){ + .value = 0, + .ret = 1, + }; +} + +static u8 dcpep_cb_create_backlight_service(struct apple_dcp *dcp) +{ + return dcp_has_panel(dcp); +} + +TRAMPOLINE_VOID(trampoline_nop, dcpep_cb_nop); +TRAMPOLINE_OUT(trampoline_true, dcpep_cb_true, u8); +TRAMPOLINE_OUT(trampoline_false, dcpep_cb_false, u8); +TRAMPOLINE_OUT(trampoline_zero, dcpep_cb_zero, u32); +TRAMPOLINE_IN(trampoline_swap_complete, dcpep_cb_swap_complete, + struct DCP_FW_NAME(dc_swap_complete_resp)); +TRAMPOLINE_INOUT(trampoline_get_uint_prop, dcpep_cb_get_uint_prop, + struct dcp_get_uint_prop_req, struct dcp_get_uint_prop_resp); +TRAMPOLINE_IN(trampoline_set_fx_prop, iomfbep_cb_set_fx_prop, + struct iomfb_set_fx_prop_req) +TRAMPOLINE_INOUT(trampoline_map_piodma, dcpep_cb_map_piodma, + struct dcp_map_buf_req, struct dcp_map_buf_resp); +TRAMPOLINE_IN(trampoline_unmap_piodma, dcpep_cb_unmap_piodma, + struct dcp_unmap_buf_resp); +TRAMPOLINE_INOUT(trampoline_sr_set_property_int, iomfbep_cb_sr_set_property_int, + struct iomfb_sr_set_property_int_req, u8); +TRAMPOLINE_INOUT(trampoline_allocate_buffer, dcpep_cb_allocate_buffer, + struct dcp_allocate_buffer_req, + struct dcp_allocate_buffer_resp); +TRAMPOLINE_INOUT(trampoline_map_physical, dcpep_cb_map_physical, + struct dcp_map_physical_req, struct dcp_map_physical_resp); +TRAMPOLINE_INOUT(trampoline_release_mem_desc, dcpep_cb_release_mem_desc, u32, + u8); +TRAMPOLINE_INOUT(trampoline_map_reg, dcpep_cb_map_reg, + struct DCP_FW_NAME(dcp_map_reg_req), + struct DCP_FW_NAME(dcp_map_reg_resp)); +TRAMPOLINE_INOUT(trampoline_read_edt_data, dcpep_cb_read_edt_data, + struct dcp_read_edt_data_req, struct dcp_read_edt_data_resp); +TRAMPOLINE_INOUT(trampoline_prop_start, dcpep_cb_prop_start, u32, u8); +TRAMPOLINE_INOUT(trampoline_prop_chunk, dcpep_cb_prop_chunk, + struct dcp_set_dcpav_prop_chunk_req, u8); +TRAMPOLINE_INOUT(trampoline_prop_end, dcpep_cb_prop_end, + struct dcp_set_dcpav_prop_end_req, u8); +TRAMPOLINE_INOUT(trampoline_allocate_bandwidth, dcpep_cb_allocate_bandwidth, + struct dcp_allocate_bandwidth_req, struct dcp_allocate_bandwidth_resp); +TRAMPOLINE_OUT(trampoline_rt_bandwidth, dcpep_cb_rt_bandwidth, + struct dcp_rt_bandwidth); +TRAMPOLINE_INOUT(trampoline_set_frame_sync_props, dcpep_cb_set_frame_sync_props, + struct dcp_set_frame_sync_props_req, + struct dcp_set_frame_sync_props_resp); +TRAMPOLINE_OUT(trampoline_get_frequency, dcpep_cb_get_frequency, u64); +TRAMPOLINE_OUT(trampoline_get_time, dcpep_cb_get_time, u64); +TRAMPOLINE_IN(trampoline_hotplug, dcpep_cb_hotplug, u64); +TRAMPOLINE_IN(trampoline_swap_complete_intent_gated, + dcpep_cb_swap_complete_intent_gated, + struct dcp_swap_complete_intent_gated); +TRAMPOLINE_IN(trampoline_abort_swap_ap_gated, dcpep_cb_abort_swap_ap_gated, u32); +TRAMPOLINE_IN(trampoline_enable_backlight_message_ap_gated, + iomfbep_cb_enable_backlight_message_ap_gated, u8); +TRAMPOLINE_IN(trampoline_pr_publish, iomfb_cb_pr_publish, + struct iomfb_property); +TRAMPOLINE_INOUT(trampoline_get_tiling_state, dcpep_cb_get_tiling_state, + struct dcpep_get_tiling_state_req, struct dcpep_get_tiling_state_resp); +TRAMPOLINE_OUT(trampoline_create_backlight_service, dcpep_cb_create_backlight_service, u8); + +/* + * Callback for swap requests. If a swap failed, we'll never get a swap + * complete event so we need to fake a vblank event early to avoid a hang. + */ + +static void dcp_swapped(struct apple_dcp *dcp, void *data, void *cookie) +{ + struct DCP_FW_NAME(dcp_swap_submit_resp) *resp = data; + + if (resp->ret) { + dev_err(dcp->dev, "swap failed! status %u\n", resp->ret); + dcp_drm_crtc_vblank(dcp->crtc); + return; + } + dcp->swap_start = ktime_get(); + + while (!list_empty(&dcp->swapped_out_fbs)) { + struct dcp_fb_reference *entry; + entry = list_first_entry(&dcp->swapped_out_fbs, + struct dcp_fb_reference, head); + if (entry->swap_id == dcp->last_swap_id) + break; + if (entry->fb) + drm_framebuffer_put(entry->fb); + list_del(&entry->head); + kfree(entry); + } +} + +static void dcp_swap_started(struct apple_dcp *dcp, void *data, void *cookie) +{ + struct dcp_swap_start_resp *resp = data; + + DCP_FW_UNION(dcp->swap).swap.swap_id = resp->swap_id; + + trace_iomfb_swap_submit(dcp, resp->swap_id); + dcp_swap_submit(dcp, false, &DCP_FW_UNION(dcp->swap), dcp_swapped, NULL); +} + +/* Helpers to modeset and swap, used to flush */ +static void do_swap(struct apple_dcp *dcp, void *data, void *cookie) +{ + struct dcp_swap_start_req start_req = { 0 }; + + if (dcp->connector && dcp->connector->connected) + dcp_swap_start(dcp, false, &start_req, dcp_swap_started, NULL); + else + dcp_drm_crtc_vblank(dcp->crtc); +} + +static void complete_set_digital_out_mode(struct apple_dcp *dcp, void *data, + void *cookie) +{ + struct dcp_wait_cookie *wait = cookie; + + if (wait) { + complete(&wait->done); + kref_put(&wait->refcount, release_wait_cookie); + } +} + +/* Changes to Adaptive Sync require a trip through set_digital_out_mode */ +static void dcp_on_set_adaptive_sync(struct apple_dcp *dcp, void *out, void *cookie) +{ + dcp_set_digital_out_mode(dcp, false, &dcp->mode, + complete_set_digital_out_mode, cookie); +} + +static void dcp_set_adaptive_sync(struct apple_dcp *dcp, u32 rate, void *cookie) +{ + struct dcp_set_parameter_dcp param = { + .param = IOMFBPARAM_ADAPTIVE_SYNC, + .value = { + rate, /* minRR */ + 0, /* mediaTargetRate */ + 0, /* Fractional Rate (?) */ + 0, /* unused */ + }, +#if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) + .count = 3, +#else + .count = 1, +#endif + }; + + dcp_set_parameter_dcp(dcp, false, ¶m, dcp_on_set_adaptive_sync, cookie); +} + +int DCP_FW_NAME(iomfb_modeset)(struct apple_dcp *dcp, + struct drm_crtc_state *crtc_state) +{ + struct dcp_display_mode *mode; + struct dcp_wait_cookie *cookie; + struct dcp_color_mode *cmode = NULL; + int ret; + + mode = lookup_mode(dcp, &crtc_state->mode); + if (!mode) { + dev_err(dcp->dev, "no match for " DRM_MODE_FMT "\n", + DRM_MODE_ARG(&crtc_state->mode)); + return -EIO; + } + + dev_info(dcp->dev, + "set_digital_out_mode(color:%d timing:%d) " DRM_MODE_FMT "\n", + mode->color_mode_id, mode->timing_mode_id, + DRM_MODE_ARG(&crtc_state->mode)); + if (mode->color_mode_id == mode->sdr_rgb.id) + cmode = &mode->sdr_rgb; + else if (mode->color_mode_id == mode->sdr_444.id) + cmode = &mode->sdr_444; + else if (mode->color_mode_id == mode->sdr.id) + cmode = &mode->sdr; + else if (mode->color_mode_id == mode->best.id) + cmode = &mode->best; + if (cmode) + dev_info(dcp->dev, + "set_digital_out_mode() color mode depth:%hhu format:%u " + "colorimetry:%u eotf:%u range:%u vrr:%u\n", cmode->depth, + cmode->format, cmode->colorimetry, cmode->eotf, + cmode->range, mode->vrr); + + dcp->mode = (struct dcp_set_digital_out_mode_req){ + .color_mode_id = mode->color_mode_id, + .timing_mode_id = mode->timing_mode_id + }; + + /* Use DCP swap timestamps on MacBook Pros with VRR */ + dcp->use_timestamps = mode->vrr && dcp->main_display; + + cookie = kzalloc(sizeof(*cookie), GFP_KERNEL); + if (!cookie) { + return -ENOMEM; + } + + init_completion(&cookie->done); + kref_init(&cookie->refcount); + /* increase refcount to ensure the receiver has a reference */ + kref_get(&cookie->refcount); + + dcp->during_modeset = true; + + if (mode->vrr) { + dcp_set_adaptive_sync(dcp, force_vrr ? mode->min_vrr : 0, cookie); + } else { + dcp_set_digital_out_mode(dcp, false, &dcp->mode, + complete_set_digital_out_mode, cookie); + } + + /* + * The DCP firmware has an internal timeout of ~8 seconds for + * modesets. Add an extra 500ms to safe side that the modeset + * call has returned. + */ + ret = wait_for_completion_timeout(&cookie->done, + msecs_to_jiffies(8500)); + + kref_put(&cookie->refcount, release_wait_cookie); + dcp->during_modeset = false; + dev_info(dcp->dev, "set_digital_out_mode finished:%d\n", ret); + + if (ret == 0) { + dev_info(dcp->dev, "set_digital_out_mode timed out\n"); + return -EIO; + } else if (ret < 0) { + dev_info(dcp->dev, + "waiting on set_digital_out_mode failed:%d\n", ret); + return -EIO; + + } else if (ret > 0) { + dev_dbg(dcp->dev, + "set_digital_out_mode finished with %d to spare\n", + jiffies_to_msecs(ret)); + } + dcp->valid_mode = true; + dcp->vrr_enabled = mode->vrr && force_vrr; + + return 0; +} + +/* + * DCP timestamps are expressed in system timer ticks. Approximate + * this by converting from ktime nanoseconds to 24 MHz ticks. + */ +static u64 ns_to_mach(u64 ns) +{ + return ns * 3 / 125; +} + +void DCP_FW_NAME(iomfb_flush)(struct apple_dcp *dcp, struct drm_crtc *crtc, struct drm_atomic_state *state) +{ + struct drm_plane *plane; + struct drm_plane_state *new_state, *old_state; + struct drm_crtc_state *crtc_state; + struct DCP_FW_NAME(dcp_swap_submit_req) *req = &DCP_FW_UNION(dcp->swap); + int plane_idx, l; + int has_surface = 0; + + crtc_state = drm_atomic_get_new_crtc_state(state, crtc); + + /* Reset all surfaces to defaults */ + memset(req, 0, sizeof(*req)); + for (l = 0; l < SWAP_SURFACES; l++) + req->surf_null[l] = true; +#if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) + for (l = 0; l < 5; l++) + req->surf2_null[l] = true; + req->unkU32Ptr_null = true; + req->unkU32out_null = true; +#endif + + /* + * Clear all surfaces on startup. The boot framebuffer in surface 0 + * sticks around. + */ + if (!dcp->surfaces_cleared) { + req->swap.swap_enabled = IOMFB_SET_BACKGROUND | 0x7; + req->swap.bg_color = 0xFF000000; + dcp->surfaces_cleared = true; + } + + for_each_oldnew_plane_in_state(state, plane, old_state, new_state, plane_idx) { + struct apple_plane_state *apple_state = to_apple_plane_state(new_state); + + /* skip planes not for this crtc */ + if (old_state->crtc != crtc && new_state->crtc != crtc) + continue; + + /* + * Plane order is nondeterministic for this iterator. DCP will + * almost always crash at some point if the z order of planes + * flip-flops around. Make sure we are always blending them + * in the correct order. + * + * Despite having 4 surfaces, we can only blend two. Surface 0 is + * also unusable on some machines, so ignore it. + */ + + l = MAX_BLEND_SURFACES - new_state->normalized_zpos; + + WARN_ON(l > MAX_BLEND_SURFACES); + + req->swap.swap_enabled |= BIT(l); + + if (old_state->fb && new_state->fb != old_state->fb) { + /* + * Race condition between a framebuffer unbind getting + * swapped out and GEM unreferencing a framebuffer. If + * we lose the race, the display gets IOVA faults and + * the DCP crashes. We need to extend the lifetime of + * the drm_framebuffer (and hence the GEM object) until + * after we get a swap complete for the swap unbinding + * it. + */ + struct dcp_fb_reference *entry = + kzalloc(sizeof(*entry), GFP_KERNEL); + if (entry) { + entry->fb = old_state->fb; + entry->swap_id = dcp->last_swap_id; + list_add_tail(&entry->head, + &dcp->swapped_out_fbs); + } + drm_framebuffer_get(old_state->fb); + } + + if (!new_state->fb || !new_state->visible) { + continue; + } + req->surf_null[l] = false; + has_surface = 1; + + req->swap.src_rect[l] = apple_state->src_rect; + req->swap.dst_rect[l] = apple_state->dst_rect; + + if (dcp->notch_height > 0) + req->swap.dst_rect[l].y += dcp->notch_height; + + req->surf_iova[l] = apple_state->iova; + req->surf[l].base = apple_state->surf; + + /* Use sRGB colorspace only for internal panels. External + * displays are expected to have EDID and user space can use + * the contained colorimetry information to provide native + * colors. + */ + if (dcp->connector_type == DRM_MODE_CONNECTOR_eDP && + req->surf[l].base.colorspace == DCP_COLORSPACE_BG_SRGB) + req->surf[l].base.colorspace = DCP_COLORSPACE_NATIVE; + } + + if (!has_surface && !crtc_state->color_mgmt_changed) { + if (crtc_state->enable && crtc_state->active && + !crtc_state->planes_changed) { + schedule_work(&dcp->vblank_wq); + return; + } + + /* Set black background */ + req->swap.swap_enabled |= IOMFB_SET_BACKGROUND; + req->swap.bg_color = 0xFF000000; + req->clear = 1; + } + + if (has_surface && (dcp->use_timestamps || crtc_state->vrr_enabled || force_vrr)) { + /* + * TODO: ascertain with certainty what these timestamps + * are. They are something to do with presentation timing, + * but that is all we know for sure. These values seem to + * work well with VRR. + */ + req->swap.unk_pres_ts1 = ns_to_mach(ktime_get_ns()); + req->swap.unk_pres_ts2 = ns_to_mach(ktime_to_ns(dcp->swap_start)); + req->swap.unk_pres_ts3 = req->swap.unk_pres_ts1; + } + + /* These fields should be set together */ + req->swap.swap_completed = req->swap.swap_enabled; + + /* update brightness if changed */ + if (dcp_has_panel(dcp) && dcp->brightness.update) { + req->swap.bl_unk = 1; + req->swap.bl_value = dcp->brightness.dac; + req->swap.bl_power = 0x40; + dcp->brightness.update = false; + } + + if (crtc_state->color_mgmt_changed) { + struct iomfb_set_matrix_req mat = { + .location = 9, + }; + + if (crtc_state->ctm) { + struct drm_color_ctm *ctm = (struct drm_color_ctm *)crtc_state->ctm->data; + memcpy(mat.matrix, ctm->matrix, sizeof(mat.matrix)); + } else { + mat.matrix[0] = mat.matrix[4] = mat.matrix[8] = 1LLU << 32; + } + + iomfb_set_matrix(dcp, false, &mat, do_swap, NULL); + } else + do_swap(dcp, NULL, NULL); +} + +static void res_is_main_display(struct apple_dcp *dcp, void *out, void *cookie) +{ + struct apple_connector *connector; + int result = *(int *)out; + dev_info(dcp->dev, "DCP is_main_display: %d\n", result); + + dcp->main_display = result != 0; + + connector = dcp->connector; + if (connector) { + connector->connected = dcp->nr_modes > 0; + schedule_work(&connector->hotplug_wq); + } + + dcp->active = true; + complete(&dcp->start_done); +} + +static void init_3(struct apple_dcp *dcp, void *out, void *cookie) +{ + dcp_is_main_display(dcp, false, res_is_main_display, NULL); +} + +static void init_2(struct apple_dcp *dcp, void *out, void *cookie) +{ + dcp_first_client_open(dcp, false, init_3, NULL); +} + +static void init_1(struct apple_dcp *dcp, void *out, void *cookie) +{ + u32 val = 0; + dcp_enable_disable_video_power_savings(dcp, false, &val, init_2, NULL); +} + +static void dcp_started(struct apple_dcp *dcp, void *data, void *cookie) +{ + struct iomfb_get_color_remap_mode_req color_remap = + (struct iomfb_get_color_remap_mode_req){ + .mode = 6, + }; + + dev_info(dcp->dev, "DCP booted\n"); + + iomfb_get_color_remap_mode(dcp, false, &color_remap, init_1, cookie); +} + +void DCP_FW_NAME(iomfb_shutdown)(struct apple_dcp *dcp) +{ + struct dcp_set_power_state_req req = { + /* defaults are ok */ + }; + + dcp_set_power_state(dcp, false, &req, NULL, NULL); +} diff --git a/drivers/gpu/drm/apple/iomfb_template.h b/drivers/gpu/drm/apple/iomfb_template.h new file mode 100644 index 00000000000000..e74672da712efa --- /dev/null +++ b/drivers/gpu/drm/apple/iomfb_template.h @@ -0,0 +1,167 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright 2021 Alyssa Rosenzweig */ + +/* + * This file is intended to be included multiple times with IOMFB_VER + * defined to declare DCP firmware version dependent structs. + */ + +#ifdef DCP_FW_VER + +#include + +#include + +#include "iomfb.h" +#include "iomfb_plane.h" +#include "plane.h" +#include "version_utils.h" + +struct DCP_FW_NAME(dcp_swap) { + u64 unk_pres_ts1; + u64 unk_pres_ts2; + + u64 unk_10; + u64 unk_18; + u64 ts64_unk; + u64 unk_28; + u64 unk_pres_ts3; + u64 unk_38; + + u64 flags1; + u64 flags2; + + u32 swap_id; + + u32 surf_ids[SWAP_SURFACES]; + struct dcp_rect src_rect[SWAP_SURFACES]; + u32 surf_flags[SWAP_SURFACES]; + u32 surf_unk[SWAP_SURFACES]; + struct dcp_rect dst_rect[SWAP_SURFACES]; + u32 swap_enabled; + u32 swap_completed; + + u32 bg_color; + u8 unk_110[0x1b8]; + u32 unk_2c8; + u8 unk_2cc[0x14]; + u32 unk_2e0; +#if DCP_FW_VER < DCP_FW_VERSION(13, 2, 0) + u16 unk_2e2; +#else + u8 unk_2e2[3]; +#endif + u64 bl_unk; + u32 bl_value; // min value is 0x10000000 + u8 bl_power; // constant 0x40 for on + u8 unk_2f3[0x2d]; +#if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) + u8 unk_320[0x13f]; + u64 unk_1; +#endif +} __packed; + +/* Information describing a surface */ +struct DCP_FW_NAME(dcp_surface) { + struct dcp_surface base; +#if DCP_FW_VER < DCP_FW_VERSION(13, 2, 0) + u8 padding[7]; +#else + u8 padding[47]; +#endif +} __packed; + +/* Prototypes */ + +struct DCP_FW_NAME(dcp_swap_submit_req) { + struct DCP_FW_NAME(dcp_swap) swap; + struct DCP_FW_NAME(dcp_surface) surf[SWAP_SURFACES]; + u64 surf_iova[SWAP_SURFACES]; +#if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) + u64 unk_u64_a[SWAP_SURFACES]; + struct DCP_FW_NAME(dcp_surface) surf2[5]; + u64 surf2_iova[5]; +#endif + u8 unkbool; + u64 unkdouble; +#if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) + u64 unkU64; + u8 unkbool2; +#endif + u32 clear; // or maybe switch to default fb? +#if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) + u32 unkU32Ptr; +#endif + u8 swap_null; + u8 surf_null[SWAP_SURFACES]; +#if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) + u8 surf2_null[5]; +#endif + u8 unkoutbool_null; +#if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) + u8 unkU32Ptr_null; + u8 unkU32out_null; +#endif + u8 padding[1]; +} __packed; + +struct DCP_FW_NAME(dcp_swap_submit_resp) { + u8 unkoutbool; +#if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) + u32 unkU32out; +#endif + u32 ret; + u8 padding[3]; +} __packed; + +struct DCP_FW_NAME(dc_swap_complete_resp) { + u32 swap_id; + u8 unkbool; + u64 swap_data; +#if DCP_FW_VER < DCP_FW_VERSION(13, 2, 0) + u8 swap_info[0x6c4]; +#else + u8 swap_info[0x6c5]; +#endif + u32 unkint; + u8 swap_info_null; +} __packed; + +struct DCP_FW_NAME(dcp_map_reg_req) { + char obj[4]; + u32 index; + u32 flags; +#if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) + u8 unk_u64_null; +#endif + u8 addr_null; + u8 length_null; +#if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) + u8 padding[1]; +#else + u8 padding[2]; +#endif +} __packed; + +struct DCP_FW_NAME(dcp_map_reg_resp) { +#if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) + u64 dva; +#endif + u64 addr; + u64 length; + u32 ret; +} __packed; + + +struct apple_dcp; + +int DCP_FW_NAME(iomfb_modeset)(struct apple_dcp *dcp, + struct drm_crtc_state *crtc_state); +void DCP_FW_NAME(iomfb_flush)(struct apple_dcp *dcp, struct drm_crtc *crtc, struct drm_atomic_state *state); +void DCP_FW_NAME(iomfb_poweron)(struct apple_dcp *dcp); +void DCP_FW_NAME(iomfb_poweroff)(struct apple_dcp *dcp); +void DCP_FW_NAME(iomfb_sleep)(struct apple_dcp *dcp); +void DCP_FW_NAME(iomfb_start)(struct apple_dcp *dcp); +void DCP_FW_NAME(iomfb_shutdown)(struct apple_dcp *dcp); + +#endif diff --git a/drivers/gpu/drm/apple/iomfb_v12_3.c b/drivers/gpu/drm/apple/iomfb_v12_3.c new file mode 100644 index 00000000000000..0fe08c42d64659 --- /dev/null +++ b/drivers/gpu/drm/apple/iomfb_v12_3.c @@ -0,0 +1,108 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright The Asahi Linux Contributors */ + +#include "iomfb_v12_3.h" +#include "iomfb_v13_3.h" +#include "version_utils.h" + +static const struct dcp_method_entry dcp_methods[dcpep_num_methods] = { + IOMFB_METHOD("A000", dcpep_late_init_signal), + IOMFB_METHOD("A029", dcpep_setup_video_limits), + IOMFB_METHOD("A131", iomfbep_a131_pmu_service_matched), + IOMFB_METHOD("A132", iomfbep_a132_backlight_service_matched), + IOMFB_METHOD("A357", dcpep_set_create_dfb), + IOMFB_METHOD("A358", iomfbep_a358_vi_set_temperature_hint), + IOMFB_METHOD("A401", dcpep_start_signal), + IOMFB_METHOD("A407", dcpep_swap_start), + IOMFB_METHOD("A408", dcpep_swap_submit), + IOMFB_METHOD("A410", dcpep_set_display_device), + IOMFB_METHOD("A411", dcpep_is_main_display), + IOMFB_METHOD("A412", dcpep_set_digital_out_mode), + IOMFB_METHOD("A422", iomfbep_set_matrix), + IOMFB_METHOD("A426", iomfbep_get_color_remap_mode), + IOMFB_METHOD("A439", dcpep_set_parameter_dcp), + IOMFB_METHOD("A443", dcpep_create_default_fb), + IOMFB_METHOD("A447", dcpep_enable_disable_video_power_savings), + IOMFB_METHOD("A454", dcpep_first_client_open), + IOMFB_METHOD("A455", iomfbep_last_client_close), + IOMFB_METHOD("A460", dcpep_set_display_refresh_properties), + IOMFB_METHOD("A463", dcpep_flush_supports_power), + IOMFB_METHOD("A464", iomfbep_abort_swaps_dcp), + IOMFB_METHOD("A468", dcpep_set_power_state), +}; + +#define DCP_FW v12_3 +#define DCP_FW_VER DCP_FW_VERSION(12, 3, 0) + +#include "iomfb_template.c" + +static const iomfb_cb_handler cb_handlers[IOMFB_MAX_CB] = { + [0] = trampoline_true, /* did_boot_signal */ + [1] = trampoline_true, /* did_power_on_signal */ + [2] = trampoline_nop, /* will_power_off_signal */ + [3] = trampoline_rt_bandwidth, + [100] = iomfbep_cb_match_pmu_service, + [101] = trampoline_zero, /* get_display_default_stride */ + [102] = trampoline_nop, /* set_number_property */ + [103] = trampoline_nop, /* set_boolean_property */ + [106] = trampoline_nop, /* remove_property */ + [107] = trampoline_true, /* create_provider_service */ + [108] = trampoline_true, /* create_product_service */ + [109] = trampoline_true, /* create_pmu_service */ + [110] = trampoline_true, /* create_iomfb_service */ + [111] = trampoline_create_backlight_service, + [116] = dcpep_cb_boot_1, + [117] = trampoline_false, /* is_dark_boot */ + [118] = trampoline_false, /* is_dark_boot / is_waking_from_hibernate*/ + [120] = trampoline_read_edt_data, + [122] = trampoline_prop_start, + [123] = trampoline_prop_chunk, + [124] = trampoline_prop_end, + [201] = trampoline_map_piodma, + [202] = trampoline_unmap_piodma, + [206] = iomfbep_cb_match_pmu_service_2, + [207] = iomfbep_cb_match_backlight_service, + [208] = trampoline_get_time, + [211] = trampoline_nop, /* update_backlight_factor_prop */ + [300] = trampoline_pr_publish, + [401] = trampoline_get_uint_prop, + [404] = trampoline_nop, /* sr_set_uint_prop */ + [406] = trampoline_set_fx_prop, + [408] = trampoline_get_frequency, + [411] = trampoline_map_reg, + [413] = trampoline_true, /* sr_set_property_dict */ + [414] = trampoline_sr_set_property_int, + [415] = trampoline_true, /* sr_set_property_bool */ + [451] = trampoline_allocate_buffer, + [452] = trampoline_map_physical, + [456] = trampoline_release_mem_desc, + [552] = trampoline_true, /* set_property_dict_0 */ + [561] = trampoline_true, /* set_property_dict */ + [563] = trampoline_true, /* set_property_int */ + [565] = trampoline_true, /* set_property_bool */ + [567] = trampoline_true, /* set_property_str */ + [574] = trampoline_zero, /* power_up_dart */ + [576] = trampoline_hotplug, + [577] = trampoline_nop, /* powerstate_notify */ + [582] = trampoline_true, /* create_default_fb_surface */ + [584] = trampoline_nop, /* IOMobileFramebufferAP::clear_default_surface */ + [588] = trampoline_nop, /* resize_default_fb_surface_gated */ + [589] = trampoline_swap_complete, + [591] = trampoline_swap_complete_intent_gated, + [592] = trampoline_abort_swap_ap_gated, + [593] = trampoline_enable_backlight_message_ap_gated, + [594] = trampoline_nop, /* IOMobileFramebufferAP::setSystemConsoleMode */ + [596] = trampoline_false, /* IOMobileFramebufferAP::isDFBAllocated */ + [597] = trampoline_false, /* IOMobileFramebufferAP::preserveContents */ + [598] = trampoline_nop, /* find_swap_function_gated */ +}; + +void DCP_FW_NAME(iomfb_start)(struct apple_dcp *dcp) +{ + dcp->cb_handlers = cb_handlers; + + dcp_start_signal(dcp, false, dcp_started, NULL); +} + +#undef DCP_FW_VER +#undef DCP_FW diff --git a/drivers/gpu/drm/apple/iomfb_v12_3.h b/drivers/gpu/drm/apple/iomfb_v12_3.h new file mode 100644 index 00000000000000..7359685d981fe5 --- /dev/null +++ b/drivers/gpu/drm/apple/iomfb_v12_3.h @@ -0,0 +1,17 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright The Asahi Linux Contributors */ + +#ifndef __APPLE_IOMFB_V12_3_H__ +#define __APPLE_IOMFB_V12_3_H__ + +#include "version_utils.h" + +#define DCP_FW v12_3 +#define DCP_FW_VER DCP_FW_VERSION(12, 3, 0) + +#include "iomfb_template.h" + +#undef DCP_FW_VER +#undef DCP_FW + +#endif /* __APPLE_IOMFB_V12_3_H__ */ diff --git a/drivers/gpu/drm/apple/iomfb_v13_3.c b/drivers/gpu/drm/apple/iomfb_v13_3.c new file mode 100644 index 00000000000000..0ac869d24eb01b --- /dev/null +++ b/drivers/gpu/drm/apple/iomfb_v13_3.c @@ -0,0 +1,113 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright The Asahi Linux Contributors */ + +#include "iomfb_v12_3.h" +#include "iomfb_v13_3.h" +#include "version_utils.h" + +static const struct dcp_method_entry dcp_methods[dcpep_num_methods] = { + IOMFB_METHOD("A000", dcpep_late_init_signal), + IOMFB_METHOD("A029", dcpep_setup_video_limits), + IOMFB_METHOD("A131", iomfbep_a131_pmu_service_matched), + IOMFB_METHOD("A132", iomfbep_a132_backlight_service_matched), + IOMFB_METHOD("A373", dcpep_set_create_dfb), + IOMFB_METHOD("A374", iomfbep_a358_vi_set_temperature_hint), + IOMFB_METHOD("A401", dcpep_start_signal), + IOMFB_METHOD("A407", dcpep_swap_start), + IOMFB_METHOD("A408", dcpep_swap_submit), + IOMFB_METHOD("A410", dcpep_set_display_device), + IOMFB_METHOD("A411", dcpep_is_main_display), + IOMFB_METHOD("A412", dcpep_set_digital_out_mode), + IOMFB_METHOD("A422", iomfbep_set_matrix), + IOMFB_METHOD("A426", iomfbep_get_color_remap_mode), + IOMFB_METHOD("A441", dcpep_set_parameter_dcp), + IOMFB_METHOD("A445", dcpep_create_default_fb), + IOMFB_METHOD("A449", dcpep_enable_disable_video_power_savings), + IOMFB_METHOD("A456", dcpep_first_client_open), + IOMFB_METHOD("A457", iomfbep_last_client_close), + IOMFB_METHOD("A463", dcpep_set_display_refresh_properties), + IOMFB_METHOD("A466", dcpep_flush_supports_power), + IOMFB_METHOD("A467", iomfbep_abort_swaps_dcp), + IOMFB_METHOD("A472", dcpep_set_power_state), +}; + +#define DCP_FW v13_3 +#define DCP_FW_VER DCP_FW_VERSION(13, 3, 0) + +#include "iomfb_template.c" + +static const iomfb_cb_handler cb_handlers[IOMFB_MAX_CB] = { + [0] = trampoline_true, /* did_boot_signal */ + [1] = trampoline_true, /* did_power_on_signal */ + [2] = trampoline_nop, /* will_power_off_signal */ + [3] = trampoline_rt_bandwidth, + [6] = trampoline_set_frame_sync_props, + [100] = iomfbep_cb_match_pmu_service, + [101] = trampoline_zero, /* get_display_default_stride */ + [102] = trampoline_nop, /* set_number_property */ + [103] = trampoline_nop, /* trigger_user_cal_loader */ + [104] = trampoline_nop, /* set_boolean_property */ + [107] = trampoline_nop, /* remove_property */ + [108] = trampoline_true, /* create_provider_service */ + [109] = trampoline_true, /* create_product_service */ + [110] = trampoline_true, /* create_pmu_service */ + [111] = trampoline_true, /* create_iomfb_service */ + [112] = trampoline_create_backlight_service, + [113] = trampoline_true, /* create_nvram_service? */ + [114] = trampoline_get_tiling_state, + [115] = trampoline_false, /* set_tiling_state */ + [120] = dcpep_cb_boot_1, + [121] = trampoline_false, /* is_dark_boot */ + [122] = trampoline_false, /* is_dark_boot / is_waking_from_hibernate*/ + [124] = trampoline_read_edt_data, + [126] = trampoline_prop_start, + [127] = trampoline_prop_chunk, + [128] = trampoline_prop_end, + [129] = trampoline_allocate_bandwidth, + [201] = trampoline_map_piodma, + [202] = trampoline_unmap_piodma, + [206] = iomfbep_cb_match_pmu_service_2, + [207] = iomfbep_cb_match_backlight_service, + [208] = trampoline_nop, /* update_backlight_factor_prop */ + [209] = trampoline_get_time, + [300] = trampoline_pr_publish, + [401] = trampoline_get_uint_prop, + [404] = trampoline_nop, /* sr_set_uint_prop */ + [406] = trampoline_set_fx_prop, + [408] = trampoline_get_frequency, + [411] = trampoline_map_reg, + [413] = trampoline_true, /* sr_set_property_dict */ + [414] = trampoline_sr_set_property_int, + [415] = trampoline_true, /* sr_set_property_bool */ + [451] = trampoline_allocate_buffer, + [452] = trampoline_map_physical, + [454] = trampoline_release_mem_desc, + [552] = trampoline_true, /* set_property_dict_0 */ + [561] = trampoline_true, /* set_property_dict */ + [563] = trampoline_true, /* set_property_int */ + [565] = trampoline_true, /* set_property_bool */ + [567] = trampoline_true, /* set_property_str */ + [574] = trampoline_zero, /* power_up_dart */ + [576] = trampoline_hotplug, + [577] = trampoline_nop, /* powerstate_notify */ + [582] = trampoline_true, /* create_default_fb_surface */ + [584] = trampoline_nop, /* IOMobileFramebufferAP::clear_default_surface */ + [588] = trampoline_nop, /* resize_default_fb_surface_gated */ + [589] = trampoline_swap_complete, + [591] = trampoline_swap_complete_intent_gated, + [592] = trampoline_abort_swap_ap_gated, + [593] = trampoline_enable_backlight_message_ap_gated, + [594] = trampoline_nop, /* IOMobileFramebufferAP::setSystemConsoleMode */ + [596] = trampoline_false, /* IOMobileFramebufferAP::isDFBAllocated */ + [597] = trampoline_false, /* IOMobileFramebufferAP::preserveContents */ + [598] = trampoline_nop, /* find_swap_function_gated */ +}; +void DCP_FW_NAME(iomfb_start)(struct apple_dcp *dcp) +{ + dcp->cb_handlers = cb_handlers; + + dcp_start_signal(dcp, false, dcp_started, NULL); +} + +#undef DCP_FW_VER +#undef DCP_FW diff --git a/drivers/gpu/drm/apple/iomfb_v13_3.h b/drivers/gpu/drm/apple/iomfb_v13_3.h new file mode 100644 index 00000000000000..bbb3156b40f893 --- /dev/null +++ b/drivers/gpu/drm/apple/iomfb_v13_3.h @@ -0,0 +1,17 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright The Asahi Linux Contributors */ + +#ifndef __APPLE_IOMFB_V13_3_H__ +#define __APPLE_IOMFB_V13_3_H__ + +#include "version_utils.h" + +#define DCP_FW v13_3 +#define DCP_FW_VER DCP_FW_VERSION(13, 3, 0) + +#include "iomfb_template.h" + +#undef DCP_FW_VER +#undef DCP_FW + +#endif /* __APPLE_IOMFB_V13_3_H__ */ diff --git a/drivers/gpu/drm/apple/parser.c b/drivers/gpu/drm/apple/parser.c new file mode 100644 index 00000000000000..dd1d76936fb18b --- /dev/null +++ b/drivers/gpu/drm/apple/parser.c @@ -0,0 +1,1068 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright 2021 Alyssa Rosenzweig */ + +#include +#include +#include +#include +#include + +#if IS_ENABLED(CONFIG_DRM_APPLE_AUDIO) +#include // for sound format masks +#endif + +#include "parser.h" +#include "trace.h" + +#define DCP_PARSE_HEADER 0xd3 + +enum dcp_parse_type { + DCP_TYPE_DICTIONARY = 1, + DCP_TYPE_ARRAY = 2, + DCP_TYPE_INT64 = 4, + DCP_TYPE_STRING = 9, + DCP_TYPE_BLOB = 10, + DCP_TYPE_BOOL = 11 +}; + +struct dcp_parse_tag { + unsigned int size : 24; + enum dcp_parse_type type : 5; + unsigned int padding : 2; + bool last : 1; +} __packed; + +static const void *parse_bytes(struct dcp_parse_ctx *ctx, size_t count) +{ + const void *ptr = ctx->blob + ctx->pos; + + if (ctx->pos + count > ctx->len) + return ERR_PTR(-EINVAL); + + ctx->pos += count; + return ptr; +} + +static const u32 *parse_u32(struct dcp_parse_ctx *ctx) +{ + return parse_bytes(ctx, sizeof(u32)); +} + +static const struct dcp_parse_tag *parse_tag(struct dcp_parse_ctx *ctx) +{ + const struct dcp_parse_tag *tag; + + /* Align to 32-bits */ + ctx->pos = round_up(ctx->pos, 4); + + tag = parse_bytes(ctx, sizeof(struct dcp_parse_tag)); + + if (IS_ERR(tag)) + return tag; + + if (tag->padding) + return ERR_PTR(-EINVAL); + + return tag; +} + +static const struct dcp_parse_tag *parse_tag_of_type(struct dcp_parse_ctx *ctx, + enum dcp_parse_type type) +{ + const struct dcp_parse_tag *tag = parse_tag(ctx); + + if (IS_ERR(tag)) + return tag; + + if (tag->type != type) + return ERR_PTR(-EINVAL); + + return tag; +} + +static int skip(struct dcp_parse_ctx *handle) +{ + const struct dcp_parse_tag *tag = parse_tag(handle); + int ret = 0; + int i; + + if (IS_ERR(tag)) + return PTR_ERR(tag); + + switch (tag->type) { + case DCP_TYPE_DICTIONARY: + for (i = 0; i < tag->size; ++i) { + ret |= skip(handle); /* key */ + ret |= skip(handle); /* value */ + } + + return ret; + + case DCP_TYPE_ARRAY: + for (i = 0; i < tag->size; ++i) + ret |= skip(handle); + + return ret; + + case DCP_TYPE_INT64: + handle->pos += sizeof(s64); + return 0; + + case DCP_TYPE_STRING: + case DCP_TYPE_BLOB: + handle->pos += tag->size; + return 0; + + case DCP_TYPE_BOOL: + return 0; + + default: + return -EINVAL; + } +} + +#if IS_ENABLED(CONFIG_DRM_APPLE_AUDIO) +static int skip_pair(struct dcp_parse_ctx *handle) +{ + int ret; + + ret = skip(handle); + if (ret) + return ret; + + return skip(handle); +} + +static bool consume_string(struct dcp_parse_ctx *ctx, const char *specimen) +{ + const struct dcp_parse_tag *tag; + const char *key; + ctx->pos = round_up(ctx->pos, 4); + + if (ctx->pos + sizeof(*tag) + strlen(specimen) - 1 > ctx->len) + return false; + tag = ctx->blob + ctx->pos; + key = ctx->blob + ctx->pos + sizeof(*tag); + if (tag->padding) + return false; + + if (tag->type != DCP_TYPE_STRING || + tag->size != strlen(specimen) || + strncmp(key, specimen, tag->size)) + return false; + + skip(ctx); + return true; +} +#endif + +/* Caller must free the result */ +static char *parse_string(struct dcp_parse_ctx *handle) +{ + const struct dcp_parse_tag *tag = parse_tag_of_type(handle, DCP_TYPE_STRING); + const char *in; + char *out; + + if (IS_ERR(tag)) + return (void *)tag; + + in = parse_bytes(handle, tag->size); + if (IS_ERR(in)) + return (void *)in; + + out = kmalloc(tag->size + 1, GFP_KERNEL); + + memcpy(out, in, tag->size); + out[tag->size] = '\0'; + return out; +} + +static int parse_int(struct dcp_parse_ctx *handle, s64 *value) +{ + const void *tag = parse_tag_of_type(handle, DCP_TYPE_INT64); + const s64 *in; + + if (IS_ERR(tag)) + return PTR_ERR(tag); + + in = parse_bytes(handle, sizeof(s64)); + + if (IS_ERR(in)) + return PTR_ERR(in); + + memcpy(value, in, sizeof(*value)); + return 0; +} + +/* + * DCP stores VRR refresh rates in 64-bit regions, however the number is actually an + * unsigned Q16.16 with the high 32 bits unused. + */ +static int parse_q1616(struct dcp_parse_ctx *handle, u32 *value) +{ + s64 dcp_int; + u32 in; + int ret; + + ret = parse_int(handle, &dcp_int); + if (ret) + return ret; + + in = dcp_int & 0xffffffff; + + memcpy(value, &in, sizeof(*value)); + return 0; +} + +static int parse_bool(struct dcp_parse_ctx *handle, bool *b) +{ + const struct dcp_parse_tag *tag = parse_tag_of_type(handle, DCP_TYPE_BOOL); + + if (IS_ERR(tag)) + return PTR_ERR(tag); + + *b = !!tag->size; + return 0; +} + +#if IS_ENABLED(CONFIG_DRM_APPLE_AUDIO) +static int parse_blob(struct dcp_parse_ctx *handle, size_t size, u8 const **blob) +{ + const struct dcp_parse_tag *tag = parse_tag_of_type(handle, DCP_TYPE_BLOB); + const u8 *out; + + if (IS_ERR(tag)) + return PTR_ERR(tag); + + if (tag->size < size) + return -EINVAL; + + out = parse_bytes(handle, tag->size); + + if (IS_ERR(out)) + return PTR_ERR(out); + + *blob = out; + return 0; +} +#endif + +struct iterator { + struct dcp_parse_ctx *handle; + u32 idx, len; +}; + +static int iterator_begin(struct dcp_parse_ctx *handle, struct iterator *it, + bool dict) +{ + const struct dcp_parse_tag *tag; + enum dcp_parse_type type = dict ? DCP_TYPE_DICTIONARY : DCP_TYPE_ARRAY; + + *it = (struct iterator) { + .handle = handle, + .idx = 0 + }; + + tag = parse_tag_of_type(it->handle, type); + if (IS_ERR(tag)) + return PTR_ERR(tag); + + it->len = tag->size; + return 0; +} + +#define dcp_parse_foreach_in_array(handle, it) \ + for (iterator_begin(handle, &it, false); it.idx < it.len; ++it.idx) +#define dcp_parse_foreach_in_dict(handle, it) \ + for (iterator_begin(handle, &it, true); it.idx < it.len; ++it.idx) + +int parse(const void *blob, size_t size, struct dcp_parse_ctx *ctx) +{ + const u32 *header; + + *ctx = (struct dcp_parse_ctx) { + .blob = blob, + .len = size, + .pos = 0, + }; + + header = parse_u32(ctx); + if (IS_ERR(header)) + return PTR_ERR(header); + + if (*header != DCP_PARSE_HEADER) + return -EINVAL; + + return 0; +} + +static int parse_dimension(struct dcp_parse_ctx *handle, struct dimension *dim) +{ + struct iterator it; + int ret = 0; + + dcp_parse_foreach_in_dict(handle, it) { + char *key = parse_string(it.handle); + + if (IS_ERR(key)) + ret = PTR_ERR(key); + else if (!strcmp(key, "Active")) + ret = parse_int(it.handle, &dim->active); + else if (!strcmp(key, "Total")) + ret = parse_int(it.handle, &dim->total); + else if (!strcmp(key, "FrontPorch")) + ret = parse_int(it.handle, &dim->front_porch); + else if (!strcmp(key, "SyncWidth")) + ret = parse_int(it.handle, &dim->sync_width); + else if (!strcmp(key, "PreciseSyncRate")) + ret = parse_int(it.handle, &dim->precise_sync_rate); + else + skip(it.handle); + + if (!IS_ERR_OR_NULL(key)) + kfree(key); + + if (ret) + return ret; + } + + return 0; +} + +struct color_mode { + s64 colorimetry; + s64 depth; + s64 dynamic_range; + s64 eotf; + s64 id; + s64 pixel_encoding; + s64 score; +}; + +static int fill_color_mode(struct dcp_color_mode *color, + struct color_mode *cmode) +{ + if (color->score >= cmode->score) + return 0; + + if (cmode->colorimetry < 0 || cmode->colorimetry >= DCP_COLORIMETRY_COUNT) + return -EINVAL; + if (cmode->depth < 8 || cmode->depth > 12) + return -EINVAL; + if (cmode->dynamic_range < 0 || cmode->dynamic_range >= DCP_COLOR_YCBCR_RANGE_COUNT) + return -EINVAL; + if (cmode->eotf < 0 || cmode->eotf >= DCP_EOTF_COUNT) + return -EINVAL; + if (cmode->pixel_encoding < 0 || cmode->pixel_encoding >= DCP_COLOR_FORMAT_COUNT) + return -EINVAL; + + color->score = cmode->score; + color->id = cmode->id; + color->eotf = cmode->eotf; + color->format = cmode->pixel_encoding; + color->colorimetry = cmode->colorimetry; + color->range = cmode->dynamic_range; + color->depth = cmode->depth; + + return 0; +} + +static int parse_color_modes(struct dcp_parse_ctx *handle, + struct dcp_display_mode *out) +{ + struct iterator outer_it; + int ret = 0; + out->sdr_444.score = -1; + out->sdr_rgb.score = -1; + out->sdr.score = -1; + out->best.score = -1; + + dcp_parse_foreach_in_array(handle, outer_it) { + struct iterator it; + bool is_virtual = true; + struct color_mode cmode; + + dcp_parse_foreach_in_dict(handle, it) { + char *key = parse_string(it.handle); + + if (IS_ERR(key)) + ret = PTR_ERR(key); + else if (!strcmp(key, "Colorimetry")) + ret = parse_int(it.handle, &cmode.colorimetry); + else if (!strcmp(key, "Depth")) + ret = parse_int(it.handle, &cmode.depth); + else if (!strcmp(key, "DynamicRange")) + ret = parse_int(it.handle, &cmode.dynamic_range); + else if (!strcmp(key, "EOTF")) + ret = parse_int(it.handle, &cmode.eotf); + else if (!strcmp(key, "ID")) + ret = parse_int(it.handle, &cmode.id); + else if (!strcmp(key, "IsVirtual")) + ret = parse_bool(it.handle, &is_virtual); + else if (!strcmp(key, "PixelEncoding")) + ret = parse_int(it.handle, &cmode.pixel_encoding); + else if (!strcmp(key, "Score")) + ret = parse_int(it.handle, &cmode.score); + else + skip(it.handle); + + if (!IS_ERR_OR_NULL(key)) + kfree(key); + + if (ret) + return ret; + } + + /* Skip virtual or partial entries */ + if (is_virtual || cmode.score < 0 || cmode.id < 0) + continue; + + trace_iomfb_color_mode(handle->dcp, cmode.id, cmode.score, + cmode.depth, cmode.colorimetry, + cmode.eotf, cmode.dynamic_range, + cmode.pixel_encoding); + + if (cmode.eotf == DCP_EOTF_SDR_GAMMA) { + if (cmode.pixel_encoding == DCP_COLOR_FORMAT_RGB && + cmode.depth <= 10) + fill_color_mode(&out->sdr_rgb, &cmode); + else if (cmode.pixel_encoding == DCP_COLOR_FORMAT_YCBCR444 && + cmode.depth <= 10) + fill_color_mode(&out->sdr_444, &cmode); + fill_color_mode(&out->sdr, &cmode); + } + fill_color_mode(&out->best, &cmode); + } + + return 0; +} + +/* + * Calculate the pixel clock for a mode given the 16:16 fixed-point refresh + * rate. The pixel clock is the refresh rate times the pixel count. DRM + * specifies the clock in kHz. The intermediate result may overflow a u32, so + * use a u64 where required. + */ +static u32 calculate_clock(struct dimension *horiz, struct dimension *vert) +{ + u32 pixels = horiz->total * vert->total; + u64 clock = mul_u32_u32(pixels, vert->precise_sync_rate); + + return DIV_ROUND_CLOSEST_ULL(clock >> 16, 1000); +} + +static int parse_mode(struct dcp_parse_ctx *handle, + struct dcp_display_mode *out, s64 *score, int width_mm, + int height_mm, unsigned notch_height) +{ + int ret = 0; + struct iterator it; + struct dimension horiz, vert; + s64 id = -1; + s64 best_color_mode = -1; + bool is_virtual = false; + struct drm_display_mode *mode = &out->mode; + + dcp_parse_foreach_in_dict(handle, it) { + char *key = parse_string(it.handle); + + if (IS_ERR(key)) + ret = PTR_ERR(key); + else if (is_virtual) + skip(it.handle); + else if (!strcmp(key, "HorizontalAttributes")) + ret = parse_dimension(it.handle, &horiz); + else if (!strcmp(key, "VerticalAttributes")) + ret = parse_dimension(it.handle, &vert); + else if (!strcmp(key, "MinimumVariableRefreshRate")) + ret = parse_q1616(it.handle, &out->min_vrr); + else if (!strcmp(key, "MaximumVariableRefreshRate")) + ret = parse_q1616(it.handle, &out->max_vrr); + else if (!strcmp(key, "ColorModes")) + ret = parse_color_modes(it.handle, out); + else if (!strcmp(key, "ID")) + ret = parse_int(it.handle, &id); + else if (!strcmp(key, "IsVirtual")) + ret = parse_bool(it.handle, &is_virtual); + else if (!strcmp(key, "Score")) + ret = parse_int(it.handle, score); + else + skip(it.handle); + + if (!IS_ERR_OR_NULL(key)) + kfree(key); + + if (ret) { + trace_iomfb_parse_mode_fail(id, &horiz, &vert, best_color_mode, is_virtual, *score); + return ret; + } + } + if (out->sdr_rgb.score >= 0) + best_color_mode = out->sdr_rgb.id; + else if (out->sdr_444.score >= 0) + best_color_mode = out->sdr_444.id; + else if (out->sdr.score >= 0) + best_color_mode = out->sdr.id; + else if (out->best.score >= 0) + best_color_mode = out->best.id; + + trace_iomfb_parse_mode_success(id, &horiz, &vert, best_color_mode, + is_virtual, *score); + + /* + * Reject modes without valid color mode. + */ + if (best_color_mode < 0) + return -EINVAL; + + /* + * We need to skip virtual modes. In some cases, virtual modes are "too + * big" for the monitor and can cause breakage. It is unclear why the + * DCP reports these modes at all. Treat as a recoverable error. + */ + if (is_virtual) + return -EINVAL; + + /* + * HACK: + * Mark the 120 Hz mode on j314/j316 (identified by resolution) as vrr. + * Setting timestamps in the the swap_surface message to non-zero + * values drives the display at 120 fps. + */ + if (vert.precise_sync_rate >> 16 == 120 && + ((horiz.active == 3024 && vert.active == 1964) || + (horiz.active == 3456 && vert.active == 2234))) { + out->min_vrr = 24 << 16; + out->max_vrr = 120 << 16; + out->vrr = true; + } + + if (out->min_vrr && out->max_vrr) + out->vrr = true; + + vert.active -= notch_height; + vert.sync_width += notch_height; + + /* From here we must succeed. Start filling out the mode. */ + *mode = (struct drm_display_mode) { + .type = DRM_MODE_TYPE_DRIVER, + .clock = calculate_clock(&horiz, &vert), + + .vdisplay = vert.active, + .vsync_start = vert.active + vert.front_porch, + .vsync_end = vert.active + vert.front_porch + vert.sync_width, + .vtotal = vert.total, + + .hdisplay = horiz.active, + .hsync_start = horiz.active + horiz.front_porch, + .hsync_end = horiz.active + horiz.front_porch + + horiz.sync_width, + .htotal = horiz.total, + + .width_mm = width_mm, + .height_mm = height_mm, + }; + + drm_mode_set_name(mode); + + out->timing_mode_id = id; + out->color_mode_id = best_color_mode; + + trace_iomfb_timing_mode(handle->dcp, id, *score, horiz.active, + vert.active, vert.precise_sync_rate, + best_color_mode); + + return 0; +} + +struct dcp_display_mode *enumerate_modes(struct dcp_parse_ctx *handle, + unsigned int *count, int width_mm, + int height_mm, unsigned notch_height) +{ + struct iterator it; + int ret; + struct dcp_display_mode *mode, *modes; + struct dcp_display_mode *best_mode = NULL; + s64 score, best_score = -1; + + ret = iterator_begin(handle, &it, false); + + if (ret) + return ERR_PTR(ret); + + /* Start with a worst case allocation */ + modes = kmalloc_array(it.len, sizeof(*modes), GFP_KERNEL); + *count = 0; + + if (!modes) + return ERR_PTR(-ENOMEM); + + for (; it.idx < it.len; ++it.idx) { + mode = &modes[*count]; + ret = parse_mode(it.handle, mode, &score, width_mm, height_mm, notch_height); + + /* Errors for a single mode are recoverable -- just skip it. */ + if (ret) + continue; + + /* Process a successful mode */ + (*count)++; + + if (score > best_score) { + best_score = score; + best_mode = mode; + } + } + + if (best_mode != NULL) + best_mode->mode.type |= DRM_MODE_TYPE_PREFERRED; + + return modes; +} + +int parse_display_attributes(struct dcp_parse_ctx *handle, int *width_mm, + int *height_mm) +{ + int ret = 0; + struct iterator it; + s64 width_cm = 0, height_cm = 0; + + dcp_parse_foreach_in_dict(handle, it) { + char *key = parse_string(it.handle); + + if (IS_ERR(key)) + ret = PTR_ERR(key); + else if (!strcmp(key, "MaxHorizontalImageSize")) + ret = parse_int(it.handle, &width_cm); + else if (!strcmp(key, "MaxVerticalImageSize")) + ret = parse_int(it.handle, &height_cm); + else + skip(it.handle); + + if (!IS_ERR_OR_NULL(key)) + kfree(key); + + if (ret) + return ret; + } + + /* 1cm = 10mm */ + *width_mm = 10 * width_cm; + *height_mm = 10 * height_cm; + + return 0; +} + +int parse_epic_service_init(struct dcp_parse_ctx *handle, const char **name, + const char **class, s64 *unit) +{ + int ret = 0; + struct iterator it; + bool parsed_unit = false; + bool parsed_name = false; + bool parsed_class = false; + + *name = ERR_PTR(-ENOENT); + *class = ERR_PTR(-ENOENT); + + dcp_parse_foreach_in_dict(handle, it) { + char *key = parse_string(it.handle); + + if (IS_ERR(key)) { + ret = PTR_ERR(key); + break; + } + + if (!strcmp(key, "EPICName")) { + *name = parse_string(it.handle); + if (IS_ERR(*name)) + ret = PTR_ERR(*name); + else + parsed_name = true; + } else if (!strcmp(key, "EPICProviderClass")) { + *class = parse_string(it.handle); + if (IS_ERR(*class)) + ret = PTR_ERR(*class); + else + parsed_class = true; + } else if (!strcmp(key, "EPICUnit")) { + ret = parse_int(it.handle, unit); + if (!ret) + parsed_unit = true; + } else { + skip(it.handle); + } + + kfree(key); + if (ret) + break; + } + + if (!parsed_unit || !parsed_name || !parsed_class) + ret = -ENOENT; + + if (ret) { + if (!IS_ERR(*name)) { + kfree(*name); + *name = ERR_PTR(ret); + } + if (!IS_ERR(*class)) { + kfree(*class); + *class = ERR_PTR(ret); + } + } + + return ret; +} + +#if IS_ENABLED(CONFIG_DRM_APPLE_AUDIO) +static int parse_sample_rate_bit(struct dcp_parse_ctx *handle, unsigned int *ratebit) +{ + s64 rate; + int ret = parse_int(handle, &rate); + + if (ret) + return ret; + + *ratebit = snd_pcm_rate_to_rate_bit(rate); + if (*ratebit == SNDRV_PCM_RATE_KNOT) { + /* + * The rate wasn't recognized, and unless we supply + * a supplementary constraint, the SNDRV_PCM_RATE_KNOT bit + * will allow any rate. So clear it. + */ + *ratebit = 0; + } + + return 0; +} + +static int parse_sample_fmtbit(struct dcp_parse_ctx *handle, u64 *fmtbit) +{ + s64 sample_size; + int ret = parse_int(handle, &sample_size); + + if (ret) + return ret; + + switch (sample_size) { + case 16: + *fmtbit = SNDRV_PCM_FMTBIT_S16; + break; + case 20: + *fmtbit = SNDRV_PCM_FMTBIT_S20; + break; + case 24: + *fmtbit = SNDRV_PCM_FMTBIT_S24; + break; + case 32: + *fmtbit = SNDRV_PCM_FMTBIT_S32; + break; + default: + *fmtbit = 0; + break; + } + + return 0; +} + +static struct { + const char *label; + u8 type; +} chan_position_names[] = { + { "Front Left", SNDRV_CHMAP_FL }, + { "Front Right", SNDRV_CHMAP_FR }, + { "Rear Left", SNDRV_CHMAP_RL }, + { "Rear Right", SNDRV_CHMAP_RR }, + { "Front Center", SNDRV_CHMAP_FC }, + { "Low Frequency Effects", SNDRV_CHMAP_LFE }, + { "Rear Center", SNDRV_CHMAP_RC }, + { "Front Left Center", SNDRV_CHMAP_FLC }, + { "Front Right Center", SNDRV_CHMAP_FRC }, + { "Rear Left Center", SNDRV_CHMAP_RLC }, + { "Rear Right Center", SNDRV_CHMAP_RRC }, + { "Front Left Wide", SNDRV_CHMAP_FLW }, + { "Front Right Wide", SNDRV_CHMAP_FRW }, + { "Front Left High", SNDRV_CHMAP_FLH }, + { "Front Center High", SNDRV_CHMAP_FCH }, + { "Front Right High", SNDRV_CHMAP_FRH }, + { "Top Center", SNDRV_CHMAP_TC }, +}; + +static void append_chmap(struct snd_pcm_chmap_elem *chmap, u8 type) +{ + if (!chmap || chmap->channels >= ARRAY_SIZE(chmap->map)) + return; + + chmap->map[chmap->channels] = type; + chmap->channels++; +} + +static int parse_chmap(struct dcp_parse_ctx *handle, struct snd_pcm_chmap_elem *chmap) +{ + struct iterator it; + int i, ret; + + if (!chmap) { + skip(handle); + return 0; + } + + chmap->channels = 0; + + dcp_parse_foreach_in_array(handle, it) { + for (i = 0; i < ARRAY_SIZE(chan_position_names); i++) + if (consume_string(it.handle, chan_position_names[i].label)) + break; + + if (i == ARRAY_SIZE(chan_position_names)) { + ret = skip(it.handle); + if (ret) + return ret; + + append_chmap(chmap, SNDRV_CHMAP_UNKNOWN); + continue; + } + + append_chmap(chmap, chan_position_names[i].type); + } + + return 0; +} + +static int parse_chan_layout_element(struct dcp_parse_ctx *handle, + unsigned int *nchans_out, + struct snd_pcm_chmap_elem *chmap) +{ + struct iterator it; + int ret; + s64 nchans = 0; + + dcp_parse_foreach_in_dict(handle, it) { + if (consume_string(it.handle, "ActiveChannelCount")) + ret = parse_int(it.handle, &nchans); + else if (consume_string(it.handle, "ChannelLayout")) + ret = parse_chmap(it.handle, chmap); + else + ret = skip_pair(it.handle); + + if (ret) + return ret; + } + + if (nchans_out) + *nchans_out = nchans; + + return 0; +} + +static int parse_nchans_mask(struct dcp_parse_ctx *handle, unsigned int *mask) +{ + struct iterator it; + int ret; + + *mask = 0; + + dcp_parse_foreach_in_array(handle, it) { + int nchans; + + ret = parse_chan_layout_element(it.handle, &nchans, NULL); + if (ret) + return ret; + *mask |= 1 << nchans; + } + + return 0; +} + +static int parse_avep_element(struct dcp_parse_ctx *handle, + struct dcp_sound_format_mask *sieve, + struct dcp_sound_format_mask *hits) +{ + struct dcp_sound_format_mask mask = {0, 0, 0}; + struct iterator it; + int ret; + + dcp_parse_foreach_in_dict(handle, it) { + if (consume_string(handle, "StreamSampleRate")) + ret = parse_sample_rate_bit(it.handle, &mask.rates); + else if (consume_string(handle, "SampleSize")) + ret = parse_sample_fmtbit(it.handle, &mask.formats); + else if (consume_string(handle, "AudioChannelLayoutElements")) + ret = parse_nchans_mask(it.handle, &mask.nchans); + else + ret = skip_pair(it.handle); + + if (ret) + return ret; + } + + trace_avep_sound_mode(handle->dcp, mask.rates, mask.formats, mask.nchans); + + if (!(mask.rates & sieve->rates) || !(mask.formats & sieve->formats) || + !(mask.nchans & sieve->nchans)) + return 0; + + if (hits) { + hits->rates |= mask.rates; + hits->formats |= mask.formats; + hits->nchans |= mask.nchans; + } + + return 1; +} + +static int parse_mode_in_avep_element(struct dcp_parse_ctx *handle, + unsigned int selected_nchans, + struct snd_pcm_chmap_elem *chmap, + struct dcp_sound_cookie *cookie) +{ + struct iterator it; + struct dcp_parse_ctx save_handle; + int ret; + + dcp_parse_foreach_in_dict(handle, it) { + if (consume_string(it.handle, "AudioChannelLayoutElements")) { + struct iterator inner_it; + int nchans; + + dcp_parse_foreach_in_array(it.handle, inner_it) { + save_handle = *it.handle; + ret = parse_chan_layout_element(inner_it.handle, + &nchans, NULL); + if (ret) + return ret; + + if (nchans != selected_nchans) + continue; + + /* + * Now that we know this layout matches the + * selected channel number, reread the element + * and fill in the channel map. + */ + *inner_it.handle = save_handle; + ret = parse_chan_layout_element(inner_it.handle, + NULL, chmap); + if (ret) + return ret; + } + } else if (consume_string(it.handle, "ElementData")) { + const u8 *blob; + + ret = parse_blob(it.handle, sizeof(*cookie), &blob); + if (ret) + return ret; + + if (cookie) + memcpy(cookie, blob, sizeof(*cookie)); + } else { + ret = skip_pair(it.handle); + if (ret) + return ret; + } + } + + return 0; +} + +int parse_sound_constraints(struct dcp_parse_ctx *handle, + struct dcp_sound_format_mask *sieve, + struct dcp_sound_format_mask *hits) +{ + int ret; + struct iterator it; + + if (hits) { + hits->rates = 0; + hits->formats = 0; + hits->nchans = 0; + } + + dcp_parse_foreach_in_array(handle, it) { + ret = parse_avep_element(it.handle, sieve, hits); + + if (ret < 0) + return ret; + } + + return 0; +} + +int parse_sound_mode(struct dcp_parse_ctx *handle, + struct dcp_sound_format_mask *sieve, + struct snd_pcm_chmap_elem *chmap, + struct dcp_sound_cookie *cookie) +{ + struct dcp_parse_ctx save_handle; + struct iterator it; + int ret; + + dcp_parse_foreach_in_array(handle, it) { + save_handle = *it.handle; + ret = parse_avep_element(it.handle, sieve, NULL); + + if (!ret) + continue; + + if (ret < 0) + return ret; + + ret = parse_mode_in_avep_element(&save_handle, __ffs(sieve->nchans), + chmap, cookie); + if (ret < 0) + return ret; + return 1; + } + + return 0; +} +#endif + +int parse_system_log_mnits(struct dcp_parse_ctx *handle, struct dcp_system_ev_mnits *entry) +{ + struct iterator it; + int ret; + s64 mnits = -1; + s64 idac = -1; + s64 timestamp = -1; + bool type_match = false; + + dcp_parse_foreach_in_dict(handle, it) { + char *key = parse_string(it.handle); + if (IS_ERR(key)) { + ret = PTR_ERR(key); + } else if (!strcmp(key, "mNits")) { + ret = parse_int(it.handle, &mnits); + } else if (!strcmp(key, "iDAC")) { + ret = parse_int(it.handle, &idac); + } else if (!strcmp(key, "logEvent")) { + const char * value = parse_string(it.handle); + if (!IS_ERR_OR_NULL(value)) { + type_match = strcmp(value, "Display (Event Forward)") == 0; + kfree(value); + } + } else if (!strcmp(key, "timestamp")) { + ret = parse_int(it.handle, ×tamp); + } else { + skip(it.handle); + } + + if (!IS_ERR_OR_NULL(key)) + kfree(key); + + if (ret) { + pr_err("dcp parser: failed to parse mNits sys event\n"); + return ret; + } + } + + if (!type_match || mnits < 0 || idac < 0 || timestamp < 0) + return -EINVAL; + + entry->millinits = mnits; + entry->idac = idac; + entry->timestamp = timestamp; + + return 0; +} diff --git a/drivers/gpu/drm/apple/parser.h b/drivers/gpu/drm/apple/parser.h new file mode 100644 index 00000000000000..a7af17bfe35d36 --- /dev/null +++ b/drivers/gpu/drm/apple/parser.h @@ -0,0 +1,141 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright 2021 Alyssa Rosenzweig */ + +#ifndef __APPLE_DCP_PARSER_H__ +#define __APPLE_DCP_PARSER_H__ + +/* For mode parsing */ +#include + +struct apple_dcp; + +struct dcp_parse_ctx { + struct apple_dcp *dcp; + const void *blob; + u32 pos, len; +}; + +enum dcp_color_eotf { + DCP_EOTF_SDR_GAMMA = 0, // "SDR gamma" + DCP_EOTF_HDR_GAMMA = 1, // "HDR gamma" + DCP_EOTF_ST_2084 = 2, // "ST 2084 (PQ)" + DCP_EOTF_BT_2100 = 3, // "BT.2100 (HLG)" + DCP_EOTF_COUNT +}; + +enum dcp_color_format { + DCP_COLOR_FORMAT_RGB = 0, // "RGB" + DCP_COLOR_FORMAT_YCBCR420 = 1, // "YUV 4:2:0" + DCP_COLOR_FORMAT_YCBCR422 = 3, // "YUV 4:2:2" + DCP_COLOR_FORMAT_YCBCR444 = 2, // "YUV 4:4:4" + DCP_COLOR_FORMAT_DV_NATIVE = 4, // "DolbyVision (native)" + DCP_COLOR_FORMAT_DV_HDMI = 5, // "DolbyVision (HDMI)" + DCP_COLOR_FORMAT_YCBCR422_DP = 6, // "YCbCr 4:2:2 (DP tunnel)" + DCP_COLOR_FORMAT_YCBCR422_HDMI = 7, // "YCbCr 4:2:2 (HDMI tunnel)" + DCP_COLOR_FORMAT_DV_LL_YCBCR422 = 8, // "DolbyVision LL YCbCr 4:2:2" + DCP_COLOR_FORMAT_DV_LL_YCBCR422_DP = 9, // "DolbyVision LL YCbCr 4:2:2 (DP)" + DCP_COLOR_FORMAT_DV_LL_YCBCR422_HDMI = 10, // "DolbyVision LL YCbCr 4:2:2 (HDMI)" + DCP_COLOR_FORMAT_DV_LL_YCBCR444 = 11, // "DolbyVision LL YCbCr 4:4:4" + DCP_COLOR_FORMAT_DV_LL_RGB422 = 12, // "DolbyVision LL RGB 4:2:2" + DCP_COLOR_FORMAT_GRGB_BLUE_422 = 13, // "GRGB as YCbCr422 (Even line blue)" + DCP_COLOR_FORMAT_GRGB_RED_422 = 14, // "GRGB as YCbCr422 (Even line red)" + DCP_COLOR_FORMAT_COUNT +}; + +enum dcp_colorimetry { + DCP_COLORIMETRY_BT601 = 0, // "SMPTE 170M/BT.601" + DCP_COLORIMETRY_BT709 = 1, // "BT.701" + DCP_COLORIMETRY_XVYCC_601 = 2, // "xvYCC601" + DCP_COLORIMETRY_XVYCC_709 = 3, // "xvYCC709" + DCP_COLORIMETRY_SYCC_601 = 4, // "sYCC601" + DCP_COLORIMETRY_ADOBE_YCC_601 = 5, // "AdobeYCC601" + DCP_COLORIMETRY_BT2020_CYCC = 6, // "BT.2020 (c)" + DCP_COLORIMETRY_BT2020_YCC = 7, // "BT.2020 (nc)" + DCP_COLORIMETRY_VSVDB = 8, // "DolbyVision VSVDB" + DCP_COLORIMETRY_BT2020_RGB = 9, // "BT.2020 (RGB)" + DCP_COLORIMETRY_SRGB = 10, // "sRGB" + DCP_COLORIMETRY_SCRGB = 11, // "scRGB" + DCP_COLORIMETRY_SCRGB_FIXED = 12, // "scRGBfixed" + DCP_COLORIMETRY_ADOBE_RGB = 13, // "AdobeRGB" + DCP_COLORIMETRY_DCI_P3_RGB_D65 = 14, // "DCI-P3 (D65)" + DCP_COLORIMETRY_DCI_P3_RGB_THEATER = 15, // "DCI-P3 (Theater)" + DCP_COLORIMETRY_RGB = 16, // "Default RGB" + DCP_COLORIMETRY_COUNT +}; + +enum dcp_color_range { + DCP_COLOR_YCBCR_RANGE_FULL = 0, + DCP_COLOR_YCBCR_RANGE_LIMITED = 1, + DCP_COLOR_YCBCR_RANGE_COUNT +}; + +struct dcp_color_mode { + s64 score; + u32 id; + enum dcp_color_eotf eotf; + enum dcp_color_format format; + enum dcp_colorimetry colorimetry; + enum dcp_color_range range; + u8 depth; +}; + +/* + * Represents a single display mode. These mode objects are populated at + * runtime based on the TimingElements dictionary sent by the DCP. + */ +struct dcp_display_mode { + struct drm_display_mode mode; + u32 color_mode_id; + u32 timing_mode_id; + struct dcp_color_mode sdr_rgb; + struct dcp_color_mode sdr_444; + struct dcp_color_mode sdr; + struct dcp_color_mode best; + bool vrr; + u32 min_vrr; + u32 max_vrr; +}; + +struct dimension { + s64 total, front_porch, sync_width, active; + s64 precise_sync_rate; +}; + +int parse(const void *blob, size_t size, struct dcp_parse_ctx *ctx); +struct dcp_display_mode *enumerate_modes(struct dcp_parse_ctx *handle, + unsigned int *count, int width_mm, + int height_mm, unsigned notch_height); +int parse_display_attributes(struct dcp_parse_ctx *handle, int *width_mm, + int *height_mm); +int parse_epic_service_init(struct dcp_parse_ctx *handle, const char **name, + const char **class, s64 *unit); + +struct dcp_sound_format_mask { + u64 formats; /* SNDRV_PCM_FMTBIT_* */ + unsigned int rates; /* SNDRV_PCM_RATE_* */ + unsigned int nchans; +}; + +struct dcp_sound_cookie { + u8 data[24]; +}; + +struct snd_pcm_chmap_elem; +int parse_sound_constraints(struct dcp_parse_ctx *handle, + struct dcp_sound_format_mask *sieve, + struct dcp_sound_format_mask *hits); +int parse_sound_mode(struct dcp_parse_ctx *handle, + struct dcp_sound_format_mask *sieve, + struct snd_pcm_chmap_elem *chmap, + struct dcp_sound_cookie *cookie); + +struct dcp_system_ev_mnits { + u32 timestamp; + u32 millinits; + u32 idac; +}; + +int parse_system_log_mnits(struct dcp_parse_ctx *handle, + struct dcp_system_ev_mnits *entry); + +#endif diff --git a/drivers/gpu/drm/apple/plane.c b/drivers/gpu/drm/apple/plane.c new file mode 100644 index 00000000000000..2f0b76ad84ad65 --- /dev/null +++ b/drivers/gpu/drm/apple/plane.c @@ -0,0 +1,491 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Copyright (C) The Asahi Linux Contributors + */ + +#include "plane.h" + +#include "iomfb_internal.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#define FRAC_16_16(mult, div) (((mult) << 16) / (div)) + +static int apple_plane_atomic_check(struct drm_plane *plane, + struct drm_atomic_state *state) +{ + struct drm_plane_state *new_plane_state; + struct drm_crtc_state *crtc_state; + struct drm_rect *dst; + int ret; + + new_plane_state = drm_atomic_get_new_plane_state(state, plane); + + if (!new_plane_state->crtc) + return 0; + + crtc_state = drm_atomic_get_crtc_state(state, new_plane_state->crtc); + if (IS_ERR(crtc_state)) + return PTR_ERR(crtc_state); + + /* + * DCP limits downscaling to 2x and upscaling to 4x. Attempting to + * scale outside these bounds errors out when swapping. + * + * This function also takes care of clipping the src/dest rectangles, + * which is required for correct operation. Partially off-screen + * surfaces may appear corrupted. + * + * DCP does not distinguish plane types in the hardware, so we set + * can_position. If the primary plane does not fill the screen, the + * hardware will fill in zeroes (black). + */ + ret = drm_atomic_helper_check_plane_state(new_plane_state, crtc_state, + FRAC_16_16(1, 2), + FRAC_16_16(4, 1), + true, true); + if (ret < 0) + return ret; + + if (!new_plane_state->visible) + return 0; + + /* + * DCP does not allow a surface to clip off the screen, and will crash + * if any blended surface is smaller than 32x32. Reject the atomic op + * if the plane will crash DCP. + * + * This is most pertinent to cursors. Userspace should fall back to + * software cursors if the plane check is rejected. + */ + dst = &new_plane_state->dst; + if (drm_rect_width(dst) < 32 || drm_rect_height(dst) < 32) { + dev_err_once(state->dev->dev, + "Plane operation would have crashed DCP! Rejected!\n\ + DCP requires 32x32 of every plane to be within screen space.\n\ + Your compositor asked to overlay [%dx%d, %dx%d] on %dx%d.\n\ + This is not supported, and your compositor should have\n\ + switched to software compositing when this operation failed.\n\ + You should not have noticed this at all. If your screen\n\ + froze/hitched, or your compositor crashed, please report\n\ + this to the your compositor's developers. We will not\n\ + throw this error again until you next reboot.\n", + dst->x1, dst->y1, dst->x2, dst->y2, + crtc_state->mode.hdisplay, crtc_state->mode.vdisplay); + return -EINVAL; + } + + /* + * Pitches have to be 64-byte aligned. + */ + for (u32 i = 0; i < new_plane_state->fb->format->num_planes; i++) + if (new_plane_state->fb->pitches[i] & 63) + return -EINVAL; + + /* + * FIXME: dcp can currently only use multi-planar buffers using the same + * object for all planes. It has a mandatory iommu so it should + * be no problem to map multiple objects "linearly" into DCP + * virtual address space and calculate the offsets accordingly. + * Or maybe it can accept multiple BOs via the per plane field + * `base`. + */ + if (new_plane_state->fb->format->num_planes > 1) { + const struct drm_gem_object *first = new_plane_state->fb->obj[0]; + for (u32 i = 1; i < new_plane_state->fb->format->num_planes; i++) + if (new_plane_state->fb->obj[i] != NULL && + new_plane_state->fb->obj[i] != first) + return -EINVAL; + } + + return 0; +} + +/* + * DRM specifies rectangles as start and end coordinates. DCP specifies + * rectangles as a start coordinate and a width/height. Convert a DRM rectangle + * to a DCP rectangle. + */ +static struct dcp_rect drm_to_dcp_rect(const struct drm_rect *rect) +{ + return (struct dcp_rect){ .x = rect->x1, + .y = rect->y1, + .w = drm_rect_width(rect), + .h = drm_rect_height(rect), + }; +} + +static struct dcp_rect drm_to_dcp_rect_fp(const struct drm_rect *fp_rect) +{ + struct drm_rect rect; + drm_rect_fp_to_int(&rect, fp_rect); + return drm_to_dcp_rect(&rect); +} + +static u32 drm_format_to_dcp(u32 drm, enum drm_color_range range) +{ + bool fr = range == DRM_COLOR_YCBCR_FULL_RANGE; + switch (drm) { + case DRM_FORMAT_XRGB8888: + case DRM_FORMAT_ARGB8888: + return DCP_FORMAT_BGRA; + + case DRM_FORMAT_XBGR8888: + case DRM_FORMAT_ABGR8888: + return DCP_FORMAT_RGBA; + + case DRM_FORMAT_XRGB2101010: + case DRM_FORMAT_ARGB2101010: + return DCP_FORMAT_L10R; + + /* semi planar YCbCr formats, limited and full range */ + case DRM_FORMAT_NV12: + return fr ? DCP_FORMAT_420F : DCP_FORMAT_420V; + case DRM_FORMAT_NV16: + return fr ? DCP_FORMAT_422F : DCP_FORMAT_422V; + case DRM_FORMAT_NV24: + return fr ? DCP_FORMAT_444F : DCP_FORMAT_444V; + + /* semi planar 10-bit YCbCr formats, limited and full range */ + case DRM_FORMAT_P010: + return fr ? DCP_FORMAT_XF20 : DCP_FORMAT_X420; + case DRM_FORMAT_P210: + return fr ? DCP_FORMAT_XF22 : DCP_FORMAT_X422; + /* + * TODO: missing DRM fourcc for P410 + */ +#if defined(DRM_FORMAT_P410) + case DRM_FORMAT_P410: + return fr ? DCP_FORMAT_XF44 : DCP_FORMAT_X444; +#endif + } + + pr_warn("DRM format %X not supported in DCP\n", drm); + return 0; +} + +static enum dcp_xfer_func get_xfer_func(bool is_yuv, enum drm_color_encoding enc) +{ + if (!is_yuv) + return DCP_XFER_FUNC_SDR; + + switch (enc) { + case DRM_COLOR_YCBCR_BT601: + return DCP_XFER_FUNC_BT601; + case DRM_COLOR_YCBCR_BT709: + case DRM_COLOR_YCBCR_BT2020: + return DCP_XFER_FUNC_BT1886; + default: + return DCP_XFER_FUNC_SDR; + } +} + +static enum dcp_colorspace get_colorspace(bool is_yuv, + enum drm_color_encoding enc) +{ + if (!is_yuv) + return DCP_COLORSPACE_NATIVE; + + switch (enc) { + case DRM_COLOR_YCBCR_BT601: + return DCP_COLORSPACE_BT601; + case DRM_COLOR_YCBCR_BT709: + return DCP_COLORSPACE_BT709; + case DRM_COLOR_YCBCR_BT2020: + return DCP_COLORSPACE_BG_BT2020; + default: + return DCP_COLORSPACE_NATIVE; + } +} + +static void apple_plane_atomic_update(struct drm_plane *plane, + struct drm_atomic_state *state) +{ + struct drm_plane_state *base = drm_atomic_get_new_plane_state(state, plane); + struct apple_plane_state *new_state; + struct drm_gem_dma_object *obj; + bool is_premultiplied = false; + + if (!base) + return; + + new_state = to_apple_plane_state(base); + + if (!base->fb) { + memset(&new_state->surf, 0, sizeof(new_state->surf)); + return; + } + + struct drm_framebuffer *fb = base->fb; + const struct drm_format_info *fmt = fb->format; + /* + * DCP doesn't support XBGR8 / XRGB8 / XBGR2101010 natively. Blending as + * pre-multiplied alpha with a black background can be used as + * workaround for the bottommost plane. + */ + if (fmt->format == DRM_FORMAT_XRGB8888 || + fmt->format == DRM_FORMAT_XBGR8888 || + fmt->format == DRM_FORMAT_XBGR2101010) + is_premultiplied = true; + + new_state->src_rect = drm_to_dcp_rect_fp(&base->src); + new_state->dst_rect = drm_to_dcp_rect(&base->dst); + + new_state->surf = (struct dcp_surface){ + .is_premultiplied = is_premultiplied, + .plane_cnt = fb->format->num_planes, + .plane_cnt2 = fb->format->num_planes, + .format = drm_format_to_dcp(fmt->format, base->color_range), + .xfer_func = get_xfer_func(fmt->is_yuv, base->color_encoding), + .colorspace = get_colorspace(fmt->is_yuv, base->color_encoding), + .stride = fb->pitches[0], + .width = fb->width, + .height = fb->height, + .buf_size = fb->height * fb->pitches[0], + // .surface_id = req->swap.surf_ids[l], + + /* Only used for compressed or multiplanar surfaces */ + .pix_size = 1, + .pel_w = 1, + .pel_h = 1, + .has_comp = 1, + .has_planes = 1, + }; + + /* Populate plane information for planar formats */ + struct dcp_surface *surf = &new_state->surf; + for (int i = 0; fb->format->num_planes && i < fb->format->num_planes; i++) { + u32 width = drm_format_info_plane_width(fb->format, fb->width, i); + u32 height = drm_format_info_plane_height(fb->format, fb->height, i); + u32 bh = drm_format_info_block_height(fb->format, i); + u32 bw = drm_format_info_block_width(fb->format, i); + + surf->planes[i] = (struct dcp_plane_info){ + .width = width, + .height = height, + .base = fb->offsets[i] - fb->offsets[0], + .offset = fb->offsets[i] - fb->offsets[0], + .stride = fb->pitches[i], + .size = height * fb->pitches[i], + .tile_size = bw * bh, + .tile_w = bw, + .tile_h = bh, + }; + + if (i > 0) + surf->buf_size += surf->planes[i].size; + } + + /* the obvious helper call drm_fb_dma_get_gem_addr() adjusts + * the address for source x/y offsets. Since IOMFB has a direct + * support source position prefer that. + */ + obj = drm_fb_dma_get_gem_obj(base->fb, 0); + if (obj) + new_state->iova = obj->dma_addr + base->fb->offsets[0]; +} + +static const struct drm_plane_helper_funcs apple_primary_plane_helper_funcs = { + .atomic_check = apple_plane_atomic_check, + .atomic_update = apple_plane_atomic_update, + .get_scanout_buffer = drm_fb_dma_get_scanout_buffer, +}; + +static const struct drm_plane_helper_funcs apple_plane_helper_funcs = { + .atomic_check = apple_plane_atomic_check, + .atomic_update = apple_plane_atomic_update, +}; + +// Duplicate drm_atomic_helper_plane_reset but allocate struct apple_plane_state +static void apple_plane_reset(struct drm_plane *plane) +{ + struct apple_plane_state *state = to_apple_plane_state(plane->state); + if (state) + __drm_atomic_helper_plane_destroy_state(&state->base); + + kfree(state); + plane->state = NULL; + state = kzalloc(sizeof(*state), GFP_KERNEL); + if (state) + __drm_atomic_helper_plane_reset(plane, &state->base); +} + +static struct drm_plane_state * +apple_plane_duplicate_state(struct drm_plane *plane) +{ + struct apple_plane_state *apple_plane_state, *old_apple_plane_state; + + if (!plane->state) + return NULL; + + old_apple_plane_state = to_apple_plane_state(plane->state); + apple_plane_state = kzalloc(sizeof(*apple_plane_state), GFP_KERNEL); + if (!apple_plane_state) + return NULL; + + __drm_atomic_helper_plane_duplicate_state(plane, &apple_plane_state->base); + + apple_plane_state->surf = old_apple_plane_state->surf; + + return &apple_plane_state->base; +} + +// void apple_plane_destroy_state(struct drm_plane *plane, +// struct drm_plane_state *state) +// { +// drm_atomic_helper_plane_destroy_state(plane, state); +// } + +static const struct drm_plane_funcs apple_plane_funcs = { + .update_plane = drm_atomic_helper_update_plane, + .disable_plane = drm_atomic_helper_disable_plane, + .reset = apple_plane_reset, + .atomic_duplicate_state = apple_plane_duplicate_state, + // .atomic_destroy_state = apple_plane_destroy_state, + .atomic_destroy_state = drm_atomic_helper_plane_destroy_state, +}; + +/* + * Table of supported formats, mapping from DRM fourccs to DCP fourccs. + * + * For future work, DCP supports more formats not listed, including YUV + * formats, an extra RGBA format, and a biplanar RGB10_A8 format (fourcc b3a8) + * used for HDR. + * + * Note: we don't have non-alpha formats but userspace breaks without XRGB. It + * doesn't matter for the primary plane, but cursors/overlays must not + * advertise formats without alpha. + */ +static const u32 dcp_primary_formats[] = { + DRM_FORMAT_XRGB2101010, + DRM_FORMAT_ARGB2101010, + DRM_FORMAT_XRGB8888, + DRM_FORMAT_ARGB8888, + DRM_FORMAT_XBGR8888, + DRM_FORMAT_ABGR8888, + DRM_FORMAT_NV12, + DRM_FORMAT_NV16, + DRM_FORMAT_NV24, + DRM_FORMAT_P010, + DRM_FORMAT_P210, +#if defined(DRM_FORMAT_P410) + DRM_FORMAT_P410, +#endif +}; + +static const u32 dcp_overlay_formats[] = { + DRM_FORMAT_ARGB2101010, + DRM_FORMAT_ARGB8888, + DRM_FORMAT_ABGR8888, + DRM_FORMAT_NV12, + DRM_FORMAT_NV16, + DRM_FORMAT_NV24, + DRM_FORMAT_P010, + DRM_FORMAT_P210, +#if defined(DRM_FORMAT_P410) + DRM_FORMAT_P410, +#endif +}; + +/* + * Formats for the 12.x firmware which does not support "l10r" / ARGB2101010 + */ +static const u32 dcp_primary_formats_12_x[] = { + DRM_FORMAT_XRGB8888, + DRM_FORMAT_ARGB8888, + DRM_FORMAT_XBGR8888, + DRM_FORMAT_ABGR8888, + DRM_FORMAT_NV12, + DRM_FORMAT_NV16, + DRM_FORMAT_NV24, + DRM_FORMAT_P010, + DRM_FORMAT_P210, +#if defined(DRM_FORMAT_P410) + DRM_FORMAT_P410, +#endif +}; + +static const u32 dcp_overlay_formats_12_x[] = { + DRM_FORMAT_ARGB8888, + DRM_FORMAT_ABGR8888, + DRM_FORMAT_NV12, + DRM_FORMAT_NV16, + DRM_FORMAT_NV24, + DRM_FORMAT_P010, + DRM_FORMAT_P210, +#if defined(DRM_FORMAT_P410) + DRM_FORMAT_P410, +#endif +}; + +u64 apple_format_modifiers[] = { + DRM_FORMAT_MOD_LINEAR, + DRM_FORMAT_MOD_INVALID +}; + +struct apple_plane { + struct drm_plane base; +}; + +struct drm_plane *apple_plane_init(struct drm_device *dev, + unsigned long possible_crtcs, + bool supports_l10r, + enum drm_plane_type type) +{ + struct apple_plane *plane; + const u32 *fmts; + u32 num_fmts; + + switch (type) { + case DRM_PLANE_TYPE_PRIMARY: + if (supports_l10r) { + fmts = dcp_primary_formats; + num_fmts = ARRAY_SIZE(dcp_primary_formats); + } else { + fmts = dcp_primary_formats_12_x; + num_fmts = ARRAY_SIZE(dcp_primary_formats_12_x); + } + plane = drmm_universal_plane_alloc(dev, struct apple_plane, base, possible_crtcs, + &apple_plane_funcs, fmts, num_fmts, + apple_format_modifiers, type, NULL); + break; + case DRM_PLANE_TYPE_OVERLAY: + case DRM_PLANE_TYPE_CURSOR: + if (supports_l10r) { + fmts = dcp_overlay_formats; + num_fmts = ARRAY_SIZE(dcp_overlay_formats); + } else { + fmts = dcp_overlay_formats_12_x; + num_fmts = ARRAY_SIZE(dcp_overlay_formats_12_x); + } + plane = drmm_universal_plane_alloc(dev, struct apple_plane, base, possible_crtcs, + &apple_plane_funcs, fmts, num_fmts, + apple_format_modifiers, type, NULL); + break; + default: + return ERR_PTR(-EINVAL); + } + + if (IS_ERR(plane)) + return ERR_PTR(PTR_ERR(plane)); + + drm_plane_create_color_properties(&plane->base, + (1 << DRM_COLOR_ENCODING_MAX) - 1, + (1 << DRM_COLOR_RANGE_MAX) - 1, + DRM_COLOR_YCBCR_BT709, + DRM_COLOR_YCBCR_LIMITED_RANGE); + + if (type == DRM_PLANE_TYPE_PRIMARY) + drm_plane_helper_add(&plane->base, &apple_primary_plane_helper_funcs); + else + drm_plane_helper_add(&plane->base, &apple_plane_helper_funcs); + + return &plane->base; +} diff --git a/drivers/gpu/drm/apple/plane.h b/drivers/gpu/drm/apple/plane.h new file mode 100644 index 00000000000000..67d15938cf0dcb --- /dev/null +++ b/drivers/gpu/drm/apple/plane.h @@ -0,0 +1,30 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Copyright (C) The Asahi Linux Contributors + */ + +#ifndef __APPLE_PLANE_H__ +#define __APPLE_PLANE_H__ + +#include + +#include + +#include "iomfb_plane.h" + +struct apple_plane_state { + struct drm_plane_state base; + struct dcp_surface surf; + struct dcp_rect src_rect; + struct dcp_rect dst_rect; + u64 iova; +}; + +#define to_apple_plane_state(x) container_of(x, struct apple_plane_state, base) + +struct drm_plane *apple_plane_init(struct drm_device *dev, + unsigned long possible_crtcs, + bool supports_l10r, + enum drm_plane_type type); + +#endif /* __APPLE_PLANE_H__ */ diff --git a/drivers/gpu/drm/apple/systemep.c b/drivers/gpu/drm/apple/systemep.c new file mode 100644 index 00000000000000..9fe7a0ce495aab --- /dev/null +++ b/drivers/gpu/drm/apple/systemep.c @@ -0,0 +1,137 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright 2022 Sven Peter */ + +#include + +#include "afk.h" +#include "dcp.h" +#include "parser.h" + +static bool enable_verbose_logging; +module_param(enable_verbose_logging, bool, 0644); +MODULE_PARM_DESC(enable_verbose_logging, "Enable DCP firmware verbose logging"); + +/* + * Serialized setProperty("gAFKConfigLogMask", 0xffff) IPC call which + * will set the DCP firmware log level to the most verbose setting + */ +#define SYSTEM_SET_PROPERTY 0x43 +static const u8 setprop_gAFKConfigLogMask_ffff[] = { + 0x14, 0x00, 0x00, 0x00, 0x67, 0x41, 0x46, 0x4b, 0x43, 0x6f, + 0x6e, 0x66, 0x69, 0x67, 0x4c, 0x6f, 0x67, 0x4d, 0x61, 0x73, + 0x6b, 0x00, 0x00, 0x00, 0xd3, 0x00, 0x00, 0x00, 0x40, 0x00, + 0x00, 0x84, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +}; + +struct systemep_work { + struct apple_epic_service *service; + struct work_struct work; +}; + +static void system_log_work(struct work_struct *work_) +{ + struct systemep_work *work = + container_of(work_, struct systemep_work, work); + + afk_send_command(work->service, SYSTEM_SET_PROPERTY, + setprop_gAFKConfigLogMask_ffff, + sizeof(setprop_gAFKConfigLogMask_ffff), NULL, + sizeof(setprop_gAFKConfigLogMask_ffff), NULL); + complete(&work->service->ep->dcp->systemep_done); + kfree(work); +} + +static void system_init(struct apple_epic_service *service, const char *name, + const char *class, s64 unit) +{ + struct systemep_work *work; + + if (!enable_verbose_logging) + return; + + /* + * We're called from the service message handler thread and can't + * dispatch blocking message from there. + */ + work = kzalloc(sizeof(*work), GFP_KERNEL); + if (!work) + return; + + work->service = service; + INIT_WORK(&work->work, system_log_work); + schedule_work(&work->work); +} + +static void powerlog_init(struct apple_epic_service *service, const char *name, + const char *class, s64 unit) +{ +} + +static int powerlog_report(struct apple_epic_service *service, enum epic_subtype type, + const void *data, size_t data_size) +{ + struct dcp_system_ev_mnits mnits; + struct dcp_parse_ctx parse_ctx; + struct apple_dcp *dcp = service->ep->dcp; + int ret; + + dev_dbg(dcp->dev, "systemep[ch:%u]: report type:%02x len:%zu\n", + service->channel, type, data_size); + + if (type != EPIC_SUBTYPE_STD_SERVICE) + return 0; + + ret = parse(data, data_size, &parse_ctx); + if (ret) { + dev_warn(service->ep->dcp->dev, "systemep: failed to parse report: %d\n", ret); + return ret; + } + + ret = parse_system_log_mnits(&parse_ctx, &mnits); + if (ret) { + /* ignore parse errors in the case dcp sends unknown log events */ + dev_dbg(dcp->dev, "systemep: failed to parse mNits event: %d\n", ret); + return 0; + } + + dev_dbg(dcp->dev, "systemep: mNits event: Nits: %u.%03u, iDAC: %u\n", + mnits.millinits / 1000, mnits.millinits % 1000, mnits.idac); + + dcp->brightness.nits = mnits.millinits / 1000; + + return 0; +} + +static const struct apple_epic_service_ops systemep_ops[] = { + { + .name = "system", + .init = system_init, + }, + { + .name = "powerlog-service", + .init = powerlog_init, + .report = powerlog_report, + }, + {} +}; + +int systemep_init(struct apple_dcp *dcp) +{ + init_completion(&dcp->systemep_done); + + dcp->systemep = afk_init(dcp, SYSTEM_ENDPOINT, systemep_ops); + afk_start(dcp->systemep); + + if (!enable_verbose_logging) + return 0; + + /* + * Timeouts aren't really fatal here: in the worst case we just weren't + * able to enable additional debug prints inside DCP + */ + if (!wait_for_completion_timeout(&dcp->systemep_done, + msecs_to_jiffies(MSEC_PER_SEC))) + dev_err(dcp->dev, "systemep: couldn't enable verbose logs\n"); + + return 0; +} diff --git a/drivers/gpu/drm/apple/trace.c b/drivers/gpu/drm/apple/trace.c new file mode 100644 index 00000000000000..6f40d5a583df01 --- /dev/null +++ b/drivers/gpu/drm/apple/trace.c @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * Tracepoints for Apple DCP driver + * + * Copyright (C) The Asahi Linux Contributors + */ + +#define CREATE_TRACE_POINTS +#include "trace.h" diff --git a/drivers/gpu/drm/apple/trace.h b/drivers/gpu/drm/apple/trace.h new file mode 100644 index 00000000000000..a13dd34fb7aab1 --- /dev/null +++ b/drivers/gpu/drm/apple/trace.h @@ -0,0 +1,608 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright (C) The Asahi Linux Contributors */ + +#undef TRACE_SYSTEM +#define TRACE_SYSTEM dcp + +#if !defined(_TRACE_DCP_H) || defined(TRACE_HEADER_MULTI_READ) +#define _TRACE_DCP_H + +#include "afk.h" +#include "dptxep.h" +#include "dcp-internal.h" +#include "parser.h" + +#include +#include +#include + +#define show_dcp_endpoint(ep) \ + __print_symbolic(ep, { SYSTEM_ENDPOINT, "system" }, \ + { TEST_ENDPOINT, "test" }, \ + { DCP_EXPERT_ENDPOINT, "dcpexpert" }, \ + { DISP0_ENDPOINT, "disp0" }, \ + { DPTX_ENDPOINT, "dptxport" }, \ + { HDCP_ENDPOINT, "hdcp" }, \ + { REMOTE_ALLOC_ENDPOINT, "remotealloc" }, \ + { IOMFB_ENDPOINT, "iomfb" }) +#define print_epic_type(etype) \ + __print_symbolic(etype, { EPIC_TYPE_NOTIFY, "notify" }, \ + { EPIC_TYPE_COMMAND, "command" }, \ + { EPIC_TYPE_REPLY, "reply" }, \ + { EPIC_TYPE_NOTIFY_ACK, "notify-ack" }) + +#define print_epic_category(ecat) \ + __print_symbolic(ecat, { EPIC_CAT_REPORT, "report" }, \ + { EPIC_CAT_NOTIFY, "notify" }, \ + { EPIC_CAT_REPLY, "reply" }, \ + { EPIC_CAT_COMMAND, "command" }) + +#define show_dptxport_apcall(idx) \ + __print_symbolic( \ + idx, { DPTX_APCALL_ACTIVATE, "activate" }, \ + { DPTX_APCALL_DEACTIVATE, "deactivate" }, \ + { DPTX_APCALL_GET_MAX_DRIVE_SETTINGS, \ + "get_max_drive_settings" }, \ + { DPTX_APCALL_SET_DRIVE_SETTINGS, "set_drive_settings" }, \ + { DPTX_APCALL_GET_DRIVE_SETTINGS, "get_drive_settings" }, \ + { DPTX_APCALL_WILL_CHANGE_LINKG_CONFIG, \ + "will_change_link_config" }, \ + { DPTX_APCALL_DID_CHANGE_LINK_CONFIG, \ + "did_change_link_config" }, \ + { DPTX_APCALL_GET_MAX_LINK_RATE, "get_max_link_rate" }, \ + { DPTX_APCALL_GET_LINK_RATE, "get_link_rate" }, \ + { DPTX_APCALL_SET_LINK_RATE, "set_link_rate" }, \ + { DPTX_APCALL_GET_MAX_LANE_COUNT, \ + "get_max_lane_count" }, \ + { DPTX_APCALL_GET_ACTIVE_LANE_COUNT, \ + "get_active_lane_count" }, \ + { DPTX_APCALL_SET_ACTIVE_LANE_COUNT, \ + "set_active_lane_count" }, \ + { DPTX_APCALL_GET_SUPPORTS_DOWN_SPREAD, \ + "get_supports_downspread" }, \ + { DPTX_APCALL_GET_DOWN_SPREAD, "get_downspread" }, \ + { DPTX_APCALL_SET_DOWN_SPREAD, "set_downspread" }, \ + { DPTX_APCALL_GET_SUPPORTS_LANE_MAPPING, \ + "get_supports_lane_mapping" }, \ + { DPTX_APCALL_SET_LANE_MAP, "set_lane_map" }, \ + { DPTX_APCALL_GET_SUPPORTS_HPD, "get_supports_hpd" }, \ + { DPTX_APCALL_FORCE_HOTPLUG_DETECT, "force_hotplug_detect" }, \ + { DPTX_APCALL_INACTIVE_SINK_DETECTED, \ + "inactive_sink_detected" }, \ + { DPTX_APCALL_SET_TILED_DISPLAY_HINTS, \ + "set_tiled_display_hints" }, \ + { DPTX_APCALL_DEVICE_NOT_RESPONDING, \ + "device_not_responding" }, \ + { DPTX_APCALL_DEVICE_BUSY_TIMEOUT, "device_busy_timeout" }, \ + { DPTX_APCALL_DEVICE_NOT_STARTED, "device_not_started" }) + +TRACE_EVENT(dcp_recv_msg, + TP_PROTO(struct apple_dcp *dcp, u8 endpoint, u64 message), + TP_ARGS(dcp, endpoint, message), + + TP_STRUCT__entry(__string(devname, dev_name(dcp->dev)) + __field(u8, endpoint) + __field(u64, message)), + + TP_fast_assign(__assign_str(devname); + __entry->endpoint = endpoint; + __entry->message = message;), + + TP_printk("%s: endpoint 0x%x (%s): received message 0x%016llx", + __get_str(devname), __entry->endpoint, + show_dcp_endpoint(__entry->endpoint), __entry->message)); + +TRACE_EVENT(dcp_send_msg, + TP_PROTO(struct apple_dcp *dcp, u8 endpoint, u64 message), + TP_ARGS(dcp, endpoint, message), + + TP_STRUCT__entry(__string(devname, dev_name(dcp->dev)) + __field(u8, endpoint) + __field(u64, message)), + + TP_fast_assign(__assign_str(devname); + __entry->endpoint = endpoint; + __entry->message = message;), + + TP_printk("%s: endpoint 0x%x (%s): will send message 0x%016llx", + __get_str(devname), __entry->endpoint, + show_dcp_endpoint(__entry->endpoint), __entry->message)); + +TRACE_EVENT( + afk_getbuf, TP_PROTO(struct apple_dcp_afkep *ep, u16 size, u16 tag), + TP_ARGS(ep, size, tag), + + TP_STRUCT__entry(__string(devname, dev_name(ep->dcp->dev)) + __field(u8, endpoint) __field(u16, size) + __field(u16, tag)), + + TP_fast_assign(__assign_str(devname); + __entry->endpoint = ep->endpoint; __entry->size = size; + __entry->tag = tag;), + + TP_printk( + "%s: endpoint 0x%x (%s): get buffer with size 0x%x and tag 0x%x", + __get_str(devname), __entry->endpoint, + show_dcp_endpoint(__entry->endpoint), __entry->size, + __entry->tag)); + +DECLARE_EVENT_CLASS(afk_rwptr_template, + TP_PROTO(struct apple_dcp_afkep *ep, u32 rptr, u32 wptr), + TP_ARGS(ep, rptr, wptr), + + TP_STRUCT__entry(__string(devname, dev_name(ep->dcp->dev)) + __field(u8, endpoint) __field(u32, rptr) + __field(u32, wptr)), + + TP_fast_assign(__assign_str(devname); + __entry->endpoint = ep->endpoint; + __entry->rptr = rptr; __entry->wptr = wptr;), + + TP_printk("%s: endpoint 0x%x (%s): rptr 0x%x, wptr 0x%x", + __get_str(devname), __entry->endpoint, + show_dcp_endpoint(__entry->endpoint), __entry->rptr, + __entry->wptr)); + +DEFINE_EVENT(afk_rwptr_template, afk_recv_rwptr_pre, + TP_PROTO(struct apple_dcp_afkep *ep, u32 rptr, u32 wptr), + TP_ARGS(ep, rptr, wptr)); +DEFINE_EVENT(afk_rwptr_template, afk_recv_rwptr_post, + TP_PROTO(struct apple_dcp_afkep *ep, u32 rptr, u32 wptr), + TP_ARGS(ep, rptr, wptr)); +DEFINE_EVENT(afk_rwptr_template, afk_send_rwptr_pre, + TP_PROTO(struct apple_dcp_afkep *ep, u32 rptr, u32 wptr), + TP_ARGS(ep, rptr, wptr)); +DEFINE_EVENT(afk_rwptr_template, afk_send_rwptr_post, + TP_PROTO(struct apple_dcp_afkep *ep, u32 rptr, u32 wptr), + TP_ARGS(ep, rptr, wptr)); + +TRACE_EVENT( + afk_recv_qe, + TP_PROTO(struct apple_dcp_afkep *ep, u32 rptr, u32 magic, u32 size), + TP_ARGS(ep, rptr, magic, size), + + TP_STRUCT__entry(__string(devname, dev_name(ep->dcp->dev)) + __field(u8, endpoint) __field(u32, rptr) + __field(u32, magic) + __field(u32, size)), + + TP_fast_assign(__assign_str(devname); + __entry->endpoint = ep->endpoint; __entry->rptr = rptr; + __entry->magic = magic; __entry->size = size;), + + TP_printk("%s: endpoint 0x%x (%s): QE rptr 0x%x, magic 0x%x, size 0x%x", + __get_str(devname), __entry->endpoint, + show_dcp_endpoint(__entry->endpoint), __entry->rptr, + __entry->magic, __entry->size)); + +TRACE_EVENT( + afk_recv_handle, + TP_PROTO(struct apple_dcp_afkep *ep, u32 channel, u32 type, + u32 data_size, struct epic_hdr *ehdr, + struct epic_sub_hdr *eshdr), + TP_ARGS(ep, channel, type, data_size, ehdr, eshdr), + + TP_STRUCT__entry(__string(devname, dev_name(ep->dcp->dev)) __field( + u8, endpoint) __field(u32, channel) __field(u32, type) + __field(u32, data_size) __field(u8, category) + __field(u16, subtype) + __field(u16, tag)), + + TP_fast_assign(__assign_str(devname); + __entry->endpoint = ep->endpoint; + __entry->channel = channel; __entry->type = type; + __entry->data_size = data_size; + __entry->category = eshdr->category, + __entry->subtype = le16_to_cpu(eshdr->type), + __entry->tag = le16_to_cpu(eshdr->tag)), + + TP_printk( + "%s: endpoint 0x%x (%s): channel 0x%x, type 0x%x (%s), data_size 0x%x, category: 0x%x (%s), subtype: 0x%x, seq: 0x%x", + __get_str(devname), __entry->endpoint, + show_dcp_endpoint(__entry->endpoint), __entry->channel, + __entry->type, print_epic_type(__entry->type), + __entry->data_size, __entry->category, + print_epic_category(__entry->category), __entry->subtype, + __entry->tag)); + +TRACE_EVENT(iomfb_callback, + TP_PROTO(struct apple_dcp *dcp, int tag, const char *name), + TP_ARGS(dcp, tag, name), + + TP_STRUCT__entry( + __string(devname, dev_name(dcp->dev)) + __field(int, tag) + __field(const char *, name) + ), + + TP_fast_assign( + __assign_str(devname); + __entry->tag = tag; __entry->name = name; + ), + + TP_printk("%s: Callback D%03d %s", __get_str(devname), __entry->tag, + __entry->name)); + +TRACE_EVENT(iomfb_push, + TP_PROTO(struct apple_dcp *dcp, + const struct dcp_method_entry *method, int context, + int offset, int depth), + TP_ARGS(dcp, method, context, offset, depth), + + TP_STRUCT__entry( + __string(devname, dev_name(dcp->dev)) + __string(name, method->name) + __field(int, context) + __field(int, offset) + __field(int, depth)), + + TP_fast_assign( + __assign_str(devname); + __assign_str(name); + __entry->context = context; __entry->offset = offset; + __entry->depth = depth; + ), + + TP_printk("%s: Method %s: context %u, offset %u, depth %u", + __get_str(devname), __get_str(name), __entry->context, + __entry->offset, __entry->depth)); + +TRACE_EVENT(iomfb_swap_submit, + TP_PROTO(struct apple_dcp *dcp, u32 swap_id), + TP_ARGS(dcp, swap_id), + TP_STRUCT__entry( + __field(u64, dcp) + __field(u32, swap_id) + ), + TP_fast_assign( + __entry->dcp = (u64)dcp; + __entry->swap_id = swap_id; + ), + TP_printk("dcp=%llx, swap_id=%d", + __entry->dcp, + __entry->swap_id) +); + +TRACE_EVENT(iomfb_swap_complete, + TP_PROTO(struct apple_dcp *dcp, u32 swap_id), + TP_ARGS(dcp, swap_id), + TP_STRUCT__entry( + __field(u64, dcp) + __field(u32, swap_id) + ), + TP_fast_assign( + __entry->dcp = (u64)dcp; + __entry->swap_id = swap_id; + ), + TP_printk("dcp=%llx, swap_id=%d", + __entry->dcp, + __entry->swap_id + ) +); + +TRACE_EVENT(iomfb_swap_complete_intent_gated, + TP_PROTO(struct apple_dcp *dcp, u32 swap_id, u32 width, u32 height), + TP_ARGS(dcp, swap_id, width, height), + TP_STRUCT__entry( + __field(u64, dcp) + __field(u32, swap_id) + __field(u32, width) + __field(u32, height) + ), + TP_fast_assign( + __entry->dcp = (u64)dcp; + __entry->swap_id = swap_id; + __entry->height = height; + __entry->width = width; + ), + TP_printk("dcp=%llx, swap_id=%u %ux%u", + __entry->dcp, + __entry->swap_id, + __entry->width, + __entry->height + ) +); + +TRACE_EVENT(iomfb_abort_swap_ap_gated, + TP_PROTO(struct apple_dcp *dcp, u32 swap_id), + TP_ARGS(dcp, swap_id), + TP_STRUCT__entry( + __field(u64, dcp) + __field(u32, swap_id) + ), + TP_fast_assign( + __entry->dcp = (u64)dcp; + __entry->swap_id = swap_id; + ), + TP_printk("dcp=%llx, swap_id=%u", + __entry->dcp, + __entry->swap_id + ) +); + +DECLARE_EVENT_CLASS(iomfb_parse_mode_template, + TP_PROTO(s64 id, struct dimension *horiz, struct dimension *vert, s64 best_color_mode, bool is_virtual, s64 score), + TP_ARGS(id, horiz, vert, best_color_mode, is_virtual, score), + + TP_STRUCT__entry(__field(s64, id) + __field_struct(struct dimension, horiz) + __field_struct(struct dimension, vert) + __field(s64, best_color_mode) + __field(bool, is_virtual) + __field(s64, score)), + + TP_fast_assign(__entry->id = id; + __entry->horiz = *horiz; + __entry->vert = *vert; + __entry->best_color_mode = best_color_mode; + __entry->is_virtual = is_virtual; + __entry->score = score;), + + TP_printk("id: %lld, best_color_mode: %lld, resolution:%lldx%lld virtual: %d, score: %lld", + __entry->id, __entry->best_color_mode, + __entry->horiz.active, __entry->vert.active, + __entry->is_virtual, __entry->score)); + +DEFINE_EVENT(iomfb_parse_mode_template, iomfb_parse_mode_success, + TP_PROTO(s64 id, struct dimension *horiz, struct dimension *vert, s64 best_color_mode, bool is_virtual, s64 score), + TP_ARGS(id, horiz, vert, best_color_mode, is_virtual, score)); + +DEFINE_EVENT(iomfb_parse_mode_template, iomfb_parse_mode_fail, + TP_PROTO(s64 id, struct dimension *horiz, struct dimension *vert, s64 best_color_mode, bool is_virtual, s64 score), + TP_ARGS(id, horiz, vert, best_color_mode, is_virtual, score)); + +TRACE_EVENT(dcpavserv_init, TP_PROTO(struct apple_dcp *dcp, u64 unit), + TP_ARGS(dcp, unit), + + TP_STRUCT__entry(__string(devname, dev_name(dcp->dev)) + __field(u64, unit)), + + TP_fast_assign(__assign_str(devname); + __entry->unit = unit;), + + TP_printk("%s: dcpav-service unit %lld initialized", __get_str(devname), + __entry->unit)); + +TRACE_EVENT(dptxport_init, TP_PROTO(struct apple_dcp *dcp, u64 unit), + TP_ARGS(dcp, unit), + + TP_STRUCT__entry(__string(devname, dev_name(dcp->dev)) + __field(u64, unit)), + + TP_fast_assign(__assign_str(devname); + __entry->unit = unit;), + + TP_printk("%s: dptxport unit %lld initialized", __get_str(devname), + __entry->unit)); + +TRACE_EVENT( + dptxport_apcall, + TP_PROTO(struct dptx_port *dptx, int idx, size_t len), + TP_ARGS(dptx, idx, len), + + TP_STRUCT__entry(__string(devname, dev_name(dptx->service->ep->dcp->dev)) + __field(u32, unit) __field(int, idx) __field(size_t, len)), + + TP_fast_assign(__assign_str(devname); + __entry->unit = dptx->unit; __entry->idx = idx; __entry->len = len;), + + TP_printk("%s: dptx%d: AP Call %d (%s) with len %lu", __get_str(devname), + __entry->unit, + __entry->idx, show_dptxport_apcall(__entry->idx), __entry->len)); + +TRACE_EVENT( + dptxport_validate_connection, + TP_PROTO(struct dptx_port *dptx, u8 core, u8 atc, u8 die), + TP_ARGS(dptx, core, atc, die), + + TP_STRUCT__entry(__string(devname, dev_name(dptx->service->ep->dcp->dev)) + __field(u32, unit) __field(u8, core) __field(u8, atc) __field(u8, die)), + + TP_fast_assign(__assign_str(devname); + __entry->unit = dptx->unit; __entry->core = core; __entry->atc = atc; __entry->die = die;), + + TP_printk("%s: dptx%d: core %d, atc %d, die %d", __get_str(devname), + __entry->unit, __entry->core, __entry->atc, __entry->die)); + +TRACE_EVENT( + dptxport_connect, + TP_PROTO(struct dptx_port *dptx, u8 core, u8 atc, u8 die), + TP_ARGS(dptx, core, atc, die), + + TP_STRUCT__entry(__string(devname, dev_name(dptx->service->ep->dcp->dev)) + __field(u32, unit) __field(u8, core) __field(u8, atc) __field(u8, die)), + + TP_fast_assign(__assign_str(devname); + __entry->unit = dptx->unit; __entry->core = core; __entry->atc = atc; __entry->die = die;), + + TP_printk("%s: dptx%d: core %d, atc %d, die %d", __get_str(devname), + __entry->unit, __entry->core, __entry->atc, __entry->die)); + +TRACE_EVENT( + dptxport_call_set_link_rate, + TP_PROTO(struct dptx_port *dptx, u32 link_rate), + TP_ARGS(dptx, link_rate), + + TP_STRUCT__entry(__string(devname, dev_name(dptx->service->ep->dcp->dev)) + __field(u32, unit) + __field(u32, link_rate)), + + TP_fast_assign(__assign_str(devname); + __entry->unit = dptx->unit; + __entry->link_rate = link_rate;), + + TP_printk("%s: dptx%d: link rate 0x%x", __get_str(devname), __entry->unit, + __entry->link_rate)); + +TRACE_EVENT(iomfb_brightness, + TP_PROTO(struct apple_dcp *dcp, u32 nits), + TP_ARGS(dcp, nits), + TP_STRUCT__entry( + __field(u64, dcp) + __field(u32, nits) + ), + TP_fast_assign( + __entry->dcp = (u64)dcp; + __entry->nits = nits; + ), + TP_printk("dcp=%llx, nits=%u (raw=0x%05x)", + __entry->dcp, + __entry->nits >> 16, + __entry->nits + ) +); + +#define show_eotf(eotf) \ + __print_symbolic(eotf, { 0, "SDR gamma"}, \ + { 1, "HDR gamma"}, \ + { 2, "ST 2084 (PQ)"}, \ + { 3, "BT.2100 (HLG)"}, \ + { 4, "unexpected"}) + +#define show_encoding(enc) \ + __print_symbolic(enc, { 0, "RGB"}, \ + { 1, "YUV 4:2:0"}, \ + { 3, "YUV 4:2:2"}, \ + { 2, "YUV 4:4:4"}, \ + { 4, "DolbyVision (native)"}, \ + { 5, "DolbyVision (HDMI)"}, \ + { 6, "YCbCr 4:2:2 (DP tunnel)"}, \ + { 7, "YCbCr 4:2:2 (HDMI tunnel)"}, \ + { 8, "DolbyVision LL YCbCr 4:2:2"}, \ + { 9, "DolbyVision LL YCbCr 4:2:2 (DP)"}, \ + {10, "DolbyVision LL YCbCr 4:2:2 (HDMI)"}, \ + {11, "DolbyVision LL YCbCr 4:4:4"}, \ + {12, "DolbyVision LL RGB 4:2:2"}, \ + {13, "GRGB as YCbCr422 (Even line blue)"}, \ + {14, "GRGB as YCbCr422 (Even line red)"}, \ + {15, "unexpected"}) + +#define show_colorimetry(col) \ + __print_symbolic(col, { 0, "SMPTE 170M/BT.601"}, \ + { 1, "BT.701"}, \ + { 2, "xvYCC601"}, \ + { 3, "xvYCC709"}, \ + { 4, "sYCC601"}, \ + { 5, "AdobeYCC601"}, \ + { 6, "BT.2020 (c)"}, \ + { 7, "BT.2020 (nc)"}, \ + { 8, "DolbyVision VSVDB"}, \ + { 9, "BT.2020 (RGB)"}, \ + {10, "sRGB"}, \ + {11, "scRGB"}, \ + {12, "scRGBfixed"}, \ + {13, "AdobeRGB"}, \ + {14, "DCI-P3 (D65)"}, \ + {15, "DCI-P3 (Theater)"}, \ + {16, "Default RGB"}, \ + {17, "unexpected"}) + +#define show_range(range) \ + __print_symbolic(range, { 0, "Full"}, \ + { 1, "Limited"}, \ + { 2, "unexpected"}) + +TRACE_EVENT(iomfb_color_mode, + TP_PROTO(struct apple_dcp *dcp, u32 id, u32 score, u32 depth, + u32 colorimetry, u32 eotf, u32 range, u32 pixel_enc), + TP_ARGS(dcp, id, score, depth, colorimetry, eotf, range, pixel_enc), + TP_STRUCT__entry( + __field(u64, dcp) + __field(u32, id) + __field(u32, score) + __field(u32, depth) + __field(u32, colorimetry) + __field(u32, eotf) + __field(u32, range) + __field(u32, pixel_enc) + ), + TP_fast_assign( + __entry->dcp = (u64)dcp; + __entry->id = id; + __entry->score = score; + __entry->depth = depth; + __entry->colorimetry = min_t(u32, colorimetry, 17U); + __entry->eotf = min_t(u32, eotf, 4U); + __entry->range = min_t(u32, range, 2U); + __entry->pixel_enc = min_t(u32, pixel_enc, 15U); + ), + TP_printk("dcp=%llx, id=%u, score=%u, depth=%u, colorimetry=%s, eotf=%s, range=%s, pixel_enc=%s", + __entry->dcp, + __entry->id, + __entry->score, + __entry->depth, + show_colorimetry(__entry->colorimetry), + show_eotf(__entry->eotf), + show_range(__entry->range), + show_encoding(__entry->pixel_enc) + ) +); + +TRACE_EVENT(iomfb_timing_mode, + TP_PROTO(struct apple_dcp *dcp, u32 id, u32 score, u32 width, + u32 height, u32 clock, u32 color_mode), + TP_ARGS(dcp, id, score, width, height, clock, color_mode), + TP_STRUCT__entry( + __field(u64, dcp) + __field(u32, id) + __field(u32, score) + __field(u32, width) + __field(u32, height) + __field(u32, clock) + __field(u32, color_mode) + ), + TP_fast_assign( + __entry->dcp = (u64)dcp; + __entry->id = id; + __entry->score = score; + __entry->width = width; + __entry->height = height; + __entry->clock = clock; + __entry->color_mode = color_mode; + ), + TP_printk("dcp=%llx, id=%u, score=%u, %ux%u@%u.%u, color_mode=%u", + __entry->dcp, + __entry->id, + __entry->score, + __entry->width, + __entry->height, + __entry->clock >> 16, + ((__entry->clock & 0xffff) * 1000) >> 16, + __entry->color_mode + ) +); + +TRACE_EVENT(avep_sound_mode, + TP_PROTO(struct apple_dcp *dcp, u32 rates, u64 formats, unsigned int nchans), + TP_ARGS(dcp, rates, formats, nchans), + TP_STRUCT__entry( + __field(u64, dcp) + __field(u32, rates) + __field(u64, formats) + __field(unsigned int, nchans) + ), + TP_fast_assign( + __entry->dcp = (u64)dcp; + __entry->rates = rates; + __entry->formats = formats; + __entry->nchans = nchans; + ), + TP_printk("dcp=%llx, rates=%#x, formats=%#llx, nchans=%#x", + __entry->dcp, + __entry->rates, + __entry->formats, + __entry->nchans + ) +); + +#endif /* _TRACE_DCP_H */ + +/* This part must be outside protection */ + +#undef TRACE_INCLUDE_FILE +#define TRACE_INCLUDE_FILE trace + +#undef TRACE_INCLUDE_PATH +#define TRACE_INCLUDE_PATH . + +#include diff --git a/drivers/gpu/drm/apple/version_utils.h b/drivers/gpu/drm/apple/version_utils.h new file mode 100644 index 00000000000000..5a33ce1db61c47 --- /dev/null +++ b/drivers/gpu/drm/apple/version_utils.h @@ -0,0 +1,15 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright The Asahi Linux Contributors */ + +#ifndef __APPLE_VERSION_UTILS_H__ +#define __APPLE_VERSION_UTILS_H__ + +#include +#include + +#define DCP_FW_UNION(u) (u).DCP_FW +#define DCP_FW_SUFFIX CONCATENATE(_, DCP_FW) +#define DCP_FW_NAME(name) CONCATENATE(name, DCP_FW_SUFFIX) +#define DCP_FW_VERSION(x, y, z) ( ((x) << 16) | ((y) << 8) | (z) ) + +#endif /*__APPLE_VERSION_UTILS_H__*/ diff --git a/drivers/gpu/drm/asahi/Kconfig b/drivers/gpu/drm/asahi/Kconfig new file mode 100644 index 00000000000000..27c1016f3ef58e --- /dev/null +++ b/drivers/gpu/drm/asahi/Kconfig @@ -0,0 +1,42 @@ +# SPDX-License-Identifier: GPL-2.0 + +config RUST_DRM_SCHED + bool + select DRM_SCHED + +config RUST_DRM_GEM_SHMEM_HELPER + bool + select DRM_GEM_SHMEM_HELPER + +config RUST_DRM_GPUVM + bool + select DRM_GPUVM + +config DRM_ASAHI + tristate "Asahi (DRM support for Apple AGX GPUs)" + depends on RUST + depends on DRM=y + depends on (ARM64 && ARCH_APPLE) || (COMPILE_TEST && !GENERIC_ATOMIC64) + depends on MMU + depends on IOMMU_SUPPORT + depends on PAGE_SIZE_16KB + select APPLE_PMP_REPORT + select RUST_DRM_SCHED + select RUST_DRM_GEM_SHMEM_HELPER + select RUST_DRM_GPUVM + select RUST_APPLE_RTKIT + select WANT_DEV_COREDUMP + help + DRM driver for Apple AGX GPUs (G13x, found in the M1 SoC family) + +config DRM_ASAHI_DEBUG_ALLOCATOR + bool "Use debug allocator" + depends on DRM_ASAHI + help + Use an alternate, simpler allocator which significantly reduces + performance, but can help find firmware- or GPU-side memory safety + issues. However, it can also trigger firmware bugs more easily, + so expect GPU crashes. + + Say N unless you are debugging firmware structures or porting to a + new firmware version. diff --git a/drivers/gpu/drm/asahi/Makefile b/drivers/gpu/drm/asahi/Makefile new file mode 100644 index 00000000000000..e6724866798760 --- /dev/null +++ b/drivers/gpu/drm/asahi/Makefile @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: GPL-2.0 + +obj-$(CONFIG_DRM_ASAHI) += asahi.o diff --git a/drivers/gpu/drm/asahi/alloc.rs b/drivers/gpu/drm/asahi/alloc.rs new file mode 100644 index 00000000000000..2711b784843300 --- /dev/null +++ b/drivers/gpu/drm/asahi/alloc.rs @@ -0,0 +1,1087 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! GPU kernel object allocator. +//! +//! This kernel driver needs to manage a large number of GPU objects, in both firmware/kernel +//! address space and user address space. This module implements a simple grow-only heap allocator +//! based on the DRM MM range allocator, and a debug allocator that allocates each object as a +//! separate GEM object. +//! +//! Allocations may optionally have debugging enabled, which adds preambles that store metadata +//! about the allocation. This is useful for live debugging using the hypervisor or postmortem +//! debugging with a GPU memory snapshot, since it makes it easier to identify use-after-free and +//! caching issues. + +use kernel::{ + drm::mm, + error::Result, + prelude::*, + str::CString, // +}; + +use crate::debug::*; +use crate::driver::{ + AsahiDevRef, + AsahiDevice, // +}; +use crate::fw::types::Zeroable; +use crate::mmu; +use crate::object::{ + GpuArray, + GpuObject, + GpuOnlyArray, + GpuStruct, + GpuWeakPointer, // +}; +use crate::util::RangeExt; + +use core::cmp::Ordering; +use core::fmt; +use core::fmt::{ + Debug, + Formatter, // +}; +use core::marker::PhantomData; +use core::mem; +use core::ops::Range; +use core::ptr::NonNull; + +const DEBUG_CLASS: DebugFlags = DebugFlags::Alloc; + +#[cfg(not(CONFIG_DRM_ASAHI_DEBUG_ALLOCATOR))] +/// The driver-global allocator type +pub(crate) type DefaultAllocator = HeapAllocator; + +#[cfg(not(CONFIG_DRM_ASAHI_DEBUG_ALLOCATOR))] +/// The driver-global allocation type +pub(crate) type DefaultAllocation = HeapAllocation; + +#[cfg(CONFIG_DRM_ASAHI_DEBUG_ALLOCATOR)] +/// The driver-global allocator type +pub(crate) type DefaultAllocator = SimpleAllocator; + +#[cfg(CONFIG_DRM_ASAHI_DEBUG_ALLOCATOR)] +/// The driver-global allocation type +pub(crate) type DefaultAllocation = SimpleAllocation; + +/// Represents a raw allocation (without any type information). +pub(crate) trait RawAllocation { + /// Returns the CPU-side pointer (if CPU mapping is enabled) as a byte non-null pointer. + fn ptr(&self) -> Option>; + /// Returns the GPU VA pointer as a u64. + fn gpu_ptr(&self) -> u64; + /// Returns the AsahiDevice that owns this allocation. + fn device(&self) -> &AsahiDevice; +} + +/// Represents a typed allocation. +pub(crate) trait Allocation: Debug { + /// Returns the typed CPU-side pointer (if CPU mapping is enabled). + fn ptr(&self) -> Option>; + /// Returns the GPU VA pointer as a u64. + fn gpu_ptr(&self) -> u64; + /// Returns the size of the allocation in bytes. + fn size(&self) -> usize; + /// Returns the AsahiDevice that owns this allocation. + fn device(&self) -> &AsahiDevice; +} + +/// A generic typed allocation wrapping a RawAllocation. +/// +/// This is currently the only Allocation implementation, since it is shared by all allocators. +/// +/// # Invariants +/// The alloaction at `alloc` must have a size equal or greater than `alloc_size` plus `debug_offset` plus `padding`. +pub(crate) struct GenericAlloc { + alloc: U, + alloc_size: usize, + debug_offset: usize, + padding: usize, + tag: u32, + pad_word: u32, + _p: PhantomData, +} + +impl Allocation for GenericAlloc { + /// Returns a pointer to the inner (usable) part of the allocation. + fn ptr(&self) -> Option> { + // SAFETY: self.debug_offset is always within the allocation per the invariant, so is safe to add + // to the base pointer. + unsafe { self.alloc.ptr().map(|p| p.add(self.debug_offset).cast()) } + } + /// Returns the GPU pointer to the inner (usable) part of the allocation. + fn gpu_ptr(&self) -> u64 { + self.alloc.gpu_ptr() + self.debug_offset as u64 + } + /// Returns the size of the inner (usable) part of the allocation. + fn size(&self) -> usize { + self.alloc_size + } + fn device(&self) -> &AsahiDevice { + self.alloc.device() + } +} + +impl Debug for GenericAlloc { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + f.debug_struct(core::any::type_name::>()) + .field("ptr", &format_args!("{:?}", self.ptr())) + .field("gpu_ptr", &format_args!("{:#X?}", self.gpu_ptr())) + .field("size", &format_args!("{:#X?}", self.size())) + .finish() + } +} + +/// Debugging data associated with an allocation, when debugging is enabled. +#[repr(C)] +struct AllocDebugData { + state: u32, + tag: u32, + size: u64, + base_gpuva: u64, + obj_gpuva: u64, + name: [u8; 0x20], +} + +/// Magic flag indicating a live allocation. +const STATE_LIVE: u32 = u32::from_le_bytes(*b"LIVE"); +/// Magic flag indicating a freed allocation. +const STATE_DEAD: u32 = u32::from_le_bytes(*b"DEAD"); + +/// Marker byte to identify when firmware/GPU write beyond the end of an allocation. +const GUARD_MARKER: u32 = 0x93939393; + +impl Drop for GenericAlloc { + fn drop(&mut self) { + let debug_len = mem::size_of::(); + if self.debug_offset >= debug_len { + if let Some(p) = self.alloc.ptr() { + // SAFETY: self.debug_offset is always greater than the alloc size per + // the invariant, and greater than debug_len as checked above. + unsafe { + let p = p.as_ptr().add(self.debug_offset - debug_len); + (p as *mut u32).write(STATE_DEAD); + } + } + } + if debug_enabled(DebugFlags::FillAllocations) { + if let Some(p) = self.ptr() { + // SAFETY: Writing to our inner base pointer with our known inner size is safe. + unsafe { (p.as_ptr() as *mut u8).write_bytes(0xde, self.size()) }; + } + } + if self.padding != 0 { + if let Some(p) = self.ptr() { + // SAFETY: Per the invariant, we have at least `self.padding` bytes trailing + // the inner base pointer, after `size()` bytes. + let guard = unsafe { + core::slice::from_raw_parts( + (p.as_ptr() as *mut u8 as *const u8).add(self.size()), + self.padding, + ) + }; + let mut first_err = None; + let mut last_err = 0; + for (i, p) in guard.iter().enumerate() { + if *p != (self.pad_word >> (8 * (i & 3))) as u8 { + if first_err.is_none() { + first_err = Some(i); + } + last_err = i; + } + } + if let Some(start) = first_err { + dev_warn!( + self.device().as_ref(), + "Allocator: Corruption after object of type {}/{:#x} at {:#x}:{:#x} + {:#x}..={:#x}\n", + core::any::type_name::(), + self.tag, + self.gpu_ptr(), + self.size(), + start, + last_err, + ); + } + } + } + } +} + +static_assert!(mem::size_of::() == 0x40); + +/// A trait representing an allocator. +pub(crate) trait Allocator { + /// The raw allocation type used by this allocator. + type Raw: RawAllocation; + // TODO: Needs associated_type_defaults + // type Allocation = GenericAlloc; + + /// Returns whether CPU-side mapping is enabled. + fn cpu_maps(&self) -> bool; + /// Returns the minimum alignment for allocations. + fn min_align(&self) -> usize; + /// Allocate an object of the given size in bytes with the given alignment. + fn alloc(&mut self, size: usize, align: usize) -> Result; + + /// Returns a tuple of (count, size) of how much garbage (freed but not yet reusable objects) + /// exists in this allocator. Optional. + fn garbage(&self) -> (usize, usize) { + (0, 0) + } + /// Collect garbage for this allocator, up to the given object count. Optional. + fn collect_garbage(&mut self, _count: usize) {} + + /// Allocate a new GpuStruct object. See [`GpuObject::new`]. + #[inline(never)] + fn new_object( + &mut self, + inner: T, + callback: impl for<'a> FnOnce(&'a T) -> T::Raw<'a>, + ) -> Result>> { + GpuObject::>::new(self.alloc_object()?, inner, callback) + } + + /// Allocate a new GpuStruct object. See [`GpuObject::new_default`]. + #[inline(never)] + fn new_default( + &mut self, + ) -> Result>> + where + for<'a> ::Raw<'a>: Default + Zeroable, + { + GpuObject::>::new_default(self.alloc_object()?) + } + + /// Allocate a new GpuStruct object. See [`GpuObject::new_init`]. + #[inline(never)] + fn new_init<'a, T: GpuStruct, R: PinInit, F>, E, F>( + &mut self, + inner_init: impl Init, + raw_init: impl FnOnce(&'a T, GpuWeakPointer) -> R, + ) -> Result>> + where + kernel::error::Error: core::convert::From, + kernel::error::Error: core::convert::From, + { + GpuObject::>::new_init_prealloc( + self.alloc_object()?, + |_p| inner_init, + raw_init, + ) + } + + /// Allocate a generic buffer of the given size and alignment, applying the debug features if + /// enabled to tag it and detect overflows. + fn alloc_generic( + &mut self, + size: usize, + align: usize, + tag: Option, + ) -> Result> { + let padding = if debug_enabled(DebugFlags::DetectOverflows) { + size + } else { + 0 + }; + + let ret: GenericAlloc = + if self.cpu_maps() && debug_enabled(debug::DebugFlags::DebugAllocations) { + let debug_align = self.min_align().max(align); + let debug_len = mem::size_of::(); + let debug_offset = (debug_len * 2 + debug_align - 1) & !(debug_align - 1); + + let alloc = self.alloc(size + debug_offset + padding, align)?; + + let mut debug = AllocDebugData { + state: STATE_LIVE, + tag: tag.unwrap_or(0), + size: size as u64, + base_gpuva: alloc.gpu_ptr(), + obj_gpuva: alloc.gpu_ptr() + debug_offset as u64, + name: [0; 0x20], + }; + + let name = core::any::type_name::().as_bytes(); + let len = name.len().min(debug.name.len() - 1); + debug.name[..len].copy_from_slice(&name[..len]); + + if let Some(p) = alloc.ptr() { + // SAFETY: Per the size calculations above, this pointer math and the + // writes never exceed the allocation size. + unsafe { + let p = p.as_ptr(); + p.write_bytes(0x42, debug_offset - 2 * debug_len); + let cur = p.add(debug_offset - debug_len) as *mut AllocDebugData; + let prev = p.add(debug_offset - 2 * debug_len) as *mut AllocDebugData; + prev.copy_from(cur, 1); + cur.copy_from(&debug, 1); + }; + } + + GenericAlloc { + alloc, + alloc_size: size, + debug_offset, + tag: tag.unwrap_or(0), + pad_word: tag.unwrap_or(GUARD_MARKER) | 0x81818181, + padding, + _p: PhantomData, + } + } else { + GenericAlloc { + alloc: self.alloc(size + padding, align)?, + alloc_size: size, + debug_offset: 0, + tag: tag.unwrap_or(0), + pad_word: tag.unwrap_or(GUARD_MARKER) | 0x81818181, + padding, + _p: PhantomData, + } + }; + + if debug_enabled(DebugFlags::FillAllocations) { + if let Some(p) = ret.ptr() { + // SAFETY: Writing to our inner base pointer with our known inner size is safe. + unsafe { (p.as_ptr() as *mut u8).write_bytes(0xaa, ret.size()) }; + } + } + + if padding != 0 { + if let Some(p) = ret.ptr() { + // SAFETY: Per the invariant, we have at least `self.padding` bytes trailing + // the inner base pointer, after `size()` bytes. + let guard = unsafe { + core::slice::from_raw_parts_mut( + (p.as_ptr() as *mut u8).add(ret.size()), + padding, + ) + }; + for (i, p) in guard.iter_mut().enumerate() { + *p = (ret.pad_word >> (8 * (i & 3))) as u8; + } + } + } + + Ok(ret) + } + + /// Allocate an object of a given type, without actually initializing the allocation. + /// + /// This is useful to directly call [`GpuObject::new_*`], without borrowing a reference to the + /// allocator for the entire duration (e.g. if further allocations need to happen inside the + /// callbacks). + fn alloc_object(&mut self) -> Result> { + let size = mem::size_of::>(); + let align = mem::align_of::>(); + + self.alloc_generic(size, align, None) + } + + /// Allocate an empty `GpuArray` of a given type and length. + fn array_empty( + &mut self, + count: usize, + ) -> Result>> { + let size = mem::size_of::() * count; + let align = mem::align_of::(); + + let alloc = self.alloc_generic(size, align, None)?; + GpuArray::>::empty(alloc, count) + } + + /// Allocate an empty `GpuArray` of a given type and length. + fn array_empty_tagged( + &mut self, + count: usize, + tag: &[u8; 4], + ) -> Result>> { + let size = mem::size_of::() * count; + let align = mem::align_of::(); + + let alloc = self.alloc_generic(size, align, Some(u32::from_le_bytes(*tag)))?; + GpuArray::>::empty(alloc, count) + } + + /// Allocate an empty `GpuOnlyArray` of a given type and length. + fn array_gpuonly( + &mut self, + count: usize, + ) -> Result>> { + let size = mem::size_of::() * count; + let align = mem::align_of::(); + + let alloc = self.alloc_generic(size, align, None)?; + GpuOnlyArray::>::new(alloc, count) + } +} + +/// A simple allocation backed by a separate GEM object. +/// +/// # Invariants +/// `ptr` is either None or a valid, non-null pointer to the CPU view of the object. +/// `gpu_ptr` is the GPU-side VA of the object. +pub(crate) struct SimpleAllocation { + dev: AsahiDevRef, + ptr: Option>, + gpu_ptr: u64, + _mapping: mmu::KernelMapping, + obj: crate::gem::ObjectRef, +} + +/// SAFETY: `SimpleAllocation` just points to raw memory and should be safe to send across threads. +unsafe impl Send for SimpleAllocation {} +/// SAFETY: `SimpleAllocation` just points to raw memory and should be safe to share across threads. +unsafe impl Sync for SimpleAllocation {} + +impl Drop for SimpleAllocation { + fn drop(&mut self) { + mod_dev_dbg!( + self.device(), + "SimpleAllocator: drop object @ {:#x}\n", + self.gpu_ptr() + ); + if debug_enabled(DebugFlags::FillAllocations) { + if let Ok(mut vmap) = self.obj.vmap() { + vmap.memset(0x42); + } + } + } +} + +impl RawAllocation for SimpleAllocation { + fn ptr(&self) -> Option> { + self.ptr + } + fn gpu_ptr(&self) -> u64 { + self.gpu_ptr + } + fn device(&self) -> &AsahiDevice { + &self.dev + } +} + +/// A simple allocator that allocates each object as its own GEM object, aligned to the end of a +/// page. +/// +/// This is very slow, but it has the advantage that over-reads by the firmware or GPU will fault on +/// the guard page after the allocation, which can be useful to validate that the firmware's or +/// GPU's idea of object size what we expect. +pub(crate) struct SimpleAllocator { + dev: AsahiDevRef, + range: Range, + prot: mmu::Prot, + vm: mmu::Vm, + min_align: usize, + cpu_maps: bool, +} + +impl SimpleAllocator { + /// Create a new `SimpleAllocator` for a given address range and `Vm`. + #[allow(dead_code)] + #[allow(clippy::too_many_arguments)] + pub(crate) fn new( + dev: &AsahiDevice, + vm: &mmu::Vm, + range: Range, + min_align: usize, + prot: mmu::Prot, + _block_size: usize, + mut cpu_maps: bool, + _name: fmt::Arguments<'_>, + _keep_garbage: bool, + ) -> Result { + if debug_enabled(DebugFlags::ForceCPUMaps) { + cpu_maps = true; + } + Ok(SimpleAllocator { + dev: dev.into(), + vm: vm.clone(), + range, + prot, + min_align, + cpu_maps, + }) + } +} + +impl Allocator for SimpleAllocator { + type Raw = SimpleAllocation; + + fn cpu_maps(&self) -> bool { + self.cpu_maps + } + + fn min_align(&self) -> usize { + self.min_align + } + + #[inline(never)] + fn alloc(&mut self, size: usize, align: usize) -> Result { + let size_aligned = (size + mmu::UAT_PGSZ - 1) & !mmu::UAT_PGMSK; + let align = self.min_align.max(align); + let offset = (size_aligned - size) & !(align - 1); + + mod_dev_dbg!( + &self.dev, + "SimpleAllocator::new: size={:#x} size_al={:#x} al={:#x} off={:#x}\n", + size, + size_aligned, + align, + offset + ); + + let mut obj = crate::gem::new_kernel_object(&self.dev, size_aligned)?; + let p = obj.vmap()?.as_mut_ptr() as *mut u8; + if debug_enabled(DebugFlags::FillAllocations) { + obj.vmap()?.memset(0xde); + } + let mapping = obj.map_into_range( + &self.vm, + self.range.clone(), + self.min_align.max(mmu::UAT_PGSZ) as u64, + self.prot, + true, + )?; + + let iova = mapping.iova(); + + // SAFETY: Per the math above to calculate `size_aligned`, this can never overflow. + let ptr = unsafe { p.add(offset) }; + let gpu_ptr = iova + offset as u64; + + mod_dev_dbg!( + &self.dev, + "SimpleAllocator::new -> {:#?} / {:#?} | {:#x} / {:#x}\n", + p, + ptr, + iova, + gpu_ptr + ); + + Ok(SimpleAllocation { + dev: self.dev.clone(), + ptr: NonNull::new(ptr), + gpu_ptr, + _mapping: mapping, + obj, + }) + } +} + +/// Inner data for an allocation from the heap allocator. +/// +/// This is wrapped in an `mm::Node`. +pub(crate) struct HeapAllocationInner { + dev: AsahiDevRef, + ptr: Option>, + real_size: usize, +} + +/// SAFETY: `HeapAllocationInner` just points to raw memory and should be safe to send across threads. +unsafe impl Send for HeapAllocationInner {} +/// SAFETY: `HeapAllocationInner` just points to raw memory and should be safe to share between threads. +unsafe impl Sync for HeapAllocationInner {} + +/// Outer view of a heap allocation. +/// +/// This uses an Option<> so we can move the internal `Node` into the garbage pool when it gets +/// dropped. +/// +/// # Invariants +/// The `Option` must always be `Some(...)` while this object is alive. +pub(crate) struct HeapAllocation(Option>); + +impl Drop for HeapAllocation { + fn drop(&mut self) { + let node = self.0.take().unwrap(); + let size = node.size(); + let alloc = node.alloc_ref(); + + alloc.with(|a| { + if let Some(garbage) = a.garbage.as_mut() { + if garbage.push(node, GFP_KERNEL).is_err() { + dev_err!( + &a.dev.as_ref(), + "HeapAllocation[{}]::drop: Failed to keep garbage\n", + &*a.name, + ); + } + a.total_garbage += size as usize; + None + } else { + // We need to ensure node survives this scope, since dropping it + // will try to take the mm lock and deadlock us + Some(node) + } + }); + } +} + +impl mm::AllocInner for HeapAllocatorInner { + fn drop_object( + &mut self, + start: u64, + _size: u64, + _color: usize, + obj: &mut HeapAllocationInner, + ) { + /* real_size == 0 means it's a guard node */ + if obj.real_size > 0 { + mod_dev_dbg!( + obj.dev, + "HeapAllocator[{}]: drop object @ {:#x} ({} bytes)\n", + &*self.name, + start, + obj.real_size, + ); + self.allocated -= obj.real_size; + } + } +} + +impl RawAllocation for HeapAllocation { + // SAFETY: This function must always return a valid pointer. + // Since the HeapAllocation contains a reference to the + // backing_objects array that contains the object backing this pointer, + // and objects are only ever added to it, this pointer is guaranteed to + // remain valid for the lifetime of the HeapAllocation. + fn ptr(&self) -> Option> { + self.0.as_ref().unwrap().ptr + } + // SAFETY: This function must always return a valid GPU pointer. + // See the explanation in ptr(). + fn gpu_ptr(&self) -> u64 { + self.0.as_ref().unwrap().start() + } + fn device(&self) -> &AsahiDevice { + &self.0.as_ref().unwrap().dev + } +} + +/// Inner data for a heap allocator which uses the DRM MM range allocator to manage the heap. +/// +/// This is wrapped by an `mm::Allocator`. +struct HeapAllocatorInner { + dev: AsahiDevRef, + allocated: usize, + backing_objects: KVec<(crate::gem::ObjectRef, mmu::KernelMapping, u64)>, + garbage: Option>>, + total_garbage: usize, + name: CString, +} + +/// A heap allocator which uses the DRM MM range allocator to manage its objects. +/// +/// The heap is composed of a series of GEM objects. This implementation only ever grows the heap, +/// never shrinks it. +pub(crate) struct HeapAllocator { + dev: AsahiDevRef, + range: Range, + top: u64, + prot: mmu::Prot, + vm: mmu::Vm, + min_align: usize, + block_size: usize, + cpu_maps: bool, + guard_nodes: KVec>, + mm: mm::Allocator, + name: CString, + garbage: Option>>, +} + +impl HeapAllocator { + /// Create a new HeapAllocator for a given `Vm` and address range. + #[allow(dead_code)] + #[allow(clippy::too_many_arguments)] + pub(crate) fn new( + dev: &AsahiDevice, + vm: &mmu::Vm, + range: Range, + min_align: usize, + prot: mmu::Prot, + block_size: usize, + mut cpu_maps: bool, + name: fmt::Arguments<'_>, + keep_garbage: bool, + ) -> Result { + if !min_align.is_power_of_two() { + return Err(EINVAL); + } + if debug_enabled(DebugFlags::ForceCPUMaps) { + cpu_maps = true; + } + + let name = CString::try_from_fmt(name)?; + + let inner = HeapAllocatorInner { + dev: dev.into(), + allocated: 0, + backing_objects: KVec::new(), + // TODO: This clearly needs a try_clone() or similar + name: CString::try_from_fmt(fmt!("{}", &*name))?, + garbage: if keep_garbage { + Some(KVec::new()) + } else { + None + }, + total_garbage: 0, + }; + + let mm = mm::Allocator::new(range.start, range.range(), inner)?; + + Ok(HeapAllocator { + dev: dev.into(), + vm: vm.clone(), + top: range.start, + range, + prot, + min_align, + block_size: block_size.max(min_align), + cpu_maps, + guard_nodes: KVec::new(), + mm, + name, + garbage: if keep_garbage { + Some({ + let mut v = KVec::new(); + v.reserve(128, GFP_KERNEL)?; + v + }) + } else { + None + }, + }) + } + + /// Add a new backing block of the given size to this heap. + /// + /// If CPU mapping is enabled, this also adds a guard node to the range allocator to ensure that + /// objects cannot straddle backing block boundaries, since we cannot easily create a contiguous + /// CPU VA mapping for them. This can create some fragmentation. If CPU mapping is disabled, we + /// skip the guard blocks, since the GPU view of the heap is always contiguous. + #[inline(never)] + fn add_block(&mut self, size: usize) -> Result { + let size_aligned = (size + mmu::UAT_PGSZ - 1) & !mmu::UAT_PGMSK; + + mod_dev_dbg!( + &self.dev, + "HeapAllocator[{}]::add_block: size={:#x} size_al={:#x}\n", + &*self.name, + size, + size_aligned, + ); + + if self.top.saturating_add(size_aligned as u64) > self.range.end { + dev_err!( + self.dev.as_ref(), + "HeapAllocator[{}]::add_block: Exhausted VA space\n", + &*self.name, + ); + } + + let mut obj = crate::gem::new_kernel_object(&self.dev, size_aligned)?; + if self.cpu_maps && debug_enabled(DebugFlags::FillAllocations) { + obj.vmap()?.memset(0xde); + } + + let gpu_ptr = self.top; + let mapping = obj + .map_at(&self.vm, gpu_ptr, self.prot, self.cpu_maps) + .inspect_err(|err| { + dev_err!( + self.dev.as_ref(), + "HeapAllocator[{}]::add_block: Failed to map at {:#x} ({:?})\n", + &*self.name, + gpu_ptr, + err + ); + })?; + + if self.cpu_maps { + // Create virtual mapping here ahead of time so that the vmap() in + // alloc_inner() does not take the the object's dma_resv lock while + // the mm lock is locked. mmu::Vm requires the opposite lock order. + obj.vmap()?; + } + + self.mm + .with_inner(|inner| inner.backing_objects.reserve(1, GFP_KERNEL))?; + + let mut new_top = self.top + size_aligned as u64; + if self.cpu_maps { + let guard = self.min_align.max(mmu::UAT_PGSZ); + mod_dev_dbg!( + self.dev, + "HeapAllocator[{}]::add_block: Adding guard node {:#x}:{:#x}\n", + &*self.name, + new_top, + guard + ); + + let inner = HeapAllocationInner { + dev: self.dev.clone(), + ptr: None, + real_size: 0, + }; + + let node = match self.mm.reserve_node(inner, new_top, guard as u64, 0) { + Ok(a) => a, + Err(a) => { + dev_err!( + self.dev.as_ref(), + "HeapAllocator[{}]::add_block: Failed to reserve guard node {:#x}:{:#x}: {:?}\n", + &*self.name, + guard, + new_top, + a + ); + return Err(EIO); + } + }; + + self.guard_nodes.push(node, GFP_KERNEL)?; + + new_top += guard as u64; + } + mod_dev_dbg!( + &self.dev, + "HeapAllocator[{}]::add_block: top={:#x}\n", + &*self.name, + new_top + ); + + self.mm.with_inner(|inner| { + inner + .backing_objects + .push((obj, mapping, gpu_ptr), GFP_KERNEL) + })?; + + self.top = new_top; + + cls_dev_dbg!( + MemStats, + &self.dev, + "{} Heap: grow to {} bytes\n", + &*self.name, + self.top - self.range.start + ); + + Ok(()) + } + + /// Find the backing object index that backs a given GPU address. + fn find_obj(&mut self, addr: u64) -> Result { + self.mm.with_inner(|inner| { + inner + .backing_objects + .binary_search_by(|obj| { + let start = obj.2; + let end = obj.2 + obj.0.size() as u64; + if start > addr { + Ordering::Greater + } else if end <= addr { + Ordering::Less + } else { + Ordering::Equal + } + }) + .or(Err(ENOENT)) + }) + } + + fn alloc_inner(&mut self, size: usize, align: usize) -> Result { + if align != 0 && !align.is_power_of_two() { + return Err(EINVAL); + } + let align = self.min_align.max(align); + let size_aligned = (size + align - 1) & !(align - 1); + + mod_dev_dbg!( + &self.dev, + "HeapAllocator[{}]::new: size={:#x} size_al={:#x}\n", + &*self.name, + size, + size_aligned, + ); + + let inner = HeapAllocationInner { + dev: self.dev.clone(), + ptr: None, + real_size: size, + }; + + let mut node = match self.mm.insert_node_generic( + inner, + size_aligned as u64, + align as u64, + 0, + mm::InsertMode::Best, + ) { + Ok(a) => a, + Err(a) => { + dev_err!( + &self.dev.as_ref(), + "HeapAllocator[{}]::new: Failed to insert node of size {:#x} / align {:#x}: {:?}\n", + &*self.name, size_aligned, align, a + ); + return Err(a); + } + }; + + self.mm.with_inner(|inner| inner.allocated += size); + + let mut new_object = false; + let start = node.start(); + let end = start + node.size(); + if end > self.top { + if start > self.top { + dev_warn!( + self.dev.as_ref(), + "HeapAllocator[{}]::alloc: top={:#x}, start={:#x}\n", + &*self.name, + self.top, + start + ); + } + let block_size = self.block_size.max((end - self.top) as usize); + self.add_block(block_size)?; + new_object = true; + } + assert!(end <= self.top); + + if self.cpu_maps { + mod_dev_dbg!( + self.dev, + "HeapAllocator[{}]::alloc: mapping to CPU\n", + &*self.name + ); + + let idx = if new_object { + None + } else { + Some(match self.find_obj(start) { + Ok(a) => a, + Err(_) => { + dev_warn!( + self.dev.as_ref(), + "HeapAllocator[{}]::alloc: Failed to find object at {:#x}\n", + &*self.name, + start + ); + return Err(EIO); + } + }) + }; + let (obj_start, obj_size, p) = self.mm.with_inner(|inner| -> Result<_> { + let idx = idx.unwrap_or(inner.backing_objects.len() - 1); + let obj = &mut inner.backing_objects[idx]; + let p = obj.0.vmap()?.as_mut_ptr() as *mut u8; + Ok((obj.2, obj.0.size(), p)) + })?; + assert!(obj_start <= start); + assert!(obj_start + obj_size as u64 >= end); + node.as_mut().inner_mut().ptr = + // SAFETY: Per the asserts above, this offset is always within the allocation. + NonNull::new(unsafe { p.add((start - obj_start) as usize) }); + mod_dev_dbg!( + self.dev, + "HeapAllocator[{}]::alloc: CPU pointer = {:?}\n", + &*self.name, + node.ptr + ); + } + + mod_dev_dbg!( + self.dev, + "HeapAllocator[{}]::alloc: Allocated {:#x} bytes @ {:#x}\n", + &*self.name, + end - start, + start + ); + + Ok(HeapAllocation(Some(node))) + } +} + +impl Allocator for HeapAllocator { + type Raw = HeapAllocation; + + fn cpu_maps(&self) -> bool { + self.cpu_maps + } + + fn min_align(&self) -> usize { + self.min_align + } + + fn alloc(&mut self, size: usize, align: usize) -> Result { + let ret = self.alloc_inner(size, align); + + if ret.is_err() { + dev_warn!( + self.dev.as_ref(), + "HeapAllocator[{}]::alloc: Allocation of {:#x}({:#x}) size object failed\n", + &*self.name, + size, + align + ); + } + ret + } + + fn garbage(&self) -> (usize, usize) { + self.mm.with_inner(|inner| { + if let Some(g) = inner.garbage.as_ref() { + (g.len(), inner.total_garbage) + } else { + (0, 0) + } + }) + } + + fn collect_garbage(&mut self, mut count: usize) { + if let Some(garbage) = self.garbage.as_mut() { + garbage.clear(); + + while count > 0 { + let block = count.min(garbage.capacity()); + assert!(block > 0); + + // Take the garbage out of the inner block, so we can safely drop it without deadlocking + self.mm.with_inner(|inner| { + if let Some(g) = inner.garbage.as_mut() { + for node in g.drain(0..block) { + inner.total_garbage -= node.size() as usize; + garbage + .push(node, GFP_KERNEL) + .expect("push() failed after reserve()"); + } + } + }); + + count -= block; + // Now drop it + garbage.clear(); + } + } + } +} + +impl Drop for HeapAllocatorInner { + fn drop(&mut self) { + mod_dev_dbg!( + self.dev, + "HeapAllocator[{}]: dropping allocator\n", + &*self.name + ); + if self.allocated > 0 { + // This should never happen + dev_crit!( + self.dev.as_ref(), + "HeapAllocator[{}]: dropping with {} bytes allocated\n", + &*self.name, + self.allocated + ); + } + } +} diff --git a/drivers/gpu/drm/asahi/asahi.rs b/drivers/gpu/drm/asahi/asahi.rs new file mode 100644 index 00000000000000..296091dced554d --- /dev/null +++ b/drivers/gpu/drm/asahi/asahi.rs @@ -0,0 +1,61 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#![recursion_limit = "2048"] + +//! Driver for the Apple AGX GPUs found in Apple Silicon SoCs. + +mod alloc; +mod buffer; +mod channel; +#[cfg(CONFIG_DEV_COREDUMP)] +mod crashdump; +mod debug; +mod driver; +mod event; +mod file; +mod float; +mod fw; +mod gem; +mod gpu; +mod hw; +mod initdata; +mod mem; +mod microseq; +mod mmu; +mod object; +mod pgtable; +mod queue; +mod regs; +mod slotalloc; +mod util; +#[cfg(CONFIG_DRM_ASAHI_MAPLE_TREE)] +mod vm; +mod workqueue; + +kernel::module_platform_driver! { + type: driver::AsahiDriver, + name: "asahi", + description: "AGX GPU driver for Apple silicon SoCs", + license: "Dual MIT/GPL", + params: { + debug_flags: u64 { + default: 0, + // permissions: 0o644, + description: "Debug flags", + }, + fault_control: u32 { + default: 0xb, + // permissions: 0, + description: "Fault control (0x0: hard faults, 0xb: macOS default)", + }, + initial_tvb_size: usize { + default: 0x8, + // permissions: 0o644, + description: "Initial TVB size in blocks", + }, + robust_isolation: u32 { + default: 0, + // permissions: 0o644, + description: "Fully isolate GPU contexts (limits performance)", + }, + }, +} diff --git a/drivers/gpu/drm/asahi/buffer.rs b/drivers/gpu/drm/asahi/buffer.rs new file mode 100644 index 00000000000000..309482441062d9 --- /dev/null +++ b/drivers/gpu/drm/asahi/buffer.rs @@ -0,0 +1,809 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Tiled Vertex Buffer management +//! +//! This module manages the Tiled Vertex Buffer, also known as the Parameter Buffer (in imgtec +//! parlance) or the tiler heap (on other architectures). This buffer holds transformed primitive +//! data between the vertex/tiling stage and the fragment stage. +//! +//! On AGX, the buffer is a heap of 128K blocks split into 32K pages (which must be aligned to a +//! multiple of 32K in VA space). The buffer can be shared between multiple render jobs, and each +//! will allocate pages from it during vertex processing and return them during fragment processing. +//! +//! If the buffer runs out of free pages, the vertex pass stops and a partial fragment pass occurs, +//! spilling the intermediate render target state to RAM (a partial render). This is all managed +//! transparently by the firmware. Since partial renders are less efficient, the kernel must grow +//! the heap in response to feedback from the firmware to avoid partial renders in the future. +//! Currently, we only ever grow the heap, and never shrink it. +//! +//! AGX also supports memoryless render targets, which can be used for intermediate results within +//! a render pass. To support partial renders, it seems the GPU/firmware has the ability to borrow +//! pages from the TVB buffer as a temporary render target buffer. Since this happens during a +//! partial render itself, if the buffer runs out of space, it requires synchronous growth in +//! response to a firmware interrupt. This is not currently supported, but may be in the future, +//! though it is unclear whether it is worth the effort. +//! +//! This module is also in charge of managing the temporary objects associated with a single render +//! pass, which includes the top-level tile array, the tail pointer cache, preemption buffers, and +//! other miscellaneous structures collectively managed as a "scene". +//! +//! To avoid runaway memory usage, there is a maximum size for buffers (at that point it's unlikely +//! that partial renders will incur much overhead over the buffer data access itself). This is +//! different depending on whether memoryless render targets are in use, and is currently hardcoded. +//! to the most common value used by macOS. + +use crate::debug::*; +use crate::fw::buffer; +use crate::fw::types::*; +use crate::util::*; +use crate::{ + alloc, + fw, + gpu, + hw, + mmu, + slotalloc, // +}; +use core::sync::atomic::Ordering; +use kernel::new_mutex; +use kernel::prelude::*; +use kernel::sync::{ + Arc, + Mutex, // +}; +use kernel::{ + c_str, + static_lock_class, // +}; + +const DEBUG_CLASS: DebugFlags = DebugFlags::Buffer; + +/// There are 127 GPU/firmware-side buffer manager slots (yes, 127, not 128). +const NUM_BUFFERS: u32 = 127; + +/// Page size bits for buffer pages (32K). VAs must be aligned to this size. +pub(crate) const PAGE_SHIFT: usize = 15; +/// Page size for buffer pages. +pub(crate) const PAGE_SIZE: usize = 1 << PAGE_SHIFT; +/// Number of pages in a buffer block, which should be contiguous in VA space. +pub(crate) const PAGES_PER_BLOCK: usize = 4; +/// Size of a buffer block. +pub(crate) const BLOCK_SIZE: usize = PAGE_SIZE * PAGES_PER_BLOCK; + +/// Metadata about the tiling configuration for a scene. This is computed in the `render` module. +/// based on dimensions, tile size, and other info. +pub(crate) struct TileInfo { + /// Tile count in the X dimension. Tiles are always 32x32. + pub(crate) tiles_x: u32, + /// Tile count in the Y dimension. Tiles are always 32x32. + pub(crate) tiles_y: u32, + /// Total tile count. + pub(crate) tiles: u32, + /// Micro-tile width (16 or 32). + pub(crate) utile_width: u32, + /// Micro-tile height (16 or 32). + pub(crate) utile_height: u32, + // Macro-tiles in the X dimension. Always 4. + //pub(crate) mtiles_x: u32, + // Macro-tiles in the Y dimension. Always 4. + //pub(crate) mtiles_y: u32, + /// Tiles per macro-tile in the X dimension. + pub(crate) tiles_per_mtile_x: u32, + /// Tiles per macro-tile in the Y dimension. + pub(crate) tiles_per_mtile_y: u32, + // Total tiles per macro-tile. + //pub(crate) tiles_per_mtile: u32, + /// Micro-tiles per macro-tile in the X dimension. + pub(crate) utiles_per_mtile_x: u32, + /// Micro-tiles per macro-tile in the Y dimension. + pub(crate) utiles_per_mtile_y: u32, + // Total micro-tiles per macro-tile. + //pub(crate) utiles_per_mtile: u32, + /// Size of the top-level tilemap, in bytes (for all layers, one cluster). + pub(crate) tilemap_size: usize, + /// Size of the Tail Pointer Cache, in bytes (for all layers * clusters). + pub(crate) tpc_size: usize, + /// Number of blocks in the clustering meta buffer (for clustering) per layer. + pub(crate) meta1_layer_stride: u32, + /// Number of blocks in the clustering meta buffer (for clustering). + pub(crate) meta1_blocks: u32, + /// Layering metadata size. + pub(crate) layermeta_size: usize, + /// Minimum number of TVB blocks for this render. + pub(crate) min_tvb_blocks: usize, + /// Tiling parameter structure passed to firmware. + pub(crate) params: fw::vertex::raw::TilingParameters, +} + +/// A single scene, representing a render pass and its required buffers. +#[versions(AGX)] +#[derive(Debug)] +pub(crate) struct Scene { + object: GpuObject, + slot: u32, + rebind: bool, + preempt2_off: usize, + preempt3_off: usize, + // Note: these are dead code only on some version variants. + // It's easier to do this than to propagate the version conditionals everywhere. + #[allow(dead_code)] + meta1_off: usize, + #[allow(dead_code)] + meta2_off: usize, + #[allow(dead_code)] + meta3_off: usize, + #[allow(dead_code)] + meta4_off: usize, +} + +#[versions(AGX)] +impl Scene::ver { + /// Returns true if the buffer was bound to a fresh manager slot, and therefore needs an init + /// command before a render. + pub(crate) fn rebind(&self) -> bool { + self.rebind + } + + /// Returns the buffer manager slot this scene's buffer was bound to. + pub(crate) fn slot(&self) -> u32 { + self.slot + } + + /// Returns the GPU pointer to the [`buffer::Scene::ver`]. + pub(crate) fn gpu_pointer(&self) -> GpuPointer<'_, buffer::Scene::ver> { + self.object.gpu_pointer() + } + + /// Returns the GPU weak pointer to the [`buffer::Scene::ver`]. + pub(crate) fn weak_pointer(&self) -> GpuWeakPointer { + self.object.weak_pointer() + } + + /// Returns the GPU weak pointer to the kernel-side temp buffer. + /// (purpose unknown...) + pub(crate) fn kernel_buffer_pointer(&self) -> GpuWeakPointer<[u8]> { + self.object.buffer.inner.lock().kernel_buffer.weak_pointer() + } + + /// Returns the GPU pointer to the `buffer::Info::ver` object associated with this Scene. + pub(crate) fn buffer_pointer(&self) -> GpuPointer<'_, buffer::Info::ver> { + // SAFETY: We can't return the strong pointer directly since its lifetime crosses a lock, + // but we know its lifetime will be valid as long as &self since we hold a reference to the + // buffer, so just construct the strong pointer with the right lifetime here. + unsafe { self.weak_buffer_pointer().upgrade() } + } + + /// Returns the GPU weak pointer to the `buffer::Info::ver` object associated with this Scene. + pub(crate) fn weak_buffer_pointer(&self) -> GpuWeakPointer { + self.object.buffer.inner.lock().info.weak_pointer() + } + + /// Returns the GPU pointer to the TVB heap metadata buffer. + pub(crate) fn tvb_heapmeta_pointer(&self) -> GpuPointer<'_, &'_ [u8]> { + self.object.tvb_heapmeta.gpu_pointer() + } + + /// Returns the GPU pointer to the layer metadata buffer. + pub(crate) fn tvb_layermeta_pointer(&self) -> GpuPointer<'_, &'_ [u8]> { + self.object.tvb_heapmeta.gpu_offset_pointer(0x200) + } + + /// Returns the GPU pointer to the top-level TVB tilemap buffer. + pub(crate) fn tvb_tilemap_pointer(&self) -> GpuPointer<'_, &'_ [u8]> { + self.object.tvb_tilemap.gpu_pointer() + } + + /// Returns the GPU pointer to the Tail Pointer Cache buffer. + pub(crate) fn tpc_pointer(&self) -> GpuPointer<'_, &'_ [u8]> { + self.object.tpc.gpu_pointer() + } + + /// Returns the GPU pointer to the first preemption scratch buffer. + pub(crate) fn preempt_buf_1_pointer(&self) -> GpuPointer<'_, &'_ [u8]> { + self.object.preempt_buf.gpu_pointer() + } + + /// Returns the GPU pointer to the second preemption scratch buffer. + pub(crate) fn preempt_buf_2_pointer(&self) -> GpuPointer<'_, &'_ [u8]> { + self.object + .preempt_buf + .gpu_offset_pointer(self.preempt2_off) + } + + /// Returns the GPU pointer to the third preemption scratch buffer. + pub(crate) fn preempt_buf_3_pointer(&self) -> GpuPointer<'_, &'_ [u8]> { + self.object + .preempt_buf + .gpu_offset_pointer(self.preempt3_off) + } + + /// Returns the GPU pointer to the per-cluster tilemap buffer, if clustering is enabled. + #[allow(dead_code)] + pub(crate) fn cluster_tilemaps_pointer(&self) -> Option> { + self.object + .clustering + .as_ref() + .map(|c| c.tilemaps.gpu_pointer()) + } + + /// Returns the GPU pointer to the clustering layer metadata buffer, if clustering is enabled. + #[allow(dead_code)] + pub(crate) fn tvb_cluster_layermeta_pointer(&self) -> Option> { + self.object + .clustering + .as_ref() + .map(|c| c.meta.gpu_pointer()) + } + + /// Returns the GPU pointer to the clustering metadata 1 buffer, if clustering is enabled. + #[allow(dead_code)] + pub(crate) fn meta_1_pointer(&self) -> Option> { + self.object + .clustering + .as_ref() + .map(|c| c.meta.gpu_offset_pointer(self.meta1_off)) + } + + /// Returns the GPU pointer to the clustering metadata 2 buffer, if clustering is enabled. + #[allow(dead_code)] + pub(crate) fn meta_2_pointer(&self) -> Option> { + self.object + .clustering + .as_ref() + .map(|c| c.meta.gpu_offset_pointer(self.meta2_off)) + } + + /// Returns the GPU pointer to the clustering metadata 3 buffer, if clustering is enabled. + #[allow(dead_code)] + pub(crate) fn meta_3_pointer(&self) -> Option> { + self.object + .clustering + .as_ref() + .map(|c| c.meta.gpu_offset_pointer(self.meta3_off)) + } + + /// Returns the GPU pointer to the clustering metadata 4 buffer, if clustering is enabled. + #[allow(dead_code)] + pub(crate) fn meta_4_pointer(&self) -> Option> { + self.object + .clustering + .as_ref() + .map(|c| c.meta.gpu_offset_pointer(self.meta4_off)) + } +} + +#[versions(AGX)] +impl Drop for Scene::ver { + fn drop(&mut self) { + let mut inner = self.object.buffer.inner.lock(); + assert_ne!(inner.active_scenes, 0); + inner.active_scenes -= 1; + + if inner.active_scenes == 0 { + mod_pr_debug!( + "Buffer: no scenes left, dropping slot {}", + inner.active_slot.take().unwrap().slot() + ); + inner.active_slot = None; + } + } +} + +/// Inner data for a single TVB buffer object. +#[versions(AGX)] +struct BufferInner { + info: GpuObject, + ualloc: Arc>, + ualloc_priv: Arc>, + blocks: KVec>, + max_blocks: usize, + max_blocks_nomemless: usize, + mgr: BufferManager::ver, + active_scenes: usize, + active_slot: Option>, + last_token: Option, + tpc: Option>>, + kernel_buffer: GpuArray, + stats: GpuObject, + cfg: &'static hw::HwConfig, + preempt1_size: usize, + preempt2_size: usize, + preempt3_size: usize, + num_clusters: usize, +} + +/// Locked and reference counted TVB buffer. +#[versions(AGX)] +pub(crate) struct Buffer { + inner: Arc>, +} + +#[versions(AGX)] +impl Buffer::ver { + /// Create a new Buffer for a given VM, given the per-VM allocators. + pub(crate) fn new( + gpu: &dyn gpu::GpuManager, + alloc: &mut gpu::KernelAllocators, + ualloc: Arc>, + ualloc_priv: Arc>, + mgr: &BufferManager::ver, + ) -> Result { + // These are the typical max numbers on macOS. + // 8GB machines have this halved. + let max_size: usize = 862_322_688; // bytes + let max_size_nomemless = max_size / 3; + + let max_blocks = max_size / BLOCK_SIZE; + let max_blocks_nomemless = max_size_nomemless / BLOCK_SIZE; + let max_pages = max_blocks * PAGES_PER_BLOCK; + let max_pages_nomemless = max_blocks_nomemless * PAGES_PER_BLOCK; + + let num_clusters = gpu.get_dyncfg().id.num_clusters as usize; + let num_clusters_adj = if num_clusters > 1 { + num_clusters + 1 + } else { + 1 + }; + + let preempt1_size = num_clusters_adj * gpu.get_cfg().preempt1_size; + let preempt2_size = num_clusters_adj * gpu.get_cfg().preempt2_size; + let preempt3_size = num_clusters_adj * gpu.get_cfg().preempt3_size; + + let shared = &mut alloc.shared; + let info = alloc.private.new_init( + { + let ualloc_priv = &ualloc_priv; + try_init!(buffer::Info::ver { + block_ctl: shared.new_default::()?, + counter: shared.new_default::()?, + page_list: ualloc_priv.lock().array_empty_tagged(max_pages, b"PLST")?, + block_list: ualloc_priv + .lock() + .array_empty_tagged(max_blocks * 2, b"BLST")?, + }) + }, + |inner, _p| { + try_init!(buffer::raw::Info::ver { + gpu_counter: 0x0, + unk_4: 0, + last_id: 0x0, + cur_id: -1, + unk_10: 0x0, + gpu_counter2: 0x0, + unk_18: 0x0, + #[ver(V < V13_0B4 || G >= G14X)] + unk_1c: 0x0, + page_list: inner.page_list.gpu_pointer(), + page_list_size: (4 * max_pages).try_into()?, + page_count: AtomicU32::new(0), + max_blocks: max_blocks.try_into()?, + block_count: AtomicU32::new(0), + unk_38: 0x0, + block_list: inner.block_list.gpu_pointer(), + block_ctl: inner.block_ctl.gpu_pointer(), + last_page: AtomicU32::new(0), + gpu_page_ptr1: 0x0, + gpu_page_ptr2: 0x0, + unk_58: 0x0, + block_size: BLOCK_SIZE as u32, + unk_60: U64(0x0), + counter: inner.counter.gpu_pointer(), + unk_70: 0x0, + unk_74: 0x0, + unk_78: 0x0, + unk_7c: 0x0, + unk_80: 0x1, + max_pages: max_pages.try_into()?, + max_pages_nomemless: max_pages_nomemless.try_into()?, + unk_8c: 0x0, + unk_90: Default::default(), + }) + }, + )?; + + // Technically similar to Scene below, let's play it safe. + let kernel_buffer = alloc.shared.array_empty_tagged(0x40, b"KBUF")?; + let stats = alloc + .shared + .new_object(Default::default(), |_inner| buffer::raw::Stats { + reset: AtomicU32::from(1), + ..Default::default() + })?; + + Ok(Buffer::ver { + inner: Arc::pin_init( + new_mutex!(BufferInner::ver { + info, + ualloc, + ualloc_priv, + blocks: KVec::new(), + max_blocks, + max_blocks_nomemless, + mgr: mgr.clone(), + active_scenes: 0, + active_slot: None, + last_token: None, + tpc: None, + kernel_buffer, + stats, + cfg: gpu.get_cfg(), + preempt1_size, + preempt2_size, + preempt3_size, + num_clusters, + }), + GFP_KERNEL, + )?, + }) + } + + /// Returns the total block count allocated to this Buffer. + pub(crate) fn block_count(&self) -> u32 { + self.inner.lock().blocks.len() as u32 + } + + /// Automatically grow the Buffer based on feedback from the statistics. + pub(crate) fn auto_grow(&self) -> Result { + let inner = self.inner.lock(); + + let used_pages = inner.stats.with(|raw, _inner| { + let used = raw.max_pages.load(Ordering::Relaxed); + raw.reset.store(1, Ordering::Release); + used as usize + }); + + let need_blocks = (used_pages * 2) + .div_ceil(PAGES_PER_BLOCK) + .min(inner.max_blocks_nomemless); + let want_blocks = (used_pages * 3) + .div_ceil(PAGES_PER_BLOCK) + .min(inner.max_blocks_nomemless); + + let cur_count = inner.blocks.len(); + + if need_blocks <= cur_count { + Ok(false) + } else { + // Grow to 3x requested size (same logic as macOS) + core::mem::drop(inner); + self.ensure_blocks(want_blocks)?; + Ok(true) + } + } + + /// Synchronously grow the Buffer. + pub(crate) fn sync_grow(&self) { + let inner = self.inner.lock(); + + let cur_count = inner.blocks.len(); + core::mem::drop(inner); + if self.ensure_blocks(cur_count + 10).is_err() { + pr_err!("BufferManager: Failed to grow buffer synchronously\n"); + } + } + + /// Ensure that the buffer has at least a certain minimum size in blocks. + pub(crate) fn ensure_blocks(&self, min_blocks: usize) -> Result { + let mut inner = self.inner.lock(); + + let cur_count = inner.blocks.len(); + if cur_count >= min_blocks { + return Ok(false); + } + if min_blocks > inner.max_blocks { + return Err(ENOMEM); + } + + let add_blocks = min_blocks - cur_count; + let new_count = min_blocks; + + let mut new_blocks: KVec> = KVec::new(); + + // Allocate the new blocks first, so if it fails they will be dropped + let mut ualloc = inner.ualloc.lock(); + for _i in 0..add_blocks { + new_blocks.push(ualloc.array_gpuonly(BLOCK_SIZE)?, GFP_KERNEL)?; + } + core::mem::drop(ualloc); + + // Then actually commit them + inner.blocks.reserve(add_blocks, GFP_KERNEL)?; + + for (i, block) in new_blocks.into_iter().enumerate() { + let page_num = (block.gpu_va().get() >> PAGE_SHIFT) as u32; + + inner + .blocks + .push(block, GFP_KERNEL) + .expect("push() failed after reserve()"); + inner.info.block_list[2 * (cur_count + i)] = page_num; + for j in 0..PAGES_PER_BLOCK { + inner.info.page_list[(cur_count + i) * PAGES_PER_BLOCK + j] = page_num + j as u32; + } + } + + inner.info.block_ctl.with(|raw, _inner| { + raw.total.store(new_count as u32, Ordering::SeqCst); + raw.wptr.store(new_count as u32, Ordering::SeqCst); + }); + + /* Only do this update if the buffer manager is idle (which means we own it) */ + if inner.active_scenes == 0 { + let page_count = (new_count * PAGES_PER_BLOCK) as u32; + inner.info.with(|raw, _inner| { + raw.page_count.store(page_count, Ordering::Relaxed); + raw.block_count.store(new_count as u32, Ordering::Relaxed); + raw.last_page.store(page_count - 1, Ordering::Relaxed); + }); + } + + Ok(true) + } + + /// Create a new [`Scene::ver`] (render pass) using this buffer. + pub(crate) fn new_scene( + &self, + alloc: &mut gpu::KernelAllocators, + tile_info: &TileInfo, + ) -> Result { + let mut inner = self.inner.lock(); + + let tilemap_size = tile_info.tilemap_size; + let tpc_size = tile_info.tpc_size; + + // TODO: what is this exactly? + mod_pr_debug!("Buffer: Allocating TVB buffers\n"); + + // This seems to be a list, with 4x2 bytes of headers and 8 bytes per entry. + // On single-cluster devices, the used length always seems to be 1. + // On M1 Ultra, it can grow and usually doesn't exceed 64 entries. + // macOS allocates a whole 64K * 0x80 for this, so let's go with + // that to be safe... + let user_buffer = inner.ualloc.lock().array_empty_tagged( + if inner.num_clusters > 1 { + 0x10080 + } else { + 0x80 + }, + b"UBUF", + )?; + + let tvb_heapmeta = inner + .ualloc + .lock() + .array_empty_tagged(0x200 + tile_info.layermeta_size, b"HMTA")?; + let tvb_tilemap = inner + .ualloc + .lock() + .array_empty_tagged(tilemap_size, b"TMAP")?; + + mod_pr_debug!("Buffer: Allocating misc buffers\n"); + let preempt_buf = inner.ualloc.lock().array_empty_tagged( + inner.preempt1_size + inner.preempt2_size + inner.preempt3_size, + b"PRMT", + )?; + + let tpc = match inner.tpc.as_ref() { + Some(buf) if buf.len() >= tpc_size => buf.clone(), + _ => { + // MacOS allocates this as shared GPU+FW, but + // priv seems to work and might be faster? + // Needs to be FW-writable anyway, so ualloc + // won't work. + let buf = Arc::new( + inner.ualloc_priv.lock().array_empty_tagged( + (tpc_size + mmu::UAT_PGMSK) & !mmu::UAT_PGMSK, + b"TPC ", + )?, + GFP_KERNEL, + )?; + inner.tpc = Some(buf.clone()); + buf + } + }; + + let mut clmeta_size = 0; + let mut meta1_size = 0; + let mut meta2_size = 0; + let mut meta3_size = 0; + + let clustering = if inner.num_clusters > 1 { + let cfg = inner.cfg.clustering.as_ref().unwrap(); + + clmeta_size = tile_info.layermeta_size * cfg.max_splits; + // Maybe: (4x4 macro tiles + 1 global page)*n, 32bit each (17*4*n) + // Unused on t602x? + meta1_size = align(tile_info.meta1_blocks as usize * cfg.meta1_blocksize, 0x80); + meta2_size = align(cfg.meta2_size, 0x80); + meta3_size = align(cfg.meta3_size, 0x80); + let meta4_size = cfg.meta4_size; + + let meta_size = clmeta_size + meta1_size + meta2_size + meta3_size + meta4_size; + + mod_pr_debug!("Buffer: Allocating clustering buffers\n"); + let tilemaps = inner + .ualloc + .lock() + .array_empty_tagged(cfg.max_splits * tilemap_size, b"CTMP")?; + let meta = inner.ualloc.lock().array_empty_tagged(meta_size, b"CMTA")?; + Some(buffer::ClusterBuffers { tilemaps, meta }) + } else { + None + }; + + // Could be made strong, but we wind up with a deadlock if we try to grab the + // pointer through the inner.buffer path inside the closure. + let stats_pointer = inner.stats.weak_pointer(); + + let _gpu = &mut alloc.gpu; + + // macOS allocates this as private. However, the firmware does not + // DC CIVAC this before reading it (like it does most other things), + // which causes odd cache incoherency bugs when combined with + // speculation on the firmware side (maybe). This doesn't happen + // on macOS because these structs are a circular pool that is mapped + // already initialized. Just mark this shared for now. + let scene = alloc.shared.new_init( + try_init!(buffer::Scene::ver { + user_buffer: user_buffer, + buffer: self.clone(), + tvb_heapmeta: tvb_heapmeta, + tvb_tilemap: tvb_tilemap, + tpc: tpc, + clustering: clustering, + preempt_buf: preempt_buf, + #[ver(G >= G14X)] + control_word: _gpu.array_empty_tagged(1, b"CWRD")?, + }), + |inner, _p| { + try_init!(buffer::raw::Scene::ver { + #[ver(G >= G14X)] + control_word: inner.control_word.gpu_pointer(), + #[ver(G >= G14X)] + control_word2: inner.control_word.gpu_pointer(), + pass_page_count: AtomicU32::new(0), + unk_4: 0, + unk_8: U64(0), + unk_10: U64(0), + user_buffer: inner.user_buffer.gpu_pointer(), + unk_20: 0, + #[ver(V >= V13_3)] + unk_28: U64(0), + stats: stats_pointer, + total_page_count: AtomicU32::new(0), + #[ver(G < G14X)] + unk_30: U64(0), + #[ver(G < G14X)] + unk_38: U64(0), + }) + }, + )?; + + let mut rebind = false; + + if inner.active_slot.is_none() { + assert_eq!(inner.active_scenes, 0); + + let slot = inner.mgr.0.get_inner(inner.last_token, |inner, mgr| { + inner.owners[mgr.slot() as usize] = Some(self.clone()); + Ok(()) + })?; + rebind = slot.changed(); + + mod_pr_debug!("Buffer: assigning slot {} (rebind={})", slot.slot(), rebind); + + inner.last_token = Some(slot.token()); + inner.active_slot = Some(slot); + } + + inner.active_scenes += 1; + + Ok(Scene::ver { + object: scene, + slot: inner.active_slot.as_ref().unwrap().slot(), + rebind, + preempt2_off: inner.preempt1_size, + preempt3_off: inner.preempt1_size + inner.preempt2_size, + meta1_off: clmeta_size, + meta2_off: clmeta_size + meta1_size, + meta3_off: clmeta_size + meta1_size + meta2_size, + meta4_off: clmeta_size + meta1_size + meta2_size + meta3_size, + }) + } + + /// Increment the buffer manager usage count. Should we done once we know the Scene is ready + /// to be committed and used in commands submitted to the GPU. + pub(crate) fn increment(&self) { + let inner = self.inner.lock(); + inner.info.counter.with(|raw, _inner| { + // We could use fetch_add, but the non-LSE atomic + // sequence Rust produces confuses the hypervisor. + // We have inner locked anyway, so this is not racy. + let v = raw.count.load(Ordering::Relaxed); + raw.count.store(v + 1, Ordering::Relaxed); + }); + } + + pub(crate) fn any_ref(&self) -> Arc { + self.inner.clone() + } +} + +#[versions(AGX)] +impl Clone for Buffer::ver { + fn clone(&self) -> Self { + Buffer::ver { + inner: self.inner.clone(), + } + } +} + +#[versions(AGX)] +struct BufferSlotInner(); + +#[versions(AGX)] +impl slotalloc::SlotItem for BufferSlotInner::ver { + type Data = BufferManagerInner::ver; + + fn release(&mut self, data: &mut Self::Data, slot: u32) { + mod_pr_debug!("BufferManager: Released slot {}\n", slot); + data.owners[slot as usize] = None; + } +} + +/// Inner data for the buffer manager, to be protected by the SlotAllocator lock. +#[versions(AGX)] +pub(crate) struct BufferManagerInner { + owners: KVec>, +} + +/// The GPU-global buffer manager, used to allocate and release buffer slots from the pool. +#[versions(AGX)] +pub(crate) struct BufferManager(slotalloc::SlotAllocator); + +#[versions(AGX)] +impl BufferManager::ver { + pub(crate) fn new() -> Result { + let mut owners = KVec::new(); + for _i in 0..(NUM_BUFFERS as usize) { + owners.push(None, GFP_KERNEL)?; + } + Ok(BufferManager::ver(slotalloc::SlotAllocator::new( + NUM_BUFFERS, + BufferManagerInner::ver { owners }, + |_inner, _slot| Some(BufferSlotInner::ver()), + c_str!("BufferManager::SlotAllocator"), + static_lock_class!(), + static_lock_class!(), + )?)) + } + + /// Signals a Buffer to synchronously grow. + pub(crate) fn grow(&self, slot: u32) { + match self + .0 + .with_inner(|inner| inner.owners[slot as usize].as_ref().cloned()) + { + Some(owner) => { + pr_err!( + "BufferManager: Unexpected grow request for slot {}. This might deadlock. Please report this bug.\n", + slot + ); + owner.sync_grow(); + } + None => { + pr_err!( + "BufferManager: Received grow request for empty slot {}\n", + slot + ); + } + } + } +} + +#[versions(AGX)] +impl Clone for BufferManager::ver { + fn clone(&self) -> Self { + BufferManager::ver(self.0.clone()) + } +} diff --git a/drivers/gpu/drm/asahi/channel.rs b/drivers/gpu/drm/asahi/channel.rs new file mode 100644 index 00000000000000..30cc0efbf3ce22 --- /dev/null +++ b/drivers/gpu/drm/asahi/channel.rs @@ -0,0 +1,631 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! GPU ring buffer channels +//! +//! The GPU firmware use a set of ring buffer channels to receive commands from the driver and send +//! it notifications and status messages. +//! +//! These ring buffers mostly follow uniform conventions, so they share the same base +//! implementation. + +use crate::debug::*; +use crate::driver::{ + AsahiDevRef, + AsahiDevice, // +}; +use crate::fw::channels::*; +use crate::fw::initdata::{ + raw, + ChannelRing, // +}; +use crate::fw::types::*; +use crate::{ + buffer, + event, + gpu, + mem, // +}; +use kernel::{ + c_str, + prelude::*, + sync::Arc, + time::{ + delay::fsleep, + Delta, + Instant, + Monotonic, // + }, +}; + +pub(crate) use crate::fw::channels::PipeType; + +/// A receive (FW->driver) channel. +pub(crate) struct RxChannel +where + for<'a> ::Raw<'a>: Debug + Default + Zeroable, +{ + ring: ChannelRing, + // FIXME: needs feature(generic_const_exprs) + //rptr: [u32; T::SUB_CHANNELS], + rptr: [u32; 6], + count: u32, +} + +impl RxChannel +where + for<'a> ::Raw<'a>: Debug + Default + Zeroable, +{ + /// Allocates a new receive channel with a given message count. + pub(crate) fn new(alloc: &mut gpu::KernelAllocators, count: usize) -> Result> { + Ok(RxChannel { + ring: ChannelRing { + state: alloc.shared.new_default()?, + ring: alloc.shared.array_empty(T::SUB_CHANNELS * count)?, + }, + rptr: Default::default(), + count: count as u32, + }) + } + + /// Receives a message on the specified sub-channel index, optionally leaving in the ring + /// buffer. + /// + /// Returns None if the channel is empty. + fn get_or_peek(&mut self, index: usize, peek: bool) -> Option { + self.ring.state.with(|raw, _inner| { + let wptr = T::wptr(raw, index); + let rptr = &mut self.rptr[index]; + if wptr == *rptr { + None + } else { + let off = self.count as usize * index; + let msg = self.ring.ring[off + *rptr as usize]; + if !peek { + *rptr = (*rptr + 1) % self.count; + T::set_rptr(raw, index, *rptr); + } + Some(msg) + } + }) + } + + /// Receives a message on the specified sub-channel index, and dequeues it from the ring buffer. + /// + /// Returns None if the channel is empty. + pub(crate) fn get(&mut self, index: usize) -> Option { + self.get_or_peek(index, false) + } + + /// Peeks a message on the specified sub-channel index, leaving it in the ring buffer. + /// + /// Returns None if the channel is empty. + pub(crate) fn peek(&mut self, index: usize) -> Option { + self.get_or_peek(index, true) + } +} + +/// A transmit (driver->FW) channel. +pub(crate) struct TxChannel +where + for<'a> ::Raw<'a>: Debug + Default + Zeroable, +{ + ring: ChannelRing, + wptr: u32, + count: u32, +} + +impl TxChannel +where + for<'a> ::Raw<'a>: Debug + Default + Zeroable, +{ + /// Allocates a new cached transmit channel with a given message count. + pub(crate) fn new(alloc: &mut gpu::KernelAllocators, count: usize) -> Result> { + Ok(TxChannel { + ring: ChannelRing { + state: alloc.shared.new_default()?, + ring: alloc.private.array_empty(count)?, + }, + wptr: 0, + count: count as u32, + }) + } + + /// Allocates a new uncached transmit channel with a given message count. + pub(crate) fn new_uncached( + alloc: &mut gpu::KernelAllocators, + count: usize, + ) -> Result> { + Ok(TxChannel { + ring: ChannelRing { + state: alloc.shared.new_default()?, + ring: alloc.shared.array_empty(count)?, + }, + wptr: 0, + count: count as u32, + }) + } + + /// Send a message to the ring, returning a cookie with the ring buffer position. + /// + /// This will poll/block if the ring is full, which we don't really expect to happen. + pub(crate) fn put(&mut self, msg: &U) -> u32 { + self.ring.state.with(|raw, _inner| { + let next_wptr = (self.wptr + 1) % self.count; + let mut rptr = T::rptr(raw); + if next_wptr == rptr { + pr_err!( + "TX ring buffer is full! Waiting... ({}, {})\n", + next_wptr, + rptr + ); + // TODO: block properly on incoming messages? + while next_wptr == rptr { + fsleep(Delta::from_millis(8)); + rptr = T::rptr(raw); + } + } + self.ring.ring[self.wptr as usize] = *msg; + mem::sync(); + T::set_wptr(raw, next_wptr); + self.wptr = next_wptr; + }); + self.wptr + } + + /// Wait for a previously submitted message to be popped off of the ring by the GPU firmware. + /// + /// This busy-loops, and is intended to be used for rare cases when we need to block for + /// completion of a cache management or invalidation operation synchronously (which + /// the firmware normally completes fast enough not to be worth sleeping for). + /// If the poll takes longer than 10ms, this switches to sleeping between polls. + pub(crate) fn wait_for(&mut self, wptr: u32, timeout_ms: i64) -> Result { + const MAX_FAST_POLL: i64 = 10; + let start = Instant::::now(); + let timeout_ms = timeout_ms.max(1); + let timeout_fast = Delta::from_millis(timeout_ms.min(MAX_FAST_POLL)); + let timeout_slow = Delta::from_millis(timeout_ms); + self.ring.state.with(|raw, _inner| { + while start.elapsed() < timeout_fast { + if T::rptr(raw) == wptr { + return Ok(()); + } + mem::sync(); + } + while start.elapsed() < timeout_slow { + if T::rptr(raw) == wptr { + return Ok(()); + } + fsleep(Delta::from_millis(5)); + mem::sync(); + } + Err(ETIMEDOUT) + }) + } +} + +/// Device Control channel for global device management commands. +#[versions(AGX)] +pub(crate) struct DeviceControlChannel { + dev: AsahiDevRef, + ch: TxChannel, +} + +#[versions(AGX)] +impl DeviceControlChannel::ver { + const COMMAND_TIMEOUT_MS: i64 = 1000; + + /// Allocate a new Device Control channel. + pub(crate) fn new( + dev: &AsahiDevice, + alloc: &mut gpu::KernelAllocators, + ) -> Result { + Ok(DeviceControlChannel::ver { + dev: dev.into(), + ch: TxChannel::::new(alloc, 0x100)?, + }) + } + + /// Returns the raw `ChannelRing` structure to pass to firmware. + pub(crate) fn to_raw(&self) -> raw::ChannelRing { + self.ch.ring.to_raw() + } + + /// Submits a Device Control command. + pub(crate) fn send(&mut self, msg: &DeviceControlMsg::ver) -> u32 { + cls_dev_dbg!(DeviceControlCh, self.dev, "DeviceControl: {:?}\n", msg); + self.ch.put(msg) + } + + /// Waits for a previously submitted Device Control command to complete. + pub(crate) fn wait_for(&mut self, wptr: u32) -> Result { + self.ch.wait_for(wptr, Self::COMMAND_TIMEOUT_MS) + } +} + +/// Pipe channel to submit WorkQueue execution requests. +#[versions(AGX)] +pub(crate) struct PipeChannel { + dev: AsahiDevRef, + ch: TxChannel, +} + +#[versions(AGX)] +impl PipeChannel::ver { + /// Allocate a new Pipe submission channel. + pub(crate) fn new( + dev: &AsahiDevice, + alloc: &mut gpu::KernelAllocators, + ) -> Result { + Ok(PipeChannel::ver { + dev: dev.into(), + ch: TxChannel::::new(alloc, 0x100)?, + }) + } + + /// Returns the raw `ChannelRing` structure to pass to firmware. + pub(crate) fn to_raw(&self) -> raw::ChannelRing { + self.ch.ring.to_raw() + } + + /// Submits a Pipe kick command to the firmware. + pub(crate) fn send(&mut self, msg: &PipeMsg::ver) { + cls_dev_dbg!(PipeCh, self.dev, "Pipe: {:?}\n", msg); + self.ch.put(msg); + } +} + +/// Firmware Control channel, used for secure cache flush requests. +pub(crate) struct FwCtlChannel { + dev: AsahiDevRef, + ch: TxChannel, +} + +impl FwCtlChannel { + const COMMAND_TIMEOUT_MS: i64 = 1000; + + /// Allocate a new Firmware Control channel. + pub(crate) fn new( + dev: &AsahiDevice, + alloc: &mut gpu::KernelAllocators, + ) -> Result { + Ok(FwCtlChannel { + dev: dev.into(), + ch: TxChannel::::new_uncached(alloc, 0x100)?, + }) + } + + /// Returns the raw `ChannelRing` structure to pass to firmware. + pub(crate) fn to_raw(&self) -> raw::ChannelRing { + self.ch.ring.to_raw() + } + + /// Submits a Firmware Control command to the firmware. + pub(crate) fn send(&mut self, msg: &FwCtlMsg) -> u32 { + cls_dev_dbg!(FwCtlCh, self.dev, "FwCtl: {:?}\n", msg); + self.ch.put(msg) + } + + /// Waits for a previously submitted Firmware Control command to complete. + pub(crate) fn wait_for(&mut self, wptr: u32) -> Result { + self.ch.wait_for(wptr, Self::COMMAND_TIMEOUT_MS) + } +} + +/// Event channel, used to notify the driver of command completions, GPU faults and errors, and +/// other events. +#[versions(AGX)] +pub(crate) struct EventChannel { + dev: AsahiDevRef, + ch: RxChannel, + ev_mgr: Arc, + buf_mgr: buffer::BufferManager::ver, + gpu: Option>, +} + +#[versions(AGX)] +impl EventChannel::ver { + /// Allocate a new Event channel. + pub(crate) fn new( + dev: &AsahiDevice, + alloc: &mut gpu::KernelAllocators, + ev_mgr: Arc, + buf_mgr: buffer::BufferManager::ver, + ) -> Result { + Ok(EventChannel::ver { + dev: dev.into(), + ch: RxChannel::::new(alloc, 0x100)?, + ev_mgr, + buf_mgr, + gpu: None, + }) + } + + /// Registers the managing `Gpu` instance that will handle events on this channel. + pub(crate) fn set_manager(&mut self, gpu: Arc) { + self.gpu = Some(gpu); + } + + /// Returns the raw `ChannelRing` structure to pass to firmware. + pub(crate) fn to_raw(&self) -> raw::ChannelRing { + self.ch.ring.to_raw() + } + + /// Polls for new Event messages on this ring. + pub(crate) fn poll(&mut self) { + while let Some(msg) = self.ch.get(0) { + // SAFETY: The raw view is always valid for all bit patterns. + let tag = unsafe { msg.raw.0 }; + match tag { + 0..=EVENT_MAX => { + // SAFETY: Since we have checked the tag to be in range, + // accessing the enum view is valid. + let msg = unsafe { msg.msg }; + + cls_dev_dbg!(EventCh, self.dev, "Event: {:?}\n", msg); + match msg { + EventMsg::Fault => match self.gpu.as_ref() { + Some(gpu) => gpu.handle_fault(), + None => { + dev_crit!( + self.dev.as_ref(), + "EventChannel: No GPU manager available!\n" + ) + } + }, + EventMsg::Timeout { + counter, + unk_8, + event_slot, + } => match self.gpu.as_ref() { + Some(gpu) => gpu.handle_timeout(counter, event_slot, unk_8), + None => { + dev_crit!( + self.dev.as_ref(), + "EventChannel: No GPU manager available!\n" + ) + } + }, + EventMsg::Flag { firing, .. } => { + for (i, flags) in firing.iter().enumerate() { + for j in 0..32 { + if flags & (1u32 << j) != 0 { + self.ev_mgr.signal((i * 32 + j) as u32); + } + } + } + } + EventMsg::GrowTVB { + vm_slot, + buffer_slot, + counter, + } => match self.gpu.as_ref() { + Some(gpu) => { + self.buf_mgr.grow(buffer_slot); + gpu.ack_grow(buffer_slot, vm_slot, counter); + } + None => { + dev_crit!( + self.dev.as_ref(), + "EventChannel: No GPU manager available!\n" + ) + } + }, + EventMsg::ChannelError { + error_type, + pipe_type, + event_slot, + event_value, + } => match self.gpu.as_ref() { + Some(gpu) => { + let error_type = match error_type { + 0 => ChannelErrorType::MemoryError, + 1 => ChannelErrorType::DMKill, + 2 => ChannelErrorType::Aborted, + 3 => ChannelErrorType::Unk3, + a => ChannelErrorType::Unknown(a), + }; + gpu.handle_channel_error( + error_type, + pipe_type, + event_slot, + event_value, + ); + } + None => { + dev_crit!( + self.dev.as_ref(), + "EventChannel: No GPU manager available!\n" + ) + } + }, + msg => { + dev_crit!(self.dev.as_ref(), "Unknown event message: {:?}\n", msg); + } + } + } + _ => { + // SAFETY: The raw view is always valid for all bit patterns. + dev_warn!(self.dev.as_ref(), "Unknown event message: {:?}\n", unsafe { + msg.raw + }); + } + } + } + } +} + +/// Firmware Log channel. This one is pretty special, since it has 6 sub-channels (for different log +/// levels), and it also uses a side buffer to actually hold the log messages, only passing around +/// pointers in the main buffer. +pub(crate) struct FwLogChannel { + dev: AsahiDevRef, + ch: RxChannel, + payload_buf: GpuArray, +} + +impl FwLogChannel { + const RING_SIZE: usize = 0x100; + const BUF_SIZE: usize = 0x100; + + /// Allocate a new Firmware Log channel. + pub(crate) fn new( + dev: &AsahiDevice, + alloc: &mut gpu::KernelAllocators, + ) -> Result { + Ok(FwLogChannel { + dev: dev.into(), + ch: RxChannel::::new(alloc, Self::RING_SIZE)?, + payload_buf: alloc + .shared + .array_empty(Self::BUF_SIZE * FwLogChannelState::SUB_CHANNELS)?, + }) + } + + /// Returns the raw `ChannelRing` structure to pass to firmware. + pub(crate) fn to_raw(&self) -> raw::ChannelRing { + self.ch.ring.to_raw() + } + + /// Returns the GPU pointers to the firmware log payload buffer. + pub(crate) fn get_buf(&self) -> GpuWeakPointer<[RawFwLogPayloadMsg]> { + self.payload_buf.weak_pointer() + } + + /// Polls for new log messages on all sub-rings. + pub(crate) fn poll(&mut self) { + for i in 0..=FwLogChannelState::SUB_CHANNELS - 1 { + while let Some(msg) = self.ch.peek(i) { + cls_dev_dbg!(FwLogCh, self.dev, "FwLog{}: {:?}\n", i, msg); + if msg.msg_type != 2 { + dev_warn!(self.dev.as_ref(), "Unknown FWLog{} message: {:?}\n", i, msg); + self.ch.get(i); + continue; + } + if msg.msg_index.0 as usize >= Self::BUF_SIZE { + dev_warn!( + self.dev.as_ref(), + "FWLog{} message index out of bounds: {:?}\n", + i, + msg + ); + self.ch.get(i); + continue; + } + let index = Self::BUF_SIZE * i + msg.msg_index.0 as usize; + let payload = &self.payload_buf.as_slice()[index]; + if payload.msg_type != 3 { + dev_warn!( + self.dev.as_ref(), + "Unknown FWLog{} payload: {:?}\n", + i, + payload + ); + self.ch.get(i); + continue; + } + let msg = if let Some(end) = payload.msg.iter().position(|&r| r == 0) { + CStr::from_bytes_with_nul(&(*payload.msg)[..end + 1]) + .unwrap_or(c_str!("cstr_err")) + } else { + dev_warn!( + self.dev.as_ref(), + "FWLog{} payload not NUL-terminated: {:?}\n", + i, + payload + ); + self.ch.get(i); + continue; + }; + match i { + 0 => dev_dbg!(self.dev.as_ref(), "FWLog: {}\n", msg), + 1 => dev_info!(self.dev.as_ref(), "FWLog: {}\n", msg), + 2 => dev_notice!(self.dev.as_ref(), "FWLog: {}\n", msg), + 3 => dev_warn!(self.dev.as_ref(), "FWLog: {}\n", msg), + 4 => dev_err!(self.dev.as_ref(), "FWLog: {}\n", msg), + 5 => dev_crit!(self.dev.as_ref(), "FWLog: {}\n", msg), + _ => (), + }; + self.ch.get(i); + } + } + } +} + +pub(crate) struct KTraceChannel { + dev: AsahiDevRef, + ch: RxChannel, +} + +/// KTrace channel, used to receive detailed execution trace markers from the firmware. +/// We currently disable this in initdata, so no messages are expected here at this time. +impl KTraceChannel { + /// Allocate a new KTrace channel. + pub(crate) fn new( + dev: &AsahiDevice, + alloc: &mut gpu::KernelAllocators, + ) -> Result { + Ok(KTraceChannel { + dev: dev.into(), + ch: RxChannel::::new(alloc, 0x200)?, + }) + } + + /// Returns the raw `ChannelRing` structure to pass to firmware. + pub(crate) fn to_raw(&self) -> raw::ChannelRing { + self.ch.ring.to_raw() + } + + /// Polls for new KTrace messages on this ring. + pub(crate) fn poll(&mut self) { + while let Some(msg) = self.ch.get(0) { + cls_dev_dbg!(KTraceCh, self.dev, "KTrace: {:?}\n", msg); + } + } +} + +/// Statistics channel, reporting power-related statistics to the driver. +/// Not really implemented other than debug logs yet... +#[versions(AGX)] +pub(crate) struct StatsChannel { + dev: AsahiDevRef, + ch: RxChannel, +} + +#[versions(AGX)] +impl StatsChannel::ver { + /// Allocate a new Statistics channel. + pub(crate) fn new( + dev: &AsahiDevice, + alloc: &mut gpu::KernelAllocators, + ) -> Result { + Ok(StatsChannel::ver { + dev: dev.into(), + ch: RxChannel::::new(alloc, 0x100)?, + }) + } + + /// Returns the raw `ChannelRing` structure to pass to firmware. + pub(crate) fn to_raw(&self) -> raw::ChannelRing { + self.ch.ring.to_raw() + } + + /// Polls for new statistics messages on this ring. + pub(crate) fn poll(&mut self) { + while let Some(msg) = self.ch.get(0) { + // SAFETY: The raw view is always valid for all bit patterns. + let tag = unsafe { msg.raw.0 }; + match tag { + 0..=STATS_MAX::ver => { + // SAFETY: Since we have checked the tag to be in range, + // accessing the enum view is valid. + let msg = unsafe { msg.msg }; + cls_dev_dbg!(StatsCh, self.dev, "Stats: {:?}\n", msg); + } + _ => { + // SAFETY: The raw view is always valid for all bit patterns. + pr_warn!("Unknown stats message: {:?}\n", unsafe { msg.raw }); + } + } + } + } +} diff --git a/drivers/gpu/drm/asahi/crashdump.rs b/drivers/gpu/drm/asahi/crashdump.rs new file mode 100644 index 00000000000000..bd9f2f1649584f --- /dev/null +++ b/drivers/gpu/drm/asahi/crashdump.rs @@ -0,0 +1,305 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! GPU crash dump formatter +//! +//! Takes a raw dump of firmware/kernel mapped pages from `pgtable` and formats it into +//! an ELF core dump suitable for dumping into userspace. + +use core::mem::size_of; + +use kernel::{ + devcoredump::DevCoreDump, + error::Result, + page::{Page, PAGE_MASK, PAGE_SHIFT, PAGE_SIZE}, + prelude::*, + types::Owned, + uapi, +}; + +use crate::hw; +use crate::pgtable::{self, DumpedPage, Prot, UAT_PGSZ}; +use crate::util::align; + +pub(crate) struct CrashDump { + headers: KVVec, + pages: KVVec>, +} + +const NOTE_NAME_AGX: &str = "AGX"; +const NOTE_AGX_DUMP_INFO: u32 = 1; + +const NOTE_NAME_RTKIT: &str = "RTKIT"; +const NOTE_RTKIT_CRASHLOG: u32 = 1; + +#[repr(C)] +pub(crate) struct AGXDumpInfo { + initdata_address: u64, + chip_id: u32, + gpu_gen: hw::GpuGen, + gpu_variant: hw::GpuVariant, + gpu_rev: hw::GpuRevision, + total_active_cores: u32, + firmware_version: [u32; 6], +} + +struct ELFNote { + name: &'static str, + ty: u32, + data: KVVec, +} + +pub(crate) struct CrashDumpBuilder { + page_dump: KVVec, + notes: KVec, +} + +/// Helper to convert ELF headers into byte slices +/// TODO: Hook this up into kernel::AsBytes somehow +/// +/// # Safety +/// +/// Types implementing this trait must have no padding bytes. +unsafe trait AsBytes: Sized { + fn as_bytes(&self) -> &[u8] { + // SAFETY: This trait is only implemented for types with no padding bytes + unsafe { core::slice::from_raw_parts(self as *const _ as *const u8, size_of::()) } + } + fn slice_as_bytes(slice: &[Self]) -> &[u8] { + // SAFETY: This trait is only implemented for types with no padding bytes + unsafe { + core::slice::from_raw_parts(slice.as_ptr() as *const u8, core::mem::size_of_val(slice)) + } + } +} + +// SAFETY: This type has no padding +unsafe impl AsBytes for uapi::Elf64_Ehdr {} +// SAFETY: This type has no padding +unsafe impl AsBytes for uapi::Elf64_Phdr {} +// SAFETY: This type has no padding +unsafe impl AsBytes for uapi::Elf64_Nhdr {} +// SAFETY: This type has no padding +unsafe impl AsBytes for AGXDumpInfo {} + +const FIRMWARE_ENTRYPOINT: u64 = 0xFFFFFF8000000000u64; + +impl CrashDumpBuilder { + pub(crate) fn new(page_dump: KVVec) -> Result { + Ok(CrashDumpBuilder { + page_dump, + notes: KVec::new(), + }) + } + + pub(crate) fn add_agx_info( + &mut self, + cfg: &hw::HwConfig, + dyncfg: &hw::DynConfig, + initdata_address: u64, + ) -> Result { + let mut info = AGXDumpInfo { + chip_id: cfg.chip_id, + gpu_gen: dyncfg.id.gpu_gen, + gpu_variant: dyncfg.id.gpu_variant, + gpu_rev: dyncfg.id.gpu_rev, + total_active_cores: dyncfg.id.total_active_cores, + firmware_version: [0; 6], + initdata_address, + }; + info.firmware_version[..dyncfg.firmware_version.len().min(6)] + .copy_from_slice(&dyncfg.firmware_version); + + let mut data = KVVec::new(); + data.extend_from_slice(info.as_bytes(), GFP_KERNEL)?; + + self.notes.push( + ELFNote { + name: NOTE_NAME_AGX, + ty: NOTE_AGX_DUMP_INFO, + data, + }, + GFP_KERNEL, + )?; + Ok(()) + } + + pub(crate) fn add_crashlog(&mut self, crashlog: &[u8]) -> Result { + let mut data = KVVec::new(); + data.extend_from_slice(crashlog, GFP_KERNEL)?; + + self.notes.push( + ELFNote { + name: NOTE_NAME_RTKIT, + ty: NOTE_RTKIT_CRASHLOG, + data, + }, + GFP_KERNEL, + )?; + + Ok(()) + } + + pub(crate) fn finalize(self) -> Result { + let CrashDumpBuilder { page_dump, notes } = self; + + let mut ehdr: uapi::Elf64_Ehdr = Default::default(); + + ehdr.e_ident[uapi::EI_MAG0 as usize..=uapi::EI_MAG3 as usize].copy_from_slice(b"\x7fELF"); + ehdr.e_ident[uapi::EI_CLASS as usize] = uapi::ELFCLASS64 as u8; + ehdr.e_ident[uapi::EI_DATA as usize] = uapi::ELFDATA2LSB as u8; + ehdr.e_ident[uapi::EI_VERSION as usize] = uapi::EV_CURRENT as u8; + ehdr.e_type = uapi::ET_CORE as u16; + ehdr.e_machine = uapi::EM_AARCH64 as u16; + ehdr.e_version = uapi::EV_CURRENT; + ehdr.e_entry = FIRMWARE_ENTRYPOINT; + ehdr.e_ehsize = core::mem::size_of::() as u16; + ehdr.e_phentsize = core::mem::size_of::() as u16; + + let phdr_offset = core::mem::size_of::(); + + // PHDRs come after the ELF header + ehdr.e_phoff = phdr_offset as u64; + + let mut phdrs = KVVec::new(); + + // First PHDR is the NOTE section + phdrs.push( + uapi::Elf64_Phdr { + p_type: uapi::PT_NOTE, + p_flags: uapi::PF_R, + p_align: 1, + ..Default::default() + }, + GFP_KERNEL, + )?; + + // Generate the page phdrs. The offset will be fixed up later. + let mut off: usize = 0; + let mut next = None; + let mut pages: KVVec> = KVVec::new(); + + for mut page in page_dump { + let vaddr = page.iova; + let paddr = page.pte & pgtable::PTE_ADDR_BITS; + let flags = Prot::from_pte(page.pte).elf_flags(); + let valid = page.data.is_some(); + let cur = (vaddr, paddr, flags, valid); + if Some(cur) != next { + phdrs.push( + uapi::Elf64_Phdr { + p_type: uapi::PT_LOAD, + p_offset: if valid { off as u64 } else { 0 }, + p_vaddr: vaddr, + p_paddr: paddr, + p_filesz: if valid { UAT_PGSZ as u64 } else { 0 }, + p_memsz: UAT_PGSZ as u64, + p_flags: flags, + p_align: UAT_PGSZ as u64, + }, + GFP_KERNEL, + )?; + if valid { + off += UAT_PGSZ; + } + } else { + let ph = phdrs.last_mut().unwrap(); + ph.p_memsz += UAT_PGSZ as u64; + if valid { + ph.p_filesz += UAT_PGSZ as u64; + off += UAT_PGSZ; + } + } + if let Some(data_page) = page.data.take() { + pages.push(data_page, GFP_KERNEL)?; + } + next = Some(( + vaddr + UAT_PGSZ as u64, + paddr + UAT_PGSZ as u64, + flags, + valid, + )); + } + + ehdr.e_phnum = phdrs.len() as u16; + + let note_offset = phdr_offset + size_of::() * phdrs.len(); + + let mut note_data: KVVec = KVVec::new(); + + for note in notes { + let hdr = uapi::Elf64_Nhdr { + n_namesz: note.name.len() as u32 + 1, + n_descsz: note.data.len() as u32, + n_type: note.ty, + }; + note_data.extend_from_slice(hdr.as_bytes(), GFP_KERNEL)?; + note_data.extend_from_slice(note.name.as_bytes(), GFP_KERNEL)?; + note_data.push(0, GFP_KERNEL)?; + while note_data.len() & 3 != 0 { + note_data.push(0, GFP_KERNEL)?; + } + note_data.extend_from_slice(¬e.data, GFP_KERNEL)?; + while note_data.len() & 3 != 0 { + note_data.push(0, GFP_KERNEL)?; + } + } + + // NOTE section comes after the PHDRs + phdrs[0].p_offset = note_offset as u64; + phdrs[0].p_filesz = note_data.len() as u64; + + // Align data section to the page size + let data_offset = align(note_offset + note_data.len(), UAT_PGSZ); + + // Fix up data PHDR offsets + for phdr in &mut phdrs[1..] { + phdr.p_offset += data_offset as u64; + } + + // Build ELF header buffer + let mut headers: KVVec = KVVec::from_elem(0, data_offset, GFP_KERNEL)?; + + headers[0..size_of::()].copy_from_slice(ehdr.as_bytes()); + headers[phdr_offset..phdr_offset + phdrs.len() * size_of::()] + .copy_from_slice(AsBytes::slice_as_bytes(&phdrs)); + headers[note_offset..note_offset + note_data.len()].copy_from_slice(¬e_data); + + Ok(CrashDump { headers, pages }) + } +} + +impl DevCoreDump for CrashDump { + fn read(&self, buf: &mut [u8], mut offset: usize) -> Result { + let mut read = 0; + let mut left = buf.len(); + if offset < self.headers.len() { + let block = left.min(self.headers.len() - offset); + buf[..block].copy_from_slice(&self.headers[offset..offset + block]); + read += block; + offset += block; + left -= block; + } + if left == 0 { + return Ok(read); + } + offset -= self.headers.len(); // Offset from the page area + + while left > 0 { + let page_index = offset >> PAGE_SHIFT; + let page_offset = offset & !PAGE_MASK; + let block = left.min(PAGE_SIZE - page_offset); + let Some(page) = self.pages.get(page_index) else { + break; + }; + let slice = &mut buf[read..read + block]; + // SAFETY: We own the page, and the slice guarantees the + // dst length is sufficient. + unsafe { page.read_raw(slice.as_mut_ptr(), page_offset, slice.len())? }; + read += block; + offset += block; + left -= block; + } + + Ok(read) + } +} diff --git a/drivers/gpu/drm/asahi/debug.rs b/drivers/gpu/drm/asahi/debug.rs new file mode 100644 index 00000000000000..50628ade5ab8e4 --- /dev/null +++ b/drivers/gpu/drm/asahi/debug.rs @@ -0,0 +1,142 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#![allow(dead_code)] + +//! Debug enable/disable flags and convenience macros + +#[allow(unused_imports)] +pub(crate) use super::{ + cls_dev_dbg, + cls_pr_debug, + debug, + mod_dev_dbg, + mod_pr_debug, // +}; +use crate::module_parameters; +use core::sync::atomic::{ + AtomicU64, + Ordering, // +}; + +static DEBUG_FLAGS: AtomicU64 = AtomicU64::new(0); + +/// Debug flag bit indices +pub(crate) enum DebugFlags { + // 0-4: Memory-related debug + Mmu = 0, + PgTable = 1, + Alloc = 2, + Gem = 3, + Object = 4, + + // 5-7: Firmware objects and resources + Event = 5, + Buffer = 6, + WorkQueue = 7, + + // 8-13: DRM interface, rendering, compute, GPU globals + Gpu = 8, + File = 9, + Queue = 10, + Render = 11, + Compute = 12, + Errors = 13, + + // 14-15: Misc stats + MemStats = 14, + TVBStats = 15, + + // 16-22: Channels + FwLogCh = 16, + KTraceCh = 17, + StatsCh = 18, + EventCh = 19, + PipeCh = 20, + DeviceControlCh = 21, + FwCtlCh = 22, + + // 32-35: Allocator debugging + FillAllocations = 32, + DebugAllocations = 33, + DetectOverflows = 34, + ForceCPUMaps = 35, + + // 36-: Behavior flags + ConservativeTlbi = 36, + KeepGpuPowered = 37, + WaitForPowerOff = 38, + NoGpuRecovery = 39, + DisableClustering = 40, + + // 48-: Misc + Debug0 = 48, + Debug1 = 49, + Debug2 = 50, + Debug3 = 51, + Debug4 = 52, + Debug5 = 53, + Debug6 = 54, + Debug7 = 55, + + VerboseFaults = 61, + AllowUnknownOverrides = 62, + OopsOnGpuCrash = 63, +} + +/// Update the cached global debug flags from the module parameter +pub(crate) fn update_debug_flags() { + let flags = *module_parameters::debug_flags.value(); + + DEBUG_FLAGS.store(flags, Ordering::Relaxed); +} + +/// Check whether debug is enabled for a given flag +#[inline(always)] +pub(crate) fn debug_enabled(flag: DebugFlags) -> bool { + DEBUG_FLAGS.load(Ordering::Relaxed) & 1 << (flag as usize) != 0 +} + +/// Run some code only if debug is enabled for the calling module +#[macro_export] +macro_rules! debug { + ($($arg:tt)*) => { + if $crate::debug::debug_enabled(DEBUG_CLASS) { + $($arg)* + } + }; +} + +/// pr_info!() if debug is enabled for the calling module +#[macro_export] +macro_rules! mod_pr_debug ( + ($($arg:tt)*) => ( + $crate::debug! { ::kernel::pr_info! ( $($arg)* ); } + ) +); + +/// dev_info!() if debug is enabled for the calling module +#[macro_export] +macro_rules! mod_dev_dbg ( + ($dev:expr, $($arg:tt)*) => ( + $crate::debug! { ::kernel::dev_info! ( $dev.as_ref(), $($arg)* ); } + ) +); + +/// pr_info!() if debug is enabled for a specific module +#[macro_export] +macro_rules! cls_pr_debug ( + ($cls:ident, $($arg:tt)*) => ( + if $crate::debug::debug_enabled($crate::debug::DebugFlags::$cls) { + ::kernel::pr_info! ( $($arg)* ); + } + ) +); + +/// dev_info!() if debug is enabled for a specific module +#[macro_export] +macro_rules! cls_dev_dbg ( + ($cls:ident, $dev:expr, $($arg:tt)*) => ( + if $crate::debug::debug_enabled($crate::debug::DebugFlags::$cls) { + ::kernel::dev_info! ( $dev.as_ref(), $($arg)* ); + } + ) +); diff --git a/drivers/gpu/drm/asahi/driver.rs b/drivers/gpu/drm/asahi/driver.rs new file mode 100644 index 00000000000000..03245a4826be71 --- /dev/null +++ b/drivers/gpu/drm/asahi/driver.rs @@ -0,0 +1,235 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Top-level GPU driver implementation. + +use kernel::{ + c_str, + device::Core, + dma::{ + Device, + DmaMask, // + }, + drm, + drm::ioctl, + of, + platform, + prelude::*, + sync::Arc, // +}; + +use crate::{ + debug, + file, + gem::AsahiObject, + gpu, + hw, + regs, // +}; + +use kernel::macros::vtable; +use kernel::types::ARef; + +/// Holds a reference to the top-level `GpuManager` object. +#[pin_data] +pub(crate) struct AsahiData { + #[pin] + pub(crate) gpu: Arc, + pub(crate) pdev: ARef, + pub(crate) resources: regs::Resources, +} + +unsafe impl Send for AsahiData {} +unsafe impl Sync for AsahiData {} + +pub(crate) struct AsahiDriver { + #[expect(unused)] + drm: ARef>, +} + +unsafe impl Send for AsahiDriver {} +unsafe impl Sync for AsahiDriver {} + +/// Convenience type alias for the DRM device type for this driver. +pub(crate) type AsahiDevice = drm::device::Device; +pub(crate) type AsahiDevRef = ARef; + +/// DRM Driver metadata +const INFO: drm::driver::DriverInfo = drm::driver::DriverInfo { + major: 0, + minor: 0, + patchlevel: 0, + name: c_str!("asahi"), + desc: c_str!("Apple AGX Graphics"), +}; + +/// DRM Driver implementation for `AsahiDriver`. +#[vtable] +impl drm::driver::Driver for AsahiDriver { + /// Our `DeviceData` type, reference-counted + type Data = AsahiData; + /// Our `File` type. + type File = file::File; + /// Our `Object` type. + type Object = drm::gem::shmem::Object; + + const INFO: drm::driver::DriverInfo = INFO; + const FEATURES: u32 = drm::driver::FEAT_GEM + | drm::driver::FEAT_RENDER + | drm::driver::FEAT_SYNCOBJ + | drm::driver::FEAT_SYNCOBJ_TIMELINE + | drm::driver::FEAT_GEM_GPUVA; + + kernel::declare_drm_ioctls! { + (ASAHI_GET_PARAMS, drm_asahi_get_params, + ioctl::RENDER_ALLOW, crate::file::File::get_params), + (ASAHI_GET_TIME, drm_asahi_get_time, + ioctl::AUTH | ioctl::RENDER_ALLOW, crate::file::File::get_time), + (ASAHI_VM_CREATE, drm_asahi_vm_create, + ioctl::AUTH | ioctl::RENDER_ALLOW, crate::file::File::vm_create), + (ASAHI_VM_DESTROY, drm_asahi_vm_destroy, + ioctl::AUTH | ioctl::RENDER_ALLOW, crate::file::File::vm_destroy), + (ASAHI_VM_BIND, drm_asahi_vm_bind, + ioctl::AUTH | ioctl::RENDER_ALLOW, crate::file::File::vm_bind), + (ASAHI_GEM_CREATE, drm_asahi_gem_create, + ioctl::AUTH | ioctl::RENDER_ALLOW, crate::file::File::gem_create), + (ASAHI_GEM_MMAP_OFFSET, drm_asahi_gem_mmap_offset, + ioctl::AUTH | ioctl::RENDER_ALLOW, crate::file::File::gem_mmap_offset), + (ASAHI_GEM_BIND_OBJECT, drm_asahi_gem_bind_object, + ioctl::AUTH | ioctl::RENDER_ALLOW, crate::file::File::gem_bind_object), + (ASAHI_QUEUE_CREATE, drm_asahi_queue_create, + ioctl::AUTH | ioctl::RENDER_ALLOW, crate::file::File::queue_create), + (ASAHI_QUEUE_DESTROY, drm_asahi_queue_destroy, + ioctl::AUTH | ioctl::RENDER_ALLOW, crate::file::File::queue_destroy), + (ASAHI_SUBMIT, drm_asahi_submit, + ioctl::AUTH | ioctl::RENDER_ALLOW, crate::file::File::submit), + } +} + +// OF Device ID table.s +kernel::of_device_table!( + OF_TABLE, + MODULE_OF_TABLE, + ::IdInfo, + [ + ( + of::DeviceId::new(c_str!("apple,agx-t8103")), + &hw::t8103::HWCONFIG + ), + ( + of::DeviceId::new(c_str!("apple,agx-t8112")), + &hw::t8112::HWCONFIG + ), + ( + of::DeviceId::new(c_str!("apple,agx-t6000")), + &hw::t600x::HWCONFIG_T6000 + ), + ( + of::DeviceId::new(c_str!("apple,agx-t6001")), + &hw::t600x::HWCONFIG_T6001 + ), + ( + of::DeviceId::new(c_str!("apple,agx-t6002")), + &hw::t600x::HWCONFIG_T6002 + ), + ( + of::DeviceId::new(c_str!("apple,agx-t6020")), + &hw::t602x::HWCONFIG_T6020 + ), + ( + of::DeviceId::new(c_str!("apple,agx-t6021")), + &hw::t602x::HWCONFIG_T6021 + ), + ( + of::DeviceId::new(c_str!("apple,agx-t6022")), + &hw::t602x::HWCONFIG_T6022 + ), + ] +); + +/// Platform Driver implementation for `AsahiDriver`. +impl platform::Driver for AsahiDriver { + type IdInfo = &'static hw::HwConfig; + const OF_ID_TABLE: Option> = Some(&OF_TABLE); + + /// Device probe function. + fn probe( + pdev: &platform::Device, + info: Option<&Self::IdInfo>, + ) -> impl PinInit { + debug::update_debug_flags(); + + dev_info!(pdev.as_ref(), "Probing...\n"); + + let cfg = info.ok_or(ENODEV)?; + + unsafe { pdev.dma_set_mask_and_coherent(DmaMask::try_new(cfg.uat_oas)?)? }; + + let res = regs::Resources::new(pdev)?; + + // Initialize misc MMIO + res.init_mmio()?; + + // Start the coprocessor CPU, so UAT can initialize the handoff + regs::Resources::start_cpu(pdev)?; + + let fwnode = pdev.as_ref().fwnode().ok_or(EIO)?; + let compat: KVec = fwnode + .property_read_array_vec(c_str!("apple,firmware-compat"), 3)? + .required_by(pdev.as_ref())?; + + // TODO: This is very temporary + // SAFETY: This should be safe as data is not touched by the driver + // untill it gets fully initialised. + // Additionally drm::device::Device::release() will not drop data and + // leaks instead. + let uninit = unsafe { + pin_init::pin_init_from_closure::(|_slot| Ok(())) + }; + let drm: ARef = drm::device::Device::new(pdev.as_ref(), uninit)?; + + let gpu = match (cfg.gpu_gen, cfg.gpu_variant, compat.as_slice()) { + (hw::GpuGen::G13, _, &[12, 3, 0]) => { + gpu::GpuManagerG13V12_3::new(&drm.clone(), &res, cfg)? as Arc + } + (hw::GpuGen::G14, hw::GpuVariant::G, &[12, 4, 0]) => { + gpu::GpuManagerG14V12_4::new(&drm.clone(), &res, cfg)? as Arc + } + (hw::GpuGen::G13, _, &[13, 5, 0]) => { + gpu::GpuManagerG13V13_5::new(&drm.clone(), &res, cfg)? as Arc + } + (hw::GpuGen::G14, hw::GpuVariant::G, &[13, 5, 0]) => { + gpu::GpuManagerG14V13_5::new(&drm.clone(), &res, cfg)? as Arc + } + (hw::GpuGen::G14, _, &[13, 5, 0]) => { + gpu::GpuManagerG14XV13_5::new(&drm.clone(), &res, cfg)? as Arc + } + _ => { + dev_info!( + pdev.as_ref(), + "Unsupported GPU/firmware combination ({:?}, {:?}, {:?})\n", + cfg.gpu_gen, + cfg.gpu_variant, + compat + ); + return Err(ENODEV); + } + }; + + let data = try_pin_init!(AsahiData { + gpu, + pdev: pdev.into(), + resources: res, + }); + + let ptr: *const AsahiData = &raw const **drm; + unsafe { + data.__pinned_init(ptr as *mut AsahiData)?; + } + + (*drm).gpu.init()?; + + drm::driver::Registration::new_foreign_owned(&drm, pdev.as_ref(), 0)?; + + Ok(Self { drm }) + } +} diff --git a/drivers/gpu/drm/asahi/event.rs b/drivers/gpu/drm/asahi/event.rs new file mode 100644 index 00000000000000..edd7d701e665cd --- /dev/null +++ b/drivers/gpu/drm/asahi/event.rs @@ -0,0 +1,257 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! GPU event manager +//! +//! The GPU firmware manages work completion by using event objects (Apple calls them "stamps"), +//! which are monotonically incrementing counters. There are a fixed number of objects, and +//! they are managed with a `SlotAllocator`. +//! +//! This module manages the set of available events and lets users compute expected values. +//! It also manages signaling owners when the GPU firmware reports that an event fired. + +use crate::debug::*; +use crate::fw::types::*; +use crate::{ + gpu, + slotalloc, + workqueue, // +}; +use core::cmp; +use core::sync::atomic::Ordering; +use kernel::prelude::*; +use kernel::sync::Arc; +use kernel::{ + c_str, + static_lock_class, // +}; + +const DEBUG_CLASS: DebugFlags = DebugFlags::Event; + +/// Number of events managed by the firmware. +const NUM_EVENTS: u32 = 128; + +/// Inner data associated with a given event slot. +pub(crate) struct EventInner { + /// CPU pointer to the driver notification event stamp + stamp: *const AtomicU32, + /// GPU pointer to the driver notification event stamp + gpu_stamp: GpuWeakPointer, + /// GPU pointer to the firmware-internal event stamp + gpu_fw_stamp: GpuWeakPointer, +} + +/// SAFETY: The event slots are safe to send across threads. +unsafe impl Send for EventInner {} + +/// Alias for an event token, which allows requesting the same event. +pub(crate) type Token = slotalloc::SlotToken; +/// Alias for an allocated `Event` that has a slot. +pub(crate) type Event = slotalloc::Guard; + +/// Represents a given stamp value for an event. +#[derive(Eq, PartialEq, Copy, Clone, Debug)] +#[repr(transparent)] +pub(crate) struct EventValue(u32); + +impl EventValue { + /// Returns the `EventValue` that succeeds this one. + pub(crate) fn next(&self) -> EventValue { + EventValue(self.0.wrapping_add(0x100)) + } + + /// Increments this `EventValue` in place. + pub(crate) fn increment(&mut self) { + self.0 = self.0.wrapping_add(0x100); + } + + /* Not used + /// Increments this `EventValue` in place by a certain count. + pub(crate) fn add(&mut self, val: u32) { + self.0 = self + .0 + .wrapping_add(val.checked_mul(0x100).expect("Adding too many events")); + } + */ + + /// Increments this `EventValue` in place by a certain count. + pub(crate) fn sub(&mut self, val: u32) { + self.0 = self + .0 + .wrapping_sub(val.checked_mul(0x100).expect("Subtracting too many events")); + } + + /// Computes the delta between this event and another event. + pub(crate) fn delta(&self, other: &EventValue) -> i32 { + (self.0.wrapping_sub(other.0) as i32) >> 8 + } +} + +impl PartialOrd for EventValue { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for EventValue { + fn cmp(&self, other: &Self) -> cmp::Ordering { + self.delta(other).cmp(&0) + } +} + +impl EventInner { + /// Returns the GPU pointer to the driver notification stamp + pub(crate) fn stamp_pointer(&self) -> GpuWeakPointer { + self.gpu_stamp + } + + /// Returns the GPU pointer to the firmware internal stamp + pub(crate) fn fw_stamp_pointer(&self) -> GpuWeakPointer { + self.gpu_fw_stamp + } + + /// Fetches the current event value from shared memory + pub(crate) fn current(&self) -> EventValue { + // SAFETY: The pointer is always valid as constructed in + // EventManager below, and outside users cannot construct + // new EventInners, nor move or copy them, and Guards as + // returned by the SlotAllocator hold a reference to the + // SlotAllocator containing the EventManagerInner, which + // keeps the GpuObject the stamp is contained within alive. + EventValue(unsafe { &*self.stamp }.load(Ordering::Acquire)) + } +} + +impl slotalloc::SlotItem for EventInner { + type Data = EventManagerInner; + + fn release(&mut self, data: &mut Self::Data, slot: u32) { + mod_pr_debug!("EventManager: Released slot {}\n", slot); + data.owners[slot as usize] = None; + } +} + +/// Inner data for the event manager, to be protected by the SlotAllocator lock. +pub(crate) struct EventManagerInner { + stamps: GpuArray, + fw_stamps: GpuArray, + // Note: Use dyn to avoid having to version this entire module. + owners: KVec>>, +} + +/// Top-level EventManager object. +pub(crate) struct EventManager { + alloc: slotalloc::SlotAllocator, +} + +impl EventManager { + /// Create a new EventManager. + #[inline(never)] + pub(crate) fn new(alloc: &mut gpu::KernelAllocators) -> Result { + let mut owners = KVec::new(); + for _i in 0..(NUM_EVENTS as usize) { + owners.push(None, GFP_KERNEL)?; + } + let inner = EventManagerInner { + stamps: alloc.shared.array_empty(NUM_EVENTS as usize)?, + fw_stamps: alloc.private.array_empty(NUM_EVENTS as usize)?, + owners, + }; + + for slot in 0..NUM_EVENTS { + inner.stamps[slot as usize] + .0 + .store(slot << 24, Ordering::Relaxed); + } + + Ok(EventManager { + alloc: slotalloc::SlotAllocator::new( + NUM_EVENTS, + inner, + |inner: &mut EventManagerInner, slot| { + Some(EventInner { + stamp: &inner.stamps[slot as usize].0, + gpu_stamp: inner.stamps.weak_item_pointer(slot as usize), + gpu_fw_stamp: inner.fw_stamps.weak_item_pointer(slot as usize), + }) + }, + c_str!("EventManager::SlotAllocator"), + static_lock_class!(), + static_lock_class!(), + )?, + }) + } + + /// Gets a free `Event`, optionally trying to reuse the last one allocated by this caller. + pub(crate) fn get( + &self, + token: Option, + owner: Arc, + ) -> Result { + let ev = self.alloc.get_inner(token, |inner, ev| { + mod_pr_debug!( + "EventManager: Registered owner {:p} on slot {}\n", + &*owner, + ev.slot() + ); + inner.owners[ev.slot() as usize] = Some(owner); + Ok(()) + })?; + Ok(ev) + } + + /// Signals an event by slot, indicating completion (of one or more commands). + pub(crate) fn signal(&self, slot: u32) { + match self + .alloc + .with_inner(|inner| inner.owners[slot as usize].as_ref().cloned()) + { + Some(owner) => { + owner.signal(); + } + None => { + mod_pr_debug!("EventManager: Received event for empty slot {}\n", slot); + } + } + } + + /// Marks the owner of an event as having lost its work due to a GPU error. + pub(crate) fn mark_error(&self, slot: u32, wait_value: u32, error: workqueue::WorkError) { + match self + .alloc + .with_inner(|inner| inner.owners[slot as usize].as_ref().cloned()) + { + Some(owner) => { + owner.mark_error(EventValue(wait_value), error); + } + None => { + pr_err!("Received error for empty slot {}\n", slot); + } + } + } + + /// Returns a reference to the workqueue owning an event. + pub(crate) fn get_owner( + &self, + slot: u32, + ) -> Option> { + self.alloc + .with_inner(|inner| inner.owners[slot as usize].as_ref().cloned()) + } + + /// Fail all commands, used when the GPU crashes. + pub(crate) fn fail_all(&self, error: workqueue::WorkError) { + let mut owners: KVec> = KVec::new(); + + self.alloc.with_inner(|inner| { + for wq in inner.owners.iter().filter_map(|o| o.as_ref()).cloned() { + if owners.push(wq, GFP_KERNEL).is_err() { + pr_err!("Failed to signal failure to WorkQueue\n"); + } + } + }); + + for wq in owners { + wq.fail_all(error); + } + } +} diff --git a/drivers/gpu/drm/asahi/file.rs b/drivers/gpu/drm/asahi/file.rs new file mode 100644 index 00000000000000..02a4ce874a92c8 --- /dev/null +++ b/drivers/gpu/drm/asahi/file.rs @@ -0,0 +1,1092 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#![allow(clippy::unusual_byte_groupings)] + +//! File implementation, which represents a single DRM client. +//! +//! This is in charge of managing the resources associated with one GPU client, including an +//! arbitrary number of submission queues and Vm objects, and reporting hardware/driver +//! information to userspace and accepting submissions. + +use crate::debug::*; +use crate::driver::AsahiDevice; +use crate::{ + alloc, + buffer, + driver, + gem, + mmu, + module_parameters, + queue, + util::{ + align, + align_down, + gcd, + AnyBitPattern, + RangeExt, + Reader, // + }, // +}; +use core::mem::MaybeUninit; +use core::ops::Deref; +use core::ops::Range; +use core::ptr::addr_of_mut; +use kernel::bindings; +use kernel::dma_fence::RawDmaFence; +use kernel::drm::gem::BaseObject; +use kernel::error::code::*; +use kernel::new_mutex; +use kernel::prelude::*; +use kernel::sync::{ + Arc, + Mutex, // +}; +use kernel::time::NSEC_PER_SEC; +use kernel::uaccess::{ + UserPtr, + UserSlice, // +}; +use kernel::{ + dma_fence, + drm, + uapi, + xarray, // +}; + +const DEBUG_CLASS: DebugFlags = DebugFlags::File; + +pub(crate) const MAX_COMMANDS_PER_SUBMISSION: u32 = 64; + +/// A client instance of an `mmu::Vm` address space. +struct Vm { + ualloc: Arc>, + ualloc_priv: Arc>, + vm: mmu::Vm, + kernel_range: Range, + _dummy_mapping: mmu::KernelMapping, +} + +impl Drop for Vm { + fn drop(&mut self) { + // When the user Vm is dropped, unmap everything in the user range + let left_range = VM_USER_RANGE.start..self.kernel_range.start; + let right_range = self.kernel_range.end..VM_USER_RANGE.end; + + if !left_range.is_empty() + && self + .vm + .unmap_range(left_range.start, left_range.range()) + .is_err() + { + pr_err!("Vm::Drop: vm.unmap_range() failed\n"); + } + if !right_range.is_empty() + && self + .vm + .unmap_range(right_range.start, right_range.range()) + .is_err() + { + pr_err!("Vm::Drop: vm.unmap_range() failed\n"); + } + } +} + +/// Sync object from userspace. +pub(crate) struct SyncItem { + pub(crate) syncobj: drm::syncobj::SyncObj, + pub(crate) fence: Option, + pub(crate) chain_fence: Option, + pub(crate) timeline_value: u64, +} + +impl SyncItem { + fn parse_one(file: &DrmFile, data: uapi::drm_asahi_sync, out: bool) -> Result { + match data.sync_type { + uapi::drm_asahi_sync_type_DRM_ASAHI_SYNC_SYNCOBJ => { + if data.timeline_value != 0 { + cls_pr_debug!(Errors, "Non-timeline sync object with a nonzero value\n"); + return Err(EINVAL); + } + let syncobj = drm::syncobj::SyncObj::lookup_handle(file, data.handle)?; + + Ok(SyncItem { + fence: if out { + None + } else { + Some(syncobj.fence_get().ok_or_else(|| { + cls_pr_debug!(Errors, "Failed to get fence from sync object\n"); + EINVAL + })?) + }, + syncobj, + chain_fence: None, + timeline_value: data.timeline_value, + }) + } + uapi::drm_asahi_sync_type_DRM_ASAHI_SYNC_TIMELINE_SYNCOBJ => { + let syncobj = drm::syncobj::SyncObj::lookup_handle(file, data.handle)?; + let fence = if out { + None + } else { + syncobj + .fence_get() + .ok_or_else(|| { + cls_pr_debug!( + Errors, + "Failed to get fence from timeline sync object\n" + ); + EINVAL + })? + .chain_find_seqno(data.timeline_value)? + }; + + Ok(SyncItem { + fence, + syncobj, + chain_fence: if out { + Some(dma_fence::FenceChain::new()?) + } else { + None + }, + timeline_value: data.timeline_value, + }) + } + _ => { + cls_pr_debug!(Errors, "Invalid sync type {}\n", data.sync_type); + Err(EINVAL) + } + } + } + + fn parse_array( + file: &DrmFile, + ptr: u64, + in_count: u32, + out_count: u32, + ) -> Result> { + let count = in_count + out_count; + let mut vec = KVec::with_capacity(count as usize, GFP_KERNEL)?; + + const STRIDE: usize = core::mem::size_of::(); + let size = STRIDE * count as usize; + + // SAFETY: We only read this once, so there are no TOCTOU issues. + let mut reader = UserSlice::new(UserPtr::from_addr(ptr as _), size).reader(); + + for i in 0..count { + let mut sync: MaybeUninit = MaybeUninit::uninit(); + + // SAFETY: The size of `sync` is STRIDE + reader.read_raw(unsafe { + core::slice::from_raw_parts_mut(sync.as_mut_ptr() as *mut MaybeUninit, STRIDE) + })?; + + // SAFETY: All bit patterns in the struct are valid + let sync = unsafe { sync.assume_init() }; + + vec.push(SyncItem::parse_one(file, sync, i >= in_count)?, GFP_KERNEL)?; + } + + Ok(vec) + } +} + +#[derive(Clone)] +pub(crate) enum Object { + TimestampBuffer(Arc), +} + +/// State associated with a client. +// #[pin_data] +pub(crate) struct File { + id: u64, + // #[pin] + vms: xarray::XArray>, + // #[pin] + queues: xarray::XArray>>>, + // #[pin] + objects: xarray::XArray>, +} + +/// Convenience type alias for our DRM `File` type. +pub(crate) type DrmFile = drm::File; + +/// Available VM range for the user +const VM_USER_RANGE: Range = mmu::IOVA_USER_USABLE_RANGE; + +/// Minimum reserved AS for kernel mappings +const VM_KERNEL_MIN_SIZE: u64 = 0x20000000; + +impl drm::file::DriverFile for File { + type Driver = driver::AsahiDriver; + + /// Create a new `File` instance for a fresh client. + fn open(device: &AsahiDevice) -> Result>> { + debug::update_debug_flags(); + + let gpu = &device.gpu; + let id = gpu.ids().file.next(); + + mod_dev_dbg!(device, "[File {}]: DRM device opened\n", id); + Ok(KBox::pin_init(File::new(id), GFP_KERNEL)?) + } + + fn as_raw(&self) -> *mut bindings::drm_file { + todo!() + } +} + +// SAFETY: All bit patterns are valid by construction. +unsafe impl AnyBitPattern for uapi::drm_asahi_gem_bind_op {} + +impl File { + fn new(id: u64) -> impl PinInit { + unsafe { + pin_init::pin_init_from_closure(move |slot: *mut Self| { + let raw_vms = addr_of_mut!((*slot).vms); + xarray::XArray::>::new(xarray::AllocKind::Alloc1) + .__pinned_init(raw_vms)?; + + let raw_queues = addr_of_mut!((*slot).queues); + xarray::XArray::>>>::new( + xarray::AllocKind::Alloc1, + ) + .__pinned_init(raw_queues)?; + + let raw_objects = addr_of_mut!((*slot).objects); + xarray::XArray::>::new(xarray::AllocKind::Alloc1) + .__pinned_init(raw_objects)?; + + (*slot).id = id; + Ok(()) + }) + } + } + + fn vms(self: Pin<&Self>) -> Pin<&xarray::XArray>> { + // SAFETY: Structural pinned projection for vms. + // We never move out of this field. + unsafe { self.map_unchecked(|s| &s.vms) } + } + + #[allow(clippy::type_complexity)] + fn queues(self: Pin<&Self>) -> Pin<&xarray::XArray>>>> { + // SAFETY: Structural pinned projection for queues. + // We never move out of this field. + unsafe { self.map_unchecked(|s| &s.queues) } + } + + fn objects(self: Pin<&Self>) -> Pin<&xarray::XArray>> { + // SAFETY: Structural pinned projection for objects. + // We never move out of this field. + unsafe { self.map_unchecked(|s| &s.objects) } + } + + /// IOCTL: get_param: Get a driver parameter value. + pub(crate) fn get_params( + device: &AsahiDevice, + data: &uapi::drm_asahi_get_params, + file: &DrmFile, + ) -> Result { + mod_dev_dbg!(device, "[File {}]: IOCTL: get_params\n", file.inner().id); + + let gpu = &device.gpu; + + if data.param_group != 0 || data.pad != 0 { + cls_pr_debug!(Errors, "get_params: Invalid arguments\n"); + return Err(EINVAL); + } + + if gpu.is_crashed() { + return Err(ENODEV); + } + + let mut params = uapi::drm_asahi_params_global { + features: 0, + + gpu_generation: gpu.get_dyncfg().id.gpu_gen as u32, + gpu_variant: gpu.get_dyncfg().id.gpu_variant as u32, + gpu_revision: gpu.get_dyncfg().id.gpu_rev as u32, + chip_id: gpu.get_cfg().chip_id, + + num_dies: gpu.get_cfg().num_dies, + num_clusters_total: gpu.get_dyncfg().id.num_clusters, + num_cores_per_cluster: gpu.get_dyncfg().id.num_cores, + core_masks: [0; uapi::DRM_ASAHI_MAX_CLUSTERS as usize], + + vm_start: VM_USER_RANGE.start, + vm_end: VM_USER_RANGE.end, + vm_kernel_min_size: VM_KERNEL_MIN_SIZE, + + max_commands_per_submission: MAX_COMMANDS_PER_SUBMISSION, + max_attachments: crate::microseq::MAX_ATTACHMENTS as u32, + max_frequency_khz: gpu.get_dyncfg().pwr.max_frequency_khz(), + + command_timestamp_frequency_hz: 1_000_000_000, // User timestamps always in nanoseconds + }; + + for (i, mask) in gpu.get_dyncfg().id.core_masks.iter().enumerate() { + *(params.core_masks.get_mut(i).ok_or(EIO)?) = (*mask).into(); + } + + if *module_parameters::fault_control.value() == 0xb { + params.features |= uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_SOFT_FAULTS as u64; + } + + let size = core::mem::size_of::().min(data.size.try_into()?); + + // SAFETY: We only write to this userptr once, so there are no TOCTOU issues. + let mut params_writer = + UserSlice::new(UserPtr::from_addr(data.pointer as _), size).writer(); + + // SAFETY: `size` is at most the sizeof of `params` + params_writer.write_slice(unsafe { + core::slice::from_raw_parts(¶ms as *const _ as *const u8, size) + })?; + + Ok(0) + } + + /// IOCTL: vm_create: Create a new `Vm`. + pub(crate) fn vm_create( + device: &AsahiDevice, + data: &mut uapi::drm_asahi_vm_create, + file: &DrmFile, + ) -> Result { + let kernel_range = data.kernel_start..data.kernel_end; + + // Validate requested kernel range + if !VM_USER_RANGE.is_superset(kernel_range.clone()) + || kernel_range.range() < VM_KERNEL_MIN_SIZE + || kernel_range.start & (mmu::UAT_PGMSK as u64) != 0 + || kernel_range.end & (mmu::UAT_PGMSK as u64) != 0 + { + cls_pr_debug!(Errors, "vm_create: Invalid kernel range\n"); + return Err(EINVAL); + } + + // Align to buffer::PAGE_SIZE so the allocators are happy + let kernel_range = align(kernel_range.start, buffer::PAGE_SIZE as u64) + ..align_down(kernel_range.end, buffer::PAGE_SIZE as u64); + + let kernel_half_size = align_down(kernel_range.range() >> 1, buffer::PAGE_SIZE as u64); + let kernel_gpu_range = kernel_range.start..(kernel_range.start + kernel_half_size); + let kernel_gpufw_range = kernel_gpu_range.end..kernel_range.end; + + let gpu = &device.gpu; + let file_id = file.inner().id; + let vm = gpu.new_vm(kernel_range.clone())?; + + let vm_xa = file.inner().vms(); + let resv = vm_xa.lock().reserve_limit(1..=u32::MAX, GFP_KERNEL)?; + let id: u32 = resv.index().try_into()?; + + mod_dev_dbg!(device, "[File {} VM {}]: VM Create\n", file_id, id); + mod_dev_dbg!( + device, + "[File {} VM {}]: Creating allocators\n", + file_id, + id + ); + let ualloc = Arc::pin_init( + new_mutex!(alloc::DefaultAllocator::new( + device, + &vm, + kernel_gpu_range, + buffer::PAGE_SIZE, + mmu::PROT_GPU_SHARED_RW, + 512 * 1024, + true, + fmt!("File {} VM {} GPU Shared", file_id, id), + false, + )?), + GFP_KERNEL, + )?; + let ualloc_priv = Arc::pin_init( + new_mutex!(alloc::DefaultAllocator::new( + device, + &vm, + kernel_gpufw_range, + buffer::PAGE_SIZE, + mmu::PROT_GPU_FW_PRIV_RW, + 64 * 1024, + true, + fmt!("File {} VM {} GPU FW Private", file_id, id), + false, + )?), + GFP_KERNEL, + )?; + + mod_dev_dbg!( + device, + "[File {} VM {}]: Creating dummy object\n", + file_id, + id + ); + let mut dummy_obj = gem::new_kernel_object(device, 0x4000)?; + dummy_obj.vmap()?.memset(0); + let dummy_mapping = + dummy_obj.map_at(&vm, mmu::IOVA_UNK_PAGE, mmu::PROT_GPU_SHARED_RW, true)?; + + mod_dev_dbg!(device, "[File {} VM {}]: VM created\n", file_id, id); + resv.fill(KBox::new( + Vm { + ualloc, + ualloc_priv, + vm, + kernel_range, + _dummy_mapping: dummy_mapping, + }, + GFP_KERNEL, + )?)?; + + data.vm_id = id; + + Ok(0) + } + + /// IOCTL: vm_destroy: Destroy a `Vm`. + pub(crate) fn vm_destroy( + _device: &AsahiDevice, + data: &mut uapi::drm_asahi_vm_destroy, + file: &DrmFile, + ) -> Result { + let vm = file.inner().vms().remove(data.vm_id as usize); + if vm.is_none() { + Err(ENOENT) + } else { + Ok(0) + } + } + + /// IOCTL: gem_create: Create a new GEM object. + pub(crate) fn gem_create( + device: &AsahiDevice, + data: &mut uapi::drm_asahi_gem_create, + file: &DrmFile, + ) -> Result { + mod_dev_dbg!( + device, + "[File {}]: IOCTL: gem_create size={:#x?}\n", + file.inner().id, + data.size + ); + + if (data.flags + & !(uapi::drm_asahi_gem_flags_DRM_ASAHI_GEM_WRITEBACK + | uapi::drm_asahi_gem_flags_DRM_ASAHI_GEM_VM_PRIVATE)) + != 0 + || (data.flags & uapi::drm_asahi_gem_flags_DRM_ASAHI_GEM_VM_PRIVATE == 0 + && data.vm_id != 0) + { + cls_pr_debug!(Errors, "gem_create: Invalid arguments\n"); + return Err(EINVAL); + } + + let resv_gem; + let resv_obj = if data.flags & uapi::drm_asahi_gem_flags_DRM_ASAHI_GEM_VM_PRIVATE != 0 { + resv_gem = file + .inner() + .vms() + .lock() + .get(data.vm_id.try_into()?) + .ok_or(ENOENT)? + .vm + .get_resv_obj(); + Some(resv_gem.deref()) + } else { + None + }; + + let gem = gem::new_object(device, data.size.try_into()?, data.flags, resv_obj)?; + + let handle = gem.create_handle(file)?; + data.handle = handle; + + mod_dev_dbg!( + device, + "[File {}]: IOCTL: gem_create size={:#x} handle={:#x?}\n", + file.inner().id, + data.size, + data.handle + ); + + Ok(0) + } + + /// IOCTL: gem_mmap_offset: Assign an mmap offset to a GEM object. + pub(crate) fn gem_mmap_offset( + device: &AsahiDevice, + data: &mut uapi::drm_asahi_gem_mmap_offset, + file: &DrmFile, + ) -> Result { + mod_dev_dbg!( + device, + "[File {}]: IOCTL: gem_mmap_offset handle={:#x?}\n", + file.inner().id, + data.handle + ); + + if data.flags != 0 { + cls_pr_debug!(Errors, "gem_mmap_offset: Unexpected flags\n"); + return Err(EINVAL); + } + + let gem = gem::Object::lookup_handle(file, data.handle)?; + data.offset = gem.create_mmap_offset()?; + Ok(0) + } + + /// IOCTL: vm_bind: Map or unmap memory into a Vm. + pub(crate) fn vm_bind( + device: &AsahiDevice, + data: &uapi::drm_asahi_vm_bind, + file: &DrmFile, + ) -> Result { + mod_dev_dbg!( + device, + "[File {} VM {}]: IOCTL: vm_bind\n", + file.inner().id, + data.vm_id, + ); + + if data.stride == 0 || data.pad != 0 { + cls_pr_debug!(Errors, "vm_bind: Unexpected headers\n"); + return Err(EINVAL); + } + + let vm_id = data.vm_id.try_into()?; + + let mut vec = KVec::new(); + let size = (data.stride * data.num_binds) as usize; + let reader = UserSlice::new(UserPtr::from_addr(data.userptr as _), size).reader(); + reader.read_all(&mut vec, GFP_KERNEL)?; + let mut reader = Reader::new(&vec); + + for _i in 0..data.num_binds { + let bind: uapi::drm_asahi_gem_bind_op = reader.read_up_to(data.stride as usize)?; + Self::do_gem_bind_unbind(vm_id, &bind, file)?; + } + + Ok(0) + } + + pub(crate) fn do_gem_bind_unbind( + vm_id: usize, + data: &uapi::drm_asahi_gem_bind_op, + file: &DrmFile, + ) -> Result { + if (data.flags & uapi::drm_asahi_bind_flags_DRM_ASAHI_BIND_UNBIND) != 0 { + Self::do_gem_unbind(vm_id, data, file) + } else { + Self::do_gem_bind(vm_id, data, file) + } + } + + pub(crate) fn do_gem_bind( + vm_id: usize, + data: &uapi::drm_asahi_gem_bind_op, + file: &DrmFile, + ) -> Result { + if (data.addr | data.range | data.offset) as usize & mmu::UAT_PGMSK != 0 { + cls_pr_debug!( + Errors, + "gem_bind: Addr/range/offset not page aligned: {:#x} {:#x}\n", + data.addr, + data.range + ); + return Err(EINVAL); // Must be page aligned + } + + if (data.flags + & !(uapi::drm_asahi_bind_flags_DRM_ASAHI_BIND_READ + | uapi::drm_asahi_bind_flags_DRM_ASAHI_BIND_WRITE + | uapi::drm_asahi_bind_flags_DRM_ASAHI_BIND_SINGLE_PAGE)) + != 0 + { + cls_pr_debug!(Errors, "gem_bind: Invalid flags {:#x}\n", data.flags); + return Err(EINVAL); + } + + let single_page = data.flags & uapi::drm_asahi_bind_flags_DRM_ASAHI_BIND_SINGLE_PAGE != 0; + + let bo = gem::Object::lookup_handle(file, data.handle)?; + + let start = data.addr; + let end = data.addr.checked_add(data.range).ok_or(EINVAL)?; + let range = start..end; + + let bo_accessed_size = if single_page { + mmu::UAT_PGMSK as u64 + } else { + data.range + }; + let end_off = data.offset.checked_add(bo_accessed_size).ok_or(EINVAL)?; + if end_off as usize > bo.size() { + return Err(EINVAL); + } + + if !VM_USER_RANGE.is_superset(range.clone()) { + cls_pr_debug!( + Errors, + "gem_bind: Invalid map range {:#x}..{:#x} (not contained in user range)\n", + start, + end + ); + return Err(EINVAL); // Invalid map range + } + + let prot = if data.flags & uapi::drm_asahi_bind_flags_DRM_ASAHI_BIND_READ != 0 { + if data.flags & uapi::drm_asahi_bind_flags_DRM_ASAHI_BIND_WRITE != 0 { + mmu::PROT_GPU_SHARED_RW + } else { + mmu::PROT_GPU_SHARED_RO + } + } else if data.flags & uapi::drm_asahi_bind_flags_DRM_ASAHI_BIND_WRITE != 0 { + mmu::PROT_GPU_SHARED_WO + } else { + cls_pr_debug!( + Errors, + "gem_bind: Must specify read or write (flags: {:#x})\n", + data.flags + ); + return Err(EINVAL); // Must specify one of DRM_ASAHI_BIND_{READ,WRITE} + }; + + let vms_xa = file.inner().vms(); + let guard = vms_xa.lock(); + let guarded_vm = guard.get(vm_id).ok_or(ENOENT)?; + + // Clone it immediately so we aren't holding the XArray lock + let vm = guarded_vm.vm.clone(); + let kernel_range = guarded_vm.kernel_range.clone(); + let _ = guarded_vm; + core::mem::drop(guard); + + if kernel_range.overlaps(range) { + cls_pr_debug!( + Errors, + "gem_bind: Invalid map range {:#x}..{:#x} (intrudes in kernel range)\n", + start, + end + ); + return Err(EINVAL); + } + + vm.bind_object(&bo, data.addr, data.range, data.offset, prot, single_page)?; + + vm.bo_deferred_cleanup(); + + Ok(0) + } + + pub(crate) fn do_gem_unbind( + vm_id: usize, + data: &uapi::drm_asahi_gem_bind_op, + file: &DrmFile, + ) -> Result { + if data.offset != 0 + || data.flags != uapi::drm_asahi_bind_flags_DRM_ASAHI_BIND_UNBIND + || data.handle != 0 + { + cls_pr_debug!(Errors, "gem_unbind: offset/flags/handle not zero\n"); + return Err(EINVAL); + } + + if (data.addr | data.range) as usize & mmu::UAT_PGMSK != 0 { + cls_pr_debug!( + Errors, + "gem_bind: Addr/range/offset not page aligned: {:#x} {:#x}\n", + data.addr, + data.range + ); + return Err(EINVAL); // Must be page aligned + } + + let start = data.addr; + let end = data.addr.checked_add(data.range).ok_or(EINVAL)?; + let range = start..end; + + if !VM_USER_RANGE.is_superset(range.clone()) { + cls_pr_debug!( + Errors, + "gem_bind: Invalid unmap range {:#x}..{:#x} (not contained in user range)\n", + start, + end + ); + return Err(EINVAL); // Invalid map range + } + + let vms_xa = file.inner().vms(); + let guard = vms_xa.lock(); + let guarded_vm = guard.get(vm_id).ok_or(ENOENT)?; + + // Clone it immediately so we aren't holding the XArray lock + let vm = guarded_vm.vm.clone(); + let kernel_range = guarded_vm.kernel_range.clone(); + let _ = guarded_vm; + core::mem::drop(guard); + + if kernel_range.overlaps(range.clone()) { + cls_pr_debug!( + Errors, + "gem_bind: Invalid unmap range {:#x}..{:#x} (intrudes in kernel range)\n", + start, + end + ); + return Err(EINVAL); + } + + vm.unmap_range(range.start, range.range())?; + + vm.bo_deferred_cleanup(); + + Ok(0) + } + + pub(crate) fn unbind_gem_object(file: &DrmFile, bo: &gem::Object) -> Result { + // TODO: use iter() + let mut index = 0; + loop { + let vms = file.inner().vms(); + let item = vms.find(index, usize::MAX); + match item { + Some((idx, file_vm)) => { + // Clone since we can't hold the xarray spinlock while + // calling drop_mappings() + let vm = file_vm.borrow().vm.clone(); + core::mem::drop(file_vm); + vm.drop_mappings(bo)?; + if idx == usize::MAX { + break; + } + index = idx + 1; + } + None => break, + } + } + Ok(()) + } + + /// IOCTL: gem_bind_object: Map or unmap a GEM object as a special object. + pub(crate) fn gem_bind_object( + device: &AsahiDevice, + data: &mut uapi::drm_asahi_gem_bind_object, + file: &DrmFile, + ) -> Result { + mod_dev_dbg!( + device, + "[File {} VM {}]: IOCTL: gem_bind_object op={:?} handle={:#x?} flags={:#x?} {:#x?}:{:#x?} object_handle={:#x?}\n", + file.inner().id, + data.vm_id, + data.op, + data.handle, + data.flags, + data.offset, + data.range, + data.object_handle + ); + + if data.pad != 0 { + cls_pr_debug!(Errors, "gem_bind_object: Unexpected pad\n"); + return Err(EINVAL); + } + + if data.vm_id != 0 { + cls_pr_debug!(Errors, "gem_bind_object: Unexpected vm_id\n"); + return Err(EINVAL); + } + + match data.op { + uapi::drm_asahi_bind_object_op_DRM_ASAHI_BIND_OBJECT_OP_BIND => { + Self::do_gem_bind_object(device, data, file) + } + uapi::drm_asahi_bind_object_op_DRM_ASAHI_BIND_OBJECT_OP_UNBIND => { + Self::do_gem_unbind_object(device, data, file) + } + _ => { + cls_pr_debug!(Errors, "gem_bind_object: Invalid op {}\n", data.op); + Err(EINVAL) + } + } + } + + pub(crate) fn do_gem_bind_object( + device: &AsahiDevice, + data: &mut uapi::drm_asahi_gem_bind_object, + file: &DrmFile, + ) -> Result { + if (data.range | data.offset) as usize & mmu::UAT_PGMSK != 0 { + cls_pr_debug!( + Errors, + "gem_bind_object: Range/offset not page aligned: {:#x} {:#x}\n", + data.range, + data.offset + ); + return Err(EINVAL); // Must be page aligned + } + + if data.flags != uapi::drm_asahi_bind_object_flags_DRM_ASAHI_BIND_OBJECT_USAGE_TIMESTAMPS { + cls_pr_debug!(Errors, "gem_bind_object: Invalid flags {:#x}\n", data.flags); + return Err(EINVAL); + } + + let offset = data.offset.try_into()?; + let end_offset = data + .offset + .checked_add(data.range) + .ok_or(EINVAL)? + .try_into()?; + let bo = gem::ObjectRef::new(gem::Object::lookup_handle(file, data.handle)?); + + let mapping = Arc::new( + device.gpu.map_timestamp_buffer(bo, offset..end_offset)?, + GFP_KERNEL, + )?; + let obj = KBox::new(Object::TimestampBuffer(mapping), GFP_KERNEL)?; + let handle = file + .inner() + .objects() + .lock() + .insert_limit(1..=u32::MAX, obj, GFP_KERNEL)? as u64; + + data.object_handle = handle as u32; + Ok(0) + } + + pub(crate) fn do_gem_unbind_object( + _device: &AsahiDevice, + data: &mut uapi::drm_asahi_gem_bind_object, + file: &DrmFile, + ) -> Result { + if data.range != 0 || data.offset != 0 { + cls_pr_debug!( + Errors, + "gem_unbind_object: Range/offset not zero: {:#x} {:#x}\n", + data.range, + data.offset + ); + return Err(EINVAL); + } + + if data.flags != 0 { + cls_pr_debug!( + Errors, + "gem_unbind_object: Invalid flags {:#x}\n", + data.flags + ); + return Err(EINVAL); + } + + if data.handle != 0 { + cls_pr_debug!( + Errors, + "gem_unbind_object: Invalid handle {}\n", + data.handle + ); + return Err(EINVAL); + } + + let object = file.inner().objects().remove(data.object_handle as usize); + if object.is_none() { + Err(ENOENT) + } else { + Ok(0) + } + } + + /// IOCTL: queue_create: Create a new command submission queue of a given type. + pub(crate) fn queue_create( + device: &AsahiDevice, + data: &mut uapi::drm_asahi_queue_create, + file: &DrmFile, + ) -> Result { + let file_id = file.inner().id; + + mod_dev_dbg!( + device, + "[File {} VM {}]: Creating queue prio={:?} flags={:#x?}\n", + file_id, + data.vm_id, + data.priority, + data.flags, + ); + + if data.flags != 0 || data.priority > uapi::drm_asahi_priority_DRM_ASAHI_PRIORITY_REALTIME { + cls_pr_debug!(Errors, "queue_create: Invalid arguments\n"); + return Err(EINVAL); + } + + // TODO: Allow with CAP_SYS_NICE + if data.priority >= uapi::drm_asahi_priority_DRM_ASAHI_PRIORITY_HIGH { + cls_pr_debug!(Errors, "queue_create: Invalid priority\n"); + return Err(EINVAL); + } + + let queues_xa = file.inner().queues(); + let resv = queues_xa.lock().reserve_limit(1..=u32::MAX, GFP_KERNEL)?; + let vms_xa = file.inner().vms(); + let guard = vms_xa.lock(); + let file_vm = guard.get(data.vm_id.try_into()?).ok_or(ENOENT)?; + let vm = file_vm.vm.clone(); + let ualloc = file_vm.ualloc.clone(); + let ualloc_priv = file_vm.ualloc_priv.clone(); + // Drop the vms lock eagerly + let _ = file_vm; + core::mem::drop(guard); + + let queue = device.gpu.new_queue( + vm, + ualloc, + ualloc_priv, + // TODO: Plumb deeper the enum + uapi::drm_asahi_priority_DRM_ASAHI_PRIORITY_REALTIME - data.priority, + data.usc_exec_base, + )?; + + data.queue_id = resv.index().try_into()?; + resv.fill(Arc::pin_init(new_mutex!(queue), GFP_KERNEL)?)?; + + Ok(0) + } + + /// IOCTL: queue_destroy: Destroy a command submission queue. + pub(crate) fn queue_destroy( + _device: &AsahiDevice, + data: &mut uapi::drm_asahi_queue_destroy, + file: &DrmFile, + ) -> Result { + // grab the queue so the xarray spinlock is dropped first + let queue = file.inner().queues().remove(data.queue_id as usize); + if queue.is_none() { + Err(ENOENT) + } else { + Ok(0) + } + } + + /// IOCTL: submit: Submit GPU work to a command submission queue. + pub(crate) fn submit( + device: &AsahiDevice, + data: &mut uapi::drm_asahi_submit, + file: &DrmFile, + ) -> Result { + debug::update_debug_flags(); + + if data.flags != 0 || data.pad != 0 { + cls_pr_debug!(Errors, "submit: Invalid arguments\n"); + return Err(EINVAL); + } + + let gpu = &device.gpu; + gpu.update_globals(); + + // Upgrade to Arc to drop the XArray lock early + let queue: Arc>> = file + .inner() + .queues() + .lock() + .get(data.queue_id.try_into()?) + .ok_or(ENOENT)? + .into(); + + let id = gpu.ids().submission.next(); + mod_dev_dbg!( + device, + "[File {} Queue {}]: IOCTL: submit (submission ID: {})\n", + file.inner().id, + data.queue_id, + id + ); + + mod_dev_dbg!( + device, + "[File {} Queue {}]: IOCTL: submit({}): Parsing syncs\n", + file.inner().id, + data.queue_id, + id + ); + let syncs = + SyncItem::parse_array(file, data.syncs, data.in_sync_count, data.out_sync_count)?; + + mod_dev_dbg!( + device, + "[File {} Queue {}]: IOCTL: submit({}): Parsing commands\n", + file.inner().id, + data.queue_id, + id + ); + + let mut vec = KVec::new(); + + // Copy the command buffer into the kernel. Because we need to iterate + // the command buffer twice, we do this in one big copy_from_user to + // avoid TOCTOU issues. + let reader = UserSlice::new( + UserPtr::from_addr(data.cmdbuf as _), + data.cmdbuf_size as usize, + ) + .reader(); + reader.read_all(&mut vec, GFP_KERNEL)?; + + let objects = file.inner().objects(); + let ret = queue + .lock() + .submit(id, syncs, data.in_sync_count as usize, &vec, objects); + + match ret { + Err(ERESTARTSYS) => Err(ERESTARTSYS), + Err(e) => { + dev_info!( + device.as_ref(), + "[File {} Queue {}]: IOCTL: submit failed! (submission ID: {} err: {:?})\n", + file.inner().id, + data.queue_id, + id, + e + ); + Err(e) + } + Ok(()) => Ok(0), + } + } + + /// IOCTL: get_time: Get the current GPU timer value. + pub(crate) fn get_time( + device: &AsahiDevice, + data: &mut uapi::drm_asahi_get_time, + _file: &DrmFile, + ) -> Result { + if data.flags != 0 { + cls_pr_debug!(Errors, "get_time: Unexpected flags\n"); + return Err(EINVAL); + } + + // TODO: Do this on device-init for perf. + let gpu = &device.gpu; + let frequency_hz = gpu.get_cfg().base_clock_hz as u64; + let ts_gcd = gcd(frequency_hz, NSEC_PER_SEC as u64); + + let num = (NSEC_PER_SEC as u64) / ts_gcd; + let den = frequency_hz / ts_gcd; + + let raw: u64; + + // SAFETY: Assembly only loads the timer + unsafe { + core::arch::asm!( + "mrs {x}, CNTPCT_EL0", + x = out(reg) raw + ); + } + + data.gpu_timestamp = (raw * num) / den; + + Ok(0) + } +} + +impl Drop for File { + fn drop(&mut self) { + mod_pr_debug!("[File {}]: Closing...\n", self.id); + } +} diff --git a/drivers/gpu/drm/asahi/float.rs b/drivers/gpu/drm/asahi/float.rs new file mode 100644 index 00000000000000..d58a3d284da124 --- /dev/null +++ b/drivers/gpu/drm/asahi/float.rs @@ -0,0 +1,392 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Basic soft floating-point support +//! +//! The GPU firmware requires a large number of power-related configuration values, many of which +//! are IEEE 754 32-bit floating point values. These values change not only between GPU/SoC +//! variants, but also between specific hardware platforms using these SoCs, so they must be +//! derived from device tree properties. There are many redundant values computed from the same +//! inputs with simple add/sub/mul/div calculations, plus a few values that are actually specific +//! to each individual device depending on its binning and fused voltage configuration, so it +//! doesn't make sense to store the final values to be passed to the firmware in the device tree. +//! +//! Therefore, we need a way to perform floating-point calculations in the kernel. +//! +//! Using the actual FPU from kernel mode is asking for trouble, since there is no way to bound +//! the execution of FPU instructions to a controlled section of code without outright putting it +//! in its own compilation unit, which is quite painful for Rust. Since these calculations only +//! have to happen at initialization time and there is no need for performance, let's use a simple +//! software float implementation instead. +//! +//! This implementation makes no attempt to be fully IEEE754 compliant, but it's good enough and +//! gives bit-identical results to macOS in the vast majority of cases, with one or two exceptions +//! related to slightly non-compliant rounding. + +use core::ops; +use kernel::{ + of, + prelude::*, // +}; + +/// An IEEE754-compatible floating point number implemented in software. +#[derive(Default, Debug, Copy, Clone)] +#[repr(transparent)] +pub(crate) struct F32(u32); + +// SAFETY: F32 is a transparent repr of `u32` and therefore zeroable +unsafe impl Zeroable for F32 {} + +#[derive(Default, Debug, Copy, Clone)] +struct F32U { + sign: bool, + exp: i32, + frac: i64, +} + +impl F32 { + /// Convert a raw 32-bit representation into an F32 + pub(crate) const fn from_bits(u: u32) -> F32 { + F32(u) + } + + // Convert a `f32` value into an F32 + // + // This must ONLY be used in const context. Use the `f32!{}` macro to do it safely. + #[doc(hidden)] + pub(crate) const fn from_f32(v: f32) -> F32 { + // Replace with to_bits() after kernel Rust minreq is >= 1.83.0 + #[allow(clippy::transmute_float_to_int)] + #[allow(unnecessary_transmutes)] + // SAFETY: Transmuting f32 to u32 is always safe + F32(unsafe { core::mem::transmute::(v) }) + } + + // Convert an F32 into a `f32` value + // + // For testing only. + #[doc(hidden)] + #[cfg(test)] + pub(crate) fn to_f32(self) -> f32 { + f32::from_bits(self.0) + } + + const fn unpack(&self) -> F32U { + F32U { + sign: self.0 & (1 << 31) != 0, + exp: ((self.0 >> 23) & 0xff) as i32 - 127, + frac: (((self.0 & 0x7fffff) | 0x800000) as i64) << 9, + } + .norm() + } +} + +/// Safely construct an `F32` out of a constant floating-point value. +/// +/// This ensures that the conversion happens in const context, so no floating point operations are +/// emitted. +#[macro_export] +macro_rules! f32 { + ([$($val:expr),*]) => {{ + [$(f32!($val)),*] + }}; + ($val:expr) => {{ + const _K: $crate::float::F32 = $crate::float::F32::from_f32($val); + _K + }}; +} + +impl ops::Neg for F32 { + type Output = F32; + + fn neg(self) -> F32 { + F32(self.0 ^ (1 << 31)) + } +} + +impl ops::Add for F32 { + type Output = F32; + + fn add(self, rhs: F32) -> F32 { + self.unpack().add(rhs.unpack()).pack() + } +} + +impl ops::Sub for F32 { + type Output = F32; + + fn sub(self, rhs: F32) -> F32 { + self.unpack().add((-rhs).unpack()).pack() + } +} + +impl ops::Mul for F32 { + type Output = F32; + + fn mul(self, rhs: F32) -> F32 { + self.unpack().mul(rhs.unpack()).pack() + } +} + +impl ops::Div for F32 { + type Output = F32; + + fn div(self, rhs: F32) -> F32 { + self.unpack().div(rhs.unpack()).pack() + } +} + +macro_rules! from_ints { + ($u:ty, $i:ty) => { + impl From<$i> for F32 { + fn from(v: $i) -> F32 { + F32U::from_i64(v as i64).pack() + } + } + impl From<$u> for F32 { + fn from(v: $u) -> F32 { + F32U::from_u64(v as u64).pack() + } + } + }; +} + +from_ints!(u8, i8); +from_ints!(u16, i16); +from_ints!(u32, i32); +from_ints!(u64, i64); + +impl F32U { + const INFINITY: F32U = f32!(f32::INFINITY).unpack(); + const NEG_INFINITY: F32U = f32!(f32::NEG_INFINITY).unpack(); + + fn from_i64(v: i64) -> F32U { + F32U { + sign: v < 0, + exp: 32, + frac: v.abs(), + } + .norm() + } + + fn from_u64(mut v: u64) -> F32U { + let mut exp = 32; + if v >= (1 << 63) { + exp = 31; + v >>= 1; + } + F32U { + sign: false, + exp, + frac: v as i64, + } + .norm() + } + + fn shr(&mut self, shift: i32) { + if shift > 63 { + self.exp = 0; + self.frac = 0; + } else { + self.frac >>= shift; + } + } + + fn align(a: &mut F32U, b: &mut F32U) { + if a.exp > b.exp { + b.shr(a.exp - b.exp); + b.exp = a.exp; + } else { + a.shr(b.exp - a.exp); + a.exp = b.exp; + } + } + + fn mul(self, other: F32U) -> F32U { + F32U { + sign: self.sign != other.sign, + exp: self.exp + other.exp, + frac: ((self.frac >> 8) * (other.frac >> 8)) >> 16, + } + } + + fn div(self, other: F32U) -> F32U { + if other.frac == 0 || self.is_inf() { + if self.sign { + F32U::NEG_INFINITY + } else { + F32U::INFINITY + } + } else { + F32U { + sign: self.sign != other.sign, + exp: self.exp - other.exp, + frac: ((self.frac << 24) / (other.frac >> 8)), + } + } + } + + fn add(mut self, mut other: F32U) -> F32U { + F32U::align(&mut self, &mut other); + if self.sign == other.sign { + self.frac += other.frac; + } else { + self.frac -= other.frac; + } + if self.frac < 0 { + self.sign = !self.sign; + self.frac = -self.frac; + } + self + } + + const fn norm(mut self) -> F32U { + let lz = self.frac.leading_zeros() as i32; + if lz > 31 { + self.frac <<= lz - 31; + self.exp -= lz - 31; + } else if lz < 31 { + self.frac >>= 31 - lz; + self.exp += 31 - lz; + } + + if self.is_zero() { + return F32U { + sign: self.sign, + frac: 0, + exp: 0, + }; + } + self + } + + const fn is_zero(&self) -> bool { + self.frac == 0 || self.exp < -126 + } + + const fn is_inf(&self) -> bool { + self.exp > 127 + } + + const fn pack(mut self) -> F32 { + self = self.norm(); + if !self.is_zero() { + self.frac += 0x100; + self = self.norm(); + } + + if self.is_inf() { + if self.sign { + return f32!(f32::NEG_INFINITY); + } else { + return f32!(f32::INFINITY); + } + } else if self.is_zero() { + if self.sign { + return f32!(-0.0); + } else { + return f32!(0.0); + } + } + + F32(if self.sign { 1u32 << 31 } else { 0u32 } + | ((self.exp + 127) as u32) << 23 + | ((self.frac >> 9) & 0x7fffff) as u32) + } +} + +impl<'a> TryFrom> for F32 { + type Error = Error; + + fn try_from(p: of::Property<'_>) -> core::result::Result { + let bits: u32 = p.try_into()?; + Ok(F32::from_bits(bits)) + } +} + +impl of::PropertyUnit for F32 { + const UNIT_SIZE: usize = 4; + + fn from_bytes(data: &[u8]) -> Result { + Ok(F32::from_bits(::from_bytes(data)?)) + } +} + +// TODO: Make this an actual test and figure out how to make it run. +#[cfg(test)] +mod tests { + #[test] + fn test_all() { + fn add(a: f32, b: f32) { + println!( + "{} + {} = {} {}", + a, + b, + (F32::from_f32(a) + F32::from_f32(b)).to_f32(), + a + b + ); + } + fn sub(a: f32, b: f32) { + println!( + "{} - {} = {} {}", + a, + b, + (F32::from_f32(a) - F32::from_f32(b)).to_f32(), + a - b + ); + } + fn mul(a: f32, b: f32) { + println!( + "{} * {} = {} {}", + a, + b, + (F32::from_f32(a) * F32::from_f32(b)).to_f32(), + a * b + ); + } + fn div(a: f32, b: f32) { + println!( + "{} / {} = {} {}", + a, + b, + (F32::from_f32(a) / F32::from_f32(b)).to_f32(), + a / b + ); + } + + fn test(a: f32, b: f32) { + add(a, b); + sub(a, b); + mul(a, b); + div(a, b); + } + + test(1.123, 7.567); + test(1.123, 1.456); + test(7.567, 1.123); + test(1.123, -7.567); + test(1.123, -1.456); + test(7.567, -1.123); + test(-1.123, -7.567); + test(-1.123, -1.456); + test(-7.567, -1.123); + test(1000.123, 0.001); + test(1000.123, 0.0000001); + test(0.0012, 1000.123); + test(0.0000001, 1000.123); + test(0., 0.); + test(0., 1.); + test(1., 0.); + test(1., 1.); + test(2., f32::INFINITY); + test(2., f32::NEG_INFINITY); + test(f32::INFINITY, 2.); + test(f32::NEG_INFINITY, 2.); + test(f32::NEG_INFINITY, 2.); + test(f32::MAX, 2.); + test(f32::MIN, 2.); + test(f32::MIN_POSITIVE, 2.); + test(2., f32::MAX); + test(2., f32::MIN); + test(2., f32::MIN_POSITIVE); + } +} diff --git a/drivers/gpu/drm/asahi/fw/buffer.rs b/drivers/gpu/drm/asahi/fw/buffer.rs new file mode 100644 index 00000000000000..b1f4974fd02902 --- /dev/null +++ b/drivers/gpu/drm/asahi/fw/buffer.rs @@ -0,0 +1,184 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! GPU tiled vertex buffer control firmware structures + +use super::types::*; +use super::workqueue; +use crate::{ + default_zeroed, + no_debug, + trivial_gpustruct, // +}; +use kernel::sync::Arc; + +pub(crate) mod raw { + use super::*; + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct BlockControl { + pub(crate) total: AtomicU32, + pub(crate) wptr: AtomicU32, + pub(crate) unk: AtomicU32, + pub(crate) pad: Pad<0x34>, + } + default_zeroed!(BlockControl); + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct Counter { + pub(crate) count: AtomicU32, + __pad: Pad<0x3c>, + } + default_zeroed!(Counter); + + #[derive(Debug, Default)] + #[repr(C)] + pub(crate) struct Stats { + pub(crate) max_pages: AtomicU32, + pub(crate) max_b: AtomicU32, + pub(crate) overflow_count: AtomicU32, + pub(crate) gpu_c: AtomicU32, + pub(crate) __pad0: Pad<0x10>, + pub(crate) reset: AtomicU32, + pub(crate) __pad1: Pad<0x1c>, + } + + #[versions(AGX)] + #[derive(Debug)] + #[repr(C)] + pub(crate) struct Info<'a> { + pub(crate) gpu_counter: u32, + pub(crate) unk_4: u32, + pub(crate) last_id: i32, + pub(crate) cur_id: i32, + pub(crate) unk_10: u32, + pub(crate) gpu_counter2: u32, + pub(crate) unk_18: u32, + + #[ver(V < V13_0B4 || G >= G14X)] + pub(crate) unk_1c: u32, + + pub(crate) page_list: GpuPointer<'a, &'a [u32]>, + pub(crate) page_list_size: u32, + pub(crate) page_count: AtomicU32, + pub(crate) max_blocks: u32, + pub(crate) block_count: AtomicU32, + pub(crate) unk_38: u32, + pub(crate) block_list: GpuPointer<'a, &'a [u32]>, + pub(crate) block_ctl: GpuPointer<'a, super::BlockControl>, + pub(crate) last_page: AtomicU32, + pub(crate) gpu_page_ptr1: u32, + pub(crate) gpu_page_ptr2: u32, + pub(crate) unk_58: u32, + pub(crate) block_size: u32, + pub(crate) unk_60: U64, + pub(crate) counter: GpuPointer<'a, super::Counter>, + pub(crate) unk_70: u32, + pub(crate) unk_74: u32, + pub(crate) unk_78: u32, + pub(crate) unk_7c: u32, + pub(crate) unk_80: u32, + pub(crate) max_pages: u32, + pub(crate) max_pages_nomemless: u32, + pub(crate) unk_8c: u32, + pub(crate) unk_90: Array<0x30, u8>, + } + + #[versions(AGX)] + #[derive(Debug)] + #[repr(C)] + pub(crate) struct Scene<'a> { + #[ver(G >= G14X)] + pub(crate) control_word: GpuPointer<'a, &'a [u32]>, + #[ver(G >= G14X)] + pub(crate) control_word2: GpuPointer<'a, &'a [u32]>, + pub(crate) pass_page_count: AtomicU32, + pub(crate) unk_4: u32, + pub(crate) unk_8: U64, + pub(crate) unk_10: U64, + pub(crate) user_buffer: GpuPointer<'a, &'a [u8]>, + pub(crate) unk_20: u32, + #[ver(V >= V13_3)] + pub(crate) unk_28: U64, + pub(crate) stats: GpuWeakPointer, + pub(crate) total_page_count: AtomicU32, + #[ver(G < G14X)] + pub(crate) unk_30: U64, // pad + #[ver(G < G14X)] + pub(crate) unk_38: U64, // pad + } + + #[versions(AGX)] + #[derive(Debug)] + #[repr(C)] + pub(crate) struct InitBuffer<'a> { + pub(crate) tag: workqueue::CommandType, + pub(crate) vm_slot: u32, + pub(crate) buffer_slot: u32, + pub(crate) unk_c: u32, + pub(crate) block_count: u32, + pub(crate) buffer: GpuPointer<'a, super::Info::ver>, + pub(crate) stamp_value: EventValue, + } +} + +trivial_gpustruct!(BlockControl); +trivial_gpustruct!(Counter); +trivial_gpustruct!(Stats); + +#[versions(AGX)] +#[derive(Debug)] +pub(crate) struct Info { + pub(crate) block_ctl: GpuObject, + pub(crate) counter: GpuObject, + pub(crate) page_list: GpuArray, + pub(crate) block_list: GpuArray, +} + +#[versions(AGX)] +impl GpuStruct for Info::ver { + type Raw<'a> = raw::Info::ver<'a>; +} + +pub(crate) struct ClusterBuffers { + pub(crate) tilemaps: GpuArray, + pub(crate) meta: GpuArray, +} + +#[versions(AGX)] +pub(crate) struct Scene { + pub(crate) user_buffer: GpuArray, + pub(crate) buffer: crate::buffer::Buffer::ver, + pub(crate) tvb_heapmeta: GpuArray, + pub(crate) tvb_tilemap: GpuArray, + pub(crate) tpc: Arc>, + pub(crate) clustering: Option, + pub(crate) preempt_buf: GpuArray, + #[ver(G >= G14X)] + pub(crate) control_word: GpuArray, +} + +#[versions(AGX)] +no_debug!(Scene::ver); + +#[versions(AGX)] +impl GpuStruct for Scene::ver { + type Raw<'a> = raw::Scene::ver<'a>; +} + +#[versions(AGX)] +pub(crate) struct InitBuffer { + pub(crate) scene: Arc, +} + +#[versions(AGX)] +no_debug!(InitBuffer::ver); + +#[versions(AGX)] +impl workqueue::Command for InitBuffer::ver {} + +#[versions(AGX)] +impl GpuStruct for InitBuffer::ver { + type Raw<'a> = raw::InitBuffer::ver<'a>; +} diff --git a/drivers/gpu/drm/asahi/fw/channels.rs b/drivers/gpu/drm/asahi/fw/channels.rs new file mode 100644 index 00000000000000..c1a7ec82aad1e2 --- /dev/null +++ b/drivers/gpu/drm/asahi/fw/channels.rs @@ -0,0 +1,443 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! GPU communication channel firmware structures (ring buffers) + +use super::types::*; +use crate::default_zeroed; +use core::sync::atomic::Ordering; +use kernel::static_assert; + +pub(crate) mod raw { + use super::*; + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct ChannelState<'a> { + pub(crate) read_ptr: AtomicU32, + __pad0: Pad<0x1c>, + pub(crate) write_ptr: AtomicU32, + __pad1: Pad<0xc>, + _p: PhantomData<&'a ()>, + } + default_zeroed!(<'a>, ChannelState<'a>); + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct FwCtlChannelState<'a> { + pub(crate) read_ptr: AtomicU32, + __pad0: Pad<0xc>, + pub(crate) write_ptr: AtomicU32, + __pad1: Pad<0xc>, + _p: PhantomData<&'a ()>, + } + default_zeroed!(<'a>, FwCtlChannelState<'a>); +} + +pub(crate) trait RxChannelState: GpuStruct + Debug + Default +where + for<'a> ::Raw<'a>: Default + Zeroable, +{ + const SUB_CHANNELS: usize; + + fn wptr(raw: &Self::Raw<'_>, index: usize) -> u32; + fn set_rptr(raw: &Self::Raw<'_>, index: usize, rptr: u32); +} + +#[derive(Debug, Default)] +pub(crate) struct ChannelState {} + +impl GpuStruct for ChannelState { + type Raw<'a> = raw::ChannelState<'a>; +} + +impl RxChannelState for ChannelState { + const SUB_CHANNELS: usize = 1; + + fn wptr(raw: &Self::Raw<'_>, _index: usize) -> u32 { + raw.write_ptr.load(Ordering::Acquire) + } + + fn set_rptr(raw: &Self::Raw<'_>, _index: usize, rptr: u32) { + raw.read_ptr.store(rptr, Ordering::Release); + } +} + +#[derive(Debug, Default)] +pub(crate) struct FwLogChannelState {} + +impl GpuStruct for FwLogChannelState { + type Raw<'a> = Array<6, raw::ChannelState<'a>>; +} + +impl RxChannelState for FwLogChannelState { + const SUB_CHANNELS: usize = 6; + + fn wptr(raw: &Self::Raw<'_>, index: usize) -> u32 { + raw[index].write_ptr.load(Ordering::Acquire) + } + + fn set_rptr(raw: &Self::Raw<'_>, index: usize, rptr: u32) { + raw[index].read_ptr.store(rptr, Ordering::Release); + } +} + +#[derive(Debug, Default)] +pub(crate) struct FwCtlChannelState {} + +impl GpuStruct for FwCtlChannelState { + type Raw<'a> = raw::FwCtlChannelState<'a>; +} + +pub(crate) trait TxChannelState: GpuStruct + Debug + Default { + fn rptr(raw: &Self::Raw<'_>) -> u32; + fn set_wptr(raw: &Self::Raw<'_>, wptr: u32); +} + +impl TxChannelState for ChannelState { + fn rptr(raw: &Self::Raw<'_>) -> u32 { + raw.read_ptr.load(Ordering::Acquire) + } + + fn set_wptr(raw: &Self::Raw<'_>, wptr: u32) { + raw.write_ptr.store(wptr, Ordering::Release); + } +} + +impl TxChannelState for FwCtlChannelState { + fn rptr(raw: &Self::Raw<'_>) -> u32 { + raw.read_ptr.load(Ordering::Acquire) + } + + fn set_wptr(raw: &Self::Raw<'_>, wptr: u32) { + raw.write_ptr.store(wptr, Ordering::Release); + } +} + +#[derive(Debug, Copy, Clone, Default, PartialEq, Eq)] +#[repr(u32)] +pub(crate) enum PipeType { + #[default] + Vertex = 0, + Fragment = 1, + Compute = 2, +} + +#[versions(AGX)] +#[derive(Debug, Copy, Clone, Default)] +#[repr(C)] +pub(crate) struct RunWorkQueueMsg { + pub(crate) pipe_type: PipeType, + pub(crate) work_queue: Option>, + pub(crate) wptr: u32, + pub(crate) event_slot: u32, + pub(crate) is_new: bool, + #[ver(V >= V13_2 && G == G14)] + pub(crate) __pad: Pad<0x2b>, + #[ver(V < V13_2 || G != G14)] + pub(crate) __pad: Pad<0x1b>, +} + +#[versions(AGX)] +pub(crate) type PipeMsg = RunWorkQueueMsg::ver; + +#[versions(AGX)] +pub(crate) const DEVICECONTROL_SZ: usize = { + #[ver(V < V13_2 || G != G14)] + { + 0x2c + } + #[ver(V >= V13_2 && G == G14)] + { + 0x3c + } +}; + +// TODO: clean up when arbitrary_enum_discriminant is stable +// https://github.com/rust-lang/rust/issues/60553 + +#[versions(AGX)] +#[derive(Debug, Copy, Clone)] +#[repr(C, u32)] +#[allow(dead_code)] +pub(crate) enum DeviceControlMsg { + Unk00(Array), + Unk01(Array), + Unk02(Array), + Unk03(Array), + Unk04(Array), + Unk05(Array), + Unk06(Array), + Unk07(Array), + Unk08(Array), + Unk09(Array), + Unk0a(Array), + Unk0b(Array), + Unk0c(Array), + #[ver(V >= V13_3)] + Unk0d(Array), + GrowTVBAck { + unk_4: u32, + buffer_slot: u32, + vm_slot: u32, + counter: u32, + subpipe: u32, + halt_count: U64, + __pad: Pad<{ DEVICECONTROL_SZ::ver - 0x1c }>, + }, + RecoverChannel { + pipe_type: u32, + work_queue: GpuWeakPointer, + event_value: u32, + __pad: Pad<{ DEVICECONTROL_SZ::ver - 0x10 }>, + }, + IdlePowerOff { + val: u32, + __pad: Pad<{ DEVICECONTROL_SZ::ver - 0x4 }>, + }, + Unk10(Array), + Unk11(Array), + Unk12(Array), + Unk13(Array), + Unk14(Array), // Init? + Unk15(Array), // Enable something + Unk16(Array), // Disable something + DestroyContext { + unk_4: u32, + ctx_23: u8, + #[ver(V < V13_3)] + __pad0: Pad<3>, + unk_c: U32, + unk_10: U32, + ctx_0: u8, + ctx_1: u8, + ctx_4: u8, + #[ver(V < V13_3)] + __pad1: Pad<1>, + #[ver(V < V13_3)] + unk_18: u32, + gpu_context: Option>, + #[ver(V < V13_3)] + __pad2: Pad<{ DEVICECONTROL_SZ::ver - 0x20 }>, + #[ver(V >= V13_3)] + __pad2: Pad<{ DEVICECONTROL_SZ::ver - 0x18 }>, + }, + Unk18(Array), + Initialize(Pad), // Update RegionC +} + +#[versions(AGX)] +static_assert!(core::mem::size_of::() == 4 + DEVICECONTROL_SZ::ver); + +#[versions(AGX)] +default_zeroed!(DeviceControlMsg::ver); + +#[derive(Copy, Clone, Default, Debug)] +#[repr(C)] +#[allow(dead_code)] +pub(crate) struct FwCtlMsg { + pub(crate) addr: U64, + pub(crate) unk_8: u32, + pub(crate) slot: u32, + pub(crate) page_count: u16, + pub(crate) unk_12: u16, +} + +pub(crate) const EVENT_SZ: usize = 0x34; + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +#[repr(C, u32)] +#[allow(dead_code)] +pub(crate) enum ChannelErrorType { + MemoryError, + DMKill, + Aborted, + Unk3, + Unknown(u32), +} + +#[derive(Debug, Copy, Clone)] +#[repr(C, u32)] +#[allow(dead_code)] +pub(crate) enum EventMsg { + Fault, + Flag { + firing: [u32; 4], + unk_14: u16, + }, + Unk2(Array), + Unk3(Array), + Timeout { + counter: u32, + unk_8: u32, + event_slot: i32, + }, + Unk5(Array), + Unk6(Array), + GrowTVB { + vm_slot: u32, + buffer_slot: u32, + counter: u32, + }, + ChannelError { + error_type: u32, + pipe_type: u32, + event_slot: u32, + event_value: u32, + }, + // Max discriminant: 0x8 +} + +static_assert!(core::mem::size_of::() == 4 + EVENT_SZ); + +pub(crate) const EVENT_MAX: u32 = 0x8; + +#[derive(Copy, Clone)] +#[repr(C)] +pub(crate) union RawEventMsg { + pub(crate) raw: (u32, Array), + pub(crate) msg: EventMsg, +} + +default_zeroed!(RawEventMsg); + +#[derive(Debug, Copy, Clone, Default)] +#[repr(C)] +pub(crate) struct RawFwLogMsg { + pub(crate) msg_type: u32, + __pad0: u32, + pub(crate) msg_index: U64, + __pad1: Pad<0x28>, +} + +#[derive(Debug, Copy, Clone, Default)] +#[repr(C)] +pub(crate) struct RawFwLogPayloadMsg { + pub(crate) msg_type: u32, + pub(crate) seq_no: u32, + pub(crate) timestamp: U64, + pub(crate) msg: Array<0xc8, u8>, +} + +#[derive(Debug, Copy, Clone, Default)] +#[repr(C)] +pub(crate) struct RawKTraceMsg { + pub(crate) msg_type: u32, + pub(crate) timestamp: U64, + pub(crate) args: Array<4, U64>, + pub(crate) code: u8, + pub(crate) channel: u8, + __pad: Pad<1>, + pub(crate) thread: u8, + pub(crate) unk_flag: U64, +} + +#[versions(AGX)] +pub(crate) const STATS_SZ: usize = { + #[ver(V < V13_0B4)] + { + 0x2c + } + #[ver(V >= V13_0B4)] + { + 0x3c + } +}; + +#[versions(AGX)] +#[derive(Debug, Copy, Clone)] +#[repr(C, u32)] +#[allow(dead_code)] +pub(crate) enum StatsMsg { + Power { + // 0x00 + __pad: Pad<0x18>, + power: U64, + }, + Unk1(Array<{ STATS_SZ::ver }, u8>), + PowerOn { + // 0x02 + off_time: U64, + }, + PowerOff { + // 0x03 + on_time: U64, + }, + Utilization { + // 0x04 + timestamp: U64, + util1: u32, + util2: u32, + util3: u32, + util4: u32, + }, + Unk5(Array<{ STATS_SZ::ver }, u8>), + Unk6(Array<{ STATS_SZ::ver }, u8>), + Unk7(Array<{ STATS_SZ::ver }, u8>), + Unk8(Array<{ STATS_SZ::ver }, u8>), + AvgPower { + // 0x09 + active_cs: U64, + unk2: u32, + unk3: u32, + unk4: u32, + avg_power: u32, + }, + Temperature { + // 0x0a + __pad: Pad<0x8>, + raw_value: u32, + scale: u32, + tmin: u32, + tmax: u32, + }, + PowerState { + // 0x0b + timestamp: U64, + last_busy_ts: U64, + active: u32, + poweroff: u32, + unk1: u32, + pstate: u32, + unk2: u32, + unk3: u32, + }, + FwBusy { + // 0x0c + timestamp: U64, + busy: u32, + }, + PState { + // 0x0d + __pad: Pad<0x8>, + ps_min: u32, + unk1: u32, + ps_max: u32, + unk2: u32, + }, + TempSensor { + // 0x0e + __pad: Pad<0x4>, + sensor_id: u32, + raw_value: u32, + scale: u32, + tmin: u32, + tmax: u32, + }, // Max discriminant: 0xe +} + +#[versions(AGX)] +static_assert!(core::mem::size_of::() == 4 + STATS_SZ::ver); + +#[versions(AGX)] +pub(crate) const STATS_MAX: u32 = 0xe; + +#[versions(AGX)] +#[derive(Copy, Clone)] +#[repr(C)] +pub(crate) union RawStatsMsg { + pub(crate) raw: (u32, Array<{ STATS_SZ::ver }, u8>), + pub(crate) msg: StatsMsg::ver, +} + +#[versions(AGX)] +default_zeroed!(RawStatsMsg::ver); diff --git a/drivers/gpu/drm/asahi/fw/compute.rs b/drivers/gpu/drm/asahi/fw/compute.rs new file mode 100644 index 00000000000000..f5f6ffa9d8d0d8 --- /dev/null +++ b/drivers/gpu/drm/asahi/fw/compute.rs @@ -0,0 +1,120 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! GPU compute job firmware structures + +use super::types::*; +use super::{ + event, + job, + workqueue, // +}; +use crate::{ + microseq, + mmu, // +}; +use kernel::sync::Arc; + +pub(crate) mod raw { + use super::*; + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct JobParameters1<'a> { + pub(crate) preempt_buf1: GpuPointer<'a, &'a [u8]>, + pub(crate) cdm_ctrl_stream_base: U64, + pub(crate) preempt_buf2: GpuPointer<'a, &'a [u8]>, + pub(crate) preempt_buf3: GpuPointer<'a, &'a [u8]>, + pub(crate) preempt_buf4: GpuPointer<'a, &'a [u8]>, + pub(crate) preempt_buf5: GpuPointer<'a, &'a [u8]>, + pub(crate) usc_exec_base_cp: U64, + pub(crate) unk_38: U64, + pub(crate) helper_program: u32, + pub(crate) unk_44: u32, + pub(crate) helper_arg: U64, + pub(crate) helper_cfg: u32, + pub(crate) unk_54: u32, + pub(crate) unk_58: u32, + pub(crate) unk_5c: u32, + pub(crate) iogpu_unk_40: u32, + pub(crate) __pad: Pad<0xfc>, + } + + #[versions(AGX)] + #[derive(Debug)] + #[repr(C)] + pub(crate) struct JobParameters2<'a> { + #[ver(V >= V13_0B4)] + pub(crate) unk_0_0: u32, + pub(crate) unk_0: Array<0x24, u8>, + pub(crate) preempt_buf1: GpuPointer<'a, &'a [u8]>, + pub(crate) cdm_ctrl_stream_end: U64, + pub(crate) unk_34: Array<0x20, u8>, + pub(crate) unk_g14x: u32, + pub(crate) unk_58: u32, + #[ver(V < V13_0B4)] + pub(crate) unk_5c: u32, + } + + #[versions(AGX)] + #[derive(Debug)] + #[repr(C)] + pub(crate) struct RunCompute<'a> { + pub(crate) tag: workqueue::CommandType, + + #[ver(V >= V13_0B4)] + pub(crate) counter: U64, + + pub(crate) unk_4: u32, + pub(crate) vm_slot: u32, + pub(crate) notifier: GpuPointer<'a, event::Notifier::ver>, + pub(crate) unk_pointee: u32, + #[ver(G < G14X)] + pub(crate) __pad0: Array<0x50, u8>, + #[ver(G < G14X)] + pub(crate) job_params1: JobParameters1<'a>, + #[ver(G >= G14X)] + pub(crate) registers: job::raw::RegisterArray, + pub(crate) __pad1: Array<0x20, u8>, + pub(crate) microsequence: GpuPointer<'a, &'a [u8]>, + pub(crate) microsequence_size: u32, + pub(crate) job_params2: JobParameters2::ver<'a>, + pub(crate) encoder_params: job::raw::EncoderParams, + pub(crate) meta: job::raw::JobMeta, + pub(crate) command_time: U64, + pub(crate) timestamp_pointers: job::raw::TimestampPointers<'a>, + pub(crate) user_timestamp_pointers: job::raw::TimestampPointers<'a>, + pub(crate) client_sequence: u8, + pub(crate) pad_2d1: Array<3, u8>, + pub(crate) unk_2d4: u32, + pub(crate) unk_2d8: u8, + #[ver(V >= V13_0B4)] + pub(crate) context_store_req: U64, + #[ver(V >= V13_0B4)] + pub(crate) context_store_compl: U64, + #[ver(V >= V13_0B4)] + pub(crate) unk_2e9: Array<0x14, u8>, + #[ver(V >= V13_0B4)] + pub(crate) unk_flag: U32, + #[ver(V >= V13_0B4)] + pub(crate) unk_pad: Array<0x10, u8>, + } +} + +#[versions(AGX)] +#[derive(Debug)] +pub(crate) struct RunCompute { + pub(crate) notifier: Arc>, + pub(crate) preempt_buf: GpuArray, + pub(crate) micro_seq: microseq::MicroSequence, + pub(crate) vm_bind: mmu::VmBind, + pub(crate) timestamps: Arc>, + pub(crate) user_timestamps: job::UserTimestamps, +} + +#[versions(AGX)] +impl GpuStruct for RunCompute::ver { + type Raw<'a> = raw::RunCompute::ver<'a>; +} + +#[versions(AGX)] +impl workqueue::Command for RunCompute::ver {} diff --git a/drivers/gpu/drm/asahi/fw/event.rs b/drivers/gpu/drm/asahi/fw/event.rs new file mode 100644 index 00000000000000..52bc456f58707d --- /dev/null +++ b/drivers/gpu/drm/asahi/fw/event.rs @@ -0,0 +1,104 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! GPU events control structures & stamps + +use super::types::*; +use crate::{ + default_zeroed, + trivial_gpustruct, // +}; +use core::sync::atomic::Ordering; + +pub(crate) mod raw { + use super::*; + + #[derive(Debug, Clone, Copy, Default)] + #[repr(C)] + pub(crate) struct LinkedListHead { + pub(crate) prev: Option>, + pub(crate) next: Option>, + } + + #[derive(Debug, Clone, Copy)] + #[repr(C)] + pub(crate) struct NotifierList { + pub(crate) list_head: LinkedListHead, + pub(crate) unkptr_10: U64, + } + default_zeroed!(NotifierList); + + #[versions(AGX)] + #[derive(Debug, Clone, Copy)] + #[repr(C)] + pub(crate) struct NotifierState { + unk_14: u32, + unk_18: U64, + unk_20: u32, + vm_slot: u32, + has_vtx: u32, + pstamp_vtx: Array<4, U64>, + has_frag: u32, + pstamp_frag: Array<4, U64>, + has_comp: u32, + pstamp_comp: Array<4, U64>, + #[ver(G >= G14 && V < V13_0B4)] + unk_98_g14_0: Array<0x14, u8>, + in_list: u32, + list_head: LinkedListHead, + #[ver(G >= G14 && V < V13_0B4)] + unk_a8_g14_0: Pad<4>, + #[ver(V >= V13_0B4)] + pub(crate) unk_buf: Array<0x8, u8>, // Init to all-ff + } + + #[versions(AGX)] + impl Default for NotifierState::ver { + fn default() -> Self { + #[allow(unused_mut)] + // SAFETY: All bit patterns are valid for this type. + let mut s: Self = unsafe { core::mem::zeroed() }; + #[ver(V >= V13_0B4)] + s.unk_buf = Array::new([0xff; 0x8]); + s + } + } + + #[derive(Debug)] + #[repr(transparent)] + pub(crate) struct Threshold(AtomicU64); + default_zeroed!(Threshold); + + impl Threshold { + pub(crate) fn increase(&self, amount: u32) { + // We could use fetch_add, but the non-LSE atomic + // sequence Rust produces confuses the hypervisor. + let v = self.0.load(Ordering::Relaxed); + self.0.store(v + (amount as u64), Ordering::Relaxed); + } + } + + #[versions(AGX)] + #[derive(Debug)] + #[repr(C)] + pub(crate) struct Notifier<'a> { + pub(crate) threshold: GpuPointer<'a, super::Threshold>, + pub(crate) generation: AtomicU32, + pub(crate) cur_count: AtomicU32, + pub(crate) unk_10: AtomicU32, + pub(crate) state: NotifierState::ver, + } +} + +trivial_gpustruct!(Threshold); +trivial_gpustruct!(NotifierList); + +#[versions(AGX)] +#[derive(Debug)] +pub(crate) struct Notifier { + pub(crate) threshold: GpuObject, +} + +#[versions(AGX)] +impl GpuStruct for Notifier::ver { + type Raw<'a> = raw::Notifier::ver<'a>; +} diff --git a/drivers/gpu/drm/asahi/fw/fragment.rs b/drivers/gpu/drm/asahi/fw/fragment.rs new file mode 100644 index 00000000000000..3daad1ae4db671 --- /dev/null +++ b/drivers/gpu/drm/asahi/fw/fragment.rs @@ -0,0 +1,296 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! GPU fragment job firmware structures + +use super::types::*; +use super::{ + event, + job, + workqueue, // +}; +use crate::{ + buffer, + fw, + microseq, + mmu, // +}; +use kernel::sync::Arc; + +pub(crate) mod raw { + use super::*; + + #[derive(Debug, Clone, Copy)] + #[repr(C)] + pub(crate) struct BackgroundProgram { + pub(crate) rsrc_spec: U64, + pub(crate) address: U64, + } + + #[derive(Debug, Clone, Copy, Default)] + #[repr(C)] + pub(crate) struct EotProgram { + pub(crate) unk_0: U64, + pub(crate) unk_8: u32, + pub(crate) rsrc_spec: u32, + pub(crate) unk_10: u32, + pub(crate) address: u32, + pub(crate) unk_18: u32, + pub(crate) unk_1c_padding: u32, + } + + impl EotProgram { + pub(crate) fn new(rsrc_spec: u32, address: u32) -> EotProgram { + EotProgram { + rsrc_spec, + address, + ..Default::default() + } + } + } + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct ArrayAddr { + pub(crate) ptr: U64, + pub(crate) unk_padding: U64, + } + + #[versions(AGX)] + #[derive(Debug, Clone, Copy)] + #[repr(C)] + pub(crate) struct AuxFBInfo { + pub(crate) isp_ctl: u32, + pub(crate) unk2: u32, + pub(crate) width: u32, + pub(crate) height: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk3: U64, + } + + #[versions(AGX)] + #[derive(Debug)] + #[repr(C)] + pub(crate) struct JobParameters1<'a> { + pub(crate) utile_config: u32, + pub(crate) unk_4: u32, + pub(crate) bg: BackgroundProgram, + pub(crate) ppp_multisamplectl: U64, + pub(crate) isp_scissor_base: U64, + pub(crate) isp_dbias_base: U64, + pub(crate) aux_fb_info: AuxFBInfo::ver, + pub(crate) isp_zls_pixels: U64, + pub(crate) isp_oclqry_base: U64, + pub(crate) zls_ctrl: U64, + + #[ver(G >= G14)] + pub(crate) unk_58_g14_0: U64, + #[ver(G >= G14)] + pub(crate) unk_58_g14_8: U64, + + pub(crate) z_load: U64, + pub(crate) z_store: U64, + pub(crate) s_load: U64, + pub(crate) s_store: U64, + + #[ver(G >= G14)] + pub(crate) unk_68_g14_0: Array<0x20, u8>, + + pub(crate) z_load_stride: U64, + pub(crate) z_store_stride: U64, + pub(crate) s_load_stride: U64, + pub(crate) s_store_stride: U64, + pub(crate) z_load_comp: U64, + pub(crate) z_load_comp_stride: U64, + pub(crate) z_store_comp: U64, + pub(crate) z_store_comp_stride: U64, + pub(crate) s_load_comp: U64, + pub(crate) s_load_comp_stride: U64, + pub(crate) s_store_comp: U64, + pub(crate) s_store_comp_stride: U64, + pub(crate) tvb_tilemap: GpuPointer<'a, &'a [u8]>, + pub(crate) tvb_layermeta: GpuPointer<'a, &'a [u8]>, + pub(crate) mtile_stride_dwords: U64, + pub(crate) tvb_heapmeta: GpuPointer<'a, &'a [u8]>, + pub(crate) tile_config: U64, + pub(crate) aux_fb: GpuPointer<'a, &'a [u8]>, + pub(crate) unk_108: Array<0x6, U64>, + pub(crate) usc_exec_base_isp: U64, + pub(crate) unk_140: U64, + pub(crate) helper_program: u32, + pub(crate) unk_14c: u32, + pub(crate) helper_arg: U64, + pub(crate) unk_158: U64, + pub(crate) unk_160: U64, + + #[ver(G < G14)] + pub(crate) __pad: Pad<0x1d8>, + #[ver(G >= G14)] + pub(crate) __pad: Pad<0x1a8>, + #[ver(V < V13_0B4)] + pub(crate) __pad1: Pad<0x8>, + } + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct JobParameters2 { + pub(crate) eot_rsrc_spec: u32, + pub(crate) eot_usc: u32, + pub(crate) unk_8: u32, + pub(crate) unk_c: u32, + pub(crate) isp_merge_upper_x: F32, + pub(crate) isp_merge_upper_y: F32, + pub(crate) unk_18: U64, + pub(crate) utiles_per_mtile_y: u16, + pub(crate) utiles_per_mtile_x: u16, + pub(crate) unk_24: u32, + pub(crate) tile_counts: u32, + pub(crate) tib_blocks: u32, + pub(crate) isp_bgobjdepth: u32, + pub(crate) isp_bgobjvals: u32, + pub(crate) unk_38: u32, + pub(crate) unk_3c: u32, + pub(crate) helper_cfg: u32, + pub(crate) __pad: Pad<0xac>, + } + + #[versions(AGX)] + #[derive(Debug)] + #[repr(C)] + pub(crate) struct JobParameters3 { + pub(crate) isp_dbias_base: ArrayAddr, + pub(crate) isp_scissor_base: ArrayAddr, + pub(crate) isp_oclqry_base: U64, + pub(crate) unk_118: U64, + pub(crate) unk_120: Array<0x25, U64>, + pub(crate) unk_partial_bg: BackgroundProgram, + pub(crate) unk_258: U64, + pub(crate) unk_260: U64, + pub(crate) unk_268: U64, + pub(crate) unk_270: U64, + pub(crate) partial_bg: BackgroundProgram, + pub(crate) zls_ctrl: U64, + pub(crate) unk_290: U64, + pub(crate) z_load: U64, + pub(crate) z_partial_stride: U64, + pub(crate) z_partial_comp_stride: U64, + pub(crate) z_store: U64, + pub(crate) z_partial: U64, + pub(crate) z_partial_comp: U64, + pub(crate) s_load: U64, + pub(crate) s_partial_stride: U64, + pub(crate) s_partial_comp_stride: U64, + pub(crate) s_store: U64, + pub(crate) s_partial: U64, + pub(crate) s_partial_comp: U64, + pub(crate) unk_2f8: Array<2, U64>, + pub(crate) tib_blocks: u32, + pub(crate) unk_30c: u32, + pub(crate) aux_fb_info: AuxFBInfo::ver, + pub(crate) tile_config: U64, + pub(crate) unk_328_padding: Array<0x8, u8>, + pub(crate) unk_partial_eot: EotProgram, + pub(crate) partial_eot: EotProgram, + pub(crate) isp_bgobjdepth: u32, + pub(crate) isp_bgobjvals: u32, + pub(crate) sample_size: u32, + pub(crate) unk_37c: u32, + pub(crate) unk_380: U64, + pub(crate) unk_388: U64, + + #[ver(V >= V13_0B4)] + pub(crate) unk_390_0: U64, + + pub(crate) isp_zls_pixels: U64, + } + + #[versions(AGX)] + #[derive(Debug)] + #[repr(C)] + pub(crate) struct RunFragment<'a> { + pub(crate) tag: workqueue::CommandType, + + #[ver(V >= V13_0B4)] + pub(crate) counter: U64, + + pub(crate) vm_slot: u32, + pub(crate) unk_8: u32, + pub(crate) microsequence: GpuPointer<'a, &'a [u8]>, + pub(crate) microsequence_size: u32, + pub(crate) notifier: GpuPointer<'a, event::Notifier::ver>, + pub(crate) buffer: GpuPointer<'a, fw::buffer::Info::ver>, + pub(crate) scene: GpuPointer<'a, fw::buffer::Scene::ver>, + pub(crate) unk_buffer_buf: GpuWeakPointer<[u8]>, + pub(crate) tvb_tilemap: GpuPointer<'a, &'a [u8]>, + pub(crate) ppp_multisamplectl: U64, + pub(crate) samples: u32, + pub(crate) tiles_per_mtile_y: u16, + pub(crate) tiles_per_mtile_x: u16, + pub(crate) unk_50: U64, + pub(crate) unk_58: U64, + pub(crate) isp_merge_upper_x: F32, + pub(crate) isp_merge_upper_y: F32, + pub(crate) unk_68: U64, + pub(crate) tile_count: U64, + + #[ver(G < G14X)] + pub(crate) job_params1: JobParameters1::ver<'a>, + #[ver(G < G14X)] + pub(crate) job_params2: JobParameters2, + #[ver(G >= G14X)] + pub(crate) registers: job::raw::RegisterArray, + + pub(crate) job_params3: JobParameters3::ver, + pub(crate) unk_758_flag: u32, + pub(crate) unk_75c_flag: u32, + pub(crate) unk_buf: Array<0x110, u8>, + pub(crate) busy_flag: u32, + pub(crate) tvb_overflow_count: u32, + pub(crate) unk_878: u32, + pub(crate) encoder_params: job::raw::EncoderParams, + pub(crate) process_empty_tiles: u32, + pub(crate) no_clear_pipeline_textures: u32, + pub(crate) msaa_zs: u32, + pub(crate) unk_pointee: u32, + #[ver(V >= V13_3)] + pub(crate) unk_v13_3: u32, + pub(crate) meta: job::raw::JobMeta, + pub(crate) unk_after_meta: u32, + pub(crate) unk_buf_0: U64, + pub(crate) unk_buf_8: U64, + pub(crate) unk_buf_10: U64, + pub(crate) command_time: U64, + pub(crate) timestamp_pointers: job::raw::TimestampPointers<'a>, + pub(crate) user_timestamp_pointers: job::raw::TimestampPointers<'a>, + pub(crate) client_sequence: u8, + pub(crate) pad_925: Array<3, u8>, + pub(crate) unk_928: u32, + pub(crate) unk_92c: u8, + + #[ver(V >= V13_0B4)] + pub(crate) unk_ts: U64, + + #[ver(V >= V13_0B4)] + pub(crate) unk_92d_8: Array<0x1b, u8>, + } +} + +#[versions(AGX)] +#[derive(Debug)] +pub(crate) struct RunFragment { + pub(crate) notifier: Arc>, + pub(crate) scene: Arc, + pub(crate) micro_seq: microseq::MicroSequence, + pub(crate) vm_bind: mmu::VmBind, + pub(crate) aux_fb: GpuArray, + pub(crate) timestamps: Arc>, + pub(crate) user_timestamps: job::UserTimestamps, +} + +#[versions(AGX)] +impl GpuStruct for RunFragment::ver { + type Raw<'a> = raw::RunFragment::ver<'a>; +} + +#[versions(AGX)] +impl workqueue::Command for RunFragment::ver {} diff --git a/drivers/gpu/drm/asahi/fw/initdata.rs b/drivers/gpu/drm/asahi/fw/initdata.rs new file mode 100644 index 00000000000000..43cd3f0dfc2560 --- /dev/null +++ b/drivers/gpu/drm/asahi/fw/initdata.rs @@ -0,0 +1,1351 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! GPU initialization / global structures + +use super::channels; +use super::types::*; +use crate::{ + default_zeroed, + gem, + mmu, + no_debug, + trivial_gpustruct, // +}; + +pub(crate) mod raw { + use super::*; + + #[derive(Debug, Default)] + #[repr(C)] + pub(crate) struct ChannelRing { + pub(crate) state: Option>, + pub(crate) ring: Option>, + } + + #[versions(AGX)] + #[derive(Debug)] + #[repr(C)] + pub(crate) struct PipeChannels { + pub(crate) vtx: ChannelRing, + pub(crate) frag: ChannelRing, + pub(crate) comp: ChannelRing, + } + #[versions(AGX)] + default_zeroed!(PipeChannels::ver); + + #[derive(Debug, Default)] + #[repr(C)] + pub(crate) struct FwStatusFlags { + pub(crate) halt_count: AtomicU64, + __pad0: Pad<0x8>, + pub(crate) halted: AtomicU32, + __pad1: Pad<0xc>, + pub(crate) resume: AtomicU32, + __pad2: Pad<0xc>, + pub(crate) unk_40: u32, + __pad3: Pad<0xc>, + pub(crate) unk_ctr: u32, + __pad4: Pad<0xc>, + pub(crate) unk_60: u32, + __pad5: Pad<0xc>, + pub(crate) unk_70: u32, + __pad6: Pad<0xc>, + } + + #[derive(Debug, Default)] + #[repr(C)] + pub(crate) struct FwStatus { + pub(crate) fwctl_channel: ChannelRing, + pub(crate) flags: FwStatusFlags, + } + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct HwDataShared1 { + pub(crate) table: Array<16, i32>, + pub(crate) unk_44: Array<0x60, u8>, + pub(crate) unk_a4: u32, + pub(crate) unk_a8: u32, + } + default_zeroed!(HwDataShared1); + + #[derive(Debug, Default)] + #[repr(C)] + pub(crate) struct HwDataShared2Curve { + pub(crate) unk_0: u32, + pub(crate) unk_4: u32, + pub(crate) t1: Array<16, u16>, + pub(crate) t2: Array<16, i16>, + pub(crate) t3: Array<8, Array<16, i32>>, + } + + #[derive(Debug, Default)] + #[repr(C)] + pub(crate) struct HwDataShared2G14 { + pub(crate) unk_0: Array<5, u32>, + pub(crate) unk_14: u32, + pub(crate) unk_18: Array<8, u32>, + pub(crate) curve1: HwDataShared2Curve, + pub(crate) curve2: HwDataShared2Curve, + } + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct HwDataShared2 { + pub(crate) table: Array<10, i32>, + pub(crate) unk_28: Array<0x10, u8>, + pub(crate) g14: HwDataShared2G14, + pub(crate) unk_500: u32, + pub(crate) unk_504: u32, + pub(crate) unk_508: u32, + pub(crate) unk_50c: u32, + } + default_zeroed!(HwDataShared2); + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct HwDataShared3 { + pub(crate) unk_0: u32, + pub(crate) unk_4: u32, + pub(crate) unk_8: u32, + pub(crate) table: Array<16, u32>, + pub(crate) unk_4c: u32, + } + default_zeroed!(HwDataShared3); + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct HwDataA130Extra { + pub(crate) unk_0: Array<0x38, u8>, + pub(crate) unk_38: u32, + pub(crate) unk_3c: u32, + pub(crate) gpu_se_inactive_threshold: u32, + pub(crate) unk_44: u32, + pub(crate) gpu_se_engagement_criteria: i32, + pub(crate) gpu_se_reset_criteria: u32, + pub(crate) unk_50: u32, + pub(crate) unk_54: u32, + pub(crate) unk_58: u32, + pub(crate) unk_5c: u32, + pub(crate) gpu_se_filter_a_neg: F32, + pub(crate) gpu_se_filter_1_a_neg: F32, + pub(crate) gpu_se_filter_a: F32, + pub(crate) gpu_se_filter_1_a: F32, + pub(crate) gpu_se_ki_dt: F32, + pub(crate) gpu_se_ki_1_dt: F32, + pub(crate) unk_78: F32, + pub(crate) unk_7c: F32, + pub(crate) gpu_se_kp: F32, + pub(crate) gpu_se_kp_1: F32, + pub(crate) unk_88: u32, + pub(crate) unk_8c: u32, + pub(crate) max_pstate_scaled_1: u32, + pub(crate) unk_94: u32, + pub(crate) unk_98: u32, + pub(crate) unk_9c: F32, + pub(crate) unk_a0: u32, + pub(crate) unk_a4: u32, + pub(crate) gpu_se_filter_time_constant_ms: u32, + pub(crate) gpu_se_filter_time_constant_1_ms: u32, + pub(crate) gpu_se_filter_time_constant_clks: U64, + pub(crate) gpu_se_filter_time_constant_1_clks: U64, + pub(crate) unk_c0: u32, + pub(crate) unk_c4: F32, + pub(crate) unk_c8: Array<0x4c, u8>, + pub(crate) unk_114: F32, + pub(crate) unk_118: u32, + pub(crate) unk_11c: u32, + pub(crate) unk_120: u32, + pub(crate) unk_124: u32, + pub(crate) max_pstate_scaled_2: u32, + pub(crate) unk_12c: Array<0x8c, u8>, + } + default_zeroed!(HwDataA130Extra); + + #[repr(C)] + pub(crate) struct T81xxData { + pub(crate) unk_d8c: u32, + pub(crate) unk_d90: u32, + pub(crate) unk_d94: u32, + pub(crate) unk_d98: u32, + pub(crate) unk_d9c: F32, + pub(crate) unk_da0: u32, + pub(crate) unk_da4: F32, + pub(crate) unk_da8: u32, + pub(crate) unk_dac: F32, + pub(crate) unk_db0: u32, + pub(crate) unk_db4: u32, + pub(crate) unk_db8: F32, + pub(crate) unk_dbc: F32, + pub(crate) unk_dc0: u32, + pub(crate) unk_dc4: u32, + pub(crate) unk_dc8: u32, + pub(crate) max_pstate_scaled: u32, + } + default_zeroed!(T81xxData); + + #[versions(AGX)] + #[derive(Default, Copy, Clone)] + #[repr(C)] + pub(crate) struct PowerZone { + pub(crate) val: F32, + pub(crate) target: u32, + pub(crate) target_off: u32, + pub(crate) filter_tc_x4: u32, + pub(crate) filter_tc_xperiod: u32, + #[ver(V >= V13_0B4)] + pub(crate) unk_10: u32, + #[ver(V >= V13_0B4)] + pub(crate) unk_14: u32, + pub(crate) filter_a_neg: F32, + pub(crate) filter_a: F32, + pub(crate) pad: u32, + } + + #[versions(AGX)] + const MAX_CORES_PER_CLUSTER: usize = { + #[ver(G >= G14X)] + { + 16 + } + #[ver(G < G14X)] + { + 8 + } + }; + + #[derive(Debug, Default)] + #[repr(C)] + pub(crate) struct AuxLeakCoef { + pub(crate) afr_1: Array<2, F32>, + pub(crate) cs_1: Array<2, F32>, + pub(crate) afr_2: Array<2, F32>, + pub(crate) cs_2: Array<2, F32>, + } + + #[versions(AGX)] + #[repr(C)] + pub(crate) struct HwDataA { + pub(crate) unk_0: u32, + pub(crate) clocks_per_period: u32, + + #[ver(V >= V13_0B4)] + pub(crate) clocks_per_period_2: u32, + + pub(crate) unk_8: u32, + pub(crate) pwr_status: AtomicU32, + pub(crate) unk_10: F32, + pub(crate) unk_14: u32, + pub(crate) unk_18: u32, + pub(crate) unk_1c: u32, + pub(crate) unk_20: u32, + pub(crate) unk_24: u32, + pub(crate) actual_pstate: u32, + pub(crate) tgt_pstate: u32, + pub(crate) unk_30: u32, + pub(crate) cur_pstate: u32, + pub(crate) unk_38: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_3c_0: u32, + + pub(crate) base_pstate_scaled: u32, + pub(crate) unk_40: u32, + pub(crate) max_pstate_scaled: u32, + pub(crate) unk_48: u32, + pub(crate) min_pstate_scaled: u32, + pub(crate) freq_mhz: F32, + pub(crate) unk_54: Array<0x20, u8>, + + #[ver(V >= V13_0B4)] + pub(crate) unk_74_0: u32, + + pub(crate) sram_k: Array<0x10, F32>, + pub(crate) unk_b4: Array<0x100, u8>, + pub(crate) unk_1b4: u32, + pub(crate) temp_c: u32, + pub(crate) avg_power_mw: u32, + pub(crate) update_ts: U64, + pub(crate) unk_1c8: u32, + pub(crate) unk_1cc: Array<0x478, u8>, + pub(crate) pad_644: Pad<0x8>, + pub(crate) unk_64c: u32, + pub(crate) unk_650: u32, + pub(crate) pad_654: u32, + pub(crate) pwr_filter_a_neg: F32, + pub(crate) pad_65c: u32, + pub(crate) pwr_filter_a: F32, + pub(crate) pad_664: u32, + pub(crate) pwr_integral_gain: F32, + pub(crate) pad_66c: u32, + pub(crate) pwr_integral_min_clamp: F32, + pub(crate) max_power_1: F32, + pub(crate) pwr_proportional_gain: F32, + pub(crate) pad_67c: u32, + pub(crate) pwr_pstate_related_k: F32, + pub(crate) pwr_pstate_max_dc_offset: i32, + pub(crate) unk_688: u32, + pub(crate) max_pstate_scaled_2: u32, + pub(crate) pad_690: u32, + pub(crate) unk_694: u32, + pub(crate) max_power_2: u32, + pub(crate) pad_69c: Pad<0x18>, + pub(crate) unk_6b4: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_6b8_0: Array<0x10, u8>, + + pub(crate) max_pstate_scaled_3: u32, + pub(crate) unk_6bc: u32, + pub(crate) pad_6c0: Pad<0x14>, + pub(crate) ppm_filter_tc_periods_x4: u32, + pub(crate) unk_6d8: u32, + pub(crate) pad_6dc: u32, + pub(crate) ppm_filter_a_neg: F32, + pub(crate) pad_6e4: u32, + pub(crate) ppm_filter_a: F32, + pub(crate) pad_6ec: u32, + pub(crate) ppm_ki_dt: F32, + pub(crate) pad_6f4: u32, + pub(crate) pwr_integral_min_clamp_2: u32, + pub(crate) unk_6fc: F32, + pub(crate) ppm_kp: F32, + pub(crate) pad_704: u32, + pub(crate) unk_708: u32, + pub(crate) pwr_min_duty_cycle: u32, + pub(crate) max_pstate_scaled_4: u32, + pub(crate) unk_714: u32, + pub(crate) pad_718: u32, + pub(crate) unk_71c: F32, + pub(crate) max_power_3: u32, + pub(crate) cur_power_mw_2: u32, + pub(crate) ppm_filter_tc_ms: u32, + pub(crate) unk_72c: u32, + + #[ver(V >= V13_0B4)] + pub(crate) ppm_filter_tc_clks: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_730_4: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_730_8: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_730_c: u32, + + pub(crate) unk_730: F32, + pub(crate) unk_734: u32, + pub(crate) unk_738: u32, + pub(crate) unk_73c: u32, + pub(crate) unk_740: u32, + pub(crate) unk_744: u32, + pub(crate) unk_748: Array<0x4, F32>, + pub(crate) unk_758: u32, + pub(crate) perf_tgt_utilization: u32, + pub(crate) pad_760: u32, + pub(crate) perf_boost_min_util: u32, + pub(crate) perf_boost_ce_step: u32, + pub(crate) perf_reset_iters: u32, + pub(crate) pad_770: u32, + pub(crate) unk_774: u32, + pub(crate) unk_778: u32, + pub(crate) perf_filter_drop_threshold: u32, + pub(crate) perf_filter_a_neg: F32, + pub(crate) perf_filter_a2_neg: F32, + pub(crate) perf_filter_a: F32, + pub(crate) perf_filter_a2: F32, + pub(crate) perf_ki: F32, + pub(crate) perf_ki2: F32, + pub(crate) perf_integral_min_clamp: F32, + pub(crate) unk_79c: F32, + pub(crate) perf_kp: F32, + pub(crate) perf_kp2: F32, + pub(crate) boost_state_unk_k: F32, + pub(crate) base_pstate_scaled_2: u32, + pub(crate) max_pstate_scaled_5: u32, + pub(crate) base_pstate_scaled_3: u32, + pub(crate) pad_7b8: u32, + pub(crate) perf_cur_utilization: F32, + pub(crate) perf_tgt_utilization_2: u32, + pub(crate) pad_7c4: Pad<0x18>, + pub(crate) unk_7dc: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_7e0_0: Array<0x10, u8>, + + pub(crate) base_pstate_scaled_4: u32, + pub(crate) pad_7e4: u32, + pub(crate) unk_7e8: Array<0x14, u8>, + pub(crate) unk_7fc: F32, + pub(crate) pwr_min_duty_cycle_2: F32, + pub(crate) max_pstate_scaled_6: F32, + pub(crate) max_freq_mhz: u32, + pub(crate) pad_80c: u32, + pub(crate) unk_810: u32, + pub(crate) pad_814: u32, + pub(crate) pwr_min_duty_cycle_3: u32, + pub(crate) unk_81c: u32, + pub(crate) pad_820: u32, + pub(crate) min_pstate_scaled_4: F32, + pub(crate) max_pstate_scaled_7: u32, + pub(crate) unk_82c: u32, + pub(crate) unk_alpha_neg: F32, + pub(crate) unk_alpha: F32, + pub(crate) unk_838: u32, + pub(crate) unk_83c: u32, + pub(crate) pad_840: Pad<0x2c>, + pub(crate) unk_86c: u32, + pub(crate) fast_die0_sensor_mask: U64, + #[ver(G >= G14X)] + pub(crate) fast_die1_sensor_mask: U64, + pub(crate) fast_die0_release_temp_cc: u32, + pub(crate) unk_87c: i32, + pub(crate) unk_880: u32, + pub(crate) unk_884: u32, + pub(crate) pad_888: u32, + pub(crate) unk_88c: u32, + pub(crate) pad_890: u32, + pub(crate) unk_894: F32, + pub(crate) pad_898: u32, + pub(crate) fast_die0_ki_dt: F32, + pub(crate) pad_8a0: u32, + pub(crate) unk_8a4: u32, + pub(crate) unk_8a8: F32, + pub(crate) fast_die0_kp: F32, + pub(crate) pad_8b0: u32, + pub(crate) unk_8b4: u32, + pub(crate) pwr_min_duty_cycle_4: u32, + pub(crate) max_pstate_scaled_8: u32, + pub(crate) max_pstate_scaled_9: u32, + pub(crate) fast_die0_prop_tgt_delta: u32, + pub(crate) unk_8c8: u32, + pub(crate) unk_8cc: u32, + pub(crate) pad_8d0: Pad<0x14>, + + #[ver(V >= V13_0B4)] + pub(crate) unk_8e4_0: Array<0x10, u8>, + + pub(crate) unk_8e4: u32, + pub(crate) unk_8e8: u32, + pub(crate) max_pstate_scaled_10: u32, + pub(crate) unk_8f0: u32, + pub(crate) unk_8f4: u32, + pub(crate) pad_8f8: u32, + pub(crate) pad_8fc: u32, + pub(crate) unk_900: Array<0x24, u8>, + + pub(crate) unk_coef_a1: Array<8, Array>, + pub(crate) unk_coef_a2: Array<8, Array>, + + pub(crate) pad_b24: Pad<0x70>, + pub(crate) max_pstate_scaled_11: u32, + pub(crate) freq_with_off: u32, + pub(crate) unk_b9c: u32, + pub(crate) unk_ba0: U64, + pub(crate) unk_ba8: U64, + pub(crate) unk_bb0: u32, + pub(crate) unk_bb4: u32, + + #[ver(V >= V13_3)] + pub(crate) pad_bb8_0: Pad<0x200>, + #[ver(V >= V13_5)] + pub(crate) pad_bb8_200: Pad<0x8>, + + pub(crate) pad_bb8: Pad<0x74>, + pub(crate) unk_c2c: u32, + pub(crate) power_zone_count: u32, + pub(crate) max_power_4: u32, + pub(crate) max_power_5: u32, + pub(crate) max_power_6: u32, + pub(crate) unk_c40: u32, + pub(crate) unk_c44: F32, + pub(crate) avg_power_target_filter_a_neg: F32, + pub(crate) avg_power_target_filter_a: F32, + pub(crate) avg_power_target_filter_tc_x4: u32, + pub(crate) avg_power_target_filter_tc_xperiod: u32, + + #[ver(V >= V13_0B4)] + pub(crate) avg_power_target_filter_tc_clks: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_c58_4: u32, + + pub(crate) power_zones: Array<5, PowerZone::ver>, + pub(crate) avg_power_filter_tc_periods_x4: u32, + pub(crate) unk_cfc: u32, + pub(crate) unk_d00: u32, + pub(crate) avg_power_filter_a_neg: F32, + pub(crate) unk_d08: u32, + pub(crate) avg_power_filter_a: F32, + pub(crate) unk_d10: u32, + pub(crate) avg_power_ki_dt: F32, + pub(crate) unk_d18: u32, + pub(crate) unk_d1c: u32, + pub(crate) unk_d20: F32, + pub(crate) avg_power_kp: F32, + pub(crate) unk_d28: u32, + pub(crate) unk_d2c: u32, + pub(crate) avg_power_min_duty_cycle: u32, + pub(crate) max_pstate_scaled_12: u32, + pub(crate) max_pstate_scaled_13: u32, + pub(crate) unk_d3c: u32, + pub(crate) max_power_7: F32, + pub(crate) max_power_8: u32, + pub(crate) unk_d48: u32, + pub(crate) avg_power_filter_tc_ms: u32, + pub(crate) unk_d50: u32, + + #[ver(V >= V13_0B4)] + pub(crate) avg_power_filter_tc_clks: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_d54_4: Array<0xc, u8>, + + pub(crate) unk_d54: Array<0x10, u8>, + pub(crate) max_pstate_scaled_14: u32, + pub(crate) unk_d68: Array<0x24, u8>, + + pub(crate) t81xx_data: T81xxData, + + pub(crate) unk_dd0: Array<0x40, u8>, + + #[ver(V >= V13_2)] + pub(crate) unk_e10_pad: Array<0x10, u8>, + + #[ver(V >= V13_0B4)] + pub(crate) unk_e10_0: HwDataA130Extra, + + pub(crate) unk_e10: Array<0xc, u8>, + + pub(crate) fast_die0_sensor_mask_2: U64, + #[ver(G >= G14X)] + pub(crate) fast_die1_sensor_mask_2: U64, + + pub(crate) unk_e24: u32, + pub(crate) unk_e28: u32, + pub(crate) unk_e2c: Pad<0x1c>, + pub(crate) unk_coef_b1: Array<8, Array>, + pub(crate) unk_coef_b2: Array<8, Array>, + + #[ver(G >= G14X)] + pub(crate) pad_1048_0: Pad<0x600>, + + pub(crate) pad_1048: Pad<0x5e4>, + + pub(crate) fast_die0_sensor_mask_alt: U64, + #[ver(G >= G14X)] + pub(crate) fast_die1_sensor_mask_alt: U64, + #[ver(V < V13_0B4)] + pub(crate) fast_die0_sensor_present: U64, + + pub(crate) unk_163c: u32, + + pub(crate) unk_1640: Array<0x2000, u8>, + + #[ver(G >= G14X)] + pub(crate) unk_3640_0: Array<0x2000, u8>, + + pub(crate) unk_3640: u32, + pub(crate) unk_3644: u32, + pub(crate) hws1: HwDataShared1, + + #[ver(V >= V13_0B4)] + pub(crate) unk_hws2: Array<16, u16>, + + pub(crate) hws2: HwDataShared2, + pub(crate) unk_3c00: u32, + pub(crate) unk_3c04: u32, + pub(crate) hws3: HwDataShared3, + pub(crate) unk_3c58: Array<0x3c, u8>, + pub(crate) unk_3c94: u32, + pub(crate) unk_3c98: U64, + pub(crate) unk_3ca0: U64, + pub(crate) unk_3ca8: U64, + pub(crate) unk_3cb0: U64, + pub(crate) ts_last_idle: U64, + pub(crate) ts_last_poweron: U64, + pub(crate) ts_last_poweroff: U64, + pub(crate) unk_3cd0: U64, + pub(crate) unk_3cd8: U64, + + #[ver(V >= V13_0B4)] + pub(crate) unk_3ce0_0: u32, + + pub(crate) unk_3ce0: u32, + pub(crate) unk_3ce4: u32, + pub(crate) unk_3ce8: u32, + pub(crate) unk_3cec: u32, + pub(crate) unk_3cf0: u32, + pub(crate) core_leak_coef: Array<8, F32>, + pub(crate) sram_leak_coef: Array<8, F32>, + + #[ver(V >= V13_0B4)] + pub(crate) aux_leak_coef: AuxLeakCoef, + #[ver(V >= V13_0B4)] + pub(crate) unk_3d34_0: Array<0x18, u8>, + + pub(crate) unk_3d34: Array<0x38, u8>, + } + #[versions(AGX)] + default_zeroed!(HwDataA::ver); + #[versions(AGX)] + no_debug!(HwDataA::ver); + + #[derive(Debug, Default, Clone, Copy)] + #[repr(C)] + pub(crate) struct IOMapping { + pub(crate) phys_addr: U64, + pub(crate) virt_addr: U64, + pub(crate) total_size: u32, + pub(crate) element_size: u32, + pub(crate) readwrite: U64, + } + + #[versions(AGX)] + const IO_MAPPING_COUNT: usize = { + #[ver(V < V13_0B4)] + { + 0x14 + } + #[ver(V >= V13_0B4 && V < V13_3)] + { + 0x17 + } + #[ver(V >= V13_3 && V < V13_5)] + { + 0x18 + } + #[ver(V >= V13_5)] + { + 0x19 + } + }; + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct HwDataBAuxPStates { + pub(crate) cs_max_pstate: u32, + pub(crate) cs_frequencies: Array<0x10, u32>, + pub(crate) cs_voltages: Array<0x10, Array<0x2, u32>>, + pub(crate) cs_voltages_sram: Array<0x10, Array<0x2, u32>>, + pub(crate) cs_unkpad: u32, + pub(crate) afr_max_pstate: u32, + pub(crate) afr_frequencies: Array<0x8, u32>, + pub(crate) afr_voltages: Array<0x8, Array<0x2, u32>>, + pub(crate) afr_voltages_sram: Array<0x8, Array<0x2, u32>>, + pub(crate) afr_unkpad: u32, + } + + #[versions(AGX)] + #[derive(Debug)] + #[repr(C)] + pub(crate) struct HwDataB { + #[ver(V < V13_0B4)] + pub(crate) unk_0: U64, + + pub(crate) unk_8: U64, + + #[ver(V < V13_0B4)] + pub(crate) unk_10: U64, + + pub(crate) unk_18: U64, + pub(crate) unk_20: U64, + pub(crate) unk_28: U64, + pub(crate) unk_30: U64, + pub(crate) timestamp_area_base: U64, + pub(crate) pad_40: Pad<0x20>, + + #[ver(V < V13_0B4)] + pub(crate) yuv_matrices: Array<0xf, Array<3, Array<4, i16>>>, + + #[ver(V >= V13_0B4)] + pub(crate) yuv_matrices: Array<0x3f, Array<3, Array<4, i16>>>, + + pub(crate) pad_1c8: Pad<0x8>, + pub(crate) io_mappings: Array, + + #[ver(V >= V13_0B4)] + pub(crate) sgx_sram_ptr: U64, + + pub(crate) chip_id: u32, + pub(crate) unk_454: u32, + pub(crate) unk_458: u32, + pub(crate) unk_45c: u32, + pub(crate) unk_460: u32, + pub(crate) unk_464: u32, + pub(crate) unk_468: u32, + pub(crate) unk_46c: u32, + pub(crate) unk_470: u32, + pub(crate) unk_474: u32, + pub(crate) unk_478: u32, + pub(crate) unk_47c: u32, + pub(crate) unk_480: u32, + pub(crate) unk_484: u32, + pub(crate) unk_488: u32, + pub(crate) unk_48c: u32, + pub(crate) base_clock_khz: u32, + pub(crate) power_sample_period: u32, + pub(crate) pad_498: Pad<0x4>, + pub(crate) unk_49c: u32, + pub(crate) unk_4a0: u32, + pub(crate) unk_4a4: u32, + pub(crate) pad_4a8: Pad<0x4>, + pub(crate) unk_4ac: u32, + pub(crate) pad_4b0: Pad<0x8>, + pub(crate) unk_4b8: u32, + pub(crate) unk_4bc: Array<0x4, u8>, + pub(crate) unk_4c0: u32, + pub(crate) unk_4c4: u32, + pub(crate) unk_4c8: u32, + pub(crate) unk_4cc: u32, + pub(crate) unk_4d0: u32, + pub(crate) unk_4d4: u32, + pub(crate) unk_4d8: Array<0x4, u8>, + pub(crate) unk_4dc: u32, + pub(crate) unk_4e0: U64, + pub(crate) unk_4e8: u32, + pub(crate) unk_4ec: u32, + pub(crate) unk_4f0: u32, + pub(crate) unk_4f4: u32, + pub(crate) unk_4f8: u32, + pub(crate) unk_4fc: u32, + pub(crate) unk_500: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_504_0: u32, + + pub(crate) unk_504: u32, + pub(crate) unk_508: u32, + pub(crate) unk_50c: u32, + pub(crate) unk_510: u32, + pub(crate) unk_514: u32, + pub(crate) unk_518: u32, + pub(crate) unk_51c: u32, + pub(crate) unk_520: u32, + pub(crate) unk_524: u32, + pub(crate) unk_528: u32, + pub(crate) unk_52c: u32, + pub(crate) unk_530: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_534_0: u32, + + pub(crate) unk_534: u32, + pub(crate) unk_538: u32, + + pub(crate) num_frags: u32, + pub(crate) unk_540: u32, + pub(crate) unk_544: u32, + pub(crate) unk_548: u32, + pub(crate) unk_54c: u32, + pub(crate) unk_550: u32, + pub(crate) unk_554: u32, + pub(crate) uat_ttb_base: U64, + pub(crate) gpu_core_id: u32, + pub(crate) gpu_rev_id: u32, + pub(crate) num_cores: u32, + pub(crate) max_pstate: u32, + + #[ver(V < V13_0B4)] + pub(crate) num_pstates: u32, + + pub(crate) frequencies: Array<0x10, u32>, + pub(crate) voltages: Array<0x10, [u32; 0x8]>, + pub(crate) voltages_sram: Array<0x10, [u32; 0x8]>, + + #[ver(V >= V13_3)] + pub(crate) unk_9f4_0: Pad<64>, + + pub(crate) sram_k: Array<0x10, F32>, + pub(crate) unk_9f4: Array<0x10, u32>, + pub(crate) rel_max_powers: Array<0x10, u32>, + pub(crate) rel_boost_freqs: Array<0x10, u32>, + + #[ver(V >= V13_3)] + pub(crate) unk_arr_0: Array<32, u32>, + + #[ver(V < V13_0B4)] + pub(crate) min_sram_volt: u32, + + #[ver(V < V13_0B4)] + pub(crate) unk_ab8: u32, + + #[ver(V < V13_0B4)] + pub(crate) unk_abc: u32, + + #[ver(V < V13_0B4)] + pub(crate) unk_ac0: u32, + + #[ver(V >= V13_0B4)] + pub(crate) aux_ps: HwDataBAuxPStates, + + #[ver(V >= V13_3)] + pub(crate) pad_ac4_0: Array<0x44c, u8>, + + pub(crate) pad_ac4: Pad<0x8>, + pub(crate) unk_acc: u32, + pub(crate) unk_ad0: u32, + pub(crate) pad_ad4: Pad<0x10>, + pub(crate) unk_ae4: Array<0x4, u32>, + pub(crate) pad_af4: Pad<0x4>, + pub(crate) unk_af8: u32, + pub(crate) pad_afc: Pad<0x8>, + pub(crate) unk_b04: u32, + pub(crate) unk_b08: u32, + pub(crate) unk_b0c: u32, + + #[ver(G >= G14X)] + pub(crate) pad_b10_0: Array<0x8, u8>, + + pub(crate) unk_b10: u32, + pub(crate) timer_offset: U64, + pub(crate) unk_b1c: u32, + pub(crate) unk_b20: u32, + pub(crate) unk_b24: u32, + pub(crate) unk_b28: u32, + pub(crate) unk_b2c: u32, + pub(crate) unk_b30: u32, + pub(crate) unk_b34: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_b38_0: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_b38_4: u32, + + #[ver(V >= V13_3)] + pub(crate) unk_b38_8: u32, + + pub(crate) unk_b38: Array<0xc, u32>, + pub(crate) unk_b68: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_b6c: Array<0xd0, u8>, + + #[ver(G >= G14X)] + pub(crate) unk_c3c_0: Array<0x8, u8>, + + #[ver(G < G14X && V >= V13_5)] + pub(crate) unk_c3c_8: Array<0x10, u8>, + + #[ver(V >= V13_5)] + pub(crate) unk_c3c_18: Array<0x20, u8>, + + #[ver(V >= V13_0B4)] + pub(crate) unk_c3c: u32, + } + #[versions(AGX)] + default_zeroed!(HwDataB::ver); + + #[derive(Debug)] + #[repr(C, packed)] + pub(crate) struct GpuStatsVtx { + // This changes all the time and we don't use it, let's just make it a big buffer + pub(crate) opaque: Array<0x3000, u8>, + } + default_zeroed!(GpuStatsVtx); + + #[versions(AGX)] + #[derive(Debug)] + #[repr(C)] + pub(crate) struct GpuStatsFrag { + // This changes all the time and we don't use it, let's just make it a big buffer + // except for these two fields which may need init. + #[ver(G >= G14X)] + pub(crate) unk1_0: Array<0x910, u8>, + pub(crate) unk1: Array<0x100, u8>, + pub(crate) cur_stamp_id: i32, + pub(crate) unk2: Array<0x14, u8>, + pub(crate) unk_id: i32, + pub(crate) unk3: Array<0x1000, u8>, + } + + #[versions(AGX)] + impl Default for GpuStatsFrag::ver { + fn default() -> Self { + Self { + #[ver(G >= G14X)] + unk1_0: Default::default(), + unk1: Default::default(), + cur_stamp_id: -1, + unk2: Default::default(), + unk_id: -1, + unk3: Default::default(), + } + } + } + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct GpuGlobalStatsVtx { + pub(crate) total_cmds: u32, + pub(crate) stats: GpuStatsVtx, + } + default_zeroed!(GpuGlobalStatsVtx); + + #[versions(AGX)] + #[derive(Debug, Default)] + #[repr(C)] + pub(crate) struct GpuGlobalStatsFrag { + pub(crate) total_cmds: u32, + pub(crate) unk_4: u32, + pub(crate) stats: GpuStatsFrag::ver, + } + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct GpuStatsComp { + // This changes all the time and we don't use it, let's just make it a big buffer + pub(crate) opaque: Array<0x3000, u8>, + } + default_zeroed!(GpuStatsComp); + + #[versions(AGX)] + #[derive(Debug)] + #[repr(C)] + pub(crate) struct RuntimeScratch { + pub(crate) unk_280: Array<0x6800, u8>, + pub(crate) unk_6a80: u32, + pub(crate) gpu_idle: u32, + pub(crate) unkpad_6a88: Pad<0x14>, + pub(crate) unk_6a9c: u32, + pub(crate) unk_ctr0: u32, + pub(crate) unk_ctr1: u32, + pub(crate) unk_6aa8: u32, + pub(crate) unk_6aac: u32, + pub(crate) unk_ctr2: u32, + pub(crate) unk_6ab4: u32, + pub(crate) unk_6ab8: u32, + pub(crate) unk_6abc: u32, + pub(crate) unk_6ac0: u32, + pub(crate) unk_6ac4: u32, + pub(crate) unk_ctr3: u32, + pub(crate) unk_6acc: u32, + pub(crate) unk_6ad0: u32, + pub(crate) unk_6ad4: u32, + pub(crate) unk_6ad8: u32, + pub(crate) unk_6adc: u32, + pub(crate) unk_6ae0: u32, + pub(crate) unk_6ae4: u32, + pub(crate) unk_6ae8: u32, + pub(crate) unk_6aec: u32, + pub(crate) unk_6af0: u32, + pub(crate) unk_ctr4: u32, + pub(crate) unk_ctr5: u32, + pub(crate) unk_6afc: u32, + pub(crate) pad_6b00: Pad<0x38>, + + #[ver(G >= G14X)] + pub(crate) pad_6b00_extra: Array<0x4800, u8>, + + pub(crate) unk_6b38: u32, + pub(crate) pad_6b3c: Pad<0x84>, + } + #[versions(AGX)] + default_zeroed!(RuntimeScratch::ver); + + #[versions(AGX)] + #[repr(C)] + pub(crate) struct RuntimePointers<'a> { + pub(crate) pipes: Array<4, PipeChannels::ver>, + + pub(crate) device_control: + ChannelRing, + pub(crate) event: ChannelRing, + pub(crate) fw_log: ChannelRing, + pub(crate) ktrace: ChannelRing, + pub(crate) stats: ChannelRing, + + pub(crate) __pad0: Pad<0x50>, + pub(crate) unk_160: U64, + pub(crate) unk_168: U64, + pub(crate) stats_vtx: GpuPointer<'a, super::GpuGlobalStatsVtx>, + pub(crate) stats_frag: GpuPointer<'a, super::GpuGlobalStatsFrag::ver>, + pub(crate) stats_comp: GpuPointer<'a, super::GpuStatsComp>, + pub(crate) hwdata_a: GpuPointer<'a, super::HwDataA::ver>, + pub(crate) unkptr_190: GpuPointer<'a, &'a [u8]>, + pub(crate) unkptr_198: GpuPointer<'a, &'a [u8]>, + pub(crate) hwdata_b: GpuPointer<'a, super::HwDataB::ver>, + pub(crate) hwdata_b_2: GpuPointer<'a, super::HwDataB::ver>, + pub(crate) fwlog_buf: Option>, + pub(crate) unkptr_1b8: GpuPointer<'a, &'a [u8]>, + + #[ver(G < G14X)] + pub(crate) unkptr_1c0: GpuPointer<'a, &'a [u8]>, + #[ver(G < G14X)] + pub(crate) unkptr_1c8: GpuPointer<'a, &'a [u8]>, + + pub(crate) unk_1d0: u32, + pub(crate) unk_1d4: u32, + pub(crate) unk_1d8: Array<0x3c, u8>, + pub(crate) buffer_mgr_ctl_gpu_addr: U64, + pub(crate) buffer_mgr_ctl_fw_addr: U64, + pub(crate) __pad1: Pad<0x5c>, + pub(crate) gpu_scratch: RuntimeScratch::ver, + } + #[versions(AGX)] + no_debug!(RuntimePointers::ver<'_>); + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct PendingStamp { + pub(crate) info: AtomicU32, + pub(crate) wait_value: AtomicU32, + } + default_zeroed!(PendingStamp); + + #[derive(Debug, Clone, Copy)] + #[repr(C, packed)] + pub(crate) struct FaultInfo { + pub(crate) unk_0: u32, + pub(crate) unk_4: u32, + pub(crate) queue_uuid: u32, + pub(crate) unk_c: u32, + pub(crate) unk_10: u32, + pub(crate) unk_14: u32, + } + default_zeroed!(FaultInfo); + + #[derive(Debug, Clone, Copy)] + #[repr(C)] + pub(crate) struct PowerZoneGlobal { + pub(crate) target: u32, + pub(crate) target_off: u32, + pub(crate) filter_tc: u32, + } + default_zeroed!(PowerZoneGlobal); + + #[versions(AGX)] + #[derive(Debug)] + #[repr(C)] + pub(crate) struct Globals { + pub(crate) ktrace_enable: u32, + pub(crate) unk_4: Array<0x20, u8>, + + #[ver(V >= V13_2)] + pub(crate) unk_24_0: u32, + + pub(crate) unk_24: u32, + + #[ver(V >= V13_0B4)] + pub(crate) debug: u32, + + #[ver(V >= V13_3)] + pub(crate) unk_28_4: u32, + + pub(crate) unk_28: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_2c_0: u32, + + pub(crate) unk_2c: u32, + pub(crate) unk_30: u32, + pub(crate) unk_34: u32, + pub(crate) unk_38: Array<0x1c, u8>, + + // pub(crate) sub: GlobalsSub::ver, + pub(crate) unk_54: u16, + pub(crate) unk_56: u16, + pub(crate) unk_58: u16, + pub(crate) unk_5a: U32, + pub(crate) unk_5e: U32, + pub(crate) unk_62: U32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_66_0: Array<0xc, u8>, + + pub(crate) unk_66: U32, + pub(crate) unk_6a: Array<0x16, u8>, + // end GlobalsSub::ver + pub(crate) unk_80: Array<0xf80, u8>, + pub(crate) unk_1000: Array<0x7000, u8>, + pub(crate) unk_8000: Array<0x900, u8>, + + #[ver(G >= G14X)] + pub(crate) unk_8900_pad: Array<0x484c, u8>, + + #[ver(V >= V13_3)] + pub(crate) unk_8900_pad2: Array<0x54, u8>, + + pub(crate) unk_8900: u32, + pub(crate) pending_submissions: AtomicU32, + pub(crate) max_power: u32, + pub(crate) max_pstate_scaled: u32, + pub(crate) max_pstate_scaled_2: u32, + pub(crate) unk_8914: u32, + pub(crate) unk_8918: u32, + pub(crate) max_pstate_scaled_3: u32, + pub(crate) unk_8920: u32, + pub(crate) power_zone_count: u32, + pub(crate) avg_power_filter_tc_periods: u32, + pub(crate) avg_power_ki_dt: F32, + pub(crate) avg_power_kp: F32, + pub(crate) avg_power_min_duty_cycle: u32, + pub(crate) avg_power_target_filter_tc: u32, + pub(crate) power_zones: Array<5, PowerZoneGlobal>, + pub(crate) unk_8978: Array<0x44, u8>, + + #[ver(V >= V13_0B4)] + pub(crate) unk_89bc_0: Array<0x3c, u8>, + + pub(crate) unk_89bc: u32, + pub(crate) fast_die0_release_temp: u32, + pub(crate) unk_89c4: i32, + pub(crate) fast_die0_prop_tgt_delta: u32, + pub(crate) fast_die0_kp: F32, + pub(crate) fast_die0_ki_dt: F32, + pub(crate) unk_89d4: Array<0xc, u8>, + pub(crate) unk_89e0: u32, + pub(crate) max_power_2: u32, + pub(crate) ppm_kp: F32, + pub(crate) ppm_ki_dt: F32, + pub(crate) unk_89f0: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_89f4_0: Array<0x8, u8>, + + #[ver(V >= V13_0B4)] + pub(crate) unk_89f4_8: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_89f4_c: Array<0x50, u8>, + + #[ver(V >= V13_3)] + pub(crate) unk_89f4_5c: Array<0xc, u8>, + + pub(crate) unk_89f4: u32, + pub(crate) hws1: HwDataShared1, + pub(crate) hws2: HwDataShared2, + + #[ver(V >= V13_0B4)] + pub(crate) idle_off_standby_timer: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_hws2_4: Array<0x8, F32>, + + #[ver(V >= V13_0B4)] + pub(crate) unk_hws2_24: u32, + + pub(crate) unk_hws2_28: u32, + + pub(crate) hws3: HwDataShared3, + pub(crate) unk_9004: Array<8, u8>, + pub(crate) unk_900c: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_9010_0: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_9010_4: Array<0x14, u8>, + + pub(crate) unk_9010: Array<0x2c, u8>, + pub(crate) unk_903c: u32, + pub(crate) unk_9040: Array<0xc0, u8>, + pub(crate) unk_9100: Array<0x6f00, u8>, + pub(crate) unk_10000: Array<0xe50, u8>, + pub(crate) unk_10e50: u32, + pub(crate) unk_10e54: Array<0x2c, u8>, + + #[ver((G >= G14X && V < V13_3) || (G <= G14 && V >= V13_3))] + pub(crate) unk_x_pad: Array<0x4, u8>, + + // bit 0: sets sgx_reg 0x17620 + // bit 1: sets sgx_reg 0x17630 + pub(crate) fault_control: u32, + pub(crate) do_init: u32, + pub(crate) unk_10e88: Array<0x188, u8>, + pub(crate) idle_ts: U64, + pub(crate) idle_unk: U64, + pub(crate) progress_check_interval_3d: u32, + pub(crate) progress_check_interval_ta: u32, + pub(crate) progress_check_interval_cl: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_1102c_0: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_1102c_4: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_1102c_8: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_1102c_c: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_1102c_10: u32, + + pub(crate) unk_1102c: u32, + pub(crate) idle_off_delay_ms: AtomicU32, + pub(crate) fender_idle_off_delay_ms: u32, + pub(crate) fw_early_wake_timeout_ms: u32, + #[ver(V == V13_3)] + pub(crate) ps_pad_0: Pad<0x8>, + pub(crate) pending_stamps: Array<0x100, PendingStamp>, + #[ver(V != V13_3)] + pub(crate) ps_pad_0: Pad<0x8>, + pub(crate) unkpad_ps: Pad<0x78>, + pub(crate) unk_117bc: u32, + pub(crate) fault_info: FaultInfo, + pub(crate) counter: u32, + pub(crate) unk_118dc: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_118e0_0: Array<0x9c, u8>, + + #[ver(G >= G14X)] + pub(crate) unk_118e0_9c: Array<0x580, u8>, + + #[ver(V >= V13_3)] + pub(crate) unk_118e0_9c_x: Array<0x8, u8>, + + pub(crate) cl_context_switch_timeout_ms: u32, + + #[ver(V >= V13_0B4)] + pub(crate) cl_kill_timeout_ms: u32, + + pub(crate) cdm_context_store_latency_threshold: u32, + pub(crate) unk_118e8: u32, + pub(crate) unk_118ec: Array<0x400, u8>, + pub(crate) unk_11cec: Array<0x54, u8>, + + #[ver(V >= V13_0B4)] + pub(crate) unk_11d40: Array<0x19c, u8>, + + #[ver(V >= V13_0B4)] + pub(crate) unk_11edc: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_11ee0: Array<0x1c, u8>, + + #[ver(V >= V13_0B4)] + pub(crate) unk_11efc: u32, + + #[ver(V >= V13_3)] + pub(crate) unk_11f00: Array<0x280, u8>, + } + #[versions(AGX)] + default_zeroed!(Globals::ver); + + #[derive(Debug, Default, Clone, Copy)] + #[repr(C, packed)] + pub(crate) struct UatLevelInfo { + pub(crate) unk_3: u8, + pub(crate) unk_1: u8, + pub(crate) unk_2: u8, + pub(crate) index_shift: u8, + pub(crate) num_entries: u16, + pub(crate) unk_4: u16, + pub(crate) unk_8: U64, + pub(crate) unk_10: U64, + pub(crate) index_mask: U64, + } + + #[versions(AGX)] + #[derive(Debug)] + #[repr(C)] + pub(crate) struct InitData<'a> { + #[ver(V >= V13_0B4)] + pub(crate) ver_info: Array<0x4, u16>, + + pub(crate) unk_buf: GpuPointer<'a, &'a [u8]>, + pub(crate) unk_8: u32, + pub(crate) unk_c: u32, + pub(crate) runtime_pointers: GpuPointer<'a, super::RuntimePointers::ver>, + pub(crate) globals: GpuPointer<'a, super::Globals::ver>, + pub(crate) fw_status: GpuPointer<'a, super::FwStatus>, + pub(crate) uat_page_size: u16, + pub(crate) uat_page_bits: u8, + pub(crate) uat_num_levels: u8, + pub(crate) uat_level_info: Array<0x3, UatLevelInfo>, + pub(crate) __pad0: Pad<0x14>, + pub(crate) host_mapped_fw_allocations: u32, + pub(crate) unk_ac: u32, + pub(crate) unk_b0: u32, + pub(crate) unk_b4: u32, + pub(crate) unk_b8: u32, + } +} + +#[derive(Debug)] +pub(crate) struct ChannelRing +where + for<'a> ::Raw<'a>: Debug, +{ + pub(crate) state: GpuObject, + pub(crate) ring: GpuArray, +} + +impl ChannelRing +where + for<'a> ::Raw<'a>: Debug, +{ + pub(crate) fn to_raw(&self) -> raw::ChannelRing { + raw::ChannelRing { + state: Some(self.state.weak_pointer()), + ring: Some(self.ring.weak_pointer()), + } + } +} + +trivial_gpustruct!(FwStatus); +trivial_gpustruct!(GpuGlobalStatsVtx); +#[versions(AGX)] +trivial_gpustruct!(GpuGlobalStatsFrag::ver); +trivial_gpustruct!(GpuStatsComp); + +#[versions(AGX)] +trivial_gpustruct!(HwDataA::ver); + +#[versions(AGX)] +trivial_gpustruct!(HwDataB::ver); + +#[versions(AGX)] +#[derive(Debug)] +pub(crate) struct Stats { + pub(crate) vtx: GpuObject, + pub(crate) frag: GpuObject, + pub(crate) comp: GpuObject, +} + +#[versions(AGX)] +#[derive(Debug)] +pub(crate) struct RuntimePointers { + pub(crate) stats: Stats::ver, + + pub(crate) hwdata_a: GpuObject, + pub(crate) unkptr_190: GpuArray, + pub(crate) unkptr_198: GpuArray, + pub(crate) hwdata_b: GpuObject, + + pub(crate) unkptr_1b8: GpuArray, + pub(crate) unkptr_1c0: GpuArray, + pub(crate) unkptr_1c8: GpuArray, + + pub(crate) buffer_mgr_ctl: gem::ObjectRef, + pub(crate) buffer_mgr_ctl_low_mapping: Option, + pub(crate) buffer_mgr_ctl_high_mapping: Option, +} + +#[versions(AGX)] +impl GpuStruct for RuntimePointers::ver { + type Raw<'a> = raw::RuntimePointers::ver<'a>; +} + +#[versions(AGX)] +trivial_gpustruct!(Globals::ver); + +#[versions(AGX)] +#[derive(Debug)] +pub(crate) struct InitData { + pub(crate) unk_buf: GpuArray, + pub(crate) runtime_pointers: GpuObject, + pub(crate) globals: GpuObject, + pub(crate) fw_status: GpuObject, +} + +#[versions(AGX)] +impl GpuStruct for InitData::ver { + type Raw<'a> = raw::InitData::ver<'a>; +} diff --git a/drivers/gpu/drm/asahi/fw/job.rs b/drivers/gpu/drm/asahi/fw/job.rs new file mode 100644 index 00000000000000..e4f2f9225ea050 --- /dev/null +++ b/drivers/gpu/drm/asahi/fw/job.rs @@ -0,0 +1,160 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Common GPU job firmware structures + +use super::types::*; +use crate::{ + default_zeroed, + mmu, + trivial_gpustruct, // +}; +use kernel::prelude::Result; +use kernel::sync::Arc; + +pub(crate) mod raw { + use super::*; + + #[derive(Debug, Clone, Copy)] + #[repr(C)] + pub(crate) struct JobMeta { + pub(crate) unk_0: u16, + pub(crate) unk_2: u8, + pub(crate) no_preemption: u8, + pub(crate) stamp: GpuWeakPointer, + pub(crate) fw_stamp: GpuWeakPointer, + pub(crate) stamp_value: EventValue, + pub(crate) stamp_slot: u32, + pub(crate) evctl_index: u32, + pub(crate) flush_stamps: u32, + pub(crate) uuid: u32, + pub(crate) event_seq: u32, + } + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct EncoderParams { + pub(crate) unk_8: u32, + pub(crate) sync_grow: u32, + pub(crate) unk_10: u32, + pub(crate) encoder_id: u32, + pub(crate) unk_18: u32, + pub(crate) unk_mask: u32, + pub(crate) sampler_array: U64, + pub(crate) sampler_count: u32, + pub(crate) sampler_max: u32, + } + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct JobTimestamps { + pub(crate) start: AtomicU64, + pub(crate) end: AtomicU64, + } + default_zeroed!(JobTimestamps); + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct RenderTimestamps { + pub(crate) vtx: JobTimestamps, + pub(crate) frag: JobTimestamps, + } + default_zeroed!(RenderTimestamps); + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct Register { + pub(crate) number: u32, + pub(crate) value: U64, + } + default_zeroed!(Register); + + impl Register { + fn new(number: u32, value: u64) -> Register { + Register { + number, + value: U64(value), + } + } + } + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct RegisterArray { + pub(crate) registers: Array<128, Register>, + pub(crate) pad: Array<0x100, u8>, + + pub(crate) addr: GpuWeakPointer>, + pub(crate) count: u16, + pub(crate) length: u16, + pub(crate) unk_pad: u32, + } + + impl RegisterArray { + pub(crate) fn new( + self_ptr: GpuWeakPointer>, + cb: impl FnOnce(&mut RegisterArray), + ) -> RegisterArray { + let mut array = RegisterArray { + registers: Default::default(), + pad: Default::default(), + addr: self_ptr, + count: 0, + length: 0, + unk_pad: 0, + }; + + cb(&mut array); + + array + } + + pub(crate) fn add(&mut self, number: u32, value: u64) { + self.registers[self.count as usize] = Register::new(number, value); + self.count += 1; + self.length += core::mem::size_of::() as u16; + } + } + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct TimestampPointers<'a> { + pub(crate) start_addr: Option>, + pub(crate) end_addr: Option>, + } +} + +trivial_gpustruct!(JobTimestamps); +trivial_gpustruct!(RenderTimestamps); + +#[derive(Debug)] +pub(crate) struct UserTimestamp { + pub(crate) mapping: Arc, + pub(crate) offset: usize, +} + +#[derive(Debug, Default)] +pub(crate) struct UserTimestamps { + pub(crate) start: Option, + pub(crate) end: Option, +} + +impl UserTimestamps { + pub(crate) fn any(&self) -> bool { + self.start.is_some() || self.end.is_some() + } + + pub(crate) fn pointers(&self) -> Result> { + Ok(raw::TimestampPointers { + start_addr: self + .start + .as_ref() + .map(|a| GpuPointer::from_mapping(&a.mapping, a.offset)) + .transpose()?, + end_addr: self + .end + .as_ref() + .map(|a| GpuPointer::from_mapping(&a.mapping, a.offset)) + .transpose()?, + }) + } +} diff --git a/drivers/gpu/drm/asahi/fw/microseq.rs b/drivers/gpu/drm/asahi/fw/microseq.rs new file mode 100644 index 00000000000000..ff59deda6be615 --- /dev/null +++ b/drivers/gpu/drm/asahi/fw/microseq.rs @@ -0,0 +1,412 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! GPU firmware microsequence operations + +use super::types::*; +use super::{ + buffer, + compute, + fragment, + initdata, + job, + vertex, + workqueue, // +}; +use crate::default_zeroed; + +pub(crate) trait Operation {} + +#[derive(Debug, Copy, Clone)] +#[repr(u32)] +enum OpCode { + WaitForIdle = 0x01, + WaitForIdle2 = 0x02, + RetireStamp = 0x18, + #[allow(dead_code)] + Timestamp = 0x19, + StartVertex = 0x22, + FinalizeVertex = 0x23, + StartFragment = 0x24, + FinalizeFragment = 0x25, + StartCompute = 0x29, + FinalizeCompute = 0x2a, +} + +#[derive(Debug, Copy, Clone)] +#[repr(u32)] +pub(crate) enum Pipe { + Vertex = 1 << 0, + Fragment = 1 << 8, + Compute = 1 << 15, +} + +pub(crate) const MAX_ATTACHMENTS: usize = 16; + +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub(crate) struct Attachment { + pub(crate) address: U64, + pub(crate) size: u32, + pub(crate) unk_c: u16, + pub(crate) unk_e: u16, +} +default_zeroed!(Attachment); + +#[derive(Debug, Clone, Copy, Default)] +#[repr(C)] +pub(crate) struct Attachments { + pub(crate) list: Array, + pub(crate) count: u32, +} + +#[derive(Debug, Copy, Clone)] +#[repr(transparent)] +pub(crate) struct OpHeader(u32); + +impl OpHeader { + const fn new(opcode: OpCode) -> OpHeader { + OpHeader(opcode as u32) + } + const fn with_args(opcode: OpCode, args: u32) -> OpHeader { + OpHeader(opcode as u32 | args) + } +} + +macro_rules! simple_op { + ($name:ident) => { + #[allow(dead_code)] + #[derive(Debug, Copy, Clone)] + pub(crate) struct $name(OpHeader); + + impl $name { + pub(crate) const HEADER: $name = $name(OpHeader::new(OpCode::$name)); + } + }; +} + +pub(crate) mod op { + use super::*; + + simple_op!(StartVertex); + simple_op!(FinalizeVertex); + simple_op!(StartFragment); + simple_op!(FinalizeFragment); + simple_op!(StartCompute); + simple_op!(FinalizeCompute); + simple_op!(WaitForIdle2); + + #[allow(dead_code)] + #[derive(Debug, Copy, Clone)] + pub(crate) struct RetireStamp(OpHeader); + impl RetireStamp { + pub(crate) const HEADER: RetireStamp = + RetireStamp(OpHeader::with_args(OpCode::RetireStamp, 0x40000000)); + } + + #[allow(dead_code)] + #[derive(Debug, Copy, Clone)] + pub(crate) struct WaitForIdle(OpHeader); + impl WaitForIdle { + pub(crate) const fn new(pipe: Pipe) -> WaitForIdle { + WaitForIdle(OpHeader::with_args(OpCode::WaitForIdle, (pipe as u32) << 8)) + } + } + + #[allow(dead_code)] + #[derive(Debug, Copy, Clone)] + pub(crate) struct Timestamp(OpHeader); + impl Timestamp { + #[allow(dead_code)] + pub(crate) const fn new(flag: bool) -> Timestamp { + Timestamp(OpHeader::with_args(OpCode::Timestamp, (flag as u32) << 31)) + } + } +} + +#[derive(Debug)] +#[repr(C)] +pub(crate) struct WaitForIdle { + pub(crate) header: op::WaitForIdle, +} + +impl Operation for WaitForIdle {} + +#[derive(Debug)] +#[repr(C)] +pub(crate) struct WaitForIdle2 { + pub(crate) header: op::WaitForIdle2, +} + +impl Operation for WaitForIdle2 {} + +#[derive(Debug)] +#[repr(C)] +pub(crate) struct RetireStamp { + pub(crate) header: op::RetireStamp, +} + +impl Operation for RetireStamp {} + +#[versions(AGX)] +#[derive(Debug)] +#[repr(C)] +pub(crate) struct Timestamp<'a> { + pub(crate) header: op::Timestamp, + pub(crate) command_time: GpuWeakPointer, + pub(crate) ts_pointers: GpuWeakPointer>, + // Unused? + pub(crate) update_ts: GpuWeakPointer>>, + pub(crate) work_queue: GpuWeakPointer, + pub(crate) user_ts_pointers: GpuWeakPointer>, + + #[ver(V >= V13_0B4)] + pub(crate) unk_ts: GpuWeakPointer, + + pub(crate) uuid: u32, + pub(crate) unk_30_padding: u32, +} + +#[versions(AGX)] +impl<'a> Operation for Timestamp::ver<'a> {} + +#[versions(AGX)] +#[derive(Debug)] +#[repr(C)] +pub(crate) struct StartVertex<'a> { + pub(crate) header: op::StartVertex, + pub(crate) tiling_params: Option>, + pub(crate) job_params1: Option>>, + #[ver(G >= G14X)] + pub(crate) registers: GpuWeakPointer, + pub(crate) buffer: GpuWeakPointer, + pub(crate) scene: GpuWeakPointer, + pub(crate) stats: GpuWeakPointer, + pub(crate) work_queue: GpuWeakPointer, + pub(crate) vm_slot: u32, + pub(crate) unk_38: u32, + pub(crate) event_generation: u32, + pub(crate) buffer_slot: u32, + pub(crate) unk_44: u32, + pub(crate) event_seq: U64, + pub(crate) unk_50: u32, + pub(crate) unk_pointer: GpuWeakPointer, + pub(crate) unk_job_buf: GpuWeakPointer, + pub(crate) unk_64: u32, + pub(crate) unk_68: u32, + pub(crate) uuid: u32, + pub(crate) attachments: Attachments, + pub(crate) padding: u32, + + #[ver(V >= V13_0B4)] + pub(crate) counter: U64, + + #[ver(V >= V13_0B4)] + pub(crate) notifier_buf: GpuWeakPointer>, + + pub(crate) unk_178: u32, +} + +#[versions(AGX)] +impl<'a> Operation for StartVertex::ver<'a> {} + +#[versions(AGX)] +#[derive(Debug)] +#[repr(C)] +pub(crate) struct FinalizeVertex { + pub(crate) header: op::FinalizeVertex, + pub(crate) scene: GpuWeakPointer, + pub(crate) buffer: GpuWeakPointer, + pub(crate) stats: GpuWeakPointer, + pub(crate) work_queue: GpuWeakPointer, + pub(crate) vm_slot: u32, + pub(crate) unk_28: u32, + pub(crate) unk_pointer: GpuWeakPointer, + pub(crate) unk_34: u32, + pub(crate) uuid: u32, + pub(crate) fw_stamp: GpuWeakPointer, + pub(crate) stamp_value: EventValue, + pub(crate) unk_48: U64, + pub(crate) unk_50: u32, + pub(crate) unk_54: u32, + pub(crate) unk_58: U64, + pub(crate) unk_60: u32, + pub(crate) unk_64: u32, + pub(crate) unk_68: u32, + + #[ver(G >= G14 && V < V13_0B4)] + pub(crate) unk_68_g14: U64, + + pub(crate) restart_branch_offset: i32, + pub(crate) has_attachments: u32, // Check DCMP errors bits 2,3 1=ktrace 2=log 3=panic + + #[ver(V >= V13_0B4)] + pub(crate) unk_74: Array<0x10, u8>, +} + +#[versions(AGX)] +impl Operation for FinalizeVertex::ver {} + +#[versions(AGX)] +#[derive(Debug)] +#[repr(C)] +pub(crate) struct StartFragment<'a> { + pub(crate) header: op::StartFragment, + pub(crate) job_params2: Option>, + pub(crate) job_params1: Option>>, + #[ver(G >= G14X)] + pub(crate) registers: GpuWeakPointer, + pub(crate) scene: GpuPointer<'a, buffer::Scene::ver>, + pub(crate) stats: GpuWeakPointer, + pub(crate) busy_flag: GpuWeakPointer, + pub(crate) tvb_overflow_count: GpuWeakPointer, + pub(crate) unk_pointer: GpuWeakPointer, + pub(crate) work_queue: GpuWeakPointer, + pub(crate) work_item: GpuWeakPointer, + pub(crate) vm_slot: u32, + pub(crate) unk_50: u32, + pub(crate) event_generation: u32, + pub(crate) buffer_slot: u32, + pub(crate) sync_grow: u32, + pub(crate) event_seq: U64, + pub(crate) unk_68: u32, + pub(crate) unk_758_flag: GpuWeakPointer, + pub(crate) unk_job_buf: GpuWeakPointer, + #[ver(V >= V13_3)] + pub(crate) unk_7c_0: U64, + pub(crate) unk_7c: u32, + pub(crate) unk_80: u32, + pub(crate) unk_84: u32, + pub(crate) uuid: u32, + pub(crate) attachments: Attachments, + pub(crate) padding: u32, + + #[ver(V >= V13_0B4)] + pub(crate) counter: U64, + + #[ver(V >= V13_0B4)] + pub(crate) notifier_buf: GpuWeakPointer>, +} + +#[versions(AGX)] +impl<'a> Operation for StartFragment::ver<'a> {} + +#[versions(AGX)] +#[derive(Debug)] +#[repr(C)] +pub(crate) struct FinalizeFragment { + pub(crate) header: op::FinalizeFragment, + pub(crate) uuid: u32, + pub(crate) unk_8: u32, + pub(crate) fw_stamp: GpuWeakPointer, + pub(crate) stamp_value: EventValue, + pub(crate) unk_18: u32, + pub(crate) scene: GpuWeakPointer, + pub(crate) buffer: GpuWeakPointer, + pub(crate) unk_2c: U64, + pub(crate) stats: GpuWeakPointer, + pub(crate) unk_pointer: GpuWeakPointer, + pub(crate) busy_flag: GpuWeakPointer, + pub(crate) work_queue: GpuWeakPointer, + pub(crate) work_item: GpuWeakPointer, + pub(crate) vm_slot: u32, + pub(crate) unk_60: u32, + pub(crate) unk_758_flag: GpuWeakPointer, + #[ver(V >= V13_3)] + pub(crate) unk_6c_0: U64, + pub(crate) unk_6c: U64, + pub(crate) unk_74: U64, + pub(crate) unk_7c: U64, + pub(crate) unk_84: U64, + pub(crate) unk_8c: U64, + + #[ver(G == G14 && V < V13_0B4)] + pub(crate) unk_8c_g14: U64, + + pub(crate) restart_branch_offset: i32, + pub(crate) has_attachments: u32, // Check DCMP errors bits 2,3 1=ktrace 2=log 3=panic + + #[ver(V >= V13_0B4)] + pub(crate) unk_9c: Array<0x10, u8>, +} + +#[versions(AGX)] +impl Operation for FinalizeFragment::ver {} + +#[versions(AGX)] +#[derive(Debug)] +#[repr(C)] +pub(crate) struct StartCompute<'a> { + pub(crate) header: op::StartCompute, + pub(crate) unk_pointer: GpuWeakPointer, + pub(crate) job_params1: Option>>, + #[ver(G >= G14X)] + pub(crate) registers: GpuWeakPointer, + pub(crate) stats: GpuWeakPointer, + pub(crate) work_queue: GpuWeakPointer, + pub(crate) vm_slot: u32, + pub(crate) unk_28: u32, + pub(crate) event_generation: u32, + pub(crate) event_seq: U64, + pub(crate) unk_38: u32, + pub(crate) job_params2: GpuWeakPointer>, + pub(crate) unk_44: u32, + pub(crate) uuid: u32, + pub(crate) attachments: Attachments, + pub(crate) padding: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_flag: GpuWeakPointer, + + #[ver(V >= V13_0B4)] + pub(crate) counter: U64, + + #[ver(V >= V13_0B4)] + pub(crate) notifier_buf: GpuWeakPointer>, +} + +#[versions(AGX)] +impl<'a> Operation for StartCompute::ver<'a> {} + +#[versions(AGX)] +#[derive(Debug)] +#[repr(C)] +pub(crate) struct FinalizeCompute<'a> { + pub(crate) header: op::FinalizeCompute, + pub(crate) stats: GpuWeakPointer, + pub(crate) work_queue: GpuWeakPointer, + pub(crate) vm_slot: u32, + #[ver(V < V13_0B4)] + pub(crate) unk_18: u32, + pub(crate) job_params2: GpuWeakPointer>, + pub(crate) unk_24: u32, + pub(crate) uuid: u32, + pub(crate) fw_stamp: GpuWeakPointer, + pub(crate) stamp_value: EventValue, + pub(crate) unk_38: u32, + pub(crate) unk_3c: u32, + pub(crate) unk_40: u32, + pub(crate) unk_44: u32, + pub(crate) unk_48: u32, + pub(crate) unk_4c: u32, + pub(crate) unk_50: u32, + pub(crate) unk_54: u32, + pub(crate) unk_58: u32, + + #[ver(G == G14 && V < V13_0B4)] + pub(crate) unk_5c_g14: U64, + + pub(crate) restart_branch_offset: i32, + pub(crate) has_attachments: u32, // Check DCMP errors bits 2,3 1=ktrace 2=log 3=panic + + #[ver(V >= V13_0B4)] + pub(crate) unk_64: Array<0xd, u8>, + + #[ver(V >= V13_0B4)] + pub(crate) unk_flag: GpuWeakPointer, + + #[ver(V >= V13_0B4)] + pub(crate) unk_79: Array<0x7, u8>, +} + +#[versions(AGX)] +impl<'a> Operation for FinalizeCompute::ver<'a> {} diff --git a/drivers/gpu/drm/asahi/fw/mod.rs b/drivers/gpu/drm/asahi/fw/mod.rs new file mode 100644 index 00000000000000..a5649aa20d3a8e --- /dev/null +++ b/drivers/gpu/drm/asahi/fw/mod.rs @@ -0,0 +1,15 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Firmware structures for Apple AGX GPUs + +pub(crate) mod buffer; +pub(crate) mod channels; +pub(crate) mod compute; +pub(crate) mod event; +pub(crate) mod fragment; +pub(crate) mod initdata; +pub(crate) mod job; +pub(crate) mod microseq; +pub(crate) mod types; +pub(crate) mod vertex; +pub(crate) mod workqueue; diff --git a/drivers/gpu/drm/asahi/fw/types.rs b/drivers/gpu/drm/asahi/fw/types.rs new file mode 100644 index 00000000000000..f55d6cec6b8ca3 --- /dev/null +++ b/drivers/gpu/drm/asahi/fw/types.rs @@ -0,0 +1,224 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Common types for firmware structure definitions + +use crate::{alloc, object}; +use core::fmt; +use core::ops::{Deref, DerefMut, Index, IndexMut}; + +pub(crate) use crate::event::EventValue; +pub(crate) use crate::object::{ + GpuPointer, + GpuStruct, + GpuWeakPointer, // +}; +pub(crate) use crate::{ + f32, + float::F32, // +}; + +pub(crate) use core::fmt::Debug; +pub(crate) use core::marker::PhantomData; +pub(crate) use core::sync::atomic::{ + AtomicI32, + AtomicU32, + AtomicU64, // +}; +pub(crate) use kernel::macros::versions; +pub(crate) use kernel::prelude::Zeroable; + +// Make the trait visible +pub(crate) use crate::alloc::Allocator as _Allocator; + +/// General allocator type used for the driver +pub(crate) type Allocator = alloc::DefaultAllocator; + +/// General GpuObject type used for the driver +pub(crate) type GpuObject = + object::GpuObject>; + +/// General GpuArray type used for the driver +pub(crate) type GpuArray = object::GpuArray>; + +/// General GpuOnlyArray type used for the driver +pub(crate) type GpuOnlyArray = + object::GpuOnlyArray>; + +/// A stamp slot that is shared between firmware and the driver. +#[derive(Debug, Default)] +#[repr(transparent)] +pub(crate) struct Stamp(pub(crate) AtomicU32); + +/// A stamp slot that is for private firmware use. +/// +/// This is a separate type to guard against pointer type confusion. +#[derive(Debug, Default)] +#[repr(transparent)] +pub(crate) struct FwStamp(pub(crate) AtomicU32); + +/// An unaligned u64 type. +/// +/// This is useful to avoid having to pack firmware structures entirely, since that is incompatible +/// with `#[derive(Debug)]` and atomics. +#[derive(Copy, Clone, Default)] +#[repr(C, packed(1))] +pub(crate) struct U64(pub(crate) u64); + +// SAFETY: U64 is zeroable just like u64 +unsafe impl Zeroable for U64 {} + +impl fmt::Debug for U64 { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let v = self.0; + f.write_fmt(format_args!("{:#x}", v)) + } +} + +/// An unaligned u32 type. +/// +/// This is useful to avoid having to pack firmware structures entirely, since that is incompatible +/// with `#[derive(Debug)]` and atomics. +#[derive(Copy, Clone, Default)] +#[repr(C, packed(1))] +pub(crate) struct U32(pub(crate) u32); + +// SAFETY: U32 is zeroable just like u32 +unsafe impl Zeroable for U32 {} + +impl fmt::Debug for U32 { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let v = self.0; + f.write_fmt(format_args!("{:#x}", v)) + } +} + +/// Create a dummy `Debug` implementation, for when we need it but it's too painful to write by +/// hand or not very useful. +#[macro_export] +macro_rules! no_debug { + ($type:ty) => { + impl ::core::fmt::Debug for $type { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!(f, "...") + } + } + }; +} + +/// Implement Zeroable for a given type (and Default along with it). +/// +/// # Safety +/// +/// This macro must only be used if a type only contains primitive types which can be +/// zero-initialized, FFI structs intended to be zero-initialized, or other types which +/// impl Zeroable. +#[macro_export] +macro_rules! default_zeroed { + (<$($lt:lifetime),*>, $type:ty) => { + impl<$($lt),*> Default for $type { + fn default() -> $type { + unsafe { core::mem::zeroed() } + } + } + // SAFETY: The user is responsible for ensuring this is safe. + unsafe impl<$($lt),*> ::pin_init::Zeroable for $type {} + }; + ($type:ty) => { + impl Default for $type { + fn default() -> $type { + unsafe { core::mem::zeroed() } + } + } + // SAFETY: The user is responsible for ensuring this is safe. + unsafe impl ::pin_init::Zeroable for $type {} + }; +} + +/// A convenience type for a number of padding bytes. Hidden from Debug formatting. +#[derive(Copy, Clone)] +#[repr(C, packed)] +pub(crate) struct Pad([u8; N]); + +/// SAFETY: Primitive type, safe to zero-init. +unsafe impl Zeroable for Pad {} + +impl Default for Pad { + fn default() -> Self { + unsafe { core::mem::zeroed() } + } +} + +impl fmt::Debug for Pad { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_fmt(format_args!("")) + } +} + +/// A convenience type for a fixed-sized array with Default/Zeroable impls. +#[derive(Copy, Clone)] +#[repr(C)] +pub(crate) struct Array([T; N]); + +impl Array { + pub(crate) fn new(data: [T; N]) -> Self { + Self(data) + } +} + +// SAFETY: Arrays of Zeroable values can be safely Zeroable. +unsafe impl Zeroable for Array {} + +impl Default for Array { + fn default() -> Self { + unsafe { core::mem::zeroed() } + } +} + +impl Index for Array { + type Output = T; + + fn index(&self, index: usize) -> &Self::Output { + &self.0[index] + } +} + +impl IndexMut for Array { + fn index_mut(&mut self, index: usize) -> &mut Self::Output { + &mut self.0[index] + } +} + +impl Deref for Array { + type Target = [T; N]; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl DerefMut for Array { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} + +impl fmt::Debug for Array { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.0.fmt(f) + } +} + +/// Convenience macro to define an identically-named trivial GpuStruct with no inner fields for a +/// given raw type name. +#[macro_export] +macro_rules! trivial_gpustruct { + ($type:ident) => { + #[derive(Debug)] + pub(crate) struct $type {} + + impl GpuStruct for $type { + type Raw<'a> = raw::$type; + } + $crate::default_zeroed!($type); + }; +} diff --git a/drivers/gpu/drm/asahi/fw/vertex.rs b/drivers/gpu/drm/asahi/fw/vertex.rs new file mode 100644 index 00000000000000..07a0e05c72112c --- /dev/null +++ b/drivers/gpu/drm/asahi/fw/vertex.rs @@ -0,0 +1,193 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! GPU vertex job firmware structures + +use super::types::*; +use super::{ + event, + job, + workqueue, // +}; +use crate::{ + buffer, + fw, + microseq, + mmu, // +}; +use kernel::sync::Arc; + +pub(crate) mod raw { + use super::*; + + #[derive(Debug, Default, Copy, Clone)] + #[repr(C)] + pub(crate) struct TilingParameters { + pub(crate) rgn_size: u32, + pub(crate) unk_4: u32, + pub(crate) ppp_ctrl: u32, + pub(crate) x_max: u16, + pub(crate) y_max: u16, + pub(crate) te_screen: u32, + pub(crate) te_mtile1: u32, + pub(crate) te_mtile2: u32, + pub(crate) tiles_per_mtile: u32, + pub(crate) tpc_stride: u32, + pub(crate) unk_24: u32, + pub(crate) unk_28: u32, + pub(crate) helper_cfg: u32, + pub(crate) __pad: Pad<0x70>, + } + + #[versions(AGX)] + #[derive(Debug)] + #[repr(C)] + pub(crate) struct JobParameters1<'a> { + pub(crate) unk_0: U64, + pub(crate) unk_8: F32, + pub(crate) unk_c: F32, + pub(crate) tvb_tilemap: GpuPointer<'a, &'a [u8]>, + #[ver(G < G14)] + pub(crate) tvb_cluster_tilemaps: Option>, + pub(crate) tpc: GpuPointer<'a, &'a [u8]>, + pub(crate) tvb_heapmeta: GpuPointer<'a, &'a [u8]>, + pub(crate) iogpu_unk_54: U64, + pub(crate) iogpu_unk_56: U64, + #[ver(G < G14)] + pub(crate) tvb_cluster_meta1: Option>, + pub(crate) utile_config: u32, + pub(crate) unk_4c: u32, + pub(crate) ppp_multisamplectl: U64, + pub(crate) tvb_layermeta: GpuPointer<'a, &'a [u8]>, + #[ver(G < G14)] + pub(crate) tvb_cluster_layermeta: Option>, + #[ver(G < G14)] + pub(crate) core_mask: Array<2, u32>, + pub(crate) preempt_buf1: GpuPointer<'a, &'a [u8]>, + pub(crate) preempt_buf2: GpuPointer<'a, &'a [u8]>, + pub(crate) unk_80: U64, + pub(crate) preempt_buf3: GpuPointer<'a, &'a [u8]>, + pub(crate) vdm_ctrl_stream_base: U64, + #[ver(G < G14)] + pub(crate) tvb_cluster_meta2: Option>, + #[ver(G < G14)] + pub(crate) tvb_cluster_meta3: Option>, + #[ver(G < G14)] + pub(crate) tiling_control: u32, + #[ver(G < G14)] + pub(crate) unk_ac: u32, + pub(crate) unk_b0: Array<6, U64>, + pub(crate) usc_exec_base_ta: U64, + #[ver(G < G14)] + pub(crate) tvb_cluster_meta4: Option>, + #[ver(G < G14)] + pub(crate) unk_f0: U64, + pub(crate) unk_f8: U64, + pub(crate) helper_program: u32, + pub(crate) unk_104: u32, + pub(crate) helper_arg: U64, + pub(crate) unk_110: U64, + pub(crate) unk_118: u32, + #[ver(G >= G14)] + pub(crate) __pad: Pad<{ 8 * 9 + 0x268 }>, + #[ver(G < G14)] + pub(crate) __pad: Pad<0x268>, + } + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct JobParameters2<'a> { + pub(crate) unk_480: Array<4, u32>, + pub(crate) unk_498: U64, + pub(crate) unk_4a0: u32, + pub(crate) preempt_buf1: GpuPointer<'a, &'a [u8]>, + pub(crate) unk_4ac: u32, + pub(crate) unk_4b0: U64, + pub(crate) unk_4b8: u32, + pub(crate) unk_4bc: U64, + pub(crate) unk_4c4_padding: Array<0x48, u8>, + pub(crate) unk_50c: u32, + pub(crate) unk_510: U64, + pub(crate) unk_518: U64, + pub(crate) unk_520: U64, + } + + #[versions(AGX)] + #[derive(Debug)] + #[repr(C)] + pub(crate) struct RunVertex<'a> { + pub(crate) tag: workqueue::CommandType, + + #[ver(V >= V13_0B4)] + pub(crate) counter: U64, + + pub(crate) vm_slot: u32, + pub(crate) unk_8: u32, + pub(crate) notifier: GpuPointer<'a, event::Notifier::ver>, + pub(crate) buffer_slot: u32, + pub(crate) unk_1c: u32, + pub(crate) buffer: GpuPointer<'a, fw::buffer::Info::ver>, + pub(crate) scene: GpuPointer<'a, fw::buffer::Scene::ver>, + pub(crate) unk_buffer_buf: GpuWeakPointer<[u8]>, + pub(crate) unk_34: u32, + + #[ver(G < G14X)] + pub(crate) job_params1: JobParameters1::ver<'a>, + #[ver(G < G14X)] + pub(crate) tiling_params: TilingParameters, + #[ver(G >= G14X)] + pub(crate) registers: job::raw::RegisterArray, + + pub(crate) tpc: GpuPointer<'a, &'a [u8]>, + pub(crate) tpc_size: U64, + pub(crate) microsequence: GpuPointer<'a, &'a [u8]>, + pub(crate) microsequence_size: u32, + pub(crate) fragment_stamp_slot: u32, + pub(crate) fragment_stamp_value: EventValue, + pub(crate) unk_pointee: u32, + pub(crate) unk_pad: u32, + pub(crate) job_params2: JobParameters2<'a>, + pub(crate) encoder_params: job::raw::EncoderParams, + pub(crate) unk_55c: u32, + pub(crate) unk_560: u32, + pub(crate) sync_grow: u32, + pub(crate) unk_568: u32, + pub(crate) uses_scratch: u32, + pub(crate) meta: job::raw::JobMeta, + pub(crate) unk_after_meta: u32, + pub(crate) unk_buf_0: U64, + pub(crate) unk_buf_8: U64, + pub(crate) unk_buf_10: U64, + pub(crate) command_time: U64, + pub(crate) timestamp_pointers: job::raw::TimestampPointers<'a>, + pub(crate) user_timestamp_pointers: job::raw::TimestampPointers<'a>, + pub(crate) client_sequence: u8, + pub(crate) pad_5d5: Array<3, u8>, + pub(crate) unk_5d8: u32, + pub(crate) unk_5dc: u8, + + #[ver(V >= V13_0B4)] + pub(crate) unk_ts: U64, + + #[ver(V >= V13_0B4)] + pub(crate) unk_5dd_8: Array<0x1b, u8>, + } +} + +#[versions(AGX)] +#[derive(Debug)] +pub(crate) struct RunVertex { + pub(crate) notifier: Arc>, + pub(crate) scene: Arc, + pub(crate) micro_seq: microseq::MicroSequence, + pub(crate) vm_bind: mmu::VmBind, + pub(crate) timestamps: Arc>, + pub(crate) user_timestamps: job::UserTimestamps, +} + +#[versions(AGX)] +impl GpuStruct for RunVertex::ver { + type Raw<'a> = raw::RunVertex::ver<'a>; +} + +#[versions(AGX)] +impl workqueue::Command for RunVertex::ver {} diff --git a/drivers/gpu/drm/asahi/fw/workqueue.rs b/drivers/gpu/drm/asahi/fw/workqueue.rs new file mode 100644 index 00000000000000..b86bd3ff3757af --- /dev/null +++ b/drivers/gpu/drm/asahi/fw/workqueue.rs @@ -0,0 +1,187 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! GPU work queue firmware structes + +use super::event; +use super::types::*; +use crate::event::EventValue; +use crate::{ + default_zeroed, + trivial_gpustruct, // +}; +use kernel::sync::Arc; + +#[derive(Debug)] +#[repr(u32)] +pub(crate) enum CommandType { + RunVertex = 0, + RunFragment = 1, + #[allow(dead_code)] + RunBlitter = 2, + RunCompute = 3, + Barrier = 4, + InitBuffer = 6, +} + +pub(crate) trait Command: GpuStruct + Send + Sync {} + +pub(crate) mod raw { + use super::*; + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct Barrier { + pub(crate) tag: CommandType, + pub(crate) wait_stamp: GpuWeakPointer, + pub(crate) wait_value: EventValue, + pub(crate) wait_slot: u32, + pub(crate) stamp_self: EventValue, + pub(crate) uuid: u32, + pub(crate) external_barrier: u32, + // G14X addition + pub(crate) internal_barrier_type: u32, + pub(crate) padding: Pad<0x1c>, + } + + #[derive(Debug, Clone, Copy)] + #[repr(C)] + pub(crate) struct GpuContextData { + pub(crate) unk_0: u8, + pub(crate) unk_1: u8, + unk_2: Array<0x2, u8>, + pub(crate) unk_4: u8, + pub(crate) unk_5: u8, + unk_6: Array<0x18, u8>, + pub(crate) unk_1e: u8, + pub(crate) unk_1f: u8, + unk_20: Array<0x3, u8>, + pub(crate) unk_23: u8, + unk_24: Array<0x1c, u8>, + } + + impl Default for GpuContextData { + fn default() -> Self { + Self { + unk_0: 0xff, + unk_1: 0xff, + unk_2: Default::default(), + unk_4: 0, + unk_5: 1, + unk_6: Default::default(), + unk_1e: 0xff, + unk_1f: 0, + unk_20: Default::default(), + unk_23: 2, + unk_24: Default::default(), + } + } + } + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct RingState { + pub(crate) gpu_doneptr: AtomicU32, + __pad0: Pad<0xc>, + pub(crate) unk_10: AtomicU32, + __pad1: Pad<0xc>, + pub(crate) unk_20: AtomicU32, + __pad2: Pad<0xc>, + pub(crate) gpu_rptr: AtomicU32, + __pad3: Pad<0xc>, + pub(crate) cpu_wptr: AtomicU32, + __pad4: Pad<0xc>, + pub(crate) rb_size: u32, + __pad5: Pad<0xc>, + // This isn't part of the structure, but it's here as a + // debugging hack so we can inspect what ring position + // the driver considered complete and freeable. + pub(crate) cpu_freeptr: AtomicU32, + __pad6: Pad<0xc>, + } + default_zeroed!(RingState); + + #[derive(Debug, Clone, Copy)] + #[repr(C)] + pub(crate) struct Priority( + pub(crate) u32, + pub(crate) u32, + pub(crate) U64, + pub(crate) u32, + pub(crate) u32, + pub(crate) u32, + ); + + pub(crate) const PRIORITY: [Priority; 4] = [ + Priority(0, 0, U64(0xffff_ffff_ffff_0000), 1, 0, 1), + Priority(1, 1, U64(0xffff_ffff_0000_0000), 0, 0, 0), + Priority(2, 2, U64(0xffff_0000_0000_0000), 0, 0, 2), + Priority(3, 3, U64(0x0000_0000_0000_0000), 0, 0, 3), + ]; + + impl Default for Priority { + fn default() -> Priority { + PRIORITY[2] + } + } + + #[versions(AGX)] + #[derive(Debug)] + #[repr(C)] + pub(crate) struct QueueInfo<'a> { + pub(crate) state: GpuPointer<'a, super::RingState>, + pub(crate) ring: GpuPointer<'a, &'a [u64]>, + pub(crate) notifier_list: GpuPointer<'a, event::NotifierList>, + pub(crate) gpu_buf: GpuPointer<'a, &'a [u8]>, + pub(crate) gpu_rptr1: AtomicU32, + pub(crate) gpu_rptr2: AtomicU32, + pub(crate) gpu_rptr3: AtomicU32, + pub(crate) event_id: AtomicI32, + pub(crate) priority: Priority, + pub(crate) unk_4c: i32, + pub(crate) uuid: u32, + pub(crate) unk_54: i32, + pub(crate) unk_58: U64, + pub(crate) busy: AtomicU32, + pub(crate) __pad: Pad<0x20>, + #[ver(V >= V13_2 && G < G14X)] + pub(crate) unk_84_0: u32, + pub(crate) unk_84_state: AtomicU32, + pub(crate) error_count: AtomicU32, + pub(crate) unk_8c: u32, + pub(crate) unk_90: u32, + pub(crate) unk_94: u32, + pub(crate) pending: AtomicU32, + pub(crate) unk_9c: u32, + pub(crate) gpu_context: GpuPointer<'a, super::GpuContextData>, + pub(crate) unk_a8: U64, + #[ver(V >= V13_2 && G < G14X)] + pub(crate) unk_b0: u32, + } +} + +trivial_gpustruct!(Barrier); +trivial_gpustruct!(RingState); + +impl Command for Barrier {} + +pub(crate) struct GpuContextData { + pub(crate) _buffer: Arc, +} +impl GpuStruct for GpuContextData { + type Raw<'a> = raw::GpuContextData; +} + +#[versions(AGX)] +#[derive(Debug)] +pub(crate) struct QueueInfo { + pub(crate) state: GpuObject, + pub(crate) ring: GpuArray, + pub(crate) gpu_buf: GpuArray, + pub(crate) notifier_list: Arc>, + pub(crate) gpu_context: Arc, +} + +#[versions(AGX)] +impl GpuStruct for QueueInfo::ver { + type Raw<'a> = raw::QueueInfo::ver<'a>; +} diff --git a/drivers/gpu/drm/asahi/gem.rs b/drivers/gpu/drm/asahi/gem.rs new file mode 100644 index 00000000000000..8affba257d956f --- /dev/null +++ b/drivers/gpu/drm/asahi/gem.rs @@ -0,0 +1,248 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Asahi driver GEM object implementation +//! +//! Basic wrappers and adaptations between generic GEM shmem objects and this driver's +//! view of what a GPU buffer object is. It is in charge of keeping track of all mappings for +//! each GEM object so we can remove them when a client (File) or a Vm are destroyed, as well as +//! implementing RTKit buffers on top of GEM objects for firmware use. + +use kernel::{ + drm, + drm::gem::{ + shmem, + shmem::VMap, + BaseObject, + DriverObject, // + }, + error::Result, + prelude::*, + types::ARef, + uapi, +}; + +use core::ops::Range; +use core::sync::atomic::{ + AtomicU64, + Ordering, // +}; + +use crate::{ + debug::*, + driver::{ + AsahiDevice, + AsahiDriver, // + }, + file, + mmu, + util::*, // +}; + +const DEBUG_CLASS: DebugFlags = DebugFlags::Gem; + +/// Represents the inner data of a GEM object for this driver. +#[pin_data] +pub(crate) struct AsahiObject { + /// ID for debug + id: u64, + /// Object creation flags. + flags: u32, + /// Whether this object can be exported. + exportable: bool, + /// Whether this is a kernel-created object. + kernel: bool, +} + +/// Type alias for the shmem GEM object type for this driver. +pub(crate) type Object = shmem::Object; + +unsafe impl Send for AsahiObject {} +unsafe impl Sync for AsahiObject {} + +// /// Type alias for the SGTable type for this driver. +// pub(crate) type SGTable = shmem::SGTable; + +/// A shared reference to a GEM object for this driver. +pub(crate) struct ObjectRef { + /// The underlying GEM object reference + pub(crate) gem: ARef, + /// The kernel-side VMap of this object, if needed + vmap: Option>, +} + +crate::no_debug!(ObjectRef); + +static GEM_ID: AtomicU64 = AtomicU64::new(0); + +impl ObjectRef { + /// Create a new wrapper for a raw GEM object reference. + pub(crate) fn new(gem: ARef) -> ObjectRef { + ObjectRef { gem, vmap: None } + } + + /// Return the `VMap` for this object, creating it if necessary. + pub(crate) fn vmap(&mut self) -> Result> { + if self.vmap.is_none() { + self.vmap = Some(self.gem.owned_vmap()?); + } + self.gem.vmap() + } + + /// Returns the size of an object in bytes + pub(crate) fn size(&self) -> usize { + self.gem.size() + } + + /// Maps an object into a given `Vm` at any free address within a given range. + pub(crate) fn map_into_range( + &mut self, + vm: &crate::mmu::Vm, + range: Range, + alignment: u64, + prot: mmu::Prot, + guard: bool, + ) -> Result { + // Only used for kernel objects now + if !self.gem.kernel { + return Err(EINVAL); + } + vm.map_in_range(&self.gem, 0..self.gem.size(), alignment, range, prot, guard) + } + + /// Maps a range within an object into a given `Vm` at any free address within a given range. + pub(crate) fn map_range_into_range( + &mut self, + vm: &crate::mmu::Vm, + obj_range: Range, + range: Range, + alignment: u64, + prot: mmu::Prot, + guard: bool, + ) -> Result { + if obj_range.end > self.gem.size() { + return Err(EINVAL); + } + if self.gem.flags & uapi::drm_asahi_gem_flags_DRM_ASAHI_GEM_VM_PRIVATE != 0 + && vm.is_extobj(&*self.gem) + { + return Err(EINVAL); + } + vm.map_in_range(&self.gem, obj_range, alignment, range, prot, guard) + } + + /// Maps an object into a given `Vm` at a specific address. + /// + /// Returns Err(ENOSPC) if the requested address is already busy. + pub(crate) fn map_at( + &mut self, + vm: &crate::mmu::Vm, + addr: u64, + prot: mmu::Prot, + guard: bool, + ) -> Result { + if self.gem.flags & uapi::drm_asahi_gem_flags_DRM_ASAHI_GEM_VM_PRIVATE != 0 + && vm.is_extobj(&*self.gem) + { + return Err(EINVAL); + } + + vm.map_at(addr, self.gem.size(), self.gem.clone(), prot, guard) + } +} + +pub(crate) struct AsahiObjConfig { + flags: u32, + exportable: bool, + kernel: bool, +} + +/// Create a new kernel-owned GEM object. +pub(crate) fn new_kernel_object(dev: &AsahiDevice, size: usize) -> Result { + let gem = shmem::Object::::new( + dev, + align(size, mmu::UAT_PGSZ), + shmem::ObjectConfig:: { + map_wc: false, + parent_resv_obj: None, + }, + AsahiObjConfig { + flags: 0, + exportable: false, + kernel: true, + }, + )?; + + mod_pr_debug!("AsahiObject new kernel object id={}\n", gem.id); + Ok(ObjectRef::new(gem)) +} + +/// Create a new user-owned GEM object with the given flags. +pub(crate) fn new_object( + dev: &AsahiDevice, + size: usize, + flags: u32, + parent_object: Option<&shmem::Object>, +) -> Result> { + if (flags & uapi::drm_asahi_gem_flags_DRM_ASAHI_GEM_VM_PRIVATE != 0) != parent_object.is_some() + { + return Err(EINVAL); + } + + let gem = shmem::Object::::new( + dev, + align(size, mmu::UAT_PGSZ), + shmem::ObjectConfig:: { + map_wc: flags & uapi::drm_asahi_gem_flags_DRM_ASAHI_GEM_WRITEBACK == 0, + parent_resv_obj: parent_object, + }, + AsahiObjConfig { + flags, + exportable: parent_object.is_none(), + kernel: false, + }, + )?; + + mod_pr_debug!("AsahiObject new user object: id={}\n", gem.id); + Ok(gem) +} + +#[vtable] +impl DriverObject for AsahiObject { + type Driver = AsahiDriver; + type Args = AsahiObjConfig; + + const HAS_EXPORT: bool = true; + + /// Callback to create the inner data of a GEM object + fn new(_dev: &AsahiDevice, _size: usize, args: Self::Args) -> impl PinInit { + let id = GEM_ID.fetch_add(1, Ordering::Relaxed); + mod_pr_debug!("AsahiObject::new id={}\n", id); + try_pin_init!(AsahiObject { + id, + flags: args.flags, + exportable: args.exportable, + kernel: args.kernel, + }) + } + + /// Callback to drop all mappings for a GEM object owned by a given `File` + fn close(obj: &::Object, file: &drm::gem::DriverFile) { + // fn close(obj: &Object, file: &DrmFile) { + mod_pr_debug!("AsahiObject::close id={}\n", obj.id); + if file::File::unbind_gem_object(file, obj).is_err() { + pr_err!("AsahiObject::close: Failed to unbind GEM object\n"); + } + } + + /// Optional handle for exporting a gem object. + fn export( + obj: &::Object, + flags: u32, + ) -> Result> { + if !obj.exportable { + return Err(EINVAL); + } + + obj.prime_export(flags) + } +} diff --git a/drivers/gpu/drm/asahi/gpu.rs b/drivers/gpu/drm/asahi/gpu.rs new file mode 100644 index 00000000000000..b6c1c64c503671 --- /dev/null +++ b/drivers/gpu/drm/asahi/gpu.rs @@ -0,0 +1,1642 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Top-level GPU manager +//! +//! This module is the root of all GPU firmware management for a given driver instance. It is +//! responsible for initialization, owning the top-level managers (events, UAT, etc.), and +//! communicating with the raw RtKit endpoints to send and receive messages to/from the GPU +//! firmware. +//! +//! It is also the point where diverging driver firmware/GPU variants (using the versions macro) +//! are unified, so that the top level of the driver itself (in `driver`) does not have to concern +//! itself with version dependence. + +use core::any::Any; +use core::ops::Range; +use core::slice; +use core::sync::atomic::{ + AtomicBool, + AtomicU64, + Ordering, // +}; + +use kernel::{ + c_str, + drm::gem::shmem, + error::code::*, + io::mem::{ + Mem, + MemFlag, // + }, + iosys_map::IoSysMapRef, + macros::versions, + new_mutex, + prelude::*, + soc::apple::rtkit, + sync::{ + lock::{ + mutex::MutexBackend, + Guard, // + }, + Arc, + Mutex, + UniqueArc, // + }, + time::{ + Delta, + Instant, + Monotonic, // + }, + types::ForeignOwnable, // +}; +#[cfg(CONFIG_DEV_COREDUMP)] +use kernel::{ + devcoredump, + time::msecs_to_jiffies, // +}; + +use crate::alloc::Allocator; +use crate::debug::*; +use crate::driver::{ + AsahiDevRef, + AsahiDevice, // +}; +use crate::fw::channels::{ + ChannelErrorType, + PipeType, // +}; +use crate::fw::types::{ + U32, + U64, // +}; +use crate::{ + alloc, + buffer, + channel, + event, + fw, + gem, + hw, + initdata, + mem, + mmu, + queue, + regs, + workqueue, // +}; + +const DEBUG_CLASS: DebugFlags = DebugFlags::Gpu; + +/// Firmware endpoint for init & incoming notifications. +const EP_FIRMWARE: u8 = 0x20; + +/// Doorbell endpoint for work/message submissions. +const EP_DOORBELL: u8 = 0x21; + +/// Initialize the GPU firmware. +const MSG_INIT: u64 = 0x81 << 48; +const INIT_DATA_MASK: u64 = (1 << 44) - 1; + +/// TX channel doorbell. +const MSG_TX_DOORBELL: u64 = 0x83 << 48; +/// Firmware control channel doorbell. +const MSG_FWCTL: u64 = 0x84 << 48; +// /// Halt the firmware (?). +// const MSG_HALT: u64 = 0x85 << 48; + +/// Receive channel doorbell notification. +const MSG_RX_DOORBELL: u64 = 0x42 << 48; + +/// Doorbell number for firmware kicks/wakeups. +const DOORBELL_KICKFW: u64 = 0x10; +/// Doorbell number for device control channel kicks. +const DOORBELL_DEVCTRL: u64 = 0x11; + +// Upper kernel half VA address ranges. +/// Private (cached) firmware structure VA range base. +const IOVA_KERN_PRIV_RANGE: Range = 0xffffffa000000000..0xffffffa600000000; +/// Private (cached) GPU-RO firmware structure VA range base. +const IOVA_KERN_GPU_RO_RANGE: Range = 0xffffffa600000000..0xffffffa800000000; +/// Shared (uncached) firmware structure VA range base. +const IOVA_KERN_SHARED_RANGE: Range = 0xffffffa800000000..0xffffffaa00000000; +/// Shared (uncached) read-only firmware structure VA range base. +const IOVA_KERN_SHARED_RO_RANGE: Range = 0xffffffaa00000000..0xffffffac00000000; +/// GPU/FW shared structure VA range base. +const IOVA_KERN_GPU_RANGE: Range = 0xffffffac00000000..0xffffffae00000000; +/// GPU/FW shared structure VA range base. +const IOVA_KERN_RTKIT_RANGE: Range = 0xffffffae00000000..0xffffffae10000000; +/// Shared (uncached) timestamp region. +pub(crate) const IOVA_KERN_TIMESTAMP_RANGE: Range = 0xffffffae10000000..0xffffffae14000000; +/// FW MMIO VA range base. +const IOVA_KERN_MMIO_RANGE: Range = 0xffffffaf00000000..0xffffffb000000000; + +/// GPU/FW buffer manager control address (context 0 low) +pub(crate) const IOVA_KERN_GPU_BUFMGR_LOW: u64 = 0x20_0000_0000; +/// GPU/FW buffer manager control address (context 0 high) +pub(crate) const IOVA_KERN_GPU_BUFMGR_HIGH: u64 = 0xffffffaeffff0000; + +/// Timeout for entering the halt state after a fault or request. +const HALT_ENTER_TIMEOUT: Delta = Delta::from_millis(100); + +/// Maximum amount of firmware-private memory garbage allowed before collection. +/// Collection flushes the FW cache and is expensive, so this needs to be +/// reasonably high. +const MAX_FW_ALLOC_GARBAGE_BYTES: usize = 16 * 1024 * 1024; +/// Maximum count of firmware-private memory garbage objects allowed before collection. +/// This works out to 16K of memory in the garbage list (8 bytes each), which keeps us +/// within the safe range for kmalloc (on 16K page systems). +const MAX_FW_ALLOC_GARBAGE_OBJECTS: usize = 2048; + +/// Global allocators used for kernel-half structures. +pub(crate) struct KernelAllocators { + pub(crate) private: alloc::DefaultAllocator, + pub(crate) shared: alloc::DefaultAllocator, + pub(crate) shared_ro: alloc::DefaultAllocator, + #[allow(dead_code)] + pub(crate) gpu: alloc::DefaultAllocator, + pub(crate) gpu_ro: alloc::DefaultAllocator, +} + +/// Receive (GPU->driver) ring buffer channels. +#[versions(AGX)] +#[pin_data] +struct RxChannels { + event: channel::EventChannel::ver, + fw_log: channel::FwLogChannel, + ktrace: channel::KTraceChannel, + stats: channel::StatsChannel::ver, +} + +/// GPU work submission pipe channels (driver->GPU). +#[versions(AGX)] +struct PipeChannels { + pub(crate) vtx: KVec>>>, + pub(crate) frag: KVec>>>, + pub(crate) comp: KVec>>>, +} + +/// Misc command transmit (driver->GPU) channels. +#[versions(AGX)] +#[pin_data] +struct TxChannels { + pub(crate) device_control: channel::DeviceControlChannel::ver, +} + +/// Number of work submission pipes per type, one for each priority level. +const NUM_PIPES: usize = 4; + +/// A generic monotonically incrementing ID used to uniquely identify object instances within the +/// driver. +pub(crate) struct ID(AtomicU64); + +impl ID { + /// Create a new ID counter with a given value. + fn new(val: u64) -> ID { + ID(AtomicU64::new(val)) + } + + /// Fetch the next unique ID. + pub(crate) fn next(&self) -> u64 { + self.0.fetch_add(1, Ordering::Relaxed) + } +} + +impl Default for ID { + /// IDs default to starting at 2, as 0/1 are considered reserved for the system. + fn default() -> Self { + Self::new(2) + } +} + +/// A guard representing one active submission on the GPU. When dropped, decrements the active +/// submission count. +pub(crate) struct OpGuard(Arc); + +impl Drop for OpGuard { + fn drop(&mut self) { + self.0.end_op(); + } +} + +/// Set of global sequence IDs used in the driver. +#[derive(Default)] +pub(crate) struct SequenceIDs { + /// `File` instance ID. + pub(crate) file: ID, + /// `Vm` instance ID. + pub(crate) vm: ID, + /// Submission instance ID. + pub(crate) submission: ID, + /// `Queue` instance ID. + pub(crate) queue: ID, +} + +/// Top-level GPU manager that owns all the global state relevant to the driver instance. +#[versions(AGX)] +#[pin_data] +pub(crate) struct GpuManager { + dev: AsahiDevRef, + cfg: &'static hw::HwConfig, + dyncfg: hw::DynConfig, + pub(crate) initdata: fw::types::GpuObject, + uat: mmu::Uat, + crashed: AtomicBool, + #[pin] + alloc: Mutex, + io_mappings: KVec, + next_mmio_iova: u64, + #[pin] + rtkit: Mutex>>, + #[pin] + rx_channels: Mutex, + #[pin] + tx_channels: Mutex, + #[pin] + fwctl_channel: Mutex, + pipes: PipeChannels::ver, + event_manager: Arc, + buffer_mgr: buffer::BufferManager::ver, + ids: SequenceIDs, + #[allow(clippy::vec_box)] + #[pin] + garbage_contexts: Mutex>>>, +} + +/// Trait used to abstract the firmware/GPU-dependent variants of the GpuManager. +pub(crate) trait GpuManager: Send + Sync { + /// Cast as an Any type. + fn as_any(&self) -> &dyn Any; + /// Cast Arc as an Any type. + fn arc_as_any(self: Arc) -> Arc; + /// Initialize the GPU. + fn init(&self) -> Result; + /// Update the GPU globals from global info + /// + /// TODO: Unclear what can and cannot be updated like this. + fn update_globals(&self); + /// Get a reference to the KernelAllocators. + fn alloc(&self) -> Guard<'_, KernelAllocators, MutexBackend>; + /// Create a new `Vm` given a unique `File` ID. + fn new_vm(&self, kernel_range: Range) -> Result; + /// Bind a `Vm` to an available slot and return the `VmBind`. + fn bind_vm(&self, vm: &mmu::Vm) -> Result; + /// Create a new user command queue. + fn new_queue( + &self, + vm: mmu::Vm, + ualloc: Arc>, + ualloc_priv: Arc>, + priority: u32, + usc_exec_base: u64, + ) -> Result>; + /// Return a reference to the global `SequenceIDs` instance. + fn ids(&self) -> &SequenceIDs; + /// Kick the firmware (wake it up if asleep). + /// + /// This should be useful to reduce latency on work submission, so we can ask the firmware to + /// wake up while we do some preparatory work for the work submission. + fn kick_firmware(&self) -> Result; + /// Flush the entire firmware cache. + /// + /// TODO: Does this actually work? + fn flush_fw_cache(&self) -> Result; + /// Handle a GPU work timeout event. + fn handle_timeout(&self, counter: u32, event_slot: i32, unk: u32); + /// Handle a GPU fault event. + fn handle_fault(&self); + /// Handle a channel error event. + fn handle_channel_error( + &self, + error_type: ChannelErrorType, + pipe_type: u32, + event_slot: u32, + event_value: u32, + ); + /// Acknowledge a Buffer grow op. + fn ack_grow(&self, buffer_slot: u32, vm_slot: u32, counter: u32); + /// Send a firmware control command (secure cache flush). + fn fwctl(&self, msg: fw::channels::FwCtlMsg) -> Result; + /// Get the static GPU configuration for this SoC. + fn get_cfg(&self) -> &'static hw::HwConfig; + /// Get the dynamic GPU configuration for this SoC. + fn get_dyncfg(&self) -> &hw::DynConfig; + /// Register an unused context as garbage + fn free_context(&self, data: KBox>); + /// Check whether the GPU is crashed + fn is_crashed(&self) -> bool; + /// Map a BO as a timestamp buffer + fn map_timestamp_buffer( + &self, + bo: gem::ObjectRef, + range: Range, + ) -> Result; +} + +/// Private generic trait for functions that don't need to escape this module. +trait GpuManagerPriv { + /// Decrement the pending submission counter. + fn end_op(&self); +} + +pub(crate) struct RtkitObject { + vmap: shmem::VMap, + mapping: mmu::KernelMapping, +} + +impl rtkit::Buffer for RtkitObject { + fn iova(&self) -> Result { + Ok(self.mapping.iova() as usize) + } + fn buf(&mut self) -> Result> { + Ok(self.vmap.get()) + } +} + +#[versions(AGX)] +#[vtable] +impl rtkit::Operations for GpuManager::ver { + type Data = Arc; + type Buffer = RtkitObject; + + fn recv_message(data: ::Borrowed<'_>, ep: u8, msg: u64) { + let dev = &data.dev; + //dev_info!(dev.as_ref(), "RtKit message: {:#x}:{:#x}\n", ep, msg); + + if ep != EP_FIRMWARE || msg != MSG_RX_DOORBELL { + dev_err!(dev.as_ref(), "Unknown message: {:#x}:{:#x}\n", ep, msg); + return; + } + + let mut ch = data.rx_channels.lock(); + + ch.fw_log.poll(); + ch.ktrace.poll(); + ch.stats.poll(); + ch.event.poll(); + } + + fn crashed(data: ::Borrowed<'_>, crashlog: Option<&[u8]>) { + let dev = &data.dev; + + data.crashed.store(true, Ordering::Relaxed); + + #[cfg(CONFIG_DEV_COREDUMP)] + if let Err(e) = data.generate_crashdump(crashlog) { + dev_err!(dev.as_ref(), "Could not generate crashdump: {:?}\n", e); + } + #[cfg(not(CONFIG_DEV_COREDUMP))] + let _ = crashlog; + + if debug_enabled(DebugFlags::OopsOnGpuCrash) { + panic!("GPU firmware crashed"); + } else { + dev_err!(dev.as_ref(), "GPU firmware crashed, failing all jobs\n"); + data.event_manager.fail_all(workqueue::WorkError::NoDevice); + } + } + + fn shmem_alloc( + data: ::Borrowed<'_>, + size: usize, + ) -> Result { + let dev = &data.dev; + mod_dev_dbg!(dev, "shmem_alloc() {:#x} bytes\n", size); + + let mut obj = gem::new_kernel_object(dev, size)?; + let vmap = obj.gem.owned_vmap()?; + let mapping = obj.map_into_range( + data.uat.kernel_vm(), + IOVA_KERN_RTKIT_RANGE, + mmu::UAT_PGSZ as u64, + mmu::PROT_FW_SHARED_RW, + true, + )?; + mod_dev_dbg!(dev, "shmem_alloc() -> VA {:#x}\n", mapping.iova()); + Ok(RtkitObject { vmap, mapping }) + } +} + +#[versions(AGX)] +impl GpuManager::ver { + /// Create a new GpuManager of this version/GPU combination. + #[inline(never)] + pub(crate) fn new( + dev: &AsahiDevice, + res: ®s::Resources, + cfg: &'static hw::HwConfig, + ) -> Result> { + let uat = Self::make_uat(dev, cfg)?; + let dyncfg = Self::make_dyncfg(dev, res, cfg, &uat)?; + + let mut alloc = KernelAllocators { + private: alloc::DefaultAllocator::new( + dev, + uat.kernel_vm(), + IOVA_KERN_PRIV_RANGE, + 0x80, + mmu::PROT_FW_PRIV_RW, + 1024 * 1024, + true, + fmt!("Kernel Private"), + true, + )?, + shared: alloc::DefaultAllocator::new( + dev, + uat.kernel_vm(), + IOVA_KERN_SHARED_RANGE, + 0x80, + mmu::PROT_FW_SHARED_RW, + 1024 * 1024, + true, + fmt!("Kernel Shared"), + false, + )?, + shared_ro: alloc::DefaultAllocator::new( + dev, + uat.kernel_vm(), + IOVA_KERN_SHARED_RO_RANGE, + 0x80, + mmu::PROT_FW_SHARED_RO, + 64 * 1024, + true, + fmt!("Kernel RO Shared"), + false, + )?, + gpu: alloc::DefaultAllocator::new( + dev, + uat.kernel_vm(), + IOVA_KERN_GPU_RANGE, + 0x80, + mmu::PROT_GPU_FW_SHARED_RW, + 64 * 1024, + true, + fmt!("Kernel GPU Shared"), + false, + )?, + gpu_ro: alloc::DefaultAllocator::new( + dev, + uat.kernel_vm(), + IOVA_KERN_GPU_RO_RANGE, + 0x80, + mmu::PROT_GPU_RO_FW_PRIV_RW, + 1024 * 1024, + true, + fmt!("Kernel GPU RO Shared"), + true, + )?, + }; + + let event_manager = Self::make_event_manager(&mut alloc)?; + let mut initdata = Self::make_initdata(dev, cfg, &dyncfg, &mut alloc)?; + + initdata.runtime_pointers.buffer_mgr_ctl_low_mapping = + Some(initdata.runtime_pointers.buffer_mgr_ctl.map_at( + uat.kernel_lower_vm(), + IOVA_KERN_GPU_BUFMGR_LOW, + mmu::PROT_GPU_SHARED_RW, + false, + )?); + initdata.runtime_pointers.buffer_mgr_ctl_high_mapping = + Some(initdata.runtime_pointers.buffer_mgr_ctl.map_at( + uat.kernel_vm(), + IOVA_KERN_GPU_BUFMGR_HIGH, + mmu::PROT_FW_SHARED_RW, + false, + )?); + + let mut mgr = Self::make_mgr(dev, cfg, dyncfg, uat, alloc, event_manager, initdata)?; + + { + let fwctl = mgr.fwctl_channel.lock(); + let p_fwctl = fwctl.to_raw(); + core::mem::drop(fwctl); + + mgr.as_mut() + .initdata_mut() + .fw_status + .with_mut(|raw, _inner| { + raw.fwctl_channel = p_fwctl; + }); + } + + { + let txc = mgr.tx_channels.lock(); + let p_device_control = txc.device_control.to_raw(); + core::mem::drop(txc); + + let rxc = mgr.rx_channels.lock(); + let p_event = rxc.event.to_raw(); + let p_fw_log = rxc.fw_log.to_raw(); + let p_ktrace = rxc.ktrace.to_raw(); + let p_stats = rxc.stats.to_raw(); + let p_fwlog_buf = rxc.fw_log.get_buf(); + core::mem::drop(rxc); + + mgr.as_mut() + .initdata_mut() + .runtime_pointers + .with_mut(|raw, _inner| { + raw.device_control = p_device_control; + raw.event = p_event; + raw.fw_log = p_fw_log; + raw.ktrace = p_ktrace; + raw.stats = p_stats; + raw.fwlog_buf = Some(p_fwlog_buf); + }); + } + + let mut p_pipes: KVec = KVec::new(); + + for ((v, f), c) in mgr + .pipes + .vtx + .iter() + .zip(&mgr.pipes.frag) + .zip(&mgr.pipes.comp) + { + p_pipes.push( + fw::initdata::raw::PipeChannels::ver { + vtx: v.lock().to_raw(), + frag: f.lock().to_raw(), + comp: c.lock().to_raw(), + }, + GFP_KERNEL, + )?; + } + + mgr.as_mut() + .initdata_mut() + .runtime_pointers + .with_mut(|raw, _inner| { + for (i, p) in p_pipes.into_iter().enumerate() { + raw.pipes[i].vtx = p.vtx; + raw.pipes[i].frag = p.frag; + raw.pipes[i].comp = p.comp; + } + }); + + for (i, map) in cfg.io_mappings.iter().enumerate() { + if let Some(map) = map.as_ref() { + Self::iomap(&mut mgr, cfg, i, map)?; + } + } + + #[ver(V >= V13_0B4)] + if let Some(base) = cfg.sram_base { + let size = cfg.sram_size.unwrap(); + let iova = mgr.as_mut().alloc_mmio_iova(size); + + let mapping = mgr + .uat + .kernel_vm() + .map_io(iova, base, size, mmu::PROT_FW_SHARED_RW)?; + + mgr.as_mut() + .initdata_mut() + .runtime_pointers + .hwdata_b + .with_mut(|raw, _| { + raw.sgx_sram_ptr = U64(mapping.iova()); + }); + + mgr.as_mut().io_mappings_mut().push(mapping, GFP_KERNEL)?; + } + + let mgr = Arc::from(mgr); + + let rtkit = rtkit::RtKit::::new(dev.as_ref(), None, 0, mgr.clone())?; + + *mgr.rtkit.lock() = Some(rtkit); + + { + let mut rxc = mgr.rx_channels.lock(); + rxc.event.set_manager(mgr.clone()); + } + + Ok(mgr) + } + + /// Return a mutable reference to the initdata member + fn initdata_mut( + self: Pin<&mut Self>, + ) -> &mut fw::types::GpuObject { + // SAFETY: initdata does not require structural pinning. + unsafe { &mut self.get_unchecked_mut().initdata } + } + + /// Return a mutable reference to the io_mappings member + fn io_mappings_mut(self: Pin<&mut Self>) -> &mut KVec { + // SAFETY: io_mappings does not require structural pinning. + unsafe { &mut self.get_unchecked_mut().io_mappings } + } + + /// Allocate an MMIO iova range + fn alloc_mmio_iova(self: Pin<&mut Self>, size: usize) -> u64 { + // SAFETY: next_mmio_iova does not require structural pinning. + let next_ref = unsafe { &mut self.get_unchecked_mut().next_mmio_iova }; + + let addr = *next_ref; + let next = addr + (size + mmu::UAT_PGSZ) as u64; + + assert!(next <= IOVA_KERN_MMIO_RANGE.end); + + *next_ref = next; + + addr + } + + /// Build the entire GPU InitData structure tree and return it as a boxed GpuObject. + fn make_initdata( + dev: &AsahiDevice, + cfg: &'static hw::HwConfig, + dyncfg: &hw::DynConfig, + alloc: &mut KernelAllocators, + ) -> Result>> { + let mut builder = initdata::InitDataBuilder::ver::new(dev, alloc, cfg, dyncfg); + builder.build() + } + + /// Create a fresh boxed Uat instance. + /// + /// Force disable inlining to avoid blowing up the stack. + #[inline(never)] + fn make_uat(dev: &AsahiDevice, cfg: &'static hw::HwConfig) -> Result> { + // G14X has a new thing in the Scene structure that unfortunately requires + // write access from user contexts. Hopefully it's not security-sensitive. + #[ver(G >= G14X)] + let map_kernel_to_user = true; + #[ver(G < G14X)] + let map_kernel_to_user = false; + + Ok(KBox::new( + mmu::Uat::new(dev, cfg, map_kernel_to_user)?, + GFP_KERNEL, + )?) + } + + /// Actually create the final GpuManager instance, as a UniqueArc. + /// + /// Force disable inlining to avoid blowing up the stack. + #[inline(never)] + fn make_mgr( + dev: &AsahiDevice, + cfg: &'static hw::HwConfig, + dyncfg: KBox, + uat: KBox, + mut alloc: KernelAllocators, + event_manager: Arc, + initdata: KBox>, + ) -> Result>> { + let mut pipes = PipeChannels::ver { + vtx: KVec::new(), + frag: KVec::new(), + comp: KVec::new(), + }; + + for _i in 0..=NUM_PIPES - 1 { + pipes.vtx.push( + KBox::pin_init( + new_mutex!(channel::PipeChannel::ver::new(dev, &mut alloc)?, "pipe_vtx",), + GFP_KERNEL, + )?, + GFP_KERNEL, + )?; + pipes.frag.push( + KBox::pin_init( + new_mutex!( + channel::PipeChannel::ver::new(dev, &mut alloc)?, + "pipe_frag", + ), + GFP_KERNEL, + )?, + GFP_KERNEL, + )?; + pipes.comp.push( + KBox::pin_init( + new_mutex!( + channel::PipeChannel::ver::new(dev, &mut alloc)?, + "pipe_comp", + ), + GFP_KERNEL, + )?, + GFP_KERNEL, + )?; + } + + let fwctl_channel = channel::FwCtlChannel::new(dev, &mut alloc)?; + + let buffer_mgr = buffer::BufferManager::ver::new()?; + let event_manager_clone = event_manager.clone(); + let buffer_mgr_clone = buffer_mgr.clone(); + let alloc_ref = &mut alloc; + let rx_channels = KBox::init( + try_init!(RxChannels::ver { + event: channel::EventChannel::ver::new( + dev, + alloc_ref, + event_manager_clone, + buffer_mgr_clone, + )?, + fw_log: channel::FwLogChannel::new(dev, alloc_ref)?, + ktrace: channel::KTraceChannel::new(dev, alloc_ref)?, + stats: channel::StatsChannel::ver::new(dev, alloc_ref)?, + }), + GFP_KERNEL, + )?; + + let alloc_ref = &mut alloc; + let tx_channels = KBox::init( + try_init!(TxChannels::ver { + device_control: channel::DeviceControlChannel::ver::new(dev, alloc_ref)?, + }), + GFP_KERNEL, + )?; + + let x = UniqueArc::pin_init( + try_pin_init!(GpuManager::ver { + dev: dev.into(), + cfg, + dyncfg: KBox::::into_inner(dyncfg), + initdata: KBox::>::into_inner(initdata), + uat: KBox::::into_inner(uat), + io_mappings: KVec::new(), + next_mmio_iova: IOVA_KERN_MMIO_RANGE.start, + rtkit <- new_mutex!(None, "rtkit"), + crashed: AtomicBool::new(false), + event_manager, + alloc <- new_mutex!(alloc, "alloc"), + fwctl_channel <- new_mutex!(fwctl_channel, "fwctl_channel"), + rx_channels <- new_mutex!(KBox::::into_inner(rx_channels), "rx_channels"), + tx_channels <- new_mutex!(KBox::::into_inner(tx_channels), "tx_channels"), + pipes, + buffer_mgr, + ids: Default::default(), + garbage_contexts <- new_mutex!(KVec::new(), "garbage_contexts"), + }), + GFP_KERNEL, + )?; + + Ok(x) + } + + fn load_hwdata_blob(dev: &AsahiDevice, name: &CStr, size_name: &CStr) -> Result> { + let of_node = dev.as_ref().of_node().ok_or(EINVAL)?; + let size: usize = dev + .as_ref() + .fwnode() + .ok_or(ENOENT)? + .property_read::(size_name) + .or(0) + .try_into()?; + let res = of_node.reserved_mem_region_to_resource_byname(name)?; + // SAFETY: No dma here, just loading init data. + let mem = unsafe { Mem::try_new(res, (MemFlag::WB).into())? }; + if size > mem.size() { + return Err(ENOENT); + } + // SAFETY: trusting the bootloader to fill it out correctly + let blob_sl = unsafe { slice::from_raw_parts(mem.ptr(), size) }; + let mut blob = KVVec::new(); + blob.extend_from_slice(blob_sl, GFP_KERNEL)?; + Ok(blob) + } + + /// Fetch and validate the GPU dynamic configuration from the device tree and hardware. + /// + /// Force disable inlining to avoid blowing up the stack. + #[inline(never)] + fn make_dyncfg( + dev: &AsahiDevice, + res: ®s::Resources, + cfg: &'static hw::HwConfig, + uat: &mmu::Uat, + ) -> Result> { + let gpu_id = res.get_gpu_id()?; + + dev_info!(dev.as_ref(), "GPU Information:\n"); + dev_info!( + dev.as_ref(), + " Type: {:?}{:?}\n", + gpu_id.gpu_gen, + gpu_id.gpu_variant + ); + dev_info!(dev.as_ref(), " Clusters: {}\n", gpu_id.num_clusters); + dev_info!( + dev.as_ref(), + " Cores: {} ({})\n", + gpu_id.num_cores, + gpu_id.num_cores * gpu_id.num_clusters + ); + dev_info!( + dev.as_ref(), + " Frags: {} ({})\n", + gpu_id.num_frags, + gpu_id.num_frags * gpu_id.num_clusters + ); + dev_info!( + dev.as_ref(), + " GPs: {} ({})\n", + gpu_id.num_gps, + gpu_id.num_gps * gpu_id.num_clusters + ); + dev_info!(dev.as_ref(), " Core masks: {:#x?}\n", gpu_id.core_masks); + dev_info!( + dev.as_ref(), + " Active cores: {}\n", + gpu_id.total_active_cores + ); + + dev_info!(dev.as_ref(), "Getting configuration from device tree...\n"); + let pwr_cfg = hw::PwrConfig::load(dev, cfg)?; + dev_info!(dev.as_ref(), "Dynamic configuration fetched\n"); + + if gpu_id.gpu_gen != cfg.gpu_gen || gpu_id.gpu_variant != cfg.gpu_variant { + dev_err!( + dev.as_ref(), + "GPU type mismatch (expected {:?}{:?}, found {:?}{:?})\n", + cfg.gpu_gen, + cfg.gpu_variant, + gpu_id.gpu_gen, + gpu_id.gpu_variant + ); + return Err(EIO); + } + if gpu_id.num_clusters > cfg.max_num_clusters { + dev_err!( + dev.as_ref(), + "Too many clusters ({} > {})\n", + gpu_id.num_clusters, + cfg.max_num_clusters + ); + return Err(EIO); + } + if gpu_id.num_cores > cfg.max_num_cores { + dev_err!( + dev.as_ref(), + "Too many cores ({} > {})\n", + gpu_id.num_cores, + cfg.max_num_cores + ); + return Err(EIO); + } + if gpu_id.num_frags > cfg.max_num_frags { + dev_err!( + dev.as_ref(), + "Too many frags ({} > {})\n", + gpu_id.num_frags, + cfg.max_num_frags + ); + return Err(EIO); + } + if gpu_id.num_gps > cfg.max_num_gps { + dev_err!( + dev.as_ref(), + "Too many GPs ({} > {})\n", + gpu_id.num_gps, + cfg.max_num_gps + ); + return Err(EIO); + } + + let fwnode = dev.as_ref().fwnode().ok_or(ENOENT)?; + + Ok(KBox::new( + hw::DynConfig { + pwr: pwr_cfg, + uat_ttb_base: uat.ttb_base(), + id: gpu_id, + firmware_version: fwnode + .property_read_array_vec(c_str!("apple,firmware-version"), 3)? + .or(kernel::kvec![0; 3]?), + + hw_data_a: Self::load_hwdata_blob( + dev, + c_str!("hw-cal-a"), + c_str!("debug,hw-cal-a-size"), + ) + .unwrap_or(KVVec::new()), + hw_data_b: Self::load_hwdata_blob( + dev, + c_str!("hw-cal-b"), + c_str!("debug,hw-cal-b-size"), + ) + .unwrap_or(KVVec::new()), + hw_globals: Self::load_hwdata_blob( + dev, + c_str!("globals"), + c_str!("debug,globals-size"), + ) + .unwrap_or(KVVec::new()), + }, + GFP_KERNEL, + )?) + } + + /// Create the global GPU event manager, and return an `Arc<>` to it. + fn make_event_manager(alloc: &mut KernelAllocators) -> Result> { + Ok(Arc::new(event::EventManager::new(alloc)?, GFP_KERNEL)?) + } + + /// Create a new MMIO mapping and add it to the mappings list in initdata at the specified + /// index. + fn iomap( + this: &mut Pin>, + cfg: &'static hw::HwConfig, + index: usize, + map: &hw::IOMapping, + ) -> Result { + let dies = if map.per_die { + cfg.num_dies as usize + } else { + 1 + }; + + let off = map.base & mmu::UAT_PGMSK; + let base = map.base - off; + let end = (map.base + map.size + mmu::UAT_PGMSK) & !mmu::UAT_PGMSK; + let map_size = end - base; + + // Array mappings must be aligned + assert!((off == 0 && map_size == map.size) || (map.count == 1 && !map.per_die)); + assert!(map.count > 0); + + let iova = this.as_mut().alloc_mmio_iova(map_size * map.count * dies); + let mut cur_iova = iova; + + for die in 0..dies { + for i in 0..map.count { + let phys_off = die * 0x20_0000_0000 + i * map.stride; + + let mapping = this.uat.kernel_vm().map_io( + cur_iova, + base + phys_off, + map_size, + if map.writable { + mmu::PROT_FW_MMIO_RW + } else { + mmu::PROT_FW_MMIO_RO + }, + )?; + + this.as_mut().io_mappings_mut().push(mapping, GFP_KERNEL)?; + cur_iova += map_size as u64; + } + } + + this.as_mut() + .initdata_mut() + .runtime_pointers + .hwdata_b + .with_mut(|raw, _| { + raw.io_mappings[index] = fw::initdata::raw::IOMapping { + phys_addr: U64(map.base as u64), + virt_addr: U64(iova + off as u64), + total_size: (map.size * map.count * dies) as u32, + element_size: map.size as u32, + readwrite: U64(map.writable as u64), + }; + }); + + Ok(()) + } + + /// Mark work associated with currently in-progress event slots as failed, after a fault or + /// timeout. + fn mark_pending_events(&self, culprit_slot: Option, error: workqueue::WorkError) { + dev_err!(self.dev.as_ref(), " Pending events:\n"); + + self.initdata.globals.with(|raw, _inner| { + for (index, i) in raw.pending_stamps.iter().enumerate() { + let info = i.info.load(Ordering::Relaxed); + let wait_value = i.wait_value.load(Ordering::Relaxed); + + if info & 1 != 0 { + #[ver(V >= V13_5)] + let slot = (info >> 4) & 0x7f; + #[ver(V < V13_5)] + let slot = (info >> 3) & 0x7f; + #[ver(V >= V13_5)] + let flags = info & 0xf; + #[ver(V < V13_5)] + let flags = info & 0x7; + dev_err!( + self.dev.as_ref(), + " [{}:{}] flags={} value={:#x}\n", + index, + slot, + flags, + wait_value + ); + let error = if culprit_slot.is_some() && culprit_slot != Some(slot) { + workqueue::WorkError::Killed + } else { + error + }; + self.event_manager.mark_error(slot, wait_value, error); + i.info.store(0, Ordering::Relaxed); + i.wait_value.store(0, Ordering::Relaxed); + } + } + }); + } + + /// Fetch the GPU MMU fault information from the hardware registers. + fn get_fault_info(&self) -> Option { + let res = &(*self.dev).resources; + + let info = res.get_fault_info(self.cfg); + if info.is_some() { + dev_err!( + self.dev.as_ref(), + " Fault info: {:#x?}\n", + info.as_ref().unwrap() + ); + } + info + } + + /// Resume the GPU firmware after it halts (due to a timeout, fault, or request). + fn recover(&self) { + self.initdata.fw_status.with(|raw, _inner| { + let halt_count = raw.flags.halt_count.load(Ordering::Relaxed); + let mut halted = raw.flags.halted.load(Ordering::Relaxed); + dev_err!(self.dev.as_ref(), " Halt count: {}\n", halt_count); + dev_err!(self.dev.as_ref(), " Halted: {}\n", halted); + + if halted == 0 { + let start = Instant::::now(); + while start.elapsed() < HALT_ENTER_TIMEOUT { + halted = raw.flags.halted.load(Ordering::Relaxed); + if halted != 0 { + break; + } + mem::sync(); + } + halted = raw.flags.halted.load(Ordering::Relaxed); + } + + if debug_enabled(DebugFlags::NoGpuRecovery) { + dev_crit!( + self.dev.as_ref(), + " GPU recovery is disabled, wedging forever!\n" + ); + } else if halted != 0 { + dev_err!(self.dev.as_ref(), " Attempting recovery...\n"); + raw.flags.halted.store(0, Ordering::SeqCst); + raw.flags.resume.store(1, Ordering::SeqCst); + } else { + dev_err!(self.dev.as_ref(), " Cannot recover.\n"); + } + }); + } + + /// Return the packed GPU enabled core masks. + // Only used for some versions + #[allow(dead_code)] + pub(crate) fn core_masks_packed(&self) -> &[u32] { + self.dyncfg.id.core_masks_packed.as_slice() + } + + /// Kick a submission pipe for a submitted job to tell the firmware to start processing it. + pub(crate) fn run_job(&self, job: workqueue::JobSubmission::ver<'_>) -> Result { + mod_dev_dbg!(self.dev, "GPU: run_job\n"); + + let pipe_type = job.pipe_type(); + mod_dev_dbg!(self.dev, "GPU: run_job: pipe_type={:?}\n", pipe_type); + + let pipes = match pipe_type { + PipeType::Vertex => &self.pipes.vtx, + PipeType::Fragment => &self.pipes.frag, + PipeType::Compute => &self.pipes.comp, + }; + + let index: usize = job.priority() as usize; + let mut pipe = pipes.get(index).ok_or(EIO)?.lock(); + + mod_dev_dbg!(self.dev, "GPU: run_job: run()\n"); + job.run(&mut pipe); + mod_dev_dbg!(self.dev, "GPU: run_job: ring doorbell\n"); + + let mut guard = self.rtkit.lock(); + let rtk = guard.as_mut().as_pin_mut().unwrap(); + rtk.send_message( + EP_DOORBELL, + MSG_TX_DOORBELL | pipe_type as u64 | ((index as u64) << 2), + )?; + mod_dev_dbg!(self.dev, "GPU: run_job: done\n"); + + Ok(()) + } + + pub(crate) fn start_op(self: &Arc) -> Result { + if self.is_crashed() { + return Err(ENODEV); + } + + let val = self + .initdata + .globals + .with(|raw, _inner| raw.pending_submissions.fetch_add(1, Ordering::Acquire)); + + mod_dev_dbg!(self.dev, "OP start (pending: {})\n", val + 1); + self.kick_firmware()?; + Ok(OpGuard(self.clone())) + } + + fn invalidate_context( + &self, + context: &fw::types::GpuObject, + ) -> Result { + mod_dev_dbg!( + self.dev, + "Invalidating GPU context @ {:?}\n", + context.weak_pointer() + ); + + if self.is_crashed() { + return Err(ENODEV); + } + + let mut guard = self.alloc.lock(); + let (garbage_count, _) = guard.private.garbage(); + let (garbage_count_gpuro, _) = guard.gpu_ro.garbage(); + + let dc = context.with( + |raw, _inner| fw::channels::DeviceControlMsg::ver::DestroyContext { + unk_4: 0, + ctx_23: raw.unk_23, + #[ver(V < V13_3)] + __pad0: Default::default(), + unk_c: U32(0), + unk_10: U32(0), + ctx_0: raw.unk_0, + ctx_1: raw.unk_1, + ctx_4: raw.unk_4, + #[ver(V < V13_3)] + __pad1: Default::default(), + #[ver(V < V13_3)] + unk_18: 0, + gpu_context: Some(context.weak_pointer()), + __pad2: Default::default(), + }, + ); + + mod_dev_dbg!(self.dev, "Context invalidation command: {:?}\n", &dc); + + let mut txch = self.tx_channels.lock(); + + let token = txch.device_control.send(&dc); + + { + let mut guard = self.rtkit.lock(); + let rtk = guard.as_mut().as_pin_mut().unwrap(); + rtk.send_message(EP_DOORBELL, MSG_TX_DOORBELL | DOORBELL_DEVCTRL)?; + } + + txch.device_control.wait_for(token)?; + + mod_dev_dbg!( + self.dev, + "GPU context invalidated: {:?}\n", + context.weak_pointer() + ); + + // The invalidation does a cache flush, so it is okay to collect garbage + guard.private.collect_garbage(garbage_count); + guard.gpu_ro.collect_garbage(garbage_count_gpuro); + + Ok(()) + } + + #[cfg(CONFIG_DEV_COREDUMP)] + fn generate_crashdump(&self, crashlog: Option<&[u8]>) -> Result { + // Lock the allocators, to block kernel/FW memory mutations (mostly) + let kalloc = self.alloc(); + let pages = self.uat.dump_kernel_pages()?; + core::mem::drop(kalloc); + + let mut crashdump = crate::crashdump::CrashDumpBuilder::new(pages)?; + let initdata_addr = self.initdata.gpu_va().get(); + crashdump.add_agx_info(self.cfg, &self.dyncfg, initdata_addr)?; + if let Some(crashlog) = crashlog { + crashdump.add_crashlog(crashlog)?; + } + let crashdump = KBox::new(crashdump.finalize()?, GFP_KERNEL)?; + + devcoredump::dev_coredump( + self.dev.as_ref(), + &crate::THIS_MODULE, + crashdump, + GFP_KERNEL, + msecs_to_jiffies(60 * 60 * 1000), + ); + + Ok(()) + } +} + +#[versions(AGX)] +impl GpuManager for GpuManager::ver { + fn as_any(&self) -> &dyn Any { + self + } + + fn arc_as_any(self: Arc) -> Arc { + self as Arc + } + + fn init(&self) -> Result { + self.tx_channels.lock().device_control.send( + &fw::channels::DeviceControlMsg::ver::Initialize(Default::default()), + ); + + let initdata = self.initdata.gpu_va().get(); + let mut guard = self.rtkit.lock(); + let mut rtk = guard.as_mut().as_pin_mut().unwrap(); + + rtk.as_mut().boot()?; + rtk.as_mut().start_endpoint(EP_FIRMWARE)?; + rtk.as_mut().start_endpoint(EP_DOORBELL)?; + rtk.as_mut() + .send_message(EP_FIRMWARE, MSG_INIT | (initdata & INIT_DATA_MASK))?; + rtk.as_mut() + .send_message(EP_DOORBELL, MSG_TX_DOORBELL | DOORBELL_DEVCTRL)?; + core::mem::drop(guard); + + self.kick_firmware()?; + Ok(()) + } + + fn update_globals(&self) { + let mut timeout: u32 = 2; + if debug_enabled(DebugFlags::WaitForPowerOff) { + timeout = 0; + } else if debug_enabled(DebugFlags::KeepGpuPowered) { + timeout = 5000; + } + + self.initdata.globals.with(|raw, _inner| { + raw.idle_off_delay_ms.store(timeout, Ordering::Relaxed); + }); + } + + fn alloc(&self) -> Guard<'_, KernelAllocators, MutexBackend> { + /* Clean up idle contexts */ + let mut garbage_ctx = KVec::new(); + core::mem::swap(&mut *self.garbage_contexts.lock(), &mut garbage_ctx); + + for ctx in garbage_ctx { + if self.invalidate_context(&ctx).is_err() { + dev_err!( + self.dev.as_ref(), + "GpuContext: Failed to invalidate GPU context!\n" + ); + if debug_enabled(DebugFlags::OopsOnGpuCrash) { + panic!("GPU firmware timed out"); + } + } + } + + let mut guard = self.alloc.lock(); + let (garbage_count, garbage_bytes) = guard.private.garbage(); + let (ro_garbage_count, ro_garbage_bytes) = guard.gpu_ro.garbage(); + + if garbage_bytes > MAX_FW_ALLOC_GARBAGE_BYTES + || ro_garbage_bytes > MAX_FW_ALLOC_GARBAGE_BYTES + || garbage_count > MAX_FW_ALLOC_GARBAGE_OBJECTS + || ro_garbage_count > MAX_FW_ALLOC_GARBAGE_OBJECTS + { + mod_dev_dbg!( + self.dev, + "Collecting kalloc garbage (private: {} objects, {} bytes, gpuro: {} objects, {} bytes)\n", + garbage_count, + garbage_bytes, + ro_garbage_count, + ro_garbage_bytes + ); + if self.flush_fw_cache().is_err() { + dev_err!(self.dev.as_ref(), "Failed to flush FW cache\n"); + } else { + guard.private.collect_garbage(garbage_count); + guard.gpu_ro.collect_garbage(ro_garbage_count); + } + } + + guard + } + + fn new_vm(&self, kernel_range: Range) -> Result { + self.uat.new_vm(self.ids.vm.next(), kernel_range) + } + + fn bind_vm(&self, vm: &mmu::Vm) -> Result { + self.uat.bind(vm) + } + + fn new_queue( + &self, + vm: mmu::Vm, + ualloc: Arc>, + ualloc_priv: Arc>, + priority: u32, + usc_exec_base: u64, + ) -> Result> { + let mut kalloc = self.alloc(); + let id = self.ids.queue.next(); + Ok(KBox::new( + queue::Queue::ver::new( + &self.dev, + vm, + &mut kalloc, + ualloc, + ualloc_priv, + self.event_manager.clone(), + &self.buffer_mgr, + id, + priority, + usc_exec_base, + )?, + GFP_KERNEL, + )?) + } + + fn kick_firmware(&self) -> Result { + if self.is_crashed() { + return Err(ENODEV); + } + + let mut guard = self.rtkit.lock(); + let rtk = guard.as_mut().as_pin_mut().unwrap(); + rtk.send_message(EP_DOORBELL, MSG_TX_DOORBELL | DOORBELL_KICKFW)?; + + Ok(()) + } + + fn flush_fw_cache(&self) -> Result { + mod_dev_dbg!(self.dev, "Flushing coprocessor data cache\n"); + + if self.is_crashed() { + return Err(ENODEV); + } + + // ctx_0 == 0xff or ctx_1 == 0xff cause no effect on context, + // but this command does a full cache flush too, so abuse it + // for that. + + let dc = fw::channels::DeviceControlMsg::ver::DestroyContext { + unk_4: 0, + + ctx_23: 0, + #[ver(V < V13_3)] + __pad0: Default::default(), + unk_c: U32(0), + unk_10: U32(0), + ctx_0: 0xff, + ctx_1: 0xff, + ctx_4: 0, + #[ver(V < V13_3)] + __pad1: Default::default(), + #[ver(V < V13_3)] + unk_18: 0, + gpu_context: None, + __pad2: Default::default(), + }; + + let mut txch = self.tx_channels.lock(); + + let token = txch.device_control.send(&dc); + { + let mut guard = self.rtkit.lock(); + let rtk = guard.as_mut().as_pin_mut().unwrap(); + rtk.send_message(EP_DOORBELL, MSG_TX_DOORBELL | DOORBELL_DEVCTRL)?; + } + + txch.device_control.wait_for(token)?; + Ok(()) + } + + fn ids(&self) -> &SequenceIDs { + &self.ids + } + + fn handle_timeout(&self, counter: u32, event_slot: i32, unk: u32) { + dev_err!(self.dev.as_ref(), " (\\________/) \n"); + dev_err!(self.dev.as_ref(), " | | \n"); + dev_err!(self.dev.as_ref(), "'.| \\ , / |.'\n"); + dev_err!(self.dev.as_ref(), "--| / (( \\ |--\n"); + dev_err!(self.dev.as_ref(), ".'| _-_- |'.\n"); + dev_err!(self.dev.as_ref(), " |________| \n"); + dev_err!(self.dev.as_ref(), "** GPU timeout nya~!!!!! **\n"); + dev_err!(self.dev.as_ref(), " Event slot: {}\n", event_slot); + dev_err!(self.dev.as_ref(), " Timeout count: {}\n", counter); + dev_err!(self.dev.as_ref(), " Unk: {}\n", unk); + + // If we have fault info, consider it a fault. + let error = match self.get_fault_info() { + Some(info) => workqueue::WorkError::Fault(info), + None => workqueue::WorkError::Timeout, + }; + self.mark_pending_events(event_slot.try_into().ok(), error); + self.recover(); + } + + fn handle_fault(&self) { + dev_err!(self.dev.as_ref(), " (\\________/) \n"); + dev_err!(self.dev.as_ref(), " | | \n"); + dev_err!(self.dev.as_ref(), "'.| \\ , / |.'\n"); + dev_err!(self.dev.as_ref(), "--| / (( \\ |--\n"); + dev_err!(self.dev.as_ref(), ".'| _-_- |'.\n"); + dev_err!(self.dev.as_ref(), " |________| \n"); + dev_err!(self.dev.as_ref(), "GPU fault nya~!!!!!\n"); + let error = match self.get_fault_info() { + Some(info) => workqueue::WorkError::Fault(info), + None => workqueue::WorkError::Unknown, + }; + self.mark_pending_events(None, error); + self.recover(); + } + + fn handle_channel_error( + &self, + error_type: ChannelErrorType, + pipe_type: u32, + event_slot: u32, + event_value: u32, + ) { + dev_err!(self.dev.as_ref(), " (\\________/) \n"); + dev_err!(self.dev.as_ref(), " | | \n"); + dev_err!(self.dev.as_ref(), "'.| \\ , / |.'\n"); + dev_err!(self.dev.as_ref(), "--| / (( \\ |--\n"); + dev_err!(self.dev.as_ref(), ".'| _-_- |'.\n"); + dev_err!(self.dev.as_ref(), " |________| \n"); + dev_err!(self.dev.as_ref(), "GPU channel error nya~!!!!!\n"); + dev_err!(self.dev.as_ref(), " Error type: {:?}\n", error_type); + dev_err!(self.dev.as_ref(), " Pipe type: {}\n", pipe_type); + dev_err!(self.dev.as_ref(), " Event slot: {}\n", event_slot); + dev_err!(self.dev.as_ref(), " Event value: {:#x?}\n", event_value); + + self.event_manager.mark_error( + event_slot, + event_value, + workqueue::WorkError::ChannelError(error_type), + ); + + let wq = match self.event_manager.get_owner(event_slot) { + Some(wq) => wq, + None => { + dev_err!( + self.dev.as_ref(), + "Workqueue not found for this event slot!\n" + ); + return; + } + }; + + let wq = match wq.as_any().downcast_ref::() { + Some(wq) => wq, + None => { + dev_crit!(self.dev.as_ref(), "GpuManager mismatched with WorkQueue!\n"); + return; + } + }; + + if debug_enabled(DebugFlags::VerboseFaults) { + wq.dump_info(); + } + + let dc = fw::channels::DeviceControlMsg::ver::RecoverChannel { + pipe_type, + work_queue: wq.info_pointer(), + event_value, + __pad: Default::default(), + }; + + mod_dev_dbg!(self.dev, "Recover Channel command: {:?}\n", &dc); + let mut txch = self.tx_channels.lock(); + + let token = txch.device_control.send(&dc); + { + let mut guard = self.rtkit.lock(); + let rtk = guard.as_mut().as_pin_mut().unwrap(); + if rtk + .send_message(EP_DOORBELL, MSG_TX_DOORBELL | DOORBELL_DEVCTRL) + .is_err() + { + dev_err!( + self.dev.as_ref(), + "Failed to send Recover Channel command\n" + ); + } + } + + if txch.device_control.wait_for(token).is_err() { + dev_err!( + self.dev.as_ref(), + "Timed out waiting for Recover Channel command\n" + ); + } + + if debug_enabled(DebugFlags::VerboseFaults) { + wq.dump_info(); + } + } + + fn ack_grow(&self, buffer_slot: u32, vm_slot: u32, counter: u32) { + let halt_count = self + .initdata + .fw_status + .with(|raw, _inner| raw.flags.halt_count.load(Ordering::Relaxed)); + + let dc = fw::channels::DeviceControlMsg::ver::GrowTVBAck { + unk_4: 1, + buffer_slot, + vm_slot, + counter, + subpipe: 0, // TODO + halt_count: U64(halt_count), + __pad: Default::default(), + }; + + mod_dev_dbg!(self.dev, "TVB Grow Ack command: {:?}\n", &dc); + + let mut txch = self.tx_channels.lock(); + + txch.device_control.send(&dc); + { + let mut guard = self.rtkit.lock(); + let rtk = guard.as_mut().as_pin_mut().unwrap(); + if rtk + .send_message(EP_DOORBELL, MSG_TX_DOORBELL | DOORBELL_DEVCTRL) + .is_err() + { + dev_err!(self.dev.as_ref(), "Failed to send TVB Grow Ack command\n"); + } + } + } + + fn fwctl(&self, msg: fw::channels::FwCtlMsg) -> Result { + if self.is_crashed() { + return Err(ENODEV); + } + + let mut fwctl = self.fwctl_channel.lock(); + let token = fwctl.send(&msg); + { + let mut guard = self.rtkit.lock(); + let rtk = guard.as_mut().as_pin_mut().unwrap(); + rtk.send_message(EP_DOORBELL, MSG_FWCTL)?; + } + fwctl.wait_for(token)?; + Ok(()) + } + + fn get_cfg(&self) -> &'static hw::HwConfig { + self.cfg + } + + fn get_dyncfg(&self) -> &hw::DynConfig { + &self.dyncfg + } + + fn free_context(&self, ctx: KBox>) { + let mut garbage = self.garbage_contexts.lock(); + + if garbage.push(ctx, GFP_KERNEL).is_err() { + dev_err!( + self.dev.as_ref(), + "Failed to reserve space for freed context, deadlock possible.\n" + ); + } + } + + fn is_crashed(&self) -> bool { + self.crashed.load(Ordering::Relaxed) + } + + fn map_timestamp_buffer( + &self, + mut bo: gem::ObjectRef, + range: Range, + ) -> Result { + bo.map_range_into_range( + self.uat.kernel_vm(), + range, + IOVA_KERN_TIMESTAMP_RANGE, + mmu::UAT_PGSZ as u64, + mmu::PROT_FW_SHARED_RW, + false, + ) + } +} + +#[versions(AGX)] +impl GpuManagerPriv for GpuManager::ver { + fn end_op(&self) { + let val = self + .initdata + .globals + .with(|raw, _inner| raw.pending_submissions.fetch_sub(1, Ordering::Release)); + + mod_dev_dbg!(self.dev, "OP end (pending: {})\n", val - 1); + } +} diff --git a/drivers/gpu/drm/asahi/hw/mod.rs b/drivers/gpu/drm/asahi/hw/mod.rs new file mode 100644 index 00000000000000..611764b5463b59 --- /dev/null +++ b/drivers/gpu/drm/asahi/hw/mod.rs @@ -0,0 +1,657 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Per-SoC hardware configuration structures +//! +//! This module contains the definitions used to store per-GPU and per-SoC configuration data. + +use crate::driver::AsahiDevice; +use crate::fw::types::*; +use kernel::c_str; +use kernel::prelude::*; + +const MAX_POWERZONES: usize = 5; + +pub(crate) mod t600x; +pub(crate) mod t602x; +pub(crate) mod t8103; +pub(crate) mod t8112; + +/// GPU generation enumeration. Note: Part of the UABI. +#[derive(Debug, PartialEq, Copy, Clone)] +#[repr(u32)] +pub(crate) enum GpuGen { + G13 = 13, + G14 = 14, +} + +/// GPU variant enumeration. Note: Part of the UABI. +#[derive(Debug, PartialEq, Copy, Clone)] +#[repr(u32)] +pub(crate) enum GpuVariant { + P = 'P' as u32, + G = 'G' as u32, + S = 'S' as u32, + C = 'C' as u32, + D = 'D' as u32, +} + +/// GPU revision enumeration. Note: Part of the UABI. +#[derive(Debug, PartialEq, Copy, Clone)] +#[repr(u32)] +pub(crate) enum GpuRevision { + A0 = 0x00, + A1 = 0x01, + B0 = 0x10, + B1 = 0x11, + C0 = 0x20, + C1 = 0x21, +} + +/// GPU core type enumeration. Note: Part of the firmware ABI. +#[derive(Debug, Copy, Clone)] +#[repr(u32)] +pub(crate) enum GpuCore { + // Unknown = 0, + // G5P = 1, + // G5G = 2, + // G9P = 3, + // G9G = 4, + // G10P = 5, + // G11P = 6, + // G11M = 7, + // G11G = 8, + // G12P = 9, + // G13P = 10, + G13G = 11, + G13S = 12, + G13C = 13, + // G14P = 14, + G14G = 15, + G14S = 16, + G14C = 17, + G14D = 18, // Split out, unlike G13D +} + +/// GPU revision ID. Note: Part of the firmware ABI. +#[derive(Debug, PartialEq, Copy, Clone)] +#[repr(u32)] +pub(crate) enum GpuRevisionID { + // Unknown = 0, + A0 = 1, + A1 = 2, + B0 = 3, + B1 = 4, + C0 = 5, + C1 = 6, +} + +/// A single performance state of the GPU. +#[derive(Debug)] +pub(crate) struct PState { + /// Voltage in millivolts, per GPU cluster. + pub(crate) volt_mv: KVec, + /// Frequency in hertz. + pub(crate) freq_hz: u32, + /// Maximum power consumption of the GPU at this pstate, in milliwatts. + pub(crate) pwr_mw: u32, +} + +impl PState { + pub(crate) fn max_volt_mv(&self) -> u32 { + *self.volt_mv.iter().max().expect("No voltages") + } +} + +/// A power zone definition (we have no idea what this is but Apple puts them in the DT). +#[allow(missing_docs)] +#[derive(Debug, Copy, Clone)] +pub(crate) struct PowerZone { + pub(crate) target: u32, + pub(crate) target_offset: u32, + pub(crate) filter_tc: u32, +} + +/// An MMIO mapping used by the firmware. +#[derive(Debug, Copy, Clone)] +pub(crate) struct IOMapping { + /// Base physical address of the mapping. + pub(crate) base: usize, + /// Whether this mapping should be replicated to all dies + pub(crate) per_die: bool, + /// Number of mappings. + pub(crate) count: usize, + /// Size of one mapping. + pub(crate) size: usize, + /// Stride between mappings. + pub(crate) stride: usize, + /// Whether the mapping should be writable. + pub(crate) writable: bool, +} + +impl IOMapping { + /// Convenience constructor for a new IOMapping. + pub(crate) const fn new( + base: usize, + per_die: bool, + count: usize, + size: usize, + stride: usize, + writable: bool, + ) -> IOMapping { + IOMapping { + base, + per_die, + count, + size, + stride, + writable, + } + } +} + +/// Unknown HwConfigA fields that vary from SoC to SoC. +#[allow(missing_docs)] +#[derive(Debug, Copy, Clone)] +pub(crate) struct HwConfigA { + pub(crate) unk_87c: i32, + pub(crate) unk_8cc: u32, + pub(crate) unk_e24: u32, +} + +/// Unknown HwConfigB fields that vary from SoC to SoC. +#[allow(missing_docs)] +#[derive(Debug, Copy, Clone)] +pub(crate) struct HwConfigB { + pub(crate) unk_454: u32, + pub(crate) unk_4e0: u64, + pub(crate) unk_534: u32, + pub(crate) unk_ab8: u32, + pub(crate) unk_abc: u32, + pub(crate) unk_b30: u32, +} + +/// Render command configs that vary from SoC to SoC. +#[derive(Debug, Copy, Clone)] +pub(crate) struct HwRenderConfig { + /// Vertex/tiling-related configuration register (lsb: disable clustering) + pub(crate) tiling_control: u32, +} + +#[derive(Debug)] +pub(crate) struct HwConfigShared2Curves { + pub(crate) t1_coef: u32, + pub(crate) t2: &'static [i16], + pub(crate) t3_coefs: &'static [u32], + pub(crate) t3_scales: &'static [u32], +} + +/// Static hardware clustering configuration for multi-cluster SoCs. +#[derive(Debug)] +pub(crate) struct HwClusteringConfig { + pub(crate) meta1_blocksize: usize, + pub(crate) meta2_size: usize, + pub(crate) meta3_size: usize, + pub(crate) meta4_size: usize, + pub(crate) max_splits: usize, +} + +/// Static hardware configuration for a given SoC model. +#[derive(Debug)] +pub(crate) struct HwConfig { + /// Chip ID in hex format (e.g. 0x8103 for t8103). + pub(crate) chip_id: u32, + /// GPU generation. + pub(crate) gpu_gen: GpuGen, + /// GPU variant type. + pub(crate) gpu_variant: GpuVariant, + /// GPU core type ID (as known by the firmware). + pub(crate) gpu_core: GpuCore, + + /// Base clock used used for timekeeping. + pub(crate) base_clock_hz: u32, + /// Output address space for the UAT on this SoC. + pub(crate) uat_oas: u32, + /// Number of dies on this SoC. + pub(crate) num_dies: u32, + /// Maximum number of clusters on this SoC. + pub(crate) max_num_clusters: u32, + /// Maximum number of cores per cluster for this GPU. + pub(crate) max_num_cores: u32, + /// Maximum number of frags per cluster for this GPU. + pub(crate) max_num_frags: u32, + /// Maximum number of GPs per cluster for this GPU. + pub(crate) max_num_gps: u32, + + /// Required size of the first preemption buffer. + pub(crate) preempt1_size: usize, + /// Required size of the second preemption buffer. + pub(crate) preempt2_size: usize, + /// Required size of the third preemption buffer. + pub(crate) preempt3_size: usize, + + /// Required size of the compute preemption buffer. + pub(crate) compute_preempt1_size: usize, + + pub(crate) clustering: Option, + + /// Rendering-relevant configuration. + pub(crate) render: HwRenderConfig, + + /// Misc HWDataA field values. + pub(crate) da: HwConfigA, + /// Misc HWDataB field values. + pub(crate) db: HwConfigB, + /// HwDataShared1.table. + pub(crate) shared1_tab: &'static [i32], + /// HwDataShared1.unk_a4. + pub(crate) shared1_a4: u32, + /// HwDataShared2.table. + pub(crate) shared2_tab: &'static [i32], + /// HwDataShared2.unk_508. + pub(crate) shared2_unk_508: u32, + /// HwDataShared2.unk_508. + pub(crate) shared2_curves: Option, + + /// HwDataShared3.unk_8. + pub(crate) shared3_unk: u32, + /// HwDataShared3.table. + pub(crate) shared3_tab: &'static [u32], + + /// Globals.idle_off_standby_timer. + pub(crate) idle_off_standby_timer_default: u32, + /// Globals.unk_hws2_4. + pub(crate) unk_hws2_4: Option<[F32; 8]>, + /// Globals.unk_hws2_24. + pub(crate) unk_hws2_24: u32, + /// Globals.unk_54 + pub(crate) global_unk_54: u16, + + /// Constant related to SRAM voltages. + pub(crate) sram_k: F32, + /// Unknown per-cluster coefficients 1. + pub(crate) unk_coef_a: &'static [&'static [F32]], + /// Unknown per-cluster coefficients 2. + pub(crate) unk_coef_b: &'static [&'static [F32]], + /// Unknown table in Global struct. + pub(crate) global_tab: Option<&'static [u8]>, + /// Whether this GPU has CS/AFR performance states + pub(crate) has_csafr: bool, + + /// Temperature sensor list (8 bits per sensor). + pub(crate) fast_sensor_mask: [u64; 2], + /// Temperature sensor list (alternate). + pub(crate) fast_sensor_mask_alt: [u64; 2], + /// Temperature sensor present bitmask. + pub(crate) fast_die0_sensor_present: u32, + /// Required MMIO mappings for this GPU/firmware. + pub(crate) io_mappings: &'static [Option], + /// SRAM base + pub(crate) sram_base: Option, + /// SRAM size + pub(crate) sram_size: Option, +} + +/// Dynamic (fetched from hardware/DT) configuration. +#[derive(Debug)] +pub(crate) struct DynConfig { + /// Base physical address of the UAT TTB (from DT reserved memory region). + pub(crate) uat_ttb_base: u64, + /// GPU ID configuration read from hardware. + pub(crate) id: GpuIdConfig, + /// Power calibration configuration for this specific chip/device. + pub(crate) pwr: PwrConfig, + /// Firmware version. + #[allow(dead_code)] + pub(crate) firmware_version: KVec, + + pub(crate) hw_data_a: KVVec, + pub(crate) hw_data_b: KVVec, + pub(crate) hw_globals: KVVec, +} + +/// Specific GPU ID configuration fetched from SGX MMIO registers. +#[derive(Debug)] +pub(crate) struct GpuIdConfig { + /// GPU generation (should match static config). + pub(crate) gpu_gen: GpuGen, + /// GPU variant type (should match static config). + pub(crate) gpu_variant: GpuVariant, + /// GPU silicon revision. + pub(crate) gpu_rev: GpuRevision, + /// GPU silicon revision ID (firmware enum). + pub(crate) gpu_rev_id: GpuRevisionID, + /// Total number of GPU clusters. + pub(crate) num_clusters: u32, + /// Maximum number of GPU cores per cluster. + pub(crate) num_cores: u32, + /// Number of frags per cluster. + pub(crate) num_frags: u32, + /// Number of GPs per cluster. + pub(crate) num_gps: u32, + /// Total number of active cores for the whole GPU. + pub(crate) total_active_cores: u32, + /// Mask of active cores per cluster. + pub(crate) core_masks: KVec, + /// Packed mask of all active cores. + pub(crate) core_masks_packed: KVec, +} + +/// Configurable CS/AFR GPU power settings from the device tree. +#[derive(Debug)] +pub(crate) struct CsAfrPwrConfig { + /// GPU CS performance state list. + pub(crate) perf_states_cs: KVec, + /// GPU AFR performance state list. + pub(crate) perf_states_afr: KVec, + + /// CS leakage coefficient per die. + pub(crate) leak_coef_cs: KVec, + /// AFR leakage coefficient per die. + pub(crate) leak_coef_afr: KVec, + + /// Minimum voltage for the CS/AFR SRAM power domain in microvolts. + pub(crate) min_sram_microvolt: u32, +} + +/// Configurable GPU power settings from the device tree. +#[derive(Debug)] +pub(crate) struct PwrConfig { + /// GPU performance state list. + pub(crate) perf_states: KVec, + /// GPU power zone list. + pub(crate) power_zones: KVec, + + /// Core leakage coefficient per cluster. + pub(crate) core_leak_coef: KVec, + /// SRAM leakage coefficient per cluster. + pub(crate) sram_leak_coef: KVec, + + pub(crate) csafr: Option, + + /// Maximum total power of the GPU in milliwatts. + pub(crate) max_power_mw: u32, + /// Maximum frequency of the GPU in megahertz. + pub(crate) max_freq_mhz: u32, + + /// Minimum performance state to start at. + pub(crate) perf_base_pstate: u32, + /// Maximum enabled performance state. + pub(crate) perf_max_pstate: u32, + + /// Minimum voltage for the SRAM power domain in microvolts. + pub(crate) min_sram_microvolt: u32, + + // Most of these fields are just named after Apple ADT property names and we don't fully + // understand them. They configure various power-related PID loops and filters. + /// Average power filter time constant in milliseconds. + pub(crate) avg_power_filter_tc_ms: u32, + /// Average power filter PID integral gain? + pub(crate) avg_power_ki_only: F32, + /// Average power filter PID proportional gain? + pub(crate) avg_power_kp: F32, + pub(crate) avg_power_min_duty_cycle: u32, + /// Average power target filter time constant in periods. + pub(crate) avg_power_target_filter_tc: u32, + /// "Fast die0" (temperature?) PID integral gain. + pub(crate) fast_die0_integral_gain: F32, + /// "Fast die0" (temperature?) PID proportional gain. + pub(crate) fast_die0_proportional_gain: F32, + pub(crate) fast_die0_prop_tgt_delta: u32, + pub(crate) fast_die0_release_temp: u32, + /// Delay from the fender (?) becoming idle to powerdown + pub(crate) fender_idle_off_delay_ms: u32, + /// Timeout from firmware early wake to sleep if no work was submitted (?) + pub(crate) fw_early_wake_timeout_ms: u32, + /// Delay from the GPU becoming idle to powerdown + pub(crate) idle_off_delay_ms: u32, + /// Related to the above? + pub(crate) idle_off_standby_timer: u32, + /// Percent? + pub(crate) perf_boost_ce_step: u32, + /// Minimum utilization before performance state is increased in %. + pub(crate) perf_boost_min_util: u32, + pub(crate) perf_filter_drop_threshold: u32, + /// Performance PID filter time constant? (periods?) + pub(crate) perf_filter_time_constant: u32, + /// Performance PID filter time constant 2? (periods?) + pub(crate) perf_filter_time_constant2: u32, + /// Performance PID integral gain. + pub(crate) perf_integral_gain: F32, + /// Performance PID integral gain 2 (?). + pub(crate) perf_integral_gain2: F32, + pub(crate) perf_integral_min_clamp: u32, + /// Performance PID proportional gain. + pub(crate) perf_proportional_gain: F32, + /// Performance PID proportional gain 2 (?). + pub(crate) perf_proportional_gain2: F32, + pub(crate) perf_reset_iters: u32, + /// Target GPU utilization for the performance controller in %. + pub(crate) perf_tgt_utilization: u32, + /// Power sampling period in milliseconds. + pub(crate) power_sample_period: u32, + /// PPM (?) filter time constant in milliseconds. + pub(crate) ppm_filter_time_constant_ms: u32, + /// PPM (?) filter PID integral gain. + pub(crate) ppm_ki: F32, + /// PPM (?) filter PID proportional gain. + pub(crate) ppm_kp: F32, + /// Power consumption filter time constant (periods?) + pub(crate) pwr_filter_time_constant: u32, + /// Power consumption filter PID integral gain. + pub(crate) pwr_integral_gain: F32, + pub(crate) pwr_integral_min_clamp: u32, + pub(crate) pwr_min_duty_cycle: u32, + pub(crate) pwr_proportional_gain: F32, + /// Power sample period in base clocks, used when not an integer number of ms + pub(crate) pwr_sample_period_aic_clks: u32, + + pub(crate) se_engagement_criteria: i32, + pub(crate) se_filter_time_constant: u32, + pub(crate) se_filter_time_constant_1: u32, + pub(crate) se_inactive_threshold: u32, + pub(crate) se_ki: F32, + pub(crate) se_ki_1: F32, + pub(crate) se_kp: F32, + pub(crate) se_kp_1: F32, + pub(crate) se_reset_criteria: u32, +} + +impl PwrConfig { + fn load_opp( + dev: &AsahiDevice, + name: &CStr, + cfg: &HwConfig, + is_main: bool, + ) -> Result> { + let mut perf_states = KVec::new(); + + let node = dev.as_ref().of_node().ok_or(EIO)?; + let opps = node.parse_phandle(name, 0).ok_or(EIO)?; + + for opp in opps.children() { + let freq_hz: u64 = opp.get_property(c_str!("opp-hz"))?; + let mut volt_uv: KVec = opp.get_property(c_str!("opp-microvolt"))?; + let pwr_uw: u32 = if is_main { + opp.get_property(c_str!("opp-microwatt"))? + } else { + 0 + }; + + let voltage_count = if is_main { + cfg.max_num_clusters + } else { + cfg.num_dies + }; + + if volt_uv.len() != voltage_count as usize { + dev_err!( + dev.as_ref(), + "Invalid opp-microvolt length (expected {}, got {})\n", + voltage_count, + volt_uv.len() + ); + return Err(EINVAL); + } + + volt_uv.iter_mut().for_each(|a| *a /= 1000); + let volt_mv = volt_uv; + + let pwr_mw = pwr_uw / 1000; + + perf_states.push( + PState { + freq_hz: freq_hz.try_into()?, + volt_mv, + pwr_mw, + }, + GFP_KERNEL, + )?; + } + + if perf_states.is_empty() { + Err(EINVAL) + } else { + Ok(perf_states) + } + } + + /// Load the GPU power configuration from the device tree. + pub(crate) fn load(dev: &AsahiDevice, cfg: &HwConfig) -> Result { + let perf_states = Self::load_opp(dev, c_str!("operating-points-v2"), cfg, true)?; + let node = dev.as_ref().of_node().ok_or(EIO)?; + + macro_rules! prop { + ($prop:expr, $default:expr) => {{ + node.get_opt_property(c_str!($prop)) + .map_err(|e| { + dev_err!(dev.as_ref(), "Error reading property {}: {:?}\n", $prop, e); + e + })? + .unwrap_or($default) + }}; + ($prop:expr) => {{ + node.get_property(c_str!($prop)).map_err(|e| { + dev_err!(dev.as_ref(), "Error reading property {}: {:?}\n", $prop, e); + e + })? + }}; + } + + let pz_data = prop!("apple,power-zones", KVec::new()); + + if pz_data.len() > 3 * MAX_POWERZONES || pz_data.len() % 3 != 0 { + dev_err!(dev.as_ref(), "Invalid apple,power-zones value\n"); + return Err(EINVAL); + } + + let pz_count = pz_data.len() / 3; + let mut power_zones = KVec::new(); + for i in (0..pz_count).step_by(3) { + power_zones.push( + PowerZone { + target: pz_data[i], + target_offset: pz_data[i + 1], + filter_tc: pz_data[i + 2], + }, + GFP_KERNEL, + )?; + } + + let core_leak_coef: KVec = prop!("apple,core-leak-coef"); + let sram_leak_coef: KVec = prop!("apple,sram-leak-coef"); + + if core_leak_coef.len() != cfg.max_num_clusters as usize { + dev_err!(dev.as_ref(), "Invalid apple,core-leak-coef\n"); + return Err(EINVAL); + } + if sram_leak_coef.len() != cfg.max_num_clusters as usize { + dev_err!(dev.as_ref(), "Invalid apple,sram_leak_coef\n"); + return Err(EINVAL); + } + + let csafr = if cfg.has_csafr { + Some(CsAfrPwrConfig { + perf_states_cs: Self::load_opp(dev, c_str!("apple,cs-opp"), cfg, false)?, + perf_states_afr: Self::load_opp(dev, c_str!("apple,afr-opp"), cfg, false)?, + leak_coef_cs: prop!("apple,cs-leak-coef"), + leak_coef_afr: prop!("apple,afr-leak-coef"), + min_sram_microvolt: prop!("apple,csafr-min-sram-microvolt"), + }) + } else { + None + }; + + let power_sample_period: u32 = prop!("apple,power-sample-period"); + + Ok(PwrConfig { + core_leak_coef, + sram_leak_coef, + + max_power_mw: perf_states.iter().map(|a| a.pwr_mw).max().unwrap(), + max_freq_mhz: perf_states.iter().map(|a| a.freq_hz).max().unwrap() / 1_000_000, + + perf_base_pstate: prop!("apple,perf-base-pstate", 1), + perf_max_pstate: perf_states.len() as u32 - 1, + min_sram_microvolt: prop!("apple,min-sram-microvolt"), + + avg_power_filter_tc_ms: prop!("apple,avg-power-filter-tc-ms"), + avg_power_ki_only: prop!("apple,avg-power-ki-only"), + avg_power_kp: prop!("apple,avg-power-kp"), + avg_power_min_duty_cycle: prop!("apple,avg-power-min-duty-cycle"), + avg_power_target_filter_tc: prop!("apple,avg-power-target-filter-tc"), + fast_die0_integral_gain: prop!("apple,fast-die0-integral-gain"), + fast_die0_proportional_gain: prop!("apple,fast-die0-proportional-gain"), + fast_die0_prop_tgt_delta: prop!("apple,fast-die0-prop-tgt-delta", 0), + fast_die0_release_temp: prop!("apple,fast-die0-release-temp", 80), + fender_idle_off_delay_ms: prop!("apple,fender-idle-off-delay-ms", 40), + fw_early_wake_timeout_ms: prop!("apple,fw-early-wake-timeout-ms", 5), + idle_off_delay_ms: prop!("apple,idle-off-delay-ms", 2), + idle_off_standby_timer: prop!( + "apple,idleoff-standby-timer", + cfg.idle_off_standby_timer_default + ), + perf_boost_ce_step: prop!("apple,perf-boost-ce-step", 25), + perf_boost_min_util: prop!("apple,perf-boost-min-util", 100), + perf_filter_drop_threshold: prop!("apple,perf-filter-drop-threshold"), + perf_filter_time_constant2: prop!("apple,perf-filter-time-constant2"), + perf_filter_time_constant: prop!("apple,perf-filter-time-constant"), + perf_integral_gain2: prop!("apple,perf-integral-gain2"), + perf_integral_gain: prop!("apple,perf-integral-gain", f32!(7.8956833)), + perf_integral_min_clamp: prop!("apple,perf-integral-min-clamp"), + perf_proportional_gain2: prop!("apple,perf-proportional-gain2"), + perf_proportional_gain: prop!("apple,perf-proportional-gain", f32!(14.707963)), + perf_reset_iters: prop!("apple,perf-reset-iters", 6), + perf_tgt_utilization: prop!("apple,perf-tgt-utilization"), + power_sample_period, + ppm_filter_time_constant_ms: prop!("apple,ppm-filter-time-constant-ms"), + ppm_ki: prop!("apple,ppm-ki"), + ppm_kp: prop!("apple,ppm-kp"), + pwr_filter_time_constant: prop!("apple,pwr-filter-time-constant", 313), + pwr_integral_gain: prop!("apple,pwr-integral-gain", f32!(0.0202129)), + pwr_integral_min_clamp: prop!("apple,pwr-integral-min-clamp", 0), + pwr_min_duty_cycle: prop!("apple,pwr-min-duty-cycle"), + pwr_proportional_gain: prop!("apple,pwr-proportional-gain", f32!(5.2831855)), + pwr_sample_period_aic_clks: prop!( + "apple,pwr-sample-period-aic-clks", + cfg.base_clock_hz / 1000 * power_sample_period + ), + se_engagement_criteria: prop!("apple,se-engagement-criteria", -1), + se_filter_time_constant: prop!("apple,se-filter-time-constant", 9), + se_filter_time_constant_1: prop!("apple,se-filter-time-constant-1", 3), + se_inactive_threshold: prop!("apple,se-inactive-threshold", 2500), + se_ki: prop!("apple,se-ki", f32!(-50.0)), + se_ki_1: prop!("apple,se-ki-1", f32!(-100.0)), + se_kp: prop!("apple,se-kp", f32!(-5.0)), + se_kp_1: prop!("apple,se-kp-1", f32!(-10.0)), + se_reset_criteria: prop!("apple,se-reset-criteria", 50), + + perf_states, + power_zones, + csafr, + }) + } + + pub(crate) fn max_frequency_khz(&self) -> u32 { + self.perf_states[self.perf_max_pstate as usize].freq_hz / 1000 + } +} diff --git a/drivers/gpu/drm/asahi/hw/t600x.rs b/drivers/gpu/drm/asahi/hw/t600x.rs new file mode 100644 index 00000000000000..58665f985ec38e --- /dev/null +++ b/drivers/gpu/drm/asahi/hw/t600x.rs @@ -0,0 +1,161 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Hardware configuration for t600x (M1 Pro/Max/Ultra) platforms. + +use crate::f32; + +use super::*; + +const fn iomaps(mcc_count: usize, has_die1: bool) -> [Option; 20] { + [ + Some(IOMapping::new(0x404d00000, false, 1, 0x1c000, 0, true)), // Fender + Some(IOMapping::new(0x20e100000, false, 1, 0x4000, 0, false)), // AICTimer + Some(IOMapping::new(0x28e104000, false, 1, 0x4000, 0, true)), // AICSWInt + Some(IOMapping::new(0x404000000, false, 1, 0x20000, 0, true)), // RGX + None, // UVD + None, // unused + None, // DisplayUnderrunWA + Some(IOMapping::new(0x28e494000, true, 1, 0x4000, 0, false)), // AnalogTempSensorControllerRegs + None, // PMPDoorbell + Some(IOMapping::new(0x404d80000, false, 1, 0x8000, 0, true)), // MetrologySensorRegs + Some(IOMapping::new(0x204d61000, false, 1, 0x1000, 0, true)), // GMGIFAFRegs + Some(IOMapping::new( + 0x200000000, + true, + mcc_count, + 0xd8000, + 0x1000000, + true, + )), // MCache registers + None, // AICBankedRegisters + None, // PMGRScratch + Some(IOMapping::new(0x2643c4000, false, 1, 0x1000, 0, true)), // NIA Special agent idle register die 0 + if has_die1 { + // NIA Special agent idle register die 1 + Some(IOMapping::new(0x22643c4000, false, 1, 0x1000, 0, true)) + } else { + None + }, + None, // CRE registers + None, // Streaming codec registers + Some(IOMapping::new(0x28e3d0000, false, 1, 0x1000, 0, true)), // ? + Some(IOMapping::new(0x28e3c0000, false, 1, 0x2000, 0, false)), // ? + ] +} + +pub(crate) const HWCONFIG_T6002: super::HwConfig = HwConfig { + chip_id: 0x6002, + gpu_gen: GpuGen::G13, + gpu_variant: GpuVariant::D, + gpu_core: GpuCore::G13C, + + base_clock_hz: 24_000_000, + uat_oas: 42, + num_dies: 2, + max_num_clusters: 8, + max_num_cores: 8, + max_num_frags: 8, + max_num_gps: 4, + + preempt1_size: 0x540, + preempt2_size: 0x280, + preempt3_size: 0x20, + compute_preempt1_size: 0x3bd00, + clustering: Some(HwClusteringConfig { + meta1_blocksize: 0x44, + meta2_size: 0xc0 * 8, + meta3_size: 0x280 * 8, + meta4_size: 0x30 * 16, + max_splits: 16, + }), + + render: HwRenderConfig { + tiling_control: 0xa540, + }, + + da: HwConfigA { + unk_87c: 900, + unk_8cc: 11000, + unk_e24: 125, + }, + db: HwConfigB { + unk_454: 1, + unk_4e0: 4, + unk_534: 1, + unk_ab8: 0x2084, + unk_abc: 0x80, + unk_b30: 0, + }, + shared1_tab: &[ + 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, + 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, + ], + shared1_a4: 0xffff, + shared2_tab: &[-1, -1, -1, -1, 0x2aa, 0xaaa, -1, -1, 0, 0], + shared2_unk_508: 0xcc00001, + shared2_curves: None, + shared3_unk: 0, + shared3_tab: &[], + idle_off_standby_timer_default: 0, + unk_hws2_4: None, + unk_hws2_24: 0, + global_unk_54: 0xffff, + sram_k: f32!(1.02), + unk_coef_a: &[ + &f32!([9.838]), + &f32!([9.819]), + &f32!([9.826]), + &f32!([9.799]), + &f32!([9.799]), + &f32!([9.826]), + &f32!([9.819]), + &f32!([9.838]), + ], + unk_coef_b: &[ + &f32!([13.0]), + &f32!([13.0]), + &f32!([13.0]), + &f32!([13.0]), + &f32!([13.0]), + &f32!([13.0]), + &f32!([13.0]), + &f32!([13.0]), + ], + global_tab: Some(&[ + 0, 1, 2, 1, 1, 90, 75, 1, 1, 1, 2, 90, 75, 1, 1, 1, 1, 90, 75, 1, 1, + ]), + has_csafr: false, + fast_sensor_mask: [0x8080808080808080, 0], + fast_sensor_mask_alt: [0x9090909090909090, 0], + fast_die0_sensor_present: 0xff, + io_mappings: &iomaps(8, true), + sram_base: None, + sram_size: None, +}; + +pub(crate) const HWCONFIG_T6001: super::HwConfig = HwConfig { + chip_id: 0x6001, + gpu_variant: GpuVariant::C, + gpu_core: GpuCore::G13C, + + num_dies: 1, + max_num_clusters: 4, + fast_sensor_mask: [0x80808080, 0], + fast_sensor_mask_alt: [0x90909090, 0], + fast_die0_sensor_present: 0x0f, + io_mappings: &iomaps(8, false), + ..HWCONFIG_T6002 +}; + +pub(crate) const HWCONFIG_T6000: super::HwConfig = HwConfig { + chip_id: 0x6000, + gpu_variant: GpuVariant::S, + gpu_core: GpuCore::G13S, + + max_num_clusters: 2, + fast_sensor_mask: [0x8080, 0], + fast_sensor_mask_alt: [0x9090, 0], + fast_die0_sensor_present: 0x03, + io_mappings: &iomaps(4, false), + ..HWCONFIG_T6001 +}; diff --git a/drivers/gpu/drm/asahi/hw/t602x.rs b/drivers/gpu/drm/asahi/hw/t602x.rs new file mode 100644 index 00000000000000..98a7ac2b76e571 --- /dev/null +++ b/drivers/gpu/drm/asahi/hw/t602x.rs @@ -0,0 +1,179 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Hardware configuration for t600x (M1 Pro/Max/Ultra) platforms. + +use crate::f32; + +use super::*; + +const fn iomaps(chip_id: u32, mcc_count: usize) -> [Option; 24] { + [ + Some(IOMapping::new(0x404d00000, false, 1, 0x144000, 0, true)), // Fender + Some(IOMapping::new(0x20e100000, false, 1, 0x4000, 0, false)), // AICTimer + Some(IOMapping::new(0x28e106000, false, 1, 0x4000, 0, true)), // AICSWInt + Some(IOMapping::new(0x404000000, false, 1, 0x20000, 0, true)), // RGX + None, // UVD + None, // unused + None, // DisplayUnderrunWA + Some(match chip_id { + 0x6020 => IOMapping::new(0x28e460000, true, 1, 0x4000, 0, false), + _ => IOMapping::new(0x28e478000, true, 1, 0x4000, 0, false), + }), // AnalogTempSensorControllerRegs + None, // PMPDoorbell + Some(IOMapping::new(0x404e08000, false, 1, 0x8000, 0, true)), // MetrologySensorRegs + None, // GMGIFAFRegs + Some(IOMapping::new( + 0x200000000, + true, + mcc_count, + 0xd8000, + 0x1000000, + true, + )), // MCache registers + Some(IOMapping::new(0x28e118000, false, 1, 0x4000, 0, false)), // AICBankedRegisters + None, // PMGRScratch + None, // NIA Special agent idle register die 0 + None, // NIA Special agent idle register die 1 + None, // CRE registers + None, // Streaming codec registers + Some(IOMapping::new(0x28e3d0000, false, 1, 0x4000, 0, true)), // ? + Some(IOMapping::new(0x28e3c0000, false, 1, 0x4000, 0, false)), // ? + Some(IOMapping::new(0x28e3d8000, false, 1, 0x4000, 0, true)), // ? + Some(IOMapping::new(0x404eac000, true, 1, 0x4000, 0, true)), // ? + None, + None, + ] +} + +// TODO: Tentative +pub(crate) const HWCONFIG_T6022: super::HwConfig = HwConfig { + chip_id: 0x6022, + gpu_gen: GpuGen::G14, + gpu_variant: GpuVariant::D, + gpu_core: GpuCore::G14D, + + base_clock_hz: 24_000_000, + uat_oas: 42, + num_dies: 2, + max_num_clusters: 8, + max_num_cores: 10, + max_num_frags: 10, + max_num_gps: 4, + + preempt1_size: 0x540, + preempt2_size: 0x280, + preempt3_size: 0x40, + compute_preempt1_size: 0x25980 * 2, // Conservative guess + clustering: Some(HwClusteringConfig { + meta1_blocksize: 0x44, + meta2_size: 0xc0 * 16, + meta3_size: 0x280 * 16, + meta4_size: 0x10 * 128, + max_splits: 64, + }), + + render: HwRenderConfig { + tiling_control: 0x180340, + }, + + da: HwConfigA { + unk_87c: 500, + unk_8cc: 11000, + unk_e24: 125, + }, + db: HwConfigB { + unk_454: 1, + unk_4e0: 4, + unk_534: 0, + unk_ab8: 0, // Unused + unk_abc: 0, // Unused + unk_b30: 0, + }, + shared1_tab: &[ + 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, + 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, + ], + shared1_a4: 0, + shared2_tab: &[0x800, 0x1555, -1, -1, -1, -1, -1, -1, 0xaaaaa, 0], + shared2_unk_508: 0xc00007, + shared2_curves: Some(HwConfigShared2Curves { + t1_coef: 11000, + t2: &[ + 0xf07, 0x4c0, 0x680, 0x8c0, 0xa80, 0xc40, 0xd80, 0xec0, 0xf40, + ], + t3_coefs: &[0, 20, 27, 36, 43, 50, 55, 60, 62], + t3_scales: &[9, 3209, 10400], + }), + shared3_unk: 8, + shared3_tab: &[ + 125, 125, 125, 125, 125, 125, 125, 125, 7500, 125, 125, 125, 125, 125, 125, 125, + ], + idle_off_standby_timer_default: 700, + unk_hws2_4: Some(f32!([1.0, 0.8, 0.2, 0.9, 0.1, 0.25, 0.5, 0.9])), + unk_hws2_24: 6, + global_unk_54: 4000, + sram_k: f32!(1.02), + unk_coef_a: &[ + &f32!([0.0, 8.2, 0.0, 6.9, 6.9]), + &f32!([0.0, 0.0, 0.0, 6.9, 6.9]), + &f32!([0.0, 8.2, 0.0, 6.9, 0.0]), + &f32!([0.0, 0.0, 0.0, 6.9, 0.0]), + &f32!([0.0, 0.0, 0.0, 6.9, 0.0]), + &f32!([0.0, 8.2, 0.0, 6.9, 0.0]), + &f32!([0.0, 0.0, 0.0, 6.9, 6.9]), + &f32!([0.0, 8.2, 0.0, 6.9, 6.9]), + ], + unk_coef_b: &[ + &f32!([0.0, 9.0, 0.0, 8.0, 8.0]), + &f32!([0.0, 0.0, 0.0, 8.0, 8.0]), + &f32!([0.0, 9.0, 0.0, 8.0, 0.0]), + &f32!([0.0, 0.0, 0.0, 8.0, 0.0]), + &f32!([0.0, 0.0, 0.0, 8.0, 0.0]), + &f32!([0.0, 9.0, 0.0, 8.0, 0.0]), + &f32!([0.0, 0.0, 0.0, 8.0, 8.0]), + &f32!([0.0, 9.0, 0.0, 8.0, 8.0]), + ], + global_tab: Some(&[ + 0, 2, 2, 1, 1, 90, 75, 1, 1, 1, 2, 90, 75, 1, 1, 1, 2, 90, 75, 1, 1, 1, 1, 90, 75, 1, 1, + ]), + has_csafr: true, + fast_sensor_mask: [0x40005000c000d00, 0xd000c0005000400], + // Apple typo? Should probably be 0x140015001c001d00 + fast_sensor_mask_alt: [0x140015001d001d00, 0x1d001c0015001400], + fast_die0_sensor_present: 0, // Unused + io_mappings: &iomaps(0x6022, 8), + sram_base: Some(0x404d60000), + sram_size: Some(0x20000), +}; + +pub(crate) const HWCONFIG_T6021: super::HwConfig = HwConfig { + chip_id: 0x6021, + gpu_variant: GpuVariant::C, + gpu_core: GpuCore::G14C, + + num_dies: 1, + max_num_clusters: 4, + compute_preempt1_size: 0x25980, + unk_hws2_4: Some(f32!([1.0, 0.8, 0.2, 0.9, 0.1, 0.25, 0.7, 0.9])), + fast_sensor_mask: [0x40005000c000d00, 0], + fast_sensor_mask_alt: [0x140015001d001d00, 0], + io_mappings: &iomaps(0x6021, 8), + ..HWCONFIG_T6022 +}; + +pub(crate) const HWCONFIG_T6020: super::HwConfig = HwConfig { + chip_id: 0x6020, + gpu_variant: GpuVariant::S, + gpu_core: GpuCore::G14S, + + db: HwConfigB { + unk_454: 0, + ..HWCONFIG_T6021.db + }, + + max_num_clusters: 2, + fast_sensor_mask: [0xc000d00, 0], + fast_sensor_mask_alt: [0x1d001d00, 0], + io_mappings: &iomaps(0x6020, 4), + ..HWCONFIG_T6021 +}; diff --git a/drivers/gpu/drm/asahi/hw/t8103.rs b/drivers/gpu/drm/asahi/hw/t8103.rs new file mode 100644 index 00000000000000..484bf6c3414f2f --- /dev/null +++ b/drivers/gpu/drm/asahi/hw/t8103.rs @@ -0,0 +1,92 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Hardware configuration for t8103 platforms (M1). + +use crate::f32; + +use super::*; + +pub(crate) const HWCONFIG: super::HwConfig = HwConfig { + chip_id: 0x8103, + gpu_gen: GpuGen::G13, + gpu_variant: GpuVariant::G, + gpu_core: GpuCore::G13G, + + base_clock_hz: 24_000_000, + uat_oas: 40, + num_dies: 1, + max_num_clusters: 1, + max_num_cores: 8, + max_num_frags: 8, + max_num_gps: 4, + + preempt1_size: 0x540, + preempt2_size: 0x280, + preempt3_size: 0x20, + compute_preempt1_size: 0x7f80, + clustering: None, + + render: HwRenderConfig { + // bit 0: disable clustering (always) + tiling_control: 0xa041, + }, + + da: HwConfigA { + unk_87c: -220, + unk_8cc: 9880, + unk_e24: 112, + }, + db: HwConfigB { + unk_454: 1, + unk_4e0: 0, + unk_534: 0, + unk_ab8: 0x48, + unk_abc: 0x8, + unk_b30: 0, + }, + shared1_tab: &[ + -1, 0x7282, 0x50ea, 0x370a, 0x25be, 0x1c1f, 0x16fb, -1, -1, -1, -1, -1, -1, -1, -1, -1, + ], + shared1_a4: 0xffff, + shared2_tab: &[0x800, 0x1555, -1, -1, -1, -1, -1, -1, 0, 0], + shared2_unk_508: 0xc00007, + shared2_curves: None, + shared3_unk: 0, + shared3_tab: &[], + idle_off_standby_timer_default: 0, + unk_hws2_4: None, + unk_hws2_24: 0, + global_unk_54: 0xffff, + sram_k: f32!(1.02), + unk_coef_a: &[], + unk_coef_b: &[], + global_tab: None, + has_csafr: false, + fast_sensor_mask: [0x12, 0], + fast_sensor_mask_alt: [0x12, 0], + fast_die0_sensor_present: 0x01, + io_mappings: &[ + Some(IOMapping::new(0x204d00000, false, 1, 0x1c000, 0, true)), // Fender + Some(IOMapping::new(0x20e100000, false, 1, 0x4000, 0, false)), // AICTimer + Some(IOMapping::new(0x23b104000, false, 1, 0x4000, 0, true)), // AICSWInt + Some(IOMapping::new(0x204000000, false, 1, 0x20000, 0, true)), // RGX + None, // UVD + None, // unused + None, // DisplayUnderrunWA + Some(IOMapping::new(0x23b2e8000, false, 1, 0x1000, 0, false)), // AnalogTempSensorControllerRegs + Some(IOMapping::new(0x23bc00000, false, 1, 0x1000, 0, true)), // PMPDoorbell + Some(IOMapping::new(0x204d80000, false, 1, 0x5000, 0, true)), // MetrologySensorRegs + Some(IOMapping::new(0x204d61000, false, 1, 0x1000, 0, true)), // GMGIFAFRegs + Some(IOMapping::new(0x200000000, false, 1, 0xd6400, 0, true)), // MCache registers + None, // AICBankedRegisters + Some(IOMapping::new(0x23b738000, false, 1, 0x1000, 0, true)), // PMGRScratch + None, // NIA Special agent idle register die 0 + None, // NIA Special agent idle register die 1 + None, // CRE registers + None, // Streaming codec registers + None, // + None, // + ], + sram_base: None, + sram_size: None, +}; diff --git a/drivers/gpu/drm/asahi/hw/t8112.rs b/drivers/gpu/drm/asahi/hw/t8112.rs new file mode 100644 index 00000000000000..3eba0457d76ac9 --- /dev/null +++ b/drivers/gpu/drm/asahi/hw/t8112.rs @@ -0,0 +1,105 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Hardware configuration for t8112 platforms (M2). + +use crate::f32; + +use super::*; + +pub(crate) const HWCONFIG: super::HwConfig = HwConfig { + chip_id: 0x8112, + gpu_gen: GpuGen::G14, + gpu_variant: GpuVariant::G, + gpu_core: GpuCore::G14G, + + base_clock_hz: 24_000_000, + uat_oas: 40, + num_dies: 1, + max_num_clusters: 1, + max_num_cores: 10, + max_num_frags: 10, + max_num_gps: 4, + + preempt1_size: 0x540, + preempt2_size: 0x280, + preempt3_size: 0x20, + compute_preempt1_size: 0x10000, // TODO: Check + clustering: None, + + render: HwRenderConfig { + // TODO: this is unused here, may be present in newer FW + tiling_control: 0xa041, + }, + + da: HwConfigA { + unk_87c: 900, + unk_8cc: 11000, + unk_e24: 125, + }, + db: HwConfigB { + unk_454: 1, + unk_4e0: 4, + unk_534: 0, + unk_ab8: 0x2048, + unk_abc: 0x4000, + unk_b30: 1, + }, + shared1_tab: &[ + 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, + 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, + ], + shared1_a4: 0, + shared2_tab: &[-1, -1, -1, -1, -1, -1, -1, -1, 0xaa5aa, 0], + shared2_unk_508: 0xc00000, + shared2_curves: Some(HwConfigShared2Curves { + t1_coef: 7200, + t2: &[ + 0xf07, 0x4c0, 0x6c0, 0x8c0, 0xac0, 0xc40, 0xdc0, 0xec0, 0xf80, + ], + t3_coefs: &[0, 20, 28, 36, 44, 50, 56, 60, 63], + t3_scales: &[9, 3209, 10400], + }), + shared3_unk: 5, + shared3_tab: &[ + 10700, 10700, 10700, 10700, 10700, 6000, 1000, 1000, 1000, 10700, 10700, 10700, 10700, + 10700, 10700, 10700, + ], + idle_off_standby_timer_default: 0, + unk_hws2_4: None, + unk_hws2_24: 0, + global_unk_54: 0xffff, + + sram_k: f32!(1.02), + // 13.2: last coef changed from 6.6 to 5.3, assuming that was a fix we can backport + unk_coef_a: &[&f32!([0.0, 0.0, 0.0, 0.0, 5.3, 0.0, 5.3, /*6.6*/ 5.3])], + unk_coef_b: &[&f32!([0.0, 0.0, 0.0, 0.0, 5.3, 0.0, 5.3, /*6.6*/ 5.3])], + global_tab: None, + has_csafr: false, + fast_sensor_mask: [0x6800, 0], + fast_sensor_mask_alt: [0x6800, 0], + fast_die0_sensor_present: 0x02, + io_mappings: &[ + Some(IOMapping::new(0x204d00000, false, 1, 0x14000, 0, true)), // Fender + Some(IOMapping::new(0x20e100000, false, 1, 0x4000, 0, false)), // AICTimer + Some(IOMapping::new(0x23b0c4000, false, 1, 0x4000, 0, true)), // AICSWInt + Some(IOMapping::new(0x204000000, false, 1, 0x20000, 0, true)), // RGX + None, // UVD + None, // unused + None, // DisplayUnderrunWA + Some(IOMapping::new(0x23b2c0000, false, 1, 0x1000, 0, false)), // AnalogTempSensorControllerRegs + None, // PMPDoorbell + Some(IOMapping::new(0x204d80000, false, 1, 0x8000, 0, true)), // MetrologySensorRegs + Some(IOMapping::new(0x204d61000, false, 1, 0x1000, 0, true)), // GMGIFAFRegs + Some(IOMapping::new(0x200000000, false, 1, 0xd6400, 0, true)), // MCache registers + None, // AICBankedRegisters + None, // PMGRScratch + None, // NIA Special agent idle register die 0 + None, // NIA Special agent idle register die 1 + Some(IOMapping::new(0x204e00000, false, 1, 0x10000, 0, true)), // CRE registers + Some(IOMapping::new(0x27d050000, false, 1, 0x4000, 0, true)), // Streaming codec registers + Some(IOMapping::new(0x23b3d0000, false, 1, 0x1000, 0, true)), // + Some(IOMapping::new(0x23b3c0000, false, 1, 0x1000, 0, false)), // + ], + sram_base: None, + sram_size: None, +}; diff --git a/drivers/gpu/drm/asahi/initdata.rs b/drivers/gpu/drm/asahi/initdata.rs new file mode 100644 index 00000000000000..b190a7da6fe85f --- /dev/null +++ b/drivers/gpu/drm/asahi/initdata.rs @@ -0,0 +1,1021 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#![allow(clippy::unusual_byte_groupings)] + +//! GPU initialization data builder. +//! +//! The root of all interaction between the GPU firmware and the host driver is a complex set of +//! nested structures that we call InitData. This includes both GPU hardware/firmware configuration +//! and the pointers to the ring buffers and global data fields that are used for communication at +//! runtime. +//! +//! Many of these structures are poorly understood, so there are lots of hardcoded unknown values +//! derived from observing the InitData structures that macOS generates. + +use crate::f32; +use crate::fw::initdata::*; +use crate::fw::types::*; +use crate::module_parameters; +use crate::{ + driver::AsahiDevice, + gem, + gpu, + hw, + mmu, // +}; +use kernel::error::{ + Error, + Result, // +}; +use kernel::macros::versions; +use kernel::prelude::*; +use kernel::try_init; + +use ::pin_init; +use ::pin_init::Init; + +/// Builder helper for the global GPU InitData. +#[versions(AGX)] +pub(crate) struct InitDataBuilder<'a> { + dev: &'a AsahiDevice, + alloc: &'a mut gpu::KernelAllocators, + cfg: &'static hw::HwConfig, + dyncfg: &'a hw::DynConfig, +} + +#[versions(AGX)] +impl<'a> InitDataBuilder::ver<'a> { + /// Create a new InitData builder + pub(crate) fn new( + dev: &'a AsahiDevice, + alloc: &'a mut gpu::KernelAllocators, + cfg: &'static hw::HwConfig, + dyncfg: &'a hw::DynConfig, + ) -> InitDataBuilder::ver<'a> { + InitDataBuilder::ver { + dev, + alloc, + cfg, + dyncfg, + } + } + + /// Create the HwDataShared1 structure, which is used in two places in InitData. + fn hw_shared1(cfg: &'static hw::HwConfig) -> impl Init { + init!(raw::HwDataShared1 { + unk_a4: cfg.shared1_a4, + ..Zeroable::init_zeroed() + }) + .chain(|ret| { + for (i, val) in cfg.shared1_tab.iter().enumerate() { + ret.table[i] = *val; + } + Ok(()) + }) + } + + fn init_curve( + curve: &mut raw::HwDataShared2Curve, + unk_0: u32, + unk_4: u32, + t1: &[u16], + t2: &[i16], + t3: &[KVec], + ) { + curve.unk_0 = unk_0; + curve.unk_4 = unk_4; + (*curve.t1)[..t1.len()].copy_from_slice(t1); + (*curve.t1)[t1.len()..].fill(t1[0]); + (*curve.t2)[..t2.len()].copy_from_slice(t2); + (*curve.t2)[t2.len()..].fill(t2[0]); + for (i, a) in curve.t3.iter_mut().enumerate() { + a.fill(0x3ffffff); + if i < t3.len() { + let b = &t3[i]; + (**a)[..b.len()].copy_from_slice(b); + } + } + } + + /// Create the HwDataShared2 structure, which is used in two places in InitData. + fn hw_shared2( + cfg: &'static hw::HwConfig, + dyncfg: &'a hw::DynConfig, + ) -> impl Init + 'a { + try_init!(raw::HwDataShared2 { + unk_28: Array::new([0xff; 16]), + g14: Default::default(), + unk_508: cfg.shared2_unk_508, + ..Zeroable::init_zeroed() + }) + .chain(|ret| { + for (i, val) in cfg.shared2_tab.iter().enumerate() { + ret.table[i] = *val; + } + + let curve_cfg = match cfg.shared2_curves.as_ref() { + None => return Ok(()), + Some(a) => a, + }; + + let mut t1 = KVec::new(); + let mut t3 = KVec::new(); + + for _ in 0..curve_cfg.t3_scales.len() { + t3.push(KVec::new(), GFP_KERNEL)?; + } + + for (i, ps) in dyncfg.pwr.perf_states.iter().enumerate() { + let t3_coef = curve_cfg.t3_coefs[i]; + if t3_coef == 0 { + t1.push(0xffff, GFP_KERNEL)?; + for j in t3.iter_mut() { + j.push(0x3ffffff, GFP_KERNEL)?; + } + continue; + } + + let f_khz = (ps.freq_hz / 1000) as u64; + let v_max = ps.max_volt_mv() as u64; + + t1.push( + (1000000000 * (curve_cfg.t1_coef as u64) / (f_khz * v_max)) + .try_into() + .unwrap(), + GFP_KERNEL, + )?; + + for (j, scale) in curve_cfg.t3_scales.iter().enumerate() { + t3[j].push( + (t3_coef as u64 * 1000000100 * *scale as u64 / (f_khz * v_max * 6)) + .try_into() + .unwrap(), + GFP_KERNEL, + )?; + } + } + + ret.g14.unk_14 = 0x6000000; + Self::init_curve( + &mut ret.g14.curve1, + 0, + 0x20000000, + &[0xffff], + &[0x0f07], + &[], + ); + Self::init_curve(&mut ret.g14.curve2, 7, 0x80000000, &t1, curve_cfg.t2, &t3); + + Ok(()) + }) + } + + /// Create the HwDataShared3 structure, which is used in two places in InitData. + fn hw_shared3(cfg: &'static hw::HwConfig) -> impl Init { + pin_init::init_zeroed::().chain(|ret| { + if !cfg.shared3_tab.is_empty() { + ret.unk_0 = 1; + ret.unk_4 = 500; + ret.unk_8 = cfg.shared3_unk; + ret.table.copy_from_slice(cfg.shared3_tab); + ret.unk_4c = 1; + } + Ok(()) + }) + } + + /// Create an unknown T81xx-specific data structure. + fn t81xx_data( + cfg: &'static hw::HwConfig, + dyncfg: &'a hw::DynConfig, + ) -> impl Init { + let _perf_max_pstate = dyncfg.pwr.perf_max_pstate; + + pin_init::init_zeroed::().chain(move |_ret| { + match cfg.chip_id { + 0x8103 | 0x8112 => { + #[ver(V < V13_3)] + { + _ret.unk_d8c = 0x80000000; + _ret.unk_d90 = 4; + _ret.unk_d9c = f32!(0.6); + _ret.unk_da4 = f32!(0.4); + _ret.unk_dac = f32!(0.38552); + _ret.unk_db8 = f32!(65536.0); + _ret.unk_dbc = f32!(13.56); + _ret.max_pstate_scaled = 100 * _perf_max_pstate; + } + } + _ => (), + } + Ok(()) + }) + } + + /// Create the HwDataA structure. This mostly contains power-related configuration. + fn hwdata_a(&mut self) -> Result> { + let pwr = &self.dyncfg.pwr; + let period_ms = pwr.power_sample_period; + let period_s = F32::from(period_ms) / f32!(1000.0); + let ppm_filter_tc_periods = pwr.ppm_filter_time_constant_ms / period_ms; + #[ver(V >= V13_0B4)] + let ppm_filter_tc_ms_rounded = ppm_filter_tc_periods * period_ms; + let ppm_filter_a = f32!(1.0) / ppm_filter_tc_periods.into(); + let perf_filter_a = f32!(1.0) / pwr.perf_filter_time_constant.into(); + let perf_filter_a2 = f32!(1.0) / pwr.perf_filter_time_constant2.into(); + let avg_power_target_filter_a = f32!(1.0) / pwr.avg_power_target_filter_tc.into(); + let avg_power_filter_tc_periods = pwr.avg_power_filter_tc_ms / period_ms; + #[ver(V >= V13_0B4)] + let avg_power_filter_tc_ms_rounded = avg_power_filter_tc_periods * period_ms; + let avg_power_filter_a = f32!(1.0) / avg_power_filter_tc_periods.into(); + let pwr_filter_a = f32!(1.0) / pwr.pwr_filter_time_constant.into(); + + let base_ps = pwr.perf_base_pstate; + let base_ps_scaled = 100 * base_ps; + let max_ps = pwr.perf_max_pstate; + let max_ps_scaled = 100 * max_ps; + let boost_ps_count = max_ps - base_ps; + + #[allow(unused_variables)] + let base_clock_khz = self.cfg.base_clock_hz / 1000; + let v_clocks_per_period = pwr.pwr_sample_period_aic_clks; + + #[allow(unused_variables)] + let clocks_per_period_coarse = self.cfg.base_clock_hz / 1000 * pwr.power_sample_period; + + self.alloc + .private + .new_init(pin_init::init_zeroed(), |_inner, _ptr| { + let cfg = &self.cfg; + let dyncfg = &self.dyncfg; + try_init!(raw::HwDataA::ver { + clocks_per_period: v_clocks_per_period, + #[ver(V >= V13_0B4)] + clocks_per_period_2: v_clocks_per_period, + pwr_status: AtomicU32::new(4), + unk_10: f32!(1.0), + actual_pstate: 1, + tgt_pstate: 1, + base_pstate_scaled: base_ps_scaled, + unk_40: 1, + max_pstate_scaled: max_ps_scaled, + min_pstate_scaled: 100, + unk_64c: 625, + pwr_filter_a_neg: f32!(1.0) - pwr_filter_a, + pwr_filter_a: pwr_filter_a, + pwr_integral_gain: pwr.pwr_integral_gain, + pwr_integral_min_clamp: pwr.pwr_integral_min_clamp.into(), + max_power_1: pwr.max_power_mw.into(), + pwr_proportional_gain: pwr.pwr_proportional_gain, + pwr_pstate_related_k: -F32::from(max_ps_scaled) / pwr.max_power_mw.into(), + pwr_pstate_max_dc_offset: pwr.pwr_min_duty_cycle as i32 - max_ps_scaled as i32, + max_pstate_scaled_2: max_ps_scaled, + max_power_2: pwr.max_power_mw, + max_pstate_scaled_3: max_ps_scaled, + ppm_filter_tc_periods_x4: ppm_filter_tc_periods * 4, + ppm_filter_a_neg: f32!(1.0) - ppm_filter_a, + ppm_filter_a: ppm_filter_a, + ppm_ki_dt: pwr.ppm_ki * period_s, + unk_6fc: f32!(65536.0), + ppm_kp: pwr.ppm_kp, + pwr_min_duty_cycle: pwr.pwr_min_duty_cycle, + max_pstate_scaled_4: max_ps_scaled, + unk_71c: f32!(0.0), + max_power_3: pwr.max_power_mw, + cur_power_mw_2: 0x0, + ppm_filter_tc_ms: pwr.ppm_filter_time_constant_ms, + #[ver(V >= V13_0B4)] + ppm_filter_tc_clks: ppm_filter_tc_ms_rounded * base_clock_khz, + perf_tgt_utilization: pwr.perf_tgt_utilization, + perf_boost_min_util: pwr.perf_boost_min_util, + perf_boost_ce_step: pwr.perf_boost_ce_step, + perf_reset_iters: pwr.perf_reset_iters, + unk_774: 6, + unk_778: 1, + perf_filter_drop_threshold: pwr.perf_filter_drop_threshold, + perf_filter_a_neg: f32!(1.0) - perf_filter_a, + perf_filter_a2_neg: f32!(1.0) - perf_filter_a2, + perf_filter_a: perf_filter_a, + perf_filter_a2: perf_filter_a2, + perf_ki: pwr.perf_integral_gain, + perf_ki2: pwr.perf_integral_gain2, + perf_integral_min_clamp: pwr.perf_integral_min_clamp.into(), + unk_79c: f32!(95.0), + perf_kp: pwr.perf_proportional_gain, + perf_kp2: pwr.perf_proportional_gain2, + boost_state_unk_k: F32::from(boost_ps_count) / f32!(0.95), + base_pstate_scaled_2: base_ps_scaled, + max_pstate_scaled_5: max_ps_scaled, + base_pstate_scaled_3: base_ps_scaled, + perf_tgt_utilization_2: pwr.perf_tgt_utilization, + base_pstate_scaled_4: base_ps_scaled, + unk_7fc: f32!(65536.0), + pwr_min_duty_cycle_2: pwr.pwr_min_duty_cycle.into(), + max_pstate_scaled_6: max_ps_scaled.into(), + max_freq_mhz: pwr.max_freq_mhz, + pwr_min_duty_cycle_3: pwr.pwr_min_duty_cycle, + min_pstate_scaled_4: f32!(100.0), + max_pstate_scaled_7: max_ps_scaled, + unk_alpha_neg: f32!(0.8), + unk_alpha: f32!(0.2), + fast_die0_sensor_mask: U64(cfg.fast_sensor_mask[0]), + #[ver(G >= G14X)] + fast_die1_sensor_mask: U64(cfg.fast_sensor_mask[1]), + fast_die0_release_temp_cc: 100 * pwr.fast_die0_release_temp, + unk_87c: cfg.da.unk_87c, + unk_880: 0x4, + unk_894: f32!(1.0), + + fast_die0_ki_dt: pwr.fast_die0_integral_gain * period_s, + unk_8a8: f32!(65536.0), + fast_die0_kp: pwr.fast_die0_proportional_gain, + pwr_min_duty_cycle_4: pwr.pwr_min_duty_cycle, + max_pstate_scaled_8: max_ps_scaled, + max_pstate_scaled_9: max_ps_scaled, + fast_die0_prop_tgt_delta: 100 * pwr.fast_die0_prop_tgt_delta, + unk_8cc: cfg.da.unk_8cc, + max_pstate_scaled_10: max_ps_scaled, + max_pstate_scaled_11: max_ps_scaled, + unk_c2c: 1, + power_zone_count: pwr.power_zones.len() as u32, + max_power_4: pwr.max_power_mw, + max_power_5: pwr.max_power_mw, + max_power_6: pwr.max_power_mw, + avg_power_target_filter_a_neg: f32!(1.0) - avg_power_target_filter_a, + avg_power_target_filter_a: avg_power_target_filter_a, + avg_power_target_filter_tc_x4: 4 * pwr.avg_power_target_filter_tc, + avg_power_target_filter_tc_xperiod: period_ms * pwr.avg_power_target_filter_tc, + #[ver(V >= V13_0B4)] + avg_power_target_filter_tc_clks: period_ms + * pwr.avg_power_target_filter_tc + * base_clock_khz, + avg_power_filter_tc_periods_x4: 4 * avg_power_filter_tc_periods, + avg_power_filter_a_neg: f32!(1.0) - avg_power_filter_a, + avg_power_filter_a: avg_power_filter_a, + avg_power_ki_dt: pwr.avg_power_ki_only * period_s, + unk_d20: f32!(65536.0), + avg_power_kp: pwr.avg_power_kp, + avg_power_min_duty_cycle: pwr.avg_power_min_duty_cycle, + max_pstate_scaled_12: max_ps_scaled, + max_pstate_scaled_13: max_ps_scaled, + max_power_7: pwr.max_power_mw.into(), + max_power_8: pwr.max_power_mw, + avg_power_filter_tc_ms: pwr.avg_power_filter_tc_ms, + #[ver(V >= V13_0B4)] + avg_power_filter_tc_clks: avg_power_filter_tc_ms_rounded * base_clock_khz, + max_pstate_scaled_14: max_ps_scaled, + t81xx_data <- Self::t81xx_data(cfg, dyncfg), + #[ver(V >= V13_0B4)] + unk_e10_0 <- { + let filter_a = f32!(1.0) / pwr.se_filter_time_constant.into(); + let filter_1_a = f32!(1.0) / pwr.se_filter_time_constant_1.into(); + try_init!(raw::HwDataA130Extra { + unk_38: 4, + unk_3c: 8000, + gpu_se_inactive_threshold: pwr.se_inactive_threshold, + gpu_se_engagement_criteria: pwr.se_engagement_criteria, + gpu_se_reset_criteria: pwr.se_reset_criteria, + unk_54: 50, + unk_58: 0x1, + gpu_se_filter_a_neg: f32!(1.0) - filter_a, + gpu_se_filter_1_a_neg: f32!(1.0) - filter_1_a, + gpu_se_filter_a: filter_a, + gpu_se_filter_1_a: filter_1_a, + gpu_se_ki_dt: pwr.se_ki * period_s, + gpu_se_ki_1_dt: pwr.se_ki_1 * period_s, + unk_7c: f32!(65536.0), + gpu_se_kp: pwr.se_kp, + gpu_se_kp_1: pwr.se_kp_1, + + #[ver(V >= V13_3)] + unk_8c: 100, + #[ver(V < V13_3)] + unk_8c: 40, + + max_pstate_scaled_1: max_ps_scaled, + unk_9c: f32!(8000.0), + unk_a0: 1400, + gpu_se_filter_time_constant_ms: pwr.se_filter_time_constant * period_ms, + gpu_se_filter_time_constant_1_ms: pwr.se_filter_time_constant_1 + * period_ms, + gpu_se_filter_time_constant_clks: U64((pwr.se_filter_time_constant + * clocks_per_period_coarse) + .into()), + gpu_se_filter_time_constant_1_clks: U64((pwr + .se_filter_time_constant_1 + * clocks_per_period_coarse) + .into()), + unk_c4: f32!(65536.0), + unk_114: f32!(65536.0), + unk_124: 40, + max_pstate_scaled_2: max_ps_scaled, + ..Zeroable::init_zeroed() + }) + }, + fast_die0_sensor_mask_2: U64(cfg.fast_sensor_mask[0]), + #[ver(G >= G14X)] + fast_die1_sensor_mask_2: U64(cfg.fast_sensor_mask[1]), + unk_e24: cfg.da.unk_e24, + unk_e28: 1, + fast_die0_sensor_mask_alt: U64(cfg.fast_sensor_mask_alt[0]), + #[ver(G >= G14X)] + fast_die1_sensor_mask_alt: U64(cfg.fast_sensor_mask_alt[1]), + #[ver(V < V13_0B4)] + fast_die0_sensor_present: U64(cfg.fast_die0_sensor_present as u64), + unk_163c: 1, + unk_3644: 0, + hws1 <- Self::hw_shared1(cfg), + hws2 <- Self::hw_shared2(cfg, dyncfg), + hws3 <- Self::hw_shared3(cfg), + unk_3ce8: 1, + ..Zeroable::init_zeroed() + }) + .chain(|raw| { + for i in 0..self.dyncfg.pwr.perf_states.len() { + raw.sram_k[i] = self.cfg.sram_k; + } + + for (i, coef) in pwr.core_leak_coef.iter().enumerate() { + raw.core_leak_coef[i] = *coef; + } + + for (i, coef) in pwr.sram_leak_coef.iter().enumerate() { + raw.sram_leak_coef[i] = *coef; + } + + #[ver(V >= V13_0B4)] + if let Some(csafr) = pwr.csafr.as_ref() { + for (i, coef) in csafr.leak_coef_afr.iter().enumerate() { + raw.aux_leak_coef.cs_1[i] = *coef; + raw.aux_leak_coef.cs_2[i] = *coef; + } + + for (i, coef) in csafr.leak_coef_cs.iter().enumerate() { + raw.aux_leak_coef.afr_1[i] = *coef; + raw.aux_leak_coef.afr_2[i] = *coef; + } + } + + for i in 0..self.dyncfg.id.num_clusters as usize { + if let Some(coef_a) = self.cfg.unk_coef_a.get(i) { + (*raw.unk_coef_a1[i])[..coef_a.len()].copy_from_slice(coef_a); + (*raw.unk_coef_a2[i])[..coef_a.len()].copy_from_slice(coef_a); + } + if let Some(coef_b) = self.cfg.unk_coef_b.get(i) { + (*raw.unk_coef_b1[i])[..coef_b.len()].copy_from_slice(coef_b); + (*raw.unk_coef_b2[i])[..coef_b.len()].copy_from_slice(coef_b); + } + } + + for (i, pz) in pwr.power_zones.iter().enumerate() { + raw.power_zones[i].target = pz.target; + raw.power_zones[i].target_off = pz.target - pz.target_offset; + raw.power_zones[i].filter_tc_x4 = 4 * pz.filter_tc; + raw.power_zones[i].filter_tc_xperiod = period_ms * pz.filter_tc; + let filter_a = f32!(1.0) / pz.filter_tc.into(); + raw.power_zones[i].filter_a = filter_a; + raw.power_zones[i].filter_a_neg = f32!(1.0) - filter_a; + #[ver(V >= V13_0B4)] + raw.power_zones[i].unk_10 = 1320000000; + } + + #[ver(V >= V13_0B4 && G >= G14X)] + for (i, j) in raw.hws2.g14.curve2.t1.iter().enumerate() { + raw.unk_hws2[i] = if *j == 0xffff { 0 } else { j / 2 }; + } + + if !dyncfg.hw_data_b.is_empty() { + unsafe { + let mut matches: bool = true; + let sla = core::slice::from_raw_parts( + raw as *const raw::HwDataA::ver as *const u8, + core::mem::size_of::(), + ); + if sla.len() != dyncfg.hw_data_a.len() { + matches = false; + dev_err!( + self.dev.as_ref(), + "!!! Hwdata A size mismatch: {} {}", + sla.len(), + dyncfg.hw_data_a.len(), + ); + } + for i in 0..core::cmp::min(sla.len(), dyncfg.hw_data_a.len()) { + if sla[i] != dyncfg.hw_data_a[i] { + matches = false; + dev_err!(self.dev.as_ref(), "!!! Hwdata A first mismatch: {i}"); + break; + } + } + if matches { + dev_info!(self.dev.as_ref(), "!!! Hwdata A match"); + } + } + } + + Ok(()) + }) + }) + } + + /// Create the HwDataB structure. This mostly contains GPU-related configuration. + fn hwdata_b(&mut self) -> Result> { + self.alloc + .private + .new_init(pin_init::init_zeroed(), |_inner, _ptr| { + let cfg = &self.cfg; + let dyncfg = &self.dyncfg; + try_init!(raw::HwDataB::ver { + // Userspace VA map related + #[ver(V < V13_0B4)] + unk_0: U64(0x13_00000000), + unk_8: U64(0x14_00000000), + #[ver(V < V13_0B4)] + unk_10: U64(0x1_00000000), + unk_18: U64(0xffc00000), + // USC start + unk_20: U64(0), // U64(0x11_00000000), + unk_28: U64(0), // U64(0x11_00000000), + // Unknown page + //unk_30: U64(0x6f_ffff8000), + unk_30: U64(mmu::IOVA_UNK_PAGE), + timestamp_area_base: U64(gpu::IOVA_KERN_TIMESTAMP_RANGE.start), + // TODO: yuv matrices + chip_id: cfg.chip_id, + unk_454: cfg.db.unk_454, + unk_458: 0x1, + unk_460: 0x1, + unk_464: 0x1, + unk_468: 0x1, + unk_47c: 0x1, + unk_484: 0x1, + unk_48c: 0x1, + base_clock_khz: cfg.base_clock_hz / 1000, + power_sample_period: dyncfg.pwr.power_sample_period, + unk_49c: 0x1, + unk_4a0: 0x1, + unk_4a4: 0x1, + unk_4c0: 0x1f, + unk_4e0: U64(cfg.db.unk_4e0), + unk_4f0: 0x1, + unk_4f4: 0x1, + unk_504: 0x31, + unk_524: 0x1, // use_secure_cache_flush + unk_534: cfg.db.unk_534, + num_frags: dyncfg.id.num_frags * dyncfg.id.num_clusters, + unk_554: 0x1, + uat_ttb_base: U64(dyncfg.uat_ttb_base), + gpu_core_id: cfg.gpu_core as u32, + gpu_rev_id: dyncfg.id.gpu_rev_id as u32, + num_cores: dyncfg.id.num_cores * dyncfg.id.num_clusters, + max_pstate: dyncfg.pwr.perf_states.len() as u32 - 1, + #[ver(V < V13_0B4)] + num_pstates: dyncfg.pwr.perf_states.len() as u32, + #[ver(V < V13_0B4)] + min_sram_volt: dyncfg.pwr.min_sram_microvolt / 1000, + #[ver(V < V13_0B4)] + unk_ab8: cfg.db.unk_ab8, + #[ver(V < V13_0B4)] + unk_abc: cfg.db.unk_abc, + #[ver(V < V13_0B4)] + unk_ac0: 0x1020, + + #[ver(V >= V13_0B4)] + unk_ae4: Array::new([0x0, 0x3, 0x7, 0x7]), + #[ver(V < V13_0B4)] + unk_ae4: Array::new([0x0, 0xf, 0x3f, 0x3f]), + unk_b10: 0x1, + timer_offset: U64(0), + unk_b24: 0x1, + unk_b28: 0x1, + unk_b2c: 0x1, + unk_b30: cfg.db.unk_b30, + #[ver(V >= V13_0B4)] + unk_b38_0: 1, + #[ver(V >= V13_0B4)] + unk_b38_4: 1, + unk_b38: Array::new([0xffffffff; 12]), + #[ver(V >= V13_0B4 && V < V13_3)] + unk_c3c: 0x19, + #[ver(V >= V13_3)] + unk_c3c: 0x1a, + ..Zeroable::init_zeroed() + }) + .chain(|raw| { + #[ver(V >= V13_3)] + for i in 0..16 { + raw.unk_arr_0[i] = i as u32; + } + + let base_ps = self.dyncfg.pwr.perf_base_pstate as usize; + let max_ps = self.dyncfg.pwr.perf_max_pstate as usize; + let base_freq = self.dyncfg.pwr.perf_states[base_ps].freq_hz; + let max_freq = self.dyncfg.pwr.perf_states[max_ps].freq_hz; + + for (i, ps) in self.dyncfg.pwr.perf_states.iter().enumerate() { + raw.frequencies[i] = ps.freq_hz / 1000000; + for (j, mv) in ps.volt_mv.iter().enumerate() { + let sram_mv = (*mv).max(self.dyncfg.pwr.min_sram_microvolt / 1000); + raw.voltages[i][j] = *mv; + raw.voltages_sram[i][j] = sram_mv; + } + for j in ps.volt_mv.len()..raw.voltages[i].len() { + raw.voltages[i][j] = raw.voltages[i][0]; + raw.voltages_sram[i][j] = raw.voltages_sram[i][0]; + } + raw.sram_k[i] = self.cfg.sram_k; + raw.rel_max_powers[i] = ps.pwr_mw * 100 / self.dyncfg.pwr.max_power_mw; + raw.rel_boost_freqs[i] = if i > base_ps { + (ps.freq_hz - base_freq) / ((max_freq - base_freq) / 100) + } else { + 0 + }; + } + + #[ver(V >= V13_0B4)] + if let Some(csafr) = self.dyncfg.pwr.csafr.as_ref() { + let aux = &mut raw.aux_ps; + aux.cs_max_pstate = (csafr.perf_states_cs.len() - 1).try_into()?; + aux.afr_max_pstate = (csafr.perf_states_afr.len() - 1).try_into()?; + + for (i, ps) in csafr.perf_states_cs.iter().enumerate() { + aux.cs_frequencies[i] = ps.freq_hz / 1000000; + for (j, mv) in ps.volt_mv.iter().enumerate() { + let sram_mv = (*mv).max(csafr.min_sram_microvolt / 1000); + aux.cs_voltages[i][j] = *mv; + aux.cs_voltages_sram[i][j] = sram_mv; + } + } + + for (i, ps) in csafr.perf_states_afr.iter().enumerate() { + aux.afr_frequencies[i] = ps.freq_hz / 1000000; + for (j, mv) in ps.volt_mv.iter().enumerate() { + let sram_mv = (*mv).max(csafr.min_sram_microvolt / 1000); + aux.afr_voltages[i][j] = *mv; + aux.afr_voltages_sram[i][j] = sram_mv; + } + } + } + + // Special case override for T602x + #[ver(G == G14X)] + if dyncfg.id.gpu_rev_id == hw::GpuRevisionID::B1 { + raw.gpu_rev_id = hw::GpuRevisionID::B0 as u32; + } + + if !dyncfg.hw_data_b.is_empty() { + unsafe { + let mut matches: bool = true; + let sla = core::slice::from_raw_parts( + raw as *const raw::HwDataB::ver as *const u8, + core::mem::size_of::(), + ); + if sla.len() != dyncfg.hw_data_b.len() { + matches = false; + dev_err!( + self.dev.as_ref(), + "!!! Hwdata B size mismatch: {} {}", + sla.len(), + dyncfg.hw_data_b.len(), + ); + } + for i in 0..core::cmp::min(sla.len(), dyncfg.hw_data_b.len()) { + if sla[i] != dyncfg.hw_data_b[i] { + matches = false; + dev_err!(self.dev.as_ref(), "!!! Hwdata B first mismatch: {i}"); + break; + } + } + if matches { + dev_info!(self.dev.as_ref(), "!!! Hwdata B match"); + } + } + } + + Ok(()) + }) + }) + } + + /// Create the Globals structure, which contains global firmware config including more power + /// configuration data and globals used to exchange state between the firmware and driver. + fn globals(&mut self) -> Result> { + self.alloc + .private + .new_init(pin_init::init_zeroed(), |_inner, _ptr| { + let cfg = &self.cfg; + let dyncfg = &self.dyncfg; + let pwr = &dyncfg.pwr; + let period_ms = pwr.power_sample_period; + let period_s = F32::from(period_ms) / f32!(1000.0); + let avg_power_filter_tc_periods = pwr.avg_power_filter_tc_ms / period_ms; + + let max_ps = pwr.perf_max_pstate; + let max_ps_scaled = 100 * max_ps; + + try_init!(raw::Globals::ver { + //ktrace_enable: 0xffffffff, + ktrace_enable: 0, + #[ver(V >= V13_2)] + unk_24_0: 3000, + unk_24: 0, + #[ver(V >= V13_0B4)] + debug: 0, + unk_28: 1, + #[ver(G >= G14X)] + unk_2c_0: 1, + #[ver(V >= V13_0B4 && G < G14X)] + unk_2c_0: 0, + unk_2c: 1, + unk_30: 0, + unk_34: 120, + // sub <- try_init!(raw::GlobalsSub::ver { + unk_54: cfg.global_unk_54, + unk_56: 40, + unk_58: 0xffff, + unk_5e: U32(1), + unk_66: U32(1), + // ..Zeroable::init_zeroed() + // }), + unk_8900: 1, + pending_submissions: AtomicU32::new(0), + max_power: pwr.max_power_mw, + max_pstate_scaled: max_ps_scaled, + max_pstate_scaled_2: max_ps_scaled, + max_pstate_scaled_3: max_ps_scaled, + power_zone_count: pwr.power_zones.len() as u32, + avg_power_filter_tc_periods: avg_power_filter_tc_periods, + avg_power_ki_dt: pwr.avg_power_ki_only * period_s, + avg_power_kp: pwr.avg_power_kp, + avg_power_min_duty_cycle: pwr.avg_power_min_duty_cycle, + avg_power_target_filter_tc: pwr.avg_power_target_filter_tc, + unk_89bc: cfg.da.unk_8cc, + fast_die0_release_temp: 100 * pwr.fast_die0_release_temp, + unk_89c4: cfg.da.unk_87c, + fast_die0_prop_tgt_delta: 100 * pwr.fast_die0_prop_tgt_delta, + fast_die0_kp: pwr.fast_die0_proportional_gain, + fast_die0_ki_dt: pwr.fast_die0_integral_gain * period_s, + unk_89e0: 1, + max_power_2: pwr.max_power_mw, + ppm_kp: pwr.ppm_kp, + ppm_ki_dt: pwr.ppm_ki * period_s, + #[ver(V >= V13_0B4)] + unk_89f4_8: 1, + unk_89f4: 0, + hws1 <- Self::hw_shared1(cfg), + hws2 <- Self::hw_shared2(cfg, dyncfg), + hws3 <- Self::hw_shared3(cfg), + #[ver(V >= V13_0B4)] + idle_off_standby_timer: pwr.idle_off_standby_timer, + #[ver(V >= V13_0B4)] + unk_hws2_4: cfg.unk_hws2_4.map(Array::new).unwrap_or_default(), + #[ver(V >= V13_0B4)] + unk_hws2_24: cfg.unk_hws2_24, + unk_900c: 1, + #[ver(V >= V13_0B4)] + unk_9010_0: 1, + #[ver(V >= V13_0B4)] + unk_903c: 1, + #[ver(V < V13_0B4)] + unk_903c: 0, + fault_control: *module_parameters::fault_control.value(), + do_init: 1, + progress_check_interval_3d: 40, + progress_check_interval_ta: 10, + progress_check_interval_cl: 250, + #[ver(V >= V13_0B4)] + unk_1102c_0: 1, + #[ver(V >= V13_0B4)] + unk_1102c_4: 1, + #[ver(V >= V13_0B4)] + unk_1102c_8: 100, + #[ver(V >= V13_0B4)] + unk_1102c_c: 1, + idle_off_delay_ms: AtomicU32::new(pwr.idle_off_delay_ms), + fender_idle_off_delay_ms: pwr.fender_idle_off_delay_ms, + fw_early_wake_timeout_ms: pwr.fw_early_wake_timeout_ms, + cl_context_switch_timeout_ms: 40, + #[ver(V >= V13_0B4)] + cl_kill_timeout_ms: 50, + #[ver(V >= V13_0B4)] + unk_11edc: 0, + #[ver(V >= V13_0B4)] + unk_11efc: 0, + ..Zeroable::init_zeroed() + }) + .chain(|raw| { + for (i, pz) in self.dyncfg.pwr.power_zones.iter().enumerate() { + raw.power_zones[i].target = pz.target; + raw.power_zones[i].target_off = pz.target - pz.target_offset; + raw.power_zones[i].filter_tc = pz.filter_tc; + } + + if let Some(tab) = self.cfg.global_tab.as_ref() { + for (i, x) in tab.iter().enumerate() { + raw.unk_118ec[i] = *x; + } + raw.unk_118e8 = 1; + } + + if !dyncfg.hw_globals.is_empty() { + unsafe { + let mut matches: bool = true; + let sla = core::slice::from_raw_parts( + raw as *const raw::Globals::ver as *const u8, + core::mem::size_of::(), + ); + if sla.len() != dyncfg.hw_globals.len() { + matches = false; + dev_err!( + self.dev.as_ref(), + "!!! Globals size mismatch: {} {}", + sla.len(), + dyncfg.hw_globals.len(), + ); + } + for i in 0..core::cmp::min(sla.len(), dyncfg.hw_globals.len()) { + if sla[i] != dyncfg.hw_globals[i] { + matches = false; + dev_err!(self.dev.as_ref(), "!!! Globals first mismatch: {i}"); + break; + } + } + if matches { + dev_info!(self.dev.as_ref(), "!!! Globals match"); + } + } + } + + Ok(()) + }) + }) + } + + /// Create the RuntimePointers structure, which contains pointers to most of the other + /// structures including the ring buffer channels, statistics structures, and HwDataA/HwDataB. + fn runtime_pointers(&mut self) -> Result> { + let hwa = self.hwdata_a()?; + let hwb = self.hwdata_b()?; + + let mut buffer_mgr_ctl = gem::new_kernel_object(self.dev, 0x4000)?; + buffer_mgr_ctl.vmap()?.memset(0); + + GpuObject::new_init_prealloc( + self.alloc.private.alloc_object()?, + |_ptr| { + let alloc = &mut *self.alloc; + try_init!(RuntimePointers::ver { + stats <- { + let alloc = &mut *alloc; + try_init!(Stats::ver { + vtx: alloc.private.new_default::()?, + frag: alloc.private.new_init( + pin_init::init_zeroed::(), + |_inner, _ptr| { + try_init!(raw::GpuGlobalStatsFrag::ver { + total_cmds: 0, + unk_4: 0, + stats: Default::default(), + }) + } + )?, + comp: alloc.private.new_default::()?, + }) + }, + + hwdata_a: hwa, + unkptr_190: alloc.private.array_empty_tagged(0x80, b"I190")?, + unkptr_198: alloc.private.array_empty_tagged(0xc0, b"I198")?, + hwdata_b: hwb, + + unkptr_1b8: alloc.private.array_empty_tagged(0x1000, b"I1B8")?, + unkptr_1c0: alloc.private.array_empty_tagged(0x300, b"I1C0")?, + unkptr_1c8: alloc.private.array_empty_tagged(0x1000, b"I1C8")?, + + buffer_mgr_ctl, + buffer_mgr_ctl_low_mapping: None, + buffer_mgr_ctl_high_mapping: None, + }) + }, + |inner, _ptr| { + try_init!(raw::RuntimePointers::ver { + pipes: Default::default(), + device_control: Default::default(), + event: Default::default(), + fw_log: Default::default(), + ktrace: Default::default(), + stats: Default::default(), + + stats_vtx: inner.stats.vtx.gpu_pointer(), + stats_frag: inner.stats.frag.gpu_pointer(), + stats_comp: inner.stats.comp.gpu_pointer(), + + hwdata_a: inner.hwdata_a.gpu_pointer(), + unkptr_190: inner.unkptr_190.gpu_pointer(), + unkptr_198: inner.unkptr_198.gpu_pointer(), + hwdata_b: inner.hwdata_b.gpu_pointer(), + hwdata_b_2: inner.hwdata_b.gpu_pointer(), + + fwlog_buf: None, + + unkptr_1b8: inner.unkptr_1b8.gpu_pointer(), + + #[ver(G < G14X)] + unkptr_1c0: inner.unkptr_1c0.gpu_pointer(), + #[ver(G < G14X)] + unkptr_1c8: inner.unkptr_1c8.gpu_pointer(), + + buffer_mgr_ctl_gpu_addr: U64(gpu::IOVA_KERN_GPU_BUFMGR_LOW), + buffer_mgr_ctl_fw_addr: U64(gpu::IOVA_KERN_GPU_BUFMGR_HIGH), + + __pad0: Default::default(), + unk_160: U64(0), + unk_168: U64(0), + unk_1d0: 0, + unk_1d4: 0, + unk_1d8: Default::default(), + + __pad1: Default::default(), + gpu_scratch: raw::RuntimeScratch::ver { + unk_6b38: 0xff, + ..Default::default() + }, + }) + }, + ) + } + + /// Create the FwStatus structure, which is used to coordinate the firmware halt state between + /// the firmware and the driver. + fn fw_status(&mut self) -> Result> { + self.alloc + .shared + .new_object(Default::default(), |_inner| Default::default()) + } + + /// Create one UatLevelInfo structure, which describes one level of translation for the UAT MMU. + fn uat_level_info( + cfg: &'static hw::HwConfig, + index_shift: usize, + num_entries: usize, + ) -> raw::UatLevelInfo { + raw::UatLevelInfo { + index_shift: index_shift as _, + unk_1: 14, + unk_2: 14, + unk_3: 8, + unk_4: 0x4000, + num_entries: num_entries as _, + unk_8: U64(1), + unk_10: U64(((1u64 << cfg.uat_oas) - 1) & !(mmu::UAT_PGMSK as u64)), + index_mask: U64(((num_entries - 1) << index_shift) as u64), + } + } + + /// Build the top-level InitData object. + #[inline(never)] + pub(crate) fn build(&mut self) -> Result>> { + let runtime_pointers = self.runtime_pointers()?; + let globals = self.globals()?; + let fw_status = self.fw_status()?; + let shared_ro = &mut self.alloc.shared_ro; + + let obj = self.alloc.private.new_init( + try_init!(InitData::ver { + unk_buf: shared_ro.array_empty_tagged(0x4000, b"IDTA")?, + runtime_pointers, + globals, + fw_status, + }), + |inner, _ptr| { + let cfg = &self.cfg; + try_init!(raw::InitData::ver { + #[ver(V == V13_5 && G != G14X)] + ver_info: Array::new([0x6ba0, 0x1f28, 0x601, 0xb0]), + #[ver(V == V13_5 && G == G14X)] + ver_info: Array::new([0xb390, 0x70f8, 0x601, 0xb0]), + unk_buf: inner.unk_buf.gpu_pointer(), + unk_8: 0, + unk_c: 0, + runtime_pointers: inner.runtime_pointers.gpu_pointer(), + globals: inner.globals.gpu_pointer(), + fw_status: inner.fw_status.gpu_pointer(), + uat_page_size: 0x4000, + uat_page_bits: 14, + uat_num_levels: 3, + uat_level_info: Array::new([ + Self::uat_level_info(cfg, 36, 8), + Self::uat_level_info(cfg, 25, 2048), + Self::uat_level_info(cfg, 14, 2048), + ]), + __pad0: Default::default(), + host_mapped_fw_allocations: 1, + unk_ac: 0, + unk_b0: 0, + unk_b4: 0, + unk_b8: 0, + }) + }, + )?; + Ok(KBox::new(obj, GFP_KERNEL)?) + } +} diff --git a/drivers/gpu/drm/asahi/mem.rs b/drivers/gpu/drm/asahi/mem.rs new file mode 100644 index 00000000000000..60a64e23a161c5 --- /dev/null +++ b/drivers/gpu/drm/asahi/mem.rs @@ -0,0 +1,144 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! ARM64 low level memory operations. +//! +//! This GPU uses CPU-side `tlbi` outer-shareable instructions to manage its TLBs. +//! Yes, really. Even though the VA address spaces are unrelated. +//! +//! Right now we pick our own ASIDs and don't coordinate with the CPU. This might result +//! in needless TLB shootdowns on the CPU side... TODO: fix this. + +use core::arch::asm; +use core::cmp::min; + +use crate::debug::*; +use crate::mmu; + +type Asid = u8; + +/// Invalidate the entire GPU TLB. +#[inline(always)] +pub(crate) fn tlbi_all() { + // SAFETY: tlbi is always safe by definition + unsafe { + asm!(".arch armv8.4-a", "tlbi vmalle1os",); + } +} + +/// Invalidate all TLB entries for a given ASID. +#[inline(always)] +pub(crate) fn tlbi_asid(asid: Asid) { + if debug_enabled(DebugFlags::ConservativeTlbi) { + tlbi_all(); + sync(); + return; + } + + // SAFETY: tlbi is always safe by definition + unsafe { + asm!( + ".arch armv8.4-a", + "tlbi aside1os, {x}", + x = in(reg) ((asid as u64) << 48) + ); + } +} + +/// Invalidate a single page for a given ASID. +#[inline(always)] +pub(crate) fn tlbi_page(asid: Asid, va: usize) { + if debug_enabled(DebugFlags::ConservativeTlbi) { + tlbi_all(); + sync(); + return; + } + + let val: u64 = ((asid as u64) << 48) | ((va as u64 >> 12) & 0xffffffffffc); + // SAFETY: tlbi is always safe by definition + unsafe { + asm!( + ".arch armv8.4-a", + "tlbi vae1os, {x}", + x = in(reg) val + ); + } +} + +/// Invalidate a range of pages for a given ASID. +#[inline(always)] +pub(crate) fn tlbi_range(asid: Asid, va: usize, len: usize) { + if debug_enabled(DebugFlags::ConservativeTlbi) { + tlbi_all(); + sync(); + return; + } + + if len == 0 { + return; + } + + let start_pg = va >> mmu::UAT_PGBIT; + let end_pg = (va + len + mmu::UAT_PGMSK) >> mmu::UAT_PGBIT; + + let mut val: u64 = ((asid as u64) << 48) | (2 << 46) | (start_pg as u64 & 0x1fffffffff); + let pages = end_pg - start_pg; + + // Guess? It's possible that the page count is in terms of 4K pages + // when the CPU is in 4K mode... + #[cfg(CONFIG_ARM64_4K_PAGES)] + let pages = 4 * pages; + + if pages == 1 { + tlbi_page(asid, va); + return; + } + + // Page count is always in units of 2 + let num = ((pages + 1) >> 1) as u64; + // base: 5 bits + // exp: 2 bits + // pages = (base + 1) << (5 * exp + 1) + // 0:00000 -> 2 pages = 2 << 0 + // 0:11111 -> 32 * 2 pages = 2 << 5 + // 1:00000 -> 1 * 32 * 2 pages = 2 << 5 + // 1:11111 -> 32 * 32 * 2 pages = 2 << 10 + // 2:00000 -> 1 * 32 * 32 * 2 pages = 2 << 10 + // 2:11111 -> 32 * 32 * 32 * 2 pages = 2 << 15 + // 3:00000 -> 1 * 32 * 32 * 32 * 2 pages = 2 << 15 + // 3:11111 -> 32 * 32 * 32 * 32 * 2 pages = 2 << 20 + let exp = min(3, (64 - num.leading_zeros()) / 5); + let bits = 5 * exp; + let mut base = (num + (1 << bits) - 1) >> bits; + + val |= (exp as u64) << 44; + + while base > 32 { + // SAFETY: tlbi is always safe by definition + unsafe { + asm!( + ".arch armv8.4-a", + "tlbi rvae1os, {x}", + x = in(reg) val | (31 << 39) + ); + } + base -= 32; + } + + // SAFETY: tlbi is always safe by definition + unsafe { + asm!( + ".arch armv8.4-a", + "tlbi rvae1os, {x}", + x = in(reg) val | ((base - 1) << 39) + ); + } +} + +/// Issue a memory barrier (`dsb sy`). +#[inline(always)] +pub(crate) fn sync() { + // SAFETY: Barriers are always safe + unsafe { + asm!("dsb sy"); + } +} diff --git a/drivers/gpu/drm/asahi/microseq.rs b/drivers/gpu/drm/asahi/microseq.rs new file mode 100644 index 00000000000000..cbdb5de62e9218 --- /dev/null +++ b/drivers/gpu/drm/asahi/microseq.rs @@ -0,0 +1,63 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! GPU Micro operation sequence builder +//! +//! As part of a single job submisssion to the GPU, the GPU firmware interprets a sequence of +//! commands that we call a "microsequence". These are responsible for setting up the job execution, +//! timestamping the process, waiting for completion, tearing up any resources, and signaling +//! completion to the driver via the event stamp mechanism. +//! +//! Although the microsequences used by the macOS driver are usually quite uniform and simple, the +//! firmware actually implements enough operations to make this interpreter Turing-complete (!). +//! Most of those aren't implemented yet, since we don't need them, but they could come in handy in +//! the future to do strange things or work around firmware bugs... +//! +//! This module simply implements a collection of microsequence operations that can be appended to +//! and later concatenated into one buffer, ready for firmware execution. + +use crate::fw::microseq; +pub(crate) use crate::fw::microseq::*; +use crate::fw::types::*; +use kernel::prelude::*; + +/// MicroSequence object type, which is just an opaque byte array. +pub(crate) type MicroSequence = GpuArray; + +/// MicroSequence builder. +pub(crate) struct Builder { + ops: KVec, +} + +impl Builder { + /// Create a new Builder object + pub(crate) fn new() -> Builder { + Builder { ops: KVec::new() } + } + + /// Get the relative offset from the current pointer to a given target offset. + /// + /// Used for relative jumps. + pub(crate) fn offset_to(&self, target: i32) -> i32 { + target - self.ops.len() as i32 + } + + /// Add an operation to the end of the sequence. + pub(crate) fn add(&mut self, op: T) -> Result { + let off = self.ops.len(); + let p: *const T = &op; + let p: *const u8 = p as *const u8; + // SAFETY: Microseq operations always have no padding bytes, so it is safe to + // access them as a byte slice. + let s: &[u8] = unsafe { core::slice::from_raw_parts(p, core::mem::size_of::()) }; + self.ops.extend_from_slice(s, GFP_KERNEL)?; + Ok(off as i32) + } + + /// Collect all submitted operations into a finalized GPU object. + pub(crate) fn build(self, alloc: &mut Allocator) -> Result { + let mut array = alloc.array_empty::(self.ops.len())?; + + array.as_mut_slice().clone_from_slice(self.ops.as_slice()); + Ok(array) + } +} diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs new file mode 100644 index 00000000000000..93692e27f2061f --- /dev/null +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -0,0 +1,1602 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! GPU UAT (MMU) management +//! +//! AGX GPUs use an MMU called the UAT, which is largely compatible with the ARM64 page table +//! format. This module manages the global MMU structures, including a shared handoff structure +//! that is used to coordinate VM management operations with the firmware, the TTBAT which points +//! to currently active GPU VM contexts, as well as the individual `Vm` operations to map and +//! unmap buffer objects into a single user or kernel address space. +//! +//! The actual page table management is in the `pt` module. + +use core::fmt::Debug; +use core::mem::size_of; +use core::num::NonZeroUsize; +use core::ops::Range; +use core::sync::atomic::{ + fence, + AtomicU32, + AtomicU64, + AtomicU8, + Ordering, // +}; + +use kernel::{ + addr::PhysicalAddr, + bindings::drm_gpuvm_flags_DRM_GPUVM_IMMEDIATE_MODE, + c_str, + device, + drm::{ + gem::shmem, + gpuvm, + mm, // + }, + error::Result, + io, + new_mutex, + prelude::*, + static_lock_class, + sync::{ + lock::{ + mutex::MutexBackend, + Guard, // + }, + Arc, Mutex, + }, + time::{ + delay::fsleep, + Delta, + Instant, + Monotonic, // + }, + types::ARef, // +}; + +use crate::debug::*; +use crate::module_parameters; +use crate::no_debug; +use crate::{ + driver, + fw, + gem, + hw, + mem, + pgtable, + slotalloc, + util::RangeExt, // +}; + +// KernelMapping protection types +pub(crate) use crate::pgtable::Prot; +pub(crate) use pgtable::prot::*; +pub(crate) use pgtable::{ + UatPageTable, + UAT_PGBIT, + UAT_PGMSK, + UAT_PGSZ, // +}; + +use pgtable::UAT_IAS; + +use pin_init; + +const DEBUG_CLASS: DebugFlags = DebugFlags::Mmu; + +/// PPL magic number for the handoff region +const PPL_MAGIC: u64 = 0x4b1d000000000002; + +/// Number of supported context entries in the TTBAT +const UAT_NUM_CTX: usize = 64; +/// First context available for users +const UAT_USER_CTX_START: usize = 1; +/// Number of available user contexts +const UAT_USER_CTX: usize = UAT_NUM_CTX - UAT_USER_CTX_START; + +/// Lower/user base VA +pub(crate) const IOVA_USER_BASE: u64 = UAT_PGSZ as u64; +/// Lower/user top VA +pub(crate) const IOVA_USER_TOP: u64 = 1 << (UAT_IAS as u64); +/// Lower/user VA range +pub(crate) const IOVA_USER_RANGE: Range = IOVA_USER_BASE..IOVA_USER_TOP; + +/// Upper/kernel base VA +#[cfg(CONFIG_DEV_COREDUMP)] +const IOVA_TTBR1_BASE: u64 = 0xffffff8000000000; +/// Driver-managed kernel base VA +const IOVA_KERN_BASE: u64 = 0xffffffa000000000; +/// Driver-managed kernel top VA +const IOVA_KERN_TOP: u64 = 0xffffffb000000000; +/// Driver-managed kernel VA range +const IOVA_KERN_RANGE: Range = IOVA_KERN_BASE..IOVA_KERN_TOP; +/// Full kernel VA range +#[cfg(CONFIG_DEV_COREDUMP)] +const IOVA_KERN_FULL_RANGE: Range = IOVA_TTBR1_BASE..(!UAT_PGMSK as u64); + +const TTBR_VALID: u64 = 0x1; // BIT(0) +const TTBR_ASID_SHIFT: usize = 48; + +/// Address of a special dummy page? +//const IOVA_UNK_PAGE: u64 = 0x6f_ffff8000; +pub(crate) const IOVA_UNK_PAGE: u64 = IOVA_USER_TOP - 2 * UAT_PGSZ as u64; +/// User VA range excluding the unk page +pub(crate) const IOVA_USER_USABLE_RANGE: Range = IOVA_USER_BASE..IOVA_UNK_PAGE; + +/// A pre-allocated memory region for UAT management +struct UatRegion { + base: PhysicalAddr, + map: io::mem::Mem, +} + +/// SAFETY: It's safe to share UAT region records across threads. +unsafe impl Send for UatRegion {} +/// SAFETY: It's safe to share UAT region records across threads. +unsafe impl Sync for UatRegion {} + +/// Handoff region flush info structure +#[repr(C)] +struct FlushInfo { + state: AtomicU64, + addr: AtomicU64, + size: AtomicU64, +} + +/// UAT Handoff region layout +#[repr(C)] +struct Handoff { + magic_ap: AtomicU64, + magic_fw: AtomicU64, + + lock_ap: AtomicU8, + lock_fw: AtomicU8, + // Implicit padding: 2 bytes + turn: AtomicU32, + cur_slot: AtomicU32, + // Implicit padding: 4 bytes + flush: [FlushInfo; UAT_NUM_CTX + 1], + + unk2: AtomicU8, + // Implicit padding: 7 bytes + unk3: AtomicU64, +} + +const HANDOFF_SIZE: usize = size_of::(); + +/// One VM slot in the TTBAT +#[repr(C)] +struct SlotTTBS { + ttb0: AtomicU64, + ttb1: AtomicU64, +} + +const SLOTS_SIZE: usize = UAT_NUM_CTX * size_of::(); + +// We need at least page 0 (ttb0) +const PAGETABLES_SIZE: usize = UAT_PGSZ; + +/// Inner data for a Vm instance. This is reference-counted by the outer Vm object. +struct VmInner { + dev: driver::AsahiDevRef, + is_kernel: bool, + va_range: Range, + page_table: UatPageTable, + mm: mm::Allocator<(), KernelMappingInner>, + uat_inner: Arc, + binding: Arc>, + id: u64, +} + +/// Slot binding-related inner data for a Vm instance. +struct VmBinding { + active_users: usize, + binding: Option>, + bind_token: Option, + ttb: u64, +} + +/// Data associated with a VM <=> BO pairing +#[pin_data] +struct VmBo { + #[pin] + sgt: Mutex>>, +} + +impl gpuvm::DriverGpuVmBo for VmBo { + fn new() -> impl PinInit { + pin_init!(VmBo { + sgt <- new_mutex!(None, "VmBinding"), + }) + } +} + +#[derive(Default)] +struct StepContext { + new_va: Option>>>, + prev_va: Option>>>, + next_va: Option>>>, + vm_bo: Option>>, + prot: Prot, +} + +impl gpuvm::DriverGpuVm for VmInner { + type Driver = driver::AsahiDriver; + type GpuVmBo = VmBo; + type StepContext = StepContext; + + fn step_map( + self: &mut gpuvm::UpdatingGpuVm<'_, Self>, + op: &mut gpuvm::OpMap, + ctx: &mut Self::StepContext, + ) -> Result { + let mut iova = op.addr(); + let mut left = op.range() as usize; + let mut offset = op.offset() as usize; + + let bo = ctx.vm_bo.as_ref().expect("step_map with no BO"); + + let one_page = op.flags().contains(gpuvm::GpuVaFlags::REPEAT); + + let guard = bo.inner().sgt.lock(); + for range in guard.as_ref().expect("step_map with no SGT").iter() { + // TODO: proper DMA address/length handling + let mut addr = range.dma_address() as usize; + let mut len: usize = range.dma_len() as usize; + + if left == 0 { + break; + } + + if offset > 0 { + let skip = len.min(offset); + addr += skip; + len -= skip; + offset -= skip; + } + + if len == 0 { + continue; + } + + assert!(offset == 0); + + if one_page { + len = left; + } else { + len = len.min(left); + } + + mod_dev_dbg!( + self.dev, + "MMU: map: {:#x}:{:#x} -> {:#x} [OP={}]\n", + addr, + len, + iova, + one_page + ); + + self.page_table.map_pages( + iova..(iova + len as u64), + addr as PhysicalAddr, + ctx.prot, + one_page, + )?; + + left -= len; + iova += len as u64; + } + + let gpuva = ctx.new_va.take().expect("Multiple step_map calls"); + + if op + .map_and_link_va( + self, + gpuva, + ctx.vm_bo.as_ref().expect("step_map with no BO"), + ) + .is_err() + { + dev_err!( + self.dev.as_ref(), + "map_and_link_va failed: {:#x} [{:#x}] -> {:#x}\n", + op.offset(), + op.range(), + op.addr() + ); + return Err(EINVAL); + } + Ok(()) + } + fn step_unmap( + self: &mut gpuvm::UpdatingGpuVm<'_, Self>, + op: &mut gpuvm::OpUnMap, + _ctx: &mut Self::StepContext, + ) -> Result { + let va = op.va().expect("step_unmap: missing VA"); + + mod_dev_dbg!(self.dev, "MMU: unmap: {:#x}:{:#x}\n", va.addr(), va.range()); + + self.page_table + .unmap_pages(va.addr()..(va.addr() + va.range()))?; + + if let Some(asid) = self.slot() { + fence(Ordering::SeqCst); + mem::tlbi_range(asid as u8, va.addr() as usize, va.range() as usize); + mod_dev_dbg!( + self.dev, + "MMU: flush range: asid={:#x} start={:#x} len={:#x}\n", + asid, + va.addr(), + va.range(), + ); + mem::sync(); + } + + if op.unmap_and_unlink_va_defer().is_none() { + dev_err!(self.dev.as_ref(), "step_unmap: could not unlink gpuva"); + } + Ok(()) + } + fn step_remap( + self: &mut gpuvm::UpdatingGpuVm<'_, Self>, + op: &mut gpuvm::OpReMap, + vm_bo: &gpuvm::GpuVmBo, + ctx: &mut Self::StepContext, + ) -> Result { + let va = op.unmap().va().expect("No previous VA"); + let orig_addr = va.addr(); + let orig_range = va.range(); + + // Only unmap the hole between prev/next, if they exist + let unmap_start = if let Some(op) = op.prev_map() { + op.addr() + op.range() + } else { + orig_addr + }; + + let unmap_end = if let Some(op) = op.next_map() { + op.addr() + } else { + orig_addr + orig_range + }; + + mod_dev_dbg!( + self.dev, + "MMU: unmap for remap: {:#x}..{:#x} (from {:#x}:{:#x})\n", + unmap_start, + unmap_end, + orig_addr, + orig_range + ); + + let unmap_range = unmap_end - unmap_start; + + self.page_table.unmap_pages(unmap_start..unmap_end)?; + + if let Some(asid) = self.slot() { + fence(Ordering::SeqCst); + mem::tlbi_range(asid as u8, unmap_start as usize, unmap_range as usize); + mod_dev_dbg!( + self.dev, + "MMU: flush range: asid={:#x} start={:#x} len={:#x}\n", + asid, + unmap_start, + unmap_range, + ); + mem::sync(); + } + + if op.unmap().unmap_and_unlink_va_defer().is_none() { + dev_err!(self.dev.as_ref(), "step_unmap: could not unlink gpuva"); + } + + if let Some(prev_op) = op.prev_map() { + let prev_gpuva = ctx + .prev_va + .take() + .expect("Multiple step_remap calls with prev_op"); + if prev_op.map_and_link_va(self, prev_gpuva, vm_bo).is_err() { + dev_err!(self.dev.as_ref(), "step_remap: could not relink prev gpuva"); + return Err(EINVAL); + } + } + + if let Some(next_op) = op.next_map() { + let next_gpuva = ctx + .next_va + .take() + .expect("Multiple step_remap calls with next_op"); + if next_op.map_and_link_va(self, next_gpuva, vm_bo).is_err() { + dev_err!(self.dev.as_ref(), "step_remap: could not relink next gpuva"); + return Err(EINVAL); + } + } + + Ok(()) + } +} + +impl VmInner { + /// Returns the slot index, if this VM is bound. + fn slot(&self) -> Option { + if self.is_kernel { + // The GFX ASC does not care about the ASID. Pick an arbitrary one. + // TODO: This needs to be a persistently reserved ASID once we integrate + // with the ARM64 kernel ASID machinery to avoid overlap. + Some(0) + } else { + // We don't check whether we lost the slot, which could cause unnecessary + // invalidations against another Vm. However, this situation should be very + // rare (e.g. a Vm lost its slot, which means 63 other Vms bound in the + // interim, and then it gets killed / drops its mappings without doing any + // final rendering). Anything doing active maps/unmaps is probably also + // rendering and therefore likely bound. + self.binding + .lock() + .bind_token + .as_ref() + .map(|token| token.last_slot() + UAT_USER_CTX_START as u32) + } + } + + /// Returns the translation table base for this Vm + fn ttb(&self) -> u64 { + self.page_table.ttb() + } + + /// Map an `mm::Node` representing an mapping in VA space. + fn map_node(&mut self, node: &mm::Node<(), KernelMappingInner>, prot: Prot) -> Result { + let mut iova = node.start(); + let guard = node.bo.as_ref().ok_or(EINVAL)?.inner().sgt.lock(); + let sgt = guard.as_ref().ok_or(EINVAL)?; + let mut offset = node.offset; + let mut left = node.mapped_size; + + for range in sgt.iter() { + if left == 0 { + break; + } + + // TODO: proper DMA address/length handling + let mut addr = range.dma_address() as usize; + let mut len: usize = range.dma_len() as usize; + + if (offset | addr | len | iova as usize) & UAT_PGMSK != 0 { + dev_err!( + self.dev.as_ref(), + "MMU: KernelMapping {:#x}:{:#x} -> {:#x} is not page-aligned\n", + addr, + len, + iova + ); + return Err(EINVAL); + } + + if offset > 0 { + let skip = len.min(offset); + addr += skip; + len -= skip; + offset -= skip; + } + + len = len.min(left); + + if len == 0 { + continue; + } + + mod_dev_dbg!( + self.dev, + "MMU: map: {:#x}:{:#x} -> {:#x}\n", + addr, + len, + iova + ); + + self.page_table.map_pages( + iova..(iova + len as u64), + addr as PhysicalAddr, + prot, + false, + )?; + + iova += len as u64; + left -= len; + } + Ok(()) + } +} + +/// Shared reference to a virtual memory address space ([`Vm`]). +#[derive(Clone)] +pub(crate) struct Vm { + id: u64, + inner: ARef>, + dummy_obj: ARef, + binding: Arc>, +} +no_debug!(Vm); + +/// Slot data for a [`Vm`] slot (nothing, we only care about the indices). +pub(crate) struct SlotInner(); + +impl slotalloc::SlotItem for SlotInner { + type Data = (); +} + +/// Represents a single user of a binding of a [`Vm`] to a slot. +/// +/// The number of users is counted, and the slot will be freed when it drops to 0. +#[derive(Debug)] +pub(crate) struct VmBind(Vm, u32); + +impl VmBind { + /// Returns the slot that this `Vm` is bound to. + pub(crate) fn slot(&self) -> u32 { + self.1 + } +} + +impl Drop for VmBind { + fn drop(&mut self) { + let mut binding = self.0.binding.lock(); + + assert_ne!(binding.active_users, 0); + binding.active_users -= 1; + mod_pr_debug!( + "MMU: slot {} active users {}\n", + self.1, + binding.active_users + ); + if binding.active_users == 0 { + binding.binding = None; + } + } +} + +impl Clone for VmBind { + fn clone(&self) -> VmBind { + let mut binding = self.0.binding.lock(); + + binding.active_users += 1; + mod_pr_debug!( + "MMU: slot {} active users {}\n", + self.1, + binding.active_users + ); + VmBind(self.0.clone(), self.1) + } +} + +/// Inner data required for an object mapping into a [`Vm`]. +pub(crate) struct KernelMappingInner { + // Drop order matters: + // - Drop the GpuVmBo first, which resv locks its BO and drops a GpuVm reference + // - Drop the GEM BO next, since BO free can take the resv lock itself + // - Drop the owner GpuVm last, since that again can take resv locks when the refcount drops to 0 + bo: Option>>, + _gem: Option>, + owner: ARef>, + uat_inner: Arc, + prot: Prot, + offset: usize, + mapped_size: usize, +} + +/// An object mapping into a [`Vm`], which reserves the address range from use by other mappings. +pub(crate) struct KernelMapping(mm::Node<(), KernelMappingInner>); + +impl KernelMapping { + /// Returns the IOVA base of this mapping + pub(crate) fn iova(&self) -> u64 { + self.0.start() + } + + /// Returns the size of this mapping in bytes + pub(crate) fn size(&self) -> usize { + self.0.mapped_size + } + + /// Returns the IOVA base of this mapping + pub(crate) fn iova_range(&self) -> Range { + self.0.start()..(self.0.start() + self.0.mapped_size as u64) + } + + /// Remap a cached mapping as uncached, then synchronously flush that range of VAs from the + /// coprocessor cache. This is required to safely unmap cached/private mappings. + fn remap_uncached_and_flush(&mut self) { + let mut owner = self + .0 + .owner + .exec_lock(None, false) + .expect("Failed to exec_lock in remap_uncached_and_flush"); + + mod_dev_dbg!( + owner.dev, + "MMU: remap as uncached {:#x}:{:#x}\n", + self.iova(), + self.size() + ); + + // Remap in-place as uncached. + // Do not try to unmap the guard page (-1) + let prot = self.0.prot.as_uncached(); + if owner + .page_table + .reprot_pages(self.iova_range(), prot) + .is_err() + { + dev_err!( + owner.dev.as_ref(), + "MMU: remap {:#x}:{:#x} failed\n", + self.iova(), + self.size() + ); + } + fence(Ordering::SeqCst); + + // If we don't have (and have never had) a VM slot, just return + let slot = match owner.slot() { + None => return, + Some(slot) => slot, + }; + + let flush_slot = if owner.is_kernel { + // If this is a kernel mapping, always flush on index 64 + UAT_NUM_CTX as u32 + } else { + // Otherwise, check if this slot is the active one, otherwise return + // Also check that we actually own this slot + let ttb = owner.ttb() | TTBR_VALID | (slot as u64) << TTBR_ASID_SHIFT; + + let uat_inner = self.0.uat_inner.lock(); + uat_inner.handoff().lock(); + let cur_slot = uat_inner.handoff().current_slot(); + let ttb_cur = uat_inner.ttbs()[slot as usize].ttb0.load(Ordering::Relaxed); + uat_inner.handoff().unlock(); + if cur_slot == Some(slot) && ttb_cur == ttb { + slot + } else { + return; + } + }; + + // FIXME: There is a race here, though it'll probably never happen in practice. + // In theory, it's possible for the ASC to finish using our slot, whatever command + // it was processing to complete, the slot to be lost to another context, and the ASC + // to begin using it again with a different page table, thus faulting when it gets a + // flush request here. In practice, the chance of this happening is probably vanishingly + // small, as all 62 other slots would have to be recycled or in use before that slot can + // be reused, and the ASC using user contexts at all is very rare. + + // Still, the locking around UAT/Handoff/TTBs should probably be redesigned to better + // model the interactions with the firmware and avoid these races. + // Possibly TTB changes should be tied to slot locks: + + // Flush: + // - Can early check handoff here (no need to lock). + // If user slot and it doesn't match the active ASC slot, + // we can elide the flush as the ASC guarantees it flushes + // TLBs/caches when it switches context. We just need a + // barrier to ensure ordering. + // - Lock TTB slot + // - If user ctx: + // - Lock handoff AP-side + // - Lock handoff dekker + // - Check TTB & handoff cur ctx + // - Perform flush if necessary + // - This implies taking the fwring lock + // + // TTB change: + // - lock TTB slot + // - lock handoff AP-side + // - lock handoff dekker + // change TTB + + // Lock this flush slot, and write the range to it + let flush = self.0.uat_inner.lock_flush(flush_slot); + let pages = self.size() >> UAT_PGBIT; + flush.begin_flush(self.iova(), self.size() as u64); + if pages >= 0x10000 { + dev_err!( + owner.dev.as_ref(), + "MMU: Flush too big ({:#x} pages))\n", + pages + ); + } + + let cmd = fw::channels::FwCtlMsg { + addr: fw::types::U64(self.iova()), + unk_8: 0, + slot: flush_slot, + page_count: pages as u16, + unk_12: 2, // ? + }; + + // Tell the firmware to do a cache flush + if let Err(e) = (*owner.dev).gpu.fwctl(cmd) { + dev_err!( + owner.dev.as_ref(), + "MMU: ASC cache flush {:#x}:{:#x} failed (err: {:?})\n", + self.iova(), + self.size(), + e + ); + } + + // Finish the flush + flush.end_flush(); + + // Slot is unlocked here + } +} +no_debug!(KernelMapping); + +impl Drop for KernelMapping { + fn drop(&mut self) { + // This is the main unmap function for UAT mappings. + // The sequence of operations here is finicky, due to the interaction + // between cached GFX ASC mappings and the page tables. These mappings + // always have to be flushed from the cache before being unmapped. + + // For uncached mappings, just unmapping and flushing the TLB is sufficient. + + // For cached mappings, this is the required sequence: + // 1. Remap it as uncached + // 2. Flush the TLB range + // 3. If kernel VA mapping OR user VA mapping and handoff.current_slot() == slot: + // a. Take a lock for this slot + // b. Write the flush range to the right context slot in handoff area + // c. Issue a cache invalidation request via FwCtl queue + // d. Poll for completion via queue + // e. Check for completion flag in the handoff area + // f. Drop the lock + // 4. Unmap + // 5. Flush the TLB range again + + if self.0.prot.is_cached_noncoherent() { + mod_pr_debug!( + "MMU: remap as uncached {:#x}:{:#x}\n", + self.iova(), + self.size() + ); + self.remap_uncached_and_flush(); + } + + let mut owner = self + .0 + .owner + .exec_lock(None, false) + .expect("exec_lock failed in KernelMapping::drop"); + mod_dev_dbg!( + owner.dev, + "MMU: unmap {:#x}:{:#x}\n", + self.iova(), + self.size() + ); + + if owner.page_table.unmap_pages(self.iova_range()).is_err() { + dev_err!( + owner.dev.as_ref(), + "MMU: unmap {:#x}:{:#x} failed\n", + self.iova(), + self.size() + ); + } + + if let Some(asid) = owner.slot() { + fence(Ordering::SeqCst); + mem::tlbi_range(asid as u8, self.iova() as usize, self.size()); + mod_dev_dbg!( + owner.dev, + "MMU: flush range: asid={:#x} start={:#x} len={:#x}\n", + asid, + self.iova(), + self.size() + ); + mem::sync(); + } + } +} + +/// Shared UAT global data structures +struct UatShared { + kernel_ttb1: u64, + map_kernel_to_user: bool, + handoff_rgn: UatRegion, + ttbs_rgn: UatRegion, +} + +impl UatShared { + /// Returns the handoff region area + fn handoff(&self) -> &Handoff { + // SAFETY: pointer is non-null per the type invariant + unsafe { (self.handoff_rgn.map.ptr() as *mut Handoff).as_ref() }.unwrap() + } + + /// Returns the TTBAT area + fn ttbs(&self) -> &[SlotTTBS; UAT_NUM_CTX] { + // SAFETY: pointer is non-null per the type invariant + unsafe { (self.ttbs_rgn.map.ptr() as *mut [SlotTTBS; UAT_NUM_CTX]).as_ref() }.unwrap() + } +} + +// SAFETY: Nothing here is unsafe to send across threads. +unsafe impl Send for UatShared {} + +/// Inner data for the top-level UAT instance. +#[pin_data] +struct UatInner { + #[pin] + shared: Mutex, + #[pin] + handoff_flush: [Mutex; UAT_NUM_CTX + 1], +} + +impl UatInner { + /// Take the lock on the shared data and return the guard. + fn lock(&self) -> Guard<'_, UatShared, MutexBackend> { + self.shared.lock() + } + + /// Take a lock on a handoff flush slot and return the guard. + fn lock_flush(&self, slot: u32) -> Guard<'_, HandoffFlush, MutexBackend> { + self.handoff_flush[slot as usize].lock() + } +} + +/// Top-level UAT manager object +pub(crate) struct Uat { + dev: driver::AsahiDevRef, + cfg: &'static hw::HwConfig, + + inner: Arc, + slots: slotalloc::SlotAllocator, + + kernel_vm: Vm, + kernel_lower_vm: Vm, +} + +impl Handoff { + /// Lock the handoff region from firmware access + fn lock(&self) { + self.lock_ap.store(1, Ordering::Relaxed); + fence(Ordering::SeqCst); + + while self.lock_fw.load(Ordering::Relaxed) != 0 { + if self.turn.load(Ordering::Relaxed) != 0 { + self.lock_ap.store(0, Ordering::Relaxed); + while self.turn.load(Ordering::Relaxed) != 0 {} + self.lock_ap.store(1, Ordering::Relaxed); + fence(Ordering::SeqCst); + } + } + fence(Ordering::Acquire); + } + + /// Unlock the handoff region, allowing firmware access + fn unlock(&self) { + self.turn.store(1, Ordering::Relaxed); + self.lock_ap.store(0, Ordering::Release); + } + + /// Returns the current Vm slot mapped by the firmware for lower/unprivileged access, if any. + fn current_slot(&self) -> Option { + let slot = self.cur_slot.load(Ordering::Relaxed); + if slot == 0 || slot == u32::MAX { + None + } else { + Some(slot) + } + } + + /// Initialize the handoff region + fn init(&self) -> Result { + self.magic_ap.store(PPL_MAGIC, Ordering::Relaxed); + self.cur_slot.store(0, Ordering::Relaxed); + self.unk3.store(0, Ordering::Relaxed); + fence(Ordering::SeqCst); + + let start = Instant::::now(); + const TIMEOUT: Delta = Delta::from_millis(1000); + + self.lock(); + while start.elapsed() < TIMEOUT { + if self.magic_fw.load(Ordering::Relaxed) == PPL_MAGIC { + break; + } else { + self.unlock(); + fsleep(Delta::from_millis(10)); + self.lock(); + } + } + + if self.magic_fw.load(Ordering::Relaxed) != PPL_MAGIC { + self.unlock(); + pr_err!("Handoff: Failed to initialize (firmware not running?)\n"); + return Err(EIO); + } + + self.unlock(); + + for i in 0..=UAT_NUM_CTX { + self.flush[i].state.store(0, Ordering::Relaxed); + self.flush[i].addr.store(0, Ordering::Relaxed); + self.flush[i].size.store(0, Ordering::Relaxed); + } + fence(Ordering::SeqCst); + Ok(()) + } +} + +/// Represents a single flush info slot in the handoff region. +/// +/// # Invariants +/// The pointer is valid and there is no aliasing HandoffFlush instance. +struct HandoffFlush(*const FlushInfo); + +// SAFETY: These pointers are safe to send across threads. +unsafe impl Send for HandoffFlush {} + +impl HandoffFlush { + /// Set up a flush operation for the coprocessor + fn begin_flush(&self, start: u64, size: u64) { + // SAFETY: Per the type invariant, this is safe + let flush = unsafe { self.0.as_ref().unwrap() }; + + let state = flush.state.load(Ordering::Relaxed); + if state != 0 { + pr_err!("Handoff: expected flush state 0, got {}\n", state); + } + flush.addr.store(start, Ordering::Relaxed); + flush.size.store(size, Ordering::Relaxed); + flush.state.store(1, Ordering::Relaxed); + } + + /// Complete a flush operation for the coprocessor + fn end_flush(&self) { + // SAFETY: Per the type invariant, this is safe + let flush = unsafe { self.0.as_ref().unwrap() }; + let state = flush.state.load(Ordering::Relaxed); + if state != 2 { + pr_err!("Handoff: expected flush state 2, got {}\n", state); + } + flush.state.store(0, Ordering::Relaxed); + } +} + +impl Vm { + /// Create a new virtual memory address space + fn new( + dev: &driver::AsahiDevice, + uat_inner: Arc, + kernel_range: Range, + cfg: &'static hw::HwConfig, + ttb: Option, + id: u64, + ) -> Result { + let dummy_obj = gem::new_kernel_object(dev, UAT_PGSZ)?; + let is_kernel = ttb.is_some(); + + let page_table = if let Some(ttb) = ttb { + UatPageTable::new_with_ttb(ttb, IOVA_KERN_RANGE, cfg.uat_oas)? + } else { + UatPageTable::new(cfg.uat_oas)? + }; + + let (va_range, gpuvm_range) = if is_kernel { + (IOVA_KERN_RANGE, kernel_range.clone()) + } else { + (IOVA_USER_RANGE, IOVA_USER_USABLE_RANGE) + }; + + let mm = mm::Allocator::new(va_range.start, va_range.range(), ())?; + + let binding = Arc::pin_init( + new_mutex!( + VmBinding { + binding: None, + bind_token: None, + active_users: 0, + ttb: page_table.ttb(), + }, + "VmBinding", + ), + GFP_KERNEL, + )?; + + let binding_clone = binding.clone(); + Ok(Vm { + id, + dummy_obj: dummy_obj.gem.clone(), + inner: gpuvm::GpuVm::new( + c_str!("Asahi::GpuVm"), + // TODO: should we using DRM_GPUVM_RESV_PROTECTED as well? + drm_gpuvm_flags_DRM_GPUVM_IMMEDIATE_MODE, + dev, + dummy_obj.gem.clone(), + gpuvm_range, + kernel_range, + init!(VmInner { + dev: dev.into(), + va_range, + is_kernel, + page_table, + mm, + uat_inner, + binding: binding_clone, + id, + }), + )?, + binding, + }) + } + + /// Get the translation table base for this Vm + fn ttb(&self) -> u64 { + self.binding.lock().ttb + } + + /// Map a GEM object (using its `SGTable`) into this Vm at a free address in a given range. + #[allow(clippy::too_many_arguments)] + pub(crate) fn map_in_range( + &self, + gem: &gem::Object, + object_range: Range, + alignment: u64, + range: Range, + prot: Prot, + guard: bool, + ) -> Result { + let size = object_range.range(); + let sgt = gem.owned_sg_table()?; + let mut inner = self.inner.exec_lock(Some(gem), false)?; + let vm_bo = self.inner.obtain_bo(gem)?; + + let mut vm_bo_guard = vm_bo.inner().sgt.lock(); + if vm_bo_guard.is_none() { + vm_bo_guard.replace(sgt); + } + core::mem::drop(vm_bo_guard); + + let uat_inner = inner.uat_inner.clone(); + let node = inner.mm.insert_node_in_range( + KernelMappingInner { + owner: self.inner.clone(), + uat_inner, + prot, + bo: Some(vm_bo), + _gem: Some(gem.into()), + offset: object_range.start, + mapped_size: size, + }, + (size + if guard { UAT_PGSZ } else { 0 }) as u64, // Add guard page + alignment, + 0, + range.start, + range.end, + mm::InsertMode::Best, + )?; + + let ret = inner.map_node(&node, prot); + // Drop the exec_lock first, so that if map_node failed the + // KernelMappingInner destructur does not deadlock. + core::mem::drop(inner); + ret?; + Ok(KernelMapping(node)) + } + + /// Map a GEM object into this Vm at a specific address. + #[allow(clippy::too_many_arguments)] + pub(crate) fn map_at( + &self, + addr: u64, + size: usize, + gem: ARef, + prot: Prot, + guard: bool, + ) -> Result { + let sgt = gem.owned_sg_table()?; + let mut inner = self.inner.exec_lock(Some(&gem), false)?; + + let vm_bo = self.inner.obtain_bo(&gem)?; + + let mut vm_bo_guard = vm_bo.inner().sgt.lock(); + if vm_bo_guard.is_none() { + vm_bo_guard.replace(sgt); + } + core::mem::drop(vm_bo_guard); + + let uat_inner = inner.uat_inner.clone(); + let node = inner.mm.reserve_node( + KernelMappingInner { + owner: self.inner.clone(), + uat_inner, + prot, + bo: Some(vm_bo), + _gem: Some(gem.clone()), + offset: 0, + mapped_size: size, + }, + addr, + (size + if guard { UAT_PGSZ } else { 0 }) as u64, // Add guard page + 0, + )?; + + let ret = inner.map_node(&node, prot); + // Drop the exec_lock first, so that if map_node failed the + // KernelMappingInner destructur does not deadlock. + core::mem::drop(inner); + ret?; + Ok(KernelMapping(node)) + } + + /// Map a range of a GEM object into this Vm using GPUVM. + #[allow(clippy::too_many_arguments)] + pub(crate) fn bind_object( + &self, + gem: &gem::Object, + addr: u64, + size: u64, + offset: u64, + prot: Prot, + single_page: bool, + ) -> Result { + // Mapping needs a complete context + let mut ctx = StepContext { + new_va: Some(gpuvm::GpuVa::::new(pin_init::default())?), + prev_va: Some(gpuvm::GpuVa::::new(pin_init::default())?), + next_va: Some(gpuvm::GpuVa::::new(pin_init::default())?), + prot, + ..Default::default() + }; + + let sgt = gem.owned_sg_table()?; + let mut inner = self.inner.exec_lock(Some(gem), true)?; + + // Preallocate the page tables, to fail early if we ENOMEM + inner.page_table.alloc_pages(addr..(addr + size))?; + + let vm_bo = self.inner.obtain_bo(gem)?; + + let mut vm_bo_guard = vm_bo.inner().sgt.lock(); + if vm_bo_guard.is_none() { + vm_bo_guard.replace(sgt); + } + core::mem::drop(vm_bo_guard); + + ctx.vm_bo = Some(vm_bo); + + if (addr | size | offset) & (UAT_PGMSK as u64) != 0 { + dev_err!( + inner.dev.as_ref(), + "MMU: Map step {:#x} [{:#x}] -> {:#x} is not page-aligned\n", + offset, + size, + addr + ); + return Err(EINVAL); + } + + let (flags, gem_range) = if single_page { + (gpuvm::GpuVaFlags::REPEAT, UAT_PGSZ as u32) + } else { + (gpuvm::GpuVaFlags::NONE, 0u32) + }; + + mod_dev_dbg!( + inner.dev, + "MMU: sm_map: {:#x} [{:#x}] -> {:#x}\n", + offset, + size, + addr + ); + inner.sm_map(&mut ctx, addr, size, offset, gem_range, flags) + } + + /// Add a direct MMIO mapping to this Vm at a free address. + pub(crate) fn map_io( + &self, + iova: u64, + phys: usize, + size: usize, + prot: Prot, + ) -> Result { + let mut inner = self.inner.exec_lock(None, false)?; + + if (iova as usize | phys | size) & UAT_PGMSK != 0 { + dev_err!( + inner.dev.as_ref(), + "MMU: KernelMapping {:#x}:{:#x} -> {:#x} is not page-aligned\n", + phys, + size, + iova + ); + return Err(EINVAL); + } + + dev_info!( + inner.dev.as_ref(), + "MMU: IO map: {:#x}:{:#x} -> {:#x}\n", + phys, + size, + iova + ); + + let uat_inner = inner.uat_inner.clone(); + let node = inner.mm.reserve_node( + KernelMappingInner { + owner: self.inner.clone(), + uat_inner, + prot, + bo: None, + _gem: None, + offset: 0, + mapped_size: size, + }, + iova, + size as u64, + 0, + )?; + + let ret = inner.page_table.map_pages( + iova..(iova + size as u64), + phys as PhysicalAddr, + prot, + false, + ); + // Drop the exec_lock first, so that if map_node failed the + // KernelMappingInner destructur does not deadlock. + core::mem::drop(inner); + ret?; + Ok(KernelMapping(node)) + } + + /// Unmap everything in an address range. + pub(crate) fn unmap_range(&self, iova: u64, size: u64) -> Result { + // Unmapping a range can only do a single split, so just preallocate + // the prev and next GpuVas + let mut ctx = StepContext { + prev_va: Some(gpuvm::GpuVa::::new(pin_init::default())?), + next_va: Some(gpuvm::GpuVa::::new(pin_init::default())?), + ..Default::default() + }; + + let mut inner = self.inner.exec_lock(None, false)?; + + mod_dev_dbg!(inner.dev, "MMU: sm_unmap: {:#x}:{:#x}\n", iova, size); + inner.sm_unmap(&mut ctx, iova, size) + } + + /// Drop mappings for a given bo. + pub(crate) fn drop_mappings(&self, gem: &gem::Object) -> Result { + // Removing whole mappings only does unmaps, so no preallocated VAs + let mut ctx = Default::default(); + + let inner = self.inner.exec_lock(Some(gem), false)?; + + if let Some(bo) = self.inner.find_bo(gem) { + mod_dev_dbg!(inner.dev, "MMU: bo_unmap\n"); + self.inner.bo_unmap(&mut ctx, &bo)?; + mod_dev_dbg!(inner.dev, "MMU: bo_unmap done\n"); + // We need to drop the exec_lock first, then the GpuVmBo since that will take the lock itself. + core::mem::drop(inner); + core::mem::drop(bo); + } + + Ok(()) + } + + /// Returns the dummy GEM object used to hold the shared DMA reservation locks + pub(crate) fn get_resv_obj(&self) -> ARef { + self.dummy_obj.clone() + } + + /// Check whether an object is external to this GpuVm + pub(crate) fn is_extobj(&self, gem: &gem::Object) -> bool { + self.inner.is_extobj(gem) + } + + /// Check whether an object is external to this GpuVm + pub(crate) fn bo_deferred_cleanup(&self) { + self.inner.bo_deferred_cleanup() + } +} + +impl Drop for VmInner { + fn drop(&mut self) { + let mut binding = self.binding.lock(); + assert_eq!(binding.active_users, 0); + + mod_pr_debug!( + "VmInner::Drop [{}]: bind_token={:?}\n", + self.id, + binding.bind_token + ); + + // Make sure this VM is not mapped to a TTB if it was + if let Some(token) = binding.bind_token.take() { + let idx = (token.last_slot() as usize) + UAT_USER_CTX_START; + let ttb = self.ttb() | TTBR_VALID | (idx as u64) << TTBR_ASID_SHIFT; + + let uat_inner = self.uat_inner.lock(); + uat_inner.handoff().lock(); + let handoff_cur = uat_inner.handoff().current_slot(); + let ttb_cur = uat_inner.ttbs()[idx].ttb0.load(Ordering::SeqCst); + let inval = ttb_cur == ttb; + if inval { + if handoff_cur == Some(idx as u32) { + pr_err!( + "VmInner::drop owning slot {}, but it is currently in use by the ASC?\n", + idx + ); + } + uat_inner.ttbs()[idx].ttb0.store(0, Ordering::SeqCst); + uat_inner.ttbs()[idx].ttb1.store(0, Ordering::SeqCst); + } + uat_inner.handoff().unlock(); + core::mem::drop(uat_inner); + + // In principle we dropped all the KernelMappings already, but we might as + // well play it safe and invalidate the whole ASID. + if inval { + mod_pr_debug!( + "VmInner::Drop [{}]: need inval for ASID {:#x}\n", + self.id, + idx + ); + mem::tlbi_asid(idx as u8); + mem::sync(); + } + } + } +} + +impl Uat { + /// Map a bootloader-preallocated memory region + fn map_region( + dev: &device::Device, + name: &CStr, + size: usize, + cached: bool, + ) -> Result { + let of_node = dev.of_node().ok_or(EINVAL)?; + let res = of_node.reserved_mem_region_to_resource_byname(name)?; + let base = res.start(); + let res_size = res.size().try_into()?; + + if size > res_size { + dev_err!( + dev, + "Region {} is too small (expected {}, got {})\n", + name, + size, + res_size + ); + return Err(ENOMEM); + } + + let flags = if cached { + io::mem::MemFlag::WB + } else { + io::mem::MemFlag::WC + }; + + // SAFETY: The safety of this operation hinges on the correctness of + // much of this file and also the `pgtable` module, so it is difficult + // to prove in a single safety comment. Such is life with raw GPU + // page table management... + let map = unsafe { io::mem::Mem::try_new(res, flags.into()) }.inspect_err(|_| { + dev_err!(dev, "Failed to remap {} mem resource\n", name); + })?; + + Ok(UatRegion { base, map }) + } + + /// Returns a reference to the global kernel (upper half) `Vm` + pub(crate) fn kernel_vm(&self) -> &Vm { + &self.kernel_vm + } + + /// Returns a reference to the local kernel (lower half) `Vm` + pub(crate) fn kernel_lower_vm(&self) -> &Vm { + &self.kernel_lower_vm + } + + #[cfg(CONFIG_DEV_COREDUMP)] + pub(crate) fn dump_kernel_pages(&self) -> Result> { + let mut inner = self.kernel_vm.inner.exec_lock(None, false)?; + inner.page_table.dump_pages(IOVA_KERN_FULL_RANGE) + } + + /// Returns the base physical address of the TTBAT region. + pub(crate) fn ttb_base(&self) -> u64 { + let inner = self.inner.lock(); + + inner.ttbs_rgn.base + } + + /// Binds a `Vm` to a slot, preferring the last used one. + pub(crate) fn bind(&self, vm: &Vm) -> Result { + let mut binding = vm.binding.lock(); + + if binding.binding.is_none() { + assert_eq!(binding.active_users, 0); + + let isolation = *module_parameters::robust_isolation.value() != 0; + + self.slots.set_limit(if isolation { + NonZeroUsize::new(1) + } else { + None + }); + + let slot = self.slots.get(binding.bind_token)?; + if slot.changed() { + mod_pr_debug!("Vm Bind [{}]: bind_token={:?}\n", vm.id, slot.token(),); + let idx = (slot.slot() as usize) + UAT_USER_CTX_START; + let ttb = binding.ttb | TTBR_VALID | (idx as u64) << TTBR_ASID_SHIFT; + + let uat_inner = self.inner.lock(); + + let ttb1 = if uat_inner.map_kernel_to_user { + uat_inner.kernel_ttb1 | TTBR_VALID | (idx as u64) << TTBR_ASID_SHIFT + } else { + 0 + }; + + let ttbs = uat_inner.ttbs(); + uat_inner.handoff().lock(); + if uat_inner.handoff().current_slot() == Some(idx as u32) { + pr_err!( + "Vm::bind to slot {}, but it is currently in use by the ASC?\n", + idx + ); + } + ttbs[idx].ttb0.store(ttb, Ordering::Release); + ttbs[idx].ttb1.store(ttb1, Ordering::Release); + uat_inner.handoff().unlock(); + core::mem::drop(uat_inner); + + // Make sure all TLB entries from the previous owner of this ASID are gone + mem::tlbi_asid(idx as u8); + mem::sync(); + } + + binding.bind_token = Some(slot.token()); + binding.binding = Some(slot); + } + + binding.active_users += 1; + + let slot = binding.binding.as_ref().unwrap().slot() + UAT_USER_CTX_START as u32; + mod_pr_debug!("MMU: slot {} active users {}\n", slot, binding.active_users); + Ok(VmBind(vm.clone(), slot)) + } + + /// Creates a new `Vm` linked to this UAT. + pub(crate) fn new_vm(&self, id: u64, kernel_range: Range) -> Result { + Vm::new( + &self.dev, + self.inner.clone(), + kernel_range, + self.cfg, + None, + id, + ) + } + + /// Creates the reference-counted inner data for a new `Uat` instance. + #[inline(never)] + fn make_inner(dev: &driver::AsahiDevice) -> Result> { + let handoff_rgn = Self::map_region(dev.as_ref(), c_str!("handoff"), HANDOFF_SIZE, true)?; + let ttbs_rgn = Self::map_region(dev.as_ref(), c_str!("ttbs"), SLOTS_SIZE, true)?; + + // SAFETY: The Handoff struct layout matches the firmware's view of memory at this address, + // and the region is at least large enough per the size specified above. + let handoff = unsafe { &(handoff_rgn.map.ptr() as *mut Handoff).as_ref().unwrap() }; + + dev_info!(dev.as_ref(), "MMU: Initializing kernel page table\n"); + + Arc::pin_init( + try_pin_init!(UatInner { + handoff_flush <- pin_init::pin_init_array_from_fn(|i| { + new_mutex!(HandoffFlush(&handoff.flush[i]), "handoff_flush") + }), + shared <- new_mutex!( + UatShared { + kernel_ttb1: 0, + map_kernel_to_user: false, + handoff_rgn, + ttbs_rgn, + }, + "uat_shared" + ), + }), + GFP_KERNEL, + ) + } + + /// Creates a new `Uat` instance given the relevant hardware config. + #[inline(never)] + pub(crate) fn new( + dev: &driver::AsahiDevice, + cfg: &'static hw::HwConfig, + map_kernel_to_user: bool, + ) -> Result { + dev_info!(dev.as_ref(), "MMU: Initializing...\n"); + + let inner = Self::make_inner(dev)?; + + let of_node = dev.as_ref().of_node().ok_or(EINVAL)?; + let res = of_node.reserved_mem_region_to_resource_byname(c_str!("pagetables"))?; + let ttb1 = res.start(); + let ttb1size: usize = res.size().try_into()?; + + if ttb1size < PAGETABLES_SIZE { + dev_err!(dev.as_ref(), "MMU: Pagetables region is too small\n"); + return Err(ENOMEM); + } + + dev_info!(dev.as_ref(), "MMU: Creating kernel page tables\n"); + let kernel_lower_vm = Vm::new(dev, inner.clone(), IOVA_USER_RANGE, cfg, None, 1)?; + let kernel_vm = Vm::new(dev, inner.clone(), IOVA_KERN_RANGE, cfg, Some(ttb1), 0)?; + + dev_info!(dev.as_ref(), "MMU: Kernel page tables created\n"); + + let ttb0 = kernel_lower_vm.ttb(); + + let uat = Self { + dev: dev.into(), + cfg, + kernel_vm, + kernel_lower_vm, + inner, + slots: slotalloc::SlotAllocator::new( + UAT_USER_CTX as u32, + (), + |_inner, _slot| Some(SlotInner()), + c_str!("Uat::SlotAllocator"), + static_lock_class!(), + static_lock_class!(), + )?, + }; + + let mut inner = uat.inner.lock(); + + inner.map_kernel_to_user = map_kernel_to_user; + inner.kernel_ttb1 = ttb1; + + inner.handoff().init()?; + + dev_info!(dev.as_ref(), "MMU: Initializing TTBs\n"); + + inner.handoff().lock(); + + let ttbs = inner.ttbs(); + + ttbs[0].ttb0.store(ttb0 | TTBR_VALID, Ordering::SeqCst); + ttbs[0].ttb1.store(ttb1 | TTBR_VALID, Ordering::SeqCst); + + for ctx in &ttbs[1..] { + ctx.ttb0.store(0, Ordering::Relaxed); + ctx.ttb1.store(0, Ordering::Relaxed); + } + + inner.handoff().unlock(); + + core::mem::drop(inner); + + dev_info!(dev.as_ref(), "MMU: initialized\n"); + + Ok(uat) + } +} + +impl Drop for Uat { + fn drop(&mut self) { + // Make sure we flush the TLBs + fence(Ordering::SeqCst); + mem::tlbi_all(); + mem::sync(); + } +} diff --git a/drivers/gpu/drm/asahi/object.rs b/drivers/gpu/drm/asahi/object.rs new file mode 100644 index 00000000000000..38a2268137effb --- /dev/null +++ b/drivers/gpu/drm/asahi/object.rs @@ -0,0 +1,733 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Asahi GPU object model +//! +//! The AGX GPU includes a coprocessor that uses a large number of shared memory structures to +//! communicate with the driver. These structures contain GPU VA pointers to each other, which are +//! directly dereferenced by the firmware and are expected to always be valid for the usage +//! lifetime of the containing struct (which is an implicit contract, not explicitly managed). +//! Any faults cause an unrecoverable firmware crash, requiring a full system reboot. +//! +//! In order to manage this complexity safely, we implement a GPU object model using Rust's type +//! system to enforce GPU object lifetime relationships. GPU objects represent an allocated piece +//! of memory of a given type, mapped to the GPU (and usually also the CPU). On the CPU side, +//! these objects are associated with a pure Rust structure that contains the objects it depends +//! on (or references to them). This allows us to map Rust lifetimes into the GPU object model +//! system. Then, GPU VA pointers also inherit those lifetimes, which means the Rust borrow checker +//! can ensure that all pointers are assigned an address that is guaranteed to outlive the GPU +//! object it points to. +//! +//! Since the firmware object model does have self-referencing pointers (and there is of course no +//! underlying revocability mechanism to make it safe), we must have an escape hatch. GPU pointers +//! can be weak pointers, which do not enforce lifetimes. In those cases, it is the user's +//! responsibility to ensure that lifetime requirements are met. +//! +//! In other words, the model is necessarily leaky and there is no way to fully map Rust safety to +//! GPU firmware object safety. The goal of the model is to make it easy to model the lifetimes of +//! GPU objects and have the compiler help in avoiding mistakes, rather than to guarantee safety +//! 100% of the time as would be the case for CPU-side Rust code. + +// TODO: There is a fundamental soundness issue with sharing memory with the GPU (that even affects +// C code too). Since the GPU is free to mutate that memory at any time, normal reference invariants +// cannot be enforced on the CPU side. For example, the compiler could perform an optimization that +// assumes that a given memory location does not change between two reads, and causes UB otherwise, +// and then the GPU could mutate that memory out from under the CPU. +// +// For cases where we *expect* this to happen, we use atomic types, which avoid this issue. However, +// doing so for every single field of every type is a non-starter. Right now, there seems to be no +// good solution for this that does not come with significant performance or ergonomics downsides. +// +// In *practice* we are almost always only writing GPU memory, and only reading from atomics, so the +// chances of this actually triggering UB (e.g. a security issue that can be triggered from the GPU +// side) due to a compiler optimization are very slim. +// +// Further discussion: https://github.com/rust-lang/unsafe-code-guidelines/issues/152 + +use kernel::{ + error::code::*, + prelude::*, + sync::Arc, // +}; + +use core::fmt; +use core::fmt::Debug; +use core::fmt::Formatter; +use core::marker::PhantomData; +use core::mem::MaybeUninit; +use core::num::NonZeroU64; +use core::ops::{ + Deref, + DerefMut, + Index, + IndexMut, // +}; +use core::{mem, ptr, slice}; + +use crate::alloc::Allocation; +use crate::debug::*; +use crate::fw::types::Zeroable; +use crate::mmu; + +const DEBUG_CLASS: DebugFlags = DebugFlags::Object; + +/// A GPU-side strong pointer, which is a 64-bit non-zero VA with an associated lifetime. +/// +/// In rare cases these pointers are not aligned, so this is `packed(1)`. +#[repr(C, packed(1))] +pub(crate) struct GpuPointer<'a, T: ?Sized>(NonZeroU64, PhantomData<&'a T>); + +impl<'a, T: ?Sized> GpuPointer<'a, T> { + /// Logical OR the pointer with an arbitrary `u64`. This is used when GPU struct fields contain + /// misc flag fields in the upper bits. The lifetime is retained. This is GPU-unsafe in + /// principle, but we assert that only non-implemented address bits are touched, which is safe + /// for pointers used by the GPU (not by firmware). + pub(crate) fn or(&self, other: u64) -> GpuPointer<'a, T> { + // This will fail for kernel-half pointers, which should not be ORed. + assert_eq!(self.0.get() & other, 0); + // Assert that we only touch the high bits. + assert_eq!(other & 0xffffffffff, 0); + GpuPointer(self.0 | other, PhantomData) + } + + /// Add an arbitrary offset to the pointer. This is not safe (from the GPU perspective), and + /// should only be used via the `inner_ptr` macro to get pointers to inner fields, hence we mark + /// it `unsafe` to discourage direct use. + /// + /// # Safety + /// Do not use directly, only via `inner_ptr`. + // NOTE: The third argument is a type inference hack. + pub(crate) unsafe fn offset(&self, off: usize, _: *const U) -> GpuPointer<'a, U> { + GpuPointer::<'a, U>( + NonZeroU64::new(self.0.get() + (off as u64)).unwrap(), + PhantomData, + ) + } +} + +impl<'a, T> GpuPointer<'a, T> { + /// Create a GPU pointer from a KernelMapping and an offset. + /// TODO: Change all GPU pointers to point to the raw types so size_of here is GPU-sound. + pub(crate) fn from_mapping( + mapping: &'a Arc, + offset: usize, + ) -> Result> { + let addr = mapping.iova().checked_add(offset as u64).ok_or(EINVAL)?; + let end = offset + .checked_add(core::mem::size_of::()) + .ok_or(EINVAL)?; + if end > mapping.size() { + Err(ERANGE) + } else { + Ok(Self(addr.try_into().unwrap(), PhantomData)) + } + } +} + +impl<'a, T: ?Sized> Debug for GpuPointer<'a, T> { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + let val = self.0; + f.write_fmt(format_args!("{:#x} ({})", val, core::any::type_name::())) + } +} + +impl<'a, T: ?Sized> From> for u64 { + fn from(value: GpuPointer<'a, T>) -> Self { + value.0.get() + } +} + +/// Take a pointer to a sub-field within a structure pointed to by a GpuPointer, keeping the +/// lifetime. +#[macro_export] +macro_rules! inner_ptr { + ($gpuva:expr, $($f:tt)*) => ({ + // This mirrors kernel::offset_of(), except we use type inference to avoid having to know + // the type of the pointer explicitly. + fn uninit_from(_: GpuPointer<'_, T>) -> core::mem::MaybeUninit> { + core::mem::MaybeUninit::uninit() + } + let tmp = uninit_from($gpuva); + let outer = tmp.as_ptr(); + // SAFETY: The pointer is valid and aligned, just not initialised; `addr_of` ensures that + // we don't actually read from `outer` (which would be UB) nor create an intermediate + // reference. + let p: *const _ = unsafe { core::ptr::addr_of!((*outer).$($f)*) }; + let inner = p as *const u8; + // SAFETY: The two pointers are within the same allocation block. + let off = unsafe { inner.offset_from(outer as *const u8) }; + // SAFETY: The resulting pointer is guaranteed to point to valid memory within the outer + // object. + unsafe { $gpuva.offset(off.try_into().unwrap(), p) } + }) +} + +/// A GPU-side weak pointer, which is a 64-bit non-zero VA with no lifetime. +/// +/// In rare cases these pointers are not aligned, so this is `packed(1)`. +#[repr(C, packed(1))] +pub(crate) struct GpuWeakPointer(NonZeroU64, PhantomData<*const T>); + +/// SAFETY: GPU weak pointers are always safe to share between threads. +unsafe impl Send for GpuWeakPointer {} +/// SAFETY: GPU weak pointers are always safe to share between threads. +unsafe impl Sync for GpuWeakPointer {} + +// Weak pointers can be copied/cloned regardless of their target type. +impl Copy for GpuWeakPointer {} + +impl Clone for GpuWeakPointer { + fn clone(&self) -> Self { + *self + } +} + +impl GpuWeakPointer { + /// Add an arbitrary offset to the pointer. This is not safe (from the GPU perspective), and + /// should only be used via the `inner_weak_ptr` macro to get pointers to inner fields, hence we + /// mark it `unsafe` to discourage direct use. + /// + /// # Safety + /// Do not use directly, only via `inner_weak_ptr`. + // NOTE: The third argument is a type inference hack. + pub(crate) unsafe fn offset(&self, off: usize, _: *const U) -> GpuWeakPointer { + GpuWeakPointer::( + NonZeroU64::new(self.0.get() + (off as u64)).unwrap(), + PhantomData, + ) + } + + /// Upgrade a weak pointer into a strong pointer. This is not considered safe from the GPU + /// perspective. + /// + /// # Safety + /// The caller must ensure tht the data pointed to lives in the GPU at least as long as the + /// returned lifetime. + pub(crate) unsafe fn upgrade<'a>(&self) -> GpuPointer<'a, T> { + GpuPointer(self.0, PhantomData) + } +} + +impl From> for u64 { + fn from(value: GpuWeakPointer) -> Self { + value.0.get() + } +} + +impl Debug for GpuWeakPointer { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + let val = self.0; + f.write_fmt(format_args!("{:#x} ({})", val, core::any::type_name::())) + } +} + +/// Take a pointer to a sub-field within a structure pointed to by a GpuWeakPointer. +#[macro_export] +macro_rules! inner_weak_ptr { + ($gpuva:expr, $($f:tt)*) => ({ + // See inner_ptr() + fn uninit_from(_: GpuWeakPointer) -> core::mem::MaybeUninit> { + core::mem::MaybeUninit::uninit() + } + let tmp = uninit_from($gpuva); + let outer = tmp.as_ptr(); + // SAFETY: The pointer is valid and aligned, just not initialised; `addr_of` ensures that + // we don't actually read from `outer` (which would be UB) nor create an intermediate + // reference. + let p: *const _ = unsafe { core::ptr::addr_of!((*outer).$($f)*) }; + let inner = p as *const u8; + // SAFETY: The two pointers are within the same allocation block. + let off = unsafe { inner.offset_from(outer as *const u8) }; + // SAFETY: The resulting pointer is guaranteed to point to valid memory within the outer + // object. + unsafe { $gpuva.offset(off.try_into().unwrap(), p) } + }) +} + +/// Types that implement this trait represent a GPU structure from the CPU side. +/// +/// The `Raw` type represents the actual raw structure definition on the GPU side. +/// +/// Types implementing [`GpuStruct`] must have fields owning any objects (or strong references +/// to them) that GPU pointers in the `Raw` structure point to. This mechanism is used to enforce +/// lifetimes. +pub(crate) trait GpuStruct: 'static { + /// The type of the GPU-side structure definition representing the firmware struct layout. + type Raw<'a>; +} + +/// An instance of a GPU object in memory. +/// +/// # Invariants +/// `raw` must point to a valid mapping of the `T::Raw` type associated with the `alloc` allocation. +/// `gpu_ptr` must be the GPU address of the same object. +pub(crate) struct GpuObject> { + raw: *mut T::Raw<'static>, + alloc: U, + gpu_ptr: GpuWeakPointer, + inner: KBox, +} + +impl> GpuObject { + /// Create a new GpuObject given an allocator and the inner data (a type implementing + /// GpuStruct). + /// + /// The caller passes a closure that constructs the `T::Raw` type given a reference to the + /// `GpuStruct`. This is the mechanism used to enforce lifetimes. + pub(crate) fn new( + alloc: U, + inner: T, + callback: impl for<'a> FnOnce(&'a T) -> T::Raw<'a>, + ) -> Result { + let size = mem::size_of::>(); + if size > 0x1000 { + dev_crit!( + alloc.device().as_ref(), + "Allocating {} of size {:#x}, with new, please use new_boxed!\n", + core::any::type_name::(), + size + ); + } + if alloc.size() < size { + return Err(ENOMEM); + } + let gpu_ptr = + GpuWeakPointer::(NonZeroU64::new(alloc.gpu_ptr()).ok_or(EINVAL)?, PhantomData); + mod_dev_dbg!( + alloc.device(), + "Allocating {} @ {:#x}\n", + core::any::type_name::(), + alloc.gpu_ptr() + ); + let p = alloc.ptr().ok_or(EINVAL)?.as_ptr() as *mut T::Raw<'static>; + let mut raw = callback(&inner); + // SAFETY: `p` is guaranteed to be valid per the Allocation invariant, and the type is + // identical to the type of `raw` other than the lifetime. + unsafe { p.copy_from(&mut raw as *mut _ as *mut u8 as *mut _, 1) }; + mem::forget(raw); + Ok(Self { + raw: p, + gpu_ptr, + alloc, + inner: KBox::new(inner, GFP_KERNEL)?, + }) + } + + /// Create a new GpuObject given an allocator and the boxed inner data (a type implementing + /// GpuStruct). + /// + /// The caller passes a closure that initializes the `T::Raw` type given a reference to the + /// `GpuStruct` and a `MaybeUninit`. This is intended to be used with the place!() + /// macro to avoid constructing the whole `T::Raw` object on the stack. + pub(crate) fn new_boxed( + alloc: U, + inner: KBox, + callback: impl for<'a> FnOnce( + &'a T, + &'a mut MaybeUninit>, + ) -> Result<&'a mut T::Raw<'a>>, + ) -> Result { + if alloc.size() < mem::size_of::>() { + return Err(ENOMEM); + } + let gpu_ptr = + GpuWeakPointer::(NonZeroU64::new(alloc.gpu_ptr()).ok_or(EINVAL)?, PhantomData); + mod_dev_dbg!( + alloc.device(), + "Allocating {} @ {:#x}\n", + core::any::type_name::(), + alloc.gpu_ptr() + ); + let p = alloc.ptr().ok_or(EINVAL)?.as_ptr() as *mut MaybeUninit>; + // SAFETY: `p` is guaranteed to be valid per the Allocation invariant. + let raw = callback(&inner, unsafe { &mut *p })?; + if p as *mut T::Raw<'_> != raw as *mut _ { + dev_err!( + alloc.device().as_ref(), + "Allocation callback returned a mismatched reference ({})\n", + core::any::type_name::(), + ); + return Err(EINVAL); + } + Ok(Self { + raw: p as *mut u8 as *mut T::Raw<'static>, + gpu_ptr, + alloc, + inner, + }) + } + + /// Create a new GpuObject given an allocator and the inner data (a type implementing + /// GpuStruct). + /// + /// The caller passes a closure that initializes the `T::Raw` type given a reference to the + /// `GpuStruct` and a `MaybeUninit`. This is intended to be used with the place!() + /// macro to avoid constructing the whole `T::Raw` object on the stack. + pub(crate) fn new_inplace( + alloc: U, + inner: T, + callback: impl for<'a> FnOnce( + &'a T, + &'a mut MaybeUninit>, + ) -> Result<&'a mut T::Raw<'a>>, + ) -> Result { + GpuObject::::new_boxed(alloc, KBox::new(inner, GFP_KERNEL)?, callback) + } + + /// Create a new GpuObject given an allocator and the boxed inner data (a type implementing + /// GpuStruct). + /// + /// The caller passes a closure that initializes the `T::Raw` type given a reference to the + /// `GpuStruct` and a `MaybeUninit`. This is intended to be used with the place!() + /// macro to avoid constructing the whole `T::Raw` object on the stack. + pub(crate) fn new_init_prealloc<'a, I: Init, R: PinInit, F>, E, F>( + alloc: U, + inner_init: impl FnOnce(GpuWeakPointer) -> I, + raw_init: impl FnOnce(&'a T, GpuWeakPointer) -> R, + ) -> Result + where + kernel::error::Error: core::convert::From, + kernel::error::Error: core::convert::From, + { + if alloc.size() < mem::size_of::>() { + return Err(ENOMEM); + } + let gpu_ptr = + GpuWeakPointer::(NonZeroU64::new(alloc.gpu_ptr()).ok_or(EINVAL)?, PhantomData); + mod_dev_dbg!( + alloc.device(), + "Allocating {} @ {:#x}\n", + core::any::type_name::(), + alloc.gpu_ptr() + ); + let inner = inner_init(gpu_ptr); + let p = alloc.ptr().ok_or(EINVAL)?.as_ptr() as *mut T::Raw<'_>; + let ret = Self { + raw: p as *mut u8 as *mut T::Raw<'static>, + gpu_ptr, + alloc, + inner: KBox::init(inner, GFP_KERNEL)?, + }; + let q = &*ret.inner as *const T; + // SAFETY: `p` is guaranteed to be valid per the Allocation invariant. + unsafe { raw_init(&*q, gpu_ptr).__pinned_init(p) }?; + Ok(ret) + } + + /// Returns the GPU VA of this object (as a raw [`NonZeroU64`]) + pub(crate) fn gpu_va(&self) -> NonZeroU64 { + self.gpu_ptr.0 + } + + /// Returns a strong GPU pointer to this object, with a lifetime. + pub(crate) fn gpu_pointer(&self) -> GpuPointer<'_, T> { + GpuPointer(self.gpu_ptr.0, PhantomData) + } + + /// Returns a weak GPU pointer to this object, with no lifetime. + pub(crate) fn weak_pointer(&self) -> GpuWeakPointer { + GpuWeakPointer(self.gpu_ptr.0, PhantomData) + } + + /// Perform a mutation to the inner `Raw` data given a user-supplied callback. + /// + /// The callback gets a mutable reference to the `GpuStruct` type. + pub(crate) fn with_mut( + &mut self, + callback: impl for<'a> FnOnce(&'a mut ::Raw<'a>, &'a mut T) -> RetVal, + ) -> RetVal { + // SAFETY: `self.raw` is valid per the type invariant, and the second half is just + // converting lifetimes. + unsafe { callback(&mut *self.raw, &mut *(&mut *self.inner as *mut _)) } + } + + /// Access the inner `Raw` data given a user-supplied callback. + /// + /// The callback gets a reference to the `GpuStruct` type. + pub(crate) fn with( + &self, + callback: impl for<'a> FnOnce(&'a ::Raw<'a>, &'a T) -> RetVal, + ) -> RetVal { + // SAFETY: `self.raw` is valid per the type invariant, and the second half is just + // converting lifetimes. + unsafe { callback(&*self.raw, &*(&*self.inner as *const _)) } + } +} + +impl> Deref for GpuObject { + type Target = T; + + fn deref(&self) -> &Self::Target { + &self.inner + } +} + +impl> DerefMut for GpuObject { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.inner + } +} + +impl> Debug for GpuObject +where + ::Raw<'static>: Debug, +{ + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + f.debug_struct(core::any::type_name::()) + // SAFETY: `self.raw` is valid per the type invariant. + .field("raw", &format_args!("{:#X?}", unsafe { &*self.raw })) + .field("inner", &format_args!("{:#X?}", &self.inner)) + .field("alloc", &format_args!("{:?}", &self.alloc)) + .finish() + } +} + +impl> GpuObject +where + for<'a> ::Raw<'a>: Default + Zeroable, +{ + /// Create a new GpuObject with default data. `T` must implement `Default` and `T::Raw` must + /// implement `Zeroable`, since the GPU-side memory is initialized by zeroing. + pub(crate) fn new_default(alloc: U) -> Result { + GpuObject::::new_inplace(alloc, Default::default(), |_inner, raw| { + // SAFETY: `raw` is valid here, and `T::Raw` implements `Zeroable`. + Ok(unsafe { + ptr::write_bytes(raw, 0, 1); + (*raw).assume_init_mut() + }) + }) + } +} + +impl> Drop for GpuObject { + fn drop(&mut self) { + mod_dev_dbg!( + self.alloc.device(), + "Dropping {} @ {:?}\n", + core::any::type_name::(), + self.gpu_pointer() + ); + } +} + +// SAFETY: GpuObjects are Send as long as the GpuStruct itself is Send +unsafe impl> Send for GpuObject {} +// SAFETY: GpuObjects are Send as long as the GpuStruct itself is Send +unsafe impl> Sync for GpuObject {} + +/// Trait used to erase the type of a GpuObject, used when we need to keep a list of heterogenous +/// objects around. +pub(crate) trait OpaqueGpuObject: Send + Sync { + fn gpu_va(&self) -> NonZeroU64; +} + +impl> OpaqueGpuObject for GpuObject { + fn gpu_va(&self) -> NonZeroU64 { + Self::gpu_va(self) + } +} + +/// An array of raw GPU objects that is only accessible to the GPU (no CPU-side mapping required). +/// +/// This must necessarily be uninitialized as far as the GPU is concerned, so it cannot be used +/// when initialization is required. +/// +/// # Invariants +/// +/// `alloc` is valid and at least as large as `len` times the size of one `T`. +/// `gpu_ptr` is valid and points to the allocation start. +pub(crate) struct GpuOnlyArray> { + len: usize, + alloc: U, + gpu_ptr: NonZeroU64, + _p: PhantomData, +} + +impl> GpuOnlyArray { + /// Allocate a new GPU-only array with the given length. + pub(crate) fn new(alloc: U, count: usize) -> Result> { + let bytes = count * mem::size_of::(); + let gpu_ptr = NonZeroU64::new(alloc.gpu_ptr()).ok_or(EINVAL)?; + if alloc.size() < bytes { + return Err(ENOMEM); + } + Ok(Self { + len: count, + alloc, + gpu_ptr, + _p: PhantomData, + }) + } + + /// Returns the GPU VA of this arraw (as a raw [`NonZeroU64`]) + pub(crate) fn gpu_va(&self) -> NonZeroU64 { + self.gpu_ptr + } + + /// Returns a strong GPU pointer to this array, with a lifetime. + pub(crate) fn gpu_pointer(&self) -> GpuPointer<'_, &'_ [T]> { + GpuPointer(self.gpu_ptr, PhantomData) + } + + /// Returns a weak GPU pointer to this array, with no lifetime. + pub(crate) fn weak_pointer(&self) -> GpuWeakPointer<[T]> { + GpuWeakPointer(self.gpu_ptr, PhantomData) + } + + /// Returns a pointer to an offset within the array (as a subslice). + pub(crate) fn gpu_offset_pointer(&self, offset: usize) -> GpuPointer<'_, &'_ [T]> { + if offset > self.len { + panic!("Index {} out of bounds (len: {})", offset, self.len); + } + GpuPointer( + NonZeroU64::new(self.gpu_ptr.get() + (offset * mem::size_of::()) as u64).unwrap(), + PhantomData, + ) + } + + /* Not used yet + /// Returns a weak pointer to an offset within the array (as a subslice). + pub(crate) fn weak_offset_pointer(&self, offset: usize) -> GpuWeakPointer<[T]> { + if offset > self.len { + panic!("Index {} out of bounds (len: {})", offset, self.len); + } + GpuWeakPointer( + NonZeroU64::new(self.gpu_ptr.get() + (offset * mem::size_of::()) as u64).unwrap(), + PhantomData, + ) + } + + /// Returns a pointer to an element within the array. + pub(crate) fn gpu_item_pointer(&self, index: usize) -> GpuPointer<'_, &'_ T> { + if index >= self.len { + panic!("Index {} out of bounds (len: {})", index, self.len); + } + GpuPointer( + NonZeroU64::new(self.gpu_ptr.get() + (index * mem::size_of::()) as u64).unwrap(), + PhantomData, + ) + } + */ + + /// Returns a weak pointer to an element within the array. + pub(crate) fn weak_item_pointer(&self, index: usize) -> GpuWeakPointer { + if index >= self.len { + panic!("Index {} out of bounds (len: {})", index, self.len); + } + GpuWeakPointer( + NonZeroU64::new(self.gpu_ptr.get() + (index * mem::size_of::()) as u64).unwrap(), + PhantomData, + ) + } + + /// Returns the length of the array. + pub(crate) fn len(&self) -> usize { + self.len + } +} + +impl> Debug for GpuOnlyArray { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + f.debug_struct(core::any::type_name::()) + .field("len", &format_args!("{:#X?}", self.len())) + .finish() + } +} + +impl> Drop for GpuOnlyArray { + fn drop(&mut self) { + mod_dev_dbg!( + self.alloc.device(), + "Dropping {} @ {:?}\n", + core::any::type_name::(), + self.gpu_pointer() + ); + } +} + +/// An array of raw GPU objects that is also CPU-accessible. +/// +/// # Invariants +/// +/// `raw` is valid and points to the CPU-side view of the array (which must have one). +pub(crate) struct GpuArray> { + raw: *mut T, + array: GpuOnlyArray, +} + +impl> GpuArray { + /// Allocate a new GPU array, initializing each element to its default. + pub(crate) fn empty(alloc: U, count: usize) -> Result> { + let p = alloc.ptr().ok_or(EINVAL)?.as_ptr(); + let inner = GpuOnlyArray::new(alloc, count)?; + let mut pi = p; + for _i in 0..count { + // SAFETY: `pi` is valid per the Allocation type invariant, and GpuOnlyArray guarantees + // that it can never iterate beyond the buffer length. + unsafe { + pi.write(Default::default()); + pi = pi.add(1); + } + } + Ok(Self { + raw: p, + array: inner, + }) + } +} + +impl> GpuArray { + /// Get a slice view of the array contents. + pub(crate) fn as_slice(&self) -> &[T] { + // SAFETY: self.raw / self.len are valid per the type invariant + unsafe { slice::from_raw_parts(self.raw, self.len) } + } + + /// Get a mutable slice view of the array contents. + pub(crate) fn as_mut_slice(&mut self) -> &mut [T] { + // SAFETY: self.raw / self.len are valid per the type invariant + unsafe { slice::from_raw_parts_mut(self.raw, self.len) } + } +} + +impl> Deref for GpuArray { + type Target = GpuOnlyArray; + + fn deref(&self) -> &GpuOnlyArray { + &self.array + } +} + +impl> Index for GpuArray { + type Output = T; + + fn index(&self, index: usize) -> &T { + if index >= self.len { + panic!("Index {} out of bounds (len: {})", index, self.len); + } + // SAFETY: This is bounds checked above + unsafe { &*(self.raw.add(index)) } + } +} + +impl> IndexMut for GpuArray { + fn index_mut(&mut self, index: usize) -> &mut T { + if index >= self.len { + panic!("Index {} out of bounds (len: {})", index, self.len); + } + // SAFETY: This is bounds checked above + unsafe { &mut *(self.raw.add(index)) } + } +} + +// SAFETY: GpuArray are Send as long as the contained type itself is Send +unsafe impl> Send for GpuArray {} +// SAFETY: GpuArray are Sync as long as the contained type itself is Sync +unsafe impl> Sync for GpuArray {} + +impl> Debug for GpuArray { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + f.debug_struct(core::any::type_name::()) + .field("array", &format_args!("{:#X?}", self.as_slice())) + .finish() + } +} diff --git a/drivers/gpu/drm/asahi/pgtable.rs b/drivers/gpu/drm/asahi/pgtable.rs new file mode 100644 index 00000000000000..7c9b54c2242515 --- /dev/null +++ b/drivers/gpu/drm/asahi/pgtable.rs @@ -0,0 +1,690 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! UAT Page Table management +//! +//! AGX GPUs use an MMU called the UAT, which is largely compatible with the ARM64 page table +//! format. This module manages the actual page tables by allocating raw memory pages from +//! the kernel page allocator. + +use core::fmt::Debug; +use core::mem::size_of; +use core::ops::Range; +use core::sync::atomic::{ + AtomicU64, + Ordering, // +}; + +use kernel::{ + addr::PhysicalAddr, + error::Result, + page::Page, + prelude::*, // +}; +#[cfg(CONFIG_DEV_COREDUMP)] +use kernel::{ + types::Owned, + uapi::{ + PF_R, + PF_W, + PF_X, // + }, +}; + +use crate::debug::*; +use crate::util::align; + +const DEBUG_CLASS: DebugFlags = DebugFlags::PgTable; + +/// Number of bits in a page offset. +pub(crate) const UAT_PGBIT: usize = 14; +/// UAT page size. +pub(crate) const UAT_PGSZ: usize = 1 << UAT_PGBIT; +/// UAT page offset mask. +pub(crate) const UAT_PGMSK: usize = UAT_PGSZ - 1; + +type Pte = AtomicU64; + +const PTE_BIT: usize = 3; // log2(sizeof(Pte)) +const PTE_SIZE: usize = 1 << PTE_BIT; + +/// Number of PTEs per page. +const UAT_NPTE: usize = UAT_PGSZ / size_of::(); + +/// Number of address bits to address a level +const UAT_LVBIT: usize = UAT_PGBIT - PTE_BIT; +/// Number of entries per level +const UAT_LVSZ: usize = UAT_NPTE; +/// Mask of level bits +const UAT_LVMSK: u64 = (UAT_LVSZ - 1) as u64; + +const UAT_LEVELS: usize = 3; + +/// UAT input address space +pub(crate) const UAT_IAS: usize = 39; +const UAT_IASMSK: u64 = (1u64 << UAT_IAS) - 1; + +const PTE_TYPE_BITS: u64 = 3; +const PTE_TYPE_LEAF_TABLE: u64 = 3; + +const UAT_NON_GLOBAL: u64 = 1 << 11; +const UAT_AP_SHIFT: u32 = 6; +const UAT_AP_BITS: u64 = 3 << UAT_AP_SHIFT; +const UAT_HIGH_BITS_SHIFT: u32 = 52; +const UAT_HIGH_BITS: u64 = 0xfff << UAT_HIGH_BITS_SHIFT; +const UAT_MEMATTR_SHIFT: u32 = 2; +const UAT_MEMATTR_BITS: u64 = 7 << UAT_MEMATTR_SHIFT; + +const UAT_PROT_BITS: u64 = UAT_AP_BITS | UAT_MEMATTR_BITS | UAT_HIGH_BITS; + +const UAT_AF: u64 = 1 << 10; + +const MEMATTR_CACHED: u8 = 0; +const MEMATTR_DEV: u8 = 1; +const MEMATTR_UNCACHED: u8 = 2; + +const AP_FW_GPU: u8 = 0; +const AP_FW: u8 = 1; +const AP_GPU: u8 = 2; + +const HIGH_BITS_PXN: u16 = 1 << 1; +const HIGH_BITS_UXN: u16 = 1 << 2; +const HIGH_BITS_GPU_ACCESS: u16 = 1 << 3; + +#[cfg(CONFIG_DEV_COREDUMP)] +pub(crate) const PTE_ADDR_BITS: u64 = (!UAT_PGMSK as u64) & (!UAT_HIGH_BITS); + +#[derive(Debug, Copy, Clone)] +pub(crate) struct Prot { + memattr: u8, + ap: u8, + high_bits: u16, +} + +// Firmware + GPU access +const PROT_FW_GPU_NA: Prot = Prot::from_bits(AP_FW_GPU, 0, 0); +const _PROT_FW_GPU_RO: Prot = Prot::from_bits(AP_FW_GPU, 0, 1); +const _PROT_FW_GPU_WO: Prot = Prot::from_bits(AP_FW_GPU, 1, 0); +const PROT_FW_GPU_RW: Prot = Prot::from_bits(AP_FW_GPU, 1, 1); + +// Firmware only access +const PROT_FW_RO: Prot = Prot::from_bits(AP_FW, 0, 0); +const _PROT_FW_NA: Prot = Prot::from_bits(AP_FW, 0, 1); +const PROT_FW_RW: Prot = Prot::from_bits(AP_FW, 1, 0); +const PROT_FW_RW_GPU_RO: Prot = Prot::from_bits(AP_FW, 1, 1); + +// GPU only access +const PROT_GPU_RO: Prot = Prot::from_bits(AP_GPU, 0, 0); +const PROT_GPU_WO: Prot = Prot::from_bits(AP_GPU, 0, 1); +const PROT_GPU_RW: Prot = Prot::from_bits(AP_GPU, 1, 0); +const _PROT_GPU_NA: Prot = Prot::from_bits(AP_GPU, 1, 1); + +#[cfg(CONFIG_DEV_COREDUMP)] +const PF_RW: u32 = PF_R | PF_W; +#[cfg(CONFIG_DEV_COREDUMP)] +const PF_RX: u32 = PF_R | PF_X; + +// For crash dumps +#[cfg(CONFIG_DEV_COREDUMP)] +const PROT_TO_PERMS_FW: [[u32; 4]; 4] = [ + [0, 0, 0, PF_RW], + [0, PF_RW, 0, PF_RW], + [PF_RX, PF_RX, 0, PF_R], + [PF_RX, PF_RW, 0, PF_R], +]; +#[cfg(CONFIG_DEV_COREDUMP)] +const PROT_TO_PERMS_OS: [[u32; 4]; 4] = [ + [0, PF_R, PF_W, PF_RW], + [PF_R, 0, PF_RW, PF_RW], + [0, 0, 0, 0], + [0, 0, 0, 0], +]; + +pub(crate) mod prot { + pub(crate) use super::Prot; + use super::*; + + /// Firmware MMIO R/W + pub(crate) const PROT_FW_MMIO_RW: Prot = PROT_FW_RW.memattr(MEMATTR_DEV); + /// Firmware MMIO R/O + pub(crate) const PROT_FW_MMIO_RO: Prot = PROT_FW_RO.memattr(MEMATTR_DEV); + /// Firmware shared (uncached) RW + pub(crate) const PROT_FW_SHARED_RW: Prot = PROT_FW_RW.memattr(MEMATTR_UNCACHED); + /// Firmware shared (uncached) RO + pub(crate) const PROT_FW_SHARED_RO: Prot = PROT_FW_RO.memattr(MEMATTR_UNCACHED); + /// Firmware private (cached) RW + pub(crate) const PROT_FW_PRIV_RW: Prot = PROT_FW_RW.memattr(MEMATTR_CACHED); + /// Firmware/GPU shared (uncached) RW + pub(crate) const PROT_GPU_FW_SHARED_RW: Prot = PROT_FW_GPU_RW.memattr(MEMATTR_UNCACHED); + /// Firmware/GPU shared (private) RW + pub(crate) const PROT_GPU_FW_PRIV_RW: Prot = PROT_FW_GPU_RW.memattr(MEMATTR_CACHED); + /// Firmware-RW/GPU-RO shared (private) RW + pub(crate) const PROT_GPU_RO_FW_PRIV_RW: Prot = PROT_FW_RW_GPU_RO.memattr(MEMATTR_CACHED); + /// GPU shared/coherent RW + pub(crate) const PROT_GPU_SHARED_RW: Prot = PROT_GPU_RW.memattr(MEMATTR_UNCACHED); + /// GPU shared/coherent RO + pub(crate) const PROT_GPU_SHARED_RO: Prot = PROT_GPU_RO.memattr(MEMATTR_UNCACHED); + /// GPU shared/coherent WO + pub(crate) const PROT_GPU_SHARED_WO: Prot = PROT_GPU_WO.memattr(MEMATTR_UNCACHED); +} + +impl Prot { + const fn from_bits(ap: u8, uxn: u16, pxn: u16) -> Self { + assert!(uxn <= 1); + assert!(pxn <= 1); + assert!(ap <= 3); + + Prot { + high_bits: HIGH_BITS_GPU_ACCESS | (pxn * HIGH_BITS_PXN) | (uxn * HIGH_BITS_UXN), + memattr: 0, + ap, + } + } + + #[cfg(CONFIG_DEV_COREDUMP)] + pub(crate) const fn from_pte(pte: u64) -> Self { + Prot { + high_bits: (pte >> UAT_HIGH_BITS_SHIFT) as u16, + ap: ((pte & UAT_AP_BITS) >> UAT_AP_SHIFT) as u8, + memattr: ((pte & UAT_MEMATTR_BITS) >> UAT_MEMATTR_SHIFT) as u8, + } + } + + #[cfg(CONFIG_DEV_COREDUMP)] + pub(crate) const fn elf_flags(&self) -> u32 { + let ap = (self.ap & 3) as usize; + let uxn = if self.high_bits & HIGH_BITS_UXN != 0 { + 1 + } else { + 0 + }; + let pxn = if self.high_bits & HIGH_BITS_PXN != 0 { + 1 + } else { + 0 + }; + let gpu = self.high_bits & HIGH_BITS_GPU_ACCESS != 0; + + // Format: + // [12 top bits of PTE] [12 bottom bits of PTE] [5 bits pad] [ELF RWX] + let mut perms = if gpu { + PROT_TO_PERMS_OS[ap][(uxn << 1) | pxn] + } else { + PROT_TO_PERMS_FW[ap][(uxn << 1) | pxn] + }; + + perms |= ((self.as_pte() >> 52) << 20) as u32; + perms |= ((self.as_pte() & 0xfff) << 8) as u32; + + perms + } + + const fn memattr(&self, memattr: u8) -> Self { + Self { memattr, ..*self } + } + + const fn as_pte(&self) -> u64 { + (self.ap as u64) << UAT_AP_SHIFT + | (self.high_bits as u64) << UAT_HIGH_BITS_SHIFT + | (self.memattr as u64) << UAT_MEMATTR_SHIFT + | UAT_AF + } + + pub(crate) const fn is_cached_noncoherent(&self) -> bool { + self.ap != AP_GPU && self.memattr == MEMATTR_CACHED + } + + pub(crate) const fn as_uncached(&self) -> Self { + self.memattr(MEMATTR_UNCACHED) + } +} + +impl Default for Prot { + fn default() -> Self { + PROT_FW_GPU_NA + } +} + +#[cfg(CONFIG_DEV_COREDUMP)] +pub(crate) struct DumpedPage { + pub(crate) iova: u64, + pub(crate) pte: u64, + pub(crate) data: Option>, +} + +pub(crate) struct UatPageTable { + ttb: PhysicalAddr, + ttb_owned: bool, + va_range: Range, + oas_mask: u64, +} + +impl UatPageTable { + pub(crate) fn new(oas: u32) -> Result { + mod_pr_debug!("UATPageTable::new: oas={}\n", oas); + let ttb_page = Page::alloc_page(GFP_KERNEL | __GFP_ZERO)?; + let ttb = Page::into_phys(ttb_page); + Ok(UatPageTable { + ttb, + ttb_owned: true, + va_range: 0..(1u64 << UAT_IAS), + oas_mask: (1u64 << oas) - 1, + }) + } + + pub(crate) fn new_with_ttb(ttb: PhysicalAddr, va_range: Range, oas: u32) -> Result { + mod_pr_debug!( + "UATPageTable::new_with_ttb: ttb={:#x} range={:#x?} oas={}\n", + ttb, + va_range, + oas + ); + if ttb & (UAT_PGMSK as PhysicalAddr) != 0 { + return Err(EINVAL); + } + if (va_range.start | va_range.end) & (UAT_PGMSK as u64) != 0 { + return Err(EINVAL); + } + // SAFETY: The TTB is should remain valid (if properly mapped), as it is bootloader-managed. + if unsafe { Page::borrow_phys(&ttb) }.is_none() { + pr_err!( + "UATPageTable::new_with_ttb: ttb at {:#x} is not mapped (DT using no-map?)\n", + ttb + ); + return Err(EIO); + } + + Ok(UatPageTable { + ttb, + ttb_owned: false, + va_range, + oas_mask: (1u64 << oas) - 1, + }) + } + + pub(crate) fn ttb(&self) -> PhysicalAddr { + self.ttb + } + + fn with_pages( + &mut self, + iova_range: Range, + alloc: bool, + free: bool, + mut cb: F, + ) -> Result + where + F: FnMut(u64, &[Pte]) -> Result, + { + mod_pr_debug!( + "UATPageTable::with_pages: {:#x?} alloc={} free={}\n", + iova_range, + alloc, + free + ); + if (iova_range.start | iova_range.end) & (UAT_PGMSK as u64) != 0 { + pr_err!( + "UATPageTable::with_pages: iova range not aligned: {:#x?}\n", + iova_range + ); + return Err(EINVAL); + } + + if iova_range.is_empty() { + return Ok(()); + } + + let mut iova = iova_range.start & UAT_IASMSK; + let mut last_iova = iova; + // Handle the case where iova_range.end is just at the top boundary of the IAS + let end = ((iova_range.end - 1) & UAT_IASMSK) + 1; + + let mut pt_addr: [Option; UAT_LEVELS] = Default::default(); + pt_addr[UAT_LEVELS - 1] = Some(self.ttb); + + 'outer: while iova < end { + mod_pr_debug!("UATPageTable::with_pages: iova={:#x}\n", iova); + let addr_diff = last_iova ^ iova; + for level in (0..UAT_LEVELS - 1).rev() { + // If the iova has changed at this level or above, invalidate the physaddr + if addr_diff & !((1 << (UAT_PGBIT + (level + 1) * UAT_LVBIT)) - 1) != 0 { + if let Some(phys) = pt_addr[level].take() { + if free { + mod_pr_debug!( + "UATPageTable::with_pages: free level {} {:#x?}\n", + level, + phys + ); + // SAFETY: Page tables for our VA ranges always come from Page::into_phys(). + unsafe { Page::from_phys(phys) }; + } + mod_pr_debug!("UATPageTable::with_pages: invalidate level {}\n", level); + } + } + } + last_iova = iova; + for level in (0..UAT_LEVELS - 1).rev() { + // Fetch the page table base address for this level + if pt_addr[level].is_none() { + let phys = pt_addr[level + 1].unwrap(); + mod_pr_debug!( + "UATPageTable::with_pages: need level {}, parent phys {:#x}\n", + level, + phys + ); + let upidx = ((iova >> (UAT_PGBIT + (level + 1) * UAT_LVBIT) as u64) & UAT_LVMSK) + as usize; + // SAFETY: Page table addresses are either allocated by us, or + // firmware-managed and safe to borrow a struct page from. + let upt = unsafe { Page::borrow_phys_unchecked(&phys) }; + mod_pr_debug!("UATPageTable::with_pages: borrowed phys {:#x}\n", phys); + pt_addr[level] = + upt.with_pointer_into_page(upidx * PTE_SIZE, PTE_SIZE, |p| { + let uptep = p as *const _ as *const Pte; + // SAFETY: with_pointer_into_page() ensures the pointer is valid, + // and our index is aligned so it is safe to deref as an AtomicU64. + let upte = unsafe { &*uptep }; + let mut upte_val = upte.load(Ordering::Relaxed); + // Allocate if requested + if upte_val == 0 && alloc { + let pt_page = Page::alloc_page(GFP_KERNEL | __GFP_ZERO)?; + mod_pr_debug!("UATPageTable::with_pages: alloc PT at {:#x}\n", pt_page.phys()); + let pt_paddr = Page::into_phys(pt_page); + upte_val = pt_paddr | PTE_TYPE_LEAF_TABLE; + upte.store(upte_val, Ordering::Relaxed); + } + if upte_val & PTE_TYPE_BITS == PTE_TYPE_LEAF_TABLE { + Ok(Some(upte_val & self.oas_mask & (!UAT_PGMSK as u64))) + } else if upte_val == 0 || (!alloc && !free) { + mod_pr_debug!("UATPageTable::with_pages: no level {}\n", level); + Ok(None) + } else { + pr_err!("UATPageTable::with_pages: Unexpected Table PTE value {:#x} at iova {:#x} index {} phys {:#x}\n", upte_val, + iova, level + 1, phys + ((upidx * PTE_SIZE) as PhysicalAddr)); + Ok(None) + } + })?; + mod_pr_debug!( + "UATPageTable::with_pages: level {} PT {:#x?}\n", + level, + pt_addr[level] + ); + } + // If we don't have a page table, skip this entire level + if pt_addr[level].is_none() { + let block = 1 << (UAT_PGBIT + UAT_LVBIT * (level + 1)); + let old = iova; + iova = align(iova + 1, block); + mod_pr_debug!( + "UATPageTable::with_pages: skip {:#x} {:#x} -> {:#x}\n", + block, + old, + iova + ); + continue 'outer; + } + } + + let idx = ((iova >> UAT_PGBIT as u64) & UAT_LVMSK) as usize; + let max_count = UAT_NPTE - idx; + let count = (((end - iova) >> UAT_PGBIT) as usize).min(max_count); + let phys = pt_addr[0].unwrap(); + mod_pr_debug!( + "UATPageTable::with_pages: leaf PT at {:#x} idx {:#x} count {:#x} iova {:#x}\n", + phys, + idx, + count, + iova + ); + // SAFETY: Page table addresses are either allocated by us, or + // firmware-managed and safe to borrow a struct page from. + let pt = unsafe { Page::borrow_phys_unchecked(&phys) }; + pt.with_pointer_into_page(idx * PTE_SIZE, count * PTE_SIZE, |p| { + let ptep = p as *const _ as *const Pte; + // SAFETY: We know this is a valid pointer to PTEs and the range is valid and + // checked by with_pointer_into_page(). + let ptes = unsafe { core::slice::from_raw_parts(ptep, count) }; + cb(iova, ptes)?; + Ok(()) + })?; + + let block = 1 << (UAT_PGBIT + UAT_LVBIT); + iova = align(iova + 1, block); + } + + if free { + for level in (0..UAT_LEVELS - 1).rev() { + if let Some(phys) = pt_addr[level] { + mod_pr_debug!( + "UATPageTable::with_pages: free level {} {:#x?}\n", + level, + phys + ); + // SAFETY: Page tables for our VA ranges always come from Page::into_phys(). + unsafe { Page::from_phys(phys) }; + } + } + } + + Ok(()) + } + + pub(crate) fn alloc_pages(&mut self, iova_range: Range) -> Result { + mod_pr_debug!("UATPageTable::alloc_pages: {:#x?}\n", iova_range); + self.with_pages(iova_range, true, false, |_, _| Ok(())) + } + + fn pte_bits(&self) -> u64 { + if self.ttb_owned { + // Owned page tables are userspace, so non-global + PTE_TYPE_LEAF_TABLE | UAT_NON_GLOBAL + } else { + // The sole non-owned page table is kernelspace, so global + PTE_TYPE_LEAF_TABLE + } + } + + pub(crate) fn map_pages( + &mut self, + iova_range: Range, + mut phys: PhysicalAddr, + prot: Prot, + one_page: bool, + ) -> Result { + mod_pr_debug!( + "UATPageTable::map_pages: {:#x?} {:#x?} {:?}\n", + iova_range, + phys, + prot + ); + if phys & (UAT_PGMSK as PhysicalAddr) != 0 { + pr_err!("UATPageTable::map_pages: phys not aligned: {:#x?}\n", phys); + return Err(EINVAL); + } + + let pte_bits = self.pte_bits(); + + self.with_pages(iova_range, true, false, |iova, ptes| { + for (idx, pte) in ptes.iter().enumerate() { + let ptev = pte.load(Ordering::Relaxed); + if ptev != 0 { + pr_err!( + "UATPageTable::map_pages: Page at IOVA {:#x} is mapped (PTE: {:#x})\n", + iova + (idx * UAT_PGSZ) as u64, + ptev + ); + } + pte.store(phys | prot.as_pte() | pte_bits, Ordering::Relaxed); + if !one_page { + phys += UAT_PGSZ as PhysicalAddr; + } + } + Ok(()) + }) + } + + pub(crate) fn reprot_pages(&mut self, iova_range: Range, prot: Prot) -> Result { + mod_pr_debug!( + "UATPageTable::reprot_pages: {:#x?} {:?}\n", + iova_range, + prot + ); + self.with_pages(iova_range, true, false, |iova, ptes| { + for (idx, pte) in ptes.iter().enumerate() { + let ptev = pte.load(Ordering::Relaxed); + if ptev & PTE_TYPE_BITS != PTE_TYPE_LEAF_TABLE { + pr_err!( + "UATPageTable::reprot_pages: Page at IOVA {:#x} is unmapped (PTE: {:#x})\n", + iova + (idx * UAT_PGSZ) as u64, + ptev + ); + continue; + } + pte.store((ptev & !UAT_PROT_BITS) | prot.as_pte(), Ordering::Relaxed); + } + Ok(()) + }) + } + + pub(crate) fn unmap_pages(&mut self, iova_range: Range) -> Result { + mod_pr_debug!("UATPageTable::unmap_pages: {:#x?}\n", iova_range); + self.with_pages(iova_range, false, false, |iova, ptes| { + for (idx, pte) in ptes.iter().enumerate() { + if pte.load(Ordering::Relaxed) & PTE_TYPE_LEAF_TABLE == 0 { + pr_err!( + "UATPageTable::unmap_pages: Page at IOVA {:#x} already unmapped\n", + iova + (idx * UAT_PGSZ) as u64 + ); + } + pte.store(0, Ordering::Relaxed); + } + Ok(()) + }) + } + + #[cfg(CONFIG_DEV_COREDUMP)] + pub(crate) fn dump_pages(&mut self, iova_range: Range) -> Result> { + let mut pages = KVVec::new(); + let oas_mask = self.oas_mask; + let iova_base = self.va_range.start & !UAT_IASMSK; + self.with_pages(iova_range, false, false, |iova, ptes| { + let iova = iova | iova_base; + for (idx, ppte) in ptes.iter().enumerate() { + let pte = ppte.load(Ordering::Relaxed); + if (pte & PTE_TYPE_LEAF_TABLE) != PTE_TYPE_LEAF_TABLE { + continue; + } + let memattr = ((pte & UAT_MEMATTR_BITS) >> UAT_MEMATTR_SHIFT) as u8; + + if !(memattr == MEMATTR_CACHED || memattr == MEMATTR_UNCACHED) { + pages.push( + DumpedPage { + iova: iova + (idx * UAT_PGSZ) as u64, + pte, + data: None, + }, + GFP_KERNEL, + )?; + continue; + } + let phys = pte & oas_mask & (!UAT_PGMSK as u64); + // SAFETY: GPU pages are either firmware/preallocated pages + // (which the kernel isn't concerned with and are either in + // the page map or not, and if they aren't, borrow_phys() + // will fail), or GPU page table pages (which we own), + // or GEM buffer pages (which are locked while they are + // mapped in the page table), so they should be safe to + // borrow. + // + // This does trust the firmware not to have any weird + // mappings in its own internal page tables, but since + // those are managed by the uPPL which is privileged anyway, + // this trust does not actually extend any trust boundary. + let src_page = match unsafe { Page::borrow_phys(&phys) } { + Some(page) => page, + None => { + pages.push( + DumpedPage { + iova: iova + (idx * UAT_PGSZ) as u64, + pte, + data: None, + }, + GFP_KERNEL, + )?; + continue; + } + }; + let dst_page = Page::alloc_page(GFP_KERNEL)?; + src_page.with_page_mapped(|psrc| -> Result { + // SAFETY: This could technically still have a data race with the firmware + // or other driver code (or even userspace with timestamp buffers), but while + // the Rust language technically says this is UB, in the real world, using + // atomic reads for this is guaranteed to never cause any harmful effects + // other than possibly reading torn/unreliable data. At least on ARM64 anyway. + // + // (Yes, I checked with Rust people about this. ~~ Lina) + // + let src_items = unsafe { + core::slice::from_raw_parts( + psrc as *const AtomicU64, + UAT_PGSZ / core::mem::size_of::(), + ) + }; + dst_page.with_page_mapped(|pdst| -> Result { + // SAFETY: We own the destination page, so it is safe to view its contents + // as a u64 slice. + let dst_items = unsafe { + core::slice::from_raw_parts_mut( + pdst as *mut u64, + UAT_PGSZ / core::mem::size_of::(), + ) + }; + for (si, di) in src_items.iter().zip(dst_items.iter_mut()) { + *di = si.load(Ordering::Relaxed); + } + Ok(()) + })?; + Ok(()) + })?; + pages.push( + DumpedPage { + iova: iova + (idx * UAT_PGSZ) as u64, + pte, + data: Some(dst_page), + }, + GFP_KERNEL, + )?; + } + Ok(()) + })?; + Ok(pages) + } +} + +impl Drop for UatPageTable { + fn drop(&mut self) { + mod_pr_debug!("UATPageTable::drop range: {:#x?}\n", &self.va_range); + if self + .with_pages(self.va_range.clone(), false, true, |iova, ptes| { + for (idx, pte) in ptes.iter().enumerate() { + if pte.load(Ordering::Relaxed) != 0 { + pr_err!( + "UATPageTable::drop: Leaked page at IOVA {:#x}\n", + iova + (idx * UAT_PGSZ) as u64 + ); + } + } + Ok(()) + }) + .is_err() + { + pr_err!("UATPageTable::drop failed to free page tables\n",); + } + if self.ttb_owned { + mod_pr_debug!("UATPageTable::drop: Free TTB {:#x}\n", self.ttb); + // SAFETY: If we own the ttb, it was allocated with Page::into_phys(). + unsafe { + Page::from_phys(self.ttb); + } + } + } +} diff --git a/drivers/gpu/drm/asahi/queue/common.rs b/drivers/gpu/drm/asahi/queue/common.rs new file mode 100644 index 00000000000000..a68352828cfbc3 --- /dev/null +++ b/drivers/gpu/drm/asahi/queue/common.rs @@ -0,0 +1,42 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Common queue functionality. +//! +//! Shared helpers used by the submission logic for multiple command types. + +use crate::file; +use crate::fw::job::UserTimestamp; + +use kernel::prelude::*; +use kernel::uapi; +use kernel::xarray; + +pub(super) fn get_timestamp_object( + objects: Pin<&xarray::XArray>>, + timestamp: uapi::drm_asahi_timestamp, +) -> Result> { + if timestamp.handle == 0 { + return Ok(None); + } + + let guard = objects.lock(); + let object = guard + .get(timestamp.handle.try_into()?) + .ok_or(ENOENT)? + .clone(); + core::mem::drop(guard); + + #[allow(irrefutable_let_patterns)] + if let file::Object::TimestampBuffer(mapping) = object { + let offset = timestamp.offset; + if (offset.checked_add(8).ok_or(EINVAL)?) as usize > mapping.size() { + return Err(ERANGE); + } + Ok(Some(UserTimestamp { + mapping: mapping.clone(), + offset: offset as usize, + })) + } else { + Err(EINVAL) + } +} diff --git a/drivers/gpu/drm/asahi/queue/compute.rs b/drivers/gpu/drm/asahi/queue/compute.rs new file mode 100644 index 00000000000000..62afc561806703 --- /dev/null +++ b/drivers/gpu/drm/asahi/queue/compute.rs @@ -0,0 +1,385 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#![allow(clippy::unusual_byte_groupings)] + +//! Compute work queue. +//! +//! A compute queue consists of one underlying WorkQueue. +//! This module is in charge of creating all of the firmware structures required to submit compute +//! work to the GPU, based on the userspace command buffer. + +use super::common; +use crate::alloc::Allocator; +use crate::debug::*; +use crate::fw::types::*; +use crate::gpu::GpuManager; +use crate::{ + file, + fw, + gpu, + microseq, // +}; +use crate::{ + inner_ptr, + inner_weak_ptr, // +}; +use core::sync::atomic::Ordering; +use kernel::dma_fence::RawDmaFence; +use kernel::drm::sched::Job; +use kernel::prelude::*; +use kernel::sync::Arc; +use kernel::uapi; +use kernel::xarray; + +const DEBUG_CLASS: DebugFlags = DebugFlags::Compute; + +#[versions(AGX)] +impl super::QueueInner::ver { + /// Submit work to a compute queue. + pub(super) fn submit_compute( + &self, + job: &mut Job, + cmdbuf: &uapi::drm_asahi_cmd_compute, + attachments: µseq::Attachments, + objects: Pin<&xarray::XArray>>, + id: u64, + flush_stamps: bool, + ) -> Result { + let gpu = match (*self.dev) + .gpu + .as_any() + .downcast_ref::() + { + Some(gpu) => gpu, + None => { + dev_crit!(self.dev.as_ref(), "GpuManager mismatched with Queue!\n"); + return Err(EIO); + } + }; + + let mut alloc = gpu.alloc(); + let kalloc = &mut *alloc; + + mod_dev_dbg!(self.dev, "[Submission {}] Compute!\n", id); + + if cmdbuf.flags != 0 { + return Err(EINVAL); + } + + let mut user_timestamps: fw::job::UserTimestamps = Default::default(); + user_timestamps.start = common::get_timestamp_object(objects, cmdbuf.ts.start)?; + user_timestamps.end = common::get_timestamp_object(objects, cmdbuf.ts.end)?; + + // This sequence number increases per new client/VM? assigned to some slot, + // but it's unclear *which* slot... + let slot_client_seq: u8 = (self.id & 0xff) as u8; + + let vm_bind = job.vm_bind.clone(); + + mod_dev_dbg!( + self.dev, + "[Submission {}] VM slot = {}\n", + id, + vm_bind.slot() + ); + + let notifier = self.notifier.clone(); + + let fence = job.fence.clone(); + let comp_job = job.get_comp()?; + let ev_comp = comp_job.event_info(); + + let preempt2_off = gpu.get_cfg().compute_preempt1_size; + let preempt3_off = preempt2_off + 8; + let preempt4_off = preempt3_off + 8; + let preempt5_off = preempt4_off + 8; + let preempt_size = preempt5_off + 8; + + let preempt_buf = self + .ualloc + .lock() + .array_empty_tagged(preempt_size, b"CPMT")?; + + mod_dev_dbg!( + self.dev, + "[Submission {}] Event #{} {:#x?} -> {:#x?}\n", + id, + ev_comp.slot, + ev_comp.value, + ev_comp.value.next(), + ); + + let timestamps = Arc::new( + kalloc.shared.new_default::()?, + GFP_KERNEL, + )?; + + let uuid = 0; + mod_dev_dbg!(self.dev, "[Submission {}] UUID = {:#x?}\n", id, uuid); + + // TODO: check + #[ver(V >= V13_0B4)] + let count = self.counter.fetch_add(1, Ordering::Relaxed); + + let comp = GpuObject::new_init_prealloc( + kalloc.gpu_ro.alloc_object()?, + |ptr: GpuWeakPointer| { + let notifier = notifier.clone(); + let vm_bind = vm_bind.clone(); + try_init!(fw::compute::RunCompute::ver { + preempt_buf: preempt_buf, + micro_seq: { + let mut builder = microseq::Builder::new(); + + let stats = gpu.initdata.runtime_pointers.stats.comp.weak_pointer(); + + let start_comp = builder.add(microseq::StartCompute::ver { + header: microseq::op::StartCompute::HEADER, + unk_pointer: inner_weak_ptr!(ptr, unk_pointee), + #[ver(G < G14X)] + job_params1: Some(inner_weak_ptr!(ptr, job_params1)), + #[ver(G >= G14X)] + job_params1: None, + #[ver(G >= G14X)] + registers: inner_weak_ptr!(ptr, registers), + stats, + work_queue: ev_comp.info_ptr, + vm_slot: vm_bind.slot(), + unk_28: 0x1, + event_generation: self.id as u32, + event_seq: U64(ev_comp.event_seq), + unk_38: 0x0, + job_params2: inner_weak_ptr!(ptr, job_params2), + unk_44: 0x0, + uuid, + attachments: *attachments, + padding: Default::default(), + #[ver(V >= V13_0B4)] + unk_flag: inner_weak_ptr!(ptr, unk_flag), + #[ver(V >= V13_0B4)] + counter: U64(count), + #[ver(V >= V13_0B4)] + notifier_buf: inner_weak_ptr!(notifier.weak_pointer(), state.unk_buf), + })?; + + if user_timestamps.any() { + builder.add(microseq::Timestamp::ver { + header: microseq::op::Timestamp::new(true), + command_time: inner_weak_ptr!(ptr, command_time), + ts_pointers: inner_weak_ptr!(ptr, timestamp_pointers), + update_ts: inner_weak_ptr!(ptr, timestamp_pointers.start_addr), + work_queue: ev_comp.info_ptr, + user_ts_pointers: inner_weak_ptr!(ptr, user_timestamp_pointers), + #[ver(V >= V13_0B4)] + unk_ts: inner_weak_ptr!(ptr, context_store_req), + uuid, + unk_30_padding: 0, + })?; + } + + #[ver(G < G14X)] + builder.add(microseq::WaitForIdle { + header: microseq::op::WaitForIdle::new(microseq::Pipe::Compute), + })?; + #[ver(G >= G14X)] + builder.add(microseq::WaitForIdle2 { + header: microseq::op::WaitForIdle2::HEADER, + })?; + + if user_timestamps.any() { + builder.add(microseq::Timestamp::ver { + header: microseq::op::Timestamp::new(false), + command_time: inner_weak_ptr!(ptr, command_time), + ts_pointers: inner_weak_ptr!(ptr, timestamp_pointers), + update_ts: inner_weak_ptr!(ptr, timestamp_pointers.end_addr), + work_queue: ev_comp.info_ptr, + user_ts_pointers: inner_weak_ptr!(ptr, user_timestamp_pointers), + #[ver(V >= V13_0B4)] + unk_ts: inner_weak_ptr!(ptr, context_store_req), + uuid, + unk_30_padding: 0, + })?; + } + + let off = builder.offset_to(start_comp); + builder.add(microseq::FinalizeCompute::ver { + header: microseq::op::FinalizeCompute::HEADER, + stats, + work_queue: ev_comp.info_ptr, + vm_slot: vm_bind.slot(), + #[ver(V < V13_0B4)] + unk_18: 0, + job_params2: inner_weak_ptr!(ptr, job_params2), + unk_24: 0, + uuid, + fw_stamp: ev_comp.fw_stamp_pointer, + stamp_value: ev_comp.value.next(), + unk_38: 0, + unk_3c: 0, + unk_40: 0, + unk_44: 0, + unk_48: 0, + unk_4c: 0, + unk_50: 0, + unk_54: 0, + unk_58: 0, + #[ver(G == G14 && V < V13_0B4)] + unk_5c_g14: U64(0), + restart_branch_offset: off, + has_attachments: (attachments.count > 0) as u32, + #[ver(V >= V13_0B4)] + unk_64: Default::default(), + #[ver(V >= V13_0B4)] + unk_flag: inner_weak_ptr!(ptr, unk_flag), + #[ver(V >= V13_0B4)] + unk_79: Default::default(), + })?; + + builder.add(microseq::RetireStamp { + header: microseq::op::RetireStamp::HEADER, + })?; + builder.build(&mut kalloc.private)? + }, + notifier, + vm_bind, + timestamps, + user_timestamps, + }) + }, + |inner, _ptr| { + let vm_slot = vm_bind.slot(); + try_init!(fw::compute::raw::RunCompute::ver { + tag: fw::workqueue::CommandType::RunCompute, + #[ver(V >= V13_0B4)] + counter: U64(count), + unk_4: 0, + vm_slot, + notifier: inner.notifier.gpu_pointer(), + unk_pointee: Default::default(), + #[ver(G < G14X)] + __pad0: Default::default(), + #[ver(G < G14X)] + job_params1 <- try_init!(fw::compute::raw::JobParameters1 { + preempt_buf1: inner.preempt_buf.gpu_pointer(), + cdm_ctrl_stream_base: U64(cmdbuf.cdm_ctrl_stream_base), + // buf2-5 Only if internal program is used + preempt_buf2: inner.preempt_buf.gpu_offset_pointer(preempt2_off), + preempt_buf3: inner.preempt_buf.gpu_offset_pointer(preempt3_off), + preempt_buf4: inner.preempt_buf.gpu_offset_pointer(preempt4_off), + preempt_buf5: inner.preempt_buf.gpu_offset_pointer(preempt5_off), + usc_exec_base_cp: U64(self.usc_exec_base), + unk_38: U64(0x8c60), + helper_program: cmdbuf.helper.binary, // Internal program addr | 1 + unk_44: 0, + helper_arg: U64(cmdbuf.helper.data), // Only if internal program used + helper_cfg: cmdbuf.helper.cfg, // 0x40 if internal program used + unk_54: 0, + unk_58: 1, + unk_5c: 0, + iogpu_unk_40: 0, // 0x1c if internal program used + __pad: Default::default(), + }), + #[ver(G >= G14X)] + registers: fw::job::raw::RegisterArray::new( + inner_weak_ptr!(_ptr, registers.registers), + |r| { + r.add(0x1a510, inner.preempt_buf.gpu_pointer().into()); + r.add(0x1a420, cmdbuf.cdm_ctrl_stream_base); + // buf2-5 Only if internal program is used + r.add(0x1a4d0, inner.preempt_buf.gpu_offset_pointer(preempt2_off).into()); + r.add(0x1a4d8, inner.preempt_buf.gpu_offset_pointer(preempt3_off).into()); + r.add(0x1a4e0, inner.preempt_buf.gpu_offset_pointer(preempt4_off).into()); + r.add(0x1a4e8, inner.preempt_buf.gpu_offset_pointer(preempt5_off).into()); + r.add(0x10071, self.usc_exec_base); // USC_EXEC_BASE_CP + r.add(0x11841, cmdbuf.helper.binary.into()); + r.add(0x11849, cmdbuf.helper.data); + r.add(0x11f81, cmdbuf.helper.cfg.into()); + r.add(0x1a440, 0x24201); + r.add(0x12091, 0 /* iogpu_unk_40 */); + /* + r.add(0x10201, 0x100); // Some kind of counter?? Does this matter? + r.add(0x10428, 0x100); // Some kind of counter?? Does this matter? + */ + } + ), + __pad1: Default::default(), + microsequence: inner.micro_seq.gpu_pointer(), + microsequence_size: inner.micro_seq.len() as u32, + job_params2 <- try_init!(fw::compute::raw::JobParameters2::ver { + #[ver(V >= V13_0B4)] + unk_0_0: 0, + unk_0: Default::default(), + preempt_buf1: inner.preempt_buf.gpu_pointer(), + cdm_ctrl_stream_end: U64(cmdbuf.cdm_ctrl_stream_end), + unk_34: Default::default(), + #[ver(G < G14X)] + unk_g14x: 0, + #[ver(G >= G14X)] + unk_g14x: 0x24201, + unk_58: 0, + #[ver(V < V13_0B4)] + unk_5c: 0, + }), + encoder_params <- try_init!(fw::job::raw::EncoderParams { + unk_8: 0x0, // fixed + sync_grow: 0x0, // check! + unk_10: 0x0, // fixed + encoder_id: 0, + unk_18: 0x0, // fixed + unk_mask: 0xffffffff, + sampler_array: U64(cmdbuf.sampler_heap), + sampler_count: cmdbuf.sampler_count as u32, + sampler_max: (cmdbuf.sampler_count as u32) + 1, + }), + meta <- try_init!(fw::job::raw::JobMeta { + unk_0: 0, + unk_2: 0, + no_preemption: 0, + stamp: ev_comp.stamp_pointer, + fw_stamp: ev_comp.fw_stamp_pointer, + stamp_value: ev_comp.value.next(), + stamp_slot: ev_comp.slot, + evctl_index: 0, // fixed + flush_stamps: flush_stamps as u32, + uuid, + event_seq: ev_comp.event_seq as u32, + }), + command_time: U64(0), + timestamp_pointers <- try_init!(fw::job::raw::TimestampPointers { + start_addr: Some(inner_ptr!(inner.timestamps.gpu_pointer(), start)), + end_addr: Some(inner_ptr!(inner.timestamps.gpu_pointer(), end)), + }), + user_timestamp_pointers: inner.user_timestamps.pointers()?, + client_sequence: slot_client_seq, + pad_2d1: Default::default(), + unk_2d4: 0, + unk_2d8: 0, + #[ver(V >= V13_0B4)] + context_store_req: U64(0), + #[ver(V >= V13_0B4)] + context_store_compl: U64(0), + #[ver(V >= V13_0B4)] + unk_2e9: Default::default(), + #[ver(V >= V13_0B4)] + unk_flag: U32(0), + #[ver(V >= V13_0B4)] + unk_pad: Default::default(), + }) + }, + )?; + + core::mem::drop(alloc); + + fence.add_command(); + comp_job.add_cb(comp, vm_bind.slot(), move |error| { + if let Some(err) = error { + fence.set_error(err.into()) + } + + fence.command_complete(); + })?; + + comp_job.next_seq(); + + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/queue/mod.rs b/drivers/gpu/drm/asahi/queue/mod.rs new file mode 100644 index 00000000000000..caae3e19f994c6 --- /dev/null +++ b/drivers/gpu/drm/asahi/queue/mod.rs @@ -0,0 +1,937 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Submission queue management +//! +//! This module implements the userspace view of submission queues and the logic to map userspace +//! submissions to firmware queues. + +use kernel::dma_fence::*; +use kernel::prelude::*; +use kernel::{ + c_str, + dma_fence, + drm::sched, + macros::versions, + sync::{ + Arc, + LockClassKey, + Mutex, // + }, + uapi, + xarray, // +}; + +use crate::alloc::Allocator; +use crate::debug::*; +use crate::driver::{AsahiDevRef, AsahiDevice}; +use crate::file::MAX_COMMANDS_PER_SUBMISSION; +use crate::fw::types::*; +use crate::gpu::GpuManager; +use crate::inner_weak_ptr; +use crate::microseq; +use crate::module_parameters; +use crate::util::{ + AnyBitPattern, + Reader, // +}; +use crate::{ + alloc, + buffer, + channel, + event, + file, + fw, + gpu, + mmu, + workqueue, // +}; + +use core::sync::atomic::{ + AtomicU64, + Ordering, // +}; + +const DEBUG_CLASS: DebugFlags = DebugFlags::Queue; + +const WQ_SIZE: u32 = 0x500; + +mod common; +mod compute; +mod render; + +/// Trait implemented by all versioned queues. +pub(crate) trait Queue: Send + Sync { + fn submit( + &mut self, + id: u64, + syncs: KVec, + in_sync_count: usize, + cmdbuf_raw: &[u8], + objects: Pin<&xarray::XArray>>, + ) -> Result; +} + +#[versions(AGX)] +struct SubQueue { + wq: Arc, +} + +#[versions(AGX)] +impl SubQueue::ver { + fn new_job(&mut self, fence: dma_fence::Fence) -> SubQueueJob::ver { + SubQueueJob::ver { + wq: self.wq.clone(), + fence: Some(fence), + job: None, + } + } +} + +#[versions(AGX)] +struct SubQueueJob { + wq: Arc, + job: Option, + fence: Option, +} + +#[versions(AGX)] +impl SubQueueJob::ver { + fn get(&mut self) -> Result<&mut workqueue::Job::ver> { + if self.job.is_none() { + mod_pr_debug!("SubQueueJob: Creating {:?} job\n", self.wq.pipe_type()); + self.job + .replace(self.wq.new_job(self.fence.take().unwrap())?); + } + Ok(self.job.as_mut().expect("expected a Job")) + } + + fn commit(&mut self) -> Result { + match self.job.as_mut() { + Some(job) => job.commit(), + None => Ok(()), + } + } + + fn can_submit(&self) -> Option { + self.job.as_ref().and_then(|job| job.can_submit()) + } +} + +#[versions(AGX)] +pub(crate) struct Queue { + dev: AsahiDevRef, + _sched: sched::Scheduler, + entity: sched::Entity, + vm: mmu::Vm, + q_vtx: Option, + q_frag: Option, + q_comp: Option, + fence_ctx: FenceContexts, + inner: QueueInner::ver, +} + +#[versions(AGX)] +pub(crate) struct QueueInner { + dev: AsahiDevRef, + ualloc: Arc>, + buffer: buffer::Buffer::ver, + gpu_context: Arc, + notifier_list: Arc>, + notifier: Arc>, + usc_exec_base: u64, + id: u64, + #[ver(V >= V13_0B4)] + counter: AtomicU64, +} + +#[versions(AGX)] +#[derive(Default)] +pub(crate) struct JobFence { + id: u64, + pending: AtomicU64, +} + +#[versions(AGX)] +impl JobFence::ver { + fn add_command(self: &FenceObject) { + self.pending.fetch_add(1, Ordering::Relaxed); + } + + fn command_complete(self: &FenceObject) { + let remain = self.pending.fetch_sub(1, Ordering::Relaxed) - 1; + mod_pr_debug!( + "JobFence[{}]: Command complete (remain: {})\n", + self.id, + remain + ); + if remain == 0 { + mod_pr_debug!("JobFence[{}]: Signaling\n", self.id); + if self.signal().is_err() { + pr_err!("JobFence[{}]: Fence signal failed\n", self.id); + } + } + } +} + +#[versions(AGX)] +#[vtable] +impl dma_fence::FenceOps for JobFence::ver { + fn get_driver_name<'a>(self: &'a FenceObject) -> &'a CStr { + c_str!("asahi") + } + fn get_timeline_name<'a>(self: &'a FenceObject) -> &'a CStr { + c_str!("queue") + } +} + +#[versions(AGX)] +pub(crate) struct QueueJob { + dev: AsahiDevRef, + vm_bind: mmu::VmBind, + op_guard: Option, + sj_vtx: Option, + sj_frag: Option, + sj_comp: Option, + fence: UserFence, + notifier: Arc>, + notification_count: u32, + did_run: bool, + id: u64, +} + +#[versions(AGX)] +impl QueueJob::ver { + fn get_vtx(&mut self) -> Result<&mut workqueue::Job::ver> { + self.sj_vtx + .as_mut() + .ok_or_else(|| { + cls_pr_debug!(Errors, "No vertex queue\n"); + EINVAL + })? + .get() + } + fn get_frag(&mut self) -> Result<&mut workqueue::Job::ver> { + self.sj_frag + .as_mut() + .ok_or_else(|| { + cls_pr_debug!(Errors, "No fragment queue\n"); + EINVAL + })? + .get() + } + fn get_comp(&mut self) -> Result<&mut workqueue::Job::ver> { + self.sj_comp + .as_mut() + .ok_or_else(|| { + cls_pr_debug!(Errors, "No compute queue\n"); + EINVAL + })? + .get() + } + + fn commit(&mut self) -> Result { + mod_dev_dbg!(self.dev, "QueueJob {}: Committing\n", self.id); + + self.sj_vtx.as_mut().map(|a| a.commit()).unwrap_or(Ok(()))?; + self.sj_frag + .as_mut() + .map(|a| a.commit()) + .unwrap_or(Ok(()))?; + self.sj_comp.as_mut().map(|a| a.commit()).unwrap_or(Ok(())) + } +} + +#[versions(AGX)] +impl sched::JobImpl for QueueJob::ver { + fn prepare(job: &mut sched::Job) -> Option { + mod_dev_dbg!(job.dev, "QueueJob {}: Checking runnability\n", job.id); + + if let Some(sj) = job.sj_vtx.as_ref() { + if let Some(fence) = sj.can_submit() { + mod_dev_dbg!( + job.dev, + "QueueJob {}: Blocking due to vertex queue full\n", + job.id + ); + return Some(fence); + } + } + if let Some(sj) = job.sj_frag.as_ref() { + if let Some(fence) = sj.can_submit() { + mod_dev_dbg!( + job.dev, + "QueueJob {}: Blocking due to fragment queue full\n", + job.id + ); + return Some(fence); + } + } + if let Some(sj) = job.sj_comp.as_ref() { + if let Some(fence) = sj.can_submit() { + mod_dev_dbg!( + job.dev, + "QueueJob {}: Blocking due to compute queue full\n", + job.id + ); + return Some(fence); + } + } + None + } + + #[allow(unused_assignments)] + fn run(job: &mut sched::Job) -> Result> { + mod_dev_dbg!(job.dev, "QueueJob {}: Running Job\n", job.id); + + // We can only increase the notifier threshold here, now that we are + // actually running the job. We cannot increase it while queueing the + // job without introducing subtle race conditions. Suppose we did, as + // early versions of drm/asahi did: + // + // 1. When processing the ioctl submit, a job is queued to drm_sched. + // Incorrectly, the notifier threshold is increased, gating firmware + // events. + // 2. When DRM schedules an event, the hardware is kicked. + // 3. When the number of processed jobs equals the threshold, the + // firmware signals the complete event to the kernel + // 4. When the kernel gets a complete event, we signal the out-syncs. + // + // Does that work? There are a few scenarios. + // + // 1. There is nothing else ioctl submitted before the job completes. + // The job is scheduled, completes, and signals immediately. + // Everything works. + // 2. There is nontrivial sync across different queues. Since each queue + // has a separate own notifier threshold, submitting one does not + // block scheduling of the other. Everything works the way you'd + // expect. drm/sched handles the wait/signal ordering. + // 3. Two ioctls are submitted back-to-back. The first signals a fence + // that the second waits on. Due to the notifier threshold increment, + // the first job's completion event is deferred. But in good + // conditions, drm/sched will schedule the second submit anyway + // because it kills the pointless intra-queue sync. Then both + // commands execute and are signalled together. + // 4. Two ioctls are submitted back-to-back as above, but conditions are + // bad. Reporting completion of the first job is still masked by the + // notifier threshold, but the intra-queue fences are not optimized + // out in drm/sched... drm/sched doesn't schedule the second job + // until the first is signalled, but the first isn't signalled until + // the second is completed, but the second can't complete until it's + // scheduled. We hang! + // + // In good conditions, everything works properly and/or we win the race + // to mask the issue. So the issue here is challenging to hit. + // Nevertheless, we do need to get it right. + // + // The intention with drm/sched is that jobs that are not yet scheduled + // are "invisible" to the firmware. Incrementing the notifier threshold + // earlier than this violates that which leads to circles like the + // above. Deferring the increment to submit solves the race. + job.notifier.threshold.with(|raw, _inner| { + raw.increase(job.notification_count); + }); + + let gpu = match (*job.dev) + .gpu + .clone() + .arc_as_any() + .downcast::() + { + Ok(gpu) => gpu, + Err(_) => { + dev_crit!(job.dev.as_ref(), "GpuManager mismatched with QueueJob!\n"); + return Err(EIO); + } + }; + + if job.op_guard.is_none() { + job.op_guard = Some(gpu.start_op()?); + } + + // First submit all the commands for each queue. This can fail. + + let mut frag_job = None; + let mut frag_sub = None; + if let Some(sj) = job.sj_frag.as_mut() { + frag_job = sj.job.take(); + if let Some(wqjob) = frag_job.as_mut() { + mod_dev_dbg!(job.dev, "QueueJob {}: Submit fragment\n", job.id); + frag_sub = Some(wqjob.submit()?); + } + } + + let mut vtx_job = None; + let mut vtx_sub = None; + if let Some(sj) = job.sj_vtx.as_mut() { + vtx_job = sj.job.take(); + if let Some(wqjob) = vtx_job.as_mut() { + mod_dev_dbg!(job.dev, "QueueJob {}: Submit vertex\n", job.id); + vtx_sub = Some(wqjob.submit()?); + } + } + + let mut comp_job = None; + let mut comp_sub = None; + if let Some(sj) = job.sj_comp.as_mut() { + comp_job = sj.job.take(); + if let Some(wqjob) = comp_job.as_mut() { + mod_dev_dbg!(job.dev, "QueueJob {}: Submit compute\n", job.id); + comp_sub = Some(wqjob.submit()?); + } + } + + // Now we fully commit to running the job + mod_dev_dbg!(job.dev, "QueueJob {}: Run fragment\n", job.id); + frag_sub.map(|a| gpu.run_job(a)).transpose()?; + + mod_dev_dbg!(job.dev, "QueueJob {}: Run vertex\n", job.id); + vtx_sub.map(|a| gpu.run_job(a)).transpose()?; + + mod_dev_dbg!(job.dev, "QueueJob {}: Run compute\n", job.id); + comp_sub.map(|a| gpu.run_job(a)).transpose()?; + + mod_dev_dbg!(job.dev, "QueueJob {}: Drop compute job\n", job.id); + core::mem::drop(comp_job); + mod_dev_dbg!(job.dev, "QueueJob {}: Drop vertex job\n", job.id); + core::mem::drop(vtx_job); + mod_dev_dbg!(job.dev, "QueueJob {}: Drop fragment job\n", job.id); + core::mem::drop(frag_job); + + job.did_run = true; + + Ok(Some(Fence::from_fence(&job.fence))) + } + + fn timed_out(job: &mut sched::Job) -> sched::Status { + // FIXME: Handle timeouts properly + dev_err!( + job.dev.as_ref(), + "QueueJob {}: Job timed out on the DRM scheduler, things will probably break (ran: {})\n", + job.id, job.did_run + ); + sched::Status::NoDevice + } + + fn cancel(job: &mut sched::Job) { + dev_info!( + job.dev.as_ref(), + "QueueJob {}: Job canceled on DRM scheduler teardown\n", + job.id + ); + } +} + +#[versions(AGX)] +impl Drop for QueueJob::ver { + fn drop(&mut self) { + mod_dev_dbg!(self.dev, "QueueJob {}: Dropping\n", self.id); + } +} + +static QUEUE_NAME: &CStr = c_str!("asahi_fence"); +static QUEUE_CLASS_KEY: Pin<&LockClassKey> = kernel::static_lock_class!(); + +#[versions(AGX)] +impl Queue::ver { + /// Create a new user queue. + #[allow(clippy::too_many_arguments)] + pub(crate) fn new( + dev: &AsahiDevice, + vm: mmu::Vm, + alloc: &mut gpu::KernelAllocators, + ualloc: Arc>, + ualloc_priv: Arc>, + event_manager: Arc, + mgr: &buffer::BufferManager::ver, + id: u64, + priority: u32, + usc_exec_base: u64, + ) -> Result { + mod_dev_dbg!(dev, "[Queue {}] Creating queue\n", id); + + // Must be shared, no cache management on this one! + let mut notifier_list = alloc.shared.new_default::()?; + + let self_ptr = notifier_list.weak_pointer(); + notifier_list.with_mut(|raw, _inner| { + raw.list_head.next = Some(inner_weak_ptr!(self_ptr, list_head)); + }); + + let threshold = alloc.shared.new_default::()?; + + let notifier: Arc> = Arc::new( + alloc.private.new_init( + /*try_*/ init!(fw::event::Notifier::ver { threshold }), + |inner, _p| { + try_init!(fw::event::raw::Notifier::ver { + threshold: inner.threshold.gpu_pointer(), + generation: AtomicU32::new(id as u32), + cur_count: AtomicU32::new(0), + unk_10: AtomicU32::new(0x50), + state: Default::default() + }) + }, + )?, + GFP_KERNEL, + )?; + + // Priorities are handled by the AGX scheduler, there is no meaning within a + // per-queue scheduler. Use a single run queue wth Kernel priority. + let sched = + sched::Scheduler::new(dev.as_ref(), 1, WQ_SIZE, 0, 100000, c_str!("asahi_sched"))?; + let entity = sched::Entity::new(&sched, sched::Priority::Kernel)?; + + let buffer = + buffer::Buffer::ver::new(&*(*dev).gpu, alloc, ualloc.clone(), ualloc_priv, mgr)?; + + let mut ret = Queue::ver { + dev: dev.into(), + _sched: sched, + entity, + vm, + q_vtx: None, + q_frag: None, + q_comp: None, + fence_ctx: FenceContexts::new(1, QUEUE_NAME, QUEUE_CLASS_KEY)?, + inner: QueueInner::ver { + dev: dev.into(), + ualloc, + gpu_context: Arc::new( + workqueue::GpuContext::new(dev, alloc, buffer.any_ref())?, + GFP_KERNEL, + )?, + + buffer, + notifier_list: Arc::new(notifier_list, GFP_KERNEL)?, + notifier, + usc_exec_base, + id, + #[ver(V >= V13_0B4)] + counter: AtomicU64::new(0), + }, + }; + + // Rendering structures + let tvb_blocks = *module_parameters::initial_tvb_size.value(); + + ret.inner.buffer.ensure_blocks(tvb_blocks)?; + + ret.q_vtx = Some(SubQueue::ver { + wq: workqueue::WorkQueue::ver::new( + dev, + alloc, + event_manager.clone(), + ret.inner.gpu_context.clone(), + ret.inner.notifier_list.clone(), + channel::PipeType::Vertex, + id, + priority, + WQ_SIZE, + )?, + }); + + ret.q_frag = Some(SubQueue::ver { + wq: workqueue::WorkQueue::ver::new( + dev, + alloc, + event_manager.clone(), + ret.inner.gpu_context.clone(), + ret.inner.notifier_list.clone(), + channel::PipeType::Fragment, + id, + priority, + WQ_SIZE, + )?, + }); + + // Compute structures + ret.q_comp = Some(SubQueue::ver { + wq: workqueue::WorkQueue::ver::new( + dev, + alloc, + event_manager, + ret.inner.gpu_context.clone(), + ret.inner.notifier_list.clone(), + channel::PipeType::Compute, + id, + priority, + WQ_SIZE, + )?, + }); + + mod_dev_dbg!(dev, "[Queue {}] Queue created\n", id); + Ok(ret) + } +} + +const SQ_RENDER: usize = 0; +const SQ_COMPUTE: usize = 1; +const SQ_COUNT: usize = 2; + +// SAFETY: All bit patterns are valid by construction. +unsafe impl AnyBitPattern for uapi::drm_asahi_cmd_header {} +unsafe impl AnyBitPattern for uapi::drm_asahi_cmd_render {} +unsafe impl AnyBitPattern for uapi::drm_asahi_cmd_compute {} +unsafe impl AnyBitPattern for uapi::drm_asahi_attachment {} + +fn build_attachments(reader: &mut Reader<'_>, size: usize) -> Result { + const STRIDE: usize = core::mem::size_of::(); + let count = size / STRIDE; + + if count > microseq::MAX_ATTACHMENTS { + return Err(EINVAL); + } + + let mut attachments: microseq::Attachments = Default::default(); + attachments.count = count as u32; + + for i in 0..count { + let att: uapi::drm_asahi_attachment = reader.read()?; + + if att.flags != 0 || att.pad != 0 { + return Err(EINVAL); + } + + // Some kind of power-of-2 exponent related to attachment size, in + // bounds [1, 6]? We don't know what this is exactly yet. + let unk_e = 1; + + let cache_lines = (att.size + 127) >> 7; + attachments.list[i as usize] = microseq::Attachment { + address: U64(att.pointer), + size: cache_lines.try_into()?, + unk_c: 0x17, + unk_e: unk_e as u16, + }; + } + + Ok(attachments) +} + +#[versions(AGX)] +impl Queue for Queue::ver { + fn submit( + &mut self, + id: u64, + mut syncs: KVec, + in_sync_count: usize, + cmdbuf_raw: &[u8], + objects: Pin<&xarray::XArray>>, + ) -> Result { + let gpu = match (*self.dev) + .gpu + .clone() + .arc_as_any() + .downcast::() + { + Ok(gpu) => gpu, + Err(_) => { + dev_crit!(self.dev.as_ref(), "GpuManager mismatched with JobImpl!\n"); + return Err(EIO); + } + }; + + mod_dev_dbg!(self.dev, "[Submission {}] Submit job\n", id); + + if gpu.is_crashed() { + dev_err!( + self.dev.as_ref(), + "[Submission {}] GPU is crashed, cannot submit\n", + id + ); + return Err(ENODEV); + } + + let op_guard = if in_sync_count > 0 { + Some(gpu.start_op()?) + } else { + None + }; + + let mut events: [KVec>; SQ_COUNT] = + Default::default(); + + events[SQ_RENDER].push( + self.q_frag.as_ref().and_then(|a| a.wq.event_info()), + GFP_KERNEL, + )?; + events[SQ_COMPUTE].push( + self.q_comp.as_ref().and_then(|a| a.wq.event_info()), + GFP_KERNEL, + )?; + + let vm_bind = gpu.bind_vm(&self.vm)?; + let vm_slot = vm_bind.slot(); + + mod_dev_dbg!(self.dev, "[Submission {}] Creating job\n", id); + + // FIXME: I think this can violate the fence seqno ordering contract. + // If we have e.g. a render submission with no barriers and then a compute submission + // with no barriers, it's possible for the compute submission to complete first, and + // therefore its fence. Maybe we should have separate fence contexts for render + // and compute, and then do a ? (Vert+frag should be fine since there is no vert + // without frag, and frag always serializes.) + let fence: UserFence = self + .fence_ctx + .new_fence::( + 0, + JobFence::ver { + id, + pending: Default::default(), + }, + )? + .into(); + + let mut cmdbuf = Reader::new(cmdbuf_raw); + + // First, parse the headers to determine the number of compute/render + // commands. This will be used to determine when to flush stamps. + // + // We also use it to determine how many notifications the job will + // generate. We could calculate that in the second pass since we don't + // need until much later, but it's convenient to gather everything at + // the same time. + let mut nr_commands = 0; + let mut last_compute = 0; + let mut last_render = 0; + let mut nr_render = 0; + let mut nr_compute = 0; + + while !cmdbuf.is_empty() { + let header: uapi::drm_asahi_cmd_header = cmdbuf.read()?; + cmdbuf.skip(header.size as usize); + nr_commands += 1; + + match header.cmd_type as u32 { + uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_RENDER => { + last_compute = nr_commands; + nr_render += 1; + } + uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_COMPUTE => { + last_render = nr_commands; + nr_compute += 1; + } + _ => {} + } + } + + let mut job = self.entity.new_job( + 1, + QueueJob::ver { + dev: self.dev.clone(), + vm_bind, + op_guard, + sj_vtx: self + .q_vtx + .as_mut() + .map(|a| a.new_job(Fence::from_fence(&fence))), + sj_frag: self + .q_frag + .as_mut() + .map(|a| a.new_job(Fence::from_fence(&fence))), + sj_comp: self + .q_comp + .as_mut() + .map(|a| a.new_job(Fence::from_fence(&fence))), + fence, + notifier: self.inner.notifier.clone(), + + // Each render command generates 2 notifications: 1 for the + // vertex part, 1 for the fragment part. Each compute command + // generates 1 notification. Sum up to calculate the total + // notification count for the job. + notification_count: (2 * nr_render) + nr_compute, + + did_run: false, + id, + }, + )?; + + mod_dev_dbg!( + self.dev, + "[Submission {}] Adding {} in_syncs\n", + id, + in_sync_count + ); + for sync in syncs.drain(0..in_sync_count) { + if let Some(fence) = sync.fence { + job.add_dependency(fence)?; + } + } + + // Validate the number of hardware commands, ignoring software commands + let nr_hw_commands = nr_render + nr_compute; + if nr_hw_commands == 0 || nr_hw_commands > MAX_COMMANDS_PER_SUBMISSION { + cls_pr_debug!( + Errors, + "submit: Command count {} out of valid range [1, {}]\n", + nr_hw_commands, + MAX_COMMANDS_PER_SUBMISSION - 1 + ); + return Err(EINVAL); + } + + cmdbuf.rewind(); + + let mut command_index = 0; + let mut vertex_attachments: microseq::Attachments = Default::default(); + let mut fragment_attachments: microseq::Attachments = Default::default(); + let mut compute_attachments: microseq::Attachments = Default::default(); + + // Parse the full command buffer submitting as we go + while !cmdbuf.is_empty() { + let header: uapi::drm_asahi_cmd_header = cmdbuf.read()?; + let header_size = header.size as usize; + + // Pre-increment command index to match last_compute/last_render + command_index += 1; + + for (queue_idx, index) in [header.vdm_barrier, header.cdm_barrier].iter().enumerate() { + if *index == uapi::DRM_ASAHI_BARRIER_NONE as u16 { + continue; + } + if let Some(event) = events[queue_idx].get(*index as usize).ok_or_else(|| { + cls_pr_debug!(Errors, "Invalid barrier #{}: {}\n", queue_idx, index); + EINVAL + })? { + let mut alloc = gpu.alloc(); + let queue_job = match header.cmd_type as u32 { + uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_RENDER => job.get_vtx()?, + uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_COMPUTE => job.get_comp()?, + _ => return Err(EINVAL), + }; + mod_dev_dbg!(self.dev, "[Submission {}] Create Explicit Barrier\n", id); + let barrier = alloc.private.new_init( + pin_init::zeroed::(), + |_inner, _p| { + let queue_job = &queue_job; + try_init!(fw::workqueue::raw::Barrier { + tag: fw::workqueue::CommandType::Barrier, + wait_stamp: event.fw_stamp_pointer, + wait_value: event.value, + wait_slot: event.slot, + stamp_self: queue_job.event_info().value.next(), + uuid: 0xffffbbbb, + external_barrier: 0, + internal_barrier_type: 1, + padding: Default::default(), + }) + }, + )?; + mod_dev_dbg!(self.dev, "[Submission {}] Add Explicit Barrier\n", id); + queue_job.add(barrier, vm_slot)?; + } else { + assert!(*index == 0); + } + } + + match header.cmd_type as u32 { + uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_RENDER => { + let render: uapi::drm_asahi_cmd_render = cmdbuf.read_up_to(header_size)?; + + self.inner.submit_render( + &mut job, + &render, + &vertex_attachments, + &fragment_attachments, + objects, + id, + command_index == last_render, + )?; + events[SQ_RENDER].push( + Some( + job.sj_frag + .as_ref() + .expect("No frag queue?") + .job + .as_ref() + .expect("No frag job?") + .event_info(), + ), + GFP_KERNEL, + )?; + } + uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_COMPUTE => { + let compute: uapi::drm_asahi_cmd_compute = cmdbuf.read_up_to(header_size)?; + + self.inner.submit_compute( + &mut job, + &compute, + &compute_attachments, + objects, + id, + command_index == last_compute, + )?; + events[SQ_COMPUTE].push( + Some( + job.sj_comp + .as_ref() + .expect("No comp queue?") + .job + .as_ref() + .expect("No comp job?") + .event_info(), + ), + GFP_KERNEL, + )?; + } + uapi::drm_asahi_cmd_type_DRM_ASAHI_SET_VERTEX_ATTACHMENTS => { + vertex_attachments = build_attachments(&mut cmdbuf, header_size)?; + } + uapi::drm_asahi_cmd_type_DRM_ASAHI_SET_FRAGMENT_ATTACHMENTS => { + fragment_attachments = build_attachments(&mut cmdbuf, header_size)?; + } + uapi::drm_asahi_cmd_type_DRM_ASAHI_SET_COMPUTE_ATTACHMENTS => { + compute_attachments = build_attachments(&mut cmdbuf, header_size)?; + } + _ => { + cls_pr_debug!(Errors, "Unknown command type {}\n", header.cmd_type); + return Err(EINVAL); + } + } + } + + mod_dev_dbg!( + self.dev, + "Queue {}: Committing job {}\n", + self.inner.id, + job.id + ); + job.commit()?; + + mod_dev_dbg!(self.dev, "Queue {}: Arming job {}\n", self.inner.id, job.id); + let mut job = job.arm(); + let out_fence = job.fences().finished(); + mod_dev_dbg!( + self.dev, + "Queue {}: Pushing job {}\n", + self.inner.id, + job.id + ); + job.push(); + + mod_dev_dbg!( + self.dev, + "Queue {}: Adding {} out_syncs\n", + self.inner.id, + syncs.len() + ); + for mut sync in syncs { + if let Some(chain) = sync.chain_fence.take() { + sync.syncobj + .add_point(chain, &out_fence, sync.timeline_value); + } else { + sync.syncobj.replace_fence(Some(&out_fence)); + } + } + + Ok(()) + } +} + +#[versions(AGX)] +impl Drop for Queue::ver { + fn drop(&mut self) { + mod_dev_dbg!(self.dev, "[Queue {}] Dropping queue\n", self.inner.id); + } +} diff --git a/drivers/gpu/drm/asahi/queue/render.rs b/drivers/gpu/drm/asahi/queue/render.rs new file mode 100644 index 00000000000000..32273b2975b505 --- /dev/null +++ b/drivers/gpu/drm/asahi/queue/render.rs @@ -0,0 +1,1400 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#![allow(clippy::unusual_byte_groupings)] + +//! Render work queue. +//! +//! A render queue consists of two underlying WorkQueues, one for vertex and one for fragment work. +//! This module is in charge of creating all of the firmware structures required to submit 3D +//! rendering work to the GPU, based on the userspace command buffer. + +use super::common; +use crate::alloc::Allocator; +use crate::debug::*; +use crate::fw::types::*; +use crate::gpu::GpuManager; +use crate::util::*; +use crate::{ + buffer, + file, + fw, + gpu, + microseq, // +}; +use crate::{ + inner_ptr, + inner_weak_ptr, // +}; +use core::sync::atomic::Ordering; +use kernel::dma_fence::RawDmaFence; +use kernel::drm::sched::Job; +use kernel::prelude::*; +use kernel::sync::Arc; +use kernel::uapi; +use kernel::xarray; + +const DEBUG_CLASS: DebugFlags = DebugFlags::Render; + +/// Tiling/Vertex control bit to disable using more than one GPU cluster. This results in decreased +/// throughput but also less latency, which is probably desirable for light vertex loads where the +/// overhead of clustering/merging would exceed the time it takes to just run the job on one +/// cluster. +const TILECTL_DISABLE_CLUSTERING: u32 = 1u32 << 0; + +#[versions(AGX)] +impl super::QueueInner::ver { + /// Get the appropriate tiling parameters for a given userspace command buffer. + fn get_tiling_params( + cmdbuf: &uapi::drm_asahi_cmd_render, + num_clusters: u32, + ) -> Result { + let width: u32 = cmdbuf.width_px as u32; + let height: u32 = cmdbuf.height_px as u32; + let layers: u32 = cmdbuf.layers as u32; + + if layers == 0 || layers > 2048 { + cls_pr_debug!(Errors, "Layer count invalid ({})\n", layers); + return Err(EINVAL); + } + + // This is overflow safe: all these calculations are done in u32. + // At 64Kx64K max dimensions above, this is 2**32 pixels max. + // In terms of tiles that are always larger than one pixel, + // this can never overflow. Note that real actual dimensions + // are limited to 16K * 16K below anyway. + // + // Once we multiply by the layer count, then we need to check + // for overflow or use u64. + + let tile_width = 32u32; + let tile_height = 32u32; + + let utile_width = cmdbuf.utile_width_px as u32; + let utile_height = cmdbuf.utile_height_px as u32; + + match (utile_width, utile_height) { + (32, 32) | (32, 16) | (16, 16) => (), + _ => { + cls_pr_debug!( + Errors, + "uTile size invalid ({} x {})\n", + utile_width, + utile_height + ); + return Err(EINVAL); + } + }; + + let utiles_per_tile_x = tile_width / utile_width; + let utiles_per_tile_y = tile_height / utile_height; + + let utiles_per_tile = utiles_per_tile_x * utiles_per_tile_y; + + let tiles_x = width.div_ceil(tile_width); + let tiles_y = height.div_ceil(tile_height); + let tiles = tiles_x * tiles_y; + + let mtiles_x = 4u32; + let mtiles_y = 4u32; + let mtiles = mtiles_x * mtiles_y; + + let tiles_per_mtile_x = align(tiles_x.div_ceil(mtiles_x), 4); + let tiles_per_mtile_y = align(tiles_y.div_ceil(mtiles_y), 4); + let tiles_per_mtile = tiles_per_mtile_x * tiles_per_mtile_y; + + let mtile_x1 = tiles_per_mtile_x; + let mtile_x2 = 2 * tiles_per_mtile_x; + let mtile_x3 = 3 * tiles_per_mtile_x; + + let mtile_y1 = tiles_per_mtile_y; + let mtile_y2 = 2 * tiles_per_mtile_y; + let mtile_y3 = 3 * tiles_per_mtile_y; + + let rgn_entry_size = 5; + // Macrotile stride in 32-bit words + let rgn_size = align(rgn_entry_size * tiles_per_mtile * utiles_per_tile, 4) / 4; + let tilemap_size = (4 * rgn_size * mtiles) as usize * layers as usize; + + let tpc_entry_size = 8; + // TPC stride in 32-bit words + let tpc_mtile_stride = tpc_entry_size * utiles_per_tile * tiles_per_mtile / 4; + let tpc_size = + (4 * tpc_mtile_stride * mtiles) as usize * layers as usize * num_clusters as usize; + + // No idea where this comes from, but it fits what macOS does... + // GUESS: Number of 32K heap blocks to fit a 5-byte region header/pointer per tile? + // That would make a ton of sense... + let meta1_layer_stride = if num_clusters > 1 { + (align(tiles_x, 2) * align(tiles_y, 4) * utiles_per_tile).div_ceil(0x1980) + } else { + 0 + }; + + let mut min_tvb_blocks = align((tiles_x * tiles_y).div_ceil(128), 8); + + if num_clusters > 1 { + min_tvb_blocks = min_tvb_blocks.max(7 + 2 * layers); + } + + Ok(buffer::TileInfo { + tiles_x, + tiles_y, + tiles, + utile_width, + utile_height, + //mtiles_x, + //mtiles_y, + tiles_per_mtile_x, + tiles_per_mtile_y, + //tiles_per_mtile, + utiles_per_mtile_x: tiles_per_mtile_x * utiles_per_tile_x, + utiles_per_mtile_y: tiles_per_mtile_y * utiles_per_tile_y, + //utiles_per_mtile: tiles_per_mtile * utiles_per_tile, + tilemap_size, + tpc_size, + meta1_layer_stride, + #[ver(G < G14X)] + meta1_blocks: meta1_layer_stride * (cmdbuf.layers as u32), + #[ver(G >= G14X)] + meta1_blocks: meta1_layer_stride, + layermeta_size: if layers > 1 { 0x100 } else { 0 }, + min_tvb_blocks: min_tvb_blocks as usize, + params: fw::vertex::raw::TilingParameters { + rgn_size, + unk_4: 0x88, + ppp_ctrl: cmdbuf.ppp_ctrl, + x_max: (width - 1) as u16, + y_max: (height - 1) as u16, + te_screen: ((tiles_y - 1) << 12) | (tiles_x - 1), + te_mtile1: mtile_x3 | (mtile_x2 << 9) | (mtile_x1 << 18), + te_mtile2: mtile_y3 | (mtile_y2 << 9) | (mtile_y1 << 18), + tiles_per_mtile, + tpc_stride: tpc_mtile_stride, + unk_24: 0x100, + unk_28: if layers > 1 { + 0xe000 | (layers - 1) + } else { + 0x8000 + }, + helper_cfg: cmdbuf.vertex_helper.cfg, + __pad: Default::default(), + }, + }) + } + + /// Submit work to a render queue. + pub(super) fn submit_render( + &self, + job: &mut Job, + cmdbuf: &uapi::drm_asahi_cmd_render, + vertex_attachments: µseq::Attachments, + fragment_attachments: µseq::Attachments, + objects: Pin<&xarray::XArray>>, + id: u64, + flush_stamps: bool, + ) -> Result { + mod_dev_dbg!(self.dev, "[Submission {}] Render!\n", id); + + if cmdbuf.flags + & !(uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_VERTEX_SCRATCH + | uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_PROCESS_EMPTY_TILES + | uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_NO_VERTEX_CLUSTERING + | uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_DBIAS_IS_INT) as u32 + != 0 + { + cls_pr_debug!(Errors, "Invalid flags ({:#x})\n", cmdbuf.flags); + return Err(EINVAL); + } + + if cmdbuf.width_px == 0 + || cmdbuf.height_px == 0 + || cmdbuf.width_px > 16384 + || cmdbuf.height_px > 16384 + { + cls_pr_debug!( + Errors, + "Invalid dimensions ({}x{})\n", + cmdbuf.width_px, + cmdbuf.height_px + ); + return Err(EINVAL); + } + + let mut vtx_user_timestamps: fw::job::UserTimestamps = Default::default(); + let mut frg_user_timestamps: fw::job::UserTimestamps = Default::default(); + + vtx_user_timestamps.start = common::get_timestamp_object(objects, cmdbuf.ts_vtx.start)?; + vtx_user_timestamps.end = common::get_timestamp_object(objects, cmdbuf.ts_vtx.end)?; + frg_user_timestamps.start = common::get_timestamp_object(objects, cmdbuf.ts_frag.start)?; + frg_user_timestamps.end = common::get_timestamp_object(objects, cmdbuf.ts_frag.end)?; + + let gpu = match (*self.dev) + .gpu + .as_any() + .downcast_ref::() + { + Some(gpu) => gpu, + None => { + dev_crit!(self.dev.as_ref(), "GpuManager mismatched with Queue!\n"); + return Err(EIO); + } + }; + + let nclusters = gpu.get_dyncfg().id.num_clusters; + + // Can be set to false to disable clustering (for simpler jobs), but then the + // core masks below should be adjusted to cover a single rolling cluster. + let mut clustering = nclusters > 1; + + if debug_enabled(debug::DebugFlags::DisableClustering) + || cmdbuf.flags + & uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_NO_VERTEX_CLUSTERING as u32 + != 0 + { + clustering = false; + } + + #[ver(G != G14)] + let tiling_control = { + let render_cfg = gpu.get_cfg().render; + let mut tiling_control = render_cfg.tiling_control; + + if !clustering { + tiling_control |= TILECTL_DISABLE_CLUSTERING; + } + tiling_control + }; + + let mut alloc = gpu.alloc(); + let kalloc = &mut *alloc; + + // This sequence number increases per new client/VM? assigned to some slot, + // but it's unclear *which* slot... + let slot_client_seq: u8 = (self.id & 0xff) as u8; + + let tile_info = Self::get_tiling_params(&cmdbuf, if clustering { nclusters } else { 1 })?; + + let buffer = &self.buffer; + let notifier = self.notifier.clone(); + + let tvb_autogrown = buffer.auto_grow()?; + if tvb_autogrown { + let new_size = buffer.block_count() as usize; + cls_dev_dbg!( + TVBStats, + &self.dev, + "[Submission {}] TVB grew to {} bytes ({} blocks) due to overflows\n", + id, + new_size * buffer::BLOCK_SIZE, + new_size, + ); + } + + let tvb_grown = buffer.ensure_blocks(tile_info.min_tvb_blocks)?; + if tvb_grown { + cls_dev_dbg!( + TVBStats, + &self.dev, + "[Submission {}] TVB grew to {} bytes ({} blocks) due to dimensions ({}x{})\n", + id, + tile_info.min_tvb_blocks * buffer::BLOCK_SIZE, + tile_info.min_tvb_blocks, + cmdbuf.width_px, + cmdbuf.height_px + ); + } + + let scene = Arc::new(buffer.new_scene(kalloc, &tile_info)?, GFP_KERNEL)?; + + let vm_bind = job.vm_bind.clone(); + + mod_dev_dbg!( + self.dev, + "[Submission {}] VM slot = {}\n", + id, + vm_bind.slot() + ); + + let ev_vtx = job.get_vtx()?.event_info(); + let ev_frag = job.get_frag()?.event_info(); + + mod_dev_dbg!( + self.dev, + "[Submission {}] Vert event #{} -> {:#x?}\n", + id, + ev_vtx.slot, + ev_vtx.value.next(), + ); + mod_dev_dbg!( + self.dev, + "[Submission {}] Frag event #{} -> {:#x?}\n", + id, + ev_frag.slot, + ev_frag.value.next(), + ); + + let uuid_3d = 0; + let uuid_ta = 0; + + mod_dev_dbg!( + self.dev, + "[Submission {}] Vert UUID = {:#x?}\n", + id, + uuid_ta + ); + mod_dev_dbg!( + self.dev, + "[Submission {}] Frag UUID = {:#x?}\n", + id, + uuid_3d + ); + + let fence = job.fence.clone(); + let frag_job = job.get_frag()?; + + mod_dev_dbg!(self.dev, "[Submission {}] Create Barrier\n", id); + let barrier = kalloc.private.new_init( + pin_init::zeroed::(), + |_inner, _p| { + try_init!(fw::workqueue::raw::Barrier { + tag: fw::workqueue::CommandType::Barrier, + wait_stamp: ev_vtx.fw_stamp_pointer, + wait_value: ev_vtx.value.next(), + wait_slot: ev_vtx.slot, + stamp_self: ev_frag.value.next(), + uuid: uuid_3d, + external_barrier: 0, + internal_barrier_type: 0, + padding: Default::default(), + }) + }, + )?; + + mod_dev_dbg!(self.dev, "[Submission {}] Add Barrier\n", id); + frag_job.add(barrier, vm_bind.slot())?; + + let timestamps = Arc::new( + kalloc.shared.new_default::()?, + GFP_KERNEL, + )?; + + let unk1 = false; + + let mut tile_config: u64 = 0; + if !unk1 { + tile_config |= 0x280; + } + if cmdbuf.layers > 1 { + tile_config |= 1; + } + if cmdbuf.flags & uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_PROCESS_EMPTY_TILES as u32 + != 0 + { + tile_config |= 0x10000; + } + + let samples_log2 = match cmdbuf.samples { + 1 => 0, + 2 => 1, + 4 => 2, + _ => { + cls_pr_debug!(Errors, "Invalid sample count {}\n", cmdbuf.samples); + return Err(EINVAL); + } + }; + + let utile_config = ((tile_info.utile_width / 16) << 12) + | ((tile_info.utile_height / 16) << 14) + | samples_log2; + + // Calculate the number of 2KiB blocks to allocate per utile. This is + // just a bit of dimensional analysis. + let pixels_per_utile: u32 = + (cmdbuf.utile_width_px as u32) * (cmdbuf.utile_height_px as u32); + let samples_per_utile: u32 = pixels_per_utile << samples_log2; + let utile_size_bytes: u32 = (cmdbuf.sample_size_B as u32) * samples_per_utile; + let block_size_bytes: u32 = 2048; + let blocks_per_utile: u32 = utile_size_bytes.div_ceil(block_size_bytes); + + #[ver(G >= G14X)] + let frg_tilecfg = 0x0000000_00036011 + | (((tile_info.tiles_x - 1) as u64) << 44) + | (((tile_info.tiles_y - 1) as u64) << 53) + | (if unk1 { 0 } else { 0x20_00000000 }) + | (if cmdbuf.layers > 1 { 0x1_00000000 } else { 0 }) + | ((utile_config as u64 & 0xf000) << 28); + + // TODO: check + #[ver(V >= V13_0B4)] + let count_frag = self.counter.fetch_add(2, Ordering::Relaxed); + #[ver(V >= V13_0B4)] + let count_vtx = count_frag + 1; + + // Unknowns handling + + #[ver(G >= G14)] + let g14_unk = 0x4040404; + #[ver(G < G14)] + let g14_unk = 0; + #[ver(G < G14X)] + let frg_unk_140 = 0x8c60; + let frg_unk_158 = 0x1c; + #[ver(G >= G14)] + let load_bgobjvals = cmdbuf.isp_bgobjvals as u64; + #[ver(G < G14)] + let load_bgobjvals = cmdbuf.isp_bgobjvals as u64 | 0x400; + let reload_zlsctrl = cmdbuf.zls_ctrl; + let iogpu_unk54 = 0x3a0012006b0003; + let iogpu_unk56 = 1; + #[ver(G < G14)] + let tiling_control_2 = 0; + #[ver(G >= G14X)] + let tiling_control_2 = 4; + #[ver(G >= G14X)] + let vtx_unk_f0 = 0x1c; + #[ver(G < G14)] + let vtx_unk_f0 = 0x1c + (align(tile_info.meta1_blocks, 4) as u64); + let vtx_unk_118 = 0x1c; + + // DRM_ASAHI_RENDER_DBIAS_IS_INT chosen to match hardware bit. + let isp_ctl = 0xc000u32 + | (cmdbuf.flags & uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_DBIAS_IS_INT as u32); + + // Always allow preemption at the UAPI level + let no_preemption = false; + + mod_dev_dbg!(self.dev, "[Submission {}] Create Frag\n", id); + let frag = GpuObject::new_init_prealloc( + kalloc.gpu_ro.alloc_object()?, + |ptr: GpuWeakPointer| { + let scene = scene.clone(); + let notifier = notifier.clone(); + let vm_bind = vm_bind.clone(); + let timestamps = timestamps.clone(); + let private = &mut kalloc.private; + try_init!(fw::fragment::RunFragment::ver { + micro_seq: { + let mut builder = microseq::Builder::new(); + + let stats = inner_weak_ptr!( + gpu.initdata.runtime_pointers.stats.frag.weak_pointer(), + stats + ); + + let start_frag = builder.add(microseq::StartFragment::ver { + header: microseq::op::StartFragment::HEADER, + #[ver(G < G14X)] + job_params2: Some(inner_weak_ptr!(ptr, job_params2)), + #[ver(G < G14X)] + job_params1: Some(inner_weak_ptr!(ptr, job_params1)), + #[ver(G >= G14X)] + job_params1: None, + #[ver(G >= G14X)] + job_params2: None, + #[ver(G >= G14X)] + registers: inner_weak_ptr!(ptr, registers), + scene: scene.gpu_pointer(), + stats, + busy_flag: inner_weak_ptr!(ptr, busy_flag), + tvb_overflow_count: inner_weak_ptr!(ptr, tvb_overflow_count), + unk_pointer: inner_weak_ptr!(ptr, unk_pointee), + work_queue: ev_frag.info_ptr, + work_item: ptr, + vm_slot: vm_bind.slot(), + unk_50: 0x1, // fixed + event_generation: self.id as u32, + buffer_slot: scene.slot(), + sync_grow: 0, + event_seq: U64(ev_frag.event_seq), + unk_68: 0, + unk_758_flag: inner_weak_ptr!(ptr, unk_758_flag), + unk_job_buf: inner_weak_ptr!(ptr, unk_buf_0), + #[ver(V >= V13_3)] + unk_7c_0: U64(0), + unk_7c: 0, + unk_80: 0, + unk_84: unk1.into(), + uuid: uuid_3d, + attachments: *fragment_attachments, + padding: 0, + #[ver(V >= V13_0B4)] + counter: U64(count_frag), + #[ver(V >= V13_0B4)] + notifier_buf: inner_weak_ptr!(notifier.weak_pointer(), state.unk_buf), + })?; + + if frg_user_timestamps.any() { + builder.add(microseq::Timestamp::ver { + header: microseq::op::Timestamp::new(true), + command_time: inner_weak_ptr!(ptr, command_time), + ts_pointers: inner_weak_ptr!(ptr, timestamp_pointers), + update_ts: inner_weak_ptr!(ptr, timestamp_pointers.start_addr), + work_queue: ev_frag.info_ptr, + user_ts_pointers: inner_weak_ptr!(ptr, user_timestamp_pointers), + #[ver(V >= V13_0B4)] + unk_ts: inner_weak_ptr!(ptr, unk_ts), + uuid: uuid_3d, + unk_30_padding: 0, + })?; + } + + #[ver(G < G14X)] + builder.add(microseq::WaitForIdle { + header: microseq::op::WaitForIdle::new(microseq::Pipe::Fragment), + })?; + #[ver(G >= G14X)] + builder.add(microseq::WaitForIdle2 { + header: microseq::op::WaitForIdle2::HEADER, + })?; + + if frg_user_timestamps.any() { + builder.add(microseq::Timestamp::ver { + header: microseq::op::Timestamp::new(false), + command_time: inner_weak_ptr!(ptr, command_time), + ts_pointers: inner_weak_ptr!(ptr, timestamp_pointers), + update_ts: inner_weak_ptr!(ptr, timestamp_pointers.end_addr), + work_queue: ev_frag.info_ptr, + user_ts_pointers: inner_weak_ptr!(ptr, user_timestamp_pointers), + #[ver(V >= V13_0B4)] + unk_ts: inner_weak_ptr!(ptr, unk_ts), + uuid: uuid_3d, + unk_30_padding: 0, + })?; + } + + let off = builder.offset_to(start_frag); + builder.add(microseq::FinalizeFragment::ver { + header: microseq::op::FinalizeFragment::HEADER, + uuid: uuid_3d, + unk_8: 0, + fw_stamp: ev_frag.fw_stamp_pointer, + stamp_value: ev_frag.value.next(), + unk_18: 0, + scene: scene.weak_pointer(), + buffer: scene.weak_buffer_pointer(), + unk_2c: U64(1), + stats, + unk_pointer: inner_weak_ptr!(ptr, unk_pointee), + busy_flag: inner_weak_ptr!(ptr, busy_flag), + work_queue: ev_frag.info_ptr, + work_item: ptr, + vm_slot: vm_bind.slot(), + unk_60: 0, + unk_758_flag: inner_weak_ptr!(ptr, unk_758_flag), + #[ver(V >= V13_3)] + unk_6c_0: U64(0), + unk_6c: U64(0), + unk_74: U64(0), + unk_7c: U64(0), + unk_84: U64(0), + unk_8c: U64(0), + #[ver(G == G14 && V < V13_0B4)] + unk_8c_g14: U64(0), + restart_branch_offset: off, + has_attachments: (fragment_attachments.count > 0) as u32, + #[ver(V >= V13_0B4)] + unk_9c: Default::default(), + })?; + + builder.add(microseq::RetireStamp { + header: microseq::op::RetireStamp::HEADER, + })?; + + builder.build(private)? + }, + notifier, + scene, + vm_bind, + aux_fb: self.ualloc.lock().array_empty_tagged(0x8000, b"AXFB")?, + timestamps, + user_timestamps: frg_user_timestamps, + }) + }, + |inner, _ptr| { + let vm_slot = vm_bind.slot(); + let aux_fb_info = fw::fragment::raw::AuxFBInfo::ver { + isp_ctl: isp_ctl, + unk2: 0, + width: cmdbuf.width_px as u32, + height: cmdbuf.height_px as u32, + #[ver(V >= V13_0B4)] + unk3: U64(0x100000), + }; + + try_init!(fw::fragment::raw::RunFragment::ver { + tag: fw::workqueue::CommandType::RunFragment, + #[ver(V >= V13_0B4)] + counter: U64(count_frag), + vm_slot, + unk_8: 0, + microsequence: inner.micro_seq.gpu_pointer(), + microsequence_size: inner.micro_seq.len() as u32, + notifier: inner.notifier.gpu_pointer(), + buffer: inner.scene.buffer_pointer(), + scene: inner.scene.gpu_pointer(), + unk_buffer_buf: inner.scene.kernel_buffer_pointer(), + tvb_tilemap: inner.scene.tvb_tilemap_pointer(), + ppp_multisamplectl: U64(cmdbuf.ppp_multisamplectl), + samples: cmdbuf.samples as u32, + tiles_per_mtile_y: tile_info.tiles_per_mtile_y as u16, + tiles_per_mtile_x: tile_info.tiles_per_mtile_x as u16, + unk_50: U64(0), + unk_58: U64(0), + isp_merge_upper_x: F32::from_bits(cmdbuf.isp_merge_upper_x), + isp_merge_upper_y: F32::from_bits(cmdbuf.isp_merge_upper_y), + unk_68: U64(0), + tile_count: U64(tile_info.tiles as u64), + #[ver(G < G14X)] + job_params1 <- try_init!(fw::fragment::raw::JobParameters1::ver { + utile_config, + unk_4: 0, + bg: fw::fragment::raw::BackgroundProgram { + rsrc_spec: U64(cmdbuf.bg.rsrc_spec as u64), + address: U64(cmdbuf.bg.usc as u64), + }, + ppp_multisamplectl: U64(cmdbuf.ppp_multisamplectl), + isp_scissor_base: U64(cmdbuf.isp_scissor_base), + isp_dbias_base: U64(cmdbuf.isp_dbias_base), + isp_oclqry_base: U64(cmdbuf.isp_oclqry_base), + aux_fb_info, + isp_zls_pixels: U64(cmdbuf.isp_zls_pixels as u64), + zls_ctrl: U64(cmdbuf.zls_ctrl), + #[ver(G >= G14)] + unk_58_g14_0: U64(g14_unk), + #[ver(G >= G14)] + unk_58_g14_8: U64(0), + z_load: U64(cmdbuf.depth.base), + z_store: U64(cmdbuf.depth.base), + s_load: U64(cmdbuf.stencil.base), + s_store: U64(cmdbuf.stencil.base), + #[ver(G >= G14)] + unk_68_g14_0: Default::default(), + z_load_stride: U64(cmdbuf.depth.stride as u64), + z_store_stride: U64(cmdbuf.depth.stride as u64), + s_load_stride: U64(cmdbuf.stencil.stride as u64), + s_store_stride: U64(cmdbuf.stencil.stride as u64), + z_load_comp: U64(cmdbuf.depth.comp_base), + z_load_comp_stride: U64(cmdbuf.depth.comp_stride as u64), + z_store_comp: U64(cmdbuf.depth.comp_base), + z_store_comp_stride: U64(cmdbuf.depth.comp_stride as u64), + s_load_comp: U64(cmdbuf.stencil.comp_base), + s_load_comp_stride: U64(cmdbuf.stencil.comp_stride as u64), + s_store_comp: U64(cmdbuf.stencil.comp_base), + s_store_comp_stride: U64(cmdbuf.stencil.comp_stride as u64), + tvb_tilemap: inner.scene.tvb_tilemap_pointer(), + tvb_layermeta: inner.scene.tvb_layermeta_pointer(), + mtile_stride_dwords: U64((4 * tile_info.params.rgn_size as u64) << 24), + tvb_heapmeta: inner.scene.tvb_heapmeta_pointer(), + tile_config: U64(tile_config), + aux_fb: inner.aux_fb.gpu_pointer(), + unk_108: Default::default(), + usc_exec_base_isp: U64(self.usc_exec_base), + unk_140: U64(frg_unk_140), + helper_program: cmdbuf.fragment_helper.binary, + unk_14c: 0, + helper_arg: U64(cmdbuf.fragment_helper.data), + unk_158: U64(frg_unk_158), + unk_160: U64(0), + __pad: Default::default(), + #[ver(V < V13_0B4)] + __pad1: Default::default(), + }), + #[ver(G < G14X)] + job_params2 <- try_init!(fw::fragment::raw::JobParameters2 { + eot_rsrc_spec: cmdbuf.eot.rsrc_spec, + eot_usc: cmdbuf.eot.usc, + unk_8: 0x0, + unk_c: 0x0, + isp_merge_upper_x: F32::from_bits(cmdbuf.isp_merge_upper_x), + isp_merge_upper_y: F32::from_bits(cmdbuf.isp_merge_upper_y), + unk_18: U64(0x0), + utiles_per_mtile_y: tile_info.utiles_per_mtile_y as u16, + utiles_per_mtile_x: tile_info.utiles_per_mtile_x as u16, + unk_24: 0x0, + tile_counts: ((tile_info.tiles_y - 1) << 12) | (tile_info.tiles_x - 1), + tib_blocks: blocks_per_utile, + isp_bgobjdepth: cmdbuf.isp_bgobjdepth, + // TODO: does this flag need to be exposed to userspace? + isp_bgobjvals: load_bgobjvals as u32, + unk_38: 0x0, + unk_3c: 0x1, + helper_cfg: cmdbuf.fragment_helper.cfg, + __pad: Default::default(), + }), + #[ver(G >= G14X)] + registers: fw::job::raw::RegisterArray::new( + inner_weak_ptr!(_ptr, registers.registers), + |r| { + r.add(0x1739, 1); + r.add(0x10009, utile_config.into()); + r.add(0x15379, cmdbuf.eot.rsrc_spec.into()); + r.add(0x15381, cmdbuf.eot.usc.into()); + r.add(0x15369, cmdbuf.bg.rsrc_spec.into()); + r.add(0x15371, cmdbuf.bg.usc.into()); + r.add(0x15131, cmdbuf.isp_merge_upper_x.into()); + r.add(0x15139, cmdbuf.isp_merge_upper_y.into()); + r.add(0x100a1, 0); + r.add(0x15069, 0); + r.add(0x15071, 0); // pointer + r.add(0x16058, 0); + r.add(0x10019, cmdbuf.ppp_multisamplectl); + let isp_mtile_size = (tile_info.utiles_per_mtile_y + | (tile_info.utiles_per_mtile_x << 16)) + .into(); + r.add(0x100b1, isp_mtile_size); // ISP_MTILE_SIZE + r.add(0x16030, isp_mtile_size); // ISP_MTILE_SIZE + r.add( + 0x100d9, + (((tile_info.tiles_y - 1) << 12) | (tile_info.tiles_x - 1)).into(), + ); // TE_SCREEN + r.add(0x16098, inner.scene.tvb_heapmeta_pointer().into()); + r.add(0x15109, cmdbuf.isp_scissor_base); // ISP_SCISSOR_BASE + r.add(0x15101, cmdbuf.isp_dbias_base); // ISP_DBIAS_BASE + r.add(0x15021, isp_ctl.into()); // aux_fb_info.unk_1 + r.add( + 0x15211, + ((cmdbuf.height_px as u64) << 32) | cmdbuf.width_px as u64, + ); // aux_fb_info.{width, heigh + r.add(0x15049, 0x100000); // s2.aux_fb_info.unk3 + r.add(0x10051, blocks_per_utile.into()); // s1.unk_2c + r.add(0x15321, cmdbuf.isp_zls_pixels.into()); // ISP_ZLS_PIXELS + r.add(0x15301, cmdbuf.isp_bgobjdepth.into()); // ISP_BGOBJDEPTH + r.add(0x15309, load_bgobjvals); // ISP_BGOBJVALS + r.add(0x15311, cmdbuf.isp_oclqry_base); // ISP_OCLQRY_BASE + r.add(0x15319, cmdbuf.zls_ctrl); // ISP_ZLSCTL + r.add(0x15349, g14_unk); // s2.unk_58_g14_0 + r.add(0x15351, 0); // s2.unk_58_g14_8 + r.add(0x15329, cmdbuf.depth.base); // ISP_ZLOAD_BASE + r.add(0x15331, cmdbuf.depth.base); // ISP_ZSTORE_BASE + r.add(0x15339, cmdbuf.stencil.base); // ISP_STENCIL_LOAD_BASE + r.add(0x15341, cmdbuf.stencil.base); // ISP_STENCIL_STORE_BASE + r.add(0x15231, 0); + r.add(0x15221, 0); + r.add(0x15239, 0); + r.add(0x15229, 0); + r.add(0x15401, cmdbuf.depth.stride as u64); // load + r.add(0x15421, cmdbuf.depth.stride as u64); // store + r.add(0x15409, cmdbuf.stencil.stride as u64); // load + r.add(0x15429, cmdbuf.stencil.stride as u64); + r.add(0x153c1, cmdbuf.depth.comp_base); // load + r.add(0x15411, cmdbuf.depth.comp_stride as u64); // load + r.add(0x153c9, cmdbuf.depth.comp_base); // store + r.add(0x15431, cmdbuf.depth.comp_stride as u64); // store + r.add(0x153d1, cmdbuf.stencil.comp_base); // load + r.add(0x15419, cmdbuf.stencil.comp_stride as u64); // load + r.add(0x153d9, cmdbuf.stencil.comp_base); // store + r.add(0x15439, cmdbuf.stencil.comp_stride as u64); // store + r.add(0x16429, inner.scene.tvb_tilemap_pointer().into()); + r.add(0x16060, inner.scene.tvb_layermeta_pointer().into()); + r.add(0x16431, (4 * tile_info.params.rgn_size as u64) << 24); // ISP_RGN? + r.add(0x10039, tile_config); // tile_config ISP_CTL? + r.add(0x16451, 0x0); // ISP_RENDER_ORIGIN + r.add(0x11821, cmdbuf.fragment_helper.binary.into()); + r.add(0x11829, cmdbuf.fragment_helper.data); + r.add(0x11f79, cmdbuf.fragment_helper.cfg.into()); + r.add(0x15359, 0); + r.add(0x10069, self.usc_exec_base); // frag; USC_EXEC_BASE_ISP + r.add(0x16020, 0); + r.add(0x16461, inner.aux_fb.gpu_pointer().into()); + r.add(0x16090, inner.aux_fb.gpu_pointer().into()); + r.add(0x120a1, frg_unk_158); + r.add(0x160a8, 0); + r.add(0x16068, frg_tilecfg); + r.add(0x160b8, 0x0); + /* + r.add(0x10201, 0x100); // Some kind of counter?? Does this matter? + r.add(0x10428, 0x100); // Some kind of counter?? Does this matter? + r.add(0x1c838, 1); // ? + r.add(0x1ca28, 0x1502960f00); // ?? + r.add(0x1731, 0x1); // ?? + */ + } + ), + job_params3 <- try_init!(fw::fragment::raw::JobParameters3::ver { + isp_dbias_base: fw::fragment::raw::ArrayAddr { + ptr: U64(cmdbuf.isp_dbias_base), + unk_padding: U64(0), + }, + isp_scissor_base: fw::fragment::raw::ArrayAddr { + ptr: U64(cmdbuf.isp_scissor_base), + unk_padding: U64(0), + }, + isp_oclqry_base: U64(cmdbuf.isp_oclqry_base), + unk_118: U64(0x0), + unk_120: Default::default(), + unk_partial_bg: fw::fragment::raw::BackgroundProgram { + rsrc_spec: U64(cmdbuf.partial_bg.rsrc_spec as u64), + address: U64(cmdbuf.partial_bg.usc as u64), + }, + unk_258: U64(0), + unk_260: U64(0), + unk_268: U64(0), + unk_270: U64(0), + partial_bg: fw::fragment::raw::BackgroundProgram { + rsrc_spec: U64(cmdbuf.partial_bg.rsrc_spec as u64), + address: U64(cmdbuf.partial_bg.usc as u64), + }, + zls_ctrl: U64(reload_zlsctrl), + unk_290: U64(g14_unk), + z_load: U64(cmdbuf.depth.base), + z_partial_stride: U64(cmdbuf.depth.stride as u64), + z_partial_comp_stride: U64(cmdbuf.depth.comp_stride as u64), + z_store: U64(cmdbuf.depth.base), + z_partial: U64(cmdbuf.depth.base), + z_partial_comp: U64(cmdbuf.depth.comp_base), + s_load: U64(cmdbuf.stencil.base), + s_partial_stride: U64(cmdbuf.stencil.stride as u64), + s_partial_comp_stride: U64(cmdbuf.stencil.comp_stride as u64), + s_store: U64(cmdbuf.stencil.base), + s_partial: U64(cmdbuf.stencil.base), + s_partial_comp: U64(cmdbuf.stencil.comp_base), + unk_2f8: Default::default(), + tib_blocks: blocks_per_utile, + unk_30c: 0x0, + aux_fb_info, + tile_config: U64(tile_config), + unk_328_padding: Default::default(), + unk_partial_eot: fw::fragment::raw::EotProgram::new( + cmdbuf.partial_eot.rsrc_spec, + cmdbuf.partial_eot.usc + ), + partial_eot: fw::fragment::raw::EotProgram::new( + cmdbuf.partial_eot.rsrc_spec, + cmdbuf.partial_eot.usc + ), + isp_bgobjdepth: cmdbuf.isp_bgobjdepth, + isp_bgobjvals: cmdbuf.isp_bgobjvals, + sample_size: cmdbuf.sample_size_B as u32, + unk_37c: 0x0, + unk_380: U64(0x0), + unk_388: U64(0x0), + #[ver(V >= V13_0B4)] + unk_390_0: U64(0x0), + isp_zls_pixels: U64(cmdbuf.isp_zls_pixels as u64), + }), + unk_758_flag: 0, + unk_75c_flag: 0, + unk_buf: Default::default(), + busy_flag: 0, + tvb_overflow_count: 0, + unk_878: 0, + encoder_params <- try_init!(fw::job::raw::EncoderParams { + // Maybe set when reloading z/s? + unk_8: 0, + sync_grow: 0, + unk_10: 0x0, // fixed + encoder_id: 0, + unk_18: 0x0, // fixed + unk_mask: 0xffffffffu32, + sampler_array: U64(cmdbuf.sampler_heap), + sampler_count: cmdbuf.sampler_count as u32, + sampler_max: (cmdbuf.sampler_count as u32) + 1, + }), + process_empty_tiles: (cmdbuf.flags + & uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_PROCESS_EMPTY_TILES as u32 + != 0) as u32, + // TODO: needs to be investigated + no_clear_pipeline_textures: 1, + // TODO: needs to be investigated + msaa_zs: 0, + unk_pointee: 0, + #[ver(V >= V13_3)] + unk_v13_3: 0, + meta <- try_init!(fw::job::raw::JobMeta { + unk_0: 0, + unk_2: 0, + no_preemption: no_preemption as u8, + stamp: ev_frag.stamp_pointer, + fw_stamp: ev_frag.fw_stamp_pointer, + stamp_value: ev_frag.value.next(), + stamp_slot: ev_frag.slot, + evctl_index: 0, // fixed + flush_stamps: flush_stamps as u32, + uuid: uuid_3d, + event_seq: ev_frag.event_seq as u32, + }), + unk_after_meta: unk1.into(), + unk_buf_0: U64(0), + unk_buf_8: U64(0), + #[ver(G < G14X)] + unk_buf_10: U64(1), + #[ver(G >= G14X)] + unk_buf_10: U64(0), + command_time: U64(0), + timestamp_pointers <- try_init!(fw::job::raw::TimestampPointers { + start_addr: Some(inner_ptr!(inner.timestamps.gpu_pointer(), frag.start)), + end_addr: Some(inner_ptr!(inner.timestamps.gpu_pointer(), frag.end)), + }), + user_timestamp_pointers: inner.user_timestamps.pointers()?, + client_sequence: slot_client_seq, + pad_925: Default::default(), + unk_928: 0, + unk_92c: 0, + #[ver(V >= V13_0B4)] + unk_ts: U64(0), + #[ver(V >= V13_0B4)] + unk_92d_8: Default::default(), + }) + }, + )?; + + mod_dev_dbg!(self.dev, "[Submission {}] Add Frag\n", id); + fence.add_command(); + + frag_job.add_cb(frag, vm_bind.slot(), move |error| { + if let Some(err) = error { + fence.set_error(err.into()); + } + + fence.command_complete(); + })?; + + let fence = job.fence.clone(); + let vtx_job = job.get_vtx()?; + + if scene.rebind() || tvb_grown || tvb_autogrown { + mod_dev_dbg!(self.dev, "[Submission {}] Create Bind Buffer\n", id); + let bind_buffer = kalloc.private.new_init( + { + let scene = scene.clone(); + try_init!(fw::buffer::InitBuffer::ver { scene }) + }, + |inner, _ptr| { + let vm_slot = vm_bind.slot(); + try_init!(fw::buffer::raw::InitBuffer::ver { + tag: fw::workqueue::CommandType::InitBuffer, + vm_slot, + buffer_slot: inner.scene.slot(), + unk_c: 0, + block_count: buffer.block_count(), + buffer: inner.scene.buffer_pointer(), + stamp_value: ev_vtx.value.next(), + }) + }, + )?; + + mod_dev_dbg!(self.dev, "[Submission {}] Add Bind Buffer\n", id); + vtx_job.add(bind_buffer, vm_bind.slot())?; + } + + mod_dev_dbg!(self.dev, "[Submission {}] Create Vertex\n", id); + let vtx = GpuObject::new_init_prealloc( + kalloc.gpu_ro.alloc_object()?, + |ptr: GpuWeakPointer| { + let scene = scene.clone(); + let vm_bind = vm_bind.clone(); + let timestamps = timestamps.clone(); + let private = &mut kalloc.private; + try_init!(fw::vertex::RunVertex::ver { + micro_seq: { + let mut builder = microseq::Builder::new(); + + let stats = inner_weak_ptr!( + gpu.initdata.runtime_pointers.stats.vtx.weak_pointer(), + stats + ); + + let start_vtx = builder.add(microseq::StartVertex::ver { + header: microseq::op::StartVertex::HEADER, + #[ver(G < G14X)] + tiling_params: Some(inner_weak_ptr!(ptr, tiling_params)), + #[ver(G < G14X)] + job_params1: Some(inner_weak_ptr!(ptr, job_params1)), + #[ver(G >= G14X)] + tiling_params: None, + #[ver(G >= G14X)] + job_params1: None, + #[ver(G >= G14X)] + registers: inner_weak_ptr!(ptr, registers), + buffer: scene.weak_buffer_pointer(), + scene: scene.weak_pointer(), + stats, + work_queue: ev_vtx.info_ptr, + vm_slot: vm_bind.slot(), + unk_38: 1, // fixed + event_generation: self.id as u32, + buffer_slot: scene.slot(), + unk_44: 0, + event_seq: U64(ev_vtx.event_seq), + unk_50: 0, + unk_pointer: inner_weak_ptr!(ptr, unk_pointee), + unk_job_buf: inner_weak_ptr!(ptr, unk_buf_0), + unk_64: 0x0, // fixed + unk_68: unk1.into(), + uuid: uuid_ta, + attachments: *vertex_attachments, + padding: 0, + #[ver(V >= V13_0B4)] + counter: U64(count_vtx), + #[ver(V >= V13_0B4)] + notifier_buf: inner_weak_ptr!(notifier.weak_pointer(), state.unk_buf), + #[ver(V < V13_0B4)] + unk_178: 0x0, // padding? + #[ver(V >= V13_0B4)] + unk_178: (!clustering) as u32, + })?; + + if vtx_user_timestamps.any() { + builder.add(microseq::Timestamp::ver { + header: microseq::op::Timestamp::new(true), + command_time: inner_weak_ptr!(ptr, command_time), + ts_pointers: inner_weak_ptr!(ptr, timestamp_pointers), + update_ts: inner_weak_ptr!(ptr, timestamp_pointers.start_addr), + work_queue: ev_vtx.info_ptr, + user_ts_pointers: inner_weak_ptr!(ptr, user_timestamp_pointers), + #[ver(V >= V13_0B4)] + unk_ts: inner_weak_ptr!(ptr, unk_ts), + uuid: uuid_ta, + unk_30_padding: 0, + })?; + } + + #[ver(G < G14X)] + builder.add(microseq::WaitForIdle { + header: microseq::op::WaitForIdle::new(microseq::Pipe::Vertex), + })?; + #[ver(G >= G14X)] + builder.add(microseq::WaitForIdle2 { + header: microseq::op::WaitForIdle2::HEADER, + })?; + + if vtx_user_timestamps.any() { + builder.add(microseq::Timestamp::ver { + header: microseq::op::Timestamp::new(false), + command_time: inner_weak_ptr!(ptr, command_time), + ts_pointers: inner_weak_ptr!(ptr, timestamp_pointers), + update_ts: inner_weak_ptr!(ptr, timestamp_pointers.end_addr), + work_queue: ev_vtx.info_ptr, + user_ts_pointers: inner_weak_ptr!(ptr, user_timestamp_pointers), + #[ver(V >= V13_0B4)] + unk_ts: inner_weak_ptr!(ptr, unk_ts), + uuid: uuid_ta, + unk_30_padding: 0, + })?; + } + + let off = builder.offset_to(start_vtx); + builder.add(microseq::FinalizeVertex::ver { + header: microseq::op::FinalizeVertex::HEADER, + scene: scene.weak_pointer(), + buffer: scene.weak_buffer_pointer(), + stats, + work_queue: ev_vtx.info_ptr, + vm_slot: vm_bind.slot(), + unk_28: 0x0, // fixed + unk_pointer: inner_weak_ptr!(ptr, unk_pointee), + unk_34: 0x0, // fixed + uuid: uuid_ta, + fw_stamp: ev_vtx.fw_stamp_pointer, + stamp_value: ev_vtx.value.next(), + unk_48: U64(0x0), // fixed + unk_50: 0x0, // fixed + unk_54: 0x0, // fixed + unk_58: U64(0x0), // fixed + unk_60: 0x0, // fixed + unk_64: 0x0, // fixed + unk_68: 0x0, // fixed + #[ver(G >= G14 && V < V13_0B4)] + unk_68_g14: U64(0), + restart_branch_offset: off, + has_attachments: (vertex_attachments.count > 0) as u32, + #[ver(V >= V13_0B4)] + unk_74: Default::default(), // Ventura + })?; + + builder.add(microseq::RetireStamp { + header: microseq::op::RetireStamp::HEADER, + })?; + builder.build(private)? + }, + notifier, + scene, + vm_bind, + timestamps, + user_timestamps: vtx_user_timestamps, + }) + }, + |inner, _ptr| { + let vm_slot = vm_bind.slot(); + #[ver(G < G14)] + let core_masks = gpu.core_masks_packed(); + + try_init!(fw::vertex::raw::RunVertex::ver { + tag: fw::workqueue::CommandType::RunVertex, + #[ver(V >= V13_0B4)] + counter: U64(count_vtx), + vm_slot, + unk_8: 0, + notifier: inner.notifier.gpu_pointer(), + buffer_slot: inner.scene.slot(), + unk_1c: 0, + buffer: inner.scene.buffer_pointer(), + scene: inner.scene.gpu_pointer(), + unk_buffer_buf: inner.scene.kernel_buffer_pointer(), + unk_34: 0, + #[ver(G < G14X)] + job_params1 <- try_init!(fw::vertex::raw::JobParameters1::ver { + unk_0: U64(if unk1 { 0 } else { 0x200 }), // sometimes 0 + unk_8: f32!(1e-20), // fixed + unk_c: f32!(1e-20), // fixed + tvb_tilemap: inner.scene.tvb_tilemap_pointer(), + #[ver(G < G14)] + tvb_cluster_tilemaps: inner.scene.cluster_tilemaps_pointer(), + tpc: inner.scene.tpc_pointer(), + tvb_heapmeta: inner.scene.tvb_heapmeta_pointer().or(0x8000_0000_0000_0000), + iogpu_unk_54: U64(iogpu_unk54), // fixed + iogpu_unk_56: U64(iogpu_unk56), // fixed + #[ver(G < G14)] + tvb_cluster_meta1: inner + .scene + .meta_1_pointer() + .map(|x| x.or((tile_info.meta1_layer_stride as u64) << 50)), + utile_config, + unk_4c: 0, + ppp_multisamplectl: U64(cmdbuf.ppp_multisamplectl), // fixed + tvb_layermeta: inner.scene.tvb_layermeta_pointer(), + #[ver(G < G14)] + tvb_cluster_layermeta: inner.scene.tvb_cluster_layermeta_pointer(), + #[ver(G < G14)] + core_mask: Array::new([ + *core_masks.first().unwrap_or(&0), + *core_masks.get(1).unwrap_or(&0), + ]), + preempt_buf1: inner.scene.preempt_buf_1_pointer(), + preempt_buf2: inner.scene.preempt_buf_2_pointer(), + unk_80: U64(0x1), // fixed + preempt_buf3: inner.scene.preempt_buf_3_pointer().or(0x4_0000_0000_0000), // check + vdm_ctrl_stream_base: U64(cmdbuf.vdm_ctrl_stream_base), + #[ver(G < G14)] + tvb_cluster_meta2: inner.scene.meta_2_pointer(), + #[ver(G < G14)] + tvb_cluster_meta3: inner.scene.meta_3_pointer(), + #[ver(G < G14)] + tiling_control, + #[ver(G < G14)] + unk_ac: tiling_control_2 as u32, // fixed + unk_b0: Default::default(), // fixed + usc_exec_base_ta: U64(self.usc_exec_base), + #[ver(G < G14)] + tvb_cluster_meta4: inner + .scene + .meta_4_pointer() + .map(|x| x.or(0x3000_0000_0000_0000)), + #[ver(G < G14)] + unk_f0: U64(vtx_unk_f0), + unk_f8: U64(0x8c60), // fixed + helper_program: cmdbuf.vertex_helper.binary, + unk_104: 0, + helper_arg: U64(cmdbuf.vertex_helper.data), + unk_110: Default::default(), // fixed + unk_118: vtx_unk_118 as u32, // fixed + __pad: Default::default(), + }), + #[ver(G < G14X)] + tiling_params: tile_info.params, + #[ver(G >= G14X)] + registers: fw::job::raw::RegisterArray::new( + inner_weak_ptr!(_ptr, registers.registers), + |r| { + r.add(0x10141, if unk1 { 0 } else { 0x200 }); // s2.unk_0 + r.add(0x1c039, inner.scene.tvb_tilemap_pointer().into()); + r.add(0x1c9c8, inner.scene.tvb_tilemap_pointer().into()); + + let cl_tilemaps_ptr = inner + .scene + .cluster_tilemaps_pointer() + .map_or(0, |a| a.into()); + r.add(0x1c041, cl_tilemaps_ptr); + r.add(0x1c9d0, cl_tilemaps_ptr); + r.add(0x1c0a1, inner.scene.tpc_pointer().into()); // TE_TPC_ADDR + + let tvb_heapmeta_ptr = inner + .scene + .tvb_heapmeta_pointer() + .or(0x8000_0000_0000_0000) + .into(); + r.add(0x1c031, tvb_heapmeta_ptr); + r.add(0x1c9c0, tvb_heapmeta_ptr); + r.add(0x1c051, iogpu_unk54); // iogpu_unk_54/55 + r.add(0x1c061, iogpu_unk56); // iogpu_unk_56 + r.add(0x10149, utile_config.into()); // s2.unk_48 utile_config + r.add(0x10139, cmdbuf.ppp_multisamplectl); // PPP_MULTISAMPLECTL + r.add(0x10111, inner.scene.preempt_buf_1_pointer().into()); + r.add(0x1c9b0, inner.scene.preempt_buf_1_pointer().into()); + r.add(0x10119, inner.scene.preempt_buf_2_pointer().into()); + r.add(0x1c9b8, inner.scene.preempt_buf_2_pointer().into()); + r.add(0x1c958, 1); // s2.unk_80 + r.add( + 0x1c950, + inner + .scene + .preempt_buf_3_pointer() + .or(0x4_0000_0000_0000) + .into(), + ); + r.add(0x1c930, 0); // VCE related addr, lsb to enable + r.add(0x1c880, cmdbuf.vdm_ctrl_stream_base); // VDM_CTRL_STREAM_BASE + r.add(0x1c898, 0x0); // if lsb set, faults in UL1C0, possibly missing addr. + r.add( + 0x1c948, + inner.scene.meta_2_pointer().map_or(0, |a| a.into()), + ); // tvb_cluster_meta2 + r.add( + 0x1c888, + inner.scene.meta_3_pointer().map_or(0, |a| a.into()), + ); // tvb_cluster_meta3 + r.add(0x1c890, tiling_control.into()); // tvb_tiling_control + r.add(0x1c918, tiling_control_2); + r.add(0x1c079, inner.scene.tvb_layermeta_pointer().into()); + r.add(0x1c9d8, inner.scene.tvb_layermeta_pointer().into()); + let cl_layermeta_pointer = + inner.scene.tvb_cluster_layermeta_pointer().map_or(0, |a| a.into()); + r.add(0x1c089, cl_layermeta_pointer); + r.add(0x1c9e0, cl_layermeta_pointer); + let cl_meta_4_pointer = + inner.scene.meta_4_pointer().map_or(0, |a| a.into()); + r.add(0x16c41, cl_meta_4_pointer); // tvb_cluster_meta4 + r.add(0x1ca40, cl_meta_4_pointer); // tvb_cluster_meta4 + r.add(0x1c9a8, vtx_unk_f0); // + meta1_blocks? min_free_tvb_pages? + r.add( + 0x1c920, + inner.scene.meta_1_pointer().map_or(0, |a| a.into()), + ); // ??? | meta1_blocks? + r.add(0x10151, 0); + r.add(0x1c199, 0); + r.add(0x1c1a1, 0); + r.add(0x1c1a9, 0); // 0x10151 bit 1 enables + r.add(0x1c1b1, 0); + r.add(0x1c1b9, 0); + r.add(0x10061, self.usc_exec_base); // USC_EXEC_BASE_TA + r.add(0x11801, cmdbuf.vertex_helper.binary.into()); + r.add(0x11809, cmdbuf.vertex_helper.data); + r.add(0x11f71, cmdbuf.vertex_helper.cfg.into()); + r.add(0x1c0b1, tile_info.params.rgn_size.into()); // TE_PSG + r.add(0x1c850, tile_info.params.rgn_size.into()); + r.add(0x10131, tile_info.params.unk_4.into()); + r.add(0x10121, tile_info.params.ppp_ctrl.into()); // PPP_CTRL + r.add( + 0x10129, + tile_info.params.x_max as u64 + | ((tile_info.params.y_max as u64) << 16), + ); // PPP_SCREEN + r.add(0x101b9, tile_info.params.te_screen.into()); // TE_SCREEN + r.add(0x1c069, tile_info.params.te_mtile1.into()); // TE_MTILE1 + r.add(0x1c071, tile_info.params.te_mtile2.into()); // TE_MTILE2 + r.add(0x1c081, tile_info.params.tiles_per_mtile.into()); // TE_MTILE + r.add(0x1c0a9, tile_info.params.tpc_stride.into()); // TE_TPC + r.add(0x10171, tile_info.params.unk_24.into()); + r.add(0x10169, tile_info.params.unk_28.into()); // TA_RENDER_TARGET_MAX + r.add(0x12099, vtx_unk_118); + r.add(0x1c9e8, (tile_info.params.unk_28 & 0x4fff).into()); + /* + r.add(0x10209, 0x100); // Some kind of counter?? Does this matter? + r.add(0x1c9f0, 0x100); // Some kind of counter?? Does this matter? + r.add(0x1c830, 1); // ? + r.add(0x1ca30, 0x1502960e60); // ? + r.add(0x16c39, 0x1502960e60); // ? + r.add(0x1c910, 0xa0000b011d); // ? + r.add(0x1c8e0, 0xff); // cluster mask + r.add(0x1c8e8, 0); // ? + */ + } + ), + tpc: inner.scene.tpc_pointer(), + tpc_size: U64(tile_info.tpc_size as u64), + microsequence: inner.micro_seq.gpu_pointer(), + microsequence_size: inner.micro_seq.len() as u32, + fragment_stamp_slot: ev_frag.slot, + fragment_stamp_value: ev_frag.value.next(), + unk_pointee: 0, + unk_pad: 0, + job_params2 <- try_init!(fw::vertex::raw::JobParameters2 { + unk_480: Default::default(), // fixed + unk_498: U64(0x0), // fixed + unk_4a0: 0x0, // fixed + preempt_buf1: inner.scene.preempt_buf_1_pointer(), + unk_4ac: 0x0, // fixed + unk_4b0: U64(0x0), // fixed + unk_4b8: 0x0, // fixed + unk_4bc: U64(0x0), // fixed + unk_4c4_padding: Default::default(), + unk_50c: 0x0, // fixed + unk_510: U64(0x0), // fixed + unk_518: U64(0x0), // fixed + unk_520: U64(0x0), // fixed + }), + encoder_params <- try_init!(fw::job::raw::EncoderParams { + unk_8: 0x0, // fixed + sync_grow: 0x0, // fixed + unk_10: 0x0, // fixed + encoder_id: 0, + unk_18: 0x0, // fixed + unk_mask: 0xffffffffu32, + sampler_array: U64(cmdbuf.sampler_heap), + sampler_count: cmdbuf.sampler_count as u32, + sampler_max: (cmdbuf.sampler_count as u32) + 1, + }), + unk_55c: 0, + unk_560: 0, + sync_grow: 0, + unk_568: 0, + uses_scratch: (cmdbuf.flags + & uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_VERTEX_SCRATCH as u32 + != 0) as u32, + meta <- try_init!(fw::job::raw::JobMeta { + unk_0: 0, + unk_2: 0, + no_preemption: no_preemption as u8, + stamp: ev_vtx.stamp_pointer, + fw_stamp: ev_vtx.fw_stamp_pointer, + stamp_value: ev_vtx.value.next(), + stamp_slot: ev_vtx.slot, + evctl_index: 0, // fixed + flush_stamps: flush_stamps as u32, + uuid: uuid_ta, + event_seq: ev_vtx.event_seq as u32, + }), + unk_after_meta: unk1.into(), + unk_buf_0: U64(0), + unk_buf_8: U64(0), + unk_buf_10: U64(0), + command_time: U64(0), + timestamp_pointers <- try_init!(fw::job::raw::TimestampPointers { + start_addr: Some(inner_ptr!(inner.timestamps.gpu_pointer(), vtx.start)), + end_addr: Some(inner_ptr!(inner.timestamps.gpu_pointer(), vtx.end)), + }), + user_timestamp_pointers: inner.user_timestamps.pointers()?, + client_sequence: slot_client_seq, + pad_5d5: Default::default(), + unk_5d8: 0, + unk_5dc: 0, + #[ver(V >= V13_0B4)] + unk_ts: U64(0), + #[ver(V >= V13_0B4)] + unk_5dd_8: Default::default(), + }) + }, + )?; + + core::mem::drop(alloc); + + mod_dev_dbg!(self.dev, "[Submission {}] Add Vertex\n", id); + fence.add_command(); + vtx_job.add_cb(vtx, vm_bind.slot(), move |error| { + if let Some(err) = error { + fence.set_error(err.into()) + } + + fence.command_complete(); + })?; + + mod_dev_dbg!(self.dev, "[Submission {}] Increment counters\n", id); + + // TODO: handle rollbacks, move to job submit? + buffer.increment(); + + job.get_vtx()?.next_seq(); + job.get_frag()?.next_seq(); + + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/regs.rs b/drivers/gpu/drm/asahi/regs.rs new file mode 100644 index 00000000000000..6ebbaa56f48c81 --- /dev/null +++ b/drivers/gpu/drm/asahi/regs.rs @@ -0,0 +1,491 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! GPU MMIO register abstraction +//! +//! Since the vast majority of the interactions with the GPU are brokered through the firmware, +//! there is very little need to interact directly with GPU MMIO register. This module abstracts +//! the few operations that require that, mainly reading the MMU fault status, reading GPU ID +//! information, and starting the GPU firmware coprocessor. + +use crate::hw; +use kernel::{ + c_str, + device::Core, + devres::Devres, + io::mem::IoMem, + platform, + prelude::*, + types::ARef, // +}; + +/// Size of the ASC control MMIO region. +pub(crate) const ASC_CTL_SIZE: usize = 0x4000; + +/// Size of the SGX MMIO region. +pub(crate) const SGX_SIZE: usize = 0x1000000; + +const CPU_CONTROL: usize = 0x44; +const CPU_RUN: u32 = 0x1 << 4; // BIT(4) + +const FAULT_INFO: usize = 0x17030; + +const ID_VERSION: usize = 0xd04000; +const ID_UNK08: usize = 0xd04008; +const ID_COUNTS_1: usize = 0xd04010; +const ID_COUNTS_2: usize = 0xd04014; +const ID_UNK18: usize = 0xd04018; +const ID_CLUSTERS: usize = 0xd0401c; + +const CORE_MASK_0: usize = 0xd01500; +const CORE_MASK_1: usize = 0xd01514; + +const CORE_MASKS_G14X: usize = 0xe01500; +const FAULT_INFO_G14X: usize = 0xd8c0; +const FAULT_ADDR_G14X: usize = 0xd8c8; + +/// Enum representing the unit that caused an MMU fault. +#[allow(non_camel_case_types)] +#[allow(clippy::upper_case_acronyms)] +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +pub(crate) enum FaultUnit { + /// Decompress / pixel fetch + DCMP(u8), + /// USC L1 Cache (device loads/stores) + UL1C(u8), + /// Compress / pixel store + CMP(u8), + GSL1(u8), + IAP(u8), + VCE(u8), + /// Tiling Engine + TE(u8), + RAS(u8), + /// Vertex Data Master + VDM(u8), + PPP(u8), + /// ISP Parameter Fetch + IPF(u8), + IPF_CPF(u8), + VF(u8), + VF_CPF(u8), + /// Depth/Stencil load/store + ZLS(u8), + + /// Parameter Management + dPM, + /// Compute Data Master + dCDM_KS(u8), + dIPP, + dIPP_CS, + // Vertex Data Master + dVDM_CSD, + dVDM_SSD, + dVDM_ILF, + dVDM_ILD, + dRDE(u8), + FC, + GSL2, + + /// Graphics L2 Cache Control? + GL2CC_META(u8), + GL2CC_MB, + + /// Parameter Management + gPM_SP(u8), + /// Vertex Data Master - CSD + gVDM_CSD_SP(u8), + gVDM_SSD_SP(u8), + gVDM_ILF_SP(u8), + gVDM_TFP_SP(u8), + gVDM_MMB_SP(u8), + /// Compute Data Master + gCDM_CS_KS0_SP(u8), + gCDM_CS_KS1_SP(u8), + gCDM_CS_KS2_SP(u8), + gCDM_KS0_SP(u8), + gCDM_KS1_SP(u8), + gCDM_KS2_SP(u8), + gIPP_SP(u8), + gIPP_CS_SP(u8), + gRDE0_SP(u8), + gRDE1_SP(u8), + + gCDM_CS, + gCDM_ID, + gCDM_CSR, + gCDM_CSW, + gCDM_CTXR, + gCDM_CTXW, + gIPP, + gIPP_CS, + gKSM_RCE, + + Unknown(u8), +} + +/// Reason for an MMU fault. +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +pub(crate) enum FaultReason { + Unmapped, + AfFault, + WriteOnly, + ReadOnly, + NoAccess, + Unknown(u8), +} + +/// Collection of information about an MMU fault. +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +pub(crate) struct FaultInfo { + pub(crate) address: u64, + pub(crate) sideband: u8, + pub(crate) vm_slot: u32, + pub(crate) unit_code: u8, + pub(crate) unit: FaultUnit, + pub(crate) level: u8, + pub(crate) unk_5: u8, + pub(crate) read: bool, + pub(crate) reason: FaultReason, +} + +/// Device resources for this GPU instance. +pub(crate) struct Resources { + dev: ARef, + sgx: Pin>>>, +} + +impl Resources { + /// Map the required resources given our platform device. + pub(crate) fn new(pdev: &platform::Device) -> Result { + let sgx_req = pdev.io_request_by_name(c_str!("sgx")).ok_or(EINVAL)?; + let sgx_iomem = KBox::pin_init(sgx_req.iomap_sized::(), GFP_KERNEL)?; + + Ok(Resources { + // SAFETY: This device does DMA via the UAT IOMMU. + dev: pdev.into(), + sgx: sgx_iomem, + }) + } + + fn sgx_read32(&self) -> u32 { + if let Some(sgx) = self.sgx.try_access() { + sgx.read32_relaxed(OFF) + } else { + 0 + } + } + + /* Not yet used + fn sgx_write32(&self, val: u32) { + if let Some(sgx) = self.sgx.try_access() { + sgx.write32_relaxed(val, OFF) + } + } + */ + + fn sgx_read64(&self) -> u64 { + if let Some(sgx) = self.sgx.try_access() { + sgx.read64_relaxed(OFF) + } else { + 0 + } + } + + /* Not yet used + fn sgx_write64(&self, val: u64) { + if let Some(sgx) = self.sgx.try_access() { + sgx.write64_relaxed(val, OFF) + } + } + */ + + /// Initialize the MMIO registers for the GPU. + pub(crate) fn init_mmio(&self) -> Result { + // Nothing to do for now... + + Ok(()) + } + + /// Start the ASC coprocessor CPU. + pub(crate) fn start_cpu(pdev: &platform::Device) -> Result { + let asc_req = pdev.io_request_by_name(c_str!("asc")).ok_or(EINVAL)?; + let asc_iomem = KBox::pin_init(asc_req.iomap_sized::(), GFP_KERNEL)?; + let res = asc_iomem.access(pdev.as_ref())?; + + let val = res.read32_relaxed(CPU_CONTROL); + res.write32_relaxed(val | CPU_RUN, CPU_CONTROL); + Ok(()) + } + + /// Get the GPU identification info from registers. + /// + /// See [`hw::GpuIdConfig`] for the result. + pub(crate) fn get_gpu_id(&self) -> Result { + let id_version = self.sgx_read32::(); + let id_unk08 = self.sgx_read32::(); + let id_counts_1 = self.sgx_read32::(); + let id_counts_2 = self.sgx_read32::(); + let id_unk18 = self.sgx_read32::(); + let id_clusters = self.sgx_read32::(); + + dev_info!( + self.dev.as_ref(), + "GPU ID registers: {:#x} {:#x} {:#x} {:#x} {:#x} {:#x}\n", + id_version, + id_unk08, + id_counts_1, + id_counts_2, + id_unk18, + id_clusters + ); + + let gpu_gen = (id_version >> 24) & 0xff; + + let mut core_mask_regs = KVec::new(); + + let num_clusters = match gpu_gen { + 4 | 5 => { + // G13 | G14G + core_mask_regs.push(self.sgx_read32::(), GFP_KERNEL)?; + core_mask_regs.push(self.sgx_read32::(), GFP_KERNEL)?; + (id_clusters >> 12) & 0xff + } + 6 => { + // G14X + core_mask_regs.push(self.sgx_read32::(), GFP_KERNEL)?; + core_mask_regs.push(self.sgx_read32::<{ CORE_MASKS_G14X + 4 }>(), GFP_KERNEL)?; + core_mask_regs.push(self.sgx_read32::<{ CORE_MASKS_G14X + 8 }>(), GFP_KERNEL)?; + // Clusters per die * num dies + ((id_counts_1 >> 8) & 0xff) * ((id_counts_1 >> 16) & 0xf) + } + a => { + dev_err!(self.dev.as_ref(), "Unknown GPU generation {}\n", a); + return Err(ENODEV); + } + }; + + let mut core_masks_packed = KVec::new(); + core_masks_packed.extend_from_slice(&core_mask_regs, GFP_KERNEL)?; + + dev_info!(self.dev.as_ref(), "Core masks: {:#x?}\n", core_masks_packed); + + let num_cores = id_counts_1 & 0xff; + + if num_cores > 32 { + dev_err!( + self.dev.as_ref(), + "Too many cores per cluster ({} > 32)\n", + num_cores + ); + return Err(ENODEV); + } + + if num_cores * num_clusters > (core_mask_regs.len() * 32) as u32 { + dev_err!( + self.dev.as_ref(), + "Too many total cores ({} x {} > {})\n", + num_clusters, + num_cores, + core_mask_regs.len() * 32 + ); + return Err(ENODEV); + } + + let mut core_masks = KVec::new(); + let mut total_active_cores: u32 = 0; + + let max_core_mask = ((1u64 << num_cores) - 1) as u32; + for _ in 0..num_clusters { + let mask = core_mask_regs[0] & max_core_mask; + core_masks.push(mask, GFP_KERNEL)?; + for i in 0..core_mask_regs.len() { + core_mask_regs[i] >>= num_cores; + if i < (core_mask_regs.len() - 1) { + core_mask_regs[i] |= core_mask_regs[i + 1] << (32 - num_cores); + } + } + total_active_cores += mask.count_ones(); + } + + if core_mask_regs.iter().any(|a| *a != 0) { + dev_err!( + self.dev.as_ref(), + "Leftover core mask: {:#x?}\n", + core_mask_regs + ); + return Err(EIO); + } + + let (gpu_rev, gpu_rev_id) = match (id_version >> 8) & 0xff { + 0x00 => (hw::GpuRevision::A0, hw::GpuRevisionID::A0), + 0x01 => (hw::GpuRevision::A1, hw::GpuRevisionID::A1), + 0x10 => (hw::GpuRevision::B0, hw::GpuRevisionID::B0), + 0x11 => (hw::GpuRevision::B1, hw::GpuRevisionID::B1), + 0x20 => (hw::GpuRevision::C0, hw::GpuRevisionID::C0), + 0x21 => (hw::GpuRevision::C1, hw::GpuRevisionID::C1), + a => { + dev_err!(self.dev.as_ref(), "Unknown GPU revision {}\n", a); + return Err(ENODEV); + } + }; + + Ok(hw::GpuIdConfig { + gpu_gen: match (id_version >> 24) & 0xff { + 4 => hw::GpuGen::G13, + 5 => hw::GpuGen::G14, + 6 => hw::GpuGen::G14, // G14X has a separate ID + a => { + dev_err!(self.dev.as_ref(), "Unknown GPU generation {}\n", a); + return Err(ENODEV); + } + }, + gpu_variant: match (id_version >> 16) & 0xff { + 1 => hw::GpuVariant::P, // Guess + 2 => hw::GpuVariant::G, + 3 => hw::GpuVariant::S, + 4 => { + if num_clusters > 4 { + hw::GpuVariant::D + } else { + hw::GpuVariant::C + } + } + a => { + dev_err!(self.dev.as_ref(), "Unknown GPU variant {}\n", a); + return Err(ENODEV); + } + }, + gpu_rev, + gpu_rev_id, + num_clusters, + num_cores, + num_frags: num_cores, // Used to be id_counts_1[15:8] but does not work for G14X + num_gps: (id_counts_2 >> 16) & 0xff, + total_active_cores, + core_masks, + core_masks_packed, + }) + } + + /// Get the fault information from the MMU status register, if one occurred. + pub(crate) fn get_fault_info(&self, cfg: &'static hw::HwConfig) -> Option { + let g14x = cfg.gpu_core as u32 >= hw::GpuCore::G14S as u32; + + let fault_info = if g14x { + self.sgx_read64::() + } else { + self.sgx_read64::() + }; + + if fault_info & 1 == 0 { + return None; + } + + let fault_addr = if g14x { + self.sgx_read64::() + } else { + fault_info >> 30 + }; + + let unit_code = ((fault_info >> 9) & 0xff) as u8; + let unit = match unit_code { + 0x00..=0x9f => match unit_code & 0xf { + 0x0 => FaultUnit::DCMP(unit_code >> 4), + 0x1 => FaultUnit::UL1C(unit_code >> 4), + 0x2 => FaultUnit::CMP(unit_code >> 4), + 0x3 => FaultUnit::GSL1(unit_code >> 4), + 0x4 => FaultUnit::IAP(unit_code >> 4), + 0x5 => FaultUnit::VCE(unit_code >> 4), + 0x6 => FaultUnit::TE(unit_code >> 4), + 0x7 => FaultUnit::RAS(unit_code >> 4), + 0x8 => FaultUnit::VDM(unit_code >> 4), + 0x9 => FaultUnit::PPP(unit_code >> 4), + 0xa => FaultUnit::IPF(unit_code >> 4), + 0xb => FaultUnit::IPF_CPF(unit_code >> 4), + 0xc => FaultUnit::VF(unit_code >> 4), + 0xd => FaultUnit::VF_CPF(unit_code >> 4), + 0xe => FaultUnit::ZLS(unit_code >> 4), + _ => FaultUnit::Unknown(unit_code), + }, + 0xa1 => FaultUnit::dPM, + 0xa2 => FaultUnit::dCDM_KS(0), + 0xa3 => FaultUnit::dCDM_KS(1), + 0xa4 => FaultUnit::dCDM_KS(2), + 0xa5 => FaultUnit::dIPP, + 0xa6 => FaultUnit::dIPP_CS, + 0xa7 => FaultUnit::dVDM_CSD, + 0xa8 => FaultUnit::dVDM_SSD, + 0xa9 => FaultUnit::dVDM_ILF, + 0xaa => FaultUnit::dVDM_ILD, + 0xab => FaultUnit::dRDE(0), + 0xac => FaultUnit::dRDE(1), + 0xad => FaultUnit::FC, + 0xae => FaultUnit::GSL2, + 0xb0..=0xb7 => FaultUnit::GL2CC_META(unit_code & 0xf), + 0xb8 => FaultUnit::GL2CC_MB, + 0xd0..=0xdf if g14x => match unit_code & 0xf { + 0x0 => FaultUnit::gCDM_CS, + 0x1 => FaultUnit::gCDM_ID, + 0x2 => FaultUnit::gCDM_CSR, + 0x3 => FaultUnit::gCDM_CSW, + 0x4 => FaultUnit::gCDM_CTXR, + 0x5 => FaultUnit::gCDM_CTXW, + 0x6 => FaultUnit::gIPP, + 0x7 => FaultUnit::gIPP_CS, + 0x8 => FaultUnit::gKSM_RCE, + _ => FaultUnit::Unknown(unit_code), + }, + 0xe0..=0xff if g14x => match unit_code & 0xf { + 0x0 => FaultUnit::gPM_SP((unit_code >> 4) & 1), + 0x1 => FaultUnit::gVDM_CSD_SP((unit_code >> 4) & 1), + 0x2 => FaultUnit::gVDM_SSD_SP((unit_code >> 4) & 1), + 0x3 => FaultUnit::gVDM_ILF_SP((unit_code >> 4) & 1), + 0x4 => FaultUnit::gVDM_TFP_SP((unit_code >> 4) & 1), + 0x5 => FaultUnit::gVDM_MMB_SP((unit_code >> 4) & 1), + 0x6 => FaultUnit::gRDE0_SP((unit_code >> 4) & 1), + _ => FaultUnit::Unknown(unit_code), + }, + 0xe0..=0xff if !g14x => match unit_code & 0xf { + 0x0 => FaultUnit::gPM_SP((unit_code >> 4) & 1), + 0x1 => FaultUnit::gVDM_CSD_SP((unit_code >> 4) & 1), + 0x2 => FaultUnit::gVDM_SSD_SP((unit_code >> 4) & 1), + 0x3 => FaultUnit::gVDM_ILF_SP((unit_code >> 4) & 1), + 0x4 => FaultUnit::gVDM_TFP_SP((unit_code >> 4) & 1), + 0x5 => FaultUnit::gVDM_MMB_SP((unit_code >> 4) & 1), + 0x6 => FaultUnit::gCDM_CS_KS0_SP((unit_code >> 4) & 1), + 0x7 => FaultUnit::gCDM_CS_KS1_SP((unit_code >> 4) & 1), + 0x8 => FaultUnit::gCDM_CS_KS2_SP((unit_code >> 4) & 1), + 0x9 => FaultUnit::gCDM_KS0_SP((unit_code >> 4) & 1), + 0xa => FaultUnit::gCDM_KS1_SP((unit_code >> 4) & 1), + 0xb => FaultUnit::gCDM_KS2_SP((unit_code >> 4) & 1), + 0xc => FaultUnit::gIPP_SP((unit_code >> 4) & 1), + 0xd => FaultUnit::gIPP_CS_SP((unit_code >> 4) & 1), + 0xe => FaultUnit::gRDE0_SP((unit_code >> 4) & 1), + 0xf => FaultUnit::gRDE1_SP((unit_code >> 4) & 1), + _ => FaultUnit::Unknown(unit_code), + }, + _ => FaultUnit::Unknown(unit_code), + }; + + let reason = match (fault_info >> 1) & 0x7 { + 0 => FaultReason::Unmapped, + 1 => FaultReason::AfFault, + 2 => FaultReason::WriteOnly, + 3 => FaultReason::ReadOnly, + 4 => FaultReason::NoAccess, + a => FaultReason::Unknown(a as u8), + }; + + Some(FaultInfo { + address: fault_addr << 6, + sideband: ((fault_info >> 23) & 0x7f) as u8, + vm_slot: ((fault_info >> 17) & 0x3f) as u32, + unit_code, + unit, + level: ((fault_info >> 7) & 3) as u8, + unk_5: ((fault_info >> 5) & 3) as u8, + read: (fault_info & (1 << 4)) != 0, + reason, + }) + } +} diff --git a/drivers/gpu/drm/asahi/slotalloc.rs b/drivers/gpu/drm/asahi/slotalloc.rs new file mode 100644 index 00000000000000..fde7470fe57791 --- /dev/null +++ b/drivers/gpu/drm/asahi/slotalloc.rs @@ -0,0 +1,323 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Generic slot allocator +//! +//! This is a simple allocator to manage fixed-size pools of GPU resources that are transiently +//! required during command execution. Each item resides in a "slot" at a given index. Users borrow +//! and return free items from the available pool. +//! +//! Allocations are "sticky", and return a token that callers can use to request the same slot +//! again later. This allows slots to be lazily invalidated, so that multiple uses by the same user +//! avoid any actual cleanup work. +//! +//! The allocation policy is currently a simple LRU mechanism, doing a full linear scan over the +//! slots when no token was previously provided. This is probably good enough, since in the absence +//! of serious system contention most allocation requests will be immediately fulfilled from the +//! previous slot without doing an LRU scan. + +use core::num::NonZeroUsize; +use core::ops::{ + Deref, + DerefMut, // +}; +use kernel::{ + error::{ + code::*, + Result, // + }, + prelude::*, + str::CStr, + sync::{ + Arc, + CondVar, + LockClassKey, + Mutex, // + }, +}; + +/// Trait representing a single item within a slot. +pub(crate) trait SlotItem { + /// Arbitrary user data associated with the SlotAllocator. + type Data; + + /// Called eagerly when this item is released back into the available pool. + fn release(&mut self, _data: &mut Self::Data, _slot: u32) {} +} + +/// Trivial implementation for users which do not require any slot data nor any allocator data. +impl SlotItem for () { + type Data = (); +} + +/// Represents a current or previous allocation of an item from a slot. Users keep `SlotToken`s +/// around across allocations to request that, if possible, the same slot be reused. +#[derive(Copy, Clone, Debug)] +pub(crate) struct SlotToken { + time: u64, + slot: u32, +} + +impl SlotToken { + /// Returns the slot index that this token represents a past assignment to. + pub(crate) fn last_slot(&self) -> u32 { + self.slot + } +} + +/// A guard representing active ownership of a slot. +pub(crate) struct Guard { + item: Option, + changed: bool, + token: SlotToken, + alloc: Arc>, +} + +impl Guard { + /// Returns the active slot owned by this `Guard`. + pub(crate) fn slot(&self) -> u32 { + self.token.slot + } + + /// Returns `true` if the slot changed since the last allocation (or no `SlotToken` was + /// provided), or `false` if the previously allocated slot was successfully re-acquired with + /// no other users in the interim. + pub(crate) fn changed(&self) -> bool { + self.changed + } + + /// Returns a `SlotToken` that can be used to re-request the same slot at a later time, after + /// this `Guard` is dropped. + pub(crate) fn token(&self) -> SlotToken { + self.token + } +} + +impl Deref for Guard { + type Target = T; + + fn deref(&self) -> &Self::Target { + self.item.as_ref().expect("SlotItem Guard lost our item!") + } +} + +impl DerefMut for Guard { + fn deref_mut(&mut self) -> &mut Self::Target { + self.item.as_mut().expect("SlotItem Guard lost our item!") + } +} + +/// A slot item that is currently free. +struct Entry { + item: T, + get_time: u64, + drop_time: u64, +} + +/// Inner data for the `SlotAllocator`, protected by a `Mutex`. +struct SlotAllocatorInner { + data: T::Data, + slots: KVec>>, + get_count: u64, + drop_count: u64, + slot_limit: usize, +} + +/// A single slot allocator instance. +#[pin_data] +struct SlotAllocatorOuter { + #[pin] + inner: Mutex>, + #[pin] + cond: CondVar, +} + +/// A shared reference to a slot allocator instance. +pub(crate) struct SlotAllocator(Arc>); + +impl SlotAllocator { + /// Creates a new `SlotAllocator`, with a fixed number of slots and arbitrary associated data. + /// + /// The caller provides a constructor callback which takes a reference to the `T::Data` and + /// creates a single slot. This is called during construction to create all the initial + /// items, which then live the lifetime of the `SlotAllocator`. + pub(crate) fn new( + num_slots: u32, + mut data: T::Data, + mut constructor: impl FnMut(&mut T::Data, u32) -> Option, + name: &'static CStr, + lock_key1: Pin<&'static LockClassKey>, + lock_key2: Pin<&'static LockClassKey>, + ) -> Result> { + let mut slots = KVec::with_capacity(num_slots as usize, GFP_KERNEL)?; + + for i in 0..num_slots { + slots + .push( + constructor(&mut data, i).map(|item| Entry { + item, + get_time: 0, + drop_time: 0, + }), + GFP_KERNEL, + ) + .expect("try_push() failed after reservation"); + } + + let inner = SlotAllocatorInner { + data, + slots, + get_count: 0, + drop_count: 0, + slot_limit: usize::MAX, + }; + + let alloc = Arc::pin_init( + pin_init!(SlotAllocatorOuter { + // SAFETY: `mutex_init!` is called below. + inner <- Mutex::new(inner, name, lock_key1), + // SAFETY: `condvar_init!` is called below. + cond <- CondVar::new(name, lock_key2), + }), + GFP_KERNEL, + )?; + + Ok(SlotAllocator(alloc)) + } + + /// Calls a callback on the inner data associated with this allocator, taking the lock. + pub(crate) fn with_inner(&self, cb: impl FnOnce(&mut T::Data) -> RetVal) -> RetVal { + let mut inner = self.0.inner.lock(); + cb(&mut inner.data) + } + + /// Set the slot limit for this allocator. New bindings will not use slots above + /// this threshold. + pub(crate) fn set_limit(&self, limit: Option) { + let mut inner = self.0.inner.lock(); + inner.slot_limit = limit.unwrap_or(NonZeroUsize::MAX).get(); + } + + /// Gets a fresh slot, optionally reusing a previous allocation if a `SlotToken` is provided. + /// + /// Blocks if no slots are free. + pub(crate) fn get(&self, token: Option) -> Result> { + self.get_inner(token, |_a, _b| Ok(())) + } + + /// Gets a fresh slot, optionally reusing a previous allocation if a `SlotToken` is provided. + /// + /// Blocks if no slots are free. + /// + /// This version allows the caller to pass in a callback that gets a mutable reference to the + /// user data for the allocator and the freshly acquired slot, which is called before the + /// allocator lock is released. This can be used to perform bookkeeping associated with + /// specific slots (such as tracking their current owner). + pub(crate) fn get_inner( + &self, + token: Option, + cb: impl FnOnce(&mut T::Data, &mut Guard) -> Result<()>, + ) -> Result> { + let mut inner = self.0.inner.lock(); + + if let Some(token) = token { + if (token.slot as usize) < inner.slot_limit { + let slot = &mut inner.slots[token.slot as usize]; + if slot.is_some() { + let count = slot.as_ref().unwrap().get_time; + if count == token.time { + let mut guard = Guard { + item: Some(slot.take().unwrap().item), + token, + changed: false, + alloc: self.0.clone(), + }; + cb(&mut inner.data, &mut guard)?; + return Ok(guard); + } + } + } + } + + let mut first = true; + let slot = loop { + let mut oldest_time = u64::MAX; + let mut oldest_slot = 0u32; + + for (i, slot) in inner.slots.iter().enumerate() { + if i >= inner.slot_limit { + break; + } + if let Some(slot) = slot.as_ref() { + if slot.drop_time < oldest_time { + oldest_slot = i as u32; + oldest_time = slot.drop_time; + } + } + } + + if oldest_time == u64::MAX { + if first && inner.slot_limit == usize::MAX { + pr_warn!( + "{}: out of slots, blocking\n", + core::any::type_name::() + ); + } + first = false; + if self.0.cond.wait_interruptible(&mut inner) { + return Err(ERESTARTSYS); + } + } else { + break oldest_slot; + } + }; + + inner.get_count += 1; + + let item = inner.slots[slot as usize] + .take() + .expect("Someone stole our slot?") + .item; + + let mut guard = Guard { + item: Some(item), + changed: true, + token: SlotToken { + time: inner.get_count, + slot, + }, + alloc: self.0.clone(), + }; + + cb(&mut inner.data, &mut guard)?; + Ok(guard) + } +} + +impl Clone for SlotAllocator { + fn clone(&self) -> Self { + SlotAllocator(self.0.clone()) + } +} + +impl Drop for Guard { + fn drop(&mut self) { + let mut inner = self.alloc.inner.lock(); + if inner.slots[self.token.slot as usize].is_some() { + pr_crit!( + "{}: tried to return an item into a full slot ({})\n", + core::any::type_name::(), + self.token.slot + ); + } else { + inner.drop_count += 1; + let mut item = self.item.take().expect("Guard lost its item"); + item.release(&mut inner.data, self.token.slot); + inner.slots[self.token.slot as usize] = Some(Entry { + item, + get_time: self.token.time, + drop_time: inner.drop_count, + }); + self.alloc.cond.notify_one(); + } + } +} diff --git a/drivers/gpu/drm/asahi/util.rs b/drivers/gpu/drm/asahi/util.rs new file mode 100644 index 00000000000000..1a41d8f16d4432 --- /dev/null +++ b/drivers/gpu/drm/asahi/util.rs @@ -0,0 +1,141 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Miscellaneous utility functions + +use core::ops::{ + Add, + BitAnd, + Div, + Not, + Sub, // +}; +use kernel::prelude::*; + +/// Aligns an integer type to a power of two. +pub(crate) fn align(a: T, b: T) -> T +where + T: Copy + + Default + + BitAnd + + Not + + Add + + Sub + + Div + + core::cmp::PartialEq, +{ + let def: T = Default::default(); + #[allow(clippy::eq_op)] + let one: T = !def / !def; + + assert!((b & (b - one)) == def); + + (a + b - one) & !(b - one) +} + +/// Aligns an integer type down to a power of two. +pub(crate) fn align_down(a: T, b: T) -> T +where + T: Copy + + Default + + BitAnd + + Not + + Sub + + Div + + core::cmp::PartialEq, +{ + let def: T = Default::default(); + #[allow(clippy::eq_op)] + let one: T = !def / !def; + + assert!((b & (b - one)) == def); + + a & !(b - one) +} + +pub(crate) trait RangeExt { + fn overlaps(&self, other: Self) -> bool; + fn is_superset(&self, other: Self) -> bool; + // fn len(&self) -> usize; + fn range(&self) -> T; +} + +impl + Default + Copy + Sub> RangeExt for core::ops::Range +where + usize: core::convert::TryFrom, + >::Error: core::fmt::Debug, +{ + fn overlaps(&self, other: Self) -> bool { + !(self.is_empty() || other.is_empty() || self.end <= other.start || other.end <= self.start) + } + fn is_superset(&self, other: Self) -> bool { + !self.is_empty() + && (other.is_empty() || (other.start >= self.start && other.end <= self.end)) + } + fn range(&self) -> T { + if self.is_empty() { + Default::default() + } else { + self.end - self.start + } + } + // fn len(&self) -> usize { + // self.range().try_into().unwrap() + // } +} + +pub(crate) fn gcd(in_n: u64, in_m: u64) -> u64 { + let mut n = in_n; + let mut m = in_m; + + while n != 0 { + let remainder = m % n; + m = n; + n = remainder; + } + + m +} + +pub(crate) unsafe trait AnyBitPattern: Default + Sized + Copy + 'static {} + +pub(crate) struct Reader<'a> { + buffer: &'a [u8], + offset: usize, +} + +impl<'a> Reader<'a> { + pub(crate) fn new(buffer: &'a [u8]) -> Self { + Reader { buffer, offset: 0 } + } + + pub(crate) fn read_up_to(&mut self, max_size: usize) -> Result { + let mut obj: T = Default::default(); + let size: usize = core::mem::size_of::().min(max_size); + let range = self.offset..self.offset + size; + let src = self.buffer.get(range).ok_or(EINVAL)?; + + // SAFETY: The output pointer is valid, and the size does not exceed + // the type size, and all bit patterns are valid. + let dst = unsafe { core::slice::from_raw_parts_mut(&mut obj as *mut _ as *mut u8, size) }; + + dst.copy_from_slice(src); + self.offset += size; + Ok(obj) + } + + pub(crate) fn read(&mut self) -> Result { + self.read_up_to(!0) + } + + pub(crate) fn is_empty(&self) -> bool { + self.offset >= self.buffer.len() + } + + pub(crate) fn skip(&mut self, size: usize) { + self.offset += size + } + + pub(crate) fn rewind(&mut self) { + self.offset = 0 + } +} diff --git a/drivers/gpu/drm/asahi/vm/mod.rs b/drivers/gpu/drm/asahi/vm/mod.rs new file mode 100644 index 00000000000000..63cf8a76cd5ce4 --- /dev/null +++ b/drivers/gpu/drm/asahi/vm/mod.rs @@ -0,0 +1,5 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Virtual address space management + +mod range; diff --git a/drivers/gpu/drm/asahi/vm/range.rs b/drivers/gpu/drm/asahi/vm/range.rs new file mode 100644 index 00000000000000..727d29a5cccf02 --- /dev/null +++ b/drivers/gpu/drm/asahi/vm/range.rs @@ -0,0 +1,125 @@ +// SPDX-License-Identifier: GPL-2.0 or MIT + +// Copied from tyr's mmu/vm/range.rs + +//! Range allocator. +//! +//! This module allows you to search for unused ranges to store GEM objects. + +use kernel::alloc::Flags; +use kernel::maple_tree::MapleTreeAlloc; +use kernel::prelude::*; +use kernel::sync::Arc; + +use core::ops::Range; + +/// The actual storage for the ranges. +/// +/// All ranges must fit within the `range` field. +/// +/// The implementation is different on 32-bit and 64-bit cpus. On 64-bit, the 64-bit addresses are +/// stored directly in the maple tree, but on 32-bit, the maple tree stores the ranges translated +/// in the range zero until `range.end-range.start`. This is done because the maple tree uses +/// unsigned long as its address type, which is too small to store the 64-bit address directly on +/// 32-bit machines. +#[pin_data] +struct RangeAllocInner { + #[pin] + maple: MapleTreeAlloc<()>, + range: Range, +} + +/// This object allows you to allocate ranges on the inner maple tree. +pub(crate) struct RangeAlloc { + inner: Arc, +} + +/// Represents a live range in the maple tree. +/// +/// The destructor removes the range from the maple tree, allowing others to allocate it in the +/// future. +pub(crate) struct LiveRange { + inner: Arc, + offset: u64, + size: usize, +} + +impl RangeAlloc { + pub(crate) fn new(start: u64, end: u64, gfp: Flags) -> Result { + if end < start { + return Err(EINVAL); + } + + let inner = Arc::pin_init( + try_pin_init!(RangeAllocInner { + maple <- MapleTreeAlloc::new(), + range: start..end, + }), + gfp, + )?; + + Ok(RangeAlloc { inner }) + } + + pub(crate) fn allocate(&self, size: usize, gfp: Flags) -> Result { + let maple_start = self.inner.range.start as usize; + let maple_end = self.inner.range.end as usize; + + let offset = self + .inner + .maple + .alloc_range(size, (), maple_start..maple_end, gfp)?; + + Ok(LiveRange { + inner: self.inner.clone(), + offset: offset as u64, + size, + }) + } + + pub(crate) fn insert(&self, start: u64, end: u64, gfp: Flags) -> Result { + if end < start { + return Err(EINVAL); + } + if start < self.inner.range.start { + return Err(EINVAL); + } + if end > self.inner.range.end { + return Err(EINVAL); + } + + self.inner + .maple + .insert_range(start as usize..end as usize, (), gfp)?; + + Ok(LiveRange { + inner: self.inner.clone(), + offset: start, + size: (end - start) as usize, + }) + } +} + +impl LiveRange { + pub(crate) fn size(&self) -> usize { + self.size + } + + pub(crate) fn start(&self) -> u64 { + self.offset + } + + pub(crate) fn end(&self) -> u64 { + self.offset + self.size as u64 + } + + pub(crate) fn range(&self) -> Range { + self.start()..self.end() + } +} + +impl Drop for LiveRange { + fn drop(&mut self) { + self.inner.maple.erase(self.offset as usize); + } +} diff --git a/drivers/gpu/drm/asahi/workqueue.rs b/drivers/gpu/drm/asahi/workqueue.rs new file mode 100644 index 00000000000000..55b6058d50224f --- /dev/null +++ b/drivers/gpu/drm/asahi/workqueue.rs @@ -0,0 +1,1015 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! GPU command execution queues +//! +//! The AGX GPU firmware schedules GPU work commands out of work queues, which are ring buffers of +//! pointers to work commands. There can be an arbitrary number of work queues. Work queues have an +//! associated type (vertex, fragment, or compute) and may only contain generic commands or commands +//! specific to that type. +//! +//! This module manages queueing work commands into a work queue and submitting them for execution +//! by the firmware. An active work queue needs an event to signal completion of its work, which is +//! owned by what we call a batch. This event then notifies the work queue when work is completed, +//! and that triggers freeing of all resources associated with that work. An idle work queue gives +//! up its associated event. + +use crate::debug::*; +use crate::fw::channels::{ + ChannelErrorType, + PipeType, // +}; +use crate::fw::types::*; +use crate::fw::workqueue::*; +use crate::no_debug; +use crate::object::OpaqueGpuObject; +use crate::{ + channel, + driver, + event, + fw, + gpu, + regs, // +}; +use core::any::Any; +use core::num::NonZeroU64; +use core::sync::atomic::Ordering; +use kernel::{ + dma_fence, + error::code::*, + new_mutex, + prelude::*, + sync::{ + lock::{ + mutex::MutexBackend, + Guard, // + }, + Arc, + Mutex, // + }, + workqueue::{ + self, + impl_has_work, + new_work, + Work, + WorkItem, // + }, // +}; + +pub(crate) trait OpaqueCommandObject: OpaqueGpuObject {} + +impl OpaqueCommandObject for GpuObject where T: Command {} + +const DEBUG_CLASS: DebugFlags = DebugFlags::WorkQueue; + +const MAX_JOB_SLOTS: u32 = 127; + +/// An enum of possible errors that might cause a piece of work to fail execution. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) enum WorkError { + /// GPU timeout (command execution took too long). + Timeout, + /// GPU MMU fault (invalid access). + Fault(regs::FaultInfo), + /// Work failed due to an error caused by other concurrent GPU work. + Killed, + /// Channel error + ChannelError(ChannelErrorType), + /// The GPU crashed. + NoDevice, + /// Unknown reason. + Unknown, +} + +impl From for kernel::error::Error { + fn from(err: WorkError) -> Self { + match err { + WorkError::Timeout => ETIMEDOUT, + // Not EFAULT because that's for userspace faults + WorkError::Fault(_) => EIO, + WorkError::Unknown => ENODATA, + WorkError::Killed => ECANCELED, + WorkError::NoDevice => ENODEV, + WorkError::ChannelError(_) => EIO, + } + } +} + +/// A GPU context tracking structure, which must be explicitly invalidated when dropped. +pub(crate) struct GpuContext { + dev: driver::AsahiDevRef, + data: Option>>, +} +no_debug!(GpuContext); + +impl GpuContext { + /// Allocate a new GPU context. + pub(crate) fn new( + dev: &driver::AsahiDevice, + alloc: &mut gpu::KernelAllocators, + buffer: Arc, + ) -> Result { + Ok(GpuContext { + dev: dev.into(), + data: Some(KBox::new( + alloc.shared.new_object( + fw::workqueue::GpuContextData { _buffer: buffer }, + |_inner| Default::default(), + )?, + GFP_KERNEL, + )?), + }) + } + + /// Returns the GPU pointer to the inner GPU context data structure. + pub(crate) fn gpu_pointer(&self) -> GpuPointer<'_, fw::workqueue::GpuContextData> { + self.data.as_ref().unwrap().gpu_pointer() + } +} + +impl Drop for GpuContext { + fn drop(&mut self) { + mod_dev_dbg!(self.dev, "GpuContext: Freeing GPU context\n"); + let data = self.data.take().unwrap(); + (*self.dev).gpu.free_context(data); + } +} + +struct SubmittedWork +where + O: OpaqueCommandObject, + C: FnOnce(Option) + Send + Sync + 'static, +{ + object: O, + value: EventValue, + error: Option, + wptr: u32, + vm_slot: u32, + callback: Option, + fence: dma_fence::Fence, +} + +pub(crate) trait GenSubmittedWork: Send + Sync { + fn gpu_va(&self) -> NonZeroU64; + fn value(&self) -> event::EventValue; + fn wptr(&self) -> u32; + fn set_wptr(&mut self, wptr: u32); + fn mark_error(&mut self, error: WorkError); + fn complete(&mut self); + fn get_fence(&self) -> dma_fence::Fence; +} + +#[pin_data] +struct SubmittedWorkContainer { + #[pin] + work: Work, + inner: KBox, +} + +impl_has_work! { + impl HasWork for SubmittedWorkContainer { self.work } +} + +impl WorkItem for SubmittedWorkContainer { + type Pointer = Pin>; + + fn run(this: Pin>) { + mod_pr_debug!("WorkQueue: Freeing command @ {:?}\n", this.inner.gpu_va()); + } +} + +impl SubmittedWorkContainer { + fn inner_mut(self: Pin<&mut Self>) -> &mut KBox { + // SAFETY: inner does not require structural pinning. + unsafe { &mut self.get_unchecked_mut().inner } + } +} + +impl) + Send + Sync> GenSubmittedWork + for SubmittedWork +{ + fn gpu_va(&self) -> NonZeroU64 { + self.object.gpu_va() + } + + fn value(&self) -> event::EventValue { + self.value + } + + fn wptr(&self) -> u32 { + self.wptr + } + + fn set_wptr(&mut self, wptr: u32) { + self.wptr = wptr; + } + + fn complete(&mut self) { + if let Some(cb) = self.callback.take() { + cb(self.error); + } + } + + fn mark_error(&mut self, error: WorkError) { + mod_pr_debug!("WorkQueue: Command at value {:#x?} failed\n", self.value); + self.error = Some(match error { + WorkError::Fault(info) if info.vm_slot != self.vm_slot => WorkError::Killed, + err => err, + }); + } + + fn get_fence(&self) -> dma_fence::Fence { + self.fence.clone() + } +} + +/// Inner data for managing a single work queue. +#[versions(AGX)] +struct WorkQueueInner { + dev: driver::AsahiDevRef, + event_manager: Arc, + info: GpuObject, + new: bool, + pipe_type: PipeType, + size: u32, + wptr: u32, + pending: KVec>>, + last_token: Option, + pending_jobs: usize, + last_submitted: Option, + last_completed: Option, + event: Option<(event::Event, event::EventValue)>, + priority: u32, + commit_seq: u64, + submit_seq: u64, + event_seq: u64, +} + +/// An instance of a work queue. +#[versions(AGX)] +#[pin_data] +pub(crate) struct WorkQueue { + info_pointer: GpuWeakPointer, + #[pin] + inner: Mutex, +} + +#[versions(AGX)] +impl WorkQueueInner::ver { + /// Return the GPU done pointer, representing how many work items have been completed by the + /// GPU. + fn doneptr(&self) -> u32 { + self.info + .state + .with(|raw, _inner| raw.gpu_doneptr.load(Ordering::Acquire)) + } +} + +#[versions(AGX)] +#[derive(Copy, Clone)] +pub(crate) struct QueueEventInfo { + pub(crate) stamp_pointer: GpuWeakPointer, + pub(crate) fw_stamp_pointer: GpuWeakPointer, + pub(crate) slot: u32, + pub(crate) value: event::EventValue, + pub(crate) cmd_seq: u64, + pub(crate) event_seq: u64, + pub(crate) info_ptr: GpuWeakPointer, +} + +#[versions(AGX)] +pub(crate) struct Job { + wq: Arc, + event_info: QueueEventInfo::ver, + start_value: EventValue, + pending: KVec>>, + committed: bool, + submitted: bool, + event_count: usize, + fence: dma_fence::Fence, +} + +#[versions(AGX)] +pub(crate) struct JobSubmission<'a> { + inner: Option>, + wptr: u32, + event_count: usize, + command_count: usize, +} + +#[versions(AGX)] +impl Job::ver { + pub(crate) fn event_info(&self) -> QueueEventInfo::ver { + let mut info = self.event_info; + info.cmd_seq += self.pending.len() as u64; + info.event_seq += self.event_count as u64; + + info + } + + pub(crate) fn next_seq(&mut self) { + self.event_count += 1; + self.event_info.value.increment(); + } + + pub(crate) fn add( + &mut self, + command: O, + vm_slot: u32, + ) -> Result { + self.add_cb(command, vm_slot, |_| {}) + } + + pub(crate) fn add_cb( + &mut self, + command: O, + vm_slot: u32, + callback: impl FnOnce(Option) + Sync + Send + 'static, + ) -> Result { + if self.committed { + pr_err!("WorkQueue: Tried to mutate committed Job\n"); + return Err(EINVAL); + } + + let fence = self.fence.clone(); + let value = self.event_info.value.next(); + + self.pending.push( + KBox::try_pin_init( + try_pin_init!(SubmittedWorkContainer { + work <- new_work!("SubmittedWorkWrapper::work"), + inner: KBox::new(SubmittedWork::<_, _> { + object: command, + value, + error: None, + callback: Some(callback), + wptr: 0, + vm_slot, + fence, + }, GFP_KERNEL)? + }), + GFP_KERNEL, + )?, + GFP_KERNEL, + )?; + + Ok(()) + } + + pub(crate) fn commit(&mut self) -> Result { + if self.committed { + pr_err!("WorkQueue: Tried to commit committed Job\n"); + return Err(EINVAL); + } + + if self.pending.is_empty() { + pr_err!("WorkQueue: Job::commit() with no commands\n"); + return Err(EINVAL); + } + + let mut inner = self.wq.inner.lock(); + + let ev = inner.event.as_mut().expect("WorkQueue: Job lost its event"); + + if ev.1 != self.start_value { + pr_err!( + "WorkQueue: Job::commit() out of order (event slot {} {:?} != {:?}\n", + ev.0.slot(), + ev.1, + self.start_value + ); + return Err(EINVAL); + } + + ev.1 = self.event_info.value; + inner.commit_seq += self.pending.len() as u64; + inner.event_seq += self.event_count as u64; + self.committed = true; + + Ok(()) + } + + pub(crate) fn can_submit(&self) -> Option { + let inner = self.wq.inner.lock(); + if inner.free_slots() > self.event_count && inner.free_space() > self.pending.len() { + None + } else if let Some(work) = inner.pending.first() { + Some(work.inner.get_fence()) + } else { + pr_err!( + "WorkQueue: Cannot submit, but queue is empty? {} > {}, {} > {} (pend={} ls={:#x?} lc={:#x?}) ev={:#x?} cur={:#x?} slot {:?}\n", + inner.free_slots(), + self.event_count, + inner.free_space(), + self.pending.len(), + inner.pending.len(), + inner.last_submitted, + inner.last_completed, + inner.event.as_ref().map(|a| a.1), + inner.event.as_ref().map(|a| a.0.current()), + inner.event.as_ref().map(|a| a.0.slot()), + ); + None + } + } + + pub(crate) fn submit(&mut self) -> Result> { + if !self.committed { + pr_err!("WorkQueue: Tried to submit uncommitted Job\n"); + return Err(EINVAL); + } + + if self.submitted { + pr_err!("WorkQueue: Tried to submit Job twice\n"); + return Err(EINVAL); + } + + if self.pending.is_empty() { + pr_err!("WorkQueue: Job::submit() with no commands\n"); + return Err(EINVAL); + } + + let mut inner = self.wq.inner.lock(); + + if inner.submit_seq != self.event_info.cmd_seq { + pr_err!( + "WorkQueue: Job::submit() out of order (submit_seq {} != {})\n", + inner.submit_seq, + self.event_info.cmd_seq + ); + return Err(EINVAL); + } + + if inner.commit_seq < (self.event_info.cmd_seq + self.pending.len() as u64) { + pr_err!( + "WorkQueue: Job::submit() out of order (commit_seq {} != {})\n", + inner.commit_seq, + (self.event_info.cmd_seq + self.pending.len() as u64) + ); + return Err(EINVAL); + } + + let mut wptr = inner.wptr; + let command_count = self.pending.len(); + + if inner.free_space() <= command_count { + pr_err!("WorkQueue: Job does not fit in ring buffer\n"); + return Err(EBUSY); + } + + inner.pending.reserve(command_count, GFP_KERNEL)?; + + inner.last_submitted = Some(self.event_info.value); + mod_dev_dbg!( + inner.dev, + "WorkQueue: submitting {} cmds at {:#x?}, lc {:#x?}, cur {:#x?}, pending {}, events {}\n", + self.pending.len(), + inner.last_submitted, + inner.last_completed, + inner.event.as_ref().map(|a| a.0.current()), + inner.pending.len(), + self.event_count, + ); + + for mut command in self.pending.drain(..) { + command.as_mut().inner_mut().set_wptr(wptr); + + let next_wptr = (wptr + 1) % inner.size; + assert!(inner.doneptr() != next_wptr); + inner.info.ring[wptr as usize] = command.inner.gpu_va().get(); + wptr = next_wptr; + + // Cannot fail, since we did a reserve(1) above + inner + .pending + .push(command, GFP_KERNEL) + .expect("push() failed after reserve()"); + } + + self.submitted = true; + + Ok(JobSubmission::ver { + inner: Some(inner), + wptr, + command_count, + event_count: self.event_count, + }) + } +} + +#[versions(AGX)] +impl<'a> JobSubmission::ver<'a> { + pub(crate) fn run(mut self, channel: &mut channel::PipeChannel::ver) { + let command_count = self.command_count; + let mut inner = self.inner.take().expect("No inner?"); + let wptr = self.wptr; + core::mem::forget(self); + + inner + .info + .state + .with(|raw, _inner| raw.cpu_wptr.store(wptr, Ordering::Release)); + + inner.wptr = wptr; + + let event = inner.event.as_mut().expect("JobSubmission lost its event"); + + let event_slot = event.0.slot(); + + let msg = fw::channels::RunWorkQueueMsg::ver { + pipe_type: inner.pipe_type, + work_queue: Some(inner.info.weak_pointer()), + wptr: inner.wptr, + event_slot, + is_new: inner.new, + __pad: Default::default(), + }; + channel.send(&msg); + inner.new = false; + + inner.submit_seq += command_count as u64; + } + + pub(crate) fn pipe_type(&self) -> PipeType { + self.inner.as_ref().expect("No inner?").pipe_type + } + + pub(crate) fn priority(&self) -> u32 { + self.inner.as_ref().expect("No inner?").priority + } +} + +#[versions(AGX)] +impl Drop for Job::ver { + fn drop(&mut self) { + mod_pr_debug!("WorkQueue: Dropping Job\n"); + let mut inner = self.wq.inner.lock(); + + if !self.committed { + pr_info!( + "WorkQueue: Dropping uncommitted job with {} events\n", + self.event_count + ); + } + + if self.committed && !self.submitted { + let pipe_type = inner.pipe_type; + let event = inner.event.as_mut().expect("Job lost its event"); + pr_info!( + "WorkQueue({:?}): Roll back {} events (slot {} val {:#x?}) and {} commands\n", + pipe_type, + self.event_count, + event.0.slot(), + event.1, + self.pending.len() + ); + event.1.sub(self.event_count as u32); + inner.commit_seq -= self.pending.len() as u64; + inner.event_seq -= self.event_count as u64; + } + + inner.pending_jobs -= 1; + + if inner.pending.is_empty() && inner.pending_jobs == 0 { + mod_pr_debug!("WorkQueue({:?}): Dropping event\n", inner.pipe_type); + inner.event = None; + inner.last_submitted = None; + inner.last_completed = None; + } + mod_pr_debug!("WorkQueue({:?}): Dropped Job\n", inner.pipe_type); + } +} + +#[versions(AGX)] +impl<'a> Drop for JobSubmission::ver<'a> { + fn drop(&mut self) { + let inner = self.inner.as_mut().expect("No inner?"); + mod_pr_debug!("WorkQueue({:?}): Dropping JobSubmission\n", inner.pipe_type); + + let new_len = inner.pending.len() - self.command_count; + inner.pending.truncate(new_len); + + let pipe_type = inner.pipe_type; + let event = inner.event.as_mut().expect("JobSubmission lost its event"); + pr_info!( + "WorkQueue({:?}): JobSubmission: Roll back {} events (slot {} val {:#x?}) and {} commands\n", + pipe_type, + self.event_count, + event.0.slot(), + event.1, + self.command_count + ); + event.1.sub(self.event_count as u32); + let val = event.1; + inner.commit_seq -= self.command_count as u64; + inner.event_seq -= self.event_count as u64; + inner.last_submitted = Some(val); + mod_pr_debug!("WorkQueue({:?}): Dropped JobSubmission\n", inner.pipe_type); + } +} + +#[versions(AGX)] +impl WorkQueueInner::ver { + /// Return the number of free entries in the workqueue + pub(crate) fn free_space(&self) -> usize { + self.size as usize - self.pending.len() - 1 + } + + pub(crate) fn free_slots(&self) -> usize { + let busy_slots = if let Some(ls) = self.last_submitted { + let lc = self + .last_completed + .expect("last_submitted but not completed?"); + ls.delta(&lc) + } else { + 0 + }; + + ((MAX_JOB_SLOTS as i32) - busy_slots).max(0) as usize + } +} + +#[versions(AGX)] +impl WorkQueue::ver { + /// Create a new WorkQueue of a given type and priority. + #[allow(clippy::too_many_arguments)] + pub(crate) fn new( + dev: &driver::AsahiDevice, + alloc: &mut gpu::KernelAllocators, + event_manager: Arc, + gpu_context: Arc, + notifier_list: Arc>, + pipe_type: PipeType, + id: u64, + priority: u32, + size: u32, + ) -> Result> { + let gpu_buf = alloc.private.array_empty_tagged(0x2c18, b"GPBF")?; + let mut state = alloc.shared.new_default::()?; + let ring = alloc.shared.array_empty(size as usize)?; + let mut prio = *raw::PRIORITY.get(priority as usize).ok_or(EINVAL)?; + + if pipe_type == PipeType::Compute && !debug_enabled(DebugFlags::Debug0) { + // Hack to disable compute preemption until we fix it + prio.0 = 0; + prio.5 = 1; + } + + let inner = WorkQueueInner::ver { + dev: dev.into(), + event_manager, + // Use shared (coherent) state with verbose faults so we can dump state correctly + info: if debug_enabled(DebugFlags::VerboseFaults) { + &mut alloc.shared + } else { + &mut alloc.private + } + .new_init( + try_init!(QueueInfo::ver { + state: { + state.with_mut(|raw, _inner| { + raw.rb_size = size; + }); + state + }, + ring, + gpu_buf, + notifier_list: notifier_list, + gpu_context: gpu_context, + }), + |inner, _p| { + try_init!(raw::QueueInfo::ver { + state: inner.state.gpu_pointer(), + ring: inner.ring.gpu_pointer(), + notifier_list: inner.notifier_list.gpu_pointer(), + gpu_buf: inner.gpu_buf.gpu_pointer(), + gpu_rptr1: Default::default(), + gpu_rptr2: Default::default(), + gpu_rptr3: Default::default(), + event_id: AtomicI32::new(-1), + priority: prio, + unk_4c: -1, + uuid: id as u32, + unk_54: -1, + unk_58: Default::default(), + busy: Default::default(), + __pad: Default::default(), + #[ver(V >= V13_2 && G < G14X)] + unk_84_0: 0, + unk_84_state: Default::default(), + error_count: Default::default(), + unk_8c: 0, + unk_90: 0, + unk_94: 0, + pending: Default::default(), + unk_9c: 0, + gpu_context: inner.gpu_context.gpu_pointer(), + unk_a8: Default::default(), + #[ver(V >= V13_2 && G < G14X)] + unk_b0: 0, + }) + }, + )?, + new: true, + pipe_type, + size, + wptr: 0, + pending: KVec::new(), + last_token: None, + event: None, + priority, + pending_jobs: 0, + commit_seq: 0, + submit_seq: 0, + event_seq: 0, + last_completed: None, + last_submitted: None, + }; + + let info_pointer = inner.info.weak_pointer(); + + Arc::pin_init( + pin_init!(Self { + info_pointer, + inner <- match pipe_type { + PipeType::Vertex => new_mutex!(inner, "WorkQueue::inner (Vertex)"), + PipeType::Fragment => new_mutex!(inner, "WorkQueue::inner (Fragment)"), + PipeType::Compute => new_mutex!(inner, "WorkQueue::inner (Compute)"), + }, + }), + GFP_KERNEL, + ) + } + + pub(crate) fn event_info(&self) -> Option { + let inner = self.inner.lock(); + + inner.event.as_ref().map(|ev| QueueEventInfo::ver { + stamp_pointer: ev.0.stamp_pointer(), + fw_stamp_pointer: ev.0.fw_stamp_pointer(), + slot: ev.0.slot(), + value: ev.1, + cmd_seq: inner.commit_seq, + event_seq: inner.event_seq, + info_ptr: self.info_pointer, + }) + } + + pub(crate) fn new_job(self: &Arc, fence: dma_fence::Fence) -> Result { + let mut inner = self.inner.lock(); + + if inner.event.is_none() { + mod_pr_debug!("WorkQueue({:?}): Grabbing event\n", inner.pipe_type); + let event = inner.event_manager.get(inner.last_token, self.clone())?; + let cur = event.current(); + inner.last_token = Some(event.token()); + mod_pr_debug!( + "WorkQueue({:?}): Grabbed event slot {}: {:#x?}\n", + inner.pipe_type, + event.slot(), + cur + ); + inner.event = Some((event, cur)); + inner.last_submitted = Some(cur); + inner.last_completed = Some(cur); + } + + inner.pending_jobs += 1; + + let ev = &inner.event.as_ref().unwrap(); + + mod_pr_debug!( + "WorkQueue({:?}): New job at value {:#x?} slot {}\n", + inner.pipe_type, + ev.1, + ev.0.slot() + ); + Ok(Job::ver { + wq: self.clone(), + event_info: QueueEventInfo::ver { + stamp_pointer: ev.0.stamp_pointer(), + fw_stamp_pointer: ev.0.fw_stamp_pointer(), + slot: ev.0.slot(), + value: ev.1, + cmd_seq: inner.commit_seq, + event_seq: inner.event_seq, + info_ptr: self.info_pointer, + }, + start_value: ev.1, + pending: KVec::new(), + event_count: 0, + committed: false, + submitted: false, + fence, + }) + } + + pub(crate) fn pipe_type(&self) -> PipeType { + self.inner.lock().pipe_type + } + + pub(crate) fn dump_info(&self) { + pr_info!("WorkQueue @ {:?}:", self.info_pointer); + self.inner.lock().info.with(|raw, _inner| { + pr_info!(" GPU rptr1: {:#x}", raw.gpu_rptr1.load(Ordering::Relaxed)); + pr_info!(" GPU rptr1: {:#x}", raw.gpu_rptr2.load(Ordering::Relaxed)); + pr_info!(" GPU rptr1: {:#x}", raw.gpu_rptr3.load(Ordering::Relaxed)); + pr_info!(" Event ID: {:#x}", raw.event_id.load(Ordering::Relaxed)); + pr_info!(" Busy: {:#x}", raw.busy.load(Ordering::Relaxed)); + pr_info!(" Unk 84: {:#x}", raw.unk_84_state.load(Ordering::Relaxed)); + pr_info!( + " Error count: {:#x}", + raw.error_count.load(Ordering::Relaxed) + ); + pr_info!(" Pending: {:#x}", raw.pending.load(Ordering::Relaxed)); + }); + } + + pub(crate) fn info_pointer(&self) -> GpuWeakPointer { + self.info_pointer + } +} + +/// Trait used to erase the version-specific type of WorkQueues, to avoid leaking +/// version-specificity into the event module. +pub(crate) trait WorkQueue { + /// Cast as an Any type. + fn as_any(&self) -> &dyn Any; + + fn signal(&self) -> bool; + fn mark_error(&self, value: event::EventValue, error: WorkError); + fn fail_all(&self, error: WorkError); +} + +#[versions(AGX)] +impl WorkQueue for WorkQueue::ver { + fn as_any(&self) -> &dyn Any { + self + } + + /// Signal a workqueue that some work was completed. + /// + /// This will check the event stamp value to find out exactly how many commands were processed. + fn signal(&self) -> bool { + let mut inner = self.inner.lock(); + let event = inner.event.as_ref(); + let value = match event { + None => { + mod_pr_debug!("WorkQueue: signal() called but no event?\n"); + + if inner.pending_jobs > 0 || !inner.pending.is_empty() { + pr_crit!("WorkQueue: signal() called with no event and pending jobs.\n"); + } + return true; + } + Some(event) => event.0.current(), + }; + + if let Some(lc) = inner.last_completed { + if value < lc { + pr_err!( + "WorkQueue: event rolled back? cur {:#x?}, lc {:#x?}, ls {:#x?}", + value, + inner.last_completed, + inner.last_submitted + ); + } + } else { + pr_crit!("WorkQueue: signal() called with no last_completed.\n"); + } + inner.last_completed = Some(value); + + mod_pr_debug!( + "WorkQueue({:?}): Signaling event {:?} value {:#x?}\n", + inner.pipe_type, + inner.last_token, + value + ); + + let mut completed_commands: usize = 0; + + for cmd in inner.pending.iter() { + if cmd.inner.value() <= value { + mod_pr_debug!( + "WorkQueue({:?}): Command at value {:#x?} complete\n", + inner.pipe_type, + cmd.inner.value() + ); + completed_commands += 1; + } else { + break; + } + } + + if completed_commands == 0 { + return inner.pending.is_empty(); + } + + let last_wptr = inner.pending[completed_commands - 1].inner.wptr(); + let pipe_type = inner.pipe_type; + + for mut cmd in inner.pending.drain(..completed_commands) { + mod_pr_debug!( + "WorkQueue({:?}): Queueing command @ {:?} for cleanup\n", + pipe_type, + cmd.inner.gpu_va() + ); + cmd.as_mut().inner_mut().complete(); + workqueue::system().enqueue(cmd); + } + + mod_pr_debug!( + "WorkQueue({:?}): Completed {} commands, left pending {}, ls {:#x?}, lc {:#x?}\n", + inner.pipe_type, + completed_commands, + inner.pending.len(), + inner.last_submitted, + inner.last_completed, + ); + + inner + .info + .state + .with(|raw, _inner| raw.cpu_freeptr.store(last_wptr, Ordering::Release)); + + let empty = inner.pending.is_empty(); + if empty && inner.pending_jobs == 0 { + inner.event = None; + inner.last_submitted = None; + inner.last_completed = None; + } + + empty + } + + /// Mark this queue's work up to a certain stamp value as having failed. + fn mark_error(&self, value: event::EventValue, error: WorkError) { + // If anything is marked completed, we can consider it successful + // at this point, even if we didn't get the signal event yet. + self.signal(); + + let mut inner = self.inner.lock(); + + if inner.event.is_none() { + mod_pr_debug!("WorkQueue: signal_fault() called but no event?\n"); + + if inner.pending_jobs > 0 || !inner.pending.is_empty() { + pr_crit!("WorkQueue: signal_fault() called with no event and pending jobs.\n"); + } + return; + } + + mod_pr_debug!( + "WorkQueue({:?}): Signaling fault for event {:?} at value {:#x?}\n", + inner.pipe_type, + inner.last_token, + value + ); + + for cmd in inner.pending.iter_mut() { + if cmd.inner.value() <= value { + cmd.as_mut().inner_mut().mark_error(error); + } else { + break; + } + } + } + + /// Mark all of this queue's work as having failed, and complete it. + fn fail_all(&self, error: WorkError) { + // If anything is marked completed, we can consider it successful + // at this point, even if we didn't get the signal event yet. + self.signal(); + + let mut inner = self.inner.lock(); + + if inner.event.is_none() { + mod_pr_debug!("WorkQueue: fail_all() called but no event?\n"); + + if inner.pending_jobs > 0 || !inner.pending.is_empty() { + pr_crit!("WorkQueue: fail_all() called with no event and pending jobs.\n"); + } + return; + } + + mod_pr_debug!( + "WorkQueue({:?}): Failing all jobs {:?}\n", + inner.pipe_type, + error + ); + + let mut cmds = KVec::new(); + + core::mem::swap(&mut inner.pending, &mut cmds); + + if inner.pending_jobs == 0 { + inner.event = None; + } + + core::mem::drop(inner); + + for mut cmd in cmds { + cmd.as_mut().inner_mut().mark_error(error); + cmd.as_mut().inner_mut().complete(); + } + } +} diff --git a/drivers/gpu/drm/ast/ast_cursor.c b/drivers/gpu/drm/ast/ast_cursor.c index 2d3ad7610c2e9c..7da0a2d463e6c4 100644 --- a/drivers/gpu/drm/ast/ast_cursor.c +++ b/drivers/gpu/drm/ast/ast_cursor.c @@ -92,12 +92,17 @@ static void ast_set_cursor_image(struct ast_device *ast, const u8 *src, unsigned int width, unsigned int height) { u8 __iomem *dst = ast_plane_vaddr(&ast->cursor_plane.base); - u32 csum; - - csum = ast_cursor_calculate_checksum(src, width, height); + u32 csum = ast_cursor_calculate_checksum(src, width, height); /* write pixel data */ +#if defined(__BIG_ENDIAN) + unsigned int i; + + for (i = 0; i < AST_HWC_SIZE; i += 2) + writew(swab16(*(const __u16 *)&src[i]), &dst[i]); +#else memcpy_toio(dst, src, AST_HWC_SIZE); +#endif /* write checksum + signature */ dst += AST_HWC_SIZE; diff --git a/drivers/gpu/drm/ast/ast_dp501.c b/drivers/gpu/drm/ast/ast_dp501.c index 9e19d8c1773083..677c52c0d99a3c 100644 --- a/drivers/gpu/drm/ast/ast_dp501.c +++ b/drivers/gpu/drm/ast/ast_dp501.c @@ -436,7 +436,7 @@ static void ast_init_analog(struct ast_device *ast) /* Finally, clear bits [17:16] of SCU2c */ data = ast_read32(ast, 0x1202c); data &= 0xfffcffff; - ast_write32(ast, 0, data); + ast_write32(ast, 0x1202c, data); /* Disable DVO */ ast_set_index_reg_mask(ast, AST_IO_VGACRI, 0xa3, 0xcf, 0x00); diff --git a/drivers/gpu/drm/ast/ast_mode.c b/drivers/gpu/drm/ast/ast_mode.c index cd08990a10f936..57c6fbc3232b04 100644 --- a/drivers/gpu/drm/ast/ast_mode.c +++ b/drivers/gpu/drm/ast/ast_mode.c @@ -526,12 +526,18 @@ static int ast_primary_plane_helper_atomic_check(struct drm_plane *plane, static void ast_handle_damage(struct ast_plane *ast_plane, struct iosys_map *src, struct drm_framebuffer *fb, - const struct drm_rect *clip) + const struct drm_rect *clip, + struct drm_format_conv_state *fmtcnv_state) { struct iosys_map dst = IOSYS_MAP_INIT_VADDR_IOMEM(ast_plane_vaddr(ast_plane)); iosys_map_incr(&dst, drm_fb_clip_offset(fb->pitches[0], fb->format, clip)); + +#if defined(__BIG_ENDIAN) + drm_fb_swab(&dst, fb->pitches, src, fb, clip, !src[0].is_iomem, fmtcnv_state); +#else drm_fb_memcpy(&dst, fb->pitches, src, fb, clip); +#endif } static void ast_primary_plane_helper_atomic_update(struct drm_plane *plane, @@ -561,7 +567,8 @@ static void ast_primary_plane_helper_atomic_update(struct drm_plane *plane, if (drm_gem_fb_begin_cpu_access(fb, DMA_FROM_DEVICE) == 0) { drm_atomic_helper_damage_iter_init(&iter, old_plane_state, plane_state); drm_atomic_for_each_plane_damage(&iter, &damage) { - ast_handle_damage(ast_plane, shadow_plane_state->data, fb, &damage); + ast_handle_damage(ast_plane, shadow_plane_state->data, fb, &damage, + &shadow_plane_state->fmtcnv_state); } drm_gem_fb_end_cpu_access(fb, DMA_FROM_DEVICE); diff --git a/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_plane.c b/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_plane.c index 92132be9823f1e..e55e88d44e8299 100644 --- a/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_plane.c +++ b/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_plane.c @@ -79,8 +79,6 @@ drm_plane_state_to_atmel_hlcdc_plane_state(struct drm_plane_state *s) return container_of(s, struct atmel_hlcdc_plane_state, base); } -#define SUBPIXEL_MASK 0xffff - static uint32_t rgb_formats[] = { DRM_FORMAT_C8, DRM_FORMAT_XRGB4444, @@ -745,24 +743,15 @@ static int atmel_hlcdc_plane_atomic_check(struct drm_plane *p, if (ret || !s->visible) return ret; - hstate->src_x = s->src.x1; - hstate->src_y = s->src.y1; - hstate->src_w = drm_rect_width(&s->src); - hstate->src_h = drm_rect_height(&s->src); + hstate->src_x = s->src.x1 >> 16; + hstate->src_y = s->src.y1 >> 16; + hstate->src_w = drm_rect_width(&s->src) >> 16; + hstate->src_h = drm_rect_height(&s->src) >> 16; hstate->crtc_x = s->dst.x1; hstate->crtc_y = s->dst.y1; hstate->crtc_w = drm_rect_width(&s->dst); hstate->crtc_h = drm_rect_height(&s->dst); - if ((hstate->src_x | hstate->src_y | hstate->src_w | hstate->src_h) & - SUBPIXEL_MASK) - return -EINVAL; - - hstate->src_x >>= 16; - hstate->src_y >>= 16; - hstate->src_w >>= 16; - hstate->src_h >>= 16; - hstate->nplanes = fb->format->num_planes; if (hstate->nplanes > ATMEL_HLCDC_LAYER_MAX_PLANES) return -EINVAL; @@ -1155,32 +1144,6 @@ static int atmel_hlcdc_plane_alloc_dscrs(struct drm_plane *p, return -ENOMEM; } -static void atmel_hlcdc_plane_reset(struct drm_plane *p) -{ - struct atmel_hlcdc_plane_state *state; - - if (p->state) { - state = drm_plane_state_to_atmel_hlcdc_plane_state(p->state); - - if (state->base.fb) - drm_framebuffer_put(state->base.fb); - - kfree(state); - p->state = NULL; - } - - state = kzalloc(sizeof(*state), GFP_KERNEL); - if (state) { - if (atmel_hlcdc_plane_alloc_dscrs(p, state)) { - kfree(state); - drm_err(p->dev, - "Failed to allocate initial plane state\n"); - return; - } - __drm_atomic_helper_plane_reset(p, &state->base); - } -} - static struct drm_plane_state * atmel_hlcdc_plane_atomic_duplicate_state(struct drm_plane *p) { @@ -1197,8 +1160,7 @@ atmel_hlcdc_plane_atomic_duplicate_state(struct drm_plane *p) return NULL; } - if (copy->base.fb) - drm_framebuffer_get(copy->base.fb); + __drm_atomic_helper_plane_duplicate_state(p, ©->base); return ©->base; } @@ -1216,12 +1178,37 @@ static void atmel_hlcdc_plane_atomic_destroy_state(struct drm_plane *p, state->dscrs[i]->self); } - if (s->fb) - drm_framebuffer_put(s->fb); + __drm_atomic_helper_plane_destroy_state(s); kfree(state); } +static void atmel_hlcdc_plane_reset(struct drm_plane *p) +{ + struct atmel_hlcdc_plane_state *state; + struct atmel_hlcdc_dc *dc = p->dev->dev_private; + struct atmel_hlcdc_plane *plane = drm_plane_to_atmel_hlcdc_plane(p); + + if (p->state) { + atmel_hlcdc_plane_atomic_destroy_state(p, p->state); + p->state = NULL; + } + + state = kzalloc(sizeof(*state), GFP_KERNEL); + if (state) { + if (atmel_hlcdc_plane_alloc_dscrs(p, state)) { + kfree(state); + drm_err(p->dev, + "Failed to allocate initial plane state\n"); + return; + } + __drm_atomic_helper_plane_reset(p, &state->base); + } + + if (plane->layer.desc->layout.csc) + dc->desc->ops->lcdc_csc_init(plane, plane->layer.desc); +} + static const struct drm_plane_funcs layer_plane_funcs = { .update_plane = drm_atomic_helper_update_plane, .disable_plane = drm_atomic_helper_disable_plane, diff --git a/drivers/gpu/drm/bridge/analogix/anx7625.c b/drivers/gpu/drm/bridge/analogix/anx7625.c index 6f3fdcb6afdb9d..4e49e4f28d5525 100644 --- a/drivers/gpu/drm/bridge/analogix/anx7625.c +++ b/drivers/gpu/drm/bridge/analogix/anx7625.c @@ -1801,7 +1801,7 @@ static const struct drm_edid *anx7625_edid_read(struct anx7625_data *ctx) return NULL; } - ctx->cached_drm_edid = drm_edid_alloc(edid_buf, FOUR_BLOCK_SIZE); + ctx->cached_drm_edid = drm_edid_alloc(edid_buf, edid_num * ONE_BLOCK_SIZE); kfree(edid_buf); out: diff --git a/drivers/gpu/drm/bridge/samsung-dsim.c b/drivers/gpu/drm/bridge/samsung-dsim.c index eabc4c32f6ab4a..ad8c6aa49d48a6 100644 --- a/drivers/gpu/drm/bridge/samsung-dsim.c +++ b/drivers/gpu/drm/bridge/samsung-dsim.c @@ -1881,6 +1881,14 @@ static int samsung_dsim_register_te_irq(struct samsung_dsim *dsi, struct device return 0; } +static void samsung_dsim_unregister_te_irq(struct samsung_dsim *dsi) +{ + if (dsi->te_gpio) { + free_irq(gpiod_to_irq(dsi->te_gpio), dsi); + gpiod_put(dsi->te_gpio); + } +} + static int samsung_dsim_host_attach(struct mipi_dsi_host *host, struct mipi_dsi_device *device) { @@ -1955,13 +1963,13 @@ static int samsung_dsim_host_attach(struct mipi_dsi_host *host, if (!(device->mode_flags & MIPI_DSI_MODE_VIDEO)) { ret = samsung_dsim_register_te_irq(dsi, &device->dev); if (ret) - return ret; + goto err_remove_bridge; } if (pdata->host_ops && pdata->host_ops->attach) { ret = pdata->host_ops->attach(dsi, device); if (ret) - return ret; + goto err_unregister_te_irq; } dsi->lanes = device->lanes; @@ -1969,14 +1977,13 @@ static int samsung_dsim_host_attach(struct mipi_dsi_host *host, dsi->mode_flags = device->mode_flags; return 0; -} -static void samsung_dsim_unregister_te_irq(struct samsung_dsim *dsi) -{ - if (dsi->te_gpio) { - free_irq(gpiod_to_irq(dsi->te_gpio), dsi); - gpiod_put(dsi->te_gpio); - } +err_unregister_te_irq: + if (!(device->mode_flags & MIPI_DSI_MODE_VIDEO)) + samsung_dsim_unregister_te_irq(dsi); +err_remove_bridge: + drm_bridge_remove(&dsi->bridge); + return ret; } static int samsung_dsim_host_detach(struct mipi_dsi_host *host, diff --git a/drivers/gpu/drm/bridge/synopsys/dw-dp.c b/drivers/gpu/drm/bridge/synopsys/dw-dp.c index 43234245248473..07f7a2e0d9f2a9 100644 --- a/drivers/gpu/drm/bridge/synopsys/dw-dp.c +++ b/drivers/gpu/drm/bridge/synopsys/dw-dp.c @@ -2049,7 +2049,9 @@ struct dw_dp *dw_dp_bind(struct device *dev, struct drm_encoder *encoder, bridge->type = DRM_MODE_CONNECTOR_DisplayPort; bridge->ycbcr_420_allowed = true; - devm_drm_bridge_add(dev, bridge); + ret = devm_drm_bridge_add(dev, bridge); + if (ret) + return ERR_PTR(ret); dp->aux.dev = dev; dp->aux.drm_dev = encoder->dev; diff --git a/drivers/gpu/drm/bridge/synopsys/dw-hdmi-qp.c b/drivers/gpu/drm/bridge/synopsys/dw-hdmi-qp.c index 60166919c5b548..ace9d8bcdd197c 100644 --- a/drivers/gpu/drm/bridge/synopsys/dw-hdmi-qp.c +++ b/drivers/gpu/drm/bridge/synopsys/dw-hdmi-qp.c @@ -838,7 +838,7 @@ static int dw_hdmi_qp_config_audio_infoframe(struct dw_hdmi_qp *hdmi, regmap_bulk_write(hdmi->regm, PKT_AUDI_CONTENTS0, &header_bytes, 1); regmap_bulk_write(hdmi->regm, PKT_AUDI_CONTENTS1, &buffer[3], 1); - regmap_bulk_write(hdmi->regm, PKT_AUDI_CONTENTS2, &buffer[4], 1); + regmap_bulk_write(hdmi->regm, PKT_AUDI_CONTENTS2, &buffer[7], 1); /* Enable ACR, AUDI, AMD */ dw_hdmi_qp_mod(hdmi, diff --git a/drivers/gpu/drm/bridge/ti-sn65dsi83.c b/drivers/gpu/drm/bridge/ti-sn65dsi83.c index fffb47b62f437b..43344c2e15b778 100644 --- a/drivers/gpu/drm/bridge/ti-sn65dsi83.c +++ b/drivers/gpu/drm/bridge/ti-sn65dsi83.c @@ -351,9 +351,9 @@ static u8 sn65dsi83_get_dsi_range(struct sn65dsi83 *ctx, * DSI_CLK = mode clock * bpp / dsi_data_lanes / 2 * the 2 is there because the bus is DDR. */ - return DIV_ROUND_UP(clamp((unsigned int)mode->clock * - mipi_dsi_pixel_format_to_bpp(ctx->dsi->format) / - ctx->dsi->lanes / 2, 40000U, 500000U), 5000U); + return clamp((unsigned int)mode->clock * + mipi_dsi_pixel_format_to_bpp(ctx->dsi->format) / + ctx->dsi->lanes / 2, 40000U, 500000U) / 5000U; } static u8 sn65dsi83_get_dsi_div(struct sn65dsi83 *ctx) @@ -474,6 +474,7 @@ static void sn65dsi83_atomic_pre_enable(struct drm_bridge *bridge, struct drm_atomic_state *state) { struct sn65dsi83 *ctx = bridge_to_sn65dsi83(bridge); + const unsigned int dual_factor = ctx->lvds_dual_link ? 2 : 1; const struct drm_bridge_state *bridge_state; const struct drm_crtc_state *crtc_state; const struct drm_display_mode *mode; @@ -606,18 +607,18 @@ static void sn65dsi83_atomic_pre_enable(struct drm_bridge *bridge, /* 32 + 1 pixel clock to ensure proper operation */ le16val = cpu_to_le16(32 + 1); regmap_bulk_write(ctx->regmap, REG_VID_CHA_SYNC_DELAY_LOW, &le16val, 2); - le16val = cpu_to_le16(mode->hsync_end - mode->hsync_start); + le16val = cpu_to_le16((mode->hsync_end - mode->hsync_start) / dual_factor); regmap_bulk_write(ctx->regmap, REG_VID_CHA_HSYNC_PULSE_WIDTH_LOW, &le16val, 2); le16val = cpu_to_le16(mode->vsync_end - mode->vsync_start); regmap_bulk_write(ctx->regmap, REG_VID_CHA_VSYNC_PULSE_WIDTH_LOW, &le16val, 2); regmap_write(ctx->regmap, REG_VID_CHA_HORIZONTAL_BACK_PORCH, - mode->htotal - mode->hsync_end); + (mode->htotal - mode->hsync_end) / dual_factor); regmap_write(ctx->regmap, REG_VID_CHA_VERTICAL_BACK_PORCH, mode->vtotal - mode->vsync_end); regmap_write(ctx->regmap, REG_VID_CHA_HORIZONTAL_FRONT_PORCH, - mode->hsync_start - mode->hdisplay); + (mode->hsync_start - mode->hdisplay) / dual_factor); regmap_write(ctx->regmap, REG_VID_CHA_VERTICAL_FRONT_PORCH, mode->vsync_start - mode->vdisplay); regmap_write(ctx->regmap, REG_VID_CHA_TEST_PATTERN, 0x00); diff --git a/drivers/gpu/drm/bridge/ti-sn65dsi86.c b/drivers/gpu/drm/bridge/ti-sn65dsi86.c index 276d05d25ad8b8..98d64ad791d040 100644 --- a/drivers/gpu/drm/bridge/ti-sn65dsi86.c +++ b/drivers/gpu/drm/bridge/ti-sn65dsi86.c @@ -1415,6 +1415,7 @@ static int ti_sn_bridge_probe(struct auxiliary_device *adev, { struct ti_sn65dsi86 *pdata = dev_get_drvdata(adev->dev.parent); struct device_node *np = pdata->dev->of_node; + const struct i2c_client *client = to_i2c_client(pdata->dev); int ret; pdata->next_bridge = devm_drm_of_get_bridge(&adev->dev, np, 1, 0); @@ -1433,8 +1434,9 @@ static int ti_sn_bridge_probe(struct auxiliary_device *adev, ? DRM_MODE_CONNECTOR_DisplayPort : DRM_MODE_CONNECTOR_eDP; if (pdata->bridge.type == DRM_MODE_CONNECTOR_DisplayPort) { - pdata->bridge.ops = DRM_BRIDGE_OP_EDID | DRM_BRIDGE_OP_DETECT | - DRM_BRIDGE_OP_HPD; + pdata->bridge.ops = DRM_BRIDGE_OP_EDID | DRM_BRIDGE_OP_DETECT; + if (client->irq) + pdata->bridge.ops |= DRM_BRIDGE_OP_HPD; /* * If comms were already enabled they would have been enabled * with the wrong value of HPD_DISABLE. Update it now. Comms diff --git a/drivers/gpu/drm/display/drm_dp_mst_topology.c b/drivers/gpu/drm/display/drm_dp_mst_topology.c index 64e5c176d5cce9..be749dcad3b585 100644 --- a/drivers/gpu/drm/display/drm_dp_mst_topology.c +++ b/drivers/gpu/drm/display/drm_dp_mst_topology.c @@ -4572,7 +4572,8 @@ int drm_dp_atomic_release_time_slots(struct drm_atomic_state *state, if (!payload->delete) { payload->pbn = 0; payload->delete = true; - topology_state->payload_mask &= ~BIT(payload->vcpi - 1); + if (payload->vcpi > 0) + topology_state->payload_mask &= ~BIT(payload->vcpi - 1); } return 0; diff --git a/drivers/gpu/drm/drm_atomic.c b/drivers/gpu/drm/drm_atomic.c index 67e095e398a34d..06f0205664fc5c 100644 --- a/drivers/gpu/drm/drm_atomic.c +++ b/drivers/gpu/drm/drm_atomic.c @@ -641,6 +641,38 @@ drm_atomic_get_colorop_state(struct drm_atomic_state *state, } EXPORT_SYMBOL(drm_atomic_get_colorop_state); +/** + * drm_atomic_get_old_colorop_state - get colorop state, if it exists + * @state: global atomic state object + * @colorop: colorop to grab + * + * This function returns the old colorop state for the given colorop, or + * NULL if the colorop is not part of the global atomic state. + */ +struct drm_colorop_state * +drm_atomic_get_old_colorop_state(struct drm_atomic_state *state, + struct drm_colorop *colorop) +{ + return state->colorops[drm_colorop_index(colorop)].old_state; +} +EXPORT_SYMBOL(drm_atomic_get_old_colorop_state); + +/** + * drm_atomic_get_new_colorop_state - get colorop state, if it exists + * @state: global atomic state object + * @colorop: colorop to grab + * + * This function returns the new colorop state for the given colorop, or + * NULL if the colorop is not part of the global atomic state. + */ +struct drm_colorop_state * +drm_atomic_get_new_colorop_state(struct drm_atomic_state *state, + struct drm_colorop *colorop) +{ + return state->colorops[drm_colorop_index(colorop)].new_state; +} +EXPORT_SYMBOL(drm_atomic_get_new_colorop_state); + static bool plane_switching_crtc(const struct drm_plane_state *old_plane_state, const struct drm_plane_state *new_plane_state) diff --git a/drivers/gpu/drm/drm_atomic_helper.c b/drivers/gpu/drm/drm_atomic_helper.c index 5beea645035f20..cc1f0c102414f1 100644 --- a/drivers/gpu/drm/drm_atomic_helper.c +++ b/drivers/gpu/drm/drm_atomic_helper.c @@ -34,6 +34,7 @@ #include #include #include +#include #include #include #include diff --git a/drivers/gpu/drm/drm_bridge.c b/drivers/gpu/drm/drm_bridge.c index 8f355df883d8ac..250bf8fa516779 100644 --- a/drivers/gpu/drm/drm_bridge.c +++ b/drivers/gpu/drm/drm_bridge.c @@ -1465,11 +1465,17 @@ EXPORT_SYMBOL(devm_drm_put_bridge); static void drm_bridge_debugfs_show_bridge(struct drm_printer *p, struct drm_bridge *bridge, unsigned int idx, - bool lingering) + bool lingering, + bool scoped) { + unsigned int refcount = kref_read(&bridge->refcount); + + if (scoped) + refcount--; + drm_printf(p, "bridge[%u]: %ps\n", idx, bridge->funcs); - drm_printf(p, "\trefcount: %u%s\n", kref_read(&bridge->refcount), + drm_printf(p, "\trefcount: %u%s\n", refcount, lingering ? " [lingering]" : ""); drm_printf(p, "\ttype: [%d] %s\n", @@ -1503,10 +1509,10 @@ static int allbridges_show(struct seq_file *m, void *data) mutex_lock(&bridge_lock); list_for_each_entry(bridge, &bridge_list, list) - drm_bridge_debugfs_show_bridge(&p, bridge, idx++, false); + drm_bridge_debugfs_show_bridge(&p, bridge, idx++, false, false); list_for_each_entry(bridge, &bridge_lingering_list, list) - drm_bridge_debugfs_show_bridge(&p, bridge, idx++, true); + drm_bridge_debugfs_show_bridge(&p, bridge, idx++, true, false); mutex_unlock(&bridge_lock); @@ -1521,7 +1527,7 @@ static int encoder_bridges_show(struct seq_file *m, void *data) unsigned int idx = 0; drm_for_each_bridge_in_chain_scoped(encoder, bridge) - drm_bridge_debugfs_show_bridge(&p, bridge, idx++, false); + drm_bridge_debugfs_show_bridge(&p, bridge, idx++, false, true); return 0; } diff --git a/drivers/gpu/drm/drm_buddy.c b/drivers/gpu/drm/drm_buddy.c index 2f279b46bd2cf8..fd34d3755f7c5c 100644 --- a/drivers/gpu/drm/drm_buddy.c +++ b/drivers/gpu/drm/drm_buddy.c @@ -420,6 +420,7 @@ void drm_buddy_fini(struct drm_buddy *mm) for_each_free_tree(i) kfree(mm->free_trees[i]); + kfree(mm->free_trees); kfree(mm->roots); } EXPORT_SYMBOL(drm_buddy_fini); @@ -1155,6 +1156,15 @@ int drm_buddy_alloc_blocks(struct drm_buddy *mm, order = fls(pages) - 1; min_order = ilog2(min_block_size) - ilog2(mm->chunk_size); + if (order > mm->max_order || size > mm->size) { + if ((flags & DRM_BUDDY_CONTIGUOUS_ALLOCATION) && + !(flags & DRM_BUDDY_RANGE_ALLOCATION)) + return __alloc_contig_try_harder(mm, original_size, + original_min_size, blocks); + + return -EINVAL; + } + do { order = min(order, (unsigned int)fls(pages) - 1); BUG_ON(order > mm->max_order); diff --git a/drivers/gpu/drm/drm_client_modeset.c b/drivers/gpu/drm/drm_client_modeset.c index fc4caf7da5fcd3..4a72f323e83e36 100644 --- a/drivers/gpu/drm/drm_client_modeset.c +++ b/drivers/gpu/drm/drm_client_modeset.c @@ -930,7 +930,8 @@ int drm_client_modeset_probe(struct drm_client_dev *client, unsigned int width, mutex_unlock(&client->modeset_mutex); out: kfree(crtcs); - modes_destroy(dev, modes, connector_count); + if (modes) + modes_destroy(dev, modes, connector_count); kfree(modes); kfree(offsets); kfree(enabled); diff --git a/drivers/gpu/drm/drm_gem_shmem_helper.c b/drivers/gpu/drm/drm_gem_shmem_helper.c index f13eb5f36e8a97..0db3fe08a57b75 100644 --- a/drivers/gpu/drm/drm_gem_shmem_helper.c +++ b/drivers/gpu/drm/drm_gem_shmem_helper.c @@ -15,6 +15,8 @@ #include #endif +#include + #include #include #include @@ -894,6 +896,67 @@ struct drm_gem_object *drm_gem_shmem_prime_import_no_map(struct drm_device *dev, } EXPORT_SYMBOL_GPL(drm_gem_shmem_prime_import_no_map); +/* + * Kunit helpers + */ + +#if IS_ENABLED(CONFIG_KUNIT) +int drm_gem_shmem_vmap(struct drm_gem_shmem_object *shmem, struct iosys_map *map) +{ + struct drm_gem_object *obj = &shmem->base; + int ret; + + ret = dma_resv_lock_interruptible(obj->resv, NULL); + if (ret) + return ret; + ret = drm_gem_shmem_vmap_locked(shmem, map); + dma_resv_unlock(obj->resv); + + return ret; +} +EXPORT_SYMBOL_IF_KUNIT(drm_gem_shmem_vmap); + +void drm_gem_shmem_vunmap(struct drm_gem_shmem_object *shmem, struct iosys_map *map) +{ + struct drm_gem_object *obj = &shmem->base; + + dma_resv_lock_interruptible(obj->resv, NULL); + drm_gem_shmem_vunmap_locked(shmem, map); + dma_resv_unlock(obj->resv); +} +EXPORT_SYMBOL_IF_KUNIT(drm_gem_shmem_vunmap); + +int drm_gem_shmem_madvise(struct drm_gem_shmem_object *shmem, int madv) +{ + struct drm_gem_object *obj = &shmem->base; + int ret; + + ret = dma_resv_lock_interruptible(obj->resv, NULL); + if (ret) + return ret; + ret = drm_gem_shmem_madvise_locked(shmem, madv); + dma_resv_unlock(obj->resv); + + return ret; +} +EXPORT_SYMBOL_IF_KUNIT(drm_gem_shmem_madvise); + +int drm_gem_shmem_purge(struct drm_gem_shmem_object *shmem) +{ + struct drm_gem_object *obj = &shmem->base; + int ret; + + ret = dma_resv_lock_interruptible(obj->resv, NULL); + if (ret) + return ret; + drm_gem_shmem_purge_locked(shmem); + dma_resv_unlock(obj->resv); + + return 0; +} +EXPORT_SYMBOL_IF_KUNIT(drm_gem_shmem_purge); +#endif + MODULE_DESCRIPTION("DRM SHMEM memory-management helpers"); MODULE_IMPORT_NS("DMA_BUF"); MODULE_LICENSE("GPL"); diff --git a/drivers/gpu/drm/drm_gpuvm.c b/drivers/gpu/drm/drm_gpuvm.c index 0de47e83d84df1..c19cfb00f1fcec 100644 --- a/drivers/gpu/drm/drm_gpuvm.c +++ b/drivers/gpu/drm/drm_gpuvm.c @@ -2352,6 +2352,7 @@ op_map_cb(const struct drm_gpuvm_ops *fn, void *priv, op.map.va.range = req->map.va.range; op.map.gem.obj = req->map.gem.obj; op.map.gem.offset = req->map.gem.offset; + op.map.flags = req->map.flags; return fn->sm_step_map(&op, priv); } @@ -2390,6 +2391,97 @@ op_unmap_cb(const struct drm_gpuvm_ops *fn, void *priv, return fn->sm_step_unmap(&op, priv); } +static bool can_merge_flags(struct drm_gpuvm *gpuvm, enum drm_gpuva_flags a, + enum drm_gpuva_flags b) +{ + if (gpuvm->ops->sm_can_merge_flags) + return gpuvm->ops->sm_can_merge_flags(a, b); + return a == b; +} + +static bool __can_merge(struct drm_gpuvm *gpuvm, const struct drm_gpuva_op_map *a, + const struct drm_gpuva_op_map *b) +{ + /* Only GEM-based mappings can be merged, and they must point to + * the same GEM object. + */ + if (a->gem.obj != b->gem.obj || !a->gem.obj) + return false; + + if (can_merge_flags(gpuvm, a->flags, b->flags)) + return false; + + /* Order VAs for the rest of the checks. */ + if (a->va.addr > b->va.addr) + swap(a, b); + + /* We assume the caller already checked that VAs overlap or are + * contiguous. + */ + if (drm_WARN_ON(gpuvm->drm, b->va.addr > a->va.addr + a->va.range)) + return false; + + if (a->flags & DRM_GPUVA_REPEAT) { + u64 va_diff = b->va.addr - a->va.addr; + + /* If this is a repeated mapping, both the GEM range + * and offset must match. + */ + if (a->gem.range != b->gem.range || + a->gem.offset != b->gem.offset) + return false; + + /* The difference between the VA addresses must be a + * multiple of the repeated range, otherwise there's + * a shift. + */ + if (do_div(va_diff, a->gem.range)) + return false; + + return true; + } + + /* We intentionally ignore u64 underflows because all we care about + * here is whether the VA diff matches the GEM offset diff. + */ + return b->va.addr - a->va.addr == b->gem.offset - a->gem.offset; +} + +static bool can_merge(struct drm_gpuvm *gpuvm, const struct drm_gpuva *a, + const struct drm_gpuva_op_map *b) +{ + struct drm_gpuva_op_map tmp = { + .va.addr = a->va.addr, + .va.range = a->va.range, + .gem.offset = a->gem.offset, + .gem.obj = a->gem.obj, + .flags = a->flags, + }; + + return __can_merge(gpuvm, &tmp, b); +} + +static int validate_map_request(struct drm_gpuvm *gpuvm, + const struct drm_gpuva_op_map *req) +{ + if (unlikely(!drm_gpuvm_range_valid(gpuvm, req->va.addr, req->va.range))) + return -EINVAL; + + if (req->flags & DRM_GPUVA_REPEAT) { + u64 va_range = req->va.range; + + /* For a repeated mapping, GEM range must be > 0 + * and a multiple of the VA range. + */ + if (unlikely(!req->gem.range || + va_range < req->gem.range || + do_div(va_range, req->gem.range))) + return -EINVAL; + } + + return 0; +} + static int __drm_gpuvm_sm_map(struct drm_gpuvm *gpuvm, const struct drm_gpuvm_ops *ops, void *priv, @@ -2405,7 +2497,8 @@ __drm_gpuvm_sm_map(struct drm_gpuvm *gpuvm, u64 req_end = req_addr + req_range; int ret; - if (unlikely(!drm_gpuvm_range_valid(gpuvm, req_addr, req_range))) + ret = validate_map_request(gpuvm, &req->map); + if (unlikely(ret)) return -EINVAL; drm_gpuvm_for_each_va_range_safe(va, next, gpuvm, req_addr, req_end) { @@ -2414,7 +2507,7 @@ __drm_gpuvm_sm_map(struct drm_gpuvm *gpuvm, u64 addr = va->va.addr; u64 range = va->va.range; u64 end = addr + range; - bool merge = !!va->gem.obj; + bool merge = can_merge(gpuvm, va, &req->map); if (madvise && obj) continue; @@ -2442,7 +2535,10 @@ __drm_gpuvm_sm_map(struct drm_gpuvm *gpuvm, .va.addr = req_end, .va.range = range - req_range, .gem.obj = obj, - .gem.offset = offset + req_range, + .gem.range = va->gem.range, + .gem.offset = offset + + (va->flags & DRM_GPUVA_REPEAT ? 0 : req_range), + .flags = va->flags, }; struct drm_gpuva_op_unmap u = { .va = va, @@ -2463,7 +2559,9 @@ __drm_gpuvm_sm_map(struct drm_gpuvm *gpuvm, .va.addr = addr, .va.range = ls_range, .gem.obj = obj, + .gem.range = va->gem.range, .gem.offset = offset, + .flags = va->flags, }; struct drm_gpuva_op_unmap u = { .va = va }; @@ -2505,8 +2603,10 @@ __drm_gpuvm_sm_map(struct drm_gpuvm *gpuvm, .va.addr = req_end, .va.range = end - req_end, .gem.obj = obj, - .gem.offset = offset + ls_range + - req_range, + .gem.range = va->gem.range, + .gem.offset = offset + + (va->flags & DRM_GPUVA_REPEAT ? 0 : ls_range + req_range), + .flags = va->flags, }; ret = op_remap_cb(ops, priv, &p, &n, &u); @@ -2543,7 +2643,10 @@ __drm_gpuvm_sm_map(struct drm_gpuvm *gpuvm, .va.addr = req_end, .va.range = end - req_end, .gem.obj = obj, - .gem.offset = offset + req_end - addr, + .gem.range = va->gem.range, + .gem.offset = offset + + (va->flags & DRM_GPUVA_REPEAT ? 0 : req_end - addr), + .flags = va->flags, }; struct drm_gpuva_op_unmap u = { .va = va, @@ -2594,7 +2697,9 @@ __drm_gpuvm_sm_unmap(struct drm_gpuvm *gpuvm, prev.va.addr = addr; prev.va.range = req_addr - addr; prev.gem.obj = obj; + prev.gem.range = va->gem.range; prev.gem.offset = offset; + prev.flags = va->flags; prev_split = true; } @@ -2603,7 +2708,10 @@ __drm_gpuvm_sm_unmap(struct drm_gpuvm *gpuvm, next.va.addr = req_end; next.va.range = end - req_end; next.gem.obj = obj; - next.gem.offset = offset + (req_end - addr); + prev.gem.range = va->gem.range; + next.gem.offset = offset + + (va->flags & DRM_GPUVA_REPEAT ? 0 : req_end - addr); + next.flags = va->flags; next_split = true; } @@ -3146,6 +3254,55 @@ drm_gpuvm_prefetch_ops_create(struct drm_gpuvm *gpuvm, } EXPORT_SYMBOL_GPL(drm_gpuvm_prefetch_ops_create); +/** + * drm_gpuvm_bo_unmap() - unmaps a GEM + * @vm_bo: the &drm_gpuvm_bo abstraction + * + * This function calls the unmap callback for every GPUVA attached to a GEM. + * + * It is the callers responsibility to protect the GEMs GPUVA list against + * concurrent access using the GEMs dma_resv lock. + * + * Returns: a pointer to the &drm_gpuva_ops on success, an ERR_PTR on failure + */ +int +drm_gpuvm_bo_unmap(struct drm_gpuvm_bo *vm_bo, void *priv) +{ + struct drm_gpuva_ops *ops; + struct drm_gpuva_op *op; + int ret; + + if (unlikely(!vm_bo->vm)) + return -EINVAL; + + const struct drm_gpuvm_ops *vm_ops = vm_bo->vm->ops; + + if (unlikely(!(vm_ops && vm_ops->sm_step_unmap))) + return -EINVAL; + + if (drm_gpuvm_immediate_mode(vm_bo->vm)) { + guard(mutex)(&vm_bo->obj->gpuva.lock); + ops = drm_gpuvm_bo_unmap_ops_create(vm_bo); + } else { + ops = drm_gpuvm_bo_unmap_ops_create(vm_bo); + } + if (IS_ERR(ops)) + return PTR_ERR(ops); + + drm_gpuva_for_each_op(op, ops) { + drm_WARN_ON(vm_bo->vm->drm, op->op != DRM_GPUVA_OP_UNMAP); + + ret = op_unmap_cb(vm_ops, priv, op->unmap.va, false, false); + if (ret) + goto cleanup; + } + +cleanup: + drm_gpuva_ops_free(vm_bo->vm, ops); + return ret; +} +EXPORT_SYMBOL_GPL(drm_gpuvm_bo_unmap); + /** * drm_gpuvm_bo_unmap_ops_create() - creates the &drm_gpuva_ops to unmap a GEM * @vm_bo: the &drm_gpuvm_bo abstraction diff --git a/drivers/gpu/drm/drm_ioc32.c b/drivers/gpu/drm/drm_ioc32.c index e6b5b06de1487b..f3e40d1e6098b8 100644 --- a/drivers/gpu/drm/drm_ioc32.c +++ b/drivers/gpu/drm/drm_ioc32.c @@ -28,6 +28,7 @@ * IN THE SOFTWARE. */ #include +#include #include #include @@ -374,6 +375,7 @@ long drm_compat_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) if (nr >= ARRAY_SIZE(drm_compat_ioctls)) return drm_ioctl(filp, cmd, arg); + nr = array_index_nospec(nr, ARRAY_SIZE(drm_compat_ioctls)); fn = drm_compat_ioctls[nr].fn; if (!fn) return drm_ioctl(filp, cmd, arg); diff --git a/drivers/gpu/drm/drm_plane.c b/drivers/gpu/drm/drm_plane.c index b143589717e648..bed2562bf911b2 100644 --- a/drivers/gpu/drm/drm_plane.c +++ b/drivers/gpu/drm/drm_plane.c @@ -1867,9 +1867,9 @@ int drm_plane_create_color_pipeline_property(struct drm_plane *plane, prop = drm_property_create_enum(plane->dev, DRM_MODE_PROP_ATOMIC, "COLOR_PIPELINE", all_pipelines, len); - if (IS_ERR(prop)) { + if (!prop) { kfree(all_pipelines); - return PTR_ERR(prop); + return -ENOMEM; } drm_object_attach_property(&plane->base, prop, 0); diff --git a/drivers/gpu/drm/drm_property.c b/drivers/gpu/drm/drm_property.c index 596272149a3599..3c88b5fbdf28c5 100644 --- a/drivers/gpu/drm/drm_property.c +++ b/drivers/gpu/drm/drm_property.c @@ -562,7 +562,7 @@ drm_property_create_blob(struct drm_device *dev, size_t length, if (!length || length > INT_MAX - sizeof(struct drm_property_blob)) return ERR_PTR(-EINVAL); - blob = kvzalloc(sizeof(struct drm_property_blob)+length, GFP_KERNEL); + blob = kvzalloc(sizeof(struct drm_property_blob) + length, GFP_KERNEL_ACCOUNT); if (!blob) return ERR_PTR(-ENOMEM); diff --git a/drivers/gpu/drm/drm_syncobj.c b/drivers/gpu/drm/drm_syncobj.c index e1b0fa4000cdd5..7eb2cdbc574a02 100644 --- a/drivers/gpu/drm/drm_syncobj.c +++ b/drivers/gpu/drm/drm_syncobj.c @@ -900,7 +900,7 @@ drm_syncobj_handle_to_fd_ioctl(struct drm_device *dev, void *data, return drm_syncobj_export_sync_file(file_private, args->handle, point, &args->fd); - if (args->point) + if (point) return -EINVAL; return drm_syncobj_handle_to_fd(file_private, args->handle, @@ -934,7 +934,7 @@ drm_syncobj_fd_to_handle_ioctl(struct drm_device *dev, void *data, args->handle, point); - if (args->point) + if (point) return -EINVAL; return drm_syncobj_fd_to_handle(file_private, args->fd, diff --git a/drivers/gpu/drm/exynos/exynos_drm_drv.h b/drivers/gpu/drm/exynos/exynos_drm_drv.h index 23646e55f142c3..06c29ff2aac0e1 100644 --- a/drivers/gpu/drm/exynos/exynos_drm_drv.h +++ b/drivers/gpu/drm/exynos/exynos_drm_drv.h @@ -199,6 +199,7 @@ struct drm_exynos_file_private { struct exynos_drm_private { struct device *g2d_dev; struct device *dma_dev; + struct device *vidi_dev; void *mapping; /* for atomic commit */ diff --git a/drivers/gpu/drm/exynos/exynos_drm_vidi.c b/drivers/gpu/drm/exynos/exynos_drm_vidi.c index 64c69dd2966ec9..67bbf9b8bc0ef8 100644 --- a/drivers/gpu/drm/exynos/exynos_drm_vidi.c +++ b/drivers/gpu/drm/exynos/exynos_drm_vidi.c @@ -187,29 +187,37 @@ static ssize_t vidi_store_connection(struct device *dev, const char *buf, size_t len) { struct vidi_context *ctx = dev_get_drvdata(dev); - int ret; + int ret, new_connected; - ret = kstrtoint(buf, 0, &ctx->connected); + ret = kstrtoint(buf, 0, &new_connected); if (ret) return ret; - - if (ctx->connected > 1) + if (new_connected > 1) return -EINVAL; + mutex_lock(&ctx->lock); + /* * Use fake edid data for test. If raw_edid is set then it can't be * tested. */ if (ctx->raw_edid) { DRM_DEV_DEBUG_KMS(dev, "edid data is not fake data.\n"); - return -EINVAL; + ret = -EINVAL; + goto fail; } + ctx->connected = new_connected; + mutex_unlock(&ctx->lock); + DRM_DEV_DEBUG_KMS(dev, "requested connection.\n"); drm_helper_hpd_irq_event(ctx->drm_dev); return len; +fail: + mutex_unlock(&ctx->lock); + return ret; } static DEVICE_ATTR(connection, 0644, vidi_show_connection, @@ -224,9 +232,14 @@ ATTRIBUTE_GROUPS(vidi); int vidi_connection_ioctl(struct drm_device *drm_dev, void *data, struct drm_file *file_priv) { - struct vidi_context *ctx = dev_get_drvdata(drm_dev->dev); + struct exynos_drm_private *priv = drm_dev->dev_private; + struct device *dev = priv ? priv->vidi_dev : NULL; + struct vidi_context *ctx = dev ? dev_get_drvdata(dev) : NULL; struct drm_exynos_vidi_connection *vidi = data; + if (!ctx) + return -ENODEV; + if (!vidi) { DRM_DEV_DEBUG_KMS(ctx->dev, "user data for vidi is null.\n"); @@ -239,21 +252,38 @@ int vidi_connection_ioctl(struct drm_device *drm_dev, void *data, return -EINVAL; } + mutex_lock(&ctx->lock); if (ctx->connected == vidi->connection) { + mutex_unlock(&ctx->lock); DRM_DEV_DEBUG_KMS(ctx->dev, "same connection request.\n"); return -EINVAL; } + mutex_unlock(&ctx->lock); if (vidi->connection) { const struct drm_edid *drm_edid; - const struct edid *raw_edid; + const void __user *edid_userptr = u64_to_user_ptr(vidi->edid); + void *edid_buf; + struct edid hdr; size_t size; - raw_edid = (const struct edid *)(unsigned long)vidi->edid; - size = (raw_edid->extensions + 1) * EDID_LENGTH; + if (copy_from_user(&hdr, edid_userptr, sizeof(hdr))) + return -EFAULT; + + size = (hdr.extensions + 1) * EDID_LENGTH; + + edid_buf = kmalloc(size, GFP_KERNEL); + if (!edid_buf) + return -ENOMEM; - drm_edid = drm_edid_alloc(raw_edid, size); + if (copy_from_user(edid_buf, edid_userptr, size)) { + kfree(edid_buf); + return -EFAULT; + } + + drm_edid = drm_edid_alloc(edid_buf, size); + kfree(edid_buf); if (!drm_edid) return -ENOMEM; @@ -263,14 +293,21 @@ int vidi_connection_ioctl(struct drm_device *drm_dev, void *data, "edid data is invalid.\n"); return -EINVAL; } + mutex_lock(&ctx->lock); ctx->raw_edid = drm_edid; + mutex_unlock(&ctx->lock); } else { /* with connection = 0, free raw_edid */ + mutex_lock(&ctx->lock); drm_edid_free(ctx->raw_edid); ctx->raw_edid = NULL; + mutex_unlock(&ctx->lock); } + mutex_lock(&ctx->lock); ctx->connected = vidi->connection; + mutex_unlock(&ctx->lock); + drm_helper_hpd_irq_event(ctx->drm_dev); return 0; @@ -285,7 +322,7 @@ static enum drm_connector_status vidi_detect(struct drm_connector *connector, * connection request would come from user side * to do hotplug through specific ioctl. */ - return ctx->connected ? connector_status_connected : + return READ_ONCE(ctx->connected) ? connector_status_connected : connector_status_disconnected; } @@ -308,11 +345,15 @@ static int vidi_get_modes(struct drm_connector *connector) const struct drm_edid *drm_edid; int count; + mutex_lock(&ctx->lock); + if (ctx->raw_edid) drm_edid = drm_edid_dup(ctx->raw_edid); else drm_edid = drm_edid_alloc(fake_edid_info, sizeof(fake_edid_info)); + mutex_unlock(&ctx->lock); + drm_edid_connector_update(connector, drm_edid); count = drm_edid_connector_add_modes(connector); @@ -372,6 +413,7 @@ static int vidi_bind(struct device *dev, struct device *master, void *data) { struct vidi_context *ctx = dev_get_drvdata(dev); struct drm_device *drm_dev = data; + struct exynos_drm_private *priv = drm_dev->dev_private; struct drm_encoder *encoder = &ctx->encoder; struct exynos_drm_plane *exynos_plane; struct exynos_drm_plane_config plane_config = { 0 }; @@ -379,6 +421,8 @@ static int vidi_bind(struct device *dev, struct device *master, void *data) int ret; ctx->drm_dev = drm_dev; + if (priv) + priv->vidi_dev = dev; plane_config.pixel_formats = formats; plane_config.num_pixel_formats = ARRAY_SIZE(formats); @@ -424,8 +468,12 @@ static int vidi_bind(struct device *dev, struct device *master, void *data) static void vidi_unbind(struct device *dev, struct device *master, void *data) { struct vidi_context *ctx = dev_get_drvdata(dev); + struct drm_device *drm_dev = data; + struct exynos_drm_private *priv = drm_dev->dev_private; timer_delete_sync(&ctx->timer); + if (priv) + priv->vidi_dev = NULL; } static const struct component_ops vidi_component_ops = { @@ -457,9 +505,13 @@ static void vidi_remove(struct platform_device *pdev) { struct vidi_context *ctx = platform_get_drvdata(pdev); + mutex_lock(&ctx->lock); + drm_edid_free(ctx->raw_edid); ctx->raw_edid = NULL; + mutex_unlock(&ctx->lock); + component_del(&pdev->dev, &vidi_component_ops); } diff --git a/drivers/gpu/drm/hisilicon/hibmc/dp/dp_comm.h b/drivers/gpu/drm/hisilicon/hibmc/dp/dp_comm.h index 4add05c7f161a0..f9ee7ebfec55c3 100644 --- a/drivers/gpu/drm/hisilicon/hibmc/dp/dp_comm.h +++ b/drivers/gpu/drm/hisilicon/hibmc/dp/dp_comm.h @@ -40,6 +40,10 @@ struct hibmc_dp_dev { struct mutex lock; /* protects concurrent RW in hibmc_dp_reg_write_field() */ struct hibmc_dp_link link; u8 dpcd[DP_RECEIVER_CAP_SIZE]; + u8 downstream_ports[DP_MAX_DOWNSTREAM_PORTS]; + struct drm_dp_desc desc; + bool is_branch; + int hpd_status; void __iomem *serdes_base; }; diff --git a/drivers/gpu/drm/hisilicon/hibmc/dp/dp_config.h b/drivers/gpu/drm/hisilicon/hibmc/dp/dp_config.h index 08f9e1caf7fcbb..efb30a7584758c 100644 --- a/drivers/gpu/drm/hisilicon/hibmc/dp/dp_config.h +++ b/drivers/gpu/drm/hisilicon/hibmc/dp/dp_config.h @@ -17,5 +17,7 @@ #define HIBMC_DP_LINK_RATE_CAL 27 #define HIBMC_DP_SYNC_DELAY(lanes) ((lanes) == 0x2 ? 86 : 46) #define HIBMC_DP_INT_ENABLE 0xc +/* HIBMC_DP_LINK_RATE_CAL * 10000 * 80% = 216000 */ +#define DP_MODE_VALI_CAL 216000 #endif diff --git a/drivers/gpu/drm/hisilicon/hibmc/dp/dp_hw.c b/drivers/gpu/drm/hisilicon/hibmc/dp/dp_hw.c index 8f0daec7d17490..d5bd3c45649b2f 100644 --- a/drivers/gpu/drm/hisilicon/hibmc/dp/dp_hw.c +++ b/drivers/gpu/drm/hisilicon/hibmc/dp/dp_hw.c @@ -2,6 +2,7 @@ // Copyright (c) 2024 Hisilicon Limited. #include +#include #include #include "dp_config.h" #include "dp_comm.h" @@ -176,13 +177,18 @@ int hibmc_dp_hw_init(struct hibmc_dp *dp) dp_dev->link.cap.lanes = 0x2; dp_dev->link.cap.link_rate = DP_LINK_BW_8_1; - /* hdcp data */ - writel(HIBMC_DP_HDCP, dp_dev->base + HIBMC_DP_HDCP_CFG); /* int init */ writel(0, dp_dev->base + HIBMC_DP_INTR_ENABLE); writel(HIBMC_DP_INT_RST, dp_dev->base + HIBMC_DP_INTR_ORIGINAL_STATUS); + /* clr colorbar */ + writel(0, dp_dev->base + HIBMC_DP_COLOR_BAR_CTRL); /* rst */ + writel(0, dp_dev->base + HIBMC_DP_DPTX_RST_CTRL); + usleep_range(30, 50); + /* de-rst */ writel(HIBMC_DP_DPTX_RST, dp_dev->base + HIBMC_DP_DPTX_RST_CTRL); + /* hdcp data */ + writel(HIBMC_DP_HDCP, dp_dev->base + HIBMC_DP_HDCP_CFG); /* clock enable */ writel(HIBMC_DP_CLK_EN, dp_dev->base + HIBMC_DP_DPTX_CLK_CTRL); @@ -263,6 +269,16 @@ void hibmc_dp_reset_link(struct hibmc_dp *dp) dp->dp_dev->link.status.channel_equalized = false; } +u8 hibmc_dp_get_link_rate(struct hibmc_dp *dp) +{ + return dp->dp_dev->link.cap.link_rate; +} + +u8 hibmc_dp_get_lanes(struct hibmc_dp *dp) +{ + return dp->dp_dev->link.cap.lanes; +} + static const struct hibmc_dp_color_raw g_rgb_raw[] = { {CBAR_COLOR_BAR, 0x000, 0x000, 0x000}, {CBAR_WHITE, 0xfff, 0xfff, 0xfff}, @@ -305,3 +321,21 @@ void hibmc_dp_set_cbar(struct hibmc_dp *dp, const struct hibmc_dp_cbar_cfg *cfg) hibmc_dp_reg_write_field(dp_dev, HIBMC_DP_COLOR_BAR_CTRL, BIT(0), cfg->enable); writel(HIBMC_DP_SYNC_EN_MASK, dp_dev->base + HIBMC_DP_TIMING_SYNC_CTRL); } + +bool hibmc_dp_check_hpd_status(struct hibmc_dp *dp, int exp_status) +{ + u32 status; + int ret; + + ret = readl_poll_timeout(dp->dp_dev->base + HIBMC_DP_HPD_STATUS, status, + FIELD_GET(HIBMC_DP_HPD_CUR_STATE, status) == exp_status, + 1000, 100000); /* DP spec says 100ms */ + if (ret) { + drm_dbg_dp(dp->drm_dev, "wait hpd status timeout"); + return false; + } + + dp->dp_dev->hpd_status = exp_status; + + return true; +} diff --git a/drivers/gpu/drm/hisilicon/hibmc/dp/dp_hw.h b/drivers/gpu/drm/hisilicon/hibmc/dp/dp_hw.h index 665f5b166dfb5c..31316fe1ea8dbf 100644 --- a/drivers/gpu/drm/hisilicon/hibmc/dp/dp_hw.h +++ b/drivers/gpu/drm/hisilicon/hibmc/dp/dp_hw.h @@ -14,6 +14,11 @@ struct hibmc_dp_dev; +enum hibmc_hpd_status { + HIBMC_HPD_OUT, + HIBMC_HPD_IN, +}; + enum hibmc_dp_cbar_pattern { CBAR_COLOR_BAR, CBAR_WHITE, @@ -60,5 +65,8 @@ void hibmc_dp_reset_link(struct hibmc_dp *dp); void hibmc_dp_hpd_cfg(struct hibmc_dp *dp); void hibmc_dp_enable_int(struct hibmc_dp *dp); void hibmc_dp_disable_int(struct hibmc_dp *dp); +bool hibmc_dp_check_hpd_status(struct hibmc_dp *dp, int exp_status); +u8 hibmc_dp_get_link_rate(struct hibmc_dp *dp); +u8 hibmc_dp_get_lanes(struct hibmc_dp *dp); #endif diff --git a/drivers/gpu/drm/hisilicon/hibmc/dp/dp_reg.h b/drivers/gpu/drm/hisilicon/hibmc/dp/dp_reg.h index 394b1e933c3ae7..64306abcd98666 100644 --- a/drivers/gpu/drm/hisilicon/hibmc/dp/dp_reg.h +++ b/drivers/gpu/drm/hisilicon/hibmc/dp/dp_reg.h @@ -24,6 +24,9 @@ #define HIBMC_DP_CFG_AUX_READY_DATA_BYTE GENMASK(16, 12) #define HIBMC_DP_CFG_AUX GENMASK(24, 17) +#define HIBMC_DP_HPD_STATUS 0x98 +#define HIBMC_DP_HPD_CUR_STATE GENMASK(7, 4) + #define HIBMC_DP_PHYIF_CTRL0 0xa0 #define HIBMC_DP_CFG_SCRAMBLE_EN BIT(0) #define HIBMC_DP_CFG_PAT_SEL GENMASK(7, 4) diff --git a/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_dp.c b/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_dp.c index d06832e62e966e..616821e3c933bc 100644 --- a/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_dp.c +++ b/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_dp.c @@ -12,6 +12,8 @@ #include "hibmc_drm_drv.h" #include "dp/dp_hw.h" +#include "dp/dp_comm.h" +#include "dp/dp_config.h" #define DP_MASKED_SINK_HPD_PLUG_INT BIT(2) @@ -31,17 +33,76 @@ static int hibmc_dp_connector_get_modes(struct drm_connector *connector) return count; } +static bool hibmc_dp_get_dpcd(struct hibmc_dp_dev *dp_dev) +{ + int ret; + + ret = drm_dp_read_dpcd_caps(dp_dev->aux, dp_dev->dpcd); + if (ret) + return false; + + dp_dev->is_branch = drm_dp_is_branch(dp_dev->dpcd); + + ret = drm_dp_read_desc(dp_dev->aux, &dp_dev->desc, dp_dev->is_branch); + if (ret) + return false; + + ret = drm_dp_read_downstream_info(dp_dev->aux, dp_dev->dpcd, dp_dev->downstream_ports); + if (ret) + return false; + + return true; +} + static int hibmc_dp_detect(struct drm_connector *connector, struct drm_modeset_acquire_ctx *ctx, bool force) { - mdelay(200); + struct hibmc_dp *dp = to_hibmc_dp(connector); + struct hibmc_dp_dev *dp_dev = dp->dp_dev; + int ret; + + if (dp->irq_status) { + if (dp_dev->hpd_status != HIBMC_HPD_IN) + return connector_status_disconnected; + } + + if (!hibmc_dp_get_dpcd(dp_dev)) + return connector_status_disconnected; + + if (!dp_dev->is_branch) + return connector_status_connected; + + if (drm_dp_read_sink_count_cap(connector, dp_dev->dpcd, &dp_dev->desc) && + dp_dev->downstream_ports[0] & DP_DS_PORT_HPD) { + ret = drm_dp_read_sink_count(dp_dev->aux); + if (ret > 0) + return connector_status_connected; + } + + return connector_status_disconnected; +} + +static int hibmc_dp_mode_valid(struct drm_connector *connector, + const struct drm_display_mode *mode, + struct drm_modeset_acquire_ctx *ctx, + enum drm_mode_status *status) +{ + struct hibmc_dp *dp = to_hibmc_dp(connector); + u64 cur_val, max_val; - return drm_connector_helper_detect_from_ddc(connector, ctx, force); + /* check DP link BW */ + cur_val = (u64)mode->clock * HIBMC_DP_BPP; + max_val = (u64)hibmc_dp_get_link_rate(dp) * DP_MODE_VALI_CAL * hibmc_dp_get_lanes(dp); + + *status = cur_val > max_val ? MODE_CLOCK_HIGH : MODE_OK; + + return 0; } static const struct drm_connector_helper_funcs hibmc_dp_conn_helper_funcs = { .get_modes = hibmc_dp_connector_get_modes, .detect_ctx = hibmc_dp_detect, + .mode_valid_ctx = hibmc_dp_mode_valid, }; static int hibmc_dp_late_register(struct drm_connector *connector) @@ -115,7 +176,7 @@ irqreturn_t hibmc_dp_hpd_isr(int irq, void *arg) { struct drm_device *dev = (struct drm_device *)arg; struct hibmc_drm_private *priv = to_hibmc_drm_private(dev); - int idx; + int idx, exp_status; if (!drm_dev_enter(dev, &idx)) return -ENODEV; @@ -123,12 +184,14 @@ irqreturn_t hibmc_dp_hpd_isr(int irq, void *arg) if (priv->dp.irq_status & DP_MASKED_SINK_HPD_PLUG_INT) { drm_dbg_dp(&priv->dev, "HPD IN isr occur!\n"); hibmc_dp_hpd_cfg(&priv->dp); + exp_status = HIBMC_HPD_IN; } else { drm_dbg_dp(&priv->dev, "HPD OUT isr occur!\n"); hibmc_dp_reset_link(&priv->dp); + exp_status = HIBMC_HPD_OUT; } - if (dev->registered) + if (hibmc_dp_check_hpd_status(&priv->dp, exp_status)) drm_connector_helper_hpd_irq_event(&priv->dp.connector); drm_dev_exit(idx); diff --git a/drivers/gpu/drm/i915/display/g4x_dp.c b/drivers/gpu/drm/i915/display/g4x_dp.c index a3ff21b2f69fe0..4ad36ed1872498 100644 --- a/drivers/gpu/drm/i915/display/g4x_dp.c +++ b/drivers/gpu/drm/i915/display/g4x_dp.c @@ -137,7 +137,7 @@ static void intel_dp_prepare(struct intel_encoder *encoder, intel_dp->DP |= DP_SYNC_VS_HIGH; intel_dp->DP |= DP_LINK_TRAIN_OFF_CPT; - if (drm_dp_enhanced_frame_cap(intel_dp->dpcd)) + if (pipe_config->enhanced_framing) intel_dp->DP |= DP_ENHANCED_FRAMING; intel_dp->DP |= DP_PIPE_SEL_IVB(crtc->pipe); diff --git a/drivers/gpu/drm/i915/display/icl_dsi.c b/drivers/gpu/drm/i915/display/icl_dsi.c index 9230792960f29b..bc53449cdd8957 100644 --- a/drivers/gpu/drm/i915/display/icl_dsi.c +++ b/drivers/gpu/drm/i915/display/icl_dsi.c @@ -889,7 +889,7 @@ gen11_dsi_set_transcoder_timings(struct intel_encoder *encoder, * non-compressed link speeds, and simplifies down to the ratio between * compressed and non-compressed bpp. */ - if (crtc_state->dsc.compression_enable) { + if (is_vid_mode(intel_dsi) && crtc_state->dsc.compression_enable) { mul = fxp_q4_to_int(crtc_state->dsc.compressed_bpp_x16); div = mipi_dsi_pixel_format_to_bpp(intel_dsi->pixel_format); } @@ -1503,7 +1503,7 @@ static void gen11_dsi_get_timings(struct intel_encoder *encoder, struct drm_display_mode *adjusted_mode = &pipe_config->hw.adjusted_mode; - if (pipe_config->dsc.compressed_bpp_x16) { + if (is_vid_mode(intel_dsi) && pipe_config->dsc.compressed_bpp_x16) { int div = fxp_q4_to_int(pipe_config->dsc.compressed_bpp_x16); int mul = mipi_dsi_pixel_format_to_bpp(intel_dsi->pixel_format); diff --git a/drivers/gpu/drm/i915/display/intel_acpi.c b/drivers/gpu/drm/i915/display/intel_acpi.c index 68c01932f7b4f6..e06f324027bec9 100644 --- a/drivers/gpu/drm/i915/display/intel_acpi.c +++ b/drivers/gpu/drm/i915/display/intel_acpi.c @@ -96,6 +96,7 @@ static void intel_dsm_platform_mux_info(acpi_handle dhandle) if (!pkg->package.count) { DRM_DEBUG_DRIVER("no connection in _DSM\n"); + ACPI_FREE(pkg); return; } diff --git a/drivers/gpu/drm/i915/display/intel_alpm.c b/drivers/gpu/drm/i915/display/intel_alpm.c index 6372f533f65b52..affa3daefd594d 100644 --- a/drivers/gpu/drm/i915/display/intel_alpm.c +++ b/drivers/gpu/drm/i915/display/intel_alpm.c @@ -43,12 +43,6 @@ bool intel_alpm_is_alpm_aux_less(struct intel_dp *intel_dp, void intel_alpm_init(struct intel_dp *intel_dp) { - u8 dpcd; - - if (drm_dp_dpcd_readb(&intel_dp->aux, DP_RECEIVER_ALPM_CAP, &dpcd) < 0) - return; - - intel_dp->alpm_dpcd = dpcd; mutex_init(&intel_dp->alpm.lock); } @@ -564,12 +558,7 @@ void intel_alpm_disable(struct intel_dp *intel_dp) mutex_lock(&intel_dp->alpm.lock); intel_de_rmw(display, ALPM_CTL(display, cpu_transcoder), - ALPM_CTL_ALPM_ENABLE | ALPM_CTL_LOBF_ENABLE | - ALPM_CTL_ALPM_AUX_LESS_ENABLE, 0); - - intel_de_rmw(display, - PORT_ALPM_CTL(cpu_transcoder), - PORT_ALPM_CTL_ALPM_AUX_LESS_ENABLE, 0); + ALPM_CTL_ALPM_ENABLE | ALPM_CTL_LOBF_ENABLE, 0); drm_dbg_kms(display->drm, "Disabling ALPM\n"); mutex_unlock(&intel_dp->alpm.lock); diff --git a/drivers/gpu/drm/i915/display/intel_cdclk.c b/drivers/gpu/drm/i915/display/intel_cdclk.c index 37801c744b053c..13e4dde3daa978 100644 --- a/drivers/gpu/drm/i915/display/intel_cdclk.c +++ b/drivers/gpu/drm/i915/display/intel_cdclk.c @@ -2930,6 +2930,53 @@ static int intel_cdclk_update_crtc_min_cdclk(struct intel_atomic_state *state, return 0; } +static int intel_cdclk_update_crtc_min_voltage_level(struct intel_atomic_state *state, + struct intel_crtc *crtc, + u8 old_min_voltage_level, + u8 new_min_voltage_level, + bool *need_cdclk_calc) +{ + struct intel_display *display = to_intel_display(state); + struct intel_cdclk_state *cdclk_state; + bool allow_voltage_level_decrease = intel_any_crtc_needs_modeset(state); + int ret; + + if (new_min_voltage_level == old_min_voltage_level) + return 0; + + if (!allow_voltage_level_decrease && + new_min_voltage_level < old_min_voltage_level) + return 0; + + cdclk_state = intel_atomic_get_cdclk_state(state); + if (IS_ERR(cdclk_state)) + return PTR_ERR(cdclk_state); + + old_min_voltage_level = cdclk_state->min_voltage_level[crtc->pipe]; + + if (new_min_voltage_level == old_min_voltage_level) + return 0; + + if (!allow_voltage_level_decrease && + new_min_voltage_level < old_min_voltage_level) + return 0; + + cdclk_state->min_voltage_level[crtc->pipe] = new_min_voltage_level; + + ret = intel_atomic_lock_global_state(&cdclk_state->base); + if (ret) + return ret; + + *need_cdclk_calc = true; + + drm_dbg_kms(display->drm, + "[CRTC:%d:%s] min voltage level: %d -> %d\n", + crtc->base.base.id, crtc->base.name, + old_min_voltage_level, new_min_voltage_level); + + return 0; +} + int intel_cdclk_update_dbuf_bw_min_cdclk(struct intel_atomic_state *state, int old_min_cdclk, int new_min_cdclk, bool *need_cdclk_calc) @@ -3345,6 +3392,13 @@ static int intel_crtcs_calc_min_cdclk(struct intel_atomic_state *state, need_cdclk_calc); if (ret) return ret; + + ret = intel_cdclk_update_crtc_min_voltage_level(state, crtc, + old_crtc_state->min_voltage_level, + new_crtc_state->min_voltage_level, + need_cdclk_calc); + if (ret) + return ret; } return 0; diff --git a/drivers/gpu/drm/i915/display/intel_colorop.c b/drivers/gpu/drm/i915/display/intel_colorop.c index f2fc0d8780ceea..1d84933f05aa9f 100644 --- a/drivers/gpu/drm/i915/display/intel_colorop.c +++ b/drivers/gpu/drm/i915/display/intel_colorop.c @@ -2,7 +2,9 @@ /* * Copyright © 2025 Intel Corporation */ + #include "intel_colorop.h" +#include "intel_display_types.h" struct intel_colorop *to_intel_colorop(struct drm_colorop *colorop) { diff --git a/drivers/gpu/drm/i915/display/intel_colorop.h b/drivers/gpu/drm/i915/display/intel_colorop.h index 21d58eb9f3d0f6..9276eee6e75a3e 100644 --- a/drivers/gpu/drm/i915/display/intel_colorop.h +++ b/drivers/gpu/drm/i915/display/intel_colorop.h @@ -6,7 +6,9 @@ #ifndef __INTEL_COLOROP_H__ #define __INTEL_COLOROP_H__ -#include "intel_display_types.h" +enum intel_color_block; +struct drm_colorop; +struct intel_colorop; struct intel_colorop *to_intel_colorop(struct drm_colorop *colorop); struct intel_colorop *intel_colorop_alloc(void); diff --git a/drivers/gpu/drm/i915/display/intel_display.c b/drivers/gpu/drm/i915/display/intel_display.c index 095a319f8bc902..2b8a77ddcefc31 100644 --- a/drivers/gpu/drm/i915/display/intel_display.c +++ b/drivers/gpu/drm/i915/display/intel_display.c @@ -1614,7 +1614,6 @@ static void hsw_configure_cpu_transcoder(const struct intel_crtc_state *crtc_sta } intel_set_transcoder_timings(crtc_state); - intel_vrr_set_transcoder_timings(crtc_state); if (cpu_transcoder != TRANSCODER_EDP) intel_de_write(display, TRANS_MULT(display, cpu_transcoder), @@ -4579,6 +4578,7 @@ intel_crtc_prepare_cleared_state(struct intel_atomic_state *state, struct intel_crtc_state *crtc_state = intel_atomic_get_new_crtc_state(state, crtc); struct intel_crtc_state *saved_state; + int err; saved_state = intel_crtc_state_alloc(crtc); if (!saved_state) @@ -4587,7 +4587,12 @@ intel_crtc_prepare_cleared_state(struct intel_atomic_state *state, /* free the old crtc_state->hw members */ intel_crtc_free_hw_state(crtc_state); - intel_dp_tunnel_atomic_clear_stream_bw(state, crtc_state); + err = intel_dp_tunnel_atomic_clear_stream_bw(state, crtc_state); + if (err) { + kfree(saved_state); + + return err; + } /* FIXME: before the switch to atomic started, a new pipe_config was * kzalloc'd. Code that depends on any field being zero should be diff --git a/drivers/gpu/drm/i915/display/intel_display_device.h b/drivers/gpu/drm/i915/display/intel_display_device.h index b559ef43d54704..6944d081f0ad3b 100644 --- a/drivers/gpu/drm/i915/display/intel_display_device.h +++ b/drivers/gpu/drm/i915/display/intel_display_device.h @@ -189,6 +189,7 @@ struct intel_display_platforms { #define HAS_MSO(__display) (DISPLAY_VER(__display) >= 12) #define HAS_OVERLAY(__display) (DISPLAY_INFO(__display)->has_overlay) #define HAS_PIPEDMC(__display) (DISPLAY_VER(__display) >= 12) +#define HAS_PIXEL_NORMALIZER(__display) (DISPLAY_VER(__display) >= 35) #define HAS_PSR(__display) (DISPLAY_INFO(__display)->has_psr) #define HAS_PSR_HW_TRACKING(__display) (DISPLAY_INFO(__display)->has_psr_hw_tracking) #define HAS_PSR2_SEL_FETCH(__display) (DISPLAY_VER(__display) >= 12) diff --git a/drivers/gpu/drm/i915/display/intel_display_power_well.c b/drivers/gpu/drm/i915/display/intel_display_power_well.c index f4f7e73acc8742..dfdd417eb6b56e 100644 --- a/drivers/gpu/drm/i915/display/intel_display_power_well.c +++ b/drivers/gpu/drm/i915/display/intel_display_power_well.c @@ -798,7 +798,7 @@ void gen9_set_dc_state(struct intel_display *display, u32 state) power_domains->dc_state, val & mask); enable_dc6 = state & DC_STATE_EN_UPTO_DC6; - dc6_was_enabled = val & DC_STATE_EN_UPTO_DC6; + dc6_was_enabled = power_domains->dc_state & DC_STATE_EN_UPTO_DC6; if (!dc6_was_enabled && enable_dc6) intel_dmc_update_dc6_allowed_count(display, true); diff --git a/drivers/gpu/drm/i915/display/intel_display_types.h b/drivers/gpu/drm/i915/display/intel_display_types.h index 06bf8f7c0989b0..1a7188970f24e8 100644 --- a/drivers/gpu/drm/i915/display/intel_display_types.h +++ b/drivers/gpu/drm/i915/display/intel_display_types.h @@ -34,6 +34,7 @@ #include #include #include +#include #include #include #include @@ -1159,6 +1160,7 @@ struct intel_crtc_state { u32 dc3co_exitline; u16 su_y_granularity; u8 active_non_psr_pipes; + u8 entry_setup_frames; const char *no_psr_reason; /* diff --git a/drivers/gpu/drm/i915/display/intel_dmc.c b/drivers/gpu/drm/i915/display/intel_dmc.c index 6ebbd97e6351b2..12c2f8adc5d73b 100644 --- a/drivers/gpu/drm/i915/display/intel_dmc.c +++ b/drivers/gpu/drm/i915/display/intel_dmc.c @@ -1591,8 +1591,7 @@ static bool intel_dmc_get_dc6_allowed_count(struct intel_display *display, u32 * return false; mutex_lock(&power_domains->lock); - dc6_enabled = intel_de_read(display, DC_STATE_EN) & - DC_STATE_EN_UPTO_DC6; + dc6_enabled = power_domains->dc_state & DC_STATE_EN_UPTO_DC6; if (dc6_enabled) intel_dmc_update_dc6_allowed_count(display, false); diff --git a/drivers/gpu/drm/i915/display/intel_dp.c b/drivers/gpu/drm/i915/display/intel_dp.c index 0ec82fcbcf48ef..b6ce11267b92df 100644 --- a/drivers/gpu/drm/i915/display/intel_dp.c +++ b/drivers/gpu/drm/i915/display/intel_dp.c @@ -2605,16 +2605,30 @@ intel_dp_compute_config_link_bpp_limits(struct intel_dp *intel_dp, return true; } -static void -intel_dp_dsc_compute_pipe_bpp_limits(struct intel_dp *intel_dp, +static bool +intel_dp_dsc_compute_pipe_bpp_limits(struct intel_connector *connector, struct link_config_limits *limits) { - struct intel_display *display = to_intel_display(intel_dp); + struct intel_display *display = to_intel_display(connector); + const struct link_config_limits orig_limits = *limits; int dsc_min_bpc = intel_dp_dsc_min_src_input_bpc(); int dsc_max_bpc = intel_dp_dsc_max_src_input_bpc(display); - limits->pipe.max_bpp = clamp(limits->pipe.max_bpp, dsc_min_bpc * 3, dsc_max_bpc * 3); - limits->pipe.min_bpp = clamp(limits->pipe.min_bpp, dsc_min_bpc * 3, dsc_max_bpc * 3); + limits->pipe.min_bpp = max(limits->pipe.min_bpp, dsc_min_bpc * 3); + limits->pipe.max_bpp = min(limits->pipe.max_bpp, dsc_max_bpc * 3); + + if (limits->pipe.min_bpp <= 0 || + limits->pipe.min_bpp > limits->pipe.max_bpp) { + drm_dbg_kms(display->drm, + "[CONNECTOR:%d:%s] Invalid DSC src/sink input BPP (src:%d-%d pipe:%d-%d)\n", + connector->base.base.id, connector->base.name, + dsc_min_bpc * 3, dsc_max_bpc * 3, + orig_limits.pipe.min_bpp, orig_limits.pipe.max_bpp); + + return false; + } + + return true; } bool @@ -2625,6 +2639,7 @@ intel_dp_compute_config_limits(struct intel_dp *intel_dp, bool dsc, struct link_config_limits *limits) { + struct intel_display *display = to_intel_display(intel_dp); bool is_mst = intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DP_MST); struct intel_connector *connector = to_intel_connector(conn_state->connector); @@ -2637,8 +2652,7 @@ intel_dp_compute_config_limits(struct intel_dp *intel_dp, limits->min_lane_count = intel_dp_min_lane_count(intel_dp); limits->max_lane_count = intel_dp_max_lane_count(intel_dp); - limits->pipe.min_bpp = intel_dp_in_hdr_mode(conn_state) ? 30 : - intel_dp_min_bpp(crtc_state->output_format); + limits->pipe.min_bpp = intel_dp_min_bpp(crtc_state->output_format); if (is_mst) { /* * FIXME: If all the streams can't fit into the link with their @@ -2654,8 +2668,21 @@ intel_dp_compute_config_limits(struct intel_dp *intel_dp, respect_downstream_limits); } - if (dsc) - intel_dp_dsc_compute_pipe_bpp_limits(intel_dp, limits); + if (!dsc && intel_dp_in_hdr_mode(conn_state)) { + if (intel_dp_supports_dsc(intel_dp, connector, crtc_state) && + limits->pipe.max_bpp >= 30) + limits->pipe.min_bpp = max(limits->pipe.min_bpp, 30); + else + drm_dbg_kms(display->drm, + "[CONNECTOR:%d:%s] Can't force 30 bpp for HDR (pipe bpp: %d-%d DSC-support: %s)\n", + connector->base.base.id, connector->base.name, + limits->pipe.min_bpp, limits->pipe.max_bpp, + str_yes_no(intel_dp_supports_dsc(intel_dp, connector, + crtc_state))); + } + + if (dsc && !intel_dp_dsc_compute_pipe_bpp_limits(connector, limits)) + return false; if (is_mst || intel_dp->use_max_params) { /* @@ -2784,10 +2811,11 @@ intel_dp_compute_link_config(struct intel_encoder *encoder, } drm_dbg_kms(display->drm, - "DP lane count %d clock %d bpp input %d compressed " FXP_Q4_FMT " link rate required %d available %d\n", + "DP lane count %d clock %d bpp input %d compressed " FXP_Q4_FMT " HDR %s link rate required %d available %d\n", pipe_config->lane_count, pipe_config->port_clock, pipe_config->pipe_bpp, FXP_Q4_ARGS(pipe_config->dsc.compressed_bpp_x16), + str_yes_no(intel_dp_in_hdr_mode(conn_state)), intel_dp_config_required_rate(pipe_config), intel_dp_max_link_data_rate(intel_dp, pipe_config->port_clock, @@ -4519,6 +4547,7 @@ static bool intel_edp_init_dpcd(struct intel_dp *intel_dp, struct intel_connector *connector) { struct intel_display *display = to_intel_display(intel_dp); + int ret; /* this function is meant to be called only once */ drm_WARN_ON(display->drm, intel_dp->dpcd[DP_DPCD_REV] != 0); @@ -4558,6 +4587,12 @@ intel_edp_init_dpcd(struct intel_dp *intel_dp, struct intel_connector *connector */ intel_dp_init_source_oui(intel_dp); + /* Read the ALPM DPCD caps */ + ret = drm_dp_dpcd_read_byte(&intel_dp->aux, DP_RECEIVER_ALPM_CAP, + &intel_dp->alpm_dpcd); + if (ret < 0) + return false; + /* * This has to be called after intel_dp->edp_dpcd is filled, PSR checks * for SET_POWER_CAPABLE bit in intel_dp->edp_dpcd[1] diff --git a/drivers/gpu/drm/i915/display/intel_dp_tunnel.c b/drivers/gpu/drm/i915/display/intel_dp_tunnel.c index faa2b7a46699d3..0d15739eaa05de 100644 --- a/drivers/gpu/drm/i915/display/intel_dp_tunnel.c +++ b/drivers/gpu/drm/i915/display/intel_dp_tunnel.c @@ -622,19 +622,27 @@ int intel_dp_tunnel_atomic_compute_stream_bw(struct intel_atomic_state *state, * * Clear any DP tunnel stream BW requirement set by * intel_dp_tunnel_atomic_compute_stream_bw(). + * + * Returns 0 in case of success, a negative error code otherwise. */ -void intel_dp_tunnel_atomic_clear_stream_bw(struct intel_atomic_state *state, - struct intel_crtc_state *crtc_state) +int intel_dp_tunnel_atomic_clear_stream_bw(struct intel_atomic_state *state, + struct intel_crtc_state *crtc_state) { struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); + int err; if (!crtc_state->dp_tunnel_ref.tunnel) - return; + return 0; + + err = drm_dp_tunnel_atomic_set_stream_bw(&state->base, + crtc_state->dp_tunnel_ref.tunnel, + crtc->pipe, 0); + if (err) + return err; - drm_dp_tunnel_atomic_set_stream_bw(&state->base, - crtc_state->dp_tunnel_ref.tunnel, - crtc->pipe, 0); drm_dp_tunnel_ref_put(&crtc_state->dp_tunnel_ref); + + return 0; } /** diff --git a/drivers/gpu/drm/i915/display/intel_dp_tunnel.h b/drivers/gpu/drm/i915/display/intel_dp_tunnel.h index 7f0f720e8dcadf..10ab9eebcef694 100644 --- a/drivers/gpu/drm/i915/display/intel_dp_tunnel.h +++ b/drivers/gpu/drm/i915/display/intel_dp_tunnel.h @@ -40,8 +40,8 @@ int intel_dp_tunnel_atomic_compute_stream_bw(struct intel_atomic_state *state, struct intel_dp *intel_dp, const struct intel_connector *connector, struct intel_crtc_state *crtc_state); -void intel_dp_tunnel_atomic_clear_stream_bw(struct intel_atomic_state *state, - struct intel_crtc_state *crtc_state); +int intel_dp_tunnel_atomic_clear_stream_bw(struct intel_atomic_state *state, + struct intel_crtc_state *crtc_state); int intel_dp_tunnel_atomic_add_state_for_crtc(struct intel_atomic_state *state, struct intel_crtc *crtc); @@ -88,9 +88,12 @@ intel_dp_tunnel_atomic_compute_stream_bw(struct intel_atomic_state *state, return 0; } -static inline void +static inline int intel_dp_tunnel_atomic_clear_stream_bw(struct intel_atomic_state *state, - struct intel_crtc_state *crtc_state) {} + struct intel_crtc_state *crtc_state) +{ + return 0; +} static inline int intel_dp_tunnel_atomic_add_state_for_crtc(struct intel_atomic_state *state, diff --git a/drivers/gpu/drm/i915/display/intel_fbc.c b/drivers/gpu/drm/i915/display/intel_fbc.c index 437d2fda20a7e7..8d387709f25cc5 100644 --- a/drivers/gpu/drm/i915/display/intel_fbc.c +++ b/drivers/gpu/drm/i915/display/intel_fbc.c @@ -1120,13 +1120,15 @@ static bool xe3p_lpd_fbc_pixel_format_is_valid(const struct intel_plane_state *p } } -bool -intel_fbc_is_enable_pixel_normalizer(const struct intel_plane_state *plane_state) +bool intel_fbc_need_pixel_normalizer(const struct intel_plane_state *plane_state) { struct intel_display *display = to_intel_display(plane_state); - return DISPLAY_VER(display) >= 35 && - xe3p_lpd_fbc_fp16_format_is_valid(plane_state); + if (HAS_PIXEL_NORMALIZER(display) && + xe3p_lpd_fbc_fp16_format_is_valid(plane_state)) + return true; + + return false; } static bool pixel_format_is_valid(const struct intel_plane_state *plane_state) diff --git a/drivers/gpu/drm/i915/display/intel_fbc.h b/drivers/gpu/drm/i915/display/intel_fbc.h index 91424563206a31..7e2416b29a0ea2 100644 --- a/drivers/gpu/drm/i915/display/intel_fbc.h +++ b/drivers/gpu/drm/i915/display/intel_fbc.h @@ -53,7 +53,6 @@ void intel_fbc_prepare_dirty_rect(struct intel_atomic_state *state, struct intel_crtc *crtc); void intel_fbc_dirty_rect_update_noarm(struct intel_dsb *dsb, struct intel_plane *plane); -bool -intel_fbc_is_enable_pixel_normalizer(const struct intel_plane_state *plane_state); +bool intel_fbc_need_pixel_normalizer(const struct intel_plane_state *plane_state); #endif /* __INTEL_FBC_H__ */ diff --git a/drivers/gpu/drm/i915/display/intel_gmbus.c b/drivers/gpu/drm/i915/display/intel_gmbus.c index 795012d7c24c23..5a941bea81cad0 100644 --- a/drivers/gpu/drm/i915/display/intel_gmbus.c +++ b/drivers/gpu/drm/i915/display/intel_gmbus.c @@ -498,8 +498,10 @@ gmbus_xfer_read_chunk(struct intel_display *display, val = intel_de_read_fw(display, GMBUS3(display)); do { - if (extra_byte_added && len == 1) + if (extra_byte_added && len == 1) { + len--; break; + } *buf++ = val & 0xff; val >>= 8; diff --git a/drivers/gpu/drm/i915/display/intel_plane.c b/drivers/gpu/drm/i915/display/intel_plane.c index ab6a58530b396e..79910d52dc8fbe 100644 --- a/drivers/gpu/drm/i915/display/intel_plane.c +++ b/drivers/gpu/drm/i915/display/intel_plane.c @@ -433,11 +433,16 @@ void intel_plane_copy_hw_state(struct intel_plane_state *plane_state, drm_framebuffer_get(plane_state->hw.fb); } +static void unlink_nv12_plane(struct intel_crtc_state *crtc_state, + struct intel_plane_state *plane_state); + void intel_plane_set_invisible(struct intel_crtc_state *crtc_state, struct intel_plane_state *plane_state) { struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane); + unlink_nv12_plane(crtc_state, plane_state); + crtc_state->active_planes &= ~BIT(plane->id); crtc_state->scaled_planes &= ~BIT(plane->id); crtc_state->nv12_planes &= ~BIT(plane->id); @@ -1511,6 +1516,9 @@ static void unlink_nv12_plane(struct intel_crtc_state *crtc_state, struct intel_display *display = to_intel_display(plane_state); struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane); + if (!plane_state->planar_linked_plane) + return; + plane_state->planar_linked_plane = NULL; if (!plane_state->is_y_plane) @@ -1548,8 +1556,7 @@ static int icl_check_nv12_planes(struct intel_atomic_state *state, if (plane->pipe != crtc->pipe) continue; - if (plane_state->planar_linked_plane) - unlink_nv12_plane(crtc_state, plane_state); + unlink_nv12_plane(crtc_state, plane_state); } if (!crtc_state->nv12_planes) diff --git a/drivers/gpu/drm/i915/display/intel_psr.c b/drivers/gpu/drm/i915/display/intel_psr.c index 08bca45739749e..6dddeb4421ba2c 100644 --- a/drivers/gpu/drm/i915/display/intel_psr.c +++ b/drivers/gpu/drm/i915/display/intel_psr.c @@ -857,7 +857,12 @@ static void intel_psr_enable_sink(struct intel_dp *intel_dp, void intel_psr_panel_replay_enable_sink(struct intel_dp *intel_dp) { - if (CAN_PANEL_REPLAY(intel_dp)) + /* + * NOTE: We might want to trigger mode set when + * disabling/enabling Panel Replay via debugfs interface to + * ensure this bit is cleared/set accordingly. + */ + if (CAN_PANEL_REPLAY(intel_dp) && panel_replay_global_enabled(intel_dp)) drm_dp_dpcd_writeb(&intel_dp->aux, PANEL_REPLAY_CONFIG, DP_PANEL_REPLAY_ENABLE); } @@ -1706,7 +1711,7 @@ static bool _psr_compute_config(struct intel_dp *intel_dp, entry_setup_frames = intel_psr_entry_setup_frames(intel_dp, adjusted_mode); if (entry_setup_frames >= 0) { - intel_dp->psr.entry_setup_frames = entry_setup_frames; + crtc_state->entry_setup_frames = entry_setup_frames; } else { crtc_state->no_psr_reason = "PSR setup timing not met"; drm_dbg_kms(display->drm, @@ -1787,7 +1792,7 @@ static bool intel_psr_needs_wa_18037818876(struct intel_dp *intel_dp, { struct intel_display *display = to_intel_display(intel_dp); - return (DISPLAY_VER(display) == 20 && intel_dp->psr.entry_setup_frames > 0 && + return (DISPLAY_VER(display) == 20 && crtc_state->entry_setup_frames > 0 && !crtc_state->has_sel_update); } @@ -2162,6 +2167,7 @@ static void intel_psr_enable_locked(struct intel_dp *intel_dp, intel_dp->psr.pkg_c_latency_used = crtc_state->pkg_c_latency_used; intel_dp->psr.io_wake_lines = crtc_state->alpm_state.io_wake_lines; intel_dp->psr.fast_wake_lines = crtc_state->alpm_state.fast_wake_lines; + intel_dp->psr.entry_setup_frames = crtc_state->entry_setup_frames; if (!psr_interrupt_error_check(intel_dp)) return; @@ -2592,6 +2598,12 @@ void intel_psr2_program_trans_man_trk_ctl(struct intel_dsb *dsb, intel_de_write_dsb(display, dsb, PIPE_SRCSZ_ERLY_TPT(crtc->pipe), crtc_state->pipe_srcsz_early_tpt); + + if (!crtc_state->dsc.compression_enable) + return; + + intel_dsc_su_et_parameters_configure(dsb, encoder, crtc_state, + drm_rect_height(&crtc_state->psr2_su_area)); } static void psr2_man_trk_ctl_calc(struct intel_crtc_state *crtc_state, @@ -2644,9 +2656,9 @@ static u32 psr2_pipe_srcsz_early_tpt_calc(struct intel_crtc_state *crtc_state, static void clip_area_update(struct drm_rect *overlap_damage_area, struct drm_rect *damage_area, - struct drm_rect *pipe_src) + struct drm_rect *display_area) { - if (!drm_rect_intersect(damage_area, pipe_src)) + if (!drm_rect_intersect(damage_area, display_area)) return; if (overlap_damage_area->y1 == -1) { @@ -2662,11 +2674,12 @@ static void clip_area_update(struct drm_rect *overlap_damage_area, overlap_damage_area->y2 = damage_area->y2; } -static void intel_psr2_sel_fetch_pipe_alignment(struct intel_crtc_state *crtc_state) +static bool intel_psr2_sel_fetch_pipe_alignment(struct intel_crtc_state *crtc_state) { struct intel_display *display = to_intel_display(crtc_state); const struct drm_dsc_config *vdsc_cfg = &crtc_state->dsc.config; u16 y_alignment; + bool su_area_changed = false; /* ADLP aligns the SU region to vdsc slice height in case dsc is enabled */ if (crtc_state->dsc.compression_enable && @@ -2675,10 +2688,18 @@ static void intel_psr2_sel_fetch_pipe_alignment(struct intel_crtc_state *crtc_st else y_alignment = crtc_state->su_y_granularity; - crtc_state->psr2_su_area.y1 -= crtc_state->psr2_su_area.y1 % y_alignment; - if (crtc_state->psr2_su_area.y2 % y_alignment) + if (crtc_state->psr2_su_area.y1 % y_alignment) { + crtc_state->psr2_su_area.y1 -= crtc_state->psr2_su_area.y1 % y_alignment; + su_area_changed = true; + } + + if (crtc_state->psr2_su_area.y2 % y_alignment) { crtc_state->psr2_su_area.y2 = ((crtc_state->psr2_su_area.y2 / y_alignment) + 1) * y_alignment; + su_area_changed = true; + } + + return su_area_changed; } /* @@ -2688,6 +2709,7 @@ static void intel_psr2_sel_fetch_pipe_alignment(struct intel_crtc_state *crtc_st static void intel_psr2_sel_fetch_et_alignment(struct intel_atomic_state *state, struct intel_crtc *crtc, + struct drm_rect *display_area, bool *cursor_in_su_area) { struct intel_crtc_state *crtc_state = intel_atomic_get_new_crtc_state(state, crtc); @@ -2715,7 +2737,7 @@ intel_psr2_sel_fetch_et_alignment(struct intel_atomic_state *state, continue; clip_area_update(&crtc_state->psr2_su_area, &new_plane_state->uapi.dst, - &crtc_state->pipe_src); + display_area); *cursor_in_su_area = true; } } @@ -2811,7 +2833,13 @@ int intel_psr2_sel_fetch_update(struct intel_atomic_state *state, struct intel_crtc_state *crtc_state = intel_atomic_get_new_crtc_state(state, crtc); struct intel_plane_state *new_plane_state, *old_plane_state; struct intel_plane *plane; - bool full_update = false, cursor_in_su_area = false; + struct drm_rect display_area = { + .x1 = 0, + .y1 = 0, + .x2 = crtc_state->hw.adjusted_mode.crtc_hdisplay, + .y2 = crtc_state->hw.adjusted_mode.crtc_vdisplay, + }; + bool full_update = false, su_area_changed; int i, ret; if (!crtc_state->enable_psr2_sel_fetch) @@ -2824,7 +2852,7 @@ int intel_psr2_sel_fetch_update(struct intel_atomic_state *state, crtc_state->psr2_su_area.x1 = 0; crtc_state->psr2_su_area.y1 = -1; - crtc_state->psr2_su_area.x2 = drm_rect_width(&crtc_state->pipe_src); + crtc_state->psr2_su_area.x2 = drm_rect_width(&display_area); crtc_state->psr2_su_area.y2 = -1; /* @@ -2862,14 +2890,14 @@ int intel_psr2_sel_fetch_update(struct intel_atomic_state *state, damaged_area.y1 = old_plane_state->uapi.dst.y1; damaged_area.y2 = old_plane_state->uapi.dst.y2; clip_area_update(&crtc_state->psr2_su_area, &damaged_area, - &crtc_state->pipe_src); + &display_area); } if (new_plane_state->uapi.visible) { damaged_area.y1 = new_plane_state->uapi.dst.y1; damaged_area.y2 = new_plane_state->uapi.dst.y2; clip_area_update(&crtc_state->psr2_su_area, &damaged_area, - &crtc_state->pipe_src); + &display_area); } continue; } else if (new_plane_state->uapi.alpha != old_plane_state->uapi.alpha) { @@ -2877,7 +2905,7 @@ int intel_psr2_sel_fetch_update(struct intel_atomic_state *state, damaged_area.y1 = new_plane_state->uapi.dst.y1; damaged_area.y2 = new_plane_state->uapi.dst.y2; clip_area_update(&crtc_state->psr2_su_area, &damaged_area, - &crtc_state->pipe_src); + &display_area); continue; } @@ -2893,7 +2921,7 @@ int intel_psr2_sel_fetch_update(struct intel_atomic_state *state, damaged_area.x1 += new_plane_state->uapi.dst.x1 - src.x1; damaged_area.x2 += new_plane_state->uapi.dst.x1 - src.x1; - clip_area_update(&crtc_state->psr2_su_area, &damaged_area, &crtc_state->pipe_src); + clip_area_update(&crtc_state->psr2_su_area, &damaged_area, &display_area); } /* @@ -2918,15 +2946,33 @@ int intel_psr2_sel_fetch_update(struct intel_atomic_state *state, if (ret) return ret; - /* - * Adjust su area to cover cursor fully as necessary (early - * transport). This needs to be done after - * drm_atomic_add_affected_planes to ensure visible cursor is added into - * affected planes even when cursor is not updated by itself. - */ - intel_psr2_sel_fetch_et_alignment(state, crtc, &cursor_in_su_area); + do { + bool cursor_in_su_area; + + /* + * Adjust su area to cover cursor fully as necessary + * (early transport). This needs to be done after + * drm_atomic_add_affected_planes to ensure visible + * cursor is added into affected planes even when + * cursor is not updated by itself. + */ + intel_psr2_sel_fetch_et_alignment(state, crtc, &display_area, + &cursor_in_su_area); + + su_area_changed = intel_psr2_sel_fetch_pipe_alignment(crtc_state); - intel_psr2_sel_fetch_pipe_alignment(crtc_state); + /* + * If the cursor was outside the SU area before + * alignment, the alignment step (which only expands + * SU) may pull the cursor partially inside, so we + * must run ET alignment again to fully cover it. But + * if the cursor was already fully inside before + * alignment, expanding the SU area won't change that, + * so no further work is needed. + */ + if (cursor_in_su_area) + break; + } while (su_area_changed); /* * Now that we have the pipe damaged area check if it intersect with @@ -2986,6 +3032,10 @@ int intel_psr2_sel_fetch_update(struct intel_atomic_state *state, } skip_sel_fetch_set_loop: + if (full_update) + clip_area_update(&crtc_state->psr2_su_area, &display_area, + &display_area); + psr2_man_trk_ctl_calc(crtc_state, full_update); crtc_state->pipe_srcsz_early_tpt = psr2_pipe_srcsz_early_tpt_calc(crtc_state, full_update); @@ -3045,6 +3095,8 @@ void intel_psr_pre_plane_update(struct intel_atomic_state *state, * - Display WA #1136: skl, bxt */ if (intel_crtc_needs_modeset(new_crtc_state) || + new_crtc_state->update_m_n || + new_crtc_state->update_lrr || !new_crtc_state->has_psr || !new_crtc_state->active_planes || new_crtc_state->has_sel_update != psr->sel_update_enabled || diff --git a/drivers/gpu/drm/i915/display/intel_quirks.c b/drivers/gpu/drm/i915/display/intel_quirks.c index d2e16b79d6be1e..1abbdd426e5873 100644 --- a/drivers/gpu/drm/i915/display/intel_quirks.c +++ b/drivers/gpu/drm/i915/display/intel_quirks.c @@ -239,7 +239,7 @@ static struct intel_quirk intel_quirks[] = { { 0x0f31, 0x103c, 0x220f, quirk_invert_brightness }, /* Dell XPS 13 7390 2-in-1 */ - { 0x8a12, 0x1028, 0x08b0, quirk_edp_limit_rate_hbr2 }, + { 0x8a52, 0x1028, 0x08b0, quirk_edp_limit_rate_hbr2 }, }; static const struct intel_dpcd_quirk intel_dpcd_quirks[] = { diff --git a/drivers/gpu/drm/i915/display/intel_vdsc.c b/drivers/gpu/drm/i915/display/intel_vdsc.c index 0e727fc5e80c11..b08e677fa2b306 100644 --- a/drivers/gpu/drm/i915/display/intel_vdsc.c +++ b/drivers/gpu/drm/i915/display/intel_vdsc.c @@ -767,6 +767,29 @@ void intel_dsc_dp_pps_write(struct intel_encoder *encoder, sizeof(dp_dsc_pps_sdp)); } +void intel_dsc_su_et_parameters_configure(struct intel_dsb *dsb, struct intel_encoder *encoder, + const struct intel_crtc_state *crtc_state, int su_lines) +{ + struct intel_display *display = to_intel_display(crtc_state); + struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); + const struct drm_dsc_config *vdsc_cfg = &crtc_state->dsc.config; + enum pipe pipe = crtc->pipe; + int vdsc_instances_per_pipe = intel_dsc_get_vdsc_per_pipe(crtc_state); + int slice_row_per_frame = su_lines / vdsc_cfg->slice_height; + u32 val; + + drm_WARN_ON_ONCE(display->drm, su_lines % vdsc_cfg->slice_height); + drm_WARN_ON_ONCE(display->drm, vdsc_instances_per_pipe > 2); + + val = DSC_SUPS0_SU_SLICE_ROW_PER_FRAME(slice_row_per_frame); + val |= DSC_SUPS0_SU_PIC_HEIGHT(su_lines); + + intel_de_write_dsb(display, dsb, LNL_DSC0_SU_PARAMETER_SET_0(pipe), val); + + if (vdsc_instances_per_pipe == 2) + intel_de_write_dsb(display, dsb, LNL_DSC1_SU_PARAMETER_SET_0(pipe), val); +} + static i915_reg_t dss_ctl1_reg(struct intel_crtc *crtc, enum transcoder cpu_transcoder) { return is_pipe_dsc(crtc, cpu_transcoder) ? diff --git a/drivers/gpu/drm/i915/display/intel_vdsc.h b/drivers/gpu/drm/i915/display/intel_vdsc.h index 99f64ac54b2734..99bb9042592a42 100644 --- a/drivers/gpu/drm/i915/display/intel_vdsc.h +++ b/drivers/gpu/drm/i915/display/intel_vdsc.h @@ -13,6 +13,7 @@ struct drm_printer; enum transcoder; struct intel_crtc; struct intel_crtc_state; +struct intel_dsb; struct intel_encoder; bool intel_dsc_source_support(const struct intel_crtc_state *crtc_state); @@ -31,6 +32,8 @@ void intel_dsc_dsi_pps_write(struct intel_encoder *encoder, const struct intel_crtc_state *crtc_state); void intel_dsc_dp_pps_write(struct intel_encoder *encoder, const struct intel_crtc_state *crtc_state); +void intel_dsc_su_et_parameters_configure(struct intel_dsb *dsb, struct intel_encoder *encoder, + const struct intel_crtc_state *crtc_state, int su_lines); void intel_vdsc_state_dump(struct drm_printer *p, int indent, const struct intel_crtc_state *crtc_state); int intel_vdsc_min_cdclk(const struct intel_crtc_state *crtc_state); diff --git a/drivers/gpu/drm/i915/display/intel_vdsc_regs.h b/drivers/gpu/drm/i915/display/intel_vdsc_regs.h index 2d478a84b07c44..2b2e3c1b8138a9 100644 --- a/drivers/gpu/drm/i915/display/intel_vdsc_regs.h +++ b/drivers/gpu/drm/i915/display/intel_vdsc_regs.h @@ -196,6 +196,18 @@ #define DSC_PPS18_NSL_BPG_OFFSET(offset) REG_FIELD_PREP(DSC_PPS18_NSL_BPG_OFFSET_MASK, offset) #define DSC_PPS18_SL_OFFSET_ADJ(offset) REG_FIELD_PREP(DSC_PPS18_SL_OFFSET_ADJ_MASK, offset) +#define _LNL_DSC0_SU_PARAMETER_SET_0_PA 0x78064 +#define _LNL_DSC1_SU_PARAMETER_SET_0_PA 0x78164 +#define _LNL_DSC0_SU_PARAMETER_SET_0_PB 0x78264 +#define _LNL_DSC1_SU_PARAMETER_SET_0_PB 0x78364 +#define LNL_DSC0_SU_PARAMETER_SET_0(pipe) _MMIO_PIPE((pipe), _LNL_DSC0_SU_PARAMETER_SET_0_PA, _LNL_DSC0_SU_PARAMETER_SET_0_PB) +#define LNL_DSC1_SU_PARAMETER_SET_0(pipe) _MMIO_PIPE((pipe), _LNL_DSC1_SU_PARAMETER_SET_0_PA, _LNL_DSC1_SU_PARAMETER_SET_0_PB) + +#define DSC_SUPS0_SU_SLICE_ROW_PER_FRAME_MASK REG_GENMASK(31, 20) +#define DSC_SUPS0_SU_SLICE_ROW_PER_FRAME(rows) REG_FIELD_PREP(DSC_SUPS0_SU_SLICE_ROW_PER_FRAME_MASK, (rows)) +#define DSC_SUPS0_SU_PIC_HEIGHT_MASK REG_GENMASK(15, 0) +#define DSC_SUPS0_SU_PIC_HEIGHT(h) REG_FIELD_PREP(DSC_SUPS0_SU_PIC_HEIGHT_MASK, (h)) + /* Icelake Rate Control Buffer Threshold Registers */ #define DSCA_RC_BUF_THRESH_0 _MMIO(0x6B230) #define DSCA_RC_BUF_THRESH_0_UDW _MMIO(0x6B230 + 4) diff --git a/drivers/gpu/drm/i915/display/intel_vrr.c b/drivers/gpu/drm/i915/display/intel_vrr.c index b92c42fde937f6..81128a27d4550b 100644 --- a/drivers/gpu/drm/i915/display/intel_vrr.c +++ b/drivers/gpu/drm/i915/display/intel_vrr.c @@ -528,6 +528,18 @@ void intel_vrr_set_transcoder_timings(const struct intel_crtc_state *crtc_state) if (!HAS_VRR(display)) return; + /* + * Bspec says: + * "(note: VRR needs to be programmed after + * TRANS_DDI_FUNC_CTL and before TRANS_CONF)." + * + * In practice it turns out that ICL can hang if + * TRANS_VRR_VMAX/FLIPLINE are written before + * enabling TRANS_DDI_FUNC_CTL. + */ + drm_WARN_ON(display->drm, + !(intel_de_read(display, TRANS_DDI_FUNC_CTL(display, cpu_transcoder)) & TRANS_DDI_FUNC_ENABLE)); + /* * This bit seems to have two meanings depending on the platform: * TGL: generate VRR "safe window" for DSB vblank waits @@ -754,6 +766,8 @@ void intel_vrr_transcoder_enable(const struct intel_crtc_state *crtc_state) { struct intel_display *display = to_intel_display(crtc_state); + intel_vrr_set_transcoder_timings(crtc_state); + if (!intel_vrr_possible(crtc_state)) return; diff --git a/drivers/gpu/drm/i915/display/skl_universal_plane.c b/drivers/gpu/drm/i915/display/skl_universal_plane.c index ee8e24497d2cfd..ed14b9ea2ad2d2 100644 --- a/drivers/gpu/drm/i915/display/skl_universal_plane.c +++ b/drivers/gpu/drm/i915/display/skl_universal_plane.c @@ -892,23 +892,20 @@ static void icl_plane_disable_sel_fetch_arm(struct intel_dsb *dsb, intel_de_write_dsb(display, dsb, SEL_FETCH_PLANE_CTL(pipe, plane->id), 0); } -static void x3p_lpd_plane_update_pixel_normalizer(struct intel_dsb *dsb, - struct intel_plane *plane, - bool enable) +static bool plane_has_normalizer(struct intel_plane *plane) { struct intel_display *display = to_intel_display(plane); - enum intel_fbc_id fbc_id = skl_fbc_id_for_pipe(plane->pipe); - u32 val; - /* Only HDR planes have pixel normalizer and don't matter if no FBC */ - if (!skl_plane_has_fbc(display, fbc_id, plane->id)) - return; + return HAS_PIXEL_NORMALIZER(display) && icl_is_hdr_plane(display, plane->id); +} - val = enable ? PLANE_PIXEL_NORMALIZE_NORM_FACTOR(PLANE_PIXEL_NORMALIZE_NORM_FACTOR_1_0) | - PLANE_PIXEL_NORMALIZE_ENABLE : 0; +static u32 pixel_normalizer_value(const struct intel_plane_state *plane_state) +{ + if (!intel_fbc_need_pixel_normalizer(plane_state)) + return 0; - intel_de_write_dsb(display, dsb, - PLANE_PIXEL_NORMALIZE(plane->pipe, plane->id), val); + return PLANE_PIXEL_NORMALIZE_ENABLE | + PLANE_PIXEL_NORMALIZE_NORM_FACTOR(PLANE_PIXEL_NORMALIZE_NORM_FACTOR_1_0); } static void @@ -927,8 +924,9 @@ icl_plane_disable_arm(struct intel_dsb *dsb, icl_plane_disable_sel_fetch_arm(dsb, plane, crtc_state); - if (DISPLAY_VER(display) >= 35) - x3p_lpd_plane_update_pixel_normalizer(dsb, plane, false); + if (plane_has_normalizer(plane)) + intel_de_write_dsb(display, dsb, + PLANE_PIXEL_NORMALIZE(plane->pipe, plane->id), 0); intel_de_write_dsb(display, dsb, PLANE_CTL(pipe, plane_id), 0); intel_de_write_dsb(display, dsb, PLANE_SURF(pipe, plane_id), 0); @@ -1677,11 +1675,13 @@ icl_plane_update_arm(struct intel_dsb *dsb, /* * In order to have FBC for fp16 formats pixel normalizer block must be - * active. Check if pixel normalizer block need to be enabled for FBC. - * If needed, use normalization factor as 1.0 and enable the block. + * active. For FP16 formats, use normalization factor as 1.0 and enable + * the block. */ - if (intel_fbc_is_enable_pixel_normalizer(plane_state)) - x3p_lpd_plane_update_pixel_normalizer(dsb, plane, true); + if (plane_has_normalizer(plane)) + intel_de_write_dsb(display, dsb, + PLANE_PIXEL_NORMALIZE(plane->pipe, plane->id), + pixel_normalizer_value(plane_state)); /* * The control register self-arms if the plane was previously diff --git a/drivers/gpu/drm/i915/gem/i915_gem_shmem.c b/drivers/gpu/drm/i915/gem/i915_gem_shmem.c index 26dda55a07ff41..6c0c4cf1517ad7 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_shmem.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_shmem.c @@ -151,8 +151,12 @@ int shmem_sg_alloc_table(struct drm_i915_private *i915, struct sg_table *st, } } while (1); - nr_pages = min_t(unsigned long, - folio_nr_pages(folio), page_count - i); + nr_pages = min_array(((unsigned long[]) { + folio_nr_pages(folio), + page_count - i, + max_segment / PAGE_SIZE, + }), 3); + if (!i || sg->length >= max_segment || folio_pfn(folio) != next_pfn) { @@ -162,7 +166,9 @@ int shmem_sg_alloc_table(struct drm_i915_private *i915, struct sg_table *st, st->nents++; sg_set_folio(sg, folio, nr_pages * PAGE_SIZE, 0); } else { - /* XXX: could overflow? */ + nr_pages = min_t(unsigned long, nr_pages, + (max_segment - sg->length) / PAGE_SIZE); + sg->length += nr_pages * PAGE_SIZE; } next_pfn = folio_pfn(folio) + nr_pages; diff --git a/drivers/gpu/drm/i915/gt/intel_engine_cs.c b/drivers/gpu/drm/i915/gt/intel_engine_cs.c index b721bbd2335674..ce8cdd517daa1e 100644 --- a/drivers/gpu/drm/i915/gt/intel_engine_cs.c +++ b/drivers/gpu/drm/i915/gt/intel_engine_cs.c @@ -1969,7 +1969,8 @@ void intel_engines_reset_default_submission(struct intel_gt *gt) if (engine->sanitize) engine->sanitize(engine); - engine->set_default_submission(engine); + if (engine->set_default_submission) + engine->set_default_submission(engine); } } diff --git a/drivers/gpu/drm/i915/gt/intel_engine_heartbeat.c b/drivers/gpu/drm/i915/gt/intel_engine_heartbeat.c index b279878dca2932..6424ecce8bcbea 100644 --- a/drivers/gpu/drm/i915/gt/intel_engine_heartbeat.c +++ b/drivers/gpu/drm/i915/gt/intel_engine_heartbeat.c @@ -148,10 +148,12 @@ static void heartbeat(struct work_struct *wrk) /* Just in case everything has gone horribly wrong, give it a kick */ intel_engine_flush_submission(engine); - rq = engine->heartbeat.systole; - if (rq && i915_request_completed(rq)) { - i915_request_put(rq); - engine->heartbeat.systole = NULL; + rq = xchg(&engine->heartbeat.systole, NULL); + if (rq) { + if (i915_request_completed(rq)) + i915_request_put(rq); + else + engine->heartbeat.systole = rq; } if (!intel_engine_pm_get_if_awake(engine)) @@ -232,8 +234,11 @@ static void heartbeat(struct work_struct *wrk) unlock: mutex_unlock(&ce->timeline->mutex); out: - if (!engine->i915->params.enable_hangcheck || !next_heartbeat(engine)) - i915_request_put(fetch_and_zero(&engine->heartbeat.systole)); + if (!engine->i915->params.enable_hangcheck || !next_heartbeat(engine)) { + rq = xchg(&engine->heartbeat.systole, NULL); + if (rq) + i915_request_put(rq); + } intel_engine_pm_put(engine); } @@ -247,8 +252,13 @@ void intel_engine_unpark_heartbeat(struct intel_engine_cs *engine) void intel_engine_park_heartbeat(struct intel_engine_cs *engine) { - if (cancel_delayed_work(&engine->heartbeat.work)) - i915_request_put(fetch_and_zero(&engine->heartbeat.systole)); + if (cancel_delayed_work(&engine->heartbeat.work)) { + struct i915_request *rq; + + rq = xchg(&engine->heartbeat.systole, NULL); + if (rq) + i915_request_put(rq); + } } void intel_gt_unpark_heartbeats(struct intel_gt *gt) diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c index 4c82c9544b9328..72fe91ed1c7403 100644 --- a/drivers/gpu/drm/i915/i915_gem.c +++ b/drivers/gpu/drm/i915/i915_gem.c @@ -520,7 +520,7 @@ ggtt_write(struct io_mapping *mapping, /* We can use the cpu mem copy function because this is X86. */ vaddr = io_mapping_map_atomic_wc(mapping, base); - unwritten = __copy_from_user_inatomic_nocache((void __force *)vaddr + offset, + unwritten = copy_from_user_inatomic_nontemporal((void __force *)vaddr + offset, user_data, length); io_mapping_unmap_atomic(vaddr); if (unwritten) { diff --git a/drivers/gpu/drm/i915/i915_wait_util.h b/drivers/gpu/drm/i915/i915_wait_util.h index 7376898e3bf836..e1ed7921ec701d 100644 --- a/drivers/gpu/drm/i915/i915_wait_util.h +++ b/drivers/gpu/drm/i915/i915_wait_util.h @@ -25,9 +25,9 @@ might_sleep(); \ for (;;) { \ const bool expired__ = ktime_after(ktime_get_raw(), end__); \ - OP; \ /* Guarantee COND check prior to timeout */ \ barrier(); \ + OP; \ if (COND) { \ ret__ = 0; \ break; \ diff --git a/drivers/gpu/drm/i915/intel_wakeref.c b/drivers/gpu/drm/i915/intel_wakeref.c index b1883dccc22aff..98e7cee4e1dcc7 100644 --- a/drivers/gpu/drm/i915/intel_wakeref.c +++ b/drivers/gpu/drm/i915/intel_wakeref.c @@ -80,7 +80,7 @@ void __intel_wakeref_put_last(struct intel_wakeref *wf, unsigned long flags) /* Assume we are not in process context and so cannot sleep. */ if (flags & INTEL_WAKEREF_PUT_ASYNC || !mutex_trylock(&wf->mutex)) { mod_delayed_work(wf->i915->unordered_wq, &wf->work, - FIELD_GET(INTEL_WAKEREF_PUT_DELAY, flags)); + FIELD_GET(INTEL_WAKEREF_PUT_DELAY_MASK, flags)); return; } diff --git a/drivers/gpu/drm/i915/intel_wakeref.h b/drivers/gpu/drm/i915/intel_wakeref.h index a2894a56e18fc3..81308bac34babe 100644 --- a/drivers/gpu/drm/i915/intel_wakeref.h +++ b/drivers/gpu/drm/i915/intel_wakeref.h @@ -128,17 +128,16 @@ intel_wakeref_get_if_active(struct intel_wakeref *wf) return atomic_inc_not_zero(&wf->count); } -enum { - INTEL_WAKEREF_PUT_ASYNC_BIT = 0, - __INTEL_WAKEREF_PUT_LAST_BIT__ -}; - static inline void intel_wakeref_might_get(struct intel_wakeref *wf) { might_lock(&wf->mutex); } +/* flags for __intel_wakeref_put() and __intel_wakeref_put_last */ +#define INTEL_WAKEREF_PUT_ASYNC BIT(0) +#define INTEL_WAKEREF_PUT_DELAY_MASK GENMASK(BITS_PER_LONG - 1, 1) + /** * __intel_wakeref_put: Release the wakeref * @wf: the wakeref @@ -154,9 +153,6 @@ intel_wakeref_might_get(struct intel_wakeref *wf) */ static inline void __intel_wakeref_put(struct intel_wakeref *wf, unsigned long flags) -#define INTEL_WAKEREF_PUT_ASYNC BIT(INTEL_WAKEREF_PUT_ASYNC_BIT) -#define INTEL_WAKEREF_PUT_DELAY \ - GENMASK(BITS_PER_LONG - 1, __INTEL_WAKEREF_PUT_LAST_BIT__) { INTEL_WAKEREF_BUG_ON(atomic_read(&wf->count) <= 0); if (unlikely(!atomic_add_unless(&wf->count, -1, 1))) @@ -181,7 +177,7 @@ intel_wakeref_put_delay(struct intel_wakeref *wf, unsigned long delay) { __intel_wakeref_put(wf, INTEL_WAKEREF_PUT_ASYNC | - FIELD_PREP(INTEL_WAKEREF_PUT_DELAY, delay)); + FIELD_PREP(INTEL_WAKEREF_PUT_DELAY_MASK, delay)); } static inline void diff --git a/drivers/gpu/drm/imagination/pvr_device.c b/drivers/gpu/drm/imagination/pvr_device.c index 78d6b8a0a4506c..b2275cb7360e4e 100644 --- a/drivers/gpu/drm/imagination/pvr_device.c +++ b/drivers/gpu/drm/imagination/pvr_device.c @@ -224,29 +224,12 @@ static irqreturn_t pvr_device_irq_thread_handler(int irq, void *data) } if (pvr_dev->has_safety_events) { - int err; - - /* - * Ensure the GPU is powered on since some safety events (such - * as ECC faults) can happen outside of job submissions, which - * are otherwise the only time a power reference is held. - */ - err = pvr_power_get(pvr_dev); - if (err) { - drm_err_ratelimited(drm_dev, - "%s: could not take power reference (%d)\n", - __func__, err); - return ret; - } - while (pvr_device_safety_irq_pending(pvr_dev)) { pvr_device_safety_irq_clear(pvr_dev); pvr_device_handle_safety_events(pvr_dev); ret = IRQ_HANDLED; } - - pvr_power_put(pvr_dev); } return ret; diff --git a/drivers/gpu/drm/imagination/pvr_power.c b/drivers/gpu/drm/imagination/pvr_power.c index b9f801c63260cb..50ce3668f05161 100644 --- a/drivers/gpu/drm/imagination/pvr_power.c +++ b/drivers/gpu/drm/imagination/pvr_power.c @@ -90,7 +90,7 @@ pvr_power_request_pwr_off(struct pvr_device *pvr_dev) } static int -pvr_power_fw_disable(struct pvr_device *pvr_dev, bool hard_reset) +pvr_power_fw_disable(struct pvr_device *pvr_dev, bool hard_reset, bool rpm_suspend) { if (!hard_reset) { int err; @@ -106,6 +106,11 @@ pvr_power_fw_disable(struct pvr_device *pvr_dev, bool hard_reset) return err; } + if (rpm_suspend) { + /* Wait for late processing of GPU or firmware IRQs in other cores */ + synchronize_irq(pvr_dev->irq); + } + return pvr_fw_stop(pvr_dev); } @@ -361,7 +366,7 @@ pvr_power_device_suspend(struct device *dev) return -EIO; if (pvr_dev->fw_dev.booted) { - err = pvr_power_fw_disable(pvr_dev, false); + err = pvr_power_fw_disable(pvr_dev, false, true); if (err) goto err_drm_dev_exit; } @@ -510,7 +515,16 @@ pvr_power_reset(struct pvr_device *pvr_dev, bool hard_reset) } /* Disable IRQs for the duration of the reset. */ - disable_irq(pvr_dev->irq); + if (hard_reset) { + disable_irq(pvr_dev->irq); + } else { + /* + * Soft reset is triggered as a response to a FW command to the Host and is + * processed from the threaded IRQ handler. This code cannot (nor needs to) + * wait for any IRQ processing to complete. + */ + disable_irq_nosync(pvr_dev->irq); + } do { if (hard_reset) { @@ -518,7 +532,7 @@ pvr_power_reset(struct pvr_device *pvr_dev, bool hard_reset) queues_disabled = true; } - err = pvr_power_fw_disable(pvr_dev, hard_reset); + err = pvr_power_fw_disable(pvr_dev, hard_reset, false); if (!err) { if (hard_reset) { pvr_dev->fw_dev.booted = false; diff --git a/drivers/gpu/drm/imx/ipuv3/parallel-display.c b/drivers/gpu/drm/imx/ipuv3/parallel-display.c index 6fbf505d2801d6..590120a33fa074 100644 --- a/drivers/gpu/drm/imx/ipuv3/parallel-display.c +++ b/drivers/gpu/drm/imx/ipuv3/parallel-display.c @@ -256,7 +256,9 @@ static int imx_pd_probe(struct platform_device *pdev) platform_set_drvdata(pdev, imxpd); - devm_drm_bridge_add(dev, &imxpd->bridge); + ret = devm_drm_bridge_add(dev, &imxpd->bridge); + if (ret) + return ret; return component_add(dev, &imx_pd_ops); } diff --git a/drivers/gpu/drm/logicvc/logicvc_drm.c b/drivers/gpu/drm/logicvc/logicvc_drm.c index 204b0fee55d0b2..bbebf4fc7f51a8 100644 --- a/drivers/gpu/drm/logicvc/logicvc_drm.c +++ b/drivers/gpu/drm/logicvc/logicvc_drm.c @@ -92,7 +92,6 @@ static int logicvc_drm_config_parse(struct logicvc_drm *logicvc) struct device *dev = drm_dev->dev; struct device_node *of_node = dev->of_node; struct logicvc_drm_config *config = &logicvc->config; - struct device_node *layers_node; int ret; logicvc_of_property_parse_bool(of_node, LOGICVC_OF_PROPERTY_DITHERING, @@ -128,7 +127,8 @@ static int logicvc_drm_config_parse(struct logicvc_drm *logicvc) if (ret) return ret; - layers_node = of_get_child_by_name(of_node, "layers"); + struct device_node *layers_node __free(device_node) = + of_get_child_by_name(of_node, "layers"); if (!layers_node) { drm_err(drm_dev, "Missing non-optional layers node\n"); return -EINVAL; diff --git a/drivers/gpu/drm/mediatek/mtk_dsi.c b/drivers/gpu/drm/mediatek/mtk_dsi.c index d7726091819c47..acee2227275b73 100644 --- a/drivers/gpu/drm/mediatek/mtk_dsi.c +++ b/drivers/gpu/drm/mediatek/mtk_dsi.c @@ -1232,6 +1232,11 @@ static int mtk_dsi_probe(struct platform_device *pdev) dsi->host.ops = &mtk_dsi_ops; dsi->host.dev = dev; + + init_waitqueue_head(&dsi->irq_wait_queue); + + platform_set_drvdata(pdev, dsi); + ret = mipi_dsi_host_register(&dsi->host); if (ret < 0) return dev_err_probe(dev, ret, "Failed to register DSI host\n"); @@ -1243,10 +1248,6 @@ static int mtk_dsi_probe(struct platform_device *pdev) return dev_err_probe(&pdev->dev, ret, "Failed to request DSI irq\n"); } - init_waitqueue_head(&dsi->irq_wait_queue); - - platform_set_drvdata(pdev, dsi); - dsi->bridge.of_node = dev->of_node; dsi->bridge.type = DRM_MODE_CONNECTOR_DSI; diff --git a/drivers/gpu/drm/msm/adreno/a2xx_gpu.c b/drivers/gpu/drm/msm/adreno/a2xx_gpu.c index 1b1ee14b65cf03..f6e48bb8cc69ee 100644 --- a/drivers/gpu/drm/msm/adreno/a2xx_gpu.c +++ b/drivers/gpu/drm/msm/adreno/a2xx_gpu.c @@ -77,7 +77,10 @@ static bool a2xx_me_init(struct msm_gpu *gpu) /* Vertex and Pixel Shader Start Addresses in instructions * (3 DWORDS per instruction) */ - OUT_RING(ring, 0x80000180); + if (adreno_is_a225(adreno_gpu)) + OUT_RING(ring, 0x80000300); + else + OUT_RING(ring, 0x80000180); /* Maximum Contexts */ OUT_RING(ring, 0x00000001); /* Write Confirm Interval and The CP will wait the diff --git a/drivers/gpu/drm/msm/adreno/a2xx_gpummu.c b/drivers/gpu/drm/msm/adreno/a2xx_gpummu.c index 0407c9bc8c1bf4..4467b04527cd16 100644 --- a/drivers/gpu/drm/msm/adreno/a2xx_gpummu.c +++ b/drivers/gpu/drm/msm/adreno/a2xx_gpummu.c @@ -78,7 +78,7 @@ static void a2xx_gpummu_destroy(struct msm_mmu *mmu) { struct a2xx_gpummu *gpummu = to_a2xx_gpummu(mmu); - dma_free_attrs(mmu->dev, TABLE_SIZE, gpummu->table, gpummu->pt_base, + dma_free_attrs(mmu->dev, TABLE_SIZE + 32, gpummu->table, gpummu->pt_base, DMA_ATTR_FORCE_CONTIGUOUS); kfree(gpummu); diff --git a/drivers/gpu/drm/msm/adreno/a6xx_catalog.c b/drivers/gpu/drm/msm/adreno/a6xx_catalog.c index 4c042133261c96..38561f26837e33 100644 --- a/drivers/gpu/drm/msm/adreno/a6xx_catalog.c +++ b/drivers/gpu/drm/msm/adreno/a6xx_catalog.c @@ -1689,7 +1689,7 @@ static const struct adreno_reglist_pipe x285_nonctxt_regs[] = { { REG_A7XX_SP_READ_SEL, 0x0001ff00, BIT(PIPE_NONE) }, { REG_A6XX_TPL1_DBG_ECO_CNTL, 0x10000000, BIT(PIPE_NONE) }, /* BIT(26): Disable final clamp for bicubic filtering */ - { REG_A6XX_TPL1_DBG_ECO_CNTL1, 0x00000720, BIT(PIPE_NONE) }, + { REG_A6XX_TPL1_DBG_ECO_CNTL1, 0x04000720, BIT(PIPE_NONE) }, { REG_A6XX_UCHE_MODE_CNTL, 0x80080000, BIT(PIPE_NONE) }, { REG_A8XX_UCHE_CCHE_MODE_CNTL, 0x00001000, BIT(PIPE_NONE) }, { REG_A8XX_UCHE_CCHE_CACHE_WAYS, 0x00000800, BIT(PIPE_NONE) }, @@ -1759,7 +1759,7 @@ static const u32 x285_protect_regs[] = { A6XX_PROTECT_NORDWR(0x27c06, 0x0000), }; -DECLARE_ADRENO_PROTECT(x285_protect, 64); +DECLARE_ADRENO_PROTECT(x285_protect, 15); static const struct adreno_reglist_pipe a840_nonctxt_regs[] = { { REG_A8XX_CP_SMMU_STREAM_ID_LPAC, 0x00000101, BIT(PIPE_NONE) }, @@ -1966,5 +1966,4 @@ static inline __always_unused void __build_asserts(void) BUILD_BUG_ON(a660_protect.count > a660_protect.count_max); BUILD_BUG_ON(a690_protect.count > a690_protect.count_max); BUILD_BUG_ON(a730_protect.count > a730_protect.count_max); - BUILD_BUG_ON(a840_protect.count > a840_protect.count_max); } diff --git a/drivers/gpu/drm/msm/adreno/a8xx_gpu.c b/drivers/gpu/drm/msm/adreno/a8xx_gpu.c index 30de078e9dfd2c..3b17ddac075328 100644 --- a/drivers/gpu/drm/msm/adreno/a8xx_gpu.c +++ b/drivers/gpu/drm/msm/adreno/a8xx_gpu.c @@ -306,11 +306,21 @@ static void a8xx_set_ubwc_config(struct msm_gpu *gpu) hbb = cfg->highest_bank_bit - 13; hbb_hi = hbb >> 2; hbb_lo = hbb & 3; - a8xx_write_pipe(gpu, PIPE_BV, REG_A8XX_GRAS_NC_MODE_CNTL, hbb << 5); - a8xx_write_pipe(gpu, PIPE_BR, REG_A8XX_GRAS_NC_MODE_CNTL, hbb << 5); + + a8xx_write_pipe(gpu, PIPE_BV, REG_A8XX_GRAS_NC_MODE_CNTL, + hbb << 5 | + level3_swizzling_dis << 4 | + level2_swizzling_dis << 3); + + a8xx_write_pipe(gpu, PIPE_BR, REG_A8XX_GRAS_NC_MODE_CNTL, + hbb << 5 | + level3_swizzling_dis << 4 | + level2_swizzling_dis << 3); a8xx_write_pipe(gpu, PIPE_BR, REG_A8XX_RB_CCU_NC_MODE_CNTL, yuvnotcomptofc << 6 | + level3_swizzling_dis << 5 | + level2_swizzling_dis << 4 | hbb_hi << 3 | hbb_lo << 1); diff --git a/drivers/gpu/drm/msm/adreno/adreno_gpu.c b/drivers/gpu/drm/msm/adreno/adreno_gpu.c index 1c80909e63cab4..d5fe6f6f0decc4 100644 --- a/drivers/gpu/drm/msm/adreno/adreno_gpu.c +++ b/drivers/gpu/drm/msm/adreno/adreno_gpu.c @@ -376,8 +376,7 @@ int adreno_get_param(struct msm_gpu *gpu, struct msm_context *ctx, *value = adreno_gpu->info->gmem; return 0; case MSM_PARAM_GMEM_BASE: - if (adreno_is_a650_family(adreno_gpu) || - adreno_is_a740_family(adreno_gpu)) + if (adreno_gpu->info->family >= ADRENO_6XX_GEN4) *value = 0; else *value = 0x100000; diff --git a/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_3_0_msm8998.h b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_3_0_msm8998.h index f91220496082bd..b1b03d8b30fa0c 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_3_0_msm8998.h +++ b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_3_0_msm8998.h @@ -42,24 +42,19 @@ static const struct dpu_ctl_cfg msm8998_ctl[] = { .name = "ctl_0", .id = CTL_0, .base = 0x1000, .len = 0x94, .features = BIT(DPU_CTL_SPLIT_DISPLAY), - .intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 9), }, { .name = "ctl_1", .id = CTL_1, .base = 0x1200, .len = 0x94, - .intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 10), }, { .name = "ctl_2", .id = CTL_2, .base = 0x1400, .len = 0x94, .features = BIT(DPU_CTL_SPLIT_DISPLAY), - .intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 11), }, { .name = "ctl_3", .id = CTL_3, .base = 0x1600, .len = 0x94, - .intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 12), }, { .name = "ctl_4", .id = CTL_4, .base = 0x1800, .len = 0x94, - .intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 13), }, }; diff --git a/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_3_2_sdm660.h b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_3_2_sdm660.h index 8f9a097147c02b..64df4e80ea43de 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_3_2_sdm660.h +++ b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_3_2_sdm660.h @@ -37,24 +37,19 @@ static const struct dpu_ctl_cfg sdm660_ctl[] = { .name = "ctl_0", .id = CTL_0, .base = 0x1000, .len = 0x94, .features = BIT(DPU_CTL_SPLIT_DISPLAY), - .intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 9), }, { .name = "ctl_1", .id = CTL_1, .base = 0x1200, .len = 0x94, - .intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 10), }, { .name = "ctl_2", .id = CTL_2, .base = 0x1400, .len = 0x94, .features = BIT(DPU_CTL_SPLIT_DISPLAY), - .intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 11), }, { .name = "ctl_3", .id = CTL_3, .base = 0x1600, .len = 0x94, - .intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 12), }, { .name = "ctl_4", .id = CTL_4, .base = 0x1800, .len = 0x94, - .intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 13), }, }; diff --git a/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_3_3_sdm630.h b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_3_3_sdm630.h index 0ad18bd273ff8c..b409af89991820 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_3_3_sdm630.h +++ b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_3_3_sdm630.h @@ -36,24 +36,19 @@ static const struct dpu_ctl_cfg sdm630_ctl[] = { .name = "ctl_0", .id = CTL_0, .base = 0x1000, .len = 0x94, .features = BIT(DPU_CTL_SPLIT_DISPLAY), - .intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 9), }, { .name = "ctl_1", .id = CTL_1, .base = 0x1200, .len = 0x94, - .intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 10), }, { .name = "ctl_2", .id = CTL_2, .base = 0x1400, .len = 0x94, .features = BIT(DPU_CTL_SPLIT_DISPLAY), - .intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 11), }, { .name = "ctl_3", .id = CTL_3, .base = 0x1600, .len = 0x94, - .intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 12), }, { .name = "ctl_4", .id = CTL_4, .base = 0x1800, .len = 0x94, - .intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 13), }, }; diff --git a/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_7_2_sc7280.h b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_7_2_sc7280.h index 8f978b9c345202..2f8688224f3430 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_7_2_sc7280.h +++ b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_7_2_sc7280.h @@ -13,6 +13,7 @@ static const struct dpu_caps sc7280_dpu_caps = { .has_dim_layer = true, .has_idle_pc = true, .max_linewidth = 2400, + .has_3d_merge = true, .pixel_ram_size = DEFAULT_PIXEL_RAM_SIZE, }; @@ -134,17 +135,24 @@ static const struct dpu_pingpong_cfg sc7280_pp[] = { .name = "pingpong_2", .id = PINGPONG_2, .base = 0x6b000, .len = 0, .sblk = &sc7280_pp_sblk, - .merge_3d = 0, + .merge_3d = MERGE_3D_1, .intr_done = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 10), }, { .name = "pingpong_3", .id = PINGPONG_3, .base = 0x6c000, .len = 0, .sblk = &sc7280_pp_sblk, - .merge_3d = 0, + .merge_3d = MERGE_3D_1, .intr_done = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 11), }, }; +static const struct dpu_merge_3d_cfg sc7280_merge_3d[] = { + { + .name = "merge_3d_1", .id = MERGE_3D_1, + .base = 0x4f000, .len = 0x8, + }, +}; + /* NOTE: sc7280 only has one DSC hard slice encoder */ static const struct dpu_dsc_cfg sc7280_dsc[] = { { @@ -247,6 +255,8 @@ const struct dpu_mdss_cfg dpu_sc7280_cfg = { .mixer = sc7280_lm, .pingpong_count = ARRAY_SIZE(sc7280_pp), .pingpong = sc7280_pp, + .merge_3d_count = ARRAY_SIZE(sc7280_merge_3d), + .merge_3d = sc7280_merge_3d, .dsc_count = ARRAY_SIZE(sc7280_dsc), .dsc = sc7280_dsc, .wb_count = ARRAY_SIZE(sc7280_wb), diff --git a/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_8_0_sc8280xp.h b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_8_0_sc8280xp.h index 303d33dc7783ac..9f2bceca1789ed 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_8_0_sc8280xp.h +++ b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_8_0_sc8280xp.h @@ -133,7 +133,7 @@ static const struct dpu_sspp_cfg sc8280xp_sspp[] = { static const struct dpu_lm_cfg sc8280xp_lm[] = { { .name = "lm_0", .id = LM_0, - .base = 0x44000, .len = 0x320, + .base = 0x44000, .len = 0x400, .features = MIXER_MSM8998_MASK, .sblk = &sdm845_lm_sblk, .lm_pair = LM_1, @@ -141,7 +141,7 @@ static const struct dpu_lm_cfg sc8280xp_lm[] = { .dspp = DSPP_0, }, { .name = "lm_1", .id = LM_1, - .base = 0x45000, .len = 0x320, + .base = 0x45000, .len = 0x400, .features = MIXER_MSM8998_MASK, .sblk = &sdm845_lm_sblk, .lm_pair = LM_0, @@ -149,7 +149,7 @@ static const struct dpu_lm_cfg sc8280xp_lm[] = { .dspp = DSPP_1, }, { .name = "lm_2", .id = LM_2, - .base = 0x46000, .len = 0x320, + .base = 0x46000, .len = 0x400, .features = MIXER_MSM8998_MASK, .sblk = &sdm845_lm_sblk, .lm_pair = LM_3, @@ -157,7 +157,7 @@ static const struct dpu_lm_cfg sc8280xp_lm[] = { .dspp = DSPP_2, }, { .name = "lm_3", .id = LM_3, - .base = 0x47000, .len = 0x320, + .base = 0x47000, .len = 0x400, .features = MIXER_MSM8998_MASK, .sblk = &sdm845_lm_sblk, .lm_pair = LM_2, @@ -165,14 +165,14 @@ static const struct dpu_lm_cfg sc8280xp_lm[] = { .dspp = DSPP_3, }, { .name = "lm_4", .id = LM_4, - .base = 0x48000, .len = 0x320, + .base = 0x48000, .len = 0x400, .features = MIXER_MSM8998_MASK, .sblk = &sdm845_lm_sblk, .lm_pair = LM_5, .pingpong = PINGPONG_4, }, { .name = "lm_5", .id = LM_5, - .base = 0x49000, .len = 0x320, + .base = 0x49000, .len = 0x400, .features = MIXER_MSM8998_MASK, .sblk = &sdm845_lm_sblk, .lm_pair = LM_4, diff --git a/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_8_1_sm8450.h b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_8_1_sm8450.h index b09a6af4c474aa..04b22167f93d66 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_8_1_sm8450.h +++ b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_8_1_sm8450.h @@ -134,7 +134,7 @@ static const struct dpu_sspp_cfg sm8450_sspp[] = { static const struct dpu_lm_cfg sm8450_lm[] = { { .name = "lm_0", .id = LM_0, - .base = 0x44000, .len = 0x320, + .base = 0x44000, .len = 0x400, .features = MIXER_MSM8998_MASK, .sblk = &sdm845_lm_sblk, .lm_pair = LM_1, @@ -142,7 +142,7 @@ static const struct dpu_lm_cfg sm8450_lm[] = { .dspp = DSPP_0, }, { .name = "lm_1", .id = LM_1, - .base = 0x45000, .len = 0x320, + .base = 0x45000, .len = 0x400, .features = MIXER_MSM8998_MASK, .sblk = &sdm845_lm_sblk, .lm_pair = LM_0, @@ -150,7 +150,7 @@ static const struct dpu_lm_cfg sm8450_lm[] = { .dspp = DSPP_1, }, { .name = "lm_2", .id = LM_2, - .base = 0x46000, .len = 0x320, + .base = 0x46000, .len = 0x400, .features = MIXER_MSM8998_MASK, .sblk = &sdm845_lm_sblk, .lm_pair = LM_3, @@ -158,7 +158,7 @@ static const struct dpu_lm_cfg sm8450_lm[] = { .dspp = DSPP_2, }, { .name = "lm_3", .id = LM_3, - .base = 0x47000, .len = 0x320, + .base = 0x47000, .len = 0x400, .features = MIXER_MSM8998_MASK, .sblk = &sdm845_lm_sblk, .lm_pair = LM_2, @@ -166,14 +166,14 @@ static const struct dpu_lm_cfg sm8450_lm[] = { .dspp = DSPP_3, }, { .name = "lm_4", .id = LM_4, - .base = 0x48000, .len = 0x320, + .base = 0x48000, .len = 0x400, .features = MIXER_MSM8998_MASK, .sblk = &sdm845_lm_sblk, .lm_pair = LM_5, .pingpong = PINGPONG_4, }, { .name = "lm_5", .id = LM_5, - .base = 0x49000, .len = 0x320, + .base = 0x49000, .len = 0x400, .features = MIXER_MSM8998_MASK, .sblk = &sdm845_lm_sblk, .lm_pair = LM_4, diff --git a/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_8_4_sa8775p.h b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_8_4_sa8775p.h index 0f7b4a224e4c97..42cf3bd5a12adc 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_8_4_sa8775p.h +++ b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_8_4_sa8775p.h @@ -366,8 +366,8 @@ static const struct dpu_intf_cfg sa8775p_intf[] = { .type = INTF_NONE, .controller_id = MSM_DP_CONTROLLER_0, /* pair with intf_0 for DP MST */ .prog_fetch_lines_worst_case = 24, - .intr_underrun = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 17), - .intr_vsync = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 16), + .intr_underrun = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 16), + .intr_vsync = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 17), }, { .name = "intf_7", .id = INTF_7, .base = 0x3b000, .len = 0x280, diff --git a/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_9_0_sm8550.h b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_9_0_sm8550.h index 465b6460f8754d..4c7eb55d474c51 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_9_0_sm8550.h +++ b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_9_0_sm8550.h @@ -131,7 +131,7 @@ static const struct dpu_sspp_cfg sm8550_sspp[] = { static const struct dpu_lm_cfg sm8550_lm[] = { { .name = "lm_0", .id = LM_0, - .base = 0x44000, .len = 0x320, + .base = 0x44000, .len = 0x400, .features = MIXER_MSM8998_MASK, .sblk = &sdm845_lm_sblk, .lm_pair = LM_1, @@ -139,7 +139,7 @@ static const struct dpu_lm_cfg sm8550_lm[] = { .dspp = DSPP_0, }, { .name = "lm_1", .id = LM_1, - .base = 0x45000, .len = 0x320, + .base = 0x45000, .len = 0x400, .features = MIXER_MSM8998_MASK, .sblk = &sdm845_lm_sblk, .lm_pair = LM_0, @@ -147,7 +147,7 @@ static const struct dpu_lm_cfg sm8550_lm[] = { .dspp = DSPP_1, }, { .name = "lm_2", .id = LM_2, - .base = 0x46000, .len = 0x320, + .base = 0x46000, .len = 0x400, .features = MIXER_MSM8998_MASK, .sblk = &sdm845_lm_sblk, .lm_pair = LM_3, @@ -155,7 +155,7 @@ static const struct dpu_lm_cfg sm8550_lm[] = { .dspp = DSPP_2, }, { .name = "lm_3", .id = LM_3, - .base = 0x47000, .len = 0x320, + .base = 0x47000, .len = 0x400, .features = MIXER_MSM8998_MASK, .sblk = &sdm845_lm_sblk, .lm_pair = LM_2, @@ -163,14 +163,14 @@ static const struct dpu_lm_cfg sm8550_lm[] = { .dspp = DSPP_3, }, { .name = "lm_4", .id = LM_4, - .base = 0x48000, .len = 0x320, + .base = 0x48000, .len = 0x400, .features = MIXER_MSM8998_MASK, .sblk = &sdm845_lm_sblk, .lm_pair = LM_5, .pingpong = PINGPONG_4, }, { .name = "lm_5", .id = LM_5, - .base = 0x49000, .len = 0x320, + .base = 0x49000, .len = 0x400, .features = MIXER_MSM8998_MASK, .sblk = &sdm845_lm_sblk, .lm_pair = LM_4, diff --git a/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_9_1_sar2130p.h b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_9_1_sar2130p.h index 6caa7d40f36880..dec83ea8167d1d 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_9_1_sar2130p.h +++ b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_9_1_sar2130p.h @@ -131,7 +131,7 @@ static const struct dpu_sspp_cfg sar2130p_sspp[] = { static const struct dpu_lm_cfg sar2130p_lm[] = { { .name = "lm_0", .id = LM_0, - .base = 0x44000, .len = 0x320, + .base = 0x44000, .len = 0x400, .features = MIXER_MSM8998_MASK, .sblk = &sdm845_lm_sblk, .lm_pair = LM_1, @@ -139,7 +139,7 @@ static const struct dpu_lm_cfg sar2130p_lm[] = { .dspp = DSPP_0, }, { .name = "lm_1", .id = LM_1, - .base = 0x45000, .len = 0x320, + .base = 0x45000, .len = 0x400, .features = MIXER_MSM8998_MASK, .sblk = &sdm845_lm_sblk, .lm_pair = LM_0, @@ -147,7 +147,7 @@ static const struct dpu_lm_cfg sar2130p_lm[] = { .dspp = DSPP_1, }, { .name = "lm_2", .id = LM_2, - .base = 0x46000, .len = 0x320, + .base = 0x46000, .len = 0x400, .features = MIXER_MSM8998_MASK, .sblk = &sdm845_lm_sblk, .lm_pair = LM_3, @@ -155,7 +155,7 @@ static const struct dpu_lm_cfg sar2130p_lm[] = { .dspp = DSPP_2, }, { .name = "lm_3", .id = LM_3, - .base = 0x47000, .len = 0x320, + .base = 0x47000, .len = 0x400, .features = MIXER_MSM8998_MASK, .sblk = &sdm845_lm_sblk, .lm_pair = LM_2, @@ -163,14 +163,14 @@ static const struct dpu_lm_cfg sar2130p_lm[] = { .dspp = DSPP_3, }, { .name = "lm_4", .id = LM_4, - .base = 0x48000, .len = 0x320, + .base = 0x48000, .len = 0x400, .features = MIXER_MSM8998_MASK, .sblk = &sdm845_lm_sblk, .lm_pair = LM_5, .pingpong = PINGPONG_4, }, { .name = "lm_5", .id = LM_5, - .base = 0x49000, .len = 0x320, + .base = 0x49000, .len = 0x400, .features = MIXER_MSM8998_MASK, .sblk = &sdm845_lm_sblk, .lm_pair = LM_4, diff --git a/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_9_2_x1e80100.h b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_9_2_x1e80100.h index 7243eebb85f36f..52ff4baa668a4b 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_9_2_x1e80100.h +++ b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_9_2_x1e80100.h @@ -130,7 +130,7 @@ static const struct dpu_sspp_cfg x1e80100_sspp[] = { static const struct dpu_lm_cfg x1e80100_lm[] = { { .name = "lm_0", .id = LM_0, - .base = 0x44000, .len = 0x320, + .base = 0x44000, .len = 0x400, .features = MIXER_MSM8998_MASK, .sblk = &sdm845_lm_sblk, .lm_pair = LM_1, @@ -138,7 +138,7 @@ static const struct dpu_lm_cfg x1e80100_lm[] = { .dspp = DSPP_0, }, { .name = "lm_1", .id = LM_1, - .base = 0x45000, .len = 0x320, + .base = 0x45000, .len = 0x400, .features = MIXER_MSM8998_MASK, .sblk = &sdm845_lm_sblk, .lm_pair = LM_0, @@ -146,7 +146,7 @@ static const struct dpu_lm_cfg x1e80100_lm[] = { .dspp = DSPP_1, }, { .name = "lm_2", .id = LM_2, - .base = 0x46000, .len = 0x320, + .base = 0x46000, .len = 0x400, .features = MIXER_MSM8998_MASK, .sblk = &sdm845_lm_sblk, .lm_pair = LM_3, @@ -154,7 +154,7 @@ static const struct dpu_lm_cfg x1e80100_lm[] = { .dspp = DSPP_2, }, { .name = "lm_3", .id = LM_3, - .base = 0x47000, .len = 0x320, + .base = 0x47000, .len = 0x400, .features = MIXER_MSM8998_MASK, .sblk = &sdm845_lm_sblk, .lm_pair = LM_2, @@ -162,14 +162,14 @@ static const struct dpu_lm_cfg x1e80100_lm[] = { .dspp = DSPP_3, }, { .name = "lm_4", .id = LM_4, - .base = 0x48000, .len = 0x320, + .base = 0x48000, .len = 0x400, .features = MIXER_MSM8998_MASK, .sblk = &sdm845_lm_sblk, .lm_pair = LM_5, .pingpong = PINGPONG_4, }, { .name = "lm_5", .id = LM_5, - .base = 0x49000, .len = 0x320, + .base = 0x49000, .len = 0x400, .features = MIXER_MSM8998_MASK, .sblk = &sdm845_lm_sblk, .lm_pair = LM_4, diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.c index 9f3957f24c6a3d..eba1d52211f68f 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.c +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.c @@ -785,24 +785,24 @@ static void _dpu_encoder_update_vsync_source(struct dpu_encoder_virt *dpu_enc, return; } + vsync_cfg.vsync_source = disp_info->vsync_source; + vsync_cfg.frame_rate = drm_mode_vrefresh(&dpu_enc->base.crtc->state->adjusted_mode); + if (hw_mdptop->ops.setup_vsync_source) { for (i = 0; i < dpu_enc->num_phys_encs; i++) vsync_cfg.ppnumber[i] = dpu_enc->hw_pp[i]->idx; vsync_cfg.pp_count = dpu_enc->num_phys_encs; - vsync_cfg.frame_rate = drm_mode_vrefresh(&dpu_enc->base.crtc->state->adjusted_mode); - - vsync_cfg.vsync_source = disp_info->vsync_source; hw_mdptop->ops.setup_vsync_source(hw_mdptop, &vsync_cfg); + } - for (i = 0; i < dpu_enc->num_phys_encs; i++) { - phys_enc = dpu_enc->phys_encs[i]; + for (i = 0; i < dpu_enc->num_phys_encs; i++) { + phys_enc = dpu_enc->phys_encs[i]; - if (phys_enc->has_intf_te && phys_enc->hw_intf->ops.vsync_sel) - phys_enc->hw_intf->ops.vsync_sel(phys_enc->hw_intf, - vsync_cfg.vsync_source); - } + if (phys_enc->has_intf_te && phys_enc->hw_intf->ops.vsync_sel) + phys_enc->hw_intf->ops.vsync_sel(phys_enc->hw_intf, + &vsync_cfg); } } diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder_phys_cmd.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder_phys_cmd.c index 0ec6d67c7c70b1..93db1484f60698 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder_phys_cmd.c +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder_phys_cmd.c @@ -681,10 +681,11 @@ static int dpu_encoder_phys_cmd_wait_for_commit_done( if (!dpu_encoder_phys_cmd_is_master(phys_enc)) return 0; - if (phys_enc->hw_ctl->ops.is_started(phys_enc->hw_ctl)) - return dpu_encoder_phys_cmd_wait_for_tx_complete(phys_enc); + if (phys_enc->irq[INTR_IDX_CTL_START] && + !phys_enc->hw_ctl->ops.is_started(phys_enc->hw_ctl)) + return _dpu_encoder_phys_cmd_wait_for_ctl_start(phys_enc); - return _dpu_encoder_phys_cmd_wait_for_ctl_start(phys_enc); + return dpu_encoder_phys_cmd_wait_for_tx_complete(phys_enc); } static void dpu_encoder_phys_cmd_handle_post_kickoff( diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_intf.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_intf.c index a80ac82a96255d..7e620f59098499 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_intf.c +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_intf.c @@ -67,6 +67,10 @@ #define INTF_MISR_CTRL 0x180 #define INTF_MISR_SIGNATURE 0x184 +#define INTF_WD_TIMER_0_CTL 0x230 +#define INTF_WD_TIMER_0_CTL2 0x234 +#define INTF_WD_TIMER_0_LOAD_VALUE 0x238 + #define INTF_MUX 0x25C #define INTF_STATUS 0x26C #define INTF_AVR_CONTROL 0x270 @@ -475,7 +479,20 @@ static int dpu_hw_intf_get_vsync_info(struct dpu_hw_intf *intf, } static void dpu_hw_intf_vsync_sel(struct dpu_hw_intf *intf, - enum dpu_vsync_source vsync_source) + struct dpu_vsync_source_cfg *cfg) +{ + struct dpu_hw_blk_reg_map *c; + + if (!intf) + return; + + c = &intf->hw; + + DPU_REG_WRITE(c, INTF_TEAR_MDP_VSYNC_SEL, (cfg->vsync_source & 0xf)); +} + +static void dpu_hw_intf_vsync_sel_v8(struct dpu_hw_intf *intf, + struct dpu_vsync_source_cfg *cfg) { struct dpu_hw_blk_reg_map *c; @@ -484,7 +501,30 @@ static void dpu_hw_intf_vsync_sel(struct dpu_hw_intf *intf, c = &intf->hw; - DPU_REG_WRITE(c, INTF_TEAR_MDP_VSYNC_SEL, (vsync_source & 0xf)); + if (cfg->vsync_source >= DPU_VSYNC_SOURCE_WD_TIMER_4 && + cfg->vsync_source <= DPU_VSYNC_SOURCE_WD_TIMER_1) { + pr_warn_once("DPU 8.x supports only GPIOs and timer0 as TE sources\n"); + return; + } + + if (cfg->vsync_source == DPU_VSYNC_SOURCE_WD_TIMER_0) { + u32 reg; + + DPU_REG_WRITE(c, INTF_WD_TIMER_0_LOAD_VALUE, + CALCULATE_WD_LOAD_VALUE(cfg->frame_rate)); + + DPU_REG_WRITE(c, INTF_WD_TIMER_0_CTL, BIT(0)); /* clear timer */ + + reg = BIT(8); /* enable heartbeat timer */ + reg |= BIT(0); /* enable WD timer */ + reg |= BIT(1); /* select default 16 clock ticks */ + DPU_REG_WRITE(c, INTF_WD_TIMER_0_CTL2, reg); + + /* make sure that timers are enabled/disabled for vsync state */ + wmb(); + } + + dpu_hw_intf_vsync_sel(intf, cfg); } static void dpu_hw_intf_disable_autorefresh(struct dpu_hw_intf *intf, @@ -598,7 +638,10 @@ struct dpu_hw_intf *dpu_hw_intf_init(struct drm_device *dev, c->ops.enable_tearcheck = dpu_hw_intf_enable_te; c->ops.disable_tearcheck = dpu_hw_intf_disable_te; c->ops.connect_external_te = dpu_hw_intf_connect_external_te; - c->ops.vsync_sel = dpu_hw_intf_vsync_sel; + if (mdss_rev->core_major_ver >= 8) + c->ops.vsync_sel = dpu_hw_intf_vsync_sel_v8; + else + c->ops.vsync_sel = dpu_hw_intf_vsync_sel; c->ops.disable_autorefresh = dpu_hw_intf_disable_autorefresh; } diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_intf.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_intf.h index 5a19cd74fa9471..f6ef2c21b66d42 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_intf.h +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_intf.h @@ -12,6 +12,7 @@ #include "dpu_hw_util.h" struct dpu_hw_intf; +struct dpu_vsync_source_cfg; /* intf timing settings */ struct dpu_hw_intf_timing_params { @@ -104,7 +105,7 @@ struct dpu_hw_intf_ops { int (*connect_external_te)(struct dpu_hw_intf *intf, bool enable_external_te); - void (*vsync_sel)(struct dpu_hw_intf *intf, enum dpu_vsync_source vsync_source); + void (*vsync_sel)(struct dpu_hw_intf *intf, struct dpu_vsync_source_cfg *cfg); void (*disable_autorefresh)(struct dpu_hw_intf *intf, uint32_t encoder_id, u16 vdisplay); diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_sspp.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_sspp.c index 6f1fc790ad6d81..f275b14da4ffd0 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_sspp.c +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_sspp.c @@ -72,6 +72,8 @@ #define SSPP_EXCL_REC_XY_REC1 0x188 #define SSPP_EXCL_REC_SIZE 0x1B4 #define SSPP_EXCL_REC_XY 0x1B8 +#define SSPP_UBWC_STATIC_CTRL_REC1 0x1c0 +#define SSPP_UBWC_ERROR_STATUS_REC1 0x1c8 #define SSPP_CLK_CTRL 0x330 /* SSPP_SRC_OP_MODE & OP_MODE_REC1 */ @@ -215,7 +217,7 @@ static void dpu_hw_sspp_setup_format(struct dpu_sw_pipe *pipe, u32 chroma_samp, unpack, src_format; u32 opmode = 0; u32 fast_clear = 0; - u32 op_mode_off, unpack_pat_off, format_off; + u32 op_mode_off, unpack_pat_off, format_off, ubwc_ctrl_off, ubwc_error_off; if (!ctx || !fmt) return; @@ -225,10 +227,21 @@ static void dpu_hw_sspp_setup_format(struct dpu_sw_pipe *pipe, op_mode_off = SSPP_SRC_OP_MODE; unpack_pat_off = SSPP_SRC_UNPACK_PATTERN; format_off = SSPP_SRC_FORMAT; + ubwc_ctrl_off = SSPP_UBWC_STATIC_CTRL; + ubwc_error_off = SSPP_UBWC_ERROR_STATUS; } else { op_mode_off = SSPP_SRC_OP_MODE_REC1; unpack_pat_off = SSPP_SRC_UNPACK_PATTERN_REC1; format_off = SSPP_SRC_FORMAT_REC1; + + /* reg wasn't present before DPU 8.0 */ + if (ctx->mdss_ver->core_major_ver >= 8) { + ubwc_ctrl_off = SSPP_UBWC_STATIC_CTRL_REC1; + ubwc_error_off = SSPP_UBWC_ERROR_STATUS_REC1; + } else { + ubwc_ctrl_off = SSPP_UBWC_STATIC_CTRL; + ubwc_error_off = SSPP_UBWC_ERROR_STATUS; + } } c = &ctx->hw; @@ -270,36 +283,41 @@ static void dpu_hw_sspp_setup_format(struct dpu_sw_pipe *pipe, ((fmt->bpp - 1) << 9); if (fmt->fetch_mode != MDP_FETCH_LINEAR) { + u32 hbb = ctx->ubwc->highest_bank_bit - 13; + u32 ctrl_val; + if (MSM_FORMAT_IS_UBWC(fmt)) opmode |= MDSS_MDP_OP_BWC_EN; src_format |= (fmt->fetch_mode & 3) << 30; /*FRAME_FORMAT */ DPU_REG_WRITE(c, SSPP_FETCH_CONFIG, DPU_FETCH_CONFIG_RESET_VALUE | - ctx->ubwc->highest_bank_bit << 18); - switch (ctx->ubwc->ubwc_enc_version) { - case UBWC_1_0: + hbb << 18); + + if (ctx->ubwc->ubwc_enc_version == UBWC_1_0) { fast_clear = fmt->alpha_enable ? BIT(31) : 0; - DPU_REG_WRITE(c, SSPP_UBWC_STATIC_CTRL, - fast_clear | (ctx->ubwc->ubwc_swizzle & 0x1) | - BIT(8) | - (ctx->ubwc->highest_bank_bit << 4)); - break; - case UBWC_2_0: + ctrl_val = fast_clear | (ctx->ubwc->ubwc_swizzle & 0x1) | + BIT(8) | (hbb << 4); + } else if (ctx->ubwc->ubwc_enc_version == UBWC_2_0) { fast_clear = fmt->alpha_enable ? BIT(31) : 0; - DPU_REG_WRITE(c, SSPP_UBWC_STATIC_CTRL, - fast_clear | (ctx->ubwc->ubwc_swizzle) | - (ctx->ubwc->highest_bank_bit << 4)); - break; - case UBWC_3_0: - DPU_REG_WRITE(c, SSPP_UBWC_STATIC_CTRL, - BIT(30) | (ctx->ubwc->ubwc_swizzle) | - (ctx->ubwc->highest_bank_bit << 4)); - break; - case UBWC_4_0: - DPU_REG_WRITE(c, SSPP_UBWC_STATIC_CTRL, - MSM_FORMAT_IS_YUV(fmt) ? 0 : BIT(30)); - break; + ctrl_val = fast_clear | ctx->ubwc->ubwc_swizzle | (hbb << 4); + } else if (ctx->ubwc->ubwc_enc_version == UBWC_3_0) { + ctrl_val = BIT(30) | (ctx->ubwc->ubwc_swizzle) | (hbb << 4); + } else if (ctx->ubwc->ubwc_enc_version == UBWC_4_0) { + ctrl_val = MSM_FORMAT_IS_YUV(fmt) ? 0 : BIT(30); + } else if (ctx->ubwc->ubwc_enc_version <= UBWC_6_0) { + if (MSM_FORMAT_IS_YUV(fmt)) + ctrl_val = 0; + else if (MSM_FORMAT_IS_DX(fmt)) /* or FP16, but it's unsupported */ + ctrl_val = BIT(30); + else + ctrl_val = BIT(30) | BIT(31); + /* SDE also sets bits for lossy formats, but we don't support them yet */ + } else { + DRM_WARN_ONCE("Unsupported UBWC version %x\n", ctx->ubwc->ubwc_enc_version); + ctrl_val = 0; } + + DPU_REG_WRITE(c, ubwc_ctrl_off, ctrl_val); } opmode |= MDSS_MDP_OP_PE_OVERRIDE; @@ -325,7 +343,7 @@ static void dpu_hw_sspp_setup_format(struct dpu_sw_pipe *pipe, DPU_REG_WRITE(c, op_mode_off, opmode); /* clear previous UBWC error */ - DPU_REG_WRITE(c, SSPP_UBWC_ERROR_STATUS, BIT(31)); + DPU_REG_WRITE(c, ubwc_error_off, BIT(31)); } static void dpu_hw_sspp_setup_pe_config(struct dpu_hw_sspp *ctx, diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_top.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_top.c index 96dc10589bee6c..1ebd75d4f9be8d 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_top.c +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_top.c @@ -22,13 +22,6 @@ #define TRAFFIC_SHAPER_WR_CLIENT(num) (0x060 + (num * 4)) #define TRAFFIC_SHAPER_FIXPOINT_FACTOR 4 -#define MDP_TICK_COUNT 16 -#define XO_CLK_RATE 19200 -#define MS_TICKS_IN_SEC 1000 - -#define CALCULATE_WD_LOAD_VALUE(fps) \ - ((uint32_t)((MS_TICKS_IN_SEC * XO_CLK_RATE)/(MDP_TICK_COUNT * fps))) - static void dpu_hw_setup_split_pipe(struct dpu_hw_mdp *mdp, struct split_pipe_cfg *cfg) { diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_util.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_util.h index 67b08e99335dcf..6fe65bc3bff4e8 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_util.h +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_util.h @@ -21,6 +21,13 @@ #define TO_S15D16(_x_)((_x_) << 7) +#define MDP_TICK_COUNT 16 +#define XO_CLK_RATE 19200 +#define MS_TICKS_IN_SEC 1000 + +#define CALCULATE_WD_LOAD_VALUE(fps) \ + ((uint32_t)((MS_TICKS_IN_SEC * XO_CLK_RATE)/(MDP_TICK_COUNT * fps))) + extern const struct dpu_csc_cfg dpu_csc_YUV2RGB_601L; extern const struct dpu_csc_cfg dpu_csc10_YUV2RGB_601L; extern const struct dpu_csc_cfg dpu_csc10_rgb2yuv_601l; diff --git a/drivers/gpu/drm/msm/disp/mdp_format.c b/drivers/gpu/drm/msm/disp/mdp_format.c index 426782d50cb49d..eebedb1a2636e7 100644 --- a/drivers/gpu/drm/msm/disp/mdp_format.c +++ b/drivers/gpu/drm/msm/disp/mdp_format.c @@ -479,25 +479,25 @@ static const struct msm_format mdp_formats[] = { 0, BPC8, BPC8, BPC8, C2_R_Cr, C0_G_Y, C1_B_Cb, C0_G_Y, false, CHROMA_H2V1, 4, 2, MSM_FORMAT_FLAG_YUV, - MDP_FETCH_LINEAR, 2), + MDP_FETCH_LINEAR, 1), INTERLEAVED_YUV_FMT(UYVY, 0, BPC8, BPC8, BPC8, C1_B_Cb, C0_G_Y, C2_R_Cr, C0_G_Y, false, CHROMA_H2V1, 4, 2, MSM_FORMAT_FLAG_YUV, - MDP_FETCH_LINEAR, 2), + MDP_FETCH_LINEAR, 1), INTERLEAVED_YUV_FMT(YUYV, 0, BPC8, BPC8, BPC8, C0_G_Y, C1_B_Cb, C0_G_Y, C2_R_Cr, false, CHROMA_H2V1, 4, 2, MSM_FORMAT_FLAG_YUV, - MDP_FETCH_LINEAR, 2), + MDP_FETCH_LINEAR, 1), INTERLEAVED_YUV_FMT(YVYU, 0, BPC8, BPC8, BPC8, C0_G_Y, C2_R_Cr, C0_G_Y, C1_B_Cb, false, CHROMA_H2V1, 4, 2, MSM_FORMAT_FLAG_YUV, - MDP_FETCH_LINEAR, 2), + MDP_FETCH_LINEAR, 1), /* 3 plane YUV */ PLANAR_YUV_FMT(YUV420, diff --git a/drivers/gpu/drm/msm/dp/dp_ctrl.c b/drivers/gpu/drm/msm/dp/dp_ctrl.c index cbcc7c2f0ffc46..94411870a5e0a5 100644 --- a/drivers/gpu/drm/msm/dp/dp_ctrl.c +++ b/drivers/gpu/drm/msm/dp/dp_ctrl.c @@ -2395,20 +2395,32 @@ static void msm_dp_ctrl_config_msa(struct msm_dp_ctrl_private *ctrl, bool is_ycbcr_420) { u32 pixel_m, pixel_n; - u32 mvid, nvid, pixel_div = 0, dispcc_input_rate; + u32 mvid, nvid, pixel_div, dispcc_input_rate; u32 const nvid_fixed = DP_LINK_CONSTANT_N_VALUE; u32 const link_rate_hbr2 = 540000; u32 const link_rate_hbr3 = 810000; unsigned long den, num; - if (rate == link_rate_hbr3) + switch (rate) { + case link_rate_hbr3: pixel_div = 6; - else if (rate == 162000 || rate == 270000) - pixel_div = 2; - else if (rate == link_rate_hbr2) + break; + case link_rate_hbr2: pixel_div = 4; - else + break; + case 162000: + case 270000: + pixel_div = 2; + break; + default: + /* + * This cannot be reached but the compiler is not able to know + * that statically so return early to avoid a possibly invalid + * division. + */ DRM_ERROR("Invalid pixel mux divider\n"); + return; + } dispcc_input_rate = (rate * 10) / pixel_div; diff --git a/drivers/gpu/drm/msm/dp/dp_display.c b/drivers/gpu/drm/msm/dp/dp_display.c index 9bd9cd5c1e03cc..a082f4d3ebe27d 100644 --- a/drivers/gpu/drm/msm/dp/dp_display.c +++ b/drivers/gpu/drm/msm/dp/dp_display.c @@ -141,8 +141,8 @@ static const struct msm_dp_desc msm_dp_desc_glymur[] = { static const struct msm_dp_desc msm_dp_desc_sa8775p[] = { { .io_start = 0x0af54000, .id = MSM_DP_CONTROLLER_0, .wide_bus_supported = true }, { .io_start = 0x0af5c000, .id = MSM_DP_CONTROLLER_1, .wide_bus_supported = true }, - { .io_start = 0x22154000, .id = MSM_DP_CONTROLLER_2, .wide_bus_supported = true }, - { .io_start = 0x2215c000, .id = MSM_DP_CONTROLLER_3, .wide_bus_supported = true }, + { .io_start = 0x22154000, .id = MSM_DP_CONTROLLER_0, .wide_bus_supported = true }, + { .io_start = 0x2215c000, .id = MSM_DP_CONTROLLER_1, .wide_bus_supported = true }, {} }; diff --git a/drivers/gpu/drm/msm/dsi/dsi_host.c b/drivers/gpu/drm/msm/dsi/dsi_host.c index e0de545d40775f..db6da99375a185 100644 --- a/drivers/gpu/drm/msm/dsi/dsi_host.c +++ b/drivers/gpu/drm/msm/dsi/dsi_host.c @@ -584,13 +584,30 @@ void dsi_link_clk_disable_v2(struct msm_dsi_host *msm_host) * FIXME: Reconsider this if/when CMD mode handling is rewritten to use * transfer time and data overhead as a starting point of the calculations. */ -static unsigned long dsi_adjust_pclk_for_compression(const struct drm_display_mode *mode, - const struct drm_dsc_config *dsc) +static unsigned long +dsi_adjust_pclk_for_compression(const struct drm_display_mode *mode, + const struct drm_dsc_config *dsc, + bool is_bonded_dsi) { - int new_hdisplay = DIV_ROUND_UP(mode->hdisplay * drm_dsc_get_bpp_int(dsc), - dsc->bits_per_component * 3); + int hdisplay, new_hdisplay, new_htotal; - int new_htotal = mode->htotal - mode->hdisplay + new_hdisplay; + /* + * For bonded DSI, split hdisplay across two links and round up each + * half separately, passing the full hdisplay would only round up once. + * This also aligns with the hdisplay we program later in + * dsi_timing_setup() + */ + hdisplay = mode->hdisplay; + if (is_bonded_dsi) + hdisplay /= 2; + + new_hdisplay = DIV_ROUND_UP(hdisplay * drm_dsc_get_bpp_int(dsc), + dsc->bits_per_component * 3); + + if (is_bonded_dsi) + new_hdisplay *= 2; + + new_htotal = mode->htotal - mode->hdisplay + new_hdisplay; return mult_frac(mode->clock * 1000u, new_htotal, mode->htotal); } @@ -603,7 +620,7 @@ static unsigned long dsi_get_pclk_rate(const struct drm_display_mode *mode, pclk_rate = mode->clock * 1000u; if (dsc) - pclk_rate = dsi_adjust_pclk_for_compression(mode, dsc); + pclk_rate = dsi_adjust_pclk_for_compression(mode, dsc, is_bonded_dsi); /* * For bonded DSI mode, the current DRM mode has the complete width of the @@ -993,7 +1010,7 @@ static void dsi_timing_setup(struct msm_dsi_host *msm_host, bool is_bonded_dsi) if (msm_host->dsc) { struct drm_dsc_config *dsc = msm_host->dsc; - u32 bytes_per_pclk; + u32 bits_per_pclk; /* update dsc params with timing params */ if (!dsc || !mode->hdisplay || !mode->vdisplay) { @@ -1015,7 +1032,9 @@ static void dsi_timing_setup(struct msm_dsi_host *msm_host, bool is_bonded_dsi) /* * DPU sends 3 bytes per pclk cycle to DSI. If widebus is - * enabled, bus width is extended to 6 bytes. + * enabled, MDP always sends out 48-bit compressed data per + * pclk and on average, DSI consumes an amount of compressed + * data equivalent to the uncompressed pixel depth per pclk. * * Calculate the number of pclks needed to transmit one line of * the compressed data. @@ -1027,12 +1046,12 @@ static void dsi_timing_setup(struct msm_dsi_host *msm_host, bool is_bonded_dsi) * unused anyway. */ h_total -= hdisplay; - if (wide_bus_enabled && !(msm_host->mode_flags & MIPI_DSI_MODE_VIDEO)) - bytes_per_pclk = 6; + if (wide_bus_enabled) + bits_per_pclk = mipi_dsi_pixel_format_to_bpp(msm_host->format); else - bytes_per_pclk = 3; + bits_per_pclk = 24; - hdisplay = DIV_ROUND_UP(msm_dsc_get_bytes_per_line(msm_host->dsc), bytes_per_pclk); + hdisplay = DIV_ROUND_UP(msm_dsc_get_bytes_per_line(msm_host->dsc) * 8, bits_per_pclk); h_total += hdisplay; ha_end = ha_start + hdisplay; diff --git a/drivers/gpu/drm/msm/dsi/phy/dsi_phy_14nm.c b/drivers/gpu/drm/msm/dsi/phy/dsi_phy_14nm.c index fdefcbd9c2848a..a156c7e7cea832 100644 --- a/drivers/gpu/drm/msm/dsi/phy/dsi_phy_14nm.c +++ b/drivers/gpu/drm/msm/dsi/phy/dsi_phy_14nm.c @@ -628,12 +628,7 @@ static int dsi_pll_14nm_postdiv_determine_rate(struct clk_hw *hw, DBG("DSI%d PLL parent rate=%lu", pll_14nm->phy->id, req->rate); - req->rate = divider_round_rate(hw, req->rate, &req->best_parent_rate, - NULL, - postdiv->width, - postdiv->flags); - - return 0; + return divider_determine_rate(hw, req, NULL, postdiv->width, postdiv->flags); } static int dsi_pll_14nm_postdiv_set_rate(struct clk_hw *hw, unsigned long rate, diff --git a/drivers/gpu/drm/msm/msm_mdss.c b/drivers/gpu/drm/msm/msm_mdss.c index bf9a33e925ac83..910242f5a71f24 100644 --- a/drivers/gpu/drm/msm/msm_mdss.c +++ b/drivers/gpu/drm/msm/msm_mdss.c @@ -229,7 +229,7 @@ static void msm_mdss_setup_ubwc_dec_50(struct msm_mdss *msm_mdss) { const struct qcom_ubwc_cfg_data *data = msm_mdss->mdss_data; u32 value = MDSS_UBWC_STATIC_UBWC_SWIZZLE(data->ubwc_swizzle) | - MDSS_UBWC_STATIC_HIGHEST_BANK_BIT(data->highest_bank_bit); + MDSS_UBWC_STATIC_HIGHEST_BANK_BIT(data->highest_bank_bit - 13); if (data->ubwc_bank_spread) value |= MDSS_UBWC_STATIC_UBWC_BANK_SPREAD; diff --git a/drivers/gpu/drm/nouveau/nouveau_connector.c b/drivers/gpu/drm/nouveau/nouveau_connector.c index 63621b1510f605..902e0e93e9682b 100644 --- a/drivers/gpu/drm/nouveau/nouveau_connector.c +++ b/drivers/gpu/drm/nouveau/nouveau_connector.c @@ -1230,6 +1230,9 @@ nouveau_connector_aux_xfer(struct drm_dp_aux *obj, struct drm_dp_aux_msg *msg) u8 size = msg->size; int ret; + if (pm_runtime_suspended(nv_connector->base.dev->dev)) + return -EBUSY; + nv_encoder = find_encoder(&nv_connector->base, DCB_OUTPUT_DP); if (!nv_encoder) return -ENODEV; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r535/gsp.c b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r535/gsp.c index 7fb13434c051de..a575a8dbf727df 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r535/gsp.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r535/gsp.c @@ -737,8 +737,8 @@ r535_gsp_acpi_caps(acpi_handle handle, CAPS_METHOD_DATA *caps) if (!obj) goto done; - if (WARN_ON(obj->type != ACPI_TYPE_BUFFER) || - WARN_ON(obj->buffer.length != 4)) + if (obj->type != ACPI_TYPE_BUFFER || + obj->buffer.length != 4) goto done; caps->status = 0; @@ -773,8 +773,8 @@ r535_gsp_acpi_jt(acpi_handle handle, JT_METHOD_DATA *jt) if (!obj) goto done; - if (WARN_ON(obj->type != ACPI_TYPE_BUFFER) || - WARN_ON(obj->buffer.length != 4)) + if (obj->type != ACPI_TYPE_BUFFER || + obj->buffer.length != 4) goto done; jt->status = 0; @@ -861,8 +861,8 @@ r535_gsp_acpi_dod(acpi_handle handle, DOD_METHOD_DATA *dod) _DOD = output.pointer; - if (WARN_ON(_DOD->type != ACPI_TYPE_PACKAGE) || - WARN_ON(_DOD->package.count > ARRAY_SIZE(dod->acpiIdList))) + if (_DOD->type != ACPI_TYPE_PACKAGE || + _DOD->package.count > ARRAY_SIZE(dod->acpiIdList)) return; for (int i = 0; i < _DOD->package.count; i++) { diff --git a/drivers/gpu/drm/nova/driver.rs b/drivers/gpu/drm/nova/driver.rs index 2246d8e104e083..b01954773255b1 100644 --- a/drivers/gpu/drm/nova/driver.rs +++ b/drivers/gpu/drm/nova/driver.rs @@ -63,6 +63,8 @@ impl drm::Driver for NovaDriver { const INFO: drm::DriverInfo = INFO; + const FEATURES: u32 = drm::driver::FEAT_GEM; + kernel::declare_drm_ioctls! { (NOVA_GETPARAM, drm_nova_getparam, ioctl::RENDER_ALLOW, File::get_param), (NOVA_GEM_CREATE, drm_nova_gem_create, ioctl::AUTH | ioctl::RENDER_ALLOW, File::gem_create), diff --git a/drivers/gpu/drm/nova/file.rs b/drivers/gpu/drm/nova/file.rs index a3b7bd36792c1b..a8288bc014c250 100644 --- a/drivers/gpu/drm/nova/file.rs +++ b/drivers/gpu/drm/nova/file.rs @@ -4,6 +4,7 @@ use crate::driver::{NovaDevice, NovaDriver}; use crate::gem::NovaObject; use kernel::{ alloc::flags::*, + bindings, drm::{self, gem::BaseObject}, pci, prelude::*, @@ -18,6 +19,10 @@ impl drm::file::DriverFile for File { fn open(_dev: &NovaDevice) -> Result>> { Ok(KBox::new(Self, GFP_KERNEL)?.into()) } + + fn as_raw(&self) -> *mut bindings::drm_file { + todo!() + } } impl File { diff --git a/drivers/gpu/drm/nova/gem.rs b/drivers/gpu/drm/nova/gem.rs index 2760ba4f3450be..680f91919e8421 100644 --- a/drivers/gpu/drm/nova/gem.rs +++ b/drivers/gpu/drm/nova/gem.rs @@ -16,10 +16,12 @@ use crate::{ #[pin_data] pub(crate) struct NovaObject {} +#[vtable] impl gem::DriverObject for NovaObject { type Driver = NovaDriver; + type Args = (); - fn new(_dev: &NovaDevice, _size: usize) -> impl PinInit { + fn new(_dev: &NovaDevice, _size: usize, _args: Self::Args) -> impl PinInit { try_pin_init!(NovaObject {}) } } @@ -33,7 +35,7 @@ impl NovaObject { return Err(EINVAL); } - gem::Object::new(dev, aligned_size) + gem::Object::new(dev, aligned_size, ()) } /// Look up a GEM object handle for a `File` and return an `ObjectRef` for it. diff --git a/drivers/gpu/drm/panel/panel-edp.c b/drivers/gpu/drm/panel/panel-edp.c index 415b894890ad7c..679f4af5246d8a 100644 --- a/drivers/gpu/drm/panel/panel-edp.c +++ b/drivers/gpu/drm/panel/panel-edp.c @@ -1730,6 +1730,12 @@ static const struct panel_delay delay_200_500_p2e100 = { .prepare_to_enable = 100, }; +static const struct panel_delay delay_200_500_p2e200 = { + .hpd_absent = 200, + .unprepare = 500, + .prepare_to_enable = 200, +}; + static const struct panel_delay delay_200_500_e50 = { .hpd_absent = 200, .unprepare = 500, @@ -1880,6 +1886,7 @@ static const struct panel_delay delay_80_500_e50_d50 = { */ static const struct edp_panel_entry edp_panels[] = { EDP_PANEL_ENTRY('A', 'U', 'O', 0x04a4, &delay_200_500_e50, "B122UAN01.0"), + EDP_PANEL_ENTRY('A', 'U', 'O', 0x0ba4, &delay_200_500_e50, "B140QAX01.H"), EDP_PANEL_ENTRY('A', 'U', 'O', 0x105c, &delay_200_500_e50, "B116XTN01.0"), EDP_PANEL_ENTRY('A', 'U', 'O', 0x1062, &delay_200_500_e50, "B120XAN01.0"), EDP_PANEL_ENTRY('A', 'U', 'O', 0x125c, &delay_200_500_e50, "Unknown"), @@ -1904,6 +1911,7 @@ static const struct edp_panel_entry edp_panels[] = { EDP_PANEL_ENTRY('A', 'U', 'O', 0x615c, &delay_200_500_e50, "B116XAN06.1"), EDP_PANEL_ENTRY('A', 'U', 'O', 0x635c, &delay_200_500_e50, "B116XAN06.3"), EDP_PANEL_ENTRY('A', 'U', 'O', 0x639c, &delay_200_500_e50, "B140HAK02.7"), + EDP_PANEL_ENTRY('A', 'U', 'O', 0x643d, &delay_200_500_e50, "B140HAN06.4"), EDP_PANEL_ENTRY('A', 'U', 'O', 0x723c, &delay_200_500_e50, "B140XTN07.2"), EDP_PANEL_ENTRY('A', 'U', 'O', 0x73aa, &delay_200_500_e50, "B116XTN02.3"), EDP_PANEL_ENTRY('A', 'U', 'O', 0x8594, &delay_200_500_e50, "B133UAN01.0"), @@ -1975,6 +1983,7 @@ static const struct edp_panel_entry edp_panels[] = { EDP_PANEL_ENTRY('B', 'O', 'E', 0x0b56, &delay_200_500_e80, "NT140FHM-N47"), EDP_PANEL_ENTRY('B', 'O', 'E', 0x0b66, &delay_200_500_e80, "NE140WUM-N6G"), EDP_PANEL_ENTRY('B', 'O', 'E', 0x0c20, &delay_200_500_e80, "NT140FHM-N47"), + EDP_PANEL_ENTRY('B', 'O', 'E', 0x0c26, &delay_200_500_p2e200, "NV140WUM-T08"), EDP_PANEL_ENTRY('B', 'O', 'E', 0x0c93, &delay_200_500_e200, "Unknown"), EDP_PANEL_ENTRY('B', 'O', 'E', 0x0cb6, &delay_200_500_e200, "NT116WHM-N44"), EDP_PANEL_ENTRY('B', 'O', 'E', 0x0cf2, &delay_200_500_e200, "NV156FHM-N4S"), @@ -2033,6 +2042,7 @@ static const struct edp_panel_entry edp_panels[] = { EDP_PANEL_ENTRY('C', 'S', 'W', 0x1462, &delay_200_500_e50, "MNE007QS5-2"), EDP_PANEL_ENTRY('C', 'S', 'W', 0x1468, &delay_200_500_e50, "MNE007QB2-2"), EDP_PANEL_ENTRY('C', 'S', 'W', 0x146e, &delay_80_500_e50_d50, "MNE007QB3-1"), + EDP_PANEL_ENTRY('C', 'S', 'W', 0x147c, &delay_200_500_e50_d100, "MNE007QB3-1"), EDP_PANEL_ENTRY('C', 'S', 'W', 0x1519, &delay_200_500_e80_d50, "MNF601BS1-3"), EDP_PANEL_ENTRY('E', 'T', 'C', 0x0000, &delay_50_500_e200_d200_po2e335, "LP079QX1-SP0V"), diff --git a/drivers/gpu/drm/panel/panel-jdi-lpm102a188a.c b/drivers/gpu/drm/panel/panel-jdi-lpm102a188a.c index 23462065d726b2..ea975170fafff4 100644 --- a/drivers/gpu/drm/panel/panel-jdi-lpm102a188a.c +++ b/drivers/gpu/drm/panel/panel-jdi-lpm102a188a.c @@ -434,8 +434,10 @@ static void jdi_panel_dsi_remove(struct mipi_dsi_device *dsi) int err; /* only detach from host for the DSI-LINK2 interface */ - if (!jdi) + if (!jdi) { mipi_dsi_detach(dsi); + return; + } err = jdi_panel_disable(&jdi->base); if (err < 0) diff --git a/drivers/gpu/drm/panel/panel-lg-sw43408.c b/drivers/gpu/drm/panel/panel-lg-sw43408.c index 46a56ea92ad9ff..6e307fba658f78 100644 --- a/drivers/gpu/drm/panel/panel-lg-sw43408.c +++ b/drivers/gpu/drm/panel/panel-lg-sw43408.c @@ -294,10 +294,6 @@ static void sw43408_remove(struct mipi_dsi_device *dsi) struct sw43408_panel *ctx = mipi_dsi_get_drvdata(dsi); int ret; - ret = sw43408_unprepare(&ctx->base); - if (ret < 0) - dev_err(&dsi->dev, "failed to unprepare panel: %d\n", ret); - ret = mipi_dsi_detach(dsi); if (ret < 0) dev_err(&dsi->dev, "failed to detach from DSI host: %d\n", ret); diff --git a/drivers/gpu/drm/panthor/panthor_fw.c b/drivers/gpu/drm/panthor/panthor_fw.c index 1a5e3c1a27fbc0..9533b1a31820e6 100644 --- a/drivers/gpu/drm/panthor/panthor_fw.c +++ b/drivers/gpu/drm/panthor/panthor_fw.c @@ -1187,7 +1187,6 @@ void panthor_fw_pre_reset(struct panthor_device *ptdev, bool on_hang) else ptdev->reset.fast = true; } - panthor_fw_stop(ptdev); panthor_job_irq_suspend(&ptdev->fw->irq); panthor_fw_stop(ptdev); @@ -1261,10 +1260,6 @@ void panthor_fw_unplug(struct panthor_device *ptdev) if (ptdev->fw->irq.irq) panthor_job_irq_suspend(&ptdev->fw->irq); - panthor_fw_halt_mcu(ptdev); - if (!panthor_fw_wait_mcu_halted(ptdev)) - drm_warn(&ptdev->base, "Failed to halt MCU on unplug"); - panthor_fw_stop(ptdev); } diff --git a/drivers/gpu/drm/panthor/panthor_gpu.c b/drivers/gpu/drm/panthor/panthor_gpu.c index 06b231b2460ab5..ff5231269518ef 100644 --- a/drivers/gpu/drm/panthor/panthor_gpu.c +++ b/drivers/gpu/drm/panthor/panthor_gpu.c @@ -51,7 +51,7 @@ struct panthor_gpu { static void panthor_gpu_coherency_set(struct panthor_device *ptdev) { gpu_write(ptdev, GPU_COHERENCY_PROTOCOL, - ptdev->coherent ? GPU_COHERENCY_PROT_BIT(ACE_LITE) : GPU_COHERENCY_NONE); + ptdev->coherent ? GPU_COHERENCY_ACE_LITE : GPU_COHERENCY_NONE); } static void panthor_gpu_l2_config_set(struct panthor_device *ptdev) @@ -289,38 +289,42 @@ int panthor_gpu_l2_power_on(struct panthor_device *ptdev) int panthor_gpu_flush_caches(struct panthor_device *ptdev, u32 l2, u32 lsc, u32 other) { - bool timedout = false; unsigned long flags; + int ret = 0; /* Serialize cache flush operations. */ guard(mutex)(&ptdev->gpu->cache_flush_lock); spin_lock_irqsave(&ptdev->gpu->reqs_lock, flags); - if (!drm_WARN_ON(&ptdev->base, - ptdev->gpu->pending_reqs & GPU_IRQ_CLEAN_CACHES_COMPLETED)) { + if (!(ptdev->gpu->pending_reqs & GPU_IRQ_CLEAN_CACHES_COMPLETED)) { ptdev->gpu->pending_reqs |= GPU_IRQ_CLEAN_CACHES_COMPLETED; gpu_write(ptdev, GPU_CMD, GPU_FLUSH_CACHES(l2, lsc, other)); + } else { + ret = -EIO; } spin_unlock_irqrestore(&ptdev->gpu->reqs_lock, flags); + if (ret) + return ret; + if (!wait_event_timeout(ptdev->gpu->reqs_acked, !(ptdev->gpu->pending_reqs & GPU_IRQ_CLEAN_CACHES_COMPLETED), msecs_to_jiffies(100))) { spin_lock_irqsave(&ptdev->gpu->reqs_lock, flags); if ((ptdev->gpu->pending_reqs & GPU_IRQ_CLEAN_CACHES_COMPLETED) != 0 && !(gpu_read(ptdev, GPU_INT_RAWSTAT) & GPU_IRQ_CLEAN_CACHES_COMPLETED)) - timedout = true; + ret = -ETIMEDOUT; else ptdev->gpu->pending_reqs &= ~GPU_IRQ_CLEAN_CACHES_COMPLETED; spin_unlock_irqrestore(&ptdev->gpu->reqs_lock, flags); } - if (timedout) { + if (ret) { + panthor_device_schedule_reset(ptdev); drm_err(&ptdev->base, "Flush caches timeout"); - return -ETIMEDOUT; } - return 0; + return ret; } /** @@ -360,6 +364,7 @@ int panthor_gpu_soft_reset(struct panthor_device *ptdev) return -ETIMEDOUT; } + ptdev->gpu->pending_reqs = 0; return 0; } diff --git a/drivers/gpu/drm/panthor/panthor_mmu.c b/drivers/gpu/drm/panthor/panthor_mmu.c index f6339963e4960d..c15d2a3906db9a 100644 --- a/drivers/gpu/drm/panthor/panthor_mmu.c +++ b/drivers/gpu/drm/panthor/panthor_mmu.c @@ -510,27 +510,29 @@ static int wait_ready(struct panthor_device *ptdev, u32 as_nr) return ret; } -static int write_cmd(struct panthor_device *ptdev, u32 as_nr, u32 cmd) +static int as_send_cmd_and_wait(struct panthor_device *ptdev, u32 as_nr, u32 cmd) { int status; /* write AS_COMMAND when MMU is ready to accept another command */ status = wait_ready(ptdev, as_nr); - if (!status) + if (!status) { gpu_write(ptdev, AS_COMMAND(as_nr), cmd); + status = wait_ready(ptdev, as_nr); + } return status; } -static void lock_region(struct panthor_device *ptdev, u32 as_nr, - u64 region_start, u64 size) +static int lock_region(struct panthor_device *ptdev, u32 as_nr, + u64 region_start, u64 size) { u8 region_width; u64 region; u64 region_end = region_start + size; if (!size) - return; + return 0; /* * The locked region is a naturally aligned power of 2 block encoded as @@ -553,7 +555,7 @@ static void lock_region(struct panthor_device *ptdev, u32 as_nr, /* Lock the region that needs to be updated */ gpu_write64(ptdev, AS_LOCKADDR(as_nr), region); - write_cmd(ptdev, as_nr, AS_COMMAND_LOCK); + return as_send_cmd_and_wait(ptdev, as_nr, AS_COMMAND_LOCK); } static int mmu_hw_do_operation_locked(struct panthor_device *ptdev, int as_nr, @@ -586,9 +588,7 @@ static int mmu_hw_do_operation_locked(struct panthor_device *ptdev, int as_nr, * power it up */ - lock_region(ptdev, as_nr, iova, size); - - ret = wait_ready(ptdev, as_nr); + ret = lock_region(ptdev, as_nr, iova, size); if (ret) return ret; @@ -601,10 +601,7 @@ static int mmu_hw_do_operation_locked(struct panthor_device *ptdev, int as_nr, * at the end of the GPU_CONTROL cache flush command, unlike * AS_COMMAND_FLUSH_MEM or AS_COMMAND_FLUSH_PT. */ - write_cmd(ptdev, as_nr, AS_COMMAND_UNLOCK); - - /* Wait for the unlock command to complete */ - return wait_ready(ptdev, as_nr); + return as_send_cmd_and_wait(ptdev, as_nr, AS_COMMAND_UNLOCK); } static int mmu_hw_do_operation(struct panthor_vm *vm, @@ -633,7 +630,7 @@ static int panthor_mmu_as_enable(struct panthor_device *ptdev, u32 as_nr, gpu_write64(ptdev, AS_MEMATTR(as_nr), memattr); gpu_write64(ptdev, AS_TRANSCFG(as_nr), transcfg); - return write_cmd(ptdev, as_nr, AS_COMMAND_UPDATE); + return as_send_cmd_and_wait(ptdev, as_nr, AS_COMMAND_UPDATE); } static int panthor_mmu_as_disable(struct panthor_device *ptdev, u32 as_nr) @@ -648,7 +645,7 @@ static int panthor_mmu_as_disable(struct panthor_device *ptdev, u32 as_nr) gpu_write64(ptdev, AS_MEMATTR(as_nr), 0); gpu_write64(ptdev, AS_TRANSCFG(as_nr), AS_TRANSCFG_ADRMODE_UNMAPPED); - return write_cmd(ptdev, as_nr, AS_COMMAND_UPDATE); + return as_send_cmd_and_wait(ptdev, as_nr, AS_COMMAND_UPDATE); } static u32 panthor_mmu_fault_mask(struct panthor_device *ptdev, u32 value) @@ -1503,6 +1500,10 @@ static void panthor_vm_destroy(struct panthor_vm *vm) vm->destroyed = true; + /* Tell scheduler to stop all GPU work related to this VM */ + if (refcount_read(&vm->as.active_cnt) > 0) + panthor_sched_prepare_for_vm_destruction(vm->ptdev); + mutex_lock(&vm->heaps.lock); panthor_heap_pool_destroy(vm->heaps.pool); vm->heaps.pool = NULL; diff --git a/drivers/gpu/drm/panthor/panthor_sched.c b/drivers/gpu/drm/panthor/panthor_sched.c index a6b8024e1a3cd5..5c55ba593b2445 100644 --- a/drivers/gpu/drm/panthor/panthor_sched.c +++ b/drivers/gpu/drm/panthor/panthor_sched.c @@ -23,6 +23,7 @@ #include #include #include +#include #include "panthor_devfreq.h" #include "panthor_device.h" @@ -939,6 +940,9 @@ static void group_release_work(struct work_struct *work) release_work); u32 i; + /* dma-fences may still be accessing group->queues under rcu lock. */ + synchronize_rcu(); + for (i = 0; i < group->queue_count; i++) group_free_queue(group, group->queues[i]); @@ -1072,18 +1076,6 @@ group_is_idle(struct panthor_group *group) return hweight32(inactive_queues) == group->queue_count; } -static void -queue_reset_timeout_locked(struct panthor_queue *queue) -{ - lockdep_assert_held(&queue->fence_ctx.lock); - - if (queue->timeout.remaining != MAX_SCHEDULE_TIMEOUT) { - mod_delayed_work(queue->scheduler.timeout_wq, - &queue->timeout.work, - msecs_to_jiffies(JOB_TIMEOUT_MS)); - } -} - static bool group_can_run(struct panthor_group *group) { @@ -1100,6 +1092,18 @@ queue_timeout_is_suspended(struct panthor_queue *queue) return queue->timeout.remaining != MAX_SCHEDULE_TIMEOUT; } +static void +queue_reset_timeout_locked(struct panthor_queue *queue) +{ + lockdep_assert_held(&queue->fence_ctx.lock); + + if (!queue_timeout_is_suspended(queue)) { + mod_delayed_work(queue->scheduler.timeout_wq, + &queue->timeout.work, + msecs_to_jiffies(JOB_TIMEOUT_MS)); + } +} + static void queue_suspend_timeout_locked(struct panthor_queue *queue) { @@ -2020,10 +2024,10 @@ struct panthor_sched_tick_ctx { struct list_head groups[PANTHOR_CSG_PRIORITY_COUNT]; u32 idle_group_count; u32 group_count; - enum panthor_csg_priority min_priority; struct panthor_vm *vms[MAX_CS_PER_CSG]; u32 as_count; bool immediate_tick; + bool stop_tick; u32 csg_upd_failed_mask; }; @@ -2066,17 +2070,21 @@ tick_ctx_pick_groups_from_list(const struct panthor_scheduler *sched, if (!owned_by_tick_ctx) group_get(group); - list_move_tail(&group->run_node, &ctx->groups[group->priority]); ctx->group_count++; + + /* If we have more than one active group with the same priority, + * we need to keep ticking to rotate the CSG priority. + */ if (group_is_idle(group)) ctx->idle_group_count++; + else if (!list_empty(&ctx->groups[group->priority])) + ctx->stop_tick = false; + + list_move_tail(&group->run_node, &ctx->groups[group->priority]); if (i == ctx->as_count) ctx->vms[ctx->as_count++] = group->vm; - if (ctx->min_priority > group->priority) - ctx->min_priority = group->priority; - if (tick_ctx_is_full(sched, ctx)) return; } @@ -2085,31 +2093,22 @@ tick_ctx_pick_groups_from_list(const struct panthor_scheduler *sched, static void tick_ctx_insert_old_group(struct panthor_scheduler *sched, struct panthor_sched_tick_ctx *ctx, - struct panthor_group *group, - bool full_tick) + struct panthor_group *group) { struct panthor_csg_slot *csg_slot = &sched->csg_slots[group->csg_id]; struct panthor_group *other_group; - if (!full_tick) { - list_add_tail(&group->run_node, &ctx->old_groups[group->priority]); - return; - } - - /* Rotate to make sure groups with lower CSG slot - * priorities have a chance to get a higher CSG slot - * priority next time they get picked. This priority - * has an impact on resource request ordering, so it's - * important to make sure we don't let one group starve - * all other groups with the same group priority. - */ + /* Class groups in descending priority order so we can easily rotate. */ list_for_each_entry(other_group, &ctx->old_groups[csg_slot->group->priority], run_node) { struct panthor_csg_slot *other_csg_slot = &sched->csg_slots[other_group->csg_id]; - if (other_csg_slot->priority > csg_slot->priority) { - list_add_tail(&csg_slot->group->run_node, &other_group->run_node); + /* Our group has a higher prio than the one we're testing against, + * place it just before. + */ + if (csg_slot->priority > other_csg_slot->priority) { + list_add_tail(&group->run_node, &other_group->run_node); return; } } @@ -2119,8 +2118,7 @@ tick_ctx_insert_old_group(struct panthor_scheduler *sched, static void tick_ctx_init(struct panthor_scheduler *sched, - struct panthor_sched_tick_ctx *ctx, - bool full_tick) + struct panthor_sched_tick_ctx *ctx) { struct panthor_device *ptdev = sched->ptdev; struct panthor_csg_slots_upd_ctx upd_ctx; @@ -2130,7 +2128,7 @@ tick_ctx_init(struct panthor_scheduler *sched, memset(ctx, 0, sizeof(*ctx)); csgs_upd_ctx_init(&upd_ctx); - ctx->min_priority = PANTHOR_CSG_PRIORITY_COUNT; + ctx->stop_tick = true; for (i = 0; i < ARRAY_SIZE(ctx->groups); i++) { INIT_LIST_HEAD(&ctx->groups[i]); INIT_LIST_HEAD(&ctx->old_groups[i]); @@ -2158,7 +2156,7 @@ tick_ctx_init(struct panthor_scheduler *sched, group->fatal_queues |= GENMASK(group->queue_count - 1, 0); } - tick_ctx_insert_old_group(sched, ctx, group, full_tick); + tick_ctx_insert_old_group(sched, ctx, group); csgs_upd_ctx_queue_reqs(ptdev, &upd_ctx, i, csg_iface->output->ack ^ CSG_STATUS_UPDATE, CSG_STATUS_UPDATE); @@ -2442,32 +2440,18 @@ static u64 tick_ctx_update_resched_target(struct panthor_scheduler *sched, const struct panthor_sched_tick_ctx *ctx) { - /* We had space left, no need to reschedule until some external event happens. */ - if (!tick_ctx_is_full(sched, ctx)) - goto no_tick; - - /* If idle groups were scheduled, no need to wake up until some external - * event happens (group unblocked, new job submitted, ...). - */ - if (ctx->idle_group_count) - goto no_tick; + u64 resched_target; - if (drm_WARN_ON(&sched->ptdev->base, ctx->min_priority >= PANTHOR_CSG_PRIORITY_COUNT)) + if (ctx->stop_tick) goto no_tick; - /* If there are groups of the same priority waiting, we need to - * keep the scheduler ticking, otherwise, we'll just wait for - * new groups with higher priority to be queued. - */ - if (!list_empty(&sched->groups.runnable[ctx->min_priority])) { - u64 resched_target = sched->last_tick + sched->tick_period; + resched_target = sched->last_tick + sched->tick_period; - if (time_before64(sched->resched_target, sched->last_tick) || - time_before64(resched_target, sched->resched_target)) - sched->resched_target = resched_target; + if (time_before64(sched->resched_target, sched->last_tick) || + time_before64(resched_target, sched->resched_target)) + sched->resched_target = resched_target; - return sched->resched_target - sched->last_tick; - } + return sched->resched_target - sched->last_tick; no_tick: sched->resched_target = U64_MAX; @@ -2480,9 +2464,11 @@ static void tick_work(struct work_struct *work) tick_work.work); struct panthor_device *ptdev = sched->ptdev; struct panthor_sched_tick_ctx ctx; + u64 resched_target = sched->resched_target; u64 remaining_jiffies = 0, resched_delay; u64 now = get_jiffies_64(); int prio, ret, cookie; + bool full_tick; if (!drm_dev_enter(&ptdev->base, &cookie)) return; @@ -2491,18 +2477,24 @@ static void tick_work(struct work_struct *work) if (drm_WARN_ON(&ptdev->base, ret)) goto out_dev_exit; - if (time_before64(now, sched->resched_target)) - remaining_jiffies = sched->resched_target - now; + /* If the tick is stopped, calculate when the next tick would be */ + if (resched_target == U64_MAX) + resched_target = sched->last_tick + sched->tick_period; + + if (time_before64(now, resched_target)) + remaining_jiffies = resched_target - now; + + full_tick = remaining_jiffies == 0; mutex_lock(&sched->lock); if (panthor_device_reset_is_pending(sched->ptdev)) goto out_unlock; - tick_ctx_init(sched, &ctx, remaining_jiffies != 0); + tick_ctx_init(sched, &ctx); if (ctx.csg_upd_failed_mask) goto out_cleanup_ctx; - if (remaining_jiffies) { + if (!full_tick) { /* Scheduling forced in the middle of a tick. Only RT groups * can preempt non-RT ones. Currently running RT groups can't be * preempted. @@ -2524,9 +2516,29 @@ static void tick_work(struct work_struct *work) for (prio = PANTHOR_CSG_PRIORITY_COUNT - 1; prio >= 0 && !tick_ctx_is_full(sched, &ctx); prio--) { + struct panthor_group *old_highest_prio_group = + list_first_entry_or_null(&ctx.old_groups[prio], + struct panthor_group, run_node); + + /* Pull out the group with the highest prio for rotation. */ + if (old_highest_prio_group) + list_del(&old_highest_prio_group->run_node); + + /* Re-insert old active groups so they get a chance to run with higher prio. */ + tick_ctx_pick_groups_from_list(sched, &ctx, &ctx.old_groups[prio], true, true); + + /* Fill the remaining slots with runnable groups. */ tick_ctx_pick_groups_from_list(sched, &ctx, &sched->groups.runnable[prio], true, false); - tick_ctx_pick_groups_from_list(sched, &ctx, &ctx.old_groups[prio], true, true); + + /* Re-insert the old group with the highest prio, and give it a chance to be + * scheduled again (but with a lower prio) if there's room left. + */ + if (old_highest_prio_group) { + list_add_tail(&old_highest_prio_group->run_node, &ctx.old_groups[prio]); + tick_ctx_pick_groups_from_list(sched, &ctx, &ctx.old_groups[prio], + true, true); + } } /* If we have free CSG slots left, pick idle groups */ @@ -2651,14 +2663,33 @@ static void sync_upd_work(struct work_struct *work) sched_queue_delayed_work(sched, tick, 0); } +static void sched_resume_tick(struct panthor_device *ptdev) +{ + struct panthor_scheduler *sched = ptdev->scheduler; + u64 delay_jiffies, now; + + drm_WARN_ON(&ptdev->base, sched->resched_target != U64_MAX); + + /* Scheduler tick was off, recalculate the resched_target based on the + * last tick event, and queue the scheduler work. + */ + now = get_jiffies_64(); + sched->resched_target = sched->last_tick + sched->tick_period; + if (sched->used_csg_slot_count == sched->csg_slot_count && + time_before64(now, sched->resched_target)) + delay_jiffies = min_t(unsigned long, sched->resched_target - now, ULONG_MAX); + else + delay_jiffies = 0; + + sched_queue_delayed_work(sched, tick, delay_jiffies); +} + static void group_schedule_locked(struct panthor_group *group, u32 queue_mask) { struct panthor_device *ptdev = group->ptdev; struct panthor_scheduler *sched = ptdev->scheduler; struct list_head *queue = &sched->groups.runnable[group->priority]; - u64 delay_jiffies = 0; bool was_idle; - u64 now; if (!group_can_run(group)) return; @@ -2703,13 +2734,7 @@ static void group_schedule_locked(struct panthor_group *group, u32 queue_mask) /* Scheduler tick was off, recalculate the resched_target based on the * last tick event, and queue the scheduler work. */ - now = get_jiffies_64(); - sched->resched_target = sched->last_tick + sched->tick_period; - if (sched->used_csg_slot_count == sched->csg_slot_count && - time_before64(now, sched->resched_target)) - delay_jiffies = min_t(unsigned long, sched->resched_target - now, ULONG_MAX); - - sched_queue_delayed_work(sched, tick, delay_jiffies); + sched_resume_tick(ptdev); } static void queue_stop(struct panthor_queue *queue, @@ -2784,6 +2809,20 @@ void panthor_sched_report_mmu_fault(struct panthor_device *ptdev) panthor_sched_immediate_tick(ptdev); } +void panthor_sched_prepare_for_vm_destruction(struct panthor_device *ptdev) +{ + /* FW can write out internal state, like the heap context, during CSG + * suspend. It is therefore important that the scheduler has fully + * evicted any pending and related groups before VM destruction can + * safely continue. Failure to do so can lead to GPU page faults. + * A controlled termination of a Panthor instance involves destroying + * the group(s) before the VM. This means any relevant group eviction + * has already been initiated by this point, and we just need to + * ensure that any pending tick_work() has been completed. + */ + flush_work(&ptdev->scheduler->tick_work.work); +} + void panthor_sched_resume(struct panthor_device *ptdev) { /* Force a tick to re-evaluate after a resume. */ @@ -3343,6 +3382,18 @@ queue_run_job(struct drm_sched_job *sched_job) if (group->csg_id < 0) { group_schedule_locked(group, BIT(job->queue_idx)); } else { + u32 queue_mask = BIT(job->queue_idx); + bool resume_tick = group_is_idle(group) && + (group->idle_queues & queue_mask) && + !(group->blocked_queues & queue_mask) && + sched->resched_target == U64_MAX; + + /* We just added something to the queue, so it's no longer idle. */ + group->idle_queues &= ~queue_mask; + + if (resume_tick) + sched_resume_tick(ptdev); + gpu_write(ptdev, CSF_DOORBELL(queue->doorbell_id), 1); if (!sched->pm.has_ref && !(group->blocked_queues & BIT(job->queue_idx))) { diff --git a/drivers/gpu/drm/panthor/panthor_sched.h b/drivers/gpu/drm/panthor/panthor_sched.h index f4a475aa34c0aa..9a8692de8adedb 100644 --- a/drivers/gpu/drm/panthor/panthor_sched.h +++ b/drivers/gpu/drm/panthor/panthor_sched.h @@ -50,6 +50,7 @@ void panthor_sched_suspend(struct panthor_device *ptdev); void panthor_sched_resume(struct panthor_device *ptdev); void panthor_sched_report_mmu_fault(struct panthor_device *ptdev); +void panthor_sched_prepare_for_vm_destruction(struct panthor_device *ptdev); void panthor_sched_report_fw_events(struct panthor_device *ptdev, u32 events); void panthor_fdinfo_gather_group_samples(struct panthor_file *pfile); diff --git a/drivers/gpu/drm/qxl/qxl_ioctl.c b/drivers/gpu/drm/qxl/qxl_ioctl.c index 336cbff2608915..26545a08cdf771 100644 --- a/drivers/gpu/drm/qxl/qxl_ioctl.c +++ b/drivers/gpu/drm/qxl/qxl_ioctl.c @@ -184,7 +184,7 @@ static int qxl_process_single_command(struct qxl_device *qdev, /* TODO copy slow path code from i915 */ fb_cmd = qxl_bo_kmap_atomic_page(qdev, cmd_bo, (release->release_offset & PAGE_MASK)); - unwritten = __copy_from_user_inatomic_nocache + unwritten = copy_from_user_inatomic_nontemporal (fb_cmd + sizeof(union qxl_release_info) + (release->release_offset & ~PAGE_MASK), u64_to_user_ptr(cmd->command), cmd->command_size); diff --git a/drivers/gpu/drm/radeon/si_dpm.c b/drivers/gpu/drm/radeon/si_dpm.c index 9deb91970d4df2..0342d095d44cec 100644 --- a/drivers/gpu/drm/radeon/si_dpm.c +++ b/drivers/gpu/drm/radeon/si_dpm.c @@ -2915,9 +2915,11 @@ static void si_apply_state_adjust_rules(struct radeon_device *rdev, if (rdev->family == CHIP_HAINAN) { if ((rdev->pdev->revision == 0x81) || (rdev->pdev->revision == 0xC3) || + (rdev->pdev->device == 0x6660) || (rdev->pdev->device == 0x6664) || (rdev->pdev->device == 0x6665) || - (rdev->pdev->device == 0x6667)) { + (rdev->pdev->device == 0x6667) || + (rdev->pdev->device == 0x666F)) { max_sclk = 75000; } if ((rdev->pdev->revision == 0xC3) || @@ -2925,6 +2927,11 @@ static void si_apply_state_adjust_rules(struct radeon_device *rdev, max_sclk = 60000; max_mclk = 80000; } + if ((rdev->pdev->device == 0x666f) && + (rdev->pdev->revision == 0x00)) { + max_sclk = 80000; + max_mclk = 95000; + } } else if (rdev->family == CHIP_OLAND) { if ((rdev->pdev->revision == 0xC7) || (rdev->pdev->revision == 0x80) || diff --git a/drivers/gpu/drm/renesas/rz-du/rzg2l_mipi_dsi.c b/drivers/gpu/drm/renesas/rz-du/rzg2l_mipi_dsi.c index 3b52dfc0ea1e04..b164e3a62cc2f7 100644 --- a/drivers/gpu/drm/renesas/rz-du/rzg2l_mipi_dsi.c +++ b/drivers/gpu/drm/renesas/rz-du/rzg2l_mipi_dsi.c @@ -646,6 +646,13 @@ static void rzg2l_mipi_dsi_atomic_disable(struct drm_bridge *bridge, rzg2l_mipi_dsi_stop_video(dsi); rzg2l_mipi_dsi_stop_hs_clock(dsi); +} + +static void rzg2l_mipi_dsi_atomic_post_disable(struct drm_bridge *bridge, + struct drm_atomic_state *state) +{ + struct rzg2l_mipi_dsi *dsi = bridge_to_rzg2l_mipi_dsi(bridge); + rzg2l_mipi_dsi_stop(dsi); } @@ -681,6 +688,7 @@ static const struct drm_bridge_funcs rzg2l_mipi_dsi_bridge_ops = { .atomic_pre_enable = rzg2l_mipi_dsi_atomic_pre_enable, .atomic_enable = rzg2l_mipi_dsi_atomic_enable, .atomic_disable = rzg2l_mipi_dsi_atomic_disable, + .atomic_post_disable = rzg2l_mipi_dsi_atomic_post_disable, .mode_valid = rzg2l_mipi_dsi_bridge_mode_valid, }; diff --git a/drivers/gpu/drm/rockchip/dw_hdmi_qp-rockchip.c b/drivers/gpu/drm/rockchip/dw_hdmi_qp-rockchip.c index 8604342f994327..c7158b1b8c59e8 100644 --- a/drivers/gpu/drm/rockchip/dw_hdmi_qp-rockchip.c +++ b/drivers/gpu/drm/rockchip/dw_hdmi_qp-rockchip.c @@ -280,12 +280,7 @@ static irqreturn_t dw_hdmi_qp_rk3576_hardirq(int irq, void *dev_id) static irqreturn_t dw_hdmi_qp_rk3576_irq(int irq, void *dev_id) { struct rockchip_hdmi_qp *hdmi = dev_id; - u32 intr_stat, val; - - regmap_read(hdmi->regmap, RK3576_IOC_HDMI_HPD_STATUS, &intr_stat); - - if (!intr_stat) - return IRQ_NONE; + u32 val; val = FIELD_PREP_WM16(RK3576_HDMI_HPD_INT_CLR, 1); regmap_write(hdmi->regmap, RK3576_IOC_MISC_CON0, val); diff --git a/drivers/gpu/drm/scheduler/sched_entity.c b/drivers/gpu/drm/scheduler/sched_entity.c index fe174a4857be7f..e1920f3f920aee 100644 --- a/drivers/gpu/drm/scheduler/sched_entity.c +++ b/drivers/gpu/drm/scheduler/sched_entity.c @@ -420,7 +420,12 @@ static bool drm_sched_entity_add_dependency_cb(struct drm_sched_entity *entity, /* * Fence is from the same scheduler, only need to wait for - * it to be scheduled + * it to be scheduled. + * + * Note: s_fence->sched could have been freed and reallocated + * as another scheduler. This false positive case is okay, as if + * the old scheduler was freed all of its jobs must have + * signaled their completion fences. */ fence = dma_fence_get(&s_fence->scheduled); dma_fence_put(entity->dependency); diff --git a/drivers/gpu/drm/scheduler/sched_fence.c b/drivers/gpu/drm/scheduler/sched_fence.c index 9391d6f0dc01d7..d05ab041a581a6 100644 --- a/drivers/gpu/drm/scheduler/sched_fence.c +++ b/drivers/gpu/drm/scheduler/sched_fence.c @@ -92,7 +92,7 @@ static const char *drm_sched_fence_get_driver_name(struct dma_fence *fence) static const char *drm_sched_fence_get_timeline_name(struct dma_fence *f) { struct drm_sched_fence *fence = to_drm_sched_fence(f); - return (const char *)fence->sched->name; + return (const char *)fence->sched_name; } static void drm_sched_fence_free_rcu(struct rcu_head *rcu) @@ -228,6 +228,8 @@ void drm_sched_fence_init(struct drm_sched_fence *fence, unsigned seq; fence->sched = entity->rq->sched; + strscpy(fence->sched_name, entity->rq->sched->name, + sizeof(fence->sched_name)); seq = atomic_inc_return(&entity->fence_seq); dma_fence_init(&fence->scheduled, &drm_sched_fence_ops_scheduled, &fence->lock, entity->fence_context, seq); diff --git a/drivers/gpu/drm/scheduler/sched_main.c b/drivers/gpu/drm/scheduler/sched_main.c index 1d4f1b822e7b76..2d70c06113cfee 100644 --- a/drivers/gpu/drm/scheduler/sched_main.c +++ b/drivers/gpu/drm/scheduler/sched_main.c @@ -361,6 +361,7 @@ static void drm_sched_run_free_queue(struct drm_gpu_scheduler *sched) /** * drm_sched_job_done - complete a job * @s_job: pointer to the job which is done + * @result: 0 on success, -ERRNO on error * * Finish the job's fence and resubmit the work items. */ diff --git a/drivers/gpu/drm/sitronix/st7586.c b/drivers/gpu/drm/sitronix/st7586.c index b57ebf37a664c9..16b6b4e368af85 100644 --- a/drivers/gpu/drm/sitronix/st7586.c +++ b/drivers/gpu/drm/sitronix/st7586.c @@ -347,6 +347,12 @@ static int st7586_probe(struct spi_device *spi) if (ret) return ret; + /* + * Override value set by mipi_dbi_spi_init(). This driver is a bit + * non-standard, so best to set it explicitly here. + */ + dbi->write_memory_bpw = 8; + /* Cannot read from this controller via SPI */ dbi->read_commands = NULL; @@ -356,15 +362,6 @@ static int st7586_probe(struct spi_device *spi) if (ret) return ret; - /* - * we are using 8-bit data, so we are not actually swapping anything, - * but setting mipi->swap_bytes makes mipi_dbi_typec3_command() do the - * right thing and not use 16-bit transfers (which results in swapped - * bytes on little-endian systems and causes out of order data to be - * sent to the display). - */ - dbi->swap_bytes = true; - drm_mode_config_reset(drm); ret = drm_dev_register(drm, 0); diff --git a/drivers/gpu/drm/solomon/ssd130x.c b/drivers/gpu/drm/solomon/ssd130x.c index 96cf393201372b..33ceed86ed3626 100644 --- a/drivers/gpu/drm/solomon/ssd130x.c +++ b/drivers/gpu/drm/solomon/ssd130x.c @@ -737,6 +737,7 @@ static int ssd130x_update_rect(struct ssd130x_device *ssd130x, unsigned int height = drm_rect_height(rect); unsigned int line_length = DIV_ROUND_UP(width, 8); unsigned int page_height = SSD130X_PAGE_HEIGHT; + u8 page_start = ssd130x->page_offset + y / page_height; unsigned int pages = DIV_ROUND_UP(height, page_height); struct drm_device *drm = &ssd130x->drm; u32 array_idx = 0; @@ -774,14 +775,11 @@ static int ssd130x_update_rect(struct ssd130x_device *ssd130x, */ if (!ssd130x->page_address_mode) { - u8 page_start; - /* Set address range for horizontal addressing mode */ ret = ssd130x_set_col_range(ssd130x, ssd130x->col_offset + x, width); if (ret < 0) return ret; - page_start = ssd130x->page_offset + y / page_height; ret = ssd130x_set_page_range(ssd130x, page_start, pages); if (ret < 0) return ret; @@ -813,7 +811,7 @@ static int ssd130x_update_rect(struct ssd130x_device *ssd130x, */ if (ssd130x->page_address_mode) { ret = ssd130x_set_page_pos(ssd130x, - ssd130x->page_offset + i, + page_start + i, ssd130x->col_offset + x); if (ret < 0) return ret; diff --git a/drivers/gpu/drm/sysfb/efidrm.c b/drivers/gpu/drm/sysfb/efidrm.c index 1b683d55d6ea48..ac48bfa47e0815 100644 --- a/drivers/gpu/drm/sysfb/efidrm.c +++ b/drivers/gpu/drm/sysfb/efidrm.c @@ -151,7 +151,6 @@ static struct efidrm_device *efidrm_device_create(struct drm_driver *drv, struct drm_sysfb_device *sysfb; struct drm_device *dev; struct resource *mem = NULL; - void __iomem *screen_base = NULL; struct drm_plane *primary_plane; struct drm_crtc *crtc; struct drm_encoder *encoder; @@ -236,21 +235,38 @@ static struct efidrm_device *efidrm_device_create(struct drm_driver *drv, mem_flags = efidrm_get_mem_flags(dev, res->start, vsize); - if (mem_flags & EFI_MEMORY_WC) - screen_base = devm_ioremap_wc(&pdev->dev, mem->start, resource_size(mem)); - else if (mem_flags & EFI_MEMORY_UC) - screen_base = devm_ioremap(&pdev->dev, mem->start, resource_size(mem)); - else if (mem_flags & EFI_MEMORY_WT) - screen_base = devm_memremap(&pdev->dev, mem->start, resource_size(mem), - MEMREMAP_WT); - else if (mem_flags & EFI_MEMORY_WB) - screen_base = devm_memremap(&pdev->dev, mem->start, resource_size(mem), - MEMREMAP_WB); - else + if (mem_flags & EFI_MEMORY_WC) { + void __iomem *screen_base = devm_ioremap_wc(&pdev->dev, mem->start, + resource_size(mem)); + + if (!screen_base) + return ERR_PTR(-ENXIO); + iosys_map_set_vaddr_iomem(&sysfb->fb_addr, screen_base); + } else if (mem_flags & EFI_MEMORY_UC) { + void __iomem *screen_base = devm_ioremap(&pdev->dev, mem->start, + resource_size(mem)); + + if (!screen_base) + return ERR_PTR(-ENXIO); + iosys_map_set_vaddr_iomem(&sysfb->fb_addr, screen_base); + } else if (mem_flags & EFI_MEMORY_WT) { + void *screen_base = devm_memremap(&pdev->dev, mem->start, + resource_size(mem), MEMREMAP_WT); + + if (IS_ERR(screen_base)) + return ERR_CAST(screen_base); + iosys_map_set_vaddr(&sysfb->fb_addr, screen_base); + } else if (mem_flags & EFI_MEMORY_WB) { + void *screen_base = devm_memremap(&pdev->dev, mem->start, + resource_size(mem), MEMREMAP_WB); + + if (IS_ERR(screen_base)) + return ERR_CAST(screen_base); + iosys_map_set_vaddr(&sysfb->fb_addr, screen_base); + } else { drm_err(dev, "invalid mem_flags: 0x%llx\n", mem_flags); - if (!screen_base) - return ERR_PTR(-ENOMEM); - iosys_map_set_vaddr_iomem(&sysfb->fb_addr, screen_base); + return ERR_PTR(-EINVAL); + } /* * Modesetting diff --git a/drivers/gpu/drm/sysfb/simpledrm.c b/drivers/gpu/drm/sysfb/simpledrm.c index 7a95d2dacd9d27..2ab6d96b4ec9e7 100644 --- a/drivers/gpu/drm/sysfb/simpledrm.c +++ b/drivers/gpu/drm/sysfb/simpledrm.c @@ -841,6 +841,12 @@ static int simpledrm_probe(struct platform_device *pdev) struct drm_device *dev; int ret; + ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); + if (ret) + ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); + if (ret) + return dev_err_probe(&pdev->dev, ret, "Failed to set dma mask\n"); + sdev = simpledrm_device_create(&simpledrm_driver, pdev); if (IS_ERR(sdev)) return PTR_ERR(sdev); diff --git a/drivers/gpu/drm/tegra/dsi.c b/drivers/gpu/drm/tegra/dsi.c index 175f5f9937b01e..8ee96b59fdbc8b 100644 --- a/drivers/gpu/drm/tegra/dsi.c +++ b/drivers/gpu/drm/tegra/dsi.c @@ -1542,11 +1542,9 @@ static int tegra_dsi_ganged_probe(struct tegra_dsi *dsi) return -EPROBE_DEFER; dsi->slave = platform_get_drvdata(gangster); - - if (!dsi->slave) { - put_device(&gangster->dev); + put_device(&gangster->dev); + if (!dsi->slave) return -EPROBE_DEFER; - } dsi->slave->master = dsi; } diff --git a/drivers/gpu/drm/tests/drm_gem_shmem_test.c b/drivers/gpu/drm/tests/drm_gem_shmem_test.c index 68f2c316235473..4b459f21acfd95 100644 --- a/drivers/gpu/drm/tests/drm_gem_shmem_test.c +++ b/drivers/gpu/drm/tests/drm_gem_shmem_test.c @@ -19,6 +19,8 @@ #include #include +MODULE_IMPORT_NS("EXPORTED_FOR_KUNIT_TESTING"); + #define TEST_SIZE SZ_1M #define TEST_BYTE 0xae @@ -34,6 +36,9 @@ KUNIT_DEFINE_ACTION_WRAPPER(sg_free_table_wrapper, sg_free_table, KUNIT_DEFINE_ACTION_WRAPPER(drm_gem_shmem_free_wrapper, drm_gem_shmem_free, struct drm_gem_shmem_object *); +KUNIT_DEFINE_ACTION_WRAPPER(drm_gem_shmem_unpin_wrapper, drm_gem_shmem_unpin, + struct drm_gem_shmem_object *); + /* * Test creating a shmem GEM object backed by shmem buffer. The test * case succeeds if the GEM object is successfully allocated with the @@ -173,7 +178,7 @@ static void drm_gem_shmem_test_vmap(struct kunit *test) ret = kunit_add_action_or_reset(test, drm_gem_shmem_free_wrapper, shmem); KUNIT_ASSERT_EQ(test, ret, 0); - ret = drm_gem_shmem_vmap_locked(shmem, &map); + ret = drm_gem_shmem_vmap(shmem, &map); KUNIT_ASSERT_EQ(test, ret, 0); KUNIT_ASSERT_NOT_NULL(test, shmem->vaddr); KUNIT_ASSERT_FALSE(test, iosys_map_is_null(&map)); @@ -183,7 +188,7 @@ static void drm_gem_shmem_test_vmap(struct kunit *test) for (i = 0; i < TEST_SIZE; i++) KUNIT_EXPECT_EQ(test, iosys_map_rd(&map, i, u8), TEST_BYTE); - drm_gem_shmem_vunmap_locked(shmem, &map); + drm_gem_shmem_vunmap(shmem, &map); KUNIT_EXPECT_NULL(test, shmem->vaddr); KUNIT_EXPECT_EQ(test, refcount_read(&shmem->vmap_use_count), 0); } @@ -194,7 +199,7 @@ static void drm_gem_shmem_test_vmap(struct kunit *test) * scatter/gather table large enough to accommodate the backing memory * is successfully exported. */ -static void drm_gem_shmem_test_get_pages_sgt(struct kunit *test) +static void drm_gem_shmem_test_get_sg_table(struct kunit *test) { struct drm_device *drm_dev = test->priv; struct drm_gem_shmem_object *shmem; @@ -212,6 +217,9 @@ static void drm_gem_shmem_test_get_pages_sgt(struct kunit *test) ret = drm_gem_shmem_pin(shmem); KUNIT_ASSERT_EQ(test, ret, 0); + ret = kunit_add_action_or_reset(test, drm_gem_shmem_unpin_wrapper, shmem); + KUNIT_ASSERT_EQ(test, ret, 0); + sgt = drm_gem_shmem_get_sg_table(shmem); KUNIT_ASSERT_NOT_ERR_OR_NULL(test, sgt); KUNIT_EXPECT_NULL(test, shmem->sgt); @@ -236,7 +244,7 @@ static void drm_gem_shmem_test_get_pages_sgt(struct kunit *test) * backing pages are pinned and a scatter/gather table large enough to * accommodate the backing memory is successfully exported. */ -static void drm_gem_shmem_test_get_sg_table(struct kunit *test) +static void drm_gem_shmem_test_get_pages_sgt(struct kunit *test) { struct drm_device *drm_dev = test->priv; struct drm_gem_shmem_object *shmem; @@ -284,17 +292,17 @@ static void drm_gem_shmem_test_madvise(struct kunit *test) ret = kunit_add_action_or_reset(test, drm_gem_shmem_free_wrapper, shmem); KUNIT_ASSERT_EQ(test, ret, 0); - ret = drm_gem_shmem_madvise_locked(shmem, 1); + ret = drm_gem_shmem_madvise(shmem, 1); KUNIT_EXPECT_TRUE(test, ret); KUNIT_ASSERT_EQ(test, shmem->madv, 1); /* Set madv to a negative value */ - ret = drm_gem_shmem_madvise_locked(shmem, -1); + ret = drm_gem_shmem_madvise(shmem, -1); KUNIT_EXPECT_FALSE(test, ret); KUNIT_ASSERT_EQ(test, shmem->madv, -1); /* Check that madv cannot be set back to a positive value */ - ret = drm_gem_shmem_madvise_locked(shmem, 0); + ret = drm_gem_shmem_madvise(shmem, 0); KUNIT_EXPECT_FALSE(test, ret); KUNIT_ASSERT_EQ(test, shmem->madv, -1); } @@ -322,7 +330,7 @@ static void drm_gem_shmem_test_purge(struct kunit *test) ret = drm_gem_shmem_is_purgeable(shmem); KUNIT_EXPECT_FALSE(test, ret); - ret = drm_gem_shmem_madvise_locked(shmem, 1); + ret = drm_gem_shmem_madvise(shmem, 1); KUNIT_EXPECT_TRUE(test, ret); /* The scatter/gather table will be freed by drm_gem_shmem_free */ @@ -332,7 +340,9 @@ static void drm_gem_shmem_test_purge(struct kunit *test) ret = drm_gem_shmem_is_purgeable(shmem); KUNIT_EXPECT_TRUE(test, ret); - drm_gem_shmem_purge_locked(shmem); + ret = drm_gem_shmem_purge(shmem); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_NULL(test, shmem->pages); KUNIT_EXPECT_NULL(test, shmem->sgt); KUNIT_EXPECT_EQ(test, shmem->madv, -1); @@ -366,8 +376,8 @@ static struct kunit_case drm_gem_shmem_test_cases[] = { KUNIT_CASE(drm_gem_shmem_test_obj_create_private), KUNIT_CASE(drm_gem_shmem_test_pin_pages), KUNIT_CASE(drm_gem_shmem_test_vmap), - KUNIT_CASE(drm_gem_shmem_test_get_pages_sgt), KUNIT_CASE(drm_gem_shmem_test_get_sg_table), + KUNIT_CASE(drm_gem_shmem_test_get_pages_sgt), KUNIT_CASE(drm_gem_shmem_test_madvise), KUNIT_CASE(drm_gem_shmem_test_purge), {} diff --git a/drivers/gpu/drm/tiny/sharp-memory.c b/drivers/gpu/drm/tiny/sharp-memory.c index 64272cd0f6e226..cbf69460ebf32d 100644 --- a/drivers/gpu/drm/tiny/sharp-memory.c +++ b/drivers/gpu/drm/tiny/sharp-memory.c @@ -541,8 +541,8 @@ static int sharp_memory_probe(struct spi_device *spi) smd = devm_drm_dev_alloc(dev, &sharp_memory_drm_driver, struct sharp_memory_device, drm); - if (!smd) - return -ENOMEM; + if (IS_ERR(smd)) + return PTR_ERR(smd); spi_set_drvdata(spi, smd); diff --git a/drivers/gpu/drm/ttm/tests/ttm_bo_test.c b/drivers/gpu/drm/ttm/tests/ttm_bo_test.c index d468f832207209..f3103307b5df98 100644 --- a/drivers/gpu/drm/ttm/tests/ttm_bo_test.c +++ b/drivers/gpu/drm/ttm/tests/ttm_bo_test.c @@ -222,13 +222,13 @@ static void ttm_bo_reserve_interrupted(struct kunit *test) KUNIT_FAIL(test, "Couldn't create ttm bo reserve task\n"); /* Take a lock so the threaded reserve has to wait */ - mutex_lock(&bo->base.resv->lock.base); + dma_resv_lock(bo->base.resv, NULL); wake_up_process(task); msleep(20); err = kthread_stop(task); - mutex_unlock(&bo->base.resv->lock.base); + dma_resv_unlock(bo->base.resv); KUNIT_ASSERT_EQ(test, err, -ERESTARTSYS); } diff --git a/drivers/gpu/drm/ttm/ttm_pool_internal.h b/drivers/gpu/drm/ttm/ttm_pool_internal.h index 82c4b7e56a99d5..24c179fd69d1af 100644 --- a/drivers/gpu/drm/ttm/ttm_pool_internal.h +++ b/drivers/gpu/drm/ttm/ttm_pool_internal.h @@ -17,7 +17,7 @@ static inline bool ttm_pool_uses_dma32(struct ttm_pool *pool) return pool->alloc_flags & TTM_ALLOCATION_POOL_USE_DMA32; } -static inline bool ttm_pool_beneficial_order(struct ttm_pool *pool) +static inline unsigned int ttm_pool_beneficial_order(struct ttm_pool *pool) { return pool->alloc_flags & 0xff; } diff --git a/drivers/gpu/drm/tyr/driver.rs b/drivers/gpu/drm/tyr/driver.rs index 0389c558c03675..d4a9225df9ed6c 100644 --- a/drivers/gpu/drm/tyr/driver.rs +++ b/drivers/gpu/drm/tyr/driver.rs @@ -76,7 +76,7 @@ fn issue_soft_reset(dev: &Device, iomem: &Devres) -> Result { dev_err!(dev, "GPU reset failed with errno\n"); dev_err!( dev, - "GPU_INT_RAWSTAT is {}\n", + "GPU_IRQ_RAWSTAT is {}\n", regs::GPU_IRQ_RAWSTAT.read(dev, iomem)? ); @@ -186,6 +186,8 @@ impl drm::Driver for TyrDriver { const INFO: drm::DriverInfo = INFO; + const FEATURES: u32 = drm::driver::FEAT_GEM; + kernel::declare_drm_ioctls! { (PANTHOR_DEV_QUERY, drm_panthor_dev_query, ioctl::RENDER_ALLOW, File::dev_query), } diff --git a/drivers/gpu/drm/tyr/file.rs b/drivers/gpu/drm/tyr/file.rs index 0ef432947b73d5..8fc26e03a9034b 100644 --- a/drivers/gpu/drm/tyr/file.rs +++ b/drivers/gpu/drm/tyr/file.rs @@ -1,5 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 or MIT +use kernel::bindings; use kernel::drm; use kernel::prelude::*; use kernel::uaccess::UserSlice; @@ -20,6 +21,10 @@ impl drm::file::DriverFile for File { fn open(_dev: &drm::Device) -> Result>> { KBox::try_pin_init(try_pin_init!(Self {}), GFP_KERNEL) } + + fn as_raw(&self) -> *mut bindings::drm_file { + todo!() + } } impl File { diff --git a/drivers/gpu/drm/tyr/gem.rs b/drivers/gpu/drm/tyr/gem.rs index 1273bf89dbd5d7..83493904a13f5e 100644 --- a/drivers/gpu/drm/tyr/gem.rs +++ b/drivers/gpu/drm/tyr/gem.rs @@ -9,10 +9,12 @@ use kernel::prelude::*; #[pin_data] pub(crate) struct TyrObject {} +#[vtable] impl gem::DriverObject for TyrObject { type Driver = TyrDriver; + type Args = (); - fn new(_dev: &TyrDevice, _size: usize) -> impl PinInit { + fn new(_dev: &TyrDevice, _size: usize, _args: ()) -> impl PinInit { try_pin_init!(TyrObject {}) } } diff --git a/drivers/gpu/drm/v3d/v3d_drv.c b/drivers/gpu/drm/v3d/v3d_drv.c index e8a46c8bad8a27..f469de456f9bb4 100644 --- a/drivers/gpu/drm/v3d/v3d_drv.c +++ b/drivers/gpu/drm/v3d/v3d_drv.c @@ -378,6 +378,8 @@ static int v3d_platform_drm_probe(struct platform_device *pdev) if (ret) goto clk_disable; + dma_set_max_seg_size(&pdev->dev, UINT_MAX); + v3d->va_width = 30 + V3D_GET_FIELD(mmu_debug, V3D_MMU_VA_WIDTH); ident1 = V3D_READ(V3D_HUB_IDENT1); diff --git a/drivers/gpu/drm/vc4/vc4_bo.c b/drivers/gpu/drm/vc4/vc4_bo.c index 46b4474ac41d46..44b1f2b00f9b05 100644 --- a/drivers/gpu/drm/vc4/vc4_bo.c +++ b/drivers/gpu/drm/vc4/vc4_bo.c @@ -739,12 +739,15 @@ static int vc4_gem_object_mmap(struct drm_gem_object *obj, struct vm_area_struct return -EINVAL; } + mutex_lock(&bo->madv_lock); if (bo->madv != VC4_MADV_WILLNEED) { DRM_DEBUG("mmapping of %s BO not allowed\n", bo->madv == VC4_MADV_DONTNEED ? "purgeable" : "purged"); + mutex_unlock(&bo->madv_lock); return -EINVAL; } + mutex_unlock(&bo->madv_lock); return drm_gem_dma_mmap(&bo->base, vma); } diff --git a/drivers/gpu/drm/vc4/vc4_gem.c b/drivers/gpu/drm/vc4/vc4_gem.c index ab16164b5edaf3..840aadb14b518a 100644 --- a/drivers/gpu/drm/vc4/vc4_gem.c +++ b/drivers/gpu/drm/vc4/vc4_gem.c @@ -62,6 +62,7 @@ vc4_free_hang_state(struct drm_device *dev, struct vc4_hang_state *state) for (i = 0; i < state->user_state.bo_count; i++) drm_gem_object_put(state->bo[i]); + kfree(state->bo); kfree(state); } @@ -170,10 +171,8 @@ vc4_save_hang_state(struct drm_device *dev) spin_lock_irqsave(&vc4->job_lock, irqflags); exec[0] = vc4_first_bin_job(vc4); exec[1] = vc4_first_render_job(vc4); - if (!exec[0] && !exec[1]) { - spin_unlock_irqrestore(&vc4->job_lock, irqflags); - return; - } + if (!exec[0] && !exec[1]) + goto err_free_state; /* Get the bos from both binner and renderer into hang state. */ state->bo_count = 0; @@ -190,10 +189,8 @@ vc4_save_hang_state(struct drm_device *dev) kernel_state->bo = kcalloc(state->bo_count, sizeof(*kernel_state->bo), GFP_ATOMIC); - if (!kernel_state->bo) { - spin_unlock_irqrestore(&vc4->job_lock, irqflags); - return; - } + if (!kernel_state->bo) + goto err_free_state; k = 0; for (i = 0; i < 2; i++) { @@ -285,6 +282,12 @@ vc4_save_hang_state(struct drm_device *dev) vc4->hang_state = kernel_state; spin_unlock_irqrestore(&vc4->job_lock, irqflags); } + + return; + +err_free_state: + spin_unlock_irqrestore(&vc4->job_lock, irqflags); + kfree(kernel_state); } static void diff --git a/drivers/gpu/drm/vc4/vc4_hdmi.c b/drivers/gpu/drm/vc4/vc4_hdmi.c index 1798d1156d10bd..d89a0ec5d77289 100644 --- a/drivers/gpu/drm/vc4/vc4_hdmi.c +++ b/drivers/gpu/drm/vc4/vc4_hdmi.c @@ -2349,17 +2349,23 @@ static int vc4_hdmi_hotplug_init(struct vc4_hdmi *vc4_hdmi) int ret; if (vc4_hdmi->variant->external_irq_controller) { - unsigned int hpd_con = platform_get_irq_byname(pdev, "hpd-connected"); - unsigned int hpd_rm = platform_get_irq_byname(pdev, "hpd-removed"); + int hpd = platform_get_irq_byname(pdev, "hpd-connected"); - ret = devm_request_threaded_irq(&pdev->dev, hpd_con, + if (hpd < 0) + return hpd; + + ret = devm_request_threaded_irq(&pdev->dev, hpd, NULL, vc4_hdmi_hpd_irq_thread, IRQF_ONESHOT, "vc4 hdmi hpd connected", vc4_hdmi); if (ret) return ret; - ret = devm_request_threaded_irq(&pdev->dev, hpd_rm, + hpd = platform_get_irq_byname(pdev, "hpd-removed"); + if (hpd < 0) + return hpd; + + ret = devm_request_threaded_irq(&pdev->dev, hpd, NULL, vc4_hdmi_hpd_irq_thread, IRQF_ONESHOT, "vc4 hdmi hpd disconnected", vc4_hdmi); diff --git a/drivers/gpu/drm/vc4/vc4_v3d.c b/drivers/gpu/drm/vc4/vc4_v3d.c index 3ffe09bc89d273..d31b906cb8e787 100644 --- a/drivers/gpu/drm/vc4/vc4_v3d.c +++ b/drivers/gpu/drm/vc4/vc4_v3d.c @@ -481,6 +481,7 @@ static int vc4_v3d_bind(struct device *dev, struct device *master, void *data) pm_runtime_use_autosuspend(dev); pm_runtime_set_autosuspend_delay(dev, 40); /* a little over 2 frames. */ + pm_runtime_put_autosuspend(dev); return 0; diff --git a/drivers/gpu/drm/vkms/vkms_composer.c b/drivers/gpu/drm/vkms/vkms_composer.c index 3cf3f26e0d8eae..cd85de4ffd03d2 100644 --- a/drivers/gpu/drm/vkms/vkms_composer.c +++ b/drivers/gpu/drm/vkms/vkms_composer.c @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include diff --git a/drivers/gpu/drm/vkms/vkms_drv.c b/drivers/gpu/drm/vkms/vkms_drv.c index dd1402f4377369..434c295f44ba65 100644 --- a/drivers/gpu/drm/vkms/vkms_drv.c +++ b/drivers/gpu/drm/vkms/vkms_drv.c @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c b/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c index 3057f8baa7d25b..e1f18020170ab8 100644 --- a/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c +++ b/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c @@ -1143,7 +1143,7 @@ static int vmw_translate_mob_ptr(struct vmw_private *dev_priv, ret = vmw_user_bo_lookup(sw_context->filp, handle, &vmw_bo); if (ret != 0) { drm_dbg(&dev_priv->drm, "Could not find or use MOB buffer.\n"); - return PTR_ERR(vmw_bo); + return ret; } vmw_bo_placement_set(vmw_bo, VMW_BO_DOMAIN_MOB, VMW_BO_DOMAIN_MOB); ret = vmw_validation_add_bo(sw_context->ctx, vmw_bo); @@ -1199,7 +1199,7 @@ static int vmw_translate_guest_ptr(struct vmw_private *dev_priv, ret = vmw_user_bo_lookup(sw_context->filp, handle, &vmw_bo); if (ret != 0) { drm_dbg(&dev_priv->drm, "Could not find or use GMR region.\n"); - return PTR_ERR(vmw_bo); + return ret; } vmw_bo_placement_set(vmw_bo, VMW_BO_DOMAIN_GMR | VMW_BO_DOMAIN_VRAM, VMW_BO_DOMAIN_GMR | VMW_BO_DOMAIN_VRAM); diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c index bc51b5d55e38a9..35c7277521a90f 100644 --- a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c +++ b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c @@ -771,7 +771,8 @@ static struct drm_framebuffer *vmw_kms_fb_create(struct drm_device *dev, ret = vmw_bo_dirty_add(bo); if (!ret && surface && surface->res.func->dirty_alloc) { surface->res.coherent = true; - ret = surface->res.func->dirty_alloc(&surface->res); + if (surface->res.dirty == NULL) + ret = surface->res.func->dirty_alloc(&surface->res); } ttm_bo_unreserve(&bo->tbo); } diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_page_dirty.c b/drivers/gpu/drm/vmwgfx/vmwgfx_page_dirty.c index fd4e76486f2d1b..45561bc1c9effe 100644 --- a/drivers/gpu/drm/vmwgfx/vmwgfx_page_dirty.c +++ b/drivers/gpu/drm/vmwgfx/vmwgfx_page_dirty.c @@ -260,6 +260,13 @@ int vmw_bo_dirty_add(struct vmw_bo *vbo) return ret; } +static void vmw_bo_dirty_free(struct kref *kref) +{ + struct vmw_bo_dirty *dirty = container_of(kref, struct vmw_bo_dirty, ref_count); + + kvfree(dirty); +} + /** * vmw_bo_dirty_release - Release a dirty-tracking user from a buffer object * @vbo: The buffer object @@ -274,7 +281,7 @@ void vmw_bo_dirty_release(struct vmw_bo *vbo) { struct vmw_bo_dirty *dirty = vbo->dirty; - if (dirty && kref_put(&dirty->ref_count, (void *)kvfree)) + if (dirty && kref_put(&dirty->ref_count, vmw_bo_dirty_free)) vbo->dirty = NULL; } diff --git a/drivers/gpu/drm/xe/display/xe_fb_pin.c b/drivers/gpu/drm/xe/display/xe_fb_pin.c index 1fd4a815e784be..b18d15cc3c53dc 100644 --- a/drivers/gpu/drm/xe/display/xe_fb_pin.c +++ b/drivers/gpu/drm/xe/display/xe_fb_pin.c @@ -378,7 +378,7 @@ intel_fb_pin_to_ggtt(const struct drm_framebuffer *fb, { *out_flags = 0; - return __xe_pin_fb_vma(to_intel_framebuffer(fb), view, phys_alignment); + return __xe_pin_fb_vma(to_intel_framebuffer(fb), view, alignment); } void intel_fb_unpin_vma(struct i915_vma *vma, unsigned long flags) diff --git a/drivers/gpu/drm/xe/regs/xe_engine_regs.h b/drivers/gpu/drm/xe/regs/xe_engine_regs.h index 68172b0248a6e4..dc5a4fafa70cfb 100644 --- a/drivers/gpu/drm/xe/regs/xe_engine_regs.h +++ b/drivers/gpu/drm/xe/regs/xe_engine_regs.h @@ -96,6 +96,12 @@ #define ENABLE_SEMAPHORE_POLL_BIT REG_BIT(13) #define RING_CMD_CCTL(base) XE_REG((base) + 0xc4, XE_REG_OPTION_MASKED) + +#define CS_MMIO_GROUP_INSTANCE_SELECT(base) XE_REG((base) + 0xcc) +#define SELECTIVE_READ_ADDRESSING REG_BIT(30) +#define SELECTIVE_READ_GROUP REG_GENMASK(29, 23) +#define SELECTIVE_READ_INSTANCE REG_GENMASK(22, 16) + /* * CMD_CCTL read/write fields take a MOCS value and _not_ a table index. * The lsb of each can be considered a separate enabling bit for encryption. diff --git a/drivers/gpu/drm/xe/regs/xe_gt_regs.h b/drivers/gpu/drm/xe/regs/xe_gt_regs.h index 917a088c28f24a..ec1ae2dc6cabe2 100644 --- a/drivers/gpu/drm/xe/regs/xe_gt_regs.h +++ b/drivers/gpu/drm/xe/regs/xe_gt_regs.h @@ -544,6 +544,7 @@ #define ENABLE_SMP_LD_RENDER_SURFACE_CONTROL REG_BIT(44 - 32) #define FORCE_SLM_FENCE_SCOPE_TO_TILE REG_BIT(42 - 32) #define FORCE_UGM_FENCE_SCOPE_TO_TILE REG_BIT(41 - 32) +#define L3_128B_256B_WRT_DIS REG_BIT(40 - 32) #define MAXREQS_PER_BANK REG_GENMASK(39 - 32, 37 - 32) #define DISABLE_128B_EVICTION_COMMAND_UDW REG_BIT(36 - 32) diff --git a/drivers/gpu/drm/xe/regs/xe_guc_regs.h b/drivers/gpu/drm/xe/regs/xe_guc_regs.h index 2118f7dec287fc..87984713dd1261 100644 --- a/drivers/gpu/drm/xe/regs/xe_guc_regs.h +++ b/drivers/gpu/drm/xe/regs/xe_guc_regs.h @@ -90,6 +90,9 @@ #define GUC_SEND_INTERRUPT XE_REG(0xc4c8) #define GUC_SEND_TRIGGER REG_BIT(0) +#define GUC_INTR_CHICKEN XE_REG(0xc50c) +#define DISABLE_SIGNALING_ENGINES REG_BIT(1) + #define GUC_BCS_RCS_IER XE_REG(0xc550) #define GUC_VCS2_VCS1_IER XE_REG(0xc554) #define GUC_WD_VECS_IER XE_REG(0xc558) diff --git a/drivers/gpu/drm/xe/xe_bo.c b/drivers/gpu/drm/xe/xe_bo.c index 71acd45aa33b00..579e98dec7492f 100644 --- a/drivers/gpu/drm/xe/xe_bo.c +++ b/drivers/gpu/drm/xe/xe_bo.c @@ -1942,7 +1942,7 @@ static vm_fault_t xe_bo_cpu_fault(struct vm_fault *vmf) int err = 0; int idx; - if (!drm_dev_enter(&xe->drm, &idx)) + if (xe_device_wedged(xe) || !drm_dev_enter(&xe->drm, &idx)) return ttm_bo_vm_dummy_page(vmf, vmf->vma->vm_page_prot); ret = xe_bo_cpu_fault_fastpath(vmf, xe, bo, needs_rpm); diff --git a/drivers/gpu/drm/xe/xe_configfs.c b/drivers/gpu/drm/xe/xe_configfs.c index 82edd046600557..4afdfd69899aad 100644 --- a/drivers/gpu/drm/xe/xe_configfs.c +++ b/drivers/gpu/drm/xe/xe_configfs.c @@ -830,6 +830,7 @@ static void xe_config_device_release(struct config_item *item) mutex_destroy(&dev->lock); + kfree(dev->config.ctx_restore_mid_bb[0].cs); kfree(dev->config.ctx_restore_post_bb[0].cs); kfree(dev); } diff --git a/drivers/gpu/drm/xe/xe_configfs.h b/drivers/gpu/drm/xe/xe_configfs.h index fed57be0b90e14..f3683bc7eb90cc 100644 --- a/drivers/gpu/drm/xe/xe_configfs.h +++ b/drivers/gpu/drm/xe/xe_configfs.h @@ -21,9 +21,11 @@ bool xe_configfs_primary_gt_allowed(struct pci_dev *pdev); bool xe_configfs_media_gt_allowed(struct pci_dev *pdev); u64 xe_configfs_get_engines_allowed(struct pci_dev *pdev); bool xe_configfs_get_psmi_enabled(struct pci_dev *pdev); -u32 xe_configfs_get_ctx_restore_mid_bb(struct pci_dev *pdev, enum xe_engine_class, +u32 xe_configfs_get_ctx_restore_mid_bb(struct pci_dev *pdev, + enum xe_engine_class class, const u32 **cs); -u32 xe_configfs_get_ctx_restore_post_bb(struct pci_dev *pdev, enum xe_engine_class, +u32 xe_configfs_get_ctx_restore_post_bb(struct pci_dev *pdev, + enum xe_engine_class class, const u32 **cs); #ifdef CONFIG_PCI_IOV unsigned int xe_configfs_get_max_vfs(struct pci_dev *pdev); @@ -37,9 +39,11 @@ static inline bool xe_configfs_primary_gt_allowed(struct pci_dev *pdev) { return static inline bool xe_configfs_media_gt_allowed(struct pci_dev *pdev) { return true; } static inline u64 xe_configfs_get_engines_allowed(struct pci_dev *pdev) { return U64_MAX; } static inline bool xe_configfs_get_psmi_enabled(struct pci_dev *pdev) { return false; } -static inline u32 xe_configfs_get_ctx_restore_mid_bb(struct pci_dev *pdev, enum xe_engine_class, +static inline u32 xe_configfs_get_ctx_restore_mid_bb(struct pci_dev *pdev, + enum xe_engine_class class, const u32 **cs) { return 0; } -static inline u32 xe_configfs_get_ctx_restore_post_bb(struct pci_dev *pdev, enum xe_engine_class, +static inline u32 xe_configfs_get_ctx_restore_post_bb(struct pci_dev *pdev, + enum xe_engine_class class, const u32 **cs) { return 0; } static inline unsigned int xe_configfs_get_max_vfs(struct pci_dev *pdev) { return UINT_MAX; } #endif diff --git a/drivers/gpu/drm/xe/xe_device.c b/drivers/gpu/drm/xe/xe_device.c index 9a6d49fcd8e49b..ef5acef32f4167 100644 --- a/drivers/gpu/drm/xe/xe_device.c +++ b/drivers/gpu/drm/xe/xe_device.c @@ -976,6 +976,7 @@ int xe_device_probe(struct xe_device *xe) err_unregister_display: xe_display_unregister(xe); + drm_dev_unregister(&xe->drm); return err; } diff --git a/drivers/gpu/drm/xe/xe_exec_queue.c b/drivers/gpu/drm/xe/xe_exec_queue.c index 779d7e7e2d2ec8..1e774fa1fa1908 100644 --- a/drivers/gpu/drm/xe/xe_exec_queue.c +++ b/drivers/gpu/drm/xe/xe_exec_queue.c @@ -185,6 +185,16 @@ static struct xe_exec_queue *__xe_exec_queue_alloc(struct xe_device *xe, return q; } +static void __xe_exec_queue_fini(struct xe_exec_queue *q) +{ + int i; + + q->ops->fini(q); + + for (i = 0; i < q->width; ++i) + xe_lrc_put(q->lrc[i]); +} + static int __xe_exec_queue_init(struct xe_exec_queue *q, u32 exec_queue_flags) { int i, err; @@ -239,21 +249,10 @@ static int __xe_exec_queue_init(struct xe_exec_queue *q, u32 exec_queue_flags) return 0; err_lrc: - for (i = i - 1; i >= 0; --i) - xe_lrc_put(q->lrc[i]); + __xe_exec_queue_fini(q); return err; } -static void __xe_exec_queue_fini(struct xe_exec_queue *q) -{ - int i; - - q->ops->fini(q); - - for (i = 0; i < q->width; ++i) - xe_lrc_put(q->lrc[i]); -} - struct xe_exec_queue *xe_exec_queue_create(struct xe_device *xe, struct xe_vm *vm, u32 logical_mask, u16 width, struct xe_hw_engine *hwe, u32 flags, diff --git a/drivers/gpu/drm/xe/xe_ggtt.c b/drivers/gpu/drm/xe/xe_ggtt.c index 793d7324a395d2..f3d76c639aac4f 100644 --- a/drivers/gpu/drm/xe/xe_ggtt.c +++ b/drivers/gpu/drm/xe/xe_ggtt.c @@ -274,6 +274,8 @@ static void dev_fini_ggtt(void *arg) { struct xe_ggtt *ggtt = arg; + scoped_guard(mutex, &ggtt->lock) + ggtt->flags &= ~XE_GGTT_FLAGS_ONLINE; drain_workqueue(ggtt->wq); } @@ -332,6 +334,7 @@ int xe_ggtt_init_early(struct xe_ggtt *ggtt) if (err) return err; + ggtt->flags |= XE_GGTT_FLAGS_ONLINE; err = devm_add_action_or_reset(xe->drm.dev, dev_fini_ggtt, ggtt); if (err) return err; @@ -365,13 +368,10 @@ static void xe_ggtt_initial_clear(struct xe_ggtt *ggtt) static void ggtt_node_remove(struct xe_ggtt_node *node) { struct xe_ggtt *ggtt = node->ggtt; - struct xe_device *xe = tile_to_xe(ggtt->tile); bool bound; - int idx; - - bound = drm_dev_enter(&xe->drm, &idx); mutex_lock(&ggtt->lock); + bound = ggtt->flags & XE_GGTT_FLAGS_ONLINE; if (bound) xe_ggtt_clear(ggtt, node->base.start, node->base.size); drm_mm_remove_node(&node->base); @@ -384,8 +384,6 @@ static void ggtt_node_remove(struct xe_ggtt_node *node) if (node->invalidate_on_remove) xe_ggtt_invalidate(ggtt); - drm_dev_exit(idx); - free_node: xe_ggtt_node_fini(node); } @@ -396,9 +394,8 @@ static void ggtt_node_remove_work_func(struct work_struct *work) delayed_removal_work); struct xe_device *xe = tile_to_xe(node->ggtt->tile); - xe_pm_runtime_get(xe); + guard(xe_pm_runtime)(xe); ggtt_node_remove(node); - xe_pm_runtime_put(xe); } /** diff --git a/drivers/gpu/drm/xe/xe_ggtt_types.h b/drivers/gpu/drm/xe/xe_ggtt_types.h index dacd796f818445..53ba753417ebe1 100644 --- a/drivers/gpu/drm/xe/xe_ggtt_types.h +++ b/drivers/gpu/drm/xe/xe_ggtt_types.h @@ -25,11 +25,14 @@ struct xe_ggtt { /** @size: Total size of this GGTT */ u64 size; -#define XE_GGTT_FLAGS_64K BIT(0) +#define XE_GGTT_FLAGS_64K BIT(0) +#define XE_GGTT_FLAGS_ONLINE BIT(1) /** * @flags: Flags for this GGTT * Acceptable flags: * - %XE_GGTT_FLAGS_64K - if PTE size is 64K. Otherwise, regular is 4K. + * - %XE_GGTT_FLAGS_ONLINE - is GGTT online, protected by ggtt->lock + * after init */ unsigned int flags; /** @scratch: Internal object allocation used as a scratch page */ diff --git a/drivers/gpu/drm/xe/xe_gsc_proxy.c b/drivers/gpu/drm/xe/xe_gsc_proxy.c index 464282a89eef39..a6f6f0ea56526f 100644 --- a/drivers/gpu/drm/xe/xe_gsc_proxy.c +++ b/drivers/gpu/drm/xe/xe_gsc_proxy.c @@ -435,16 +435,12 @@ static int proxy_channel_alloc(struct xe_gsc *gsc) return 0; } -static void xe_gsc_proxy_remove(void *arg) +static void xe_gsc_proxy_stop(struct xe_gsc *gsc) { - struct xe_gsc *gsc = arg; struct xe_gt *gt = gsc_to_gt(gsc); struct xe_device *xe = gt_to_xe(gt); unsigned int fw_ref = 0; - if (!gsc->proxy.component_added) - return; - /* disable HECI2 IRQs */ xe_pm_runtime_get(xe); fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GSC); @@ -458,6 +454,30 @@ static void xe_gsc_proxy_remove(void *arg) xe_pm_runtime_put(xe); xe_gsc_wait_for_worker_completion(gsc); + gsc->proxy.started = false; +} + +static void xe_gsc_proxy_remove(void *arg) +{ + struct xe_gsc *gsc = arg; + struct xe_gt *gt = gsc_to_gt(gsc); + struct xe_device *xe = gt_to_xe(gt); + + if (!gsc->proxy.component_added) + return; + + /* + * GSC proxy start is an async process that can be ongoing during + * Xe module load/unload. Using devm managed action to register + * xe_gsc_proxy_stop could cause issues if Xe module unload has + * already started when the action is registered, potentially leading + * to the cleanup being called at the wrong time. Therefore, instead + * of registering a separate devm action to undo what is done in + * proxy start, we call it from here, but only if the start has + * completed successfully (tracked with the 'started' flag). + */ + if (gsc->proxy.started) + xe_gsc_proxy_stop(gsc); component_del(xe->drm.dev, &xe_gsc_proxy_component_ops); gsc->proxy.component_added = false; @@ -513,6 +533,7 @@ int xe_gsc_proxy_init(struct xe_gsc *gsc) */ int xe_gsc_proxy_start(struct xe_gsc *gsc) { + struct xe_gt *gt = gsc_to_gt(gsc); int err; /* enable the proxy interrupt in the GSC shim layer */ @@ -524,12 +545,18 @@ int xe_gsc_proxy_start(struct xe_gsc *gsc) */ err = xe_gsc_proxy_request_handler(gsc); if (err) - return err; + goto err_irq_disable; if (!xe_gsc_proxy_init_done(gsc)) { - xe_gt_err(gsc_to_gt(gsc), "GSC FW reports proxy init not completed\n"); - return -EIO; + xe_gt_err(gt, "GSC FW reports proxy init not completed\n"); + err = -EIO; + goto err_irq_disable; } + gsc->proxy.started = true; return 0; + +err_irq_disable: + gsc_proxy_irq_toggle(gsc, false); + return err; } diff --git a/drivers/gpu/drm/xe/xe_gsc_types.h b/drivers/gpu/drm/xe/xe_gsc_types.h index 97c056656df05b..5aaa2a75861fd2 100644 --- a/drivers/gpu/drm/xe/xe_gsc_types.h +++ b/drivers/gpu/drm/xe/xe_gsc_types.h @@ -58,6 +58,8 @@ struct xe_gsc { struct mutex mutex; /** @proxy.component_added: whether the component has been added */ bool component_added; + /** @proxy.started: whether the proxy has been started */ + bool started; /** @proxy.bo: object to store message to and from the GSC */ struct xe_bo *bo; /** @proxy.to_gsc: map of the memory used to send messages to the GSC */ diff --git a/drivers/gpu/drm/xe/xe_gt.c b/drivers/gpu/drm/xe/xe_gt.c index cdce210e36f25c..e89cbe498c427c 100644 --- a/drivers/gpu/drm/xe/xe_gt.c +++ b/drivers/gpu/drm/xe/xe_gt.c @@ -187,11 +187,15 @@ static int emit_nop_job(struct xe_gt *gt, struct xe_exec_queue *q) return ret; } +/* Dwords required to emit a RMW of a register */ +#define EMIT_RMW_DW 20 + static int emit_wa_job(struct xe_gt *gt, struct xe_exec_queue *q) { - struct xe_reg_sr *sr = &q->hwe->reg_lrc; + struct xe_hw_engine *hwe = q->hwe; + struct xe_reg_sr *sr = &hwe->reg_lrc; struct xe_reg_sr_entry *entry; - int count_rmw = 0, count = 0, ret; + int count_rmw = 0, count_rmw_mcr = 0, count = 0, ret; unsigned long idx; struct xe_bb *bb; size_t bb_len = 0; @@ -201,6 +205,8 @@ static int emit_wa_job(struct xe_gt *gt, struct xe_exec_queue *q) xa_for_each(&sr->xa, idx, entry) { if (entry->reg.masked || entry->clr_bits == ~0) ++count; + else if (entry->reg.mcr) + ++count_rmw_mcr; else ++count_rmw; } @@ -208,17 +214,35 @@ static int emit_wa_job(struct xe_gt *gt, struct xe_exec_queue *q) if (count) bb_len += count * 2 + 1; - if (count_rmw) - bb_len += count_rmw * 20 + 7; + /* + * RMW of MCR registers is the same as a normal RMW, except an + * additional LRI (3 dwords) is required per register to steer the read + * to a nom-terminated instance. + * + * We could probably shorten the batch slightly by eliding the + * steering for consecutive MCR registers that have the same + * group/instance target, but it's not worth the extra complexity to do + * so. + */ + bb_len += count_rmw * EMIT_RMW_DW; + bb_len += count_rmw_mcr * (EMIT_RMW_DW + 3); + + /* + * After doing all RMW, we need 7 trailing dwords to clean up, + * plus an additional 3 dwords to reset steering if any of the + * registers were MCR. + */ + if (count_rmw || count_rmw_mcr) + bb_len += 7 + (count_rmw_mcr ? 3 : 0); - if (q->hwe->class == XE_ENGINE_CLASS_RENDER) + if (hwe->class == XE_ENGINE_CLASS_RENDER) /* * Big enough to emit all of the context's 3DSTATE via * xe_lrc_emit_hwe_state_instructions() */ - bb_len += xe_gt_lrc_size(gt, q->hwe->class) / sizeof(u32); + bb_len += xe_gt_lrc_size(gt, hwe->class) / sizeof(u32); - xe_gt_dbg(gt, "LRC %s WA job: %zu dwords\n", q->hwe->name, bb_len); + xe_gt_dbg(gt, "LRC %s WA job: %zu dwords\n", hwe->name, bb_len); bb = xe_bb_new(gt, bb_len, false); if (IS_ERR(bb)) @@ -253,13 +277,23 @@ static int emit_wa_job(struct xe_gt *gt, struct xe_exec_queue *q) } } - if (count_rmw) { - /* Emit MI_MATH for each RMW reg: 20dw per reg + 7 trailing dw */ - + if (count_rmw || count_rmw_mcr) { xa_for_each(&sr->xa, idx, entry) { if (entry->reg.masked || entry->clr_bits == ~0) continue; + if (entry->reg.mcr) { + struct xe_reg_mcr reg = { .__reg.raw = entry->reg.raw }; + u8 group, instance; + + xe_gt_mcr_get_nonterminated_steering(gt, reg, &group, &instance); + *cs++ = MI_LOAD_REGISTER_IMM | MI_LRI_NUM_REGS(1); + *cs++ = CS_MMIO_GROUP_INSTANCE_SELECT(hwe->mmio_base).addr; + *cs++ = SELECTIVE_READ_ADDRESSING | + REG_FIELD_PREP(SELECTIVE_READ_GROUP, group) | + REG_FIELD_PREP(SELECTIVE_READ_INSTANCE, instance); + } + *cs++ = MI_LOAD_REGISTER_REG | MI_LRR_DST_CS_MMIO; *cs++ = entry->reg.addr; *cs++ = CS_GPR_REG(0, 0).addr; @@ -285,8 +319,9 @@ static int emit_wa_job(struct xe_gt *gt, struct xe_exec_queue *q) *cs++ = CS_GPR_REG(0, 0).addr; *cs++ = entry->reg.addr; - xe_gt_dbg(gt, "REG[%#x] = ~%#x|%#x\n", - entry->reg.addr, entry->clr_bits, entry->set_bits); + xe_gt_dbg(gt, "REG[%#x] = ~%#x|%#x%s\n", + entry->reg.addr, entry->clr_bits, entry->set_bits, + entry->reg.mcr ? " (MCR)" : ""); } /* reset used GPR */ @@ -298,6 +333,13 @@ static int emit_wa_job(struct xe_gt *gt, struct xe_exec_queue *q) *cs++ = 0; *cs++ = CS_GPR_REG(0, 2).addr; *cs++ = 0; + + /* reset steering */ + if (count_rmw_mcr) { + *cs++ = MI_LOAD_REGISTER_IMM | MI_LRI_NUM_REGS(1); + *cs++ = CS_MMIO_GROUP_INSTANCE_SELECT(q->hwe->mmio_base).addr; + *cs++ = 0; + } } cs = xe_lrc_emit_hwe_state_instructions(q, cs); diff --git a/drivers/gpu/drm/xe/xe_gt_ccs_mode.c b/drivers/gpu/drm/xe/xe_gt_ccs_mode.c index 50fffc9ebf62a1..852e73173070b0 100644 --- a/drivers/gpu/drm/xe/xe_gt_ccs_mode.c +++ b/drivers/gpu/drm/xe/xe_gt_ccs_mode.c @@ -12,6 +12,7 @@ #include "xe_gt_printk.h" #include "xe_gt_sysfs.h" #include "xe_mmio.h" +#include "xe_pm.h" #include "xe_sriov.h" static void __xe_gt_apply_ccs_mode(struct xe_gt *gt, u32 num_engines) @@ -150,6 +151,7 @@ ccs_mode_store(struct device *kdev, struct device_attribute *attr, xe_gt_info(gt, "Setting compute mode to %d\n", num_engines); gt->ccs_mode = num_engines; xe_gt_record_user_engines(gt); + guard(xe_pm_runtime)(xe); xe_gt_reset(gt); } diff --git a/drivers/gpu/drm/xe/xe_guc.c b/drivers/gpu/drm/xe/xe_guc.c index edb939f2626851..2eaa009ba2d8df 100644 --- a/drivers/gpu/drm/xe/xe_guc.c +++ b/drivers/gpu/drm/xe/xe_guc.c @@ -1121,14 +1121,14 @@ static int guc_wait_ucode(struct xe_guc *guc) struct xe_guc_pc *guc_pc = >->uc.guc.pc; u32 before_freq, act_freq, cur_freq; u32 status = 0, tries = 0; + int load_result, ret; ktime_t before; u64 delta_ms; - int ret; before_freq = xe_guc_pc_get_act_freq(guc_pc); before = ktime_get(); - ret = poll_timeout_us(ret = guc_load_done(gt, &status, &tries), ret, + ret = poll_timeout_us(load_result = guc_load_done(gt, &status, &tries), load_result, 10 * USEC_PER_MSEC, GUC_LOAD_TIMEOUT_SEC * USEC_PER_SEC, false); @@ -1136,7 +1136,7 @@ static int guc_wait_ucode(struct xe_guc *guc) act_freq = xe_guc_pc_get_act_freq(guc_pc); cur_freq = xe_guc_pc_get_cur_freq_fw(guc_pc); - if (ret) { + if (ret || load_result <= 0) { xe_gt_err(gt, "load failed: status = 0x%08X, time = %lldms, freq = %dMHz (req %dMHz)\n", status, delta_ms, xe_guc_pc_get_act_freq(guc_pc), xe_guc_pc_get_cur_freq_fw(guc_pc)); diff --git a/drivers/gpu/drm/xe/xe_guc_ct.c b/drivers/gpu/drm/xe/xe_guc_ct.c index a5019d1e741b3b..791d42e455a04d 100644 --- a/drivers/gpu/drm/xe/xe_guc_ct.c +++ b/drivers/gpu/drm/xe/xe_guc_ct.c @@ -265,6 +265,7 @@ static void guc_action_disable_ct(void *arg) { struct xe_guc_ct *ct = arg; + xe_guc_ct_stop(ct); guc_ct_change_state(ct, XE_GUC_CT_STATE_DISABLED); } diff --git a/drivers/gpu/drm/xe/xe_guc_pc.c b/drivers/gpu/drm/xe/xe_guc_pc.c index 951a49fb1d3e49..7f09cf0e495fee 100644 --- a/drivers/gpu/drm/xe/xe_guc_pc.c +++ b/drivers/gpu/drm/xe/xe_guc_pc.c @@ -1214,6 +1214,36 @@ int xe_guc_pc_set_power_profile(struct xe_guc_pc *pc, const char *buf) return ret; } +static int pc_action_set_dcc(struct xe_guc_pc *pc, bool enable) +{ + int ret; + + ret = pc_action_set_param(pc, + SLPC_PARAM_TASK_ENABLE_DCC, + enable); + if (!ret) + return pc_action_set_param(pc, + SLPC_PARAM_TASK_DISABLE_DCC, + !enable); + else + return ret; +} + +static int pc_modify_defaults(struct xe_guc_pc *pc) +{ + struct xe_device *xe = pc_to_xe(pc); + struct xe_gt *gt = pc_to_gt(pc); + int ret = 0; + + if (xe->info.platform == XE_PANTHERLAKE) { + ret = pc_action_set_dcc(pc, false); + if (unlikely(ret)) + xe_gt_err(gt, "Failed to modify DCC default: %pe\n", ERR_PTR(ret)); + } + + return ret; +} + /** * xe_guc_pc_start - Start GuC's Power Conservation component * @pc: Xe_GuC_PC instance @@ -1271,6 +1301,10 @@ int xe_guc_pc_start(struct xe_guc_pc *pc) ktime_ms_delta(ktime_get(), earlier)); } + ret = pc_modify_defaults(pc); + if (ret) + return ret; + ret = pc_init_freqs(pc); if (ret) goto out; diff --git a/drivers/gpu/drm/xe/xe_guc_submit.c b/drivers/gpu/drm/xe/xe_guc_submit.c index f6ba2b0f074d23..ea9e3d506392a4 100644 --- a/drivers/gpu/drm/xe/xe_guc_submit.c +++ b/drivers/gpu/drm/xe/xe_guc_submit.c @@ -1304,7 +1304,7 @@ guc_exec_queue_timedout_job(struct drm_sched_job *drm_job) if (exec_queue_reset(q)) err = -EIO; - if (!exec_queue_destroyed(q)) { + if (!exec_queue_destroyed(q) && xe_uc_fw_is_running(&guc->fw)) { /* * Wait for any pending G2H to flush out before * modifying state @@ -1339,6 +1339,7 @@ guc_exec_queue_timedout_job(struct drm_sched_job *drm_job) */ smp_rmb(); ret = wait_event_timeout(guc->ct.wq, + !xe_uc_fw_is_running(&guc->fw) || !exec_queue_pending_disable(q) || xe_guc_read_stopped(guc) || vf_recovery(guc), HZ * 5); @@ -2409,8 +2410,7 @@ void xe_guc_submit_pause_abort(struct xe_guc *guc) continue; xe_sched_submission_start(sched); - if (exec_queue_killed_or_banned_or_wedged(q)) - xe_guc_exec_queue_trigger_cleanup(q); + guc_exec_queue_kill(q); } mutex_unlock(&guc->submission_state.lock); } diff --git a/drivers/gpu/drm/xe/xe_hw_engine.c b/drivers/gpu/drm/xe/xe_hw_engine.c index 6a9e2a4272ddea..3e928b6c098f29 100644 --- a/drivers/gpu/drm/xe/xe_hw_engine.c +++ b/drivers/gpu/drm/xe/xe_hw_engine.c @@ -596,9 +596,8 @@ static void adjust_idledly(struct xe_hw_engine *hwe) maxcnt *= maxcnt_units_ns; if (xe_gt_WARN_ON(gt, idledly >= maxcnt || inhibit_switch)) { - idledly = DIV_ROUND_CLOSEST(((maxcnt - 1) * maxcnt_units_ns), + idledly = DIV_ROUND_CLOSEST(((maxcnt - 1) * 1000), idledly_units_ps); - idledly = DIV_ROUND_CLOSEST(idledly, 1000); xe_mmio_write32(>->mmio, RING_IDLEDLY(hwe->mmio_base), idledly); } } diff --git a/drivers/gpu/drm/xe/xe_lrc.h b/drivers/gpu/drm/xe/xe_lrc.h index 2fb628da5c4331..96ae31df3359f6 100644 --- a/drivers/gpu/drm/xe/xe_lrc.h +++ b/drivers/gpu/drm/xe/xe_lrc.h @@ -73,7 +73,8 @@ static inline struct xe_lrc *xe_lrc_get(struct xe_lrc *lrc) */ static inline void xe_lrc_put(struct xe_lrc *lrc) { - kref_put(&lrc->refcount, xe_lrc_destroy); + if (lrc) + kref_put(&lrc->refcount, xe_lrc_destroy); } /** diff --git a/drivers/gpu/drm/xe/xe_mmio.c b/drivers/gpu/drm/xe/xe_mmio.c index 350dca1f092596..3d59440ec44dd7 100644 --- a/drivers/gpu/drm/xe/xe_mmio.c +++ b/drivers/gpu/drm/xe/xe_mmio.c @@ -260,11 +260,11 @@ u64 xe_mmio_read64_2x32(struct xe_mmio *mmio, struct xe_reg reg) struct xe_reg reg_udw = { .addr = reg.addr + 0x4 }; u32 ldw, udw, oldudw, retries; - reg.addr = xe_mmio_adjusted_addr(mmio, reg.addr); - reg_udw.addr = xe_mmio_adjusted_addr(mmio, reg_udw.addr); - - /* we shouldn't adjust just one register address */ - xe_tile_assert(mmio->tile, reg_udw.addr == reg.addr + 0x4); + /* + * The two dwords of a 64-bit register can never straddle the offset + * adjustment cutoff. + */ + xe_tile_assert(mmio->tile, !in_range(mmio->adj_limit, reg.addr + 1, 7)); oldudw = xe_mmio_read32(mmio, reg_udw); for (retries = 5; retries; --retries) { diff --git a/drivers/gpu/drm/xe/xe_module.h b/drivers/gpu/drm/xe/xe_module.h index 5a3bfea8b7b4c4..b6684953927016 100644 --- a/drivers/gpu/drm/xe/xe_module.h +++ b/drivers/gpu/drm/xe/xe_module.h @@ -12,7 +12,7 @@ struct xe_modparam { bool force_execlist; bool probe_display; - u32 force_vram_bar_size; + int force_vram_bar_size; int guc_log_level; char *guc_firmware_path; char *huc_firmware_path; diff --git a/drivers/gpu/drm/xe/xe_oa.c b/drivers/gpu/drm/xe/xe_oa.c index f8bb28ab81248b..164e18f4959e39 100644 --- a/drivers/gpu/drm/xe/xe_oa.c +++ b/drivers/gpu/drm/xe/xe_oa.c @@ -543,8 +543,7 @@ static ssize_t xe_oa_read(struct file *file, char __user *buf, size_t offset = 0; int ret; - /* Can't read from disabled streams */ - if (!stream->enabled || !stream->sample) + if (!stream->sample) return -EINVAL; if (!(file->f_flags & O_NONBLOCK)) { @@ -1460,6 +1459,10 @@ static void xe_oa_stream_disable(struct xe_oa_stream *stream) if (stream->sample) hrtimer_cancel(&stream->poll_check_timer); + + /* Update stream->oa_buffer.tail to allow any final reports to be read */ + if (xe_oa_buffer_check_unlocked(stream)) + wake_up(&stream->poll_wq); } static int xe_oa_enable_preempt_timeslice(struct xe_oa_stream *stream) diff --git a/drivers/gpu/drm/xe/xe_pagefault.c b/drivers/gpu/drm/xe/xe_pagefault.c index afb06598b6e1ab..0b625a52a59844 100644 --- a/drivers/gpu/drm/xe/xe_pagefault.c +++ b/drivers/gpu/drm/xe/xe_pagefault.c @@ -187,6 +187,12 @@ static int xe_pagefault_service(struct xe_pagefault *pf) goto unlock_vm; } + if (xe_vma_read_only(vma) && + pf->consumer.access_type != XE_PAGEFAULT_ACCESS_TYPE_READ) { + err = -EPERM; + goto unlock_vm; + } + atomic = xe_pagefault_access_is_atomic(pf->consumer.access_type); if (xe_vma_is_cpu_addr_mirror(vma)) diff --git a/drivers/gpu/drm/xe/xe_pci.c b/drivers/gpu/drm/xe/xe_pci.c index 2aa883f5ef797e..38ea9d7dad091e 100644 --- a/drivers/gpu/drm/xe/xe_pci.c +++ b/drivers/gpu/drm/xe/xe_pci.c @@ -533,6 +533,12 @@ static int read_gmdid(struct xe_device *xe, enum xe_gmdid_type type, u32 *ver, u struct xe_gt *gt __free(kfree) = NULL; int err; + /* Don't try to read media ver if media GT is not allowed */ + if (type == GMDID_MEDIA && !xe_configfs_media_gt_allowed(to_pci_dev(xe->drm.dev))) { + *ver = *revid = 0; + return 0; + } + gt = kzalloc(sizeof(*gt), GFP_KERNEL); if (!gt) return -ENOMEM; diff --git a/drivers/gpu/drm/xe/xe_pt.c b/drivers/gpu/drm/xe/xe_pt.c index 884127b4d97ddc..9b674147c1566a 100644 --- a/drivers/gpu/drm/xe/xe_pt.c +++ b/drivers/gpu/drm/xe/xe_pt.c @@ -1442,9 +1442,9 @@ static int op_check_svm_userptr(struct xe_vm *vm, struct xe_vma_op *op, err = vma_check_userptr(vm, op->map.vma, pt_update); break; case DRM_GPUVA_OP_REMAP: - if (op->remap.prev) + if (op->remap.prev && !op->remap.skip_prev) err = vma_check_userptr(vm, op->remap.prev, pt_update); - if (!err && op->remap.next) + if (!err && op->remap.next && !op->remap.skip_next) err = vma_check_userptr(vm, op->remap.next, pt_update); break; case DRM_GPUVA_OP_UNMAP: @@ -2029,12 +2029,12 @@ static int op_prepare(struct xe_vm *vm, err = unbind_op_prepare(tile, pt_update_ops, old); - if (!err && op->remap.prev) { + if (!err && op->remap.prev && !op->remap.skip_prev) { err = bind_op_prepare(vm, tile, pt_update_ops, op->remap.prev, false); pt_update_ops->wait_vm_bookkeep = true; } - if (!err && op->remap.next) { + if (!err && op->remap.next && !op->remap.skip_next) { err = bind_op_prepare(vm, tile, pt_update_ops, op->remap.next, false); pt_update_ops->wait_vm_bookkeep = true; @@ -2258,10 +2258,10 @@ static void op_commit(struct xe_vm *vm, unbind_op_commit(vm, tile, pt_update_ops, old, fence, fence2); - if (op->remap.prev) + if (op->remap.prev && !op->remap.skip_prev) bind_op_commit(vm, tile, pt_update_ops, op->remap.prev, fence, fence2, false); - if (op->remap.next) + if (op->remap.next && !op->remap.skip_next) bind_op_commit(vm, tile, pt_update_ops, op->remap.next, fence, fence2, false); break; diff --git a/drivers/gpu/drm/xe/xe_pxp.c b/drivers/gpu/drm/xe/xe_pxp.c index bdbdbbf6a6781a..9261a8412b64ff 100644 --- a/drivers/gpu/drm/xe/xe_pxp.c +++ b/drivers/gpu/drm/xe/xe_pxp.c @@ -532,7 +532,7 @@ static int __exec_queue_add(struct xe_pxp *pxp, struct xe_exec_queue *q) static int pxp_start(struct xe_pxp *pxp, u8 type) { int ret = 0; - bool restart = false; + bool restart; if (!xe_pxp_is_enabled(pxp)) return -ENODEV; @@ -561,6 +561,8 @@ static int pxp_start(struct xe_pxp *pxp, u8 type) msecs_to_jiffies(PXP_ACTIVATION_TIMEOUT_MS))) return -ETIMEDOUT; + restart = false; + mutex_lock(&pxp->mutex); /* If PXP is not already active, turn it on */ @@ -603,6 +605,7 @@ static int pxp_start(struct xe_pxp *pxp, u8 type) drm_err(&pxp->xe->drm, "PXP termination failed before start\n"); mutex_lock(&pxp->mutex); pxp->status = XE_PXP_ERROR; + complete_all(&pxp->termination); goto out_unlock; } @@ -890,11 +893,6 @@ int xe_pxp_pm_suspend(struct xe_pxp *pxp) pxp->key_instance++; needs_queue_inval = true; break; - default: - drm_err(&pxp->xe->drm, "unexpected state during PXP suspend: %u", - pxp->status); - ret = -EIO; - goto out; } /* @@ -919,7 +917,6 @@ int xe_pxp_pm_suspend(struct xe_pxp *pxp) pxp->last_suspend_key_instance = pxp->key_instance; -out: return ret; } diff --git a/drivers/gpu/drm/xe/xe_reg_sr.c b/drivers/gpu/drm/xe/xe_reg_sr.c index fc8447a838c4f9..6b9edc7ca41158 100644 --- a/drivers/gpu/drm/xe/xe_reg_sr.c +++ b/drivers/gpu/drm/xe/xe_reg_sr.c @@ -101,10 +101,12 @@ int xe_reg_sr_add(struct xe_reg_sr *sr, *pentry = *e; ret = xa_err(xa_store(&sr->xa, idx, pentry, GFP_KERNEL)); if (ret) - goto fail; + goto fail_free; return 0; +fail_free: + kfree(pentry); fail: xe_gt_err(gt, "discarding save-restore reg %04lx (clear: %08x, set: %08x, masked: %s, mcr: %s): ret=%d\n", diff --git a/drivers/gpu/drm/xe/xe_ring_ops.c b/drivers/gpu/drm/xe/xe_ring_ops.c index ac0c6dcffe156b..803c652f5af918 100644 --- a/drivers/gpu/drm/xe/xe_ring_ops.c +++ b/drivers/gpu/drm/xe/xe_ring_ops.c @@ -267,6 +267,9 @@ static void __emit_job_gen12_simple(struct xe_sched_job *job, struct xe_lrc *lrc i = emit_bb_start(batch_addr, ppgtt_flag, dw, i); + /* Don't preempt fence signaling */ + dw[i++] = MI_ARB_ON_OFF | MI_ARB_DISABLE; + if (job->user_fence.used) { i = emit_flush_dw(dw, i); i = emit_store_imm_ppgtt_posted(job->user_fence.addr, @@ -332,6 +335,9 @@ static void __emit_job_gen12_video(struct xe_sched_job *job, struct xe_lrc *lrc, i = emit_bb_start(batch_addr, ppgtt_flag, dw, i); + /* Don't preempt fence signaling */ + dw[i++] = MI_ARB_ON_OFF | MI_ARB_DISABLE; + if (job->user_fence.used) { i = emit_flush_dw(dw, i); i = emit_store_imm_ppgtt_posted(job->user_fence.addr, @@ -384,6 +390,9 @@ static void __emit_job_gen12_render_compute(struct xe_sched_job *job, i = emit_bb_start(batch_addr, ppgtt_flag, dw, i); + /* Don't preempt fence signaling */ + dw[i++] = MI_ARB_ON_OFF | MI_ARB_DISABLE; + i = emit_render_cache_flush(job, dw, i); if (job->user_fence.used) diff --git a/drivers/gpu/drm/xe/xe_sriov_packet.c b/drivers/gpu/drm/xe/xe_sriov_packet.c index bab99469689649..111877b6d44cf2 100644 --- a/drivers/gpu/drm/xe/xe_sriov_packet.c +++ b/drivers/gpu/drm/xe/xe_sriov_packet.c @@ -342,6 +342,8 @@ ssize_t xe_sriov_packet_write_single(struct xe_device *xe, unsigned int vfid, ret = xe_sriov_pf_migration_restore_produce(xe, vfid, *data); if (ret) { xe_sriov_packet_free(*data); + *data = NULL; + return ret; } diff --git a/drivers/gpu/drm/xe/xe_sriov_pf_sysfs.c b/drivers/gpu/drm/xe/xe_sriov_pf_sysfs.c index c0b767ac735cf2..d1c1f6c2956641 100644 --- a/drivers/gpu/drm/xe/xe_sriov_pf_sysfs.c +++ b/drivers/gpu/drm/xe/xe_sriov_pf_sysfs.c @@ -349,18 +349,33 @@ static const struct attribute_group *xe_sriov_vf_attr_groups[] = { /* no user serviceable parts below */ -static struct kobject *create_xe_sriov_kobj(struct xe_device *xe, unsigned int vfid) +static void action_put_kobject(void *arg) +{ + struct kobject *kobj = arg; + + kobject_put(kobj); +} + +static struct kobject *create_xe_sriov_kobj(struct xe_device *xe, unsigned int vfid, + const struct kobj_type *ktype) { struct xe_sriov_kobj *vkobj; + int err; xe_sriov_pf_assert_vfid(xe, vfid); vkobj = kzalloc(sizeof(*vkobj), GFP_KERNEL); if (!vkobj) - return NULL; + return ERR_PTR(-ENOMEM); vkobj->xe = xe; vkobj->vfid = vfid; + kobject_init(&vkobj->base, ktype); + + err = devm_add_action_or_reset(xe->drm.dev, action_put_kobject, &vkobj->base); + if (err) + return ERR_PTR(err); + return &vkobj->base; } @@ -471,28 +486,17 @@ static void pf_sysfs_note(struct xe_device *xe, int err, const char *what) xe_sriov_dbg(xe, "Failed to setup sysfs %s (%pe)\n", what, ERR_PTR(err)); } -static void action_put_kobject(void *arg) -{ - struct kobject *kobj = arg; - - kobject_put(kobj); -} - static int pf_setup_root(struct xe_device *xe) { struct kobject *parent = &xe->drm.dev->kobj; struct kobject *root; int err; - root = create_xe_sriov_kobj(xe, PFID); - if (!root) - return pf_sysfs_error(xe, -ENOMEM, "root obj"); - - err = devm_add_action_or_reset(xe->drm.dev, action_put_kobject, root); - if (err) - return pf_sysfs_error(xe, err, "root action"); + root = create_xe_sriov_kobj(xe, PFID, &xe_sriov_dev_ktype); + if (IS_ERR(root)) + return pf_sysfs_error(xe, PTR_ERR(root), "root obj"); - err = kobject_init_and_add(root, &xe_sriov_dev_ktype, parent, "sriov_admin"); + err = kobject_add(root, parent, "sriov_admin"); if (err) return pf_sysfs_error(xe, err, "root init"); @@ -513,20 +517,14 @@ static int pf_setup_tree(struct xe_device *xe) root = xe->sriov.pf.sysfs.root; for (n = 0; n <= totalvfs; n++) { - kobj = create_xe_sriov_kobj(xe, VFID(n)); - if (!kobj) - return pf_sysfs_error(xe, -ENOMEM, "tree obj"); - - err = devm_add_action_or_reset(xe->drm.dev, action_put_kobject, root); - if (err) - return pf_sysfs_error(xe, err, "tree action"); + kobj = create_xe_sriov_kobj(xe, VFID(n), &xe_sriov_vf_ktype); + if (IS_ERR(kobj)) + return pf_sysfs_error(xe, PTR_ERR(kobj), "tree obj"); if (n) - err = kobject_init_and_add(kobj, &xe_sriov_vf_ktype, - root, "vf%u", n); + err = kobject_add(kobj, root, "vf%u", n); else - err = kobject_init_and_add(kobj, &xe_sriov_vf_ktype, - root, "pf"); + err = kobject_add(kobj, root, "pf"); if (err) return pf_sysfs_error(xe, err, "tree init"); diff --git a/drivers/gpu/drm/xe/xe_sync.c b/drivers/gpu/drm/xe/xe_sync.c index ff74528ca0c6f2..2b1f92ef7188b8 100644 --- a/drivers/gpu/drm/xe/xe_sync.c +++ b/drivers/gpu/drm/xe/xe_sync.c @@ -146,8 +146,10 @@ int xe_sync_entry_parse(struct xe_device *xe, struct xe_file *xef, if (!signal) { sync->fence = drm_syncobj_fence_get(sync->syncobj); - if (XE_IOCTL_DBG(xe, !sync->fence)) - return -EINVAL; + if (XE_IOCTL_DBG(xe, !sync->fence)) { + err = -EINVAL; + goto free_sync; + } } break; @@ -167,17 +169,21 @@ int xe_sync_entry_parse(struct xe_device *xe, struct xe_file *xef, if (signal) { sync->chain_fence = dma_fence_chain_alloc(); - if (!sync->chain_fence) - return -ENOMEM; + if (!sync->chain_fence) { + err = -ENOMEM; + goto free_sync; + } } else { sync->fence = drm_syncobj_fence_get(sync->syncobj); - if (XE_IOCTL_DBG(xe, !sync->fence)) - return -EINVAL; + if (XE_IOCTL_DBG(xe, !sync->fence)) { + err = -EINVAL; + goto free_sync; + } err = dma_fence_chain_find_seqno(&sync->fence, sync_in.timeline_value); if (err) - return err; + goto free_sync; } break; @@ -200,8 +206,10 @@ int xe_sync_entry_parse(struct xe_device *xe, struct xe_file *xef, if (XE_IOCTL_DBG(xe, IS_ERR(sync->ufence))) return PTR_ERR(sync->ufence); sync->ufence_chain_fence = dma_fence_chain_alloc(); - if (!sync->ufence_chain_fence) - return -ENOMEM; + if (!sync->ufence_chain_fence) { + err = -ENOMEM; + goto free_sync; + } sync->ufence_syncobj = ufence_syncobj; } @@ -216,6 +224,10 @@ int xe_sync_entry_parse(struct xe_device *xe, struct xe_file *xef, sync->timeline_value = sync_in.timeline_value; return 0; + +free_sync: + xe_sync_entry_cleanup(sync); + return err; } ALLOW_ERROR_INJECTION(xe_sync_entry_parse, ERRNO); diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c index 095bb197e8b05d..e340e48cd103ab 100644 --- a/drivers/gpu/drm/xe/xe_vm.c +++ b/drivers/gpu/drm/xe/xe_vm.c @@ -2451,8 +2451,17 @@ static struct xe_vma *new_vma(struct xe_vm *vm, struct drm_gpuva_op_map *op, if (IS_ERR(vma)) return vma; - if (xe_vma_is_userptr(vma)) + if (xe_vma_is_userptr(vma)) { err = xe_vma_userptr_pin_pages(to_userptr_vma(vma)); + /* + * -EBUSY has dedicated meaning that a user fence + * attached to the VMA is busy, in practice + * xe_vma_userptr_pin_pages can only fail with -EBUSY if + * we are low on memory so convert this to -ENOMEM. + */ + if (err == -EBUSY) + err = -ENOMEM; + } } if (err) { prep_vma_destroy(vm, vma, false); @@ -2523,7 +2532,6 @@ static int xe_vma_op_commit(struct xe_vm *vm, struct xe_vma_op *op) if (!err && op->remap.skip_prev) { op->remap.prev->tile_present = tile_present; - op->remap.prev = NULL; } } if (op->remap.next) { @@ -2533,11 +2541,13 @@ static int xe_vma_op_commit(struct xe_vm *vm, struct xe_vma_op *op) if (!err && op->remap.skip_next) { op->remap.next->tile_present = tile_present; - op->remap.next = NULL; } } - /* Adjust for partial unbind after removing VMA from VM */ + /* + * Adjust for partial unbind after removing VMA from VM. In case + * of unwind we might need to undo this later. + */ if (!err) { op->base.remap.unmap->va->va.addr = op->remap.start; op->base.remap.unmap->va->va.range = op->remap.range; @@ -2656,6 +2666,8 @@ static int vm_bind_ioctl_ops_parse(struct xe_vm *vm, struct drm_gpuva_ops *ops, op->remap.start = xe_vma_start(old); op->remap.range = xe_vma_size(old); + op->remap.old_start = op->remap.start; + op->remap.old_range = op->remap.range; flags |= op->base.remap.unmap->va->flags & XE_VMA_CREATE_MASK; if (op->base.remap.prev) { @@ -2803,8 +2815,19 @@ static void xe_vma_op_unwind(struct xe_vm *vm, struct xe_vma_op *op, xe_svm_notifier_lock(vm); vma->gpuva.flags &= ~XE_VMA_DESTROYED; xe_svm_notifier_unlock(vm); - if (post_commit) + if (post_commit) { + /* + * Restore the old va range, in case of the + * prev/next skip optimisation. Otherwise what + * we re-insert here could be smaller than the + * original range. + */ + op->base.remap.unmap->va->va.addr = + op->remap.old_start; + op->base.remap.unmap->va->va.range = + op->remap.old_range; xe_vm_insert_vma(vm, vma); + } } break; } @@ -3214,7 +3237,8 @@ static void op_add_ufence(struct xe_vm *vm, struct xe_vma_op *op, { switch (op->base.op) { case DRM_GPUVA_OP_MAP: - vma_add_ufence(op->map.vma, ufence); + if (!xe_vma_is_cpu_addr_mirror(op->map.vma)) + vma_add_ufence(op->map.vma, ufence); break; case DRM_GPUVA_OP_REMAP: if (op->remap.prev) diff --git a/drivers/gpu/drm/xe/xe_vm_madvise.c b/drivers/gpu/drm/xe/xe_vm_madvise.c index cad3cf627c3f2f..9dc801f6571297 100644 --- a/drivers/gpu/drm/xe/xe_vm_madvise.c +++ b/drivers/gpu/drm/xe/xe_vm_madvise.c @@ -268,8 +268,13 @@ static bool madvise_args_are_sane(struct xe_device *xe, const struct drm_xe_madv break; case DRM_XE_MEM_RANGE_ATTR_PAT: { - u16 coh_mode = xe_pat_index_get_coh_mode(xe, args->pat_index.val); + u16 pat_index, coh_mode; + if (XE_IOCTL_DBG(xe, args->pat_index.val >= xe->pat.n_entries)) + return false; + + pat_index = array_index_nospec(args->pat_index.val, xe->pat.n_entries); + coh_mode = xe_pat_index_get_coh_mode(xe, pat_index); if (XE_IOCTL_DBG(xe, !coh_mode)) return false; @@ -385,7 +390,7 @@ int xe_vm_madvise_ioctl(struct drm_device *dev, void *data, struct drm_file *fil madvise_range.num_vmas, args->atomic.val)) { err = -EINVAL; - goto unlock_vm; + goto free_vmas; } } @@ -421,6 +426,7 @@ int xe_vm_madvise_ioctl(struct drm_device *dev, void *data, struct drm_file *fil err_fini: if (madvise_range.has_bo_vmas) drm_exec_fini(&exec); +free_vmas: kfree(madvise_range.vmas); madvise_range.vmas = NULL; unlock_vm: diff --git a/drivers/gpu/drm/xe/xe_vm_types.h b/drivers/gpu/drm/xe/xe_vm_types.h index 2168ef052499ec..016bc423f8c3d8 100644 --- a/drivers/gpu/drm/xe/xe_vm_types.h +++ b/drivers/gpu/drm/xe/xe_vm_types.h @@ -360,6 +360,10 @@ struct xe_vma_op_remap { u64 start; /** @range: range of the VMA unmap */ u64 range; + /** @old_start: Original start of the VMA we unmap */ + u64 old_start; + /** @old_range: Original range of the VMA we unmap */ + u64 old_range; /** @skip_prev: skip prev rebind */ bool skip_prev; /** @skip_next: skip next rebind */ diff --git a/drivers/gpu/drm/xe/xe_wa.c b/drivers/gpu/drm/xe/xe_wa.c index e32dd2fde6f1c5..c15b0288e0ff50 100644 --- a/drivers/gpu/drm/xe/xe_wa.c +++ b/drivers/gpu/drm/xe/xe_wa.c @@ -15,6 +15,7 @@ #include "regs/xe_engine_regs.h" #include "regs/xe_gt_regs.h" +#include "regs/xe_guc_regs.h" #include "regs/xe_regs.h" #include "xe_device_types.h" #include "xe_force_wake.h" @@ -254,12 +255,14 @@ static const struct xe_rtp_entry_sr gt_was[] = { { XE_RTP_NAME("16025250150"), XE_RTP_RULES(GRAPHICS_VERSION(2001)), - XE_RTP_ACTIONS(SET(LSN_VC_REG2, - LSN_LNI_WGT(1) | - LSN_LNE_WGT(1) | - LSN_DIM_X_WGT(1) | - LSN_DIM_Y_WGT(1) | - LSN_DIM_Z_WGT(1))) + XE_RTP_ACTIONS(FIELD_SET(LSN_VC_REG2, + LSN_LNI_WGT_MASK | LSN_LNE_WGT_MASK | + LSN_DIM_X_WGT_MASK | LSN_DIM_Y_WGT_MASK | + LSN_DIM_Z_WGT_MASK, + LSN_LNI_WGT(1) | LSN_LNE_WGT(1) | + LSN_DIM_X_WGT(1) | LSN_DIM_Y_WGT(1) | + LSN_DIM_Z_WGT(1)), + SET(LSC_CHICKEN_BIT_0_UDW, L3_128B_256B_WRT_DIS)) }, /* Xe2_HPM */ @@ -315,6 +318,10 @@ static const struct xe_rtp_entry_sr gt_was[] = { XE_RTP_ACTIONS(SET(VDBOX_CGCTL3F10(0), RAMDFTUNIT_CLKGATE_DIS)), XE_RTP_ENTRY_FLAG(FOREACH_ENGINE), }, + { XE_RTP_NAME("16028005424"), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(3000, 3005)), + XE_RTP_ACTIONS(SET(GUC_INTR_CHICKEN, DISABLE_SIGNALING_ENGINES)) + }, }; static const struct xe_rtp_entry_sr engine_was[] = { @@ -567,16 +574,6 @@ static const struct xe_rtp_entry_sr engine_was[] = { FUNC(xe_rtp_match_first_render_or_compute)), XE_RTP_ACTIONS(SET(ROW_CHICKEN, EARLY_EOT_DIS)) }, - { XE_RTP_NAME("14019988906"), - XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, 2002), - FUNC(xe_rtp_match_first_render_or_compute)), - XE_RTP_ACTIONS(SET(XEHP_PSS_CHICKEN, FLSH_IGNORES_PSD)) - }, - { XE_RTP_NAME("14019877138"), - XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, 2002), - FUNC(xe_rtp_match_first_render_or_compute)), - XE_RTP_ACTIONS(SET(XEHP_PSS_CHICKEN, FD_END_COLLECT)) - }, { XE_RTP_NAME("14020338487"), XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, 2002), FUNC(xe_rtp_match_first_render_or_compute)), @@ -873,6 +870,14 @@ static const struct xe_rtp_entry_sr lrc_was[] = { XE_RTP_RULES(GRAPHICS_VERSION(2001), ENGINE_CLASS(RENDER)), XE_RTP_ACTIONS(SET(WM_CHICKEN3, HIZ_PLANE_COMPRESSION_DIS)) }, + { XE_RTP_NAME("14019988906"), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, 2002), ENGINE_CLASS(RENDER)), + XE_RTP_ACTIONS(SET(XEHP_PSS_CHICKEN, FLSH_IGNORES_PSD)) + }, + { XE_RTP_NAME("14019877138"), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, 2002), ENGINE_CLASS(RENDER)), + XE_RTP_ACTIONS(SET(XEHP_PSS_CHICKEN, FD_END_COLLECT)) + }, { XE_RTP_NAME("14021490052"), XE_RTP_RULES(GRAPHICS_VERSION(2001), ENGINE_CLASS(RENDER)), XE_RTP_ACTIONS(SET(FF_MODE, diff --git a/drivers/gpu/nova-core/falcon.rs b/drivers/gpu/nova-core/falcon.rs index 82c661aef594ff..3ab33ea36d9c8c 100644 --- a/drivers/gpu/nova-core/falcon.rs +++ b/drivers/gpu/nova-core/falcon.rs @@ -8,7 +8,10 @@ use hal::FalconHal; use kernel::{ device, - dma::DmaAddress, + dma::{ + DmaAddress, + DmaMask, // + }, io::poll::read_poll_timeout, prelude::*, sync::aref::ARef, @@ -472,6 +475,12 @@ impl Falcon { return Err(EINVAL); } + // The DMATRFBASE/1 register pair only supports a 49-bit address. + if dma_start > DmaMask::new::<49>().value() { + dev_err!(self.dev, "DMA address {:#x} exceeds 49 bits\n", dma_start); + return Err(ERANGE); + } + // DMA transfers can only be done in units of 256 bytes. Compute how many such transfers we // need to perform. let num_transfers = load_offsets.len.div_ceil(DMA_LEN); diff --git a/drivers/hid/Kconfig b/drivers/hid/Kconfig index 920a64b66b25b3..a9218323d5a3b1 100644 --- a/drivers/hid/Kconfig +++ b/drivers/hid/Kconfig @@ -369,6 +369,7 @@ config HID_ELECOM - EX-G Trackballs (M-XT3DRBK, M-XT3URBK) - DEFT Trackballs (M-DT1DRBK, M-DT1URBK, M-DT2DRBK, M-DT2URBK) - HUGE Trackballs (M-HT1DRBK, M-HT1URBK) + - HUGE Plus Trackball (M-HT1MRBK) config HID_ELO tristate "ELO USB 4000/4500 touchscreen" @@ -731,7 +732,8 @@ config HID_MAGICMOUSE Support for the Apple Magic Mouse/Trackpad multi-touch. Say Y here if you want support for the multi-touch features of the - Apple Wireless "Magic" Mouse and the Apple Wireless "Magic" Trackpad. + Apple Wireless "Magic" Mouse, the Apple Wireless "Magic" Trackpad and + force touch Trackpads in Macbooks starting from 2015. config HID_MALTRON tristate "Maltron L90 keyboard" @@ -1448,4 +1450,8 @@ endif # HID source "drivers/hid/usbhid/Kconfig" +source "drivers/hid/spi-hid/Kconfig" + +source "drivers/hid/dockchannel-hid/Kconfig" + endif # HID_SUPPORT diff --git a/drivers/hid/Makefile b/drivers/hid/Makefile index 361a7daedeb854..2ed4942821539f 100644 --- a/drivers/hid/Makefile +++ b/drivers/hid/Makefile @@ -173,6 +173,12 @@ obj-$(CONFIG_INTEL_ISH_HID) += intel-ish-hid/ obj-$(CONFIG_AMD_SFH_HID) += amd-sfh-hid/ +obj-$(CONFIG_HID_DOCKCHANNEL) += dockchannel-hid/ + +obj-$(CONFIG_SPI_HID_APPLE_CORE) += spi-hid/ + +obj-$(CONFIG_HID_DOCKCHANNEL) += dockchannel-hid/ + obj-$(CONFIG_SURFACE_HID_CORE) += surface-hid/ obj-$(CONFIG_INTEL_THC_HID) += intel-thc-hid/ diff --git a/drivers/hid/amd-sfh-hid/amd_sfh_pcie.c b/drivers/hid/amd-sfh-hid/amd_sfh_pcie.c index 1d9f955573aa43..4b81cebdc33590 100644 --- a/drivers/hid/amd-sfh-hid/amd_sfh_pcie.c +++ b/drivers/hid/amd-sfh-hid/amd_sfh_pcie.c @@ -413,7 +413,8 @@ static void sfh_init_work(struct work_struct *work) rc = amd_sfh_hid_client_init(mp2); if (rc) { amd_sfh_clear_intr(mp2); - dev_err(&pdev->dev, "amd_sfh_hid_client_init failed err %d\n", rc); + if (rc != -EOPNOTSUPP) + dev_err(&pdev->dev, "amd_sfh_hid_client_init failed err %d\n", rc); return; } diff --git a/drivers/hid/bpf/hid_bpf_dispatch.c b/drivers/hid/bpf/hid_bpf_dispatch.c index 9a06f9b0e4ef33..cf465a5fe43aff 100644 --- a/drivers/hid/bpf/hid_bpf_dispatch.c +++ b/drivers/hid/bpf/hid_bpf_dispatch.c @@ -447,6 +447,8 @@ hid_bpf_hw_request(struct hid_bpf_ctx *ctx, __u8 *buf, size_t buf__sz, (u64)(long)ctx, true); /* prevent infinite recursions */ + if (ret > size) + ret = size; if (ret > 0) memcpy(buf, dma_data, ret); diff --git a/drivers/hid/dockchannel-hid/Kconfig b/drivers/hid/dockchannel-hid/Kconfig new file mode 100644 index 00000000000000..254961ad15e19c --- /dev/null +++ b/drivers/hid/dockchannel-hid/Kconfig @@ -0,0 +1,13 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +menu "DockChannel HID support" + depends on APPLE_DOCKCHANNEL + +config HID_DOCKCHANNEL + tristate "HID over DockChannel transport layer for Apple Silicon SoCs" + depends on APPLE_DOCKCHANNEL && INPUT && OF && HID + help + Say Y here if you use an M2 or later Apple Silicon based laptop. + The keyboard and touchpad are HID based devices connected via the + proprietary DockChannel interface. + +endmenu diff --git a/drivers/hid/dockchannel-hid/Makefile b/drivers/hid/dockchannel-hid/Makefile new file mode 100644 index 00000000000000..7dba766b047fcc --- /dev/null +++ b/drivers/hid/dockchannel-hid/Makefile @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +# +# Makefile for DockChannel HID transport drivers +# + +obj-$(CONFIG_HID_DOCKCHANNEL) += dockchannel-hid.o diff --git a/drivers/hid/dockchannel-hid/dockchannel-hid.c b/drivers/hid/dockchannel-hid/dockchannel-hid.c new file mode 100644 index 00000000000000..a712a724ded30b --- /dev/null +++ b/drivers/hid/dockchannel-hid/dockchannel-hid.c @@ -0,0 +1,1213 @@ +/* + * SPDX-License-Identifier: GPL-2.0 OR MIT + * + * Apple DockChannel HID transport driver + * + * Copyright The Asahi Linux Contributors + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "../hid-ids.h" + +#define COMMAND_TIMEOUT_MS 1000 +#define START_TIMEOUT_MS 2000 + +#define MAX_INTERFACES 16 + +/* Data + checksum */ +#define MAX_PKT_SIZE (0xffff + 4) + +#define DCHID_CHANNEL_CMD 0x11 +#define DCHID_CHANNEL_REPORT 0x12 + +struct dchid_hdr { + u8 hdr_len; + u8 channel; + u16 length; + u8 seq; + u8 iface; + u16 pad; +} __packed; + +#define IFACE_COMM 0 + +#define FLAGS_GROUP GENMASK(7, 6) +#define FLAGS_REQ GENMASK(5, 0) + +#define REQ_SET_REPORT 0 +#define REQ_GET_REPORT 1 + +struct dchid_subhdr { + u8 flags; + u8 unk; + u16 length; + u32 retcode; +} __packed; + +#define EVENT_GPIO_CMD 0xa0 +#define EVENT_INIT 0xf0 +#define EVENT_READY 0xf1 + +struct dchid_init_hdr { + u8 type; + u8 unk1; + u8 unk2; + u8 iface; + char name[16]; + u8 more_packets; + u8 unkpad; +} __packed; + +#define INIT_HID_DESCRIPTOR 0 +#define INIT_GPIO_REQUEST 1 +#define INIT_TERMINATOR 2 +#define INIT_PRODUCT_NAME 7 + +#define CMD_RESET_INTERFACE 0x40 +#define CMD_SEND_FIRMWARE 0x95 +#define CMD_ENABLE_INTERFACE 0xb4 +#define CMD_ACK_GPIO_CMD 0xa1 + +struct dchid_init_block_hdr { + u16 type; + u16 length; +} __packed; + +#define MAX_GPIO_NAME 32 + +struct dchid_gpio_request { + u16 unk; + u16 id; + char name[MAX_GPIO_NAME]; +} __packed; + +struct dchid_gpio_cmd { + u8 type; + u8 iface; + u8 gpio; + u8 unk; + u8 cmd; +} __packed; + +struct dchid_gpio_ack { + u8 type; + u32 retcode; + u8 cmd[]; +} __packed; + +#define STM_REPORT_ID 0x10 +#define STM_REPORT_SERIAL 0x11 +#define STM_REPORT_KEYBTYPE 0x14 + +struct dchid_stm_id { + u8 unk; + u16 vendor_id; + u16 product_id; + u16 version_number; + u8 unk2; + u8 unk3; + u8 keyboard_type; + u8 serial_length; + /* Serial follows, but we grab it with a different report. */ +} __packed; + +#define FW_MAGIC 0x46444948 +#define FW_VER 1 + +struct fw_header { + u32 magic; + u32 version; + u32 hdr_length; + u32 data_length; + u32 iface_offset; +} __packed; + +struct dchid_work { + struct work_struct work; + struct dchid_iface *iface; + + struct dchid_hdr hdr; + u8 data[]; +}; + +struct dchid_iface { + struct dockchannel_hid *dchid; + struct hid_device *hid; + struct workqueue_struct *wq; + + bool creating; + struct work_struct create_work; + + int index; + const char *name; + const struct device_node *of_node; + + uint8_t tx_seq; + bool deferred; + bool starting; + bool open; + struct completion ready; + + void *hid_desc; + size_t hid_desc_len; + + struct gpio_desc *gpio; + char gpio_name[MAX_GPIO_NAME]; + int gpio_id; + + struct mutex out_mutex; + u32 out_flags; + int out_report; + u32 retcode; + void *resp_buf; + size_t resp_size; + struct completion out_complete; + + u32 keyboard_layout_id; +}; + +struct dockchannel_hid { + struct device *dev; + struct dockchannel *dc; + struct device_link *helper_link; + + bool id_ready; + struct dchid_stm_id device_id; + char serial[64]; + + struct dchid_iface *comm; + struct dchid_iface *ifaces[MAX_INTERFACES]; + + u8 pkt_buf[MAX_PKT_SIZE]; + + /* Workqueue to asynchronously create HID devices */ + struct workqueue_struct *new_iface_wq; +}; + +static ssize_t apple_layout_id_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct hid_device *hdev = to_hid_device(dev); + struct dchid_iface *iface = hdev->driver_data; + + return scnprintf(buf, PAGE_SIZE, "%d\n", iface->keyboard_layout_id); +} + +static DEVICE_ATTR_RO(apple_layout_id); + +static struct dchid_iface * +dchid_get_interface(struct dockchannel_hid *dchid, int index, const char *name) +{ + struct dchid_iface *iface; + + if (index >= MAX_INTERFACES) { + dev_err(dchid->dev, "Interface index %d out of range\n", index); + return NULL; + } + + if (dchid->ifaces[index]) + return dchid->ifaces[index]; + + iface = devm_kzalloc(dchid->dev, sizeof(struct dchid_iface), GFP_KERNEL); + if (!iface) + return NULL; + + iface->index = index; + iface->name = devm_kstrdup(dchid->dev, name, GFP_KERNEL); + iface->dchid = dchid; + iface->out_report= -1; + init_completion(&iface->out_complete); + init_completion(&iface->ready); + mutex_init(&iface->out_mutex); + iface->wq = alloc_ordered_workqueue("dchid-%s", WQ_MEM_RECLAIM, iface->name); + if (!iface->wq) + return NULL; + + /* Comm is not a HID subdevice */ + if (!strcmp(name, "comm")) { + dchid->ifaces[index] = iface; + return iface; + } + + iface->of_node = of_get_child_by_name(dchid->dev->of_node, name); + if (!iface->of_node) { + dev_warn(dchid->dev, "No OF node for subdevice %s, ignoring.", name); + return NULL; + } + + dchid->ifaces[index] = iface; + return iface; +} + +static u32 dchid_checksum(void *p, size_t length) +{ + u32 sum = 0; + + while (length >= 4) { + sum += get_unaligned_le32(p); + p += 4; + length -= 4; + } + + WARN_ON_ONCE(length); + return sum; +} + +static int dchid_send(struct dchid_iface *iface, u32 flags, void *msg, size_t size) +{ + u32 checksum = 0xffffffff; + size_t wsize = round_down(size, 4); + size_t tsize = size - wsize; + int ret; + struct { + struct dchid_hdr hdr; + struct dchid_subhdr sub; + } __packed h; + + memset(&h, 0, sizeof(h)); + h.hdr.hdr_len = sizeof(h.hdr); + h.hdr.channel = DCHID_CHANNEL_CMD; + h.hdr.length = round_up(size, 4) + sizeof(h.sub); + h.hdr.seq = iface->tx_seq; + h.hdr.iface = iface->index; + h.sub.flags = flags; + h.sub.length = size; + + ret = dockchannel_send(iface->dchid->dc, &h, sizeof(h)); + if (ret < 0) + return ret; + checksum -= dchid_checksum(&h, sizeof(h)); + + ret = dockchannel_send(iface->dchid->dc, msg, wsize); + if (ret < 0) + return ret; + checksum -= dchid_checksum(msg, wsize); + + if (tsize) { + u8 tail[4] = {0, 0, 0, 0}; + + memcpy(tail, msg + wsize, tsize); + ret = dockchannel_send(iface->dchid->dc, tail, sizeof(tail)); + if (ret < 0) + return ret; + checksum -= dchid_checksum(tail, sizeof(tail)); + } + + ret = dockchannel_send(iface->dchid->dc, &checksum, sizeof(checksum)); + if (ret < 0) + return ret; + + return 0; +} + +static int dchid_cmd(struct dchid_iface *iface, u32 type, u32 req, + void *data, size_t size, void *resp_buf, size_t resp_size) +{ + int ret; + int report_id = *(u8*)data; + + mutex_lock(&iface->out_mutex); + + WARN_ON(iface->out_report != -1); + iface->out_report = report_id; + iface->out_flags = FIELD_PREP(FLAGS_GROUP, type) | FIELD_PREP(FLAGS_REQ, req); + iface->resp_buf = resp_buf; + iface->resp_size = resp_size; + reinit_completion(&iface->out_complete); + + ret = dchid_send(iface, iface->out_flags, data, size); + if (ret < 0) + goto done; + + if (!wait_for_completion_timeout(&iface->out_complete, msecs_to_jiffies(COMMAND_TIMEOUT_MS))) { + dev_err(iface->dchid->dev, "output report 0x%x to iface %d (%s) timed out\n", + report_id, iface->index, iface->name); + ret = -ETIMEDOUT; + goto done; + } + + ret = iface->resp_size; + if (iface->retcode) { + dev_err(iface->dchid->dev, + "output report 0x%x to iface %d (%s) failed with err 0x%x\n", + report_id, iface->index, iface->name, iface->retcode); + ret = -EIO; + } + +done: + iface->tx_seq++; + iface->out_report = -1; + iface->out_flags = 0; + iface->resp_buf = NULL; + iface->resp_size = 0; + mutex_unlock(&iface->out_mutex); + return ret; +} + +static int dchid_comm_cmd(struct dockchannel_hid *dchid, void *cmd, size_t size) +{ + return dchid_cmd(dchid->comm, HID_FEATURE_REPORT, REQ_SET_REPORT, cmd, size, NULL, 0); +} + +static int dchid_enable_interface(struct dchid_iface *iface) +{ + u8 msg[] = { CMD_ENABLE_INTERFACE, iface->index }; + + return dchid_comm_cmd(iface->dchid, msg, sizeof(msg)); +} + +static int dchid_reset_interface(struct dchid_iface *iface, int state) +{ + u8 msg[] = { CMD_RESET_INTERFACE, 1, iface->index, state }; + + return dchid_comm_cmd(iface->dchid, msg, sizeof(msg)); +} + +static int dchid_send_firmware(struct dchid_iface *iface, void *firmware, size_t size) +{ + struct { + u8 cmd; + u8 unk1; + u8 unk2; + u8 iface; + u64 addr; + u32 size; + } __packed msg = { + .cmd = CMD_SEND_FIRMWARE, + .unk1 = 2, + .unk2 = 0, + .iface = iface->index, + .size = size, + }; + dma_addr_t addr; + void *buf = dmam_alloc_coherent(iface->dchid->dev, size, &addr, GFP_KERNEL); + + if (IS_ERR_OR_NULL(buf)) + return buf ? PTR_ERR(buf) : -ENOMEM; + + msg.addr = addr; + memcpy(buf, firmware, size); + wmb(); + + return dchid_comm_cmd(iface->dchid, &msg, sizeof(msg)); +} + +static int dchid_get_firmware(struct dchid_iface *iface, void **firmware, size_t *size) +{ + int ret; + const char *fw_name; + const struct firmware *fw; + struct fw_header *hdr; + u8 *fw_data; + + ret = of_property_read_string(iface->of_node, "firmware-name", &fw_name); + if (ret) { + /* Firmware is only for some devices */ + *firmware = NULL; + *size = 0; + return 0; + } + + ret = request_firmware(&fw, fw_name, iface->dchid->dev); + if (ret) + return ret; + + hdr = (struct fw_header *)fw->data; + + if (hdr->magic != FW_MAGIC || hdr->version != FW_VER || + hdr->hdr_length < sizeof(*hdr) || hdr->hdr_length > fw->size || + (hdr->hdr_length + (size_t)hdr->data_length) > fw->size || + hdr->iface_offset >= hdr->data_length) { + dev_warn(iface->dchid->dev, "%s: invalid firmware header\n", + fw_name); + ret = -EINVAL; + goto done; + } + + fw_data = devm_kmemdup(iface->dchid->dev, fw->data + hdr->hdr_length, + hdr->data_length, GFP_KERNEL); + if (!fw_data) { + ret = -ENOMEM; + goto done; + } + + if (hdr->iface_offset) + fw_data[hdr->iface_offset] = iface->index; + + *firmware = fw_data; + *size = hdr->data_length; + +done: + release_firmware(fw); + return ret; +} + +static int dchid_request_gpio(struct dchid_iface *iface) +{ + char prop_name[MAX_GPIO_NAME + 16]; + + if (iface->gpio) + return 0; + + dev_info(iface->dchid->dev, "Requesting GPIO %s#%d: %s\n", + iface->name, iface->gpio_id, iface->gpio_name); + + snprintf(prop_name, sizeof(prop_name), "apple,%s", iface->gpio_name); + + iface->gpio = devm_gpiod_get_index(iface->dchid->dev, prop_name, 0, GPIOD_OUT_LOW); + + if (IS_ERR_OR_NULL(iface->gpio)) { + dev_err(iface->dchid->dev, "Failed to request GPIO %s-gpios\n", prop_name); + iface->gpio = NULL; + return -1; + } + + return 0; +} + +static int dchid_start_interface(struct dchid_iface *iface) +{ + void *fw; + size_t size; + int ret; + + if (iface->starting) { + dev_warn(iface->dchid->dev, "Interface %s is already starting", iface->name); + return -EINPROGRESS; + } + + dev_info(iface->dchid->dev, "Starting interface %s\n", iface->name); + + iface->starting = true; + + /* Look to see if we need firmware */ + ret = dchid_get_firmware(iface, &fw, &size); + if (ret < 0) + goto err; + + /* If we need a GPIO, make sure we have it. */ + if (iface->gpio_id) { + ret = dchid_request_gpio(iface); + if (ret < 0) + goto err; + } + + /* Only multi-touch has firmware */ + if (fw && size) { + + /* Send firmware to the device */ + dev_info(iface->dchid->dev, "Sending firmware for %s\n", iface->name); + ret = dchid_send_firmware(iface, fw, size); + if (ret < 0) { + dev_err(iface->dchid->dev, "Failed to send %s firmwareS", iface->name); + goto err; + } + + /* After loading firmware, multi-touch needs a reset */ + dev_info(iface->dchid->dev, "Resetting %s\n", iface->name); + dchid_reset_interface(iface, 0); + dchid_reset_interface(iface, 2); + } + + return 0; + +err: + iface->starting = false; + return ret; +} + +static int dchid_start(struct hid_device *hdev) +{ + struct dchid_iface *iface = hdev->driver_data; + + if (iface->keyboard_layout_id) { + int ret = device_create_file(&hdev->dev, &dev_attr_apple_layout_id); + if (ret) { + dev_warn(iface->dchid->dev, "Failed to create apple_layout_id: %d", ret); + iface->keyboard_layout_id = 0; + } + } + + return 0; +}; + +static void dchid_stop(struct hid_device *hdev) +{ + struct dchid_iface *iface = hdev->driver_data; + + if (iface->keyboard_layout_id) + device_remove_file(&hdev->dev, &dev_attr_apple_layout_id); +} + +static int dchid_open(struct hid_device *hdev) +{ + struct dchid_iface *iface = hdev->driver_data; + int ret; + + if (!completion_done(&iface->ready)) { + ret = dchid_start_interface(iface); + if (ret < 0) + return ret; + + if (!wait_for_completion_timeout(&iface->ready, msecs_to_jiffies(START_TIMEOUT_MS))) { + dev_err(iface->dchid->dev, "iface %s start timed out\n", iface->name); + return -ETIMEDOUT; + } + } + + iface->open = true; + return 0; +} + +static void dchid_close(struct hid_device *hdev) +{ + struct dchid_iface *iface = hdev->driver_data; + + iface->open = false; +} + +static int dchid_parse(struct hid_device *hdev) +{ + struct dchid_iface *iface = hdev->driver_data; + + return hid_parse_report(hdev, iface->hid_desc, iface->hid_desc_len); +} + +/* Note: buf excludes report number! For ease of fetching strings/etc. */ +static int dchid_get_report_cmd(struct dchid_iface *iface, u8 reportnum, void *buf, size_t len) +{ + int ret = dchid_cmd(iface, HID_FEATURE_REPORT, REQ_GET_REPORT, &reportnum, 1, buf, len); + + return ret <= 0 ? ret : ret - 1; +} + +/* Note: buf includes report number! */ +static int dchid_set_report(struct dchid_iface *iface, void *buf, size_t len) +{ + return dchid_cmd(iface, HID_OUTPUT_REPORT, REQ_SET_REPORT, buf, len, NULL, 0); +} + +static int dchid_raw_request(struct hid_device *hdev, + unsigned char reportnum, __u8 *buf, size_t len, + unsigned char rtype, int reqtype) +{ + struct dchid_iface *iface = hdev->driver_data; + + switch (reqtype) { + case HID_REQ_GET_REPORT: + buf[0] = reportnum; + return dchid_cmd(iface, rtype, REQ_GET_REPORT, &reportnum, 1, buf + 1, len - 1); + case HID_REQ_SET_REPORT: + return dchid_set_report(iface, buf, len); + default: + return -EIO; + } + + return 0; +} + +static struct hid_ll_driver dchid_ll = { + .start = &dchid_start, + .stop = &dchid_stop, + .open = &dchid_open, + .close = &dchid_close, + .parse = &dchid_parse, + .raw_request = &dchid_raw_request, +}; + +static void dchid_create_interface_work(struct work_struct *ws) +{ + struct dchid_iface *iface = container_of(ws, struct dchid_iface, create_work); + struct dockchannel_hid *dchid = iface->dchid; + struct hid_device *hid; + int ret; + + if (iface->hid) { + dev_warn(dchid->dev, "Interface %s already created!\n", + iface->name); + return; + } + + dev_info(dchid->dev, "New interface %s\n", iface->name); + + /* Start the interface. This is not the entire init process, as firmware is loaded later on device open. */ + ret = dchid_enable_interface(iface); + if (ret < 0) { + dev_warn(dchid->dev, "Failed to enable %s: %d\n", iface->name, ret); + return; + } + + iface->deferred = false; + + hid = hid_allocate_device(); + if (IS_ERR(hid)) + return; + + snprintf(hid->name, sizeof(hid->name), "Apple MTP %s", iface->name); + snprintf(hid->phys, sizeof(hid->phys), "%s.%d (%s)", + dev_name(dchid->dev), iface->index, iface->name); + strscpy(hid->uniq, dchid->serial, sizeof(hid->uniq)); + + hid->ll_driver = &dchid_ll; + hid->bus = BUS_HOST; + hid->vendor = dchid->device_id.vendor_id; + hid->product = dchid->device_id.product_id; + hid->version = dchid->device_id.version_number; + hid->type = HID_TYPE_OTHER; + if (!strcmp(iface->name, "multi-touch")) { + hid->type = HID_TYPE_SPI_MOUSE; + } else if (!strcmp(iface->name, "keyboard")) { + u32 country_code = 0; + + hid->type = HID_TYPE_SPI_KEYBOARD; + + /* + * We have to get the country code from the device tree, since the + * device provides no reliable way to get this info. + */ + if (!of_property_read_u32(iface->of_node, "hid-country-code", &country_code)) + hid->country = country_code; + + of_property_read_u32(iface->of_node, "apple,keyboard-layout-id", + &iface->keyboard_layout_id); + } + + hid->dev.parent = iface->dchid->dev; + hid->driver_data = iface; + + iface->hid = hid; + + ret = hid_add_device(hid); + if (ret < 0) { + iface->hid = NULL; + hid_destroy_device(hid); + dev_warn(iface->dchid->dev, "Failed to register hid device %s", iface->name); + } +} + +static int dchid_create_interface(struct dchid_iface *iface) +{ + if (iface->creating) + return -EBUSY; + + iface->creating = true; + INIT_WORK(&iface->create_work, dchid_create_interface_work); + return queue_work(iface->dchid->new_iface_wq, &iface->create_work); +} + +static void dchid_handle_descriptor(struct dchid_iface *iface, void *hid_desc, size_t desc_len) +{ + if (iface->hid) { + dev_warn(iface->dchid->dev, "Tried to initialize already started interface %s!\n", + iface->name); + return; + } + + iface->hid_desc = devm_kmemdup(iface->dchid->dev, hid_desc, desc_len, GFP_KERNEL); + if (!iface->hid_desc) + return; + + iface->hid_desc_len = desc_len; +} + +static void dchid_handle_ready(struct dockchannel_hid *dchid, void *data, size_t length) +{ + struct dchid_iface *iface; + u8 *pkt = data; + u8 index; + int i, ret; + + if (length < 2) { + dev_err(dchid->dev, "Bad length for ready message: %zu\n", length); + return; + } + + index = pkt[1]; + + if (index >= MAX_INTERFACES) { + dev_err(dchid->dev, "Got ready notification for bad iface %d\n", index); + return; + } + + iface = dchid->ifaces[index]; + if (!iface) { + dev_err(dchid->dev, "Got ready notification for unknown iface %d\n", index); + return; + } + + dev_info(dchid->dev, "Interface %s is now ready\n", iface->name); + complete_all(&iface->ready); + + /* When STM is ready, grab global device info */ + if (!strcmp(iface->name, "stm")) { + ret = dchid_get_report_cmd(iface, STM_REPORT_ID, &dchid->device_id, + sizeof(dchid->device_id)); + if (ret < sizeof(dchid->device_id)) { + dev_warn(iface->dchid->dev, "Failed to get device ID from STM!\n"); + /* Fake it and keep going. Things might still work... */ + memset(&dchid->device_id, 0, sizeof(dchid->device_id)); + dchid->device_id.vendor_id = HOST_VENDOR_ID_APPLE; + } + ret = dchid_get_report_cmd(iface, STM_REPORT_SERIAL, dchid->serial, + sizeof(dchid->serial) - 1); + if (ret < 0) { + dev_warn(iface->dchid->dev, "Failed to get serial from STM!\n"); + dchid->serial[0] = 0; + } + + dchid->id_ready = true; + for (i = 0; i < MAX_INTERFACES; i++) { + if (!dchid->ifaces[i] || !dchid->ifaces[i]->deferred) + continue; + dchid_create_interface(dchid->ifaces[i]); + } + } +} + +static void dchid_handle_init(struct dockchannel_hid *dchid, void *data, size_t length) +{ + struct dchid_init_hdr *hdr = data; + struct dchid_iface *iface; + struct dchid_init_block_hdr *blk; + + if (length < sizeof(*hdr)) + return; + + iface = dchid_get_interface(dchid, hdr->iface, hdr->name); + if (!iface) + return; + + data += sizeof(*hdr); + length -= sizeof(*hdr); + + while (length >= sizeof(*blk)) { + blk = data; + data += sizeof(*blk); + length -= sizeof(*blk); + + if (blk->length > length) + break; + + switch (blk->type) { + case INIT_HID_DESCRIPTOR: + dchid_handle_descriptor(iface, data, blk->length); + break; + + case INIT_GPIO_REQUEST: { + struct dchid_gpio_request *req = data; + + if (sizeof(*req) > length) + break; + + if (iface->gpio_id) { + dev_err(dchid->dev, + "Cannot request more than one GPIO per interface!\n"); + break; + } + + strscpy(iface->gpio_name, req->name, MAX_GPIO_NAME); + iface->gpio_id = req->id; + break; + } + + case INIT_TERMINATOR: + break; + + case INIT_PRODUCT_NAME: { + char *product = data; + + if (product[blk->length - 1] != 0) { + dev_warn(dchid->dev, "Unterminated product name for %s\n", + iface->name); + } else { + dev_info(dchid->dev, "Product name for %s: %s\n", + iface->name, product); + } + break; + } + + default: + dev_warn(dchid->dev, "Unknown init packet %d for %s\n", + blk->type, iface->name); + break; + } + + data += blk->length; + length -= blk->length; + + if (blk->type == INIT_TERMINATOR) + break; + } + + if (hdr->more_packets) + return; + + /* We need to enable STM first, since it'll give us the device IDs */ + if (iface->dchid->id_ready || !strcmp(iface->name, "stm")) { + dchid_create_interface(iface); + } else { + iface->deferred = true; + } +} + +static void dchid_handle_gpio(struct dockchannel_hid *dchid, void *data, size_t length) +{ + struct dchid_gpio_cmd *cmd = data; + struct dchid_iface *iface; + u32 retcode = 0xe000f00d; /* Give it a random Apple-style error code */ + struct dchid_gpio_ack *ack; + + if (length < sizeof(*cmd)) + return; + + if (cmd->iface >= MAX_INTERFACES || !(iface = dchid->ifaces[cmd->iface])) { + dev_err(dchid->dev, "Got GPIO command for bad inteface %d\n", cmd->iface); + goto err; + } + + if (dchid_request_gpio(iface) < 0) + goto err; + + if (!iface->gpio || cmd->gpio != iface->gpio_id) { + dev_err(dchid->dev, "Got GPIO command for bad GPIO %s#%d\n", + iface->name, cmd->gpio); + goto err; + } + + dev_info(dchid->dev, "GPIO command: %s#%d: %d\n", iface->name, cmd->gpio, cmd->cmd); + + switch (cmd->cmd) { + case 3: + /* Pulse. */ + gpiod_set_value_cansleep(iface->gpio, 1); + msleep(10); /* Random guess... */ + gpiod_set_value_cansleep(iface->gpio, 0); + retcode = 0; + break; + default: + dev_err(dchid->dev, "Unknown GPIO command %d\n", cmd->cmd ); + break; + } + +err: + /* Ack it */ + ack = kzalloc(sizeof(*ack) + length, GFP_KERNEL); + if (!ack) + return; + + ack->type = CMD_ACK_GPIO_CMD; + ack->retcode = retcode; + memcpy(ack->cmd, data, length); + + if (dchid_comm_cmd(dchid, ack, sizeof(*ack) + length) < 0) + dev_err(dchid->dev, "Failed to ACK GPIO command\n"); + + kfree(ack); +} + +static void dchid_handle_event(struct dockchannel_hid *dchid, void *data, size_t length) +{ + u8 *p = data; + switch (*p) { + case EVENT_INIT: + dchid_handle_init(dchid, data, length); + break; + case EVENT_READY: + dchid_handle_ready(dchid, data, length); + break; + case EVENT_GPIO_CMD: + dchid_handle_gpio(dchid, data, length); + break; + } +} + +static void dchid_handle_report(struct dchid_iface *iface, void *data, size_t length) +{ + struct dockchannel_hid *dchid = iface->dchid; + + if (!iface->hid) { + dev_warn(dchid->dev, "Report received but %s is not initialized!\n", iface->name); + return; + } + + if (!iface->open) + return; + + hid_input_report(iface->hid, HID_INPUT_REPORT, data, length, 1); +} + +static void dchid_packet_work(struct work_struct *ws) +{ + struct dchid_work *work = container_of(ws, struct dchid_work, work); + struct dchid_subhdr *shdr = (void *)work->data; + struct dockchannel_hid *dchid = work->iface->dchid; + int type = FIELD_GET(FLAGS_GROUP, shdr->flags); + u8 *payload = work->data + sizeof(*shdr); + + if (shdr->length + sizeof(*shdr) > work->hdr.length) { + dev_err(dchid->dev, "Bad sub header length (%d > %zu)\n", + shdr->length, work->hdr.length - sizeof(*shdr)); + return; + } + + switch (type) { + case HID_INPUT_REPORT: + if (work->hdr.iface == IFACE_COMM) + dchid_handle_event(dchid, payload, shdr->length); + else + dchid_handle_report(work->iface, payload, shdr->length); + break; + default: + dev_err(dchid->dev, "Received unknown packet type %d\n", type); + break; + } + + kfree(work); +} + +static void dchid_handle_ack(struct dchid_iface *iface, struct dchid_hdr *hdr, void *data) +{ + struct dchid_subhdr *shdr = (void *)data; + u8 *payload = data + sizeof(*shdr); + + if (shdr->length + sizeof(*shdr) > hdr->length) { + dev_err(iface->dchid->dev, "Bad sub header length (%d > %ld)\n", + shdr->length, hdr->length - sizeof(*shdr)); + return; + } + if (shdr->flags != iface->out_flags) { + dev_err(iface->dchid->dev, + "Received unexpected flags 0x%x on ACK channel (expFected 0x%x)\n", + shdr->flags, iface->out_flags); + return; + } + + if (shdr->length < 1) { + dev_err(iface->dchid->dev, "Received length 0 output report ack\n"); + return; + } + if (iface->tx_seq != hdr->seq) { + dev_err(iface->dchid->dev, "Received ACK with bad seq (expected %d, got %d)\n", + iface->tx_seq, hdr->seq); + return; + } + if (iface->out_report != payload[0]) { + dev_err(iface->dchid->dev, "Received ACK with bad report (expected %d, got %d\n", + iface->out_report, payload[0]); + return; + } + + if (iface->resp_buf && iface->resp_size) + memcpy(iface->resp_buf, payload + 1, min((size_t)shdr->length - 1, iface->resp_size)); + + iface->resp_size = shdr->length; + iface->out_report = -1; + iface->retcode = shdr->retcode; + complete(&iface->out_complete); +} + +static void dchid_handle_packet(void *cookie, size_t avail) +{ + struct dockchannel_hid *dchid = cookie; + struct dchid_hdr hdr; + struct dchid_work *work; + struct dchid_iface *iface; + u32 checksum; + + if (dockchannel_recv(dchid->dc, &hdr, sizeof(hdr)) != sizeof(hdr)) { + dev_err(dchid->dev, "Read failed (header)\n"); + return; + } + + if (hdr.hdr_len != sizeof(hdr)) { + dev_err(dchid->dev, "Bad header length %d\n", hdr.hdr_len); + goto done; + } + + if (dockchannel_recv(dchid->dc, dchid->pkt_buf, hdr.length + 4) != (hdr.length + 4)) { + dev_err(dchid->dev, "Read failed (body)\n"); + goto done; + } + + checksum = dchid_checksum(&hdr, sizeof(hdr)); + checksum += dchid_checksum(dchid->pkt_buf, hdr.length + 4); + + if (checksum != 0xffffffff) { + dev_err(dchid->dev, "Checksum mismatch (iface %d): 0x%08x != 0xffffffff\n", + hdr.iface, checksum); + goto done; + } + + + if (hdr.iface >= MAX_INTERFACES) { + dev_err(dchid->dev, "Bad iface %d\n", hdr.iface); + } + + iface = dchid->ifaces[hdr.iface]; + + if (!iface) { + dev_err(dchid->dev, "Received packet for uninitialized iface %d\n", hdr.iface); + goto done; + } + + switch (hdr.channel) { + case DCHID_CHANNEL_CMD: + dchid_handle_ack(iface, &hdr, dchid->pkt_buf); + goto done; + case DCHID_CHANNEL_REPORT: + break; + default: + dev_warn(dchid->dev, "Unknown channel 0x%x, treating as report...\n", + hdr.channel); + break; + } + + work = kzalloc(sizeof(*work) + hdr.length, GFP_KERNEL); + if (!work) + return; + + work->hdr = hdr; + work->iface = iface; + memcpy(work->data, dchid->pkt_buf, hdr.length); + INIT_WORK(&work->work, dchid_packet_work); + + queue_work(iface->wq, &work->work); + +done: + dockchannel_await(dchid->dc, dchid_handle_packet, dchid, sizeof(struct dchid_hdr)); +} + +static int dockchannel_hid_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct dockchannel_hid *dchid; + struct device_node *child, *helper; + struct platform_device *helper_pdev; + struct property *prop; + int ret; + + ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); + if (ret) + return ret; + + dchid = devm_kzalloc(dev, sizeof(*dchid), GFP_KERNEL); + if (!dchid) { + return -ENOMEM; + } + + dchid->dev = dev; + + /* + * First make sure all the GPIOs are available, in cased we need to defer. + * This is necessary because MTP will request them by name later, and by then + * it's too late to defer the probe. + */ + + for_each_child_of_node(dev->of_node, child) { + for_each_property_of_node(child, prop) { + size_t len = strlen(prop->name); + struct gpio_desc *gpio; + + if (len < 12 || strncmp("apple,", prop->name, 6) || + strcmp("-gpios", prop->name + len - 6)) + continue; + + gpio = fwnode_gpiod_get_index(&child->fwnode, prop->name, 0, GPIOD_ASIS, + prop->name); + if (IS_ERR_OR_NULL(gpio)) { + if (PTR_ERR(gpio) == -EPROBE_DEFER) { + of_node_put(child); + return -EPROBE_DEFER; + } + } else { + gpiod_put(gpio); + } + } + } + + /* + * Make sure we also have the MTP coprocessor available, and + * defer probe if the helper hasn't probed yet. + */ + helper = of_parse_phandle(dev->of_node, "apple,helper-cpu", 0); + if (!helper) { + dev_err(dev, "Missing apple,helper-cpu property"); + return -EINVAL; + } + + helper_pdev = of_find_device_by_node(helper); + of_node_put(helper); + if (!helper_pdev) { + dev_err(dev, "Failed to find helper device"); + return -EINVAL; + } + + dchid->helper_link = device_link_add(dev, &helper_pdev->dev, + DL_FLAG_AUTOREMOVE_CONSUMER); + put_device(&helper_pdev->dev); + if (!dchid->helper_link) { + dev_err(dev, "Failed to link to helper device"); + return -EINVAL; + } + + if (dchid->helper_link->supplier->links.status != DL_DEV_DRIVER_BOUND) + return -EPROBE_DEFER; + + /* Now it is safe to begin initializing */ + dchid->dc = dockchannel_init(pdev); + if (IS_ERR_OR_NULL(dchid->dc)) { + return PTR_ERR(dchid->dc); + } + dchid->new_iface_wq = alloc_workqueue("dchid-new", WQ_MEM_RECLAIM, 0); + if (!dchid->new_iface_wq) + return -ENOMEM; + + dchid->comm = dchid_get_interface(dchid, IFACE_COMM, "comm"); + if (!dchid->comm) { + dev_err(dchid->dev, "Failed to initialize comm interface"); + return -EIO; + } + + dev_info(dchid->dev, "Initialized, awaiting packets\n"); + dockchannel_await(dchid->dc, dchid_handle_packet, dchid, sizeof(struct dchid_hdr)); + + return 0; +} + +static void dockchannel_hid_remove(struct platform_device *pdev) +{ + BUG_ON(1); +} + +static const struct of_device_id dockchannel_hid_of_match[] = { + { .compatible = "apple,dockchannel-hid" }, + {}, +}; +MODULE_DEVICE_TABLE(of, dockchannel_hid_of_match); +MODULE_FIRMWARE("apple/tpmtfw-*.bin"); + +static struct platform_driver dockchannel_hid_driver = { + .driver = { + .name = "dockchannel-hid", + .of_match_table = dockchannel_hid_of_match, + }, + .probe = dockchannel_hid_probe, + .remove = dockchannel_hid_remove, +}; +module_platform_driver(dockchannel_hid_driver); + +MODULE_DESCRIPTION("Apple DockChannel HID transport driver"); +MODULE_AUTHOR("Hector Martin "); +MODULE_LICENSE("Dual MIT/GPL"); diff --git a/drivers/hid/hid-alps.c b/drivers/hid/hid-alps.c index ba00f6e6324b09..8c3f83532ce926 100644 --- a/drivers/hid/hid-alps.c +++ b/drivers/hid/hid-alps.c @@ -437,6 +437,9 @@ static int alps_raw_event(struct hid_device *hdev, int ret = 0; struct alps_dev *hdata = hid_get_drvdata(hdev); + if (!(hdev->claimed & HID_CLAIMED_INPUT) || !hdata->input) + return 0; + switch (hdev->product) { case HID_PRODUCT_ID_T4_BTNLESS: ret = t4_raw_event(hdata, data, size); diff --git a/drivers/hid/hid-apple.c b/drivers/hid/hid-apple.c index 57da4f86a9fa7f..a5ca6a44f33aec 100644 --- a/drivers/hid/hid-apple.c +++ b/drivers/hid/hid-apple.c @@ -54,10 +54,16 @@ #define APPLE_MAGIC_REPORT_ID_POWER 3 #define APPLE_MAGIC_REPORT_ID_BRIGHTNESS 1 +// DO NOT UPSTREAM: +// temporary Fn key mode until xkeyboard-config has keyboard layouts with media +// key mappings. At that point auto mode can drop function key mappings and this +// mode can be dropped. +#define FKEYS_IGNORE 5 + static unsigned int fnmode = 3; module_param(fnmode, uint, 0644); MODULE_PARM_DESC(fnmode, "Mode of fn key on Apple keyboards (0 = disabled, " - "1 = fkeyslast, 2 = fkeysfirst, [3] = auto, 4 = fkeysdisabled)"); + "1 = fkeyslast, 2 = fkeysfirst, [3] = auto, 4 = fkeysdisabled, 5 = fkeysignore))"); static int iso_layout = -1; module_param(iso_layout, int, 0644); @@ -277,6 +283,16 @@ static const struct apple_key_translation apple_fn_keys[] = { { } }; +static const struct apple_key_translation apple_fn_keys_minimal[] = { + { KEY_BACKSPACE, KEY_DELETE }, + { KEY_ENTER, KEY_INSERT }, + { KEY_UP, KEY_PAGEUP }, + { KEY_DOWN, KEY_PAGEDOWN }, + { KEY_LEFT, KEY_HOME }, + { KEY_RIGHT, KEY_END }, + { } +}; + static const struct apple_key_translation powerbook_fn_keys[] = { { KEY_BACKSPACE, KEY_DELETE }, { KEY_F1, KEY_BRIGHTNESSDOWN, APPLE_FLAG_FKEY }, @@ -354,6 +370,7 @@ static const struct apple_key_translation swapped_fn_leftctrl_keys[] = { }; static const struct apple_non_apple_keyboard non_apple_keyboards[] = { + { "SONiX KN85 Keyboard" }, { "SONiX USB DEVICE" }, { "SONiX AK870 PRO" }, { "Keychron" }, @@ -364,6 +381,9 @@ static const struct apple_non_apple_keyboard non_apple_keyboards[] = { { "A3R" }, { "hfd.cn" }, { "WKB603" }, + { "TH87" }, /* EPOMAKER TH87 BT mode */ + { "HFD Epomaker TH87" }, /* EPOMAKER TH87 USB mode */ + { "2.4G Wireless Receiver" }, /* EPOMAKER TH87 dongle */ }; static bool apple_is_non_apple_keyboard(struct hid_device *hdev) @@ -433,6 +453,8 @@ static int hidinput_apple_event(struct hid_device *hid, struct input_dev *input, real_fnmode = 2; else real_fnmode = 1; + } else if (fnmode == FKEYS_IGNORE) { + real_fnmode = 2; } else { real_fnmode = fnmode; } @@ -473,6 +495,23 @@ static int hidinput_apple_event(struct hid_device *hid, struct input_dev *input, asc->fn_on = !!value; if (real_fnmode) { + switch (hid->bus) { + case BUS_HOST: + case BUS_SPI: + switch (hid->product) { + case SPI_DEVICE_ID_APPLE_MACBOOK_PRO13_2020: + case HOST_DEVICE_ID_APPLE_MACBOOK_PRO13_2022: + table = macbookpro_dedicated_esc_fn_keys; + break; + default: + if (fnmode == FKEYS_IGNORE) + table = apple_fn_keys_minimal; + else + table = magic_keyboard_2021_and_2024_fn_keys; + break; + } + break; + default: switch (hid->product) { case USB_DEVICE_ID_APPLE_ALU_WIRELESS_ANSI: case USB_DEVICE_ID_APPLE_ALU_WIRELESS_ISO: @@ -521,6 +560,7 @@ static int hidinput_apple_event(struct hid_device *hid, struct input_dev *input, else table = apple_fn_keys; } + } trans = apple_find_translation(table, code); @@ -685,9 +725,7 @@ static const __u8 *apple_report_fixup(struct hid_device *hdev, __u8 *rdesc, hid_info(hdev, "fixing up Magic Keyboard battery report descriptor\n"); *rsize = *rsize - 1; - rdesc = kmemdup(rdesc + 1, *rsize, GFP_KERNEL); - if (!rdesc) - return NULL; + rdesc = rdesc + 1; rdesc[0] = 0x05; rdesc[1] = 0x01; @@ -704,6 +742,7 @@ static void apple_setup_input(struct input_dev *input) /* Enable all needed keys */ apple_setup_key_translation(input, apple_fn_keys); + apple_setup_key_translation(input, apple_fn_keys_minimal); apple_setup_key_translation(input, powerbook_fn_keys); apple_setup_key_translation(input, powerbook_numlock_keys); apple_setup_key_translation(input, apple_iso_keyboard); @@ -938,6 +977,15 @@ static int apple_probe(struct hid_device *hdev, struct apple_sc *asc; int ret; + if ((id->bus == BUS_SPI || id->bus == BUS_HOST) && id->vendor == SPI_VENDOR_ID_APPLE && + hdev->type != HID_TYPE_SPI_KEYBOARD) + return -ENODEV; + + // key remapping will happen in xkeyboard-config so ignore + // APPLE_ISO_TILDE_QUIRK + if ((id->bus == BUS_SPI || id->bus == BUS_HOST) && fnmode == FKEYS_IGNORE) + quirks &= ~APPLE_ISO_TILDE_QUIRK; + asc = devm_kzalloc(&hdev->dev, sizeof(*asc), GFP_KERNEL); if (asc == NULL) { hid_err(hdev, "can't alloc apple descriptor\n"); @@ -1192,6 +1240,8 @@ static const struct hid_device_id apple_devices[] = { .driver_data = APPLE_NUMLOCK_EMULATION | APPLE_HAS_FN }, { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_GEYSER1_TP_ONLY), .driver_data = APPLE_NUMLOCK_EMULATION | APPLE_HAS_FN }, + { HID_SPI_DEVICE(SPI_VENDOR_ID_APPLE, HID_ANY_ID), + .driver_data = APPLE_HAS_FN | APPLE_ISO_TILDE_QUIRK }, // TODO: remove APPLE_ISO_TILDE_QUIRK { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_MAGIC_KEYBOARD_2021), .driver_data = APPLE_HAS_FN | APPLE_ISO_TILDE_QUIRK | APPLE_RDESC_BATTERY }, { HID_BLUETOOTH_DEVICE(BT_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_MAGIC_KEYBOARD_2021), @@ -1204,6 +1254,8 @@ static const struct hid_device_id apple_devices[] = { .driver_data = APPLE_HAS_FN | APPLE_ISO_TILDE_QUIRK | APPLE_RDESC_BATTERY }, { HID_BLUETOOTH_DEVICE(BT_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_MAGIC_KEYBOARD_NUMPAD_2021), .driver_data = APPLE_HAS_FN | APPLE_ISO_TILDE_QUIRK }, + { HID_DEVICE(BUS_HOST, HID_GROUP_ANY, HOST_VENDOR_ID_APPLE, HID_ANY_ID), + .driver_data = APPLE_HAS_FN | APPLE_ISO_TILDE_QUIRK }, // TODO: remove APPLE_ISO_TILDE_QUIRK { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_MAGIC_KEYBOARD_2024), .driver_data = APPLE_HAS_FN | APPLE_ISO_TILDE_QUIRK | APPLE_RDESC_BATTERY }, { HID_BLUETOOTH_DEVICE(BT_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_MAGIC_KEYBOARD_2024), diff --git a/drivers/hid/hid-appletb-kbd.c b/drivers/hid/hid-appletb-kbd.c index b00687e67ce8e5..0b10cff465e179 100644 --- a/drivers/hid/hid-appletb-kbd.c +++ b/drivers/hid/hid-appletb-kbd.c @@ -477,7 +477,7 @@ static int appletb_kbd_suspend(struct hid_device *hdev, pm_message_t msg) return 0; } -static int appletb_kbd_reset_resume(struct hid_device *hdev) +static int appletb_kbd_resume(struct hid_device *hdev) { struct appletb_kbd *kbd = hid_get_drvdata(hdev); @@ -503,7 +503,8 @@ static struct hid_driver appletb_kbd_hid_driver = { .input_configured = appletb_kbd_input_configured, #ifdef CONFIG_PM .suspend = appletb_kbd_suspend, - .reset_resume = appletb_kbd_reset_resume, + .resume = appletb_kbd_resume, + .reset_resume = appletb_kbd_resume, #endif .driver.dev_groups = appletb_kbd_groups, }; diff --git a/drivers/hid/hid-asus.c b/drivers/hid/hid-asus.c index 472bca54642b90..b1ad4e9f20c85a 100644 --- a/drivers/hid/hid-asus.c +++ b/drivers/hid/hid-asus.c @@ -1306,14 +1306,21 @@ static const __u8 *asus_report_fixup(struct hid_device *hdev, __u8 *rdesc, */ if (*rsize == rsize_orig && rdesc[offs] == 0x09 && rdesc[offs + 1] == 0x76) { - *rsize = rsize_orig + 1; - rdesc = kmemdup(rdesc, *rsize, GFP_KERNEL); - if (!rdesc) - return NULL; + __u8 *new_rdesc; + + new_rdesc = devm_kzalloc(&hdev->dev, rsize_orig + 1, + GFP_KERNEL); + if (!new_rdesc) + return rdesc; hid_info(hdev, "Fixing up %s keyb report descriptor\n", drvdata->quirks & QUIRK_T100CHI ? "T100CHI" : "T90CHI"); + + memcpy(new_rdesc, rdesc, rsize_orig); + *rsize = rsize_orig + 1; + rdesc = new_rdesc; + memmove(rdesc + offs + 4, rdesc + offs + 2, 12); rdesc[offs] = 0x19; rdesc[offs + 1] = 0x00; @@ -1397,6 +1404,9 @@ static const struct hid_device_id asus_devices[] = { { HID_USB_DEVICE(USB_VENDOR_ID_ASUSTEK, USB_DEVICE_ID_ASUSTEK_ROG_NKEY_ALLY_X), QUIRK_USE_KBD_BACKLIGHT | QUIRK_ROG_NKEY_KEYBOARD | QUIRK_ROG_ALLY_XPAD }, + { HID_USB_DEVICE(USB_VENDOR_ID_ASUSTEK, + USB_DEVICE_ID_ASUSTEK_XGM_2023), + }, { HID_USB_DEVICE(USB_VENDOR_ID_ASUSTEK, USB_DEVICE_ID_ASUSTEK_ROG_CLAYMORE_II_KEYBOARD), QUIRK_ROG_CLAYMORE_II_KEYBOARD }, diff --git a/drivers/hid/hid-cmedia.c b/drivers/hid/hid-cmedia.c index 528d7f36121576..8bf5649b0c793e 100644 --- a/drivers/hid/hid-cmedia.c +++ b/drivers/hid/hid-cmedia.c @@ -99,7 +99,7 @@ static int cmhid_raw_event(struct hid_device *hid, struct hid_report *report, { struct cmhid *cm = hid_get_drvdata(hid); - if (len != CM6533_JD_RAWEV_LEN) + if (len != CM6533_JD_RAWEV_LEN || !(hid->claimed & HID_CLAIMED_INPUT)) goto out; if (memcmp(data+CM6533_JD_SFX_OFFSET, ji_sfx, sizeof(ji_sfx))) goto out; diff --git a/drivers/hid/hid-core.c b/drivers/hid/hid-core.c index a5b3a8ca2fcbc8..81358dc2c83dde 100644 --- a/drivers/hid/hid-core.c +++ b/drivers/hid/hid-core.c @@ -71,6 +71,9 @@ static u32 s32ton(__s32 value, unsigned int n) if (!value || !n) return 0; + if (n > 32) + n = 32; + a = value >> (n - 1); if (a && a != -1) return value < 0 ? 1 << (n - 1) : (1 << (n - 1)) - 1; @@ -468,7 +471,10 @@ static int hid_parser_global(struct hid_parser *parser, struct hid_item *item) case HID_GLOBAL_ITEM_TAG_REPORT_SIZE: parser->global.report_size = item_udata(item); - if (parser->global.report_size > 256) { + /* Arbitrary maximum. Some Apple devices have 16384 here. + * This * HID_MAX_USAGES must fit in a signed integer. + */ + if (parser->global.report_size > 16384) { hid_err(parser->device, "invalid report_size %d\n", parser->global.report_size); return -1; @@ -2057,9 +2063,10 @@ int hid_report_raw_event(struct hid_device *hid, enum hid_report_type type, u8 * rsize = max_buffer_size; if (csize < rsize) { - dbg_hid("report %d is too short, (%d < %d)\n", report->id, - csize, rsize); - memset(cdata + csize, 0, rsize - csize); + hid_warn_ratelimited(hid, "Event data for report %d was too short (%d vs %d)\n", + report->id, rsize, csize); + ret = -EINVAL; + goto out; } if ((hid->claimed & HID_CLAIMED_HIDDEV) && hid->hiddev_report_event) @@ -2319,6 +2326,12 @@ int hid_connect(struct hid_device *hdev, unsigned int connect_mask) case BUS_SDW: bus = "SOUNDWIRE"; break; + case BUS_SPI: + bus = "SPI"; + break; + case BUS_HOST: + bus = "HOST"; + break; case BUS_VIRTUAL: bus = "VIRTUAL"; break; diff --git a/drivers/hid/hid-creative-sb0540.c b/drivers/hid/hid-creative-sb0540.c index b4c8e7a5d3e025..dfd6add353d19a 100644 --- a/drivers/hid/hid-creative-sb0540.c +++ b/drivers/hid/hid-creative-sb0540.c @@ -153,7 +153,7 @@ static int creative_sb0540_raw_event(struct hid_device *hid, u64 code, main_code; int key; - if (len != 6) + if (len != 6 || !(hid->claimed & HID_CLAIMED_INPUT)) return 0; /* From daemons/hw_hiddev.c sb0540_rec() in lirc */ diff --git a/drivers/hid/hid-elecom.c b/drivers/hid/hid-elecom.c index 2003d2dcda7cc7..37d88ce57f6718 100644 --- a/drivers/hid/hid-elecom.c +++ b/drivers/hid/hid-elecom.c @@ -5,6 +5,7 @@ * - EX-G Trackballs (M-XT3DRBK, M-XT3URBK, M-XT4DRBK) * - DEFT Trackballs (M-DT1DRBK, M-DT1URBK, M-DT2DRBK, M-DT2URBK) * - HUGE Trackballs (M-HT1DRBK, M-HT1URBK) + * - HUGE Plus Trackball (M-HT1MRBK) * * Copyright (c) 2010 Richard Nauber * Copyright (c) 2016 Yuxuan Shui @@ -123,12 +124,25 @@ static const __u8 *elecom_report_fixup(struct hid_device *hdev, __u8 *rdesc, */ mouse_button_fixup(hdev, rdesc, *rsize, 22, 30, 24, 16, 8); break; + case USB_DEVICE_ID_ELECOM_M_HT1MRBK: + case USB_DEVICE_ID_ELECOM_M_HT1MRBK_01AB: + case USB_DEVICE_ID_ELECOM_M_HT1MRBK_01AC: + /* + * Report descriptor format: + * 24: button bit count + * 28: padding bit count + * 22: button report size + * 16: button usage maximum + */ + mouse_button_fixup(hdev, rdesc, *rsize, 24, 28, 22, 16, 8); + break; } return rdesc; } static const struct hid_device_id elecom_devices[] = { { HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_BM084) }, + { HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_HT1MRBK_01AC) }, { HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_XGL20DLBK) }, { HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_XT3URBK_00FB) }, { HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_XT3URBK_018F) }, @@ -142,6 +156,8 @@ static const struct hid_device_id elecom_devices[] = { { HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_HT1URBK_019B) }, { HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_HT1DRBK_010D) }, { HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_HT1DRBK_011C) }, + { HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_HT1MRBK) }, + { HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_HT1MRBK_01AB) }, { } }; MODULE_DEVICE_TABLE(hid, elecom_devices); diff --git a/drivers/hid/hid-ids.h b/drivers/hid/hid-ids.h index 9c2bf584d9f6f2..c12883c54affd9 100644 --- a/drivers/hid/hid-ids.h +++ b/drivers/hid/hid-ids.h @@ -22,6 +22,9 @@ #define USB_DEVICE_ID_3M2256 0x0502 #define USB_DEVICE_ID_3M3266 0x0506 +#define USB_VENDOR_ID_8BITDO 0x2dc8 +#define USB_DEVICE_ID_8BITDO_PRO_3 0x6009 + #define USB_VENDOR_ID_A4TECH 0x09da #define USB_DEVICE_ID_A4TECH_WCP32PU 0x0006 #define USB_DEVICE_ID_A4TECH_X5_005D 0x000a @@ -93,6 +96,8 @@ #define USB_VENDOR_ID_APPLE 0x05ac #define BT_VENDOR_ID_APPLE 0x004c +#define SPI_VENDOR_ID_APPLE 0x05ac +#define HOST_VENDOR_ID_APPLE 0x05ac #define USB_DEVICE_ID_APPLE_MIGHTYMOUSE 0x0304 #define USB_DEVICE_ID_APPLE_MAGICMOUSE 0x030d #define USB_DEVICE_ID_APPLE_MAGICMOUSE2 0x0269 @@ -197,6 +202,14 @@ #define USB_DEVICE_ID_APPLE_IRCONTROL5 0x8243 #define USB_DEVICE_ID_APPLE_TOUCHBAR_BACKLIGHT 0x8102 #define USB_DEVICE_ID_APPLE_TOUCHBAR_DISPLAY 0x8302 +#define SPI_DEVICE_ID_APPLE_MACBOOK_AIR_2020 0x0281 +#define SPI_DEVICE_ID_APPLE_MACBOOK_PRO13_2020 0x0341 +#define SPI_DEVICE_ID_APPLE_MACBOOK_PRO14_2021 0x0342 +#define SPI_DEVICE_ID_APPLE_MACBOOK_PRO16_2021 0x0343 +#define HOST_DEVICE_ID_APPLE_MACBOOK_AIR13_2022 0x0351 +#define HOST_DEVICE_ID_APPLE_MACBOOK_PRO14_2023 0x0352 +#define HOST_DEVICE_ID_APPLE_MACBOOK_PRO16_2023 0x0353 +#define HOST_DEVICE_ID_APPLE_MACBOOK_PRO13_2022 0x0354 #define USB_VENDOR_ID_ASETEK 0x2433 #define USB_DEVICE_ID_ASETEK_INVICTA 0xf300 @@ -229,6 +242,7 @@ #define USB_DEVICE_ID_ASUSTEK_ROG_NKEY_ALLY_X 0x1b4c #define USB_DEVICE_ID_ASUSTEK_ROG_CLAYMORE_II_KEYBOARD 0x196b #define USB_DEVICE_ID_ASUSTEK_FX503VD_KEYBOARD 0x1869 +#define USB_DEVICE_ID_ASUSTEK_XGM_2023 0x1a9a #define USB_VENDOR_ID_ATEN 0x0557 #define USB_DEVICE_ID_ATEN_UC100KM 0x2004 @@ -437,6 +451,7 @@ #define USB_DEVICE_ID_DWAV_EGALAX_MULTITOUCH_7349 0x7349 #define USB_DEVICE_ID_DWAV_EGALAX_MULTITOUCH_73F7 0x73f7 #define USB_DEVICE_ID_DWAV_EGALAX_MULTITOUCH_A001 0xa001 +#define USB_DEVICE_ID_DWAV_EGALAX_MULTITOUCH_C000 0xc000 #define USB_DEVICE_ID_DWAV_EGALAX_MULTITOUCH_C002 0xc002 #define USB_VENDOR_ID_EDIFIER 0x2d99 @@ -465,6 +480,9 @@ #define USB_DEVICE_ID_ELECOM_M_HT1URBK_019B 0x019b #define USB_DEVICE_ID_ELECOM_M_HT1DRBK_010D 0x010d #define USB_DEVICE_ID_ELECOM_M_HT1DRBK_011C 0x011c +#define USB_DEVICE_ID_ELECOM_M_HT1MRBK 0x01aa +#define USB_DEVICE_ID_ELECOM_M_HT1MRBK_01AB 0x01ab +#define USB_DEVICE_ID_ELECOM_M_HT1MRBK_01AC 0x01ac #define USB_VENDOR_ID_DREAM_CHEEKY 0x1d34 #define USB_DEVICE_ID_DREAM_CHEEKY_WN 0x0004 @@ -841,6 +859,7 @@ #define USB_DEVICE_ID_LENOVO_X1_TAB3 0x60b5 #define USB_DEVICE_ID_LENOVO_X12_TAB 0x60fe #define USB_DEVICE_ID_LENOVO_X12_TAB2 0x61ae +#define USB_DEVICE_ID_LENOVO_YOGABOOK9I 0x6161 #define USB_DEVICE_ID_LENOVO_OPTICAL_USB_MOUSE_600E 0x600e #define USB_DEVICE_ID_LENOVO_PIXART_USB_MOUSE_608D 0x608d #define USB_DEVICE_ID_LENOVO_PIXART_USB_MOUSE_6019 0x6019 diff --git a/drivers/hid/hid-logitech-hidpp.c b/drivers/hid/hid-logitech-hidpp.c index e871f1729d4b30..65bfad405ac5b4 100644 --- a/drivers/hid/hid-logitech-hidpp.c +++ b/drivers/hid/hid-logitech-hidpp.c @@ -4314,7 +4314,7 @@ static int hidpp_get_report_length(struct hid_device *hdev, int id) re = &(hdev->report_enum[HID_OUTPUT_REPORT]); report = re->report_id_hash[id]; - if (!report) + if (!report || !report->maxfield) return 0; return report->field[0]->report_count + 1; @@ -4487,10 +4487,12 @@ static int hidpp_probe(struct hid_device *hdev, const struct hid_device_id *id) if (!ret) ret = hidpp_ff_init(hidpp, &data); - if (ret) + if (ret) { hid_warn(hidpp->hid_dev, "Unable to initialize force feedback support, errno %d\n", ret); + ret = 0; + } } /* @@ -4666,6 +4668,10 @@ static const struct hid_device_id hidpp_devices[] = { HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_LOGITECH, 0xb037) }, { /* MX Anywhere 3SB mouse over Bluetooth */ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_LOGITECH, 0xb038) }, + { /* Slim Solar+ K980 Keyboard over Bluetooth */ + HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_LOGITECH, 0xb391) }, + { /* MX Master 4 mouse over Bluetooth */ + HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_LOGITECH, 0xb042) }, {} }; diff --git a/drivers/hid/hid-magicmouse.c b/drivers/hid/hid-magicmouse.c index 7d4a25c6de0eb7..82b2a0954a2040 100644 --- a/drivers/hid/hid-magicmouse.c +++ b/drivers/hid/hid-magicmouse.c @@ -60,8 +60,14 @@ MODULE_PARM_DESC(report_undeciphered, "Report undeciphered multi-touch state fie #define MOUSE_REPORT_ID 0x29 #define MOUSE2_REPORT_ID 0x12 #define DOUBLE_REPORT_ID 0xf7 +#define SPI_REPORT_ID 0x02 +#define SPI_RESET_REPORT_ID 0x60 +#define MTP_REPORT_ID 0x75 +#define SENSOR_DIMENSIONS_REPORT_ID 0xd9 #define USB_BATTERY_TIMEOUT_SEC 60 +#define MAX_CONTACTS 16 + /* These definitions are not precise, but they're close enough. (Bits * 0x03 seem to indicate the aspect ratio of the touch, bits 0x70 seem * to be some kind of bit mask -- 0x20 may be a near-field reading, @@ -112,30 +118,53 @@ MODULE_PARM_DESC(report_undeciphered, "Report undeciphered multi-touch state fie #define TRACKPAD2_RES_Y \ ((TRACKPAD2_MAX_Y - TRACKPAD2_MIN_Y) / (TRACKPAD2_DIMENSION_Y / 100)) +/* These are fallback values, since the real values will be queried from the device. */ +#define J314_TP_DIMENSION_X (float)13000 +#define J314_TP_MIN_X -5900 +#define J314_TP_MAX_X 6500 +#define J314_TP_RES_X \ + ((J314_TP_MAX_X - J314_TP_MIN_X) / (J314_TP_DIMENSION_X / 100)) +#define J314_TP_DIMENSION_Y (float)8100 +#define J314_TP_MIN_Y -200 +#define J314_TP_MAX_Y 7400 +#define J314_TP_RES_Y \ + ((J314_TP_MAX_Y - J314_TP_MIN_Y) / (J314_TP_DIMENSION_Y / 100)) + +#define J314_TP_MAX_FINGER_ORIENTATION 16384 + +struct magicmouse_input_ops { + int (*raw_event)(struct hid_device *hdev, + struct hid_report *report, u8 *data, int size); + int (*setup_input)(struct input_dev *input, struct hid_device *hdev); +}; + /** * struct magicmouse_sc - Tracks Magic Mouse-specific data. * @input: Input device through which we report events. * @quirks: Currently unused. + * @query_dimensions: Whether to query and update dimensions on first open * @ntouches: Number of touches in most recent touch report. * @scroll_accel: Number of consecutive scroll motions. * @scroll_jiffies: Time of last scroll motion. + * @pos: multi touch position data of the last report. * @touches: Most recent data for a touch, indexed by tracking ID. * @tracking_ids: Mapping of current touch input data to @touches. * @hdev: Pointer to the underlying HID device. * @work: Workqueue to handle initialization retry for quirky devices. * @battery_timer: Timer for obtaining battery level information. + * @input_ops: Input ops based on device type. */ struct magicmouse_sc { struct input_dev *input; unsigned long quirks; + bool query_dimensions; int ntouches; int scroll_accel; unsigned long scroll_jiffies; + struct input_mt_pos pos[MAX_CONTACTS]; struct { - short x; - short y; short scroll_x; short scroll_y; short scroll_x_hr; @@ -143,14 +172,112 @@ struct magicmouse_sc { u8 size; bool scroll_x_active; bool scroll_y_active; - } touches[16]; - int tracking_ids[16]; + } touches[MAX_CONTACTS]; + int tracking_ids[MAX_CONTACTS]; struct hid_device *hdev; struct delayed_work work; struct timer_list battery_timer; + struct magicmouse_input_ops input_ops; }; +static int magicmouse_enable_multitouch(struct hid_device *hdev); + +static inline int le16_to_int(__le16 x) +{ + return (signed short)le16_to_cpu(x); +} + +static int magicmouse_open(struct input_dev *dev) +{ + struct hid_device *hdev = input_get_drvdata(dev); + struct magicmouse_sc *msc = hid_get_drvdata(hdev); + int ret; + + ret = hid_hw_open(hdev); + if (ret) + return ret; + + /* + * Some devices repond with 'invalid report id' when feature + * report switching it into multitouch mode is sent to it. + * + * This results in -EIO from the _raw low-level transport callback, + * but there seems to be no other way of switching the mode. + * Thus the super-ugly hacky success check below. + * + * MTP devices do not need this. + */ + if (hdev->bus != BUS_HOST) { + ret = magicmouse_enable_multitouch(hdev); + if (ret != -EIO && ret < 0) { + hid_err(hdev, "unable to request touch data (%d)\n", ret); + return ret; + } + if (ret == -EIO && (hdev->product == USB_DEVICE_ID_APPLE_MAGICMOUSE2 || + hdev->product == USB_DEVICE_ID_APPLE_MAGICMOUSE2_USBC)) { + schedule_delayed_work(&msc->work, msecs_to_jiffies(500)); + } + } + + /* + * For Apple Silicon trackpads, we want to query the dimensions on + * device open. This is because doing so requires the firmware, but + * we don't want to force a firmware load until the device is opened + * for the first time. So do that here and update the input properties + * just in time before userspace queries them. + */ + if (msc->query_dimensions) { + struct input_dev *input = msc->input; + u8 buf[32]; + struct { + __le32 width; + __le32 height; + __le16 min_x; + __le16 min_y; + __le16 max_x; + __le16 max_y; + } dim; + uint32_t x_span, y_span; + + ret = hid_hw_raw_request(hdev, SENSOR_DIMENSIONS_REPORT_ID, buf, sizeof(buf), HID_FEATURE_REPORT, HID_REQ_GET_REPORT); + if (ret < (int)(1 + sizeof(dim))) { + hid_err(hdev, "unable to request dimensions (%d)\n", ret); + return ret; + } + + memcpy(&dim, buf + 1, sizeof(dim)); + + /* finger position */ + input_set_abs_params(input, ABS_MT_POSITION_X, + le16_to_int(dim.min_x), le16_to_int(dim.max_x), 0, 0); + /* Y axis is inverted */ + input_set_abs_params(input, ABS_MT_POSITION_Y, + -le16_to_int(dim.max_y), -le16_to_int(dim.min_y), 0, 0); + x_span = le16_to_int(dim.max_x) - le16_to_int(dim.min_x); + y_span = le16_to_int(dim.max_y) - le16_to_int(dim.min_y); + + /* X/Y resolution */ + input_abs_set_res(input, ABS_MT_POSITION_X, 100 * x_span / le32_to_cpu(dim.width) ); + input_abs_set_res(input, ABS_MT_POSITION_Y, 100 * y_span / le32_to_cpu(dim.height) ); + + /* copy info, as input_mt_init_slots() does */ + dev->absinfo[ABS_X] = dev->absinfo[ABS_MT_POSITION_X]; + dev->absinfo[ABS_Y] = dev->absinfo[ABS_MT_POSITION_Y]; + + msc->query_dimensions = false; + } + + return 0; +} + +static void magicmouse_close(struct input_dev *dev) +{ + struct hid_device *hdev = input_get_drvdata(dev); + + hid_hw_close(hdev); +} + static int magicmouse_firm_touch(struct magicmouse_sc *msc) { int touch = -1; @@ -192,7 +319,7 @@ static void magicmouse_emit_buttons(struct magicmouse_sc *msc, int state) } else if (last_state != 0) { state = last_state; } else if ((id = magicmouse_firm_touch(msc)) >= 0) { - int x = msc->touches[id].x; + int x = msc->pos[id].x; if (x < middle_button_start) state = 1; else if (x > middle_button_stop) @@ -256,8 +383,8 @@ static void magicmouse_emit_touch(struct magicmouse_sc *msc, int raw_id, u8 *tda /* Store tracking ID and other fields. */ msc->tracking_ids[raw_id] = id; - msc->touches[id].x = x; - msc->touches[id].y = y; + msc->pos[id].x = x; + msc->pos[id].y = y; msc->touches[id].size = size; /* If requested, emulate a scroll wheel by detecting small @@ -387,6 +514,14 @@ static int magicmouse_raw_event(struct hid_device *hdev, struct hid_report *report, u8 *data, int size) { struct magicmouse_sc *msc = hid_get_drvdata(hdev); + + return msc->input_ops.raw_event(hdev, report, data, size); +} + +static int magicmouse_raw_event_usb(struct hid_device *hdev, + struct hid_report *report, u8 *data, int size) +{ + struct magicmouse_sc *msc = hid_get_drvdata(hdev); struct input_dev *input = msc->input; int x = 0, y = 0, ii, clicks = 0, npoints; @@ -518,6 +653,174 @@ static int magicmouse_raw_event(struct hid_device *hdev, return 1; } +/** + * struct tp_finger - single trackpad finger structure, le16-aligned + * + * @unknown1: unknown + * @unknown2: unknown + * @abs_x: absolute x coordinate + * @abs_y: absolute y coordinate + * @rel_x: relative x coordinate + * @rel_y: relative y coordinate + * @tool_major: tool area, major axis + * @tool_minor: tool area, minor axis + * @orientation: 16384 when point, else 15 bit angle + * @touch_major: touch area, major axis + * @touch_minor: touch area, minor axis + * @unused: zeros + * @pressure: pressure on forcetouch touchpad + * @multi: one finger: varies, more fingers: constant + */ +struct tp_finger { + __le16 unknown1; + __le16 unknown2; + __le16 abs_x; + __le16 abs_y; + __le16 rel_x; + __le16 rel_y; + __le16 tool_major; + __le16 tool_minor; + __le16 orientation; + __le16 touch_major; + __le16 touch_minor; + __le16 unused[2]; + __le16 pressure; + __le16 multi; +} __attribute__((packed, aligned(2))); + +/** + * vendor trackpad report + * + * @num_fingers: the number of fingers being reported in @fingers + * @buttons: same as HID buttons + */ +struct tp_header { + // HID vendor part, up to 1751 bytes + u8 unknown[22]; + u8 num_fingers; + u8 buttons; + u8 unknown3[14]; +}; + +/** + * standard HID mouse report + * + * @report_id: reportid + * @buttons: HID Usage Buttons 3 1-bit reports + */ +struct tp_mouse_report { + // HID mouse report + u8 report_id; + u8 buttons; + u8 rel_x; + u8 rel_y; + u8 padding[4]; +}; + +static void report_finger_data(struct input_dev *input, int slot, + const struct input_mt_pos *pos, + const struct tp_finger *f) +{ + input_mt_slot(input, slot); + input_mt_report_slot_state(input, MT_TOOL_FINGER, true); + + input_report_abs(input, ABS_MT_TOUCH_MAJOR, + le16_to_int(f->touch_major) << 1); + input_report_abs(input, ABS_MT_TOUCH_MINOR, + le16_to_int(f->touch_minor) << 1); + input_report_abs(input, ABS_MT_WIDTH_MAJOR, + le16_to_int(f->tool_major) << 1); + input_report_abs(input, ABS_MT_WIDTH_MINOR, + le16_to_int(f->tool_minor) << 1); + input_report_abs(input, ABS_MT_ORIENTATION, + J314_TP_MAX_FINGER_ORIENTATION - le16_to_int(f->orientation)); + input_report_abs(input, ABS_MT_PRESSURE, le16_to_int(f->pressure)); + input_report_abs(input, ABS_MT_POSITION_X, pos->x); + input_report_abs(input, ABS_MT_POSITION_Y, pos->y); +} + +static int magicmouse_raw_event_mtp(struct hid_device *hdev, + struct hid_report *report, u8 *data, int size) +{ + struct magicmouse_sc *msc = hid_get_drvdata(hdev); + struct input_dev *input = msc->input; + struct tp_header *tp_hdr; + struct tp_finger *f; + int i, n; + u32 npoints; + const size_t hdr_sz = sizeof(struct tp_header); + const size_t touch_sz = sizeof(struct tp_finger); + u8 map_contacs[MAX_CONTACTS]; + + // hid_warn(hdev, "%s\n", __func__); + // print_hex_dump_debug("appleft ev: ", DUMP_PREFIX_OFFSET, 16, 1, data, + // size, false); + + /* Expect 46 bytes of prefix, and N * 30 bytes of touch data. */ + if (size < hdr_sz || ((size - hdr_sz) % touch_sz) != 0) + return 0; + + tp_hdr = (struct tp_header *)data; + + npoints = (size - hdr_sz) / touch_sz; + if (npoints < tp_hdr->num_fingers || npoints > MAX_CONTACTS) { + hid_warn(hdev, + "unexpected number of touches (%u) for " + "report\n", + npoints); + return 0; + } + + n = 0; + for (i = 0; i < tp_hdr->num_fingers; i++) { + f = (struct tp_finger *)(data + hdr_sz + i * touch_sz); + if (le16_to_int(f->touch_major) == 0) + continue; + + hid_dbg(hdev, "ev x:%04x y:%04x\n", le16_to_int(f->abs_x), + le16_to_int(f->abs_y)); + msc->pos[n].x = le16_to_int(f->abs_x); + msc->pos[n].y = -le16_to_int(f->abs_y); + map_contacs[n] = i; + n++; + } + + input_mt_assign_slots(input, msc->tracking_ids, msc->pos, n, 0); + + for (i = 0; i < n; i++) { + int idx = map_contacs[i]; + f = (struct tp_finger *)(data + hdr_sz + idx * touch_sz); + report_finger_data(input, msc->tracking_ids[i], &msc->pos[i], f); + } + + input_mt_sync_frame(input); + input_report_key(input, BTN_MOUSE, tp_hdr->buttons & 1); + + input_sync(input); + return 1; +} + +static int magicmouse_raw_event_spi(struct hid_device *hdev, + struct hid_report *report, u8 *data, int size) +{ + struct magicmouse_sc *msc = hid_get_drvdata(hdev); + const size_t hdr_sz = sizeof(struct tp_mouse_report); + + if (!size) + return 0; + + if (data[0] == SPI_RESET_REPORT_ID) { + hid_info(hdev, "Touch controller was reset, re-enabling touch mode\n"); + schedule_delayed_work(&msc->work, msecs_to_jiffies(10)); + return 1; + } + + if (data[0] != SPI_REPORT_ID || size < hdr_sz) + return 0; + + return magicmouse_raw_event_mtp(hdev, report, data + hdr_sz, size - hdr_sz); +} + static int magicmouse_event(struct hid_device *hdev, struct hid_field *field, struct hid_usage *usage, __s32 value) { @@ -536,7 +839,17 @@ static int magicmouse_event(struct hid_device *hdev, struct hid_field *field, return 0; } -static int magicmouse_setup_input(struct input_dev *input, struct hid_device *hdev) + +static int magicmouse_setup_input(struct input_dev *input, + struct hid_device *hdev) +{ + struct magicmouse_sc *msc = hid_get_drvdata(hdev); + + return msc->input_ops.setup_input(input, hdev); +} + +static int magicmouse_setup_input_usb(struct input_dev *input, + struct hid_device *hdev) { int error; int mt_flags = 0; @@ -615,7 +928,7 @@ static int magicmouse_setup_input(struct input_dev *input, struct hid_device *hd __set_bit(EV_ABS, input->evbit); - error = input_mt_init_slots(input, 16, mt_flags); + error = input_mt_init_slots(input, MAX_CONTACTS, mt_flags); if (error) return error; input_set_abs_params(input, ABS_MT_TOUCH_MAJOR, 0, 255 << 2, @@ -695,6 +1008,109 @@ static int magicmouse_setup_input(struct input_dev *input, struct hid_device *hd */ __clear_bit(EV_REP, input->evbit); + /* + * This isn't strictly speaking needed for USB, but enabling MT on + * device open is probably more robust than only doing it once on probe + * even if USB devices are not known to suffer from the SPI reset issue. + */ + input->open = magicmouse_open; + input->close = magicmouse_close; + return 0; +} + +static int magicmouse_setup_input_mtp(struct input_dev *input, + struct hid_device *hdev) +{ + int error; + int mt_flags = 0; + struct magicmouse_sc *msc = hid_get_drvdata(hdev); + + __set_bit(INPUT_PROP_BUTTONPAD, input->propbit); + __clear_bit(BTN_0, input->keybit); + __clear_bit(BTN_RIGHT, input->keybit); + __clear_bit(BTN_MIDDLE, input->keybit); + __clear_bit(EV_REL, input->evbit); + __clear_bit(REL_X, input->relbit); + __clear_bit(REL_Y, input->relbit); + + mt_flags = INPUT_MT_POINTER | INPUT_MT_DROP_UNUSED | INPUT_MT_TRACK; + + /* finger touch area */ + input_set_abs_params(input, ABS_MT_TOUCH_MAJOR, 0, 5000, 0, 0); + input_set_abs_params(input, ABS_MT_TOUCH_MINOR, 0, 5000, 0, 0); + + /* finger approach area */ + input_set_abs_params(input, ABS_MT_WIDTH_MAJOR, 0, 5000, 0, 0); + input_set_abs_params(input, ABS_MT_WIDTH_MINOR, 0, 5000, 0, 0); + + /* Note: Touch Y position from the device is inverted relative + * to how pointer motion is reported (and relative to how USB + * HID recommends the coordinates work). This driver keeps + * the origin at the same position, and just uses the additive + * inverse of the reported Y. + */ + + input_set_abs_params(input, ABS_MT_PRESSURE, 0, 6000, 0, 0); + + /* + * This makes libinput recognize this as a PressurePad and + * stop trying to use pressure for touch size. Pressure unit + * seems to be ~grams on these touchpads. + */ + input_abs_set_res(input, ABS_MT_PRESSURE, 1); + + /* finger orientation */ + input_set_abs_params(input, ABS_MT_ORIENTATION, -J314_TP_MAX_FINGER_ORIENTATION, + J314_TP_MAX_FINGER_ORIENTATION, 0, 0); + + /* finger position */ + input_set_abs_params(input, ABS_MT_POSITION_X, J314_TP_MIN_X, J314_TP_MAX_X, + 0, 0); + /* Y axis is inverted */ + input_set_abs_params(input, ABS_MT_POSITION_Y, -J314_TP_MAX_Y, -J314_TP_MIN_Y, + 0, 0); + + /* X/Y resolution */ + input_abs_set_res(input, ABS_MT_POSITION_X, J314_TP_RES_X); + input_abs_set_res(input, ABS_MT_POSITION_Y, J314_TP_RES_Y); + + input_set_events_per_packet(input, 60); + + /* touchpad button */ + input_set_capability(input, EV_KEY, BTN_MOUSE); + + /* + * hid-input may mark device as using autorepeat, but the trackpad does + * not actually want it. + */ + __clear_bit(EV_REP, input->evbit); + + error = input_mt_init_slots(input, MAX_CONTACTS, mt_flags); + if (error) + return error; + + /* + * Override the default input->open function to send the MT + * enable every time the device is opened. This ensures it works + * even if we missed a reset event due to the device being closed. + * input->close is overridden for symmetry. + * + * This also takes care of the dimensions query. + */ + input->open = magicmouse_open; + input->close = magicmouse_close; + msc->query_dimensions = true; + + return 0; +} + +static int magicmouse_setup_input_spi(struct input_dev *input, + struct hid_device *hdev) +{ + int ret = magicmouse_setup_input_mtp(input, hdev); + if (ret) + return ret; + return 0; } @@ -725,6 +1141,11 @@ static int magicmouse_input_configured(struct hid_device *hdev, struct magicmouse_sc *msc = hid_get_drvdata(hdev); int ret; + if (!msc->input) { + hid_err(hdev, "magicmouse setup input failed (no input)"); + return -EINVAL; + } + ret = magicmouse_setup_input(msc->input, hdev); if (ret) { hid_err(hdev, "magicmouse setup input failed (%d)\n", ret); @@ -748,6 +1169,10 @@ static int magicmouse_enable_multitouch(struct hid_device *hdev) int feature_size; switch (hdev->product) { + case SPI_DEVICE_ID_APPLE_MACBOOK_AIR_2020: + case SPI_DEVICE_ID_APPLE_MACBOOK_PRO13_2020: + case SPI_DEVICE_ID_APPLE_MACBOOK_PRO14_2021: + case SPI_DEVICE_ID_APPLE_MACBOOK_PRO16_2021: case USB_DEVICE_ID_APPLE_MAGICTRACKPAD2: case USB_DEVICE_ID_APPLE_MAGICTRACKPAD2_USBC: switch (hdev->vendor) { @@ -755,7 +1180,7 @@ static int magicmouse_enable_multitouch(struct hid_device *hdev) feature_size = sizeof(feature_mt_trackpad2_bt); feature = feature_mt_trackpad2_bt; break; - default: /* USB_VENDOR_ID_APPLE */ + default: /* USB_VENDOR_ID_APPLE || SPI_VENDOR_ID_APPLE */ feature_size = sizeof(feature_mt_trackpad2_usb); feature = feature_mt_trackpad2_usb; } @@ -852,12 +1277,29 @@ static int magicmouse_probe(struct hid_device *hdev, struct hid_report *report; int ret; + if ((id->bus == BUS_SPI || id->bus == BUS_HOST) && id->vendor == SPI_VENDOR_ID_APPLE && + hdev->type != HID_TYPE_SPI_MOUSE) + return -ENODEV; + msc = devm_kzalloc(&hdev->dev, sizeof(*msc), GFP_KERNEL); if (msc == NULL) { hid_err(hdev, "can't alloc magicmouse descriptor\n"); return -ENOMEM; } + // internal trackpad use a data format use input ops to avoid + // conflicts with the report ID. + if (id->bus == BUS_HOST) { + msc->input_ops.raw_event = magicmouse_raw_event_mtp; + msc->input_ops.setup_input = magicmouse_setup_input_mtp; + } else if (id->bus == BUS_SPI) { + msc->input_ops.raw_event = magicmouse_raw_event_spi; + msc->input_ops.setup_input = magicmouse_setup_input_spi; + } else { + msc->input_ops.raw_event = magicmouse_raw_event_usb; + msc->input_ops.setup_input = magicmouse_setup_input_usb; + } + msc->scroll_accel = SCROLL_ACCEL_DEFAULT; msc->hdev = hdev; INIT_DEFERRABLE_WORK(&msc->work, magicmouse_enable_mt_work); @@ -916,11 +1358,20 @@ static int magicmouse_probe(struct hid_device *hdev, TRACKPAD2_USB_REPORT_ID, 0); } break; - default: /* USB_DEVICE_ID_APPLE_MAGICTRACKPAD */ - report = hid_register_report(hdev, HID_INPUT_REPORT, - TRACKPAD_REPORT_ID, 0); - report = hid_register_report(hdev, HID_INPUT_REPORT, - DOUBLE_REPORT_ID, 0); + default: + switch (id->bus) { + case BUS_HOST: + report = hid_register_report(hdev, HID_INPUT_REPORT, MTP_REPORT_ID, 0); + break; + case BUS_SPI: + report = hid_register_report(hdev, HID_INPUT_REPORT, SPI_REPORT_ID, 0); + break; + default: /* USB_DEVICE_ID_APPLE_MAGICTRACKPAD */ + report = hid_register_report(hdev, HID_INPUT_REPORT, + TRACKPAD_REPORT_ID, 0); + report = hid_register_report(hdev, HID_INPUT_REPORT, + DOUBLE_REPORT_ID, 0); + } } if (!report) { @@ -930,22 +1381,14 @@ static int magicmouse_probe(struct hid_device *hdev, } report->size = 6; - /* - * Some devices repond with 'invalid report id' when feature - * report switching it into multitouch mode is sent to it. - * - * This results in -EIO from the _raw low-level transport callback, - * but there seems to be no other way of switching the mode. - * Thus the super-ugly hacky success check below. - */ - ret = magicmouse_enable_multitouch(hdev); - if (ret != -EIO && ret < 0) { - hid_err(hdev, "unable to request touch data (%d)\n", ret); - goto err_stop_hw; - } - if (ret == -EIO && (id->product == USB_DEVICE_ID_APPLE_MAGICMOUSE2 || - id->product == USB_DEVICE_ID_APPLE_MAGICMOUSE2_USBC)) { - schedule_delayed_work(&msc->work, msecs_to_jiffies(500)); + /* MTP devices do not need the MT enable, this is handled by the MTP driver */ + if (id->bus == BUS_HOST) + return 0; + + /* SPI devices need to watch for reset events to re-send the MT enable */ + if (id->bus == BUS_SPI) { + report = hid_register_report(hdev, HID_INPUT_REPORT, SPI_RESET_REPORT_ID, 0); + report->size = 2; } return 0; @@ -985,13 +1428,11 @@ static const __u8 *magicmouse_report_fixup(struct hid_device *hdev, __u8 *rdesc, */ if ((is_usb_magicmouse2(hdev->vendor, hdev->product) || is_usb_magictrackpad2(hdev->vendor, hdev->product)) && - *rsize == 83 && rdesc[46] == 0x84 && rdesc[58] == 0x85) { + *rsize >= 83 && rdesc[46] == 0x84 && rdesc[58] == 0x85) { hid_info(hdev, "fixing up magicmouse battery report descriptor\n"); *rsize = *rsize - 1; - rdesc = kmemdup(rdesc + 1, *rsize, GFP_KERNEL); - if (!rdesc) - return NULL; + rdesc = rdesc + 1; rdesc[0] = 0x05; rdesc[1] = 0x01; @@ -1023,10 +1464,24 @@ static const struct hid_device_id magic_mice[] = { USB_DEVICE_ID_APPLE_MAGICTRACKPAD2_USBC), .driver_data = 0 }, { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_MAGICTRACKPAD2_USBC), .driver_data = 0 }, + { HID_SPI_DEVICE(SPI_VENDOR_ID_APPLE, HID_ANY_ID), + .driver_data = 0 }, + { HID_DEVICE(BUS_HOST, HID_GROUP_ANY, HOST_VENDOR_ID_APPLE, + HID_ANY_ID), .driver_data = 0 }, { } }; MODULE_DEVICE_TABLE(hid, magic_mice); +#ifdef CONFIG_PM +static int magicmouse_reset_resume(struct hid_device *hdev) +{ + if (hdev->bus == BUS_SPI) + return magicmouse_enable_multitouch(hdev); + + return 0; +} +#endif + static struct hid_driver magicmouse_driver = { .name = "magicmouse", .id_table = magic_mice, @@ -1037,6 +1492,10 @@ static struct hid_driver magicmouse_driver = { .event = magicmouse_event, .input_mapping = magicmouse_input_mapping, .input_configured = magicmouse_input_configured, +#ifdef CONFIG_PM + .reset_resume = magicmouse_reset_resume, +#endif + }; module_hid_driver(magicmouse_driver); diff --git a/drivers/hid/hid-mcp2221.c b/drivers/hid/hid-mcp2221.c index 33603b019f975e..ef3b5c77c38e32 100644 --- a/drivers/hid/hid-mcp2221.c +++ b/drivers/hid/hid-mcp2221.c @@ -353,6 +353,8 @@ static int mcp_i2c_smbus_read(struct mcp2221 *mcp, usleep_range(90, 100); retries++; } else { + usleep_range(980, 1000); + mcp_cancel_last_cmd(mcp); return ret; } } else { diff --git a/drivers/hid/hid-multitouch.c b/drivers/hid/hid-multitouch.c index b1c3ef12905877..e82a3c4e5b44ef 100644 --- a/drivers/hid/hid-multitouch.c +++ b/drivers/hid/hid-multitouch.c @@ -76,6 +76,8 @@ MODULE_LICENSE("GPL"); #define MT_QUIRK_DISABLE_WAKEUP BIT(21) #define MT_QUIRK_ORIENTATION_INVERT BIT(22) #define MT_QUIRK_APPLE_TOUCHBAR BIT(23) +#define MT_QUIRK_YOGABOOK9I BIT(24) +#define MT_QUIRK_KEEP_LATENCY_ON_CLOSE BIT(25) #define MT_INPUTMODE_TOUCHSCREEN 0x02 #define MT_INPUTMODE_TOUCHPAD 0x03 @@ -213,6 +215,7 @@ static void mt_post_parse(struct mt_device *td, struct mt_application *app); #define MT_CLS_WIN_8_DISABLE_WAKEUP 0x0016 #define MT_CLS_WIN_8_NO_STICKY_FINGERS 0x0017 #define MT_CLS_WIN_8_FORCE_MULTI_INPUT_NSMU 0x0018 +#define MT_CLS_WIN_8_KEEP_LATENCY_ON_CLOSE 0x0019 /* vendor specific classes */ #define MT_CLS_3M 0x0101 @@ -231,6 +234,8 @@ static void mt_post_parse(struct mt_device *td, struct mt_application *app); #define MT_CLS_RAZER_BLADE_STEALTH 0x0112 #define MT_CLS_SMART_TECH 0x0113 #define MT_CLS_APPLE_TOUCHBAR 0x0114 +#define MT_CLS_YOGABOOK9I 0x0115 +#define MT_CLS_EGALAX_P80H84 0x0116 #define MT_CLS_SIS 0x0457 #define MT_DEFAULT_MAXCONTACT 10 @@ -332,6 +337,15 @@ static const struct mt_class mt_classes[] = { MT_QUIRK_CONTACT_CNT_ACCURATE | MT_QUIRK_WIN8_PTP_BUTTONS, .export_all_inputs = true }, + { .name = MT_CLS_WIN_8_KEEP_LATENCY_ON_CLOSE, + .quirks = MT_QUIRK_ALWAYS_VALID | + MT_QUIRK_IGNORE_DUPLICATES | + MT_QUIRK_HOVERING | + MT_QUIRK_CONTACT_CNT_ACCURATE | + MT_QUIRK_STICKY_FINGERS | + MT_QUIRK_WIN8_PTP_BUTTONS | + MT_QUIRK_KEEP_LATENCY_ON_CLOSE, + .export_all_inputs = true }, /* * vendor specific classes @@ -427,6 +441,19 @@ static const struct mt_class mt_classes[] = { .quirks = MT_QUIRK_NOT_SEEN_MEANS_UP | MT_QUIRK_ALWAYS_VALID | MT_QUIRK_CONTACT_CNT_ACCURATE, + }, + { .name = MT_CLS_YOGABOOK9I, + .quirks = MT_QUIRK_ALWAYS_VALID | + MT_QUIRK_FORCE_MULTI_INPUT | + MT_QUIRK_SEPARATE_APP_REPORT | + MT_QUIRK_HOVERING | + MT_QUIRK_YOGABOOK9I, + .export_all_inputs = true + }, + { .name = MT_CLS_EGALAX_P80H84, + .quirks = MT_QUIRK_ALWAYS_VALID | + MT_QUIRK_IGNORE_DUPLICATES | + MT_QUIRK_CONTACT_CNT_ACCURATE, }, { } }; @@ -499,12 +526,19 @@ static void mt_get_feature(struct hid_device *hdev, struct hid_report *report) dev_warn(&hdev->dev, "failed to fetch feature %d\n", report->id); } else { + /* The report ID in the request and the response should match */ + if (report->id != buf[0]) { + hid_err(hdev, "Returned feature report did not match the request\n"); + goto free; + } + ret = hid_report_raw_event(hdev, HID_FEATURE_REPORT, buf, size, 0); if (ret) dev_warn(&hdev->dev, "failed to report feature\n"); } +free: kfree(buf); } @@ -839,7 +873,8 @@ static int mt_touch_input_mapping(struct hid_device *hdev, struct hid_input *hi, if ((cls->name == MT_CLS_WIN_8 || cls->name == MT_CLS_WIN_8_FORCE_MULTI_INPUT || cls->name == MT_CLS_WIN_8_FORCE_MULTI_INPUT_NSMU || - cls->name == MT_CLS_WIN_8_DISABLE_WAKEUP) && + cls->name == MT_CLS_WIN_8_DISABLE_WAKEUP || + cls->name == MT_CLS_WIN_8_KEEP_LATENCY_ON_CLOSE) && (field->application == HID_DG_TOUCHPAD || field->application == HID_DG_TOUCHSCREEN)) app->quirks |= MT_QUIRK_CONFIDENCE; @@ -1576,6 +1611,38 @@ static void mt_report(struct hid_device *hid, struct hid_report *report) if (rdata && rdata->is_mt_collection) return mt_touch_report(hid, rdata); + /* Lenovo Yoga Book 9i requires consuming and dropping certain bogus reports */ + if (rdata && rdata->application && + (rdata->application->quirks & MT_QUIRK_YOGABOOK9I)) { + + bool all_zero_report = true; + + for (int f = 0; f < report->maxfield && all_zero_report; f++) { + struct hid_field *fld = report->field[f]; + + for (int i = 0; i < fld->report_count; i++) { + unsigned int usage = fld->usage[i].hid; + + if (usage == HID_DG_INRANGE || + usage == HID_DG_TIPSWITCH || + usage == HID_DG_BARRELSWITCH || + usage == HID_DG_BARRELSWITCH2 || + usage == HID_DG_CONTACTID || + usage == HID_DG_TILT_X || + usage == HID_DG_TILT_Y) { + + if (fld->value[i] != 0) { + all_zero_report = false; + break; + } + } + } + } + + if (all_zero_report) + return; + } + if (field && field->hidinput && field->hidinput->input) input_sync(field->hidinput->input); } @@ -1720,7 +1787,8 @@ static int mt_input_configured(struct hid_device *hdev, struct hid_input *hi) int ret; if (td->is_haptic_touchpad && (td->mtclass.name == MT_CLS_WIN_8 || - td->mtclass.name == MT_CLS_WIN_8_FORCE_MULTI_INPUT)) { + td->mtclass.name == MT_CLS_WIN_8_FORCE_MULTI_INPUT || + td->mtclass.name == MT_CLS_WIN_8_KEEP_LATENCY_ON_CLOSE)) { if (hid_haptic_input_configured(hdev, td->haptic, hi) == 0) td->is_haptic_touchpad = false; } else { @@ -1772,6 +1840,30 @@ static int mt_input_configured(struct hid_device *hdev, struct hid_input *hi) break; } + /* Lenovo Yoga Book 9i requires custom naming to allow differentiation in udev */ + if (hi->report && td->mtclass.quirks & MT_QUIRK_YOGABOOK9I) { + switch (hi->report->id) { + case 48: + suffix = "Touchscreen Top"; + break; + case 56: + suffix = "Touchscreen Bottom"; + break; + case 20: + suffix = "Stylus Top"; + break; + case 40: + suffix = "Stylus Bottom"; + break; + case 80: + suffix = "Emulated Touchpad"; + break; + default: + suffix = ""; + break; + } + } + if (suffix) { hi->input->name = devm_kasprintf(&hdev->dev, GFP_KERNEL, "%s %s", hdev->name, suffix); @@ -2009,7 +2101,12 @@ static void mt_on_hid_hw_open(struct hid_device *hdev) static void mt_on_hid_hw_close(struct hid_device *hdev) { - mt_set_modes(hdev, HID_LATENCY_HIGH, TOUCHPAD_REPORT_NONE); + struct mt_device *td = hid_get_drvdata(hdev); + + if (td->mtclass.quirks & MT_QUIRK_KEEP_LATENCY_ON_CLOSE) + mt_set_modes(hdev, HID_LATENCY_NORMAL, TOUCHPAD_REPORT_NONE); + else + mt_set_modes(hdev, HID_LATENCY_HIGH, TOUCHPAD_REPORT_NONE); } /* @@ -2146,8 +2243,12 @@ static const struct hid_device_id mt_devices[] = { { .driver_data = MT_CLS_EGALAX_SERIAL, MT_USB_DEVICE(USB_VENDOR_ID_DWAV, USB_DEVICE_ID_DWAV_EGALAX_MULTITOUCH_A001) }, - { .driver_data = MT_CLS_EGALAX, + { .driver_data = MT_CLS_EGALAX_SERIAL, MT_USB_DEVICE(USB_VENDOR_ID_DWAV, + USB_DEVICE_ID_DWAV_EGALAX_MULTITOUCH_C000) }, + { .driver_data = MT_CLS_EGALAX_P80H84, + HID_DEVICE(HID_BUS_ANY, HID_GROUP_MULTITOUCH_WIN_8, + USB_VENDOR_ID_DWAV, USB_DEVICE_ID_DWAV_EGALAX_MULTITOUCH_C002) }, /* Elan devices */ @@ -2277,6 +2378,12 @@ static const struct hid_device_id mt_devices[] = { USB_VENDOR_ID_LENOVO, USB_DEVICE_ID_LENOVO_X12_TAB2) }, + /* Lenovo Yoga Book 9i */ + { .driver_data = MT_CLS_YOGABOOK9I, + HID_DEVICE(BUS_USB, HID_GROUP_MULTITOUCH_WIN_8, + USB_VENDOR_ID_LENOVO, + USB_DEVICE_ID_LENOVO_YOGABOOK9I) }, + /* Logitech devices */ { .driver_data = MT_CLS_NSMU, HID_DEVICE(BUS_BLUETOOTH, HID_GROUP_MULTITOUCH_WIN_8, @@ -2386,6 +2493,14 @@ static const struct hid_device_id mt_devices[] = { MT_USB_DEVICE(USB_VENDOR_ID_UNITEC, USB_DEVICE_ID_UNITEC_USB_TOUCH_0A19) }, + /* Uniwill touchpads */ + { .driver_data = MT_CLS_WIN_8_KEEP_LATENCY_ON_CLOSE, + HID_DEVICE(BUS_I2C, HID_GROUP_MULTITOUCH_WIN_8, + USB_VENDOR_ID_PIXART, 0x0255) }, + { .driver_data = MT_CLS_WIN_8_KEEP_LATENCY_ON_CLOSE, + HID_DEVICE(BUS_I2C, HID_GROUP_MULTITOUCH_WIN_8, + USB_VENDOR_ID_PIXART, 0x0274) }, + /* VTL panels */ { .driver_data = MT_CLS_VTL, MT_USB_DEVICE(USB_VENDOR_ID_VTL, diff --git a/drivers/hid/hid-pl.c b/drivers/hid/hid-pl.c index 3c8827081deae2..dc11d5322fc0f9 100644 --- a/drivers/hid/hid-pl.c +++ b/drivers/hid/hid-pl.c @@ -194,9 +194,14 @@ static int pl_probe(struct hid_device *hdev, const struct hid_device_id *id) goto err; } - plff_init(hdev); + ret = plff_init(hdev); + if (ret) + goto stop; return 0; + +stop: + hid_hw_stop(hdev); err: return ret; } diff --git a/drivers/hid/hid-playstation.c b/drivers/hid/hid-playstation.c index e4dfcf26b04e74..2ec6d4445e84ba 100644 --- a/drivers/hid/hid-playstation.c +++ b/drivers/hid/hid-playstation.c @@ -774,7 +774,9 @@ ps_gamepad_create(struct hid_device *hdev, #if IS_ENABLED(CONFIG_PLAYSTATION_FF) if (play_effect) { input_set_capability(gamepad, EV_FF, FF_RUMBLE); - input_ff_create_memless(gamepad, NULL, play_effect); + ret = input_ff_create_memless(gamepad, NULL, play_effect); + if (ret) + return ERR_PTR(ret); } #endif diff --git a/drivers/hid/hid-prodikeys.c b/drivers/hid/hid-prodikeys.c index 74bddb2c3e82eb..6e413df38358a4 100644 --- a/drivers/hid/hid-prodikeys.c +++ b/drivers/hid/hid-prodikeys.c @@ -378,6 +378,10 @@ static int pcmidi_handle_report4(struct pcmidi_snd *pm, u8 *data) bit_mask = (bit_mask << 8) | data[2]; bit_mask = (bit_mask << 8) | data[3]; + /* robustness in case input_mapping hook does not get called */ + if (!pm->input_ep82) + return 0; + /* break keys */ for (bit_index = 0; bit_index < 24; bit_index++) { if (!((0x01 << bit_index) & bit_mask)) { diff --git a/drivers/hid/hid-quirks.c b/drivers/hid/hid-quirks.c index 11438039cdb7f7..f6be3ffee02326 100644 --- a/drivers/hid/hid-quirks.c +++ b/drivers/hid/hid-quirks.c @@ -25,6 +25,7 @@ */ static const struct hid_device_id hid_quirks[] = { + { HID_USB_DEVICE(USB_VENDOR_ID_8BITDO, USB_DEVICE_ID_8BITDO_PRO_3), HID_QUIRK_ALWAYS_POLL }, { HID_USB_DEVICE(USB_VENDOR_ID_AASHIMA, USB_DEVICE_ID_AASHIMA_GAMEPAD), HID_QUIRK_BADPAD }, { HID_USB_DEVICE(USB_VENDOR_ID_AASHIMA, USB_DEVICE_ID_AASHIMA_PREDATOR), HID_QUIRK_BADPAD }, { HID_USB_DEVICE(USB_VENDOR_ID_ADATA_XPG, USB_VENDOR_ID_ADATA_XPG_WL_GAMING_MOUSE), HID_QUIRK_ALWAYS_POLL }, @@ -420,6 +421,7 @@ static const struct hid_device_id hid_have_special_driver[] = { #if IS_ENABLED(CONFIG_HID_ELECOM) { HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_BM084) }, { HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_XGL20DLBK) }, + { HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_HT1MRBK_01AC) }, { HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_XT3URBK_00FB) }, { HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_XT3URBK_018F) }, { HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_XT3DRBK_00FC) }, @@ -432,6 +434,8 @@ static const struct hid_device_id hid_have_special_driver[] = { { HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_HT1URBK_019B) }, { HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_HT1DRBK_010D) }, { HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_HT1DRBK_011C) }, + { HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_HT1MRBK) }, + { HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_HT1MRBK_01AB) }, #endif #if IS_ENABLED(CONFIG_HID_ELO) { HID_USB_DEVICE(USB_VENDOR_ID_ELO, 0x0009) }, diff --git a/drivers/hid/hid-roccat.c b/drivers/hid/hid-roccat.c index c7f7562e22e562..e413662f750824 100644 --- a/drivers/hid/hid-roccat.c +++ b/drivers/hid/hid-roccat.c @@ -257,6 +257,7 @@ int roccat_report_event(int minor, u8 const *data) if (!new_value) return -ENOMEM; + mutex_lock(&device->readers_lock); mutex_lock(&device->cbuf_lock); report = &device->cbuf[device->cbuf_end]; @@ -279,6 +280,7 @@ int roccat_report_event(int minor, u8 const *data) } mutex_unlock(&device->cbuf_lock); + mutex_unlock(&device->readers_lock); wake_up_interruptible(&device->wait); return 0; diff --git a/drivers/hid/hid-zydacron.c b/drivers/hid/hid-zydacron.c index 3bdb26f4559257..1aae80f848f503 100644 --- a/drivers/hid/hid-zydacron.c +++ b/drivers/hid/hid-zydacron.c @@ -114,7 +114,7 @@ static int zc_raw_event(struct hid_device *hdev, struct hid_report *report, unsigned key; unsigned short index; - if (report->id == data[0]) { + if (report->id == data[0] && (hdev->claimed & HID_CLAIMED_INPUT)) { /* break keys */ for (index = 0; index < 4; index++) { diff --git a/drivers/hid/i2c-hid/i2c-hid-of-elan.c b/drivers/hid/i2c-hid/i2c-hid-of-elan.c index 0215f217f6d863..b81fcc6ff49eef 100644 --- a/drivers/hid/i2c-hid/i2c-hid-of-elan.c +++ b/drivers/hid/i2c-hid/i2c-hid-of-elan.c @@ -168,6 +168,13 @@ static const struct elan_i2c_hid_chip_data elan_ekth6a12nay_chip_data = { .power_after_backlight = true, }; +static const struct elan_i2c_hid_chip_data focaltech_ft8112_chip_data = { + .post_power_delay_ms = 10, + .post_gpio_reset_on_delay_ms = 150, + .hid_descriptor_address = 0x0001, + .main_supply_name = "vcc33", +}; + static const struct elan_i2c_hid_chip_data ilitek_ili9882t_chip_data = { .post_power_delay_ms = 1, .post_gpio_reset_on_delay_ms = 200, @@ -191,6 +198,7 @@ static const struct elan_i2c_hid_chip_data ilitek_ili2901_chip_data = { static const struct of_device_id elan_i2c_hid_of_match[] = { { .compatible = "elan,ekth6915", .data = &elan_ekth6915_chip_data }, { .compatible = "elan,ekth6a12nay", .data = &elan_ekth6a12nay_chip_data }, + { .compatible = "focaltech,ft8112", .data = &focaltech_ft8112_chip_data }, { .compatible = "ilitek,ili9882t", .data = &ilitek_ili9882t_chip_data }, { .compatible = "ilitek,ili2901", .data = &ilitek_ili2901_chip_data }, { } diff --git a/drivers/hid/intel-ish-hid/ipc/hw-ish.h b/drivers/hid/intel-ish-hid/ipc/hw-ish.h index fa5d68c3631341..27389971b96ccf 100644 --- a/drivers/hid/intel-ish-hid/ipc/hw-ish.h +++ b/drivers/hid/intel-ish-hid/ipc/hw-ish.h @@ -39,6 +39,8 @@ #define PCI_DEVICE_ID_INTEL_ISH_PTL_H 0xE345 #define PCI_DEVICE_ID_INTEL_ISH_PTL_P 0xE445 #define PCI_DEVICE_ID_INTEL_ISH_WCL 0x4D45 +#define PCI_DEVICE_ID_INTEL_ISH_NVL_H 0xD354 +#define PCI_DEVICE_ID_INTEL_ISH_NVL_S 0x6E78 #define REVISION_ID_CHT_A0 0x6 #define REVISION_ID_CHT_Ax_SI 0x0 diff --git a/drivers/hid/intel-ish-hid/ipc/pci-ish.c b/drivers/hid/intel-ish-hid/ipc/pci-ish.c index 1612e8cb23f0c4..ed3405c05e73c5 100644 --- a/drivers/hid/intel-ish-hid/ipc/pci-ish.c +++ b/drivers/hid/intel-ish-hid/ipc/pci-ish.c @@ -28,11 +28,15 @@ enum ishtp_driver_data_index { ISHTP_DRIVER_DATA_LNL_M, ISHTP_DRIVER_DATA_PTL, ISHTP_DRIVER_DATA_WCL, + ISHTP_DRIVER_DATA_NVL_H, + ISHTP_DRIVER_DATA_NVL_S, }; #define ISH_FW_GEN_LNL_M "lnlm" #define ISH_FW_GEN_PTL "ptl" #define ISH_FW_GEN_WCL "wcl" +#define ISH_FW_GEN_NVL_H "nvlh" +#define ISH_FW_GEN_NVL_S "nvls" #define ISH_FIRMWARE_PATH(gen) "intel/ish/ish_" gen ".bin" #define ISH_FIRMWARE_PATH_ALL "intel/ish/ish_*.bin" @@ -47,6 +51,12 @@ static struct ishtp_driver_data ishtp_driver_data[] = { [ISHTP_DRIVER_DATA_WCL] = { .fw_generation = ISH_FW_GEN_WCL, }, + [ISHTP_DRIVER_DATA_NVL_H] = { + .fw_generation = ISH_FW_GEN_NVL_H, + }, + [ISHTP_DRIVER_DATA_NVL_S] = { + .fw_generation = ISH_FW_GEN_NVL_S, + }, }; static const struct pci_device_id ish_pci_tbl[] = { @@ -76,6 +86,8 @@ static const struct pci_device_id ish_pci_tbl[] = { {PCI_DEVICE_DATA(INTEL, ISH_PTL_H, ISHTP_DRIVER_DATA_PTL)}, {PCI_DEVICE_DATA(INTEL, ISH_PTL_P, ISHTP_DRIVER_DATA_PTL)}, {PCI_DEVICE_DATA(INTEL, ISH_WCL, ISHTP_DRIVER_DATA_WCL)}, + {PCI_DEVICE_DATA(INTEL, ISH_NVL_H, ISHTP_DRIVER_DATA_NVL_H)}, + {PCI_DEVICE_DATA(INTEL, ISH_NVL_S, ISHTP_DRIVER_DATA_NVL_S)}, {} }; MODULE_DEVICE_TABLE(pci, ish_pci_tbl); diff --git a/drivers/hid/intel-ish-hid/ishtp/bus.c b/drivers/hid/intel-ish-hid/ishtp/bus.c index c3915f3a060ead..b890fbf97a75cd 100644 --- a/drivers/hid/intel-ish-hid/ishtp/bus.c +++ b/drivers/hid/intel-ish-hid/ishtp/bus.c @@ -730,7 +730,7 @@ void ishtp_bus_remove_all_clients(struct ishtp_device *ishtp_dev, spin_lock_irqsave(&ishtp_dev->cl_list_lock, flags); list_for_each_entry(cl, &ishtp_dev->cl_list, link) { cl->state = ISHTP_CL_DISCONNECTED; - if (warm_reset && cl->device->reference_count) + if (warm_reset && cl->device && cl->device->reference_count) continue; /* diff --git a/drivers/hid/intel-thc-hid/intel-quickspi/pci-quickspi.c b/drivers/hid/intel-thc-hid/intel-quickspi/pci-quickspi.c index ad6bd59963b288..b6a69995692cba 100644 --- a/drivers/hid/intel-thc-hid/intel-quickspi/pci-quickspi.c +++ b/drivers/hid/intel-thc-hid/intel-quickspi/pci-quickspi.c @@ -37,6 +37,10 @@ struct quickspi_driver_data arl = { .max_packet_size_value = MAX_PACKET_SIZE_VALUE_MTL, }; +struct quickspi_driver_data nvl = { + .max_packet_size_value = MAX_PACKET_SIZE_VALUE_LNL, +}; + /* THC QuickSPI ACPI method to get device properties */ /* HIDSPI Method: {6e2ac436-0fcf-41af-a265-b32a220dcfab} */ static guid_t hidspi_guid = @@ -982,6 +986,8 @@ static const struct pci_device_id quickspi_pci_tbl[] = { {PCI_DEVICE_DATA(INTEL, THC_WCL_DEVICE_ID_SPI_PORT2, &ptl), }, {PCI_DEVICE_DATA(INTEL, THC_ARL_DEVICE_ID_SPI_PORT1, &arl), }, {PCI_DEVICE_DATA(INTEL, THC_ARL_DEVICE_ID_SPI_PORT2, &arl), }, + {PCI_DEVICE_DATA(INTEL, THC_NVL_H_DEVICE_ID_SPI_PORT1, &nvl), }, + {PCI_DEVICE_DATA(INTEL, THC_NVL_H_DEVICE_ID_SPI_PORT2, &nvl), }, {} }; MODULE_DEVICE_TABLE(pci, quickspi_pci_tbl); diff --git a/drivers/hid/intel-thc-hid/intel-quickspi/quickspi-dev.h b/drivers/hid/intel-thc-hid/intel-quickspi/quickspi-dev.h index c30e1a42eb0984..bf5e18f5a5f425 100644 --- a/drivers/hid/intel-thc-hid/intel-quickspi/quickspi-dev.h +++ b/drivers/hid/intel-thc-hid/intel-quickspi/quickspi-dev.h @@ -23,6 +23,8 @@ #define PCI_DEVICE_ID_INTEL_THC_WCL_DEVICE_ID_SPI_PORT2 0x4D4B #define PCI_DEVICE_ID_INTEL_THC_ARL_DEVICE_ID_SPI_PORT1 0x7749 #define PCI_DEVICE_ID_INTEL_THC_ARL_DEVICE_ID_SPI_PORT2 0x774B +#define PCI_DEVICE_ID_INTEL_THC_NVL_H_DEVICE_ID_SPI_PORT1 0xD349 +#define PCI_DEVICE_ID_INTEL_THC_NVL_H_DEVICE_ID_SPI_PORT2 0xD34B /* HIDSPI special ACPI parameters DSM methods */ #define ACPI_QUICKSPI_REVISION_NUM 2 diff --git a/drivers/hid/intel-thc-hid/intel-thc/intel-thc-dev.c b/drivers/hid/intel-thc-hid/intel-thc/intel-thc-dev.c index 7e220a4c5ded7b..d8e195189e4bfd 100644 --- a/drivers/hid/intel-thc-hid/intel-thc/intel-thc-dev.c +++ b/drivers/hid/intel-thc-hid/intel-thc/intel-thc-dev.c @@ -1597,6 +1597,7 @@ int thc_i2c_set_rx_max_size(struct thc_device *dev, u32 max_rx_size) if (ret) return ret; + val = val & ~THC_M_PRT_SPI_ICRRD_OPCODE_I2C_MAX_SIZE; val |= FIELD_PREP(THC_M_PRT_SPI_ICRRD_OPCODE_I2C_MAX_SIZE, max_rx_size); ret = regmap_write(dev->thc_regmap, THC_M_PRT_SPI_ICRRD_OPCODE_OFFSET, val); @@ -1667,6 +1668,7 @@ int thc_i2c_set_rx_int_delay(struct thc_device *dev, u32 delay_us) return ret; /* THC hardware counts at 10us unit */ + val = val & ~THC_M_PRT_SPI_ICRRD_OPCODE_I2C_INTERVAL; val |= FIELD_PREP(THC_M_PRT_SPI_ICRRD_OPCODE_I2C_INTERVAL, DIV_ROUND_UP(delay_us, 10)); ret = regmap_write(dev->thc_regmap, THC_M_PRT_SPI_ICRRD_OPCODE_OFFSET, val); diff --git a/drivers/hid/spi-hid/Kconfig b/drivers/hid/spi-hid/Kconfig new file mode 100644 index 00000000000000..59076c6ebeed9b --- /dev/null +++ b/drivers/hid/spi-hid/Kconfig @@ -0,0 +1,24 @@ +# SPDX-License-Identifier: GPL-2.0-only +menu "SPI HID support" + depends on SPI + +config SPI_HID_APPLE_OF + tristate "HID over SPI transport layer for Apple Silicon SoCs" + depends on INPUT && OF + select SPI_HID_APPLE_CORE + help + Say Y here if you use Apple Silicon based laptop. The keyboard and + touchpad are HID based devices connected via SPI. + + If unsure, say N. + + This support is also available as a module. If so, the module + will be called spi-hid-apple-of. It will also build/depend on the + module spi-hid-apple. + +endmenu + +config SPI_HID_APPLE_CORE + tristate + select HID + select CRC16 diff --git a/drivers/hid/spi-hid/Makefile b/drivers/hid/spi-hid/Makefile new file mode 100644 index 00000000000000..f276ee12cb94fc --- /dev/null +++ b/drivers/hid/spi-hid/Makefile @@ -0,0 +1,10 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Makefile for SPI HID tarnsport drivers +# + +obj-$(CONFIG_SPI_HID_APPLE_CORE) += spi-hid-apple.o + +spi-hid-apple-objs = spi-hid-apple-core.o + +obj-$(CONFIG_SPI_HID_APPLE_OF) += spi-hid-apple-of.o diff --git a/drivers/hid/spi-hid/spi-hid-apple-core.c b/drivers/hid/spi-hid/spi-hid-apple-core.c new file mode 100644 index 00000000000000..2ed909895391c8 --- /dev/null +++ b/drivers/hid/spi-hid/spi-hid-apple-core.c @@ -0,0 +1,1194 @@ +/* + * SPDX-License-Identifier: GPL-2.0 + * + * Apple SPI HID transport driver + * + * Copyright (C) The Asahi Linux Contributors + * + * Based on: drivers/input/applespi.c + * + * MacBook (Pro) SPI keyboard and touchpad driver + * + * Copyright (c) 2015-2018 Federico Lorenzi + * Copyright (c) 2017-2018 Ronald Tschalär + * + */ + +//#define DEBUG 2 + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "spi-hid-apple.h" + +#define SPIHID_DEF_WAIT msecs_to_jiffies(1000) + +#define SPIHID_MAX_INPUT_REPORT_SIZE 0x800 + +/* support only keyboard, trackpad and management dev for now */ +#define SPIHID_MAX_DEVICES 3 + +#define SPIHID_DEVICE_ID_MNGT 0x0 +#define SPIHID_DEVICE_ID_KBD 0x1 +#define SPIHID_DEVICE_ID_TP 0x2 +#define SPIHID_DEVICE_ID_INFO 0xd0 + +#define SPIHID_READ_PACKET 0x20 +#define SPIHID_WRITE_PACKET 0x40 + +#define SPIHID_DESC_MAX 512 + +#define SPIHID_SET_LEDS 0x0151 /* caps lock */ + +#define SPI_RW_CHG_DELAY_US 200 /* 'Inter Stage Us'? */ + +static const u8 spi_hid_apple_booted[4] = { 0xa0, 0x80, 0x00, 0x00 }; +static const u8 spi_hid_apple_status_ok[4] = { 0xac, 0x27, 0x68, 0xd5 }; + +struct spihid_interface { + struct hid_device *hid; + u8 *hid_desc; + u32 hid_desc_len; + u32 id; + unsigned country; + u32 max_control_report_len; + u32 max_input_report_len; + u32 max_output_report_len; + u8 name[32]; + u8 reply_buf[SPIHID_DESC_MAX]; + u32 reply_len; + bool ready; +}; + +struct spihid_input_report { + u8 *buf; + u32 length; + u32 offset; + u8 device; + u8 flags; +}; + +struct spihid_apple { + struct spi_device *spidev; + + struct spihid_apple_ops *ops; + + struct spihid_interface mngt; + struct spihid_interface kbd; + struct spihid_interface tp; + + wait_queue_head_t wait; + struct mutex tx_lock; //< protects against concurrent SPI writes + + struct spi_message rx_msg; + struct spi_message tx_msg; + struct spi_transfer rx_transfer; + struct spi_transfer tx_transfer; + struct spi_transfer status_transfer; + + u8 *rx_buf; + u8 *tx_buf; + u8 *status_buf; + + u8 vendor[32]; + u8 product[64]; + u8 serial[32]; + + u32 num_devices; + + u32 vendor_id; + u32 product_id; + u32 version_number; + + u8 msg_id; + + /* fragmented HID report */ + struct spihid_input_report report; + + /* state tracking flags */ + bool status_booted; + +#ifdef IRQ_WAKE_SUPPORT + bool irq_wake_enabled; +#endif +}; + +/** + * struct spihid_msg_hdr - common header of protocol messages. + * + * Each message begins with fixed header, followed by a message-type specific + * payload, and ends with a 16-bit crc. Because of the varying lengths of the + * payload, the crc is defined at the end of each payload struct, rather than + * in this struct. + * + * @unknown0: request type? output, input (0x10), feature, protocol + * @unknown1: maybe report id? + * @unknown2: mostly zero, in info request maybe device num + * @id: incremented on each message, rolls over after 255; there is a + * separate counter for each message type. + * @rsplen: response length (the exact nature of this field is quite + * speculative). On a request/write this is often the same as + * @length, though in some cases it has been seen to be much larger + * (e.g. 0x400); on a response/read this the same as on the + * request; for reads that are not responses it is 0. + * @length: length of the remainder of the data in the whole message + * structure (after re-assembly in case of being split over + * multiple spi-packets), minus the trailing crc. The total size + * of a message is therefore @length + 10. + */ + +struct spihid_msg_hdr { + u8 unknown0; + u8 unknown1; + u8 unknown2; + u8 id; + __le16 rsplen; + __le16 length; +}; + +/** + * struct spihid_transfer_packet - a complete spi packet; always 256 bytes. This carries + * the (parts of the) message in the data. But note that this does not + * necessarily contain a complete message, as in some cases (e.g. many + * fingers pressed) the message is split over multiple packets (see the + * @offset, @remain, and @length fields). In general the data parts in + * spihid_transfer_packet's are concatenated until @remaining is 0, and the + * result is an message. + * + * @flags: 0x40 = write (to device), 0x20 = read (from device); note that + * the response to a write still has 0x40. + * @device: 1 = keyboard, 2 = touchpad + * @offset: specifies the offset of this packet's data in the complete + * message; i.e. > 0 indicates this is a continuation packet (in + * the second packet for a message split over multiple packets + * this would then be the same as the @length in the first packet) + * @remain: number of message bytes remaining in subsequents packets (in + * the first packet of a message split over two packets this would + * then be the same as the @length in the second packet) + * @length: length of the valid data in the @data in this packet + * @data: all or part of a message + * @crc16: crc over this whole structure minus this @crc16 field. This + * covers just this packet, even on multi-packet messages (in + * contrast to the crc in the message). + */ +struct spihid_transfer_packet { + u8 flags; + u8 device; + __le16 offset; + __le16 remain; + __le16 length; + u8 data[246]; + __le16 crc16; +}; + +/* + * how HID is mapped onto the protocol is not fully clear. This are the known + * reports/request: + * + * pkt.flags pkt.dev? msg.u0 msg.u1 msg.u2 + * info 0x40 0xd0 0x20 0x01 0xd0 + * + * info mngt: 0x40 0xd0 0x20 0x10 0x00 + * info kbd: 0x40 0xd0 0x20 0x10 0x01 + * info tp: 0x40 0xd0 0x20 0x10 0x02 + * + * desc kbd: 0x40 0xd0 0x20 0x10 0x01 + * desc trackpad: 0x40 0xd0 0x20 0x10 0x02 + * + * mt mode: 0x40 0x02 0x52 0x02 0x00 set protocol? + * capslock led 0x40 0x01 0x51 0x01 0x00 output report + * + * report kbd: 0x20 0x01 0x10 0x01 0x00 input report + * report tp: 0x20 0x02 0x10 0x02 0x00 input report + * + */ + + +static int spihid_apple_request(struct spihid_apple *spihid, u8 target, u8 unk0, + u8 unk1, u8 unk2, u16 resp_len, u8 *buf, + size_t len) +{ + struct spihid_transfer_packet *pkt; + struct spihid_msg_hdr *hdr; + u16 crc; + int err; + + /* know reports are small enoug to fit in a single packet */ + if (len > sizeof(pkt->data) - sizeof(*hdr) - sizeof(__le16)) + return -EINVAL; + + err = mutex_lock_interruptible(&spihid->tx_lock); + if (err < 0) + return err; + + pkt = (struct spihid_transfer_packet *)spihid->tx_buf; + + memset(pkt, 0, sizeof(*pkt)); + pkt->flags = SPIHID_WRITE_PACKET; + pkt->device = target; + pkt->length = cpu_to_le16(sizeof(*hdr) + len + sizeof(__le16)); + + hdr = (struct spihid_msg_hdr *)&pkt->data[0]; + hdr->unknown0 = unk0; + hdr->unknown1 = unk1; + hdr->unknown2 = unk2; + hdr->id = spihid->msg_id++; + hdr->rsplen = cpu_to_le16(resp_len); + hdr->length = cpu_to_le16(len); + + if (len) + memcpy(pkt->data + sizeof(*hdr), buf, len); + crc = crc16(0, &pkt->data[0], sizeof(*hdr) + len); + put_unaligned_le16(crc, pkt->data + sizeof(*hdr) + len); + + pkt->crc16 = cpu_to_le16(crc16(0, spihid->tx_buf, + offsetof(struct spihid_transfer_packet, crc16))); + + memset(spihid->status_buf, 0, sizeof(spi_hid_apple_status_ok)); + + err = spi_sync(spihid->spidev, &spihid->tx_msg); + + if (memcmp(spihid->status_buf, spi_hid_apple_status_ok, + sizeof(spi_hid_apple_status_ok))) { + u8 *b = spihid->status_buf; + dev_warn_ratelimited(&spihid->spidev->dev, "status message " + "mismatch: %02x %02x %02x %02x\n", + b[0], b[1], b[2], b[3]); + } + mutex_unlock(&spihid->tx_lock); + if (err < 0) + return err; + + return (int)len; +} + +static struct spihid_apple *spihid_get_data(struct spihid_interface *idev) +{ + switch (idev->id) { + case SPIHID_DEVICE_ID_KBD: + return container_of(idev, struct spihid_apple, kbd); + case SPIHID_DEVICE_ID_TP: + return container_of(idev, struct spihid_apple, tp); + default: + return NULL; + } +} + +static int apple_ll_start(struct hid_device *hdev) +{ + /* no-op SPI transport is already setup */ + return 0; +}; + +static void apple_ll_stop(struct hid_device *hdev) +{ + /* no-op, devices will be desstroyed on driver destruction */ +} + +static int apple_ll_open(struct hid_device *hdev) +{ + struct spihid_apple *spihid; + struct spihid_interface *idev = hdev->driver_data; + + if (idev->hid_desc_len == 0) { + spihid = spihid_get_data(idev); + dev_warn(&spihid->spidev->dev, + "HID descriptor missing for dev %u", idev->id); + } else + idev->ready = true; + + return 0; +} + +static void apple_ll_close(struct hid_device *hdev) +{ + struct spihid_interface *idev = hdev->driver_data; + idev->ready = false; +} + +static int apple_ll_parse(struct hid_device *hdev) +{ + struct spihid_interface *idev = hdev->driver_data; + + return hid_parse_report(hdev, idev->hid_desc, idev->hid_desc_len); +} + +static int apple_ll_raw_request(struct hid_device *hdev, + unsigned char reportnum, __u8 *buf, size_t len, + unsigned char rtype, int reqtype) +{ + struct spihid_interface *idev = hdev->driver_data; + struct spihid_apple *spihid = spihid_get_data(idev); + int ret; + + dev_dbg(&spihid->spidev->dev, + "apple_ll_raw_request: device:%u reportnum:%hhu rtype:%hhu", + idev->id, reportnum, rtype); + + switch (reqtype) { + case HID_REQ_GET_REPORT: + if (rtype != HID_FEATURE_REPORT) + return -EINVAL; + + idev->reply_len = 0; + ret = spihid_apple_request(spihid, idev->id, 0x32, reportnum, 0x00, len, NULL, 0); + if (ret < 0) + return ret; + + ret = wait_event_interruptible_timeout(spihid->wait, idev->reply_len, + SPIHID_DEF_WAIT); + if (ret == 0) + ret = -ETIMEDOUT; + if (ret < 0) { + dev_err(&spihid->spidev->dev, "waiting for get report failed: %d", ret); + return ret; + } + memcpy(buf, idev->reply_buf, max_t(size_t, len, idev->reply_len)); + return idev->reply_len; + + case HID_REQ_SET_REPORT: + if (buf[0] != reportnum) + return -EINVAL; + if (reportnum != idev->id) { + dev_warn(&spihid->spidev->dev, + "device:%u reportnum:" + "%hhu mismatch", + idev->id, reportnum); + return -EINVAL; + } + return spihid_apple_request(spihid, idev->id, 0x52, reportnum, 0x00, 2, buf, len); + default: + return -EIO; + } +} + +static int apple_ll_output_report(struct hid_device *hdev, __u8 *buf, + size_t len) +{ + struct spihid_interface *idev = hdev->driver_data; + struct spihid_apple *spihid = spihid_get_data(idev); + if (!spihid) + return -1; + + dev_dbg(&spihid->spidev->dev, + "apple_ll_output_report: device:%u len:%zu:", + idev->id, len); + // second idev->id should maybe be buf[0]? + return spihid_apple_request(spihid, idev->id, 0x51, idev->id, 0x00, 0, buf, len); +} + +static struct hid_ll_driver apple_hid_ll = { + .start = &apple_ll_start, + .stop = &apple_ll_stop, + .open = &apple_ll_open, + .close = &apple_ll_close, + .parse = &apple_ll_parse, + .raw_request = &apple_ll_raw_request, + .output_report = &apple_ll_output_report, + .max_buffer_size = SPIHID_MAX_INPUT_REPORT_SIZE, +}; + +static struct spihid_interface *spihid_get_iface(struct spihid_apple *spihid, + u32 iface) +{ + switch (iface) { + case SPIHID_DEVICE_ID_MNGT: + return &spihid->mngt; + case SPIHID_DEVICE_ID_KBD: + return &spihid->kbd; + case SPIHID_DEVICE_ID_TP: + return &spihid->tp; + default: + return NULL; + } +} + +static int spihid_verify_msg(struct spihid_apple *spihid, u8 *buf, size_t len) +{ + u16 msg_crc, crc; + struct device *dev = &spihid->spidev->dev; + + crc = crc16(0, buf, len - sizeof(__le16)); + msg_crc = get_unaligned_le16(buf + len - sizeof(__le16)); + if (crc != msg_crc) { + dev_warn_ratelimited(dev, "Read message crc mismatch\n"); + return 0; + } + return 1; +} + +static bool spihid_status_report(struct spihid_apple *spihid, u8 *pl, + size_t len) +{ + struct device *dev = &spihid->spidev->dev; + dev_dbg(dev, "%s: len: %zu", __func__, len); + if (len == 5 && pl[0] == 0xe0) + return true; + + return false; +} + +static bool spihid_process_input_report(struct spihid_apple *spihid, u32 device, + struct spihid_msg_hdr *hdr, u8 *payload, + size_t len) +{ + //dev_dbg(&spihid>spidev->dev, "input report: req:%hx iface:%u ", hdr->unknown0, device); + if (hdr->unknown0 != 0x10) + return false; + + /* HID device as well but Vendor usage only, handle it internally for now */ + if (device == 0) { + if (hdr->unknown1 == 0xe0) { + return spihid_status_report(spihid, payload, len); + } + } else if (device < SPIHID_MAX_DEVICES) { + struct spihid_interface *iface = + spihid_get_iface(spihid, device); + if (iface && iface->hid && iface->ready) { + hid_input_report(iface->hid, HID_INPUT_REPORT, payload, + len, 1); + return true; + } + } else + dev_dbg(&spihid->spidev->dev, + "unexpected iface:%u for input report", device); + + return false; +} + +struct spihid_device_info { + __le16 u0[2]; + __le16 num_devices; + __le16 vendor_id; + __le16 product_id; + __le16 version_number; + __le16 vendor_str[2]; //< offset and string length + __le16 product_str[2]; //< offset and string length + __le16 serial_str[2]; //< offset and string length +}; + +static bool spihid_process_device_info(struct spihid_apple *spihid, u32 iface, + u8 *payload, size_t len) +{ + struct device *dev = &spihid->spidev->dev; + + if (iface != SPIHID_DEVICE_ID_INFO) + return false; + + if (spihid->vendor_id == 0 && + len >= sizeof(struct spihid_device_info)) { + struct spihid_device_info *info = + (struct spihid_device_info *)payload; + u16 voff, vlen, poff, plen, soff, slen; + u32 num_devices; + + num_devices = __le16_to_cpu(info->num_devices); + + if (num_devices < SPIHID_MAX_DEVICES) { + dev_err(dev, + "Device info reports %u devices, expecting at least 3", + num_devices); + return false; + } + spihid->num_devices = num_devices; + + if (spihid->num_devices > SPIHID_MAX_DEVICES) { + dev_info( + dev, + "limiting the number of devices to mngt, kbd and mouse"); + spihid->num_devices = SPIHID_MAX_DEVICES; + } + + spihid->vendor_id = __le16_to_cpu(info->vendor_id); + spihid->product_id = __le16_to_cpu(info->product_id); + spihid->version_number = __le16_to_cpu(info->version_number); + + voff = __le16_to_cpu(info->vendor_str[0]); + vlen = __le16_to_cpu(info->vendor_str[1]); + + if (voff < len && vlen <= len - voff && + vlen < sizeof(spihid->vendor)) { + memcpy(spihid->vendor, payload + voff, vlen); + spihid->vendor[vlen] = '\0'; + } + + poff = __le16_to_cpu(info->product_str[0]); + plen = __le16_to_cpu(info->product_str[1]); + + if (poff < len && plen <= len - poff && + plen < sizeof(spihid->product)) { + memcpy(spihid->product, payload + poff, plen); + spihid->product[plen] = '\0'; + } + + soff = __le16_to_cpu(info->serial_str[0]); + slen = __le16_to_cpu(info->serial_str[1]); + + if (soff < len && slen <= len - soff && + slen < sizeof(spihid->serial)) { + memcpy(spihid->vendor, payload + soff, slen); + spihid->serial[slen] = '\0'; + } + + wake_up_interruptible(&spihid->wait); + } + return true; +} + +struct spihid_iface_info { + u8 u_0; + u8 interface_num; + u8 u_2; + u8 u_3; + u8 u_4; + u8 country_code; + __le16 max_input_report_len; + __le16 max_output_report_len; + __le16 max_control_report_len; + __le16 name_offset; + __le16 name_length; +}; + +static bool spihid_process_iface_info(struct spihid_apple *spihid, u32 num, + u8 *payload, size_t len) +{ + struct spihid_iface_info *info; + struct spihid_interface *iface = spihid_get_iface(spihid, num); + u32 name_off, name_len; + + if (!iface) + return false; + + if (!iface->max_input_report_len) { + if (len < sizeof(*info)) + return false; + + info = (struct spihid_iface_info *)payload; + + iface->max_input_report_len = + le16_to_cpu(info->max_input_report_len); + iface->max_output_report_len = + le16_to_cpu(info->max_output_report_len); + iface->max_control_report_len = + le16_to_cpu(info->max_control_report_len); + iface->country = info->country_code; + + name_off = le16_to_cpu(info->name_offset); + name_len = le16_to_cpu(info->name_length); + + if (name_off < len && name_len <= len - name_off && + name_len < sizeof(iface->name)) { + memcpy(iface->name, payload + name_off, name_len); + iface->name[name_len] = '\0'; + } + + dev_dbg(&spihid->spidev->dev, "Info for %s, country code: 0x%x", + iface->name, iface->country); + + wake_up_interruptible(&spihid->wait); + } + + return true; +} + +static int spihid_register_hid_device(struct spihid_apple *spihid, + struct spihid_interface *idev, u8 device); + +static bool spihid_process_iface_hid_report_desc(struct spihid_apple *spihid, + u32 num, u8 *payload, + size_t len) +{ + struct spihid_interface *iface = spihid_get_iface(spihid, num); + + if (!iface) + return false; + + if (iface->hid_desc_len == 0) { + if (len > SPIHID_DESC_MAX) + return false; + memcpy(iface->hid_desc, payload, len); + iface->hid_desc_len = len; + + /* do not register the mngt iface as HID device */ + if (num > 0) + spihid_register_hid_device(spihid, iface, num); + + wake_up_interruptible(&spihid->wait); + } + return true; +} + +static bool spihid_process_iface_get_report(struct spihid_apple *spihid, + u32 device, u8 report, + u8 *payload, size_t len) +{ + struct spihid_interface *iface = spihid_get_iface(spihid, device); + + if (!iface) + return false; + + if (len > sizeof(iface->reply_buf) || len < 1) + return false; + + memcpy(iface->reply_buf, payload, len); + iface->reply_len = len; + + wake_up_interruptible(&spihid->wait); + + return true; +} + +static bool spihid_process_response(struct spihid_apple *spihid, u32 device, + struct spihid_msg_hdr *hdr, u8 *payload, + size_t len) +{ + if (hdr->unknown0 == 0x20) { + switch (hdr->unknown1) { + case 0x01: + return spihid_process_device_info(spihid, hdr->unknown2, + payload, len); + case 0x02: + return spihid_process_iface_info(spihid, hdr->unknown2, + payload, len); + case 0x10: + return spihid_process_iface_hid_report_desc( + spihid, hdr->unknown2, payload, len); + default: + break; + } + } + + if (hdr->unknown0 == 0x32) { + return spihid_process_iface_get_report(spihid, device, hdr->unknown1, payload, len); + } + + return false; +} + +static void spihid_process_message(struct spihid_apple *spihid, u8 *data, + size_t length, u8 device, u8 flags) +{ + struct device *dev = &spihid->spidev->dev; + struct spihid_msg_hdr *hdr; + bool handled = false; + size_t payload_len; + u8 *payload; + + if (!spihid_verify_msg(spihid, data, length)) + return; + + hdr = (struct spihid_msg_hdr *)data; + payload_len = le16_to_cpu(hdr->length); + + if (payload_len == 0 || + (payload_len + sizeof(struct spihid_msg_hdr) + 2) > length) + return; + + payload = data + sizeof(struct spihid_msg_hdr); + + switch (flags) { + case SPIHID_READ_PACKET: + handled = spihid_process_input_report(spihid, device, hdr, + payload, payload_len); + break; + case SPIHID_WRITE_PACKET: + handled = spihid_process_response(spihid, device, hdr, payload, + payload_len); + break; + default: + break; + } + +#if defined(DEBUG) && DEBUG > 1 + { + dev_dbg(dev, + "R msg: req:%02hhx rep:%02hhx dev:%02hhx id:%hu len:%hu\n", + hdr->unknown0, hdr->unknown1, hdr->unknown2, hdr->id, + hdr->length); + print_hex_dump_debug("spihid msg: ", DUMP_PREFIX_OFFSET, 16, 1, + payload, le16_to_cpu(hdr->length), true); + } +#else + if (!handled) { + dev_dbg(dev, + "R unhandled msg: req:%02hhx rep:%02hhx dev:%02hhx id:%hu len:%hu\n", + hdr->unknown0, hdr->unknown1, hdr->unknown2, hdr->id, + hdr->length); + print_hex_dump_debug("spihid msg: ", DUMP_PREFIX_OFFSET, 16, 1, + payload, le16_to_cpu(hdr->length), true); + } +#endif +} + +static void spihid_assemble_message(struct spihid_apple *spihid, + struct spihid_transfer_packet *pkt) +{ + size_t length, offset, remain; + struct device *dev = &spihid->spidev->dev; + struct spihid_input_report *rep = &spihid->report; + + length = le16_to_cpu(pkt->length); + remain = le16_to_cpu(pkt->remain); + offset = le16_to_cpu(pkt->offset); + + if (offset + length + remain > U16_MAX) { + return; + } + + if (pkt->device != rep->device || pkt->flags != rep->flags || + offset != rep->offset) { + rep->device = 0; + rep->flags = 0; + rep->offset = 0; + rep->length = 0; + } + + if (offset == 0) { + if (rep->offset != 0) { + dev_warn(dev, "incomplete report off:%u len:%u", + rep->offset, rep->length); + } + memcpy(rep->buf, pkt->data, length); + rep->offset = length; + rep->length = length + remain; + rep->device = pkt->device; + rep->flags = pkt->flags; + } else if (offset == rep->offset) { + if (offset + length + remain != rep->length) { + dev_warn(dev, "incomplete report off:%u len:%u", + rep->offset, rep->length); + return; + } + memcpy(rep->buf + offset, pkt->data, length); + rep->offset += length; + + if (rep->offset == rep->length) { + spihid_process_message(spihid, rep->buf, rep->length, + rep->device, rep->flags); + rep->device = 0; + rep->flags = 0; + rep->offset = 0; + rep->length = 0; + } + } +} + +static void spihid_process_read(struct spihid_apple *spihid) +{ + u16 crc; + size_t length; + struct device *dev = &spihid->spidev->dev; + struct spihid_transfer_packet *pkt; + + pkt = (struct spihid_transfer_packet *)spihid->rx_buf; + + /* check transfer packet crc */ + crc = crc16(0, spihid->rx_buf, + offsetof(struct spihid_transfer_packet, crc16)); + if (crc != le16_to_cpu(pkt->crc16)) { + dev_warn_ratelimited(dev, "Read package crc mismatch\n"); + return; + } + + length = le16_to_cpu(pkt->length); + + if (length < sizeof(struct spihid_msg_hdr) + 2) { + if (length == sizeof(spi_hid_apple_booted) && + !memcmp(pkt->data, spi_hid_apple_booted, length)) { + if (!spihid->status_booted) { + spihid->status_booted = true; + wake_up_interruptible(&spihid->wait); + } + } else { + dev_info(dev, "R short packet: len:%zu\n", length); + print_hex_dump(KERN_INFO, "spihid pkt:", + DUMP_PREFIX_OFFSET, 16, 1, pkt->data, + length, false); + } + return; + } + +#if defined(DEBUG) && DEBUG > 1 + dev_dbg(dev, + "R pkt: flags:%02hhx dev:%02hhx off:%hu remain:%hu, len:%zu\n", + pkt->flags, pkt->device, pkt->offset, pkt->remain, length); +#if defined(DEBUG) && DEBUG > 2 + print_hex_dump_debug("spihid pkt: ", DUMP_PREFIX_OFFSET, 16, 1, + spihid->rx_buf, + sizeof(struct spihid_transfer_packet), true); +#endif +#endif + + if (length > sizeof(pkt->data)) { + dev_warn_ratelimited(dev, "Invalid pkt len:%zu", length); + return; + } + + /* short message */ + if (pkt->offset == 0 && pkt->remain == 0) { + spihid_process_message(spihid, pkt->data, length, pkt->device, + pkt->flags); + } else { + spihid_assemble_message(spihid, pkt); + } +} + +static void spihid_read_packet_sync(struct spihid_apple *spihid) +{ + int err; + + err = spi_sync(spihid->spidev, &spihid->rx_msg); + if (!err) { + spihid_process_read(spihid); + } else { + dev_warn(&spihid->spidev->dev, "RX failed: %d\n", err); + } +} + +irqreturn_t spihid_apple_core_irq(int irq, void *data) +{ + struct spi_device *spi = data; + struct spihid_apple *spihid = spi_get_drvdata(spi); + + spihid_read_packet_sync(spihid); + + return IRQ_HANDLED; +} +EXPORT_SYMBOL_GPL(spihid_apple_core_irq); + +static void spihid_apple_setup_spi_msgs(struct spihid_apple *spihid) +{ + memset(&spihid->rx_transfer, 0, sizeof(spihid->rx_transfer)); + + spihid->rx_transfer.rx_buf = spihid->rx_buf; + spihid->rx_transfer.len = sizeof(struct spihid_transfer_packet); + + spi_message_init(&spihid->rx_msg); + spi_message_add_tail(&spihid->rx_transfer, &spihid->rx_msg); + + memset(&spihid->tx_transfer, 0, sizeof(spihid->rx_transfer)); + memset(&spihid->status_transfer, 0, sizeof(spihid->status_transfer)); + + spihid->tx_transfer.tx_buf = spihid->tx_buf; + spihid->tx_transfer.len = sizeof(struct spihid_transfer_packet); + spihid->tx_transfer.delay.unit = SPI_DELAY_UNIT_USECS; + spihid->tx_transfer.delay.value = SPI_RW_CHG_DELAY_US; + + spihid->status_transfer.rx_buf = spihid->status_buf; + spihid->status_transfer.len = sizeof(spi_hid_apple_status_ok); + + spi_message_init(&spihid->tx_msg); + spi_message_add_tail(&spihid->tx_transfer, &spihid->tx_msg); + spi_message_add_tail(&spihid->status_transfer, &spihid->tx_msg); +} + +static int spihid_apple_setup_spi(struct spihid_apple *spihid) +{ + spihid_apple_setup_spi_msgs(spihid); + + return spihid->ops->power_on(spihid->ops); +} + +static int spihid_register_hid_device(struct spihid_apple *spihid, + struct spihid_interface *iface, u8 device) +{ + int ret; + char *suffix; + struct hid_device *hid; + + iface->id = device; + + hid = hid_allocate_device(); + if (IS_ERR(hid)) + return PTR_ERR(hid); + + /* + * Use 'Apple SPI Keyboard' and 'Apple SPI Trackpad' as input device + * names. The device names need to be distinct since at least Kwin uses + * the tripple Vendor ID, Product ID, Name to identify devices. + */ + snprintf(hid->name, sizeof(hid->name), "Apple SPI %s", iface->name); + // strip ' / Boot' suffix from the name + suffix = strstr(hid->name, " / Boot"); + if (suffix) + suffix[0] = '\0'; + snprintf(hid->phys, sizeof(hid->phys), "%s (%hhx)", + dev_name(&spihid->spidev->dev), device); + strscpy(hid->uniq, spihid->serial, sizeof(hid->uniq)); + + hid->ll_driver = &apple_hid_ll; + hid->bus = BUS_SPI; + hid->vendor = spihid->vendor_id; + hid->product = spihid->product_id; + hid->version = spihid->version_number; + + if (device == SPIHID_DEVICE_ID_KBD) + hid->type = HID_TYPE_SPI_KEYBOARD; + else if (device == SPIHID_DEVICE_ID_TP) + hid->type = HID_TYPE_SPI_MOUSE; + + hid->country = iface->country; + hid->dev.parent = &spihid->spidev->dev; + hid->driver_data = iface; + + ret = hid_add_device(hid); + if (ret < 0) { + hid_destroy_device(hid); + dev_warn(&spihid->spidev->dev, + "Failed to register hid device %hhu", device); + return ret; + } + + iface->hid = hid; + + return 0; +} + +static void spihid_destroy_hid_device(struct spihid_interface *iface) +{ + if (iface->hid) { + hid_destroy_device(iface->hid); + iface->hid = NULL; + } + iface->ready = false; +} + +int spihid_apple_core_probe(struct spi_device *spi, struct spihid_apple_ops *ops) +{ + struct device *dev = &spi->dev; + struct spihid_apple *spihid; + int err, i; + + if (!ops || !ops->power_on || !ops->power_off || !ops->enable_irq || !ops->disable_irq) + return -EINVAL; + + spihid = devm_kzalloc(dev, sizeof(*spihid), GFP_KERNEL); + if (!spihid) + return -ENOMEM; + + spihid->ops = ops; + spihid->spidev = spi; + + // init spi + spi_set_drvdata(spi, spihid); + + /* + * allocate SPI buffers + * Overallocate the receice buffer since it passed directly into + * hid_input_report / hid_report_raw_event. The later expects the buffer + * to be HID_MAX_BUFFER_SIZE (16k) or hid_ll_driver.max_buffer_size if + * set. + */ + spihid->rx_buf = devm_kmalloc( + &spi->dev, SPIHID_MAX_INPUT_REPORT_SIZE, GFP_KERNEL); + spihid->tx_buf = devm_kmalloc( + &spi->dev, sizeof(struct spihid_transfer_packet), GFP_KERNEL); + spihid->status_buf = devm_kmalloc( + &spi->dev, sizeof(spi_hid_apple_status_ok), GFP_KERNEL); + + if (!spihid->rx_buf || !spihid->tx_buf || !spihid->status_buf) + return -ENOMEM; + + spihid->report.buf = + devm_kmalloc(dev, SPIHID_MAX_INPUT_REPORT_SIZE, GFP_KERNEL); + + spihid->kbd.hid_desc = devm_kmalloc(dev, SPIHID_DESC_MAX, GFP_KERNEL); + spihid->tp.hid_desc = devm_kmalloc(dev, SPIHID_DESC_MAX, GFP_KERNEL); + + if (!spihid->report.buf || !spihid->kbd.hid_desc || + !spihid->tp.hid_desc) + return -ENOMEM; + + init_waitqueue_head(&spihid->wait); + + mutex_init(&spihid->tx_lock); + + /* Init spi transfer buffers and power device on */ + err = spihid_apple_setup_spi(spihid); + if (err < 0) + goto error; + + /* enable HID irq */ + spihid->ops->enable_irq(spihid->ops); + + // wait for boot message + err = wait_event_interruptible_timeout(spihid->wait, + spihid->status_booted, + msecs_to_jiffies(1000)); + if (err == 0) + err = -ENODEV; + if (err < 0) { + dev_err(dev, "waiting for device boot failed: %d", err); + goto error; + } + + /* request device information */ + dev_dbg(dev, "request device info"); + spihid_apple_request(spihid, 0xd0, 0x20, 0x01, 0xd0, 0, NULL, 0); + err = wait_event_interruptible_timeout(spihid->wait, spihid->vendor_id, + SPIHID_DEF_WAIT); + if (err == 0) + err = -ENODEV; + if (err < 0) { + dev_err(dev, "waiting for device info failed: %d", err); + goto error; + } + + /* request interface information */ + for (i = 0; i < spihid->num_devices; i++) { + struct spihid_interface *iface = spihid_get_iface(spihid, i); + if (!iface) + continue; + dev_dbg(dev, "request interface info 0x%02x", i); + spihid_apple_request(spihid, 0xd0, 0x20, 0x02, i, + SPIHID_DESC_MAX, NULL, 0); + err = wait_event_interruptible_timeout( + spihid->wait, iface->max_input_report_len, + SPIHID_DEF_WAIT); + } + + /* request HID report descriptors */ + for (i = 1; i < spihid->num_devices; i++) { + struct spihid_interface *iface = spihid_get_iface(spihid, i); + if (!iface) + continue; + dev_dbg(dev, "request hid report desc 0x%02x", i); + spihid_apple_request(spihid, 0xd0, 0x20, 0x10, i, + SPIHID_DESC_MAX, NULL, 0); + wait_event_interruptible_timeout( + spihid->wait, iface->hid_desc_len, SPIHID_DEF_WAIT); + } + + return 0; +error: + return err; +} +EXPORT_SYMBOL_GPL(spihid_apple_core_probe); + +void spihid_apple_core_remove(struct spi_device *spi) +{ + struct spihid_apple *spihid = spi_get_drvdata(spi); + + /* destroy input devices */ + + spihid_destroy_hid_device(&spihid->tp); + spihid_destroy_hid_device(&spihid->kbd); + + /* disable irq */ + spihid->ops->disable_irq(spihid->ops); + + /* power SPI device down */ + spihid->ops->power_off(spihid->ops); +} +EXPORT_SYMBOL_GPL(spihid_apple_core_remove); + +void spihid_apple_core_shutdown(struct spi_device *spi) +{ + struct spihid_apple *spihid = spi_get_drvdata(spi); + + /* disable irq */ + spihid->ops->disable_irq(spihid->ops); + + /* power SPI device down */ + spihid->ops->power_off(spihid->ops); +} +EXPORT_SYMBOL_GPL(spihid_apple_core_shutdown); + +#ifdef CONFIG_PM_SLEEP +static int spihid_apple_core_suspend(struct device *dev) +{ + int ret; +#ifdef IRQ_WAKE_SUPPORT + int wake_status; +#endif + struct spihid_apple *spihid = spi_get_drvdata(to_spi_device(dev)); + + if (spihid->tp.hid) { + ret = hid_driver_suspend(spihid->tp.hid, PMSG_SUSPEND); + if (ret < 0) + return ret; + } + + if (spihid->kbd.hid) { + ret = hid_driver_suspend(spihid->kbd.hid, PMSG_SUSPEND); + if (ret < 0) { + if (spihid->tp.hid) + hid_driver_resume(spihid->tp.hid); + return ret; + } + } + + /* Save some power */ + spihid->ops->disable_irq(spihid->ops); + +#ifdef IRQ_WAKE_SUPPORT + if (device_may_wakeup(dev)) { + wake_status = spihid->ops->enable_irq_wake(spihid->ops); + if (!wake_status) + spihid->irq_wake_enabled = true; + else + dev_warn(dev, "Failed to enable irq wake: %d\n", + wake_status); + } else { + spihid->ops->power_off(spihid->ops); + } +#else + spihid->ops->power_off(spihid->ops); +#endif + + return 0; +} + +static int spihid_apple_core_resume(struct device *dev) +{ + int ret_tp = 0, ret_kbd = 0; + struct spihid_apple *spihid = spi_get_drvdata(to_spi_device(dev)); +#ifdef IRQ_WAKE_SUPPORT + int wake_status; + + if (!device_may_wakeup(dev)) { + spihid->ops->power_on(spihid->ops); + } else if (spihid->irq_wake_enabled) { + wake_status = spihid->ops->disable_irq_wake(spihid->ops); + if (!wake_status) + spihid->irq_wake_enabled = false; + else + dev_warn(dev, "Failed to disable irq wake: %d\n", + wake_status); + } +#endif + + spihid->ops->enable_irq(spihid->ops); + spihid->ops->power_on(spihid->ops); + + if (spihid->tp.hid) + ret_tp = hid_driver_reset_resume(spihid->tp.hid); + if (spihid->kbd.hid) + ret_kbd = hid_driver_reset_resume(spihid->kbd.hid); + + if (ret_tp < 0) + return ret_tp; + + return ret_kbd; +} +#endif + +const struct dev_pm_ops spihid_apple_core_pm = { + SET_SYSTEM_SLEEP_PM_OPS(spihid_apple_core_suspend, + spihid_apple_core_resume) +}; +EXPORT_SYMBOL_GPL(spihid_apple_core_pm); + +MODULE_DESCRIPTION("Apple SPI HID transport driver"); +MODULE_AUTHOR("Janne Grunau "); +MODULE_LICENSE("GPL"); diff --git a/drivers/hid/spi-hid/spi-hid-apple-of.c b/drivers/hid/spi-hid/spi-hid-apple-of.c new file mode 100644 index 00000000000000..b631212b836d30 --- /dev/null +++ b/drivers/hid/spi-hid/spi-hid-apple-of.c @@ -0,0 +1,153 @@ +/* + * SPDX-License-Identifier: GPL-2.0 + * + * Apple SPI HID transport driver - Open Firmware + * + * Copyright (C) The Asahi Linux Contributors + */ + +#include +#include +#include +#include + +#include "spi-hid-apple.h" + + +struct spihid_apple_of { + struct spihid_apple_ops ops; + + struct gpio_desc *enable_gpio; + int irq; +}; + +static int spihid_apple_of_power_on(struct spihid_apple_ops *ops) +{ + struct spihid_apple_of *sh_of = container_of(ops, struct spihid_apple_of, ops); + + /* reset the controller on boot */ + gpiod_direction_output(sh_of->enable_gpio, 1); + msleep(5); + gpiod_direction_output(sh_of->enable_gpio, 0); + msleep(5); + /* turn SPI device on */ + gpiod_direction_output(sh_of->enable_gpio, 1); + msleep(50); + + return 0; +} + +static int spihid_apple_of_power_off(struct spihid_apple_ops *ops) +{ + struct spihid_apple_of *sh_of = container_of(ops, struct spihid_apple_of, ops); + + /* turn SPI device off */ + gpiod_direction_output(sh_of->enable_gpio, 0); + + return 0; +} + +static int spihid_apple_of_enable_irq(struct spihid_apple_ops *ops) +{ + struct spihid_apple_of *sh_of = container_of(ops, struct spihid_apple_of, ops); + + enable_irq(sh_of->irq); + + return 0; +} + +static int spihid_apple_of_disable_irq(struct spihid_apple_ops *ops) +{ + struct spihid_apple_of *sh_of = container_of(ops, struct spihid_apple_of, ops); + + disable_irq(sh_of->irq); + + return 0; +} + +static int spihid_apple_of_enable_irq_wake(struct spihid_apple_ops *ops) +{ + struct spihid_apple_of *sh_of = container_of(ops, struct spihid_apple_of, ops); + + return enable_irq_wake(sh_of->irq); +} + +static int spihid_apple_of_disable_irq_wake(struct spihid_apple_ops *ops) +{ + struct spihid_apple_of *sh_of = container_of(ops, struct spihid_apple_of, ops); + + return disable_irq_wake(sh_of->irq); +} + +static int spihid_apple_of_probe(struct spi_device *spi) +{ + struct device *dev = &spi->dev; + struct spihid_apple_of *spihid_of; + int err; + + spihid_of = devm_kzalloc(dev, sizeof(*spihid_of), GFP_KERNEL); + if (!spihid_of) + return -ENOMEM; + + spihid_of->ops.power_on = spihid_apple_of_power_on; + spihid_of->ops.power_off = spihid_apple_of_power_off; + spihid_of->ops.enable_irq = spihid_apple_of_enable_irq; + spihid_of->ops.disable_irq = spihid_apple_of_disable_irq; + spihid_of->ops.enable_irq_wake = spihid_apple_of_enable_irq_wake; + spihid_of->ops.disable_irq_wake = spihid_apple_of_disable_irq_wake; + + spihid_of->enable_gpio = devm_gpiod_get_index(dev, "spien", 0, 0); + if (IS_ERR(spihid_of->enable_gpio)) { + err = PTR_ERR(spihid_of->enable_gpio); + dev_err(dev, "failed to get 'spien' gpio pin: %d", err); + return err; + } + + spihid_of->irq = of_irq_get(dev->of_node, 0); + if (spihid_of->irq < 0) { + err = spihid_of->irq; + dev_err(dev, "failed to get 'extended-irq': %d", err); + return err; + } + err = devm_request_threaded_irq(dev, spihid_of->irq, NULL, + spihid_apple_core_irq, IRQF_ONESHOT | IRQF_NO_AUTOEN, + "spi-hid-apple-irq", spi); + if (err < 0) { + dev_err(dev, "failed to request extended-irq %d: %d", + spihid_of->irq, err); + return err; + } + + return spihid_apple_core_probe(spi, &spihid_of->ops); +} + +static const struct of_device_id spihid_apple_of_match[] = { + { .compatible = "apple,spi-hid-transport" }, + {}, +}; +MODULE_DEVICE_TABLE(of, spihid_apple_of_match); + +static struct spi_device_id spihid_apple_of_id[] = { + { "spi-hid-transport", 0 }, + {} +}; +MODULE_DEVICE_TABLE(spi, spihid_apple_of_id); + +static struct spi_driver spihid_apple_of_driver = { + .driver = { + .name = "spi-hid-apple-of", + .pm = &spihid_apple_core_pm, + .of_match_table = of_match_ptr(spihid_apple_of_match), + }, + + .id_table = spihid_apple_of_id, + .probe = spihid_apple_of_probe, + .remove = spihid_apple_core_remove, + .shutdown = spihid_apple_core_shutdown, +}; + +module_spi_driver(spihid_apple_of_driver); + +MODULE_DESCRIPTION("Apple SPI HID transport driver for OpenFirmware systems"); +MODULE_AUTHOR("Janne Grunau "); +MODULE_LICENSE("GPL"); diff --git a/drivers/hid/spi-hid/spi-hid-apple.h b/drivers/hid/spi-hid/spi-hid-apple.h new file mode 100644 index 00000000000000..9abecd1ba78028 --- /dev/null +++ b/drivers/hid/spi-hid/spi-hid-apple.h @@ -0,0 +1,35 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ + +#ifndef SPI_HID_APPLE_H +#define SPI_HID_APPLE_H + +#include +#include + +/** + * struct spihid_apple_ops - Ops to control the device from the core driver. + * + * @power_on: reset and power the device on. + * @power_off: power the device off. + * @enable_irq: enable irq or ACPI gpe. + * @disable_irq: disable irq or ACPI gpe. + */ + +struct spihid_apple_ops { + int (*power_on)(struct spihid_apple_ops *ops); + int (*power_off)(struct spihid_apple_ops *ops); + int (*enable_irq)(struct spihid_apple_ops *ops); + int (*disable_irq)(struct spihid_apple_ops *ops); + int (*enable_irq_wake)(struct spihid_apple_ops *ops); + int (*disable_irq_wake)(struct spihid_apple_ops *ops); +}; + +irqreturn_t spihid_apple_core_irq(int irq, void *data); + +int spihid_apple_core_probe(struct spi_device *spi, struct spihid_apple_ops *ops); +void spihid_apple_core_remove(struct spi_device *spi); +void spihid_apple_core_shutdown(struct spi_device *spi); + +extern const struct dev_pm_ops spihid_apple_core_pm; + +#endif /* SPI_HID_APPLE_H */ diff --git a/drivers/hid/usbhid/hid-pidff.c b/drivers/hid/usbhid/hid-pidff.c index 95377c5f63356b..56d6af39ba81ea 100644 --- a/drivers/hid/usbhid/hid-pidff.c +++ b/drivers/hid/usbhid/hid-pidff.c @@ -523,9 +523,19 @@ static void pidff_set_effect_report(struct pidff_device *pidff, pidff_set_duration(&pidff->set_effect[PID_DURATION], effect->replay.length); - pidff->set_effect[PID_TRIGGER_BUTTON].value[0] = effect->trigger.button; - pidff_set_time(&pidff->set_effect[PID_TRIGGER_REPEAT_INT], - effect->trigger.interval); + /* Some games set this to random values that can be out of range */ + s32 trigger_button_max = + pidff->set_effect[PID_TRIGGER_BUTTON].field->logical_maximum; + if (effect->trigger.button <= trigger_button_max) { + pidff->set_effect[PID_TRIGGER_BUTTON].value[0] = + effect->trigger.button; + pidff_set_time(&pidff->set_effect[PID_TRIGGER_REPEAT_INT], + effect->trigger.interval); + } else { + pidff->set_effect[PID_TRIGGER_BUTTON].value[0] = 0; + pidff->set_effect[PID_TRIGGER_REPEAT_INT].value[0] = 0; + } + pidff->set_effect[PID_GAIN].value[0] = pidff->set_effect[PID_GAIN].field->logical_maximum; @@ -1442,10 +1452,13 @@ static int pidff_init_fields(struct pidff_device *pidff, struct input_dev *dev) hid_warn(pidff->hid, "unknown ramp effect layout\n"); if (PIDFF_FIND_FIELDS(set_condition, PID_SET_CONDITION, 1)) { - if (test_and_clear_bit(FF_SPRING, dev->ffbit) || - test_and_clear_bit(FF_DAMPER, dev->ffbit) || - test_and_clear_bit(FF_FRICTION, dev->ffbit) || - test_and_clear_bit(FF_INERTIA, dev->ffbit)) + bool test = false; + + test |= test_and_clear_bit(FF_SPRING, dev->ffbit); + test |= test_and_clear_bit(FF_DAMPER, dev->ffbit); + test |= test_and_clear_bit(FF_FRICTION, dev->ffbit); + test |= test_and_clear_bit(FF_INERTIA, dev->ffbit); + if (test) hid_warn(pidff->hid, "unknown condition effect layout\n"); } diff --git a/drivers/hid/wacom_wac.c b/drivers/hid/wacom_wac.c index 9b2c710f8da182..da1f0ea85625dc 100644 --- a/drivers/hid/wacom_wac.c +++ b/drivers/hid/wacom_wac.c @@ -1208,10 +1208,20 @@ static int wacom_intuos_bt_irq(struct wacom_wac *wacom, size_t len) switch (data[0]) { case 0x04: + if (len < 32) { + dev_warn(wacom->pen_input->dev.parent, + "Report 0x04 too short: %zu bytes\n", len); + break; + } wacom_intuos_bt_process_data(wacom, data + i); i += 10; fallthrough; case 0x03: + if (i == 1 && len < 22) { + dev_warn(wacom->pen_input->dev.parent, + "Report 0x03 too short: %zu bytes\n", len); + break; + } wacom_intuos_bt_process_data(wacom, data + i); i += 10; wacom_intuos_bt_process_data(wacom, data + i); diff --git a/drivers/hv/mshv_eventfd.c b/drivers/hv/mshv_eventfd.c index 0b75ff1edb7359..188923fce40b46 100644 --- a/drivers/hv/mshv_eventfd.c +++ b/drivers/hv/mshv_eventfd.c @@ -87,8 +87,9 @@ static void mshv_irqfd_resampler_ack(struct mshv_irq_ack_notifier *mian) idx = srcu_read_lock(&partition->pt_irq_srcu); - hlist_for_each_entry_rcu(irqfd, &resampler->rsmplr_irqfd_list, - irqfd_resampler_hnode) { + hlist_for_each_entry_srcu(irqfd, &resampler->rsmplr_irqfd_list, + irqfd_resampler_hnode, + srcu_read_lock_held(&partition->pt_irq_srcu)) { if (hv_should_clear_interrupt(irqfd->irqfd_lapic_irq.lapic_control.interrupt_type)) hv_call_clear_virtual_interrupt(partition->pt_id); @@ -247,12 +248,13 @@ static void mshv_irqfd_shutdown(struct work_struct *work) { struct mshv_irqfd *irqfd = container_of(work, struct mshv_irqfd, irqfd_shutdown); + u64 cnt; /* * Synchronize with the wait-queue and unhook ourselves to prevent * further events. */ - remove_wait_queue(irqfd->irqfd_wqh, &irqfd->irqfd_wait); + eventfd_ctx_remove_wait_queue(irqfd->irqfd_eventfd_ctx, &irqfd->irqfd_wait, &cnt); if (irqfd->irqfd_resampler) { mshv_irqfd_resampler_shutdown(irqfd); @@ -371,8 +373,6 @@ static void mshv_irqfd_queue_proc(struct file *file, wait_queue_head_t *wqh, struct mshv_irqfd *irqfd = container_of(polltbl, struct mshv_irqfd, irqfd_polltbl); - irqfd->irqfd_wqh = wqh; - /* * TODO: Ensure there isn't already an exclusive, priority waiter, e.g. * that the irqfd isn't already bound to another partition. Only the diff --git a/drivers/hv/mshv_eventfd.h b/drivers/hv/mshv_eventfd.h index 332e7670a3442e..464c6b81ab3364 100644 --- a/drivers/hv/mshv_eventfd.h +++ b/drivers/hv/mshv_eventfd.h @@ -32,7 +32,6 @@ struct mshv_irqfd { struct mshv_lapic_irq irqfd_lapic_irq; struct hlist_node irqfd_hnode; poll_table irqfd_polltbl; - wait_queue_head_t *irqfd_wqh; wait_queue_entry_t irqfd_wait; struct work_struct irqfd_shutdown; struct mshv_irqfd_resampler *irqfd_resampler; diff --git a/drivers/hv/mshv_regions.c b/drivers/hv/mshv_regions.c index adba3564d9f1ad..baa864cac375a7 100644 --- a/drivers/hv/mshv_regions.c +++ b/drivers/hv/mshv_regions.c @@ -314,15 +314,17 @@ int mshv_region_pin(struct mshv_mem_region *region) ret = pin_user_pages_fast(userspace_addr, nr_pages, FOLL_WRITE | FOLL_LONGTERM, pages); - if (ret < 0) + if (ret != nr_pages) goto release_pages; } return 0; release_pages: + if (ret > 0) + done_count += ret; mshv_region_invalidate_pages(region, 0, done_count); - return ret; + return ret < 0 ? ret : -ENOMEM; } static int mshv_region_chunk_unmap(struct mshv_mem_region *region, diff --git a/drivers/hv/mshv_root_hv_call.c b/drivers/hv/mshv_root_hv_call.c index 598eaff4ff2994..1f93b94d7580c3 100644 --- a/drivers/hv/mshv_root_hv_call.c +++ b/drivers/hv/mshv_root_hv_call.c @@ -813,6 +813,13 @@ hv_call_notify_port_ring_empty(u32 sint_index) return hv_result_to_errno(status); } +/* + * Equivalent of hv_call_map_stats_page() for cases when the caller provides + * the map location. + * + * NOTE: This is a newer hypercall that always supports SELF and PARENT stats + * areas, unlike hv_call_map_stats_page(). + */ static int hv_call_map_stats_page2(enum hv_stats_object_type type, const union hv_stats_object_identity *identity, u64 map_location) @@ -855,6 +862,34 @@ static int hv_call_map_stats_page2(enum hv_stats_object_type type, return ret; } +static int +hv_stats_get_area_type(enum hv_stats_object_type type, + const union hv_stats_object_identity *identity) +{ + switch (type) { + case HV_STATS_OBJECT_HYPERVISOR: + return identity->hv.stats_area_type; + case HV_STATS_OBJECT_LOGICAL_PROCESSOR: + return identity->lp.stats_area_type; + case HV_STATS_OBJECT_PARTITION: + return identity->partition.stats_area_type; + case HV_STATS_OBJECT_VP: + return identity->vp.stats_area_type; + } + + return -EINVAL; +} + +/* + * Map a stats page, where the page location is provided by the hypervisor. + * + * NOTE: The concept of separate SELF and PARENT stats areas does not exist on + * older hypervisor versions. All the available stats information can be found + * on the SELF page. When attempting to map the PARENT area on a hypervisor + * that doesn't support it, return "success" but with a NULL address. The + * caller should check for this case and instead fallback to the SELF area + * alone. + */ static int hv_call_map_stats_page(enum hv_stats_object_type type, const union hv_stats_object_identity *identity, void **addr) @@ -863,7 +898,7 @@ static int hv_call_map_stats_page(enum hv_stats_object_type type, struct hv_input_map_stats_page *input; struct hv_output_map_stats_page *output; u64 status, pfn; - int ret = 0; + int hv_status, ret = 0; do { local_irq_save(flags); @@ -878,11 +913,20 @@ static int hv_call_map_stats_page(enum hv_stats_object_type type, pfn = output->map_location; local_irq_restore(flags); - if (hv_result(status) != HV_STATUS_INSUFFICIENT_MEMORY) { - ret = hv_result_to_errno(status); + + hv_status = hv_result(status); + if (hv_status != HV_STATUS_INSUFFICIENT_MEMORY) { if (hv_result_success(status)) break; - return ret; + + if (hv_stats_get_area_type(type, identity) == HV_STATS_AREA_PARENT && + hv_status == HV_STATUS_INVALID_PARAMETER) { + *addr = NULL; + return 0; + } + + hv_status_debug(status, "\n"); + return hv_result_to_errno(status); } ret = hv_call_deposit_pages(NUMA_NO_NODE, diff --git a/drivers/hv/mshv_root_main.c b/drivers/hv/mshv_root_main.c index 681b58154d5eae..5611be36f6a8ea 100644 --- a/drivers/hv/mshv_root_main.c +++ b/drivers/hv/mshv_root_main.c @@ -642,7 +642,7 @@ static bool mshv_handle_gpa_intercept(struct mshv_vp *vp) { struct mshv_partition *p = vp->vp_partition; struct mshv_mem_region *region; - bool ret; + bool ret = false; u64 gfn; #if defined(CONFIG_X86_64) struct hv_x64_memory_intercept_message *msg = @@ -653,6 +653,8 @@ static bool mshv_handle_gpa_intercept(struct mshv_vp *vp) (struct hv_arm64_memory_intercept_message *) vp->vp_intercept_msg_page->u.payload; #endif + enum hv_intercept_access_type access_type = + msg->header.intercept_access_type; gfn = HVPFN_DOWN(msg->guest_physical_address); @@ -660,12 +662,19 @@ static bool mshv_handle_gpa_intercept(struct mshv_vp *vp) if (!region) return false; + if (access_type == HV_INTERCEPT_ACCESS_WRITE && + !(region->hv_map_flags & HV_MAP_GPA_WRITABLE)) + goto put_region; + + if (access_type == HV_INTERCEPT_ACCESS_EXECUTE && + !(region->hv_map_flags & HV_MAP_GPA_EXECUTABLE)) + goto put_region; + /* Only movable memory ranges are supported for GPA intercepts */ if (region->type == MSHV_REGION_TYPE_MEM_MOVABLE) ret = mshv_region_handle_gfn_fault(region, gfn); - else - ret = false; +put_region: mshv_region_put(region); return ret; @@ -993,6 +1002,9 @@ static int mshv_vp_stats_map(u64 partition_id, u32 vp_index, if (err) goto unmap_self; + if (!stats_pages[HV_STATS_AREA_PARENT]) + stats_pages[HV_STATS_AREA_PARENT] = stats_pages[HV_STATS_AREA_SELF]; + return 0; unmap_self: @@ -1331,7 +1343,7 @@ mshv_map_user_memory(struct mshv_partition *partition, return 0; errout: - vfree(region); + mshv_region_put(region); return ret; } diff --git a/drivers/hv/vmbus_drv.c b/drivers/hv/vmbus_drv.c index a53af6fe81a657..1d5cba142828e7 100644 --- a/drivers/hv/vmbus_drv.c +++ b/drivers/hv/vmbus_drv.c @@ -25,6 +25,7 @@ #include #include #include +#include #include #include @@ -1350,7 +1351,7 @@ static void vmbus_message_sched(struct hv_per_cpu_context *hv_cpu, void *message } } -void vmbus_isr(void) +static void __vmbus_isr(void) { struct hv_per_cpu_context *hv_cpu = this_cpu_ptr(hv_context.cpu_context); @@ -1363,6 +1364,53 @@ void vmbus_isr(void) add_interrupt_randomness(vmbus_interrupt); } + +static DEFINE_PER_CPU(bool, vmbus_irq_pending); +static DEFINE_PER_CPU(struct task_struct *, vmbus_irqd); + +static void vmbus_irqd_wake(void) +{ + struct task_struct *tsk = __this_cpu_read(vmbus_irqd); + + __this_cpu_write(vmbus_irq_pending, true); + wake_up_process(tsk); +} + +static void vmbus_irqd_setup(unsigned int cpu) +{ + sched_set_fifo(current); +} + +static int vmbus_irqd_should_run(unsigned int cpu) +{ + return __this_cpu_read(vmbus_irq_pending); +} + +static void run_vmbus_irqd(unsigned int cpu) +{ + __this_cpu_write(vmbus_irq_pending, false); + __vmbus_isr(); +} + +static bool vmbus_irq_initialized; + +static struct smp_hotplug_thread vmbus_irq_threads = { + .store = &vmbus_irqd, + .setup = vmbus_irqd_setup, + .thread_should_run = vmbus_irqd_should_run, + .thread_fn = run_vmbus_irqd, + .thread_comm = "vmbus_irq/%u", +}; + +void vmbus_isr(void) +{ + if (IS_ENABLED(CONFIG_PREEMPT_RT)) { + vmbus_irqd_wake(); + } else { + lockdep_hardirq_threaded(); + __vmbus_isr(); + } +} EXPORT_SYMBOL_FOR_MODULES(vmbus_isr, "mshv_vtl"); static irqreturn_t vmbus_percpu_isr(int irq, void *dev_id) @@ -1462,6 +1510,13 @@ static int vmbus_bus_init(void) * the VMbus interrupt handler. */ + if (IS_ENABLED(CONFIG_PREEMPT_RT) && !vmbus_irq_initialized) { + ret = smpboot_register_percpu_thread(&vmbus_irq_threads); + if (ret) + goto err_kthread; + vmbus_irq_initialized = true; + } + if (vmbus_irq == -1) { hv_setup_vmbus_handler(vmbus_isr); } else { @@ -1507,6 +1562,11 @@ static int vmbus_bus_init(void) free_percpu(vmbus_evt); } err_setup: + if (IS_ENABLED(CONFIG_PREEMPT_RT) && vmbus_irq_initialized) { + smpboot_unregister_percpu_thread(&vmbus_irq_threads); + vmbus_irq_initialized = false; + } +err_kthread: bus_unregister(&hv_bus); return ret; } @@ -2976,6 +3036,10 @@ static void __exit vmbus_exit(void) free_percpu_irq(vmbus_irq, vmbus_evt); free_percpu(vmbus_evt); } + if (IS_ENABLED(CONFIG_PREEMPT_RT) && vmbus_irq_initialized) { + smpboot_unregister_percpu_thread(&vmbus_irq_threads); + vmbus_irq_initialized = false; + } for_each_online_cpu(cpu) { struct hv_per_cpu_context *hv_cpu = per_cpu_ptr(hv_context.cpu_context, cpu); diff --git a/drivers/hwmon/adm1177.c b/drivers/hwmon/adm1177.c index 8b2c965480e3fb..7888afe8dafd66 100644 --- a/drivers/hwmon/adm1177.c +++ b/drivers/hwmon/adm1177.c @@ -10,6 +10,8 @@ #include #include #include +#include +#include #include #include @@ -33,7 +35,7 @@ struct adm1177_state { struct i2c_client *client; u32 r_sense_uohm; - u32 alert_threshold_ua; + u64 alert_threshold_ua; bool vrange_high; }; @@ -48,7 +50,7 @@ static int adm1177_write_cmd(struct adm1177_state *st, u8 cmd) } static int adm1177_write_alert_thr(struct adm1177_state *st, - u32 alert_threshold_ua) + u64 alert_threshold_ua) { u64 val; int ret; @@ -91,8 +93,8 @@ static int adm1177_read(struct device *dev, enum hwmon_sensor_types type, *val = div_u64((105840000ull * dummy), 4096 * st->r_sense_uohm); return 0; - case hwmon_curr_max_alarm: - *val = st->alert_threshold_ua; + case hwmon_curr_max: + *val = div_u64(st->alert_threshold_ua, 1000); return 0; default: return -EOPNOTSUPP; @@ -126,9 +128,10 @@ static int adm1177_write(struct device *dev, enum hwmon_sensor_types type, switch (type) { case hwmon_curr: switch (attr) { - case hwmon_curr_max_alarm: - adm1177_write_alert_thr(st, val); - return 0; + case hwmon_curr_max: + val = clamp_val(val, 0, + div_u64(105840000ULL, st->r_sense_uohm)); + return adm1177_write_alert_thr(st, (u64)val * 1000); default: return -EOPNOTSUPP; } @@ -156,7 +159,7 @@ static umode_t adm1177_is_visible(const void *data, if (st->r_sense_uohm) return 0444; return 0; - case hwmon_curr_max_alarm: + case hwmon_curr_max: if (st->r_sense_uohm) return 0644; return 0; @@ -170,7 +173,7 @@ static umode_t adm1177_is_visible(const void *data, static const struct hwmon_channel_info * const adm1177_info[] = { HWMON_CHANNEL_INFO(curr, - HWMON_C_INPUT | HWMON_C_MAX_ALARM), + HWMON_C_INPUT | HWMON_C_MAX), HWMON_CHANNEL_INFO(in, HWMON_I_INPUT), NULL @@ -192,7 +195,8 @@ static int adm1177_probe(struct i2c_client *client) struct device *dev = &client->dev; struct device *hwmon_dev; struct adm1177_state *st; - u32 alert_threshold_ua; + u64 alert_threshold_ua; + u32 prop; int ret; st = devm_kzalloc(dev, sizeof(*st), GFP_KERNEL); @@ -208,22 +212,26 @@ static int adm1177_probe(struct i2c_client *client) if (device_property_read_u32(dev, "shunt-resistor-micro-ohms", &st->r_sense_uohm)) st->r_sense_uohm = 0; - if (device_property_read_u32(dev, "adi,shutdown-threshold-microamp", - &alert_threshold_ua)) { - if (st->r_sense_uohm) - /* - * set maximum default value from datasheet based on - * shunt-resistor - */ - alert_threshold_ua = div_u64(105840000000, - st->r_sense_uohm); - else - alert_threshold_ua = 0; + if (!device_property_read_u32(dev, "adi,shutdown-threshold-microamp", + &prop)) { + alert_threshold_ua = prop; + } else if (st->r_sense_uohm) { + /* + * set maximum default value from datasheet based on + * shunt-resistor + */ + alert_threshold_ua = div_u64(105840000000ULL, + st->r_sense_uohm); + } else { + alert_threshold_ua = 0; } st->vrange_high = device_property_read_bool(dev, "adi,vrange-high-enable"); - if (alert_threshold_ua && st->r_sense_uohm) - adm1177_write_alert_thr(st, alert_threshold_ua); + if (alert_threshold_ua && st->r_sense_uohm) { + ret = adm1177_write_alert_thr(st, alert_threshold_ua); + if (ret) + return ret; + } ret = adm1177_write_cmd(st, ADM1177_CMD_V_CONT | ADM1177_CMD_I_CONT | diff --git a/drivers/hwmon/aht10.c b/drivers/hwmon/aht10.c index 007befdba9776f..4ce019d2cc80ed 100644 --- a/drivers/hwmon/aht10.c +++ b/drivers/hwmon/aht10.c @@ -37,7 +37,9 @@ #define AHT10_CMD_MEAS 0b10101100 #define AHT10_CMD_RST 0b10111010 -#define DHT20_CMD_INIT 0x71 +#define AHT20_CMD_INIT 0b10111110 + +#define DHT20_CMD_INIT 0b01110001 /* * Flags in the answer byte/command @@ -341,7 +343,7 @@ static int aht10_probe(struct i2c_client *client) data->meas_size = AHT20_MEAS_SIZE; data->crc8 = true; crc8_populate_msb(crc8_table, AHT20_CRC8_POLY); - data->init_cmd = AHT10_CMD_INIT; + data->init_cmd = AHT20_CMD_INIT; break; case dht20: data->meas_size = AHT20_MEAS_SIZE; diff --git a/drivers/hwmon/asus-ec-sensors.c b/drivers/hwmon/asus-ec-sensors.c index 61b18b88ee8ffc..b685d9954df439 100644 --- a/drivers/hwmon/asus-ec-sensors.c +++ b/drivers/hwmon/asus-ec-sensors.c @@ -111,6 +111,8 @@ enum ec_sensors { ec_sensor_temp_mb, /* "T_Sensor" temperature sensor reading [℃] */ ec_sensor_temp_t_sensor, + /* like ec_sensor_temp_t_sensor, but at an alternate address [℃] */ + ec_sensor_temp_t_sensor_alt1, /* VRM temperature [℃] */ ec_sensor_temp_vrm, /* VRM east (right) temperature [℃] */ @@ -160,6 +162,7 @@ enum ec_sensors { #define SENSOR_TEMP_CPU_PACKAGE BIT(ec_sensor_temp_cpu_package) #define SENSOR_TEMP_MB BIT(ec_sensor_temp_mb) #define SENSOR_TEMP_T_SENSOR BIT(ec_sensor_temp_t_sensor) +#define SENSOR_TEMP_T_SENSOR_ALT1 BIT(ec_sensor_temp_t_sensor_alt1) #define SENSOR_TEMP_VRM BIT(ec_sensor_temp_vrm) #define SENSOR_TEMP_VRME BIT(ec_sensor_temp_vrme) #define SENSOR_TEMP_VRMW BIT(ec_sensor_temp_vrmw) @@ -279,6 +282,8 @@ static const struct ec_sensor_info sensors_family_amd_600[] = { EC_SENSOR("VRM", hwmon_temp, 1, 0x00, 0x33), [ec_sensor_temp_t_sensor] = EC_SENSOR("T_Sensor", hwmon_temp, 1, 0x00, 0x36), + [ec_sensor_temp_t_sensor_alt1] = + EC_SENSOR("T_Sensor", hwmon_temp, 1, 0x00, 0x37), [ec_sensor_fan_cpu_opt] = EC_SENSOR("CPU_Opt", hwmon_fan, 2, 0x00, 0xb0), [ec_sensor_temp_water_in] = @@ -509,7 +514,7 @@ static const struct ec_board_info board_info_prime_x570_pro = { static const struct ec_board_info board_info_prime_x670e_pro_wifi = { .sensors = SENSOR_TEMP_CPU | SENSOR_TEMP_CPU_PACKAGE | SENSOR_TEMP_MB | SENSOR_TEMP_VRM | - SENSOR_TEMP_T_SENSOR | SENSOR_FAN_CPU_OPT, + SENSOR_TEMP_T_SENSOR_ALT1 | SENSOR_FAN_CPU_OPT, .mutex_path = ACPI_GLOBAL_LOCK_PSEUDO_PATH, .family = family_amd_600_series, }; @@ -793,6 +798,8 @@ static const struct dmi_system_id dmi_table[] = { &board_info_pro_art_x870E_creator_wifi), DMI_EXACT_MATCH_ASUS_BOARD_NAME("Pro WS TRX50-SAGE WIFI", &board_info_pro_ws_trx50_sage_wifi), + DMI_EXACT_MATCH_ASUS_BOARD_NAME("Pro WS TRX50-SAGE WIFI A", + &board_info_pro_ws_trx50_sage_wifi), DMI_EXACT_MATCH_ASUS_BOARD_NAME("Pro WS WRX90E-SAGE SE", &board_info_pro_ws_wrx90e_sage_se), DMI_EXACT_MATCH_ASUS_BOARD_NAME("Pro WS X570-ACE", diff --git a/drivers/hwmon/axi-fan-control.c b/drivers/hwmon/axi-fan-control.c index b7bb325c3ad966..01590dfa55e60d 100644 --- a/drivers/hwmon/axi-fan-control.c +++ b/drivers/hwmon/axi-fan-control.c @@ -507,7 +507,7 @@ static int axi_fan_control_probe(struct platform_device *pdev) ret = devm_request_threaded_irq(&pdev->dev, ctl->irq, NULL, axi_fan_control_irq_handler, IRQF_ONESHOT | IRQF_TRIGGER_HIGH, - pdev->driver_override, ctl); + NULL, ctl); if (ret) return dev_err_probe(&pdev->dev, ret, "failed to request an irq\n"); diff --git a/drivers/hwmon/dell-smm-hwmon.c b/drivers/hwmon/dell-smm-hwmon.c index 93143cfc157cf7..038edffc1ac747 100644 --- a/drivers/hwmon/dell-smm-hwmon.c +++ b/drivers/hwmon/dell-smm-hwmon.c @@ -1325,6 +1325,13 @@ static const struct dmi_system_id i8k_dmi_table[] __initconst = { DMI_MATCH(DMI_PRODUCT_NAME, "MP061"), }, }, + { + .ident = "Dell OptiPlex 7080", + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."), + DMI_EXACT_MATCH(DMI_PRODUCT_NAME, "OptiPlex 7080"), + }, + }, { .ident = "Dell OptiPlex 7060", .matches = { diff --git a/drivers/hwmon/emc2305.c b/drivers/hwmon/emc2305.c index ceae96c07ac45b..67e82021da2103 100644 --- a/drivers/hwmon/emc2305.c +++ b/drivers/hwmon/emc2305.c @@ -578,6 +578,7 @@ static int emc2305_of_parse_pwm_child(struct device *dev, data->pwm_output_mask |= EMC2305_OPEN_DRAIN << ch; } + of_node_put(args.np); return 0; } diff --git a/drivers/hwmon/f71882fg.c b/drivers/hwmon/f71882fg.c index df83f9866fbcf7..204059d2de6cd2 100644 --- a/drivers/hwmon/f71882fg.c +++ b/drivers/hwmon/f71882fg.c @@ -51,6 +51,7 @@ #define SIO_F81866_ID 0x1010 /* Chipset ID */ #define SIO_F71858AD_ID 0x0903 /* Chipset ID */ #define SIO_F81966_ID 0x1502 /* Chipset ID */ +#define SIO_F81968_ID 0x1806 /* Chipset ID */ #define REGION_LENGTH 8 #define ADDR_REG_OFFSET 5 @@ -2570,6 +2571,7 @@ static int __init f71882fg_find(int sioaddr, struct f71882fg_sio_data *sio_data) break; case SIO_F81866_ID: case SIO_F81966_ID: + case SIO_F81968_ID: sio_data->type = f81866a; break; default: @@ -2599,9 +2601,9 @@ static int __init f71882fg_find(int sioaddr, struct f71882fg_sio_data *sio_data) address &= ~(REGION_LENGTH - 1); /* Ignore 3 LSB */ err = address; - pr_info("Found %s chip at %#x, revision %d\n", + pr_info("Found %s chip at %#x, revision %d, devid: %04x\n", f71882fg_names[sio_data->type], (unsigned int)address, - (int)superio_inb(sioaddr, SIO_REG_DEVREV)); + (int)superio_inb(sioaddr, SIO_REG_DEVREV), devid); exit: superio_exit(sioaddr); return err; diff --git a/drivers/hwmon/ibmpex.c b/drivers/hwmon/ibmpex.c index 129f3a9e8fe965..228c5f6c6f3836 100644 --- a/drivers/hwmon/ibmpex.c +++ b/drivers/hwmon/ibmpex.c @@ -277,9 +277,6 @@ static ssize_t ibmpex_high_low_store(struct device *dev, { struct ibmpex_bmc_data *data = dev_get_drvdata(dev); - if (!data) - return -ENODEV; - ibmpex_reset_high_low_data(data); return count; @@ -511,9 +508,6 @@ static void ibmpex_bmc_delete(struct ibmpex_bmc_data *data) { int i, j; - hwmon_device_unregister(data->hwmon_dev); - dev_set_drvdata(data->bmc_device, NULL); - device_remove_file(data->bmc_device, &sensor_dev_attr_reset_high_low.dev_attr); device_remove_file(data->bmc_device, &dev_attr_name.attr); @@ -527,7 +521,8 @@ static void ibmpex_bmc_delete(struct ibmpex_bmc_data *data) } list_del(&data->list); - + dev_set_drvdata(data->bmc_device, NULL); + hwmon_device_unregister(data->hwmon_dev); ipmi_destroy_user(data->user); kfree(data->sensors); kfree(data); diff --git a/drivers/hwmon/it87.c b/drivers/hwmon/it87.c index e233aafa8856c9..5cfb98a0512f00 100644 --- a/drivers/hwmon/it87.c +++ b/drivers/hwmon/it87.c @@ -3590,10 +3590,13 @@ static int it87_resume(struct device *dev) { struct platform_device *pdev = to_platform_device(dev); struct it87_data *data = dev_get_drvdata(dev); + int err; it87_resume_sio(pdev); - it87_lock(data); + err = it87_lock(data); + if (err) + return err; it87_check_pwm(dev); it87_check_limit_regs(data); diff --git a/drivers/hwmon/macsmc-hwmon.c b/drivers/hwmon/macsmc-hwmon.c index 1c0bbec7e8ebc4..1500ec2cc9f83d 100644 --- a/drivers/hwmon/macsmc-hwmon.c +++ b/drivers/hwmon/macsmc-hwmon.c @@ -22,6 +22,7 @@ #include #include +#include #include #include #include @@ -130,7 +131,7 @@ static int macsmc_hwmon_read_ioft_scaled(struct apple_smc *smc, smc_key key, if (ret < 0) return ret; - *p = mult_frac(val, scale, 65536); + *p = mul_u64_u32_div(val, scale, 65536); return 0; } @@ -140,7 +141,7 @@ static int macsmc_hwmon_read_ioft_scaled(struct apple_smc *smc, smc_key key, * them. */ static int macsmc_hwmon_read_f32_scaled(struct apple_smc *smc, smc_key key, - int *p, int scale) + long *p, int scale) { u32 fval; u64 val; @@ -162,21 +163,21 @@ static int macsmc_hwmon_read_f32_scaled(struct apple_smc *smc, smc_key key, val = 0; else if (exp < 0) val >>= -exp; - else if (exp != 0 && (val & ~((1UL << (64 - exp)) - 1))) /* overflow */ + else if (exp != 0 && (val & ~((1ULL << (64 - exp)) - 1))) /* overflow */ val = U64_MAX; else val <<= exp; if (fval & FLT_SIGN_MASK) { - if (val > (-(s64)INT_MIN)) - *p = INT_MIN; + if (val > (u64)LONG_MAX + 1) + *p = LONG_MIN; else - *p = -val; + *p = -(long)val; } else { - if (val > INT_MAX) - *p = INT_MAX; + if (val > (u64)LONG_MAX) + *p = LONG_MAX; else - *p = val; + *p = (long)val; } return 0; @@ -195,7 +196,7 @@ static int macsmc_hwmon_read_key(struct apple_smc *smc, switch (sensor->info.type_code) { /* 32-bit IEEE 754 float */ case __SMC_KEY('f', 'l', 't', ' '): { - u32 flt_ = 0; + long flt_ = 0; ret = macsmc_hwmon_read_f32_scaled(smc, sensor->macsmc_key, &flt_, scale); @@ -214,7 +215,10 @@ static int macsmc_hwmon_read_key(struct apple_smc *smc, if (ret) return ret; - *val = (long)ioft; + if (ioft > LONG_MAX) + *val = LONG_MAX; + else + *val = (long)ioft; break; } default: @@ -224,29 +228,26 @@ static int macsmc_hwmon_read_key(struct apple_smc *smc, return 0; } -static int macsmc_hwmon_write_f32(struct apple_smc *smc, smc_key key, int value) +static int macsmc_hwmon_write_f32(struct apple_smc *smc, smc_key key, long value) { u64 val; u32 fval = 0; - int exp = 0, neg; + int exp, neg; + neg = value < 0; val = abs(value); - neg = val != value; if (val) { - int msb = __fls(val) - exp; - - if (msb > 23) { - val >>= msb - FLT_MANT_BIAS; - exp -= msb - FLT_MANT_BIAS; - } else if (msb < 23) { - val <<= FLT_MANT_BIAS - msb; - exp += msb; - } + exp = __fls(val); + + if (exp > 23) + val >>= exp - 23; + else + val <<= 23 - exp; fval = FIELD_PREP(FLT_SIGN_MASK, neg) | FIELD_PREP(FLT_EXP_MASK, exp + FLT_EXP_BIAS) | - FIELD_PREP(FLT_MANT_MASK, val); + FIELD_PREP(FLT_MANT_MASK, val & FLT_MANT_MASK); } return apple_smc_write_u32(smc, key, fval); @@ -663,8 +664,8 @@ static int macsmc_hwmon_populate_sensors(struct macsmc_hwmon *hwmon, if (!hwmon->volt.sensors) return -ENOMEM; - for_each_child_of_node_with_prefix(hwmon_node, key_node, "volt-") { - sensor = &hwmon->temp.sensors[hwmon->temp.count]; + for_each_child_of_node_with_prefix(hwmon_node, key_node, "voltage-") { + sensor = &hwmon->volt.sensors[hwmon->volt.count]; if (!macsmc_hwmon_create_sensor(hwmon->dev, hwmon->smc, key_node, sensor)) { sensor->attrs = HWMON_I_INPUT; diff --git a/drivers/hwmon/max16065.c b/drivers/hwmon/max16065.c index 4c9e7892a73c1b..43fbb9b26b102b 100644 --- a/drivers/hwmon/max16065.c +++ b/drivers/hwmon/max16065.c @@ -151,27 +151,27 @@ static struct max16065_data *max16065_update_device(struct device *dev) int i; for (i = 0; i < data->num_adc; i++) - data->adc[i] - = max16065_read_adc(client, MAX16065_ADC(i)); + WRITE_ONCE(data->adc[i], + max16065_read_adc(client, MAX16065_ADC(i))); if (data->have_current) { - data->adc[MAX16065_NUM_ADC] - = max16065_read_adc(client, MAX16065_CSP_ADC); - data->curr_sense - = i2c_smbus_read_byte_data(client, - MAX16065_CURR_SENSE); + WRITE_ONCE(data->adc[MAX16065_NUM_ADC], + max16065_read_adc(client, MAX16065_CSP_ADC)); + WRITE_ONCE(data->curr_sense, + i2c_smbus_read_byte_data(client, MAX16065_CURR_SENSE)); } for (i = 0; i < 2; i++) - data->fault[i] - = i2c_smbus_read_byte_data(client, MAX16065_FAULT(i)); + WRITE_ONCE(data->fault[i], + i2c_smbus_read_byte_data(client, MAX16065_FAULT(i))); /* * MAX16067 and MAX16068 have separate undervoltage and * overvoltage alarm bits. Squash them together. */ if (data->chip == max16067 || data->chip == max16068) - data->fault[0] |= data->fault[1]; + WRITE_ONCE(data->fault[0], + data->fault[0] | data->fault[1]); data->last_updated = jiffies; data->valid = true; @@ -185,7 +185,7 @@ static ssize_t max16065_alarm_show(struct device *dev, { struct sensor_device_attribute_2 *attr2 = to_sensor_dev_attr_2(da); struct max16065_data *data = max16065_update_device(dev); - int val = data->fault[attr2->nr]; + int val = READ_ONCE(data->fault[attr2->nr]); if (val < 0) return val; @@ -203,7 +203,7 @@ static ssize_t max16065_input_show(struct device *dev, { struct sensor_device_attribute *attr = to_sensor_dev_attr(da); struct max16065_data *data = max16065_update_device(dev); - int adc = data->adc[attr->index]; + int adc = READ_ONCE(data->adc[attr->index]); if (unlikely(adc < 0)) return adc; @@ -216,7 +216,7 @@ static ssize_t max16065_current_show(struct device *dev, struct device_attribute *da, char *buf) { struct max16065_data *data = max16065_update_device(dev); - int curr_sense = data->curr_sense; + int curr_sense = READ_ONCE(data->curr_sense); if (unlikely(curr_sense < 0)) return curr_sense; diff --git a/drivers/hwmon/max6639.c b/drivers/hwmon/max6639.c index 99140a2ca9955e..b6b32286d967a9 100644 --- a/drivers/hwmon/max6639.c +++ b/drivers/hwmon/max6639.c @@ -232,7 +232,7 @@ static int max6639_read_fan(struct device *dev, u32 attr, int channel, static int max6639_set_ppr(struct max6639_data *data, int channel, u8 ppr) { /* Decrement the PPR value and shift left by 6 to match the register format */ - return regmap_write(data->regmap, MAX6639_REG_FAN_PPR(channel), ppr-- << 6); + return regmap_write(data->regmap, MAX6639_REG_FAN_PPR(channel), --ppr << 6); } static int max6639_write_fan(struct device *dev, u32 attr, int channel, @@ -524,8 +524,8 @@ static int max6639_probe_child_from_dt(struct i2c_client *client, { struct device *dev = &client->dev; - u32 i; - int err, val; + u32 i, val; + int err; err = of_property_read_u32(child, "reg", &i); if (err) { @@ -540,8 +540,8 @@ static int max6639_probe_child_from_dt(struct i2c_client *client, err = of_property_read_u32(child, "pulses-per-revolution", &val); if (!err) { - if (val < 1 || val > 5) { - dev_err(dev, "invalid pulses-per-revolution %d of %pOFn\n", val, child); + if (val < 1 || val > 4) { + dev_err(dev, "invalid pulses-per-revolution %u of %pOFn\n", val, child); return -EINVAL; } data->ppr[i] = val; @@ -610,7 +610,7 @@ static int max6639_init_client(struct i2c_client *client, return err; /* Fans PWM polarity high by default */ - err = regmap_write(data->regmap, MAX6639_REG_FAN_CONFIG2a(i), 0x00); + err = regmap_write(data->regmap, MAX6639_REG_FAN_CONFIG2a(i), 0x02); if (err) return err; diff --git a/drivers/hwmon/nct6683.c b/drivers/hwmon/nct6683.c index 6cda35388b24c9..4a838041403862 100644 --- a/drivers/hwmon/nct6683.c +++ b/drivers/hwmon/nct6683.c @@ -181,6 +181,7 @@ superio_exit(int ioreg) #define NCT6683_CUSTOMER_ID_ASROCK2 0xe1b #define NCT6683_CUSTOMER_ID_ASROCK3 0x1631 #define NCT6683_CUSTOMER_ID_ASROCK4 0x163e +#define NCT6683_CUSTOMER_ID_ASROCK5 0x1621 #define NCT6683_REG_BUILD_YEAR 0x604 #define NCT6683_REG_BUILD_MONTH 0x605 @@ -1242,6 +1243,8 @@ static int nct6683_probe(struct platform_device *pdev) break; case NCT6683_CUSTOMER_ID_ASROCK4: break; + case NCT6683_CUSTOMER_ID_ASROCK5: + break; default: if (!force) return -ENODEV; diff --git a/drivers/hwmon/nct6775-platform.c b/drivers/hwmon/nct6775-platform.c index c3a719aef1ace2..555029dfe713f2 100644 --- a/drivers/hwmon/nct6775-platform.c +++ b/drivers/hwmon/nct6775-platform.c @@ -1357,6 +1357,7 @@ static const char * const asus_msi_boards[] = { "Pro WS W680-ACE IPMI", "Pro WS W790-ACE", "Pro WS W790E-SAGE SE", + "Pro WS WRX90E-SAGE SE", "ProArt B650-CREATOR", "ProArt B660-CREATOR D4", "ProArt B760-CREATOR D4", diff --git a/drivers/hwmon/nct7363.c b/drivers/hwmon/nct7363.c index 71cef794835df7..47fc1b4a0f3f9e 100644 --- a/drivers/hwmon/nct7363.c +++ b/drivers/hwmon/nct7363.c @@ -349,6 +349,7 @@ static int nct7363_present_pwm_fanin(struct device *dev, if (ret) return ret; + of_node_put(args.np); if (args.args[0] >= NCT7363_PWM_COUNT) return -EINVAL; data->pwm_mask |= BIT(args.args[0]); diff --git a/drivers/hwmon/occ/common.c b/drivers/hwmon/occ/common.c index 89928d38831b61..42cc6068bb08be 100644 --- a/drivers/hwmon/occ/common.c +++ b/drivers/hwmon/occ/common.c @@ -420,6 +420,12 @@ static ssize_t occ_show_freq_2(struct device *dev, return sysfs_emit(buf, "%u\n", val); } +static u64 occ_get_powr_avg(u64 accum, u32 samples) +{ + return (samples == 0) ? 0 : + mul_u64_u32_div(accum, 1000000UL, samples); +} + static ssize_t occ_show_power_1(struct device *dev, struct device_attribute *attr, char *buf) { @@ -441,9 +447,8 @@ static ssize_t occ_show_power_1(struct device *dev, val = get_unaligned_be16(&power->sensor_id); break; case 1: - val = get_unaligned_be32(&power->accumulator) / - get_unaligned_be32(&power->update_tag); - val *= 1000000ULL; + val = occ_get_powr_avg(get_unaligned_be32(&power->accumulator), + get_unaligned_be32(&power->update_tag)); break; case 2: val = (u64)get_unaligned_be32(&power->update_tag) * @@ -459,12 +464,6 @@ static ssize_t occ_show_power_1(struct device *dev, return sysfs_emit(buf, "%llu\n", val); } -static u64 occ_get_powr_avg(u64 accum, u32 samples) -{ - return (samples == 0) ? 0 : - mul_u64_u32_div(accum, 1000000UL, samples); -} - static ssize_t occ_show_power_2(struct device *dev, struct device_attribute *attr, char *buf) { @@ -725,7 +724,7 @@ static ssize_t occ_show_extended(struct device *dev, switch (sattr->nr) { case 0: if (extn->flags & EXTN_FLAG_SENSOR_ID) { - rc = sysfs_emit(buf, "%u", + rc = sysfs_emit(buf, "%u\n", get_unaligned_be32(&extn->sensor_id)); } else { rc = sysfs_emit(buf, "%4phN\n", extn->name); diff --git a/drivers/hwmon/peci/cputemp.c b/drivers/hwmon/peci/cputemp.c index b2fc936851e145..457089c561b401 100644 --- a/drivers/hwmon/peci/cputemp.c +++ b/drivers/hwmon/peci/cputemp.c @@ -131,7 +131,7 @@ static int get_temp_target(struct peci_cputemp *priv, enum peci_temp_target_type *val = priv->temp.target.tjmax; break; case crit_hyst_type: - *val = priv->temp.target.tjmax - priv->temp.target.tcontrol; + *val = priv->temp.target.tcontrol; break; default: ret = -EOPNOTSUPP; @@ -319,7 +319,7 @@ static umode_t cputemp_is_visible(const void *data, enum hwmon_sensor_types type { const struct peci_cputemp *priv = data; - if (channel > CPUTEMP_CHANNEL_NUMS) + if (channel >= CPUTEMP_CHANNEL_NUMS) return 0; if (channel < channel_core) diff --git a/drivers/hwmon/pmbus/ina233.c b/drivers/hwmon/pmbus/ina233.c index dde1e16783943a..7aebd854763a62 100644 --- a/drivers/hwmon/pmbus/ina233.c +++ b/drivers/hwmon/pmbus/ina233.c @@ -67,10 +67,13 @@ static int ina233_read_word_data(struct i2c_client *client, int page, switch (reg) { case PMBUS_VIRT_READ_VMON: ret = pmbus_read_word_data(client, 0, 0xff, MFR_READ_VSHUNT); + if (ret < 0) + return ret; /* Adjust returned value to match VIN coefficients */ /* VIN: 1.25 mV VSHUNT: 2.5 uV LSB */ - ret = DIV_ROUND_CLOSEST(ret * 25, 12500); + ret = clamp_val(DIV_ROUND_CLOSEST((s16)ret * 25, 12500), + S16_MIN, S16_MAX) & 0xffff; break; default: ret = -ENODATA; diff --git a/drivers/hwmon/pmbus/isl68137.c b/drivers/hwmon/pmbus/isl68137.c index 97b61836f53a45..3e3a887aad050f 100644 --- a/drivers/hwmon/pmbus/isl68137.c +++ b/drivers/hwmon/pmbus/isl68137.c @@ -96,10 +96,21 @@ static ssize_t isl68137_avs_enable_show_page(struct i2c_client *client, int page, char *buf) { - int val = pmbus_read_byte_data(client, page, PMBUS_OPERATION); + int val; - return sprintf(buf, "%d\n", - (val & ISL68137_VOUT_AVS) == ISL68137_VOUT_AVS ? 1 : 0); + val = pmbus_lock_interruptible(client); + if (val) + return val; + + val = pmbus_read_byte_data(client, page, PMBUS_OPERATION); + + pmbus_unlock(client); + + if (val < 0) + return val; + + return sysfs_emit(buf, "%d\n", + (val & ISL68137_VOUT_AVS) == ISL68137_VOUT_AVS); } static ssize_t isl68137_avs_enable_store_page(struct i2c_client *client, @@ -115,6 +126,10 @@ static ssize_t isl68137_avs_enable_store_page(struct i2c_client *client, op_val = result ? ISL68137_VOUT_AVS : 0; + rc = pmbus_lock_interruptible(client); + if (rc) + return rc; + /* * Writes to VOUT setpoint over AVSBus will persist after the VRM is * switched to PMBus control. Switching back to AVSBus control @@ -126,17 +141,20 @@ static ssize_t isl68137_avs_enable_store_page(struct i2c_client *client, rc = pmbus_read_word_data(client, page, 0xff, PMBUS_VOUT_COMMAND); if (rc < 0) - return rc; + goto unlock; rc = pmbus_write_word_data(client, page, PMBUS_VOUT_COMMAND, rc); if (rc < 0) - return rc; + goto unlock; } rc = pmbus_update_byte_data(client, page, PMBUS_OPERATION, ISL68137_VOUT_AVS, op_val); +unlock: + pmbus_unlock(client); + return (rc < 0) ? rc : count; } diff --git a/drivers/hwmon/pmbus/ltc4286.c b/drivers/hwmon/pmbus/ltc4286.c index aabd0bcdfeee39..8715d380784a02 100644 --- a/drivers/hwmon/pmbus/ltc4286.c +++ b/drivers/hwmon/pmbus/ltc4286.c @@ -173,3 +173,4 @@ module_i2c_driver(ltc4286_driver); MODULE_AUTHOR("Delphine CC Chiu "); MODULE_DESCRIPTION("PMBUS driver for LTC4286 and compatibles"); MODULE_LICENSE("GPL"); +MODULE_IMPORT_NS("PMBUS"); diff --git a/drivers/hwmon/pmbus/mp2869.c b/drivers/hwmon/pmbus/mp2869.c index cc69a1e91dfe8a..4647892e511214 100644 --- a/drivers/hwmon/pmbus/mp2869.c +++ b/drivers/hwmon/pmbus/mp2869.c @@ -165,7 +165,7 @@ static int mp2869_read_byte_data(struct i2c_client *client, int page, int reg) { const struct pmbus_driver_info *info = pmbus_get_driver_info(client); struct mp2869_data *data = to_mp2869_data(info); - int ret; + int ret, mfr; switch (reg) { case PMBUS_VOUT_MODE: @@ -188,11 +188,14 @@ static int mp2869_read_byte_data(struct i2c_client *client, int page, int reg) if (ret < 0) return ret; + mfr = pmbus_read_byte_data(client, page, + PMBUS_STATUS_MFR_SPECIFIC); + if (mfr < 0) + return mfr; + ret = (ret & ~GENMASK(2, 2)) | FIELD_PREP(GENMASK(2, 2), - FIELD_GET(GENMASK(1, 1), - pmbus_read_byte_data(client, page, - PMBUS_STATUS_MFR_SPECIFIC))); + FIELD_GET(GENMASK(1, 1), mfr)); break; case PMBUS_STATUS_TEMPERATURE: /* @@ -207,15 +210,16 @@ static int mp2869_read_byte_data(struct i2c_client *client, int page, int reg) if (ret < 0) return ret; + mfr = pmbus_read_byte_data(client, page, + PMBUS_STATUS_MFR_SPECIFIC); + if (mfr < 0) + return mfr; + ret = (ret & ~GENMASK(7, 6)) | FIELD_PREP(GENMASK(6, 6), - FIELD_GET(GENMASK(1, 1), - pmbus_read_byte_data(client, page, - PMBUS_STATUS_MFR_SPECIFIC))) | + FIELD_GET(GENMASK(1, 1), mfr)) | FIELD_PREP(GENMASK(7, 7), - FIELD_GET(GENMASK(1, 1), - pmbus_read_byte_data(client, page, - PMBUS_STATUS_MFR_SPECIFIC))); + FIELD_GET(GENMASK(1, 1), mfr)); break; default: ret = -ENODATA; @@ -230,7 +234,7 @@ static int mp2869_read_word_data(struct i2c_client *client, int page, int phase, { const struct pmbus_driver_info *info = pmbus_get_driver_info(client); struct mp2869_data *data = to_mp2869_data(info); - int ret; + int ret, mfr; switch (reg) { case PMBUS_STATUS_WORD: @@ -246,11 +250,14 @@ static int mp2869_read_word_data(struct i2c_client *client, int page, int phase, if (ret < 0) return ret; + mfr = pmbus_read_byte_data(client, page, + PMBUS_STATUS_MFR_SPECIFIC); + if (mfr < 0) + return mfr; + ret = (ret & ~GENMASK(2, 2)) | FIELD_PREP(GENMASK(2, 2), - FIELD_GET(GENMASK(1, 1), - pmbus_read_byte_data(client, page, - PMBUS_STATUS_MFR_SPECIFIC))); + FIELD_GET(GENMASK(1, 1), mfr)); break; case PMBUS_READ_VIN: /* diff --git a/drivers/hwmon/pmbus/mp2975.c b/drivers/hwmon/pmbus/mp2975.c index c31982d8519623..d0bc47b12cb07d 100644 --- a/drivers/hwmon/pmbus/mp2975.c +++ b/drivers/hwmon/pmbus/mp2975.c @@ -313,6 +313,8 @@ static int mp2973_read_word_data(struct i2c_client *client, int page, case PMBUS_STATUS_WORD: /* MP2973 & MP2971 return PGOOD instead of PB_STATUS_POWER_GOOD_N. */ ret = pmbus_read_word_data(client, page, phase, reg); + if (ret < 0) + return ret; ret ^= PB_STATUS_POWER_GOOD_N; break; case PMBUS_OT_FAULT_LIMIT: diff --git a/drivers/hwmon/pmbus/mpq8785.c b/drivers/hwmon/pmbus/mpq8785.c index 1f56aaf4dde807..87bd039c77b9b3 100644 --- a/drivers/hwmon/pmbus/mpq8785.c +++ b/drivers/hwmon/pmbus/mpq8785.c @@ -47,6 +47,33 @@ static int mpq8785_identify(struct i2c_client *client, return 0; }; +static int mpq8785_read_byte_data(struct i2c_client *client, int page, int reg) +{ + int ret; + + switch (reg) { + case PMBUS_VOUT_MODE: + ret = pmbus_read_byte_data(client, page, reg); + if (ret < 0) + return ret; + + if ((ret >> 5) == 1) { + /* + * The MPQ8785 chip reports VOUT_MODE as VID mode, but the driver + * treats VID as direct mode. Without this, identification would fail + * due to mode mismatch. + * This override ensures the reported mode matches the driver + * configuration, allowing successful initialization. + */ + return PB_VOUT_MODE_DIRECT; + } + + return ret; + default: + return -ENODATA; + } +} + static int mpm82504_read_word_data(struct i2c_client *client, int page, int phase, int reg) { @@ -129,6 +156,7 @@ static int mpq8785_probe(struct i2c_client *client) break; case mpq8785: info->identify = mpq8785_identify; + info->read_byte_data = mpq8785_read_byte_data; break; default: return -ENODEV; diff --git a/drivers/hwmon/pmbus/pmbus_core.c b/drivers/hwmon/pmbus/pmbus_core.c index be6d05def1152b..572be3ebc03df1 100644 --- a/drivers/hwmon/pmbus/pmbus_core.c +++ b/drivers/hwmon/pmbus/pmbus_core.c @@ -6,6 +6,7 @@ * Copyright (c) 2012 Guenter Roeck */ +#include #include #include #include @@ -21,8 +22,8 @@ #include #include #include -#include #include +#include #include "pmbus.h" /* @@ -112,6 +113,11 @@ struct pmbus_data { struct mutex update_lock; +#if IS_ENABLED(CONFIG_REGULATOR) + atomic_t regulator_events[PMBUS_PAGES]; + struct work_struct regulator_notify_work; +#endif + bool has_status_word; /* device uses STATUS_WORD register */ int (*read_status)(struct i2c_client *client, int page); @@ -1209,6 +1215,12 @@ static ssize_t pmbus_show_boolean(struct device *dev, return sysfs_emit(buf, "%d\n", val); } +static ssize_t pmbus_show_zero(struct device *dev, + struct device_attribute *devattr, char *buf) +{ + return sysfs_emit(buf, "0\n"); +} + static ssize_t pmbus_show_sensor(struct device *dev, struct device_attribute *devattr, char *buf) { @@ -1407,7 +1419,7 @@ static struct pmbus_sensor *pmbus_add_sensor(struct pmbus_data *data, int reg, enum pmbus_sensor_classes class, bool update, bool readonly, - bool convert) + bool writeonly, bool convert) { struct pmbus_sensor *sensor; struct device_attribute *a; @@ -1436,7 +1448,8 @@ static struct pmbus_sensor *pmbus_add_sensor(struct pmbus_data *data, sensor->data = -ENODATA; pmbus_dev_attr_init(a, sensor->name, readonly ? 0444 : 0644, - pmbus_show_sensor, pmbus_set_sensor); + writeonly ? pmbus_show_zero : pmbus_show_sensor, + pmbus_set_sensor); if (pmbus_add_attribute(data, &a->attr)) return NULL; @@ -1495,8 +1508,10 @@ static int pmbus_add_label(struct pmbus_data *data, struct pmbus_limit_attr { u16 reg; /* Limit register */ u16 sbit; /* Alarm attribute status bit */ - bool update; /* True if register needs updates */ - bool low; /* True if low limit; for limits with compare functions only */ + bool readonly:1; /* True if the attribute is read-only */ + bool writeonly:1; /* True if the attribute is write-only */ + bool update:1; /* True if register needs updates */ + bool low:1; /* True if low limit; for limits with compare functions only */ const char *attr; /* Attribute name */ const char *alarm; /* Alarm attribute name */ }; @@ -1511,9 +1526,9 @@ struct pmbus_sensor_attr { u8 nlimit; /* # of limit registers */ enum pmbus_sensor_classes class;/* sensor class */ const char *label; /* sensor label */ - bool paged; /* true if paged sensor */ - bool update; /* true if update needed */ - bool compare; /* true if compare function needed */ + bool paged:1; /* true if paged sensor */ + bool update:1; /* true if update needed */ + bool compare:1; /* true if compare function needed */ u32 func; /* sensor mask */ u32 sfunc; /* sensor status mask */ int sreg; /* status register */ @@ -1544,7 +1559,7 @@ static int pmbus_add_limit_attrs(struct i2c_client *client, curr = pmbus_add_sensor(data, name, l->attr, index, page, 0xff, l->reg, attr->class, attr->update || l->update, - false, true); + l->readonly, l->writeonly, true); if (!curr) return -ENOMEM; if (l->sbit && (info->func[page] & attr->sfunc)) { @@ -1584,7 +1599,7 @@ static int pmbus_add_sensor_attrs_one(struct i2c_client *client, return ret; } base = pmbus_add_sensor(data, name, "input", index, page, phase, - attr->reg, attr->class, true, true, true); + attr->reg, attr->class, true, true, false, true); if (!base) return -ENOMEM; /* No limit and alarm attributes for phase specific sensors */ @@ -1707,23 +1722,29 @@ static const struct pmbus_limit_attr vin_limit_attrs[] = { }, { .reg = PMBUS_VIRT_READ_VIN_AVG, .update = true, + .readonly = true, .attr = "average", }, { .reg = PMBUS_VIRT_READ_VIN_MIN, .update = true, + .readonly = true, .attr = "lowest", }, { .reg = PMBUS_VIRT_READ_VIN_MAX, .update = true, + .readonly = true, .attr = "highest", }, { .reg = PMBUS_VIRT_RESET_VIN_HISTORY, + .writeonly = true, .attr = "reset_history", }, { .reg = PMBUS_MFR_VIN_MIN, + .readonly = true, .attr = "rated_min", }, { .reg = PMBUS_MFR_VIN_MAX, + .readonly = true, .attr = "rated_max", }, }; @@ -1776,23 +1797,29 @@ static const struct pmbus_limit_attr vout_limit_attrs[] = { }, { .reg = PMBUS_VIRT_READ_VOUT_AVG, .update = true, + .readonly = true, .attr = "average", }, { .reg = PMBUS_VIRT_READ_VOUT_MIN, .update = true, + .readonly = true, .attr = "lowest", }, { .reg = PMBUS_VIRT_READ_VOUT_MAX, .update = true, + .readonly = true, .attr = "highest", }, { .reg = PMBUS_VIRT_RESET_VOUT_HISTORY, + .writeonly = true, .attr = "reset_history", }, { .reg = PMBUS_MFR_VOUT_MIN, + .readonly = true, .attr = "rated_min", }, { .reg = PMBUS_MFR_VOUT_MAX, + .readonly = true, .attr = "rated_max", }, }; @@ -1852,20 +1879,25 @@ static const struct pmbus_limit_attr iin_limit_attrs[] = { }, { .reg = PMBUS_VIRT_READ_IIN_AVG, .update = true, + .readonly = true, .attr = "average", }, { .reg = PMBUS_VIRT_READ_IIN_MIN, .update = true, + .readonly = true, .attr = "lowest", }, { .reg = PMBUS_VIRT_READ_IIN_MAX, .update = true, + .readonly = true, .attr = "highest", }, { .reg = PMBUS_VIRT_RESET_IIN_HISTORY, + .writeonly = true, .attr = "reset_history", }, { .reg = PMBUS_MFR_IIN_MAX, + .readonly = true, .attr = "rated_max", }, }; @@ -1889,20 +1921,25 @@ static const struct pmbus_limit_attr iout_limit_attrs[] = { }, { .reg = PMBUS_VIRT_READ_IOUT_AVG, .update = true, + .readonly = true, .attr = "average", }, { .reg = PMBUS_VIRT_READ_IOUT_MIN, .update = true, + .readonly = true, .attr = "lowest", }, { .reg = PMBUS_VIRT_READ_IOUT_MAX, .update = true, + .readonly = true, .attr = "highest", }, { .reg = PMBUS_VIRT_RESET_IOUT_HISTORY, + .writeonly = true, .attr = "reset_history", }, { .reg = PMBUS_MFR_IOUT_MAX, + .readonly = true, .attr = "rated_max", }, }; @@ -1943,20 +1980,25 @@ static const struct pmbus_limit_attr pin_limit_attrs[] = { }, { .reg = PMBUS_VIRT_READ_PIN_AVG, .update = true, + .readonly = true, .attr = "average", }, { .reg = PMBUS_VIRT_READ_PIN_MIN, .update = true, + .readonly = true, .attr = "input_lowest", }, { .reg = PMBUS_VIRT_READ_PIN_MAX, .update = true, + .readonly = true, .attr = "input_highest", }, { .reg = PMBUS_VIRT_RESET_PIN_HISTORY, + .writeonly = true, .attr = "reset_history", }, { .reg = PMBUS_MFR_PIN_MAX, + .readonly = true, .attr = "rated_max", }, }; @@ -1980,20 +2022,25 @@ static const struct pmbus_limit_attr pout_limit_attrs[] = { }, { .reg = PMBUS_VIRT_READ_POUT_AVG, .update = true, + .readonly = true, .attr = "average", }, { .reg = PMBUS_VIRT_READ_POUT_MIN, .update = true, + .readonly = true, .attr = "input_lowest", }, { .reg = PMBUS_VIRT_READ_POUT_MAX, .update = true, + .readonly = true, .attr = "input_highest", }, { .reg = PMBUS_VIRT_RESET_POUT_HISTORY, + .writeonly = true, .attr = "reset_history", }, { .reg = PMBUS_MFR_POUT_MAX, + .readonly = true, .attr = "rated_max", }, }; @@ -2049,18 +2096,23 @@ static const struct pmbus_limit_attr temp_limit_attrs[] = { .sbit = PB_TEMP_OT_FAULT, }, { .reg = PMBUS_VIRT_READ_TEMP_MIN, + .readonly = true, .attr = "lowest", }, { .reg = PMBUS_VIRT_READ_TEMP_AVG, + .readonly = true, .attr = "average", }, { .reg = PMBUS_VIRT_READ_TEMP_MAX, + .readonly = true, .attr = "highest", }, { .reg = PMBUS_VIRT_RESET_TEMP_HISTORY, + .writeonly = true, .attr = "reset_history", }, { .reg = PMBUS_MFR_MAX_TEMP_1, + .readonly = true, .attr = "rated_max", }, }; @@ -2090,18 +2142,23 @@ static const struct pmbus_limit_attr temp_limit_attrs2[] = { .sbit = PB_TEMP_OT_FAULT, }, { .reg = PMBUS_VIRT_READ_TEMP2_MIN, + .readonly = true, .attr = "lowest", }, { .reg = PMBUS_VIRT_READ_TEMP2_AVG, + .readonly = true, .attr = "average", }, { .reg = PMBUS_VIRT_READ_TEMP2_MAX, + .readonly = true, .attr = "highest", }, { .reg = PMBUS_VIRT_RESET_TEMP2_HISTORY, + .writeonly = true, .attr = "reset_history", }, { .reg = PMBUS_MFR_MAX_TEMP_2, + .readonly = true, .attr = "rated_max", }, }; @@ -2131,6 +2188,7 @@ static const struct pmbus_limit_attr temp_limit_attrs3[] = { .sbit = PB_TEMP_OT_FAULT, }, { .reg = PMBUS_MFR_MAX_TEMP_3, + .readonly = true, .attr = "rated_max", }, }; @@ -2214,7 +2272,7 @@ static int pmbus_add_fan_ctrl(struct i2c_client *client, sensor = pmbus_add_sensor(data, "fan", "target", index, page, 0xff, PMBUS_VIRT_FAN_TARGET_1 + id, PSC_FAN, - false, false, true); + false, false, false, true); if (!sensor) return -ENOMEM; @@ -2225,14 +2283,14 @@ static int pmbus_add_fan_ctrl(struct i2c_client *client, sensor = pmbus_add_sensor(data, "pwm", NULL, index, page, 0xff, PMBUS_VIRT_PWM_1 + id, PSC_PWM, - false, false, true); + false, false, false, true); if (!sensor) return -ENOMEM; sensor = pmbus_add_sensor(data, "pwm", "enable", index, page, 0xff, PMBUS_VIRT_PWM_ENABLE_1 + id, PSC_PWM, - true, false, false); + true, false, false, false); if (!sensor) return -ENOMEM; @@ -2274,7 +2332,7 @@ static int pmbus_add_fan_attributes(struct i2c_client *client, if (pmbus_add_sensor(data, "fan", "input", index, page, 0xff, pmbus_fan_registers[f], - PSC_FAN, true, true, true) == NULL) + PSC_FAN, true, true, false, true) == NULL) return -ENOMEM; /* Fan control */ @@ -3176,12 +3234,19 @@ static int pmbus_regulator_get_voltage(struct regulator_dev *rdev) .class = PSC_VOLTAGE_OUT, .convert = true, }; + int ret; + mutex_lock(&data->update_lock); s.data = _pmbus_read_word_data(client, s.page, 0xff, PMBUS_READ_VOUT); - if (s.data < 0) - return s.data; + if (s.data < 0) { + ret = s.data; + goto unlock; + } - return (int)pmbus_reg2data(data, &s) * 1000; /* unit is uV */ + ret = (int)pmbus_reg2data(data, &s) * 1000; /* unit is uV */ +unlock: + mutex_unlock(&data->update_lock); + return ret; } static int pmbus_regulator_set_voltage(struct regulator_dev *rdev, int min_uv, @@ -3198,16 +3263,22 @@ static int pmbus_regulator_set_voltage(struct regulator_dev *rdev, int min_uv, }; int val = DIV_ROUND_CLOSEST(min_uv, 1000); /* convert to mV */ int low, high; + int ret; *selector = 0; + mutex_lock(&data->update_lock); low = pmbus_regulator_get_low_margin(client, s.page); - if (low < 0) - return low; + if (low < 0) { + ret = low; + goto unlock; + } high = pmbus_regulator_get_high_margin(client, s.page); - if (high < 0) - return high; + if (high < 0) { + ret = high; + goto unlock; + } /* Make sure we are within margins */ if (low > val) @@ -3217,7 +3288,10 @@ static int pmbus_regulator_set_voltage(struct regulator_dev *rdev, int min_uv, val = pmbus_data2reg(data, &s, val); - return _pmbus_write_word_data(client, s.page, PMBUS_VOUT_COMMAND, (u16)val); + ret = _pmbus_write_word_data(client, s.page, PMBUS_VOUT_COMMAND, (u16)val); +unlock: + mutex_unlock(&data->update_lock); + return ret; } static int pmbus_regulator_list_voltage(struct regulator_dev *rdev, @@ -3227,6 +3301,7 @@ static int pmbus_regulator_list_voltage(struct regulator_dev *rdev, struct i2c_client *client = to_i2c_client(dev->parent); struct pmbus_data *data = i2c_get_clientdata(client); int val, low, high; + int ret; if (data->flags & PMBUS_VOUT_PROTECTED) return 0; @@ -3239,18 +3314,29 @@ static int pmbus_regulator_list_voltage(struct regulator_dev *rdev, val = DIV_ROUND_CLOSEST(rdev->desc->min_uV + (rdev->desc->uV_step * selector), 1000); /* convert to mV */ + mutex_lock(&data->update_lock); + low = pmbus_regulator_get_low_margin(client, rdev_get_id(rdev)); - if (low < 0) - return low; + if (low < 0) { + ret = low; + goto unlock; + } high = pmbus_regulator_get_high_margin(client, rdev_get_id(rdev)); - if (high < 0) - return high; + if (high < 0) { + ret = high; + goto unlock; + } - if (val >= low && val <= high) - return val * 1000; /* unit is uV */ + if (val >= low && val <= high) { + ret = val * 1000; /* unit is uV */ + goto unlock; + } - return 0; + ret = 0; +unlock: + mutex_unlock(&data->update_lock); + return ret; } const struct regulator_ops pmbus_regulator_ops = { @@ -3281,12 +3367,42 @@ int pmbus_regulator_init_cb(struct regulator_dev *rdev, } EXPORT_SYMBOL_NS_GPL(pmbus_regulator_init_cb, "PMBUS"); +static void pmbus_regulator_notify_work_cancel(void *data) +{ + struct pmbus_data *pdata = data; + + cancel_work_sync(&pdata->regulator_notify_work); +} + +static void pmbus_regulator_notify_worker(struct work_struct *work) +{ + struct pmbus_data *data = + container_of(work, struct pmbus_data, regulator_notify_work); + int i, j; + + for (i = 0; i < data->info->pages; i++) { + int event; + + event = atomic_xchg(&data->regulator_events[i], 0); + if (!event) + continue; + + for (j = 0; j < data->info->num_regulators; j++) { + if (i == rdev_get_id(data->rdevs[j])) { + regulator_notifier_call_chain(data->rdevs[j], + event, NULL); + break; + } + } + } +} + static int pmbus_regulator_register(struct pmbus_data *data) { struct device *dev = data->dev; const struct pmbus_driver_info *info = data->info; const struct pmbus_platform_data *pdata = dev_get_platdata(dev); - int i; + int i, ret; data->rdevs = devm_kzalloc(dev, sizeof(struct regulator_dev *) * info->num_regulators, GFP_KERNEL); @@ -3310,19 +3426,19 @@ static int pmbus_regulator_register(struct pmbus_data *data) info->reg_desc[i].name); } + INIT_WORK(&data->regulator_notify_work, pmbus_regulator_notify_worker); + + ret = devm_add_action_or_reset(dev, pmbus_regulator_notify_work_cancel, data); + if (ret) + return ret; + return 0; } static void pmbus_regulator_notify(struct pmbus_data *data, int page, int event) { - int j; - - for (j = 0; j < data->info->num_regulators; j++) { - if (page == rdev_get_id(data->rdevs[j])) { - regulator_notifier_call_chain(data->rdevs[j], event, NULL); - break; - } - } + atomic_or(event, &data->regulator_events[page]); + schedule_work(&data->regulator_notify_work); } #else static int pmbus_regulator_register(struct pmbus_data *data) diff --git a/drivers/hwmon/pmbus/pxe1610.c b/drivers/hwmon/pmbus/pxe1610.c index 6a4a978eca7e89..24c1f961c76689 100644 --- a/drivers/hwmon/pmbus/pxe1610.c +++ b/drivers/hwmon/pmbus/pxe1610.c @@ -104,7 +104,10 @@ static int pxe1610_probe(struct i2c_client *client) * By default this device doesn't boot to page 0, so set page 0 * to access all pmbus registers. */ - i2c_smbus_write_byte_data(client, PMBUS_PAGE, 0); + ret = i2c_smbus_write_byte_data(client, PMBUS_PAGE, 0); + if (ret < 0) + return dev_err_probe(&client->dev, ret, + "Failed to set page 0\n"); /* Read Manufacturer id */ ret = i2c_smbus_read_block_data(client, PMBUS_MFR_ID, buf); diff --git a/drivers/hwmon/pmbus/q54sj108a2.c b/drivers/hwmon/pmbus/q54sj108a2.c index 4d7086d83aa366..7b0d292a425aef 100644 --- a/drivers/hwmon/pmbus/q54sj108a2.c +++ b/drivers/hwmon/pmbus/q54sj108a2.c @@ -78,7 +78,8 @@ static ssize_t q54sj108a2_debugfs_read(struct file *file, char __user *buf, int idx = *idxp; struct q54sj108a2_data *psu = to_psu(idxp, idx); char data[I2C_SMBUS_BLOCK_MAX + 2] = { 0 }; - char data_char[I2C_SMBUS_BLOCK_MAX + 2] = { 0 }; + char data_char[I2C_SMBUS_BLOCK_MAX * 2 + 2] = { 0 }; + char *out = data; char *res; switch (idx) { @@ -149,27 +150,27 @@ static ssize_t q54sj108a2_debugfs_read(struct file *file, char __user *buf, if (rc < 0) return rc; - res = bin2hex(data, data_char, 32); - rc = res - data; - + res = bin2hex(data_char, data, rc); + rc = res - data_char; + out = data_char; break; case Q54SJ108A2_DEBUGFS_FLASH_KEY: rc = i2c_smbus_read_block_data(psu->client, PMBUS_FLASH_KEY_WRITE, data); if (rc < 0) return rc; - res = bin2hex(data, data_char, 4); - rc = res - data; - + res = bin2hex(data_char, data, rc); + rc = res - data_char; + out = data_char; break; default: return -EINVAL; } - data[rc] = '\n'; + out[rc] = '\n'; rc += 2; - return simple_read_from_buffer(buf, count, ppos, data, rc); + return simple_read_from_buffer(buf, count, ppos, out, rc); } static ssize_t q54sj108a2_debugfs_write(struct file *file, const char __user *buf, diff --git a/drivers/hwmon/pmbus/tps53679.c b/drivers/hwmon/pmbus/tps53679.c index ca2bfa25eb04cc..249974c13aa399 100644 --- a/drivers/hwmon/pmbus/tps53679.c +++ b/drivers/hwmon/pmbus/tps53679.c @@ -103,10 +103,10 @@ static int tps53679_identify_chip(struct i2c_client *client, } ret = i2c_smbus_read_block_data(client, PMBUS_IC_DEVICE_ID, buf); - if (ret < 0) - return ret; + if (ret <= 0) + return ret < 0 ? ret : -EIO; - /* Adjust length if null terminator if present */ + /* Adjust length if null terminator is present */ buf_len = (buf[ret - 1] != '\x00' ? ret : ret - 1); id_len = strlen(id); @@ -175,8 +175,8 @@ static int tps53676_identify(struct i2c_client *client, ret = i2c_smbus_read_block_data(client, PMBUS_IC_DEVICE_ID, buf); if (ret < 0) return ret; - if (strncmp("TI\x53\x67\x60", buf, 5)) { - dev_err(&client->dev, "Unexpected device ID: %s\n", buf); + if (ret != 6 || memcmp(buf, "TI\x53\x67\x60\x00", 6)) { + dev_err(&client->dev, "Unexpected device ID: %*ph\n", ret, buf); return -ENODEV; } diff --git a/drivers/hwmon/powerz.c b/drivers/hwmon/powerz.c index 4e663d5b4e330b..a75b941bd6e2fc 100644 --- a/drivers/hwmon/powerz.c +++ b/drivers/hwmon/powerz.c @@ -108,6 +108,9 @@ static int powerz_read_data(struct usb_device *udev, struct powerz_priv *priv) { int ret; + if (!priv->urb) + return -ENODEV; + priv->status = -ETIMEDOUT; reinit_completion(&priv->completion); @@ -224,6 +227,8 @@ static int powerz_probe(struct usb_interface *intf, mutex_init(&priv->mutex); init_completion(&priv->completion); + usb_set_intfdata(intf, priv); + hwmon_dev = devm_hwmon_device_register_with_info(parent, DRIVER_NAME, priv, &powerz_chip_info, NULL); @@ -232,8 +237,6 @@ static int powerz_probe(struct usb_interface *intf, return PTR_ERR(hwmon_dev); } - usb_set_intfdata(intf, priv); - return 0; } @@ -244,6 +247,7 @@ static void powerz_disconnect(struct usb_interface *intf) mutex_lock(&priv->mutex); usb_kill_urb(priv->urb); usb_free_urb(priv->urb); + priv->urb = NULL; mutex_unlock(&priv->mutex); } diff --git a/drivers/hwspinlock/omap_hwspinlock.c b/drivers/hwspinlock/omap_hwspinlock.c index 27b47b8623c09c..2d8de835bc2429 100644 --- a/drivers/hwspinlock/omap_hwspinlock.c +++ b/drivers/hwspinlock/omap_hwspinlock.c @@ -88,7 +88,9 @@ static int omap_hwspinlock_probe(struct platform_device *pdev) * make sure the module is enabled and clocked before reading * the module SYSSTATUS register */ - devm_pm_runtime_enable(&pdev->dev); + ret = devm_pm_runtime_enable(&pdev->dev); + if (ret) + return ret; ret = pm_runtime_resume_and_get(&pdev->dev); if (ret < 0) return ret; diff --git a/drivers/hwtracing/coresight/coresight-etm3x-core.c b/drivers/hwtracing/coresight/coresight-etm3x-core.c index a5e809589d3e38..0c011b7041696a 100644 --- a/drivers/hwtracing/coresight/coresight-etm3x-core.c +++ b/drivers/hwtracing/coresight/coresight-etm3x-core.c @@ -795,16 +795,16 @@ static int __init etm_hp_setup(void) { int ret; - ret = cpuhp_setup_state_nocalls_cpuslocked(CPUHP_AP_ARM_CORESIGHT_STARTING, - "arm/coresight:starting", - etm_starting_cpu, etm_dying_cpu); + ret = cpuhp_setup_state_nocalls(CPUHP_AP_ARM_CORESIGHT_STARTING, + "arm/coresight:starting", + etm_starting_cpu, etm_dying_cpu); if (ret) return ret; - ret = cpuhp_setup_state_nocalls_cpuslocked(CPUHP_AP_ONLINE_DYN, - "arm/coresight:online", - etm_online_cpu, NULL); + ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, + "arm/coresight:online", + etm_online_cpu, NULL); /* HP dyn state ID returned in ret on success */ if (ret > 0) { diff --git a/drivers/hwtracing/coresight/coresight-tmc-etr.c b/drivers/hwtracing/coresight/coresight-tmc-etr.c index e0d83ee01b77a5..fc0a946053ddef 100644 --- a/drivers/hwtracing/coresight/coresight-tmc-etr.c +++ b/drivers/hwtracing/coresight/coresight-tmc-etr.c @@ -1306,6 +1306,19 @@ static int tmc_enable_etr_sink_sysfs(struct coresight_device *csdev) raw_spin_lock_irqsave(&drvdata->spinlock, flags); + /* + * Since the sysfs buffer allocation and the hardware enablement is not + * in the same critical region, it's possible to race with the perf. + */ + if (coresight_get_mode(csdev) == CS_MODE_PERF) { + drvdata->sysfs_buf = NULL; + raw_spin_unlock_irqrestore(&drvdata->spinlock, flags); + + /* Free allocated memory out side of the spinlock */ + tmc_etr_free_sysfs_buf(sysfs_buf); + return -EBUSY; + } + /* * In sysFS mode we can have multiple writers per sink. Since this * sink is already enabled no memory is needed and the HW need not be diff --git a/drivers/hwtracing/stm/Kconfig b/drivers/hwtracing/stm/Kconfig index eda6b11d40a1f9..cd7f0b0f3fbebc 100644 --- a/drivers/hwtracing/stm/Kconfig +++ b/drivers/hwtracing/stm/Kconfig @@ -13,7 +13,7 @@ if STM config STM_PROTO_BASIC tristate "Basic STM framing protocol driver" - default CONFIG_STM + default STM help This is a simple framing protocol for sending data over STM devices. This was the protocol that the STM framework used @@ -28,7 +28,7 @@ config STM_PROTO_BASIC config STM_PROTO_SYS_T tristate "MIPI SyS-T STM framing protocol driver" - default CONFIG_STM + default STM help This is an implementation of MIPI SyS-T protocol to be used over the STP transport. In addition to the data payload, it diff --git a/drivers/i2c/busses/Kconfig b/drivers/i2c/busses/Kconfig index 09ba55bae1fac0..7d0afdc7d88627 100644 --- a/drivers/i2c/busses/Kconfig +++ b/drivers/i2c/busses/Kconfig @@ -1220,6 +1220,8 @@ config I2C_TEGRA tristate "NVIDIA Tegra internal I2C controller" depends on ARCH_TEGRA || (COMPILE_TEST && (ARC || ARM || ARM64 || M68K || RISCV || SUPERH || SPARC)) # COMPILE_TEST needs architectures with readsX()/writesX() primitives + depends on PINCTRL + # ARCH_TEGRA implies PINCTRL, but the COMPILE_TEST side doesn't. help If you say yes to this option, support will be included for the I2C controller embedded in NVIDIA Tegra SOCs diff --git a/drivers/i2c/busses/i2c-cp2615.c b/drivers/i2c/busses/i2c-cp2615.c index e7720ea4045eb6..7b62ba115eb9f1 100644 --- a/drivers/i2c/busses/i2c-cp2615.c +++ b/drivers/i2c/busses/i2c-cp2615.c @@ -298,6 +298,9 @@ cp2615_i2c_probe(struct usb_interface *usbif, const struct usb_device_id *id) if (!adap) return -ENOMEM; + if (!usbdev->serial) + return -EINVAL; + strscpy(adap->name, usbdev->serial, sizeof(adap->name)); adap->owner = THIS_MODULE; adap->dev.parent = &usbif->dev; diff --git a/drivers/i2c/busses/i2c-designware-amdisp.c b/drivers/i2c/busses/i2c-designware-amdisp.c index 450793d5f83921..e0c3669bab08e5 100644 --- a/drivers/i2c/busses/i2c-designware-amdisp.c +++ b/drivers/i2c/busses/i2c-designware-amdisp.c @@ -7,6 +7,7 @@ #include #include +#include #include #include @@ -82,22 +83,20 @@ static int amd_isp_dw_i2c_plat_probe(struct platform_device *pdev) if (isp_i2c_dev->shared_with_punit) pm_runtime_get_noresume(&pdev->dev); - pm_runtime_enable(&pdev->dev); - pm_runtime_get_sync(&pdev->dev); - + dev_pm_genpd_resume(&pdev->dev); ret = i2c_dw_probe(isp_i2c_dev); if (ret) { dev_err_probe(&pdev->dev, ret, "i2c_dw_probe failed\n"); goto error_release_rpm; } - - pm_runtime_put_sync(&pdev->dev); + dev_pm_genpd_suspend(&pdev->dev); + pm_runtime_set_suspended(&pdev->dev); + pm_runtime_enable(&pdev->dev); return 0; error_release_rpm: amd_isp_dw_i2c_plat_pm_cleanup(isp_i2c_dev); - pm_runtime_put_sync(&pdev->dev); return ret; } diff --git a/drivers/i2c/busses/i2c-fsi.c b/drivers/i2c/busses/i2c-fsi.c index ae016a9431dac3..6a9245423d2b58 100644 --- a/drivers/i2c/busses/i2c-fsi.c +++ b/drivers/i2c/busses/i2c-fsi.c @@ -728,6 +728,7 @@ static int fsi_i2c_probe(struct device *dev) rc = i2c_add_adapter(&port->adapter); if (rc < 0) { dev_err(dev, "Failed to register adapter: %d\n", rc); + of_node_put(np); kfree(port); continue; } diff --git a/drivers/i2c/busses/i2c-i801.c b/drivers/i2c/busses/i2c-i801.c index 9e1789725edf7e..32a3cef02c7b54 100644 --- a/drivers/i2c/busses/i2c-i801.c +++ b/drivers/i2c/busses/i2c-i801.c @@ -310,9 +310,10 @@ struct i801_priv { /* * If set to true the host controller registers are reserved for - * ACPI AML use. + * ACPI AML use. Needs extra protection by acpi_lock. */ bool acpi_reserved; + struct mutex acpi_lock; }; #define FEATURE_SMBUS_PEC BIT(0) @@ -894,8 +895,11 @@ static s32 i801_access(struct i2c_adapter *adap, u16 addr, int hwpec, ret; struct i801_priv *priv = i2c_get_adapdata(adap); - if (priv->acpi_reserved) + mutex_lock(&priv->acpi_lock); + if (priv->acpi_reserved) { + mutex_unlock(&priv->acpi_lock); return -EBUSY; + } pm_runtime_get_sync(&priv->pci_dev->dev); @@ -935,6 +939,7 @@ static s32 i801_access(struct i2c_adapter *adap, u16 addr, iowrite8(SMBHSTSTS_INUSE_STS | STATUS_FLAGS, SMBHSTSTS(priv)); pm_runtime_put_autosuspend(&priv->pci_dev->dev); + mutex_unlock(&priv->acpi_lock); return ret; } @@ -1465,7 +1470,7 @@ i801_acpi_io_handler(u32 function, acpi_physical_address address, u32 bits, * further access from the driver itself. This device is now owned * by the system firmware. */ - i2c_lock_bus(&priv->adapter, I2C_LOCK_SEGMENT); + mutex_lock(&priv->acpi_lock); if (!priv->acpi_reserved && i801_acpi_is_smbus_ioport(priv, address)) { priv->acpi_reserved = true; @@ -1485,7 +1490,7 @@ i801_acpi_io_handler(u32 function, acpi_physical_address address, u32 bits, else status = acpi_os_write_port(address, (u32)*value, bits); - i2c_unlock_bus(&priv->adapter, I2C_LOCK_SEGMENT); + mutex_unlock(&priv->acpi_lock); return status; } @@ -1545,6 +1550,7 @@ static int i801_probe(struct pci_dev *dev, const struct pci_device_id *id) priv->adapter.dev.parent = &dev->dev; acpi_use_parent_companion(&priv->adapter.dev); priv->adapter.retries = 3; + mutex_init(&priv->acpi_lock); priv->pci_dev = dev; priv->features = id->driver_data; diff --git a/drivers/i2c/busses/i2c-imx-lpi2c.c b/drivers/i2c/busses/i2c-imx-lpi2c.c index d882126c1778cb..519a1ac832a41e 100644 --- a/drivers/i2c/busses/i2c-imx-lpi2c.c +++ b/drivers/i2c/busses/i2c-imx-lpi2c.c @@ -5,6 +5,7 @@ * Copyright 2016 Freescale Semiconductor, Inc. */ +#include #include #include #include @@ -90,6 +91,7 @@ #define MRDR_RXEMPTY BIT(14) #define MDER_TDDE BIT(0) #define MDER_RDDE BIT(1) +#define MSR_RDF_ASSERTED(x) FIELD_GET(MSR_RDF, (x)) #define SCR_SEN BIT(0) #define SCR_RST BIT(1) @@ -461,7 +463,7 @@ static bool lpi2c_imx_write_txfifo(struct lpi2c_imx_struct *lpi2c_imx, bool atom static bool lpi2c_imx_read_rxfifo(struct lpi2c_imx_struct *lpi2c_imx, bool atomic) { - unsigned int blocklen, remaining; + unsigned int remaining; unsigned int temp, data; do { @@ -472,15 +474,6 @@ static bool lpi2c_imx_read_rxfifo(struct lpi2c_imx_struct *lpi2c_imx, bool atomi lpi2c_imx->rx_buf[lpi2c_imx->delivered++] = data & 0xff; } while (1); - /* - * First byte is the length of remaining packet in the SMBus block - * data read. Add it to msgs->len. - */ - if (lpi2c_imx->block_data) { - blocklen = lpi2c_imx->rx_buf[0]; - lpi2c_imx->msglen += blocklen; - } - remaining = lpi2c_imx->msglen - lpi2c_imx->delivered; if (!remaining) { @@ -493,12 +486,7 @@ static bool lpi2c_imx_read_rxfifo(struct lpi2c_imx_struct *lpi2c_imx, bool atomi lpi2c_imx_set_rx_watermark(lpi2c_imx); /* multiple receive commands */ - if (lpi2c_imx->block_data) { - lpi2c_imx->block_data = 0; - temp = remaining; - temp |= (RECV_DATA << 8); - writel(temp, lpi2c_imx->base + LPI2C_MTDR); - } else if (!(lpi2c_imx->delivered & 0xff)) { + if (!(lpi2c_imx->delivered & 0xff)) { temp = (remaining > CHUNK_DATA ? CHUNK_DATA : remaining) - 1; temp |= (RECV_DATA << 8); writel(temp, lpi2c_imx->base + LPI2C_MTDR); @@ -536,18 +524,77 @@ static int lpi2c_imx_write_atomic(struct lpi2c_imx_struct *lpi2c_imx, return err; } -static void lpi2c_imx_read_init(struct lpi2c_imx_struct *lpi2c_imx, - struct i2c_msg *msgs) +static unsigned int lpi2c_SMBus_block_read_length_byte(struct lpi2c_imx_struct *lpi2c_imx) { - unsigned int temp; + unsigned int data; + + data = readl(lpi2c_imx->base + LPI2C_MRDR); + lpi2c_imx->rx_buf[lpi2c_imx->delivered++] = data & 0xff; + + return data; +} + +static int lpi2c_imx_read_init(struct lpi2c_imx_struct *lpi2c_imx, + struct i2c_msg *msgs) +{ + unsigned int temp, val, block_len; + int ret; lpi2c_imx->rx_buf = msgs->buf; lpi2c_imx->block_data = msgs->flags & I2C_M_RECV_LEN; lpi2c_imx_set_rx_watermark(lpi2c_imx); - temp = msgs->len > CHUNK_DATA ? CHUNK_DATA - 1 : msgs->len - 1; - temp |= (RECV_DATA << 8); - writel(temp, lpi2c_imx->base + LPI2C_MTDR); + + if (!lpi2c_imx->block_data) { + temp = msgs->len > CHUNK_DATA ? CHUNK_DATA - 1 : msgs->len - 1; + temp |= (RECV_DATA << 8); + writel(temp, lpi2c_imx->base + LPI2C_MTDR); + } else { + /* + * The LPI2C controller automatically sends a NACK after the last byte of a + * receive command, unless the next command in MTDR is also a receive command. + * If MTDR is empty when a receive completes, a NACK is sent by default. + * + * To comply with the SMBus block read spec, we start with a 2-byte read: + * The first byte in RXFIFO is the block length. Once this byte arrives, the + * controller immediately updates MTDR with the next read command, ensuring + * continuous ACK instead of NACK. + * + * The second byte is the first block data byte. Therefore, the subsequent + * read command should request (block_len - 1) bytes, since one data byte + * has already been read. + */ + + writel((RECV_DATA << 8) | 0x01, lpi2c_imx->base + LPI2C_MTDR); + + ret = readl_poll_timeout(lpi2c_imx->base + LPI2C_MSR, val, + MSR_RDF_ASSERTED(val), 1, 1000); + if (ret) { + dev_err(&lpi2c_imx->adapter.dev, "SMBus read count failed %d\n", ret); + return ret; + } + + /* Read block length byte and confirm this SMBus transfer meets protocol */ + block_len = lpi2c_SMBus_block_read_length_byte(lpi2c_imx); + if (block_len == 0 || block_len > I2C_SMBUS_BLOCK_MAX) { + dev_err(&lpi2c_imx->adapter.dev, "Invalid SMBus block read length\n"); + return -EPROTO; + } + + /* + * When block_len shows more bytes need to be read, update second read command to + * keep MTDR non-empty and ensuring continuous ACKs. Only update command register + * here. All block bytes will be read out at IRQ handler or lpi2c_imx_read_atomic() + * function. + */ + if (block_len > 1) + writel((RECV_DATA << 8) | (block_len - 2), lpi2c_imx->base + LPI2C_MTDR); + + lpi2c_imx->msglen += block_len; + msgs->len += block_len; + } + + return 0; } static bool lpi2c_imx_read_chunk_atomic(struct lpi2c_imx_struct *lpi2c_imx) @@ -592,6 +639,10 @@ static bool is_use_dma(struct lpi2c_imx_struct *lpi2c_imx, struct i2c_msg *msg) if (!lpi2c_imx->can_use_dma) return false; + /* DMA is not suitable for SMBus block read */ + if (msg->flags & I2C_M_RECV_LEN) + return false; + /* * A system-wide suspend or resume transition is in progress. LPI2C should use PIO to * transfer data to avoid issue caused by no ready DMA HW resource. @@ -609,10 +660,14 @@ static bool is_use_dma(struct lpi2c_imx_struct *lpi2c_imx, struct i2c_msg *msg) static int lpi2c_imx_pio_xfer(struct lpi2c_imx_struct *lpi2c_imx, struct i2c_msg *msg) { + int ret; + reinit_completion(&lpi2c_imx->complete); if (msg->flags & I2C_M_RD) { - lpi2c_imx_read_init(lpi2c_imx, msg); + ret = lpi2c_imx_read_init(lpi2c_imx, msg); + if (ret) + return ret; lpi2c_imx_intctrl(lpi2c_imx, MIER_RDIE | MIER_NDIE); } else { lpi2c_imx_write(lpi2c_imx, msg); @@ -624,8 +679,12 @@ static int lpi2c_imx_pio_xfer(struct lpi2c_imx_struct *lpi2c_imx, static int lpi2c_imx_pio_xfer_atomic(struct lpi2c_imx_struct *lpi2c_imx, struct i2c_msg *msg) { + int ret; + if (msg->flags & I2C_M_RD) { - lpi2c_imx_read_init(lpi2c_imx, msg); + ret = lpi2c_imx_read_init(lpi2c_imx, msg); + if (ret) + return ret; return lpi2c_imx_read_atomic(lpi2c_imx, msg); } diff --git a/drivers/i2c/busses/i2c-imx.c b/drivers/i2c/busses/i2c-imx.c index 85f554044cf1ee..a208fefd3c3b35 100644 --- a/drivers/i2c/busses/i2c-imx.c +++ b/drivers/i2c/busses/i2c-imx.c @@ -401,7 +401,7 @@ static void i2c_imx_reset_regs(struct imx_i2c_struct *i2c_imx) static int i2c_imx_dma_request(struct imx_i2c_struct *i2c_imx, dma_addr_t phy_addr) { struct imx_i2c_dma *dma; - struct dma_slave_config dma_sconfig; + struct dma_slave_config dma_sconfig = {}; struct device *dev = i2c_imx->adapter.dev.parent; int ret; @@ -1018,8 +1018,9 @@ static inline int i2c_imx_isr_read(struct imx_i2c_struct *i2c_imx) return 0; } -static inline void i2c_imx_isr_read_continue(struct imx_i2c_struct *i2c_imx) +static inline enum imx_i2c_state i2c_imx_isr_read_continue(struct imx_i2c_struct *i2c_imx) { + enum imx_i2c_state next_state = IMX_I2C_STATE_READ_CONTINUE; unsigned int temp; if ((i2c_imx->msg->len - 1) == i2c_imx->msg_buf_idx) { @@ -1033,18 +1034,20 @@ static inline void i2c_imx_isr_read_continue(struct imx_i2c_struct *i2c_imx) i2c_imx->stopped = 1; temp &= ~(I2CR_MSTA | I2CR_MTX); imx_i2c_write_reg(temp, i2c_imx, IMX_I2C_I2CR); - } else { - /* - * For i2c master receiver repeat restart operation like: - * read -> repeat MSTA -> read/write - * The controller must set MTX before read the last byte in - * the first read operation, otherwise the first read cost - * one extra clock cycle. - */ - temp = imx_i2c_read_reg(i2c_imx, IMX_I2C_I2CR); - temp |= I2CR_MTX; - imx_i2c_write_reg(temp, i2c_imx, IMX_I2C_I2CR); + + return IMX_I2C_STATE_DONE; } + /* + * For i2c master receiver repeat restart operation like: + * read -> repeat MSTA -> read/write + * The controller must set MTX before read the last byte in + * the first read operation, otherwise the first read cost + * one extra clock cycle. + */ + temp = imx_i2c_read_reg(i2c_imx, IMX_I2C_I2CR); + temp |= I2CR_MTX; + imx_i2c_write_reg(temp, i2c_imx, IMX_I2C_I2CR); + next_state = IMX_I2C_STATE_DONE; } else if (i2c_imx->msg_buf_idx == (i2c_imx->msg->len - 2)) { temp = imx_i2c_read_reg(i2c_imx, IMX_I2C_I2CR); temp |= I2CR_TXAK; @@ -1052,6 +1055,7 @@ static inline void i2c_imx_isr_read_continue(struct imx_i2c_struct *i2c_imx) } i2c_imx->msg->buf[i2c_imx->msg_buf_idx++] = imx_i2c_read_reg(i2c_imx, IMX_I2C_I2DR); + return next_state; } static inline void i2c_imx_isr_read_block_data_len(struct imx_i2c_struct *i2c_imx) @@ -1088,11 +1092,9 @@ static irqreturn_t i2c_imx_master_isr(struct imx_i2c_struct *i2c_imx, unsigned i break; case IMX_I2C_STATE_READ_CONTINUE: - i2c_imx_isr_read_continue(i2c_imx); - if (i2c_imx->msg_buf_idx == i2c_imx->msg->len) { - i2c_imx->state = IMX_I2C_STATE_DONE; + i2c_imx->state = i2c_imx_isr_read_continue(i2c_imx); + if (i2c_imx->state == IMX_I2C_STATE_DONE) wake_up(&i2c_imx->queue); - } break; case IMX_I2C_STATE_READ_BLOCK_DATA: @@ -1490,6 +1492,7 @@ static int i2c_imx_read(struct imx_i2c_struct *i2c_imx, struct i2c_msg *msgs, bool is_lastmsg) { int block_data = msgs->flags & I2C_M_RECV_LEN; + int ret = 0; dev_dbg(&i2c_imx->adapter.dev, "<%s> write slave address: addr=0x%x\n", @@ -1522,10 +1525,20 @@ static int i2c_imx_read(struct imx_i2c_struct *i2c_imx, struct i2c_msg *msgs, dev_err(&i2c_imx->adapter.dev, "<%s> read timedout\n", __func__); return -ETIMEDOUT; } - if (!i2c_imx->stopped) - return i2c_imx_bus_busy(i2c_imx, 0, false); + if (i2c_imx->is_lastmsg) { + if (!i2c_imx->stopped) + ret = i2c_imx_bus_busy(i2c_imx, 0, false); + /* + * Only read the last byte of the last message after the bus is + * not busy. Else the controller generates another clock which + * might confuse devices. + */ + if (!ret) + i2c_imx->msg->buf[i2c_imx->msg_buf_idx++] = imx_i2c_read_reg(i2c_imx, + IMX_I2C_I2DR); + } - return 0; + return ret; } static int i2c_imx_xfer_common(struct i2c_adapter *adapter, diff --git a/drivers/i2c/busses/i2c-pxa.c b/drivers/i2c/busses/i2c-pxa.c index 09af3b3625f110..f55840b2eb9ab7 100644 --- a/drivers/i2c/busses/i2c-pxa.c +++ b/drivers/i2c/busses/i2c-pxa.c @@ -268,6 +268,7 @@ struct pxa_i2c { struct pinctrl *pinctrl; struct pinctrl_state *pinctrl_default; struct pinctrl_state *pinctrl_recovery; + bool reset_before_xfer; }; #define _IBMR(i2c) ((i2c)->reg_ibmr) @@ -1144,6 +1145,11 @@ static int i2c_pxa_xfer(struct i2c_adapter *adap, { struct pxa_i2c *i2c = adap->algo_data; + if (i2c->reset_before_xfer) { + i2c_pxa_reset(i2c); + i2c->reset_before_xfer = false; + } + return i2c_pxa_internal_xfer(i2c, msgs, num, i2c_pxa_do_xfer); } @@ -1521,7 +1527,16 @@ static int i2c_pxa_probe(struct platform_device *dev) } } - i2c_pxa_reset(i2c); + /* + * Skip reset on Armada 3700 when recovery is used to avoid + * controller hang due to the pinctrl state changes done by + * the generic recovery initialization code. The reset will + * be performed later, prior to the first transfer. + */ + if (i2c_type == REGS_A3700 && i2c->adap.bus_recovery_info) + i2c->reset_before_xfer = true; + else + i2c_pxa_reset(i2c); ret = i2c_add_numbered_adapter(&i2c->adap); if (ret < 0) diff --git a/drivers/i2c/busses/i2c-s3c2410.c b/drivers/i2c/busses/i2c-s3c2410.c index 8138f5ef40f06b..15e14a6fe6dce9 100644 --- a/drivers/i2c/busses/i2c-s3c2410.c +++ b/drivers/i2c/busses/i2c-s3c2410.c @@ -503,8 +503,13 @@ static void i2c_s3c_irq_nextbyte(struct s3c24xx_i2c *i2c, unsigned long iicstat) i2c->msg->buf[i2c->msg_ptr++] = byte; /* Add actual length to read for smbus block read */ - if (i2c->msg->flags & I2C_M_RECV_LEN && i2c->msg->len == 1) + if (i2c->msg->flags & I2C_M_RECV_LEN && i2c->msg->len == 1) { + if (byte == 0 || byte > I2C_SMBUS_BLOCK_MAX) { + s3c24xx_i2c_stop(i2c, -EPROTO); + break; + } i2c->msg->len += byte; + } prepare_read: if (is_msglast(i2c)) { /* last byte of buffer */ diff --git a/drivers/i2c/busses/i2c-tegra.c b/drivers/i2c/busses/i2c-tegra.c index e533460bccc39e..a9aed411e3190c 100644 --- a/drivers/i2c/busses/i2c-tegra.c +++ b/drivers/i2c/busses/i2c-tegra.c @@ -1837,8 +1837,11 @@ static int tegra_i2c_probe(struct platform_device *pdev) * * VI I2C device shouldn't be marked as IRQ-safe because VI I2C won't * be used for atomic transfers. ACPI device is not IRQ safe also. + * + * Devices with pinctrl states cannot be marked IRQ-safe as the pinctrl + * state transitions during runtime PM require mutexes. */ - if (!IS_VI(i2c_dev) && !has_acpi_companion(i2c_dev->dev)) + if (!IS_VI(i2c_dev) && !has_acpi_companion(i2c_dev->dev) && !i2c_dev->dev->pins) pm_runtime_irq_safe(i2c_dev->dev); pm_runtime_enable(i2c_dev->dev); diff --git a/drivers/i3c/master.c b/drivers/i3c/master.c index 7f606c8716480c..5408332861a1bf 100644 --- a/drivers/i3c/master.c +++ b/drivers/i3c/master.c @@ -618,7 +618,8 @@ static int i3c_set_hotjoin(struct i3c_master_controller *master, bool enable) else ret = master->ops->disable_hotjoin(master); - master->hotjoin = enable; + if (!ret) + master->hotjoin = enable; i3c_bus_normaluse_unlock(&master->bus); @@ -2881,7 +2882,6 @@ int i3c_master_register(struct i3c_master_controller *master, INIT_LIST_HEAD(&master->boardinfo.i3c); device_initialize(&master->dev); - dev_set_name(&master->dev, "i3c-%d", i3cbus->id); master->dev.dma_mask = parent->dma_mask; master->dev.coherent_dma_mask = parent->coherent_dma_mask; @@ -2891,6 +2891,8 @@ int i3c_master_register(struct i3c_master_controller *master, if (ret) goto err_put_dev; + dev_set_name(&master->dev, "i3c-%d", i3cbus->id); + ret = of_populate_i3c_bus(master); if (ret) goto err_put_dev; diff --git a/drivers/i3c/master/dw-i3c-master.c b/drivers/i3c/master/dw-i3c-master.c index 889e2ed5bc8303..1368c834ca5e85 100644 --- a/drivers/i3c/master/dw-i3c-master.c +++ b/drivers/i3c/master/dw-i3c-master.c @@ -1010,7 +1010,7 @@ static int dw_i3c_master_reattach_i3c_dev(struct i3c_dev_desc *dev, master->free_pos &= ~BIT(pos); } - writel(DEV_ADDR_TABLE_DYNAMIC_ADDR(dev->info.dyn_addr), + writel(DEV_ADDR_TABLE_DYNAMIC_ADDR(dev->info.dyn_addr) | DEV_ADDR_TABLE_SIR_REJECT, master->regs + DEV_ADDR_TABLE_LOC(master->datstartaddr, data->index)); @@ -1039,7 +1039,7 @@ static int dw_i3c_master_attach_i3c_dev(struct i3c_dev_desc *dev) master->free_pos &= ~BIT(pos); i3c_dev_set_master_data(dev, data); - writel(DEV_ADDR_TABLE_DYNAMIC_ADDR(master->devs[pos].addr), + writel(DEV_ADDR_TABLE_DYNAMIC_ADDR(master->devs[pos].addr) | DEV_ADDR_TABLE_SIR_REJECT, master->regs + DEV_ADDR_TABLE_LOC(master->datstartaddr, data->index)); @@ -1099,6 +1099,7 @@ static int dw_i3c_master_i2c_xfers(struct i2c_dev_desc *dev, dev_err(master->dev, "<%s> cannot resume i3c bus master, err: %d\n", __func__, ret); + dw_i3c_master_free_xfer(xfer); return ret; } @@ -1570,6 +1571,8 @@ int dw_i3c_common_probe(struct dw_i3c_master *master, spin_lock_init(&master->xferqueue.lock); INIT_LIST_HEAD(&master->xferqueue.list); + spin_lock_init(&master->devs_lock); + writel(INTR_ALL, master->regs + INTR_STATUS); irq = platform_get_irq(pdev, 0); ret = devm_request_irq(&pdev->dev, irq, @@ -1611,6 +1614,8 @@ int dw_i3c_common_probe(struct dw_i3c_master *master, pm_runtime_get_noresume(&pdev->dev); INIT_WORK(&master->hj_work, dw_i3c_hj_work); + + device_set_of_node_from_dev(&master->base.i2c.dev, &pdev->dev); ret = i3c_master_register(&master->base, &pdev->dev, &dw_mipi_i3c_ops, false); if (ret) diff --git a/drivers/i3c/master/mipi-i3c-hci/cmd.h b/drivers/i3c/master/mipi-i3c-hci/cmd.h index 1d6dd2c5d01a53..b1bf87daa65166 100644 --- a/drivers/i3c/master/mipi-i3c-hci/cmd.h +++ b/drivers/i3c/master/mipi-i3c-hci/cmd.h @@ -17,6 +17,7 @@ #define CMD_0_TOC W0_BIT_(31) #define CMD_0_ROC W0_BIT_(30) #define CMD_0_ATTR W0_MASK(2, 0) +#define CMD_0_TID W0_MASK(6, 3) /* * Response Descriptor Structure diff --git a/drivers/i3c/master/mipi-i3c-hci/cmd_v1.c b/drivers/i3c/master/mipi-i3c-hci/cmd_v1.c index eb8a3ae2990d77..efb7a1f92641ca 100644 --- a/drivers/i3c/master/mipi-i3c-hci/cmd_v1.c +++ b/drivers/i3c/master/mipi-i3c-hci/cmd_v1.c @@ -336,7 +336,7 @@ static int hci_cmd_v1_daa(struct i3c_hci *hci) hci->io->queue_xfer(hci, xfer, 1); if (!wait_for_completion_timeout(&done, HZ) && hci->io->dequeue_xfer(hci, xfer, 1)) { - ret = -ETIME; + ret = -ETIMEDOUT; break; } if ((RESP_STATUS(xfer->response) == RESP_ERR_ADDR_HEADER || diff --git a/drivers/i3c/master/mipi-i3c-hci/cmd_v2.c b/drivers/i3c/master/mipi-i3c-hci/cmd_v2.c index efb4326a25b73e..5fc2e4c55ebb04 100644 --- a/drivers/i3c/master/mipi-i3c-hci/cmd_v2.c +++ b/drivers/i3c/master/mipi-i3c-hci/cmd_v2.c @@ -277,7 +277,7 @@ static int hci_cmd_v2_daa(struct i3c_hci *hci) hci->io->queue_xfer(hci, xfer, 2); if (!wait_for_completion_timeout(&done, HZ) && hci->io->dequeue_xfer(hci, xfer, 2)) { - ret = -ETIME; + ret = -ETIMEDOUT; break; } if (RESP_STATUS(xfer[0].response) != RESP_SUCCESS) { diff --git a/drivers/i3c/master/mipi-i3c-hci/core.c b/drivers/i3c/master/mipi-i3c-hci/core.c index 607d77ab0e5469..c529c527d7224a 100644 --- a/drivers/i3c/master/mipi-i3c-hci/core.c +++ b/drivers/i3c/master/mipi-i3c-hci/core.c @@ -230,7 +230,7 @@ static int i3c_hci_send_ccc_cmd(struct i3c_master_controller *m, goto out; if (!wait_for_completion_timeout(&done, HZ) && hci->io->dequeue_xfer(hci, xfer, nxfers)) { - ret = -ETIME; + ret = -ETIMEDOUT; goto out; } for (i = prefixed; i < nxfers; i++) { @@ -309,7 +309,7 @@ static int i3c_hci_i3c_xfers(struct i3c_dev_desc *dev, goto out; if (!wait_for_completion_timeout(&done, HZ) && hci->io->dequeue_xfer(hci, xfer, nxfers)) { - ret = -ETIME; + ret = -ETIMEDOUT; goto out; } for (i = 0; i < nxfers; i++) { @@ -357,7 +357,7 @@ static int i3c_hci_i2c_xfers(struct i2c_dev_desc *dev, goto out; if (!wait_for_completion_timeout(&done, m->i2c.timeout) && hci->io->dequeue_xfer(hci, xfer, nxfers)) { - ret = -ETIME; + ret = -ETIMEDOUT; goto out; } for (i = 0; i < nxfers; i++) { @@ -631,6 +631,9 @@ static int i3c_hci_init(struct i3c_hci *hci) if (ret) return ret; + spin_lock_init(&hci->lock); + mutex_init(&hci->control_mutex); + /* * Now let's reset the hardware. * SOFT_RST must be clear before we write to it. diff --git a/drivers/i3c/master/mipi-i3c-hci/dma.c b/drivers/i3c/master/mipi-i3c-hci/dma.c index c401a9425cdc59..fe8894f6fe607d 100644 --- a/drivers/i3c/master/mipi-i3c-hci/dma.c +++ b/drivers/i3c/master/mipi-i3c-hci/dma.c @@ -133,7 +133,6 @@ struct hci_rh_data { unsigned int xfer_struct_sz, resp_struct_sz, ibi_status_sz, ibi_chunk_sz; unsigned int done_ptr, ibi_chunk_ptr; struct hci_xfer **src_xfers; - spinlock_t lock; struct completion op_done; }; @@ -240,7 +239,6 @@ static int hci_dma_init(struct i3c_hci *hci) goto err_out; rh = &rings->headers[i]; rh->regs = hci->base_regs + offset; - spin_lock_init(&rh->lock); init_completion(&rh->op_done); rh->xfer_entries = XFER_RING_ENTRIES; @@ -342,6 +340,14 @@ static int hci_dma_init(struct i3c_hci *hci) rh_reg_write(INTR_SIGNAL_ENABLE, regval); ring_ready: + /* + * The MIPI I3C HCI specification does not document reset values for + * RING_OPERATION1 fields and some controllers (e.g. Intel controllers) + * do not reset the values, so ensure the ring pointers are set to zero + * here. + */ + rh_reg_write(RING_OPERATION1, 0); + rh_reg_write(RING_CONTROL, RING_CTRL_ENABLE | RING_CTRL_RUN_STOP); } @@ -367,6 +373,33 @@ static void hci_dma_unmap_xfer(struct i3c_hci *hci, } } +static struct i3c_dma *hci_dma_map_xfer(struct device *dev, struct hci_xfer *xfer) +{ + enum dma_data_direction dir = xfer->rnw ? DMA_FROM_DEVICE : DMA_TO_DEVICE; + bool need_bounce = device_iommu_mapped(dev) && xfer->rnw && (xfer->data_len & 3); + + return i3c_master_dma_map_single(dev, xfer->data, xfer->data_len, need_bounce, dir); +} + +static int hci_dma_map_xfer_list(struct i3c_hci *hci, struct device *dev, + struct hci_xfer *xfer_list, int n) +{ + for (int i = 0; i < n; i++) { + struct hci_xfer *xfer = xfer_list + i; + + if (!xfer->data) + continue; + + xfer->dma = hci_dma_map_xfer(dev, xfer); + if (!xfer->dma) { + hci_dma_unmap_xfer(hci, xfer_list, i); + return -ENOMEM; + } + } + + return 0; +} + static int hci_dma_queue_xfer(struct i3c_hci *hci, struct hci_xfer *xfer_list, int n) { @@ -374,6 +407,11 @@ static int hci_dma_queue_xfer(struct i3c_hci *hci, struct hci_rh_data *rh; unsigned int i, ring, enqueue_ptr; u32 op1_val, op2_val; + int ret; + + ret = hci_dma_map_xfer_list(hci, rings->sysdev, xfer_list, n); + if (ret) + return ret; /* For now we only use ring 0 */ ring = 0; @@ -384,9 +422,6 @@ static int hci_dma_queue_xfer(struct i3c_hci *hci, for (i = 0; i < n; i++) { struct hci_xfer *xfer = xfer_list + i; u32 *ring_data = rh->xfer + rh->xfer_struct_sz * enqueue_ptr; - enum dma_data_direction dir = xfer->rnw ? DMA_FROM_DEVICE : - DMA_TO_DEVICE; - bool need_bounce; /* store cmd descriptor */ *ring_data++ = xfer->cmd_desc[0]; @@ -405,18 +440,6 @@ static int hci_dma_queue_xfer(struct i3c_hci *hci, /* 2nd and 3rd words of Data Buffer Descriptor Structure */ if (xfer->data) { - need_bounce = device_iommu_mapped(rings->sysdev) && - xfer->rnw && - xfer->data_len != ALIGN(xfer->data_len, 4); - xfer->dma = i3c_master_dma_map_single(rings->sysdev, - xfer->data, - xfer->data_len, - need_bounce, - dir); - if (!xfer->dma) { - hci_dma_unmap_xfer(hci, xfer_list, i); - return -ENOMEM; - } *ring_data++ = lower_32_bits(xfer->dma->addr); *ring_data++ = upper_32_bits(xfer->dma->addr); } else { @@ -439,18 +462,18 @@ static int hci_dma_queue_xfer(struct i3c_hci *hci, op2_val = rh_reg_read(RING_OPERATION2); if (enqueue_ptr == FIELD_GET(RING_OP2_CR_DEQ_PTR, op2_val)) { /* the ring is full */ - hci_dma_unmap_xfer(hci, xfer_list, i + 1); + hci_dma_unmap_xfer(hci, xfer_list, n); return -EBUSY; } } /* take care to update the hardware enqueue pointer atomically */ - spin_lock_irq(&rh->lock); + spin_lock_irq(&hci->lock); op1_val = rh_reg_read(RING_OPERATION1); op1_val &= ~RING_OP1_CR_ENQ_PTR; op1_val |= FIELD_PREP(RING_OP1_CR_ENQ_PTR, enqueue_ptr); rh_reg_write(RING_OPERATION1, op1_val); - spin_unlock_irq(&rh->lock); + spin_unlock_irq(&hci->lock); return 0; } @@ -462,16 +485,25 @@ static bool hci_dma_dequeue_xfer(struct i3c_hci *hci, struct hci_rh_data *rh = &rings->headers[xfer_list[0].ring_number]; unsigned int i; bool did_unqueue = false; - - /* stop the ring */ - rh_reg_write(RING_CONTROL, RING_CTRL_ABORT); - if (wait_for_completion_timeout(&rh->op_done, HZ) == 0) { - /* - * We're deep in it if ever this condition is ever met. - * Hardware might still be writing to memory, etc. - */ - dev_crit(&hci->master.dev, "unable to abort the ring\n"); - WARN_ON(1); + u32 ring_status; + + guard(mutex)(&hci->control_mutex); + + ring_status = rh_reg_read(RING_STATUS); + if (ring_status & RING_STATUS_RUNNING) { + /* stop the ring */ + reinit_completion(&rh->op_done); + rh_reg_write(RING_CONTROL, RING_CTRL_ENABLE | RING_CTRL_ABORT); + wait_for_completion_timeout(&rh->op_done, HZ); + ring_status = rh_reg_read(RING_STATUS); + if (ring_status & RING_STATUS_RUNNING) { + /* + * We're deep in it if ever this condition is ever met. + * Hardware might still be writing to memory, etc. + */ + dev_crit(&hci->master.dev, "unable to abort the ring\n"); + WARN_ON(1); + } } for (i = 0; i < n; i++) { @@ -487,7 +519,7 @@ static bool hci_dma_dequeue_xfer(struct i3c_hci *hci, u32 *ring_data = rh->xfer + rh->xfer_struct_sz * idx; /* store no-op cmd descriptor */ - *ring_data++ = FIELD_PREP(CMD_0_ATTR, 0x7); + *ring_data++ = FIELD_PREP(CMD_0_ATTR, 0x7) | FIELD_PREP(CMD_0_TID, xfer->cmd_tid); *ring_data++ = 0; if (hci->cmd == &mipi_i3c_hci_cmd_v2) { *ring_data++ = 0; @@ -505,7 +537,9 @@ static bool hci_dma_dequeue_xfer(struct i3c_hci *hci, } /* restart the ring */ + mipi_i3c_hci_resume(hci); rh_reg_write(RING_CONTROL, RING_CTRL_ENABLE); + rh_reg_write(RING_CONTROL, RING_CTRL_ENABLE | RING_CTRL_RUN_STOP); return did_unqueue; } @@ -548,12 +582,12 @@ static void hci_dma_xfer_done(struct i3c_hci *hci, struct hci_rh_data *rh) } /* take care to update the software dequeue pointer atomically */ - spin_lock(&rh->lock); + spin_lock(&hci->lock); op1_val = rh_reg_read(RING_OPERATION1); op1_val &= ~RING_OP1_CR_SW_DEQ_PTR; op1_val |= FIELD_PREP(RING_OP1_CR_SW_DEQ_PTR, done_ptr); rh_reg_write(RING_OPERATION1, op1_val); - spin_unlock(&rh->lock); + spin_unlock(&hci->lock); } static int hci_dma_request_ibi(struct i3c_hci *hci, struct i3c_dev_desc *dev, @@ -734,12 +768,12 @@ static void hci_dma_process_ibi(struct i3c_hci *hci, struct hci_rh_data *rh) done: /* take care to update the ibi dequeue pointer atomically */ - spin_lock(&rh->lock); + spin_lock(&hci->lock); op1_val = rh_reg_read(RING_OPERATION1); op1_val &= ~RING_OP1_IBI_DEQ_PTR; op1_val |= FIELD_PREP(RING_OP1_IBI_DEQ_PTR, deq_ptr); rh_reg_write(RING_OPERATION1, op1_val); - spin_unlock(&rh->lock); + spin_unlock(&hci->lock); /* update the chunk pointer */ rh->ibi_chunk_ptr += ibi_chunks; diff --git a/drivers/i3c/master/mipi-i3c-hci/ext_caps.c b/drivers/i3c/master/mipi-i3c-hci/ext_caps.c index 7714f00ea9cc09..533a495e14c869 100644 --- a/drivers/i3c/master/mipi-i3c-hci/ext_caps.c +++ b/drivers/i3c/master/mipi-i3c-hci/ext_caps.c @@ -272,7 +272,7 @@ int i3c_hci_parse_ext_caps(struct i3c_hci *hci) cap_length = FIELD_GET(CAP_HEADER_LENGTH, cap_header); dev_dbg(&hci->master.dev, "id=0x%02x length=%d", cap_id, cap_length); - if (!cap_length) + if (!cap_id || !cap_length) break; if (curr_cap + cap_length * 4 >= end) { dev_err(&hci->master.dev, diff --git a/drivers/i3c/master/mipi-i3c-hci/hci.h b/drivers/i3c/master/mipi-i3c-hci/hci.h index 249ccb13c90928..32c8aecde9f76f 100644 --- a/drivers/i3c/master/mipi-i3c-hci/hci.h +++ b/drivers/i3c/master/mipi-i3c-hci/hci.h @@ -45,6 +45,8 @@ struct i3c_hci { const struct hci_io_ops *io; void *io_data; const struct hci_cmd_ops *cmd; + spinlock_t lock; + struct mutex control_mutex; atomic_t next_cmd_tid; u32 caps; unsigned int quirks; diff --git a/drivers/i3c/master/mipi-i3c-hci/pio.c b/drivers/i3c/master/mipi-i3c-hci/pio.c index 710faa46a00faa..67dc34163d51bd 100644 --- a/drivers/i3c/master/mipi-i3c-hci/pio.c +++ b/drivers/i3c/master/mipi-i3c-hci/pio.c @@ -124,7 +124,6 @@ struct hci_pio_ibi_data { }; struct hci_pio_data { - spinlock_t lock; struct hci_xfer *curr_xfer, *xfer_queue; struct hci_xfer *curr_rx, *rx_queue; struct hci_xfer *curr_tx, *tx_queue; @@ -146,7 +145,6 @@ static int hci_pio_init(struct i3c_hci *hci) return -ENOMEM; hci->io_data = pio; - spin_lock_init(&pio->lock); size_val = pio_reg_read(QUEUE_SIZE); dev_info(&hci->master.dev, "CMD/RESP FIFO = %ld entries\n", @@ -609,7 +607,7 @@ static int hci_pio_queue_xfer(struct i3c_hci *hci, struct hci_xfer *xfer, int n) xfer[i].data_left = xfer[i].data_len; } - spin_lock_irq(&pio->lock); + spin_lock_irq(&hci->lock); prev_queue_tail = pio->xfer_queue; pio->xfer_queue = &xfer[n - 1]; if (pio->curr_xfer) { @@ -623,7 +621,7 @@ static int hci_pio_queue_xfer(struct i3c_hci *hci, struct hci_xfer *xfer, int n) pio_reg_read(INTR_STATUS), pio_reg_read(INTR_SIGNAL_ENABLE)); } - spin_unlock_irq(&pio->lock); + spin_unlock_irq(&hci->lock); return 0; } @@ -694,14 +692,14 @@ static bool hci_pio_dequeue_xfer(struct i3c_hci *hci, struct hci_xfer *xfer, int struct hci_pio_data *pio = hci->io_data; int ret; - spin_lock_irq(&pio->lock); + spin_lock_irq(&hci->lock); dev_dbg(&hci->master.dev, "n=%d status=%#x/%#x", n, pio_reg_read(INTR_STATUS), pio_reg_read(INTR_SIGNAL_ENABLE)); dev_dbg(&hci->master.dev, "main_status = %#x/%#x", readl(hci->base_regs + 0x20), readl(hci->base_regs + 0x28)); ret = hci_pio_dequeue_xfer_common(hci, pio, xfer, n); - spin_unlock_irq(&pio->lock); + spin_unlock_irq(&hci->lock); return ret; } @@ -994,13 +992,13 @@ static bool hci_pio_irq_handler(struct i3c_hci *hci) struct hci_pio_data *pio = hci->io_data; u32 status; - spin_lock(&pio->lock); + spin_lock(&hci->lock); status = pio_reg_read(INTR_STATUS); dev_dbg(&hci->master.dev, "PIO_INTR_STATUS %#x/%#x", status, pio->enabled_irqs); status &= pio->enabled_irqs | STAT_LATENCY_WARNINGS; if (!status) { - spin_unlock(&pio->lock); + spin_unlock(&hci->lock); return false; } @@ -1036,7 +1034,7 @@ static bool hci_pio_irq_handler(struct i3c_hci *hci) pio_reg_write(INTR_SIGNAL_ENABLE, pio->enabled_irqs); dev_dbg(&hci->master.dev, "PIO_INTR_STATUS %#x/%#x", pio_reg_read(INTR_STATUS), pio_reg_read(INTR_SIGNAL_ENABLE)); - spin_unlock(&pio->lock); + spin_unlock(&hci->lock); return true; } diff --git a/drivers/i3c/master/svc-i3c-master.c b/drivers/i3c/master/svc-i3c-master.c index a62f22ff8b5762..857504d36e1867 100644 --- a/drivers/i3c/master/svc-i3c-master.c +++ b/drivers/i3c/master/svc-i3c-master.c @@ -533,8 +533,8 @@ static int svc_i3c_master_handle_ibi_won(struct svc_i3c_master *master, u32 msta static void svc_i3c_master_ibi_isr(struct svc_i3c_master *master) { struct svc_i3c_i2c_dev_data *data; + struct i3c_dev_desc *dev = NULL; unsigned int ibitype, ibiaddr; - struct i3c_dev_desc *dev; u32 status, val; int ret; @@ -627,7 +627,7 @@ static void svc_i3c_master_ibi_isr(struct svc_i3c_master *master) * for the slave to interrupt again. */ if (svc_i3c_master_error(master)) { - if (master->ibi.tbq_slot) { + if (master->ibi.tbq_slot && dev) { data = i3c_dev_get_master_data(dev); i3c_generic_ibi_recycle_slot(data->ibi_pool, master->ibi.tbq_slot); diff --git a/drivers/iio/accel/adxl313_core.c b/drivers/iio/accel/adxl313_core.c index 9f5d4d2cb325b3..83dcac17a0425f 100644 --- a/drivers/iio/accel/adxl313_core.c +++ b/drivers/iio/accel/adxl313_core.c @@ -998,6 +998,8 @@ static int adxl313_buffer_predisable(struct iio_dev *indio_dev) ret = regmap_write(data->regmap, ADXL313_REG_FIFO_CTL, FIELD_PREP(ADXL313_REG_FIFO_CTL_MODE_MSK, ADXL313_FIFO_BYPASS)); + if (ret) + return ret; ret = regmap_write(data->regmap, ADXL313_REG_INT_ENABLE, 0); if (ret) diff --git a/drivers/iio/accel/adxl355_core.c b/drivers/iio/accel/adxl355_core.c index 5fc7f814b90772..5aadc04771025d 100644 --- a/drivers/iio/accel/adxl355_core.c +++ b/drivers/iio/accel/adxl355_core.c @@ -745,7 +745,7 @@ static const struct iio_chan_spec adxl355_channels[] = { BIT(IIO_CHAN_INFO_OFFSET), .scan_index = 3, .scan_type = { - .sign = 's', + .sign = 'u', .realbits = 12, .storagebits = 16, .endianness = IIO_BE, diff --git a/drivers/iio/accel/adxl380.c b/drivers/iio/accel/adxl380.c index aef5109c1ddd9d..f716bd2154bc46 100644 --- a/drivers/iio/accel/adxl380.c +++ b/drivers/iio/accel/adxl380.c @@ -860,7 +860,7 @@ static int adxl380_set_fifo_samples(struct adxl380_state *st) ret = regmap_update_bits(st->regmap, ADXL380_FIFO_CONFIG_0_REG, ADXL380_FIFO_SAMPLES_8_MSK, FIELD_PREP(ADXL380_FIFO_SAMPLES_8_MSK, - (fifo_samples & BIT(8)))); + !!(fifo_samples & BIT(8)))); if (ret) return ret; @@ -949,6 +949,7 @@ static irqreturn_t adxl380_irq_handler(int irq, void *p) if (ret) return IRQ_HANDLED; + fifo_entries = rounddown(fifo_entries, st->fifo_set_size); for (i = 0; i < fifo_entries; i += st->fifo_set_size) { ret = regmap_noinc_read(st->regmap, ADXL380_FIFO_DATA, &st->fifo_buf[i], diff --git a/drivers/iio/accel/bma180.c b/drivers/iio/accel/bma180.c index 8925f5279e627a..7bc6761f51354f 100644 --- a/drivers/iio/accel/bma180.c +++ b/drivers/iio/accel/bma180.c @@ -986,8 +986,9 @@ static int bma180_probe(struct i2c_client *client) } ret = devm_request_irq(dev, client->irq, - iio_trigger_generic_data_rdy_poll, IRQF_TRIGGER_RISING, - "bma180_event", data->trig); + iio_trigger_generic_data_rdy_poll, + IRQF_TRIGGER_RISING | IRQF_NO_THREAD, + "bma180_event", data->trig); if (ret) { dev_err(dev, "unable to request IRQ\n"); goto err_trigger_free; diff --git a/drivers/iio/accel/sca3000.c b/drivers/iio/accel/sca3000.c index bfa8a3f5a92f43..9ef4d6e2746697 100644 --- a/drivers/iio/accel/sca3000.c +++ b/drivers/iio/accel/sca3000.c @@ -1489,7 +1489,11 @@ static int sca3000_probe(struct spi_device *spi) if (ret) goto error_free_irq; - return iio_device_register(indio_dev); + ret = iio_device_register(indio_dev); + if (ret) + goto error_free_irq; + + return 0; error_free_irq: if (spi->irq) diff --git a/drivers/iio/adc/ad7766.c b/drivers/iio/adc/ad7766.c index 4d570383ef0251..1e6bfe8765ab33 100644 --- a/drivers/iio/adc/ad7766.c +++ b/drivers/iio/adc/ad7766.c @@ -261,7 +261,7 @@ static int ad7766_probe(struct spi_device *spi) * don't enable the interrupt to avoid extra load on the system */ ret = devm_request_irq(&spi->dev, spi->irq, ad7766_irq, - IRQF_TRIGGER_FALLING | IRQF_NO_AUTOEN, + IRQF_TRIGGER_FALLING | IRQF_NO_AUTOEN | IRQF_NO_THREAD, dev_name(&spi->dev), ad7766->trig); if (ret < 0) diff --git a/drivers/iio/adc/ade9000.c b/drivers/iio/adc/ade9000.c index 2de8a718d62af0..3c2a93f1146098 100644 --- a/drivers/iio/adc/ade9000.c +++ b/drivers/iio/adc/ade9000.c @@ -787,7 +787,7 @@ static int ade9000_iio_push_streaming(struct iio_dev *indio_dev) ADE9000_MIDDLE_PAGE_BIT); if (ret) { dev_err_ratelimited(dev, "IRQ0 WFB write fail"); - return IRQ_HANDLED; + return ret; } ade9000_configure_scan(indio_dev, ADE9000_REG_WF_BUFF); @@ -1123,7 +1123,7 @@ static int ade9000_write_raw(struct iio_dev *indio_dev, tmp &= ~ADE9000_PHASE_C_POS_BIT; switch (tmp) { - case ADE9000_REG_AWATTOS: + case ADE9000_REG_AWATT: return regmap_write(st->regmap, ADE9000_ADDR_ADJUST(ADE9000_REG_AWATTOS, chan->channel), val); @@ -1706,19 +1706,19 @@ static int ade9000_probe(struct spi_device *spi) init_completion(&st->reset_completion); - ret = ade9000_request_irq(dev, "irq0", ade9000_irq0_thread, indio_dev); + ret = devm_mutex_init(dev, &st->lock); if (ret) return ret; - ret = ade9000_request_irq(dev, "irq1", ade9000_irq1_thread, indio_dev); + ret = ade9000_request_irq(dev, "irq0", ade9000_irq0_thread, indio_dev); if (ret) return ret; - ret = ade9000_request_irq(dev, "dready", ade9000_dready_thread, indio_dev); + ret = ade9000_request_irq(dev, "irq1", ade9000_irq1_thread, indio_dev); if (ret) return ret; - ret = devm_mutex_init(dev, &st->lock); + ret = ade9000_request_irq(dev, "dready", ade9000_dready_thread, indio_dev); if (ret) return ret; diff --git a/drivers/iio/adc/aspeed_adc.c b/drivers/iio/adc/aspeed_adc.c index bf2bfd6bdc412d..9df6e7f68f19c7 100644 --- a/drivers/iio/adc/aspeed_adc.c +++ b/drivers/iio/adc/aspeed_adc.c @@ -415,6 +415,7 @@ static int aspeed_adc_vref_config(struct iio_dev *indio_dev) } adc_engine_control_reg_val = readl(data->base + ASPEED_REG_ENGINE_CONTROL); + adc_engine_control_reg_val &= ~ASPEED_ADC_REF_VOLTAGE; ret = devm_regulator_get_enable_read_voltage(data->dev, "vref"); if (ret < 0 && ret != -ENODEV) diff --git a/drivers/iio/adc/ti-adc161s626.c b/drivers/iio/adc/ti-adc161s626.c index 28aa6b80160c8b..be1cc2e77862b8 100644 --- a/drivers/iio/adc/ti-adc161s626.c +++ b/drivers/iio/adc/ti-adc161s626.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -70,8 +71,7 @@ struct ti_adc_data { u8 read_size; u8 shift; - - u8 buffer[16] __aligned(IIO_DMA_MINALIGN); + u8 buf[3] __aligned(IIO_DMA_MINALIGN); }; static int ti_adc_read_measurement(struct ti_adc_data *data, @@ -80,26 +80,20 @@ static int ti_adc_read_measurement(struct ti_adc_data *data, int ret; switch (data->read_size) { - case 2: { - __be16 buf; - - ret = spi_read(data->spi, (void *) &buf, 2); + case 2: + ret = spi_read(data->spi, data->buf, 2); if (ret) return ret; - *val = be16_to_cpu(buf); + *val = get_unaligned_be16(data->buf); break; - } - case 3: { - __be32 buf; - - ret = spi_read(data->spi, (void *) &buf, 3); + case 3: + ret = spi_read(data->spi, data->buf, 3); if (ret) return ret; - *val = be32_to_cpu(buf) >> 8; + *val = get_unaligned_be24(data->buf); break; - } default: return -EINVAL; } @@ -114,15 +108,20 @@ static irqreturn_t ti_adc_trigger_handler(int irq, void *private) struct iio_poll_func *pf = private; struct iio_dev *indio_dev = pf->indio_dev; struct ti_adc_data *data = iio_priv(indio_dev); - int ret; + struct { + s16 data; + aligned_s64 timestamp; + } scan = { }; + int ret, val; + + ret = ti_adc_read_measurement(data, &indio_dev->channels[0], &val); + if (ret) + goto exit_notify_done; - ret = ti_adc_read_measurement(data, &indio_dev->channels[0], - (int *) &data->buffer); - if (!ret) - iio_push_to_buffers_with_timestamp(indio_dev, - data->buffer, - iio_get_time_ns(indio_dev)); + scan.data = val; + iio_push_to_buffers_with_timestamp(indio_dev, &scan, iio_get_time_ns(indio_dev)); + exit_notify_done: iio_trigger_notify_done(indio_dev->trig); return IRQ_HANDLED; diff --git a/drivers/iio/adc/ti-ads1119.c b/drivers/iio/adc/ti-ads1119.c index c9cedc59cdcdb1..79be71b4de963a 100644 --- a/drivers/iio/adc/ti-ads1119.c +++ b/drivers/iio/adc/ti-ads1119.c @@ -274,12 +274,15 @@ static int ads1119_single_conversion(struct ads1119_state *st, ret = pm_runtime_resume_and_get(dev); if (ret) - goto pdown; + return ret; ret = ads1119_configure_channel(st, mux, gain, datarate); if (ret) goto pdown; + if (st->client->irq) + reinit_completion(&st->completion); + ret = i2c_smbus_write_byte(st->client, ADS1119_CMD_START_SYNC); if (ret) goto pdown; @@ -735,10 +738,8 @@ static int ads1119_probe(struct i2c_client *client) return dev_err_probe(dev, ret, "Failed to setup IIO buffer\n"); if (client->irq > 0) { - ret = devm_request_threaded_irq(dev, client->irq, - ads1119_irq_handler, - NULL, IRQF_ONESHOT, - "ads1119", indio_dev); + ret = devm_request_irq(dev, client->irq, ads1119_irq_handler, + IRQF_NO_THREAD, "ads1119", indio_dev); if (ret) return dev_err_probe(dev, ret, "Failed to allocate irq\n"); diff --git a/drivers/iio/adc/ti-ads7950.c b/drivers/iio/adc/ti-ads7950.c index bbe1ce57778958..cdc62488955933 100644 --- a/drivers/iio/adc/ti-ads7950.c +++ b/drivers/iio/adc/ti-ads7950.c @@ -427,13 +427,15 @@ static int ti_ads7950_set(struct gpio_chip *chip, unsigned int offset, static int ti_ads7950_get(struct gpio_chip *chip, unsigned int offset) { struct ti_ads7950_state *st = gpiochip_get_data(chip); + bool state; int ret; mutex_lock(&st->slock); /* If set as output, return the output */ if (st->gpio_cmd_settings_bitmask & BIT(offset)) { - ret = st->cmd_settings_bitmask & BIT(offset); + state = st->cmd_settings_bitmask & BIT(offset); + ret = 0; goto out; } @@ -444,7 +446,7 @@ static int ti_ads7950_get(struct gpio_chip *chip, unsigned int offset) if (ret) goto out; - ret = ((st->single_rx >> 12) & BIT(offset)) ? 1 : 0; + state = (st->single_rx >> 12) & BIT(offset); /* Revert back to original settings */ st->cmd_settings_bitmask &= ~TI_ADS7950_CR_GPIO_DATA; @@ -456,7 +458,7 @@ static int ti_ads7950_get(struct gpio_chip *chip, unsigned int offset) out: mutex_unlock(&st->slock); - return ret; + return ret ?: state; } static int ti_ads7950_get_direction(struct gpio_chip *chip, diff --git a/drivers/iio/chemical/bme680_core.c b/drivers/iio/chemical/bme680_core.c index 70f81c4a96bacd..24e0b59e2fdf0b 100644 --- a/drivers/iio/chemical/bme680_core.c +++ b/drivers/iio/chemical/bme680_core.c @@ -613,7 +613,7 @@ static int bme680_wait_for_eoc(struct bme680_data *data) * + heater duration */ int wait_eoc_us = ((data->oversampling_temp + data->oversampling_press + - data->oversampling_humid) * 1936) + (477 * 4) + + data->oversampling_humid) * 1963) + (477 * 4) + (477 * 5) + 1000 + (data->heater_dur * 1000); fsleep(wait_eoc_us); diff --git a/drivers/iio/chemical/sps30_i2c.c b/drivers/iio/chemical/sps30_i2c.c index f692c089d17b40..c92f04990c34c2 100644 --- a/drivers/iio/chemical/sps30_i2c.c +++ b/drivers/iio/chemical/sps30_i2c.c @@ -171,7 +171,7 @@ static int sps30_i2c_read_meas(struct sps30_state *state, __be32 *meas, size_t n if (!sps30_i2c_meas_ready(state)) return -ETIMEDOUT; - return sps30_i2c_command(state, SPS30_I2C_READ_MEAS, NULL, 0, meas, sizeof(num) * num); + return sps30_i2c_command(state, SPS30_I2C_READ_MEAS, NULL, 0, meas, sizeof(*meas) * num); } static int sps30_i2c_clean_fan(struct sps30_state *state) diff --git a/drivers/iio/chemical/sps30_serial.c b/drivers/iio/chemical/sps30_serial.c index 008bc88590f370..a5e6bc08d5fd41 100644 --- a/drivers/iio/chemical/sps30_serial.c +++ b/drivers/iio/chemical/sps30_serial.c @@ -303,7 +303,7 @@ static int sps30_serial_read_meas(struct sps30_state *state, __be32 *meas, size_ if (msleep_interruptible(1000)) return -EINTR; - ret = sps30_serial_command(state, SPS30_SERIAL_READ_MEAS, NULL, 0, meas, num * sizeof(num)); + ret = sps30_serial_command(state, SPS30_SERIAL_READ_MEAS, NULL, 0, meas, num * sizeof(*meas)); if (ret < 0) return ret; /* if measurements aren't ready sensor returns empty frame */ diff --git a/drivers/iio/common/Kconfig b/drivers/iio/common/Kconfig index 1ccb5ccf370660..e3818ef567822b 100644 --- a/drivers/iio/common/Kconfig +++ b/drivers/iio/common/Kconfig @@ -3,6 +3,7 @@ # IIO common modules # +source "drivers/iio/common/aop_sensors/Kconfig" source "drivers/iio/common/cros_ec_sensors/Kconfig" source "drivers/iio/common/hid-sensors/Kconfig" source "drivers/iio/common/inv_sensors/Kconfig" diff --git a/drivers/iio/common/Makefile b/drivers/iio/common/Makefile index d3e952239a6219..5f99a429725d66 100644 --- a/drivers/iio/common/Makefile +++ b/drivers/iio/common/Makefile @@ -8,6 +8,7 @@ # # When adding new entries keep the list in alphabetical order +obj-y += aop_sensors/ obj-y += cros_ec_sensors/ obj-y += hid-sensors/ obj-y += inv_sensors/ diff --git a/drivers/iio/common/aop_sensors/Kconfig b/drivers/iio/common/aop_sensors/Kconfig new file mode 100644 index 00000000000000..b2c4397787dbfd --- /dev/null +++ b/drivers/iio/common/aop_sensors/Kconfig @@ -0,0 +1,22 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT + +config IIO_AOP_SENSOR_LAS + tristate "AOP Lid angle sensor" + depends on ARCH_APPLE || COMPILE_TEST + depends on RUST + depends on SYSFS + select APPLE_AOP + help + Module to handle the lid angle sensor attached to the AOP + coprocessor on Apple laptops. + +config IIO_AOP_SENSOR_ALS + tristate "AOP Ambient light sensor" + depends on ARCH_APPLE || COMPILE_TEST + depends on RUST + depends on SYSFS + select APPLE_AOP + select RUST_FW_LOADER_ABSTRACTIONS + help + Module to handle the ambient light sensor attached to the AOP + coprocessor on Apple laptops. diff --git a/drivers/iio/common/aop_sensors/Makefile b/drivers/iio/common/aop_sensors/Makefile new file mode 100644 index 00000000000000..8da5a19efe0f0c --- /dev/null +++ b/drivers/iio/common/aop_sensors/Makefile @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT + +obj-$(CONFIG_IIO_AOP_SENSOR_LAS) += aop_las.o +obj-$(CONFIG_IIO_AOP_SENSOR_ALS) += aop_als.o diff --git a/drivers/iio/common/aop_sensors/aop_als.rs b/drivers/iio/common/aop_sensors/aop_als.rs new file mode 100644 index 00000000000000..87a4686f3ca03a --- /dev/null +++ b/drivers/iio/common/aop_sensors/aop_als.rs @@ -0,0 +1,145 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Apple AOP ambient light sensor driver +//! +//! Copyright (C) The Asahi Linux Contributors + +use kernel::{ + bindings, c_str, + device::Core, + firmware::Firmware, + iio::common::aop_sensors::{AopSensorData, IIORegistration, MessageProcessor}, + module_platform_driver, of, platform, + prelude::*, + soc::apple::aop::{EPICService, AOP}, + sync::Arc, + types::ForeignOwnable, +}; + +const EPIC_SUBTYPE_GET_AOP_PROPERTY: u16 = 0xa; +const EPIC_SUBTYPE_SET_ALS_PROPERTY: u16 = 0x4; +const LUX_OFFSET_CT720: usize = 0x1d; +const LUX_OFFSET_VD6286: usize = 0x28; + +fn get_lux_offset(aop: &dyn AOP, dev: &platform::Device, svc: &EPICService) -> Result { + let name = get_aop_property(aop, svc, 0xf, 16)?.1; + match name.as_slice() { + b"Redbird\0" => Ok(LUX_OFFSET_VD6286), + b"FireFish2\0" => Ok(LUX_OFFSET_CT720), + _ => { + dev_warn!( + dev.as_ref(), + "Unknown sensor type {:?}", + core::str::from_utf8(&name) + ); + Err(EIO) + } + } +} + +fn enable_als(aop: &dyn AOP, dev: &platform::Device, svc: &EPICService) -> Result<()> { + let fw = Firmware::request(c_str!("apple/aop-als-cal.bin"), dev.as_ref())?; + set_als_property(aop, svc, 0xb, fw.data())?; + set_als_property(aop, svc, 0, &200000u32.to_le_bytes())?; + + Ok(()) +} + +fn get_aop_property( + aop: &dyn AOP, + svc: &EPICService, + tag: u32, + data_len: usize, +) -> Result<(u32, KVec)> { + let mut buf = KVec::new(); + buf.resize(8, 0, GFP_KERNEL)?; + buf[4..8].copy_from_slice(&tag.to_le_bytes()); + aop.epic_call_ret(svc, EPIC_SUBTYPE_GET_AOP_PROPERTY, &buf, data_len) +} + +fn set_als_property(aop: &dyn AOP, svc: &EPICService, tag: u32, data: &[u8]) -> Result { + let mut buf = KVec::new(); + buf.resize(data.len() + 8, 0, GFP_KERNEL)?; + buf[8..].copy_from_slice(data); + buf[4..8].copy_from_slice(&tag.to_le_bytes()); + aop.epic_call(svc, EPIC_SUBTYPE_SET_ALS_PROPERTY, &buf) +} + +fn f32_to_u32(f: u32) -> u32 { + if f & 0x80000000 != 0 { + return 0; + } + let exp = ((f & 0x7f800000) >> 23) as i32 - 127; + if exp < 0 { + return 0; + } + if exp == 128 && f & 0x7fffff != 0 { + return 0; + } + let mant = f & 0x7fffff | 0x800000; + if exp <= 23 { + return mant >> (23 - exp); + } + if exp >= 32 { + return u32::MAX; + } + mant << (exp - 23) +} + +struct MsgProc(usize); + +impl MessageProcessor for MsgProc { + fn process(&self, message: &[u8]) -> u32 { + let offset = self.0; + let raw = u32::from_le_bytes(message[offset..offset + 4].try_into().unwrap()); + f32_to_u32(raw) + } +} + +#[repr(transparent)] +struct IIOAopAlsDriver(IIORegistration); + +kernel::of_device_table!( + OF_TABLE, + MODULE_OF_TABLE, + (), + [(of::DeviceId::new(c_str!("apple,aop-als")), ())] +); + +impl platform::Driver for IIOAopAlsDriver { + type IdInfo = (); + + const OF_ID_TABLE: Option> = Some(&OF_TABLE); + + fn probe(pdev: &platform::Device, _info: Option<&()>) -> impl PinInit { + let dev = pdev.as_ref(); + let parent = dev.parent().unwrap(); + // SAFETY: our parent is AOP, and AopDriver is repr(transparent) for Arc + let adata_ptr = unsafe { Pin::>>::borrow(parent.get_drvdata()) }; + let adata = (&*adata_ptr).clone(); + // SAFETY: AOP sets the platform data correctly + let service = unsafe { *((*dev.as_raw()).platform_data as *const EPICService) }; + let ty = bindings::BINDINGS_IIO_LIGHT; + let offset = get_lux_offset(adata.as_ref(), pdev, &service)?; + let data = AopSensorData::new(dev.into(), ty, MsgProc(offset))?; + adata.add_fakehid_listener(service, data.clone())?; + enable_als(adata.as_ref(), pdev, &service)?; + let info_mask = 1 << bindings::BINDINGS_IIO_CHAN_INFO_PROCESSED; + Ok(IIOAopAlsDriver(IIORegistration::::new( + data, + c"aop-sensors-als", + ty, + info_mask, + &THIS_MODULE, + )?)) + } +} + +module_platform_driver! { + type: IIOAopAlsDriver, + name: "iio_aop_als", + description: "AOP ambient light sensor driver", + license: "Dual MIT/GPL", + alias: ["platform:iio_aop_als"], + firmware: ["apple/aop-als-cal.bin"], +} diff --git a/drivers/iio/common/aop_sensors/aop_las.rs b/drivers/iio/common/aop_sensors/aop_las.rs new file mode 100644 index 00000000000000..9256d61aed9d83 --- /dev/null +++ b/drivers/iio/common/aop_sensors/aop_las.rs @@ -0,0 +1,70 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Apple AOP lid angle sensor driver +//! +//! Copyright (C) The Asahi Linux Contributors + +use kernel::{ + bindings, c_str, + device::Core, + iio::common::aop_sensors::{AopSensorData, IIORegistration, MessageProcessor}, + module_platform_driver, of, platform, + prelude::*, + soc::apple::aop::{EPICService, AOP}, + sync::Arc, + types::ForeignOwnable, +}; + +struct MsgProc; + +impl MessageProcessor for MsgProc { + fn process(&self, message: &[u8]) -> u32 { + message[1] as u32 + } +} + +#[repr(transparent)] +struct IIOAopLasDriver(IIORegistration); + +kernel::of_device_table!( + OF_TABLE, + MODULE_OF_TABLE, + (), + [(of::DeviceId::new(c_str!("apple,aop-las")), ())] +); + +impl platform::Driver for IIOAopLasDriver { + type IdInfo = (); + + const OF_ID_TABLE: Option> = Some(&OF_TABLE); + + fn probe(pdev: &platform::Device, _info: Option<&()>) -> impl PinInit { + let dev = pdev.as_ref(); + let parent = dev.parent().unwrap(); + // SAFETY: our parent is AOP, and AopDriver is repr(transparent) for Arc + let adata_ptr = unsafe { Pin::>>::borrow(parent.get_drvdata()) }; + let adata = (&*adata_ptr).clone(); + // SAFETY: AOP sets the platform data correctly + let service = unsafe { *((*dev.as_raw()).platform_data as *const EPICService) }; + + let ty = bindings::BINDINGS_IIO_ANGL; + let data = AopSensorData::new(dev.into(), ty, MsgProc)?; + adata.add_fakehid_listener(service, data.clone())?; + let info_mask = 1 << bindings::BINDINGS_IIO_CHAN_INFO_RAW; + Ok(IIOAopLasDriver(IIORegistration::::new( + data, + c"aop-sensors-las", + ty, + info_mask, + &THIS_MODULE, + )?)) + } +} + +module_platform_driver! { + type: IIOAopLasDriver, + name: "iio_aop_las", + description: "AOP lid angle sensor driver", + license: "Dual MIT/GPL", + alias: ["platform:iio_aop_las"], +} diff --git a/drivers/iio/dac/ad5770r.c b/drivers/iio/dac/ad5770r.c index cd47cb1c685c4d..6027e8d88b2781 100644 --- a/drivers/iio/dac/ad5770r.c +++ b/drivers/iio/dac/ad5770r.c @@ -322,7 +322,7 @@ static int ad5770r_read_raw(struct iio_dev *indio_dev, chan->address, st->transf_buf, 2); if (ret) - return 0; + return ret; buf16 = get_unaligned_le16(st->transf_buf); *val = buf16 >> 2; diff --git a/drivers/iio/dac/ds4424.c b/drivers/iio/dac/ds4424.c index a8198ba4f98a62..059acca45f64fa 100644 --- a/drivers/iio/dac/ds4424.c +++ b/drivers/iio/dac/ds4424.c @@ -141,7 +141,7 @@ static int ds4424_write_raw(struct iio_dev *indio_dev, switch (mask) { case IIO_CHAN_INFO_RAW: - if (val < S8_MIN || val > S8_MAX) + if (val <= S8_MIN || val > S8_MAX) return -EINVAL; if (val > 0) { diff --git a/drivers/iio/frequency/adf4377.c b/drivers/iio/frequency/adf4377.c index 08833b7035e4ba..48aa4b015a141e 100644 --- a/drivers/iio/frequency/adf4377.c +++ b/drivers/iio/frequency/adf4377.c @@ -501,7 +501,7 @@ static int adf4377_soft_reset(struct adf4377_state *st) return ret; return regmap_read_poll_timeout(st->regmap, 0x0, read_val, - !(read_val & (ADF4377_0000_SOFT_RESET_R_MSK | + !(read_val & (ADF4377_0000_SOFT_RESET_MSK | ADF4377_0000_SOFT_RESET_R_MSK)), 200, 200 * 100); } diff --git a/drivers/iio/gyro/itg3200_buffer.c b/drivers/iio/gyro/itg3200_buffer.c index a624400a239cbe..cf97adfa97274b 100644 --- a/drivers/iio/gyro/itg3200_buffer.c +++ b/drivers/iio/gyro/itg3200_buffer.c @@ -118,11 +118,9 @@ int itg3200_probe_trigger(struct iio_dev *indio_dev) if (!st->trig) return -ENOMEM; - ret = request_irq(st->i2c->irq, - &iio_trigger_generic_data_rdy_poll, - IRQF_TRIGGER_RISING, - "itg3200_data_rdy", - st->trig); + ret = request_irq(st->i2c->irq, &iio_trigger_generic_data_rdy_poll, + IRQF_TRIGGER_RISING | IRQF_NO_THREAD, + "itg3200_data_rdy", st->trig); if (ret) goto error_free_trig; diff --git a/drivers/iio/gyro/itg3200_core.c b/drivers/iio/gyro/itg3200_core.c index cd8a2dae56cd90..bfe95ec1abda9b 100644 --- a/drivers/iio/gyro/itg3200_core.c +++ b/drivers/iio/gyro/itg3200_core.c @@ -93,6 +93,8 @@ static int itg3200_read_raw(struct iio_dev *indio_dev, case IIO_CHAN_INFO_RAW: reg = (u8)chan->address; ret = itg3200_read_reg_s16(indio_dev, reg, val); + if (ret) + return ret; return IIO_VAL_INT; case IIO_CHAN_INFO_SCALE: *val = 0; diff --git a/drivers/iio/gyro/mpu3050-core.c b/drivers/iio/gyro/mpu3050-core.c index 67ae7d1012bc27..d84e04e4b43142 100644 --- a/drivers/iio/gyro/mpu3050-core.c +++ b/drivers/iio/gyro/mpu3050-core.c @@ -322,7 +322,9 @@ static int mpu3050_read_raw(struct iio_dev *indio_dev, } case IIO_CHAN_INFO_RAW: /* Resume device */ - pm_runtime_get_sync(mpu3050->dev); + ret = pm_runtime_resume_and_get(mpu3050->dev); + if (ret) + return ret; mutex_lock(&mpu3050->lock); ret = mpu3050_set_8khz_samplerate(mpu3050); @@ -647,14 +649,20 @@ static irqreturn_t mpu3050_trigger_handler(int irq, void *p) static int mpu3050_buffer_preenable(struct iio_dev *indio_dev) { struct mpu3050 *mpu3050 = iio_priv(indio_dev); + int ret; - pm_runtime_get_sync(mpu3050->dev); + ret = pm_runtime_resume_and_get(mpu3050->dev); + if (ret) + return ret; /* Unless we have OUR trigger active, run at full speed */ - if (!mpu3050->hw_irq_trigger) - return mpu3050_set_8khz_samplerate(mpu3050); + if (!mpu3050->hw_irq_trigger) { + ret = mpu3050_set_8khz_samplerate(mpu3050); + if (ret) + pm_runtime_put_autosuspend(mpu3050->dev); + } - return 0; + return ret; } static int mpu3050_buffer_postdisable(struct iio_dev *indio_dev) @@ -1121,11 +1129,16 @@ static int mpu3050_trigger_probe(struct iio_dev *indio_dev, int irq) ret = iio_trigger_register(mpu3050->trig); if (ret) - return ret; + goto err_iio_trigger; indio_dev->trig = iio_trigger_get(mpu3050->trig); return 0; + +err_iio_trigger: + free_irq(mpu3050->irq, mpu3050->trig); + + return ret; } int mpu3050_common_probe(struct device *dev, @@ -1162,10 +1175,8 @@ int mpu3050_common_probe(struct device *dev, mpu3050->regs[1].supply = mpu3050_reg_vlogic; ret = devm_regulator_bulk_get(dev, ARRAY_SIZE(mpu3050->regs), mpu3050->regs); - if (ret) { - dev_err(dev, "Cannot get regulators\n"); - return ret; - } + if (ret) + return dev_err_probe(dev, ret, "Cannot get regulators\n"); ret = mpu3050_power_up(mpu3050); if (ret) @@ -1215,12 +1226,6 @@ int mpu3050_common_probe(struct device *dev, goto err_power_down; } - ret = iio_device_register(indio_dev); - if (ret) { - dev_err(dev, "device register failed\n"); - goto err_cleanup_buffer; - } - dev_set_drvdata(dev, indio_dev); /* Check if we have an assigned IRQ to use as trigger */ @@ -1243,9 +1248,20 @@ int mpu3050_common_probe(struct device *dev, pm_runtime_use_autosuspend(dev); pm_runtime_put(dev); + ret = iio_device_register(indio_dev); + if (ret) { + dev_err(dev, "device register failed\n"); + goto err_iio_device_register; + } + return 0; -err_cleanup_buffer: +err_iio_device_register: + pm_runtime_get_sync(dev); + pm_runtime_put_noidle(dev); + pm_runtime_disable(dev); + if (irq) + free_irq(mpu3050->irq, mpu3050->trig); iio_triggered_buffer_cleanup(indio_dev); err_power_down: mpu3050_power_down(mpu3050); @@ -1258,13 +1274,13 @@ void mpu3050_common_remove(struct device *dev) struct iio_dev *indio_dev = dev_get_drvdata(dev); struct mpu3050 *mpu3050 = iio_priv(indio_dev); + iio_device_unregister(indio_dev); pm_runtime_get_sync(dev); pm_runtime_put_noidle(dev); pm_runtime_disable(dev); - iio_triggered_buffer_cleanup(indio_dev); if (mpu3050->irq) - free_irq(mpu3050->irq, mpu3050); - iio_device_unregister(indio_dev); + free_irq(mpu3050->irq, mpu3050->trig); + iio_triggered_buffer_cleanup(indio_dev); mpu3050_power_down(mpu3050); } diff --git a/drivers/iio/gyro/mpu3050-i2c.c b/drivers/iio/gyro/mpu3050-i2c.c index 092878f2c88693..6549b22e643d81 100644 --- a/drivers/iio/gyro/mpu3050-i2c.c +++ b/drivers/iio/gyro/mpu3050-i2c.c @@ -19,8 +19,7 @@ static int mpu3050_i2c_bypass_select(struct i2c_mux_core *mux, u32 chan_id) struct mpu3050 *mpu3050 = i2c_mux_priv(mux); /* Just power up the device, that is all that is needed */ - pm_runtime_get_sync(mpu3050->dev); - return 0; + return pm_runtime_resume_and_get(mpu3050->dev); } static int mpu3050_i2c_bypass_deselect(struct i2c_mux_core *mux, u32 chan_id) diff --git a/drivers/iio/imu/adis.c b/drivers/iio/imu/adis.c index d160147cce0ba7..a2bc1d14ed91da 100644 --- a/drivers/iio/imu/adis.c +++ b/drivers/iio/imu/adis.c @@ -526,7 +526,7 @@ int adis_init(struct adis *adis, struct iio_dev *indio_dev, adis->spi = spi; adis->data = data; - if (!adis->ops->write && !adis->ops->read && !adis->ops->reset) + if (!adis->ops) adis->ops = &adis_default_ops; else if (!adis->ops->write || !adis->ops->read || !adis->ops->reset) return -EINVAL; diff --git a/drivers/iio/imu/adis16550.c b/drivers/iio/imu/adis16550.c index 28f0dbd0226cbe..1f2af506f4bdd5 100644 --- a/drivers/iio/imu/adis16550.c +++ b/drivers/iio/imu/adis16550.c @@ -643,12 +643,12 @@ static int adis16550_read_raw(struct iio_dev *indio_dev, case IIO_CHAN_INFO_LOW_PASS_FILTER_3DB_FREQUENCY: switch (chan->type) { case IIO_ANGL_VEL: - ret = adis16550_get_accl_filter_freq(st, val); + ret = adis16550_get_gyro_filter_freq(st, val); if (ret) return ret; return IIO_VAL_INT; case IIO_ACCEL: - ret = adis16550_get_gyro_filter_freq(st, val); + ret = adis16550_get_accl_filter_freq(st, val); if (ret) return ret; return IIO_VAL_INT; @@ -681,9 +681,9 @@ static int adis16550_write_raw(struct iio_dev *indio_dev, case IIO_CHAN_INFO_LOW_PASS_FILTER_3DB_FREQUENCY: switch (chan->type) { case IIO_ANGL_VEL: - return adis16550_set_accl_filter_freq(st, val); - case IIO_ACCEL: return adis16550_set_gyro_filter_freq(st, val); + case IIO_ACCEL: + return adis16550_set_accl_filter_freq(st, val); default: return -EINVAL; } diff --git a/drivers/iio/imu/bmi160/bmi160_core.c b/drivers/iio/imu/bmi160/bmi160_core.c index 5f47708b4c5dc5..4abb83b75e2e87 100644 --- a/drivers/iio/imu/bmi160/bmi160_core.c +++ b/drivers/iio/imu/bmi160/bmi160_core.c @@ -573,12 +573,16 @@ static int bmi160_config_pin(struct regmap *regmap, enum bmi160_int_pin pin, int_out_ctrl_shift = BMI160_INT1_OUT_CTRL_SHIFT; int_latch_mask = BMI160_INT1_LATCH_MASK; int_map_mask = BMI160_INT1_MAP_DRDY_EN; + pin_name = "INT1"; break; case BMI160_PIN_INT2: int_out_ctrl_shift = BMI160_INT2_OUT_CTRL_SHIFT; int_latch_mask = BMI160_INT2_LATCH_MASK; int_map_mask = BMI160_INT2_MAP_DRDY_EN; + pin_name = "INT2"; break; + default: + return -EINVAL; } int_out_ctrl_mask = BMI160_INT_OUT_CTRL_MASK << int_out_ctrl_shift; @@ -612,17 +616,8 @@ static int bmi160_config_pin(struct regmap *regmap, enum bmi160_int_pin pin, ret = bmi160_write_conf_reg(regmap, BMI160_REG_INT_MAP, int_map_mask, int_map_mask, write_usleep); - if (ret) { - switch (pin) { - case BMI160_PIN_INT1: - pin_name = "INT1"; - break; - case BMI160_PIN_INT2: - pin_name = "INT2"; - break; - } + if (ret) dev_err(dev, "Failed to configure %s IRQ pin", pin_name); - } return ret; } diff --git a/drivers/iio/imu/bmi270/bmi270_i2c.c b/drivers/iio/imu/bmi270/bmi270_i2c.c index b909a421ad0176..b92da4e0776fa5 100644 --- a/drivers/iio/imu/bmi270/bmi270_i2c.c +++ b/drivers/iio/imu/bmi270/bmi270_i2c.c @@ -37,6 +37,7 @@ static const struct i2c_device_id bmi270_i2c_id[] = { { "bmi270", (kernel_ulong_t)&bmi270_chip_info }, { } }; +MODULE_DEVICE_TABLE(i2c, bmi270_i2c_id); static const struct acpi_device_id bmi270_acpi_match[] = { /* GPD Win Mini, Aya Neo AIR Pro, OXP Mini Pro, etc. */ @@ -45,12 +46,14 @@ static const struct acpi_device_id bmi270_acpi_match[] = { { "BMI0260", (kernel_ulong_t)&bmi260_chip_info }, { } }; +MODULE_DEVICE_TABLE(acpi, bmi270_acpi_match); static const struct of_device_id bmi270_of_match[] = { { .compatible = "bosch,bmi260", .data = &bmi260_chip_info }, { .compatible = "bosch,bmi270", .data = &bmi270_chip_info }, { } }; +MODULE_DEVICE_TABLE(of, bmi270_of_match); static struct i2c_driver bmi270_i2c_driver = { .driver = { diff --git a/drivers/iio/imu/bno055/bno055.c b/drivers/iio/imu/bno055/bno055.c index 303bc308f80a84..c96fec2ebb3e78 100644 --- a/drivers/iio/imu/bno055/bno055.c +++ b/drivers/iio/imu/bno055/bno055.c @@ -64,7 +64,7 @@ #define BNO055_GRAVITY_DATA_X_LSB_REG 0x2E #define BNO055_GRAVITY_DATA_Y_LSB_REG 0x30 #define BNO055_GRAVITY_DATA_Z_LSB_REG 0x32 -#define BNO055_SCAN_CH_COUNT ((BNO055_GRAVITY_DATA_Z_LSB_REG - BNO055_ACC_DATA_X_LSB_REG) / 2) +#define BNO055_SCAN_CH_COUNT ((BNO055_GRAVITY_DATA_Z_LSB_REG - BNO055_ACC_DATA_X_LSB_REG) / 2 + 1) #define BNO055_TEMP_REG 0x34 #define BNO055_CALIB_STAT_REG 0x35 #define BNO055_CALIB_STAT_MAGN_SHIFT 0 diff --git a/drivers/iio/imu/inv_icm42600/inv_icm42600_accel.c b/drivers/iio/imu/inv_icm42600/inv_icm42600_accel.c index 54760d8f92a279..0ab6eddf0543fe 100644 --- a/drivers/iio/imu/inv_icm42600/inv_icm42600_accel.c +++ b/drivers/iio/imu/inv_icm42600/inv_icm42600_accel.c @@ -651,6 +651,8 @@ static int inv_icm42600_accel_write_odr(struct iio_dev *indio_dev, return -EINVAL; conf.odr = inv_icm42600_accel_odr_conv[idx / 2]; + if (conf.odr == st->conf.accel.odr) + return 0; pm_runtime_get_sync(dev); mutex_lock(&st->lock); diff --git a/drivers/iio/imu/inv_icm42600/inv_icm42600_buffer.c b/drivers/iio/imu/inv_icm42600/inv_icm42600_buffer.c index ada968be954d48..68a39575803188 100644 --- a/drivers/iio/imu/inv_icm42600/inv_icm42600_buffer.c +++ b/drivers/iio/imu/inv_icm42600/inv_icm42600_buffer.c @@ -371,6 +371,8 @@ static int inv_icm42600_buffer_predisable(struct iio_dev *indio_dev) static int inv_icm42600_buffer_postdisable(struct iio_dev *indio_dev) { struct inv_icm42600_state *st = iio_device_get_drvdata(indio_dev); + struct inv_icm42600_sensor_state *sensor_st = iio_priv(indio_dev); + struct inv_sensors_timestamp *ts = &sensor_st->ts; struct device *dev = regmap_get_device(st->map); unsigned int sensor; unsigned int *watermark; @@ -392,6 +394,8 @@ static int inv_icm42600_buffer_postdisable(struct iio_dev *indio_dev) mutex_lock(&st->lock); + inv_sensors_timestamp_apply_odr(ts, 0, 0, 0); + ret = inv_icm42600_buffer_set_fifo_en(st, st->fifo.en & ~sensor); if (ret) goto out_unlock; diff --git a/drivers/iio/imu/inv_icm42600/inv_icm42600_gyro.c b/drivers/iio/imu/inv_icm42600/inv_icm42600_gyro.c index 7ef0a25ec74f6b..11339ddf1da36c 100644 --- a/drivers/iio/imu/inv_icm42600/inv_icm42600_gyro.c +++ b/drivers/iio/imu/inv_icm42600/inv_icm42600_gyro.c @@ -358,6 +358,8 @@ static int inv_icm42600_gyro_write_odr(struct iio_dev *indio_dev, return -EINVAL; conf.odr = inv_icm42600_gyro_odr_conv[idx / 2]; + if (conf.odr == st->conf.gyro.odr) + return 0; pm_runtime_get_sync(dev); mutex_lock(&st->lock); diff --git a/drivers/iio/imu/inv_icm45600/inv_icm45600.h b/drivers/iio/imu/inv_icm45600/inv_icm45600.h index c5b5446f6c3b43..1c796d4b2a4038 100644 --- a/drivers/iio/imu/inv_icm45600/inv_icm45600.h +++ b/drivers/iio/imu/inv_icm45600/inv_icm45600.h @@ -205,7 +205,7 @@ struct inv_icm45600_sensor_state { #define INV_ICM45600_SPI_SLEW_RATE_38NS 0 #define INV_ICM45600_REG_INT1_CONFIG2 0x0018 -#define INV_ICM45600_INT1_CONFIG2_PUSH_PULL BIT(2) +#define INV_ICM45600_INT1_CONFIG2_OPEN_DRAIN BIT(2) #define INV_ICM45600_INT1_CONFIG2_LATCHED BIT(1) #define INV_ICM45600_INT1_CONFIG2_ACTIVE_HIGH BIT(0) #define INV_ICM45600_INT1_CONFIG2_ACTIVE_LOW 0x00 diff --git a/drivers/iio/imu/inv_icm45600/inv_icm45600_core.c b/drivers/iio/imu/inv_icm45600/inv_icm45600_core.c index 25bd9757a594d0..d49053161a6572 100644 --- a/drivers/iio/imu/inv_icm45600/inv_icm45600_core.c +++ b/drivers/iio/imu/inv_icm45600/inv_icm45600_core.c @@ -637,8 +637,8 @@ static int inv_icm45600_irq_init(struct inv_icm45600_state *st, int irq, break; } - if (!open_drain) - val |= INV_ICM45600_INT1_CONFIG2_PUSH_PULL; + if (open_drain) + val |= INV_ICM45600_INT1_CONFIG2_OPEN_DRAIN; ret = regmap_write(st->map, INV_ICM45600_REG_INT1_CONFIG2, val); if (ret) @@ -744,6 +744,11 @@ int inv_icm45600_core_probe(struct regmap *regmap, const struct inv_icm45600_chi */ fsleep(5 * USEC_PER_MSEC); + /* set pm_runtime active early for disable vddio resource cleanup */ + ret = pm_runtime_set_active(dev); + if (ret) + return ret; + ret = inv_icm45600_enable_regulator_vddio(st); if (ret) return ret; @@ -776,7 +781,7 @@ int inv_icm45600_core_probe(struct regmap *regmap, const struct inv_icm45600_chi if (ret) return ret; - ret = devm_pm_runtime_set_active_enabled(dev); + ret = devm_pm_runtime_enable(dev); if (ret) return ret; diff --git a/drivers/iio/imu/inv_mpu6050/inv_mpu_core.c b/drivers/iio/imu/inv_mpu6050/inv_mpu_core.c index b2fa1f4957a5b9..5796896d54cd86 100644 --- a/drivers/iio/imu/inv_mpu6050/inv_mpu_core.c +++ b/drivers/iio/imu/inv_mpu6050/inv_mpu_core.c @@ -1943,6 +1943,14 @@ int inv_mpu_core_probe(struct regmap *regmap, int irq, const char *name, irq_type); return -EINVAL; } + + /* + * Acking interrupts by status register does not work reliably + * but seem to work when this bit is set. + */ + if (st->chip_type == INV_MPU9150) + st->irq_mask |= INV_MPU6050_INT_RD_CLEAR; + device_set_wakeup_capable(dev, true); st->vdd_supply = devm_regulator_get(dev, "vdd"); diff --git a/drivers/iio/imu/inv_mpu6050/inv_mpu_iio.h b/drivers/iio/imu/inv_mpu6050/inv_mpu_iio.h index 211901f8b8eb6f..6239b1a803f77a 100644 --- a/drivers/iio/imu/inv_mpu6050/inv_mpu_iio.h +++ b/drivers/iio/imu/inv_mpu6050/inv_mpu_iio.h @@ -390,6 +390,8 @@ struct inv_mpu6050_state { /* enable level triggering */ #define INV_MPU6050_LATCH_INT_EN 0x20 #define INV_MPU6050_BIT_BYPASS_EN 0x2 +/* allow acking interrupts by any register read */ +#define INV_MPU6050_INT_RD_CLEAR 0x10 /* Allowed timestamp period jitter in percent */ #define INV_MPU6050_TS_PERIOD_JITTER 4 diff --git a/drivers/iio/imu/inv_mpu6050/inv_mpu_trigger.c b/drivers/iio/imu/inv_mpu6050/inv_mpu_trigger.c index 10a47334207593..22c1ce66f99ee5 100644 --- a/drivers/iio/imu/inv_mpu6050/inv_mpu_trigger.c +++ b/drivers/iio/imu/inv_mpu6050/inv_mpu_trigger.c @@ -248,7 +248,6 @@ static irqreturn_t inv_mpu6050_interrupt_handle(int irq, void *p) switch (st->chip_type) { case INV_MPU6000: case INV_MPU6050: - case INV_MPU9150: /* * WoM is not supported and interrupt status read seems to be broken for * some chips. Since data ready is the only interrupt, bypass interrupt @@ -257,6 +256,10 @@ static irqreturn_t inv_mpu6050_interrupt_handle(int irq, void *p) wom_bits = 0; int_status = INV_MPU6050_BIT_RAW_DATA_RDY_INT; goto data_ready_interrupt; + case INV_MPU9150: + /* IRQ needs to be acked */ + wom_bits = 0; + break; case INV_MPU6500: case INV_MPU6515: case INV_MPU6880: diff --git a/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_buffer.c b/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_buffer.c index 55d877745575c2..5b28a3ffcc3d0a 100644 --- a/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_buffer.c +++ b/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_buffer.c @@ -225,6 +225,10 @@ static int st_lsm6dsx_set_fifo_odr(struct st_lsm6dsx_sensor *sensor, const struct st_lsm6dsx_reg *batch_reg; u8 data; + /* Only internal sensors have a FIFO ODR configuration register. */ + if (sensor->id >= ARRAY_SIZE(hw->settings->batch)) + return 0; + batch_reg = &hw->settings->batch[sensor->id]; if (batch_reg->addr) { int val; @@ -858,12 +862,21 @@ int st_lsm6dsx_fifo_setup(struct st_lsm6dsx_hw *hw) int i, ret; for (i = 0; i < ST_LSM6DSX_ID_MAX; i++) { + const struct iio_dev_attr **attrs; + if (!hw->iio_devs[i]) continue; + /* + * For the accelerometer, allow setting FIFO sampling frequency + * values different from the sensor sampling frequency, which + * may be needed to keep FIFO data rate low while sampling + * acceleration data at high rates for accurate event detection. + */ + attrs = i == ST_LSM6DSX_ID_ACC ? st_lsm6dsx_buffer_attrs : NULL; ret = devm_iio_kfifo_buffer_setup_ext(hw->dev, hw->iio_devs[i], &st_lsm6dsx_buffer_ops, - st_lsm6dsx_buffer_attrs); + attrs); if (ret) return ret; } diff --git a/drivers/iio/industrialio-buffer.c b/drivers/iio/industrialio-buffer.c index c6259213e15035..a09c0d263d7feb 100644 --- a/drivers/iio/industrialio-buffer.c +++ b/drivers/iio/industrialio-buffer.c @@ -228,8 +228,10 @@ static ssize_t iio_buffer_write(struct file *filp, const char __user *buf, written = 0; add_wait_queue(&rb->pollq, &wait); do { - if (!indio_dev->info) - return -ENODEV; + if (!indio_dev->info) { + ret = -ENODEV; + break; + } if (!iio_buffer_space_available(rb)) { if (signal_pending(current)) { diff --git a/drivers/iio/light/bh1780.c b/drivers/iio/light/bh1780.c index 5d3c6d5276bac6..a740d1f992a8a1 100644 --- a/drivers/iio/light/bh1780.c +++ b/drivers/iio/light/bh1780.c @@ -109,9 +109,9 @@ static int bh1780_read_raw(struct iio_dev *indio_dev, case IIO_LIGHT: pm_runtime_get_sync(&bh1780->client->dev); value = bh1780_read_word(bh1780, BH1780_REG_DLOW); + pm_runtime_put_autosuspend(&bh1780->client->dev); if (value < 0) return value; - pm_runtime_put_autosuspend(&bh1780->client->dev); *val = value; return IIO_VAL_INT; diff --git a/drivers/iio/light/si1145.c b/drivers/iio/light/si1145.c index f8eb251eca8dc3..ef0abc4499b74a 100644 --- a/drivers/iio/light/si1145.c +++ b/drivers/iio/light/si1145.c @@ -1248,7 +1248,7 @@ static int si1145_probe_trigger(struct iio_dev *indio_dev) ret = devm_request_irq(&client->dev, client->irq, iio_trigger_generic_data_rdy_poll, - IRQF_TRIGGER_FALLING, + IRQF_TRIGGER_FALLING | IRQF_NO_THREAD, "si1145_irq", trig); if (ret < 0) { diff --git a/drivers/iio/light/vcnl4035.c b/drivers/iio/light/vcnl4035.c index 963747927425f7..16aeb17067bc0c 100644 --- a/drivers/iio/light/vcnl4035.c +++ b/drivers/iio/light/vcnl4035.c @@ -103,17 +103,23 @@ static irqreturn_t vcnl4035_trigger_consumer_handler(int irq, void *p) struct iio_dev *indio_dev = pf->indio_dev; struct vcnl4035_data *data = iio_priv(indio_dev); /* Ensure naturally aligned timestamp */ - u8 buffer[ALIGN(sizeof(u16), sizeof(s64)) + sizeof(s64)] __aligned(8) = { }; + struct { + u16 als_data; + aligned_s64 timestamp; + } buffer = { }; + unsigned int val; int ret; - ret = regmap_read(data->regmap, VCNL4035_ALS_DATA, (int *)buffer); + ret = regmap_read(data->regmap, VCNL4035_ALS_DATA, &val); if (ret < 0) { dev_err(&data->client->dev, "Trigger consumer can't read from sensor.\n"); goto fail_read; } - iio_push_to_buffers_with_timestamp(indio_dev, buffer, - iio_get_time_ns(indio_dev)); + + buffer.als_data = val; + iio_push_to_buffers_with_timestamp(indio_dev, &buffer, + iio_get_time_ns(indio_dev)); fail_read: iio_trigger_notify_done(indio_dev->trig); @@ -381,7 +387,7 @@ static const struct iio_chan_spec vcnl4035_channels[] = { .sign = 'u', .realbits = 16, .storagebits = 16, - .endianness = IIO_LE, + .endianness = IIO_CPU, }, }, { @@ -395,7 +401,7 @@ static const struct iio_chan_spec vcnl4035_channels[] = { .sign = 'u', .realbits = 16, .storagebits = 16, - .endianness = IIO_LE, + .endianness = IIO_CPU, }, }, }; diff --git a/drivers/iio/light/veml6070.c b/drivers/iio/light/veml6070.c index 6d4483c85f30cf..74d7246e5225e9 100644 --- a/drivers/iio/light/veml6070.c +++ b/drivers/iio/light/veml6070.c @@ -134,9 +134,7 @@ static int veml6070_read(struct veml6070_data *data) if (ret < 0) return ret; - ret = (msb << 8) | lsb; - - return 0; + return (msb << 8) | lsb; } static const struct iio_chan_spec veml6070_channels[] = { diff --git a/drivers/iio/magnetometer/ak8975.c b/drivers/iio/magnetometer/ak8975.c index 3fd0171e5d69b6..d30315ad85ded3 100644 --- a/drivers/iio/magnetometer/ak8975.c +++ b/drivers/iio/magnetometer/ak8975.c @@ -581,7 +581,7 @@ static int ak8975_setup_irq(struct ak8975_data *data) irq = gpiod_to_irq(data->eoc_gpiod); rc = devm_request_irq(&client->dev, irq, ak8975_irq_handler, - IRQF_TRIGGER_RISING | IRQF_ONESHOT, + IRQF_TRIGGER_RISING, dev_name(&client->dev), data); if (rc < 0) { dev_err(&client->dev, "irq %d request failed: %d\n", irq, rc); diff --git a/drivers/iio/magnetometer/tlv493d.c b/drivers/iio/magnetometer/tlv493d.c index ec53fd40277b0c..e5e050af2b74c1 100644 --- a/drivers/iio/magnetometer/tlv493d.c +++ b/drivers/iio/magnetometer/tlv493d.c @@ -171,7 +171,7 @@ static s16 tlv493d_get_channel_data(u8 *b, enum tlv493d_channels ch) switch (ch) { case TLV493D_AXIS_X: val = FIELD_GET(TLV493D_BX_MAG_X_AXIS_MSB, b[TLV493D_RD_REG_BX]) << 4 | - FIELD_GET(TLV493D_BX2_MAG_X_AXIS_LSB, b[TLV493D_RD_REG_BX2]) >> 4; + FIELD_GET(TLV493D_BX2_MAG_X_AXIS_LSB, b[TLV493D_RD_REG_BX2]); break; case TLV493D_AXIS_Y: val = FIELD_GET(TLV493D_BY_MAG_Y_AXIS_MSB, b[TLV493D_RD_REG_BY]) << 4 | diff --git a/drivers/iio/orientation/hid-sensor-rotation.c b/drivers/iio/orientation/hid-sensor-rotation.c index e759f91a710a2c..5a5e6e4fbe34ba 100644 --- a/drivers/iio/orientation/hid-sensor-rotation.c +++ b/drivers/iio/orientation/hid-sensor-rotation.c @@ -19,8 +19,13 @@ struct dev_rot_state { struct hid_sensor_common common_attributes; struct hid_sensor_hub_attribute_info quaternion; struct { - s32 sampled_vals[4]; - aligned_s64 timestamp; + IIO_DECLARE_QUATERNION(s32, sampled_vals); + /* + * ABI regression avoidance: There are two copies of the same + * timestamp in case of userspace depending on broken alignment + * from older kernels. + */ + aligned_s64 timestamp[2]; } scan; int scale_pre_decml; int scale_post_decml; @@ -154,8 +159,19 @@ static int dev_rot_proc_event(struct hid_sensor_hub_device *hsdev, if (!rot_state->timestamp) rot_state->timestamp = iio_get_time_ns(indio_dev); - iio_push_to_buffers_with_timestamp(indio_dev, &rot_state->scan, - rot_state->timestamp); + /* + * ABI regression avoidance: IIO previously had an incorrect + * implementation of iio_push_to_buffers_with_timestamp() that + * put the timestamp in the last 8 bytes of the buffer, which + * was incorrect according to the IIO ABI. To avoid breaking + * userspace that may be depending on this broken behavior, we + * put the timestamp in both the correct place [0] and the old + * incorrect place [1]. + */ + rot_state->scan.timestamp[0] = rot_state->timestamp; + rot_state->scan.timestamp[1] = rot_state->timestamp; + + iio_push_to_buffers(indio_dev, &rot_state->scan); rot_state->timestamp = 0; } diff --git a/drivers/iio/potentiometer/mcp4131.c b/drivers/iio/potentiometer/mcp4131.c index ad082827aad5e0..56c9111ef5e81f 100644 --- a/drivers/iio/potentiometer/mcp4131.c +++ b/drivers/iio/potentiometer/mcp4131.c @@ -221,7 +221,7 @@ static int mcp4131_write_raw(struct iio_dev *indio_dev, mutex_lock(&data->lock); - data->buf[0] = address << MCP4131_WIPER_SHIFT; + data->buf[0] = address; data->buf[0] |= MCP4131_WRITE | (val >> 8); data->buf[1] = val & 0xFF; /* 8 bits here */ diff --git a/drivers/iio/pressure/mprls0025pa.c b/drivers/iio/pressure/mprls0025pa.c index 2336f2760eaeb7..d4133fef91fac4 100644 --- a/drivers/iio/pressure/mprls0025pa.c +++ b/drivers/iio/pressure/mprls0025pa.c @@ -59,7 +59,7 @@ * * Values given to the userspace in sysfs interface: * * raw - press_cnt - * * offset - (-1 * outputmin) - pmin / scale + * * offset - (-1 * outputmin) + pmin / scale * note: With all sensors from the datasheet pmin = 0 * which reduces the offset to (-1 * outputmin) */ @@ -160,8 +160,8 @@ static const struct iio_chan_spec mpr_channels[] = { BIT(IIO_CHAN_INFO_OFFSET), .scan_index = 0, .scan_type = { - .sign = 's', - .realbits = 32, + .sign = 'u', + .realbits = 24, .storagebits = 32, .endianness = IIO_CPU, }, @@ -313,8 +313,7 @@ static int mpr_read_raw(struct iio_dev *indio_dev, return IIO_VAL_INT_PLUS_NANO; case IIO_CHAN_INFO_OFFSET: *val = data->offset; - *val2 = data->offset2; - return IIO_VAL_INT_PLUS_NANO; + return IIO_VAL_INT; default: return -EINVAL; } @@ -330,8 +329,9 @@ int mpr_common_probe(struct device *dev, const struct mpr_ops *ops, int irq) struct mpr_data *data; struct iio_dev *indio_dev; const char *triplet; - s64 scale, offset; + s64 odelta, pdelta; u32 func; + s32 tmp; indio_dev = devm_iio_device_alloc(dev, sizeof(*data)); if (!indio_dev) @@ -405,23 +405,17 @@ int mpr_common_probe(struct device *dev, const struct mpr_ops *ops, int irq) data->outmin = mpr_func_spec[data->function].output_min; data->outmax = mpr_func_spec[data->function].output_max; - /* use 64 bit calculation for preserving a reasonable precision */ - scale = div_s64(((s64)(data->pmax - data->pmin)) * NANO, - data->outmax - data->outmin); - data->scale = div_s64_rem(scale, NANO, &data->scale2); - /* - * multiply with NANO before dividing by scale and later divide by NANO - * again. - */ - offset = ((-1LL) * (s64)data->outmin) * NANO - - div_s64(div_s64((s64)data->pmin * NANO, scale), NANO); - data->offset = div_s64_rem(offset, NANO, &data->offset2); + odelta = data->outmax - data->outmin; + pdelta = data->pmax - data->pmin; + + data->scale = div_s64_rem(div_s64(pdelta * NANO, odelta), NANO, &tmp); + data->scale2 = tmp; + + data->offset = div_s64(odelta * data->pmin, pdelta) - data->outmin; if (data->irq > 0) { - ret = devm_request_irq(dev, data->irq, mpr_eoc_handler, - IRQF_TRIGGER_RISING, - dev_name(dev), - data); + ret = devm_request_irq(dev, data->irq, mpr_eoc_handler, 0, + dev_name(dev), data); if (ret) return dev_err_probe(dev, ret, "request irq %d failed\n", data->irq); diff --git a/drivers/iio/pressure/mprls0025pa.h b/drivers/iio/pressure/mprls0025pa.h index d62a018eaff32b..b6944b30512677 100644 --- a/drivers/iio/pressure/mprls0025pa.h +++ b/drivers/iio/pressure/mprls0025pa.h @@ -53,7 +53,6 @@ enum mpr_func_id { * @scale: pressure scale * @scale2: pressure scale, decimal number * @offset: pressure offset - * @offset2: pressure offset, decimal number * @gpiod_reset: reset * @irq: end of conversion irq. used to distinguish between irq mode and * reading in a loop until data is ready @@ -75,7 +74,6 @@ struct mpr_data { int scale; int scale2; int offset; - int offset2; struct gpio_desc *gpiod_reset; int irq; struct completion completion; diff --git a/drivers/iio/pressure/mprls0025pa_spi.c b/drivers/iio/pressure/mprls0025pa_spi.c index d04102f8a4a035..cf17eb2e720830 100644 --- a/drivers/iio/pressure/mprls0025pa_spi.c +++ b/drivers/iio/pressure/mprls0025pa_spi.c @@ -8,6 +8,7 @@ * https://prod-edam.honeywell.com/content/dam/honeywell-edam/sps/siot/en-us/products/sensors/pressure-sensors/board-mount-pressure-sensors/micropressure-mpr-series/documents/sps-siot-mpr-series-datasheet-32332628-ciid-172626.pdf */ +#include #include #include #include @@ -40,17 +41,25 @@ static int mpr_spi_xfer(struct mpr_data *data, const u8 cmd, const u8 pkt_len) { struct spi_device *spi = to_spi_device(data->dev); struct mpr_spi_buf *buf = spi_get_drvdata(spi); - struct spi_transfer xfer; + struct spi_transfer xfers[2] = { }; if (pkt_len > MPR_MEASUREMENT_RD_SIZE) return -EOVERFLOW; buf->tx[0] = cmd; - xfer.tx_buf = buf->tx; - xfer.rx_buf = data->buffer; - xfer.len = pkt_len; - return spi_sync_transfer(spi, &xfer, 1); + /* + * Dummy transfer with no data, just cause a 2.5us+ delay between the CS assert + * and the first clock edge as per the datasheet tHDSS timing requirement. + */ + xfers[0].delay.value = 2500; + xfers[0].delay.unit = SPI_DELAY_UNIT_NSECS; + + xfers[1].tx_buf = buf->tx; + xfers[1].rx_buf = data->buffer; + xfers[1].len = pkt_len; + + return spi_sync_transfer(spi, xfers, ARRAY_SIZE(xfers)); } static const struct mpr_ops mpr_spi_ops = { diff --git a/drivers/iio/proximity/hx9023s.c b/drivers/iio/proximity/hx9023s.c index 2918dfc0df547c..17e00ee2b6f844 100644 --- a/drivers/iio/proximity/hx9023s.c +++ b/drivers/iio/proximity/hx9023s.c @@ -719,6 +719,9 @@ static int hx9023s_set_samp_freq(struct hx9023s_data *data, int val, int val2) struct device *dev = regmap_get_device(data->regmap); unsigned int i, period_ms; + if (!val && !val2) + return -EINVAL; + period_ms = div_u64(NANO, (val * MEGA + val2)); for (i = 0; i < ARRAY_SIZE(hx9023s_samp_freq_table); i++) { @@ -1034,9 +1037,8 @@ static int hx9023s_send_cfg(const struct firmware *fw, struct hx9023s_data *data if (!bin) return -ENOMEM; - memcpy(bin->data, fw->data, fw->size); - bin->fw_size = fw->size; + memcpy(bin->data, fw->data, bin->fw_size); bin->fw_ver = bin->data[FW_VER_OFFSET]; bin->reg_count = get_unaligned_le16(bin->data + FW_REG_CNT_OFFSET); diff --git a/drivers/iio/test/Kconfig b/drivers/iio/test/Kconfig index 6e65e929791ca2..4fc17dd0dcd770 100644 --- a/drivers/iio/test/Kconfig +++ b/drivers/iio/test/Kconfig @@ -8,7 +8,6 @@ config IIO_GTS_KUNIT_TEST tristate "Test IIO gain-time-scale helpers" if !KUNIT_ALL_TESTS depends on KUNIT select IIO_GTS_HELPER - select TEST_KUNIT_DEVICE_HELPERS default KUNIT_ALL_TESTS help build unit tests for the IIO light sensor gain-time-scale helpers. diff --git a/drivers/infiniband/core/cache.c b/drivers/infiniband/core/cache.c index 81cf3c902e8195..78bc7d83edc654 100644 --- a/drivers/infiniband/core/cache.c +++ b/drivers/infiniband/core/cache.c @@ -927,6 +927,13 @@ static int gid_table_setup_one(struct ib_device *ib_dev) if (err) return err; + /* + * Mark the device as ready for GID cache updates. This allows netdev + * event handlers to update the GID cache even before the device is + * fully registered. + */ + ib_device_enable_gid_updates(ib_dev); + rdma_roce_rescan_device(ib_dev); return err; @@ -1537,7 +1544,8 @@ static void ib_cache_event_task(struct work_struct *_work) * the cache. */ ret = ib_cache_update(work->event.device, work->event.element.port_num, - work->event.event == IB_EVENT_GID_CHANGE, + work->event.event == IB_EVENT_GID_CHANGE || + work->event.event == IB_EVENT_CLIENT_REREGISTER, work->event.event == IB_EVENT_PKEY_CHANGE, work->enforce_security); @@ -1638,6 +1646,12 @@ void ib_cache_release_one(struct ib_device *device) void ib_cache_cleanup_one(struct ib_device *device) { + /* + * Clear the GID updates mark first to prevent event handlers from + * accessing the device while it's being torn down. + */ + ib_device_disable_gid_updates(device); + /* The cleanup function waits for all in-progress workqueue * elements and cleans up the GID cache. This function should be * called after the device was removed from the devices list and diff --git a/drivers/infiniband/core/core_priv.h b/drivers/infiniband/core/core_priv.h index 05102769a918ad..a2c36666e6fcb9 100644 --- a/drivers/infiniband/core/core_priv.h +++ b/drivers/infiniband/core/core_priv.h @@ -100,6 +100,9 @@ void ib_enum_all_roce_netdevs(roce_netdev_filter filter, roce_netdev_callback cb, void *cookie); +void ib_device_enable_gid_updates(struct ib_device *device); +void ib_device_disable_gid_updates(struct ib_device *device); + typedef int (*nldev_callback)(struct ib_device *device, struct sk_buff *skb, struct netlink_callback *cb, diff --git a/drivers/infiniband/core/device.c b/drivers/infiniband/core/device.c index 1174ab7da6295f..87eaefd3794bb5 100644 --- a/drivers/infiniband/core/device.c +++ b/drivers/infiniband/core/device.c @@ -93,6 +93,7 @@ static struct workqueue_struct *ib_unreg_wq; static DEFINE_XARRAY_FLAGS(devices, XA_FLAGS_ALLOC); static DECLARE_RWSEM(devices_rwsem); #define DEVICE_REGISTERED XA_MARK_1 +#define DEVICE_GID_UPDATES XA_MARK_2 static u32 highest_client_id; #define CLIENT_REGISTERED XA_MARK_1 @@ -2441,11 +2442,42 @@ void ib_enum_all_roce_netdevs(roce_netdev_filter filter, unsigned long index; down_read(&devices_rwsem); - xa_for_each_marked (&devices, index, dev, DEVICE_REGISTERED) + xa_for_each_marked(&devices, index, dev, DEVICE_GID_UPDATES) ib_enum_roce_netdev(dev, filter, filter_cookie, cb, cookie); up_read(&devices_rwsem); } +/** + * ib_device_enable_gid_updates - Mark device as ready for GID cache updates + * @device: Device to mark + * + * Called after GID table is allocated and initialized. After this mark is set, + * netdevice event handlers can update the device's GID cache. This allows + * events that arrive during device registration to be processed, avoiding + * stale GID entries when netdev properties change during the device + * registration process. + */ +void ib_device_enable_gid_updates(struct ib_device *device) +{ + down_write(&devices_rwsem); + xa_set_mark(&devices, device->index, DEVICE_GID_UPDATES); + up_write(&devices_rwsem); +} + +/** + * ib_device_disable_gid_updates - Clear the GID updates mark + * @device: Device to unmark + * + * Called before GID table cleanup to prevent event handlers from accessing + * the device while it's being torn down. + */ +void ib_device_disable_gid_updates(struct ib_device *device) +{ + down_write(&devices_rwsem); + xa_clear_mark(&devices, device->index, DEVICE_GID_UPDATES); + up_write(&devices_rwsem); +} + /* * ib_enum_all_devs - enumerate all ib_devices * @cb: Callback to call for each found ib_device diff --git a/drivers/infiniband/core/iwcm.c b/drivers/infiniband/core/iwcm.c index 62410578dec37d..eb942ab9c40559 100644 --- a/drivers/infiniband/core/iwcm.c +++ b/drivers/infiniband/core/iwcm.c @@ -95,7 +95,6 @@ static struct workqueue_struct *iwcm_wq; struct iwcm_work { struct work_struct work; struct iwcm_id_private *cm_id; - struct list_head list; struct iw_cm_event event; struct list_head free_list; }; @@ -178,7 +177,6 @@ static int alloc_work_entries(struct iwcm_id_private *cm_id_priv, int count) return -ENOMEM; } work->cm_id = cm_id_priv; - INIT_LIST_HEAD(&work->list); put_work(work); } return 0; @@ -213,7 +211,6 @@ static void free_cm_id(struct iwcm_id_private *cm_id_priv) static bool iwcm_deref_id(struct iwcm_id_private *cm_id_priv) { if (refcount_dec_and_test(&cm_id_priv->refcount)) { - BUG_ON(!list_empty(&cm_id_priv->work_list)); free_cm_id(cm_id_priv); return true; } @@ -260,7 +257,6 @@ struct iw_cm_id *iw_create_cm_id(struct ib_device *device, refcount_set(&cm_id_priv->refcount, 1); init_waitqueue_head(&cm_id_priv->connect_wait); init_completion(&cm_id_priv->destroy_comp); - INIT_LIST_HEAD(&cm_id_priv->work_list); INIT_LIST_HEAD(&cm_id_priv->work_free_list); return &cm_id_priv->id; @@ -1007,13 +1003,13 @@ static int process_event(struct iwcm_id_private *cm_id_priv, } /* - * Process events on the work_list for the cm_id. If the callback - * function requests that the cm_id be deleted, a flag is set in the - * cm_id flags to indicate that when the last reference is - * removed, the cm_id is to be destroyed. This is necessary to - * distinguish between an object that will be destroyed by the app - * thread asleep on the destroy_comp list vs. an object destroyed - * here synchronously when the last reference is removed. + * Process events for the cm_id. If the callback function requests + * that the cm_id be deleted, a flag is set in the cm_id flags to + * indicate that when the last reference is removed, the cm_id is + * to be destroyed. This is necessary to distinguish between an + * object that will be destroyed by the app thread asleep on the + * destroy_comp list vs. an object destroyed here synchronously + * when the last reference is removed. */ static void cm_work_handler(struct work_struct *_work) { @@ -1024,35 +1020,26 @@ static void cm_work_handler(struct work_struct *_work) int ret = 0; spin_lock_irqsave(&cm_id_priv->lock, flags); - while (!list_empty(&cm_id_priv->work_list)) { - work = list_first_entry(&cm_id_priv->work_list, - struct iwcm_work, list); - list_del_init(&work->list); - levent = work->event; - put_work(work); - spin_unlock_irqrestore(&cm_id_priv->lock, flags); - - if (!test_bit(IWCM_F_DROP_EVENTS, &cm_id_priv->flags)) { - ret = process_event(cm_id_priv, &levent); - if (ret) { - destroy_cm_id(&cm_id_priv->id); - WARN_ON_ONCE(iwcm_deref_id(cm_id_priv)); - } - } else - pr_debug("dropping event %d\n", levent.event); - if (iwcm_deref_id(cm_id_priv)) - return; - spin_lock_irqsave(&cm_id_priv->lock, flags); - } + levent = work->event; + put_work(work); spin_unlock_irqrestore(&cm_id_priv->lock, flags); + + if (!test_bit(IWCM_F_DROP_EVENTS, &cm_id_priv->flags)) { + ret = process_event(cm_id_priv, &levent); + if (ret) { + destroy_cm_id(&cm_id_priv->id); + WARN_ON_ONCE(iwcm_deref_id(cm_id_priv)); + } + } else + pr_debug("dropping event %d\n", levent.event); + if (iwcm_deref_id(cm_id_priv)) + return; } /* * This function is called on interrupt context. Schedule events on * the iwcm_wq thread to allow callback functions to downcall into - * the CM and/or block. Events are queued to a per-CM_ID - * work_list. If this is the first event on the work_list, the work - * element is also queued on the iwcm_wq thread. + * the CM and/or block. * * Each event holds a reference on the cm_id. Until the last posted * event has been delivered and processed, the cm_id cannot be @@ -1094,7 +1081,6 @@ static int cm_event_handler(struct iw_cm_id *cm_id, } refcount_inc(&cm_id_priv->refcount); - list_add_tail(&work->list, &cm_id_priv->work_list); queue_work(iwcm_wq, &work->work); out: spin_unlock_irqrestore(&cm_id_priv->lock, flags); diff --git a/drivers/infiniband/core/iwcm.h b/drivers/infiniband/core/iwcm.h index bf74639be1287c..b56fb12edece40 100644 --- a/drivers/infiniband/core/iwcm.h +++ b/drivers/infiniband/core/iwcm.h @@ -50,7 +50,6 @@ struct iwcm_id_private { struct ib_qp *qp; struct completion destroy_comp; wait_queue_head_t connect_wait; - struct list_head work_list; spinlock_t lock; refcount_t refcount; struct list_head work_free_list; diff --git a/drivers/infiniband/core/rw.c b/drivers/infiniband/core/rw.c index 6354ddf2a274ce..49fbfe1cef689c 100644 --- a/drivers/infiniband/core/rw.c +++ b/drivers/infiniband/core/rw.c @@ -326,14 +326,29 @@ int rdma_rw_ctx_init(struct rdma_rw_ctx *ctx, struct ib_qp *qp, u32 port_num, if (rdma_rw_io_needs_mr(qp->device, port_num, dir, sg_cnt)) { ret = rdma_rw_init_mr_wrs(ctx, qp, port_num, sg, sg_cnt, sg_offset, remote_addr, rkey, dir); - } else if (sg_cnt > 1) { + /* + * If MR init succeeded or failed for a reason other + * than pool exhaustion, that result is final. + * + * Pool exhaustion (-EAGAIN) from the max_sgl_rd + * optimization is recoverable: fall back to + * direct SGE posting. iWARP and force_mr require + * MRs unconditionally, so -EAGAIN is terminal. + */ + if (ret != -EAGAIN || + rdma_protocol_iwarp(qp->device, port_num) || + unlikely(rdma_rw_force_mr)) + goto out; + } + + if (sg_cnt > 1) ret = rdma_rw_init_map_wrs(ctx, qp, sg, sg_cnt, sg_offset, remote_addr, rkey, dir); - } else { + else ret = rdma_rw_init_single_wr(ctx, qp, sg, sg_offset, remote_addr, rkey, dir); - } +out: if (ret < 0) goto out_unmap_sg; return ret; @@ -651,34 +666,57 @@ unsigned int rdma_rw_mr_factor(struct ib_device *device, u32 port_num, } EXPORT_SYMBOL(rdma_rw_mr_factor); +/** + * rdma_rw_max_send_wr - compute max Send WRs needed for RDMA R/W contexts + * @dev: RDMA device + * @port_num: port number + * @max_rdma_ctxs: number of rdma_rw_ctx structures + * @create_flags: QP create flags (pass IB_QP_CREATE_INTEGRITY_EN if + * data integrity will be enabled on the QP) + * + * Returns the total number of Send Queue entries needed for + * @max_rdma_ctxs. The result accounts for memory registration and + * invalidation work requests when the device requires them. + * + * ULPs use this to size Send Queues and Send CQs before creating a + * Queue Pair. + */ +unsigned int rdma_rw_max_send_wr(struct ib_device *dev, u32 port_num, + unsigned int max_rdma_ctxs, u32 create_flags) +{ + unsigned int factor = 1; + unsigned int result; + + if (create_flags & IB_QP_CREATE_INTEGRITY_EN || + rdma_rw_can_use_mr(dev, port_num)) + factor += 2; /* reg + inv */ + + if (check_mul_overflow(factor, max_rdma_ctxs, &result)) + return UINT_MAX; + return result; +} +EXPORT_SYMBOL(rdma_rw_max_send_wr); + void rdma_rw_init_qp(struct ib_device *dev, struct ib_qp_init_attr *attr) { - u32 factor; + unsigned int factor = 1; WARN_ON_ONCE(attr->port_num == 0); /* - * Each context needs at least one RDMA READ or WRITE WR. - * - * For some hardware we might need more, eventually we should ask the - * HCA driver for a multiplier here. - */ - factor = 1; - - /* - * If the device needs MRs to perform RDMA READ or WRITE operations, - * we'll need two additional MRs for the registrations and the - * invalidation. + * If the device uses MRs to perform RDMA READ or WRITE operations, + * or if data integrity is enabled, account for registration and + * invalidation work requests. */ if (attr->create_flags & IB_QP_CREATE_INTEGRITY_EN || rdma_rw_can_use_mr(dev, attr->port_num)) - factor += 2; /* inv + reg */ + factor += 2; /* reg + inv */ attr->cap.max_send_wr += factor * attr->cap.max_rdma_ctxs; /* - * But maybe we were just too high in the sky and the device doesn't - * even support all we need, and we'll have to live with what we get.. + * The device might not support all we need, and we'll have to + * live with what we get. */ attr->cap.max_send_wr = min_t(u32, attr->cap.max_send_wr, dev->attrs.max_qp_wr); diff --git a/drivers/infiniband/core/umem_dmabuf.c b/drivers/infiniband/core/umem_dmabuf.c index 0ec2e4120cc94b..17b16fe0e49d94 100644 --- a/drivers/infiniband/core/umem_dmabuf.c +++ b/drivers/infiniband/core/umem_dmabuf.c @@ -221,13 +221,11 @@ ib_umem_dmabuf_get_pinned_with_dma_device(struct ib_device *device, err = ib_umem_dmabuf_map_pages(umem_dmabuf); if (err) - goto err_unpin; + goto err_release; dma_resv_unlock(umem_dmabuf->attach->dmabuf->resv); return umem_dmabuf; -err_unpin: - dma_buf_unpin(umem_dmabuf->attach); err_release: dma_resv_unlock(umem_dmabuf->attach->dmabuf->resv); ib_umem_release(&umem_dmabuf->umem); diff --git a/drivers/infiniband/core/user_mad.c b/drivers/infiniband/core/user_mad.c index fd67fc9fe85a46..2f7e3c4483fc57 100644 --- a/drivers/infiniband/core/user_mad.c +++ b/drivers/infiniband/core/user_mad.c @@ -514,7 +514,8 @@ static ssize_t ib_umad_write(struct file *filp, const char __user *buf, struct rdma_ah_attr ah_attr; struct ib_ah *ah; __be64 *tid; - int ret, data_len, hdr_len, copy_offset, rmpp_active; + int ret, hdr_len, copy_offset, rmpp_active; + size_t data_len; u8 base_version; if (count < hdr_size(file) + IB_MGMT_RMPP_HDR) @@ -588,7 +589,10 @@ static ssize_t ib_umad_write(struct file *filp, const char __user *buf, } base_version = ((struct ib_mad_hdr *)&packet->mad.data)->base_version; - data_len = count - hdr_size(file) - hdr_len; + if (check_sub_overflow(count, hdr_size(file) + hdr_len, &data_len)) { + ret = -EINVAL; + goto err_ah; + } packet->msg = ib_create_send_mad(agent, be32_to_cpu(packet->mad.hdr.qpn), packet->mad.hdr.pkey_index, rmpp_active, diff --git a/drivers/infiniband/core/uverbs_cmd.c b/drivers/infiniband/core/uverbs_cmd.c index ce16404cdfb8cc..f4616deeca5453 100644 --- a/drivers/infiniband/core/uverbs_cmd.c +++ b/drivers/infiniband/core/uverbs_cmd.c @@ -2049,7 +2049,10 @@ static int ib_uverbs_post_send(struct uverbs_attr_bundle *attrs) if (ret) return ret; - user_wr = kmalloc(cmd.wqe_size, GFP_KERNEL); + if (cmd.wqe_size < sizeof(struct ib_uverbs_send_wr)) + return -EINVAL; + + user_wr = kmalloc(cmd.wqe_size, GFP_KERNEL | __GFP_NOWARN); if (!user_wr) return -ENOMEM; @@ -2239,7 +2242,7 @@ ib_uverbs_unmarshall_recv(struct uverbs_req_iter *iter, u32 wr_count, if (ret) return ERR_PTR(ret); - user_wr = kmalloc(wqe_size, GFP_KERNEL); + user_wr = kmalloc(wqe_size, GFP_KERNEL | __GFP_NOWARN); if (!user_wr) return ERR_PTR(-ENOMEM); diff --git a/drivers/infiniband/hw/bng_re/bng_dev.c b/drivers/infiniband/hw/bng_re/bng_dev.c index d8f8d7f7075f03..9cf73f87070ec9 100644 --- a/drivers/infiniband/hw/bng_re/bng_dev.c +++ b/drivers/infiniband/hw/bng_re/bng_dev.c @@ -54,9 +54,6 @@ static void bng_re_destroy_chip_ctx(struct bng_re_dev *rdev) { struct bng_re_chip_ctx *chip_ctx; - if (!rdev->chip_ctx) - return; - kfree(rdev->dev_attr); rdev->dev_attr = NULL; @@ -124,12 +121,6 @@ static int bng_re_net_ring_free(struct bng_re_dev *rdev, struct bnge_fw_msg fw_msg = {}; int rc = -EINVAL; - if (!rdev) - return rc; - - if (!aux_dev) - return rc; - bng_re_init_hwrm_hdr((void *)&req, HWRM_RING_FREE); req.ring_type = type; req.ring_id = cpu_to_le16(fw_ring_id); @@ -150,10 +141,7 @@ static int bng_re_net_ring_alloc(struct bng_re_dev *rdev, struct hwrm_ring_alloc_input req = {}; struct hwrm_ring_alloc_output resp; struct bnge_fw_msg fw_msg = {}; - int rc = -EINVAL; - - if (!aux_dev) - return rc; + int rc; bng_re_init_hwrm_hdr((void *)&req, HWRM_RING_ALLOC); req.enables = 0; @@ -184,10 +172,7 @@ static int bng_re_stats_ctx_free(struct bng_re_dev *rdev) struct hwrm_stat_ctx_free_input req = {}; struct hwrm_stat_ctx_free_output resp = {}; struct bnge_fw_msg fw_msg = {}; - int rc = -EINVAL; - - if (!aux_dev) - return rc; + int rc; bng_re_init_hwrm_hdr((void *)&req, HWRM_STAT_CTX_FREE); req.stat_ctx_id = cpu_to_le32(rdev->stats_ctx.fw_id); @@ -208,13 +193,10 @@ static int bng_re_stats_ctx_alloc(struct bng_re_dev *rdev) struct hwrm_stat_ctx_alloc_output resp = {}; struct hwrm_stat_ctx_alloc_input req = {}; struct bnge_fw_msg fw_msg = {}; - int rc = -EINVAL; + int rc; stats->fw_id = BNGE_INVALID_STATS_CTX_ID; - if (!aux_dev) - return rc; - bng_re_init_hwrm_hdr((void *)&req, HWRM_STAT_CTX_ALLOC); req.update_period_ms = cpu_to_le32(1000); req.stats_dma_addr = cpu_to_le64(stats->dma_map); @@ -228,7 +210,7 @@ static int bng_re_stats_ctx_alloc(struct bng_re_dev *rdev) return rc; } -static void bng_re_query_hwrm_version(struct bng_re_dev *rdev) +static int bng_re_query_hwrm_version(struct bng_re_dev *rdev) { struct bnge_auxr_dev *aux_dev = rdev->aux_dev; struct hwrm_ver_get_output ver_get_resp = {}; @@ -248,7 +230,7 @@ static void bng_re_query_hwrm_version(struct bng_re_dev *rdev) if (rc) { ibdev_err(&rdev->ibdev, "Failed to query HW version, rc = 0x%x", rc); - return; + return rc; } cctx = rdev->chip_ctx; @@ -262,6 +244,8 @@ static void bng_re_query_hwrm_version(struct bng_re_dev *rdev) if (!cctx->hwrm_cmd_max_timeout) cctx->hwrm_cmd_max_timeout = BNG_ROCE_FW_MAX_TIMEOUT; + + return 0; } static void bng_re_dev_uninit(struct bng_re_dev *rdev) @@ -303,7 +287,7 @@ static int bng_re_dev_init(struct bng_re_dev *rdev) if (rc) { ibdev_err(&rdev->ibdev, "Failed to register with netedev: %#x\n", rc); - return -EINVAL; + goto reg_netdev_fail; } set_bit(BNG_RE_FLAG_NETDEV_REGISTERED, &rdev->flags); @@ -312,37 +296,34 @@ static int bng_re_dev_init(struct bng_re_dev *rdev) ibdev_err(&rdev->ibdev, "RoCE requires minimum 2 MSI-X vectors, but only %d reserved\n", rdev->aux_dev->auxr_info->msix_requested); - bnge_unregister_dev(rdev->aux_dev); - clear_bit(BNG_RE_FLAG_NETDEV_REGISTERED, &rdev->flags); - return -EINVAL; + rc = -EINVAL; + goto msix_ctx_fail; } ibdev_dbg(&rdev->ibdev, "Got %d MSI-X vectors\n", rdev->aux_dev->auxr_info->msix_requested); rc = bng_re_setup_chip_ctx(rdev); if (rc) { - bnge_unregister_dev(rdev->aux_dev); - clear_bit(BNG_RE_FLAG_NETDEV_REGISTERED, &rdev->flags); ibdev_err(&rdev->ibdev, "Failed to get chip context\n"); - return -EINVAL; + goto msix_ctx_fail; } - bng_re_query_hwrm_version(rdev); + rc = bng_re_query_hwrm_version(rdev); + if (rc) + goto destroy_chip_ctx; rc = bng_re_alloc_fw_channel(&rdev->bng_res, &rdev->rcfw); if (rc) { ibdev_err(&rdev->ibdev, "Failed to allocate RCFW Channel: %#x\n", rc); - goto fail; + goto destroy_chip_ctx; } /* Allocate nq record memory */ rdev->nqr = kzalloc(sizeof(*rdev->nqr), GFP_KERNEL); if (!rdev->nqr) { - bng_re_destroy_chip_ctx(rdev); - bnge_unregister_dev(rdev->aux_dev); - clear_bit(BNG_RE_FLAG_NETDEV_REGISTERED, &rdev->flags); - return -ENOMEM; + rc = -ENOMEM; + goto nq_alloc_fail; } rdev->nqr->num_msix = rdev->aux_dev->auxr_info->msix_requested; @@ -411,9 +392,15 @@ static int bng_re_dev_init(struct bng_re_dev *rdev) free_ring: bng_re_net_ring_free(rdev, rdev->rcfw.creq.ring_id, type); free_rcfw: + kfree(rdev->nqr); +nq_alloc_fail: bng_re_free_rcfw_channel(&rdev->rcfw); -fail: - bng_re_dev_uninit(rdev); +destroy_chip_ctx: + bng_re_destroy_chip_ctx(rdev); +msix_ctx_fail: + bnge_unregister_dev(rdev->aux_dev); + clear_bit(BNG_RE_FLAG_NETDEV_REGISTERED, &rdev->flags); +reg_netdev_fail: return rc; } @@ -486,8 +473,7 @@ static void bng_re_remove(struct auxiliary_device *adev) rdev = dev_info->rdev; - if (rdev) - bng_re_remove_device(rdev, adev); + bng_re_remove_device(rdev, adev); kfree(dev_info); } diff --git a/drivers/infiniband/hw/efa/efa_com.c b/drivers/infiniband/hw/efa/efa_com.c index 0e979ca10d240d..e97b5f0d700388 100644 --- a/drivers/infiniband/hw/efa/efa_com.c +++ b/drivers/infiniband/hw/efa/efa_com.c @@ -1,8 +1,10 @@ // SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause /* - * Copyright 2018-2025 Amazon.com, Inc. or its affiliates. All rights reserved. + * Copyright 2018-2026 Amazon.com, Inc. or its affiliates. All rights reserved. */ +#include + #include "efa_com.h" #include "efa_regs_defs.h" @@ -21,6 +23,8 @@ #define EFA_CTRL_SUB_MINOR 1 enum efa_cmd_status { + EFA_CMD_UNUSED, + EFA_CMD_ALLOCATED, EFA_CMD_SUBMITTED, EFA_CMD_COMPLETED, }; @@ -32,7 +36,6 @@ struct efa_comp_ctx { enum efa_cmd_status status; u16 cmd_id; u8 cmd_opcode; - u8 occupied; }; static const char *efa_com_cmd_str(u8 cmd) @@ -241,7 +244,6 @@ static int efa_com_admin_init_aenq(struct efa_com_dev *edev, return 0; } -/* ID to be used with efa_com_get_comp_ctx */ static u16 efa_com_alloc_ctx_id(struct efa_com_admin_queue *aq) { u16 ctx_id; @@ -263,48 +265,59 @@ static void efa_com_dealloc_ctx_id(struct efa_com_admin_queue *aq, spin_unlock(&aq->comp_ctx_lock); } -static inline void efa_com_put_comp_ctx(struct efa_com_admin_queue *aq, - struct efa_comp_ctx *comp_ctx) +static struct efa_comp_ctx *efa_com_alloc_comp_ctx(struct efa_com_admin_queue *aq) { - u16 cmd_id = EFA_GET(&comp_ctx->user_cqe->acq_common_descriptor.command, - EFA_ADMIN_ACQ_COMMON_DESC_COMMAND_ID); - u16 ctx_id = cmd_id & (aq->depth - 1); + struct efa_comp_ctx *comp_ctx; + u16 ctx_id; - ibdev_dbg(aq->efa_dev, "Put completion command_id %#x\n", cmd_id); - comp_ctx->occupied = 0; - efa_com_dealloc_ctx_id(aq, ctx_id); + ctx_id = efa_com_alloc_ctx_id(aq); + + comp_ctx = &aq->comp_ctx[ctx_id]; + if (comp_ctx->status != EFA_CMD_UNUSED) { + efa_com_dealloc_ctx_id(aq, ctx_id); + ibdev_err_ratelimited(aq->efa_dev, + "Completion context[%u] is used[%u]\n", + ctx_id, comp_ctx->status); + return NULL; + } + + comp_ctx->status = EFA_CMD_ALLOCATED; + ibdev_dbg(aq->efa_dev, "Take completion context[%u]\n", ctx_id); + return comp_ctx; } -static struct efa_comp_ctx *efa_com_get_comp_ctx(struct efa_com_admin_queue *aq, - u16 cmd_id, bool capture) +static inline u16 efa_com_get_comp_ctx_id(struct efa_com_admin_queue *aq, + struct efa_comp_ctx *comp_ctx) { - u16 ctx_id = cmd_id & (aq->depth - 1); + return comp_ctx - aq->comp_ctx; +} - if (aq->comp_ctx[ctx_id].occupied && capture) { - ibdev_err_ratelimited( - aq->efa_dev, - "Completion context for command_id %#x is occupied\n", - cmd_id); - return NULL; - } +static inline void efa_com_dealloc_comp_ctx(struct efa_com_admin_queue *aq, + struct efa_comp_ctx *comp_ctx) +{ + u16 ctx_id = efa_com_get_comp_ctx_id(aq, comp_ctx); - if (capture) { - aq->comp_ctx[ctx_id].occupied = 1; - ibdev_dbg(aq->efa_dev, - "Take completion ctxt for command_id %#x\n", cmd_id); - } + ibdev_dbg(aq->efa_dev, "Put completion context[%u]\n", ctx_id); + comp_ctx->status = EFA_CMD_UNUSED; + efa_com_dealloc_ctx_id(aq, ctx_id); +} + +static inline struct efa_comp_ctx *efa_com_get_comp_ctx_by_cmd_id(struct efa_com_admin_queue *aq, + u16 cmd_id) +{ + u16 ctx_id = cmd_id & (aq->depth - 1); return &aq->comp_ctx[ctx_id]; } -static struct efa_comp_ctx *__efa_com_submit_admin_cmd(struct efa_com_admin_queue *aq, - struct efa_admin_aq_entry *cmd, - size_t cmd_size_in_bytes, - struct efa_admin_acq_entry *comp, - size_t comp_size_in_bytes) +static void __efa_com_submit_admin_cmd(struct efa_com_admin_queue *aq, + struct efa_comp_ctx *comp_ctx, + struct efa_admin_aq_entry *cmd, + size_t cmd_size_in_bytes, + struct efa_admin_acq_entry *comp, + size_t comp_size_in_bytes) { struct efa_admin_aq_entry *aqe; - struct efa_comp_ctx *comp_ctx; u16 queue_size_mask; u16 cmd_id; u16 ctx_id; @@ -312,24 +325,17 @@ static struct efa_comp_ctx *__efa_com_submit_admin_cmd(struct efa_com_admin_queu queue_size_mask = aq->depth - 1; pi = aq->sq.pc & queue_size_mask; - - ctx_id = efa_com_alloc_ctx_id(aq); + ctx_id = efa_com_get_comp_ctx_id(aq, comp_ctx); /* cmd_id LSBs are the ctx_id and MSBs are entropy bits from pc */ cmd_id = ctx_id & queue_size_mask; - cmd_id |= aq->sq.pc & ~queue_size_mask; + cmd_id |= aq->sq.pc << ilog2(aq->depth); cmd_id &= EFA_ADMIN_AQ_COMMON_DESC_COMMAND_ID_MASK; cmd->aq_common_descriptor.command_id = cmd_id; EFA_SET(&cmd->aq_common_descriptor.flags, EFA_ADMIN_AQ_COMMON_DESC_PHASE, aq->sq.phase); - comp_ctx = efa_com_get_comp_ctx(aq, cmd_id, true); - if (!comp_ctx) { - efa_com_dealloc_ctx_id(aq, ctx_id); - return ERR_PTR(-EINVAL); - } - comp_ctx->status = EFA_CMD_SUBMITTED; comp_ctx->comp_size = comp_size_in_bytes; comp_ctx->user_cqe = comp; @@ -350,8 +356,6 @@ static struct efa_comp_ctx *__efa_com_submit_admin_cmd(struct efa_com_admin_queu /* barrier not needed in case of writel */ writel(aq->sq.pc, aq->sq.db_addr); - - return comp_ctx; } static inline int efa_com_init_comp_ctxt(struct efa_com_admin_queue *aq) @@ -370,9 +374,9 @@ static inline int efa_com_init_comp_ctxt(struct efa_com_admin_queue *aq) } for (i = 0; i < aq->depth; i++) { - comp_ctx = efa_com_get_comp_ctx(aq, i, false); - if (comp_ctx) - init_completion(&comp_ctx->wait_event); + comp_ctx = &aq->comp_ctx[i]; + comp_ctx->status = EFA_CMD_UNUSED; + init_completion(&comp_ctx->wait_event); aq->comp_ctx_pool[i] = i; } @@ -384,28 +388,25 @@ static inline int efa_com_init_comp_ctxt(struct efa_com_admin_queue *aq) return 0; } -static struct efa_comp_ctx *efa_com_submit_admin_cmd(struct efa_com_admin_queue *aq, - struct efa_admin_aq_entry *cmd, - size_t cmd_size_in_bytes, - struct efa_admin_acq_entry *comp, - size_t comp_size_in_bytes) +static int efa_com_submit_admin_cmd(struct efa_com_admin_queue *aq, + struct efa_comp_ctx *comp_ctx, + struct efa_admin_aq_entry *cmd, + size_t cmd_size_in_bytes, + struct efa_admin_acq_entry *comp, + size_t comp_size_in_bytes) { - struct efa_comp_ctx *comp_ctx; - spin_lock(&aq->sq.lock); if (!test_bit(EFA_AQ_STATE_RUNNING_BIT, &aq->state)) { ibdev_err_ratelimited(aq->efa_dev, "Admin queue is closed\n"); spin_unlock(&aq->sq.lock); - return ERR_PTR(-ENODEV); + return -ENODEV; } - comp_ctx = __efa_com_submit_admin_cmd(aq, cmd, cmd_size_in_bytes, comp, - comp_size_in_bytes); + __efa_com_submit_admin_cmd(aq, comp_ctx, cmd, cmd_size_in_bytes, comp, + comp_size_in_bytes); spin_unlock(&aq->sq.lock); - if (IS_ERR(comp_ctx)) - clear_bit(EFA_AQ_STATE_RUNNING_BIT, &aq->state); - return comp_ctx; + return 0; } static int efa_com_handle_single_admin_completion(struct efa_com_admin_queue *aq, @@ -417,11 +418,12 @@ static int efa_com_handle_single_admin_completion(struct efa_com_admin_queue *aq cmd_id = EFA_GET(&cqe->acq_common_descriptor.command, EFA_ADMIN_ACQ_COMMON_DESC_COMMAND_ID); - comp_ctx = efa_com_get_comp_ctx(aq, cmd_id, false); - if (comp_ctx->status != EFA_CMD_SUBMITTED) { + comp_ctx = efa_com_get_comp_ctx_by_cmd_id(aq, cmd_id); + if (comp_ctx->status != EFA_CMD_SUBMITTED || comp_ctx->cmd_id != cmd_id) { ibdev_err(aq->efa_dev, - "Received completion with unexpected command id[%d], sq producer: %d, sq consumer: %d, cq consumer: %d\n", - cmd_id, aq->sq.pc, aq->sq.cc, aq->cq.cc); + "Received completion with unexpected command id[%x], status[%d] sq producer[%d], sq consumer[%d], cq consumer[%d]\n", + cmd_id, comp_ctx->status, aq->sq.pc, aq->sq.cc, + aq->cq.cc); return -EINVAL; } @@ -501,7 +503,6 @@ static int efa_com_wait_and_process_admin_cq_polling(struct efa_comp_ctx *comp_c { unsigned long timeout; unsigned long flags; - int err; timeout = jiffies + usecs_to_jiffies(aq->completion_timeout); @@ -521,24 +522,20 @@ static int efa_com_wait_and_process_admin_cq_polling(struct efa_comp_ctx *comp_c atomic64_inc(&aq->stats.no_completion); clear_bit(EFA_AQ_STATE_RUNNING_BIT, &aq->state); - err = -ETIME; - goto out; + return -ETIME; } msleep(aq->poll_interval); } - err = efa_com_comp_status_to_errno(comp_ctx->user_cqe->acq_common_descriptor.status); -out: - efa_com_put_comp_ctx(aq, comp_ctx); - return err; + return efa_com_comp_status_to_errno( + comp_ctx->user_cqe->acq_common_descriptor.status); } static int efa_com_wait_and_process_admin_cq_interrupts(struct efa_comp_ctx *comp_ctx, struct efa_com_admin_queue *aq) { unsigned long flags; - int err; wait_for_completion_timeout(&comp_ctx->wait_event, usecs_to_jiffies(aq->completion_timeout)); @@ -574,14 +571,11 @@ static int efa_com_wait_and_process_admin_cq_interrupts(struct efa_comp_ctx *com aq->cq.cc); clear_bit(EFA_AQ_STATE_RUNNING_BIT, &aq->state); - err = -ETIME; - goto out; + return -ETIME; } - err = efa_com_comp_status_to_errno(comp_ctx->user_cqe->acq_common_descriptor.status); -out: - efa_com_put_comp_ctx(aq, comp_ctx); - return err; + return efa_com_comp_status_to_errno( + comp_ctx->user_cqe->acq_common_descriptor.status); } /* @@ -631,30 +625,39 @@ int efa_com_cmd_exec(struct efa_com_admin_queue *aq, ibdev_dbg(aq->efa_dev, "%s (opcode %d)\n", efa_com_cmd_str(cmd->aq_common_descriptor.opcode), cmd->aq_common_descriptor.opcode); - comp_ctx = efa_com_submit_admin_cmd(aq, cmd, cmd_size, comp, comp_size); - if (IS_ERR(comp_ctx)) { + + comp_ctx = efa_com_alloc_comp_ctx(aq); + if (!comp_ctx) { + clear_bit(EFA_AQ_STATE_RUNNING_BIT, &aq->state); + up(&aq->avail_cmds); + return -EINVAL; + } + + err = efa_com_submit_admin_cmd(aq, comp_ctx, cmd, cmd_size, comp, comp_size); + if (err) { ibdev_err_ratelimited( aq->efa_dev, - "Failed to submit command %s (opcode %u) err %pe\n", + "Failed to submit command %s (opcode %u) err %d\n", efa_com_cmd_str(cmd->aq_common_descriptor.opcode), - cmd->aq_common_descriptor.opcode, comp_ctx); + cmd->aq_common_descriptor.opcode, err); + efa_com_dealloc_comp_ctx(aq, comp_ctx); up(&aq->avail_cmds); atomic64_inc(&aq->stats.cmd_err); - return PTR_ERR(comp_ctx); + return err; } err = efa_com_wait_and_process_admin_cq(comp_ctx, aq); if (err) { ibdev_err_ratelimited( aq->efa_dev, - "Failed to process command %s (opcode %u) comp_status %d err %d\n", + "Failed to process command %s (opcode %u) err %d\n", efa_com_cmd_str(cmd->aq_common_descriptor.opcode), - cmd->aq_common_descriptor.opcode, - comp_ctx->user_cqe->acq_common_descriptor.status, err); + cmd->aq_common_descriptor.opcode, err); atomic64_inc(&aq->stats.cmd_err); } + efa_com_dealloc_comp_ctx(aq, comp_ctx); up(&aq->avail_cmds); return err; diff --git a/drivers/infiniband/hw/efa/efa_verbs.c b/drivers/infiniband/hw/efa/efa_verbs.c index 755bba8d58bbc9..5cab7dd70aebf6 100644 --- a/drivers/infiniband/hw/efa/efa_verbs.c +++ b/drivers/infiniband/hw/efa/efa_verbs.c @@ -1663,7 +1663,7 @@ static struct efa_mr *efa_alloc_mr(struct ib_pd *ibpd, int access_flags, struct efa_mr *mr; if (udata && udata->inlen && - !ib_is_udata_cleared(udata, 0, sizeof(udata->inlen))) { + !ib_is_udata_cleared(udata, 0, udata->inlen)) { ibdev_dbg(&dev->ibdev, "Incompatible ABI params, udata not cleared\n"); return ERR_PTR(-EINVAL); diff --git a/drivers/infiniband/hw/hns/hns_roce_ah.c b/drivers/infiniband/hw/hns/hns_roce_ah.c index 0c1c32d23c8848..8a605da8a93c97 100644 --- a/drivers/infiniband/hw/hns/hns_roce_ah.c +++ b/drivers/infiniband/hw/hns/hns_roce_ah.c @@ -60,7 +60,7 @@ int hns_roce_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr, u8 tclass = get_tclass(grh); u8 priority = 0; u8 tc_mode = 0; - int ret; + int ret = 0; if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08 && udata) { ret = -EOPNOTSUPP; @@ -77,19 +77,18 @@ int hns_roce_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr, ah->av.flowlabel = grh->flow_label; ah->av.udp_sport = get_ah_udp_sport(ah_attr); ah->av.tclass = tclass; + ah->av.sl = rdma_ah_get_sl(ah_attr); - ret = hr_dev->hw->get_dscp(hr_dev, tclass, &tc_mode, &priority); - if (ret == -EOPNOTSUPP) - ret = 0; - - if (ret && grh->sgid_attr->gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) - goto err_out; + if (grh->sgid_attr->gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) { + ret = hr_dev->hw->get_dscp(hr_dev, tclass, &tc_mode, &priority); + if (ret == -EOPNOTSUPP) + ret = 0; + else if (ret) + goto err_out; - if (tc_mode == HNAE3_TC_MAP_MODE_DSCP && - grh->sgid_attr->gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) - ah->av.sl = priority; - else - ah->av.sl = rdma_ah_get_sl(ah_attr); + if (tc_mode == HNAE3_TC_MAP_MODE_DSCP) + ah->av.sl = priority; + } if (!check_sl_valid(hr_dev, ah->av.sl)) { ret = -EINVAL; diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c index 2d6ae89e525b8a..a2ae4f33e459f3 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c @@ -3739,6 +3739,23 @@ static void hns_roce_v2_write_cqc(struct hns_roce_dev *hr_dev, HNS_ROCE_V2_CQ_DEFAULT_INTERVAL); } +static bool left_sw_wc(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq) +{ + struct hns_roce_qp *hr_qp; + + list_for_each_entry(hr_qp, &hr_cq->sq_list, sq_node) { + if (hr_qp->sq.head != hr_qp->sq.tail) + return true; + } + + list_for_each_entry(hr_qp, &hr_cq->rq_list, rq_node) { + if (hr_qp->rq.head != hr_qp->rq.tail) + return true; + } + + return false; +} + static int hns_roce_v2_req_notify_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags) { @@ -3747,6 +3764,12 @@ static int hns_roce_v2_req_notify_cq(struct ib_cq *ibcq, struct hns_roce_v2_db cq_db = {}; u32 notify_flag; + if (hr_dev->state >= HNS_ROCE_DEVICE_STATE_RST_DOWN) { + if ((flags & IB_CQ_REPORT_MISSED_EVENTS) && + left_sw_wc(hr_dev, hr_cq)) + return 1; + return 0; + } /* * flags = 0, then notify_flag : next * flags = 1, then notify flag : solocited @@ -5053,20 +5076,22 @@ static int hns_roce_set_sl(struct ib_qp *ibqp, struct ib_device *ibdev = &hr_dev->ib_dev; int ret; - ret = hns_roce_hw_v2_get_dscp(hr_dev, get_tclass(&attr->ah_attr.grh), - &hr_qp->tc_mode, &hr_qp->priority); - if (ret && ret != -EOPNOTSUPP && - grh->sgid_attr->gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) { - ibdev_err_ratelimited(ibdev, - "failed to get dscp, ret = %d.\n", ret); - return ret; - } + hr_qp->sl = rdma_ah_get_sl(&attr->ah_attr); - if (hr_qp->tc_mode == HNAE3_TC_MAP_MODE_DSCP && - grh->sgid_attr->gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) - hr_qp->sl = hr_qp->priority; - else - hr_qp->sl = rdma_ah_get_sl(&attr->ah_attr); + if (grh->sgid_attr->gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) { + ret = hns_roce_hw_v2_get_dscp(hr_dev, + get_tclass(&attr->ah_attr.grh), + &hr_qp->tc_mode, &hr_qp->priority); + if (ret && ret != -EOPNOTSUPP) { + ibdev_err_ratelimited(ibdev, + "failed to get dscp, ret = %d.\n", + ret); + return ret; + } + + if (hr_qp->tc_mode == HNAE3_TC_MAP_MODE_DSCP) + hr_qp->sl = hr_qp->priority; + } if (!check_sl_valid(hr_dev, hr_qp->sl)) return -EINVAL; @@ -6956,7 +6981,8 @@ static int hns_roce_v2_init_eq_table(struct hns_roce_dev *hr_dev) INIT_WORK(&hr_dev->ecc_work, fmea_ram_ecc_work); - hr_dev->irq_workq = alloc_ordered_workqueue("hns_roce_irq_workq", 0); + hr_dev->irq_workq = alloc_ordered_workqueue("hns_roce_irq_workq", + WQ_MEM_RECLAIM); if (!hr_dev->irq_workq) { dev_err(dev, "failed to create irq workqueue.\n"); ret = -ENOMEM; diff --git a/drivers/infiniband/hw/hns/hns_roce_restrack.c b/drivers/infiniband/hw/hns/hns_roce_restrack.c index 230187dda6a07b..085791cc617c4f 100644 --- a/drivers/infiniband/hw/hns/hns_roce_restrack.c +++ b/drivers/infiniband/hw/hns/hns_roce_restrack.c @@ -51,7 +51,7 @@ int hns_roce_fill_res_cq_entry_raw(struct sk_buff *msg, struct ib_cq *ib_cq) ret = hr_dev->hw->query_cqc(hr_dev, hr_cq->cqn, &context); if (ret) - return -EINVAL; + return ret; ret = nla_put(msg, RDMA_NLDEV_ATTR_RES_RAW, sizeof(context), &context); @@ -177,7 +177,7 @@ int hns_roce_fill_res_mr_entry_raw(struct sk_buff *msg, struct ib_mr *ib_mr) ret = hr_dev->hw->query_mpt(hr_dev, hr_mr->key, &context); if (ret) - return -EINVAL; + return ret; ret = nla_put(msg, RDMA_NLDEV_ATTR_RES_RAW, sizeof(context), &context); diff --git a/drivers/infiniband/hw/ionic/ionic_controlpath.c b/drivers/infiniband/hw/ionic/ionic_controlpath.c index ea12d9b8e125fe..38d57bc2ba5259 100644 --- a/drivers/infiniband/hw/ionic/ionic_controlpath.c +++ b/drivers/infiniband/hw/ionic/ionic_controlpath.c @@ -508,6 +508,7 @@ static int ionic_build_hdr(struct ionic_ibdev *dev, { const struct ib_global_route *grh; enum rdma_network_type net; + u8 smac[ETH_ALEN]; u16 vlan; int rc; @@ -518,7 +519,7 @@ static int ionic_build_hdr(struct ionic_ibdev *dev, grh = rdma_ah_read_grh(attr); - rc = rdma_read_gid_l2_fields(grh->sgid_attr, &vlan, &hdr->eth.smac_h[0]); + rc = rdma_read_gid_l2_fields(grh->sgid_attr, &vlan, smac); if (rc) return rc; @@ -536,6 +537,7 @@ static int ionic_build_hdr(struct ionic_ibdev *dev, if (rc) return rc; + ether_addr_copy(hdr->eth.smac_h, smac); ether_addr_copy(hdr->eth.dmac_h, attr->roce.dmac); if (net == RDMA_NETWORK_IPV4) { @@ -1218,7 +1220,7 @@ int ionic_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, rdma_udata_to_drv_context(udata, struct ionic_ctx, ibctx); struct ionic_vcq *vcq = to_ionic_vcq(ibcq); struct ionic_tbl_buf buf = {}; - struct ionic_cq_resp resp; + struct ionic_cq_resp resp = {}; struct ionic_cq_req req; int udma_idx = 0, rc; diff --git a/drivers/infiniband/hw/ionic/ionic_ibdev.c b/drivers/infiniband/hw/ionic/ionic_ibdev.c index 164046d00e5d47..bd4c73e530d081 100644 --- a/drivers/infiniband/hw/ionic/ionic_ibdev.c +++ b/drivers/infiniband/hw/ionic/ionic_ibdev.c @@ -81,6 +81,8 @@ static int ionic_query_port(struct ib_device *ibdev, u32 port, return -EINVAL; ndev = ib_device_get_netdev(ibdev, port); + if (!ndev) + return -ENODEV; if (netif_running(ndev) && netif_carrier_ok(ndev)) { attr->state = IB_PORT_ACTIVE; diff --git a/drivers/infiniband/hw/irdma/cm.c b/drivers/infiniband/hw/irdma/cm.c index f4f4f92ba63aca..d14a381beb6614 100644 --- a/drivers/infiniband/hw/irdma/cm.c +++ b/drivers/infiniband/hw/irdma/cm.c @@ -2241,11 +2241,12 @@ irdma_make_cm_node(struct irdma_cm_core *cm_core, struct irdma_device *iwdev, int oldarpindex; int arpindex; struct net_device *netdev = iwdev->netdev; + int ret; /* create an hte and cm_node for this instance */ cm_node = kzalloc(sizeof(*cm_node), GFP_ATOMIC); if (!cm_node) - return NULL; + return ERR_PTR(-ENOMEM); /* set our node specific transport info */ cm_node->ipv4 = cm_info->ipv4; @@ -2348,8 +2349,10 @@ irdma_make_cm_node(struct irdma_cm_core *cm_core, struct irdma_device *iwdev, arpindex = -EINVAL; } - if (arpindex < 0) + if (arpindex < 0) { + ret = -EINVAL; goto err; + } ether_addr_copy(cm_node->rem_mac, iwdev->rf->arp_table[arpindex].mac_addr); @@ -2360,7 +2363,7 @@ irdma_make_cm_node(struct irdma_cm_core *cm_core, struct irdma_device *iwdev, err: kfree(cm_node); - return NULL; + return ERR_PTR(ret); } static void irdma_destroy_connection(struct irdma_cm_node *cm_node) @@ -3021,8 +3024,8 @@ static int irdma_create_cm_node(struct irdma_cm_core *cm_core, /* create a CM connection node */ cm_node = irdma_make_cm_node(cm_core, iwdev, cm_info, NULL); - if (!cm_node) - return -ENOMEM; + if (IS_ERR(cm_node)) + return PTR_ERR(cm_node); /* set our node side to client (active) side */ cm_node->tcp_cntxt.client = 1; @@ -3219,9 +3222,9 @@ void irdma_receive_ilq(struct irdma_sc_vsi *vsi, struct irdma_puda_buf *rbuf) cm_info.cm_id = listener->cm_id; cm_node = irdma_make_cm_node(cm_core, iwdev, &cm_info, listener); - if (!cm_node) { + if (IS_ERR(cm_node)) { ibdev_dbg(&cm_core->iwdev->ibdev, - "CM: allocate node failed\n"); + "CM: allocate node failed ret=%ld\n", PTR_ERR(cm_node)); refcount_dec(&listener->refcnt); return; } @@ -4239,21 +4242,21 @@ static void irdma_cm_event_handler(struct work_struct *work) irdma_cm_event_reset(event); break; case IRDMA_CM_EVENT_CONNECTED: - if (!event->cm_node->cm_id || - event->cm_node->state != IRDMA_CM_STATE_OFFLOADED) + if (!cm_node->cm_id || + cm_node->state != IRDMA_CM_STATE_OFFLOADED) break; irdma_cm_event_connected(event); break; case IRDMA_CM_EVENT_MPA_REJECT: - if (!event->cm_node->cm_id || + if (!cm_node->cm_id || cm_node->state == IRDMA_CM_STATE_OFFLOADED) break; irdma_send_cm_event(cm_node, cm_node->cm_id, IW_CM_EVENT_CONNECT_REPLY, -ECONNREFUSED); break; case IRDMA_CM_EVENT_ABORTED: - if (!event->cm_node->cm_id || - event->cm_node->state == IRDMA_CM_STATE_OFFLOADED) + if (!cm_node->cm_id || + cm_node->state == IRDMA_CM_STATE_OFFLOADED) break; irdma_event_connect_error(event); break; @@ -4263,7 +4266,7 @@ static void irdma_cm_event_handler(struct work_struct *work) break; } - irdma_rem_ref_cm_node(event->cm_node); + irdma_rem_ref_cm_node(cm_node); kfree(event); } diff --git a/drivers/infiniband/hw/irdma/uk.c b/drivers/infiniband/hw/irdma/uk.c index f0846b800913d0..3d6d0ee57c4c15 100644 --- a/drivers/infiniband/hw/irdma/uk.c +++ b/drivers/infiniband/hw/irdma/uk.c @@ -1442,7 +1442,7 @@ int irdma_uk_cq_poll_cmpl(struct irdma_cq_uk *cq, * irdma_round_up_wq - return round up qp wq depth * @wqdepth: wq depth in quanta to round up */ -static int irdma_round_up_wq(u32 wqdepth) +static u64 irdma_round_up_wq(u64 wqdepth) { int scount = 1; @@ -1495,15 +1495,16 @@ void irdma_get_wqe_shift(struct irdma_uk_attrs *uk_attrs, u32 sge, int irdma_get_sqdepth(struct irdma_uk_attrs *uk_attrs, u32 sq_size, u8 shift, u32 *sqdepth) { - u32 min_size = (u32)uk_attrs->min_hw_wq_size << shift; + u32 min_hw_quanta = (u32)uk_attrs->min_hw_wq_size << shift; + u64 hw_quanta = + irdma_round_up_wq(((u64)sq_size << shift) + IRDMA_SQ_RSVD); - *sqdepth = irdma_round_up_wq((sq_size << shift) + IRDMA_SQ_RSVD); - - if (*sqdepth < min_size) - *sqdepth = min_size; - else if (*sqdepth > uk_attrs->max_hw_wq_quanta) + if (hw_quanta < min_hw_quanta) + hw_quanta = min_hw_quanta; + else if (hw_quanta > uk_attrs->max_hw_wq_quanta) return -EINVAL; + *sqdepth = hw_quanta; return 0; } @@ -1517,15 +1518,16 @@ int irdma_get_sqdepth(struct irdma_uk_attrs *uk_attrs, u32 sq_size, u8 shift, int irdma_get_rqdepth(struct irdma_uk_attrs *uk_attrs, u32 rq_size, u8 shift, u32 *rqdepth) { - u32 min_size = (u32)uk_attrs->min_hw_wq_size << shift; - - *rqdepth = irdma_round_up_wq((rq_size << shift) + IRDMA_RQ_RSVD); + u32 min_hw_quanta = (u32)uk_attrs->min_hw_wq_size << shift; + u64 hw_quanta = + irdma_round_up_wq(((u64)rq_size << shift) + IRDMA_RQ_RSVD); - if (*rqdepth < min_size) - *rqdepth = min_size; - else if (*rqdepth > uk_attrs->max_hw_rq_quanta) + if (hw_quanta < min_hw_quanta) + hw_quanta = min_hw_quanta; + else if (hw_quanta > uk_attrs->max_hw_rq_quanta) return -EINVAL; + *rqdepth = hw_quanta; return 0; } @@ -1539,13 +1541,16 @@ int irdma_get_rqdepth(struct irdma_uk_attrs *uk_attrs, u32 rq_size, u8 shift, int irdma_get_srqdepth(struct irdma_uk_attrs *uk_attrs, u32 srq_size, u8 shift, u32 *srqdepth) { - *srqdepth = irdma_round_up_wq((srq_size << shift) + IRDMA_RQ_RSVD); + u32 min_hw_quanta = (u32)uk_attrs->min_hw_wq_size << shift; + u64 hw_quanta = + irdma_round_up_wq(((u64)srq_size << shift) + IRDMA_RQ_RSVD); - if (*srqdepth < ((u32)uk_attrs->min_hw_wq_size << shift)) - *srqdepth = uk_attrs->min_hw_wq_size << shift; - else if (*srqdepth > uk_attrs->max_hw_srq_quanta) + if (hw_quanta < min_hw_quanta) + hw_quanta = min_hw_quanta; + else if (hw_quanta > uk_attrs->max_hw_srq_quanta) return -EINVAL; + *srqdepth = hw_quanta; return 0; } diff --git a/drivers/infiniband/hw/irdma/utils.c b/drivers/infiniband/hw/irdma/utils.c index 13d7499131d484..89c4fe05763e40 100644 --- a/drivers/infiniband/hw/irdma/utils.c +++ b/drivers/infiniband/hw/irdma/utils.c @@ -2321,8 +2321,6 @@ void irdma_modify_qp_to_err(struct irdma_sc_qp *sc_qp) struct irdma_qp *qp = sc_qp->qp_uk.back_qp; struct ib_qp_attr attr; - if (qp->iwdev->rf->reset) - return; attr.qp_state = IB_QPS_ERR; if (rdma_protocol_roce(qp->ibqp.device, 1)) diff --git a/drivers/infiniband/hw/irdma/verbs.c b/drivers/infiniband/hw/irdma/verbs.c index 6d9af41a2884a4..496d3fedaa9e60 100644 --- a/drivers/infiniband/hw/irdma/verbs.c +++ b/drivers/infiniband/hw/irdma/verbs.c @@ -558,7 +558,8 @@ static int irdma_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) } irdma_qp_rem_ref(&iwqp->ibqp); - wait_for_completion(&iwqp->free_qp); + if (!iwdev->rf->reset) + wait_for_completion(&iwqp->free_qp); irdma_free_lsmm_rsrc(iwqp); irdma_cqp_qp_destroy_cmd(&iwdev->rf->sc_dev, &iwqp->sc_qp); @@ -1105,6 +1106,7 @@ static int irdma_create_qp(struct ib_qp *ibqp, spin_lock_init(&iwqp->sc_qp.pfpdu.lock); iwqp->sig_all = init_attr->sq_sig_type == IB_SIGNAL_ALL_WR; rf->qp_table[qp_num] = iwqp; + init_completion(&iwqp->free_qp); if (udata) { /* GEN_1 legacy support with libi40iw does not have expanded uresp struct */ @@ -1129,7 +1131,6 @@ static int irdma_create_qp(struct ib_qp *ibqp, } } - init_completion(&iwqp->free_qp); return 0; error: @@ -1462,8 +1463,6 @@ int irdma_modify_qp_roce(struct ib_qp *ibqp, struct ib_qp_attr *attr, ctx_info->remote_atomics_en = true; } - wait_event(iwqp->mod_qp_waitq, !atomic_read(&iwqp->hw_mod_qp_pend)); - ibdev_dbg(&iwdev->ibdev, "VERBS: caller: %pS qp_id=%d to_ibqpstate=%d ibqpstate=%d irdma_qpstate=%d attr_mask=0x%x\n", __builtin_return_address(0), ibqp->qp_num, attr->qp_state, @@ -1540,6 +1539,7 @@ int irdma_modify_qp_roce(struct ib_qp *ibqp, struct ib_qp_attr *attr, case IB_QPS_ERR: case IB_QPS_RESET: if (iwqp->iwarp_state == IRDMA_QP_STATE_ERROR) { + iwqp->ibqp_state = attr->qp_state; spin_unlock_irqrestore(&iwqp->lock, flags); if (udata && udata->inlen) { if (ib_copy_from_udata(&ureq, udata, @@ -1745,6 +1745,7 @@ int irdma_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask, case IB_QPS_ERR: case IB_QPS_RESET: if (iwqp->iwarp_state == IRDMA_QP_STATE_ERROR) { + iwqp->ibqp_state = attr->qp_state; spin_unlock_irqrestore(&iwqp->lock, flags); if (udata && udata->inlen) { if (ib_copy_from_udata(&ureq, udata, @@ -3720,6 +3721,7 @@ static int irdma_rereg_mr_trans(struct irdma_mr *iwmr, u64 start, u64 len, err: ib_umem_release(region); + iwmr->region = NULL; return err; } @@ -5209,7 +5211,7 @@ static int irdma_create_user_ah(struct ib_ah *ibah, #define IRDMA_CREATE_AH_MIN_RESP_LEN offsetofend(struct irdma_create_ah_resp, rsvd) struct irdma_ah *ah = container_of(ibah, struct irdma_ah, ibah); struct irdma_device *iwdev = to_iwdev(ibah->pd->device); - struct irdma_create_ah_resp uresp; + struct irdma_create_ah_resp uresp = {}; struct irdma_ah *parent_ah; int err; diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c index 40284bbb45d6dc..99974794100054 100644 --- a/drivers/infiniband/hw/mlx5/main.c +++ b/drivers/infiniband/hw/mlx5/main.c @@ -561,12 +561,20 @@ static int mlx5_query_port_roce(struct ib_device *device, u32 port_num, * of an error it will still be zeroed out. * Use native port in case of reps */ - if (dev->is_rep) - err = mlx5_query_port_ptys(mdev, out, sizeof(out), MLX5_PTYS_EN, - 1, 0); - else - err = mlx5_query_port_ptys(mdev, out, sizeof(out), MLX5_PTYS_EN, - mdev_port_num, 0); + if (dev->is_rep) { + struct mlx5_eswitch_rep *rep; + + rep = dev->port[port_num - 1].rep; + if (rep) { + mdev = mlx5_eswitch_get_core_dev(rep->esw); + WARN_ON(!mdev); + } + mdev_port_num = 1; + } + + err = mlx5_query_port_ptys(mdev, out, sizeof(out), MLX5_PTYS_EN, + mdev_port_num, 0); + if (err) goto out; ext = !!MLX5_GET_ETH_PROTO(ptys_reg, out, true, eth_proto_capability); @@ -2878,7 +2886,6 @@ static void mlx5_ib_handle_event(struct work_struct *_work) container_of(_work, struct mlx5_ib_event_work, work); struct mlx5_ib_dev *ibdev; struct ib_event ibev; - bool fatal = false; if (work->is_slave) { ibdev = mlx5_ib_get_ibdev_from_mpi(work->mpi); @@ -2889,12 +2896,6 @@ static void mlx5_ib_handle_event(struct work_struct *_work) } switch (work->event) { - case MLX5_DEV_EVENT_SYS_ERROR: - ibev.event = IB_EVENT_DEVICE_FATAL; - mlx5_ib_handle_internal_error(ibdev); - ibev.element.port_num = (u8)(unsigned long)work->param; - fatal = true; - break; case MLX5_EVENT_TYPE_PORT_CHANGE: if (handle_port_change(ibdev, work->param, &ibev)) goto out; @@ -2916,8 +2917,6 @@ static void mlx5_ib_handle_event(struct work_struct *_work) if (ibdev->ib_active) ib_dispatch_event(&ibev); - if (fatal) - ibdev->ib_active = false; out: kfree(work); } @@ -2961,6 +2960,66 @@ static int mlx5_ib_event_slave_port(struct notifier_block *nb, return NOTIFY_OK; } +static void mlx5_ib_handle_sys_error_event(struct work_struct *_work) +{ + struct mlx5_ib_event_work *work = + container_of(_work, struct mlx5_ib_event_work, work); + struct mlx5_ib_dev *ibdev = work->dev; + struct ib_event ibev; + + ibev.event = IB_EVENT_DEVICE_FATAL; + mlx5_ib_handle_internal_error(ibdev); + ibev.element.port_num = (u8)(unsigned long)work->param; + ibev.device = &ibdev->ib_dev; + + if (!rdma_is_port_valid(&ibdev->ib_dev, ibev.element.port_num)) { + mlx5_ib_warn(ibdev, "warning: event on port %d\n", ibev.element.port_num); + goto out; + } + + if (ibdev->ib_active) + ib_dispatch_event(&ibev); + + ibdev->ib_active = false; +out: + kfree(work); +} + +static int mlx5_ib_sys_error_event(struct notifier_block *nb, + unsigned long event, void *param) +{ + struct mlx5_ib_event_work *work; + + if (event != MLX5_DEV_EVENT_SYS_ERROR) + return NOTIFY_DONE; + + work = kmalloc(sizeof(*work), GFP_ATOMIC); + if (!work) + return NOTIFY_DONE; + + INIT_WORK(&work->work, mlx5_ib_handle_sys_error_event); + work->dev = container_of(nb, struct mlx5_ib_dev, sys_error_events); + work->is_slave = false; + work->param = param; + work->event = event; + + queue_work(mlx5_ib_event_wq, &work->work); + + return NOTIFY_OK; +} + +static int mlx5_ib_stage_sys_error_notifier_init(struct mlx5_ib_dev *dev) +{ + dev->sys_error_events.notifier_call = mlx5_ib_sys_error_event; + mlx5_notifier_register(dev->mdev, &dev->sys_error_events); + return 0; +} + +static void mlx5_ib_stage_sys_error_notifier_cleanup(struct mlx5_ib_dev *dev) +{ + mlx5_notifier_unregister(dev->mdev, &dev->sys_error_events); +} + static int mlx5_ib_get_plane_num(struct mlx5_core_dev *mdev, u8 *num_plane) { struct mlx5_hca_vport_context vport_ctx; @@ -4466,12 +4525,16 @@ static int mlx5_ib_stage_caps_init(struct mlx5_ib_dev *dev) MLX5_HCA_CAP_2_GENERAL_OBJECT_TYPES_RDMA_CTRL) { err = mlx5_ib_init_ucaps(dev); if (err) - return err; + goto err_ucaps; } dev->ib_dev.use_cq_dim = true; return 0; + +err_ucaps: + bitmap_free(dev->var_table.bitmap); + return err; } static const struct ib_device_ops mlx5_ib_dev_port_ops = { @@ -4807,6 +4870,9 @@ static const struct mlx5_ib_profile pf_profile = { STAGE_CREATE(MLX5_IB_STAGE_WHITELIST_UID, mlx5_ib_devx_init, mlx5_ib_devx_cleanup), + STAGE_CREATE(MLX5_IB_STAGE_SYS_ERROR_NOTIFIER, + mlx5_ib_stage_sys_error_notifier_init, + mlx5_ib_stage_sys_error_notifier_cleanup), STAGE_CREATE(MLX5_IB_STAGE_IB_REG, mlx5_ib_stage_ib_reg_init, mlx5_ib_stage_ib_reg_cleanup), @@ -4864,6 +4930,9 @@ const struct mlx5_ib_profile raw_eth_profile = { STAGE_CREATE(MLX5_IB_STAGE_WHITELIST_UID, mlx5_ib_devx_init, mlx5_ib_devx_cleanup), + STAGE_CREATE(MLX5_IB_STAGE_SYS_ERROR_NOTIFIER, + mlx5_ib_stage_sys_error_notifier_init, + mlx5_ib_stage_sys_error_notifier_cleanup), STAGE_CREATE(MLX5_IB_STAGE_IB_REG, mlx5_ib_stage_ib_reg_init, mlx5_ib_stage_ib_reg_cleanup), diff --git a/drivers/infiniband/hw/mlx5/mlx5_ib.h b/drivers/infiniband/hw/mlx5/mlx5_ib.h index 09d82d5f95e354..fbccb0362590bb 100644 --- a/drivers/infiniband/hw/mlx5/mlx5_ib.h +++ b/drivers/infiniband/hw/mlx5/mlx5_ib.h @@ -1007,6 +1007,7 @@ enum mlx5_ib_stages { MLX5_IB_STAGE_BFREG, MLX5_IB_STAGE_PRE_IB_REG_UMR, MLX5_IB_STAGE_WHITELIST_UID, + MLX5_IB_STAGE_SYS_ERROR_NOTIFIER, MLX5_IB_STAGE_IB_REG, MLX5_IB_STAGE_DEVICE_NOTIFIER, MLX5_IB_STAGE_POST_IB_REG_UMR, @@ -1165,6 +1166,7 @@ struct mlx5_ib_dev { /* protect accessing data_direct_dev */ struct mutex data_direct_lock; struct notifier_block mdev_events; + struct notifier_block sys_error_events; struct notifier_block lag_events; int num_ports; /* serialize update of capability mask diff --git a/drivers/infiniband/hw/mlx5/std_types.c b/drivers/infiniband/hw/mlx5/std_types.c index 2fcf553044e15e..1ee31611b4b3f1 100644 --- a/drivers/infiniband/hw/mlx5/std_types.c +++ b/drivers/infiniband/hw/mlx5/std_types.c @@ -195,7 +195,7 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_GET_DATA_DIRECT_SYSFS_PATH)( int out_len = uverbs_attr_get_len(attrs, MLX5_IB_ATTR_GET_DATA_DIRECT_SYSFS_PATH); u32 dev_path_len; - char *dev_path; + char *dev_path = NULL; int ret; c = to_mucontext(ib_uverbs_get_ucontext(attrs)); @@ -223,9 +223,9 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_GET_DATA_DIRECT_SYSFS_PATH)( ret = uverbs_copy_to(attrs, MLX5_IB_ATTR_GET_DATA_DIRECT_SYSFS_PATH, dev_path, dev_path_len); - kfree(dev_path); end: + kfree(dev_path); mutex_unlock(&dev->data_direct_lock); return ret; } diff --git a/drivers/infiniband/hw/mthca/mthca_provider.c b/drivers/infiniband/hw/mthca/mthca_provider.c index dd572d76866c2e..e095873b381b6f 100644 --- a/drivers/infiniband/hw/mthca/mthca_provider.c +++ b/drivers/infiniband/hw/mthca/mthca_provider.c @@ -428,6 +428,8 @@ static int mthca_create_srq(struct ib_srq *ibsrq, if (context && ib_copy_to_udata(udata, &srq->srqn, sizeof(__u32))) { mthca_free_srq(to_mdev(ibsrq->device), srq); + mthca_unmap_user_db(to_mdev(ibsrq->device), &context->uar, + context->db_tab, ucmd.db_index); return -EFAULT; } @@ -436,6 +438,7 @@ static int mthca_create_srq(struct ib_srq *ibsrq, static int mthca_destroy_srq(struct ib_srq *srq, struct ib_udata *udata) { + mthca_free_srq(to_mdev(srq->device), to_msrq(srq)); if (udata) { struct mthca_ucontext *context = rdma_udata_to_drv_context( @@ -446,8 +449,6 @@ static int mthca_destroy_srq(struct ib_srq *srq, struct ib_udata *udata) mthca_unmap_user_db(to_mdev(srq->device), &context->uar, context->db_tab, to_msrq(srq)->db_index); } - - mthca_free_srq(to_mdev(srq->device), to_msrq(srq)); return 0; } diff --git a/drivers/infiniband/sw/rdmavt/qp.c b/drivers/infiniband/sw/rdmavt/qp.c index 134a79eecfcb87..3467797b5b01d2 100644 --- a/drivers/infiniband/sw/rdmavt/qp.c +++ b/drivers/infiniband/sw/rdmavt/qp.c @@ -92,12 +92,10 @@ static int rvt_wss_llc_size(void) static void cacheless_memcpy(void *dst, void *src, size_t n) { /* - * Use the only available X64 cacheless copy. Add a __user cast - * to quiet sparse. The src agument is already in the kernel so - * there are no security issues. The extra fault recovery machinery - * is not invoked. + * Use the only available X64 cacheless copy. + * The extra fault recovery machinery is not invoked. */ - __copy_user_nocache(dst, (void __user *)src, n); + copy_to_nontemporal(dst, src, n); } void rvt_wss_exit(struct rvt_dev_info *rdi) diff --git a/drivers/infiniband/sw/rxe/rxe_comp.c b/drivers/infiniband/sw/rxe/rxe_comp.c index a5b2b62f596b05..1390e861bd1d74 100644 --- a/drivers/infiniband/sw/rxe/rxe_comp.c +++ b/drivers/infiniband/sw/rxe/rxe_comp.c @@ -119,12 +119,15 @@ void retransmit_timer(struct timer_list *t) rxe_dbg_qp(qp, "retransmit timer fired\n"); + if (!rxe_get(qp)) + return; spin_lock_irqsave(&qp->state_lock, flags); if (qp->valid) { qp->comp.timeout = 1; rxe_sched_task(&qp->send_task); } spin_unlock_irqrestore(&qp->state_lock, flags); + rxe_put(qp); } void rxe_comp_queue_pkt(struct rxe_qp *qp, struct sk_buff *skb) diff --git a/drivers/infiniband/sw/rxe/rxe_mr.c b/drivers/infiniband/sw/rxe/rxe_mr.c index b1df0523884881..8116cf0fa6da35 100644 --- a/drivers/infiniband/sw/rxe/rxe_mr.c +++ b/drivers/infiniband/sw/rxe/rxe_mr.c @@ -72,14 +72,46 @@ void rxe_mr_init_dma(int access, struct rxe_mr *mr) mr->ibmr.type = IB_MR_TYPE_DMA; } +/* + * Convert iova to page_info index. The page_info stores pages of size + * PAGE_SIZE, but MRs can have different page sizes. This function + * handles the conversion for all cases: + * + * 1. mr->page_size > PAGE_SIZE: + * The MR's iova may not be aligned to mr->page_size. We use the + * aligned base (iova & page_mask) as reference, then calculate + * which PAGE_SIZE sub-page the iova falls into. + * + * 2. mr->page_size <= PAGE_SIZE: + * Use simple shift arithmetic since each page_info entry corresponds + * to one or more MR pages. + */ static unsigned long rxe_mr_iova_to_index(struct rxe_mr *mr, u64 iova) { - return (iova >> mr->page_shift) - (mr->ibmr.iova >> mr->page_shift); + int idx; + + if (mr_page_size(mr) > PAGE_SIZE) + idx = (iova - (mr->ibmr.iova & mr->page_mask)) >> PAGE_SHIFT; + else + idx = (iova >> mr->page_shift) - + (mr->ibmr.iova >> mr->page_shift); + + WARN_ON(idx >= mr->nbuf); + return idx; } +/* + * Convert iova to offset within the page_info entry. + * + * For mr_page_size > PAGE_SIZE, the offset is within the system page. + * For mr_page_size <= PAGE_SIZE, the offset is within the MR page size. + */ static unsigned long rxe_mr_iova_to_page_offset(struct rxe_mr *mr, u64 iova) { - return iova & (mr_page_size(mr) - 1); + if (mr_page_size(mr) > PAGE_SIZE) + return iova & (PAGE_SIZE - 1); + else + return iova & (mr_page_size(mr) - 1); } static bool is_pmem_page(struct page *pg) @@ -93,37 +125,69 @@ static bool is_pmem_page(struct page *pg) static int rxe_mr_fill_pages_from_sgt(struct rxe_mr *mr, struct sg_table *sgt) { - XA_STATE(xas, &mr->page_list, 0); struct sg_page_iter sg_iter; struct page *page; bool persistent = !!(mr->access & IB_ACCESS_FLUSH_PERSISTENT); + WARN_ON(mr_page_size(mr) != PAGE_SIZE); + __sg_page_iter_start(&sg_iter, sgt->sgl, sgt->orig_nents, 0); if (!__sg_page_iter_next(&sg_iter)) return 0; - do { - xas_lock(&xas); - while (true) { - page = sg_page_iter_page(&sg_iter); - - if (persistent && !is_pmem_page(page)) { - rxe_dbg_mr(mr, "Page can't be persistent\n"); - xas_set_err(&xas, -EINVAL); - break; - } + while (true) { + page = sg_page_iter_page(&sg_iter); - xas_store(&xas, page); - if (xas_error(&xas)) - break; - xas_next(&xas); - if (!__sg_page_iter_next(&sg_iter)) - break; + if (persistent && !is_pmem_page(page)) { + rxe_dbg_mr(mr, "Page can't be persistent\n"); + return -EINVAL; } - xas_unlock(&xas); - } while (xas_nomem(&xas, GFP_KERNEL)); - return xas_error(&xas); + mr->page_info[mr->nbuf].page = page; + mr->page_info[mr->nbuf].offset = 0; + mr->nbuf++; + + if (!__sg_page_iter_next(&sg_iter)) + break; + } + + return 0; +} + +static int __alloc_mr_page_info(struct rxe_mr *mr, int num_pages) +{ + mr->page_info = kcalloc(num_pages, sizeof(struct rxe_mr_page), + GFP_KERNEL); + if (!mr->page_info) + return -ENOMEM; + + mr->max_allowed_buf = num_pages; + mr->nbuf = 0; + + return 0; +} + +static int alloc_mr_page_info(struct rxe_mr *mr, int num_pages) +{ + int ret; + + WARN_ON(mr->num_buf); + ret = __alloc_mr_page_info(mr, num_pages); + if (ret) + return ret; + + mr->num_buf = num_pages; + + return 0; +} + +static void free_mr_page_info(struct rxe_mr *mr) +{ + if (!mr->page_info) + return; + + kfree(mr->page_info); + mr->page_info = NULL; } int rxe_mr_init_user(struct rxe_dev *rxe, u64 start, u64 length, @@ -134,8 +198,6 @@ int rxe_mr_init_user(struct rxe_dev *rxe, u64 start, u64 length, rxe_mr_init(access, mr); - xa_init(&mr->page_list); - umem = ib_umem_get(&rxe->ib_dev, start, length, access); if (IS_ERR(umem)) { rxe_dbg_mr(mr, "Unable to pin memory region err = %d\n", @@ -143,46 +205,24 @@ int rxe_mr_init_user(struct rxe_dev *rxe, u64 start, u64 length, return PTR_ERR(umem); } + err = alloc_mr_page_info(mr, ib_umem_num_pages(umem)); + if (err) + goto err2; + err = rxe_mr_fill_pages_from_sgt(mr, &umem->sgt_append.sgt); - if (err) { - ib_umem_release(umem); - return err; - } + if (err) + goto err1; mr->umem = umem; mr->ibmr.type = IB_MR_TYPE_USER; mr->state = RXE_MR_STATE_VALID; return 0; -} - -static int rxe_mr_alloc(struct rxe_mr *mr, int num_buf) -{ - XA_STATE(xas, &mr->page_list, 0); - int i = 0; - int err; - - xa_init(&mr->page_list); - - do { - xas_lock(&xas); - while (i != num_buf) { - xas_store(&xas, XA_ZERO_ENTRY); - if (xas_error(&xas)) - break; - xas_next(&xas); - i++; - } - xas_unlock(&xas); - } while (xas_nomem(&xas, GFP_KERNEL)); - - err = xas_error(&xas); - if (err) - return err; - - mr->num_buf = num_buf; - - return 0; +err1: + free_mr_page_info(mr); +err2: + ib_umem_release(umem); + return err; } int rxe_mr_init_fast(int max_pages, struct rxe_mr *mr) @@ -192,7 +232,7 @@ int rxe_mr_init_fast(int max_pages, struct rxe_mr *mr) /* always allow remote access for FMRs */ rxe_mr_init(RXE_ACCESS_REMOTE, mr); - err = rxe_mr_alloc(mr, max_pages); + err = alloc_mr_page_info(mr, max_pages); if (err) goto err1; @@ -205,26 +245,43 @@ int rxe_mr_init_fast(int max_pages, struct rxe_mr *mr) return err; } +/* + * I) MRs with page_size >= PAGE_SIZE, + * Split a large MR page (mr->page_size) into multiple PAGE_SIZE + * sub-pages and store them in page_info, offset is always 0. + * + * Called when mr->page_size > PAGE_SIZE. Each call to rxe_set_page() + * represents one mr->page_size region, which we must split into + * (mr->page_size >> PAGE_SHIFT) individual pages. + * + * II) MRs with page_size < PAGE_SIZE, + * Save each PAGE_SIZE page and its offset within the system page in page_info. + */ static int rxe_set_page(struct ib_mr *ibmr, u64 dma_addr) { struct rxe_mr *mr = to_rmr(ibmr); - struct page *page = ib_virt_dma_to_page(dma_addr); bool persistent = !!(mr->access & IB_ACCESS_FLUSH_PERSISTENT); - int err; + u32 i, pages_per_mr = mr_page_size(mr) >> PAGE_SHIFT; - if (persistent && !is_pmem_page(page)) { - rxe_dbg_mr(mr, "Page cannot be persistent\n"); - return -EINVAL; - } + pages_per_mr = MAX(1, pages_per_mr); - if (unlikely(mr->nbuf == mr->num_buf)) - return -ENOMEM; + for (i = 0; i < pages_per_mr; i++) { + u64 addr = dma_addr + i * PAGE_SIZE; + struct page *sub_page = ib_virt_dma_to_page(addr); - err = xa_err(xa_store(&mr->page_list, mr->nbuf, page, GFP_KERNEL)); - if (err) - return err; + if (unlikely(mr->nbuf >= mr->max_allowed_buf)) + return -ENOMEM; + + if (persistent && !is_pmem_page(sub_page)) { + rxe_dbg_mr(mr, "Page cannot be persistent\n"); + return -EINVAL; + } + + mr->page_info[mr->nbuf].page = sub_page; + mr->page_info[mr->nbuf].offset = addr & (PAGE_SIZE - 1); + mr->nbuf++; + } - mr->nbuf++; return 0; } @@ -234,6 +291,31 @@ int rxe_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sgl, struct rxe_mr *mr = to_rmr(ibmr); unsigned int page_size = mr_page_size(mr); + /* + * Ensure page_size and PAGE_SIZE are compatible for mapping. + * We require one to be a multiple of the other for correct + * iova-to-page conversion. + */ + if (!IS_ALIGNED(page_size, PAGE_SIZE) && + !IS_ALIGNED(PAGE_SIZE, page_size)) { + rxe_dbg_mr(mr, "MR page size %u must be compatible with PAGE_SIZE %lu\n", + page_size, PAGE_SIZE); + return -EINVAL; + } + + if (mr_page_size(mr) > PAGE_SIZE) { + /* resize page_info if needed */ + u32 map_mr_pages = (page_size >> PAGE_SHIFT) * mr->num_buf; + + if (map_mr_pages > mr->max_allowed_buf) { + rxe_dbg_mr(mr, "requested pages %u exceed max %u\n", + map_mr_pages, mr->max_allowed_buf); + free_mr_page_info(mr); + if (__alloc_mr_page_info(mr, map_mr_pages)) + return -ENOMEM; + } + } + mr->nbuf = 0; mr->page_shift = ilog2(page_size); mr->page_mask = ~((u64)page_size - 1); @@ -245,30 +327,30 @@ int rxe_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sgl, static int rxe_mr_copy_xarray(struct rxe_mr *mr, u64 iova, void *addr, unsigned int length, enum rxe_mr_copy_dir dir) { - unsigned int page_offset = rxe_mr_iova_to_page_offset(mr, iova); - unsigned long index = rxe_mr_iova_to_index(mr, iova); unsigned int bytes; - struct page *page; - void *va; + u8 *va; while (length) { - page = xa_load(&mr->page_list, index); - if (!page) + unsigned long index = rxe_mr_iova_to_index(mr, iova); + struct rxe_mr_page *info = &mr->page_info[index]; + unsigned int page_offset = rxe_mr_iova_to_page_offset(mr, iova); + + if (!info->page) return -EFAULT; - bytes = min_t(unsigned int, length, - mr_page_size(mr) - page_offset); - va = kmap_local_page(page); + page_offset += info->offset; + bytes = min_t(unsigned int, length, PAGE_SIZE - page_offset); + va = kmap_local_page(info->page); + if (dir == RXE_FROM_MR_OBJ) memcpy(addr, va + page_offset, bytes); else memcpy(va + page_offset, addr, bytes); kunmap_local(va); - page_offset = 0; addr += bytes; + iova += bytes; length -= bytes; - index++; } return 0; @@ -426,9 +508,6 @@ int copy_data( static int rxe_mr_flush_pmem_iova(struct rxe_mr *mr, u64 iova, unsigned int length) { - unsigned int page_offset; - unsigned long index; - struct page *page; unsigned int bytes; int err; u8 *va; @@ -438,15 +517,17 @@ static int rxe_mr_flush_pmem_iova(struct rxe_mr *mr, u64 iova, unsigned int leng return err; while (length > 0) { - index = rxe_mr_iova_to_index(mr, iova); - page = xa_load(&mr->page_list, index); - page_offset = rxe_mr_iova_to_page_offset(mr, iova); - if (!page) + unsigned long index = rxe_mr_iova_to_index(mr, iova); + struct rxe_mr_page *info = &mr->page_info[index]; + unsigned int page_offset = rxe_mr_iova_to_page_offset(mr, iova); + + if (!info->page) return -EFAULT; - bytes = min_t(unsigned int, length, - mr_page_size(mr) - page_offset); - va = kmap_local_page(page); + page_offset += info->offset; + bytes = min_t(unsigned int, length, PAGE_SIZE - page_offset); + + va = kmap_local_page(info->page); arch_wb_cache_pmem(va + page_offset, bytes); kunmap_local(va); @@ -501,6 +582,7 @@ enum resp_states rxe_mr_do_atomic_op(struct rxe_mr *mr, u64 iova, int opcode, } else { unsigned long index; int err; + struct rxe_mr_page *info; err = mr_check_range(mr, iova, sizeof(value)); if (err) { @@ -509,9 +591,12 @@ enum resp_states rxe_mr_do_atomic_op(struct rxe_mr *mr, u64 iova, int opcode, } page_offset = rxe_mr_iova_to_page_offset(mr, iova); index = rxe_mr_iova_to_index(mr, iova); - page = xa_load(&mr->page_list, index); - if (!page) + info = &mr->page_info[index]; + if (!info->page) return RESPST_ERR_RKEY_VIOLATION; + + page_offset += info->offset; + page = info->page; } if (unlikely(page_offset & 0x7)) { @@ -550,6 +635,7 @@ enum resp_states rxe_mr_do_atomic_write(struct rxe_mr *mr, u64 iova, u64 value) } else { unsigned long index; int err; + struct rxe_mr_page *info; /* See IBA oA19-28 */ err = mr_check_range(mr, iova, sizeof(value)); @@ -559,9 +645,12 @@ enum resp_states rxe_mr_do_atomic_write(struct rxe_mr *mr, u64 iova, u64 value) } page_offset = rxe_mr_iova_to_page_offset(mr, iova); index = rxe_mr_iova_to_index(mr, iova); - page = xa_load(&mr->page_list, index); - if (!page) + info = &mr->page_info[index]; + if (!info->page) return RESPST_ERR_RKEY_VIOLATION; + + page_offset += info->offset; + page = info->page; } /* See IBA A19.4.2 */ @@ -725,5 +814,5 @@ void rxe_mr_cleanup(struct rxe_pool_elem *elem) ib_umem_release(mr->umem); if (mr->ibmr.type != IB_MR_TYPE_DMA) - xa_destroy(&mr->page_list); + free_mr_page_info(mr); } diff --git a/drivers/infiniband/sw/rxe/rxe_req.c b/drivers/infiniband/sw/rxe/rxe_req.c index 373b03f223bebc..12d03f390b0978 100644 --- a/drivers/infiniband/sw/rxe/rxe_req.c +++ b/drivers/infiniband/sw/rxe/rxe_req.c @@ -102,6 +102,8 @@ void rnr_nak_timer(struct timer_list *t) rxe_dbg_qp(qp, "nak timer fired\n"); + if (!rxe_get(qp)) + return; spin_lock_irqsave(&qp->state_lock, flags); if (qp->valid) { /* request a send queue retry */ @@ -110,6 +112,7 @@ void rnr_nak_timer(struct timer_list *t) rxe_sched_task(&qp->send_task); } spin_unlock_irqrestore(&qp->state_lock, flags); + rxe_put(qp); } static void req_check_sq_drain_done(struct rxe_qp *qp) diff --git a/drivers/infiniband/sw/rxe/rxe_srq.c b/drivers/infiniband/sw/rxe/rxe_srq.c index 2a234f26ac1044..c9a7cd38953d31 100644 --- a/drivers/infiniband/sw/rxe/rxe_srq.c +++ b/drivers/infiniband/sw/rxe/rxe_srq.c @@ -77,9 +77,6 @@ int rxe_srq_from_init(struct rxe_dev *rxe, struct rxe_srq *srq, goto err_free; } - srq->rq.queue = q; - init->attr.max_wr = srq->rq.max_wr; - if (uresp) { if (copy_to_user(&uresp->srq_num, &srq->srq_num, sizeof(uresp->srq_num))) { @@ -88,6 +85,9 @@ int rxe_srq_from_init(struct rxe_dev *rxe, struct rxe_srq *srq, } } + srq->rq.queue = q; + init->attr.max_wr = srq->rq.max_wr; + return 0; err_free: diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.h b/drivers/infiniband/sw/rxe/rxe_verbs.h index fd48075810dd10..1b8ed1031bd57f 100644 --- a/drivers/infiniband/sw/rxe/rxe_verbs.h +++ b/drivers/infiniband/sw/rxe/rxe_verbs.h @@ -335,6 +335,11 @@ static inline int rkey_is_mw(u32 rkey) return (index >= RXE_MIN_MW_INDEX) && (index <= RXE_MAX_MW_INDEX); } +struct rxe_mr_page { + struct page *page; + unsigned int offset; /* offset in system page */ +}; + struct rxe_mr { struct rxe_pool_elem elem; struct ib_mr ibmr; @@ -351,10 +356,13 @@ struct rxe_mr { unsigned int page_shift; u64 page_mask; + /* size of page_info when mr allocated */ u32 num_buf; + /* real size of page_info */ + u32 max_allowed_buf; u32 nbuf; - struct xarray page_list; + struct rxe_mr_page *page_info; }; static inline unsigned int mr_page_size(struct rxe_mr *mr) diff --git a/drivers/infiniband/sw/siw/siw_qp_rx.c b/drivers/infiniband/sw/siw/siw_qp_rx.c index a10820e3388782..e8a88b378d51d4 100644 --- a/drivers/infiniband/sw/siw/siw_qp_rx.c +++ b/drivers/infiniband/sw/siw/siw_qp_rx.c @@ -1435,7 +1435,8 @@ int siw_tcp_rx_data(read_descriptor_t *rd_desc, struct sk_buff *skb, } if (unlikely(rv != 0 && rv != -EAGAIN)) { if ((srx->state > SIW_GET_HDR || - qp->rx_fpdu->more_ddp_segs) && run_completion) + (qp->rx_fpdu && qp->rx_fpdu->more_ddp_segs)) && + run_completion) siw_rdmap_complete(qp, rv); siw_dbg_qp(qp, "rx error %d, rx state %d\n", rv, diff --git a/drivers/infiniband/ulp/rtrs/rtrs-clt.c b/drivers/infiniband/ulp/rtrs/rtrs-clt.c index 2b397a544cb936..8fa1d72bd20a4f 100644 --- a/drivers/infiniband/ulp/rtrs/rtrs-clt.c +++ b/drivers/infiniband/ulp/rtrs/rtrs-clt.c @@ -1923,7 +1923,7 @@ static int rtrs_rdma_conn_rejected(struct rtrs_clt_con *con, struct rtrs_path *s = con->c.path; const struct rtrs_msg_conn_rsp *msg; const char *rej_msg; - int status, errno; + int status, errno = -ECONNRESET; u8 data_len; status = ev->status; @@ -1945,7 +1945,7 @@ static int rtrs_rdma_conn_rejected(struct rtrs_clt_con *con, status, rej_msg); } - return -ECONNRESET; + return errno; } void rtrs_clt_close_conns(struct rtrs_clt_path *clt_path, bool wait) diff --git a/drivers/infiniband/ulp/rtrs/rtrs-srv.c b/drivers/infiniband/ulp/rtrs/rtrs-srv.c index 9ecc6343455d6c..adb798e2a54ae1 100644 --- a/drivers/infiniband/ulp/rtrs/rtrs-srv.c +++ b/drivers/infiniband/ulp/rtrs/rtrs-srv.c @@ -208,7 +208,6 @@ static int rdma_write_sg(struct rtrs_srv_op *id) size_t sg_cnt; int err, offset; bool need_inval; - u32 rkey = 0; struct ib_reg_wr rwr; struct ib_sge *plist; struct ib_sge list; @@ -240,11 +239,6 @@ static int rdma_write_sg(struct rtrs_srv_op *id) wr->wr.num_sge = 1; wr->remote_addr = le64_to_cpu(id->rd_msg->desc[0].addr); wr->rkey = le32_to_cpu(id->rd_msg->desc[0].key); - if (rkey == 0) - rkey = wr->rkey; - else - /* Only one key is actually used */ - WARN_ON_ONCE(rkey != wr->rkey); wr->wr.opcode = IB_WR_RDMA_WRITE; wr->wr.wr_cqe = &io_comp_cqe; @@ -277,7 +271,7 @@ static int rdma_write_sg(struct rtrs_srv_op *id) inv_wr.opcode = IB_WR_SEND_WITH_INV; inv_wr.wr_cqe = &io_comp_cqe; inv_wr.send_flags = 0; - inv_wr.ex.invalidate_rkey = rkey; + inv_wr.ex.invalidate_rkey = wr->rkey; } imm_wr.wr.next = NULL; @@ -601,7 +595,7 @@ static int map_cont_bufs(struct rtrs_srv_path *srv_path) srv_path->mrs_num++) { struct rtrs_srv_mr *srv_mr = &srv_path->mrs[srv_path->mrs_num]; struct scatterlist *s; - int nr, nr_sgt, chunks; + int nr, nr_sgt, chunks, ind; sgt = &srv_mr->sgt; chunks = chunks_per_mr * srv_path->mrs_num; @@ -631,7 +625,7 @@ static int map_cont_bufs(struct rtrs_srv_path *srv_path) } nr = ib_map_mr_sg(mr, sgt->sgl, nr_sgt, NULL, max_chunk_size); - if (nr != nr_sgt) { + if (nr < nr_sgt) { err = nr < 0 ? nr : -EINVAL; goto dereg_mr; } @@ -647,9 +641,24 @@ static int map_cont_bufs(struct rtrs_srv_path *srv_path) goto dereg_mr; } } - /* Eventually dma addr for each chunk can be cached */ - for_each_sg(sgt->sgl, s, nr_sgt, i) - srv_path->dma_addr[chunks + i] = sg_dma_address(s); + + /* + * Cache DMA addresses by traversing sg entries. If + * regions were merged, an inner loop is required to + * populate the DMA address array by traversing larger + * regions. + */ + ind = chunks; + for_each_sg(sgt->sgl, s, nr_sgt, i) { + unsigned int dma_len = sg_dma_len(s); + u64 dma_addr = sg_dma_address(s); + u64 dma_addr_end = dma_addr + dma_len; + + do { + srv_path->dma_addr[ind++] = dma_addr; + dma_addr += max_chunk_size; + } while (dma_addr < dma_addr_end); + } ib_update_fast_reg_key(mr, ib_inc_rkey(mr->rkey)); srv_mr->mr = mr; diff --git a/drivers/input/joystick/xpad.c b/drivers/input/joystick/xpad.c index 363d509493866a..627e8950e451e5 100644 --- a/drivers/input/joystick/xpad.c +++ b/drivers/input/joystick/xpad.c @@ -313,6 +313,8 @@ static const struct xpad_device { { 0x1532, 0x0a00, "Razer Atrox Arcade Stick", MAP_TRIGGERS_TO_BUTTONS, XTYPE_XBOXONE }, { 0x1532, 0x0a03, "Razer Wildcat", 0, XTYPE_XBOXONE }, { 0x1532, 0x0a29, "Razer Wolverine V2", 0, XTYPE_XBOXONE }, + { 0x1532, 0x0a57, "Razer Wolverine V3 Pro (Wired)", 0, XTYPE_XBOX360 }, + { 0x1532, 0x0a59, "Razer Wolverine V3 Pro (2.4 GHz Dongle)", 0, XTYPE_XBOX360 }, { 0x15e4, 0x3f00, "Power A Mini Pro Elite", 0, XTYPE_XBOX360 }, { 0x15e4, 0x3f0a, "Xbox Airflo wired controller", 0, XTYPE_XBOX360 }, { 0x15e4, 0x3f10, "Batarang Xbox 360 controller", 0, XTYPE_XBOX360 }, @@ -360,6 +362,8 @@ static const struct xpad_device { { 0x1bad, 0xfd00, "Razer Onza TE", 0, XTYPE_XBOX360 }, { 0x1bad, 0xfd01, "Razer Onza", 0, XTYPE_XBOX360 }, { 0x1ee9, 0x1590, "ZOTAC Gaming Zone", 0, XTYPE_XBOX360 }, + { 0x20bc, 0x5134, "BETOP BTP-KP50B Xinput Dongle", 0, XTYPE_XBOX360 }, + { 0x20bc, 0x514a, "BETOP BTP-KP50C Xinput Dongle", 0, XTYPE_XBOX360 }, { 0x20d6, 0x2001, "BDA Xbox Series X Wired Controller", 0, XTYPE_XBOXONE }, { 0x20d6, 0x2009, "PowerA Enhanced Wired Controller for Xbox Series X|S", 0, XTYPE_XBOXONE }, { 0x20d6, 0x2064, "PowerA Wired Controller for Xbox", MAP_SHARE_BUTTON, XTYPE_XBOXONE }, @@ -562,6 +566,7 @@ static const struct usb_device_id xpad_table[] = { XPAD_XBOX360_VENDOR(0x1a86), /* Nanjing Qinheng Microelectronics (WCH) */ XPAD_XBOX360_VENDOR(0x1bad), /* Harmonix Rock Band guitar and drums */ XPAD_XBOX360_VENDOR(0x1ee9), /* ZOTAC Technology Limited */ + XPAD_XBOX360_VENDOR(0x20bc), /* BETOP wireless dongles */ XPAD_XBOX360_VENDOR(0x20d6), /* PowerA controllers */ XPAD_XBOXONE_VENDOR(0x20d6), /* PowerA controllers */ XPAD_XBOX360_VENDOR(0x2345), /* Machenike Controllers */ diff --git a/drivers/input/misc/Kconfig b/drivers/input/misc/Kconfig index 94a753fcb64fa5..c327a21b418ce5 100644 --- a/drivers/input/misc/Kconfig +++ b/drivers/input/misc/Kconfig @@ -1003,4 +1003,15 @@ config INPUT_STPMIC1_ONKEY To compile this driver as a module, choose M here: the module will be called stpmic1_onkey. +config INPUT_MACSMC_INPUT + tristate "Apple Mac SMC lid/buttons" + depends on MFD_MACSMC + help + Say Y here if you want to use the input events delivered via the + SMC controller on Apple Mac machines using the macsmc driver. + This includes lid open/close and the power button. + + To compile this driver as a module, choose M here: the + module will be called macsmc-input. + endif diff --git a/drivers/input/misc/Makefile b/drivers/input/misc/Makefile index 415fc4e2918be8..aa148f6a0f3233 100644 --- a/drivers/input/misc/Makefile +++ b/drivers/input/misc/Makefile @@ -52,6 +52,7 @@ obj-$(CONFIG_INPUT_IQS7222) += iqs7222.o obj-$(CONFIG_INPUT_KEYSPAN_REMOTE) += keyspan_remote.o obj-$(CONFIG_INPUT_KXTJ9) += kxtj9.o obj-$(CONFIG_INPUT_M68K_BEEP) += m68kspkr.o +obj-$(CONFIG_INPUT_MACSMC_INPUT) += macsmc-input.o obj-$(CONFIG_INPUT_MAX7360_ROTARY) += max7360-rotary.o obj-$(CONFIG_INPUT_MAX77650_ONKEY) += max77650-onkey.o obj-$(CONFIG_INPUT_MAX77693_HAPTIC) += max77693-haptic.o diff --git a/drivers/input/misc/macsmc-input.c b/drivers/input/misc/macsmc-input.c new file mode 100644 index 00000000000000..2cead3b7f45fed --- /dev/null +++ b/drivers/input/misc/macsmc-input.c @@ -0,0 +1,210 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Apple SMC input event driver + * Copyright The Asahi Linux Contributors + * + * This driver exposes HID events from the SMC as an input device. + * This includes the lid open/close and power button notifications. + */ + +#include +#include +#include +#include +#include +#include + +/** + * struct macsmc_input + * @dev: Underlying struct device for the input sub-device + * @smc: Pointer to apple_smc struct of the mfd parent + * @input: Allocated input_dev; devres managed + * @nb: Notifier block used for incoming events from SMC (e.g. button pressed down) + * @wakeup_mode: Set to true when system is suspended and power button events should wake it + */ +struct macsmc_input { + struct device *dev; + struct apple_smc *smc; + struct input_dev *input; + struct notifier_block nb; + bool wakeup_mode; +}; + +#define SMC_EV_BTN 0x7201 +#define SMC_EV_LID 0x7203 + +#define BTN_POWER 0x01 /* power button on e.g. Mac Mini chasis pressed */ +#define BTN_TOUCHID 0x06 /* combined TouchID / power button on MacBooks pressed */ +#define BTN_POWER_HELD_SHORT 0xfe /* power button briefly held down */ +#define BTN_POWER_HELD_LONG 0x00 /* power button held down; sent just before forced poweroff */ + +static void macsmc_input_event_button(struct macsmc_input *smcin, unsigned long event) +{ + u8 button = (event >> 8) & 0xff; + u8 state = !!(event & 0xff); + + switch (button) { + case BTN_POWER: + case BTN_TOUCHID: + pm_wakeup_dev_event(smcin->dev, 0, (smcin->wakeup_mode && state)); + /* + * Suppress KEY_POWER reports when suspended to avoid powering down + * immediately after waking from s2idle. + * */ + if (smcin->wakeup_mode) + return; + + input_report_key(smcin->input, KEY_POWER, state); + input_sync(smcin->input); + break; + case BTN_POWER_HELD_SHORT: /* power button held down; ignore */ + break; + case BTN_POWER_HELD_LONG: + /* + * If we get here the power button has been held down for a while and + * we have about 4 seconds before forced power-off is triggered by SMC. + * Try to do an emergency shutdown to make sure the NVMe cache is + * flushed. macOS actually does this by panicing (!)... + */ + if (state) { + dev_crit(smcin->dev, "Triggering forced shutdown!\n"); + if (kernel_can_power_off()) + kernel_power_off(); + else /* Missing macsmc-reboot driver? */ + kernel_restart("SMC power button triggered restart"); + } + break; + default: + dev_warn(smcin->dev, "Unknown SMC button event: %04lx\n", event & 0xffff); + } +} + +static void macsmc_input_event_lid(struct macsmc_input *smcin, unsigned long event) +{ + u8 lid_state = !!((event >> 8) & 0xff); + + pm_wakeup_dev_event(smcin->dev, 0, (smcin->wakeup_mode && !lid_state)); + input_report_switch(smcin->input, SW_LID, lid_state); + input_sync(smcin->input); +} + +static int macsmc_input_event(struct notifier_block *nb, unsigned long event, void *data) +{ + struct macsmc_input *smcin = container_of(nb, struct macsmc_input, nb); + u16 type = event >> 16; + + switch (type) { + case SMC_EV_BTN: + macsmc_input_event_button(smcin, event); + return NOTIFY_OK; + case SMC_EV_LID: + macsmc_input_event_lid(smcin, event); + return NOTIFY_OK; + default: + /* SMC event meant for another driver */ + return NOTIFY_DONE; + } +} + +static int macsmc_input_probe(struct platform_device *pdev) +{ + struct apple_smc *smc = dev_get_drvdata(pdev->dev.parent); + struct macsmc_input *smcin; + bool have_lid, have_power; + int error; + + /* Bail early if this SMC neither supports power button nor lid events */ + have_lid = apple_smc_key_exists(smc, SMC_KEY(MSLD)); + have_power = apple_smc_key_exists(smc, SMC_KEY(bHLD)); + if (!have_lid && !have_power) + return -ENODEV; + + smcin = devm_kzalloc(&pdev->dev, sizeof(*smcin), GFP_KERNEL); + if (!smcin) + return -ENOMEM; + + smcin->dev = &pdev->dev; + smcin->smc = smc; + platform_set_drvdata(pdev, smcin); + + smcin->input = devm_input_allocate_device(&pdev->dev); + if (!smcin->input) + return -ENOMEM; + + smcin->input->phys = "macsmc-input (0)"; + smcin->input->name = "Apple SMC power/lid events"; + + if (have_lid) + input_set_capability(smcin->input, EV_SW, SW_LID); + if (have_power) + input_set_capability(smcin->input, EV_KEY, KEY_POWER); + + if (have_lid) { + u8 val; + + error = apple_smc_read_u8(smc, SMC_KEY(MSLD), &val); + if (error < 0) + dev_warn(&pdev->dev, "Failed to read initial lid state\n"); + else + input_report_switch(smcin->input, SW_LID, val); + } + + if (have_power) { + u32 val; + + error = apple_smc_read_u32(smc, SMC_KEY(bHLD), &val); + if (error < 0) + dev_warn(&pdev->dev, "Failed to read initial power button state\n"); + else + input_report_key(smcin->input, KEY_POWER, val & 1); + } + + error = input_register_device(smcin->input); + if (error) { + dev_err(&pdev->dev, "Failed to register input device: %d\n", error); + return error; + } + + input_sync(smcin->input); + + smcin->nb.notifier_call = macsmc_input_event; + blocking_notifier_chain_register(&smc->event_handlers, &smcin->nb); + + device_init_wakeup(&pdev->dev, true); + + return 0; +} + +static int macsmc_input_pm_prepare(struct device *dev) +{ + struct macsmc_input *smcin = dev_get_drvdata(dev); + + smcin->wakeup_mode = true; + return 0; +} + +static void macsmc_input_pm_complete(struct device *dev) +{ + struct macsmc_input *smcin = dev_get_drvdata(dev); + + smcin->wakeup_mode = false; +} + +static const struct dev_pm_ops macsmc_input_pm_ops = { + .prepare = macsmc_input_pm_prepare, + .complete = macsmc_input_pm_complete, +}; + +static struct platform_driver macsmc_input_driver = { + .driver = { + .name = "macsmc-input", + .pm = &macsmc_input_pm_ops, + }, + .probe = macsmc_input_probe, +}; +module_platform_driver(macsmc_input_driver); + +MODULE_AUTHOR("Hector Martin "); +MODULE_LICENSE("Dual MIT/GPL"); +MODULE_DESCRIPTION("Apple SMC input driver"); +MODULE_ALIAS("platform:macsmc-input"); diff --git a/drivers/input/misc/uinput.c b/drivers/input/misc/uinput.c index 13336a2fd49c8a..0e9544a98e672a 100644 --- a/drivers/input/misc/uinput.c +++ b/drivers/input/misc/uinput.c @@ -25,8 +25,10 @@ #include #include #include +#include #include #include +#include #include #include "../input-compat.h" @@ -57,6 +59,7 @@ struct uinput_device { struct input_dev *dev; struct mutex mutex; enum uinput_state state; + spinlock_t state_lock; wait_queue_head_t waitq; unsigned char ready; unsigned char head; @@ -75,6 +78,8 @@ static int uinput_dev_event(struct input_dev *dev, struct uinput_device *udev = input_get_drvdata(dev); struct timespec64 ts; + lockdep_assert_held(&dev->event_lock); + ktime_get_ts64(&ts); udev->buff[udev->head] = (struct input_event) { @@ -146,27 +151,26 @@ static void uinput_request_release_slot(struct uinput_device *udev, static int uinput_request_send(struct uinput_device *udev, struct uinput_request *request) { - int retval; + unsigned long flags; + int retval = 0; - retval = mutex_lock_interruptible(&udev->mutex); - if (retval) - return retval; + spin_lock(&udev->state_lock); if (udev->state != UIST_CREATED) { retval = -ENODEV; goto out; } - init_completion(&request->done); - /* * Tell our userspace application about this new request * by queueing an input event. */ + spin_lock_irqsave(&udev->dev->event_lock, flags); uinput_dev_event(udev->dev, EV_UINPUT, request->code, request->id); + spin_unlock_irqrestore(&udev->dev->event_lock, flags); out: - mutex_unlock(&udev->mutex); + spin_unlock(&udev->state_lock); return retval; } @@ -175,6 +179,13 @@ static int uinput_request_submit(struct uinput_device *udev, { int retval; + /* + * Initialize completion before allocating the request slot. + * Once the slot is allocated, uinput_flush_requests() may + * complete it at any time, so it must be initialized first. + */ + init_completion(&request->done); + retval = uinput_request_reserve_slot(udev, request); if (retval) return retval; @@ -289,7 +300,14 @@ static void uinput_destroy_device(struct uinput_device *udev) struct input_dev *dev = udev->dev; enum uinput_state old_state = udev->state; + /* + * Update state under state_lock so that concurrent + * uinput_request_send() sees the state change before we + * flush pending requests and tear down the device. + */ + spin_lock(&udev->state_lock); udev->state = UIST_NEW_DEVICE; + spin_unlock(&udev->state_lock); if (dev) { name = dev->name; @@ -366,7 +384,9 @@ static int uinput_create_device(struct uinput_device *udev) if (error) goto fail2; + spin_lock(&udev->state_lock); udev->state = UIST_CREATED; + spin_unlock(&udev->state_lock); return 0; @@ -384,6 +404,7 @@ static int uinput_open(struct inode *inode, struct file *file) return -ENOMEM; mutex_init(&newdev->mutex); + spin_lock_init(&newdev->state_lock); spin_lock_init(&newdev->requests_lock); init_waitqueue_head(&newdev->requests_waitq); init_waitqueue_head(&newdev->waitq); diff --git a/drivers/input/mouse/bcm5974.c b/drivers/input/mouse/bcm5974.c index dfdfb59cc8b597..fe52f15c0c10dd 100644 --- a/drivers/input/mouse/bcm5974.c +++ b/drivers/input/mouse/bcm5974.c @@ -286,6 +286,8 @@ struct bcm5974 { const struct tp_finger *index[MAX_FINGERS]; /* finger index data */ struct input_mt_pos pos[MAX_FINGERS]; /* position array */ int slots[MAX_FINGERS]; /* slot assignments */ + struct work_struct mode_reset_work; + unsigned long last_mode_reset; }; /* trackpad finger block data, le16-aligned */ @@ -696,6 +698,32 @@ static int bcm5974_wellspring_mode(struct bcm5974 *dev, bool on) return retval; } +/* + * Mode switches sent before the control response are ignored. + * Fixing this state requires switching to normal mode and waiting + * about 1ms before switching back to wellspring mode. + */ +static void bcm5974_mode_reset_work(struct work_struct *work) +{ + struct bcm5974 *dev = container_of(work, struct bcm5974, mode_reset_work); + int error; + + guard(mutex)(&dev->pm_mutex); + dev->last_mode_reset = jiffies; + + error = bcm5974_wellspring_mode(dev, false); + if (error) { + dev_err(&dev->intf->dev, "reset to normal mode failed\n"); + return; + } + + fsleep(1000); + + error = bcm5974_wellspring_mode(dev, true); + if (error) + dev_err(&dev->intf->dev, "mode switch after reset failed\n"); +} + static void bcm5974_irq_button(struct urb *urb) { struct bcm5974 *dev = urb->context; @@ -752,10 +780,20 @@ static void bcm5974_irq_trackpad(struct urb *urb) if (dev->tp_urb->actual_length == 2) goto exit; - if (report_tp_state(dev, dev->tp_urb->actual_length)) + if (report_tp_state(dev, dev->tp_urb->actual_length)) { dprintk(1, "bcm5974: bad trackpad package, length: %d\n", dev->tp_urb->actual_length); + /* + * Receiving a HID packet means we aren't in wellspring mode. + * If we haven't tried a reset in the last second, try now. + */ + if (dev->tp_urb->actual_length == 8 && + time_after(jiffies, dev->last_mode_reset + msecs_to_jiffies(1000))) { + schedule_work(&dev->mode_reset_work); + } + } + exit: error = usb_submit_urb(dev->tp_urb, GFP_ATOMIC); if (error) @@ -906,6 +944,7 @@ static int bcm5974_probe(struct usb_interface *iface, dev->intf = iface; dev->input = input_dev; dev->cfg = *cfg; + INIT_WORK(&dev->mode_reset_work, bcm5974_mode_reset_work); mutex_init(&dev->pm_mutex); /* setup urbs */ @@ -998,6 +1037,7 @@ static void bcm5974_disconnect(struct usb_interface *iface) { struct bcm5974 *dev = usb_get_intfdata(iface); + disable_work_sync(&dev->mode_reset_work); usb_set_intfdata(iface, NULL); input_unregister_device(dev->input); diff --git a/drivers/input/rmi4/rmi_f54.c b/drivers/input/rmi4/rmi_f54.c index ac4041a69fcd3a..61909e1a39e248 100644 --- a/drivers/input/rmi4/rmi_f54.c +++ b/drivers/input/rmi4/rmi_f54.c @@ -538,6 +538,8 @@ static void rmi_f54_work(struct work_struct *work) int error; int i; + mutex_lock(&f54->data_mutex); + report_size = rmi_f54_get_report_size(f54); if (report_size == 0) { dev_err(&fn->dev, "Bad report size, report type=%d\n", @@ -546,8 +548,6 @@ static void rmi_f54_work(struct work_struct *work) goto error; /* retry won't help */ } - mutex_lock(&f54->data_mutex); - /* * Need to check if command has completed. * If not try again later. diff --git a/drivers/input/serio/i8042-acpipnpio.h b/drivers/input/serio/i8042-acpipnpio.h index d2cf940b105a69..8ebdf4fb903080 100644 --- a/drivers/input/serio/i8042-acpipnpio.h +++ b/drivers/input/serio/i8042-acpipnpio.h @@ -1187,6 +1187,13 @@ static const struct dmi_system_id i8042_dmi_quirk_table[] __initconst = { .driver_data = (void *)(SERIO_QUIRK_NOMUX | SERIO_QUIRK_RESET_ALWAYS | SERIO_QUIRK_NOLOOP | SERIO_QUIRK_NOPNP) }, + { + .matches = { + DMI_MATCH(DMI_BOARD_NAME, "X6KK45xU_X6SP45xU"), + }, + .driver_data = (void *)(SERIO_QUIRK_NOMUX | SERIO_QUIRK_RESET_ALWAYS | + SERIO_QUIRK_NOLOOP | SERIO_QUIRK_NOPNP) + }, { .matches = { DMI_MATCH(DMI_BOARD_NAME, "WUJIE Series-X5SP4NAG"), diff --git a/drivers/interconnect/mediatek/icc-emi.c b/drivers/interconnect/mediatek/icc-emi.c index 7da740b5fa8d62..dfa3a9cd939983 100644 --- a/drivers/interconnect/mediatek/icc-emi.c +++ b/drivers/interconnect/mediatek/icc-emi.c @@ -12,6 +12,7 @@ #include #include #include +#include #include #include @@ -22,7 +23,9 @@ static int mtk_emi_icc_aggregate(struct icc_node *node, u32 tag, u32 avg_bw, { struct mtk_icc_node *in = node->data; - *agg_avg += avg_bw; + if (check_add_overflow(*agg_avg, avg_bw, agg_avg)) + *agg_avg = U32_MAX; + *agg_peak = max_t(u32, *agg_peak, peak_bw); in->sum_avg = *agg_avg; @@ -40,7 +43,7 @@ static int mtk_emi_icc_set(struct icc_node *src, struct icc_node *dst) if (unlikely(!src->provider)) return -EINVAL; - dev = src->provider->dev; + dev = src->provider->dev->parent; switch (node->ep) { case 0: @@ -97,7 +100,7 @@ int mtk_emi_icc_probe(struct platform_device *pdev) if (!data) return -ENOMEM; - provider->dev = pdev->dev.parent; + provider->dev = dev; provider->set = mtk_emi_icc_set; provider->aggregate = mtk_emi_icc_aggregate; provider->xlate = of_icc_xlate_onecell; diff --git a/drivers/interconnect/qcom/qcs8300.c b/drivers/interconnect/qcom/qcs8300.c index 70a377bbcf2930..bc403a9bf68c65 100644 --- a/drivers/interconnect/qcom/qcs8300.c +++ b/drivers/interconnect/qcom/qcs8300.c @@ -629,7 +629,7 @@ static struct qcom_icc_node qxm_nsp = { .name = "qxm_nsp", .channels = 2, .buswidth = 32, - .num_links = 1, + .num_links = 2, .link_nodes = { &qns_hcp, &qns_nsp_gemnoc }, }; diff --git a/drivers/interconnect/qcom/sm8450.c b/drivers/interconnect/qcom/sm8450.c index 669a638bf3efcd..c88327d200acc8 100644 --- a/drivers/interconnect/qcom/sm8450.c +++ b/drivers/interconnect/qcom/sm8450.c @@ -800,7 +800,7 @@ static struct qcom_icc_node qhs_compute_cfg = { .channels = 1, .buswidth = 4, .num_links = 1, - .link_nodes = { MASTER_CDSP_NOC_CFG }, + .link_nodes = { &qhm_nsp_noc_config }, }; static struct qcom_icc_node qhs_cpr_cx = { @@ -874,7 +874,7 @@ static struct qcom_icc_node qhs_lpass_cfg = { .channels = 1, .buswidth = 4, .num_links = 1, - .link_nodes = { MASTER_CNOC_LPASS_AG_NOC }, + .link_nodes = { &qhm_config_noc }, }; static struct qcom_icc_node qhs_mss_cfg = { diff --git a/drivers/iommu/amd/amd_iommu.h b/drivers/iommu/amd/amd_iommu.h index b742ef1adb352b..df1c238dc88855 100644 --- a/drivers/iommu/amd/amd_iommu.h +++ b/drivers/iommu/amd/amd_iommu.h @@ -15,7 +15,6 @@ irqreturn_t amd_iommu_int_thread(int irq, void *data); irqreturn_t amd_iommu_int_thread_evtlog(int irq, void *data); irqreturn_t amd_iommu_int_thread_pprlog(int irq, void *data); irqreturn_t amd_iommu_int_thread_galog(int irq, void *data); -irqreturn_t amd_iommu_int_handler(int irq, void *data); void amd_iommu_restart_log(struct amd_iommu *iommu, const char *evt_type, u8 cntrl_intr, u8 cntrl_log, u32 status_run_mask, u32 status_overflow_mask); diff --git a/drivers/iommu/amd/amd_iommu_types.h b/drivers/iommu/amd/amd_iommu_types.h index 320733e7d8b423..3b09da3ffb74f5 100644 --- a/drivers/iommu/amd/amd_iommu_types.h +++ b/drivers/iommu/amd/amd_iommu_types.h @@ -706,7 +706,7 @@ struct amd_iommu { u32 flags; volatile u64 *cmd_sem; - atomic64_t cmd_sem_val; + u64 cmd_sem_val; /* * Track physical address to directly use it in build_completion_wait() * and avoid adding any special checks and handling for kdump. diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c index 384c90b4f90a0a..58d6f5ae155f21 100644 --- a/drivers/iommu/amd/init.c +++ b/drivers/iommu/amd/init.c @@ -1877,7 +1877,7 @@ static int __init init_iommu_one(struct amd_iommu *iommu, struct ivhd_header *h, iommu->pci_seg = pci_seg; raw_spin_lock_init(&iommu->lock); - atomic64_set(&iommu->cmd_sem_val, 0); + iommu->cmd_sem_val = 0; /* Add IOMMU to internal data structures */ list_add_tail(&iommu->list, &amd_iommu_list); @@ -2356,12 +2356,8 @@ static int iommu_setup_msi(struct amd_iommu *iommu) if (r) return r; - r = request_threaded_irq(iommu->dev->irq, - amd_iommu_int_handler, - amd_iommu_int_thread, - 0, "AMD-Vi", - iommu); - + r = request_threaded_irq(iommu->dev->irq, NULL, amd_iommu_int_thread, + IRQF_ONESHOT, "AMD-Vi", iommu); if (r) { pci_disable_msi(iommu->dev); return r; @@ -2535,8 +2531,8 @@ static int __iommu_setup_intcapxt(struct amd_iommu *iommu, const char *devname, return irq; } - ret = request_threaded_irq(irq, amd_iommu_int_handler, - thread_fn, 0, devname, iommu); + ret = request_threaded_irq(irq, NULL, thread_fn, IRQF_ONESHOT, devname, + iommu); if (ret) { irq_domain_free_irqs(irq, 1); irq_domain_remove(domain); diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index 7c12be1b247f43..cdcce333368263 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -1151,11 +1151,6 @@ irqreturn_t amd_iommu_int_thread(int irq, void *data) return IRQ_HANDLED; } -irqreturn_t amd_iommu_int_handler(int irq, void *data) -{ - return IRQ_WAKE_THREAD; -} - /**************************************************************************** * * IOMMU command queuing functions @@ -1185,7 +1180,12 @@ static int wait_on_sem(struct amd_iommu *iommu, u64 data) { int i = 0; - while (*iommu->cmd_sem != data && i < LOOP_TIMEOUT) { + /* + * cmd_sem holds a monotonically non-decreasing completion sequence + * number. + */ + while ((__s64)(READ_ONCE(*iommu->cmd_sem) - data) < 0 && + i < LOOP_TIMEOUT) { udelay(1); i += 1; } @@ -1417,6 +1417,12 @@ static int iommu_queue_command(struct amd_iommu *iommu, struct iommu_cmd *cmd) return iommu_queue_command_sync(iommu, cmd, true); } +static u64 get_cmdsem_val(struct amd_iommu *iommu) +{ + lockdep_assert_held(&iommu->lock); + return ++iommu->cmd_sem_val; +} + /* * This function queues a completion wait command into the command * buffer of an IOMMU @@ -1431,20 +1437,19 @@ static int iommu_completion_wait(struct amd_iommu *iommu) if (!iommu->need_sync) return 0; - data = atomic64_inc_return(&iommu->cmd_sem_val); - build_completion_wait(&cmd, iommu, data); - raw_spin_lock_irqsave(&iommu->lock, flags); + data = get_cmdsem_val(iommu); + build_completion_wait(&cmd, iommu, data); + ret = __iommu_queue_command_sync(iommu, &cmd, false); + raw_spin_unlock_irqrestore(&iommu->lock, flags); + if (ret) - goto out_unlock; + return ret; ret = wait_on_sem(iommu, data); -out_unlock: - raw_spin_unlock_irqrestore(&iommu->lock, flags); - return ret; } @@ -2849,8 +2854,21 @@ static struct iommu_domain blocked_domain = { static struct protection_domain identity_domain; +static int amd_iommu_identity_attach(struct iommu_domain *dom, struct device *dev, + struct iommu_domain *old) +{ + /* + * Don't allow attaching a device to the identity domain if SNP is + * enabled. + */ + if (amd_iommu_snp_en) + return -EINVAL; + + return amd_iommu_attach_device(dom, dev, old); +} + static const struct iommu_domain_ops identity_domain_ops = { - .attach_dev = amd_iommu_attach_device, + .attach_dev = amd_iommu_identity_attach, }; void amd_iommu_init_identity_domain(void) @@ -3114,18 +3132,23 @@ static void iommu_flush_irt_and_complete(struct amd_iommu *iommu, u16 devid) return; build_inv_irt(&cmd, devid); - data = atomic64_inc_return(&iommu->cmd_sem_val); - build_completion_wait(&cmd2, iommu, data); raw_spin_lock_irqsave(&iommu->lock, flags); + data = get_cmdsem_val(iommu); + build_completion_wait(&cmd2, iommu, data); + ret = __iommu_queue_command_sync(iommu, &cmd, true); if (ret) - goto out; + goto out_err; ret = __iommu_queue_command_sync(iommu, &cmd2, false); if (ret) - goto out; + goto out_err; + raw_spin_unlock_irqrestore(&iommu->lock, flags); + wait_on_sem(iommu, data); -out: + return; + +out_err: raw_spin_unlock_irqrestore(&iommu->lock, flags); } diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index 83a5aabcd15d97..1663ce5279a8fe 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -28,6 +29,7 @@ #include #include #include +#include #include #include #include @@ -201,6 +203,7 @@ struct apple_dart_hw { * @lock: lock for hardware operations involving this dart * @pgsize: pagesize supported by this DART * @supports_bypass: indicates if this DART supports bypass mode + * @locked: indicates if this DART is locked * @sid2group: maps stream ids to iommu_groups * @iommu: iommu core device */ @@ -222,12 +225,18 @@ struct apple_dart { u32 num_streams; u32 supports_bypass : 1; u32 four_level : 1; + u32 locked : 1; + + dma_addr_t dma_min; + dma_addr_t dma_max; struct iommu_group *sid2group[DART_MAX_STREAMS]; struct iommu_device iommu; u32 save_tcr[DART_MAX_STREAMS]; u32 save_ttbr[DART_MAX_STREAMS][DART_MAX_TTBR]; + + u64 *locked_ttbr[DART_MAX_STREAMS][DART_MAX_TTBR]; }; /* @@ -267,6 +276,7 @@ struct apple_dart_domain { struct io_pgtable_ops *pgtbl_ops; bool finalized; + u64 mask; struct mutex init_lock; struct apple_dart_atomic_stream_map stream_maps[MAX_DARTS_PER_DEVICE]; @@ -284,6 +294,7 @@ struct apple_dart_domain { struct apple_dart_master_cfg { /* Intersection of DART capabilitles */ u32 supports_bypass : 1; + u32 locked : 1; struct apple_dart_stream_map stream_maps[MAX_DARTS_PER_DEVICE]; }; @@ -378,6 +389,82 @@ apple_dart_hw_clear_all_ttbrs(struct apple_dart_stream_map *stream_map) apple_dart_hw_clear_ttbr(stream_map, i); } +static int +apple_dart_hw_map_locked_ttbr(struct apple_dart_stream_map *stream_map, u8 idx) +{ + struct apple_dart *dart = stream_map->dart; + int sid; + + for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) { + u32 ttbr; + phys_addr_t phys; + u64 *l1_tbl; + + ttbr = readl(dart->regs + DART_TTBR(dart, sid, idx)); + + if (!(ttbr & dart->hw->ttbr_valid)) { + dev_err(dart->dev, "Invalid ttbr[%u] for locked dart\n", + idx); + return -EIO; + } + + ttbr &= ~dart->hw->ttbr_valid; + + if (dart->hw->ttbr_addr_field_shift) + ttbr >>= dart->hw->ttbr_addr_field_shift; + phys = ((phys_addr_t) ttbr) << dart->hw->ttbr_shift; + + l1_tbl = devm_memremap(dart->dev, phys, dart->pgsize, + MEMREMAP_WB); + if (!l1_tbl) + return -ENOMEM; + + dart->locked_ttbr[sid][idx] = l1_tbl; + } + + return 0; +} + +static int +apple_dart_hw_unmap_locked_ttbr(struct apple_dart_stream_map *stream_map, + u8 idx) +{ + struct apple_dart *dart = stream_map->dart; + int sid; + + for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) { + /* TODO: locked L1 table might need to be restored to boot state */ + if (dart->locked_ttbr[sid][idx]) { + memset(dart->locked_ttbr[sid][idx], 0, dart->pgsize); + devm_memunmap(dart->dev, dart->locked_ttbr[sid][idx]); + } + dart->locked_ttbr[sid][idx] = NULL; + } + + return 0; +} + +static int +apple_dart_hw_sync_locked(struct io_pgtable_cfg *cfg, + struct apple_dart_stream_map *stream_map) +{ + struct apple_dart *dart = stream_map->dart; + int sid; + + for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) { + for (int idx = 0; idx < dart->hw->ttbr_count; idx++) { + u64 *ttbrep = dart->locked_ttbr[sid][idx]; + u64 *ptep = cfg->apple_dart_cfg.ttbr[idx]; + if (!ttbrep || !ptep) + continue; + for (int entry = 0; entry < dart->pgsize / sizeof(*ptep); entry++) + ttbrep[entry] = ptep[entry]; + } + } + + return 0; +} + static int apple_dart_t8020_hw_stream_command(struct apple_dart_stream_map *stream_map, u32 command) @@ -464,17 +551,9 @@ apple_dart_t8110_hw_invalidate_tlb(struct apple_dart_stream_map *stream_map) static int apple_dart_hw_reset(struct apple_dart *dart) { - u32 config; struct apple_dart_stream_map stream_map; int i; - config = readl(dart->regs + dart->hw->lock); - if (config & dart->hw->lock_bit) { - dev_err(dart->dev, "DART is locked down until reboot: %08x\n", - config); - return -EINVAL; - } - stream_map.dart = dart; bitmap_zero(stream_map.sidmap, DART_MAX_STREAMS); bitmap_set(stream_map.sidmap, 0, dart->num_streams); @@ -499,6 +578,8 @@ static void apple_dart_domain_flush_tlb(struct apple_dart_domain *domain) int i, j; struct apple_dart_atomic_stream_map *domain_stream_map; struct apple_dart_stream_map stream_map; + struct io_pgtable_cfg *pgtbl_cfg = + &io_pgtable_ops_to_pgtable(domain->pgtbl_ops)->cfg; for_each_stream_map(i, domain, domain_stream_map) { stream_map.dart = domain_stream_map->dart; @@ -506,7 +587,13 @@ static void apple_dart_domain_flush_tlb(struct apple_dart_domain *domain) for (j = 0; j < BITS_TO_LONGS(stream_map.dart->num_streams); j++) stream_map.sidmap[j] = atomic_long_read(&domain_stream_map->sidmap[j]); + WARN_ON(pm_runtime_get_sync(stream_map.dart->dev) < 0); + + if (stream_map.dart->locked) + apple_dart_hw_sync_locked(pgtbl_cfg, &stream_map); + stream_map.dart->hw->invalidate_tlb(&stream_map); + pm_runtime_put(stream_map.dart->dev); } } @@ -537,7 +624,7 @@ static phys_addr_t apple_dart_iova_to_phys(struct iommu_domain *domain, if (!ops) return 0; - return ops->iova_to_phys(ops, iova); + return ops->iova_to_phys(ops, iova & dart_domain->mask); } static int apple_dart_map_pages(struct iommu_domain *domain, unsigned long iova, @@ -551,8 +638,8 @@ static int apple_dart_map_pages(struct iommu_domain *domain, unsigned long iova, if (!ops) return -ENODEV; - return ops->map_pages(ops, iova, paddr, pgsize, pgcount, prot, gfp, - mapped); + return ops->map_pages(ops, iova & dart_domain->mask, paddr, pgsize, + pgcount, prot, gfp, mapped); } static size_t apple_dart_unmap_pages(struct iommu_domain *domain, @@ -563,7 +650,8 @@ static size_t apple_dart_unmap_pages(struct iommu_domain *domain, struct apple_dart_domain *dart_domain = to_dart_domain(domain); struct io_pgtable_ops *ops = dart_domain->pgtbl_ops; - return ops->unmap_pages(ops, iova, pgsize, pgcount, gather); + return ops->unmap_pages(ops, iova & dart_domain->mask, pgsize, pgcount, + gather); } static void @@ -574,9 +662,10 @@ apple_dart_setup_translation(struct apple_dart_domain *domain, struct io_pgtable_cfg *pgtbl_cfg = &io_pgtable_ops_to_pgtable(domain->pgtbl_ops)->cfg; - for (i = 0; i < pgtbl_cfg->apple_dart_cfg.n_ttbrs; ++i) - apple_dart_hw_set_ttbr(stream_map, i, - pgtbl_cfg->apple_dart_cfg.ttbr[i]); + for (i = 0; i < pgtbl_cfg->apple_dart_cfg.n_ttbrs; ++i) { + u64 ttbr = virt_to_phys(pgtbl_cfg->apple_dart_cfg.ttbr[i]); + apple_dart_hw_set_ttbr(stream_map, i, ttbr); + } for (; i < stream_map->dart->hw->ttbr_count; ++i) apple_dart_hw_clear_ttbr(stream_map, i); @@ -585,11 +674,31 @@ apple_dart_setup_translation(struct apple_dart_domain *domain, stream_map->dart->hw->invalidate_tlb(stream_map); } +static void +apple_dart_setup_translation_locked(struct apple_dart_domain *domain, + struct apple_dart_stream_map *stream_map) +{ + int i; + struct io_pgtable_cfg *pgtbl_cfg = + &io_pgtable_ops_to_pgtable(domain->pgtbl_ops)->cfg; + + /* Locked DARTs are set up by the bootloader. */ + for (i = 0; i < pgtbl_cfg->apple_dart_cfg.n_ttbrs; ++i) + apple_dart_hw_map_locked_ttbr(stream_map, i); + for (; i < stream_map->dart->hw->ttbr_count; ++i) + apple_dart_hw_unmap_locked_ttbr(stream_map, i); + + apple_dart_hw_sync_locked(pgtbl_cfg, stream_map); + stream_map->dart->hw->invalidate_tlb(stream_map); +} + static int apple_dart_finalize_domain(struct apple_dart_domain *dart_domain, struct apple_dart_master_cfg *cfg) { struct apple_dart *dart = cfg->stream_maps[0].dart; struct io_pgtable_cfg pgtbl_cfg; + dma_addr_t dma_max = dart->dma_max; + u32 ias = min_t(u32, dart->ias, fls64(dma_max)); int ret = 0; int i, j; @@ -610,12 +719,48 @@ static int apple_dart_finalize_domain(struct apple_dart_domain *dart_domain, pgtbl_cfg = (struct io_pgtable_cfg){ .pgsize_bitmap = dart->pgsize, - .ias = dart->ias, + .ias = ias, .oas = dart->oas, .coherent_walk = 1, .iommu_dev = dart->dev, }; + if (dart->locked) { + unsigned long *sidmap; + int sid; + u32 ttbr; + + /* Locked DARTs can only have a single stream bound */ + sidmap = cfg->stream_maps[0].sidmap; + sid = find_first_bit(sidmap, dart->num_streams); + + WARN_ON((sid < 0) || bitmap_weight(sidmap, dart->num_streams) > 1); + ttbr = readl(dart->regs + DART_TTBR(dart, sid, 0)); + + WARN_ON(!(ttbr & dart->hw->ttbr_valid)); + + /* If the DART is locked, we need to keep the translation level count. */ + if (dart->hw->tcr_4level && dart->ias > 36) { + if (readl(dart->regs + DART_TCR(dart, sid)) & dart->hw->tcr_4level) { + if (ias < 37) { + dev_info(dart->dev, "Expanded to ias=37 due to lock\n"); + pgtbl_cfg.ias = 37; + } + } else if (ias > 36) { + dev_info(dart->dev, "Limited to ias=36 due to lock\n"); + pgtbl_cfg.ias = 36; + if (dart->dma_min == 0 && dma_max == DMA_BIT_MASK(dart->ias)) { + dma_max = DMA_BIT_MASK(pgtbl_cfg.ias); + } else if ((dart->dma_min ^ dma_max) & ~DMA_BIT_MASK(36)) { + dev_err(dart->dev, + "Invalid DMA range for locked 3-level PT\n"); + ret = -ENOMEM; + goto done; + } + } + } + } + dart_domain->pgtbl_ops = alloc_io_pgtable_ops(dart->hw->fmt, &pgtbl_cfg, &dart_domain->domain); if (!dart_domain->pgtbl_ops) { @@ -623,10 +768,16 @@ static int apple_dart_finalize_domain(struct apple_dart_domain *dart_domain, goto done; } + if (pgtbl_cfg.pgsize_bitmap == SZ_4K) + dart_domain->mask = DMA_BIT_MASK(min_t(u32, dart->ias, 32)); + else if (pgtbl_cfg.apple_dart_cfg.n_levels == 3) + dart_domain->mask = DMA_BIT_MASK(min_t(u32, dart->ias, 36)); + else if (pgtbl_cfg.apple_dart_cfg.n_levels == 4) + dart_domain->mask = DMA_BIT_MASK(min_t(u32, dart->ias, 47)); + dart_domain->domain.pgsize_bitmap = pgtbl_cfg.pgsize_bitmap; - dart_domain->domain.geometry.aperture_start = 0; - dart_domain->domain.geometry.aperture_end = - (dma_addr_t)DMA_BIT_MASK(pgtbl_cfg.ias); + dart_domain->domain.geometry.aperture_start = dart->dma_min; + dart_domain->domain.geometry.aperture_end = dma_max; dart_domain->domain.geometry.force_aperture = true; dart_domain->finalized = true; @@ -680,17 +831,29 @@ static int apple_dart_attach_dev_paging(struct iommu_domain *domain, struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev); struct apple_dart_domain *dart_domain = to_dart_domain(domain); + for_each_stream_map(i, cfg, stream_map) + WARN_ON(pm_runtime_get_sync(stream_map->dart->dev) < 0); + ret = apple_dart_finalize_domain(dart_domain, cfg); if (ret) - return ret; + goto err; ret = apple_dart_domain_add_streams(dart_domain, cfg); if (ret) - return ret; + goto err; + + for_each_stream_map(i, cfg, stream_map) { + if (!stream_map->dart->locked) + apple_dart_setup_translation(dart_domain, stream_map); + else + apple_dart_setup_translation_locked(dart_domain, + stream_map); + } +err: for_each_stream_map(i, cfg, stream_map) - apple_dart_setup_translation(dart_domain, stream_map); - return 0; + pm_runtime_put(stream_map->dart->dev); + return ret; } static int apple_dart_attach_dev_identity(struct iommu_domain *domain, @@ -704,8 +867,17 @@ static int apple_dart_attach_dev_identity(struct iommu_domain *domain, if (!cfg->supports_bypass) return -EINVAL; + if (cfg->locked) + return -EINVAL; + + for_each_stream_map(i, cfg, stream_map) + WARN_ON(pm_runtime_get_sync(stream_map->dart->dev) < 0); + for_each_stream_map(i, cfg, stream_map) apple_dart_hw_enable_bypass(stream_map); + + for_each_stream_map(i, cfg, stream_map) + pm_runtime_put(stream_map->dart->dev); return 0; } @@ -726,8 +898,17 @@ static int apple_dart_attach_dev_blocked(struct iommu_domain *domain, struct apple_dart_stream_map *stream_map; int i; + if (cfg->locked) + return -EINVAL; + + for_each_stream_map(i, cfg, stream_map) + WARN_ON(pm_runtime_get_sync(stream_map->dart->dev) < 0); + for_each_stream_map(i, cfg, stream_map) apple_dart_hw_disable_dma(stream_map); + + for_each_stream_map(i, cfg, stream_map) + pm_runtime_put(stream_map->dart->dev); return 0; } @@ -746,21 +927,29 @@ static struct iommu_device *apple_dart_probe_device(struct device *dev) struct apple_dart_stream_map *stream_map; int i; - if (!cfg) + if (!dev_iommu_fwspec_get(dev) || !cfg) return ERR_PTR(-ENODEV); for_each_stream_map(i, cfg, stream_map) - device_link_add( - dev, stream_map->dart->dev, - DL_FLAG_PM_RUNTIME | DL_FLAG_AUTOREMOVE_SUPPLIER); + device_link_add(dev, stream_map->dart->dev, + DL_FLAG_PM_RUNTIME | DL_FLAG_AUTOREMOVE_SUPPLIER | + DL_FLAG_RPM_ACTIVE); return &cfg->stream_maps[0].dart->iommu; } static void apple_dart_release_device(struct device *dev) { + int i, j; + struct apple_dart_stream_map *stream_map; struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev); + for_each_stream_map(j, cfg, stream_map) { + if (stream_map->dart->locked) + for (i = 0; i < stream_map->dart->hw->ttbr_count; ++i) + apple_dart_hw_unmap_locked_ttbr(stream_map, i); + } + kfree(cfg); } @@ -817,6 +1006,8 @@ static int apple_dart_of_xlate(struct device *dev, return -ENOMEM; /* Will be ANDed with DART capabilities */ cfg->supports_bypass = true; + /* Will be ORed with DART capabilities*/ + cfg->locked = false; } dev_iommu_priv_set(dev, cfg); @@ -829,6 +1020,7 @@ static int apple_dart_of_xlate(struct device *dev, } cfg->supports_bypass &= dart->supports_bypass; + cfg->locked |= dart->locked; for (i = 0; i < MAX_DARTS_PER_DEVICE; ++i) { if (cfg->stream_maps[i].dart == dart) { @@ -971,6 +1163,8 @@ static int apple_dart_def_domain_type(struct device *dev) return IOMMU_DOMAIN_IDENTITY; if (!cfg->supports_bypass) return IOMMU_DOMAIN_DMA; + if (cfg->locked) + return IOMMU_DOMAIN_DMA; return 0; } @@ -1003,12 +1197,12 @@ static void apple_dart_get_resv_regions(struct device *dev, static const struct iommu_ops apple_dart_iommu_ops = { .identity_domain = &apple_dart_identity_domain, .blocked_domain = &apple_dart_blocked_domain, + .def_domain_type = apple_dart_def_domain_type, .domain_alloc_paging = apple_dart_domain_alloc_paging, .probe_device = apple_dart_probe_device, .release_device = apple_dart_release_device, .device_group = apple_dart_device_group, .of_xlate = apple_dart_of_xlate, - .def_domain_type = apple_dart_def_domain_type, .get_resv_regions = apple_dart_get_resv_regions, .owner = THIS_MODULE, .default_domain_ops = &(const struct iommu_domain_ops) { @@ -1102,6 +1296,22 @@ static irqreturn_t apple_dart_t8110_irq(int irq, void *dev) return IRQ_HANDLED; } +static irqreturn_t apple_dart_irq(int irq, void *dev) +{ + irqreturn_t ret; + struct apple_dart *dart = dev; + + WARN_ON(pm_runtime_get_sync(dart->dev) < 0); + ret = dart->hw->irq_handler(irq, dev); + pm_runtime_put(dart->dev); + return ret; +} + +static bool apple_dart_is_locked(struct apple_dart *dart) +{ + return !!(readl(dart->regs + dart->hw->lock) & dart->hw->lock_bit); +} + static int apple_dart_probe(struct platform_device *pdev) { int ret; @@ -1109,6 +1319,7 @@ static int apple_dart_probe(struct platform_device *pdev) struct resource *res; struct apple_dart *dart; struct device *dev = &pdev->dev; + u64 dma_range[2]; dart = devm_kzalloc(dev, sizeof(*dart), GFP_KERNEL); if (!dart) @@ -1140,6 +1351,14 @@ static int apple_dart_probe(struct platform_device *pdev) if (ret) return ret; + pm_runtime_get_noresume(dev); + pm_runtime_set_active(dev); + pm_runtime_irq_safe(dev); + + ret = devm_pm_runtime_enable(dev); + if (ret) + goto err_clk_disable; + dart_params[0] = readl(dart->regs + DART_PARAMS1); dart_params[1] = readl(dart->regs + DART_PARAMS2); dart->pgsize = 1 << FIELD_GET(DART_PARAMS1_PAGE_SHIFT, dart_params[0]); @@ -1163,6 +1382,26 @@ static int apple_dart_probe(struct platform_device *pdev) break; } + dart->dma_min = 0; + dart->dma_max = DMA_BIT_MASK(dart->ias); + + ret = of_property_read_u64_array(dev->of_node, "apple,dma-range", dma_range, 2); + if (ret == -EINVAL) { + ret = 0; + } else if (ret) { + goto err_clk_disable; + } else { + dart->dma_min = dma_range[0]; + dart->dma_max = dma_range[0] + dma_range[1] - 1; + if ((dart->dma_min ^ dart->dma_max) & ~DMA_BIT_MASK(dart->ias)) { + dev_err(&pdev->dev, "Invalid DMA range for ias=%d\n", + dart->ias); + goto err_clk_disable; + } + dev_info(&pdev->dev, "Limiting DMA range to %pad..%pad\n", + &dart->dma_min, &dart->dma_max); + } + if (dart->num_streams > DART_MAX_STREAMS) { dev_err(&pdev->dev, "Too many streams (%d > %d)\n", dart->num_streams, DART_MAX_STREAMS); @@ -1170,11 +1409,14 @@ static int apple_dart_probe(struct platform_device *pdev) goto err_clk_disable; } - ret = apple_dart_hw_reset(dart); - if (ret) - goto err_clk_disable; + dart->locked = apple_dart_is_locked(dart); + if (!dart->locked) { + ret = apple_dart_hw_reset(dart); + if (ret) + goto err_clk_disable; + } - ret = request_irq(dart->irq, dart->hw->irq_handler, IRQF_SHARED, + ret = request_irq(dart->irq, apple_dart_irq, IRQF_SHARED, "apple-dart fault handler", dart); if (ret) goto err_clk_disable; @@ -1190,11 +1432,13 @@ static int apple_dart_probe(struct platform_device *pdev) if (ret) goto err_sysfs_remove; + pm_runtime_put(dev); + dev_info( &pdev->dev, - "DART [pagesize %x, %d streams, bypass support: %d, bypass forced: %d, AS %d -> %d] initialized\n", + "DART [pagesize %x, %d streams, bypass support: %d, bypass forced: %d, locked: %d, AS %d -> %d] initialized\n", dart->pgsize, dart->num_streams, dart->supports_bypass, - dart->pgsize > PAGE_SIZE, dart->ias, dart->oas); + dart->pgsize > PAGE_SIZE, dart->locked, dart->ias, dart->oas); return 0; err_sysfs_remove: @@ -1202,6 +1446,7 @@ static int apple_dart_probe(struct platform_device *pdev) err_free_irq: free_irq(dart->irq, dart); err_clk_disable: + pm_runtime_put(dev); clk_bulk_disable_unprepare(dart->num_clks, dart->clks); return ret; @@ -1211,7 +1456,9 @@ static void apple_dart_remove(struct platform_device *pdev) { struct apple_dart *dart = platform_get_drvdata(pdev); - apple_dart_hw_reset(dart); + if (!dart->locked) + apple_dart_hw_reset(dart); + free_irq(dart->irq, dart); iommu_device_unregister(&dart->iommu); @@ -1329,6 +1576,10 @@ static __maybe_unused int apple_dart_suspend(struct device *dev) struct apple_dart *dart = dev_get_drvdata(dev); unsigned int sid, idx; + /* Locked DARTs can't be restored so skip saving their registers. */ + if (dart->locked) + return 0; + for (sid = 0; sid < dart->num_streams; sid++) { dart->save_tcr[sid] = readl(dart->regs + DART_TCR(dart, sid)); for (idx = 0; idx < dart->hw->ttbr_count; idx++) @@ -1345,6 +1596,10 @@ static __maybe_unused int apple_dart_resume(struct device *dev) unsigned int sid, idx; int ret; + /* Locked DARTs can't be restored, and they should not need it */ + if (dart->locked) + return 0; + ret = apple_dart_hw_reset(dart); if (ret) { dev_err(dev, "Failed to reset DART on resume\n"); @@ -1361,7 +1616,7 @@ static __maybe_unused int apple_dart_resume(struct device *dev) return 0; } -static DEFINE_SIMPLE_DEV_PM_OPS(apple_dart_pm_ops, apple_dart_suspend, apple_dart_resume); +static DEFINE_RUNTIME_DEV_PM_OPS(apple_dart_pm_ops, apple_dart_suspend, apple_dart_resume, NULL); static const struct of_device_id apple_dart_of_match[] = { { .compatible = "apple,t8103-dart", .data = &apple_dart_hw_t8103 }, @@ -1377,7 +1632,7 @@ static struct platform_driver apple_dart_driver = { .name = "apple-dart", .of_match_table = apple_dart_of_match, .suppress_bind_attrs = true, - .pm = pm_sleep_ptr(&apple_dart_pm_ops), + .pm = pm_ptr(&apple_dart_pm_ops), }, .probe = apple_dart_probe, .remove = apple_dart_remove, diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-iommufd.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-iommufd.c index 93fdadd07431ab..823461a26659f0 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-iommufd.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-iommufd.c @@ -177,7 +177,9 @@ static int arm_smmu_attach_dev_nested(struct iommu_domain *domain, * config bit here base this off the EATS value in the STE. If the EATS * is set then the VM must generate ATC flushes. */ - state.disable_ats = !nested_domain->enable_ats; + if (FIELD_GET(STRTAB_STE_0_CFG, le64_to_cpu(nested_domain->ste[0])) == + STRTAB_STE_0_CFG_S1_TRANS) + state.disable_ats = !nested_domain->enable_ats; ret = arm_smmu_attach_prepare(&state, domain); if (ret) { mutex_unlock(&arm_smmu_asid_lock); diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c index d2671bfd37981b..b254a94b2003da 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c @@ -38,13 +38,16 @@ enum arm_smmu_test_master_feat { static bool arm_smmu_entry_differs_in_used_bits(const __le64 *entry, const __le64 *used_bits, const __le64 *target, + const __le64 *safe, unsigned int length) { bool differs = false; unsigned int i; for (i = 0; i < length; i++) { - if ((entry[i] & used_bits[i]) != target[i]) + __le64 used = used_bits[i] & ~safe[i]; + + if ((entry[i] & used) != (target[i] & used)) differs = true; } return differs; @@ -56,12 +59,24 @@ arm_smmu_test_writer_record_syncs(struct arm_smmu_entry_writer *writer) struct arm_smmu_test_writer *test_writer = container_of(writer, struct arm_smmu_test_writer, writer); __le64 *entry_used_bits; + __le64 *safe_target; + __le64 *safe_init; entry_used_bits = kunit_kzalloc( test_writer->test, sizeof(*entry_used_bits) * NUM_ENTRY_QWORDS, GFP_KERNEL); KUNIT_ASSERT_NOT_NULL(test_writer->test, entry_used_bits); + safe_target = kunit_kzalloc(test_writer->test, + sizeof(*safe_target) * NUM_ENTRY_QWORDS, + GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test_writer->test, safe_target); + + safe_init = kunit_kzalloc(test_writer->test, + sizeof(*safe_init) * NUM_ENTRY_QWORDS, + GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test_writer->test, safe_init); + pr_debug("STE value is now set to: "); print_hex_dump_debug(" ", DUMP_PREFIX_NONE, 16, 8, test_writer->entry, @@ -79,14 +94,23 @@ arm_smmu_test_writer_record_syncs(struct arm_smmu_entry_writer *writer) * configuration. */ writer->ops->get_used(test_writer->entry, entry_used_bits); + if (writer->ops->get_update_safe) + writer->ops->get_update_safe(test_writer->entry, + test_writer->init_entry, + safe_init); + if (writer->ops->get_update_safe) + writer->ops->get_update_safe(test_writer->entry, + test_writer->target_entry, + safe_target); KUNIT_EXPECT_FALSE( test_writer->test, arm_smmu_entry_differs_in_used_bits( test_writer->entry, entry_used_bits, - test_writer->init_entry, NUM_ENTRY_QWORDS) && + test_writer->init_entry, safe_init, + NUM_ENTRY_QWORDS) && arm_smmu_entry_differs_in_used_bits( test_writer->entry, entry_used_bits, - test_writer->target_entry, + test_writer->target_entry, safe_target, NUM_ENTRY_QWORDS)); } } @@ -106,6 +130,7 @@ arm_smmu_v3_test_debug_print_used_bits(struct arm_smmu_entry_writer *writer, static const struct arm_smmu_entry_writer_ops test_ste_ops = { .sync = arm_smmu_test_writer_record_syncs, .get_used = arm_smmu_get_ste_used, + .get_update_safe = arm_smmu_get_ste_update_safe, }; static const struct arm_smmu_entry_writer_ops test_cd_ops = { diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index d16d35c78c0685..d55b8e39b8e3c3 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -487,20 +487,26 @@ static void arm_smmu_cmdq_skip_err(struct arm_smmu_device *smmu) */ static void arm_smmu_cmdq_shared_lock(struct arm_smmu_cmdq *cmdq) { - int val; - /* - * We can try to avoid the cmpxchg() loop by simply incrementing the - * lock counter. When held in exclusive state, the lock counter is set - * to INT_MIN so these increments won't hurt as the value will remain - * negative. + * When held in exclusive state, the lock counter is set to INT_MIN + * so these increments won't hurt as the value will remain negative. + * The increment will also signal the exclusive locker that there are + * shared waiters. */ if (atomic_fetch_inc_relaxed(&cmdq->lock) >= 0) return; - do { - val = atomic_cond_read_relaxed(&cmdq->lock, VAL >= 0); - } while (atomic_cmpxchg_relaxed(&cmdq->lock, val, val + 1) != val); + /* + * Someone else is holding the lock in exclusive state, so wait + * for them to finish. Since we already incremented the lock counter, + * no exclusive lock can be acquired until we finish. We don't need + * the return value since we only care that the exclusive lock is + * released (i.e. the lock counter is non-negative). + * Once the exclusive locker releases the lock, the sign bit will + * be cleared and our increment will make the lock counter positive, + * allowing us to proceed. + */ + atomic_cond_read_relaxed(&cmdq->lock, VAL > 0); } static void arm_smmu_cmdq_shared_unlock(struct arm_smmu_cmdq *cmdq) @@ -527,9 +533,14 @@ static bool arm_smmu_cmdq_shared_tryunlock(struct arm_smmu_cmdq *cmdq) __ret; \ }) +/* + * Only clear the sign bit when releasing the exclusive lock this will + * allow any shared_lock() waiters to proceed without the possibility + * of entering the exclusive lock in a tight loop. + */ #define arm_smmu_cmdq_exclusive_unlock_irqrestore(cmdq, flags) \ ({ \ - atomic_set_release(&cmdq->lock, 0); \ + atomic_fetch_andnot_release(INT_MIN, &cmdq->lock); \ local_irq_restore(flags); \ }) @@ -1082,6 +1093,49 @@ void arm_smmu_get_ste_used(const __le64 *ent, __le64 *used_bits) } EXPORT_SYMBOL_IF_KUNIT(arm_smmu_get_ste_used); +VISIBLE_IF_KUNIT +void arm_smmu_get_ste_update_safe(const __le64 *cur, const __le64 *target, + __le64 *safe_bits) +{ + const __le64 eats_s1chk = + FIELD_PREP(STRTAB_STE_1_EATS, STRTAB_STE_1_EATS_S1CHK); + const __le64 eats_trans = + FIELD_PREP(STRTAB_STE_1_EATS, STRTAB_STE_1_EATS_TRANS); + + /* + * When an STE changes EATS_TRANS, the sequencing code in the attach + * logic already will have the PCI cap for ATS disabled. Thus at this + * moment we can expect that the device will not generate ATS queries + * and so we don't care about the sequencing of EATS. The purpose of + * EATS_TRANS is to protect the system from hostile untrusted devices + * that issue ATS when the PCI config space is disabled. However, if + * EATS_TRANS is being changed, then we must have already trusted the + * device as the EATS_TRANS security block is being disabled. + * + * Note: now the EATS_TRANS update is moved to the first entry_set(). + * Changing S2S and EATS might transiently result in S2S=1 and EATS=1 + * which is a bad STE (see "5.2 Stream Table Entry"). In such a case, + * we can't do a hitless update. Also, it should not be added to the + * safe bits with STRTAB_STE_1_EATS_S1CHK, because EATS=0b11 would be + * effectively an errant 0b00 configuration. + */ + if (!((cur[1] | target[1]) & cpu_to_le64(eats_s1chk)) && + !((cur[2] | target[2]) & cpu_to_le64(STRTAB_STE_2_S2S))) + safe_bits[1] |= cpu_to_le64(eats_trans); + + /* + * MEV does not meaningfully impact the operation of the HW, it only + * changes how many fault events are generated, thus we can relax it + * when computing the ordering. The spec notes the device can act like + * MEV=1 anyhow: + * + * Note: Software must expect, and be able to deal with, coalesced + * fault records even when MEV == 0. + */ + safe_bits[1] |= cpu_to_le64(STRTAB_STE_1_MEV); +} +EXPORT_SYMBOL_IF_KUNIT(arm_smmu_get_ste_update_safe); + /* * Figure out if we can do a hitless update of entry to become target. Returns a * bit mask where 1 indicates that qword needs to be set disruptively. @@ -1094,13 +1148,22 @@ static u8 arm_smmu_entry_qword_diff(struct arm_smmu_entry_writer *writer, { __le64 target_used[NUM_ENTRY_QWORDS] = {}; __le64 cur_used[NUM_ENTRY_QWORDS] = {}; + __le64 safe[NUM_ENTRY_QWORDS] = {}; u8 used_qword_diff = 0; unsigned int i; writer->ops->get_used(entry, cur_used); writer->ops->get_used(target, target_used); + if (writer->ops->get_update_safe) + writer->ops->get_update_safe(entry, target, safe); for (i = 0; i != NUM_ENTRY_QWORDS; i++) { + /* + * Safe is only used for bits that are used by both entries, + * otherwise it is sequenced according to the unused entry. + */ + safe[i] &= target_used[i] & cur_used[i]; + /* * Check that masks are up to date, the make functions are not * allowed to set a bit to 1 if the used function doesn't say it @@ -1109,6 +1172,7 @@ static u8 arm_smmu_entry_qword_diff(struct arm_smmu_entry_writer *writer, WARN_ON_ONCE(target[i] & ~target_used[i]); /* Bits can change because they are not currently being used */ + cur_used[i] &= ~safe[i]; unused_update[i] = (entry[i] & cur_used[i]) | (target[i] & ~cur_used[i]); /* @@ -1121,7 +1185,7 @@ static u8 arm_smmu_entry_qword_diff(struct arm_smmu_entry_writer *writer, return used_qword_diff; } -static bool entry_set(struct arm_smmu_entry_writer *writer, __le64 *entry, +static void entry_set(struct arm_smmu_entry_writer *writer, __le64 *entry, const __le64 *target, unsigned int start, unsigned int len) { @@ -1137,7 +1201,6 @@ static bool entry_set(struct arm_smmu_entry_writer *writer, __le64 *entry, if (changed) writer->ops->sync(writer); - return changed; } /* @@ -1207,12 +1270,9 @@ void arm_smmu_write_entry(struct arm_smmu_entry_writer *writer, __le64 *entry, entry_set(writer, entry, target, 0, 1); } else { /* - * No inuse bit changed. Sanity check that all unused bits are 0 - * in the entry. The target was already sanity checked by - * compute_qword_diff(). + * No inuse bit changed, though safe bits may have changed. */ - WARN_ON_ONCE( - entry_set(writer, entry, target, 0, NUM_ENTRY_QWORDS)); + entry_set(writer, entry, target, 0, NUM_ENTRY_QWORDS); } } EXPORT_SYMBOL_IF_KUNIT(arm_smmu_write_entry); @@ -1543,6 +1603,7 @@ static void arm_smmu_ste_writer_sync_entry(struct arm_smmu_entry_writer *writer) static const struct arm_smmu_entry_writer_ops arm_smmu_ste_writer_ops = { .sync = arm_smmu_ste_writer_sync_entry, .get_used = arm_smmu_get_ste_used, + .get_update_safe = arm_smmu_get_ste_update_safe, }; static void arm_smmu_write_ste(struct arm_smmu_master *master, u32 sid, diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h index ae23aacc384029..287e223c054d1a 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h @@ -900,6 +900,8 @@ struct arm_smmu_entry_writer { struct arm_smmu_entry_writer_ops { void (*get_used)(const __le64 *entry, __le64 *used); + void (*get_update_safe)(const __le64 *cur, const __le64 *target, + __le64 *safe_bits); void (*sync)(struct arm_smmu_entry_writer *writer); }; @@ -911,6 +913,8 @@ void arm_smmu_make_s2_domain_ste(struct arm_smmu_ste *target, #if IS_ENABLED(CONFIG_KUNIT) void arm_smmu_get_ste_used(const __le64 *ent, __le64 *used_bits); +void arm_smmu_get_ste_update_safe(const __le64 *cur, const __le64 *target, + __le64 *safe_bits); void arm_smmu_write_entry(struct arm_smmu_entry_writer *writer, __le64 *cur, const __le64 *target); void arm_smmu_get_cd_used(const __le64 *ent, __le64 *used_bits); diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu-impl.c b/drivers/iommu/arm/arm-smmu/arm-smmu-impl.c index db9b9a8e139c87..4565a58bb213f4 100644 --- a/drivers/iommu/arm/arm-smmu/arm-smmu-impl.c +++ b/drivers/iommu/arm/arm-smmu/arm-smmu-impl.c @@ -228,3 +228,17 @@ struct arm_smmu_device *arm_smmu_impl_init(struct arm_smmu_device *smmu) return smmu; } + +int __init arm_smmu_impl_module_init(void) +{ + if (IS_ENABLED(CONFIG_ARM_SMMU_QCOM)) + return qcom_smmu_module_init(); + + return 0; +} + +void __exit arm_smmu_impl_module_exit(void) +{ + if (IS_ENABLED(CONFIG_ARM_SMMU_QCOM)) + qcom_smmu_module_exit(); +} diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c b/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c index 573085349df34f..22906d2c9a2db2 100644 --- a/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c +++ b/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c @@ -774,10 +774,6 @@ struct arm_smmu_device *qcom_smmu_impl_init(struct arm_smmu_device *smmu) { const struct device_node *np = smmu->dev->of_node; const struct of_device_id *match; - static u8 tbu_registered; - - if (!tbu_registered++) - platform_driver_register(&qcom_smmu_tbu_driver); #ifdef CONFIG_ACPI if (np == NULL) { @@ -802,3 +798,13 @@ struct arm_smmu_device *qcom_smmu_impl_init(struct arm_smmu_device *smmu) return smmu; } + +int __init qcom_smmu_module_init(void) +{ + return platform_driver_register(&qcom_smmu_tbu_driver); +} + +void __exit qcom_smmu_module_exit(void) +{ + platform_driver_unregister(&qcom_smmu_tbu_driver); +} diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu.c b/drivers/iommu/arm/arm-smmu/arm-smmu.c index 5e690cf85ec969..1e218fbea35a02 100644 --- a/drivers/iommu/arm/arm-smmu/arm-smmu.c +++ b/drivers/iommu/arm/arm-smmu/arm-smmu.c @@ -2365,7 +2365,29 @@ static struct platform_driver arm_smmu_driver = { .remove = arm_smmu_device_remove, .shutdown = arm_smmu_device_shutdown, }; -module_platform_driver(arm_smmu_driver); + +static int __init arm_smmu_init(void) +{ + int ret; + + ret = platform_driver_register(&arm_smmu_driver); + if (ret) + return ret; + + ret = arm_smmu_impl_module_init(); + if (ret) + platform_driver_unregister(&arm_smmu_driver); + + return ret; +} +module_init(arm_smmu_init); + +static void __exit arm_smmu_exit(void) +{ + arm_smmu_impl_module_exit(); + platform_driver_unregister(&arm_smmu_driver); +} +module_exit(arm_smmu_exit); MODULE_DESCRIPTION("IOMMU API for ARM architected SMMU implementations"); MODULE_AUTHOR("Will Deacon "); diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu.h b/drivers/iommu/arm/arm-smmu/arm-smmu.h index 2dbf3243b5ad2d..26d2e33cd328b8 100644 --- a/drivers/iommu/arm/arm-smmu/arm-smmu.h +++ b/drivers/iommu/arm/arm-smmu/arm-smmu.h @@ -540,6 +540,11 @@ struct arm_smmu_device *arm_smmu_impl_init(struct arm_smmu_device *smmu); struct arm_smmu_device *nvidia_smmu_impl_init(struct arm_smmu_device *smmu); struct arm_smmu_device *qcom_smmu_impl_init(struct arm_smmu_device *smmu); +int __init arm_smmu_impl_module_init(void); +void __exit arm_smmu_impl_module_exit(void); +int __init qcom_smmu_module_init(void); +void __exit qcom_smmu_module_exit(void); + void arm_smmu_write_context_bank(struct arm_smmu_device *smmu, int idx); int arm_mmu500_reset(struct arm_smmu_device *smmu); diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index c92088855450ad..8ef083185fc726 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -572,8 +572,13 @@ static int iova_reserve_iommu_regions(struct device *dev, if (region->type == IOMMU_RESV_SW_MSI) continue; - lo = iova_pfn(iovad, region->start); - hi = iova_pfn(iovad, region->start + region->length - 1); + if (region->type == IOMMU_RESV_TRANSLATED) { + lo = iova_pfn(iovad, region->dva); + hi = iova_pfn(iovad, region->dva + region->length - 1); + } else { + lo = iova_pfn(iovad, region->start); + hi = iova_pfn(iovad, region->start + region->length - 1); + } reserve_iova(iovad, lo, hi); if (region->type == IOMMU_RESV_MSI) diff --git a/drivers/iommu/generic_pt/fmt/amdv1.h b/drivers/iommu/generic_pt/fmt/amdv1.h index aa8e1a8ec95fde..8d11b08291d733 100644 --- a/drivers/iommu/generic_pt/fmt/amdv1.h +++ b/drivers/iommu/generic_pt/fmt/amdv1.h @@ -191,7 +191,7 @@ static inline enum pt_entry_type amdv1pt_load_entry_raw(struct pt_state *pts) } #define pt_load_entry_raw amdv1pt_load_entry_raw -static inline void +static __always_inline void amdv1pt_install_leaf_entry(struct pt_state *pts, pt_oaddr_t oa, unsigned int oasz_lg2, const struct pt_write_attrs *attrs) @@ -354,7 +354,8 @@ static inline int amdv1pt_iommu_set_prot(struct pt_common *common, * Ideally we'd have an IOMMU_ENCRYPTED flag set by higher levels to * control this. For now if the tables use sme_set then so do the ptes. */ - if (pt_feature(common, PT_FEAT_AMDV1_ENCRYPT_TABLES)) + if (pt_feature(common, PT_FEAT_AMDV1_ENCRYPT_TABLES) && + !(iommu_prot & IOMMU_MMIO)) pte = __sme_set(pte); attrs->descriptor_bits = pte; diff --git a/drivers/iommu/generic_pt/fmt/x86_64.h b/drivers/iommu/generic_pt/fmt/x86_64.h index 210748d9d6e8aa..ed9a47cbb6e022 100644 --- a/drivers/iommu/generic_pt/fmt/x86_64.h +++ b/drivers/iommu/generic_pt/fmt/x86_64.h @@ -227,7 +227,8 @@ static inline int x86_64_pt_iommu_set_prot(struct pt_common *common, * Ideally we'd have an IOMMU_ENCRYPTED flag set by higher levels to * control this. For now if the tables use sme_set then so do the ptes. */ - if (pt_feature(common, PT_FEAT_X86_64_AMD_ENCRYPT_TABLES)) + if (pt_feature(common, PT_FEAT_X86_64_AMD_ENCRYPT_TABLES) && + !(iommu_prot & IOMMU_MMIO)) pte = __sme_set(pte); attrs->descriptor_bits = pte; diff --git a/drivers/iommu/generic_pt/iommu_pt.h b/drivers/iommu/generic_pt/iommu_pt.h index d575f3ba9d3416..7e7a6e7abdeed1 100644 --- a/drivers/iommu/generic_pt/iommu_pt.h +++ b/drivers/iommu/generic_pt/iommu_pt.h @@ -58,10 +58,9 @@ static void gather_range_pages(struct iommu_iotlb_gather *iotlb_gather, * Note that the sync frees the gather's free list, so we must * not have any pages on that list that are covered by iova/len */ - } else if (pt_feature(common, PT_FEAT_FLUSH_RANGE)) { - iommu_iotlb_gather_add_range(iotlb_gather, iova, len); } + iommu_iotlb_gather_add_range(iotlb_gather, iova, len); iommu_pages_list_splice(free_list, &iotlb_gather->freelist); } @@ -1058,7 +1057,7 @@ size_t DOMAIN_NS(unmap_pages)(struct iommu_domain *domain, unsigned long iova, pt_walk_range(&range, __unmap_range, &unmap); - gather_range_pages(iotlb_gather, iommu_table, iova, len, + gather_range_pages(iotlb_gather, iommu_table, iova, unmap.unmapped, &unmap.free_list); return unmap.unmapped; diff --git a/drivers/iommu/intel/cache.c b/drivers/iommu/intel/cache.c index 265e7290256b57..385ae5cfb30d4a 100644 --- a/drivers/iommu/intel/cache.c +++ b/drivers/iommu/intel/cache.c @@ -363,6 +363,13 @@ static void qi_batch_add_pasid_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 qi_batch_increment_index(iommu, batch); } +static bool intel_domain_use_piotlb(struct dmar_domain *domain) +{ + return domain->domain.type == IOMMU_DOMAIN_SVA || + domain->domain.type == IOMMU_DOMAIN_NESTED || + intel_domain_is_fs_paging(domain); +} + static void cache_tag_flush_iotlb(struct dmar_domain *domain, struct cache_tag *tag, unsigned long addr, unsigned long pages, unsigned long mask, int ih) @@ -370,7 +377,7 @@ static void cache_tag_flush_iotlb(struct dmar_domain *domain, struct cache_tag * struct intel_iommu *iommu = tag->iommu; u64 type = DMA_TLB_PSI_FLUSH; - if (intel_domain_is_fs_paging(domain)) { + if (intel_domain_use_piotlb(domain)) { qi_batch_add_piotlb(iommu, tag->domain_id, tag->pasid, addr, pages, ih, domain->qi_batch); return; diff --git a/drivers/iommu/intel/dmar.c b/drivers/iommu/intel/dmar.c index ec975c73cfe6cc..6938800e98845e 100644 --- a/drivers/iommu/intel/dmar.c +++ b/drivers/iommu/intel/dmar.c @@ -1314,7 +1314,6 @@ static int qi_check_fault(struct intel_iommu *iommu, int index, int wait_index) if (fault & DMA_FSTS_ITE) { head = readl(iommu->reg + DMAR_IQH_REG); head = ((head >> shift) - 1 + QI_LENGTH) % QI_LENGTH; - head |= 1; tail = readl(iommu->reg + DMAR_IQT_REG); tail = ((tail >> shift) - 1 + QI_LENGTH) % QI_LENGTH; @@ -1331,7 +1330,7 @@ static int qi_check_fault(struct intel_iommu *iommu, int index, int wait_index) do { if (qi->desc_status[head] == QI_IN_USE) qi->desc_status[head] = QI_ABORT; - head = (head - 2 + QI_LENGTH) % QI_LENGTH; + head = (head - 1 + QI_LENGTH) % QI_LENGTH; } while (head != tail); /* diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index 134302fbcd9263..705828b06e329c 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -1240,22 +1240,22 @@ static void domain_context_clear_one(struct device_domain_info *info, u8 bus, u8 } did = context_domain_id(context); - context_clear_entry(context); + context_clear_present(context); __iommu_flush_cache(iommu, context, sizeof(*context)); spin_unlock(&iommu->lock); intel_context_flush_no_pasid(info, context, did); + context_clear_entry(context); + __iommu_flush_cache(iommu, context, sizeof(*context)); } int __domain_setup_first_level(struct intel_iommu *iommu, struct device *dev, ioasid_t pasid, u16 did, phys_addr_t fsptptr, int flags, struct iommu_domain *old) { - if (!old) - return intel_pasid_setup_first_level(iommu, dev, fsptptr, pasid, - did, flags); - return intel_pasid_replace_first_level(iommu, dev, fsptptr, pasid, did, - iommu_domain_did(old, iommu), - flags); + if (old) + intel_pasid_tear_down_entry(iommu, dev, pasid, false); + + return intel_pasid_setup_first_level(iommu, dev, fsptptr, pasid, did, flags); } static int domain_setup_second_level(struct intel_iommu *iommu, @@ -1263,23 +1263,20 @@ static int domain_setup_second_level(struct intel_iommu *iommu, struct device *dev, ioasid_t pasid, struct iommu_domain *old) { - if (!old) - return intel_pasid_setup_second_level(iommu, domain, - dev, pasid); - return intel_pasid_replace_second_level(iommu, domain, dev, - iommu_domain_did(old, iommu), - pasid); + if (old) + intel_pasid_tear_down_entry(iommu, dev, pasid, false); + + return intel_pasid_setup_second_level(iommu, domain, dev, pasid); } static int domain_setup_passthrough(struct intel_iommu *iommu, struct device *dev, ioasid_t pasid, struct iommu_domain *old) { - if (!old) - return intel_pasid_setup_pass_through(iommu, dev, pasid); - return intel_pasid_replace_pass_through(iommu, dev, - iommu_domain_did(old, iommu), - pasid); + if (old) + intel_pasid_tear_down_entry(iommu, dev, pasid, false); + + return intel_pasid_setup_pass_through(iommu, dev, pasid); } static int domain_setup_first_level(struct intel_iommu *iommu, diff --git a/drivers/iommu/intel/iommu.h b/drivers/iommu/intel/iommu.h index 25c5e22096d44e..599913fb65d59e 100644 --- a/drivers/iommu/intel/iommu.h +++ b/drivers/iommu/intel/iommu.h @@ -900,7 +900,26 @@ static inline int pfn_level_offset(u64 pfn, int level) static inline void context_set_present(struct context_entry *context) { - context->lo |= 1; + u64 val; + + dma_wmb(); + val = READ_ONCE(context->lo) | 1; + WRITE_ONCE(context->lo, val); +} + +/* + * Clear the Present (P) bit (bit 0) of a context table entry. This initiates + * the transition of the entry's ownership from hardware to software. The + * caller is responsible for fulfilling the invalidation handshake recommended + * by the VT-d spec, Section 6.5.3.3 (Guidance to Software for Invalidations). + */ +static inline void context_clear_present(struct context_entry *context) +{ + u64 val; + + val = READ_ONCE(context->lo) & GENMASK_ULL(63, 1); + WRITE_ONCE(context->lo, val); + dma_wmb(); } static inline void context_set_fault_enable(struct context_entry *context) diff --git a/drivers/iommu/intel/nested.c b/drivers/iommu/intel/nested.c index a3fb8c193ca647..e9a440e9c960b2 100644 --- a/drivers/iommu/intel/nested.c +++ b/drivers/iommu/intel/nested.c @@ -136,11 +136,10 @@ static int domain_setup_nested(struct intel_iommu *iommu, struct device *dev, ioasid_t pasid, struct iommu_domain *old) { - if (!old) - return intel_pasid_setup_nested(iommu, dev, pasid, domain); - return intel_pasid_replace_nested(iommu, dev, pasid, - iommu_domain_did(old, iommu), - domain); + if (old) + intel_pasid_tear_down_entry(iommu, dev, pasid, false); + + return intel_pasid_setup_nested(iommu, dev, pasid, domain); } static int intel_nested_set_dev_pasid(struct iommu_domain *domain, diff --git a/drivers/iommu/intel/pasid.c b/drivers/iommu/intel/pasid.c index 3e2255057079c5..b63a71904cfb8b 100644 --- a/drivers/iommu/intel/pasid.c +++ b/drivers/iommu/intel/pasid.c @@ -153,6 +153,9 @@ static struct pasid_entry *intel_pasid_get_entry(struct device *dev, u32 pasid) if (!entries) return NULL; + if (!ecap_coherent(info->iommu->ecap)) + clflush_cache_range(entries, VTD_PAGE_SIZE); + /* * The pasid directory table entry won't be freed after * allocation. No worry about the race with free and @@ -165,10 +168,8 @@ static struct pasid_entry *intel_pasid_get_entry(struct device *dev, u32 pasid) iommu_free_pages(entries); goto retry; } - if (!ecap_coherent(info->iommu->ecap)) { - clflush_cache_range(entries, VTD_PAGE_SIZE); + if (!ecap_coherent(info->iommu->ecap)) clflush_cache_range(&dir[dir_index].val, sizeof(*dir)); - } } return &entries[index]; @@ -218,7 +219,7 @@ devtlb_invalidation_with_pasid(struct intel_iommu *iommu, if (!info || !info->ats_enabled) return; - if (pci_dev_is_disconnected(to_pci_dev(dev))) + if (!pci_device_is_present(to_pci_dev(dev))) return; sid = PCI_DEVID(info->bus, info->devfn); @@ -272,7 +273,7 @@ void intel_pasid_tear_down_entry(struct intel_iommu *iommu, struct device *dev, did = pasid_get_domain_id(pte); pgtt = pasid_pte_get_pgtt(pte); - intel_pasid_clear_entry(dev, pasid, fault_ignore); + pasid_clear_present(pte); spin_unlock(&iommu->lock); if (!ecap_coherent(iommu->ecap)) @@ -286,6 +287,10 @@ void intel_pasid_tear_down_entry(struct intel_iommu *iommu, struct device *dev, iommu->flush.flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH); devtlb_invalidation_with_pasid(iommu, dev, pasid); + intel_pasid_clear_entry(dev, pasid, fault_ignore); + if (!ecap_coherent(iommu->ecap)) + clflush_cache_range(pte, sizeof(*pte)); + if (!fault_ignore) intel_iommu_drain_pasid_prq(dev, pasid); } @@ -412,50 +417,6 @@ int intel_pasid_setup_first_level(struct intel_iommu *iommu, struct device *dev, return 0; } -int intel_pasid_replace_first_level(struct intel_iommu *iommu, - struct device *dev, phys_addr_t fsptptr, - u32 pasid, u16 did, u16 old_did, - int flags) -{ - struct pasid_entry *pte, new_pte; - - if (!ecap_flts(iommu->ecap)) { - pr_err("No first level translation support on %s\n", - iommu->name); - return -EINVAL; - } - - if ((flags & PASID_FLAG_FL5LP) && !cap_fl5lp_support(iommu->cap)) { - pr_err("No 5-level paging support for first-level on %s\n", - iommu->name); - return -EINVAL; - } - - pasid_pte_config_first_level(iommu, &new_pte, fsptptr, did, flags); - - spin_lock(&iommu->lock); - pte = intel_pasid_get_entry(dev, pasid); - if (!pte) { - spin_unlock(&iommu->lock); - return -ENODEV; - } - - if (!pasid_pte_is_present(pte)) { - spin_unlock(&iommu->lock); - return -EINVAL; - } - - WARN_ON(old_did != pasid_get_domain_id(pte)); - - *pte = new_pte; - spin_unlock(&iommu->lock); - - intel_pasid_flush_present(iommu, dev, pasid, old_did, pte); - intel_iommu_drain_pasid_prq(dev, pasid); - - return 0; -} - /* * Set up the scalable mode pasid entry for second only translation type. */ @@ -522,51 +483,6 @@ int intel_pasid_setup_second_level(struct intel_iommu *iommu, return 0; } -int intel_pasid_replace_second_level(struct intel_iommu *iommu, - struct dmar_domain *domain, - struct device *dev, u16 old_did, - u32 pasid) -{ - struct pasid_entry *pte, new_pte; - u16 did; - - /* - * If hardware advertises no support for second level - * translation, return directly. - */ - if (!ecap_slts(iommu->ecap)) { - pr_err("No second level translation support on %s\n", - iommu->name); - return -EINVAL; - } - - did = domain_id_iommu(domain, iommu); - - pasid_pte_config_second_level(iommu, &new_pte, domain, did); - - spin_lock(&iommu->lock); - pte = intel_pasid_get_entry(dev, pasid); - if (!pte) { - spin_unlock(&iommu->lock); - return -ENODEV; - } - - if (!pasid_pte_is_present(pte)) { - spin_unlock(&iommu->lock); - return -EINVAL; - } - - WARN_ON(old_did != pasid_get_domain_id(pte)); - - *pte = new_pte; - spin_unlock(&iommu->lock); - - intel_pasid_flush_present(iommu, dev, pasid, old_did, pte); - intel_iommu_drain_pasid_prq(dev, pasid); - - return 0; -} - /* * Set up dirty tracking on a second only or nested translation type. */ @@ -679,38 +595,6 @@ int intel_pasid_setup_pass_through(struct intel_iommu *iommu, return 0; } -int intel_pasid_replace_pass_through(struct intel_iommu *iommu, - struct device *dev, u16 old_did, - u32 pasid) -{ - struct pasid_entry *pte, new_pte; - u16 did = FLPT_DEFAULT_DID; - - pasid_pte_config_pass_through(iommu, &new_pte, did); - - spin_lock(&iommu->lock); - pte = intel_pasid_get_entry(dev, pasid); - if (!pte) { - spin_unlock(&iommu->lock); - return -ENODEV; - } - - if (!pasid_pte_is_present(pte)) { - spin_unlock(&iommu->lock); - return -EINVAL; - } - - WARN_ON(old_did != pasid_get_domain_id(pte)); - - *pte = new_pte; - spin_unlock(&iommu->lock); - - intel_pasid_flush_present(iommu, dev, pasid, old_did, pte); - intel_iommu_drain_pasid_prq(dev, pasid); - - return 0; -} - /* * Set the page snoop control for a pasid entry which has been set up. */ @@ -844,69 +728,6 @@ int intel_pasid_setup_nested(struct intel_iommu *iommu, struct device *dev, return 0; } -int intel_pasid_replace_nested(struct intel_iommu *iommu, - struct device *dev, u32 pasid, - u16 old_did, struct dmar_domain *domain) -{ - struct iommu_hwpt_vtd_s1 *s1_cfg = &domain->s1_cfg; - struct dmar_domain *s2_domain = domain->s2_domain; - u16 did = domain_id_iommu(domain, iommu); - struct pasid_entry *pte, new_pte; - - /* Address width should match the address width supported by hardware */ - switch (s1_cfg->addr_width) { - case ADDR_WIDTH_4LEVEL: - break; - case ADDR_WIDTH_5LEVEL: - if (!cap_fl5lp_support(iommu->cap)) { - dev_err_ratelimited(dev, - "5-level paging not supported\n"); - return -EINVAL; - } - break; - default: - dev_err_ratelimited(dev, "Invalid stage-1 address width %d\n", - s1_cfg->addr_width); - return -EINVAL; - } - - if ((s1_cfg->flags & IOMMU_VTD_S1_SRE) && !ecap_srs(iommu->ecap)) { - pr_err_ratelimited("No supervisor request support on %s\n", - iommu->name); - return -EINVAL; - } - - if ((s1_cfg->flags & IOMMU_VTD_S1_EAFE) && !ecap_eafs(iommu->ecap)) { - pr_err_ratelimited("No extended access flag support on %s\n", - iommu->name); - return -EINVAL; - } - - pasid_pte_config_nestd(iommu, &new_pte, s1_cfg, s2_domain, did); - - spin_lock(&iommu->lock); - pte = intel_pasid_get_entry(dev, pasid); - if (!pte) { - spin_unlock(&iommu->lock); - return -ENODEV; - } - - if (!pasid_pte_is_present(pte)) { - spin_unlock(&iommu->lock); - return -EINVAL; - } - - WARN_ON(old_did != pasid_get_domain_id(pte)); - - *pte = new_pte; - spin_unlock(&iommu->lock); - - intel_pasid_flush_present(iommu, dev, pasid, old_did, pte); - intel_iommu_drain_pasid_prq(dev, pasid); - - return 0; -} - /* * Interfaces to setup or teardown a pasid table to the scalable-mode * context table entry: @@ -1019,7 +840,7 @@ static int device_pasid_table_setup(struct device *dev, u8 bus, u8 devfn) } if (context_copied(iommu, bus, devfn)) { - context_clear_entry(context); + context_clear_present(context); __iommu_flush_cache(iommu, context, sizeof(*context)); /* @@ -1039,6 +860,9 @@ static int device_pasid_table_setup(struct device *dev, u8 bus, u8 devfn) iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH); devtlb_invalidation_with_pasid(iommu, dev, IOMMU_NO_PASID); + context_clear_entry(context); + __iommu_flush_cache(iommu, context, sizeof(*context)); + /* * At this point, the device is supposed to finish reset at * its driver probe stage, so no in-flight DMA will exist, @@ -1102,6 +926,14 @@ static void __context_flush_dev_iotlb(struct device_domain_info *info) if (!info->ats_enabled) return; + /* + * Skip dev-IOTLB flush for inaccessible PCIe devices to prevent the + * Intel IOMMU from waiting indefinitely for an ATS invalidation that + * cannot complete. + */ + if (!pci_device_is_present(to_pci_dev(info->dev))) + return; + qi_flush_dev_iotlb(info->iommu, PCI_DEVID(info->bus, info->devfn), info->pfsid, info->ats_qdep, 0, MAX_AGAW_PFN_WIDTH); diff --git a/drivers/iommu/intel/pasid.h b/drivers/iommu/intel/pasid.h index 3809793e0259f1..48d3bb6b68dea0 100644 --- a/drivers/iommu/intel/pasid.h +++ b/drivers/iommu/intel/pasid.h @@ -234,9 +234,23 @@ static inline void pasid_set_wpe(struct pasid_entry *pe) */ static inline void pasid_set_present(struct pasid_entry *pe) { + dma_wmb(); pasid_set_bits(&pe->val[0], 1 << 0, 1); } +/* + * Clear the Present (P) bit (bit 0) of a scalable-mode PASID table entry. + * This initiates the transition of the entry's ownership from hardware + * to software. The caller is responsible for fulfilling the invalidation + * handshake recommended by the VT-d spec, Section 6.5.3.3 (Guidance to + * Software for Invalidations). + */ +static inline void pasid_clear_present(struct pasid_entry *pe) +{ + pasid_set_bits(&pe->val[0], 1 << 0, 0); + dma_wmb(); +} + /* * Setup Page Walk Snoop bit (Bit 87) of a scalable mode PASID * entry. @@ -302,20 +316,6 @@ int intel_pasid_setup_pass_through(struct intel_iommu *iommu, struct device *dev, u32 pasid); int intel_pasid_setup_nested(struct intel_iommu *iommu, struct device *dev, u32 pasid, struct dmar_domain *domain); -int intel_pasid_replace_first_level(struct intel_iommu *iommu, - struct device *dev, phys_addr_t fsptptr, - u32 pasid, u16 did, u16 old_did, int flags); -int intel_pasid_replace_second_level(struct intel_iommu *iommu, - struct dmar_domain *domain, - struct device *dev, u16 old_did, - u32 pasid); -int intel_pasid_replace_pass_through(struct intel_iommu *iommu, - struct device *dev, u16 old_did, - u32 pasid); -int intel_pasid_replace_nested(struct intel_iommu *iommu, - struct device *dev, u32 pasid, - u16 old_did, struct dmar_domain *domain); - void intel_pasid_tear_down_entry(struct intel_iommu *iommu, struct device *dev, u32 pasid, bool fault_ignore); diff --git a/drivers/iommu/intel/svm.c b/drivers/iommu/intel/svm.c index 71de7947971f82..b1389df36ef825 100644 --- a/drivers/iommu/intel/svm.c +++ b/drivers/iommu/intel/svm.c @@ -164,9 +164,12 @@ static int intel_svm_set_dev_pasid(struct iommu_domain *domain, if (IS_ERR(dev_pasid)) return PTR_ERR(dev_pasid); - ret = iopf_for_domain_replace(domain, old, dev); - if (ret) - goto out_remove_dev_pasid; + /* SVA with non-IOMMU/PRI IOPF handling is allowed. */ + if (info->pri_supported) { + ret = iopf_for_domain_replace(domain, old, dev); + if (ret) + goto out_remove_dev_pasid; + } /* Setup the pasid table: */ sflags = cpu_feature_enabled(X86_FEATURE_LA57) ? PASID_FLAG_FL5LP : 0; @@ -181,7 +184,8 @@ static int intel_svm_set_dev_pasid(struct iommu_domain *domain, return 0; out_unwind_iopf: - iopf_for_domain_replace(old, domain, dev); + if (info->pri_supported) + iopf_for_domain_replace(old, domain, dev); out_remove_dev_pasid: domain_remove_dev_pasid(domain, dev, pasid); return ret; diff --git a/drivers/iommu/io-pgtable-dart.c b/drivers/iommu/io-pgtable-dart.c index 54d287cc0dd1b8..2334a2ea04bb3a 100644 --- a/drivers/iommu/io-pgtable-dart.c +++ b/drivers/iommu/io-pgtable-dart.c @@ -435,7 +435,7 @@ apple_dart_alloc_pgtable(struct io_pgtable_cfg *cfg, void *cookie) iommu_alloc_pages_sz(GFP_KERNEL, DART_GRANULE(data)); if (!data->pgd[i]) goto out_free_data; - cfg->apple_dart_cfg.ttbr[i] = virt_to_phys(data->pgd[i]); + cfg->apple_dart_cfg.ttbr[i] = data->pgd[i]; } return &data->iop; diff --git a/drivers/iommu/iommu-sva.c b/drivers/iommu/iommu-sva.c index e1e63c2be82b2a..fd735aaae9e3f0 100644 --- a/drivers/iommu/iommu-sva.c +++ b/drivers/iommu/iommu-sva.c @@ -182,13 +182,13 @@ void iommu_sva_unbind_device(struct iommu_sva *handle) iommu_detach_device_pasid(domain, dev, iommu_mm->pasid); if (--domain->users == 0) { list_del(&domain->next); - iommu_domain_free(domain); - } + if (list_empty(&iommu_mm->sva_domains)) { + list_del(&iommu_mm->mm_list_elm); + if (list_empty(&iommu_sva_mms)) + iommu_sva_present = false; + } - if (list_empty(&iommu_mm->sva_domains)) { - list_del(&iommu_mm->mm_list_elm); - if (list_empty(&iommu_sva_mms)) - iommu_sva_present = false; + iommu_domain_free(domain); } mutex_unlock(&iommu_sva_lock); diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index 2ca990dfbb884f..f267b9fcc00f55 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -90,6 +90,7 @@ static const char * const iommu_group_resv_type_string[] = { [IOMMU_RESV_RESERVED] = "reserved", [IOMMU_RESV_MSI] = "msi", [IOMMU_RESV_SW_MSI] = "msi", + [IOMMU_RESV_TRANSLATED] = "translated", }; #define IOMMU_CMD_LINE_DMA_API BIT(0) @@ -133,8 +134,8 @@ static void __iommu_group_set_domain_nofail(struct iommu_group *group, static int iommu_setup_default_domain(struct iommu_group *group, int target_type); -static int iommu_create_device_direct_mappings(struct iommu_domain *domain, - struct device *dev); +static int iommu_create_device_fw_mappings(struct iommu_domain *domain, + struct device *dev); static ssize_t iommu_group_store_type(struct iommu_group *group, const char *buf, size_t count); static struct group_device *iommu_group_alloc_device(struct iommu_group *group, @@ -640,7 +641,7 @@ static int __iommu_probe_device(struct device *dev, struct list_head *group_list list_add_tail(&gdev->list, &group->devices); WARN_ON(group->default_domain && !group->domain); if (group->default_domain) - iommu_create_device_direct_mappings(group->default_domain, dev); + iommu_create_device_fw_mappings(group->default_domain, dev); if (group->domain) { ret = __iommu_device_set_domain(group, dev, group->domain, NULL, 0); @@ -1169,8 +1170,8 @@ int iommu_group_set_name(struct iommu_group *group, const char *name) } EXPORT_SYMBOL_GPL(iommu_group_set_name); -static int iommu_create_device_direct_mappings(struct iommu_domain *domain, - struct device *dev) +static int iommu_create_device_fw_mappings(struct iommu_domain *domain, + struct device *dev) { struct iommu_resv_region *entry; struct list_head mappings; @@ -1187,27 +1188,39 @@ static int iommu_create_device_direct_mappings(struct iommu_domain *domain, /* We need to consider overlapping regions for different devices */ list_for_each_entry(entry, &mappings, list) { - dma_addr_t start, end, addr; + dma_addr_t start, end, addr, iova; size_t map_size = 0; if (entry->type == IOMMU_RESV_DIRECT) dev->iommu->require_direct = 1; + if (entry->type == IOMMU_RESV_TRANSLATED) + dev->iommu->require_translated = 1; if ((entry->type != IOMMU_RESV_DIRECT && - entry->type != IOMMU_RESV_DIRECT_RELAXABLE) || + entry->type != IOMMU_RESV_DIRECT_RELAXABLE && + entry->type != IOMMU_RESV_TRANSLATED) || !iommu_is_dma_domain(domain)) continue; start = ALIGN(entry->start, pg_size); end = ALIGN(entry->start + entry->length, pg_size); - for (addr = start; addr <= end; addr += pg_size) { + if (entry->type == IOMMU_RESV_TRANSLATED) + iova = ALIGN(entry->dva, pg_size); + else + iova = start; + + for (addr = start; addr <= end; addr += pg_size, iova += pg_size) { phys_addr_t phys_addr; if (addr == end) goto map_end; - phys_addr = iommu_iova_to_phys(domain, addr); + /* + * Return address by iommu_iova_to_phys for 0 is + * ambiguous. Offset to address 1 if addr is 0. + */ + phys_addr = iommu_iova_to_phys(domain, iova ? iova : 1); if (!phys_addr) { map_size += pg_size; continue; @@ -1215,7 +1228,7 @@ static int iommu_create_device_direct_mappings(struct iommu_domain *domain, map_end: if (map_size) { - ret = iommu_map(domain, addr - map_size, + ret = iommu_map(domain, iova - map_size, addr - map_size, map_size, entry->prot, GFP_KERNEL); if (ret) @@ -2318,6 +2331,19 @@ static int __iommu_device_set_domain(struct iommu_group *group, "Firmware has requested this device have a 1:1 IOMMU mapping, rejecting configuring the device without a 1:1 mapping. Contact your platform vendor.\n"); return -EINVAL; } + /* + * If the device requires IOMMU_RESV_TRANSLATED then we cannot allow + * the identy or blocking domain to be attached as it does not contain + * the required translated mapping. + */ + if (dev->iommu->require_translated && + (new_domain->type == IOMMU_DOMAIN_IDENTITY || + new_domain->type == IOMMU_DOMAIN_BLOCKED || + new_domain == group->blocking_domain)) { + dev_warn(dev, + "Firmware has requested this device have a translated IOMMU mapping, rejecting configuring the device without a translated mapping. Contact your platform vendor.\n"); + return -EINVAL; + } if (dev->iommu->attach_deferred) { if (new_domain == group->default_domain) @@ -2859,10 +2885,11 @@ void iommu_put_resv_regions(struct device *dev, struct list_head *list) } EXPORT_SYMBOL(iommu_put_resv_regions); -struct iommu_resv_region *iommu_alloc_resv_region(phys_addr_t start, - size_t length, int prot, - enum iommu_resv_type type, - gfp_t gfp) +struct iommu_resv_region *iommu_alloc_resv_region_tr(phys_addr_t start, + dma_addr_t dva_start, + size_t length, int prot, + enum iommu_resv_type type, + gfp_t gfp) { struct iommu_resv_region *region; @@ -2872,11 +2899,25 @@ struct iommu_resv_region *iommu_alloc_resv_region(phys_addr_t start, INIT_LIST_HEAD(®ion->list); region->start = start; + if (type == IOMMU_RESV_TRANSLATED) + region->dva = dva_start; region->length = length; region->prot = prot; region->type = type; return region; } +EXPORT_SYMBOL_GPL(iommu_alloc_resv_region_tr); + +struct iommu_resv_region *iommu_alloc_resv_region(phys_addr_t start, + size_t length, int prot, + enum iommu_resv_type type, + gfp_t gfp) +{ + if (type == IOMMU_RESV_TRANSLATED) + return NULL; + + return iommu_alloc_resv_region_tr(start, 0, length, prot, type, gfp); +} EXPORT_SYMBOL_GPL(iommu_alloc_resv_region); void iommu_set_default_passthrough(bool cmd_line) @@ -3001,7 +3042,7 @@ static int iommu_setup_default_domain(struct iommu_group *group, struct iommu_domain *old_dom = group->default_domain; struct group_device *gdev; struct iommu_domain *dom; - bool direct_failed; + bool fw_failed; int req_type; int ret; @@ -3031,10 +3072,10 @@ static int iommu_setup_default_domain(struct iommu_group *group, * mapped before their device is attached, in order to guarantee * continuity with any FW activity */ - direct_failed = false; + fw_failed = false; for_each_group_device(group, gdev) { - if (iommu_create_device_direct_mappings(dom, gdev->dev)) { - direct_failed = true; + if (iommu_create_device_fw_mappings(dom, gdev->dev)) { + fw_failed = true; dev_warn_once( gdev->dev->iommu->iommu_dev->dev, "IOMMU driver was not able to establish FW requested direct mapping."); @@ -3066,9 +3107,9 @@ static int iommu_setup_default_domain(struct iommu_group *group, * trying again after attaching. If this happens it means the device * will not continuously have the IOMMU_RESV_DIRECT map. */ - if (direct_failed) { + if (fw_failed) { for_each_group_device(group, gdev) { - ret = iommu_create_device_direct_mappings(dom, gdev->dev); + ret = iommu_create_device_fw_mappings(dom, gdev->dev); if (ret) goto err_restore_domain; } diff --git a/drivers/iommu/of_iommu.c b/drivers/iommu/of_iommu.c index 6b989a62def20e..69377addd6cebb 100644 --- a/drivers/iommu/of_iommu.c +++ b/drivers/iommu/of_iommu.c @@ -147,6 +147,8 @@ int of_iommu_configure(struct device *dev, struct device_node *master_np, of_pci_check_device_ats(dev, master_np); } else { err = of_iommu_configure_device(master_np, dev, id); + if (err == -EPROBE_DEFER) + iommu_fwspec_free(dev); } if (err && dev_iommu_present) @@ -187,9 +189,7 @@ iommu_resv_region_get_type(struct device *dev, if (start == phys->start && end == phys->end) return IOMMU_RESV_DIRECT; - dev_warn(dev, "treating non-direct mapping [%pr] -> [%pap-%pap] as reservation\n", phys, - &start, &end); - return IOMMU_RESV_RESERVED; + return IOMMU_RESV_TRANSLATED; } /** @@ -260,8 +260,13 @@ void of_iommu_get_resv_regions(struct device *dev, struct list_head *list) } type = iommu_resv_region_get_type(dev, &phys, iova, length); - region = iommu_alloc_resv_region(iova, length, prot, type, + if (type == IOMMU_RESV_TRANSLATED) + region = iommu_alloc_resv_region_tr(phys.start, iova, length, prot, type, + GFP_KERNEL); + else + region = iommu_alloc_resv_region(iova, length, prot, type, GFP_KERNEL); + if (region) list_add_tail(®ion->list, list); } diff --git a/drivers/irqchip/irq-apple-aic.c b/drivers/irqchip/irq-apple-aic.c index 3c70364e7cddd6..4607d28f23887f 100644 --- a/drivers/irqchip/irq-apple-aic.c +++ b/drivers/irqchip/irq-apple-aic.c @@ -134,8 +134,12 @@ #define AIC2_IRQ_CFG 0x2000 +/* AIC v3 registers (MMIO) */ +#define AIC3_IRQ_CFG 0x10000 + /* * AIC2 registers are laid out like this, starting at AIC2_IRQ_CFG: + * AIC3 registers use the same layout but start at AIC3_IRQ_CFG: * * Repeat for each die: * IRQ_CFG: u32 * MAX_IRQS @@ -293,6 +297,15 @@ static const struct aic_info aic2_info __initconst = { .local_fast_ipi = true, }; +static const struct aic_info aic3_info __initconst = { + .version = 3, + + .irq_cfg = AIC3_IRQ_CFG, + + .fast_ipi = true, + .local_fast_ipi = true, +}; + static const struct of_device_id aic_info_match[] = { { .compatible = "apple,t8103-aic", @@ -310,6 +323,10 @@ static const struct of_device_id aic_info_match[] = { .compatible = "apple,aic2", .data = &aic2_info, }, + { + .compatible = "apple,t8122-aic3", + .data = &aic3_info, + }, {} }; @@ -620,7 +637,7 @@ static int aic_irq_domain_map(struct irq_domain *id, unsigned int irq, u32 type = FIELD_GET(AIC_EVENT_TYPE, hw); struct irq_chip *chip = &aic_chip; - if (ic->info.version == 2) + if (ic->info.version == 2 || ic->info.version == 3) chip = &aic2_chip; if (type == AIC_EVENT_TYPE_IRQ) { @@ -991,7 +1008,7 @@ static int __init aic_of_ic_init(struct device_node *node, struct device_node *p break; } - case 2: { + case 2 ... 3: { u32 info1, info3; info1 = aic_ic_read(irqc, AIC2_INFO1); @@ -1065,7 +1082,7 @@ static int __init aic_of_ic_init(struct device_node *node, struct device_node *p off += irqc->info.die_stride; } - if (irqc->info.version == 2) { + if (irqc->info.version == 2 || irqc->info.version == 3) { u32 config = aic_ic_read(irqc, AIC2_CONFIG); config |= AIC2_CONFIG_ENABLE; @@ -1116,3 +1133,4 @@ static int __init aic_of_ic_init(struct device_node *node, struct device_node *p IRQCHIP_DECLARE(apple_aic, "apple,aic", aic_of_ic_init); IRQCHIP_DECLARE(apple_aic2, "apple,aic2", aic_of_ic_init); +IRQCHIP_DECLARE(apple_aic3, "apple,t8122-aic3", aic_of_ic_init); diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c index 2988def30972b5..a51e8e6a818190 100644 --- a/drivers/irqchip/irq-gic-v3-its.c +++ b/drivers/irqchip/irq-gic-v3-its.c @@ -3475,6 +3475,7 @@ static struct its_device *its_create_device(struct its_node *its, u32 dev_id, int lpi_base; int nr_lpis; int nr_ites; + int id_bits; int sz; if (!its_alloc_device_table(its, dev_id)) @@ -3486,7 +3487,10 @@ static struct its_device *its_create_device(struct its_node *its, u32 dev_id, /* * Even if the device wants a single LPI, the ITT must be * sized as a power of two (and you need at least one bit...). + * Also honor the ITS's own EID limit. */ + id_bits = FIELD_GET(GITS_TYPER_IDBITS, its->typer) + 1; + nvecs = min_t(unsigned int, nvecs, BIT(id_bits)); nr_ites = max(2, nvecs); sz = nr_ites * (FIELD_GET(GITS_TYPER_ITT_ENTRY_SIZE, its->typer) + 1); sz = max(sz, ITS_ITT_ALIGN); diff --git a/drivers/irqchip/irq-ls-extirq.c b/drivers/irqchip/irq-ls-extirq.c index 96f9c20621cf54..d724fe8439801e 100644 --- a/drivers/irqchip/irq-ls-extirq.c +++ b/drivers/irqchip/irq-ls-extirq.c @@ -190,8 +190,10 @@ static int ls_extirq_probe(struct platform_device *pdev) return dev_err_probe(dev, -ENOMEM, "Failed to allocate memory\n"); priv->intpcr = devm_of_iomap(dev, node, 0, NULL); - if (!priv->intpcr) - return dev_err_probe(dev, -ENOMEM, "Cannot ioremap OF node %pOF\n", node); + if (IS_ERR(priv->intpcr)) { + return dev_err_probe(dev, PTR_ERR(priv->intpcr), + "Cannot ioremap OF node %pOF\n", node); + } ret = ls_extirq_parse_map(priv, node); if (ret) diff --git a/drivers/irqchip/irq-qcom-mpm.c b/drivers/irqchip/irq-qcom-mpm.c index 83f31ea657b74a..181320528a47ac 100644 --- a/drivers/irqchip/irq-qcom-mpm.c +++ b/drivers/irqchip/irq-qcom-mpm.c @@ -306,6 +306,8 @@ static int mpm_pd_power_off(struct generic_pm_domain *genpd) if (ret < 0) return ret; + mbox_client_txdone(priv->mbox_chan, 0); + return 0; } @@ -434,6 +436,7 @@ static int qcom_mpm_probe(struct platform_device *pdev, struct device_node *pare } priv->mbox_client.dev = dev; + priv->mbox_client.knows_txdone = true; priv->mbox_chan = mbox_request_channel(&priv->mbox_client, 0); if (IS_ERR(priv->mbox_chan)) { ret = PTR_ERR(priv->mbox_chan); diff --git a/drivers/irqchip/irq-renesas-rzv2h.c b/drivers/irqchip/irq-renesas-rzv2h.c index 9b487120f01132..85eb194dfe3b20 100644 --- a/drivers/irqchip/irq-renesas-rzv2h.c +++ b/drivers/irqchip/irq-renesas-rzv2h.c @@ -567,7 +567,7 @@ static int rzv2h_icu_probe_common(struct platform_device *pdev, struct device_no return 0; pm_put: - pm_runtime_put(&pdev->dev); + pm_runtime_put_sync(&pdev->dev); return ret; } diff --git a/drivers/irqchip/irq-riscv-aplic-direct.c b/drivers/irqchip/irq-riscv-aplic-direct.c index c2a75bf3d20c64..5a9650225dd805 100644 --- a/drivers/irqchip/irq-riscv-aplic-direct.c +++ b/drivers/irqchip/irq-riscv-aplic-direct.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -171,6 +172,15 @@ static void aplic_idc_set_delivery(struct aplic_idc *idc, bool en) writel(de, idc->regs + APLIC_IDC_IDELIVERY); } +void aplic_direct_restore_states(struct aplic_priv *priv) +{ + struct aplic_direct *direct = container_of(priv, struct aplic_direct, priv); + int cpu; + + for_each_cpu(cpu, &direct->lmask) + aplic_idc_set_delivery(per_cpu_ptr(&aplic_idcs, cpu), true); +} + static int aplic_direct_dying_cpu(unsigned int cpu) { if (aplic_direct_parent_irq) diff --git a/drivers/irqchip/irq-riscv-aplic-main.c b/drivers/irqchip/irq-riscv-aplic-main.c index 93e7c51f944abe..d9afb6ae98cf56 100644 --- a/drivers/irqchip/irq-riscv-aplic-main.c +++ b/drivers/irqchip/irq-riscv-aplic-main.c @@ -12,10 +12,179 @@ #include #include #include +#include +#include #include +#include #include "irq-riscv-aplic-main.h" +static LIST_HEAD(aplics); + +static void aplic_restore_states(struct aplic_priv *priv) +{ + struct aplic_saved_regs *saved_regs = &priv->saved_hw_regs; + struct aplic_src_ctrl *srcs; + void __iomem *regs; + u32 nr_irqs, i; + + regs = priv->regs; + writel(saved_regs->domaincfg, regs + APLIC_DOMAINCFG); +#ifdef CONFIG_RISCV_M_MODE + writel(saved_regs->msiaddr, regs + APLIC_xMSICFGADDR); + writel(saved_regs->msiaddrh, regs + APLIC_xMSICFGADDRH); +#endif + /* + * The sourcecfg[i] has to be restored prior to the target[i], interrupt-pending and + * interrupt-enable bits. The AIA specification states that "Whenever interrupt source i is + * inactive in an interrupt domain, the corresponding interrupt-pending and interrupt-enable + * bits within the domain are read-only zeros, and register target[i] is also read-only + * zero." + */ + nr_irqs = priv->nr_irqs; + for (i = 0; i < nr_irqs; i++) { + srcs = &priv->saved_hw_regs.srcs[i]; + writel(srcs->sourcecfg, regs + APLIC_SOURCECFG_BASE + i * sizeof(u32)); + writel(srcs->target, regs + APLIC_TARGET_BASE + i * sizeof(u32)); + } + + for (i = 0; i <= nr_irqs; i += 32) { + srcs = &priv->saved_hw_regs.srcs[i]; + writel(-1U, regs + APLIC_CLRIE_BASE + (i / 32) * sizeof(u32)); + writel(srcs->ie, regs + APLIC_SETIE_BASE + (i / 32) * sizeof(u32)); + + /* Re-trigger the interrupts if it forwards interrupts to target harts by MSIs */ + if (!priv->nr_idcs) + writel(readl(regs + APLIC_CLRIP_BASE + (i / 32) * sizeof(u32)), + regs + APLIC_SETIP_BASE + (i / 32) * sizeof(u32)); + } + + if (priv->nr_idcs) + aplic_direct_restore_states(priv); +} + +static void aplic_save_states(struct aplic_priv *priv) +{ + struct aplic_src_ctrl *srcs; + void __iomem *regs; + u32 i, nr_irqs; + + regs = priv->regs; + nr_irqs = priv->nr_irqs; + /* The valid interrupt source IDs range from 1 to N, where N is priv->nr_irqs */ + for (i = 0; i < nr_irqs; i++) { + srcs = &priv->saved_hw_regs.srcs[i]; + srcs->target = readl(regs + APLIC_TARGET_BASE + i * sizeof(u32)); + + if (i % 32) + continue; + + srcs->ie = readl(regs + APLIC_SETIE_BASE + (i / 32) * sizeof(u32)); + } + + /* Save the nr_irqs bit if needed */ + if (!(nr_irqs % 32)) { + srcs = &priv->saved_hw_regs.srcs[nr_irqs]; + srcs->ie = readl(regs + APLIC_SETIE_BASE + (nr_irqs / 32) * sizeof(u32)); + } +} + +static int aplic_syscore_suspend(void *data) +{ + struct aplic_priv *priv; + + list_for_each_entry(priv, &aplics, head) + aplic_save_states(priv); + + return 0; +} + +static void aplic_syscore_resume(void *data) +{ + struct aplic_priv *priv; + + list_for_each_entry(priv, &aplics, head) + aplic_restore_states(priv); +} + +static struct syscore_ops aplic_syscore_ops = { + .suspend = aplic_syscore_suspend, + .resume = aplic_syscore_resume, +}; + +static struct syscore aplic_syscore = { + .ops = &aplic_syscore_ops, +}; + +static bool aplic_syscore_registered __ro_after_init; + +static void aplic_syscore_init(void) +{ + if (!aplic_syscore_registered) { + register_syscore(&aplic_syscore); + aplic_syscore_registered = true; + } +} + +static int aplic_pm_notifier(struct notifier_block *nb, unsigned long action, void *data) +{ + struct aplic_priv *priv = container_of(nb, struct aplic_priv, genpd_nb); + + switch (action) { + case GENPD_NOTIFY_PRE_OFF: + aplic_save_states(priv); + break; + case GENPD_NOTIFY_ON: + aplic_restore_states(priv); + break; + default: + break; + } + + return 0; +} + +static void aplic_pm_remove(void *data) +{ + struct aplic_priv *priv = data; + struct device *dev = priv->dev; + + list_del(&priv->head); + if (dev->pm_domain && dev->of_node) + dev_pm_genpd_remove_notifier(dev); +} + +static int aplic_pm_add(struct device *dev, struct aplic_priv *priv) +{ + struct aplic_src_ctrl *srcs; + int ret; + + srcs = devm_kzalloc(dev, (priv->nr_irqs + 1) * sizeof(*srcs), GFP_KERNEL); + if (!srcs) + return -ENOMEM; + + priv->saved_hw_regs.srcs = srcs; + list_add(&priv->head, &aplics); + if (dev->pm_domain && dev->of_node) { + priv->genpd_nb.notifier_call = aplic_pm_notifier; + ret = dev_pm_genpd_add_notifier(dev, &priv->genpd_nb); + if (ret) + goto remove_head; + + ret = devm_pm_runtime_enable(dev); + if (ret) + goto remove_notifier; + } + + return devm_add_action_or_reset(dev, aplic_pm_remove, priv); + +remove_notifier: + dev_pm_genpd_remove_notifier(dev); +remove_head: + list_del(&priv->head); + return ret; +} + void aplic_irq_unmask(struct irq_data *d) { struct aplic_priv *priv = irq_data_get_irq_chip_data(d); @@ -60,6 +229,8 @@ int aplic_irq_set_type(struct irq_data *d, unsigned int type) sourcecfg += (d->hwirq - 1) * sizeof(u32); writel(val, sourcecfg); + priv->saved_hw_regs.srcs[d->hwirq - 1].sourcecfg = val; + return 0; } @@ -82,6 +253,7 @@ int aplic_irqdomain_translate(struct irq_fwspec *fwspec, u32 gsi_base, void aplic_init_hw_global(struct aplic_priv *priv, bool msi_mode) { + struct aplic_saved_regs *saved_regs = &priv->saved_hw_regs; u32 val; #ifdef CONFIG_RISCV_M_MODE u32 valh; @@ -95,6 +267,8 @@ void aplic_init_hw_global(struct aplic_priv *priv, bool msi_mode) valh |= FIELD_PREP(APLIC_xMSICFGADDRH_HHXS, priv->msicfg.hhxs); writel(val, priv->regs + APLIC_xMSICFGADDR); writel(valh, priv->regs + APLIC_xMSICFGADDRH); + saved_regs->msiaddr = val; + saved_regs->msiaddrh = valh; } #endif @@ -106,6 +280,8 @@ void aplic_init_hw_global(struct aplic_priv *priv, bool msi_mode) writel(val, priv->regs + APLIC_DOMAINCFG); if (readl(priv->regs + APLIC_DOMAINCFG) != val) dev_warn(priv->dev, "unable to write 0x%x in domaincfg\n", val); + + saved_regs->domaincfg = val; } static void aplic_init_hw_irqs(struct aplic_priv *priv) @@ -176,7 +352,7 @@ int aplic_setup_priv(struct aplic_priv *priv, struct device *dev, void __iomem * /* Setup initial state APLIC interrupts */ aplic_init_hw_irqs(priv); - return 0; + return aplic_pm_add(dev, priv); } static int aplic_probe(struct platform_device *pdev) @@ -206,16 +382,21 @@ static int aplic_probe(struct platform_device *pdev) rc = aplic_msi_setup(dev, regs); else rc = aplic_direct_setup(dev, regs); - if (rc) + + if (rc) { dev_err_probe(dev, rc, "failed to setup APLIC in %s mode\n", msi_mode ? "MSI" : "direct"); + return rc; + } + + aplic_syscore_init(); #ifdef CONFIG_ACPI if (!acpi_disabled) acpi_dev_clear_dependencies(ACPI_COMPANION(dev)); #endif - return rc; + return 0; } static const struct of_device_id aplic_match[] = { diff --git a/drivers/irqchip/irq-riscv-aplic-main.h b/drivers/irqchip/irq-riscv-aplic-main.h index b0ad8cde69b131..2d8ad7138541ad 100644 --- a/drivers/irqchip/irq-riscv-aplic-main.h +++ b/drivers/irqchip/irq-riscv-aplic-main.h @@ -23,7 +23,25 @@ struct aplic_msicfg { u32 lhxw; }; +struct aplic_src_ctrl { + u32 sourcecfg; + u32 target; + u32 ie; +}; + +struct aplic_saved_regs { + u32 domaincfg; +#ifdef CONFIG_RISCV_M_MODE + u32 msiaddr; + u32 msiaddrh; +#endif + struct aplic_src_ctrl *srcs; +}; + struct aplic_priv { + struct list_head head; + struct notifier_block genpd_nb; + struct aplic_saved_regs saved_hw_regs; struct device *dev; u32 gsi_base; u32 nr_irqs; @@ -40,6 +58,7 @@ int aplic_irqdomain_translate(struct irq_fwspec *fwspec, u32 gsi_base, unsigned long *hwirq, unsigned int *type); void aplic_init_hw_global(struct aplic_priv *priv, bool msi_mode); int aplic_setup_priv(struct aplic_priv *priv, struct device *dev, void __iomem *regs); +void aplic_direct_restore_states(struct aplic_priv *priv); int aplic_direct_setup(struct device *dev, void __iomem *regs); #ifdef CONFIG_RISCV_APLIC_MSI int aplic_msi_setup(struct device *dev, void __iomem *regs); diff --git a/drivers/irqchip/irq-riscv-imsic-early.c b/drivers/irqchip/irq-riscv-imsic-early.c index 6bac67cc0b6d99..ba903fa689bd52 100644 --- a/drivers/irqchip/irq-riscv-imsic-early.c +++ b/drivers/irqchip/irq-riscv-imsic-early.c @@ -7,6 +7,7 @@ #define pr_fmt(fmt) "riscv-imsic: " fmt #include #include +#include #include #include #include @@ -123,14 +124,8 @@ static void imsic_handle_irq(struct irq_desc *desc) chained_irq_exit(chip, desc); } -static int imsic_starting_cpu(unsigned int cpu) +static void imsic_hw_states_init(void) { - /* Mark per-CPU IMSIC state as online */ - imsic_state_online(); - - /* Enable per-CPU parent interrupt */ - enable_percpu_irq(imsic_parent_irq, irq_get_trigger_type(imsic_parent_irq)); - /* Setup IPIs */ imsic_ipi_starting_cpu(); @@ -142,6 +137,18 @@ static int imsic_starting_cpu(unsigned int cpu) /* Enable local interrupt delivery */ imsic_local_delivery(true); +} + +static int imsic_starting_cpu(unsigned int cpu) +{ + /* Mark per-CPU IMSIC state as online */ + imsic_state_online(); + + /* Enable per-CPU parent interrupt */ + enable_percpu_irq(imsic_parent_irq, irq_get_trigger_type(imsic_parent_irq)); + + /* Initialize the IMSIC registers to enable the interrupt delivery */ + imsic_hw_states_init(); return 0; } @@ -157,6 +164,22 @@ static int imsic_dying_cpu(unsigned int cpu) return 0; } +static int imsic_pm_notifier(struct notifier_block *self, unsigned long cmd, void *v) +{ + switch (cmd) { + case CPU_PM_EXIT: + /* Initialize the IMSIC registers to enable the interrupt delivery */ + imsic_hw_states_init(); + break; + } + + return NOTIFY_OK; +} + +static struct notifier_block imsic_pm_notifier_block = { + .notifier_call = imsic_pm_notifier, +}; + static int __init imsic_early_probe(struct fwnode_handle *fwnode) { struct irq_domain *domain; @@ -194,7 +217,7 @@ static int __init imsic_early_probe(struct fwnode_handle *fwnode) cpuhp_setup_state(CPUHP_AP_IRQ_RISCV_IMSIC_STARTING, "irqchip/riscv/imsic:starting", imsic_starting_cpu, imsic_dying_cpu); - return 0; + return cpu_pm_register_notifier(&imsic_pm_notifier_block); } static int __init imsic_early_dt_init(struct device_node *node, struct device_node *parent) diff --git a/drivers/irqchip/irq-riscv-rpmi-sysmsi.c b/drivers/irqchip/irq-riscv-rpmi-sysmsi.c index 5c74c561ce3161..612f3972f7af03 100644 --- a/drivers/irqchip/irq-riscv-rpmi-sysmsi.c +++ b/drivers/irqchip/irq-riscv-rpmi-sysmsi.c @@ -250,6 +250,7 @@ static int rpmi_sysmsi_probe(struct platform_device *pdev) rc = riscv_acpi_get_gsi_info(fwnode, &priv->gsi_base, &id, &nr_irqs, NULL); if (rc) { + mbox_free_channel(priv->chan); dev_err(dev, "failed to find GSI mapping\n"); return rc; } diff --git a/drivers/irqchip/irq-sifive-plic.c b/drivers/irqchip/irq-sifive-plic.c index 210a5795963772..70058871d2fb6f 100644 --- a/drivers/irqchip/irq-sifive-plic.c +++ b/drivers/irqchip/irq-sifive-plic.c @@ -68,15 +68,17 @@ #define PLIC_QUIRK_CP100_CLAIM_REGISTER_ERRATUM 1 struct plic_priv { - struct fwnode_handle *fwnode; - struct cpumask lmask; - struct irq_domain *irqdomain; - void __iomem *regs; - unsigned long plic_quirks; - unsigned int nr_irqs; - unsigned long *prio_save; - u32 gsi_base; - int acpi_plic_id; + struct fwnode_handle *fwnode; + struct cpumask lmask; + struct irq_domain *irqdomain; + void __iomem *regs; + unsigned long plic_quirks; + /* device interrupts + 1 to compensate for the reserved hwirq 0 */ + unsigned int __private total_irqs; + unsigned int irq_groups; + unsigned long *prio_save; + u32 gsi_base; + int acpi_plic_id; }; struct plic_handler { @@ -91,6 +93,12 @@ struct plic_handler { u32 *enable_save; struct plic_priv *priv; }; + +/* + * Macro to deal with the insanity of hardware interrupt 0 being reserved */ +#define for_each_device_irq(iter, priv) \ + for (unsigned int iter = 1; iter < ACCESS_PRIVATE(priv, total_irqs); iter++) + static int plic_parent_irq __ro_after_init; static bool plic_global_setup_done __ro_after_init; static DEFINE_PER_CPU(struct plic_handler, plic_handlers); @@ -164,8 +172,13 @@ static void plic_irq_disable(struct irq_data *d) static void plic_irq_eoi(struct irq_data *d) { struct plic_handler *handler = this_cpu_ptr(&plic_handlers); + u32 __iomem *reg; + bool enabled; - if (unlikely(irqd_irq_disabled(d))) { + reg = handler->enable_base + (d->hwirq / 32) * sizeof(u32); + enabled = readl(reg) & BIT(d->hwirq % 32); + + if (unlikely(!enabled)) { plic_toggle(handler, d->hwirq, 1); writel(d->hwirq, handler->hart_base + CONTEXT_CLAIM); plic_toggle(handler, d->hwirq, 0); @@ -257,14 +270,11 @@ static int plic_irq_set_type(struct irq_data *d, unsigned int type) static int plic_irq_suspend(void *data) { - struct plic_priv *priv; + struct plic_priv *priv = this_cpu_ptr(&plic_handlers)->priv; - priv = per_cpu_ptr(&plic_handlers, smp_processor_id())->priv; - - /* irq ID 0 is reserved */ - for (unsigned int i = 1; i < priv->nr_irqs; i++) { - __assign_bit(i, priv->prio_save, - readl(priv->regs + PRIORITY_BASE + i * PRIORITY_PER_ID)); + for_each_device_irq(irq, priv) { + __assign_bit(irq, priv->prio_save, + readl(priv->regs + PRIORITY_BASE + irq * PRIORITY_PER_ID)); } return 0; @@ -272,18 +282,15 @@ static int plic_irq_suspend(void *data) static void plic_irq_resume(void *data) { - unsigned int i, index, cpu; + struct plic_priv *priv = this_cpu_ptr(&plic_handlers)->priv; + unsigned int index, cpu; unsigned long flags; u32 __iomem *reg; - struct plic_priv *priv; - priv = per_cpu_ptr(&plic_handlers, smp_processor_id())->priv; - - /* irq ID 0 is reserved */ - for (i = 1; i < priv->nr_irqs; i++) { - index = BIT_WORD(i); - writel((priv->prio_save[index] & BIT_MASK(i)) ? 1 : 0, - priv->regs + PRIORITY_BASE + i * PRIORITY_PER_ID); + for_each_device_irq(irq, priv) { + index = BIT_WORD(irq); + writel((priv->prio_save[index] & BIT_MASK(irq)) ? 1 : 0, + priv->regs + PRIORITY_BASE + irq * PRIORITY_PER_ID); } for_each_present_cpu(cpu) { @@ -293,7 +300,7 @@ static void plic_irq_resume(void *data) continue; raw_spin_lock_irqsave(&handler->enable_lock, flags); - for (i = 0; i < DIV_ROUND_UP(priv->nr_irqs, 32); i++) { + for (unsigned int i = 0; i < priv->irq_groups; i++) { reg = handler->enable_base + i * sizeof(u32); writel(handler->enable_save[i], reg); } @@ -431,7 +438,7 @@ static u32 cp100_isolate_pending_irq(int nr_irq_groups, struct plic_handler *han static irq_hw_number_t cp100_get_hwirq(struct plic_handler *handler, void __iomem *claim) { - int nr_irq_groups = DIV_ROUND_UP(handler->priv->nr_irqs, 32); + int nr_irq_groups = handler->priv->irq_groups; u32 __iomem *enable = handler->enable_base; irq_hw_number_t hwirq = 0; u32 iso_mask; @@ -614,7 +621,6 @@ static int plic_probe(struct fwnode_handle *fwnode) struct plic_handler *handler; u32 nr_irqs, parent_hwirq; struct plic_priv *priv; - irq_hw_number_t hwirq; void __iomem *regs; int id, context_id; u32 gsi_base; @@ -647,7 +653,16 @@ static int plic_probe(struct fwnode_handle *fwnode) priv->fwnode = fwnode; priv->plic_quirks = plic_quirks; - priv->nr_irqs = nr_irqs; + /* + * The firmware provides the number of device interrupts. As + * hardware interrupt 0 is reserved, the number of total interrupts + * is nr_irqs + 1. + */ + nr_irqs++; + ACCESS_PRIVATE(priv, total_irqs) = nr_irqs; + /* Precalculate the number of register groups */ + priv->irq_groups = DIV_ROUND_UP(nr_irqs, 32); + priv->regs = regs; priv->gsi_base = gsi_base; priv->acpi_plic_id = id; @@ -686,7 +701,7 @@ static int plic_probe(struct fwnode_handle *fwnode) u32 __iomem *enable_base = priv->regs + CONTEXT_ENABLE_BASE + i * CONTEXT_ENABLE_SIZE; - for (int j = 0; j <= nr_irqs / 32; j++) + for (int j = 0; j < priv->irq_groups; j++) writel(0, enable_base + j); } continue; @@ -718,23 +733,21 @@ static int plic_probe(struct fwnode_handle *fwnode) context_id * CONTEXT_ENABLE_SIZE; handler->priv = priv; - handler->enable_save = kcalloc(DIV_ROUND_UP(nr_irqs, 32), - sizeof(*handler->enable_save), GFP_KERNEL); + handler->enable_save = kcalloc(priv->irq_groups, sizeof(*handler->enable_save), + GFP_KERNEL); if (!handler->enable_save) { error = -ENOMEM; goto fail_cleanup_contexts; } done: - for (hwirq = 1; hwirq <= nr_irqs; hwirq++) { + for_each_device_irq(hwirq, priv) { plic_toggle(handler, hwirq, 0); - writel(1, priv->regs + PRIORITY_BASE + - hwirq * PRIORITY_PER_ID); + writel(1, priv->regs + PRIORITY_BASE + hwirq * PRIORITY_PER_ID); } nr_handlers++; } - priv->irqdomain = irq_domain_create_linear(fwnode, nr_irqs + 1, - &plic_irqdomain_ops, priv); + priv->irqdomain = irq_domain_create_linear(fwnode, nr_irqs, &plic_irqdomain_ops, priv); if (WARN_ON(!priv->irqdomain)) { error = -ENOMEM; goto fail_cleanup_contexts; diff --git a/drivers/leds/leds-expresswire.c b/drivers/leds/leds-expresswire.c index bb69be228a6d3c..25c6b159a6ee93 100644 --- a/drivers/leds/leds-expresswire.c +++ b/drivers/leds/leds-expresswire.c @@ -9,6 +9,7 @@ #include #include #include +#include #include #include @@ -16,37 +17,41 @@ void expresswire_power_off(struct expresswire_common_props *props) { gpiod_set_value_cansleep(props->ctrl_gpio, 0); - usleep_range(props->timing.poweroff_us, props->timing.poweroff_us * 2); + fsleep(props->timing.poweroff_us); } EXPORT_SYMBOL_NS_GPL(expresswire_power_off, "EXPRESSWIRE"); void expresswire_enable(struct expresswire_common_props *props) { + unsigned long flags; + + local_irq_save(flags); + gpiod_set_value(props->ctrl_gpio, 1); udelay(props->timing.detect_delay_us); gpiod_set_value(props->ctrl_gpio, 0); udelay(props->timing.detect_us); gpiod_set_value(props->ctrl_gpio, 1); + + local_irq_restore(flags); } EXPORT_SYMBOL_NS_GPL(expresswire_enable, "EXPRESSWIRE"); -void expresswire_start(struct expresswire_common_props *props) +static void expresswire_start(struct expresswire_common_props *props) { gpiod_set_value(props->ctrl_gpio, 1); udelay(props->timing.data_start_us); } -EXPORT_SYMBOL_NS_GPL(expresswire_start, "EXPRESSWIRE"); -void expresswire_end(struct expresswire_common_props *props) +static void expresswire_end(struct expresswire_common_props *props) { gpiod_set_value(props->ctrl_gpio, 0); udelay(props->timing.end_of_data_low_us); gpiod_set_value(props->ctrl_gpio, 1); udelay(props->timing.end_of_data_high_us); } -EXPORT_SYMBOL_NS_GPL(expresswire_end, "EXPRESSWIRE"); -void expresswire_set_bit(struct expresswire_common_props *props, bool bit) +static void expresswire_set_bit(struct expresswire_common_props *props, bool bit) { if (bit) { gpiod_set_value(props->ctrl_gpio, 0); @@ -60,13 +65,18 @@ void expresswire_set_bit(struct expresswire_common_props *props, bool bit) udelay(props->timing.short_bitset_us); } } -EXPORT_SYMBOL_NS_GPL(expresswire_set_bit, "EXPRESSWIRE"); void expresswire_write_u8(struct expresswire_common_props *props, u8 val) { + unsigned long flags; + + local_irq_save(flags); + expresswire_start(props); for (int i = 7; i >= 0; i--) expresswire_set_bit(props, val & BIT(i)); expresswire_end(props); + + local_irq_restore(flags); } EXPORT_SYMBOL_NS_GPL(expresswire_write_u8, "EXPRESSWIRE"); diff --git a/drivers/leds/rgb/leds-qcom-lpg.c b/drivers/leds/rgb/leds-qcom-lpg.c index 72da0bf469ad8b..f54851dfb42fce 100644 --- a/drivers/leds/rgb/leds-qcom-lpg.c +++ b/drivers/leds/rgb/leds-qcom-lpg.c @@ -369,7 +369,7 @@ static int lpg_lut_store(struct lpg *lpg, struct led_pattern *pattern, { unsigned int idx; u16 val; - int i; + int i, ret; idx = bitmap_find_next_zero_area(lpg->lut_bitmap, lpg->lut_size, 0, len, 0); @@ -379,8 +379,10 @@ static int lpg_lut_store(struct lpg *lpg, struct led_pattern *pattern, for (i = 0; i < len; i++) { val = pattern[i].brightness; - regmap_bulk_write(lpg->map, lpg->lut_base + LPG_LUT_REG(idx + i), - &val, sizeof(val)); + ret = regmap_bulk_write(lpg->map, lpg->lut_base + LPG_LUT_REG(idx + i), + &val, sizeof(val)); + if (ret) + return ret; } bitmap_set(lpg->lut_bitmap, idx, len); diff --git a/drivers/mailbox/bcm-flexrm-mailbox.c b/drivers/mailbox/bcm-flexrm-mailbox.c index 41f79e51d9e5a9..4255fefc3a5a05 100644 --- a/drivers/mailbox/bcm-flexrm-mailbox.c +++ b/drivers/mailbox/bcm-flexrm-mailbox.c @@ -1173,14 +1173,6 @@ static int flexrm_debugfs_stats_show(struct seq_file *file, void *offset) /* ====== FlexRM interrupt handler ===== */ -static irqreturn_t flexrm_irq_event(int irq, void *dev_id) -{ - /* We only have MSI for completions so just wakeup IRQ thread */ - /* Ring related errors will be informed via completion descriptors */ - - return IRQ_WAKE_THREAD; -} - static irqreturn_t flexrm_irq_thread(int irq, void *dev_id) { flexrm_process_completions(dev_id); @@ -1271,10 +1263,8 @@ static int flexrm_startup(struct mbox_chan *chan) ret = -ENODEV; goto fail_free_cmpl_memory; } - ret = request_threaded_irq(ring->irq, - flexrm_irq_event, - flexrm_irq_thread, - 0, dev_name(ring->mbox->dev), ring); + ret = request_threaded_irq(ring->irq, NULL, flexrm_irq_thread, + IRQF_ONESHOT, dev_name(ring->mbox->dev), ring); if (ret) { dev_err(ring->mbox->dev, "failed to request ring%d IRQ\n", ring->num); diff --git a/drivers/mailbox/imx-mailbox.c b/drivers/mailbox/imx-mailbox.c index 6778afc64a048c..003f9236c35e09 100644 --- a/drivers/mailbox/imx-mailbox.c +++ b/drivers/mailbox/imx-mailbox.c @@ -122,6 +122,7 @@ struct imx_mu_dcfg { u32 xRR; /* Receive Register0 */ u32 xSR[IMX_MU_xSR_MAX]; /* Status Registers */ u32 xCR[IMX_MU_xCR_MAX]; /* Control Registers */ + bool skip_suspend_flag; }; #define IMX_MU_xSR_GIPn(type, x) (type & IMX_MU_V2 ? BIT(x) : BIT(28 + (3 - (x)))) @@ -988,6 +989,7 @@ static const struct imx_mu_dcfg imx_mu_cfg_imx7ulp = { .xRR = 0x40, .xSR = {0x60, 0x60, 0x60, 0x60}, .xCR = {0x64, 0x64, 0x64, 0x64, 0x64}, + .skip_suspend_flag = true, }; static const struct imx_mu_dcfg imx_mu_cfg_imx8ulp = { @@ -1071,7 +1073,8 @@ static int __maybe_unused imx_mu_suspend_noirq(struct device *dev) priv->xcr[i] = imx_mu_read(priv, priv->dcfg->xCR[i]); } - priv->suspend = true; + if (!priv->dcfg->skip_suspend_flag) + priv->suspend = true; return 0; } @@ -1094,7 +1097,8 @@ static int __maybe_unused imx_mu_resume_noirq(struct device *dev) imx_mu_write(priv, priv->xcr[i], priv->dcfg->xCR[i]); } - priv->suspend = false; + if (!priv->dcfg->skip_suspend_flag) + priv->suspend = false; return 0; } diff --git a/drivers/mailbox/mailbox-mchp-ipc-sbi.c b/drivers/mailbox/mailbox-mchp-ipc-sbi.c index a6e52009a42457..b87bf2fb4b9b93 100644 --- a/drivers/mailbox/mailbox-mchp-ipc-sbi.c +++ b/drivers/mailbox/mailbox-mchp-ipc-sbi.c @@ -174,26 +174,30 @@ static irqreturn_t mchp_ipc_cluster_aggr_isr(int irq, void *data) struct mchp_ipc_msg ipc_msg; struct mchp_ipc_status status_msg; int ret; - unsigned long hartid; u32 i, chan_index, chan_id; + bool found = false; /* Find out the hart that originated the irq */ for_each_online_cpu(i) { - hartid = cpuid_to_hartid_map(i); - if (irq == ipc->cluster_cfg[hartid].irq) + if (irq == ipc->cluster_cfg[i].irq) { + found = true; break; + } } - status_msg.cluster = hartid; - memcpy(ipc->cluster_cfg[hartid].buf_base, &status_msg, sizeof(struct mchp_ipc_status)); + if (unlikely(!found)) + return IRQ_NONE; - ret = mchp_ipc_sbi_send(SBI_EXT_IPC_STATUS, ipc->cluster_cfg[hartid].buf_base_addr); + status_msg.cluster = cpuid_to_hartid_map(i); + memcpy(ipc->cluster_cfg[i].buf_base, &status_msg, sizeof(struct mchp_ipc_status)); + + ret = mchp_ipc_sbi_send(SBI_EXT_IPC_STATUS, ipc->cluster_cfg[i].buf_base_addr); if (ret < 0) { dev_err_ratelimited(ipc->dev, "could not get IHC irq status ret=%d\n", ret); return IRQ_HANDLED; } - memcpy(&status_msg, ipc->cluster_cfg[hartid].buf_base, sizeof(struct mchp_ipc_status)); + memcpy(&status_msg, ipc->cluster_cfg[i].buf_base, sizeof(struct mchp_ipc_status)); /* * Iterate over each bit set in the IHC interrupt status register (IRQ_STATUS) to identify @@ -321,13 +325,6 @@ static int mchp_ipc_startup(struct mbox_chan *chan) goto fail_free_buf_msg_rx; } - if (ret) { - dev_err(ipc->dev, "failed to register interrupt(s)\n"); - goto fail_free_buf_msg_rx; - } - - return ret; - fail_free_buf_msg_rx: kfree(chan_info->msg_buf_rx); fail_free_buf_msg_tx: @@ -385,21 +382,21 @@ static int mchp_ipc_get_cluster_aggr_irq(struct mchp_ipc_sbi_mbox *ipc) if (ret <= 0) continue; - ipc->cluster_cfg[hartid].irq = ret; - ret = devm_request_irq(ipc->dev, ipc->cluster_cfg[hartid].irq, + ipc->cluster_cfg[cpuid].irq = ret; + ret = devm_request_irq(ipc->dev, ipc->cluster_cfg[cpuid].irq, mchp_ipc_cluster_aggr_isr, IRQF_SHARED, "miv-ihc-irq", ipc); if (ret) return ret; - ipc->cluster_cfg[hartid].buf_base = devm_kmalloc(ipc->dev, - sizeof(struct mchp_ipc_status), - GFP_KERNEL); + ipc->cluster_cfg[cpuid].buf_base = devm_kmalloc(ipc->dev, + sizeof(struct mchp_ipc_status), + GFP_KERNEL); - if (!ipc->cluster_cfg[hartid].buf_base) + if (!ipc->cluster_cfg[cpuid].buf_base) return -ENOMEM; - ipc->cluster_cfg[hartid].buf_base_addr = __pa(ipc->cluster_cfg[hartid].buf_base); + ipc->cluster_cfg[cpuid].buf_base_addr = __pa(ipc->cluster_cfg[cpuid].buf_base); irq_found = true; } @@ -419,7 +416,7 @@ static int mchp_ipc_probe(struct platform_device *pdev) ret = sbi_probe_extension(SBI_EXT_MICROCHIP_TECHNOLOGY); if (ret <= 0) - return dev_err_probe(dev, ret, "Microchip SBI extension not detected\n"); + return dev_err_probe(dev, -ENODEV, "Microchip SBI extension not detected\n"); ipc = devm_kzalloc(dev, sizeof(*ipc), GFP_KERNEL); if (!ipc) diff --git a/drivers/mailbox/mailbox.c b/drivers/mailbox/mailbox.c index 2acc6ec229a458..617ba505691d37 100644 --- a/drivers/mailbox/mailbox.c +++ b/drivers/mailbox/mailbox.c @@ -489,12 +489,10 @@ EXPORT_SYMBOL_GPL(mbox_free_channel); static struct mbox_chan *fw_mbox_index_xlate(struct mbox_controller *mbox, const struct fwnode_reference_args *sp) { - int ind = sp->args[0]; - - if (ind >= mbox->num_chans) + if (sp->nargs < 1 || sp->args[0] >= mbox->num_chans) return ERR_PTR(-EINVAL); - return &mbox->chans[ind]; + return &mbox->chans[sp->args[0]]; } /** diff --git a/drivers/mailbox/pcc.c b/drivers/mailbox/pcc.c index ff292b9e0be9ee..713022aed2e2fb 100644 --- a/drivers/mailbox/pcc.c +++ b/drivers/mailbox/pcc.c @@ -305,22 +305,6 @@ static void pcc_chan_acknowledge(struct pcc_chan_info *pchan) pcc_chan_reg_read_modify_write(&pchan->db); } -static void *write_response(struct pcc_chan_info *pchan) -{ - struct pcc_header pcc_header; - void *buffer; - int data_len; - - memcpy_fromio(&pcc_header, pchan->chan.shmem, - sizeof(pcc_header)); - data_len = pcc_header.length - sizeof(u32) + sizeof(struct pcc_header); - - buffer = pchan->chan.rx_alloc(pchan->chan.mchan->cl, data_len); - if (buffer != NULL) - memcpy_fromio(buffer, pchan->chan.shmem, data_len); - return buffer; -} - /** * pcc_mbox_irq - PCC mailbox interrupt handler * @irq: interrupt number @@ -332,8 +316,6 @@ static irqreturn_t pcc_mbox_irq(int irq, void *p) { struct pcc_chan_info *pchan; struct mbox_chan *chan = p; - struct pcc_header *pcc_header = chan->active_req; - void *handle = NULL; pchan = chan->con_priv; @@ -357,17 +339,7 @@ static irqreturn_t pcc_mbox_irq(int irq, void *p) * required to avoid any possible race in updatation of this flag. */ pchan->chan_in_use = false; - - if (pchan->chan.rx_alloc) - handle = write_response(pchan); - - if (chan->active_req) { - pcc_header = chan->active_req; - if (pcc_header->flags & PCC_CMD_COMPLETION_NOTIFY) - mbox_chan_txdone(chan, 0); - } - - mbox_chan_received_data(chan, handle); + mbox_chan_received_data(chan, NULL); pcc_chan_acknowledge(pchan); @@ -411,24 +383,9 @@ pcc_mbox_request_channel(struct mbox_client *cl, int subspace_id) pcc_mchan = &pchan->chan; pcc_mchan->shmem = acpi_os_ioremap(pcc_mchan->shmem_base_addr, pcc_mchan->shmem_size); - if (!pcc_mchan->shmem) - goto err; - - pcc_mchan->manage_writes = false; - - /* This indicates that the channel is ready to accept messages. - * This needs to happen after the channel has registered - * its callback. There is no access point to do that in - * the mailbox API. That implies that the mailbox client must - * have set the allocate callback function prior to - * sending any messages. - */ - if (pchan->type == ACPI_PCCT_TYPE_EXT_PCC_SLAVE_SUBSPACE) - pcc_chan_reg_read_modify_write(&pchan->cmd_update); - - return pcc_mchan; + if (pcc_mchan->shmem) + return pcc_mchan; -err: mbox_free_channel(chan); return ERR_PTR(-ENXIO); } @@ -459,38 +416,8 @@ void pcc_mbox_free_channel(struct pcc_mbox_chan *pchan) } EXPORT_SYMBOL_GPL(pcc_mbox_free_channel); -static int pcc_write_to_buffer(struct mbox_chan *chan, void *data) -{ - struct pcc_chan_info *pchan = chan->con_priv; - struct pcc_mbox_chan *pcc_mbox_chan = &pchan->chan; - struct pcc_header *pcc_header = data; - - if (!pchan->chan.manage_writes) - return 0; - - /* The PCC header length includes the command field - * but not the other values from the header. - */ - int len = pcc_header->length - sizeof(u32) + sizeof(struct pcc_header); - u64 val; - - pcc_chan_reg_read(&pchan->cmd_complete, &val); - if (!val) { - pr_info("%s pchan->cmd_complete not set", __func__); - return -1; - } - memcpy_toio(pcc_mbox_chan->shmem, data, len); - return 0; -} - - /** - * pcc_send_data - Called from Mailbox Controller code. If - * pchan->chan.rx_alloc is set, then the command complete - * flag is checked and the data is written to the shared - * buffer io memory. - * - * If pchan->chan.rx_alloc is not set, then it is used + * pcc_send_data - Called from Mailbox Controller code. Used * here only to ring the channel doorbell. The PCC client * specific read/write is done in the client driver in * order to maintain atomicity over PCC channel once @@ -506,37 +433,17 @@ static int pcc_send_data(struct mbox_chan *chan, void *data) int ret; struct pcc_chan_info *pchan = chan->con_priv; - ret = pcc_write_to_buffer(chan, data); - if (ret) - return ret; - ret = pcc_chan_reg_read_modify_write(&pchan->cmd_update); if (ret) return ret; ret = pcc_chan_reg_read_modify_write(&pchan->db); - if (!ret && pchan->plat_irq > 0) pchan->chan_in_use = true; return ret; } - -static bool pcc_last_tx_done(struct mbox_chan *chan) -{ - struct pcc_chan_info *pchan = chan->con_priv; - u64 val; - - pcc_chan_reg_read(&pchan->cmd_complete, &val); - if (!val) - return false; - else - return true; -} - - - /** * pcc_startup - Called from Mailbox Controller code. Used here * to request the interrupt. @@ -552,7 +459,7 @@ static int pcc_startup(struct mbox_chan *chan) if (pchan->plat_irq > 0) { irqflags = pcc_chan_plat_irq_can_be_shared(pchan) ? - IRQF_SHARED | IRQF_ONESHOT : 0; + IRQF_SHARED : 0; rc = devm_request_irq(chan->mbox->dev, pchan->plat_irq, pcc_mbox_irq, irqflags, MBOX_IRQ_NAME, chan); if (unlikely(rc)) { @@ -582,7 +489,6 @@ static const struct mbox_chan_ops pcc_chan_ops = { .send_data = pcc_send_data, .startup = pcc_startup, .shutdown = pcc_shutdown, - .last_tx_done = pcc_last_tx_done, }; /** diff --git a/drivers/mailbox/sprd-mailbox.c b/drivers/mailbox/sprd-mailbox.c index ee8539dfcef549..46d0c34177ab91 100644 --- a/drivers/mailbox/sprd-mailbox.c +++ b/drivers/mailbox/sprd-mailbox.c @@ -166,6 +166,11 @@ static irqreturn_t sprd_mbox_inbox_isr(int irq, void *data) return IRQ_NONE; } + /* Clear FIFO delivery and overflow status first */ + writel(fifo_sts & + (SPRD_INBOX_FIFO_DELIVER_MASK | SPRD_INBOX_FIFO_OVERLOW_MASK), + priv->inbox_base + SPRD_MBOX_FIFO_RST); + while (send_sts) { id = __ffs(send_sts); send_sts &= (send_sts - 1); @@ -181,11 +186,6 @@ static irqreturn_t sprd_mbox_inbox_isr(int irq, void *data) mbox_chan_txdone(chan, 0); } - /* Clear FIFO delivery and overflow status */ - writel(fifo_sts & - (SPRD_INBOX_FIFO_DELIVER_MASK | SPRD_INBOX_FIFO_OVERLOW_MASK), - priv->inbox_base + SPRD_MBOX_FIFO_RST); - /* Clear irq status */ writel(SPRD_MBOX_IRQ_CLR, priv->inbox_base + SPRD_MBOX_IRQ_STS); @@ -243,21 +243,19 @@ static int sprd_mbox_startup(struct mbox_chan *chan) /* Select outbox FIFO mode and reset the outbox FIFO status */ writel(0x0, priv->outbox_base + SPRD_MBOX_FIFO_RST); - /* Enable inbox FIFO overflow and delivery interrupt */ - val = readl(priv->inbox_base + SPRD_MBOX_IRQ_MSK); - val &= ~(SPRD_INBOX_FIFO_OVERFLOW_IRQ | SPRD_INBOX_FIFO_DELIVER_IRQ); + /* Enable inbox FIFO delivery interrupt */ + val = SPRD_INBOX_FIFO_IRQ_MASK; + val &= ~SPRD_INBOX_FIFO_DELIVER_IRQ; writel(val, priv->inbox_base + SPRD_MBOX_IRQ_MSK); /* Enable outbox FIFO not empty interrupt */ - val = readl(priv->outbox_base + SPRD_MBOX_IRQ_MSK); + val = SPRD_OUTBOX_FIFO_IRQ_MASK; val &= ~SPRD_OUTBOX_FIFO_NOT_EMPTY_IRQ; writel(val, priv->outbox_base + SPRD_MBOX_IRQ_MSK); /* Enable supplementary outbox as the fundamental one */ if (priv->supp_base) { writel(0x0, priv->supp_base + SPRD_MBOX_FIFO_RST); - val = readl(priv->supp_base + SPRD_MBOX_IRQ_MSK); - val &= ~SPRD_OUTBOX_FIFO_NOT_EMPTY_IRQ; writel(val, priv->supp_base + SPRD_MBOX_IRQ_MSK); } } diff --git a/drivers/mcb/mcb-core.c b/drivers/mcb/mcb-core.c index c1367223e71a0f..3d487d75c483d7 100644 --- a/drivers/mcb/mcb-core.c +++ b/drivers/mcb/mcb-core.c @@ -85,7 +85,8 @@ static void mcb_remove(struct device *dev) struct mcb_device *mdev = to_mcb_device(dev); struct module *carrier_mod; - mdrv->remove(mdev); + if (mdrv->remove) + mdrv->remove(mdev); carrier_mod = mdev->dev.parent->driver->owner; module_put(carrier_mod); @@ -176,13 +177,13 @@ static const struct device_type mcb_carrier_device_type = { * @owner: The @mcb_driver's module * @mod_name: The name of the @mcb_driver's module * - * Register a @mcb_driver at the system. Perform some sanity checks, if - * the .probe and .remove methods are provided by the driver. + * Register a @mcb_driver at the system. Perform a sanity check, if + * .probe method is provided by the driver. */ int __mcb_register_driver(struct mcb_driver *drv, struct module *owner, const char *mod_name) { - if (!drv->probe || !drv->remove) + if (!drv->probe) return -EINVAL; drv->driver.owner = owner; diff --git a/drivers/md/bcache/super.c b/drivers/md/bcache/super.c index 238d12ffdae8d6..5005a26af363f9 100644 --- a/drivers/md/bcache/super.c +++ b/drivers/md/bcache/super.c @@ -1373,6 +1373,13 @@ static CLOSURE_CALLBACK(cached_dev_free) mutex_unlock(&bch_register_lock); + /* + * Wait for any pending sb_write to complete before free. + * The sb_bio is embedded in struct cached_dev, so we must + * ensure no I/O is in progress. + */ + closure_sync(&dc->sb_write); + if (dc->sb_disk) folio_put(virt_to_folio(dc->sb_disk)); diff --git a/drivers/md/dm-exception-store.c b/drivers/md/dm-exception-store.c index c3799757bf4a0c..88f119a0a2ae0d 100644 --- a/drivers/md/dm-exception-store.c +++ b/drivers/md/dm-exception-store.c @@ -116,7 +116,7 @@ int dm_exception_store_type_register(struct dm_exception_store_type *type) if (!__find_exception_store_type(type->name)) list_add(&type->list, &_exception_store_types); else - r = -EEXIST; + r = -EBUSY; spin_unlock(&_lock); return r; diff --git a/drivers/md/dm-integrity.c b/drivers/md/dm-integrity.c index 170bf67a2edd90..ba526310525034 100644 --- a/drivers/md/dm-integrity.c +++ b/drivers/md/dm-integrity.c @@ -2411,7 +2411,7 @@ static void dm_integrity_map_continue(struct dm_integrity_io *dio, bool from_map new_pos = find_journal_node(ic, dio->range.logical_sector, &next_sector); if (unlikely(new_pos != NOT_FOUND) || - unlikely(next_sector < dio->range.logical_sector - dio->range.n_sectors)) { + unlikely(next_sector < dio->range.logical_sector + dio->range.n_sectors)) { remove_range_unlocked(ic, &dio->range); spin_unlock_irq(&ic->endio_wait.lock); queue_work(ic->commit_wq, &ic->commit_work); @@ -3788,14 +3788,27 @@ static void dm_integrity_resume(struct dm_target *ti) struct dm_integrity_c *ic = ti->private; __u64 old_provided_data_sectors = le64_to_cpu(ic->sb->provided_data_sectors); int r; + __le32 flags; DEBUG_print("resume\n"); ic->wrote_to_journal = false; + flags = ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING); + r = sync_rw_sb(ic, REQ_OP_READ); + if (r) + dm_integrity_io_error(ic, "reading superblock", r); + if ((ic->sb->flags & flags) != flags) { + ic->sb->flags |= flags; + r = sync_rw_sb(ic, REQ_OP_WRITE | REQ_FUA); + if (unlikely(r)) + dm_integrity_io_error(ic, "writing superblock", r); + } + if (ic->provided_data_sectors != old_provided_data_sectors) { if (ic->provided_data_sectors > old_provided_data_sectors && ic->mode == 'B' && + ic->sb->flags & cpu_to_le32(SB_FLAG_DIRTY_BITMAP) && ic->sb->log2_blocks_per_bitmap_bit == ic->log2_blocks_per_bitmap_bit) { rw_journal_sectors(ic, REQ_OP_READ, 0, ic->n_bitmap_blocks * (BITMAP_BLOCK_SIZE >> SECTOR_SHIFT), NULL); diff --git a/drivers/md/dm-log.c b/drivers/md/dm-log.c index 9d85d045f9d9d5..bced5a783ee339 100644 --- a/drivers/md/dm-log.c +++ b/drivers/md/dm-log.c @@ -121,7 +121,7 @@ int dm_dirty_log_type_register(struct dm_dirty_log_type *type) if (!__find_dirty_log_type(type->name)) list_add(&type->list, &_log_types); else - r = -EEXIST; + r = -EBUSY; spin_unlock(&_lock); return r; diff --git a/drivers/md/dm-mpath.c b/drivers/md/dm-mpath.c index d5d6ef7ba8381a..aa9a88a9aa7681 100644 --- a/drivers/md/dm-mpath.c +++ b/drivers/md/dm-mpath.c @@ -225,6 +225,7 @@ static struct multipath *alloc_multipath(struct dm_target *ti) mutex_init(&m->work_mutex); m->queue_mode = DM_TYPE_NONE; + m->pg_init_delay_msecs = DM_PG_INIT_DELAY_DEFAULT; m->ti = ti; ti->private = m; @@ -251,7 +252,6 @@ static int alloc_multipath_stage2(struct dm_target *ti, struct multipath *m) set_bit(MPATHF_QUEUE_IO, &m->flags); atomic_set(&m->pg_init_in_progress, 0); atomic_set(&m->pg_init_count, 0); - m->pg_init_delay_msecs = DM_PG_INIT_DELAY_DEFAULT; init_waitqueue_head(&m->pg_init_wait); init_waitqueue_head(&m->probe_wait); @@ -960,27 +960,27 @@ static struct pgpath *parse_path(struct dm_arg_set *as, struct path_selector *ps attached_handler_name = NULL; } else { r = PTR_ERR(attached_handler_name); - goto bad; + ti->error = "error allocating handler name"; + goto bad_put_device; } } if (attached_handler_name || m->hw_handler_name) { INIT_DELAYED_WORK(&p->activate_path, activate_path_work); r = setup_scsi_dh(p->path.dev->bdev, m, &attached_handler_name, &ti->error); kfree(attached_handler_name); - if (r) { - dm_put_device(ti, p->path.dev); - goto bad; - } + if (r) + goto bad_put_device; } r = ps->type->add_path(ps, &p->path, as->argc, as->argv, &ti->error); - if (r) { - dm_put_device(ti, p->path.dev); - goto bad; - } + if (r) + goto bad_put_device; return p; - bad: + +bad_put_device: + dm_put_device(ti, p->path.dev); +bad: free_pgpath(p); return ERR_PTR(r); } diff --git a/drivers/md/dm-path-selector.c b/drivers/md/dm-path-selector.c index d0b883fabfeb6f..2b0ac200f1c02c 100644 --- a/drivers/md/dm-path-selector.c +++ b/drivers/md/dm-path-selector.c @@ -107,7 +107,7 @@ int dm_register_path_selector(struct path_selector_type *pst) if (__find_path_selector_type(pst->name)) { kfree(psi); - r = -EEXIST; + r = -EBUSY; } else list_add(&psi->list, &_path_selectors); diff --git a/drivers/md/dm-rq.c b/drivers/md/dm-rq.c index a6ca92049c10e3..923252fb57aec7 100644 --- a/drivers/md/dm-rq.c +++ b/drivers/md/dm-rq.c @@ -109,14 +109,21 @@ static void end_clone_bio(struct bio *clone) */ tio->completed += nr_bytes; + if (!is_last) + return; + /* + * At this moment we know this is the last bio of the cloned request, + * and all cloned bios have been released, so reset the clone request's + * bio pointer to avoid double free. + */ + tio->clone->bio = NULL; + exit: /* * Update the original request. * Do not use blk_mq_end_request() here, because it may complete * the original request before the clone, and break the ordering. */ - if (is_last) - exit: - blk_update_request(tio->orig, BLK_STS_OK, tio->completed); + blk_update_request(tio->orig, BLK_STS_OK, tio->completed); } static struct dm_rq_target_io *tio_from_request(struct request *rq) @@ -278,8 +285,7 @@ static void dm_complete_request(struct request *rq, blk_status_t error) struct dm_rq_target_io *tio = tio_from_request(rq); tio->error = error; - if (likely(!blk_should_fake_timeout(rq->q))) - blk_mq_complete_request(rq); + blk_mq_complete_request(rq); } /* diff --git a/drivers/md/dm-table.c b/drivers/md/dm-table.c index 0522cd700e0e2d..4b70872725d043 100644 --- a/drivers/md/dm-table.c +++ b/drivers/md/dm-table.c @@ -1237,9 +1237,6 @@ static int dm_wrappedkey_op_callback(struct dm_target *ti, struct dm_dev *dev, bdev_get_queue(bdev)->crypto_profile; int err = -EOPNOTSUPP; - if (!args->err) - return 0; - switch (args->op) { case DERIVE_SW_SECRET: err = blk_crypto_derive_sw_secret( @@ -1266,9 +1263,7 @@ static int dm_wrappedkey_op_callback(struct dm_target *ti, struct dm_dev *dev, break; } args->err = err; - - /* Try another device in case this fails. */ - return 0; + return 1; /* No need to continue the iteration. */ } static int dm_exec_wrappedkey_op(struct blk_crypto_profile *profile, @@ -1294,14 +1289,13 @@ static int dm_exec_wrappedkey_op(struct blk_crypto_profile *profile, * declared on all underlying devices. Thus, all the underlying devices * should support all wrapped key operations and they should behave * identically, i.e. work with the same keys. So, just executing the - * operation on the first device on which it works suffices for now. + * operation on the first device suffices for now. */ for (i = 0; i < t->num_targets; i++) { ti = dm_table_get_target(t, i); if (!ti->type->iterate_devices) continue; - ti->type->iterate_devices(ti, dm_wrappedkey_op_callback, args); - if (!args->err) + if (ti->type->iterate_devices(ti, dm_wrappedkey_op_callback, args) != 0) break; } out: diff --git a/drivers/md/dm-target.c b/drivers/md/dm-target.c index 8fede41adec004..1fd41289de3674 100644 --- a/drivers/md/dm-target.c +++ b/drivers/md/dm-target.c @@ -88,7 +88,7 @@ int dm_register_target(struct target_type *tt) if (__find_target_type(tt->name)) { DMERR("%s: '%s' target already registered", __func__, tt->name); - rv = -EEXIST; + rv = -EBUSY; } else { list_add(&tt->list, &_targets); } diff --git a/drivers/md/dm-unstripe.c b/drivers/md/dm-unstripe.c index e8a9432057dce1..17be483595642c 100644 --- a/drivers/md/dm-unstripe.c +++ b/drivers/md/dm-unstripe.c @@ -117,7 +117,7 @@ static void unstripe_dtr(struct dm_target *ti) static sector_t map_to_core(struct dm_target *ti, struct bio *bio) { struct unstripe_c *uc = ti->private; - sector_t sector = bio->bi_iter.bi_sector; + sector_t sector = dm_target_offset(ti, bio->bi_iter.bi_sector); sector_t tmp_sector = sector; /* Shift us up to the right "row" on the stripe */ diff --git a/drivers/md/dm-verity-fec.c b/drivers/md/dm-verity-fec.c index c79de517afee77..0ac98a620f3ac2 100644 --- a/drivers/md/dm-verity-fec.c +++ b/drivers/md/dm-verity-fec.c @@ -534,9 +534,9 @@ void verity_fec_dtr(struct dm_verity *v) mempool_exit(&f->output_pool); kmem_cache_destroy(f->cache); - if (f->data_bufio) + if (!IS_ERR_OR_NULL(f->data_bufio)) dm_bufio_client_destroy(f->data_bufio); - if (f->bufio) + if (!IS_ERR_OR_NULL(f->bufio)) dm_bufio_client_destroy(f->bufio); if (f->dev) diff --git a/drivers/md/dm-zone.c b/drivers/md/dm-zone.c index c95e417194b33c..f29acf64429a75 100644 --- a/drivers/md/dm-zone.c +++ b/drivers/md/dm-zone.c @@ -50,7 +50,7 @@ int dm_blk_report_zones(struct gendisk *disk, sector_t sector, { struct mapped_device *md = disk->private_data; struct dm_table *map; - struct dm_table *zone_revalidate_map = md->zone_revalidate_map; + struct dm_table *zone_revalidate_map = READ_ONCE(md->zone_revalidate_map); int srcu_idx, ret = -EIO; bool put_table = false; @@ -60,11 +60,13 @@ int dm_blk_report_zones(struct gendisk *disk, sector_t sector, * Zone revalidation during __bind() is in progress, but this * call is from a different process */ - if (dm_suspended_md(md)) - return -EAGAIN; - map = dm_get_live_table(md, &srcu_idx); put_table = true; + + if (dm_suspended_md(md)) { + ret = -EAGAIN; + goto do_put_table; + } } else { /* Zone revalidation during __bind() */ map = zone_revalidate_map; @@ -79,6 +81,7 @@ int dm_blk_report_zones(struct gendisk *disk, sector_t sector, ret = dm_blk_do_report_zones(md, map, nr_zones, &dm_args); } +do_put_table: if (put_table) dm_put_live_table(md, srcu_idx); diff --git a/drivers/md/dm.c b/drivers/md/dm.c index b6327920226012..8e029f6289c10c 100644 --- a/drivers/md/dm.c +++ b/drivers/md/dm.c @@ -1364,6 +1364,8 @@ void dm_submit_bio_remap(struct bio *clone, struct bio *tgt_clone) if (!tgt_clone) tgt_clone = clone; + bio_clone_blkg_association(tgt_clone, io->orig_bio); + /* * Account io->origin_bio to DM dev on behalf of target * that took ownership of IO with DM_MAPIO_SUBMITTED. diff --git a/drivers/md/md-bitmap.c b/drivers/md/md-bitmap.c index 84b7e2af6dbaa5..7bb56d0491a2fb 100644 --- a/drivers/md/md-bitmap.c +++ b/drivers/md/md-bitmap.c @@ -2453,6 +2453,7 @@ static int __bitmap_resize(struct bitmap *bitmap, sector_t blocks, memcpy(page_address(store.sb_page), page_address(bitmap->storage.sb_page), sizeof(bitmap_super_t)); + mutex_lock(&bitmap->mddev->bitmap_info.mutex); spin_lock_irq(&bitmap->counts.lock); md_bitmap_file_unmap(&bitmap->storage); bitmap->storage = store; @@ -2560,7 +2561,7 @@ static int __bitmap_resize(struct bitmap *bitmap, sector_t blocks, set_page_attr(bitmap, i, BITMAP_PAGE_DIRTY); } spin_unlock_irq(&bitmap->counts.lock); - + mutex_unlock(&bitmap->mddev->bitmap_info.mutex); if (!init) { __bitmap_unplug(bitmap); bitmap->mddev->pers->quiesce(bitmap->mddev, 0); diff --git a/drivers/md/md-cluster.c b/drivers/md/md-cluster.c index 11f1e91d387d8c..896279988dfd5f 100644 --- a/drivers/md/md-cluster.c +++ b/drivers/md/md-cluster.c @@ -549,8 +549,13 @@ static void process_metadata_update(struct mddev *mddev, struct cluster_msg *msg dlm_lock_sync(cinfo->no_new_dev_lockres, DLM_LOCK_CR); - /* daemaon thread must exist */ thread = rcu_dereference_protected(mddev->thread, true); + if (!thread) { + pr_warn("md-cluster: Received metadata update but MD thread is not ready\n"); + dlm_unlock_sync(cinfo->no_new_dev_lockres); + return; + } + wait_event(thread->wqueue, (got_lock = mddev_trylock(mddev)) || test_bit(MD_CLUSTER_HOLDING_MUTEX_FOR_RECVD, &cinfo->state)); diff --git a/drivers/md/md-llbitmap.c b/drivers/md/md-llbitmap.c index 9c1ade19b7741e..cd713a7dc27064 100644 --- a/drivers/md/md-llbitmap.c +++ b/drivers/md/md-llbitmap.c @@ -712,8 +712,10 @@ static int llbitmap_suspend_timeout(struct llbitmap *llbitmap, int page_idx) percpu_ref_kill(&pctl->active); if (!wait_event_timeout(pctl->wait, percpu_ref_is_zero(&pctl->active), - llbitmap->mddev->bitmap_info.daemon_sleep * HZ)) + llbitmap->mddev->bitmap_info.daemon_sleep * HZ)) { + percpu_ref_resurrect(&pctl->active); return -ETIMEDOUT; + } return 0; } diff --git a/drivers/md/md.c b/drivers/md/md.c index 6d73f6e196a9f9..ac71640ff3a817 100644 --- a/drivers/md/md.c +++ b/drivers/md/md.c @@ -6848,13 +6848,15 @@ static void __md_stop_writes(struct mddev *mddev) { timer_delete_sync(&mddev->safemode_timer); - if (mddev->pers && mddev->pers->quiesce) { - mddev->pers->quiesce(mddev, 1); - mddev->pers->quiesce(mddev, 0); - } + if (md_is_rdwr(mddev) || !mddev_is_dm(mddev)) { + if (mddev->pers && mddev->pers->quiesce) { + mddev->pers->quiesce(mddev, 1); + mddev->pers->quiesce(mddev, 0); + } - if (md_bitmap_enabled(mddev, true)) - mddev->bitmap_ops->flush(mddev); + if (md_bitmap_enabled(mddev, true)) + mddev->bitmap_ops->flush(mddev); + } if (md_is_rdwr(mddev) && ((!mddev->in_sync && !mddev_is_clustered(mddev)) || diff --git a/drivers/md/md.h b/drivers/md/md.h index 6985f2829bbd6a..3bfbee595156d4 100644 --- a/drivers/md/md.h +++ b/drivers/md/md.h @@ -737,8 +737,8 @@ static inline int mddev_trylock(struct mddev *mddev) int ret; ret = mutex_trylock(&mddev->reconfig_mutex); - if (!ret && test_bit(MD_DELETED, &mddev->flags)) { - ret = -ENODEV; + if (ret && test_bit(MD_DELETED, &mddev->flags)) { + ret = 0; mutex_unlock(&mddev->reconfig_mutex); } return ret; diff --git a/drivers/md/raid1.c b/drivers/md/raid1.c index 57d50465eed1b7..cc9914bd15c197 100644 --- a/drivers/md/raid1.c +++ b/drivers/md/raid1.c @@ -3254,6 +3254,7 @@ static int raid1_run(struct mddev *mddev) if (!mddev_is_dm(mddev)) { ret = raid1_set_limits(mddev); if (ret) { + md_unregister_thread(mddev, &conf->thread); if (!mddev->private) raid1_free(mddev, conf); return ret; diff --git a/drivers/md/raid10.c b/drivers/md/raid10.c index 84be4cc7e87399..3a591e60a14497 100644 --- a/drivers/md/raid10.c +++ b/drivers/md/raid10.c @@ -3402,7 +3402,6 @@ static sector_t raid10_sync_request(struct mddev *mddev, sector_t sector_nr, !test_bit(In_sync, &rdev->flags)) continue; /* This is where we read from */ - any_working = 1; sector = r10_bio->devs[j].addr; if (is_badblock(rdev, sector, max_sync, @@ -3417,6 +3416,7 @@ static sector_t raid10_sync_request(struct mddev *mddev, sector_t sector_nr, continue; } } + any_working = 1; bio = r10_bio->devs[0].bio; bio->bi_next = biolist; biolist = bio; diff --git a/drivers/md/raid5.c b/drivers/md/raid5.c index 8dc98f545969ff..bdf248db1330ae 100644 --- a/drivers/md/raid5.c +++ b/drivers/md/raid5.c @@ -3751,9 +3751,14 @@ static int need_this_block(struct stripe_head *sh, struct stripe_head_state *s, struct r5dev *dev = &sh->dev[disk_idx]; struct r5dev *fdev[2] = { &sh->dev[s->failed_num[0]], &sh->dev[s->failed_num[1]] }; + struct mddev *mddev = sh->raid_conf->mddev; + bool force_rcw = false; int i; - bool force_rcw = (sh->raid_conf->rmw_level == PARITY_DISABLE_RMW); + if (sh->raid_conf->rmw_level == PARITY_DISABLE_RMW || + (mddev->bitmap_ops && mddev->bitmap_ops->blocks_synced && + !mddev->bitmap_ops->blocks_synced(mddev, sh->sector))) + force_rcw = true; if (test_bit(R5_LOCKED, &dev->flags) || test_bit(R5_UPTODATE, &dev->flags)) @@ -8057,7 +8062,8 @@ static int raid5_run(struct mddev *mddev) goto abort; } - if (log_init(conf, journal_dev, raid5_has_ppl(conf))) + ret = log_init(conf, journal_dev, raid5_has_ppl(conf)); + if (ret) goto abort; return 0; diff --git a/drivers/media/common/videobuf2/videobuf2-dma-sg.c b/drivers/media/common/videobuf2/videobuf2-dma-sg.c index b3bf2173c14e1b..7c30731cb9a57b 100644 --- a/drivers/media/common/videobuf2/videobuf2-dma-sg.c +++ b/drivers/media/common/videobuf2/videobuf2-dma-sg.c @@ -345,6 +345,7 @@ static int vb2_dma_sg_mmap(void *buf_priv, struct vm_area_struct *vma) return err; } + vm_flags_set(vma, VM_DONTEXPAND | VM_DONTDUMP); /* * Use common vm_area operations to track buffer refcount. */ diff --git a/drivers/media/dvb-core/dmxdev.c b/drivers/media/dvb-core/dmxdev.c index 8c6f5aafda1d61..9aaae55ce7b4eb 100644 --- a/drivers/media/dvb-core/dmxdev.c +++ b/drivers/media/dvb-core/dmxdev.c @@ -168,7 +168,9 @@ static int dvb_dvr_open(struct inode *inode, struct file *file) mutex_unlock(&dmxdev->mutex); return -ENOMEM; } - dvb_ringbuffer_init(&dmxdev->dvr_buffer, mem, DVR_BUFFER_SIZE); + dmxdev->dvr_buffer.data = mem; + dmxdev->dvr_buffer.size = DVR_BUFFER_SIZE; + dvb_ringbuffer_reset(&dmxdev->dvr_buffer); if (dmxdev->may_do_mmap) dvb_vb2_init(&dmxdev->dvr_vb2_ctx, "dvr", file->f_flags & O_NONBLOCK); @@ -397,11 +399,11 @@ static int dvb_dmxdev_section_callback(const u8 *buffer1, size_t buffer1_len, if (dvb_vb2_is_streaming(&dmxdevfilter->vb2_ctx)) { ret = dvb_vb2_fill_buffer(&dmxdevfilter->vb2_ctx, buffer1, buffer1_len, - buffer_flags); + buffer_flags, true); if (ret == buffer1_len) ret = dvb_vb2_fill_buffer(&dmxdevfilter->vb2_ctx, buffer2, buffer2_len, - buffer_flags); + buffer_flags, true); } else { ret = dvb_dmxdev_buffer_write(&dmxdevfilter->buffer, buffer1, buffer1_len); @@ -452,10 +454,10 @@ static int dvb_dmxdev_ts_callback(const u8 *buffer1, size_t buffer1_len, if (dvb_vb2_is_streaming(ctx)) { ret = dvb_vb2_fill_buffer(ctx, buffer1, buffer1_len, - buffer_flags); + buffer_flags, false); if (ret == buffer1_len) ret = dvb_vb2_fill_buffer(ctx, buffer2, buffer2_len, - buffer_flags); + buffer_flags, false); } else { if (buffer->error) { spin_unlock(&dmxdevfilter->dev->lock); diff --git a/drivers/media/dvb-core/dvb_net.c b/drivers/media/dvb-core/dvb_net.c index 8bb8dd34c223e9..a2159b2bc17664 100644 --- a/drivers/media/dvb-core/dvb_net.c +++ b/drivers/media/dvb-core/dvb_net.c @@ -228,6 +228,9 @@ static int handle_one_ule_extension( struct dvb_net_priv *p ) unsigned char hlen = (p->ule_sndu_type & 0x0700) >> 8; unsigned char htype = p->ule_sndu_type & 0x00FF; + if (htype >= ARRAY_SIZE(ule_mandatory_ext_handlers)) + return -1; + /* Discriminate mandatory and optional extension headers. */ if (hlen == 0) { /* Mandatory extension header */ diff --git a/drivers/media/dvb-core/dvb_vb2.c b/drivers/media/dvb-core/dvb_vb2.c index 29edaaff7a5c9d..7444bbc2f24d9b 100644 --- a/drivers/media/dvb-core/dvb_vb2.c +++ b/drivers/media/dvb-core/dvb_vb2.c @@ -249,7 +249,8 @@ int dvb_vb2_is_streaming(struct dvb_vb2_ctx *ctx) int dvb_vb2_fill_buffer(struct dvb_vb2_ctx *ctx, const unsigned char *src, int len, - enum dmx_buffer_flags *buffer_flags) + enum dmx_buffer_flags *buffer_flags, + bool flush) { unsigned long flags = 0; void *vbuf = NULL; @@ -306,7 +307,7 @@ int dvb_vb2_fill_buffer(struct dvb_vb2_ctx *ctx, } } - if (ctx->nonblocking && ctx->buf) { + if (flush && ctx->buf) { vb2_set_plane_payload(&ctx->buf->vb, 0, ll); vb2_buffer_done(&ctx->buf->vb, VB2_BUF_STATE_DONE); list_del(&ctx->buf->list); diff --git a/drivers/media/i2c/adv7180.c b/drivers/media/i2c/adv7180.c index 378f4e6af12cba..5cbc973df684d1 100644 --- a/drivers/media/i2c/adv7180.c +++ b/drivers/media/i2c/adv7180.c @@ -507,6 +507,13 @@ static int adv7180_get_frame_interval(struct v4l2_subdev *sd, fi->interval.denominator = 25; } + /* + * If the de-interlacer is active, the chip produces full video frames + * at the field rate. + */ + if (state->field == V4L2_FIELD_NONE) + fi->interval.denominator *= 2; + return 0; } diff --git a/drivers/media/i2c/ccs/ccs-core.c b/drivers/media/i2c/ccs/ccs-core.c index f8523140784c71..8d30e808a635bc 100644 --- a/drivers/media/i2c/ccs/ccs-core.c +++ b/drivers/media/i2c/ccs/ccs-core.c @@ -2346,7 +2346,7 @@ static void ccs_set_compose_scaler(struct v4l2_subdev *subdev, * CCS_LIM(sensor, SCALER_N_MIN) / sel->r.height; max_m = crops[CCS_PAD_SINK]->width * CCS_LIM(sensor, SCALER_N_MIN) - / CCS_LIM(sensor, MIN_X_OUTPUT_SIZE); + / (CCS_LIM(sensor, MIN_X_OUTPUT_SIZE) ?: 1); a = clamp(a, CCS_LIM(sensor, SCALER_M_MIN), CCS_LIM(sensor, SCALER_M_MAX)); @@ -2940,6 +2940,8 @@ static void ccs_cleanup(struct ccs_sensor *sensor) ccs_free_controls(sensor); } +static const struct v4l2_subdev_internal_ops ccs_internal_ops; + static int ccs_init_subdev(struct ccs_sensor *sensor, struct ccs_subdev *ssd, const char *name, unsigned short num_pads, u32 function, @@ -2952,8 +2954,10 @@ static int ccs_init_subdev(struct ccs_sensor *sensor, if (!ssd) return 0; - if (ssd != sensor->src) + if (ssd != sensor->src) { v4l2_subdev_init(&ssd->sd, &ccs_ops); + ssd->sd.internal_ops = &ccs_internal_ops; + } ssd->sd.flags |= V4L2_SUBDEV_FL_HAS_DEVNODE; ssd->sd.entity.function = function; @@ -3062,6 +3066,10 @@ static const struct media_entity_operations ccs_entity_ops = { .link_validate = v4l2_subdev_link_validate, }; +static const struct v4l2_subdev_internal_ops ccs_internal_ops = { + .init_state = ccs_init_state, +}; + static const struct v4l2_subdev_internal_ops ccs_internal_src_ops = { .init_state = ccs_init_state, .registered = ccs_registered, @@ -3425,7 +3433,21 @@ static int ccs_probe(struct i2c_client *client) sensor->scale_m = CCS_LIM(sensor, SCALER_N_MIN); /* prepare PLL configuration input values */ - sensor->pll.bus_type = CCS_PLL_BUS_TYPE_CSI2_DPHY; + switch (sensor->hwcfg.csi_signalling_mode) { + case CCS_CSI_SIGNALING_MODE_CSI_2_CPHY: + sensor->pll.bus_type = CCS_PLL_BUS_TYPE_CSI2_CPHY; + break; + case CCS_CSI_SIGNALING_MODE_CSI_2_DPHY: + case SMIAPP_CSI_SIGNALLING_MODE_CCP2_DATA_CLOCK: + case SMIAPP_CSI_SIGNALLING_MODE_CCP2_DATA_STROBE: + sensor->pll.bus_type = CCS_PLL_BUS_TYPE_CSI2_DPHY; + break; + default: + dev_err(&client->dev, "unsupported signalling mode %u\n", + sensor->hwcfg.csi_signalling_mode); + rval = -EINVAL; + goto out_cleanup; + } sensor->pll.csi2.lanes = sensor->hwcfg.lanes; if (CCS_LIM(sensor, CLOCK_CALCULATION) & CCS_CLOCK_CALCULATION_LANE_SPEED) { diff --git a/drivers/media/i2c/dw9714.c b/drivers/media/i2c/dw9714.c index 1e7ad355a388cf..3288de539452eb 100644 --- a/drivers/media/i2c/dw9714.c +++ b/drivers/media/i2c/dw9714.c @@ -149,7 +149,7 @@ static int dw9714_power_up(struct dw9714_device *dw9714_dev) gpiod_set_value_cansleep(dw9714_dev->powerdown_gpio, 0); - usleep_range(1000, 2000); + usleep_range(12000, 14000); return 0; } diff --git a/drivers/media/i2c/mt9m114.c b/drivers/media/i2c/mt9m114.c index 51ebbe7ae99695..b1325e2cd1321e 100644 --- a/drivers/media/i2c/mt9m114.c +++ b/drivers/media/i2c/mt9m114.c @@ -2360,11 +2360,17 @@ static int mt9m114_parse_dt(struct mt9m114 *sensor) struct fwnode_handle *ep; int ret; + /* + * On ACPI systems the fwnode graph can be initialized by a bridge + * driver, which may not have probed yet. Wait for this. + * + * TODO: Return an error once bridge driver code will have moved + * to the ACPI core. + */ ep = fwnode_graph_get_next_endpoint(fwnode, NULL); - if (!ep) { - dev_err(&sensor->client->dev, "No endpoint found\n"); - return -EINVAL; - } + if (!ep) + return dev_err_probe(&sensor->client->dev, -EPROBE_DEFER, + "waiting for fwnode graph endpoint\n"); sensor->bus_cfg.bus_type = V4L2_MBUS_UNKNOWN; ret = v4l2_fwnode_endpoint_alloc_parse(ep, &sensor->bus_cfg); @@ -2434,7 +2440,7 @@ static int mt9m114_probe(struct i2c_client *client) goto error_ep_free; } - sensor->reset = devm_gpiod_get_optional(dev, "reset", GPIOD_OUT_LOW); + sensor->reset = devm_gpiod_get_optional(dev, "reset", GPIOD_OUT_HIGH); if (IS_ERR(sensor->reset)) { ret = PTR_ERR(sensor->reset); dev_err_probe(dev, ret, "Failed to get reset GPIO\n"); diff --git a/drivers/media/i2c/ov01a10.c b/drivers/media/i2c/ov01a10.c index 141cb6f75b5550..c1a7373a6311cf 100644 --- a/drivers/media/i2c/ov01a10.c +++ b/drivers/media/i2c/ov01a10.c @@ -16,7 +16,7 @@ #include #define OV01A10_LINK_FREQ_400MHZ 400000000ULL -#define OV01A10_SCLK 40000000LL +#define OV01A10_SCLK 80000000LL #define OV01A10_DATA_LANES 1 #define OV01A10_REG_CHIP_ID 0x300a @@ -48,7 +48,7 @@ /* analog gain controls */ #define OV01A10_REG_ANALOG_GAIN 0x3508 #define OV01A10_ANAL_GAIN_MIN 0x100 -#define OV01A10_ANAL_GAIN_MAX 0xffff +#define OV01A10_ANAL_GAIN_MAX 0x3fff #define OV01A10_ANAL_GAIN_STEP 1 /* digital gain controls */ @@ -57,7 +57,7 @@ #define OV01A10_REG_DIGITAL_GAIN_GR 0x3513 #define OV01A10_REG_DIGITAL_GAIN_R 0x3516 #define OV01A10_DGTL_GAIN_MIN 0 -#define OV01A10_DGTL_GAIN_MAX 0x3ffff +#define OV01A10_DGTL_GAIN_MAX 0x3fff #define OV01A10_DGTL_GAIN_STEP 1 #define OV01A10_DGTL_GAIN_DEFAULT 1024 @@ -75,6 +75,15 @@ #define OV01A10_REG_X_WIN 0x3811 #define OV01A10_REG_Y_WIN 0x3813 +/* + * The native ov01a10 bayer-pattern is GBRG, but there was a driver bug enabling + * hflip/mirroring by default resulting in BGGR. Because of this bug Intel's + * proprietary IPU6 userspace stack expects BGGR. So we report BGGR to not break + * userspace and fix things up by shifting the crop window-x coordinate by 1 + * when hflip is *disabled*. + */ +#define OV01A10_MEDIA_BUS_FMT MEDIA_BUS_FMT_SBGGR10_1X10 + struct ov01a10_reg { u16 address; u8 val; @@ -185,14 +194,14 @@ static const struct ov01a10_reg sensor_1280x800_setting[] = { {0x380e, 0x03}, {0x380f, 0x80}, {0x3810, 0x00}, - {0x3811, 0x08}, + {0x3811, 0x09}, {0x3812, 0x00}, {0x3813, 0x08}, {0x3814, 0x01}, {0x3815, 0x01}, {0x3816, 0x01}, {0x3817, 0x01}, - {0x3820, 0xa0}, + {0x3820, 0xa8}, {0x3822, 0x13}, {0x3832, 0x28}, {0x3833, 0x10}, @@ -240,9 +249,8 @@ static const struct ov01a10_reg sensor_1280x800_setting[] = { static const char * const ov01a10_test_pattern_menu[] = { "Disabled", "Color Bar", - "Top-Bottom Darker Color Bar", - "Right-Left Darker Color Bar", - "Color Bar type 4", + "Left-Right Darker Color Bar", + "Bottom-Top Darker Color Bar", }; static const s64 link_freq_menu_items[] = { @@ -397,10 +405,8 @@ static int ov01a10_update_digital_gain(struct ov01a10 *ov01a10, u32 d_gain) static int ov01a10_test_pattern(struct ov01a10 *ov01a10, u32 pattern) { - if (!pattern) - return 0; - - pattern = (pattern - 1) | OV01A10_TEST_PATTERN_ENABLE; + if (pattern) + pattern |= OV01A10_TEST_PATTERN_ENABLE; return ov01a10_write_reg(ov01a10, OV01A10_REG_TEST_PATTERN, 1, pattern); } @@ -411,7 +417,7 @@ static int ov01a10_set_hflip(struct ov01a10 *ov01a10, u32 hflip) int ret; u32 val, offset; - offset = hflip ? 0x9 : 0x8; + offset = hflip ? 0x8 : 0x9; ret = ov01a10_write_reg(ov01a10, OV01A10_REG_X_WIN, 1, offset); if (ret) return ret; @@ -420,8 +426,8 @@ static int ov01a10_set_hflip(struct ov01a10 *ov01a10, u32 hflip) if (ret) return ret; - val = hflip ? val | FIELD_PREP(OV01A10_HFLIP_MASK, 0x1) : - val & ~OV01A10_HFLIP_MASK; + val = hflip ? val & ~OV01A10_HFLIP_MASK : + val | FIELD_PREP(OV01A10_HFLIP_MASK, 0x1); return ov01a10_write_reg(ov01a10, OV01A10_REG_FORMAT1, 1, val); } @@ -610,7 +616,7 @@ static void ov01a10_update_pad_format(const struct ov01a10_mode *mode, { fmt->width = mode->width; fmt->height = mode->height; - fmt->code = MEDIA_BUS_FMT_SBGGR10_1X10; + fmt->code = OV01A10_MEDIA_BUS_FMT; fmt->field = V4L2_FIELD_NONE; fmt->colorspace = V4L2_COLORSPACE_RAW; } @@ -722,7 +728,7 @@ static int ov01a10_set_format(struct v4l2_subdev *sd, h_blank); } - format = v4l2_subdev_state_get_format(sd_state, fmt->stream); + format = v4l2_subdev_state_get_format(sd_state, fmt->pad); *format = fmt->format; return 0; @@ -751,7 +757,7 @@ static int ov01a10_enum_mbus_code(struct v4l2_subdev *sd, if (code->index > 0) return -EINVAL; - code->code = MEDIA_BUS_FMT_SBGGR10_1X10; + code->code = OV01A10_MEDIA_BUS_FMT; return 0; } @@ -761,7 +767,7 @@ static int ov01a10_enum_frame_size(struct v4l2_subdev *sd, struct v4l2_subdev_frame_size_enum *fse) { if (fse->index >= ARRAY_SIZE(supported_modes) || - fse->code != MEDIA_BUS_FMT_SBGGR10_1X10) + fse->code != OV01A10_MEDIA_BUS_FMT) return -EINVAL; fse->min_width = supported_modes[fse->index].width; @@ -855,6 +861,7 @@ static void ov01a10_remove(struct i2c_client *client) struct v4l2_subdev *sd = i2c_get_clientdata(client); v4l2_async_unregister_subdev(sd); + v4l2_subdev_cleanup(sd); media_entity_cleanup(&sd->entity); v4l2_ctrl_handler_free(sd->ctrl_handler); @@ -925,6 +932,7 @@ static int ov01a10_probe(struct i2c_client *client) err_pm_disable: pm_runtime_disable(dev); pm_runtime_set_suspended(&client->dev); + v4l2_subdev_cleanup(&ov01a10->sd); err_media_entity_cleanup: media_entity_cleanup(&ov01a10->sd.entity); diff --git a/drivers/media/i2c/ov5647.c b/drivers/media/i2c/ov5647.c index e193fef4fcedf4..5fb10e02ba6e24 100644 --- a/drivers/media/i2c/ov5647.c +++ b/drivers/media/i2c/ov5647.c @@ -69,11 +69,11 @@ #define OV5647_NATIVE_HEIGHT 1956U #define OV5647_PIXEL_ARRAY_LEFT 16U -#define OV5647_PIXEL_ARRAY_TOP 16U +#define OV5647_PIXEL_ARRAY_TOP 6U #define OV5647_PIXEL_ARRAY_WIDTH 2592U #define OV5647_PIXEL_ARRAY_HEIGHT 1944U -#define OV5647_VBLANK_MIN 4 +#define OV5647_VBLANK_MIN 24 #define OV5647_VTS_MAX 32767 #define OV5647_EXPOSURE_MIN 4 @@ -508,7 +508,7 @@ static const struct ov5647_mode ov5647_modes[] = { { .format = { .code = MEDIA_BUS_FMT_SBGGR10_1X10, - .colorspace = V4L2_COLORSPACE_SRGB, + .colorspace = V4L2_COLORSPACE_RAW, .field = V4L2_FIELD_NONE, .width = 2592, .height = 1944 @@ -529,7 +529,7 @@ static const struct ov5647_mode ov5647_modes[] = { { .format = { .code = MEDIA_BUS_FMT_SBGGR10_1X10, - .colorspace = V4L2_COLORSPACE_SRGB, + .colorspace = V4L2_COLORSPACE_RAW, .field = V4L2_FIELD_NONE, .width = 1920, .height = 1080 @@ -550,7 +550,7 @@ static const struct ov5647_mode ov5647_modes[] = { { .format = { .code = MEDIA_BUS_FMT_SBGGR10_1X10, - .colorspace = V4L2_COLORSPACE_SRGB, + .colorspace = V4L2_COLORSPACE_RAW, .field = V4L2_FIELD_NONE, .width = 1296, .height = 972 @@ -571,7 +571,7 @@ static const struct ov5647_mode ov5647_modes[] = { { .format = { .code = MEDIA_BUS_FMT_SBGGR10_1X10, - .colorspace = V4L2_COLORSPACE_SRGB, + .colorspace = V4L2_COLORSPACE_RAW, .field = V4L2_FIELD_NONE, .width = 640, .height = 480 @@ -582,7 +582,7 @@ static const struct ov5647_mode ov5647_modes[] = { .width = 2560, .height = 1920, }, - .pixel_rate = 55000000, + .pixel_rate = 58333000, .hts = 1852, .vts = 0x1f8, .reg_list = ov5647_640x480_10bpp, @@ -1291,6 +1291,8 @@ static int ov5647_init_controls(struct ov5647 *sensor) v4l2_ctrl_handler_init(&sensor->ctrls, 9); + sensor->ctrls.lock = &sensor->lock; + v4l2_ctrl_new_std(&sensor->ctrls, &ov5647_ctrl_ops, V4L2_CID_AUTOGAIN, 0, 1, 1, 0); @@ -1420,15 +1422,15 @@ static int ov5647_probe(struct i2c_client *client) sensor->mode = OV5647_DEFAULT_MODE; - ret = ov5647_init_controls(sensor); - if (ret) - goto mutex_destroy; - sd = &sensor->sd; v4l2_i2c_subdev_init(sd, client, &ov5647_subdev_ops); sd->internal_ops = &ov5647_subdev_internal_ops; sd->flags |= V4L2_SUBDEV_FL_HAS_DEVNODE | V4L2_SUBDEV_FL_HAS_EVENTS; + ret = ov5647_init_controls(sensor); + if (ret) + goto mutex_destroy; + sensor->pad.flags = MEDIA_PAD_FL_SOURCE; sd->entity.function = MEDIA_ENT_F_CAM_SENSOR; ret = media_entity_pads_init(&sd->entity, 1, &sensor->pad); diff --git a/drivers/media/i2c/tw9903.c b/drivers/media/i2c/tw9903.c index b996a05e56f28f..c3eafd5d5dc820 100644 --- a/drivers/media/i2c/tw9903.c +++ b/drivers/media/i2c/tw9903.c @@ -228,6 +228,7 @@ static int tw9903_probe(struct i2c_client *client) if (write_regs(sd, initial_registers) < 0) { v4l2_err(client, "error initializing TW9903\n"); + v4l2_ctrl_handler_free(hdl); return -EINVAL; } diff --git a/drivers/media/i2c/tw9906.c b/drivers/media/i2c/tw9906.c index 6220f4fddbabcd..0ab43fe42d7f43 100644 --- a/drivers/media/i2c/tw9906.c +++ b/drivers/media/i2c/tw9906.c @@ -196,6 +196,7 @@ static int tw9906_probe(struct i2c_client *client) if (write_regs(sd, initial_registers) < 0) { v4l2_err(client, "error initializing TW9906\n"); + v4l2_ctrl_handler_free(hdl); return -EINVAL; } diff --git a/drivers/media/mc/mc-request.c b/drivers/media/mc/mc-request.c index 3cca9a0c7c9731..e1d2eeb5fb48c7 100644 --- a/drivers/media/mc/mc-request.c +++ b/drivers/media/mc/mc-request.c @@ -190,6 +190,8 @@ static long media_request_ioctl_reinit(struct media_request *req) struct media_device *mdev = req->mdev; unsigned long flags; + mutex_lock(&mdev->req_queue_mutex); + spin_lock_irqsave(&req->lock, flags); if (req->state != MEDIA_REQUEST_STATE_IDLE && req->state != MEDIA_REQUEST_STATE_COMPLETE) { @@ -197,6 +199,7 @@ static long media_request_ioctl_reinit(struct media_request *req) "request: %s not in idle or complete state, cannot reinit\n", req->debug_str); spin_unlock_irqrestore(&req->lock, flags); + mutex_unlock(&mdev->req_queue_mutex); return -EBUSY; } if (req->access_count) { @@ -204,6 +207,7 @@ static long media_request_ioctl_reinit(struct media_request *req) "request: %s is being accessed, cannot reinit\n", req->debug_str); spin_unlock_irqrestore(&req->lock, flags); + mutex_unlock(&mdev->req_queue_mutex); return -EBUSY; } req->state = MEDIA_REQUEST_STATE_CLEANING; @@ -214,6 +218,7 @@ static long media_request_ioctl_reinit(struct media_request *req) spin_lock_irqsave(&req->lock, flags); req->state = MEDIA_REQUEST_STATE_IDLE; spin_unlock_irqrestore(&req->lock, flags); + mutex_unlock(&mdev->req_queue_mutex); return 0; } diff --git a/drivers/media/pci/cx23885/cx23885-alsa.c b/drivers/media/pci/cx23885/cx23885-alsa.c index 25dc8d4dc5b73a..717fc6c9ef21f8 100644 --- a/drivers/media/pci/cx23885/cx23885-alsa.c +++ b/drivers/media/pci/cx23885/cx23885-alsa.c @@ -392,8 +392,10 @@ static int snd_cx23885_hw_params(struct snd_pcm_substream *substream, ret = cx23885_risc_databuffer(chip->pci, &buf->risc, buf->sglist, chip->period_size, chip->num_periods, 1); - if (ret < 0) + if (ret < 0) { + cx23885_alsa_dma_unmap(chip); goto error; + } /* Loop back to start of program */ buf->risc.jmp[0] = cpu_to_le32(RISC_JUMP|RISC_IRQ1|RISC_CNT_INC); diff --git a/drivers/media/pci/cx25821/cx25821-alsa.c b/drivers/media/pci/cx25821/cx25821-alsa.c index a42f0c03a7ca86..f463365163b7ec 100644 --- a/drivers/media/pci/cx25821/cx25821-alsa.c +++ b/drivers/media/pci/cx25821/cx25821-alsa.c @@ -535,6 +535,7 @@ static int snd_cx25821_hw_params(struct snd_pcm_substream *substream, chip->period_size, chip->num_periods, 1); if (ret < 0) { pr_info("DEBUG: ERROR after cx25821_risc_databuffer_audio()\n"); + cx25821_alsa_dma_unmap(chip); goto error; } diff --git a/drivers/media/pci/cx25821/cx25821-core.c b/drivers/media/pci/cx25821/cx25821-core.c index 6627fa9166d301..a7336be444748a 100644 --- a/drivers/media/pci/cx25821/cx25821-core.c +++ b/drivers/media/pci/cx25821/cx25821-core.c @@ -908,6 +908,7 @@ static int cx25821_dev_setup(struct cx25821_dev *dev) if (!dev->lmmio) { CX25821_ERR("ioremap failed, maybe increasing __VMALLOC_RESERVE in page.h\n"); + release_mem_region(dev->base_io_addr, pci_resource_len(dev->pci, 0)); cx25821_iounmap(dev); return -ENOMEM; } diff --git a/drivers/media/pci/cx88/cx88-alsa.c b/drivers/media/pci/cx88/cx88-alsa.c index 29fb1311e44349..4e574d8390b4d7 100644 --- a/drivers/media/pci/cx88/cx88-alsa.c +++ b/drivers/media/pci/cx88/cx88-alsa.c @@ -483,8 +483,10 @@ static int snd_cx88_hw_params(struct snd_pcm_substream *substream, ret = cx88_risc_databuffer(chip->pci, &buf->risc, buf->sglist, chip->period_size, chip->num_periods, 1); - if (ret < 0) + if (ret < 0) { + cx88_alsa_dma_unmap(chip); goto error; + } /* Loop back to start of program */ buf->risc.jmp[0] = cpu_to_le32(RISC_JUMP | RISC_IRQ1 | RISC_CNT_INC); diff --git a/drivers/media/pci/intel/ipu6/ipu6-isys-queue.c b/drivers/media/pci/intel/ipu6/ipu6-isys-queue.c index aa2cf7287477cc..8f05987cdb4e75 100644 --- a/drivers/media/pci/intel/ipu6/ipu6-isys-queue.c +++ b/drivers/media/pci/intel/ipu6/ipu6-isys-queue.c @@ -3,6 +3,7 @@ * Copyright (C) 2013--2024 Intel Corporation */ #include +#include #include #include #include @@ -201,6 +202,8 @@ static int buffer_list_get(struct ipu6_isys_stream *stream, unsigned long flags; unsigned long buf_flag = IPU6_ISYS_BUFFER_LIST_FL_INCOMING; + lockdep_assert_held(&stream->mutex); + bl->nbufs = 0; INIT_LIST_HEAD(&bl->head); @@ -294,9 +297,8 @@ static int ipu6_isys_stream_start(struct ipu6_isys_video *av, struct ipu6_isys_buffer_list __bl; int ret; - mutex_lock(&stream->isys->stream_mutex); + guard(mutex)(&stream->isys->stream_mutex); ret = ipu6_isys_video_set_streaming(av, 1, bl); - mutex_unlock(&stream->isys->stream_mutex); if (ret) goto out_requeue; @@ -637,10 +639,10 @@ static void stop_streaming(struct vb2_queue *q) mutex_lock(&av->isys->stream_mutex); if (stream->nr_streaming == stream->nr_queues && stream->streaming) ipu6_isys_video_set_streaming(av, 0, NULL); + list_del(&aq->node); mutex_unlock(&av->isys->stream_mutex); stream->nr_streaming--; - list_del(&aq->node); stream->streaming = 0; mutex_unlock(&stream->mutex); diff --git a/drivers/media/pci/intel/ipu6/ipu6-isys-video.c b/drivers/media/pci/intel/ipu6/ipu6-isys-video.c index dec8f5ffcfa5fe..54d861aca00887 100644 --- a/drivers/media/pci/intel/ipu6/ipu6-isys-video.c +++ b/drivers/media/pci/intel/ipu6/ipu6-isys-video.c @@ -1036,11 +1036,10 @@ int ipu6_isys_video_set_streaming(struct ipu6_isys_video *av, int state, sd->name, r_pad->index, stream_mask); ret = v4l2_subdev_disable_streams(sd, r_pad->index, stream_mask); - if (ret) { + if (ret) dev_err(dev, "stream off %s failed with %d\n", sd->name, ret); - return ret; - } + close_streaming_firmware(av); } else { ret = start_stream_firmware(av, bl); @@ -1066,6 +1065,7 @@ int ipu6_isys_video_set_streaming(struct ipu6_isys_video *av, int state, out_media_entity_stop_streaming_firmware: stop_streaming_firmware(av); + close_streaming_firmware(av); return ret; } diff --git a/drivers/media/pci/intel/ipu6/ipu6-mmu.c b/drivers/media/pci/intel/ipu6/ipu6-mmu.c index 6d1c0b90169d40..85cc6d5b4dd11e 100644 --- a/drivers/media/pci/intel/ipu6/ipu6-mmu.c +++ b/drivers/media/pci/intel/ipu6/ipu6-mmu.c @@ -102,7 +102,7 @@ static void page_table_dump(struct ipu6_mmu_info *mmu_info) if (mmu_info->l1_pt[l1_idx] == mmu_info->dummy_l2_pteval) continue; - l2_phys = TBL_PHYS_ADDR(mmu_info->l1_pt[l1_idx];) + l2_phys = TBL_PHYS_ADDR(mmu_info->l1_pt[l1_idx]); dev_dbg(mmu_info->dev, "l1 entry %u; iovas 0x%8.8x-0x%8.8x, at %pap\n", l1_idx, iova, iova + ISP_PAGE_SIZE, &l2_phys); @@ -248,7 +248,7 @@ static u32 *alloc_l2_pt(struct ipu6_mmu_info *mmu_info) dev_dbg(mmu_info->dev, "alloc_l2: get_zeroed_page() = %p\n", pt); - for (i = 0; i < ISP_L1PT_PTES; i++) + for (i = 0; i < ISP_L2PT_PTES; i++) pt[i] = mmu_info->dummy_page_pteval; return pt; diff --git a/drivers/media/pci/intel/ipu6/ipu6.c b/drivers/media/pci/intel/ipu6/ipu6.c index 1f4f20b9c94dc7..a2768f44017a5e 100644 --- a/drivers/media/pci/intel/ipu6/ipu6.c +++ b/drivers/media/pci/intel/ipu6/ipu6.c @@ -630,21 +630,21 @@ static int ipu6_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) if (ret) { dev_err_probe(&isp->pdev->dev, ret, "Failed to set MMU hardware\n"); - goto out_ipu6_bus_del_devices; + goto out_ipu6_rpm_put; } ret = ipu6_buttress_map_fw_image(isp->psys, isp->cpd_fw, &isp->psys->fw_sgt); if (ret) { dev_err_probe(&isp->pdev->dev, ret, "failed to map fw image\n"); - goto out_ipu6_bus_del_devices; + goto out_ipu6_rpm_put; } ret = ipu6_cpd_create_pkg_dir(isp->psys, isp->cpd_fw->data); if (ret) { dev_err_probe(&isp->pdev->dev, ret, "failed to create pkg dir\n"); - goto out_ipu6_bus_del_devices; + goto out_ipu6_rpm_put; } ret = devm_request_threaded_irq(dev, pdev->irq, ipu6_buttress_isr, @@ -652,7 +652,7 @@ static int ipu6_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) IRQF_SHARED, IPU6_NAME, isp); if (ret) { dev_err_probe(dev, ret, "Requesting irq failed\n"); - goto out_ipu6_bus_del_devices; + goto out_ipu6_rpm_put; } ret = ipu6_buttress_authenticate(isp); @@ -683,6 +683,8 @@ static int ipu6_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) out_free_irq: devm_free_irq(dev, pdev->irq, isp); +out_ipu6_rpm_put: + pm_runtime_put_sync(&isp->psys->auxdev.dev); out_ipu6_bus_del_devices: if (isp->psys) { ipu6_cpd_free_pkg_dir(isp->psys); diff --git a/drivers/media/pci/mgb4/mgb4_trigger.c b/drivers/media/pci/mgb4/mgb4_trigger.c index 4f9a35904b4186..70cad324df608d 100644 --- a/drivers/media/pci/mgb4/mgb4_trigger.c +++ b/drivers/media/pci/mgb4/mgb4_trigger.c @@ -115,7 +115,7 @@ static int probe_trigger(struct iio_dev *indio_dev, int irq) if (!st->trig) return -ENOMEM; - ret = request_irq(irq, &iio_trigger_generic_data_rdy_poll, 0, + ret = request_irq(irq, &iio_trigger_generic_data_rdy_poll, IRQF_NO_THREAD, "mgb4-trigger", st->trig); if (ret) goto error_free_trig; diff --git a/drivers/media/pci/solo6x10/solo6x10-tw28.c b/drivers/media/pci/solo6x10/solo6x10-tw28.c index 1b7c22a9bc94f1..8f53946c67928f 100644 --- a/drivers/media/pci/solo6x10/solo6x10-tw28.c +++ b/drivers/media/pci/solo6x10/solo6x10-tw28.c @@ -166,7 +166,7 @@ static const u8 tbl_tw2865_pal_template[] = { 0x64, 0x51, 0x40, 0xaf, 0xFF, 0xF0, 0x00, 0xC0, }; -#define is_tw286x(__solo, __id) (!(__solo->tw2815 & (1 << __id))) +#define is_tw286x(__solo, __id) (!((__solo)->tw2815 & (1U << (__id)))) static u8 tw_readbyte(struct solo_dev *solo_dev, int chip_id, u8 tw6x_off, u8 tw_off) @@ -686,6 +686,9 @@ int tw28_set_ctrl_val(struct solo_dev *solo_dev, u32 ctrl, u8 ch, chip_num = ch / 4; ch %= 4; + if (chip_num >= TW_NUM_CHIP) + return -EINVAL; + if (val > 255 || val < 0) return -ERANGE; @@ -758,6 +761,9 @@ int tw28_get_ctrl_val(struct solo_dev *solo_dev, u32 ctrl, u8 ch, chip_num = ch / 4; ch %= 4; + if (chip_num >= TW_NUM_CHIP) + return -EINVAL; + switch (ctrl) { case V4L2_CID_SHARPNESS: /* Only 286x has sharpness */ diff --git a/drivers/media/platform/Kconfig b/drivers/media/platform/Kconfig index 3f0b7bb68cc940..8b31f087e7da20 100644 --- a/drivers/media/platform/Kconfig +++ b/drivers/media/platform/Kconfig @@ -65,6 +65,7 @@ config VIDEO_MUX source "drivers/media/platform/allegro-dvt/Kconfig" source "drivers/media/platform/amlogic/Kconfig" source "drivers/media/platform/amphion/Kconfig" +source "drivers/media/platform/apple/Kconfig" source "drivers/media/platform/arm/Kconfig" source "drivers/media/platform/aspeed/Kconfig" source "drivers/media/platform/atmel/Kconfig" diff --git a/drivers/media/platform/Makefile b/drivers/media/platform/Makefile index 6d5f79ddfcc37a..0e93f7d860caf4 100644 --- a/drivers/media/platform/Makefile +++ b/drivers/media/platform/Makefile @@ -8,6 +8,7 @@ obj-y += allegro-dvt/ obj-y += amlogic/ obj-y += amphion/ +obj-y += apple/ obj-y += arm/ obj-y += aspeed/ obj-y += atmel/ diff --git a/drivers/media/platform/amphion/vdec.c b/drivers/media/platform/amphion/vdec.c index c0d2aabb9e0e33..f25dbcebdccf68 100644 --- a/drivers/media/platform/amphion/vdec.c +++ b/drivers/media/platform/amphion/vdec.c @@ -724,6 +724,7 @@ static int vdec_decoder_cmd(struct file *file, void *fh, struct v4l2_decoder_cmd switch (cmd->cmd) { case V4L2_DEC_CMD_START: vdec_cmd_start(inst); + vb2_clear_last_buffer_dequeued(v4l2_m2m_get_dst_vq(inst->fh.m2m_ctx)); break; case V4L2_DEC_CMD_STOP: vdec_cmd_stop(inst); diff --git a/drivers/media/platform/amphion/vpu_v4l2.c b/drivers/media/platform/amphion/vpu_v4l2.c index 47dff9a35bb46d..1fb887b9098c6b 100644 --- a/drivers/media/platform/amphion/vpu_v4l2.c +++ b/drivers/media/platform/amphion/vpu_v4l2.c @@ -670,7 +670,6 @@ static int vpu_m2m_queue_init(void *priv, struct vb2_queue *src_vq, struct vb2_q src_vq->mem_ops = &vb2_vmalloc_memops; src_vq->drv_priv = inst; src_vq->buf_struct_size = sizeof(struct vpu_vb2_buffer); - src_vq->min_queued_buffers = 1; src_vq->dev = inst->vpu->dev; src_vq->lock = &inst->lock; ret = vb2_queue_init(src_vq); @@ -687,7 +686,6 @@ static int vpu_m2m_queue_init(void *priv, struct vb2_queue *src_vq, struct vb2_q dst_vq->mem_ops = &vb2_vmalloc_memops; dst_vq->drv_priv = inst; dst_vq->buf_struct_size = sizeof(struct vpu_vb2_buffer); - dst_vq->min_queued_buffers = 1; dst_vq->dev = inst->vpu->dev; dst_vq->lock = &inst->lock; ret = vb2_queue_init(dst_vq); diff --git a/drivers/media/platform/apple/Kconfig b/drivers/media/platform/apple/Kconfig new file mode 100644 index 00000000000000..f16508bff5242a --- /dev/null +++ b/drivers/media/platform/apple/Kconfig @@ -0,0 +1,5 @@ +# SPDX-License-Identifier: GPL-2.0-only + +comment "Apple media platform drivers" + +source "drivers/media/platform/apple/isp/Kconfig" diff --git a/drivers/media/platform/apple/Makefile b/drivers/media/platform/apple/Makefile new file mode 100644 index 00000000000000..d8fe985b0e6c37 --- /dev/null +++ b/drivers/media/platform/apple/Makefile @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: GPL-2.0-only + +obj-y += isp/ diff --git a/drivers/media/platform/apple/isp/.gitignore b/drivers/media/platform/apple/isp/.gitignore new file mode 100644 index 00000000000000..bd7fab40e0d98a --- /dev/null +++ b/drivers/media/platform/apple/isp/.gitignore @@ -0,0 +1 @@ +.clang-format diff --git a/drivers/media/platform/apple/isp/Kconfig b/drivers/media/platform/apple/isp/Kconfig new file mode 100644 index 00000000000000..8e339db43bc418 --- /dev/null +++ b/drivers/media/platform/apple/isp/Kconfig @@ -0,0 +1,12 @@ +# SPDX-License-Identifier: GPL-2.0-only + +config VIDEO_APPLE_ISP + tristate "Apple Silicon Image Signal Processor driver" + select VIDEOBUF2_CORE + select VIDEOBUF2_V4L2 + select VIDEOBUF2_DMA_SG + select APPLE_PMP_REPORT + depends on ARCH_APPLE || COMPILE_TEST + depends on OF_ADDRESS + depends on V4L_PLATFORM_DRIVERS + depends on VIDEO_DEV diff --git a/drivers/media/platform/apple/isp/Makefile b/drivers/media/platform/apple/isp/Makefile new file mode 100644 index 00000000000000..4649f32987f025 --- /dev/null +++ b/drivers/media/platform/apple/isp/Makefile @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: GPL-2.0-only +apple-isp-y := isp-cam.o isp-cmd.o isp-drv.o isp-fw.o isp-iommu.o isp-ipc.o isp-v4l2.o +obj-$(CONFIG_VIDEO_APPLE_ISP) += apple-isp.o diff --git a/drivers/media/platform/apple/isp/isp-cam.c b/drivers/media/platform/apple/isp/isp-cam.c new file mode 100644 index 00000000000000..c889173bd348f3 --- /dev/null +++ b/drivers/media/platform/apple/isp/isp-cam.c @@ -0,0 +1,498 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright 2023 Eileen Yoon */ + +#include + +#include "isp-cam.h" +#include "isp-cmd.h" +#include "isp-fw.h" +#include "isp-iommu.h" + +#define ISP_MAX_PRESETS 32 + +struct isp_setfile { + u32 version; + u32 magic; + const char *path; + size_t size; +}; + +// clang-format off +static const struct isp_setfile isp_setfiles[] = { + [ISP_IMX248_1820_01] = {0x248, 0x18200103, "apple/isp_1820_01XX.dat", 0x442c}, + [ISP_IMX248_1822_02] = {0x248, 0x18220201, "apple/isp_1822_02XX.dat", 0x442c}, + [ISP_IMX343_5221_02] = {0x343, 0x52210211, "apple/isp_5221_02XX.dat", 0x4870}, + [ISP_IMX354_9251_02] = {0x354, 0x92510208, "apple/isp_9251_02XX.dat", 0xa5ec}, + [ISP_IMX356_4820_01] = {0x356, 0x48200107, "apple/isp_4820_01XX.dat", 0x9324}, + [ISP_IMX356_4820_02] = {0x356, 0x48200206, "apple/isp_4820_02XX.dat", 0x9324}, + [ISP_IMX364_8720_01] = {0x364, 0x87200103, "apple/isp_8720_01XX.dat", 0x36ac}, + [ISP_IMX364_8723_01] = {0x364, 0x87230101, "apple/isp_8723_01XX.dat", 0x361c}, + [ISP_IMX372_3820_01] = {0x372, 0x38200108, "apple/isp_3820_01XX.dat", 0xfdb0}, + [ISP_IMX372_3820_02] = {0x372, 0x38200205, "apple/isp_3820_02XX.dat", 0xfdb0}, + [ISP_IMX372_3820_11] = {0x372, 0x38201104, "apple/isp_3820_11XX.dat", 0xfdb0}, + [ISP_IMX372_3820_12] = {0x372, 0x38201204, "apple/isp_3820_12XX.dat", 0xfdb0}, + [ISP_IMX405_9720_01] = {0x405, 0x97200102, "apple/isp_9720_01XX.dat", 0x92c8}, + [ISP_IMX405_9721_01] = {0x405, 0x97210102, "apple/isp_9721_01XX.dat", 0x9818}, + [ISP_IMX405_9723_01] = {0x405, 0x97230101, "apple/isp_9723_01XX.dat", 0x92c8}, + [ISP_IMX414_2520_01] = {0x414, 0x25200102, "apple/isp_2520_01XX.dat", 0xa444}, + [ISP_IMX503_7820_01] = {0x503, 0x78200109, "apple/isp_7820_01XX.dat", 0xb268}, + [ISP_IMX503_7820_02] = {0x503, 0x78200206, "apple/isp_7820_02XX.dat", 0xb268}, + [ISP_IMX505_3921_01] = {0x505, 0x39210102, "apple/isp_3921_01XX.dat", 0x89b0}, + [ISP_IMX514_2820_01] = {0x514, 0x28200108, "apple/isp_2820_01XX.dat", 0xa198}, + [ISP_IMX514_2820_02] = {0x514, 0x28200205, "apple/isp_2820_02XX.dat", 0xa198}, + [ISP_IMX514_2820_03] = {0x514, 0x28200305, "apple/isp_2820_03XX.dat", 0xa198}, + [ISP_IMX514_2820_04] = {0x514, 0x28200405, "apple/isp_2820_04XX.dat", 0xa198}, + [ISP_IMX558_1921_01] = {0x558, 0x19210106, "apple/isp_1921_01XX.dat", 0xad40}, + [ISP_IMX558_1922_02] = {0x558, 0x19220201, "apple/isp_1922_02XX.dat", 0xad40}, + [ISP_IMX603_7920_01] = {0x603, 0x79200109, "apple/isp_7920_01XX.dat", 0xad2c}, + [ISP_IMX603_7920_02] = {0x603, 0x79200205, "apple/isp_7920_02XX.dat", 0xad2c}, + [ISP_IMX603_7921_01] = {0x603, 0x79210104, "apple/isp_7921_01XX.dat", 0xad90}, + [ISP_IMX613_4920_01] = {0x613, 0x49200108, "apple/isp_4920_01XX.dat", 0x9324}, + [ISP_IMX613_4920_02] = {0x613, 0x49200204, "apple/isp_4920_02XX.dat", 0x9324}, + [ISP_IMX614_2921_01] = {0x614, 0x29210107, "apple/isp_2921_01XX.dat", 0xed6c}, + [ISP_IMX614_2921_02] = {0x614, 0x29210202, "apple/isp_2921_02XX.dat", 0xed6c}, + [ISP_IMX614_2922_02] = {0x614, 0x29220201, "apple/isp_2922_02XX.dat", 0xed6c}, + [ISP_IMX633_3622_01] = {0x633, 0x36220111, "apple/isp_3622_01XX.dat", 0x100d4}, + [ISP_IMX703_7721_01] = {0x703, 0x77210106, "apple/isp_7721_01XX.dat", 0x936c}, + [ISP_IMX703_7722_01] = {0x703, 0x77220106, "apple/isp_7722_01XX.dat", 0xac20}, + [ISP_IMX713_4721_01] = {0x713, 0x47210107, "apple/isp_4721_01XX.dat", 0x936c}, + [ISP_IMX713_4722_01] = {0x713, 0x47220109, "apple/isp_4722_01XX.dat", 0x9218}, + [ISP_IMX714_2022_01] = {0x714, 0x20220107, "apple/isp_2022_01XX.dat", 0xa198}, + [ISP_IMX772_3721_01] = {0x772, 0x37210106, "apple/isp_3721_01XX.dat", 0xfdf8}, + [ISP_IMX772_3721_11] = {0x772, 0x37211106, "apple/isp_3721_11XX.dat", 0xfe14}, + [ISP_IMX772_3722_01] = {0x772, 0x37220104, "apple/isp_3722_01XX.dat", 0xfca4}, + [ISP_IMX772_3723_01] = {0x772, 0x37230106, "apple/isp_3723_01XX.dat", 0xfca4}, + [ISP_IMX814_2123_01] = {0x814, 0x21230101, "apple/isp_2123_01XX.dat", 0xed54}, + [ISP_IMX853_7622_01] = {0x853, 0x76220112, "apple/isp_7622_01XX.dat", 0x247f8}, + [ISP_IMX913_7523_01] = {0x913, 0x75230107, "apple/isp_7523_01XX.dat", 0x247f8}, + [ISP_VD56G0_6221_01] = {0xd56, 0x62210102, "apple/isp_6221_01XX.dat", 0x1b80}, + [ISP_VD56G0_6222_01] = {0xd56, 0x62220102, "apple/isp_6222_01XX.dat", 0x1b80}, +}; +// clang-format on + +static int isp_ch_get_sensor_id(struct apple_isp *isp, u32 ch) +{ + struct isp_format *fmt = isp_get_format(isp, ch); + enum isp_sensor_id id; + int err = 0; + + /* TODO need more datapoints to figure out the sub-versions + * Defaulting to 1st release for now, the calib files aren't too different. + */ + switch (fmt->version) { + case 0x248: + id = ISP_IMX248_1820_01; + break; + case 0x343: + id = ISP_IMX343_5221_02; + break; + case 0x354: + id = ISP_IMX354_9251_02; + break; + case 0x356: + id = ISP_IMX356_4820_01; + break; + case 0x364: + id = ISP_IMX364_8720_01; + break; + case 0x372: + id = ISP_IMX372_3820_01; + break; + case 0x405: + id = ISP_IMX405_9720_01; + break; + case 0x414: + id = ISP_IMX414_2520_01; + break; + case 0x503: + id = ISP_IMX503_7820_01; + break; + case 0x505: + id = ISP_IMX505_3921_01; + break; + case 0x514: + id = ISP_IMX514_2820_01; + break; + case 0x558: + id = ISP_IMX558_1921_01; + break; + case 0x603: + id = ISP_IMX603_7920_01; + break; + case 0x613: + id = ISP_IMX613_4920_01; + break; + case 0x614: + id = ISP_IMX614_2921_01; + break; + case 0x633: + id = ISP_IMX633_3622_01; + break; + case 0x703: + id = ISP_IMX703_7721_01; + break; + case 0x713: + id = ISP_IMX713_4721_01; + break; + case 0x714: + id = ISP_IMX714_2022_01; + break; + case 0x772: + id = ISP_IMX772_3721_01; + break; + case 0x814: + id = ISP_IMX814_2123_01; + break; + case 0x853: + id = ISP_IMX853_7622_01; + break; + case 0x913: + id = ISP_IMX913_7523_01; + break; + case 0xd56: + id = ISP_VD56G0_6221_01; + break; + default: + err = -EINVAL; + break; + } + + if (err) + dev_err(isp->dev, "invalid sensor version: 0x%x\n", + fmt->version); + else + fmt->id = id; + + return err; +} + +static int isp_ch_get_camera_preset(struct apple_isp *isp, u32 ch, u32 ps) +{ + int err = 0; + + struct cmd_ch_camera_config *args; /* Too big to allocate on stack */ + args = kzalloc(sizeof(*args), GFP_KERNEL); + if (!args) + return -ENOMEM; + + err = isp_cmd_ch_camera_config_get(isp, ch, ps, args); + if (err) + goto exit; + + pr_info("apple-isp: ps: CISP_CMD_CH_CAMERA_CONFIG_GET: %d\n", ps); + print_hex_dump(KERN_INFO, "apple-isp: ps: ", DUMP_PREFIX_NONE, 32, 4, + args, sizeof(*args), false); + +exit: + kfree(args); + + return err; +} + +static int isp_ch_cache_sensor_info(struct apple_isp *isp, u32 ch) +{ + struct isp_format *fmt = isp_get_format(isp, ch); + int err = 0; + + struct cmd_ch_info *args; /* Too big to allocate on stack */ + args = kzalloc(sizeof(*args), GFP_KERNEL); + if (!args) + return -ENOMEM; + + err = isp_cmd_ch_info_get(isp, ch, args); + if (err) + goto exit; + + dev_info(isp->dev, "found sensor %x %s on ch %d\n", args->version, + args->module_sn, ch); + + fmt->version = args->version; + + pr_info("apple-isp: ch: CISP_CMD_CH_INFO_GET: %d\n", ch); + print_hex_dump(KERN_INFO, "apple-isp: ch: ", DUMP_PREFIX_NONE, 32, 4, + args, sizeof(*args), false); + + for (u32 ps = 0; ps < args->num_presets; ps++) { + isp_ch_get_camera_preset(isp, ch, ps); + } + + err = isp_ch_get_sensor_id(isp, ch); + if (err || + (fmt->id != ISP_IMX248_1820_01 && fmt->id != ISP_IMX558_1921_01 && + fmt->id != ISP_IMX364_8720_01)) { + dev_err(isp->dev, + "ch %d: unsupported sensor. Please file a bug report with hardware info & dmesg trace.\n", + ch); + return -ENODEV; + } + +exit: + kfree(args); + + return err; +} + +static int isp_detect_camera(struct apple_isp *isp) +{ + int err; + + struct cmd_config_get args; + memset(&args, 0, sizeof(args)); + + err = isp_cmd_config_get(isp, &args); + if (err) + return err; + + pr_info("apple-isp: CISP_CMD_CONFIG_GET: \n"); + print_hex_dump(KERN_INFO, "apple-isp: ", DUMP_PREFIX_NONE, 32, 4, &args, + sizeof(args), false); + + if (!args.num_channels) { + dev_err(isp->dev, "did not detect any channels\n"); + return -ENODEV; + } + + if (args.num_channels > ISP_MAX_CHANNELS) { + dev_warn(isp->dev, "found %d channels when maximum is %d\n", + args.num_channels, ISP_MAX_CHANNELS); + args.num_channels = ISP_MAX_CHANNELS; + } + + if (args.num_channels > 1) { + dev_warn( + isp->dev, + "warning: driver doesn't support multiple channels. Please file a bug report with hardware info & dmesg trace.\n"); + } + + isp->num_channels = args.num_channels; + isp->current_ch = 0; + + err = isp_ch_cache_sensor_info(isp, isp->current_ch); + if (err) { + dev_err(isp->dev, "failed to cache sensor info\n"); + return err; + } + + return 0; +} + +int apple_isp_detect_camera(struct apple_isp *isp) +{ + int err; + + /* RPM must be enabled prior to calling this */ + err = apple_isp_firmware_boot(isp); + if (err) { + dev_err(isp->dev, + "failed to boot firmware for initial sensor detection: %d\n", + err); + return -EPROBE_DEFER; + } + + err = isp_detect_camera(isp); + + isp_cmd_flicker_sensor_set(isp, 0); + + isp_cmd_ch_stop(isp, 0); + isp_cmd_ch_buffer_return(isp, isp->current_ch); + + apple_isp_firmware_shutdown(isp); + + return err; +} + +static int isp_ch_load_setfile(struct apple_isp *isp, u32 ch) +{ + struct isp_format *fmt = isp_get_format(isp, ch); + const struct isp_setfile *setfile = &isp_setfiles[fmt->id]; + const struct firmware *fw; + u32 magic; + int err; + + err = request_firmware(&fw, setfile->path, isp->dev); + if (err) { + dev_err(isp->dev, "failed to request setfile '%s': %d\n", + setfile->path, err); + return err; + } + + if (fw->size < setfile->size) { + dev_err(isp->dev, "setfile too small (0x%lx/0x%zx)\n", fw->size, + setfile->size); + release_firmware(fw); + return -EINVAL; + } + + magic = be32_to_cpup((__be32 *)fw->data); + if (magic != setfile->magic) { + dev_err(isp->dev, "setfile '%s' corrupted?\n", setfile->path); + release_firmware(fw); + return -EINVAL; + } + + memcpy(isp->data_surf->virt, (void *)fw->data, setfile->size); + release_firmware(fw); + + return isp_cmd_ch_set_file_load(isp, ch, isp->data_surf->iova, + setfile->size); +} + +static int isp_ch_configure_capture(struct apple_isp *isp, u32 ch) +{ + struct isp_format *fmt = isp_get_format(isp, ch); + int err; + + isp_cmd_flicker_sensor_set(isp, 0); + + /* The setfile isn't requisite but then we don't get calibration */ + err = isp_ch_load_setfile(isp, ch); + if (err) { + dev_err(isp->dev, "warning: calibration data not loaded: %d\n", + err); + + /* If this failed due to a signal, propagate */ + if (err == -EINTR) + return err; + } + + if (isp->hw->lpdp) { + err = isp_cmd_ch_lpdp_hs_receiver_tuning_set(isp, ch, 1, 15); + if (err) + return err; + } + + err = isp_cmd_ch_sbs_enable(isp, ch, 1); + if (err) + return err; + + err = isp_cmd_ch_camera_config_select(isp, ch, fmt->preset->index); + if (err) + return err; + + err = isp_cmd_ch_buffer_recycle_mode_set( + isp, ch, CISP_BUFFER_RECYCLE_MODE_EMPTY_ONLY); + if (err) + return err; + + err = isp_cmd_ch_buffer_recycle_start(isp, ch); + if (err) + return err; + + err = isp_cmd_ch_crop_set(isp, ch, fmt->preset->crop_offset.x, + fmt->preset->crop_offset.y, + fmt->preset->crop_size.x, + fmt->preset->crop_size.y); + if (err) + return err; + + err = isp_cmd_ch_output_config_set(isp, ch, fmt->preset->output_dim.x, + fmt->preset->output_dim.y, + fmt->strides, CISP_COLORSPACE_REC709, + CISP_OUTPUT_FORMAT_YUV_2PLANE); + if (err) + return err; + + err = isp_cmd_ch_preview_stream_set(isp, ch, 1); + if (err) + return err; + + err = isp_cmd_ch_cnr_start(isp, ch); + if (err) + return err; + + err = isp_cmd_ch_mbnr_enable(isp, ch, 0, ISP_MBNR_MODE_ENABLE, 1); + if (err) + return err; + + err = isp_cmd_apple_ch_ae_fd_scene_metering_config_set(isp, ch); + if (err) + return err; + + err = isp_cmd_apple_ch_ae_metering_mode_set(isp, ch, 3); + if (err) + return err; + + err = isp_cmd_ch_ae_stability_set(isp, ch, 32); + if (err) + return err; + + err = isp_cmd_ch_ae_stability_to_stable_set(isp, ch, 20); + if (err) + return err; + + err = isp_cmd_ch_sif_pixel_format_set(isp, ch); + if (err) + return err; + + err = isp_cmd_ch_ae_frame_rate_max_set(isp, ch, ISP_FRAME_RATE_DEN); + if (err) + return err; + + err = isp_cmd_ch_ae_frame_rate_min_set(isp, ch, ISP_FRAME_RATE_DEN2); + if (err) + return err; + + err = isp_cmd_apple_ch_temporal_filter_start(isp, ch, isp->temporal_filter); + if (err) + return err; + + err = isp_cmd_apple_ch_motion_history_start(isp, ch); + if (err) + return err; + + err = isp_cmd_apple_ch_temporal_filter_enable(isp, ch); + if (err) + return err; + + err = isp_cmd_ch_buffer_pool_config_set(isp, ch, CISP_POOL_TYPE_META); + if (err) + return err; + + err = isp_cmd_ch_buffer_pool_config_set(isp, ch, + CISP_POOL_TYPE_META_CAPTURE); + if (err) + return err; + + return 0; +} + +static int isp_configure_capture(struct apple_isp *isp) +{ + return isp_ch_configure_capture(isp, isp->current_ch); +} + +int apple_isp_start_camera(struct apple_isp *isp) +{ + int err; + + err = apple_isp_firmware_boot(isp); + if (err < 0) { + dev_err(isp->dev, "failed to boot firmware: %d\n", err); + return err; + } + + err = isp_configure_capture(isp); + if (err) { + dev_err(isp->dev, "failed to configure capture: %d\n", err); + apple_isp_firmware_shutdown(isp); + return err; + } + + return 0; +} + +void apple_isp_stop_camera(struct apple_isp *isp) +{ + apple_isp_firmware_shutdown(isp); +} + +int apple_isp_start_capture(struct apple_isp *isp) +{ + return isp_cmd_ch_start(isp, 0); // TODO channel mask +} + +void apple_isp_stop_capture(struct apple_isp *isp) +{ + isp_cmd_ch_stop(isp, 0); // TODO channel mask + isp_cmd_ch_buffer_return(isp, isp->current_ch); +} diff --git a/drivers/media/platform/apple/isp/isp-cam.h b/drivers/media/platform/apple/isp/isp-cam.h new file mode 100644 index 00000000000000..f4fa4224c7a934 --- /dev/null +++ b/drivers/media/platform/apple/isp/isp-cam.h @@ -0,0 +1,21 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright 2023 Eileen Yoon */ + +#ifndef __ISP_CAM_H__ +#define __ISP_CAM_H__ + +#include "isp-drv.h" + +#define ISP_FRAME_RATE_NUM 256 +#define ISP_FRAME_RATE_DEN 7680 +#define ISP_FRAME_RATE_DEN2 3840 + +int apple_isp_detect_camera(struct apple_isp *isp); + +int apple_isp_start_camera(struct apple_isp *isp); +void apple_isp_stop_camera(struct apple_isp *isp); + +int apple_isp_start_capture(struct apple_isp *isp); +void apple_isp_stop_capture(struct apple_isp *isp); + +#endif /* __ISP_CAM_H__ */ diff --git a/drivers/media/platform/apple/isp/isp-cmd.c b/drivers/media/platform/apple/isp/isp-cmd.c new file mode 100644 index 00000000000000..ee491d2cb42c5b --- /dev/null +++ b/drivers/media/platform/apple/isp/isp-cmd.c @@ -0,0 +1,634 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright 2023 Eileen Yoon */ + +#include "isp-cmd.h" +#include "isp-drv.h" +#include "isp-iommu.h" +#include "isp-ipc.h" + +#define CISP_OPCODE_SHIFT 32UL +#define CISP_OPCODE(x) (((u64)(x)) << CISP_OPCODE_SHIFT) +#define CISP_OPCODE_GET(x) (((u64)(x)) >> CISP_OPCODE_SHIFT) + +#define CISP_TIMEOUT msecs_to_jiffies(3000) +#define CISP_SEND_IN(x, a) (cisp_send((x), &(a), sizeof(a), 0, CISP_TIMEOUT)) +#define CISP_SEND_INOUT(x, a) (cisp_send((x), &(a), sizeof(a), sizeof(a), CISP_TIMEOUT)) +#define CISP_SEND_OUT(x, a) (cisp_send_read((x), (a), sizeof(*a), sizeof(*a))) +#define CISP_POST_IN(x, a) (cisp_send((x), &(a), sizeof(a), 0, 0)) +#define CISP_POST_INOUT(x, a) (cisp_send((x), &(a), sizeof(a), sizeof(a), 0)) + +static int cisp_send(struct apple_isp *isp, void *args, u32 insize, u32 outsize, int timeout) +{ + struct isp_channel *chan = isp->chan_io; + struct isp_message *req = &chan->req; + int err; + + req->arg0 = isp->cmd_iova; + req->arg1 = insize; + req->arg2 = outsize; + + memcpy(isp->cmd_virt, args, insize); + err = ipc_chan_send(isp, chan, timeout); + if (err) { + u64 opcode; + memcpy(&opcode, args, sizeof(opcode)); + dev_err(isp->dev, + "%s: failed to send OPCODE 0x%04llx: [0x%llx, 0x%llx, 0x%llx]\n", + chan->name, CISP_OPCODE_GET(opcode), req->arg0, + req->arg1, req->arg2); + } + + return err; +} + +static int cisp_send_read(struct apple_isp *isp, void *args, u32 insize, + u32 outsize) +{ + /* TODO do I need to lock the iova space? */ + int err = cisp_send(isp, args, insize, outsize, CISP_TIMEOUT); + if (err) + return err; + + memcpy(args, isp->cmd_virt, outsize); + return 0; +} + +int isp_cmd_start(struct apple_isp *isp, u32 mode) +{ + struct cmd_start args = { + .opcode = CISP_OPCODE(CISP_CMD_START), + .mode = mode, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_stop(struct apple_isp *isp, u32 mode) +{ + struct cmd_stop args = { + .opcode = CISP_OPCODE(CISP_CMD_STOP), + .mode = mode, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_power_down(struct apple_isp *isp) +{ + struct cmd_power_down args = { + .opcode = CISP_OPCODE(CISP_CMD_POWER_DOWN), + }; + return CISP_POST_INOUT(isp, args); +} + +int isp_cmd_suspend(struct apple_isp *isp) +{ + struct cmd_suspend args = { + .opcode = CISP_OPCODE(CISP_CMD_SUSPEND), + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_print_enable(struct apple_isp *isp, u32 enable) +{ + struct cmd_print_enable args = { + .opcode = CISP_OPCODE(CISP_CMD_PRINT_ENABLE), + .enable = enable, + }; + return CISP_SEND_INOUT(isp, args); +} + +int isp_cmd_trace_enable(struct apple_isp *isp, u32 enable) +{ + struct cmd_trace_enable args = { + .opcode = CISP_OPCODE(CISP_CMD_TRACE_ENABLE), + .enable = enable, + }; + return CISP_SEND_INOUT(isp, args); +} + +int isp_cmd_config_get(struct apple_isp *isp, struct cmd_config_get *args) +{ + args->opcode = CISP_OPCODE(CISP_CMD_CONFIG_GET); + return CISP_SEND_OUT(isp, args); +} + +int isp_cmd_set_isp_pmu_base(struct apple_isp *isp, u64 pmu_base) +{ + struct cmd_set_isp_pmu_base args = { + .opcode = CISP_OPCODE(CISP_CMD_SET_ISP_PMU_BASE), + .pmu_base = pmu_base, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_set_dsid_clr_req_base2(struct apple_isp *isp, u64 dsid_clr_base0, + u64 dsid_clr_base1, u64 dsid_clr_base2, + u64 dsid_clr_base3, u32 dsid_clr_range0, + u32 dsid_clr_range1, u32 dsid_clr_range2, + u32 dsid_clr_range3) +{ + struct cmd_set_dsid_clr_req_base2 args = { + .opcode = CISP_OPCODE(CISP_CMD_SET_DSID_CLR_REG_BASE2), + .dsid_clr_base0 = dsid_clr_base0, + .dsid_clr_base1 = dsid_clr_base1, + .dsid_clr_base2 = dsid_clr_base2, + .dsid_clr_base3 = dsid_clr_base3, + .dsid_clr_range0 = dsid_clr_range0, + .dsid_clr_range1 = dsid_clr_range1, + .dsid_clr_range2 = dsid_clr_range2, + .dsid_clr_range3 = dsid_clr_range3, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_set_dsid_clr_req_base(struct apple_isp *isp, u64 dsid_clr_base, + u32 dsid_clr_range) +{ + struct cmd_set_dsid_clr_req_base args = { + .opcode = CISP_OPCODE(CISP_CMD_SET_DSID_CLR_REG_BASE), + .dsid_clr_base = dsid_clr_base, + .dsid_clr_range = dsid_clr_range, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_pmp_ctrl_set(struct apple_isp *isp, u64 clock_scratch, + u64 clock_base, u8 clock_bit, u8 clock_size, + u64 bandwidth_scratch, u64 bandwidth_base, + u8 bandwidth_bit, u8 bandwidth_size) +{ + struct cmd_pmp_ctrl_set args = { + .opcode = CISP_OPCODE(CISP_CMD_PMP_CTRL_SET), + .clock_scratch = clock_scratch, + .clock_base = clock_base, + .clock_bit = clock_bit, + .clock_size = clock_size, + .clock_pad = 0, + .bandwidth_scratch = bandwidth_scratch, + .bandwidth_base = bandwidth_base, + .bandwidth_bit = bandwidth_bit, + .bandwidth_size = bandwidth_size, + .bandwidth_pad = 0, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_fid_enter(struct apple_isp *isp) +{ + struct cmd_fid_enter args = { + .opcode = CISP_OPCODE(CISP_CMD_FID_ENTER), + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_fid_exit(struct apple_isp *isp) +{ + struct cmd_fid_exit args = { + .opcode = CISP_OPCODE(CISP_CMD_FID_EXIT), + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_ch_start(struct apple_isp *isp, u32 chan) +{ + struct cmd_ch_start args = { + .opcode = CISP_OPCODE(CISP_CMD_CH_START), + .chan = chan, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_ch_stop(struct apple_isp *isp, u32 chan) +{ + struct cmd_ch_stop args = { + .opcode = CISP_OPCODE(CISP_CMD_CH_STOP), + .chan = chan, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_flicker_sensor_set(struct apple_isp *isp, u32 mode) +{ + struct cmd_flicker_sensor_set args = { + .opcode = CISP_OPCODE(CISP_CMD_FLICKER_SENSOR_SET), + .mode = mode, + }; + return CISP_SEND_INOUT(isp, args); +} + +int isp_cmd_ch_info_get(struct apple_isp *isp, u32 chan, + struct cmd_ch_info *args) +{ + args->opcode = CISP_OPCODE(CISP_CMD_CH_INFO_GET); + args->chan = chan; + return CISP_SEND_OUT(isp, args); +} + +int isp_cmd_ch_camera_config_get(struct apple_isp *isp, u32 chan, u32 preset, + struct cmd_ch_camera_config *args) +{ + args->opcode = CISP_OPCODE(CISP_CMD_CH_CAMERA_CONFIG_GET); + args->preset = preset; + args->chan = chan; + return CISP_SEND_OUT(isp, args); +} + +int isp_cmd_ch_camera_config_current_get(struct apple_isp *isp, u32 chan, + struct cmd_ch_camera_config *args) +{ + args->opcode = CISP_OPCODE(CISP_CMD_CH_CAMERA_CONFIG_CURRENT_GET); + args->chan = chan; + return CISP_SEND_OUT(isp, args); +} + +int isp_cmd_ch_camera_config_select(struct apple_isp *isp, u32 chan, u32 preset) +{ + struct cmd_ch_camera_config_select args = { + .opcode = CISP_OPCODE(CISP_CMD_CH_CAMERA_CONFIG_SELECT), + .chan = chan, + .preset = preset, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_ch_buffer_return(struct apple_isp *isp, u32 chan) +{ + struct cmd_ch_buffer_return args = { + .opcode = CISP_OPCODE(CISP_CMD_CH_BUFFER_RETURN), + .chan = chan, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_ch_set_file_load(struct apple_isp *isp, u32 chan, u64 addr, + u32 size) +{ + if (isp->fw_compat >= ISP_FIRMWARE_V_13_5) { + struct cmd_ch_set_file_load64 args = { + .opcode = CISP_OPCODE(CISP_CMD_CH_SET_FILE_LOAD), + .chan = chan, + .addr = addr, + .size = size, + }; + return CISP_SEND_IN(isp, args); + } else { + struct cmd_ch_set_file_load args = { + .opcode = CISP_OPCODE(CISP_CMD_CH_SET_FILE_LOAD), + .chan = chan, + .addr = addr, + .size = size, + }; + return CISP_SEND_IN(isp, args); + } +} + +int isp_cmd_ch_sbs_enable(struct apple_isp *isp, u32 chan, u32 enable) +{ + struct cmd_ch_sbs_enable args = { + .opcode = CISP_OPCODE(CISP_CMD_CH_SBS_ENABLE), + .chan = chan, + .enable = enable, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_ch_crop_set(struct apple_isp *isp, u32 chan, u32 x1, u32 y1, u32 x2, + u32 y2) +{ + struct cmd_ch_crop_set args = { + .opcode = CISP_OPCODE(isp->hw->scl1 ? CISP_CMD_CH_CROP_SCL1_SET + : CISP_CMD_CH_CROP_SET), + .chan = chan, + .x1 = x1, + .y1 = y1, + .x2 = x2, + .y2 = y2, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_ch_output_config_set(struct apple_isp *isp, u32 chan, u32 width, + u32 height, u32 strides[3], u32 colorspace, u32 format) +{ + struct cmd_ch_output_config_set args = { + .opcode = CISP_OPCODE(isp->hw->scl1 ? CISP_CMD_CH_OUTPUT_CONFIG_SCL1_SET + : CISP_CMD_CH_OUTPUT_CONFIG_SET), + .chan = chan, + .width = width, + .height = height, + .colorspace = colorspace, + .format = format, + .padding_rows = 0, + .unk_h0 = height, + .compress = 0, + .unk_w2 = width, + }; + memcpy(args.strides, strides, sizeof(args.strides)); + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_ch_preview_stream_set(struct apple_isp *isp, u32 chan, u32 stream) +{ + struct cmd_ch_preview_stream_set args = { + .opcode = CISP_OPCODE(CISP_CMD_CH_PREVIEW_STREAM_SET), + .chan = chan, + .stream = stream, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_ch_als_disable(struct apple_isp *isp, u32 chan) +{ + struct cmd_ch_als_disable args = { + .opcode = CISP_OPCODE(CISP_CMD_CH_ALS_DISABLE), + .chan = chan, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_ch_cnr_start(struct apple_isp *isp, u32 chan) +{ + struct cmd_ch_cnr_start args = { + .opcode = CISP_OPCODE(CISP_CMD_CH_CNR_START), + .chan = chan, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_ch_mbnr_enable(struct apple_isp *isp, u32 chan, u32 use_case, + u32 mode, u32 enable_chroma) +{ + struct cmd_ch_mbnr_enable args = { + .opcode = CISP_OPCODE(CISP_CMD_CH_MBNR_ENABLE), + .chan = chan, + .use_case = use_case, + .mode = mode, + .enable_chroma = enable_chroma, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_ch_sif_pixel_format_set(struct apple_isp *isp, u32 chan) +{ + struct cmd_ch_sif_pixel_format_set args = { + .opcode = CISP_OPCODE(CISP_CMD_CH_SIF_PIXEL_FORMAT_SET), + .chan = chan, + .format = 3, + .type = 1, + .compress = 0, + .unk_10 = 0, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_ch_buffer_recycle_mode_set(struct apple_isp *isp, u32 chan, + u32 mode) +{ + struct cmd_ch_buffer_recycle_mode_set args = { + .opcode = CISP_OPCODE(CISP_CMD_CH_BUFFER_RECYCLE_MODE_SET), + .chan = chan, + .mode = mode, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_ch_buffer_recycle_start(struct apple_isp *isp, u32 chan) +{ + struct cmd_ch_buffer_recycle_start args = { + .opcode = CISP_OPCODE(CISP_CMD_CH_BUFFER_RECYCLE_START), + .chan = chan, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_ch_buffer_pool_config_set(struct apple_isp *isp, u32 chan, u16 type) +{ + struct cmd_ch_buffer_pool_config_set args = { + .opcode = CISP_OPCODE(CISP_CMD_CH_BUFFER_POOL_CONFIG_SET), + .chan = chan, + .type = type, + .count = ISP_MAX_BUFFERS, + .meta_size0 = isp->hw->meta_size, + .meta_size1 = isp->hw->meta_size, + .unk0 = 0, + .unk1 = 0, + .unk2 = 0, + .data_blocks = 1, + .compress = 0, + }; + return CISP_SEND_INOUT(isp, args); +} + +int isp_cmd_ch_buffer_pool_return(struct apple_isp *isp, u32 chan) +{ + struct cmd_ch_buffer_pool_return args = { + .opcode = CISP_OPCODE(CISP_CMD_CH_BUFFER_POOL_RETURN), + .chan = chan, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_apple_ch_temporal_filter_start(struct apple_isp *isp, u32 chan, u32 arg) +{ + struct cmd_apple_ch_temporal_filter_start args = { + .opcode = CISP_OPCODE(CISP_CMD_APPLE_CH_TEMPORAL_FILTER_START), + .chan = chan, + .unk_c = 1, + .unk_10 = arg, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_apple_ch_temporal_filter_stop(struct apple_isp *isp, u32 chan) +{ + struct cmd_apple_ch_temporal_filter_stop args = { + .opcode = CISP_OPCODE(CISP_CMD_APPLE_CH_TEMPORAL_FILTER_STOP), + .chan = chan, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_apple_ch_motion_history_start(struct apple_isp *isp, u32 chan) +{ + struct cmd_apple_ch_motion_history_start args = { + .opcode = CISP_OPCODE(CISP_CMD_APPLE_CH_MOTION_HISTORY_START), + .chan = chan, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_apple_ch_motion_history_stop(struct apple_isp *isp, u32 chan) +{ + struct cmd_apple_ch_motion_history_stop args = { + .opcode = CISP_OPCODE(CISP_CMD_APPLE_CH_MOTION_HISTORY_STOP), + .chan = chan, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_apple_ch_temporal_filter_enable(struct apple_isp *isp, u32 chan) +{ + struct cmd_apple_ch_temporal_filter_enable args = { + .opcode = CISP_OPCODE(CISP_CMD_APPLE_CH_TEMPORAL_FILTER_ENABLE), + .chan = chan, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_apple_ch_temporal_filter_disable(struct apple_isp *isp, u32 chan) +{ + struct cmd_apple_ch_temporal_filter_disable args = { + .opcode = + CISP_OPCODE(CISP_CMD_APPLE_CH_TEMPORAL_FILTER_DISABLE), + .chan = chan, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_ch_ae_stability_set(struct apple_isp *isp, u32 chan, u32 stability) +{ + struct cmd_ch_ae_stability_set args = { + .opcode = CISP_OPCODE(CISP_CMD_CH_AE_STABILITY_SET), + .chan = chan, + .stability = stability, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_ch_ae_stability_to_stable_set(struct apple_isp *isp, u32 chan, + u32 stability) +{ + struct cmd_ch_ae_stability_to_stable_set args = { + .opcode = CISP_OPCODE(CISP_CMD_CH_AE_STABILITY_TO_STABLE_SET), + .chan = chan, + .stability = stability, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_ch_ae_frame_rate_max_get(struct apple_isp *isp, u32 chan, + struct cmd_ch_ae_frame_rate_max_get *args) +{ + args->opcode = CISP_OPCODE(CISP_CMD_CH_AE_FRAME_RATE_MAX_GET); + args->chan = chan; + return CISP_SEND_OUT(isp, args); +} + +int isp_cmd_ch_ae_frame_rate_max_set(struct apple_isp *isp, u32 chan, + u32 framerate) +{ + struct cmd_ch_ae_frame_rate_max_set args = { + .opcode = CISP_OPCODE(CISP_CMD_CH_AE_FRAME_RATE_MAX_SET), + .chan = chan, + .framerate = framerate, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_ch_ae_frame_rate_min_set(struct apple_isp *isp, u32 chan, + u32 framerate) +{ + struct cmd_ch_ae_frame_rate_min_set args = { + .opcode = CISP_OPCODE(CISP_CMD_CH_AE_FRAME_RATE_MIN_SET), + .chan = chan, + .framerate = framerate, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_apple_ch_ae_fd_scene_metering_config_set(struct apple_isp *isp, + u32 chan) +{ + struct cmd_apple_ch_ae_fd_scene_metering_config_set args = { + .opcode = CISP_OPCODE( + CISP_CMD_APPLE_CH_AE_FD_SCENE_METERING_CONFIG_SET), + .chan = chan, + .unk_c = 0xb8, + .unk_10 = 0x2000200, + .unk_14 = 0x280800, + .unk_18 = 0xe10028, + .unk_1c = 0xa0399, + .unk_20 = 0x3cc02cc, + }; + return CISP_SEND_INOUT(isp, args); +} + +int isp_cmd_apple_ch_ae_metering_mode_set(struct apple_isp *isp, u32 chan, + u32 mode) +{ + struct cmd_apple_ch_ae_metering_mode_set args = { + .opcode = CISP_OPCODE(CISP_CMD_APPLE_CH_AE_METERING_MODE_SET), + .chan = chan, + .mode = mode, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_apple_ch_ae_flicker_freq_update_current_set(struct apple_isp *isp, + u32 chan, u32 freq) +{ + struct cmd_apple_ch_ae_flicker_freq_update_current_set args = { + .opcode = CISP_OPCODE( + CISP_CMD_APPLE_CH_AE_FLICKER_FREQ_UPDATE_CURRENT_SET), + .chan = chan, + .freq = freq, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_ch_semantic_video_enable(struct apple_isp *isp, u32 chan, + u32 enable) +{ + struct cmd_ch_semantic_video_enable args = { + .opcode = CISP_OPCODE(CISP_CMD_CH_SEMANTIC_VIDEO_ENABLE), + .chan = chan, + .enable = enable, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_ch_semantic_awb_enable(struct apple_isp *isp, u32 chan, u32 enable) +{ + struct cmd_ch_semantic_awb_enable args = { + .opcode = CISP_OPCODE(CISP_CMD_CH_SEMANTIC_AWB_ENABLE), + .chan = chan, + .enable = enable, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_ch_lpdp_hs_receiver_tuning_set(struct apple_isp *isp, u32 chan, u32 unk1, u32 unk2) +{ + struct cmd_ch_lpdp_hs_receiver_tuning_set args = { + .opcode = CISP_OPCODE(CISP_CMD_CH_LPDP_HS_RECEIVER_TUNING_SET), + .chan = chan, + .unk1 = unk1, + .unk2 = unk2, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_ch_property_write(struct apple_isp *isp, u32 chan, u32 prop, u32 val) +{ + struct cmd_ch_property_write args = { + .opcode = CISP_OPCODE(CISP_CMD_CH_PROPERTY_WRITE), + .chan = chan, + .prop = prop, + .val = val, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_ch_property_read(struct apple_isp *isp, u32 chan, u32 prop, u32 *val) +{ + struct cmd_ch_property_write args = { + .opcode = CISP_OPCODE(CISP_CMD_CH_PROPERTY_READ), + .chan = chan, + .prop = prop, + .val = 0xdeadbeef, + }; + int ret = CISP_SEND_OUT(isp, &args); + + *val = args.val; + + return ret; +} diff --git a/drivers/media/platform/apple/isp/isp-cmd.h b/drivers/media/platform/apple/isp/isp-cmd.h new file mode 100644 index 00000000000000..5a3c8cd9177e48 --- /dev/null +++ b/drivers/media/platform/apple/isp/isp-cmd.h @@ -0,0 +1,691 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright 2023 Eileen Yoon */ + +#ifndef __ISP_CMD_H__ +#define __ISP_CMD_H__ + +#include "isp-drv.h" + +#define CISP_CMD_START 0x0000 +#define CISP_CMD_STOP 0x0001 +#define CISP_CMD_CONFIG_GET 0x0003 +#define CISP_CMD_PRINT_ENABLE 0x0004 +#define CISP_CMD_BUILDINFO 0x0006 +#define CISP_CMD_GET_BES_PARAM 0x000f +#define CISP_CMD_POWER_DOWN 0x0010 +#define CISP_CMD_SET_ISP_PMU_BASE 0x0011 +#define CISP_CMD_PMP_CTRL_SET 0x001c +#define CISP_CMD_TRACE_ENABLE 0x001d +#define CISP_CMD_SUSPEND 0x0021 +#define CISP_CMD_FID_ENTER 0x0022 +#define CISP_CMD_FID_EXIT 0x0023 +#define CISP_CMD_FLICKER_SENSOR_SET 0x0024 +#define CISP_CMD_CH_START 0x0100 +#define CISP_CMD_CH_STOP 0x0101 +#define CISP_CMD_CH_BUFFER_RETURN 0x0104 +#define CISP_CMD_CH_CAMERA_CONFIG_CURRENT_GET 0x0105 +#define CISP_CMD_CH_CAMERA_CONFIG_GET 0x0106 +#define CISP_CMD_CH_CAMERA_CONFIG_SELECT 0x0107 +#define CISP_CMD_CH_INFO_GET 0x010d +#define CISP_CMD_CH_BUFFER_RECYCLE_MODE_SET 0x010e +#define CISP_CMD_CH_BUFFER_RECYCLE_START 0x010f +#define CISP_CMD_CH_BUFFER_RECYCLE_STOP 0x0110 +#define CISP_CMD_CH_SET_FILE_LOAD 0x0111 +#define CISP_CMD_CH_SIF_PIXEL_FORMAT_SET 0x0115 +#define CISP_CMD_CH_BUFFER_POOL_CONFIG_GET 0x0116 +#define CISP_CMD_CH_BUFFER_POOL_CONFIG_SET 0x0117 +#define CISP_CMD_CH_CAMERA_MIPI_FREQUENCY_GET 0x011a +#define CISP_CMD_CH_CAMERA_PIX_FREQUENCY_GET 0x011f +#define CISP_CMD_CH_PROPERTY_WRITE 0x0122 +#define CISP_CMD_CH_PROPERTY_READ 0x0123 +#define CISP_CMD_CH_LOCAL_RAW_BUFFER_ENABLE 0x0125 +#define CISP_CMD_CH_META_DATA_ENABLE 0x0126 +#define CISP_CMD_CH_CAMERA_MIPI_FREQUENCY_TOTAL_GET 0x0133 +#define CISP_CMD_CH_SBS_ENABLE 0x013b +#define CISP_CMD_CH_LSC_POLYNOMIAL_COEFF_GET 0x0142 +#define CISP_CMD_CH_SET_META_DATA_REQUIRED 0x014f +#define CISP_CMD_CH_BUFFER_POOL_RETURN 0x015b +#define CISP_CMD_CH_CAMERA_AGILE_FREQ_ARRAY_CURRENT_GET 0x015e +#define CISP_CMD_CH_AE_START 0x0200 +#define CISP_CMD_CH_AE_STOP 0x0201 +#define CISP_CMD_CH_AE_FRAME_RATE_MAX_GET 0x0207 +#define CISP_CMD_CH_AE_FRAME_RATE_MAX_SET 0x0208 +#define CISP_CMD_CH_AE_FRAME_RATE_MIN_GET 0x0209 +#define CISP_CMD_CH_AE_FRAME_RATE_MIN_SET 0x020a +#define CISP_CMD_CH_AE_STABILITY_SET 0x021a +#define CISP_CMD_CH_AE_STABILITY_TO_STABLE_SET 0x0229 +#define CISP_CMD_CH_SENSOR_NVM_GET 0x0501 +#define CISP_CMD_CH_SENSOR_PERMODULE_LSC_INFO_GET 0x0507 +#define CISP_CMD_CH_SENSOR_PERMODULE_LSC_GRID_GET 0x0511 +#define CISP_CMD_CH_LPDP_HS_RECEIVER_TUNING_SET 0x051b +#define CISP_CMD_CH_FOCUS_LIMITS_GET 0x0701 +#define CISP_CMD_CH_CROP_GET 0x0800 +#define CISP_CMD_CH_CROP_SET 0x0801 +#define CISP_CMD_CH_SCALER_CROP_SET 0x080a +#define CISP_CMD_CH_CROP_SCL1_GET 0x080b +#define CISP_CMD_CH_CROP_SCL1_SET 0x080c +#define CISP_CMD_CH_SCALER_CROP_SCL1_SET 0x080d +#define CISP_CMD_CH_ALS_ENABLE 0x0a1c +#define CISP_CMD_CH_ALS_DISABLE 0x0a1d +#define CISP_CMD_CH_CNR_START 0x0a2f +#define CISP_CMD_CH_MBNR_ENABLE 0x0a3a +#define CISP_CMD_CH_OUTPUT_CONFIG_SET 0x0b01 +#define CISP_CMD_CH_OUTPUT_CONFIG_SCL1_SET 0x0b09 +#define CISP_CMD_CH_PREVIEW_STREAM_SET 0x0b0d +#define CISP_CMD_CH_SEMANTIC_VIDEO_ENABLE 0x0b17 +#define CISP_CMD_CH_SEMANTIC_AWB_ENABLE 0x0b18 +#define CISP_CMD_CH_FACE_DETECTION_START 0x0d00 +#define CISP_CMD_CH_FACE_DETECTION_STOP 0x0d01 +#define CISP_CMD_CH_FACE_DETECTION_CONFIG_GET 0x0d02 +#define CISP_CMD_CH_FACE_DETECTION_CONFIG_SET 0x0d03 +#define CISP_CMD_CH_FACE_DETECTION_DISABLE 0x0d04 +#define CISP_CMD_CH_FACE_DETECTION_ENABLE 0x0d05 +#define CISP_CMD_CH_FID_START 0x3000 +#define CISP_CMD_CH_FID_STOP 0x3001 +#define CISP_CMD_IPC_ENDPOINT_SET2 0x300c +#define CISP_CMD_IPC_ENDPOINT_UNSET2 0x300d +#define CISP_CMD_SET_DSID_CLR_REG_BASE2 0x3204 +#define CISP_CMD_SET_DSID_CLR_REG_BASE 0x3205 +#define CISP_CMD_APPLE_CH_AE_METERING_MODE_SET 0x8206 +#define CISP_CMD_APPLE_CH_AE_FD_SCENE_METERING_CONFIG_SET 0x820e +#define CISP_CMD_APPLE_CH_AE_FLICKER_FREQ_UPDATE_CURRENT_SET 0x8212 +#define CISP_CMD_APPLE_CH_TEMPORAL_FILTER_START 0xc100 +#define CISP_CMD_APPLE_CH_TEMPORAL_FILTER_STOP 0xc101 +#define CISP_CMD_APPLE_CH_MOTION_HISTORY_START 0xc102 +#define CISP_CMD_APPLE_CH_MOTION_HISTORY_STOP 0xc103 +#define CISP_CMD_APPLE_CH_TEMPORAL_FILTER_ENABLE 0xc113 +#define CISP_CMD_APPLE_CH_TEMPORAL_FILTER_DISABLE 0xc114 + +#define CISP_POOL_TYPE_META 0x0 +#define CISP_POOL_TYPE_RENDERED 0x1 +#define CISP_POOL_TYPE_FD 0x2 +#define CISP_POOL_TYPE_RAW 0x3 +#define CISP_POOL_TYPE_STAT 0x4 +#define CISP_POOL_TYPE_RAW_AUX 0x5 +#define CISP_POOL_TYPE_YCC 0x6 +#define CISP_POOL_TYPE_CAPTURE_FULL_RES 0x7 +#define CISP_POOL_TYPE_META_CAPTURE 0x8 +#define CISP_POOL_TYPE_RENDERED_SCL1 0x9 +#define CISP_POOL_TYPE_STAT_PIXELOUTPUT 0x11 +#define CISP_POOL_TYPE_FSCL 0x12 +#define CISP_POOL_TYPE_CAPTURE_FULL_RES_YCC 0x13 +#define CISP_POOL_TYPE_RENDERED_RAW 0x14 +#define CISP_POOL_TYPE_CAPTURE_PDC_RAW 0x16 +#define CISP_POOL_TYPE_FPC_DATA 0x17 +#define CISP_POOL_TYPE_AICAM_SEG 0x19 +#define CISP_POOL_TYPE_SPD 0x1a +#define CISP_POOL_TYPE_META_DEPTH 0x1c +#define CISP_POOL_TYPE_JASPER_DEPTH 0x1d +#define CISP_POOL_TYPE_RAW_SIFR 0x1f +#define CISP_POOL_TYPE_FEP_THUMBNAIL_DYNAMIC_POOL_RAW 0x21 + +#define CISP_COLORSPACE_REC709 0x1 +#define CISP_OUTPUT_FORMAT_YUV_2PLANE 0x0 +#define CISP_OUTPUT_FORMAT_YUV_1PLANE 0x1 +#define CISP_OUTPUT_FORMAT_RGB 0x2 +#define CISP_BUFFER_RECYCLE_MODE_EMPTY_ONLY 0x1 + +struct cmd_start { + u64 opcode; + u32 mode; +} __packed; +static_assert(sizeof(struct cmd_start) == 0xc); + +struct cmd_stop { + u64 opcode; + u32 mode; +} __packed; +static_assert(sizeof(struct cmd_stop) == 0xc); + +struct cmd_power_down { + u64 opcode; +} __packed; +static_assert(sizeof(struct cmd_power_down) == 0x8); + +struct cmd_suspend { + u64 opcode; +} __packed; +static_assert(sizeof(struct cmd_suspend) == 0x8); + +struct cmd_print_enable { + u64 opcode; + u32 enable; +} __packed; +static_assert(sizeof(struct cmd_print_enable) == 0xc); + +struct cmd_trace_enable { + u64 opcode; + u32 enable; +} __packed; +static_assert(sizeof(struct cmd_trace_enable) == 0xc); + +struct cmd_config_get { + u64 opcode; + u32 timestamp_freq; + u32 num_channels; + u32 unk_10; + u32 unk_14; + u32 unk_18; +} __packed; +static_assert(sizeof(struct cmd_config_get) == 0x1c); + +struct cmd_set_isp_pmu_base { + u64 opcode; + u64 pmu_base; +} __packed; +static_assert(sizeof(struct cmd_set_isp_pmu_base) == 0x10); + +struct cmd_set_dsid_clr_req_base2 { + u64 opcode; + u64 dsid_clr_base0; + u64 dsid_clr_base1; + u64 dsid_clr_base2; + u64 dsid_clr_base3; + u32 dsid_clr_range0; + u32 dsid_clr_range1; + u32 dsid_clr_range2; + u32 dsid_clr_range3; +} __packed; +static_assert(sizeof(struct cmd_set_dsid_clr_req_base2) == 0x38); + +struct cmd_set_dsid_clr_req_base { + u64 opcode; + u64 dsid_clr_base; + u32 dsid_clr_range; +} __packed; +static_assert(sizeof(struct cmd_set_dsid_clr_req_base) == 0x14); + +struct cmd_pmp_ctrl_set { + u64 opcode; + u64 clock_scratch; + u64 clock_base; + u8 clock_bit; + u8 clock_size; + u16 clock_pad; + u64 bandwidth_scratch; + u64 bandwidth_base; + u8 bandwidth_bit; + u8 bandwidth_size; + u16 bandwidth_pad; +} __packed; +static_assert(sizeof(struct cmd_pmp_ctrl_set) == 0x30); + +struct cmd_fid_enter { + u64 opcode; +} __packed; +static_assert(sizeof(struct cmd_fid_enter) == 0x8); + +struct cmd_fid_exit { + u64 opcode; +} __packed; +static_assert(sizeof(struct cmd_fid_exit) == 0x8); + +struct cmd_ipc_endpoint_set2 { + u64 opcode; + u32 unk; + u64 addr1; + u32 size1; + u64 addr2; + u32 size2; + u64 regs; + u32 unk2; +} __packed; +static_assert(sizeof(struct cmd_ipc_endpoint_set2) == 0x30); + +struct cmd_flicker_sensor_set { + u64 opcode; + u32 mode; +} __packed; +static_assert(sizeof(struct cmd_flicker_sensor_set) == 0xc); + +int isp_cmd_start(struct apple_isp *isp, u32 mode); +int isp_cmd_stop(struct apple_isp *isp, u32 mode); +int isp_cmd_power_down(struct apple_isp *isp); +int isp_cmd_suspend(struct apple_isp *isp); +int isp_cmd_print_enable(struct apple_isp *isp, u32 enable); +int isp_cmd_trace_enable(struct apple_isp *isp, u32 enable); +int isp_cmd_config_get(struct apple_isp *isp, struct cmd_config_get *args); +int isp_cmd_set_isp_pmu_base(struct apple_isp *isp, u64 pmu_base); +int isp_cmd_set_dsid_clr_req_base(struct apple_isp *isp, u64 dsid_clr_base, + u32 dsid_clr_range); +int isp_cmd_set_dsid_clr_req_base2(struct apple_isp *isp, u64 dsid_clr_base0, + u64 dsid_clr_base1, u64 dsid_clr_base2, + u64 dsid_clr_base3, u32 dsid_clr_range0, + u32 dsid_clr_range1, u32 dsid_clr_range2, + u32 dsid_clr_range3); +int isp_cmd_pmp_ctrl_set(struct apple_isp *isp, u64 clock_scratch, + u64 clock_base, u8 clock_bit, u8 clock_size, + u64 bandwidth_scratch, u64 bandwidth_base, + u8 bandwidth_bit, u8 bandwidth_size); +int isp_cmd_fid_enter(struct apple_isp *isp); +int isp_cmd_fid_exit(struct apple_isp *isp); +int isp_cmd_flicker_sensor_set(struct apple_isp *isp, u32 mode); + +struct cmd_ch_start { + u64 opcode; + u32 chan; +} __packed; +static_assert(sizeof(struct cmd_ch_start) == 0xc); + +struct cmd_ch_stop { + u64 opcode; + u32 chan; +} __packed; +static_assert(sizeof(struct cmd_ch_stop) == 0xc); + +struct cmd_ch_info { + u64 opcode; + u32 chan; + u32 unk_c; // 0x7da0001, 0x7db0001 + u32 unk_10; // 0x300ac, 0x5006d + u32 unk_14; // 0x40007, 0x10007 + u32 unk_18; // 0x5, 0x2 + u32 unk_1c; // 0x1, 0x1 + u32 version; + u32 unk_24; // 0x7, 0x9 + u32 unk_28; // 0x1, 0x1410 + u32 unk_2c; // 0x7, 0x2 + u32 pad_30[7]; + u32 unk_4c; // 0x10000, 0x50000 + u32 unk_50; // 0x1, 0x1 + u32 unk_54; // 0x0, 0x0 + u32 unk_58; // 0x4, 0x4 + u32 unk_5c; // 0x10, 0x20 + u32 num_presets; + u32 unk_64; // 0x0, 0x0 + u32 unk_68; // 0x44c0, 0x4680 + u32 unk_6c; // 0x40, 0x40 + u32 unk_70; // 0x1, 0x1 + u32 unk_74; // 0x2, 0x2 + u32 unk_78; // 0x4000, 0x4000 + u32 unk_7c; // 0x40, 0x40 + u32 unk_80; // 0x1, 0x1 + u32 pad_84[2]; + u32 unk_8c; // 0x36, 0x36 + u32 pad_90[2]; + u32 timestamp_freq; + u16 pad_9c; + char module_sn[20]; + u16 pad_b0; + u32 unk_b4; // 0x8, 0x8 + u32 pad_b8[2]; + u32 unk_c0; // 0x4, 0x1 + u32 unk_c4; // 0x0, 0x0 + u32 unk_c8; // 0x0, 0x100 + u32 pad_cc[4]; + u32 unk_dc; // 0xff0000, 0xff0000 + u32 unk_e0; // 0xc00, 0xc00 + u32 unk_e4; // 0x0, 0x0 + u32 unk_e8; // 0x1c, 0x1c + u32 unk_ec; // 0x640, 0x680 + u32 unk_f0; // 0x4, 0x4 + u32 unk_f4; // 0x4, 0x4 + u32 pad_f8[6]; + u32 unk_110; // 0x0, 0x7800000 + u32 unk_114; // 0x0, 0x780 +} __packed; +static_assert(sizeof(struct cmd_ch_info) == 0x118); + +struct cmd_ch_camera_config { + u64 opcode; + u32 chan; + u32 preset; + u16 in_width; + u16 in_height; + u16 out_width; + u16 out_height; + u32 unk_28; + u32 unk_2c; + u32 unk_30[16]; + u32 sensor_clk; + u32 unk_64[4]; + u32 timestamp_freq; + u32 unk_78[2]; + u32 unk_80[16]; + u32 in_width2; // repeated in u32?? + u32 in_height2; + u32 unk_c8[3]; + u32 out_width2; + u32 out_height2; +} __packed; +static_assert(sizeof(struct cmd_ch_camera_config) == 0xdc); + +struct cmd_ch_camera_config_select { + u64 opcode; + u32 chan; + u32 preset; +} __packed; +static_assert(sizeof(struct cmd_ch_camera_config_select) == 0x10); + +struct cmd_ch_set_file_load { + u64 opcode; + u32 chan; + u32 addr; + u32 size; +} __packed; +static_assert(sizeof(struct cmd_ch_set_file_load) == 0x14); + +struct cmd_ch_set_file_load64 { + u64 opcode; + u32 chan; + u64 addr; + u32 size; +} __packed; +static_assert(sizeof(struct cmd_ch_set_file_load64) == 0x18); + +struct cmd_ch_buffer_return { + u64 opcode; + u32 chan; +} __packed; +static_assert(sizeof(struct cmd_ch_buffer_return) == 0xc); + +struct cmd_ch_sbs_enable { + u64 opcode; + u32 chan; + u32 enable; +} __packed; +static_assert(sizeof(struct cmd_ch_sbs_enable) == 0x10); + +struct cmd_ch_crop_set { + u64 opcode; + u32 chan; + u32 x1; + u32 y1; + u32 x2; + u32 y2; +} __packed; +static_assert(sizeof(struct cmd_ch_crop_set) == 0x1c); + +struct cmd_ch_output_config_set { + u64 opcode; + u32 chan; + u32 width; + u32 height; + u32 colorspace; + u32 format; + u32 strides[3]; + u32 padding_rows; + u32 unk_h0; + u32 compress; + u32 unk_w2; +} __packed; +static_assert(sizeof(struct cmd_ch_output_config_set) == 0x38); + +struct cmd_ch_preview_stream_set { + u64 opcode; + u32 chan; + u32 stream; +} __packed; +static_assert(sizeof(struct cmd_ch_preview_stream_set) == 0x10); + +struct cmd_ch_als_disable { + u64 opcode; + u32 chan; +} __packed; +static_assert(sizeof(struct cmd_ch_als_disable) == 0xc); + +struct cmd_ch_cnr_start { + u64 opcode; + u32 chan; +} __packed; +static_assert(sizeof(struct cmd_ch_cnr_start) == 0xc); + +struct cmd_ch_mbnr_enable { + u64 opcode; + u32 chan; + u32 use_case; + u32 mode; + u32 enable_chroma; +} __packed; +static_assert(sizeof(struct cmd_ch_mbnr_enable) == 0x18); + +struct cmd_ch_sif_pixel_format_set { + u64 opcode; + u32 chan; + u8 format; + u8 type; + u16 compress; + u32 unk_10; +} __packed; +static_assert(sizeof(struct cmd_ch_sif_pixel_format_set) == 0x14); + +struct cmd_ch_lpdp_hs_receiver_tuning_set { + u64 opcode; + u32 chan; + u32 unk1; + u32 unk2; +} __packed; +static_assert(sizeof(struct cmd_ch_lpdp_hs_receiver_tuning_set) == 0x14); + +struct cmd_ch_property_write { + u64 opcode; + u32 chan; + u32 prop; + u32 val; + u32 unk1; + u32 unk2; +} __packed; +static_assert(sizeof(struct cmd_ch_property_write) == 0x1c); + +int isp_cmd_ch_start(struct apple_isp *isp, u32 chan); +int isp_cmd_ch_stop(struct apple_isp *isp, u32 chan); +int isp_cmd_ch_info_get(struct apple_isp *isp, u32 chan, + struct cmd_ch_info *args); +int isp_cmd_ch_camera_config_get(struct apple_isp *isp, u32 chan, u32 preset, + struct cmd_ch_camera_config *args); +int isp_cmd_ch_camera_config_current_get(struct apple_isp *isp, u32 chan, + struct cmd_ch_camera_config *args); +int isp_cmd_ch_camera_config_select(struct apple_isp *isp, u32 chan, + u32 preset); +int isp_cmd_ch_set_file_load(struct apple_isp *isp, u32 chan, u64 addr, + u32 size); +int isp_cmd_ch_buffer_return(struct apple_isp *isp, u32 chan); +int isp_cmd_ch_sbs_enable(struct apple_isp *isp, u32 chan, u32 enable); +int isp_cmd_ch_crop_set(struct apple_isp *isp, u32 chan, u32 x1, u32 y1, u32 x2, + u32 y2); +int isp_cmd_ch_output_config_set(struct apple_isp *isp, u32 chan, u32 width, + u32 height, u32 strides[3], u32 colorspace, u32 format); +int isp_cmd_ch_preview_stream_set(struct apple_isp *isp, u32 chan, u32 stream); +int isp_cmd_ch_als_disable(struct apple_isp *isp, u32 chan); +int isp_cmd_ch_cnr_start(struct apple_isp *isp, u32 chan); +int isp_cmd_ch_mbnr_enable(struct apple_isp *isp, u32 chan, u32 use_case, + u32 mode, u32 enable_chroma); +int isp_cmd_ch_sif_pixel_format_set(struct apple_isp *isp, u32 chan); +int isp_cmd_ch_lpdp_hs_receiver_tuning_set(struct apple_isp *isp, u32 chan, u32 unk1, u32 unk2); + +int isp_cmd_ch_property_read(struct apple_isp *isp, u32 chan, u32 prop, u32 *val); +int isp_cmd_ch_property_write(struct apple_isp *isp, u32 chan, u32 prop, u32 val); + +enum isp_mbnr_mode { + ISP_MBNR_MODE_DISABLE = 0, + ISP_MBNR_MODE_ENABLE = 1, + ISP_MBNR_MODE_BYPASS = 2, +}; + +struct cmd_ch_buffer_recycle_mode_set { + u64 opcode; + u32 chan; + u32 mode; +} __packed; +static_assert(sizeof(struct cmd_ch_buffer_recycle_mode_set) == 0x10); + +struct cmd_ch_buffer_recycle_start { + u64 opcode; + u32 chan; +} __packed; +static_assert(sizeof(struct cmd_ch_buffer_recycle_start) == 0xc); + +struct cmd_ch_buffer_pool_config_set { + u64 opcode; + u32 chan; + u16 type; + u16 count; + u32 meta_size0; + u32 meta_size1; + u64 unk0; + u64 unk1; + u64 unk2; + u32 zero[0x19]; + u32 data_blocks; + u32 compress; +} __packed; +static_assert(sizeof(struct cmd_ch_buffer_pool_config_set) == 0x9c); + +struct cmd_ch_buffer_pool_return { + u64 opcode; + u32 chan; +} __packed; +static_assert(sizeof(struct cmd_ch_buffer_pool_return) == 0xc); + +int isp_cmd_ch_buffer_recycle_mode_set(struct apple_isp *isp, u32 chan, + u32 mode); +int isp_cmd_ch_buffer_recycle_start(struct apple_isp *isp, u32 chan); +int isp_cmd_ch_buffer_pool_config_set(struct apple_isp *isp, u32 chan, + u16 type); +int isp_cmd_ch_buffer_pool_config_get(struct apple_isp *isp, u32 chan, + u16 type); +int isp_cmd_ch_buffer_pool_return(struct apple_isp *isp, u32 chan); + +struct cmd_apple_ch_temporal_filter_start { + u64 opcode; + u32 chan; + u32 unk_c; + u32 unk_10; +} __packed; +static_assert(sizeof(struct cmd_apple_ch_temporal_filter_start) == 0x14); + +struct cmd_apple_ch_temporal_filter_stop { + u64 opcode; + u32 chan; +} __packed; +static_assert(sizeof(struct cmd_apple_ch_temporal_filter_stop) == 0xc); + +struct cmd_apple_ch_motion_history_start { + u64 opcode; + u32 chan; +} __packed; +static_assert(sizeof(struct cmd_apple_ch_motion_history_start) == 0xc); + +struct cmd_apple_ch_motion_history_stop { + u64 opcode; + u32 chan; +} __packed; +static_assert(sizeof(struct cmd_apple_ch_motion_history_stop) == 0xc); + +struct cmd_apple_ch_temporal_filter_enable { + u64 opcode; + u32 chan; +} __packed; +static_assert(sizeof(struct cmd_apple_ch_temporal_filter_enable) == 0xc); + +struct cmd_apple_ch_temporal_filter_disable { + u64 opcode; + u32 chan; +} __packed; +static_assert(sizeof(struct cmd_apple_ch_temporal_filter_disable) == 0xc); + +int isp_cmd_apple_ch_temporal_filter_start(struct apple_isp *isp, u32 chan, u32 arg); +int isp_cmd_apple_ch_temporal_filter_stop(struct apple_isp *isp, u32 chan); +int isp_cmd_apple_ch_motion_history_start(struct apple_isp *isp, u32 chan); +int isp_cmd_apple_ch_motion_history_stop(struct apple_isp *isp, u32 chan); +int isp_cmd_apple_ch_temporal_filter_enable(struct apple_isp *isp, u32 chan); +int isp_cmd_apple_ch_temporal_filter_disable(struct apple_isp *isp, u32 chan); + +struct cmd_ch_ae_stability_set { + u64 opcode; + u32 chan; + u32 stability; +} __packed; +static_assert(sizeof(struct cmd_ch_ae_stability_set) == 0x10); + +struct cmd_ch_ae_stability_to_stable_set { + u64 opcode; + u32 chan; + u32 stability; +} __packed; +static_assert(sizeof(struct cmd_ch_ae_stability_to_stable_set) == 0x10); + +struct cmd_ch_ae_frame_rate_max_get { + u64 opcode; + u32 chan; + u32 framerate; +} __packed; +static_assert(sizeof(struct cmd_ch_ae_frame_rate_max_get) == 0x10); + +struct cmd_ch_ae_frame_rate_max_set { + u64 opcode; + u32 chan; + u32 framerate; +} __packed; +static_assert(sizeof(struct cmd_ch_ae_frame_rate_max_set) == 0x10); + +struct cmd_ch_ae_frame_rate_min_set { + u64 opcode; + u32 chan; + u32 framerate; +} __packed; +static_assert(sizeof(struct cmd_ch_ae_frame_rate_min_set) == 0x10); + +struct cmd_apple_ch_ae_fd_scene_metering_config_set { + u64 opcode; + u32 chan; + u32 unk_c; + u32 unk_10; + u32 unk_14; + u32 unk_18; + u32 unk_1c; + u32 unk_20; +} __packed; +static_assert(sizeof(struct cmd_apple_ch_ae_fd_scene_metering_config_set) == + 0x24); + +struct cmd_apple_ch_ae_metering_mode_set { + u64 opcode; + u32 chan; + u32 mode; +} __packed; +static_assert(sizeof(struct cmd_apple_ch_ae_metering_mode_set) == 0x10); + +struct cmd_apple_ch_ae_flicker_freq_update_current_set { + u64 opcode; + u32 chan; + u32 freq; +} __packed; +static_assert(sizeof(struct cmd_apple_ch_ae_flicker_freq_update_current_set) == + 0x10); + +int isp_cmd_ch_ae_stability_set(struct apple_isp *isp, u32 chan, u32 stability); +int isp_cmd_ch_ae_stability_to_stable_set(struct apple_isp *isp, u32 chan, + u32 stability); +int isp_cmd_ch_ae_frame_rate_max_get(struct apple_isp *isp, u32 chan, + struct cmd_ch_ae_frame_rate_max_get *args); +int isp_cmd_ch_ae_frame_rate_max_set(struct apple_isp *isp, u32 chan, + u32 framerate); +int isp_cmd_ch_ae_frame_rate_min_set(struct apple_isp *isp, u32 chan, + u32 framerate); +int isp_cmd_apple_ch_ae_fd_scene_metering_config_set(struct apple_isp *isp, + u32 chan); +int isp_cmd_apple_ch_ae_metering_mode_set(struct apple_isp *isp, u32 chan, + u32 mode); +int isp_cmd_apple_ch_ae_flicker_freq_update_current_set(struct apple_isp *isp, + u32 chan, u32 freq); + +struct cmd_ch_semantic_video_enable { + u64 opcode; + u32 chan; + u32 enable; +} __packed; +static_assert(sizeof(struct cmd_ch_semantic_video_enable) == 0x10); + +struct cmd_ch_semantic_awb_enable { + u64 opcode; + u32 chan; + u32 enable; +} __packed; +static_assert(sizeof(struct cmd_ch_semantic_awb_enable) == 0x10); + +int isp_cmd_ch_semantic_video_enable(struct apple_isp *isp, u32 chan, + u32 enable); +int isp_cmd_ch_semantic_awb_enable(struct apple_isp *isp, u32 chan, u32 enable); + +#endif /* __ISP_CMD_H__ */ diff --git a/drivers/media/platform/apple/isp/isp-drv.c b/drivers/media/platform/apple/isp/isp-drv.c new file mode 100644 index 00000000000000..848f7abd535a7f --- /dev/null +++ b/drivers/media/platform/apple/isp/isp-drv.c @@ -0,0 +1,594 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Apple Image Signal Processor driver + * + * Copyright (C) 2023 The Asahi Linux Contributors + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "isp-cam.h" +#include "isp-fw.h" +#include "isp-iommu.h" +#include "isp-v4l2.h" + +static void apple_isp_detach_genpd(struct apple_isp *isp) +{ + if (isp->pd_count <= 1) + return; + + for (int i = isp->pd_count - 1; i >= 0; i--) { + if (isp->pd_link[i]) + device_link_del(isp->pd_link[i]); + if (!IS_ERR_OR_NULL(isp->pd_dev[i])) + dev_pm_domain_detach(isp->pd_dev[i], true); + } + + return; +} + +static int apple_isp_attach_genpd(struct apple_isp *isp) +{ + struct device *dev = isp->dev; + + isp->pd_count = of_count_phandle_with_args( + dev->of_node, "power-domains", "#power-domain-cells"); + if (isp->pd_count <= 1) + return 0; + + isp->pd_dev = devm_kcalloc(dev, isp->pd_count, sizeof(*isp->pd_dev), + GFP_KERNEL); + if (!isp->pd_dev) + return -ENOMEM; + + isp->pd_link = devm_kcalloc(dev, isp->pd_count, sizeof(*isp->pd_link), + GFP_KERNEL); + if (!isp->pd_link) + return -ENOMEM; + + for (int i = 0; i < isp->pd_count; i++) { + int flags = DL_FLAG_STATELESS; + + /* Primary power domain uses RPM integration */ + if (i == 0) + flags |= DL_FLAG_PM_RUNTIME | DL_FLAG_RPM_ACTIVE; + + isp->pd_dev[i] = dev_pm_domain_attach_by_id(dev, i); + if (IS_ERR(isp->pd_dev[i])) { + apple_isp_detach_genpd(isp); + return PTR_ERR(isp->pd_dev[i]); + } + + isp->pd_link[i] = + device_link_add(dev, isp->pd_dev[i], flags); + + if (!isp->pd_link[i]) { + apple_isp_detach_genpd(isp); + return -EINVAL; + } + } + + return 0; +} + +static int apple_isp_init_iommu(struct apple_isp *isp) +{ + struct device *dev = isp->dev; + phys_addr_t heap_base; + size_t heap_size; + u64 vm_size; + int err; + int idx; + int size; + struct device_node *mem_node; + const __be32 *maps, *end; + + isp->domain = iommu_get_domain_for_dev(isp->dev); + if (!isp->domain) + return -ENODEV; + isp->shift = __ffs(isp->domain->pgsize_bitmap); + + idx = of_property_match_string(dev->of_node, "memory-region-names", + "heap"); + mem_node = of_parse_phandle(dev->of_node, "memory-region", idx); + if (!mem_node) { + dev_err(dev, "No memory-region found for heap\n"); + return -ENODEV; + } + + maps = of_get_property(mem_node, "iommu-addresses", &size); + if (!maps || !size) { + dev_err(dev, "No valid iommu-addresses found for heap\n"); + return -ENODEV; + } + + end = maps + size / sizeof(__be32); + + while (maps < end) { + maps++; + maps = of_translate_dma_region(dev->of_node, maps, &heap_base, + &heap_size); + } + + isp->fw.heap_top = heap_base + heap_size; + + err = of_property_read_u64(dev->of_node, "apple,dart-vm-size", + &vm_size); + if (err) { + dev_err(dev, "failed to read 'apple,dart-vm-size': %d\n", err); + return err; + } + + // FIXME: refactor this, maybe use regular iova stuff? + drm_mm_init(&isp->iovad, isp->fw.heap_top, + vm_size - (heap_base & 0xffffffff)); + + return 0; +} + +static void apple_isp_free_iommu(struct apple_isp *isp) +{ + drm_mm_takedown(&isp->iovad); +} + +static int isp_of_read_coord(struct device *dev, struct device_node *np, + const char *prop, struct coord *val) +{ + u32 xy[2]; + int ret; + + ret = of_property_read_u32_array(np, prop, xy, 2); + if (ret) { + dev_err(dev, "failed to read '%s' property\n", prop); + return ret; + } + + val->x = xy[0]; + val->y = xy[1]; + return 0; +} + +static int apple_isp_init_presets(struct apple_isp *isp) +{ + struct device *dev = isp->dev; + struct isp_preset *preset; + int err = 0; + + struct device_node *np __free(device_node) = + of_get_child_by_name(dev->of_node, "sensor-presets"); + if (!np) { + dev_err(dev, "failed to get DT node 'presets'\n"); + return -EINVAL; + } + + isp->num_presets = of_get_child_count(np); + if (!isp->num_presets) { + dev_err(dev, "no sensor presets found\n"); + return -EINVAL; + } + + isp->presets = devm_kzalloc( + dev, sizeof(*isp->presets) * isp->num_presets, GFP_KERNEL); + if (!isp->presets) + return -ENOMEM; + + preset = isp->presets; + for_each_child_of_node_scoped(np, child) { + u32 xywh[4]; + + err = of_property_read_u32(child, "apple,config-index", + &preset->index); + if (err) { + dev_err(dev, "no apple,config-index property\n"); + return err; + } + + err = isp_of_read_coord(dev, child, "apple,input-size", + &preset->input_dim); + if (err) + return err; + err = isp_of_read_coord(dev, child, "apple,output-size", + &preset->output_dim); + if (err) + return err; + + err = of_property_read_u32_array(child, "apple,crop", xywh, 4); + if (err) { + dev_err(dev, "failed to read 'apple,crop' property\n"); + return err; + } + preset->crop_offset.x = xywh[0]; + preset->crop_offset.y = xywh[1]; + preset->crop_size.x = xywh[2]; + preset->crop_size.y = xywh[3]; + + preset++; + } + + return 0; +} + +static const char * isp_fw2str(enum isp_firmware_version version) +{ + switch (version) { + case ISP_FIRMWARE_V_12_3: + return "12.3"; + case ISP_FIRMWARE_V_12_4: + return "12.4"; + case ISP_FIRMWARE_V_13_5: + return "13.5"; + default: + return "unknown"; + } +} + +#define ISP_FW_VERSION_MIN_LEN 3 +#define ISP_FW_VERSION_MAX_LEN 5 + +static enum isp_firmware_version isp_read_fw_version(struct device *dev, + const char *name) +{ + u32 ver[ISP_FW_VERSION_MAX_LEN]; + int len = of_property_read_variable_u32_array(dev->of_node, name, ver, + ISP_FW_VERSION_MIN_LEN, + ISP_FW_VERSION_MAX_LEN); + + switch (len) { + case 3: + if (ver[0] == 12 && ver[1] == 3 && ver[2] <= 1) + return ISP_FIRMWARE_V_12_3; + else if (ver[0] == 12 && ver[1] == 4 && ver[2] == 0) + return ISP_FIRMWARE_V_12_4; + else if (ver[0] == 13 && ver[1] == 5 && ver[2] == 0) + return ISP_FIRMWARE_V_13_5; + + dev_warn(dev, "unknown %s: %d.%d.%d\n", name, ver[0], ver[1], ver[2]); + break; + case 4: + dev_warn(dev, "unknown %s: %d.%d.%d.%d\n", name, ver[0], ver[1], + ver[2], ver[3]); + break; + case 5: + dev_warn(dev, "unknown %s: %d.%d.%d.%d.%d\n", name, ver[0], + ver[1], ver[2], ver[3], ver[4]); + break; + default: + dev_warn(dev, "could not parse %s: %d\n", name, len); + break; + } + + return ISP_FIRMWARE_V_UNKNOWN; +} + +static enum isp_firmware_version isp_check_firmware_version(struct device *dev) +{ + enum isp_firmware_version version, compat; + + /* firmware version is just informative */ + version = isp_read_fw_version(dev, "apple,firmware-version"); + compat = isp_read_fw_version(dev, "apple,firmware-compat"); + + dev_info(dev, "ISP firmware-compat: %s (FW: %s)\n", isp_fw2str(compat), + isp_fw2str(version)); + + return compat; +} + +static int apple_isp_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct apple_isp *isp; + int err; + + err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(42)); + if (err) + return err; + + isp = devm_kzalloc(dev, sizeof(*isp), GFP_KERNEL); + if (!isp) + return -ENOMEM; + + isp->dev = dev; + isp->hw = of_device_get_match_data(dev); + platform_set_drvdata(pdev, isp); + dev_set_drvdata(dev, isp); + + /* Differences between firmware versions are rather minor so try to work + * with unknown firmware. + */ + isp->fw_compat = isp_check_firmware_version(dev); + + err = of_property_read_u32(dev->of_node, "apple,platform-id", + &isp->platform_id); + if (err) { + dev_err(dev, "failed to get 'apple,platform-id' property: %d\n", + err); + return err; + } + + err = of_property_read_u32(dev->of_node, "apple,temporal-filter", + &isp->temporal_filter); + if (err) + isp->temporal_filter = 0; + + err = apple_isp_init_presets(isp); + if (err) { + dev_err(dev, "failed to initialize presets\n"); + return err; + } + + err = apple_isp_attach_genpd(isp); + if (err) { + dev_err(dev, "failed to attatch power domains\n"); + return err; + } + + isp->coproc = devm_platform_ioremap_resource_byname(pdev, "coproc"); + if (IS_ERR(isp->coproc)) { + err = PTR_ERR(isp->coproc); + goto detach_genpd; + } + + isp->mbox = devm_platform_ioremap_resource_byname(pdev, "mbox"); + if (IS_ERR(isp->mbox)) { + err = PTR_ERR(isp->mbox); + goto detach_genpd; + } + + isp->gpio = devm_platform_ioremap_resource_byname(pdev, "gpio"); + if (IS_ERR(isp->gpio)) { + err = PTR_ERR(isp->gpio); + goto detach_genpd; + } + + isp->mbox2 = devm_platform_ioremap_resource_byname(pdev, "mbox2"); + if (IS_ERR(isp->mbox2)) { + err = PTR_ERR(isp->mbox2); + goto detach_genpd; + } + + isp->irq = platform_get_irq(pdev, 0); + if (isp->irq < 0) { + err = isp->irq; + goto detach_genpd; + } + if (!isp->irq) { + err = -ENODEV; + goto detach_genpd; + } + + mutex_init(&isp->iovad_lock); + mutex_init(&isp->video_lock); + spin_lock_init(&isp->buf_lock); + init_waitqueue_head(&isp->wait); + INIT_LIST_HEAD(&isp->gc); + INIT_LIST_HEAD(&isp->bufs_pending); + INIT_LIST_HEAD(&isp->bufs_submitted); + isp->wq = alloc_workqueue("apple-isp-wq", WQ_UNBOUND, 0); + if (!isp->wq) { + dev_err(dev, "failed to create workqueue\n"); + err = -ENOMEM; + goto detach_genpd; + } + + err = apple_isp_init_iommu(isp); + if (err) { + dev_err(dev, "failed to init iommu: %d\n", err); + goto destroy_wq; + } + + err = apple_isp_alloc_firmware_surface(isp); + if (err) { + dev_err(dev, "failed to alloc firmware surface: %d\n", err); + goto free_iommu; + } + + pm_runtime_enable(dev); + + err = apple_isp_detect_camera(isp); + if (err) { + dev_err(dev, "failed to detect camera: %d\n", err); + goto free_surface; + } + + err = apple_isp_setup_video(isp); + if (err) { + dev_err(dev, "failed to register video device: %d\n", err); + goto free_surface; + } + + dev_info(dev, "apple-isp probe!\n"); + + return 0; + +free_surface: + pm_runtime_disable(dev); + apple_isp_free_firmware_surface(isp); +free_iommu: + apple_isp_free_iommu(isp); +destroy_wq: + destroy_workqueue(isp->wq); +detach_genpd: + apple_isp_detach_genpd(isp); + return err; +} + +static void apple_isp_remove(struct platform_device *pdev) +{ + struct apple_isp *isp = platform_get_drvdata(pdev); + + apple_isp_remove_video(isp); + pm_runtime_disable(isp->dev); + apple_isp_free_firmware_surface(isp); + apple_isp_free_iommu(isp); + destroy_workqueue(isp->wq); + apple_isp_detach_genpd(isp); +} + +static const struct apple_isp_hw apple_isp_hw_t8103 = { + .gen = ISP_GEN_T8103, + .pmu_base = 0x23b704000, + + .dsid_count = 4, + .dsid_clr_base0 = 0x200014000, + .dsid_clr_base1 = 0x200054000, + .dsid_clr_base2 = 0x200094000, + .dsid_clr_base3 = 0x2000d4000, + .dsid_clr_range0 = 0x1000, + .dsid_clr_range1 = 0x1000, + .dsid_clr_range2 = 0x1000, + .dsid_clr_range3 = 0x1000, + + .clock_scratch = 0x23b738010, + .clock_base = 0x23bc3c000, + .clock_bit = 0x1, + .clock_size = 0x4, + .bandwidth_scratch = 0x23b73800c, + .bandwidth_base = 0x23bc3c000, + .bandwidth_bit = 0x0, + .bandwidth_size = 0x4, + + .scl1 = false, + .lpdp = false, + .meta_size = ISP_META_SIZE_T8103, +}; + +static const struct apple_isp_hw apple_isp_hw_t6000 = { + .gen = ISP_GEN_T8103, + .pmu_base = 0x28e584000, + + .dsid_count = 1, + .dsid_clr_base0 = 0x200014000, + .dsid_clr_base1 = 0x200054000, + .dsid_clr_base2 = 0x200094000, + .dsid_clr_base3 = 0x2000d4000, + .dsid_clr_range0 = 0x1000, + .dsid_clr_range1 = 0x1000, + .dsid_clr_range2 = 0x1000, + .dsid_clr_range3 = 0x1000, + + .clock_scratch = 0x28e3d0868, + .clock_base = 0x0, + .clock_bit = 0x0, + .clock_size = 0x8, + .bandwidth_scratch = 0x28e3d0980, + .bandwidth_base = 0x0, + .bandwidth_bit = 0x0, + .bandwidth_size = 0x8, + + .scl1 = false, + .lpdp = false, + .meta_size = ISP_META_SIZE_T8103, +}; + +static const struct apple_isp_hw apple_isp_hw_t8112 = { + .gen = ISP_GEN_T8112, + .pmu_base = 0x23b704000, + + .dsid_count = 1, + .dsid_clr_base0 = 0x200f14000, + .dsid_clr_range0 = 0x1000, + + .clock_scratch = 0x23b3d0560, + .clock_base = 0x0, + .clock_bit = 0x0, + .clock_size = 0x8, + .bandwidth_scratch = 0x23b3d05d0, + .bandwidth_base = 0x0, + .bandwidth_bit = 0x0, + .bandwidth_size = 0x8, + + .scl1 = false, + .lpdp = false, + .meta_size = ISP_META_SIZE_T8112, +}; + +static const struct apple_isp_hw apple_isp_hw_t6020 = { + .gen = ISP_GEN_T8112, + .pmu_base = 0x290284000, + + .dsid_count = 1, + .dsid_clr_base0 = 0x200f14000, + .dsid_clr_range0 = 0x1000, + + .clock_scratch = 0x28e3d10a8, + .clock_base = 0x0, + .clock_bit = 0x0, + .clock_size = 0x8, + .bandwidth_scratch = 0x28e3d1200, + .bandwidth_base = 0x0, + .bandwidth_bit = 0x0, + .bandwidth_size = 0x8, + + .scl1 = true, + .lpdp = true, + .meta_size = ISP_META_SIZE_T8112, +}; + +static const struct of_device_id apple_isp_of_match[] = { + { .compatible = "apple,t8103-isp", .data = &apple_isp_hw_t8103 }, + { .compatible = "apple,t8112-isp", .data = &apple_isp_hw_t8112 }, + { .compatible = "apple,t6000-isp", .data = &apple_isp_hw_t6000 }, + { .compatible = "apple,t6020-isp", .data = &apple_isp_hw_t6020 }, + {}, +}; +MODULE_DEVICE_TABLE(of, apple_isp_of_match); + +static __maybe_unused int apple_isp_runtime_suspend(struct device *dev) +{ + /* RPM sleep is called when the V4L2 file handle is closed */ + return 0; +} + +static __maybe_unused int apple_isp_runtime_resume(struct device *dev) +{ + return 0; +} + +static __maybe_unused int apple_isp_suspend(struct device *dev) +{ + struct apple_isp *isp = dev_get_drvdata(dev); + + /* We must restore V4L2 context on system resume. If we were streaming + * before, we (essentially) stop streaming and start streaming again. + */ + apple_isp_video_suspend(isp); + + return 0; +} + +static __maybe_unused int apple_isp_resume(struct device *dev) +{ + struct apple_isp *isp = dev_get_drvdata(dev); + + apple_isp_video_resume(isp); + + return 0; +} + +static const struct dev_pm_ops apple_isp_pm_ops = { + SYSTEM_SLEEP_PM_OPS(apple_isp_suspend, apple_isp_resume) + RUNTIME_PM_OPS(apple_isp_runtime_suspend, apple_isp_runtime_resume, NULL) +}; + +static struct platform_driver apple_isp_driver = { + .driver = { + .name = "apple-isp", + .of_match_table = apple_isp_of_match, + .pm = pm_ptr(&apple_isp_pm_ops), + }, + .probe = apple_isp_probe, + .remove = apple_isp_remove, +}; +module_platform_driver(apple_isp_driver); + +MODULE_AUTHOR("Eileen Yoon "); +MODULE_DESCRIPTION("Apple ISP driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/media/platform/apple/isp/isp-drv.h b/drivers/media/platform/apple/isp/isp-drv.h new file mode 100644 index 00000000000000..96a1d0b39f860d --- /dev/null +++ b/drivers/media/platform/apple/isp/isp-drv.h @@ -0,0 +1,290 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright 2023 Eileen Yoon */ + +#ifndef __ISP_DRV_H__ +#define __ISP_DRV_H__ + +#include +#include +#include +#include + +#include +#include +#include +#include + +/* #define APPLE_ISP_DEBUG */ +#define APPLE_ISP_DEVICE_NAME "apple-isp" +#define APPLE_ISP_CARD_NAME "FaceTime HD Camera" + +#define ISP_MAX_CHANNELS 6 +#define ISP_IPC_MESSAGE_SIZE 64 +#define ISP_IPC_FLAG_ACK 0x1 +#define ISP_META_SIZE_T8103 0x4640 +#define ISP_META_SIZE_T8112 0x4840 + +/* used to limit the user space buffers to the buffer_pool_config */ +#define ISP_MAX_BUFFERS 16 + +enum isp_generation { + ISP_GEN_T8103, + ISP_GEN_T8112, +}; + +enum isp_firmware_version { + ISP_FIRMWARE_V_UNKNOWN, + ISP_FIRMWARE_V_12_3, + ISP_FIRMWARE_V_12_4, + ISP_FIRMWARE_V_13_5, +}; + +struct isp_surf { + struct drm_mm_node *mm; + struct list_head head; + u64 size; + u64 type; + u32 num_pages; + struct page **pages; + struct sg_table sgt; + dma_addr_t iova; + void *virt; + refcount_t refcount; + bool gc; + bool submitted; +}; + +struct isp_message { + u64 arg0; + u64 arg1; + u64 arg2; + u64 arg3; + u64 arg4; + u64 arg5; + u64 arg6; + u64 arg7; +} __packed; +static_assert(sizeof(struct isp_message) == ISP_IPC_MESSAGE_SIZE); + +struct isp_channel { + char *name; + u32 type; + u32 src; + u32 num; + u64 size; + dma_addr_t iova; + void *virt; + u32 doorbell; + u32 cursor; + struct mutex lock; + struct isp_message req; + struct isp_message rsp; + const struct isp_chan_ops *ops; +}; + +struct coord { + u32 x; + u32 y; +}; + +struct isp_preset { + u32 index; + struct coord input_dim; + struct coord output_dim; + struct coord crop_offset; + struct coord crop_size; +}; + +struct apple_isp_hw { + enum isp_generation gen; + u64 pmu_base; + + int dsid_count; + u64 dsid_clr_base0; + u64 dsid_clr_base1; + u64 dsid_clr_base2; + u64 dsid_clr_base3; + u32 dsid_clr_range0; + u32 dsid_clr_range1; + u32 dsid_clr_range2; + u32 dsid_clr_range3; + + u64 clock_scratch; + u64 clock_base; + u8 clock_bit; + u8 clock_size; + u64 bandwidth_scratch; + u64 bandwidth_base; + u8 bandwidth_bit; + u8 bandwidth_size; + + u32 meta_size; + bool scl1; + bool lpdp; +}; + +enum isp_sensor_id { + ISP_IMX248_1820_01, + ISP_IMX248_1822_02, + ISP_IMX343_5221_02, + ISP_IMX354_9251_02, + ISP_IMX356_4820_01, + ISP_IMX356_4820_02, + ISP_IMX364_8720_01, + ISP_IMX364_8723_01, + ISP_IMX372_3820_01, + ISP_IMX372_3820_02, + ISP_IMX372_3820_11, + ISP_IMX372_3820_12, + ISP_IMX405_9720_01, + ISP_IMX405_9721_01, + ISP_IMX405_9723_01, + ISP_IMX414_2520_01, + ISP_IMX503_7820_01, + ISP_IMX503_7820_02, + ISP_IMX505_3921_01, + ISP_IMX514_2820_01, + ISP_IMX514_2820_02, + ISP_IMX514_2820_03, + ISP_IMX514_2820_04, + ISP_IMX558_1921_01, + ISP_IMX558_1922_02, + ISP_IMX603_7920_01, + ISP_IMX603_7920_02, + ISP_IMX603_7921_01, + ISP_IMX613_4920_01, + ISP_IMX613_4920_02, + ISP_IMX614_2921_01, + ISP_IMX614_2921_02, + ISP_IMX614_2922_02, + ISP_IMX633_3622_01, + ISP_IMX703_7721_01, + ISP_IMX703_7722_01, + ISP_IMX713_4721_01, + ISP_IMX713_4722_01, + ISP_IMX714_2022_01, + ISP_IMX772_3721_01, + ISP_IMX772_3721_11, + ISP_IMX772_3722_01, + ISP_IMX772_3723_01, + ISP_IMX814_2123_01, + ISP_IMX853_7622_01, + ISP_IMX913_7523_01, + ISP_VD56G0_6221_01, + ISP_VD56G0_6222_01, +}; + +struct isp_format { + enum isp_sensor_id id; + u32 version; + struct isp_preset *preset; + unsigned int num_planes; + u32 strides[VB2_MAX_PLANES]; + size_t plane_size[VB2_MAX_PLANES]; + size_t total_size; +}; + +struct apple_isp { + struct device *dev; + const struct apple_isp_hw *hw; + enum isp_firmware_version fw_compat; + u32 platform_id; + u32 temporal_filter; + struct isp_preset *presets; + int num_presets; + + int num_channels; + struct isp_format fmts[ISP_MAX_CHANNELS]; + unsigned int current_ch; + + struct video_device vdev; + struct media_device mdev; + struct v4l2_device v4l2_dev; + struct vb2_queue vbq; + struct mutex video_lock; + unsigned int sequence; + bool multiplanar; + + int pd_count; + struct device **pd_dev; + struct device_link **pd_link; + bool pds_active; + + int irq; + + void __iomem *coproc; + void __iomem *mbox; + void __iomem *gpio; + void __iomem *mbox2; + + struct iommu_domain *domain; + unsigned long shift; + struct drm_mm iovad; /* TODO iova.c can't allocate bottom-up */ + struct mutex iovad_lock; + + struct isp_firmware { + u64 heap_top; + } fw; + + struct isp_surf *ipc_surf; + struct isp_surf *extra_surf; + struct isp_surf *data_surf; + struct isp_surf *log_surf; + struct isp_surf *bt_surf; + struct isp_surf *meta_surfs[ISP_MAX_BUFFERS]; + struct list_head gc; + struct workqueue_struct *wq; + + int num_ipc_chans; + struct isp_channel **ipc_chans; + struct isp_channel *chan_tm; /* TERMINAL */ + struct isp_channel *chan_io; /* IO */ + struct isp_channel *chan_dg; /* DEBUG */ + struct isp_channel *chan_bh; /* BUF_H2T */ + struct isp_channel *chan_bt; /* BUF_T2H */ + struct isp_channel *chan_sm; /* SHAREDMALLOC */ + struct isp_channel *chan_it; /* IO_T2H */ + + wait_queue_head_t wait; + dma_addr_t cmd_iova; + void *cmd_virt; + + unsigned long state; + spinlock_t buf_lock; + struct list_head bufs_pending; + struct list_head bufs_submitted; +}; + +struct isp_chan_ops { + int (*handle)(struct apple_isp *isp, struct isp_channel *chan); +}; + +struct isp_buffer { + struct vb2_v4l2_buffer vb; + struct list_head link; + struct isp_surf surfs[VB2_MAX_PLANES]; +}; + +#define to_isp_buffer(x) container_of((x), struct isp_buffer, vb) + +enum { + ISP_STATE_STREAMING, + ISP_STATE_LOGGING, + ISP_STATE_SLEEPING, +}; + +#ifdef APPLE_ISP_DEBUG +#define isp_dbg(isp, fmt, ...) \ + dev_info((isp)->dev, "[%s] " fmt, __func__, ##__VA_ARGS__) +#else +#define isp_dbg(isp, fmt, ...) \ + dev_dbg((isp)->dev, "[%s] " fmt, __func__, ##__VA_ARGS__) +#endif + +#define isp_err(isp, fmt, ...) \ + dev_err((isp)->dev, "[%s] " fmt, __func__, ##__VA_ARGS__) + +#define isp_get_format(isp, ch) (&(isp)->fmts[(ch)]) +#define isp_get_current_format(isp) (isp_get_format(isp, isp->current_ch)) + +#endif /* __ISP_DRV_H__ */ diff --git a/drivers/media/platform/apple/isp/isp-fw.c b/drivers/media/platform/apple/isp/isp-fw.c new file mode 100644 index 00000000000000..a39f5fb4445fa7 --- /dev/null +++ b/drivers/media/platform/apple/isp/isp-fw.c @@ -0,0 +1,788 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright 2023 Eileen Yoon */ + +#include "isp-fw.h" + +#include +#include +#include +#include + +#include "isp-cmd.h" +#include "isp-iommu.h" +#include "isp-ipc.h" +#include "isp-regs.h" +#include "isp-v4l2.h" + +#define ISP_FIRMWARE_MDELAY 1 +#define ISP_FIRMWARE_MAX_TRIES 1000 + +#define ISP_FIRMWARE_IPC_SIZE 0x1c000 +#define ISP_FIRMWARE_DATA_SIZE 0x28000 + +#define ISP_COPROC_IN_WFI 0x3 + +static inline u32 isp_coproc_read32(struct apple_isp *isp, u32 reg) +{ + return readl(isp->coproc + reg); +} + +static inline void isp_coproc_write32(struct apple_isp *isp, u32 reg, u32 val) +{ + writel(val, isp->coproc + reg); +} + +static inline u32 isp_gpio_read32(struct apple_isp *isp, u32 reg) +{ + return readl(isp->gpio + reg); +} + +static inline void isp_gpio_write32(struct apple_isp *isp, u32 reg, u32 val) +{ + writel(val, isp->gpio + reg); +} + +static int apple_isp_power_up_domains(struct apple_isp *isp) +{ + int ret; + + if (isp->pds_active) + return 0; + + for (int i = 1; i < isp->pd_count; i++) { + ret = pm_runtime_get_sync(isp->pd_dev[i]); + if (ret < 0) { + dev_err(isp->dev, + "Failed to power up power domain %d: %d\n", i, ret); + while (--i != 1) + pm_runtime_put_sync(isp->pd_dev[i]); + return ret; + } + } + + isp->pds_active = true; + + return 0; +} + +static void apple_isp_power_down_domains(struct apple_isp *isp) +{ + int ret; + + if (!isp->pds_active) + return; + + for (int i = isp->pd_count - 1; i >= 1; i--) { + ret = pm_runtime_put_sync(isp->pd_dev[i]); + if (ret < 0) + dev_err(isp->dev, + "Failed to power up power domain %d: %d\n", i, ret); + } + + isp->pds_active = false; +} + +void *apple_isp_translate(struct apple_isp *isp, struct isp_surf *surf, + dma_addr_t iova, size_t size) +{ + dma_addr_t end = iova + size; + if (!surf) { + dev_err(isp->dev, + "Failed to translate IPC iova 0x%llx (0x%zx): No surface\n", + (long long)iova, size); + return NULL; + } + + if (end < iova || iova < surf->iova || + end > (surf->iova + surf->size)) { + dev_err(isp->dev, + "Failed to translate IPC iova 0x%llx (0x%zx): Out of bounds\n", + (long long)iova, size); + return NULL; + } + + if (!surf->virt) { + dev_err(isp->dev, + "Failed to translate IPC iova 0x%llx (0x%zx): No VMap\n", + (long long)iova, size); + return NULL; + } + + return surf->virt + (iova - surf->iova); +} + +struct isp_firmware_bootargs { + u32 pad_0[2]; + u64 ipc_iova; + u64 shared_base; + u64 shared_size; + u64 extra_iova; + u64 extra_size; + u32 platform_id; + u32 pad_40; + u64 logbuf_addr; + u64 logbuf_size; + u64 logbuf_entsize; + u32 ipc_size; + u32 pad_60[5]; + u32 unk5; + u32 pad_7c[13]; + u32 pad_b0; + u32 unk7; + u32 pad_b8[5]; + u32 unk_iova1; + u32 pad_c0[47]; + u32 unk9; +} __packed; +static_assert(sizeof(struct isp_firmware_bootargs) == 0x180); + +struct isp_chan_desc { + char name[64]; + u32 type; + u32 src; + u32 num; + u32 pad; + u64 iova; + u32 padding[0x2a]; +} __packed; +static_assert(sizeof(struct isp_chan_desc) == 0x100); + +static const struct isp_chan_ops tm_ops = { + .handle = ipc_tm_handle, +}; + +static const struct isp_chan_ops sm_ops = { + .handle = ipc_sm_handle, +}; + +static const struct isp_chan_ops bt_ops = { + .handle = ipc_bt_handle, +}; + +static irqreturn_t apple_isp_isr(int irq, void *dev) +{ + struct apple_isp *isp = dev; + + isp_mbox2_write32(isp, ISP_MBOX2_IRQ_ACK, + isp_mbox_read32(isp, ISP_MBOX_IRQ_INTERRUPT)); + + return IRQ_WAKE_THREAD; +} + +static irqreturn_t apple_isp_isr_thread(int irq, void *dev) +{ + struct apple_isp *isp = dev; + + wake_up_all(&isp->wait); + + ipc_chan_handle(isp, isp->chan_sm); + wake_up_all(&isp->wait); /* Some commands depend on sm */ + + ipc_chan_handle(isp, isp->chan_tm); + + ipc_chan_handle(isp, isp->chan_bt); + wake_up_all(&isp->wait); + + return IRQ_HANDLED; +} + +static void isp_disable_irq(struct apple_isp *isp) +{ + isp_mbox_write32(isp, ISP_MBOX_IRQ_ENABLE, 0x0); + free_irq(isp->irq, isp); + isp_gpio_write32(isp, ISP_GPIO_1, 0xfeedbabe); /* real funny */ +} + +static int isp_enable_irq(struct apple_isp *isp) +{ + int err; + + err = request_threaded_irq(isp->irq, apple_isp_isr, + apple_isp_isr_thread, 0, "apple-isp", isp); + if (err < 0) { + isp_err(isp, "failed to request IRQ#%u (%d)\n", isp->irq, err); + return err; + } + + isp_dbg(isp, "about to enable interrupts...\n"); + + isp_mbox_write32(isp, ISP_MBOX_IRQ_ENABLE, 0xf); + + return 0; +} + +static int isp_reset_coproc(struct apple_isp *isp) +{ + int retries; + u32 status; + u32 val; + + isp_coproc_write32(isp, ISP_COPROC_EDPRCR, 0x2); + + isp_coproc_write32(isp, ISP_COPROC_FABRIC_0, 0xff00ff); + isp_coproc_write32(isp, ISP_COPROC_FABRIC_1, 0xff00ff); + isp_coproc_write32(isp, ISP_COPROC_FABRIC_2, 0xff00ff); + isp_coproc_write32(isp, ISP_COPROC_FABRIC_3, 0xff00ff); + + isp_coproc_write32(isp, ISP_COPROC_IRQ_MASK_0, 0xffffffff); + isp_coproc_write32(isp, ISP_COPROC_IRQ_MASK_1, 0xffffffff); + isp_coproc_write32(isp, ISP_COPROC_IRQ_MASK_2, 0xffffffff); + isp_coproc_write32(isp, ISP_COPROC_IRQ_MASK_3, 0xffffffff); + isp_coproc_write32(isp, ISP_COPROC_IRQ_MASK_4, 0xffffffff); + isp_coproc_write32(isp, ISP_COPROC_IRQ_MASK_5, 0xffffffff); + + for (retries = 0; retries < 128; retries++) { + val = isp_coproc_read32(isp, 0x818); + if (val == 0) + break; + } + + for (retries = 0; retries < 128; retries++) { + val = isp_coproc_read32(isp, 0x81c); + if (val == 0) + break; + } + + for (retries = 0; retries < ISP_FIRMWARE_MAX_TRIES; retries++) { + status = isp_coproc_read32(isp, ISP_COPROC_STATUS); + if (status & ISP_COPROC_IN_WFI) { + isp_dbg(isp, "%d: coproc in WFI (status: 0x%x)\n", + retries, status); + break; + } + mdelay(ISP_FIRMWARE_MDELAY); + } + if (retries >= ISP_FIRMWARE_MAX_TRIES) { + isp_err(isp, "coproc NOT in WFI (status: 0x%x)\n", status); + return -ENODEV; + } + + return 0; +} + +static void isp_firmware_shutdown_stage1(struct apple_isp *isp) +{ + isp_coproc_write32(isp, ISP_COPROC_CONTROL, 0x0); + + apple_isp_power_down_domains(isp); +} + +static int isp_firmware_boot_stage1(struct apple_isp *isp) +{ + int err, retries; + // u32 val; + + err = apple_isp_power_up_domains(isp); + if (err < 0) + return err; + + + isp_gpio_write32(isp, ISP_GPIO_CLOCK_EN, 0x1); + +#if 0 + /* This doesn't work well with system sleep */ + val = isp_gpio_read32(isp, ISP_GPIO_1); + if (val == 0xfeedbabe) { + err = isp_reset_coproc(isp); + if (err < 0) + return err; + } +#endif + + err = isp_reset_coproc(isp); + if (err < 0) + return err; + + isp_gpio_write32(isp, ISP_GPIO_0, 0x0); + isp_gpio_write32(isp, ISP_GPIO_1, 0x0); + isp_gpio_write32(isp, ISP_GPIO_2, 0x0); + isp_gpio_write32(isp, ISP_GPIO_3, 0x0); + isp_gpio_write32(isp, ISP_GPIO_4, 0x0); + isp_gpio_write32(isp, ISP_GPIO_5, 0x0); + isp_gpio_write32(isp, ISP_GPIO_6, 0x0); + isp_gpio_write32(isp, ISP_GPIO_7, 0x0); + + isp_mbox_write32(isp, ISP_MBOX_IRQ_ENABLE, 0x0); + + isp_coproc_write32(isp, ISP_COPROC_CONTROL, 0x0); + isp_coproc_write32(isp, ISP_COPROC_CONTROL, 0x10); + + /* Wait for ISP_GPIO_7 to 0x0 -> 0x8042006 */ + for (retries = 0; retries < ISP_FIRMWARE_MAX_TRIES; retries++) { + u32 val = isp_gpio_read32(isp, ISP_GPIO_7); + if (val == 0x8042006) { + isp_dbg(isp, + "got first magic number (0x%x) from firmware\n", + val); + break; + } + mdelay(ISP_FIRMWARE_MDELAY); + } + if (retries >= ISP_FIRMWARE_MAX_TRIES) { + isp_err(isp, + "never received first magic number from firmware\n"); + return -ENODEV; + } + + return 0; +} + +int apple_isp_alloc_firmware_surface(struct apple_isp *isp) +{ + /* These are static, so let's do it once and for all */ + isp->ipc_surf = isp_alloc_surface_vmap(isp, ISP_FIRMWARE_IPC_SIZE); + if (!isp->ipc_surf) { + isp_err(isp, "failed to alloc shared surface for ipc\n"); + return -ENOMEM; + } + dev_info(isp->dev, "IPC surface iova: 0x%llx\n", + (long long)isp->ipc_surf->iova); + + isp->data_surf = isp_alloc_surface_vmap(isp, ISP_FIRMWARE_DATA_SIZE); + if (!isp->data_surf) { + isp_err(isp, "failed to alloc shared surface for data files\n"); + isp_free_surface(isp, isp->ipc_surf); + return -ENOMEM; + } + dev_info(isp->dev, "Data surface iova: 0x%llx\n", + (long long)isp->data_surf->iova); + + return 0; +} + +void apple_isp_free_firmware_surface(struct apple_isp *isp) +{ + isp_free_surface(isp, isp->data_surf); + isp_free_surface(isp, isp->ipc_surf); +} + +static void isp_firmware_shutdown_stage2(struct apple_isp *isp) +{ + isp_free_surface(isp, isp->extra_surf); +} + +static int isp_firmware_boot_stage2(struct apple_isp *isp) +{ + struct isp_firmware_bootargs args; + dma_addr_t args_iova; + void *args_virt; + int err, retries; + + u32 num_ipc_chans = isp_gpio_read32(isp, ISP_GPIO_0); + u32 args_offset = isp_gpio_read32(isp, ISP_GPIO_1); + u32 extra_size = isp_gpio_read32(isp, ISP_GPIO_3); + isp->num_ipc_chans = num_ipc_chans; + + if (!isp->num_ipc_chans) { + dev_err(isp->dev, "No IPC channels found\n"); + return -ENODEV; + } + + if (isp->num_ipc_chans != 7) + dev_warn(isp->dev, "unexpected channel count (%d)\n", + num_ipc_chans); + + isp->extra_surf = isp_alloc_surface_vmap(isp, extra_size); + if (!isp->extra_surf) { + isp_err(isp, "failed to alloc surface for extra heap\n"); + return -ENOMEM; + } + + args_iova = isp->ipc_surf->iova + args_offset + 0x40; + args_virt = isp->ipc_surf->virt + args_offset + 0x40; + isp->cmd_iova = args_iova + sizeof(args) + 0x40; + isp->cmd_virt = args_virt + sizeof(args) + 0x40; + + memset(&args, 0, sizeof(args)); + args.ipc_iova = isp->ipc_surf->iova; + args.ipc_size = isp->ipc_surf->size; + args.shared_base = isp->fw.heap_top & 0xffffffff; + args.shared_size = 0x10000000UL - args.shared_base; + args.extra_iova = isp->extra_surf->iova; + args.extra_size = isp->extra_surf->size; + args.platform_id = isp->platform_id; + args.unk5 = 0x40; + args.unk7 = 0x1; // 0? + args.unk_iova1 = args_iova + sizeof(args) - 0xc; + args.unk9 = 0x3; + memcpy(args_virt, &args, sizeof(args)); + + isp_gpio_write32(isp, ISP_GPIO_0, args_iova); + isp_gpio_write32(isp, ISP_GPIO_1, args_iova >> 32); + dma_wmb(); + + /* Wait for ISP_GPIO_7 to 0xf7fbdff9 -> 0x8042006 */ + isp_gpio_write32(isp, ISP_GPIO_7, 0xf7fbdff9); + + for (retries = 0; retries < ISP_FIRMWARE_MAX_TRIES; retries++) { + u32 val = isp_gpio_read32(isp, ISP_GPIO_7); + if (val == 0x8042006) { + isp_dbg(isp, + "got second magic number (0x%x) from firmware\n", + val); + break; + } + mdelay(ISP_FIRMWARE_MDELAY); + } + if (retries >= ISP_FIRMWARE_MAX_TRIES) { + isp_err(isp, + "never received second magic number from firmware\n"); + err = -ENODEV; + goto free_extra; + } + + return 0; + +free_extra: + isp_free_surface(isp, isp->extra_surf); + return err; +} + +static inline struct isp_channel *isp_get_chan_index(struct apple_isp *isp, + const char *name) +{ + for (int i = 0; i < isp->num_ipc_chans; i++) { + if (!strcasecmp(isp->ipc_chans[i]->name, name)) + return isp->ipc_chans[i]; + } + return NULL; +} + +static void isp_free_channel_info(struct apple_isp *isp) +{ + for (int i = 0; i < isp->num_ipc_chans; i++) { + struct isp_channel *chan = isp->ipc_chans[i]; + if (!chan) + continue; + kfree(chan->name); + kfree(chan); + isp->ipc_chans[i] = NULL; + } + kfree(isp->ipc_chans); + isp->ipc_chans = NULL; +} + +static int isp_fill_channel_info(struct apple_isp *isp) +{ + u64 table_iova = isp_gpio_read32(isp, ISP_GPIO_0) | + ((u64)isp_gpio_read32(isp, ISP_GPIO_1)) << 32; + void *table_virt = apple_isp_ipc_translate( + isp, table_iova, + sizeof(struct isp_chan_desc) * isp->num_ipc_chans); + + if (!table_virt) { + dev_err(isp->dev, "Failed to find channel table\n"); + return -EIO; + } + + isp->ipc_chans = kcalloc(isp->num_ipc_chans, + sizeof(struct isp_channel *), GFP_KERNEL); + if (!isp->ipc_chans) + goto out; + + for (int i = 0; i < isp->num_ipc_chans; i++) { + struct isp_chan_desc desc; + void *desc_virt = table_virt + (i * sizeof(desc)); + struct isp_channel *chan = + kzalloc(sizeof(struct isp_channel), GFP_KERNEL); + if (!chan) + goto out; + isp->ipc_chans[i] = chan; + + memcpy(&desc, desc_virt, sizeof(desc)); + chan->name = kstrdup(desc.name, GFP_KERNEL); + chan->type = desc.type; + chan->src = desc.src; + chan->doorbell = 1 << chan->src; + chan->num = desc.num; + chan->size = desc.num * ISP_IPC_MESSAGE_SIZE; + chan->iova = desc.iova; + chan->virt = + apple_isp_ipc_translate(isp, desc.iova, chan->size); + chan->cursor = 0; + mutex_init(&chan->lock); + + if (!chan->virt) { + dev_err(isp->dev, "Failed to find channel buffer\n"); + goto out; + } + + if ((chan->type != ISP_IPC_CHAN_TYPE_COMMAND) && + (chan->type != ISP_IPC_CHAN_TYPE_REPLY) && + (chan->type != ISP_IPC_CHAN_TYPE_REPORT)) { + isp_err(isp, "invalid ipc chan type (%d)\n", + chan->type); + goto out; + } + + isp_dbg(isp, "chan: %s type: %d src: %d num: %d iova: 0x%llx\n", + chan->name, chan->type, chan->src, chan->num, + chan->iova); + } + + isp->chan_tm = isp_get_chan_index(isp, "TERMINAL"); + isp->chan_io = isp_get_chan_index(isp, "IO"); + isp->chan_dg = isp_get_chan_index(isp, "DEBUG"); + isp->chan_bh = isp_get_chan_index(isp, "BUF_H2T"); + isp->chan_bt = isp_get_chan_index(isp, "BUF_T2H"); + isp->chan_sm = isp_get_chan_index(isp, "SHAREDMALLOC"); + isp->chan_it = isp_get_chan_index(isp, "IO_T2H"); + + if (!isp->chan_tm || !isp->chan_io || !isp->chan_dg || !isp->chan_bh || + !isp->chan_bt || !isp->chan_sm || !isp->chan_it) { + isp_err(isp, "did not find all of the required ipc chans\n"); + goto out; + } + + isp->chan_tm->ops = &tm_ops; + isp->chan_sm->ops = &sm_ops; + isp->chan_bt->ops = &bt_ops; + + return 0; +out: + isp_free_channel_info(isp); + return -ENOMEM; +} + +static void isp_firmware_shutdown_stage3(struct apple_isp *isp) +{ + isp_free_channel_info(isp); +} + +static int isp_firmware_boot_stage3(struct apple_isp *isp) +{ + int err, retries; + + err = isp_fill_channel_info(isp); + if (err < 0) + return err; + + /* Mask the command channels to prepare for submission */ + for (int i = 0; i < isp->num_ipc_chans; i++) { + struct isp_channel *chan = isp->ipc_chans[i]; + if (chan->type != ISP_IPC_CHAN_TYPE_COMMAND) + continue; + for (int j = 0; j < chan->num; j++) { + struct isp_message msg; + void *msg_virt = chan->virt + (j * sizeof(msg)); + + memset(&msg, 0, sizeof(msg)); + msg.arg0 = ISP_IPC_FLAG_ACK; + memcpy(msg_virt, &msg, sizeof(msg)); + } + } + dma_wmb(); + + /* Wait for ISP_GPIO_3 to 0x8042006 -> 0x0 */ + isp_gpio_write32(isp, ISP_GPIO_3, 0x8042006); + + for (retries = 0; retries < ISP_FIRMWARE_MAX_TRIES; retries++) { + u32 val = isp_gpio_read32(isp, ISP_GPIO_3); + if (val == 0x0) { + isp_dbg(isp, + "got third magic number (0x%x) from firmware\n", + val); + break; + } + mdelay(ISP_FIRMWARE_MDELAY); + } + if (retries >= ISP_FIRMWARE_MAX_TRIES) { + isp_err(isp, + "never received third magic number from firmware\n"); + isp_free_channel_info(isp); + return -ENODEV; + } + + isp_dbg(isp, "firmware booted!\n"); + + return 0; +} + +static int isp_stop_command_processor(struct apple_isp *isp) +{ + int retries; + +#if 0 + int res = isp_cmd_stop(isp, 0); + if (res) { + isp_err(isp, "isp_cmd_stop() failed\n"); + return res; + } + + /* Wait for ISP_GPIO_0 to 0xf7fbdff9 -> 0x8042006 */ + isp_gpio_write32(isp, ISP_GPIO_0, 0xf7fbdff9); + + isp_cmd_power_down(isp); +#else + isp_gpio_write32(isp, ISP_GPIO_0, 0xf7fbdff9); + + int res = isp_cmd_suspend(isp); + if (res) { + isp_err(isp, "isp_cmd_suspend() failed\n"); + return res; + } +#endif + + for (retries = 0; retries < ISP_FIRMWARE_MAX_TRIES; retries++) { + u32 val = isp_gpio_read32(isp, ISP_GPIO_0); + if (val == 0x8042006) { + isp_dbg(isp, "got magic number (0x%x) from firmware\n", + val); + break; + } + mdelay(ISP_FIRMWARE_MDELAY); + } + if (retries >= ISP_FIRMWARE_MAX_TRIES) { + isp_err(isp, "never received magic number from firmware\n"); + return -ENODEV; + } + + return 0; +} + +static int isp_start_command_processor(struct apple_isp *isp) +{ + int err; + + err = isp_cmd_print_enable(isp, 1); + if (err) + return err; + + err = isp_cmd_set_isp_pmu_base(isp, isp->hw->pmu_base); + if (err) + return err; + + if (isp->hw->dsid_count == 1) { + err = isp_cmd_set_dsid_clr_req_base( + isp, isp->hw->dsid_clr_base0, isp->hw->dsid_clr_range0); + if (err) + return err; + } else { + err = isp_cmd_set_dsid_clr_req_base2( + isp, isp->hw->dsid_clr_base0, isp->hw->dsid_clr_base1, + isp->hw->dsid_clr_base2, isp->hw->dsid_clr_base3, + isp->hw->dsid_clr_range0, isp->hw->dsid_clr_range1, + isp->hw->dsid_clr_range2, isp->hw->dsid_clr_range3); + if (err) + return err; + } + + err = isp_cmd_pmp_ctrl_set( + isp, isp->hw->clock_scratch, isp->hw->clock_base, + isp->hw->clock_bit, isp->hw->clock_size, + isp->hw->bandwidth_scratch, isp->hw->bandwidth_base, + isp->hw->bandwidth_bit, isp->hw->bandwidth_size); + if (err) + return err; + + err = isp_cmd_start(isp, 0); + if (err) + return err; + + /* Now we can access CISP_CMD_CH_* commands */ + + return 0; +} + +static void isp_collect_gc_surface(struct apple_isp *isp) +{ + struct isp_surf *tmp, *surf; + + isp->log_surf = NULL; + isp->bt_surf = NULL; + + list_for_each_entry_safe_reverse(surf, tmp, &isp->gc, head) { + isp_dbg(isp, "freeing iova: 0x%llx size: 0x%llx virt: %pS\n", + surf->iova, surf->size, (void *)surf->virt); + isp_free_surface(isp, surf); + } +} + +static int isp_firmware_boot(struct apple_isp *isp) +{ + int err; + + err = isp_firmware_boot_stage1(isp); + if (err < 0) { + isp_err(isp, "failed firmware boot stage 1: %d\n", err); + goto garbage_collect; + } + + err = isp_firmware_boot_stage2(isp); + if (err < 0) { + isp_err(isp, "failed firmware boot stage 2: %d\n", err); + goto shutdown_stage1; + } + + err = isp_firmware_boot_stage3(isp); + if (err < 0) { + isp_err(isp, "failed firmware boot stage 3: %d\n", err); + goto shutdown_stage2; + } + + err = isp_enable_irq(isp); + if (err < 0) { + isp_err(isp, "failed to enable interrupts: %d\n", err); + goto shutdown_stage3; + } + + err = isp_start_command_processor(isp); + if (err < 0) { + isp_err(isp, "failed to start command processor: %d\n", err); + goto disable_irqs; + } + + flush_workqueue(isp->wq); + + return 0; + +disable_irqs: + isp_disable_irq(isp); +shutdown_stage3: + isp_firmware_shutdown_stage3(isp); +shutdown_stage2: + isp_firmware_shutdown_stage2(isp); +shutdown_stage1: + isp_firmware_shutdown_stage1(isp); +garbage_collect: + isp_collect_gc_surface(isp); + return err; +} + +static void isp_firmware_shutdown(struct apple_isp *isp) +{ + flush_workqueue(isp->wq); + isp_stop_command_processor(isp); + isp_disable_irq(isp); + isp_firmware_shutdown_stage3(isp); + isp_firmware_shutdown_stage2(isp); + isp_firmware_shutdown_stage1(isp); + isp_collect_gc_surface(isp); +} + +int apple_isp_firmware_boot(struct apple_isp *isp) +{ + int err; + + /* Needs to be power cycled for IOMMU to behave correctly */ + err = pm_runtime_resume_and_get(isp->dev); + if (err < 0) { + dev_err(isp->dev, "failed to enable power: %d\n", err); + return err; + } + + err = isp_firmware_boot(isp); + if (err) { + dev_err(isp->dev, "failed to boot firmware: %d\n", err); + pm_runtime_put_sync(isp->dev); + return err; + } + + return 0; +} + +void apple_isp_firmware_shutdown(struct apple_isp *isp) +{ + isp_firmware_shutdown(isp); + pm_runtime_put_sync(isp->dev); +} diff --git a/drivers/media/platform/apple/isp/isp-fw.h b/drivers/media/platform/apple/isp/isp-fw.h new file mode 100644 index 00000000000000..974216f0989f91 --- /dev/null +++ b/drivers/media/platform/apple/isp/isp-fw.h @@ -0,0 +1,24 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright 2023 Eileen Yoon */ + +#ifndef __ISP_FW_H__ +#define __ISP_FW_H__ + +#include "isp-drv.h" + +int apple_isp_alloc_firmware_surface(struct apple_isp *isp); +void apple_isp_free_firmware_surface(struct apple_isp *isp); + +int apple_isp_firmware_boot(struct apple_isp *isp); +void apple_isp_firmware_shutdown(struct apple_isp *isp); + +void *apple_isp_translate(struct apple_isp *isp, struct isp_surf *surf, + dma_addr_t iova, size_t size); + +static inline void *apple_isp_ipc_translate(struct apple_isp *isp, + dma_addr_t iova, size_t size) +{ + return apple_isp_translate(isp, isp->ipc_surf, iova, size); +} + +#endif /* __ISP_FW_H__ */ diff --git a/drivers/media/platform/apple/isp/isp-iommu.c b/drivers/media/platform/apple/isp/isp-iommu.c new file mode 100644 index 00000000000000..1ddd089d77355a --- /dev/null +++ b/drivers/media/platform/apple/isp/isp-iommu.c @@ -0,0 +1,250 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright 2023 Eileen Yoon */ + +#include +#include + +#include "isp-iommu.h" + +static void isp_surf_free_pages(struct isp_surf *surf) +{ + for (u32 i = 0; i < surf->num_pages && surf->pages[i] != NULL; i++) { + __free_page(surf->pages[i]); + } + kvfree(surf->pages); +} + +static int isp_surf_alloc_pages(struct isp_surf *surf) +{ + surf->pages = kvmalloc_array(surf->num_pages, sizeof(*surf->pages), + GFP_KERNEL); + if (!surf->pages) + return -ENOMEM; + + for (u32 i = 0; i < surf->num_pages; i++) { + surf->pages[i] = alloc_page(GFP_KERNEL | __GFP_ZERO); + if (surf->pages[i] == NULL) + goto free_pages; + } + + return 0; + +free_pages: + isp_surf_free_pages(surf); + return -ENOMEM; +} + +int isp_surf_vmap(struct apple_isp *isp, struct isp_surf *surf) +{ + surf->virt = vmap(surf->pages, surf->num_pages, VM_MAP, + pgprot_writecombine(PAGE_KERNEL)); + if (surf->virt == NULL) { + dev_err(isp->dev, "failed to vmap size 0x%llx\n", surf->size); + return -EINVAL; + } + + return 0; +} + +static void isp_surf_vunmap(struct apple_isp *isp, struct isp_surf *surf) +{ + if (surf->virt) + vunmap(surf->virt); + surf->virt = NULL; +} + +static void isp_surf_unreserve_iova(struct apple_isp *isp, + struct isp_surf *surf) +{ + if (surf->mm) { + mutex_lock(&isp->iovad_lock); + drm_mm_remove_node(surf->mm); + mutex_unlock(&isp->iovad_lock); + kfree(surf->mm); + } + surf->mm = NULL; +} + +static int isp_surf_reserve_iova(struct apple_isp *isp, struct isp_surf *surf) +{ + int err; + + surf->mm = kzalloc(sizeof(*surf->mm), GFP_KERNEL); + if (!surf->mm) + return -ENOMEM; + + mutex_lock(&isp->iovad_lock); + err = drm_mm_insert_node_generic(&isp->iovad, surf->mm, + ALIGN(surf->size, 1UL << isp->shift), + 1UL << isp->shift, 0, 0); + mutex_unlock(&isp->iovad_lock); + if (err < 0) { + dev_err(isp->dev, "failed to reserve 0x%llx of iova space\n", + surf->size); + goto mm_free; + } + + surf->iova = surf->mm->start; + + return 0; +mm_free: + kfree(surf->mm); + surf->mm = NULL; + return err; +} + +static void isp_surf_iommu_unmap(struct apple_isp *isp, struct isp_surf *surf) +{ + iommu_unmap(isp->domain, surf->iova, surf->size); + sg_free_table(&surf->sgt); +} + +static int isp_surf_iommu_map(struct apple_isp *isp, struct isp_surf *surf) +{ + unsigned long size; + int err; + + err = sg_alloc_table_from_pages(&surf->sgt, surf->pages, + surf->num_pages, 0, surf->size, + GFP_KERNEL); + if (err < 0) { + dev_err(isp->dev, "failed to alloc sgt from pages\n"); + return err; + } + + size = iommu_map_sgtable(isp->domain, surf->iova, &surf->sgt, + IOMMU_READ | IOMMU_WRITE | IOMMU_CACHE); + if (size < surf->size) { + dev_err(isp->dev, "failed to iommu_map sgt to iova 0x%llx\n", + surf->iova); + sg_free_table(&surf->sgt); + return -ENXIO; + } + + return 0; +} + +static void __isp_surf_init(struct apple_isp *isp, struct isp_surf *surf, + u64 size, bool gc) +{ + surf->mm = NULL; + surf->virt = NULL; + surf->size = ALIGN(size, 1UL << isp->shift); + surf->num_pages = surf->size >> isp->shift; + surf->gc = gc; +} + +struct isp_surf *__isp_alloc_surface(struct apple_isp *isp, u64 size, bool gc) +{ + int err; + + struct isp_surf *surf = kzalloc(sizeof(struct isp_surf), GFP_KERNEL); + if (!surf) + return NULL; + + __isp_surf_init(isp, surf, size, gc); + + err = isp_surf_alloc_pages(surf); + if (err < 0) { + dev_err(isp->dev, "failed to allocate %d pages\n", + surf->num_pages); + goto free_surf; + } + + err = isp_surf_reserve_iova(isp, surf); + if (err < 0) { + dev_err(isp->dev, "failed to reserve 0x%llx of iova space\n", + surf->size); + goto free_pages; + } + + err = isp_surf_iommu_map(isp, surf); + if (err < 0) { + dev_err(isp->dev, + "failed to iommu_map size 0x%llx to iova 0x%llx\n", + surf->size, surf->iova); + goto unreserve_iova; + } + + refcount_set(&surf->refcount, 1); + if (surf->gc) + list_add_tail(&surf->head, &isp->gc); + + return surf; + +unreserve_iova: + isp_surf_unreserve_iova(isp, surf); +free_pages: + isp_surf_free_pages(surf); +free_surf: + kfree(surf); + return NULL; +} + +struct isp_surf *isp_alloc_surface_vmap(struct apple_isp *isp, u64 size) +{ + int err; + + struct isp_surf *surf = __isp_alloc_surface(isp, size, false); + if (!surf) + return NULL; + + err = isp_surf_vmap(isp, surf); + if (err < 0) { + dev_err(isp->dev, "failed to vmap iova 0x%llx - 0x%llx\n", + surf->iova, surf->iova + surf->size); + isp_free_surface(isp, surf); + return NULL; + } + + return surf; +} + +void isp_free_surface(struct apple_isp *isp, struct isp_surf *surf) +{ + if (refcount_dec_and_test(&surf->refcount)) { + isp_surf_vunmap(isp, surf); + isp_surf_iommu_unmap(isp, surf); + isp_surf_unreserve_iova(isp, surf); + isp_surf_free_pages(surf); + if (surf->gc) + list_del(&surf->head); + kfree(surf); + } +} + +int apple_isp_iommu_map_sgt(struct apple_isp *isp, struct isp_surf *surf, + struct sg_table *sgt, u64 size) +{ + int err; + ssize_t mapped; + + // TODO userptr sends unaligned sizes + surf->mm = NULL; + surf->size = size; + + err = isp_surf_reserve_iova(isp, surf); + if (err < 0) { + dev_err(isp->dev, "failed to reserve 0x%llx of iova space\n", + surf->size); + return err; + } + + mapped = iommu_map_sgtable(isp->domain, surf->iova, sgt, + IOMMU_READ | IOMMU_WRITE | IOMMU_CACHE); + if (mapped < surf->size) { + dev_err(isp->dev, "failed to iommu_map sgt to iova 0x%llx\n", + surf->iova); + isp_surf_unreserve_iova(isp, surf); + return -ENXIO; + } + surf->size = mapped; + + return 0; +} + +void apple_isp_iommu_unmap_sgt(struct apple_isp *isp, struct isp_surf *surf) +{ + iommu_unmap(isp->domain, surf->iova, surf->size); + isp_surf_unreserve_iova(isp, surf); +} diff --git a/drivers/media/platform/apple/isp/isp-iommu.h b/drivers/media/platform/apple/isp/isp-iommu.h new file mode 100644 index 00000000000000..b99a182e284b72 --- /dev/null +++ b/drivers/media/platform/apple/isp/isp-iommu.h @@ -0,0 +1,20 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright 2023 Eileen Yoon */ + +#ifndef __ISP_IOMMU_H__ +#define __ISP_IOMMU_H__ + +#include "isp-drv.h" + +struct isp_surf *__isp_alloc_surface(struct apple_isp *isp, u64 size, bool gc); +#define isp_alloc_surface(isp, size) (__isp_alloc_surface(isp, size, false)) +#define isp_alloc_surface_gc(isp, size) (__isp_alloc_surface(isp, size, true)) +struct isp_surf *isp_alloc_surface_vmap(struct apple_isp *isp, u64 size); +int isp_surf_vmap(struct apple_isp *isp, struct isp_surf *surf); +void isp_free_surface(struct apple_isp *isp, struct isp_surf *surf); + +int apple_isp_iommu_map_sgt(struct apple_isp *isp, struct isp_surf *surf, + struct sg_table *sgt, u64 size); +void apple_isp_iommu_unmap_sgt(struct apple_isp *isp, struct isp_surf *surf); + +#endif /* __ISP_IOMMU_H__ */ diff --git a/drivers/media/platform/apple/isp/isp-ipc.c b/drivers/media/platform/apple/isp/isp-ipc.c new file mode 100644 index 00000000000000..a1948717a31968 --- /dev/null +++ b/drivers/media/platform/apple/isp/isp-ipc.c @@ -0,0 +1,275 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright 2023 Eileen Yoon */ + +#include "isp-iommu.h" +#include "isp-ipc.h" +#include "isp-regs.h" +#include "isp-fw.h" + +#define ISP_IPC_FLAG_TERMINAL_ACK 0x3 +#define ISP_IPC_BUFEXC_STAT_META_OFFSET 0x10 + +struct isp_sm_deferred_work { + struct work_struct work; + struct apple_isp *isp; + struct isp_surf *surf; +}; + +struct isp_bufexc_stat { + u64 unk_0; // 2 + u64 unk_8; // 2 + + u64 meta_iova; + u64 pad_20[3]; + u64 meta_size; // 0x4640 + u64 unk_38; + + u32 unk_40; // 1 + u32 unk_44; + u64 unk_48; + + u64 iova0; + u64 iova1; + u64 iova2; + u64 iova3; + u32 pad_70[4]; + + u32 unk_80; // 2 + u32 unk_84; // 1 + u32 unk_88; // 0x10 || 0x13 + u32 unk_8c; + u32 pad_90[96]; + + u32 unk_210; // 0x28 + u32 unk_214; + u32 index; + u16 bes_width; // 1296, 0x510 + u16 bes_height; // 736, 0x2e0 + + u32 unk_220; // 0x0 || 0x1 + u32 pad_224[3]; + u32 unk_230; // 0xf7ed38 + u32 unk_234; // 3 + u32 pad_238[2]; + u32 pad_240[16]; +} __packed; +static_assert(sizeof(struct isp_bufexc_stat) == ISP_IPC_BUFEXC_STAT_SIZE); + +static inline void *chan_msg_virt(struct isp_channel *chan, u32 index) +{ + return chan->virt + (index * ISP_IPC_MESSAGE_SIZE); +} + +static inline void chan_read_msg_index(struct apple_isp *isp, + struct isp_channel *chan, + struct isp_message *msg, u32 index) +{ + memcpy(msg, chan_msg_virt(chan, index), sizeof(*msg)); +} + +static inline void chan_read_msg(struct apple_isp *isp, + struct isp_channel *chan, + struct isp_message *msg) +{ + chan_read_msg_index(isp, chan, msg, chan->cursor); +} + +static inline void chan_write_msg_index(struct apple_isp *isp, + struct isp_channel *chan, + struct isp_message *msg, u32 index) +{ + u64 *p0 = chan_msg_virt(chan, index); + memcpy(p0 + 1, &msg->arg1, sizeof(*msg) - 8); + + /* Make sure we write arg0 last, since that indicates message validity. */ + + dma_wmb(); + *p0 = msg->arg0; + dma_wmb(); +} + +static inline void chan_write_msg(struct apple_isp *isp, + struct isp_channel *chan, + struct isp_message *msg) +{ + chan_write_msg_index(isp, chan, msg, chan->cursor); +} + +static inline void chan_update_cursor(struct isp_channel *chan) +{ + if (chan->cursor >= (chan->num - 1)) { + chan->cursor = 0; + } else { + chan->cursor += 1; + } +} + +static int chan_handle_once(struct apple_isp *isp, struct isp_channel *chan) +{ + int err; + + lockdep_assert_held(&chan->lock); + + err = chan->ops->handle(isp, chan); + if (err < 0) { + dev_err(isp->dev, "%s: handler failed: %d)\n", chan->name, err); + return err; + } + + chan_write_msg(isp, chan, &chan->rsp); + + isp_mbox2_write32(isp, ISP_MBOX2_IRQ_DOORBELL, chan->doorbell); + + chan_update_cursor(chan); + + return 0; +} + +static inline bool chan_rx_done(struct apple_isp *isp, struct isp_channel *chan) +{ + if (((chan->req.arg0 & 0xf) == ISP_IPC_FLAG_ACK) || + ((chan->req.arg0 & 0xf) == ISP_IPC_FLAG_TERMINAL_ACK)) { + return true; + } + return false; +} + +int ipc_chan_handle(struct apple_isp *isp, struct isp_channel *chan) +{ + int err = 0; + + mutex_lock(&chan->lock); + while (1) { + chan_read_msg(isp, chan, &chan->req); + if (chan_rx_done(isp, chan)) { + err = 0; + break; + } + err = chan_handle_once(isp, chan); + if (err < 0) { + break; + } + } + mutex_unlock(&chan->lock); + + return err; +} + +static inline bool chan_tx_done(struct apple_isp *isp, struct isp_channel *chan) +{ + dma_rmb(); + + chan_read_msg(isp, chan, &chan->rsp); + if ((chan->rsp.arg0) == (chan->req.arg0 | ISP_IPC_FLAG_ACK)) { + chan_update_cursor(chan); + return true; + } + return false; +} + +int ipc_chan_send(struct apple_isp *isp, struct isp_channel *chan, + unsigned long timeout) +{ + long t; + + chan_write_msg(isp, chan, &chan->req); + dma_wmb(); + + isp_mbox2_write32(isp, ISP_MBOX2_IRQ_DOORBELL, chan->doorbell); + + if (!timeout) + return 0; + + t = wait_event_timeout(isp->wait, chan_tx_done(isp, chan), timeout); + if (t == 0) { + dev_err(isp->dev, + "%s: timed out on request [0x%llx, 0x%llx, 0x%llx]\n", + chan->name, chan->req.arg0, chan->req.arg1, + chan->req.arg2); + return -ETIME; + } + + isp_dbg(isp, "%s: request success (%ld)\n", chan->name, t); + + return 0; +} + +int ipc_tm_handle(struct apple_isp *isp, struct isp_channel *chan) +{ + struct isp_message *rsp = &chan->rsp; + +#ifdef APPLE_ISP_DEBUG + struct isp_message *req = &chan->req; + char buf[512]; + dma_addr_t iova = req->arg0 & ~ISP_IPC_FLAG_TERMINAL_ACK; + u32 size = req->arg1; + if (iova && size && size < sizeof(buf) && + isp->log_surf) { + void *p = apple_isp_translate(isp, isp->log_surf, iova, size); + if (p) { + size = min_t(u32, size, 512); + memcpy(buf, p, size); + isp_dbg(isp, "ISPASC: %.*s", size, buf); + } + } +#endif + + rsp->arg0 = ISP_IPC_FLAG_ACK; + rsp->arg1 = 0x0; + rsp->arg2 = 0x0; + + return 0; +} + +int ipc_sm_handle(struct apple_isp *isp, struct isp_channel *chan) +{ + struct isp_message *req = &chan->req, *rsp = &chan->rsp; + int err; + + if (req->arg0 == 0x0) { + struct isp_surf *surf; + + surf = isp_alloc_surface_gc(isp, req->arg1); + if (!surf) { + isp_err(isp, "failed to alloc requested size 0x%llx\n", + req->arg1); + return -ENOMEM; + } + surf->type = req->arg2; + + rsp->arg0 = surf->iova | ISP_IPC_FLAG_ACK; + rsp->arg1 = 0x0; + rsp->arg2 = 0x0; /* macOS uses this to index surfaces */ + + switch (surf->type) { + case 0x4c4f47: /* "LOG" */ + isp->log_surf = surf; + break; + case 0x4d495343: /* "MISC" */ + /* Hacky... maybe there's a better way to identify this surface? */ + if (surf->size == 0xc000) + isp->bt_surf = surf; + break; + default: + // skip vmap + return 0; + } + + err = isp_surf_vmap(isp, surf); + if (err < 0) { + isp_err(isp, "failed to vmap iova=0x%llx size=0x%llx\n", + surf->iova, surf->size); + } + } else { + /* This should be the shared surface free request, but + * 1) The fw doesn't request to free all of what it requested + * 2) The fw continues to access the surface after + * So we link it to the gc, which runs after fw shutdown + */ + rsp->arg0 = req->arg0 | ISP_IPC_FLAG_ACK; + rsp->arg1 = 0x0; + rsp->arg2 = 0x0; + } + + return 0; +} diff --git a/drivers/media/platform/apple/isp/isp-ipc.h b/drivers/media/platform/apple/isp/isp-ipc.h new file mode 100644 index 00000000000000..0c1d681835c72f --- /dev/null +++ b/drivers/media/platform/apple/isp/isp-ipc.h @@ -0,0 +1,25 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright 2023 Eileen Yoon */ + +#ifndef __ISP_IPC_H__ +#define __ISP_IPC_H__ + +#include "isp-drv.h" + +#define ISP_IPC_CHAN_TYPE_COMMAND 0 +#define ISP_IPC_CHAN_TYPE_REPLY 1 +#define ISP_IPC_CHAN_TYPE_REPORT 2 + +#define ISP_IPC_BUFEXC_STAT_SIZE 0x280 +#define ISP_IPC_BUFEXC_FLAG_RENDER 0x10000000 +#define ISP_IPC_BUFEXC_FLAG_COMMAND 0x30000000 +#define ISP_IPC_BUFEXC_FLAG_ACK 0x80000000 + +int ipc_chan_handle(struct apple_isp *isp, struct isp_channel *chan); +int ipc_chan_send(struct apple_isp *isp, struct isp_channel *chan, + unsigned long timeout); + +int ipc_tm_handle(struct apple_isp *isp, struct isp_channel *chan); +int ipc_sm_handle(struct apple_isp *isp, struct isp_channel *chan); + +#endif /* __ISP_IPC_H__ */ diff --git a/drivers/media/platform/apple/isp/isp-regs.h b/drivers/media/platform/apple/isp/isp-regs.h new file mode 100644 index 00000000000000..7357fa10fa5483 --- /dev/null +++ b/drivers/media/platform/apple/isp/isp-regs.h @@ -0,0 +1,56 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright 2023 Eileen Yoon */ + +#ifndef __ISP_REGS_H__ +#define __ISP_REGS_H__ + +#include "isp-drv.h" + +#define ISP_COPROC_FABRIC_0 0x738 +#define ISP_COPROC_FABRIC_1 0x798 +#define ISP_COPROC_FABRIC_2 0x7f8 +#define ISP_COPROC_FABRIC_3 0x858 + +#define ISP_COPROC_RVBAR 0x1050000 +#define ISP_COPROC_EDPRCR 0x1010310 +#define ISP_COPROC_CONTROL 0x1400044 +#define ISP_COPROC_STATUS 0x1400048 + +#define ISP_COPROC_IRQ_MASK_0 0x1400a00 +#define ISP_COPROC_IRQ_MASK_1 0x1400a04 +#define ISP_COPROC_IRQ_MASK_2 0x1400a08 +#define ISP_COPROC_IRQ_MASK_3 0x1400a0c +#define ISP_COPROC_IRQ_MASK_4 0x1400a10 +#define ISP_COPROC_IRQ_MASK_5 0x1400a14 + +#define ISP_MBOX_IRQ_INTERRUPT 0x00 +#define ISP_MBOX_IRQ_ENABLE 0x04 +#define ISP_MBOX2_IRQ_DOORBELL 0x00 +#define ISP_MBOX2_IRQ_ACK 0x0c + +#define ISP_GPIO_0 0x00 +#define ISP_GPIO_1 0x04 +#define ISP_GPIO_2 0x08 +#define ISP_GPIO_3 0x0c +#define ISP_GPIO_4 0x10 +#define ISP_GPIO_5 0x14 +#define ISP_GPIO_6 0x18 +#define ISP_GPIO_7 0x1c +#define ISP_GPIO_CLOCK_EN 0x20 + +static inline u32 isp_mbox_read32(struct apple_isp *isp, u32 reg) +{ + return readl(isp->mbox + reg); +} + +static inline void isp_mbox_write32(struct apple_isp *isp, u32 reg, u32 val) +{ + writel(val, isp->mbox + reg); +} + +static inline void isp_mbox2_write32(struct apple_isp *isp, u32 reg, u32 val) +{ + writel(val, isp->mbox2 + reg); +} + +#endif /* __ISP_REGS_H__ */ diff --git a/drivers/media/platform/apple/isp/isp-v4l2.c b/drivers/media/platform/apple/isp/isp-v4l2.c new file mode 100644 index 00000000000000..0561653ea7becd --- /dev/null +++ b/drivers/media/platform/apple/isp/isp-v4l2.c @@ -0,0 +1,910 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright 2023 Eileen Yoon */ + +#include + +#include +#include +#include +#include +#include + +#include "isp-cam.h" +#include "isp-cmd.h" +#include "isp-iommu.h" +#include "isp-ipc.h" +#include "isp-fw.h" +#include "isp-v4l2.h" + +#define ISP_MIN_FRAMES 2 +#define ISP_MAX_PLANES 4 +#define ISP_MAX_PIX_FORMATS 2 +#define ISP_BUFFER_TIMEOUT msecs_to_jiffies(1500) +#define ISP_STRIDE_ALIGNMENT 64 + +static bool multiplanar = false; +module_param(multiplanar, bool, 0644); +MODULE_PARM_DESC(multiplanar, "Enable multiplanar API"); + +struct isp_buflist_buffer { + u64 iovas[ISP_MAX_PLANES]; + u32 flags[ISP_MAX_PLANES]; + u32 num_planes; + u32 pool_type; + u32 tag; + u32 pad; +} __packed; +static_assert(sizeof(struct isp_buflist_buffer) == 0x40); + +struct isp_buflist { + u64 type; + u64 num_buffers; + struct isp_buflist_buffer buffers[]; +}; + +int ipc_bt_handle(struct apple_isp *isp, struct isp_channel *chan) +{ + struct isp_message *req = &chan->req, *rsp = &chan->rsp; + struct isp_buffer *tmp, *buf; + struct isp_buflist *bl; + u32 count; + int err = 0; + + /* printk("H2T: 0x%llx 0x%llx 0x%llx\n", (long long)req->arg0, + (long long)req->arg1, (long long)req->arg2); */ + + if (req->arg1 < sizeof(struct isp_buflist)) { + dev_err(isp->dev, "%s: Bad length 0x%llx\n", chan->name, + req->arg1); + return -EIO; + } + + bl = apple_isp_translate(isp, isp->bt_surf, req->arg0, req->arg1); + + count = bl->num_buffers; + if (count > (req->arg1 - sizeof(struct isp_buffer)) / + sizeof(struct isp_buflist_buffer)) { + dev_err(isp->dev, "%s: Bad length 0x%llx\n", chan->name, + req->arg1); + return -EIO; + } + + spin_lock(&isp->buf_lock); + for (int i = 0; i < count; i++) { + struct isp_buflist_buffer *bufd = &bl->buffers[i]; + + /* printk("Return: 0x%llx (%d)\n", bufd->iovas[0], + bufd->pool_type); */ + + if (bufd->pool_type == 0) { + for (int j = 0; j < ARRAY_SIZE(isp->meta_surfs); j++) { + struct isp_surf *meta = isp->meta_surfs[j]; + if ((u32)bufd->iovas[0] == (u32)meta->iova) { + WARN_ON(!meta->submitted); + meta->submitted = false; + } + } + } else { + list_for_each_entry_safe_reverse( + buf, tmp, &isp->bufs_submitted, link) { + if ((u32)buf->surfs[0].iova == + (u32)bufd->iovas[0]) { + enum vb2_buffer_state state = + VB2_BUF_STATE_ERROR; + + buf->vb.vb2_buf.timestamp = + ktime_get_ns(); + buf->vb.sequence = isp->sequence++; + buf->vb.field = V4L2_FIELD_NONE; + if (req->arg2 == + ISP_IPC_BUFEXC_FLAG_RENDER) + state = VB2_BUF_STATE_DONE; + vb2_buffer_done(&buf->vb.vb2_buf, + state); + list_del(&buf->link); + } + } + } + } + spin_unlock(&isp->buf_lock); + + rsp->arg0 = req->arg0 | ISP_IPC_FLAG_ACK; + rsp->arg1 = 0x0; + rsp->arg2 = ISP_IPC_BUFEXC_FLAG_ACK; + + return err; +} + +static int isp_submit_buffers(struct apple_isp *isp) +{ + struct isp_format *fmt = isp_get_current_format(isp); + struct isp_channel *chan = isp->chan_bh; + struct isp_message *req = &chan->req; + struct isp_buffer *buf, *tmp; + unsigned long flags; + size_t offset; + int err; + + struct isp_buflist *bl = isp->cmd_virt; + struct isp_buflist_buffer *bufd = &bl->buffers[0]; + + bl->type = 1; + bl->num_buffers = 0; + + spin_lock_irqsave(&isp->buf_lock, flags); + for (int i = 0; i < ARRAY_SIZE(isp->meta_surfs); i++) { + struct isp_surf *meta = isp->meta_surfs[i]; + + if (meta->submitted) + continue; + + /* printk("Submit: 0x%llx .. 0x%llx (meta)\n", meta->iova, + meta->iova + meta->size); */ + + bufd->num_planes = 1; + bufd->pool_type = 0; + bufd->iovas[0] = meta->iova; + bufd->flags[0] = 0x40000000; + bufd++; + bl->num_buffers++; + + meta->submitted = true; + } + + while ((buf = list_first_entry_or_null(&isp->bufs_pending, + struct isp_buffer, link))) { + memset(bufd, 0, sizeof(*bufd)); + + bufd->num_planes = fmt->num_planes; + bufd->pool_type = isp->hw->scl1 ? CISP_POOL_TYPE_RENDERED_SCL1 : + CISP_POOL_TYPE_RENDERED; + offset = 0; + for (int j = 0; j < fmt->num_planes; j++) { + bufd->iovas[j] = buf->surfs[0].iova + offset; + bufd->flags[j] = 0x40000000; + offset += fmt->plane_size[j]; + } + + /* printk("Submit: 0x%llx .. 0x%llx (render)\n", + buf->surfs[0].iova, + buf->surfs[0].iova + buf->surfs[0].size); */ + bufd++; + bl->num_buffers++; + + /* + * Queue the buffer as submitted and release the lock for now. + * We need to do this before actually submitting to avoid a + * race with the buffer return codepath. + */ + list_move_tail(&buf->link, &isp->bufs_submitted); + } + + spin_unlock_irqrestore(&isp->buf_lock, flags); + + req->arg0 = isp->cmd_iova; + req->arg1 = max_t(u64, ISP_IPC_BUFEXC_STAT_SIZE, + ((uintptr_t)bufd - (uintptr_t)bl)); + req->arg2 = ISP_IPC_BUFEXC_FLAG_COMMAND; + + err = ipc_chan_send(isp, chan, ISP_BUFFER_TIMEOUT); + if (err) { + /* If we fail, consider the buffer not submitted. */ + dev_err(isp->dev, + "%s: failed to send bufs: [0x%llx, 0x%llx, 0x%llx]\n", + chan->name, req->arg0, req->arg1, req->arg2); + + /* + * Try to find the buffer in the list, and if it's + * still there, move it back to the pending list. + */ + spin_lock_irqsave(&isp->buf_lock, flags); + + bufd = &bl->buffers[0]; + for (int i = 0; i < bl->num_buffers; i++, bufd++) { + list_for_each_entry_safe_reverse( + buf, tmp, &isp->bufs_submitted, link) { + if (bufd->iovas[0] == buf->surfs[0].iova) { + list_move_tail(&buf->link, + &isp->bufs_pending); + } + } + for (int j = 0; j < ARRAY_SIZE(isp->meta_surfs); j++) { + struct isp_surf *meta = isp->meta_surfs[j]; + if (bufd->iovas[0] == meta->iova) { + meta->submitted = false; + } + } + } + + spin_unlock_irqrestore(&isp->buf_lock, flags); + } + + return err; +} + +/* + * Videobuf2 section + */ +static int isp_vb2_queue_setup(struct vb2_queue *vq, unsigned int *nbuffers, + unsigned int *num_planes, unsigned int sizes[], + struct device *alloc_devs[]) +{ + struct apple_isp *isp = vb2_get_drv_priv(vq); + struct isp_format *fmt = isp_get_current_format(isp); + + /* This is not strictly neccessary but makes it easy to enforce that + * at most 16 buffers are submitted at once. ISP on t6001 (FW 12.3) + * times out if more buffers are submitted than set in the buffer pool + * config before streaming is started. + */ + *nbuffers = min_t(unsigned int, *nbuffers, ISP_MAX_BUFFERS); + + if (*num_planes) { + if (sizes[0] < fmt->total_size) + return -EINVAL; + + return 0; + } + + *num_planes = 1; + sizes[0] = fmt->total_size; + + return 0; +} + +static void __isp_vb2_buf_cleanup(struct vb2_buffer *vb, unsigned int i) +{ + struct apple_isp *isp = vb2_get_drv_priv(vb->vb2_queue); + struct isp_buffer *buf = + container_of(vb, struct isp_buffer, vb.vb2_buf); + + while (i--) + apple_isp_iommu_unmap_sgt(isp, &buf->surfs[i]); +} + +static void isp_vb2_buf_cleanup(struct vb2_buffer *vb) +{ + __isp_vb2_buf_cleanup(vb, vb->num_planes); +} + +static int isp_vb2_buf_init(struct vb2_buffer *vb) +{ + struct apple_isp *isp = vb2_get_drv_priv(vb->vb2_queue); + struct isp_buffer *buf = + container_of(vb, struct isp_buffer, vb.vb2_buf); + unsigned int i; + int err; + + for (i = 0; i < vb->num_planes; i++) { + struct sg_table *sgt = vb2_dma_sg_plane_desc(vb, i); + err = apple_isp_iommu_map_sgt(isp, &buf->surfs[i], sgt, + vb2_plane_size(vb, i)); + if (err) + goto cleanup; + } + + return 0; + +cleanup: + __isp_vb2_buf_cleanup(vb, i); + return err; +} + +static int isp_vb2_buf_prepare(struct vb2_buffer *vb) +{ + struct apple_isp *isp = vb2_get_drv_priv(vb->vb2_queue); + struct isp_format *fmt = isp_get_current_format(isp); + + if (vb2_plane_size(vb, 0) < fmt->total_size) + return -EINVAL; + + vb2_set_plane_payload(vb, 0, fmt->total_size); + + return 0; +} + +static void isp_vb2_release_buffers(struct apple_isp *isp, + enum vb2_buffer_state state) +{ + struct isp_buffer *buf; + unsigned long flags; + + spin_lock_irqsave(&isp->buf_lock, flags); + list_for_each_entry(buf, &isp->bufs_submitted, link) + vb2_buffer_done(&buf->vb.vb2_buf, state); + INIT_LIST_HEAD(&isp->bufs_submitted); + list_for_each_entry(buf, &isp->bufs_pending, link) + vb2_buffer_done(&buf->vb.vb2_buf, state); + INIT_LIST_HEAD(&isp->bufs_pending); + spin_unlock_irqrestore(&isp->buf_lock, flags); +} + +static void isp_vb2_buf_queue(struct vb2_buffer *vb) +{ + struct apple_isp *isp = vb2_get_drv_priv(vb->vb2_queue); + struct isp_buffer *buf = + container_of(vb, struct isp_buffer, vb.vb2_buf); + unsigned long flags; + bool empty; + + spin_lock_irqsave(&isp->buf_lock, flags); + empty = list_empty(&isp->bufs_pending) && + list_empty(&isp->bufs_submitted); + list_add_tail(&buf->link, &isp->bufs_pending); + spin_unlock_irqrestore(&isp->buf_lock, flags); + + if (test_bit(ISP_STATE_STREAMING, &isp->state) && !empty) + isp_submit_buffers(isp); +} + +static int apple_isp_start_streaming(struct apple_isp *isp) +{ + int err; + + err = apple_isp_start_camera(isp); + if (err) { + dev_err(isp->dev, "failed to start camera: %d\n", err); + goto release_buffers; + } + + err = isp_submit_buffers(isp); + if (err) { + dev_err(isp->dev, "failed to send initial batch: %d\n", err); + goto stop_camera; + } + + err = apple_isp_start_capture(isp); + if (err) { + dev_err(isp->dev, "failed to start capture: %d\n", err); + goto stop_camera; + } + + set_bit(ISP_STATE_STREAMING, &isp->state); + + return 0; + +stop_camera: + apple_isp_stop_camera(isp); +release_buffers: + isp_vb2_release_buffers(isp, VB2_BUF_STATE_QUEUED); + return err; +} + +static void apple_isp_stop_streaming(struct apple_isp *isp) +{ + clear_bit(ISP_STATE_STREAMING, &isp->state); + apple_isp_stop_capture(isp); + apple_isp_stop_camera(isp); +} + +static int isp_vb2_start_streaming(struct vb2_queue *q, unsigned int count) +{ + struct apple_isp *isp = vb2_get_drv_priv(q); + + isp->sequence = 0; + + return apple_isp_start_streaming(isp); +} + +static void isp_vb2_stop_streaming(struct vb2_queue *q) +{ + struct apple_isp *isp = vb2_get_drv_priv(q); + + apple_isp_stop_streaming(isp); + isp_vb2_release_buffers(isp, VB2_BUF_STATE_ERROR); +} + +int apple_isp_video_suspend(struct apple_isp *isp) +{ + /* Swap into STATE_SLEEPING as isp_vb2_buf_queue() submits on + * STATE_STREAMING. + */ + if (test_bit(ISP_STATE_STREAMING, &isp->state)) { + /* Signal buffers to be recycled for clean shutdown */ + isp_vb2_release_buffers(isp, VB2_BUF_STATE_QUEUED); + apple_isp_stop_streaming(isp); + set_bit(ISP_STATE_SLEEPING, &isp->state); + } + + return 0; +} + +int apple_isp_video_resume(struct apple_isp *isp) +{ + if (test_bit(ISP_STATE_SLEEPING, &isp->state)) { + clear_bit(ISP_STATE_SLEEPING, &isp->state); + apple_isp_start_streaming(isp); + } + + return 0; +} + +static const struct vb2_ops isp_vb2_ops = { + .queue_setup = isp_vb2_queue_setup, + .buf_init = isp_vb2_buf_init, + .buf_cleanup = isp_vb2_buf_cleanup, + .buf_prepare = isp_vb2_buf_prepare, + .buf_queue = isp_vb2_buf_queue, + .start_streaming = isp_vb2_start_streaming, + .stop_streaming = isp_vb2_stop_streaming, + .wait_prepare = vb2_ops_wait_prepare, + .wait_finish = vb2_ops_wait_finish, +}; + +static int isp_set_preset(struct apple_isp *isp, struct isp_format *fmt, + struct isp_preset *preset) +{ + int i; + size_t total_size; + + fmt->preset = preset; + + /* I really fucking hope they all use NV12. */ + fmt->num_planes = 2; + fmt->strides[0] = ALIGN(preset->output_dim.x, ISP_STRIDE_ALIGNMENT); + /* UV subsampled interleaved */ + fmt->strides[1] = ALIGN(preset->output_dim.x, ISP_STRIDE_ALIGNMENT); + fmt->plane_size[0] = fmt->strides[0] * preset->output_dim.y; + fmt->plane_size[1] = fmt->strides[1] * preset->output_dim.y / 2; + + total_size = 0; + for (i = 0; i < fmt->num_planes; i++) + total_size += fmt->plane_size[i]; + fmt->total_size = total_size; + + return 0; +} + +static struct isp_preset *isp_select_preset(struct apple_isp *isp, u32 width, + u32 height) +{ + struct isp_preset *preset, *best = &isp->presets[0]; + int i, score, best_score = INT_MAX; + + /* Default if no dimensions */ + if (width == 0 || height == 0) + return &isp->presets[0]; + + for (i = 0; i < isp->num_presets; i++) { + preset = &isp->presets[i]; + score = abs((int)preset->output_dim.x - (int)width) + + abs((int)preset->output_dim.y - (int)height); + if (score < best_score) { + best = preset; + best_score = score; + } + } + + return best; +} + +/* + * V4L2 ioctl section + */ +static int isp_vidioc_querycap(struct file *file, void *priv, + struct v4l2_capability *cap) +{ + strscpy(cap->card, APPLE_ISP_CARD_NAME, sizeof(cap->card)); + strscpy(cap->driver, APPLE_ISP_DEVICE_NAME, sizeof(cap->driver)); + + return 0; +} + +static int isp_vidioc_enum_format(struct file *file, void *fh, + struct v4l2_fmtdesc *f) +{ + struct apple_isp *isp = video_drvdata(file); + + if (f->index >= ISP_MAX_PIX_FORMATS) + return -EINVAL; + + switch (f->index) { + case 0: + f->pixelformat = V4L2_PIX_FMT_NV12; + break; + case 1: + if (!isp->multiplanar) + return -EINVAL; + f->pixelformat = V4L2_PIX_FMT_NV12M; + break; + default: + return -EINVAL; + } + + return 0; +} + +static int isp_vidioc_enum_framesizes(struct file *file, void *fh, + struct v4l2_frmsizeenum *f) +{ + struct apple_isp *isp = video_drvdata(file); + + if (f->index >= isp->num_presets) + return -EINVAL; + + if ((f->pixel_format != V4L2_PIX_FMT_NV12) && + (f->pixel_format != V4L2_PIX_FMT_NV12M)) + return -EINVAL; + + f->discrete.width = isp->presets[f->index].output_dim.x; + f->discrete.height = isp->presets[f->index].output_dim.y; + f->type = V4L2_FRMSIZE_TYPE_DISCRETE; + + return 0; +} + +static int isp_vidioc_enum_frameintervals(struct file *filp, void *priv, + struct v4l2_frmivalenum *interval) +{ + if (interval->index != 0) + return -EINVAL; + + interval->type = V4L2_FRMIVAL_TYPE_DISCRETE; + interval->discrete.numerator = 1; + interval->discrete.denominator = 30; + return 0; +} + +static inline void isp_get_sp_pix_format(struct apple_isp *isp, + struct v4l2_format *f, + struct isp_format *fmt) +{ + f->fmt.pix.width = fmt->preset->output_dim.x; + f->fmt.pix.height = fmt->preset->output_dim.y; + f->fmt.pix.bytesperline = fmt->strides[0]; + f->fmt.pix.sizeimage = fmt->total_size; + + f->fmt.pix.field = V4L2_FIELD_NONE; + f->fmt.pix.pixelformat = V4L2_PIX_FMT_NV12; + f->fmt.pix.colorspace = V4L2_COLORSPACE_REC709; + f->fmt.pix.ycbcr_enc = V4L2_YCBCR_ENC_709; + f->fmt.pix.xfer_func = V4L2_XFER_FUNC_709; +} + +static inline void isp_get_mp_pix_format(struct apple_isp *isp, + struct v4l2_format *f, + struct isp_format *fmt) +{ + f->fmt.pix_mp.width = fmt->preset->output_dim.x; + f->fmt.pix_mp.height = fmt->preset->output_dim.y; + f->fmt.pix_mp.num_planes = fmt->num_planes; + for (int i = 0; i < fmt->num_planes; i++) { + f->fmt.pix_mp.plane_fmt[i].sizeimage = fmt->plane_size[i]; + f->fmt.pix_mp.plane_fmt[i].bytesperline = fmt->strides[i]; + } + + f->fmt.pix_mp.field = V4L2_FIELD_NONE; + f->fmt.pix_mp.pixelformat = V4L2_PIX_FMT_NV12M; + f->fmt.pix_mp.colorspace = V4L2_COLORSPACE_REC709; + f->fmt.pix_mp.ycbcr_enc = V4L2_YCBCR_ENC_709; + f->fmt.pix_mp.xfer_func = V4L2_XFER_FUNC_709; +} + +static int isp_vidioc_get_format(struct file *file, void *fh, + struct v4l2_format *f) +{ + struct apple_isp *isp = video_drvdata(file); + struct isp_format *fmt = isp_get_current_format(isp); + + isp_get_sp_pix_format(isp, f, fmt); + + return 0; +} + +static int isp_vidioc_set_format(struct file *file, void *fh, + struct v4l2_format *f) +{ + struct apple_isp *isp = video_drvdata(file); + struct isp_format *fmt = isp_get_current_format(isp); + struct isp_preset *preset; + int err; + + preset = isp_select_preset(isp, f->fmt.pix.width, f->fmt.pix.height); + err = isp_set_preset(isp, fmt, preset); + if (err) + return err; + + isp_get_sp_pix_format(isp, f, fmt); + + isp->vbq.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; + + return 0; +} + +static int isp_vidioc_try_format(struct file *file, void *fh, + struct v4l2_format *f) +{ + struct apple_isp *isp = video_drvdata(file); + struct isp_format fmt = *isp_get_current_format(isp); + struct isp_preset *preset; + int err; + + preset = isp_select_preset(isp, f->fmt.pix.width, f->fmt.pix.height); + err = isp_set_preset(isp, &fmt, preset); + if (err) + return err; + + isp_get_sp_pix_format(isp, f, &fmt); + + return 0; +} + +static int isp_vidioc_get_format_mplane(struct file *file, void *fh, + struct v4l2_format *f) +{ + struct apple_isp *isp = video_drvdata(file); + struct isp_format *fmt = isp_get_current_format(isp); + + if (!isp->multiplanar) + return -ENOTTY; + + isp_get_mp_pix_format(isp, f, fmt); + + return 0; +} + +static int isp_vidioc_set_format_mplane(struct file *file, void *fh, + struct v4l2_format *f) +{ + struct apple_isp *isp = video_drvdata(file); + struct isp_format *fmt = isp_get_current_format(isp); + struct isp_preset *preset; + int err; + + if (!isp->multiplanar) + return -ENOTTY; + + preset = isp_select_preset(isp, f->fmt.pix_mp.width, + f->fmt.pix_mp.height); + err = isp_set_preset(isp, fmt, preset); + if (err) + return err; + + isp_get_mp_pix_format(isp, f, fmt); + + isp->vbq.type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE; + + return 0; +} + +static int isp_vidioc_try_format_mplane(struct file *file, void *fh, + struct v4l2_format *f) +{ + struct apple_isp *isp = video_drvdata(file); + struct isp_format fmt = *isp_get_current_format(isp); + struct isp_preset *preset; + int err; + + if (!isp->multiplanar) + return -ENOTTY; + + preset = isp_select_preset(isp, f->fmt.pix_mp.width, + f->fmt.pix_mp.height); + err = isp_set_preset(isp, &fmt, preset); + if (err) + return err; + + isp_get_mp_pix_format(isp, f, &fmt); + + return 0; +} + +static int isp_vidioc_enum_input(struct file *file, void *fh, + struct v4l2_input *inp) +{ + if (inp->index) + return -EINVAL; + + strscpy(inp->name, APPLE_ISP_DEVICE_NAME, sizeof(inp->name)); + inp->type = V4L2_INPUT_TYPE_CAMERA; + + return 0; +} + +static int isp_vidioc_get_input(struct file *file, void *fh, unsigned int *i) +{ + *i = 0; + + return 0; +} + +static int isp_vidioc_set_input(struct file *file, void *fh, unsigned int i) +{ + if (i) + return -EINVAL; + + return 0; +} + +static int isp_vidioc_get_param(struct file *file, void *fh, + struct v4l2_streamparm *a) +{ + struct apple_isp *isp = video_drvdata(file); + + if (a->type != V4L2_BUF_TYPE_VIDEO_CAPTURE && + (!isp->multiplanar || + a->type != V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE)) + return -EINVAL; + + a->parm.capture.capability = V4L2_CAP_TIMEPERFRAME; + a->parm.capture.readbuffers = ISP_MIN_FRAMES; + a->parm.capture.timeperframe.numerator = ISP_FRAME_RATE_NUM; + a->parm.capture.timeperframe.denominator = ISP_FRAME_RATE_DEN; + + return 0; +} + +static int isp_vidioc_set_param(struct file *file, void *fh, + struct v4l2_streamparm *a) +{ + struct apple_isp *isp = video_drvdata(file); + + if (a->type != V4L2_BUF_TYPE_VIDEO_CAPTURE && + (!isp->multiplanar || + a->type != V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE)) + return -EINVAL; + + /* Not supporting frame rate sets. No use. Plus floats. */ + a->parm.capture.capability = V4L2_CAP_TIMEPERFRAME; + a->parm.capture.readbuffers = ISP_MIN_FRAMES; + a->parm.capture.timeperframe.numerator = ISP_FRAME_RATE_NUM; + a->parm.capture.timeperframe.denominator = ISP_FRAME_RATE_DEN; + + return 0; +} + +static const struct v4l2_ioctl_ops isp_v4l2_ioctl_ops = { + .vidioc_querycap = isp_vidioc_querycap, + + .vidioc_enum_fmt_vid_cap = isp_vidioc_enum_format, + .vidioc_g_fmt_vid_cap = isp_vidioc_get_format, + .vidioc_s_fmt_vid_cap = isp_vidioc_set_format, + .vidioc_try_fmt_vid_cap = isp_vidioc_try_format, + .vidioc_g_fmt_vid_cap_mplane = isp_vidioc_get_format_mplane, + .vidioc_s_fmt_vid_cap_mplane = isp_vidioc_set_format_mplane, + .vidioc_try_fmt_vid_cap_mplane = isp_vidioc_try_format_mplane, + + .vidioc_enum_framesizes = isp_vidioc_enum_framesizes, + .vidioc_enum_frameintervals = isp_vidioc_enum_frameintervals, + .vidioc_enum_input = isp_vidioc_enum_input, + .vidioc_g_input = isp_vidioc_get_input, + .vidioc_s_input = isp_vidioc_set_input, + .vidioc_g_parm = isp_vidioc_get_param, + .vidioc_s_parm = isp_vidioc_set_param, + + .vidioc_reqbufs = vb2_ioctl_reqbufs, + .vidioc_querybuf = vb2_ioctl_querybuf, + .vidioc_create_bufs = vb2_ioctl_create_bufs, + .vidioc_qbuf = vb2_ioctl_qbuf, + .vidioc_expbuf = vb2_ioctl_expbuf, + .vidioc_dqbuf = vb2_ioctl_dqbuf, + .vidioc_prepare_buf = vb2_ioctl_prepare_buf, + .vidioc_streamon = vb2_ioctl_streamon, + .vidioc_streamoff = vb2_ioctl_streamoff, +}; + +static const struct v4l2_file_operations isp_v4l2_fops = { + .owner = THIS_MODULE, + .open = v4l2_fh_open, + .release = vb2_fop_release, + .read = vb2_fop_read, + .poll = vb2_fop_poll, + .mmap = vb2_fop_mmap, + .unlocked_ioctl = video_ioctl2, +}; + +static const struct media_device_ops isp_media_device_ops = { + .link_notify = v4l2_pipeline_link_notify, +}; + +int apple_isp_setup_video(struct apple_isp *isp) +{ + struct video_device *vdev = &isp->vdev; + struct vb2_queue *vbq = &isp->vbq; + struct isp_format *fmt = isp_get_current_format(isp); + int err; + + err = isp_set_preset(isp, fmt, &isp->presets[0]); + if (err) { + dev_err(isp->dev, "failed to set default preset: %d\n", err); + return err; + } + + for (int i = 0; i < ARRAY_SIZE(isp->meta_surfs); i++) { + isp->meta_surfs[i] = + isp_alloc_surface_vmap(isp, isp->hw->meta_size); + if (!isp->meta_surfs[i]) { + isp_err(isp, "failed to alloc meta surface\n"); + err = -ENOMEM; + goto surf_cleanup; + } + } + + media_device_init(&isp->mdev); + isp->v4l2_dev.mdev = &isp->mdev; + isp->mdev.ops = &isp_media_device_ops; + isp->mdev.dev = isp->dev; + strscpy(isp->mdev.model, APPLE_ISP_DEVICE_NAME, + sizeof(isp->mdev.model)); + + err = media_device_register(&isp->mdev); + if (err) { + dev_err(isp->dev, "failed to register media device: %d\n", err); + goto media_cleanup; + } + + isp->multiplanar = multiplanar; + + err = v4l2_device_register(isp->dev, &isp->v4l2_dev); + if (err) { + dev_err(isp->dev, "failed to register v4l2 device: %d\n", err); + goto media_unregister; + } + + vbq->drv_priv = isp; + vbq->type = V4L2_BUF_TYPE_VIDEO_CAPTURE; + vbq->io_modes = VB2_MMAP; + vbq->dev = isp->dev; + vbq->ops = &isp_vb2_ops; + vbq->mem_ops = &vb2_dma_sg_memops; + vbq->buf_struct_size = sizeof(struct isp_buffer); + vbq->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_MONOTONIC; + vbq->min_queued_buffers = ISP_MIN_FRAMES; + vbq->lock = &isp->video_lock; + + err = vb2_queue_init(vbq); + if (err) { + dev_err(isp->dev, "failed to init vb2 queue: %d\n", err); + goto v4l2_unregister; + } + + vdev->queue = vbq; + vdev->fops = &isp_v4l2_fops; + vdev->ioctl_ops = &isp_v4l2_ioctl_ops; + vdev->device_caps = V4L2_BUF_TYPE_VIDEO_CAPTURE | V4L2_CAP_STREAMING; + if (isp->multiplanar) + vdev->device_caps |= V4L2_CAP_VIDEO_CAPTURE_MPLANE; + vdev->v4l2_dev = &isp->v4l2_dev; + vdev->vfl_type = VFL_TYPE_VIDEO; + vdev->vfl_dir = VFL_DIR_RX; + vdev->release = video_device_release_empty; + vdev->lock = &isp->video_lock; + strscpy(vdev->name, APPLE_ISP_DEVICE_NAME, sizeof(vdev->name)); + video_set_drvdata(vdev, isp); + + err = video_register_device(vdev, VFL_TYPE_VIDEO, 0); + if (err) { + dev_err(isp->dev, "failed to register video device: %d\n", err); + goto v4l2_unregister; + } + + return 0; + +v4l2_unregister: + v4l2_device_unregister(&isp->v4l2_dev); +media_unregister: + media_device_unregister(&isp->mdev); +media_cleanup: + media_device_cleanup(&isp->mdev); +surf_cleanup: + for (int i = 0; i < ARRAY_SIZE(isp->meta_surfs); i++) { + if (isp->meta_surfs[i]) + isp_free_surface(isp, isp->meta_surfs[i]); + isp->meta_surfs[i] = NULL; + } + + return err; +} + +void apple_isp_remove_video(struct apple_isp *isp) +{ + vb2_video_unregister_device(&isp->vdev); + v4l2_device_unregister(&isp->v4l2_dev); + media_device_unregister(&isp->mdev); + media_device_cleanup(&isp->mdev); + for (int i = 0; i < ARRAY_SIZE(isp->meta_surfs); i++) { + if (isp->meta_surfs[i]) + isp_free_surface(isp, isp->meta_surfs[i]); + isp->meta_surfs[i] = NULL; + } +} diff --git a/drivers/media/platform/apple/isp/isp-v4l2.h b/drivers/media/platform/apple/isp/isp-v4l2.h new file mode 100644 index 00000000000000..4d47deeb83b055 --- /dev/null +++ b/drivers/media/platform/apple/isp/isp-v4l2.h @@ -0,0 +1,16 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright 2023 Eileen Yoon */ + +#ifndef __ISP_V4L2_H__ +#define __ISP_V4L2_H__ + +#include "isp-drv.h" + +int apple_isp_setup_video(struct apple_isp *isp); +void apple_isp_remove_video(struct apple_isp *isp); +int ipc_bt_handle(struct apple_isp *isp, struct isp_channel *chan); + +int apple_isp_video_suspend(struct apple_isp *isp); +int apple_isp_video_resume(struct apple_isp *isp); + +#endif /* __ISP_V4L2_H__ */ diff --git a/drivers/media/platform/chips-media/wave5/wave5-helper.c b/drivers/media/platform/chips-media/wave5/wave5-helper.c index f03ad9c0de2215..53a0ac068c2e25 100644 --- a/drivers/media/platform/chips-media/wave5/wave5-helper.c +++ b/drivers/media/platform/chips-media/wave5/wave5-helper.c @@ -27,6 +27,11 @@ const char *state_to_str(enum vpu_instance_state state) } } +int wave5_kfifo_alloc(struct vpu_instance *inst) +{ + return kfifo_alloc(&inst->irq_status, 16 * sizeof(int), GFP_KERNEL); +} + void wave5_cleanup_instance(struct vpu_instance *inst, struct file *filp) { int i; @@ -49,7 +54,7 @@ void wave5_cleanup_instance(struct vpu_instance *inst, struct file *filp) v4l2_fh_del(&inst->v4l2_fh, filp); v4l2_fh_exit(&inst->v4l2_fh); } - list_del_init(&inst->list); + kfifo_free(&inst->irq_status); ida_free(&inst->dev->inst_ida, inst->id); kfree(inst->codec_info); kfree(inst); @@ -61,8 +66,29 @@ int wave5_vpu_release_device(struct file *filp, { struct vpu_instance *inst = file_to_vpu_inst(filp); int ret = 0; + unsigned long flags; v4l2_m2m_ctx_release(inst->v4l2_fh.m2m_ctx); + /* + * To prevent Null reference exception, the existing irq handler were + * separated to two modules. + * One is to queue interrupt reason into the irq handler, + * the other is irq_thread to call the wave5_vpu_dec_finish_decode + * to get decoded frame. + * The list of instances should be protected between all flow of the + * decoding process, but to protect the list in the irq_handler, spin lock + * should be used, and mutex should be used in the irq_thread because spin lock + * is not able to be used because mutex is already being used + * in the wave5_vpu_dec_finish_decode. + * So the spin lock and mutex were used to protect the list in the release function. + */ + ret = mutex_lock_interruptible(&inst->dev->irq_lock); + if (ret) + return ret; + spin_lock_irqsave(&inst->dev->irq_spinlock, flags); + list_del_init(&inst->list); + spin_unlock_irqrestore(&inst->dev->irq_spinlock, flags); + mutex_unlock(&inst->dev->irq_lock); if (inst->state != VPU_INST_STATE_NONE) { u32 fail_res; diff --git a/drivers/media/platform/chips-media/wave5/wave5-helper.h b/drivers/media/platform/chips-media/wave5/wave5-helper.h index 976a402e426ff3..d61fdbda359d6f 100644 --- a/drivers/media/platform/chips-media/wave5/wave5-helper.h +++ b/drivers/media/platform/chips-media/wave5/wave5-helper.h @@ -33,4 +33,5 @@ void wave5_update_pix_fmt(struct v4l2_pix_format_mplane *pix_mp, unsigned int width, unsigned int height, const struct v4l2_frmsize_stepwise *frmsize); +int wave5_kfifo_alloc(struct vpu_instance *inst); #endif diff --git a/drivers/media/platform/chips-media/wave5/wave5-vpu-dec.c b/drivers/media/platform/chips-media/wave5/wave5-vpu-dec.c index e3038c18ca3621..cff2fa17c3f598 100644 --- a/drivers/media/platform/chips-media/wave5/wave5-vpu-dec.c +++ b/drivers/media/platform/chips-media/wave5/wave5-vpu-dec.c @@ -1243,6 +1243,7 @@ static void wave5_vpu_dec_buf_queue_dst(struct vb2_buffer *vb) struct vpu_instance *inst = vb2_get_drv_priv(vb->vb2_queue); struct v4l2_m2m_ctx *m2m_ctx = inst->v4l2_fh.m2m_ctx; + pm_runtime_resume_and_get(inst->dev->dev); vbuf->sequence = inst->queued_dst_buf_num++; if (inst->state == VPU_INST_STATE_PIC_RUN) { @@ -1275,6 +1276,7 @@ static void wave5_vpu_dec_buf_queue_dst(struct vb2_buffer *vb) } else { v4l2_m2m_buf_queue(m2m_ctx, vbuf); } + pm_runtime_put_autosuspend(inst->dev->dev); } static void wave5_vpu_dec_buf_queue(struct vb2_buffer *vb) @@ -1753,8 +1755,10 @@ static int wave5_vpu_open_dec(struct file *filp) spin_lock_init(&inst->state_spinlock); inst->codec_info = kzalloc(sizeof(*inst->codec_info), GFP_KERNEL); - if (!inst->codec_info) + if (!inst->codec_info) { + kfree(inst); return -ENOMEM; + } v4l2_fh_init(&inst->v4l2_fh, vdev); v4l2_fh_add(&inst->v4l2_fh, filp); @@ -1806,6 +1810,11 @@ static int wave5_vpu_open_dec(struct file *filp) inst->xfer_func = V4L2_XFER_FUNC_DEFAULT; init_completion(&inst->irq_done); + ret = wave5_kfifo_alloc(inst); + if (ret) { + dev_err(inst->dev->dev, "failed to allocate fifo\n"); + goto cleanup_inst; + } inst->id = ida_alloc(&inst->dev->inst_ida, GFP_KERNEL); if (inst->id < 0) { @@ -1825,9 +1834,6 @@ static int wave5_vpu_open_dec(struct file *filp) if (ret) goto cleanup_inst; - if (list_empty(&dev->instances)) - pm_runtime_use_autosuspend(inst->dev->dev); - list_add_tail(&inst->list, &dev->instances); mutex_unlock(&dev->dev_lock); diff --git a/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c b/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c index 9bfaa9fb3ceb3e..24fc0d0d3f4aa7 100644 --- a/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c +++ b/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c @@ -649,6 +649,8 @@ static int wave5_vpu_enc_encoder_cmd(struct file *file, void *fh, struct v4l2_en m2m_ctx->last_src_buf = v4l2_m2m_last_src_buf(m2m_ctx); m2m_ctx->is_draining = true; + + v4l2_m2m_try_schedule(m2m_ctx); break; case V4L2_ENC_CMD_START: break; @@ -1367,7 +1369,8 @@ static int wave5_vpu_enc_start_streaming(struct vb2_queue *q, unsigned int count if (ret) goto return_buffers; } - if (inst->state == VPU_INST_STATE_OPEN && m2m_ctx->cap_q_ctx.q.streaming) { + if (inst->state == VPU_INST_STATE_OPEN && + (m2m_ctx->cap_q_ctx.q.streaming || q->type == V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE)) { ret = initialize_sequence(inst); if (ret) { dev_warn(inst->dev->dev, "Sequence not found: %d\n", ret); @@ -1578,8 +1581,10 @@ static int wave5_vpu_open_enc(struct file *filp) inst->ops = &wave5_vpu_enc_inst_ops; inst->codec_info = kzalloc(sizeof(*inst->codec_info), GFP_KERNEL); - if (!inst->codec_info) + if (!inst->codec_info) { + kfree(inst); return -ENOMEM; + } v4l2_fh_init(&inst->v4l2_fh, vdev); v4l2_fh_add(&inst->v4l2_fh, filp); @@ -1754,6 +1759,11 @@ static int wave5_vpu_open_enc(struct file *filp) inst->frame_rate = 30; init_completion(&inst->irq_done); + ret = wave5_kfifo_alloc(inst); + if (ret) { + dev_err(inst->dev->dev, "failed to allocate fifo\n"); + goto cleanup_inst; + } inst->id = ida_alloc(&inst->dev->inst_ida, GFP_KERNEL); if (inst->id < 0) { @@ -1768,9 +1778,6 @@ static int wave5_vpu_open_enc(struct file *filp) if (ret) goto cleanup_inst; - if (list_empty(&dev->instances)) - pm_runtime_use_autosuspend(inst->dev->dev); - list_add_tail(&inst->list, &dev->instances); mutex_unlock(&dev->dev_lock); diff --git a/drivers/media/platform/chips-media/wave5/wave5-vpu.c b/drivers/media/platform/chips-media/wave5/wave5-vpu.c index e1715d3f43b0d8..3216b499764473 100644 --- a/drivers/media/platform/chips-media/wave5/wave5-vpu.c +++ b/drivers/media/platform/chips-media/wave5/wave5-vpu.c @@ -51,8 +51,11 @@ static void wave5_vpu_handle_irq(void *dev_id) u32 seq_done; u32 cmd_done; u32 irq_reason; - struct vpu_instance *inst; + u32 irq_subreason; + struct vpu_instance *inst, *tmp; struct vpu_device *dev = dev_id; + int val; + unsigned long flags; irq_reason = wave5_vdi_read_register(dev, W5_VPU_VINT_REASON); seq_done = wave5_vdi_read_register(dev, W5_RET_SEQ_DONE_INSTANCE_INFO); @@ -60,7 +63,8 @@ static void wave5_vpu_handle_irq(void *dev_id) wave5_vdi_write_register(dev, W5_VPU_VINT_REASON_CLR, irq_reason); wave5_vdi_write_register(dev, W5_VPU_VINT_CLEAR, 0x1); - list_for_each_entry(inst, &dev->instances, list) { + spin_lock_irqsave(&dev->irq_spinlock, flags); + list_for_each_entry_safe(inst, tmp, &dev->instances, list) { if (irq_reason & BIT(INT_WAVE5_INIT_SEQ) || irq_reason & BIT(INT_WAVE5_ENC_SET_PARAM)) { @@ -82,22 +86,54 @@ static void wave5_vpu_handle_irq(void *dev_id) irq_reason & BIT(INT_WAVE5_ENC_PIC)) { if (cmd_done & BIT(inst->id)) { cmd_done &= ~BIT(inst->id); - wave5_vdi_write_register(dev, W5_RET_QUEUE_CMD_DONE_INST, - cmd_done); - inst->ops->finish_process(inst); + if (dev->irq >= 0) { + irq_subreason = + wave5_vdi_read_register(dev, W5_VPU_VINT_REASON); + if (!(irq_subreason & BIT(INT_WAVE5_DEC_PIC))) + wave5_vdi_write_register(dev, + W5_RET_QUEUE_CMD_DONE_INST, + cmd_done); + } + val = BIT(INT_WAVE5_DEC_PIC); + kfifo_in(&inst->irq_status, &val, sizeof(int)); } } + } + spin_unlock_irqrestore(&dev->irq_spinlock, flags); + + if (dev->irq < 0) + up(&dev->irq_sem); +} - wave5_vpu_clear_interrupt(inst, irq_reason); +static irqreturn_t wave5_vpu_irq(int irq, void *dev_id) +{ + struct vpu_device *dev = dev_id; + + if (wave5_vdi_read_register(dev, W5_VPU_VPU_INT_STS)) { + wave5_vpu_handle_irq(dev); + return IRQ_WAKE_THREAD; } + + return IRQ_HANDLED; } static irqreturn_t wave5_vpu_irq_thread(int irq, void *dev_id) { struct vpu_device *dev = dev_id; + struct vpu_instance *inst, *tmp; + int irq_status, ret; - if (wave5_vdi_read_register(dev, W5_VPU_VPU_INT_STS)) - wave5_vpu_handle_irq(dev); + mutex_lock(&dev->irq_lock); + list_for_each_entry_safe(inst, tmp, &dev->instances, list) { + while (kfifo_len(&inst->irq_status)) { + ret = kfifo_out(&inst->irq_status, &irq_status, sizeof(int)); + if (!ret) + break; + + inst->ops->finish_process(inst); + } + } + mutex_unlock(&dev->irq_lock); return IRQ_HANDLED; } @@ -121,6 +157,35 @@ static enum hrtimer_restart wave5_vpu_timer_callback(struct hrtimer *timer) return HRTIMER_RESTART; } +static int irq_thread(void *data) +{ + struct vpu_device *dev = (struct vpu_device *)data; + struct vpu_instance *inst, *tmp; + int irq_status, ret; + + while (!kthread_should_stop()) { + if (down_interruptible(&dev->irq_sem)) + continue; + + if (kthread_should_stop()) + break; + + mutex_lock(&dev->irq_lock); + list_for_each_entry_safe(inst, tmp, &dev->instances, list) { + while (kfifo_len(&inst->irq_status)) { + ret = kfifo_out(&inst->irq_status, &irq_status, sizeof(int)); + if (!ret) + break; + + inst->ops->finish_process(inst); + } + } + mutex_unlock(&dev->irq_lock); + } + + return 0; +} + static int wave5_vpu_load_firmware(struct device *dev, const char *fw_name, u32 *revision) { @@ -224,6 +289,8 @@ static int wave5_vpu_probe(struct platform_device *pdev) mutex_init(&dev->dev_lock); mutex_init(&dev->hw_lock); + mutex_init(&dev->irq_lock); + spin_lock_init(&dev->irq_spinlock); dev_set_drvdata(&pdev->dev, dev); dev->dev = &pdev->dev; @@ -266,9 +333,13 @@ static int wave5_vpu_probe(struct platform_device *pdev) } dev->product = wave5_vpu_get_product_id(dev); + INIT_LIST_HEAD(&dev->instances); + dev->irq = platform_get_irq(pdev, 0); if (dev->irq < 0) { dev_err(&pdev->dev, "failed to get irq resource, falling back to polling\n"); + sema_init(&dev->irq_sem, 1); + dev->irq_thread = kthread_run(irq_thread, dev, "irq thread"); hrtimer_setup(&dev->hrtimer, &wave5_vpu_timer_callback, CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED); dev->worker = kthread_run_worker(0, "vpu_irq_thread"); @@ -280,7 +351,7 @@ static int wave5_vpu_probe(struct platform_device *pdev) dev->vpu_poll_interval = vpu_poll_interval; kthread_init_work(&dev->work, wave5_vpu_irq_work_fn); } else { - ret = devm_request_threaded_irq(&pdev->dev, dev->irq, NULL, + ret = devm_request_threaded_irq(&pdev->dev, dev->irq, wave5_vpu_irq, wave5_vpu_irq_thread, IRQF_ONESHOT, "vpu_irq", dev); if (ret) { dev_err(&pdev->dev, "Register interrupt handler, fail: %d\n", ret); @@ -288,7 +359,6 @@ static int wave5_vpu_probe(struct platform_device *pdev) } } - INIT_LIST_HEAD(&dev->instances); ret = v4l2_device_register(&pdev->dev, &dev->v4l2_dev); if (ret) { dev_err(&pdev->dev, "v4l2_device_register, fail: %d\n", ret); @@ -322,7 +392,7 @@ static int wave5_vpu_probe(struct platform_device *pdev) dev_info(&pdev->dev, "Product Code: 0x%x\n", dev->product_code); dev_info(&pdev->dev, "Firmware Revision: %u\n", fw_revision); - pm_runtime_set_autosuspend_delay(&pdev->dev, 100); + pm_runtime_set_autosuspend_delay(&pdev->dev, 500); pm_runtime_use_autosuspend(&pdev->dev); pm_runtime_enable(&pdev->dev); wave5_vpu_sleep_wake(&pdev->dev, true, NULL, 0); @@ -351,21 +421,30 @@ static void wave5_vpu_remove(struct platform_device *pdev) { struct vpu_device *dev = dev_get_drvdata(&pdev->dev); + wave5_vpu_enc_unregister_device(dev); + wave5_vpu_dec_unregister_device(dev); + v4l2_device_unregister(&dev->v4l2_dev); + if (dev->irq < 0) { - kthread_destroy_worker(dev->worker); + if (dev->irq_thread) { + kthread_stop(dev->irq_thread); + up(&dev->irq_sem); + dev->irq_thread = NULL; + } + hrtimer_cancel(&dev->hrtimer); + kthread_cancel_work_sync(&dev->work); + kthread_destroy_worker(dev->worker); } - pm_runtime_put_sync(&pdev->dev); + pm_runtime_dont_use_autosuspend(&pdev->dev); pm_runtime_disable(&pdev->dev); mutex_destroy(&dev->dev_lock); mutex_destroy(&dev->hw_lock); + mutex_destroy(&dev->irq_lock); reset_control_assert(dev->resets); clk_bulk_disable_unprepare(dev->num_clks, dev->clks); - wave5_vpu_enc_unregister_device(dev); - wave5_vpu_dec_unregister_device(dev); - v4l2_device_unregister(&dev->v4l2_dev); wave5_vdi_release(&pdev->dev); ida_destroy(&dev->inst_ida); } diff --git a/drivers/media/platform/chips-media/wave5/wave5-vpuapi.c b/drivers/media/platform/chips-media/wave5/wave5-vpuapi.c index e5e879a13e8b89..e94d6ebc9f8162 100644 --- a/drivers/media/platform/chips-media/wave5/wave5-vpuapi.c +++ b/drivers/media/platform/chips-media/wave5/wave5-vpuapi.c @@ -207,8 +207,6 @@ int wave5_vpu_dec_close(struct vpu_instance *inst, u32 *fail_res) int retry = 0; struct vpu_device *vpu_dev = inst->dev; int i; - int inst_count = 0; - struct vpu_instance *inst_elm; *fail_res = 0; if (!inst->codec_info) @@ -250,11 +248,6 @@ int wave5_vpu_dec_close(struct vpu_instance *inst, u32 *fail_res) wave5_vdi_free_dma_memory(vpu_dev, &p_dec_info->vb_task); - list_for_each_entry(inst_elm, &vpu_dev->instances, list) - inst_count++; - if (inst_count == 1) - pm_runtime_dont_use_autosuspend(vpu_dev->dev); - unlock_and_return: mutex_unlock(&vpu_dev->hw_lock); pm_runtime_put_sync(inst->dev->dev); @@ -720,8 +713,6 @@ int wave5_vpu_enc_close(struct vpu_instance *inst, u32 *fail_res) int ret; int retry = 0; struct vpu_device *vpu_dev = inst->dev; - int inst_count = 0; - struct vpu_instance *inst_elm; *fail_res = 0; if (!inst->codec_info) @@ -764,12 +755,6 @@ int wave5_vpu_enc_close(struct vpu_instance *inst, u32 *fail_res) } wave5_vdi_free_dma_memory(vpu_dev, &p_enc_info->vb_task); - - list_for_each_entry(inst_elm, &vpu_dev->instances, list) - inst_count++; - if (inst_count == 1) - pm_runtime_dont_use_autosuspend(vpu_dev->dev); - mutex_unlock(&vpu_dev->hw_lock); pm_runtime_put_sync(inst->dev->dev); diff --git a/drivers/media/platform/chips-media/wave5/wave5-vpuapi.h b/drivers/media/platform/chips-media/wave5/wave5-vpuapi.h index 45615c15beca32..bc101397204dab 100644 --- a/drivers/media/platform/chips-media/wave5/wave5-vpuapi.h +++ b/drivers/media/platform/chips-media/wave5/wave5-vpuapi.h @@ -8,6 +8,7 @@ #ifndef VPUAPI_H_INCLUDED #define VPUAPI_H_INCLUDED +#include #include #include #include @@ -747,6 +748,7 @@ struct vpu_device { struct video_device *video_dev_enc; struct mutex dev_lock; /* lock for the src, dst v4l2 queues */ struct mutex hw_lock; /* lock hw configurations */ + struct mutex irq_lock; int irq; enum product_id product; struct vpu_attr attr; @@ -764,7 +766,10 @@ struct vpu_device { struct kthread_worker *worker; int vpu_poll_interval; int num_clks; + struct task_struct *irq_thread; + struct semaphore irq_sem; /* signal to irq_thread when interrupt happens*/ struct reset_control *resets; + spinlock_t irq_spinlock; /* protect instances list */ }; struct vpu_instance; @@ -788,6 +793,7 @@ struct vpu_instance { enum v4l2_ycbcr_encoding ycbcr_enc; enum v4l2_quantization quantization; + struct kfifo irq_status; enum vpu_instance_state state; enum vpu_instance_type type; const struct vpu_instance_ops *ops; diff --git a/drivers/media/platform/mediatek/mdp/mtk_mdp_core.c b/drivers/media/platform/mediatek/mdp/mtk_mdp_core.c index 80fdc6ff57e0ec..8432833814f319 100644 --- a/drivers/media/platform/mediatek/mdp/mtk_mdp_core.c +++ b/drivers/media/platform/mediatek/mdp/mtk_mdp_core.c @@ -194,11 +194,17 @@ static int mtk_mdp_probe(struct platform_device *pdev) } mdp->vpu_dev = vpu_get_plat_device(pdev); + if (!mdp->vpu_dev) { + dev_err(&pdev->dev, "Failed to get vpu device\n"); + ret = -ENODEV; + goto err_vpu_get_dev; + } + ret = vpu_wdt_reg_handler(mdp->vpu_dev, mtk_mdp_reset_handler, mdp, VPU_RST_MDP); if (ret) { dev_err(&pdev->dev, "Failed to register reset handler\n"); - goto err_m2m_register; + goto err_reg_handler; } platform_set_drvdata(pdev, mdp); @@ -206,7 +212,7 @@ static int mtk_mdp_probe(struct platform_device *pdev) ret = vb2_dma_contig_set_max_seg_size(&pdev->dev, DMA_BIT_MASK(32)); if (ret) { dev_err(&pdev->dev, "Failed to set vb2 dma mag seg size\n"); - goto err_m2m_register; + goto err_reg_handler; } pm_runtime_enable(dev); @@ -214,6 +220,12 @@ static int mtk_mdp_probe(struct platform_device *pdev) return 0; +err_reg_handler: + platform_device_put(mdp->vpu_dev); + +err_vpu_get_dev: + mtk_mdp_unregister_m2m_device(mdp); + err_m2m_register: v4l2_device_unregister(&mdp->v4l2_dev); @@ -242,6 +254,7 @@ static void mtk_mdp_remove(struct platform_device *pdev) pm_runtime_disable(&pdev->dev); vb2_dma_contig_clear_max_seg_size(&pdev->dev); + platform_device_put(mdp->vpu_dev); mtk_mdp_unregister_m2m_device(mdp); v4l2_device_unregister(&mdp->v4l2_dev); diff --git a/drivers/media/platform/mediatek/vcodec/decoder/mtk_vcodec_dec_stateless.c b/drivers/media/platform/mediatek/vcodec/decoder/mtk_vcodec_dec_stateless.c index d873159b9b3069..9eef3ff2b12785 100644 --- a/drivers/media/platform/mediatek/vcodec/decoder/mtk_vcodec_dec_stateless.c +++ b/drivers/media/platform/mediatek/vcodec/decoder/mtk_vcodec_dec_stateless.c @@ -502,6 +502,12 @@ static int mtk_vdec_s_ctrl(struct v4l2_ctrl *ctrl) mtk_v4l2_vdec_err(ctx, "VP9: bit_depth:%d", frame->bit_depth); return -EINVAL; } + + if (!(frame->flags & V4L2_VP9_FRAME_FLAG_X_SUBSAMPLING) || + !(frame->flags & V4L2_VP9_FRAME_FLAG_Y_SUBSAMPLING)) { + mtk_v4l2_vdec_err(ctx, "VP9: only 420 subsampling is supported"); + return -EINVAL; + } break; case V4L2_CID_STATELESS_AV1_SEQUENCE: seq = (struct v4l2_ctrl_av1_sequence *)hdr_ctrl->p_new.p; diff --git a/drivers/media/platform/mediatek/vcodec/encoder/mtk_vcodec_enc.c b/drivers/media/platform/mediatek/vcodec/encoder/mtk_vcodec_enc.c index 6faf3f659e7510..b3a0a1d8b7a8e9 100644 --- a/drivers/media/platform/mediatek/vcodec/encoder/mtk_vcodec_enc.c +++ b/drivers/media/platform/mediatek/vcodec/encoder/mtk_vcodec_enc.c @@ -850,7 +850,7 @@ static void vb2ops_venc_buf_queue(struct vb2_buffer *vb) static int vb2ops_venc_start_streaming(struct vb2_queue *q, unsigned int count) { struct mtk_vcodec_enc_ctx *ctx = vb2_get_drv_priv(q); - struct venc_enc_param param; + struct venc_enc_param param = { }; int ret; int i; @@ -1004,7 +1004,7 @@ static int mtk_venc_encode_header(void *priv) int ret; struct vb2_v4l2_buffer *src_buf, *dst_buf; struct mtk_vcodec_mem bs_buf; - struct venc_done_result enc_result; + struct venc_done_result enc_result = { }; dst_buf = v4l2_m2m_dst_buf_remove(ctx->m2m_ctx); if (!dst_buf) { @@ -1125,7 +1125,7 @@ static void mtk_venc_worker(struct work_struct *work) struct vb2_v4l2_buffer *src_buf, *dst_buf; struct venc_frm_buf frm_buf; struct mtk_vcodec_mem bs_buf; - struct venc_done_result enc_result; + struct venc_done_result enc_result = { }; int ret, i; /* check dst_buf, dst_buf may be removed in device_run diff --git a/drivers/media/platform/mediatek/vcodec/encoder/mtk_vcodec_enc_drv.c b/drivers/media/platform/mediatek/vcodec/encoder/mtk_vcodec_enc_drv.c index 82b8ff38e8f1a0..4ac667a8de4cd9 100644 --- a/drivers/media/platform/mediatek/vcodec/encoder/mtk_vcodec_enc_drv.c +++ b/drivers/media/platform/mediatek/vcodec/encoder/mtk_vcodec_enc_drv.c @@ -215,6 +215,15 @@ static int fops_vcodec_release(struct file *file) v4l2_fh_exit(&ctx->fh); v4l2_ctrl_handler_free(&ctx->ctrl_hdl); + /* + * Cancel any pending encode work before freeing the context. + * Although v4l2_m2m_ctx_release() waits for m2m job completion, + * the workqueue handler (mtk_venc_worker) may still be accessing + * the context after v4l2_m2m_job_finish() returns. Without this, + * a use-after-free occurs when the worker accesses ctx after kfree. + */ + cancel_work_sync(&ctx->encode_work); + spin_lock_irqsave(&dev->dev_ctx_lock, flags); list_del_init(&ctx->list); spin_unlock_irqrestore(&dev->dev_ctx_lock, flags); diff --git a/drivers/media/platform/qcom/camss/camss-vfe-480.c b/drivers/media/platform/qcom/camss/camss-vfe-480.c index 4feea590a47bc3..d73f733fde0459 100644 --- a/drivers/media/platform/qcom/camss/camss-vfe-480.c +++ b/drivers/media/platform/qcom/camss/camss-vfe-480.c @@ -202,11 +202,13 @@ static irqreturn_t vfe_isr(int irq, void *dev) writel_relaxed(status, vfe->base + VFE_BUS_IRQ_CLEAR(0)); writel_relaxed(1, vfe->base + VFE_BUS_IRQ_CLEAR_GLOBAL); - /* Loop through all WMs IRQs */ - for (i = 0; i < MSM_VFE_IMAGE_MASTERS_NUM; i++) { + for (i = 0; i < MAX_VFE_OUTPUT_LINES; i++) { if (status & BUS_IRQ_MASK_0_RDI_RUP(vfe, i)) vfe_isr_reg_update(vfe, i); + } + /* Loop through all WMs IRQs */ + for (i = 0; i < MSM_VFE_IMAGE_MASTERS_NUM; i++) { if (status & BUS_IRQ_MASK_0_COMP_DONE(vfe, RDI_COMP_GROUP(i))) vfe_buf_done(vfe, i); } diff --git a/drivers/media/platform/qcom/camss/camss.c b/drivers/media/platform/qcom/camss/camss.c index fcc2b2c3cba076..757c548af485a5 100644 --- a/drivers/media/platform/qcom/camss/camss.c +++ b/drivers/media/platform/qcom/camss/camss.c @@ -2704,12 +2704,11 @@ static const struct camss_subdev_resources vfe_res_8550[] = { /* VFE3 lite */ { .regulators = {}, - .clock = { "gcc_axi_hf", "cpas_ahb", "cpas_fast_ahb_clk", "vfe_lite_ahb", + .clock = { "gcc_axi_hf", "cpas_ahb", "vfe_lite_ahb", "vfe_lite", "cpas_ife_lite", "camnoc_axi" }, .clock_rate = { { 0 }, { 80000000 }, { 300000000, 400000000 }, - { 300000000, 400000000 }, { 400000000, 480000000 }, { 300000000, 400000000 }, { 300000000, 400000000 } }, @@ -2726,12 +2725,11 @@ static const struct camss_subdev_resources vfe_res_8550[] = { /* VFE4 lite */ { .regulators = {}, - .clock = { "gcc_axi_hf", "cpas_ahb", "cpas_fast_ahb_clk", "vfe_lite_ahb", + .clock = { "gcc_axi_hf", "cpas_ahb", "vfe_lite_ahb", "vfe_lite", "cpas_ife_lite", "camnoc_axi" }, .clock_rate = { { 0 }, { 80000000 }, { 300000000, 400000000 }, - { 300000000, 400000000 }, { 400000000, 480000000 }, { 300000000, 400000000 }, { 300000000, 400000000 } }, diff --git a/drivers/media/platform/qcom/iris/iris_buffer.c b/drivers/media/platform/qcom/iris/iris_buffer.c index b89b1ee06cce15..f1f003a787bf22 100644 --- a/drivers/media/platform/qcom/iris/iris_buffer.c +++ b/drivers/media/platform/qcom/iris/iris_buffer.c @@ -351,12 +351,15 @@ static int iris_create_internal_buffer(struct iris_inst *inst, buffer->index = index; buffer->buffer_size = buffers->size; buffer->dma_attrs = DMA_ATTR_WRITE_COMBINE | DMA_ATTR_NO_KERNEL_MAPPING; - list_add_tail(&buffer->list, &buffers->list); buffer->kvaddr = dma_alloc_attrs(core->dev, buffer->buffer_size, &buffer->device_addr, GFP_KERNEL, buffer->dma_attrs); - if (!buffer->kvaddr) + if (!buffer->kvaddr) { + kfree(buffer); return -ENOMEM; + } + + list_add_tail(&buffer->list, &buffers->list); return 0; } diff --git a/drivers/media/platform/qcom/iris/iris_hfi_gen1_command.c b/drivers/media/platform/qcom/iris/iris_hfi_gen1_command.c index 52da7ef7bab08f..11815f6f5bacd4 100644 --- a/drivers/media/platform/qcom/iris/iris_hfi_gen1_command.c +++ b/drivers/media/platform/qcom/iris/iris_hfi_gen1_command.c @@ -282,7 +282,7 @@ static int iris_hfi_gen1_queue_input_buffer(struct iris_inst *inst, struct iris_ com_ip_pkt.shdr.session_id = inst->session_id; com_ip_pkt.time_stamp_hi = upper_32_bits(buf->timestamp); com_ip_pkt.time_stamp_lo = lower_32_bits(buf->timestamp); - com_ip_pkt.flags = buf->flags; + com_ip_pkt.flags = 0; com_ip_pkt.mark_target = 0; com_ip_pkt.mark_data = 0; com_ip_pkt.offset = buf->data_offset; @@ -441,6 +441,8 @@ static int iris_hfi_gen1_session_unset_buffers(struct iris_inst *inst, struct ir goto exit; ret = iris_wait_for_session_response(inst, false); + if (!ret) + ret = iris_destroy_internal_buffer(inst, buf); exit: kfree(pkt); @@ -733,7 +735,7 @@ static int iris_hfi_gen1_set_resolution(struct iris_inst *inst, u32 plane) struct hfi_framesize fs; int ret; - if (!iris_drc_pending(inst)) { + if (!iris_drc_pending(inst) && !(inst->sub_state & IRIS_INST_SUB_FIRST_IPSC)) { fs.buffer_type = HFI_BUFFER_INPUT; fs.width = inst->fmt_src->fmt.pix_mp.width; fs.height = inst->fmt_src->fmt.pix_mp.height; diff --git a/drivers/media/platform/qcom/iris/iris_hfi_gen2_command.c b/drivers/media/platform/qcom/iris/iris_hfi_gen2_command.c index f9129553209922..31e8fc7f8295d6 100644 --- a/drivers/media/platform/qcom/iris/iris_hfi_gen2_command.c +++ b/drivers/media/platform/qcom/iris/iris_hfi_gen2_command.c @@ -963,6 +963,9 @@ static int iris_hfi_gen2_session_stop(struct iris_inst *inst, u32 plane) struct iris_inst_hfi_gen2 *inst_hfi_gen2 = to_iris_inst_hfi_gen2(inst); int ret = 0; + if (!inst_hfi_gen2->packet) + return -EINVAL; + reinit_completion(&inst->completion); iris_hfi_gen2_packet_session_command(inst, diff --git a/drivers/media/platform/qcom/iris/iris_platform_gen2.c b/drivers/media/platform/qcom/iris/iris_platform_gen2.c index c1989240c24860..00d1d554631794 100644 --- a/drivers/media/platform/qcom/iris/iris_platform_gen2.c +++ b/drivers/media/platform/qcom/iris/iris_platform_gen2.c @@ -915,6 +915,7 @@ const struct iris_platform_data sm8750_data = { .get_instance = iris_hfi_gen2_get_instance, .init_hfi_command_ops = iris_hfi_gen2_command_ops_init, .init_hfi_response_ops = iris_hfi_gen2_response_ops_init, + .get_vpu_buffer_size = iris_vpu33_buf_size, .vpu_ops = &iris_vpu35_ops, .set_preset_registers = iris_set_sm8550_preset_registers, .icc_tbl = sm8550_icc_table, @@ -945,6 +946,7 @@ const struct iris_platform_data sm8750_data = { .num_vpp_pipe = 4, .max_session_count = 16, .max_core_mbpf = NUM_MBS_8K * 2, + .max_core_mbps = ((7680 * 4320) / 256) * 60, .dec_input_config_params_default = sm8550_vdec_input_config_params_default, .dec_input_config_params_default_size = diff --git a/drivers/media/platform/qcom/iris/iris_vb2.c b/drivers/media/platform/qcom/iris/iris_vb2.c index db8768d8a8f61c..bf0b8400996ece 100644 --- a/drivers/media/platform/qcom/iris/iris_vb2.c +++ b/drivers/media/platform/qcom/iris/iris_vb2.c @@ -193,10 +193,14 @@ int iris_vb2_start_streaming(struct vb2_queue *q, unsigned int count) buf_type = iris_v4l2_type_to_driver(q->type); if (inst->domain == DECODER) { - if (inst->state == IRIS_INST_STREAMING) + if (buf_type == BUF_INPUT) + ret = iris_queue_deferred_buffers(inst, BUF_INPUT); + + if (!ret && inst->state == IRIS_INST_STREAMING) { ret = iris_queue_internal_deferred_buffers(inst, BUF_DPB); - if (!ret) - ret = iris_queue_deferred_buffers(inst, buf_type); + if (!ret) + ret = iris_queue_deferred_buffers(inst, BUF_OUTPUT); + } } else { if (inst->state == IRIS_INST_STREAMING) { ret = iris_queue_deferred_buffers(inst, BUF_INPUT); @@ -231,8 +235,6 @@ void iris_vb2_stop_streaming(struct vb2_queue *q) return; mutex_lock(&inst->lock); - if (inst->state == IRIS_INST_ERROR) - goto exit; if (!V4L2_TYPE_IS_OUTPUT(q->type) && !V4L2_TYPE_IS_CAPTURE(q->type)) @@ -243,10 +245,10 @@ void iris_vb2_stop_streaming(struct vb2_queue *q) goto exit; exit: - if (ret) { - iris_helper_buffers_done(inst, q->type, VB2_BUF_STATE_ERROR); + iris_helper_buffers_done(inst, q->type, VB2_BUF_STATE_ERROR); + if (ret) iris_inst_change_state(inst, IRIS_INST_ERROR); - } + mutex_unlock(&inst->lock); } diff --git a/drivers/media/platform/qcom/iris/iris_vdec.c b/drivers/media/platform/qcom/iris/iris_vdec.c index 69ffe52590d3ac..227e4e5a326fdc 100644 --- a/drivers/media/platform/qcom/iris/iris_vdec.c +++ b/drivers/media/platform/qcom/iris/iris_vdec.c @@ -231,6 +231,14 @@ int iris_vdec_s_fmt(struct iris_inst *inst, struct v4l2_format *f) if (vb2_is_busy(q)) return -EBUSY; + /* Width and height are optional, so fall back to a valid placeholder + * resolution until the real one is decoded from the bitstream. + */ + if (f->fmt.pix_mp.width == 0 && f->fmt.pix_mp.height == 0) { + f->fmt.pix_mp.width = DEFAULT_WIDTH; + f->fmt.pix_mp.height = DEFAULT_HEIGHT; + } + iris_vdec_try_fmt(inst, f); switch (f->type) { diff --git a/drivers/media/platform/qcom/iris/iris_venc.c b/drivers/media/platform/qcom/iris/iris_venc.c index 5830eba93c68b2..0ed5018f9fe332 100644 --- a/drivers/media/platform/qcom/iris/iris_venc.c +++ b/drivers/media/platform/qcom/iris/iris_venc.c @@ -382,8 +382,7 @@ int iris_venc_s_param(struct iris_inst *inst, struct v4l2_streamparm *s_parm) struct v4l2_fract *timeperframe = NULL; u32 default_rate = DEFAULT_FPS; bool is_frame_rate = false; - u64 us_per_frame, fps; - u32 max_rate; + u32 fps, max_rate; int ret = 0; @@ -405,23 +404,19 @@ int iris_venc_s_param(struct iris_inst *inst, struct v4l2_streamparm *s_parm) timeperframe->denominator = default_rate; } - us_per_frame = timeperframe->numerator * (u64)USEC_PER_SEC; - do_div(us_per_frame, timeperframe->denominator); - - if (!us_per_frame) + fps = timeperframe->denominator / timeperframe->numerator; + if (!fps) return -EINVAL; - fps = (u64)USEC_PER_SEC; - do_div(fps, us_per_frame); if (fps > max_rate) { ret = -ENOMEM; goto reset_rate; } if (is_frame_rate) - inst->frame_rate = (u32)fps; + inst->frame_rate = fps; else - inst->operating_rate = (u32)fps; + inst->operating_rate = fps; if ((s_parm->type == V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE && vb2_is_streaming(src_q)) || (s_parm->type == V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE && vb2_is_streaming(dst_q))) { diff --git a/drivers/media/platform/qcom/iris/iris_vidc.c b/drivers/media/platform/qcom/iris/iris_vidc.c index c9b881923ef183..0c9c23ef2d1800 100644 --- a/drivers/media/platform/qcom/iris/iris_vidc.c +++ b/drivers/media/platform/qcom/iris/iris_vidc.c @@ -572,9 +572,10 @@ static int iris_dec_cmd(struct file *filp, void *fh, mutex_lock(&inst->lock); - ret = v4l2_m2m_ioctl_decoder_cmd(filp, fh, dec); - if (ret) + if (dec->cmd != V4L2_DEC_CMD_STOP && dec->cmd != V4L2_DEC_CMD_START) { + ret = -EINVAL; goto unlock; + } if (inst->state == IRIS_INST_DEINIT) goto unlock; @@ -605,9 +606,10 @@ static int iris_enc_cmd(struct file *filp, void *fh, mutex_lock(&inst->lock); - ret = v4l2_m2m_ioctl_encoder_cmd(filp, fh, enc); - if (ret) + if (enc->cmd != V4L2_ENC_CMD_STOP && enc->cmd != V4L2_ENC_CMD_START) { + ret = -EINVAL; goto unlock; + } if (inst->state == IRIS_INST_DEINIT) goto unlock; diff --git a/drivers/media/platform/qcom/venus/vdec.c b/drivers/media/platform/qcom/venus/vdec.c index 4a6641fdffcf79..4cd69440e87532 100644 --- a/drivers/media/platform/qcom/venus/vdec.c +++ b/drivers/media/platform/qcom/venus/vdec.c @@ -565,7 +565,13 @@ vdec_decoder_cmd(struct file *file, void *fh, struct v4l2_decoder_cmd *cmd) fdata.buffer_type = HFI_BUFFER_INPUT; fdata.flags |= HFI_BUFFERFLAG_EOS; - if (IS_V6(inst->core) && is_fw_rev_or_older(inst->core, 1, 0, 87)) + + /* Send NULL EOS addr for only IRIS2 (SM8250),for firmware <= 1.0.87. + * SC7280 also reports "1.0." parsed as 1.0.0; restricting to IRIS2 + * avoids misapplying this quirk and breaking VP9 decode on SC7280. + */ + + if (IS_IRIS2(inst->core) && is_fw_rev_or_older(inst->core, 1, 0, 87)) fdata.device_addr = 0; else fdata.device_addr = 0xdeadb000; @@ -1440,10 +1446,10 @@ static void vdec_buf_done(struct venus_inst *inst, unsigned int buf_type, inst->drain_active = false; inst->codec_state = VENUS_DEC_STATE_STOPPED; } + } else { + if (!bytesused) + state = VB2_BUF_STATE_ERROR; } - - if (!bytesused) - state = VB2_BUF_STATE_ERROR; } else { vbuf->sequence = inst->sequence_out++; } diff --git a/drivers/media/platform/rockchip/rga/rga-buf.c b/drivers/media/platform/rockchip/rga/rga-buf.c index 730bdf98565a55..bb575873f2b249 100644 --- a/drivers/media/platform/rockchip/rga/rga-buf.c +++ b/drivers/media/platform/rockchip/rga/rga-buf.c @@ -80,6 +80,9 @@ static int rga_buf_init(struct vb2_buffer *vb) struct rga_frame *f = rga_get_frame(ctx, vb->vb2_queue->type); size_t n_desc = 0; + if (IS_ERR(f)) + return PTR_ERR(f); + n_desc = DIV_ROUND_UP(f->size, PAGE_SIZE); rbuf->n_desc = n_desc; diff --git a/drivers/media/platform/rockchip/rkisp1/rkisp1-params.c b/drivers/media/platform/rockchip/rkisp1/rkisp1-params.c index c9f88635224cc6..6442436a5e428f 100644 --- a/drivers/media/platform/rockchip/rkisp1/rkisp1-params.c +++ b/drivers/media/platform/rockchip/rkisp1/rkisp1-params.c @@ -411,12 +411,6 @@ static void rkisp1_flt_config(struct rkisp1_params *params, rkisp1_write(params->rkisp1, RKISP1_CIF_ISP_FILT_LUM_WEIGHT, arg->lum_weight); - rkisp1_write(params->rkisp1, RKISP1_CIF_ISP_FILT_MODE, - (arg->mode ? RKISP1_CIF_ISP_FLT_MODE_DNR : 0) | - RKISP1_CIF_ISP_FLT_CHROMA_V_MODE(arg->chr_v_mode) | - RKISP1_CIF_ISP_FLT_CHROMA_H_MODE(arg->chr_h_mode) | - RKISP1_CIF_ISP_FLT_GREEN_STAGE1(arg->grn_stage1)); - /* avoid to override the old enable value */ filt_mode = rkisp1_read(params->rkisp1, RKISP1_CIF_ISP_FILT_MODE); filt_mode &= RKISP1_CIF_ISP_FLT_ENA; diff --git a/drivers/media/platform/rockchip/rkvdec/rkvdec-vp9.c b/drivers/media/platform/rockchip/rkvdec/rkvdec-vp9.c index b4bf01e839eff2..8fb6a1624a14f1 100644 --- a/drivers/media/platform/rockchip/rkvdec/rkvdec-vp9.c +++ b/drivers/media/platform/rockchip/rkvdec/rkvdec-vp9.c @@ -927,7 +927,8 @@ static void rkvdec_vp9_done(struct rkvdec_ctx *ctx, update_ctx_last_info(vp9_ctx); } -static void rkvdec_init_v4l2_vp9_count_tbl(struct rkvdec_ctx *ctx) +static noinline_for_stack void +rkvdec_init_v4l2_vp9_count_tbl(struct rkvdec_ctx *ctx) { struct rkvdec_vp9_ctx *vp9_ctx = ctx->priv; struct rkvdec_vp9_intra_frame_symbol_counts *intra_cnts = vp9_ctx->count_tbl.cpu; diff --git a/drivers/media/platform/st/stm32/stm32-dcmipp/dcmipp-bytecap.c b/drivers/media/platform/st/stm32/stm32-dcmipp/dcmipp-bytecap.c index 1c1b6b48918ee2..b18e273ef4a3e6 100644 --- a/drivers/media/platform/st/stm32/stm32-dcmipp/dcmipp-bytecap.c +++ b/drivers/media/platform/st/stm32/stm32-dcmipp/dcmipp-bytecap.c @@ -512,6 +512,9 @@ static void dcmipp_bytecap_stop_streaming(struct vb2_queue *vq) /* Disable pipe */ reg_clear(vcap, DCMIPP_P0FSCR, DCMIPP_P0FSCR_PIPEN); + /* Clear any pending interrupts */ + reg_write(vcap, DCMIPP_CMFCR, DCMIPP_CMIER_P0ALL); + spin_lock_irq(&vcap->irqlock); /* Return all queued buffers to vb2 in ERROR state */ diff --git a/drivers/media/platform/st/stm32/stm32-dcmipp/dcmipp-byteproc.c b/drivers/media/platform/st/stm32/stm32-dcmipp/dcmipp-byteproc.c index db76a02a1848ab..ec1d773d5ad123 100644 --- a/drivers/media/platform/st/stm32/stm32-dcmipp/dcmipp-byteproc.c +++ b/drivers/media/platform/st/stm32/stm32-dcmipp/dcmipp-byteproc.c @@ -130,11 +130,8 @@ static void dcmipp_byteproc_adjust_compose(struct v4l2_rect *r, r->left = 0; /* Compose is not possible for JPEG or Bayer formats */ - if (fmt->code == MEDIA_BUS_FMT_JPEG_1X8 || - fmt->code == MEDIA_BUS_FMT_SBGGR8_1X8 || - fmt->code == MEDIA_BUS_FMT_SGBRG8_1X8 || - fmt->code == MEDIA_BUS_FMT_SGRBG8_1X8 || - fmt->code == MEDIA_BUS_FMT_SRGGB8_1X8) { + if (fmt->code >= MEDIA_BUS_FMT_SBGGR8_1X8 && + fmt->code <= MEDIA_BUS_FMT_JPEG_1X8) { r->width = fmt->width; r->height = fmt->height; return; diff --git a/drivers/media/platform/st/stm32/stm32-dcmipp/dcmipp-core.c b/drivers/media/platform/st/stm32/stm32-dcmipp/dcmipp-core.c index 1b7bae3266c8db..49398d07776462 100644 --- a/drivers/media/platform/st/stm32/stm32-dcmipp/dcmipp-core.c +++ b/drivers/media/platform/st/stm32/stm32-dcmipp/dcmipp-core.c @@ -526,7 +526,12 @@ static int dcmipp_probe(struct platform_device *pdev) return ret; } - kclk = devm_clk_get(&pdev->dev, "kclk"); + /* + * In case of the DCMIPP has only 1 clock (such as on MP13), the + * clock might not be named. + */ + kclk = devm_clk_get(&pdev->dev, + dcmipp->pipe_cfg->needs_mclk ? "kclk" : NULL); if (IS_ERR(kclk)) return dev_err_probe(&pdev->dev, PTR_ERR(kclk), "Unable to get kclk\n"); diff --git a/drivers/media/platform/ti/omap3isp/isppreview.c b/drivers/media/platform/ti/omap3isp/isppreview.c index e383a57654de81..5c492b31b51606 100644 --- a/drivers/media/platform/ti/omap3isp/isppreview.c +++ b/drivers/media/platform/ti/omap3isp/isppreview.c @@ -1742,22 +1742,17 @@ static void preview_try_format(struct isp_prev_device *prev, switch (pad) { case PREV_PAD_SINK: - /* When reading data from the CCDC, the input size has already - * been mangled by the CCDC output pad so it can be accepted - * as-is. - * - * When reading data from memory, clamp the requested width and - * height. The TRM doesn't specify a minimum input height, make + /* + * Clamp the requested width and height. + * The TRM doesn't specify a minimum input height, make * sure we got enough lines to enable the noise filter and color * filter array interpolation. */ - if (prev->input == PREVIEW_INPUT_MEMORY) { - fmt->width = clamp_t(u32, fmt->width, PREV_MIN_IN_WIDTH, - preview_max_out_width(prev)); - fmt->height = clamp_t(u32, fmt->height, - PREV_MIN_IN_HEIGHT, - PREV_MAX_IN_HEIGHT); - } + fmt->width = clamp_t(u32, fmt->width, PREV_MIN_IN_WIDTH, + preview_max_out_width(prev)); + fmt->height = clamp_t(u32, fmt->height, + PREV_MIN_IN_HEIGHT, + PREV_MAX_IN_HEIGHT); fmt->colorspace = V4L2_COLORSPACE_SRGB; diff --git a/drivers/media/platform/ti/omap3isp/ispvideo.c b/drivers/media/platform/ti/omap3isp/ispvideo.c index 0e7f0bf2b3463b..eb33a776f27c90 100644 --- a/drivers/media/platform/ti/omap3isp/ispvideo.c +++ b/drivers/media/platform/ti/omap3isp/ispvideo.c @@ -148,12 +148,12 @@ static unsigned int isp_video_mbus_to_pix(const struct isp_video *video, pix->width = mbus->width; pix->height = mbus->height; - for (i = 0; i < ARRAY_SIZE(formats); ++i) { + for (i = 0; i < ARRAY_SIZE(formats) - 1; ++i) { if (formats[i].code == mbus->code) break; } - if (WARN_ON(i == ARRAY_SIZE(formats))) + if (WARN_ON(i == ARRAY_SIZE(formats) - 1)) return 0; min_bpl = pix->width * formats[i].bpp; @@ -191,7 +191,7 @@ static void isp_video_pix_to_mbus(const struct v4l2_pix_format *pix, /* Skip the last format in the loop so that it will be selected if no * match is found. */ - for (i = 0; i < ARRAY_SIZE(formats) - 1; ++i) { + for (i = 0; i < ARRAY_SIZE(formats) - 2; ++i) { if (formats[i].pixelformat == pix->pixelformat) break; } @@ -1288,6 +1288,7 @@ static const struct v4l2_ioctl_ops isp_video_ioctl_ops = { static int isp_video_open(struct file *file) { struct isp_video *video = video_drvdata(file); + struct v4l2_mbus_framefmt fmt; struct isp_video_fh *handle; struct vb2_queue *queue; int ret = 0; @@ -1330,6 +1331,13 @@ static int isp_video_open(struct file *file) memset(&handle->format, 0, sizeof(handle->format)); handle->format.type = video->type; + handle->format.fmt.pix.width = 720; + handle->format.fmt.pix.height = 480; + handle->format.fmt.pix.pixelformat = V4L2_PIX_FMT_UYVY; + handle->format.fmt.pix.field = V4L2_FIELD_NONE; + handle->format.fmt.pix.colorspace = V4L2_COLORSPACE_SRGB; + isp_video_pix_to_mbus(&handle->format.fmt.pix, &fmt); + isp_video_mbus_to_pix(video, &fmt, &handle->format.fmt.pix); handle->timeperframe.denominator = 1; handle->video = video; diff --git a/drivers/media/platform/verisilicon/hantro.h b/drivers/media/platform/verisilicon/hantro.h index e0fdc4535b2d73..0353de154a1ecb 100644 --- a/drivers/media/platform/verisilicon/hantro.h +++ b/drivers/media/platform/verisilicon/hantro.h @@ -77,6 +77,7 @@ struct hantro_irq { * @double_buffer: core needs double buffering * @legacy_regs: core uses legacy register set * @late_postproc: postproc must be set up at the end of the job + * @shared_devices: an array of device ids that cannot run concurrently */ struct hantro_variant { unsigned int enc_offset; @@ -101,6 +102,7 @@ struct hantro_variant { unsigned int double_buffer : 1; unsigned int legacy_regs : 1; unsigned int late_postproc : 1; + const struct of_device_id *shared_devices; }; /** diff --git a/drivers/media/platform/verisilicon/hantro_drv.c b/drivers/media/platform/verisilicon/hantro_drv.c index 60b95b5d8565fd..94f58f4e4a4e5b 100644 --- a/drivers/media/platform/verisilicon/hantro_drv.c +++ b/drivers/media/platform/verisilicon/hantro_drv.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -1035,6 +1036,41 @@ static int hantro_disable_multicore(struct hantro_dev *vpu) return 0; } +static struct v4l2_m2m_dev *hantro_get_v4l2_m2m_dev(struct hantro_dev *vpu) +{ + struct device_node *node; + struct hantro_dev *shared_vpu; + + if (!vpu->variant || !vpu->variant->shared_devices) + goto init_new_m2m_dev; + + for_each_matching_node(node, vpu->variant->shared_devices) { + struct platform_device *pdev; + struct v4l2_m2m_dev *m2m_dev; + + pdev = of_find_device_by_node(node); + if (!pdev) + continue; + + shared_vpu = platform_get_drvdata(pdev); + if (IS_ERR_OR_NULL(shared_vpu) || shared_vpu == vpu) { + platform_device_put(pdev); + continue; + } + + v4l2_m2m_get(shared_vpu->m2m_dev); + m2m_dev = shared_vpu->m2m_dev; + platform_device_put(pdev); + + of_node_put(node); + + return m2m_dev; + } + +init_new_m2m_dev: + return v4l2_m2m_init(&vpu_m2m_ops); +} + static int hantro_probe(struct platform_device *pdev) { const struct of_device_id *match; @@ -1186,7 +1222,7 @@ static int hantro_probe(struct platform_device *pdev) } platform_set_drvdata(pdev, vpu); - vpu->m2m_dev = v4l2_m2m_init(&vpu_m2m_ops); + vpu->m2m_dev = hantro_get_v4l2_m2m_dev(vpu); if (IS_ERR(vpu->m2m_dev)) { v4l2_err(&vpu->v4l2_dev, "Failed to init mem2mem device\n"); ret = PTR_ERR(vpu->m2m_dev); @@ -1225,7 +1261,7 @@ static int hantro_probe(struct platform_device *pdev) hantro_remove_enc_func(vpu); err_m2m_rel: media_device_cleanup(&vpu->mdev); - v4l2_m2m_release(vpu->m2m_dev); + v4l2_m2m_put(vpu->m2m_dev); err_v4l2_unreg: v4l2_device_unregister(&vpu->v4l2_dev); err_clk_unprepare: @@ -1248,7 +1284,7 @@ static void hantro_remove(struct platform_device *pdev) hantro_remove_dec_func(vpu); hantro_remove_enc_func(vpu); media_device_cleanup(&vpu->mdev); - v4l2_m2m_release(vpu->m2m_dev); + v4l2_m2m_put(vpu->m2m_dev); v4l2_device_unregister(&vpu->v4l2_dev); clk_bulk_unprepare(vpu->variant->num_clocks, vpu->clocks); reset_control_assert(vpu->resets); diff --git a/drivers/media/platform/verisilicon/imx8m_vpu_hw.c b/drivers/media/platform/verisilicon/imx8m_vpu_hw.c index 5be0e2e76882f1..fa4224de4b996a 100644 --- a/drivers/media/platform/verisilicon/imx8m_vpu_hw.c +++ b/drivers/media/platform/verisilicon/imx8m_vpu_hw.c @@ -343,6 +343,12 @@ const struct hantro_variant imx8mq_vpu_variant = { .num_regs = ARRAY_SIZE(imx8mq_reg_names) }; +static const struct of_device_id imx8mq_vpu_shared_resources[] = { + { .compatible = "nxp,imx8mq-vpu-g1", }, + { .compatible = "nxp,imx8mq-vpu-g2", }, + { /* sentinel */ } +}; + const struct hantro_variant imx8mq_vpu_g1_variant = { .dec_fmts = imx8m_vpu_dec_fmts, .num_dec_fmts = ARRAY_SIZE(imx8m_vpu_dec_fmts), @@ -356,6 +362,7 @@ const struct hantro_variant imx8mq_vpu_g1_variant = { .num_irqs = ARRAY_SIZE(imx8mq_irqs), .clk_names = imx8mq_g1_clk_names, .num_clocks = ARRAY_SIZE(imx8mq_g1_clk_names), + .shared_devices = imx8mq_vpu_shared_resources, }; const struct hantro_variant imx8mq_vpu_g2_variant = { @@ -371,6 +378,7 @@ const struct hantro_variant imx8mq_vpu_g2_variant = { .num_irqs = ARRAY_SIZE(imx8mq_g2_irqs), .clk_names = imx8mq_g2_clk_names, .num_clocks = ARRAY_SIZE(imx8mq_g2_clk_names), + .shared_devices = imx8mq_vpu_shared_resources, }; const struct hantro_variant imx8mm_vpu_g1_variant = { diff --git a/drivers/media/platform/verisilicon/rockchip_vpu981_hw_av1_dec.c b/drivers/media/platform/verisilicon/rockchip_vpu981_hw_av1_dec.c index e4703bb6be7c17..e4e21ad373233e 100644 --- a/drivers/media/platform/verisilicon/rockchip_vpu981_hw_av1_dec.c +++ b/drivers/media/platform/verisilicon/rockchip_vpu981_hw_av1_dec.c @@ -72,6 +72,14 @@ : AV1_DIV_ROUND_UP_POW2((_value_), (_n_))); \ }) +enum rockchip_av1_tx_mode { + ROCKCHIP_AV1_TX_MODE_ONLY_4X4 = 0, + ROCKCHIP_AV1_TX_MODE_8X8 = 1, + ROCKCHIP_AV1_TX_MODE_16x16 = 2, + ROCKCHIP_AV1_TX_MODE_32x32 = 3, + ROCKCHIP_AV1_TX_MODE_SELECT = 4, +}; + struct rockchip_av1_film_grain { u8 scaling_lut_y[256]; u8 scaling_lut_cb[256]; @@ -373,12 +381,12 @@ int rockchip_vpu981_av1_dec_init(struct hantro_ctx *ctx) return -ENOMEM; av1_dec->global_model.size = GLOBAL_MODEL_SIZE; - av1_dec->tile_info.cpu = dma_alloc_coherent(vpu->dev, AV1_MAX_TILES, + av1_dec->tile_info.cpu = dma_alloc_coherent(vpu->dev, AV1_TILE_INFO_SIZE, &av1_dec->tile_info.dma, GFP_KERNEL); if (!av1_dec->tile_info.cpu) return -ENOMEM; - av1_dec->tile_info.size = AV1_MAX_TILES; + av1_dec->tile_info.size = AV1_TILE_INFO_SIZE; av1_dec->film_grain.cpu = dma_alloc_coherent(vpu->dev, ALIGN(sizeof(struct rockchip_av1_film_grain), 2048), @@ -1396,8 +1404,16 @@ static void rockchip_vpu981_av1_dec_set_cdef(struct hantro_ctx *ctx) u16 luma_sec_strength = 0; u32 chroma_pri_strength = 0; u16 chroma_sec_strength = 0; + bool enable_cdef; int i; + enable_cdef = !(cdef->bits == 0 && + cdef->damping_minus_3 == 0 && + cdef->y_pri_strength[0] == 0 && + cdef->y_sec_strength[0] == 0 && + cdef->uv_pri_strength[0] == 0 && + cdef->uv_sec_strength[0] == 0); + hantro_reg_write(vpu, &av1_enable_cdef, enable_cdef); hantro_reg_write(vpu, &av1_cdef_bits, cdef->bits); hantro_reg_write(vpu, &av1_cdef_damping, cdef->damping_minus_3); @@ -1927,11 +1943,26 @@ static void rockchip_vpu981_av1_dec_set_reference_frames(struct hantro_ctx *ctx) rockchip_vpu981_av1_dec_set_other_frames(ctx); } +static int rockchip_vpu981_av1_get_hardware_tx_mode(enum v4l2_av1_tx_mode tx_mode) +{ + switch (tx_mode) { + case V4L2_AV1_TX_MODE_ONLY_4X4: + return ROCKCHIP_AV1_TX_MODE_ONLY_4X4; + case V4L2_AV1_TX_MODE_LARGEST: + return ROCKCHIP_AV1_TX_MODE_32x32; + case V4L2_AV1_TX_MODE_SELECT: + return ROCKCHIP_AV1_TX_MODE_SELECT; + } + + return ROCKCHIP_AV1_TX_MODE_32x32; +} + static void rockchip_vpu981_av1_dec_set_parameters(struct hantro_ctx *ctx) { struct hantro_dev *vpu = ctx->dev; struct hantro_av1_dec_hw_ctx *av1_dec = &ctx->av1_dec; struct hantro_av1_dec_ctrls *ctrls = &av1_dec->ctrls; + int tx_mode; hantro_reg_write(vpu, &av1_skip_mode, !!(ctrls->frame->flags & V4L2_AV1_FRAME_FLAG_SKIP_MODE_PRESENT)); @@ -1953,8 +1984,6 @@ static void rockchip_vpu981_av1_dec_set_parameters(struct hantro_ctx *ctx) !!(ctrls->frame->flags & V4L2_AV1_FRAME_FLAG_SHOW_FRAME)); hantro_reg_write(vpu, &av1_switchable_motion_mode, !!(ctrls->frame->flags & V4L2_AV1_FRAME_FLAG_IS_MOTION_MODE_SWITCHABLE)); - hantro_reg_write(vpu, &av1_enable_cdef, - !!(ctrls->sequence->flags & V4L2_AV1_SEQUENCE_FLAG_ENABLE_CDEF)); hantro_reg_write(vpu, &av1_allow_masked_compound, !!(ctrls->sequence->flags & V4L2_AV1_SEQUENCE_FLAG_ENABLE_MASKED_COMPOUND)); @@ -1989,7 +2018,7 @@ static void rockchip_vpu981_av1_dec_set_parameters(struct hantro_ctx *ctx) !!(ctrls->frame->quantization.flags & V4L2_AV1_QUANTIZATION_FLAG_DELTA_Q_PRESENT)); - hantro_reg_write(vpu, &av1_idr_pic_e, !ctrls->frame->frame_type); + hantro_reg_write(vpu, &av1_idr_pic_e, IS_INTRA(ctrls->frame->frame_type)); hantro_reg_write(vpu, &av1_quant_base_qindex, ctrls->frame->quantization.base_q_idx); hantro_reg_write(vpu, &av1_bit_depth_y_minus8, ctx->bit_depth - 8); hantro_reg_write(vpu, &av1_bit_depth_c_minus8, ctx->bit_depth - 8); @@ -1999,7 +2028,9 @@ static void rockchip_vpu981_av1_dec_set_parameters(struct hantro_ctx *ctx) !!(ctrls->frame->flags & V4L2_AV1_FRAME_FLAG_ALLOW_HIGH_PRECISION_MV)); hantro_reg_write(vpu, &av1_comp_pred_mode, (ctrls->frame->flags & V4L2_AV1_FRAME_FLAG_REFERENCE_SELECT) ? 2 : 0); - hantro_reg_write(vpu, &av1_transform_mode, (ctrls->frame->tx_mode == 1) ? 3 : 4); + + tx_mode = rockchip_vpu981_av1_get_hardware_tx_mode(ctrls->frame->tx_mode); + hantro_reg_write(vpu, &av1_transform_mode, tx_mode); hantro_reg_write(vpu, &av1_max_cb_size, (ctrls->sequence->flags & V4L2_AV1_SEQUENCE_FLAG_USE_128X128_SUPERBLOCK) ? 7 : 6); diff --git a/drivers/media/radio/radio-keene.c b/drivers/media/radio/radio-keene.c index f3b57f0cb1ec40..c133305fd0194a 100644 --- a/drivers/media/radio/radio-keene.c +++ b/drivers/media/radio/radio-keene.c @@ -338,7 +338,6 @@ static int usb_keene_probe(struct usb_interface *intf, if (hdl->error) { retval = hdl->error; - v4l2_ctrl_handler_free(hdl); goto err_v4l2; } retval = v4l2_device_register(&intf->dev, &radio->v4l2_dev); @@ -384,6 +383,7 @@ static int usb_keene_probe(struct usb_interface *intf, err_vdev: v4l2_device_unregister(&radio->v4l2_dev); err_v4l2: + v4l2_ctrl_handler_free(&radio->hdl); kfree(radio->buffer); kfree(radio); err: diff --git a/drivers/media/test-drivers/vidtv/vidtv_bridge.c b/drivers/media/test-drivers/vidtv/vidtv_bridge.c index 438483c62facc2..52b2abe16dcf56 100644 --- a/drivers/media/test-drivers/vidtv/vidtv_bridge.c +++ b/drivers/media/test-drivers/vidtv/vidtv_bridge.c @@ -237,8 +237,10 @@ static int vidtv_start_feed(struct dvb_demux_feed *feed) if (dvb->nfeeds == 1) { ret = vidtv_start_streaming(dvb); - if (ret < 0) + if (ret < 0) { + dvb->nfeeds--; rc = ret; + } } mutex_unlock(&dvb->feed_lock); diff --git a/drivers/media/test-drivers/vidtv/vidtv_channel.c b/drivers/media/test-drivers/vidtv/vidtv_channel.c index 3541155c6fc635..aa177cf96b6ac6 100644 --- a/drivers/media/test-drivers/vidtv/vidtv_channel.c +++ b/drivers/media/test-drivers/vidtv/vidtv_channel.c @@ -341,6 +341,10 @@ vidtv_channel_pmt_match_sections(struct vidtv_channel *channels, tail = vidtv_psi_pmt_stream_init(tail, s->type, e_pid); + if (!tail) { + vidtv_psi_pmt_stream_destroy(head); + return; + } if (!head) head = tail; diff --git a/drivers/media/test-drivers/vidtv/vidtv_mux.c b/drivers/media/test-drivers/vidtv/vidtv_mux.c index f99878eff7acea..7dad97881fdb04 100644 --- a/drivers/media/test-drivers/vidtv/vidtv_mux.c +++ b/drivers/media/test-drivers/vidtv/vidtv_mux.c @@ -233,7 +233,7 @@ static u32 vidtv_mux_push_pcr(struct vidtv_mux *m) /* the 27Mhz clock will feed both parts of the PCR bitfield */ args.pcr = m->timing.clk; - nbytes += vidtv_ts_pcr_write_into(args); + nbytes += vidtv_ts_pcr_write_into(&args); m->mux_buf_offset += nbytes; m->num_streamed_pcr++; @@ -363,7 +363,7 @@ static u32 vidtv_mux_pad_with_nulls(struct vidtv_mux *m, u32 npkts) args.continuity_counter = &ctx->cc; for (i = 0; i < npkts; ++i) { - m->mux_buf_offset += vidtv_ts_null_write_into(args); + m->mux_buf_offset += vidtv_ts_null_write_into(&args); args.dest_offset = m->mux_buf_offset; } diff --git a/drivers/media/test-drivers/vidtv/vidtv_ts.c b/drivers/media/test-drivers/vidtv/vidtv_ts.c index ca4bb9c40b78ef..cbe9aff9ffb564 100644 --- a/drivers/media/test-drivers/vidtv/vidtv_ts.c +++ b/drivers/media/test-drivers/vidtv/vidtv_ts.c @@ -48,7 +48,7 @@ void vidtv_ts_inc_cc(u8 *continuity_counter) *continuity_counter = 0; } -u32 vidtv_ts_null_write_into(struct null_packet_write_args args) +u32 vidtv_ts_null_write_into(const struct null_packet_write_args *args) { u32 nbytes = 0; struct vidtv_mpeg_ts ts_header = {}; @@ -56,21 +56,21 @@ u32 vidtv_ts_null_write_into(struct null_packet_write_args args) ts_header.sync_byte = TS_SYNC_BYTE; ts_header.bitfield = cpu_to_be16(TS_NULL_PACKET_PID); ts_header.payload = 1; - ts_header.continuity_counter = *args.continuity_counter; + ts_header.continuity_counter = *args->continuity_counter; /* copy TS header */ - nbytes += vidtv_memcpy(args.dest_buf, - args.dest_offset + nbytes, - args.buf_sz, + nbytes += vidtv_memcpy(args->dest_buf, + args->dest_offset + nbytes, + args->buf_sz, &ts_header, sizeof(ts_header)); - vidtv_ts_inc_cc(args.continuity_counter); + vidtv_ts_inc_cc(args->continuity_counter); /* fill the rest with empty data */ - nbytes += vidtv_memset(args.dest_buf, - args.dest_offset + nbytes, - args.buf_sz, + nbytes += vidtv_memset(args->dest_buf, + args->dest_offset + nbytes, + args->buf_sz, TS_FILL_BYTE, TS_PACKET_LEN - nbytes); @@ -83,17 +83,17 @@ u32 vidtv_ts_null_write_into(struct null_packet_write_args args) return nbytes; } -u32 vidtv_ts_pcr_write_into(struct pcr_write_args args) +u32 vidtv_ts_pcr_write_into(const struct pcr_write_args *args) { u32 nbytes = 0; struct vidtv_mpeg_ts ts_header = {}; struct vidtv_mpeg_ts_adaption ts_adap = {}; ts_header.sync_byte = TS_SYNC_BYTE; - ts_header.bitfield = cpu_to_be16(args.pid); + ts_header.bitfield = cpu_to_be16(args->pid); ts_header.scrambling = 0; /* cc is not incremented, but it is needed. see 13818-1 clause 2.4.3.3 */ - ts_header.continuity_counter = *args.continuity_counter; + ts_header.continuity_counter = *args->continuity_counter; ts_header.payload = 0; ts_header.adaptation_field = 1; @@ -102,27 +102,27 @@ u32 vidtv_ts_pcr_write_into(struct pcr_write_args args) ts_adap.PCR = 1; /* copy TS header */ - nbytes += vidtv_memcpy(args.dest_buf, - args.dest_offset + nbytes, - args.buf_sz, + nbytes += vidtv_memcpy(args->dest_buf, + args->dest_offset + nbytes, + args->buf_sz, &ts_header, sizeof(ts_header)); /* write the adap after the TS header */ - nbytes += vidtv_memcpy(args.dest_buf, - args.dest_offset + nbytes, - args.buf_sz, + nbytes += vidtv_memcpy(args->dest_buf, + args->dest_offset + nbytes, + args->buf_sz, &ts_adap, sizeof(ts_adap)); /* write the PCR optional */ - nbytes += vidtv_ts_write_pcr_bits(args.dest_buf, - args.dest_offset + nbytes, - args.pcr); + nbytes += vidtv_ts_write_pcr_bits(args->dest_buf, + args->dest_offset + nbytes, + args->pcr); - nbytes += vidtv_memset(args.dest_buf, - args.dest_offset + nbytes, - args.buf_sz, + nbytes += vidtv_memset(args->dest_buf, + args->dest_offset + nbytes, + args->buf_sz, TS_FILL_BYTE, TS_PACKET_LEN - nbytes); diff --git a/drivers/media/test-drivers/vidtv/vidtv_ts.h b/drivers/media/test-drivers/vidtv/vidtv_ts.h index 09b4ffd0282921..3606398e160d9f 100644 --- a/drivers/media/test-drivers/vidtv/vidtv_ts.h +++ b/drivers/media/test-drivers/vidtv/vidtv_ts.h @@ -90,7 +90,7 @@ void vidtv_ts_inc_cc(u8 *continuity_counter); * * Return: The number of bytes written into the buffer. */ -u32 vidtv_ts_null_write_into(struct null_packet_write_args args); +u32 vidtv_ts_null_write_into(const struct null_packet_write_args *args); /** * vidtv_ts_pcr_write_into - Write a PCR packet into a buffer. @@ -101,6 +101,6 @@ u32 vidtv_ts_null_write_into(struct null_packet_write_args args); * * Return: The number of bytes written into the buffer. */ -u32 vidtv_ts_pcr_write_into(struct pcr_write_args args); +u32 vidtv_ts_pcr_write_into(const struct pcr_write_args *args); #endif //VIDTV_TS_H diff --git a/drivers/media/usb/as102/as102_usb_drv.c b/drivers/media/usb/as102/as102_usb_drv.c index e0ef66a522e23d..44565f0297cdb9 100644 --- a/drivers/media/usb/as102/as102_usb_drv.c +++ b/drivers/media/usb/as102/as102_usb_drv.c @@ -403,7 +403,9 @@ static int as102_usb_probe(struct usb_interface *intf, failed_dvb: as102_free_usb_stream_buffer(as102_dev); failed_stream: + usb_set_intfdata(intf, NULL); usb_deregister_dev(intf, &as102_usb_class_driver); + return ret; failed: usb_put_dev(as102_dev->bus_adap.usb_dev); usb_set_intfdata(intf, NULL); diff --git a/drivers/media/usb/em28xx/em28xx-video.c b/drivers/media/usb/em28xx/em28xx-video.c index 2dfa3242a7ab52..14c35995cd9521 100644 --- a/drivers/media/usb/em28xx/em28xx-video.c +++ b/drivers/media/usb/em28xx/em28xx-video.c @@ -2126,7 +2126,7 @@ static int em28xx_v4l2_open(struct file *filp) { struct video_device *vdev = video_devdata(filp); struct em28xx *dev = video_drvdata(filp); - struct em28xx_v4l2 *v4l2 = dev->v4l2; + struct em28xx_v4l2 *v4l2; enum v4l2_buf_type fh_type = 0; int ret; @@ -2143,13 +2143,19 @@ static int em28xx_v4l2_open(struct file *filp) return -EINVAL; } + if (mutex_lock_interruptible(&dev->lock)) + return -ERESTARTSYS; + + v4l2 = dev->v4l2; + if (!v4l2) { + mutex_unlock(&dev->lock); + return -ENODEV; + } + em28xx_videodbg("open dev=%s type=%s users=%d\n", video_device_node_name(vdev), v4l2_type_names[fh_type], v4l2->users); - if (mutex_lock_interruptible(&dev->lock)) - return -ERESTARTSYS; - ret = v4l2_fh_open(filp); if (ret) { dev_err(&dev->intf->dev, diff --git a/drivers/media/usb/hackrf/hackrf.c b/drivers/media/usb/hackrf/hackrf.c index 0b50de8775a38d..c3c4247194d194 100644 --- a/drivers/media/usb/hackrf/hackrf.c +++ b/drivers/media/usb/hackrf/hackrf.c @@ -1485,7 +1485,7 @@ static int hackrf_probe(struct usb_interface *intf, if (ret) { dev_err(dev->dev, "Failed to register as video device (%d)\n", ret); - goto err_v4l2_device_unregister; + goto err_v4l2_device_put; } dev_info(dev->dev, "Registered as %s\n", video_device_node_name(&dev->rx_vdev)); @@ -1513,8 +1513,9 @@ static int hackrf_probe(struct usb_interface *intf, return 0; err_video_unregister_device_rx: video_unregister_device(&dev->rx_vdev); -err_v4l2_device_unregister: - v4l2_device_unregister(&dev->v4l2_dev); +err_v4l2_device_put: + v4l2_device_put(&dev->v4l2_dev); + return ret; err_v4l2_ctrl_handler_free_tx: v4l2_ctrl_handler_free(&dev->tx_ctrl_handler); err_v4l2_ctrl_handler_free_rx: diff --git a/drivers/media/usb/pvrusb2/pvrusb2-hdw.c b/drivers/media/usb/pvrusb2/pvrusb2-hdw.c index b32bb906a9de29..5807734ae26c65 100644 --- a/drivers/media/usb/pvrusb2/pvrusb2-hdw.c +++ b/drivers/media/usb/pvrusb2/pvrusb2-hdw.c @@ -3709,6 +3709,11 @@ status); "Failed to submit read-control URB status=%d", status); hdw->ctl_read_pend_flag = 0; + if (hdw->ctl_write_pend_flag) { + usb_unlink_urb(hdw->ctl_write_urb); + while (hdw->ctl_write_pend_flag) + wait_for_completion(&hdw->ctl_done); + } goto done; } } diff --git a/drivers/media/usb/uvc/uvc_ctrl.c b/drivers/media/usb/uvc/uvc_ctrl.c index 2905505c240c06..2738ef74c7373b 100644 --- a/drivers/media/usb/uvc/uvc_ctrl.c +++ b/drivers/media/usb/uvc/uvc_ctrl.c @@ -1432,7 +1432,7 @@ static bool uvc_ctrl_is_readable(u32 which, struct uvc_control *ctrl, * auto_exposure=1, exposure_time_absolute=251. */ int uvc_ctrl_is_accessible(struct uvc_video_chain *chain, u32 v4l2_id, - const struct v4l2_ext_controls *ctrls, + const struct v4l2_ext_controls *ctrls, u32 which, unsigned long ioctl) { struct uvc_control_mapping *master_map = NULL; @@ -1442,14 +1442,24 @@ int uvc_ctrl_is_accessible(struct uvc_video_chain *chain, u32 v4l2_id, s32 val; int ret; int i; + /* + * There is no need to check the ioctl, all the ioctls except + * VIDIOC_G_EXT_CTRLS use which=V4L2_CTRL_WHICH_CUR_VAL. + */ + bool is_which_min_max = which == V4L2_CTRL_WHICH_MIN_VAL || + which == V4L2_CTRL_WHICH_MAX_VAL; if (__uvc_query_v4l2_class(chain, v4l2_id, 0) >= 0) - return -EACCES; + return is_which_min_max ? -EINVAL : -EACCES; ctrl = uvc_find_control(chain, v4l2_id, &mapping); if (!ctrl) return -EINVAL; + if ((!(ctrl->info.flags & UVC_CTRL_FLAG_GET_MIN) || + !(ctrl->info.flags & UVC_CTRL_FLAG_GET_MAX)) && is_which_min_max) + return -EINVAL; + if (ioctl == VIDIOC_G_EXT_CTRLS) return uvc_ctrl_is_readable(ctrls->which, ctrl, mapping); diff --git a/drivers/media/usb/uvc/uvc_driver.c b/drivers/media/usb/uvc/uvc_driver.c index ee4f54d6834962..aa3e8d295e0f59 100644 --- a/drivers/media/usb/uvc/uvc_driver.c +++ b/drivers/media/usb/uvc/uvc_driver.c @@ -165,28 +165,17 @@ static struct uvc_entity *uvc_entity_by_reference(struct uvc_device *dev, return NULL; } -static struct uvc_streaming *uvc_stream_by_id(struct uvc_device *dev, int id) +static struct uvc_streaming *uvc_stream_for_terminal(struct uvc_device *dev, + struct uvc_entity *term) { - struct uvc_streaming *stream, *last_stream; - unsigned int count = 0; + u16 id = UVC_HARDWARE_ENTITY_ID(term->id); + struct uvc_streaming *stream; list_for_each_entry(stream, &dev->streams, list) { - count += 1; - last_stream = stream; if (stream->header.bTerminalLink == id) return stream; } - /* - * If the streaming entity is referenced by an invalid ID, notify the - * user and use heuristics to guess the correct entity. - */ - if (count == 1 && id == UVC_INVALID_ENTITY_ID) { - dev_warn(&dev->intf->dev, - "UVC non compliance: Invalid USB header. The streaming entity has an invalid ID, guessing the correct one."); - return last_stream; - } - return NULL; } @@ -823,10 +812,12 @@ static struct uvc_entity *uvc_alloc_new_entity(struct uvc_device *dev, u16 type, } /* Per UVC 1.1+ spec 3.7.2, the ID is unique. */ - if (uvc_entity_by_id(dev, id)) { - dev_err(&dev->intf->dev, "Found multiple Units with ID %u\n", id); + if (uvc_entity_by_id(dev, UVC_HARDWARE_ENTITY_ID(id))) + dev_err(&dev->intf->dev, "Found multiple Units with ID %u\n", + UVC_HARDWARE_ENTITY_ID(id)); + + if (uvc_entity_by_id(dev, id)) id = UVC_INVALID_ENTITY_ID; - } extra_size = roundup(extra_size, sizeof(*entity->pads)); if (num_pads) @@ -982,6 +973,7 @@ static int uvc_parse_standard_control(struct uvc_device *dev, struct usb_host_interface *alts = dev->intf->cur_altsetting; unsigned int i, n, p, len; const char *type_name; + unsigned int id; u16 type; switch (buffer[2]) { @@ -1120,8 +1112,28 @@ static int uvc_parse_standard_control(struct uvc_device *dev, return 0; } + id = buffer[3]; + + /* + * Some devices, such as the Grandstream GUV3100, exhibit entity + * ID collisions between units and streaming output terminals. + * Move streaming output terminals to their own ID namespace by + * setting bit UVC_TERM_OUTPUT (15), above the ID's 8-bit value. + * The bit is ignored in uvc_stream_for_terminal() when looking + * up the streaming interface for the terminal. + * + * This hack is safe to enable unconditionally, as the ID is not + * used for any other purpose (streaming output terminals have + * no controls and are never referenced as sources in UVC + * descriptors). Other types output terminals can have controls, + * so limit usage of this separate namespace to streaming output + * terminals. + */ + if (type & UVC_TT_STREAMING) + id |= UVC_TERM_OUTPUT; + term = uvc_alloc_new_entity(dev, type | UVC_TERM_OUTPUT, - buffer[3], 1, 0); + id, 1, 0); if (IS_ERR(term)) return PTR_ERR(term); @@ -2118,8 +2130,8 @@ static int uvc_register_terms(struct uvc_device *dev, if (UVC_ENTITY_TYPE(term) != UVC_TT_STREAMING) continue; - stream = uvc_stream_by_id(dev, term->id); - if (stream == NULL) { + stream = uvc_stream_for_terminal(dev, term); + if (!stream) { dev_info(&dev->intf->dev, "No streaming interface found for terminal %u.", term->id); diff --git a/drivers/media/usb/uvc/uvc_queue.c b/drivers/media/usb/uvc/uvc_queue.c index 790184c9843d21..e838c6c1893a69 100644 --- a/drivers/media/usb/uvc/uvc_queue.c +++ b/drivers/media/usb/uvc/uvc_queue.c @@ -177,18 +177,20 @@ static int uvc_start_streaming_video(struct vb2_queue *vq, unsigned int count) ret = uvc_pm_get(stream->dev); if (ret) - return ret; + goto err_buffers; queue->buf_used = 0; ret = uvc_video_start_streaming(stream); - if (ret == 0) - return 0; + if (ret) + goto err_pm; - uvc_pm_put(stream->dev); + return 0; +err_pm: + uvc_pm_put(stream->dev); +err_buffers: uvc_queue_return_buffers(queue, UVC_BUF_STATE_QUEUED); - return ret; } diff --git a/drivers/media/usb/uvc/uvc_v4l2.c b/drivers/media/usb/uvc/uvc_v4l2.c index 9e4a251eca8808..30c160daed8cb0 100644 --- a/drivers/media/usb/uvc/uvc_v4l2.c +++ b/drivers/media/usb/uvc/uvc_v4l2.c @@ -765,14 +765,15 @@ static int uvc_ioctl_query_ext_ctrl(struct file *file, void *priv, static int uvc_ctrl_check_access(struct uvc_video_chain *chain, struct v4l2_ext_controls *ctrls, - unsigned long ioctl) + u32 which, unsigned long ioctl) { struct v4l2_ext_control *ctrl = ctrls->controls; unsigned int i; int ret = 0; for (i = 0; i < ctrls->count; ++ctrl, ++i) { - ret = uvc_ctrl_is_accessible(chain, ctrl->id, ctrls, ioctl); + ret = uvc_ctrl_is_accessible(chain, ctrl->id, ctrls, which, + ioctl); if (ret) break; } @@ -806,7 +807,7 @@ static int uvc_ioctl_g_ext_ctrls(struct file *file, void *priv, which = V4L2_CTRL_WHICH_CUR_VAL; } - ret = uvc_ctrl_check_access(chain, ctrls, VIDIOC_G_EXT_CTRLS); + ret = uvc_ctrl_check_access(chain, ctrls, which, VIDIOC_G_EXT_CTRLS); if (ret < 0) return ret; @@ -840,7 +841,8 @@ static int uvc_ioctl_s_try_ext_ctrls(struct uvc_fh *handle, if (!ctrls->count) return 0; - ret = uvc_ctrl_check_access(chain, ctrls, ioctl); + ret = uvc_ctrl_check_access(chain, ctrls, V4L2_CTRL_WHICH_CUR_VAL, + ioctl); if (ret < 0) return ret; diff --git a/drivers/media/usb/uvc/uvc_video.c b/drivers/media/usb/uvc/uvc_video.c index 2094e059d7d39a..ec76595f3c4be0 100644 --- a/drivers/media/usb/uvc/uvc_video.c +++ b/drivers/media/usb/uvc/uvc_video.c @@ -1812,7 +1812,7 @@ static int uvc_alloc_urb_buffers(struct uvc_streaming *stream, npackets = UVC_MAX_PACKETS; /* Retry allocations until one succeed. */ - for (; npackets > 1; npackets /= 2) { + for (; npackets > 0; npackets /= 2) { stream->urb_size = psize * npackets; for (i = 0; i < UVC_URBS; ++i) { @@ -1837,6 +1837,7 @@ static int uvc_alloc_urb_buffers(struct uvc_streaming *stream, uvc_dbg(stream->dev, VIDEO, "Failed to allocate URB buffers (%u bytes per packet)\n", psize); + stream->urb_size = 0; return 0; } diff --git a/drivers/media/usb/uvc/uvcvideo.h b/drivers/media/usb/uvc/uvcvideo.h index ed7bad31f75ca4..8480d65ecb85ed 100644 --- a/drivers/media/usb/uvc/uvcvideo.h +++ b/drivers/media/usb/uvc/uvcvideo.h @@ -41,7 +41,8 @@ #define UVC_EXT_GPIO_UNIT 0x7ffe #define UVC_EXT_GPIO_UNIT_ID 0x100 -#define UVC_INVALID_ENTITY_ID 0xffff +#define UVC_HARDWARE_ENTITY_ID(id) ((id) & 0xff) +#define UVC_INVALID_ENTITY_ID 0xffff /* ------------------------------------------------------------------------ * Driver specific constants. @@ -786,7 +787,7 @@ int uvc_ctrl_get(struct uvc_video_chain *chain, u32 which, struct v4l2_ext_control *xctrl); int uvc_ctrl_set(struct uvc_fh *handle, struct v4l2_ext_control *xctrl); int uvc_ctrl_is_accessible(struct uvc_video_chain *chain, u32 v4l2_id, - const struct v4l2_ext_controls *ctrls, + const struct v4l2_ext_controls *ctrls, u32 which, unsigned long ioctl); int uvc_xu_ctrl_query(struct uvc_video_chain *chain, diff --git a/drivers/media/v4l2-core/v4l2-async.c b/drivers/media/v4l2-core/v4l2-async.c index ee884a8221fbd3..1c08bba9ecb91f 100644 --- a/drivers/media/v4l2-core/v4l2-async.c +++ b/drivers/media/v4l2-core/v4l2-async.c @@ -343,7 +343,6 @@ static int v4l2_async_match_notify(struct v4l2_async_notifier *notifier, struct v4l2_subdev *sd, struct v4l2_async_connection *asc) { - struct v4l2_async_notifier *subdev_notifier; bool registered = false; int ret; @@ -389,6 +388,25 @@ static int v4l2_async_match_notify(struct v4l2_async_notifier *notifier, dev_dbg(notifier_dev(notifier), "v4l2-async: %s bound (ret %d)\n", dev_name(sd->dev), ret); + return 0; + +err_call_unbind: + v4l2_async_nf_call_unbind(notifier, sd, asc); + list_del(&asc->asc_subdev_entry); + +err_unregister_subdev: + if (registered) + v4l2_device_unregister_subdev(sd); + + return ret; +} + +static int +v4l2_async_nf_try_subdev_notifier(struct v4l2_async_notifier *notifier, + struct v4l2_subdev *sd) +{ + struct v4l2_async_notifier *subdev_notifier; + /* * See if the sub-device has a notifier. If not, return here. */ @@ -404,16 +422,6 @@ static int v4l2_async_match_notify(struct v4l2_async_notifier *notifier, subdev_notifier->parent = notifier; return v4l2_async_nf_try_all_subdevs(subdev_notifier); - -err_call_unbind: - v4l2_async_nf_call_unbind(notifier, sd, asc); - list_del(&asc->asc_subdev_entry); - -err_unregister_subdev: - if (registered) - v4l2_device_unregister_subdev(sd); - - return ret; } /* Test all async sub-devices in a notifier for a match. */ @@ -445,6 +453,10 @@ v4l2_async_nf_try_all_subdevs(struct v4l2_async_notifier *notifier) if (ret < 0) return ret; + ret = v4l2_async_nf_try_subdev_notifier(notifier, sd); + if (ret < 0) + return ret; + /* * v4l2_async_match_notify() may lead to registering a * new notifier and thus changing the async subdevs @@ -829,7 +841,11 @@ int __v4l2_async_register_subdev(struct v4l2_subdev *sd, struct module *module) ret = v4l2_async_match_notify(notifier, v4l2_dev, sd, asc); if (ret) - goto err_unbind; + goto err_unlock; + + ret = v4l2_async_nf_try_subdev_notifier(notifier, sd); + if (ret) + goto err_unbind_one; ret = v4l2_async_nf_try_complete(notifier); if (ret) @@ -853,9 +869,10 @@ int __v4l2_async_register_subdev(struct v4l2_subdev *sd, struct module *module) if (subdev_notifier) v4l2_async_nf_unbind_all_subdevs(subdev_notifier); - if (asc) - v4l2_async_unbind_subdev_one(notifier, asc); +err_unbind_one: + v4l2_async_unbind_subdev_one(notifier, asc); +err_unlock: mutex_unlock(&list_lock); sd->owner = NULL; diff --git a/drivers/media/v4l2-core/v4l2-ioctl.c b/drivers/media/v4l2-core/v4l2-ioctl.c index 98512ea4cc5b9d..113ad212a37c51 100644 --- a/drivers/media/v4l2-core/v4l2-ioctl.c +++ b/drivers/media/v4l2-core/v4l2-ioctl.c @@ -3081,13 +3081,14 @@ static long __video_do_ioctl(struct file *file, } /* - * We need to serialize streamon/off with queueing new requests. + * We need to serialize streamon/off/reqbufs with queueing new requests. * These ioctls may trigger the cancellation of a streaming * operation, and that should not be mixed with queueing a new * request at the same time. */ if (v4l2_device_supports_requests(vfd->v4l2_dev) && - (cmd == VIDIOC_STREAMON || cmd == VIDIOC_STREAMOFF)) { + (cmd == VIDIOC_STREAMON || cmd == VIDIOC_STREAMOFF || + cmd == VIDIOC_REQBUFS)) { req_queue_lock = &vfd->v4l2_dev->mdev->req_queue_mutex; if (mutex_lock_interruptible(req_queue_lock)) diff --git a/drivers/media/v4l2-core/v4l2-mem2mem.c b/drivers/media/v4l2-core/v4l2-mem2mem.c index fec93c1a923171..ae0de54d4c3e16 100644 --- a/drivers/media/v4l2-core/v4l2-mem2mem.c +++ b/drivers/media/v4l2-core/v4l2-mem2mem.c @@ -90,6 +90,7 @@ static const char * const m2m_entity_name[] = { * @job_work: worker to run queued jobs. * @job_queue_flags: flags of the queue status, %QUEUE_PAUSED. * @m2m_ops: driver callbacks + * @kref: device reference count */ struct v4l2_m2m_dev { struct v4l2_m2m_ctx *curr_ctx; @@ -109,6 +110,8 @@ struct v4l2_m2m_dev { unsigned long job_queue_flags; const struct v4l2_m2m_ops *m2m_ops; + + struct kref kref; }; static struct v4l2_m2m_queue_ctx *get_queue_ctx(struct v4l2_m2m_ctx *m2m_ctx, @@ -1200,6 +1203,7 @@ struct v4l2_m2m_dev *v4l2_m2m_init(const struct v4l2_m2m_ops *m2m_ops) INIT_LIST_HEAD(&m2m_dev->job_queue); spin_lock_init(&m2m_dev->job_spinlock); INIT_WORK(&m2m_dev->job_work, v4l2_m2m_device_run_work); + kref_init(&m2m_dev->kref); return m2m_dev; } @@ -1211,6 +1215,25 @@ void v4l2_m2m_release(struct v4l2_m2m_dev *m2m_dev) } EXPORT_SYMBOL_GPL(v4l2_m2m_release); +void v4l2_m2m_get(struct v4l2_m2m_dev *m2m_dev) +{ + kref_get(&m2m_dev->kref); +} +EXPORT_SYMBOL_GPL(v4l2_m2m_get); + +static void v4l2_m2m_release_from_kref(struct kref *kref) +{ + struct v4l2_m2m_dev *m2m_dev = container_of(kref, struct v4l2_m2m_dev, kref); + + v4l2_m2m_release(m2m_dev); +} + +void v4l2_m2m_put(struct v4l2_m2m_dev *m2m_dev) +{ + kref_put(&m2m_dev->kref, v4l2_m2m_release_from_kref); +} +EXPORT_SYMBOL_GPL(v4l2_m2m_put); + struct v4l2_m2m_ctx *v4l2_m2m_ctx_init(struct v4l2_m2m_dev *m2m_dev, void *drv_priv, int (*queue_init)(void *priv, struct vb2_queue *src_vq, struct vb2_queue *dst_vq)) diff --git a/drivers/memory/mtk-smi.c b/drivers/memory/mtk-smi.c index 733e22f695ab70..3609bfd3c64be3 100644 --- a/drivers/memory/mtk-smi.c +++ b/drivers/memory/mtk-smi.c @@ -674,6 +674,7 @@ static int mtk_smi_larb_probe(struct platform_device *pdev) err_pm_disable: pm_runtime_disable(dev); device_link_remove(dev, larb->smi_common_dev); + put_device(larb->smi_common_dev); return ret; } @@ -684,6 +685,7 @@ static void mtk_smi_larb_remove(struct platform_device *pdev) device_link_remove(&pdev->dev, larb->smi_common_dev); pm_runtime_disable(&pdev->dev); component_del(&pdev->dev, &mtk_smi_larb_component_ops); + put_device(larb->smi_common_dev); } static int __maybe_unused mtk_smi_larb_resume(struct device *dev) @@ -917,6 +919,7 @@ static void mtk_smi_common_remove(struct platform_device *pdev) if (common->plat->type == MTK_SMI_GEN2_SUB_COMM) device_link_remove(&pdev->dev, common->smi_common_dev); pm_runtime_disable(&pdev->dev); + put_device(common->smi_common_dev); } static int __maybe_unused mtk_smi_common_resume(struct device *dev) diff --git a/drivers/mfd/Kconfig b/drivers/mfd/Kconfig index aace5766b38aa5..f7f12a0428aa25 100644 --- a/drivers/mfd/Kconfig +++ b/drivers/mfd/Kconfig @@ -407,6 +407,17 @@ config MFD_CS47L92 help Support for Cirrus Logic CS42L92, CS47L92 and CS47L93 Smart Codecs +config MFD_TN48M_CPLD + tristate "Delta Networks TN48M switch CPLD driver" + depends on I2C + depends on ARCH_MVEBU || COMPILE_TEST + select MFD_SIMPLE_MFD_I2C + help + Select this option to enable support for Delta Networks TN48M switch + CPLD. It consists of reset and GPIO drivers. CPLD provides GPIOS-s + for the SFP slots as well as power supply related information. + SFP support depends on the GPIO driver being selected. + config PMIC_DA903X bool "Dialog Semiconductor DA9030/DA9034 PMIC Support" depends on I2C=y diff --git a/drivers/mfd/arizona-core.c b/drivers/mfd/arizona-core.c index 85ff8717d85047..91975536d14d21 100644 --- a/drivers/mfd/arizona-core.c +++ b/drivers/mfd/arizona-core.c @@ -1100,7 +1100,7 @@ int arizona_dev_init(struct arizona *arizona) } else if (val & 0x01) { ret = wm5102_clear_write_sequencer(arizona); if (ret) - return ret; + goto err_reset; } break; default: diff --git a/drivers/mfd/da9052-spi.c b/drivers/mfd/da9052-spi.c index 80fc5c0cac2fb0..be5f2b34e18aeb 100644 --- a/drivers/mfd/da9052-spi.c +++ b/drivers/mfd/da9052-spi.c @@ -37,7 +37,7 @@ static int da9052_spi_probe(struct spi_device *spi) spi_set_drvdata(spi, da9052); config = da9052_regmap_config; - config.write_flag_mask = 1; + config.read_flag_mask = 1; config.reg_bits = 7; config.pad_bits = 1; config.val_bits = 8; diff --git a/drivers/mfd/intel-lpss-pci.c b/drivers/mfd/intel-lpss-pci.c index 8d92c895d3aeff..713a5bfb1a3c29 100644 --- a/drivers/mfd/intel-lpss-pci.c +++ b/drivers/mfd/intel-lpss-pci.c @@ -437,6 +437,19 @@ static const struct pci_device_id intel_lpss_pci_ids[] = { { PCI_VDEVICE(INTEL, 0x5ac4), (kernel_ulong_t)&bxt_spi_info }, { PCI_VDEVICE(INTEL, 0x5ac6), (kernel_ulong_t)&bxt_spi_info }, { PCI_VDEVICE(INTEL, 0x5aee), (kernel_ulong_t)&bxt_uart_info }, + /* NVL-S */ + { PCI_VDEVICE(INTEL, 0x6e28), (kernel_ulong_t)&bxt_uart_info }, + { PCI_VDEVICE(INTEL, 0x6e29), (kernel_ulong_t)&bxt_uart_info }, + { PCI_VDEVICE(INTEL, 0x6e2a), (kernel_ulong_t)&tgl_spi_info }, + { PCI_VDEVICE(INTEL, 0x6e2b), (kernel_ulong_t)&tgl_spi_info }, + { PCI_VDEVICE(INTEL, 0x6e4c), (kernel_ulong_t)&ehl_i2c_info }, + { PCI_VDEVICE(INTEL, 0x6e4d), (kernel_ulong_t)&ehl_i2c_info }, + { PCI_VDEVICE(INTEL, 0x6e4e), (kernel_ulong_t)&ehl_i2c_info }, + { PCI_VDEVICE(INTEL, 0x6e4f), (kernel_ulong_t)&ehl_i2c_info }, + { PCI_VDEVICE(INTEL, 0x6e5c), (kernel_ulong_t)&bxt_uart_info }, + { PCI_VDEVICE(INTEL, 0x6e5e), (kernel_ulong_t)&tgl_spi_info }, + { PCI_VDEVICE(INTEL, 0x6e7a), (kernel_ulong_t)&ehl_i2c_info }, + { PCI_VDEVICE(INTEL, 0x6e7b), (kernel_ulong_t)&ehl_i2c_info }, /* ARL-H */ { PCI_VDEVICE(INTEL, 0x7725), (kernel_ulong_t)&bxt_uart_info }, { PCI_VDEVICE(INTEL, 0x7726), (kernel_ulong_t)&bxt_uart_info }, diff --git a/drivers/mfd/macsmc.c b/drivers/mfd/macsmc.c index e3893e255ce5e4..358feec2d088fc 100644 --- a/drivers/mfd/macsmc.c +++ b/drivers/mfd/macsmc.c @@ -45,8 +45,12 @@ #define SMC_TIMEOUT_MS 500 static const struct mfd_cell apple_smc_devs[] = { + MFD_CELL_NAME("macsmc-input"), + MFD_CELL_NAME("macsmc-power"), MFD_CELL_OF("macsmc-gpio", NULL, NULL, 0, 0, "apple,smc-gpio"), + MFD_CELL_OF("macsmc-hwmon", NULL, NULL, 0, 0, "apple,smc-hwmon"), MFD_CELL_OF("macsmc-reboot", NULL, NULL, 0, 0, "apple,smc-reboot"), + MFD_CELL_OF("macsmc-rtc", NULL, NULL, 0, 0, "apple,smc-rtc"), }; static int apple_smc_cmd_locked(struct apple_smc *smc, u64 cmd, u64 arg, @@ -413,6 +417,7 @@ static int apple_smc_probe(struct platform_device *pdev) if (!smc) return -ENOMEM; + mutex_init(&smc->mutex); smc->dev = &pdev->dev; smc->sram_base = devm_platform_get_and_ioremap_resource(pdev, 1, &smc->sram); if (IS_ERR(smc->sram_base)) diff --git a/drivers/mfd/mfd-core.c b/drivers/mfd/mfd-core.c index 7d14a1e7631ee8..c55223ce4327a5 100644 --- a/drivers/mfd/mfd-core.c +++ b/drivers/mfd/mfd-core.c @@ -22,6 +22,7 @@ #include static LIST_HEAD(mfd_of_node_list); +static DEFINE_MUTEX(mfd_of_node_mutex); struct mfd_of_node_entry { struct list_head list; @@ -105,9 +106,11 @@ static int mfd_match_of_node_to_dev(struct platform_device *pdev, u64 of_node_addr; /* Skip if OF node has previously been allocated to a device */ - list_for_each_entry(of_entry, &mfd_of_node_list, list) - if (of_entry->np == np) - return -EAGAIN; + scoped_guard(mutex, &mfd_of_node_mutex) { + list_for_each_entry(of_entry, &mfd_of_node_list, list) + if (of_entry->np == np) + return -EAGAIN; + } if (!cell->use_of_reg) /* No of_reg defined - allocate first free compatible match */ @@ -129,7 +132,8 @@ static int mfd_match_of_node_to_dev(struct platform_device *pdev, of_entry->dev = &pdev->dev; of_entry->np = np; - list_add_tail(&of_entry->list, &mfd_of_node_list); + scoped_guard(mutex, &mfd_of_node_mutex) + list_add_tail(&of_entry->list, &mfd_of_node_list); of_node_get(np); device_set_node(&pdev->dev, of_fwnode_handle(np)); @@ -286,11 +290,13 @@ static int mfd_add_device(struct device *parent, int id, if (cell->swnode) device_remove_software_node(&pdev->dev); fail_of_entry: - list_for_each_entry_safe(of_entry, tmp, &mfd_of_node_list, list) - if (of_entry->dev == &pdev->dev) { - list_del(&of_entry->list); - kfree(of_entry); - } + scoped_guard(mutex, &mfd_of_node_mutex) { + list_for_each_entry_safe(of_entry, tmp, &mfd_of_node_list, list) + if (of_entry->dev == &pdev->dev) { + list_del(&of_entry->list); + kfree(of_entry); + } + } fail_alias: regulator_bulk_unregister_supply_alias(&pdev->dev, cell->parent_supplies, @@ -360,11 +366,13 @@ static int mfd_remove_devices_fn(struct device *dev, void *data) if (cell->swnode) device_remove_software_node(&pdev->dev); - list_for_each_entry_safe(of_entry, tmp, &mfd_of_node_list, list) - if (of_entry->dev == &pdev->dev) { - list_del(&of_entry->list); - kfree(of_entry); - } + scoped_guard(mutex, &mfd_of_node_mutex) { + list_for_each_entry_safe(of_entry, tmp, &mfd_of_node_list, list) + if (of_entry->dev == &pdev->dev) { + list_del(&of_entry->list); + kfree(of_entry); + } + } regulator_bulk_unregister_supply_alias(dev, cell->parent_supplies, cell->num_parent_supplies); diff --git a/drivers/mfd/omap-usb-host.c b/drivers/mfd/omap-usb-host.c index a77b6fc790f2e0..4d29a6e2ed87ab 100644 --- a/drivers/mfd/omap-usb-host.c +++ b/drivers/mfd/omap-usb-host.c @@ -819,8 +819,10 @@ static void usbhs_omap_remove(struct platform_device *pdev) { pm_runtime_disable(&pdev->dev); - /* remove children */ - device_for_each_child(&pdev->dev, NULL, usbhs_omap_remove_child); + if (pdev->dev.of_node) + of_platform_depopulate(&pdev->dev); + else + device_for_each_child(&pdev->dev, NULL, usbhs_omap_remove_child); } static const struct dev_pm_ops usbhsomap_dev_pm_ops = { diff --git a/drivers/mfd/qcom-pm8xxx.c b/drivers/mfd/qcom-pm8xxx.c index 1149f7102a3656..0cf374c015ce7e 100644 --- a/drivers/mfd/qcom-pm8xxx.c +++ b/drivers/mfd/qcom-pm8xxx.c @@ -577,17 +577,11 @@ static int pm8xxx_probe(struct platform_device *pdev) return rc; } -static int pm8xxx_remove_child(struct device *dev, void *unused) -{ - platform_device_unregister(to_platform_device(dev)); - return 0; -} - static void pm8xxx_remove(struct platform_device *pdev) { struct pm_irq_chip *chip = platform_get_drvdata(pdev); - device_for_each_child(&pdev->dev, NULL, pm8xxx_remove_child); + of_platform_depopulate(&pdev->dev); irq_domain_remove(chip->irqdomain); } diff --git a/drivers/mfd/sec-irq.c b/drivers/mfd/sec-irq.c index 74ac70002d1fc5..ff2671186e89f2 100644 --- a/drivers/mfd/sec-irq.c +++ b/drivers/mfd/sec-irq.c @@ -198,6 +198,7 @@ static const struct regmap_irq_chip s2mpg10_irq_chip = { static const struct regmap_irq_chip s2mpg10_irq_chip_pmic = { .name = "s2mpg10-pmic", + .domain_suffix = "pmic", .status_base = S2MPG10_PMIC_INT1, .mask_base = S2MPG10_PMIC_INT1M, .num_regs = 6, diff --git a/drivers/mfd/simple-mfd-i2c.c b/drivers/mfd/simple-mfd-i2c.c index 8b751d8e3b5ae4..7315fad618e445 100644 --- a/drivers/mfd/simple-mfd-i2c.c +++ b/drivers/mfd/simple-mfd-i2c.c @@ -116,6 +116,7 @@ static const struct simple_mfd_data spacemit_p1 = { }; static const struct of_device_id simple_mfd_i2c_of_match[] = { + { .compatible = "delta,tn48m-cpld" }, { .compatible = "fsl,ls1028aqds-fpga" }, { .compatible = "fsl,lx2160aqds-fpga" }, { .compatible = "fsl,lx2160ardb-fpga" }, diff --git a/drivers/mfd/tps65219.c b/drivers/mfd/tps65219.c index 65a952555218dc..7275dcdb7c44fc 100644 --- a/drivers/mfd/tps65219.c +++ b/drivers/mfd/tps65219.c @@ -498,6 +498,15 @@ static int tps65219_probe(struct i2c_client *client) return ret; } + if (chip_id == TPS65214) { + ret = i2c_smbus_write_byte_data(client, TPS65214_REG_LOCK, + TPS65214_LOCK_ACCESS_CMD); + if (ret) { + dev_err(tps->dev, "Failed to unlock registers %d\n", ret); + return ret; + } + } + ret = devm_regmap_add_irq_chip(tps->dev, tps->regmap, client->irq, IRQF_ONESHOT, 0, pmic->irq_chip, &tps->irq_data); diff --git a/drivers/misc/bcm-vk/bcm_vk_msg.c b/drivers/misc/bcm-vk/bcm_vk_msg.c index 1f42d1d5a630a2..665a3888708ac7 100644 --- a/drivers/misc/bcm-vk/bcm_vk_msg.c +++ b/drivers/misc/bcm-vk/bcm_vk_msg.c @@ -1010,6 +1010,9 @@ ssize_t bcm_vk_read(struct file *p_file, struct device *dev = &vk->pdev->dev; struct bcm_vk_msg_chan *chan = &vk->to_h_msg_chan; struct bcm_vk_wkent *entry = NULL, *iter; + struct vk_msg_blk tmp_msg; + u32 tmp_usr_msg_id; + u32 tmp_blks; u32 q_num; u32 rsp_length; @@ -1034,6 +1037,9 @@ ssize_t bcm_vk_read(struct file *p_file, entry = iter; } else { /* buffer not big enough */ + tmp_msg = iter->to_h_msg[0]; + tmp_usr_msg_id = iter->usr_msg_id; + tmp_blks = iter->to_h_blks; rc = -EMSGSIZE; } goto read_loop_exit; @@ -1052,14 +1058,12 @@ ssize_t bcm_vk_read(struct file *p_file, bcm_vk_free_wkent(dev, entry); } else if (rc == -EMSGSIZE) { - struct vk_msg_blk tmp_msg = entry->to_h_msg[0]; - /* * in this case, return just the first block, so * that app knows what size it is looking for. */ - set_msg_id(&tmp_msg, entry->usr_msg_id); - tmp_msg.size = entry->to_h_blks - 1; + set_msg_id(&tmp_msg, tmp_usr_msg_id); + tmp_msg.size = tmp_blks - 1; if (copy_to_user(buf, &tmp_msg, VK_MSGQ_BLK_SIZE) != 0) { dev_err(dev, "Error return 1st block in -EMSGSIZE\n"); rc = -EFAULT; diff --git a/drivers/misc/eeprom/eeprom_93xx46.c b/drivers/misc/eeprom/eeprom_93xx46.c index 9cae6f530679b3..5230e910a1d11e 100644 --- a/drivers/misc/eeprom/eeprom_93xx46.c +++ b/drivers/misc/eeprom/eeprom_93xx46.c @@ -45,6 +45,7 @@ struct eeprom_93xx46_platform_data { #define OP_START 0x4 #define OP_WRITE (OP_START | 0x1) #define OP_READ (OP_START | 0x2) +/* The following addresses are offset for the 1K EEPROM variant in 16-bit mode */ #define ADDR_EWDS 0x00 #define ADDR_ERAL 0x20 #define ADDR_EWEN 0x30 @@ -191,10 +192,7 @@ static int eeprom_93xx46_ew(struct eeprom_93xx46_dev *edev, int is_on) bits = edev->addrlen + 3; cmd_addr = OP_START << edev->addrlen; - if (edev->pdata->flags & EE_ADDR8) - cmd_addr |= (is_on ? ADDR_EWEN : ADDR_EWDS) << 1; - else - cmd_addr |= (is_on ? ADDR_EWEN : ADDR_EWDS); + cmd_addr |= (is_on ? ADDR_EWEN : ADDR_EWDS) << (edev->addrlen - 6); if (has_quirk_instruction_length(edev)) { cmd_addr <<= 2; @@ -328,10 +326,7 @@ static int eeprom_93xx46_eral(struct eeprom_93xx46_dev *edev) bits = edev->addrlen + 3; cmd_addr = OP_START << edev->addrlen; - if (edev->pdata->flags & EE_ADDR8) - cmd_addr |= ADDR_ERAL << 1; - else - cmd_addr |= ADDR_ERAL; + cmd_addr |= ADDR_ERAL << (edev->addrlen - 6); if (has_quirk_instruction_length(edev)) { cmd_addr <<= 2; diff --git a/drivers/misc/fastrpc.c b/drivers/misc/fastrpc.c index ee652ef01534a8..83b0ddfbd5c928 100644 --- a/drivers/misc/fastrpc.c +++ b/drivers/misc/fastrpc.c @@ -1370,6 +1370,7 @@ static int fastrpc_init_create_static_process(struct fastrpc_user *fl, } err_map: fastrpc_buf_free(fl->cctx->remote_heap); + fl->cctx->remote_heap = NULL; err_name: kfree(name); err: @@ -2337,8 +2338,10 @@ static int fastrpc_rpmsg_probe(struct rpmsg_device *rpdev) if (!err) { src_perms = BIT(QCOM_SCM_VMID_HLOS); - qcom_scm_assign_mem(res.start, resource_size(&res), &src_perms, + err = qcom_scm_assign_mem(res.start, resource_size(&res), &src_perms, data->vmperms, data->vmcount); + if (err) + goto err_free_data; } } diff --git a/drivers/misc/mei/hw-me.c b/drivers/misc/mei/hw-me.c index d4612c6597844f..1e4a41ac428fd8 100644 --- a/drivers/misc/mei/hw-me.c +++ b/drivers/misc/mei/hw-me.c @@ -1337,19 +1337,13 @@ irqreturn_t mei_me_irq_thread_handler(int irq, void *dev_id) /* check if we need to start the dev */ if (!mei_host_is_ready(dev)) { if (mei_hw_is_ready(dev)) { - /* synchronized by dev mutex */ - if (waitqueue_active(&dev->wait_hw_ready)) { - dev_dbg(&dev->dev, "we need to start the dev.\n"); - dev->recvd_hw_ready = true; - wake_up(&dev->wait_hw_ready); - } else if (dev->dev_state != MEI_DEV_UNINITIALIZED && - dev->dev_state != MEI_DEV_POWERING_DOWN && - dev->dev_state != MEI_DEV_POWER_DOWN) { + if (dev->dev_state == MEI_DEV_ENABLED) { dev_dbg(&dev->dev, "Force link reset.\n"); schedule_work(&dev->reset_work); } else { - dev_dbg(&dev->dev, "Ignore this interrupt in state = %d\n", - dev->dev_state); + dev_dbg(&dev->dev, "we need to start the dev.\n"); + dev->recvd_hw_ready = true; + wake_up(&dev->wait_hw_ready); } } else { dev_dbg(&dev->dev, "Spurious Interrupt\n"); diff --git a/drivers/misc/ti_fpc202.c b/drivers/misc/ti_fpc202.c index 7964e46c744829..8eb2b5ac985062 100644 --- a/drivers/misc/ti_fpc202.c +++ b/drivers/misc/ti_fpc202.c @@ -309,7 +309,6 @@ static void fpc202_remove_port(struct fpc202_priv *priv, int port_id) static int fpc202_probe(struct i2c_client *client) { struct device *dev = &client->dev; - struct device_node *i2c_handle; struct fpc202_priv *priv; int ret, port_id; @@ -357,7 +356,7 @@ static int fpc202_probe(struct i2c_client *client) bitmap_zero(priv->probed_ports, FPC202_NUM_PORTS); - for_each_child_of_node(dev->of_node, i2c_handle) { + for_each_child_of_node_scoped(dev->of_node, i2c_handle) { ret = of_property_read_u32(i2c_handle, "reg", &port_id); if (ret) { if (ret == -EINVAL) diff --git a/drivers/mmc/host/dw_mmc-rockchip.c b/drivers/mmc/host/dw_mmc-rockchip.c index 62c68cda1e2144..ff6a52d85e5201 100644 --- a/drivers/mmc/host/dw_mmc-rockchip.c +++ b/drivers/mmc/host/dw_mmc-rockchip.c @@ -36,6 +36,8 @@ struct dw_mci_rockchip_priv_data { int default_sample_phase; int num_phases; bool internal_phase; + int sample_phase; + int drv_phase; }; /* @@ -574,9 +576,43 @@ static void dw_mci_rockchip_remove(struct platform_device *pdev) dw_mci_pltfm_remove(pdev); } +static int dw_mci_rockchip_runtime_suspend(struct device *dev) +{ + struct platform_device *pdev = to_platform_device(dev); + struct dw_mci *host = platform_get_drvdata(pdev); + struct dw_mci_rockchip_priv_data *priv = host->priv; + + if (priv->internal_phase) { + priv->sample_phase = rockchip_mmc_get_phase(host, true); + priv->drv_phase = rockchip_mmc_get_phase(host, false); + } + + return dw_mci_runtime_suspend(dev); +} + +static int dw_mci_rockchip_runtime_resume(struct device *dev) +{ + struct platform_device *pdev = to_platform_device(dev); + struct dw_mci *host = platform_get_drvdata(pdev); + struct dw_mci_rockchip_priv_data *priv = host->priv; + int ret; + + ret = dw_mci_runtime_resume(dev); + if (ret) + return ret; + + if (priv->internal_phase) { + rockchip_mmc_set_phase(host, true, priv->sample_phase); + rockchip_mmc_set_phase(host, false, priv->drv_phase); + mci_writel(host, MISC_CON, MEM_CLK_AUTOGATE_ENABLE); + } + + return ret; +} + static const struct dev_pm_ops dw_mci_rockchip_dev_pm_ops = { SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, pm_runtime_force_resume) - RUNTIME_PM_OPS(dw_mci_runtime_suspend, dw_mci_runtime_resume, NULL) + RUNTIME_PM_OPS(dw_mci_rockchip_runtime_suspend, dw_mci_rockchip_runtime_resume, NULL) }; static struct platform_driver dw_mci_rockchip_pltfm_driver = { diff --git a/drivers/mmc/host/mmci_qcom_dml.c b/drivers/mmc/host/mmci_qcom_dml.c index 3da6112fbe39db..67371389cc3311 100644 --- a/drivers/mmc/host/mmci_qcom_dml.c +++ b/drivers/mmc/host/mmci_qcom_dml.c @@ -109,6 +109,7 @@ static int of_get_dml_pipe_index(struct device_node *np, const char *name) &dma_spec)) return -ENODEV; + of_node_put(dma_spec.np); if (dma_spec.args_count) return dma_spec.args[0]; diff --git a/drivers/mmc/host/sdhci-brcmstb.c b/drivers/mmc/host/sdhci-brcmstb.c index c9442499876c32..57e45951644e30 100644 --- a/drivers/mmc/host/sdhci-brcmstb.c +++ b/drivers/mmc/host/sdhci-brcmstb.c @@ -116,7 +116,7 @@ static void sdhci_brcmstb_restore_regs(struct mmc_host *mmc, enum cfg_core_ver v writel(sr->boot_main_ctl, priv->boot_regs + SDIO_BOOT_MAIN_CTL); if (ver == SDIO_CFG_CORE_V1) { - writel(sr->sd_pin_sel, cr + SDIO_CFG_SD_PIN_SEL); + writel(sr->sd_pin_sel, cr + SDIO_CFG_V1_SD_PIN_SEL); return; } diff --git a/drivers/mmc/host/sdhci-pci-core.c b/drivers/mmc/host/sdhci-pci-core.c index 47a0a738862b58..93122133202367 100644 --- a/drivers/mmc/host/sdhci-pci-core.c +++ b/drivers/mmc/host/sdhci-pci-core.c @@ -27,6 +27,7 @@ #include #include #include +#include #include #include @@ -2129,6 +2130,7 @@ static struct sdhci_pci_slot *sdhci_pci_probe_slot( struct sdhci_host *host; int ret, bar = first_bar + slotno; size_t priv_size = chip->fixes ? chip->fixes->priv_size : 0; + u32 cd_debounce_delay_ms; if (!(pci_resource_flags(pdev, bar) & IORESOURCE_MEM)) { dev_err(&pdev->dev, "BAR %d is not iomem. Aborting.\n", bar); @@ -2195,6 +2197,10 @@ static struct sdhci_pci_slot *sdhci_pci_probe_slot( if (host->mmc->caps & MMC_CAP_CD_WAKE) device_init_wakeup(&pdev->dev, true); + if (device_property_read_u32(&pdev->dev, "cd-debounce-delay-ms", + &cd_debounce_delay_ms)) + cd_debounce_delay_ms = 200; + if (slot->cd_idx >= 0) { struct gpiod_lookup_table *cd_gpio_lookup_table; @@ -2213,7 +2219,7 @@ static struct sdhci_pci_slot *sdhci_pci_probe_slot( ret = mmc_gpiod_request_cd(host->mmc, NULL, slot->cd_idx, slot->cd_override_level, - 0); + cd_debounce_delay_ms * 1000); if (ret == -EPROBE_DEFER) goto remove; @@ -2221,6 +2227,16 @@ static struct sdhci_pci_slot *sdhci_pci_probe_slot( dev_warn(&pdev->dev, "failed to setup card detect gpio\n"); slot->cd_idx = -1; } + } else if (is_of_node(pdev->dev.fwnode)) { + /* Allow all OF systems to use a CD GPIO if provided */ + + ret = mmc_gpiod_request_cd(host->mmc, "cd", 0, + slot->cd_override_level, + cd_debounce_delay_ms * 1000); + if (ret == -EPROBE_DEFER) + goto remove; + else if (ret == 0) + slot->cd_idx = 0; } if (chip->fixes && chip->fixes->add_host) diff --git a/drivers/mmc/host/sdhci-pci-gli.c b/drivers/mmc/host/sdhci-pci-gli.c index b0f91cc9e40e43..6e4084407662a1 100644 --- a/drivers/mmc/host/sdhci-pci-gli.c +++ b/drivers/mmc/host/sdhci-pci-gli.c @@ -68,6 +68,9 @@ #define GLI_9750_MISC_TX1_DLY_VALUE 0x5 #define SDHCI_GLI_9750_MISC_SSC_OFF BIT(26) +#define SDHCI_GLI_9750_GM_BURST_SIZE 0x510 +#define SDHCI_GLI_9750_GM_BURST_SIZE_R_OSRC_LMT GENMASK(17, 16) + #define SDHCI_GLI_9750_TUNING_CONTROL 0x540 #define SDHCI_GLI_9750_TUNING_CONTROL_EN BIT(4) #define GLI_9750_TUNING_CONTROL_EN_ON 0x1 @@ -345,10 +348,16 @@ static void gli_set_9750(struct sdhci_host *host) u32 misc_value; u32 parameter_value; u32 control_value; + u32 burst_value; u16 ctrl2; gl9750_wt_on(host); + /* clear R_OSRC_Lmt to avoid DMA write corruption */ + burst_value = sdhci_readl(host, SDHCI_GLI_9750_GM_BURST_SIZE); + burst_value &= ~SDHCI_GLI_9750_GM_BURST_SIZE_R_OSRC_LMT; + sdhci_writel(host, burst_value, SDHCI_GLI_9750_GM_BURST_SIZE); + driving_value = sdhci_readl(host, SDHCI_GLI_9750_DRIVING); pll_value = sdhci_readl(host, SDHCI_GLI_9750_PLL); sw_ctrl_value = sdhci_readl(host, SDHCI_GLI_9750_SW_CTRL); diff --git a/drivers/mmc/host/sdhci.c b/drivers/mmc/host/sdhci.c index ac7e11f37af71f..fec9329e1edbed 100644 --- a/drivers/mmc/host/sdhci.c +++ b/drivers/mmc/host/sdhci.c @@ -4532,8 +4532,15 @@ int sdhci_setup_host(struct sdhci_host *host) * their platform code before calling sdhci_add_host(), and we * won't assume 8-bit width for hosts without that CAP. */ - if (!(host->quirks & SDHCI_QUIRK_FORCE_1_BIT_DATA)) + if (host->quirks & SDHCI_QUIRK_FORCE_1_BIT_DATA) { + host->caps1 &= ~(SDHCI_SUPPORT_SDR104 | SDHCI_SUPPORT_SDR50 | SDHCI_SUPPORT_DDR50); + if (host->quirks2 & SDHCI_QUIRK2_CAPS_BIT63_FOR_HS400) + host->caps1 &= ~SDHCI_SUPPORT_HS400; + mmc->caps2 &= ~(MMC_CAP2_HS200 | MMC_CAP2_HS400 | MMC_CAP2_HS400_ES); + mmc->caps &= ~(MMC_CAP_DDR | MMC_CAP_UHS); + } else { mmc->caps |= MMC_CAP_4_BIT_DATA; + } if (host->quirks2 & SDHCI_QUIRK2_HOST_NO_CMD23) mmc->caps &= ~MMC_CAP_CMD23; diff --git a/drivers/mmc/host/vub300.c b/drivers/mmc/host/vub300.c index ff49d0770506f7..3c9df27f9fa74d 100644 --- a/drivers/mmc/host/vub300.c +++ b/drivers/mmc/host/vub300.c @@ -369,11 +369,14 @@ struct vub300_mmc_host { static void vub300_delete(struct kref *kref) { /* kref callback - softirq */ struct vub300_mmc_host *vub300 = kref_to_vub300_mmc_host(kref); + struct mmc_host *mmc = vub300->mmc; + usb_free_urb(vub300->command_out_urb); vub300->command_out_urb = NULL; usb_free_urb(vub300->command_res_urb); vub300->command_res_urb = NULL; usb_put_dev(vub300->udev); + mmc_free_host(mmc); /* * and hence also frees vub300 * which is contained at the end of struct mmc @@ -2112,7 +2115,7 @@ static int vub300_probe(struct usb_interface *interface, goto error1; } /* this also allocates memory for our VUB300 mmc host device */ - mmc = devm_mmc_alloc_host(&udev->dev, sizeof(*vub300)); + mmc = mmc_alloc_host(sizeof(*vub300), &udev->dev); if (!mmc) { retval = -ENOMEM; dev_err(&udev->dev, "not enough memory for the mmc_host\n"); @@ -2269,7 +2272,7 @@ static int vub300_probe(struct usb_interface *interface, dev_err(&vub300->udev->dev, "Could not find two sets of bulk-in/out endpoint pairs\n"); retval = -EINVAL; - goto error4; + goto err_free_host; } retval = usb_control_msg(vub300->udev, usb_rcvctrlpipe(vub300->udev, 0), @@ -2278,14 +2281,14 @@ static int vub300_probe(struct usb_interface *interface, 0x0000, 0x0000, &vub300->hc_info, sizeof(vub300->hc_info), 1000); if (retval < 0) - goto error4; + goto err_free_host; retval = usb_control_msg(vub300->udev, usb_sndctrlpipe(vub300->udev, 0), SET_ROM_WAIT_STATES, USB_DIR_OUT | USB_TYPE_VENDOR | USB_RECIP_DEVICE, firmware_rom_wait_states, 0x0000, NULL, 0, 1000); if (retval < 0) - goto error4; + goto err_free_host; dev_info(&vub300->udev->dev, "operating_mode = %s %s %d MHz %s %d byte USB packets\n", (mmc->caps & MMC_CAP_SDIO_IRQ) ? "IRQs" : "POLL", @@ -2300,7 +2303,7 @@ static int vub300_probe(struct usb_interface *interface, 0x0000, 0x0000, &vub300->system_port_status, sizeof(vub300->system_port_status), 1000); if (retval < 0) { - goto error4; + goto err_free_host; } else if (sizeof(vub300->system_port_status) == retval) { vub300->card_present = (0x0001 & vub300->system_port_status.port_flags) ? 1 : 0; @@ -2308,7 +2311,7 @@ static int vub300_probe(struct usb_interface *interface, (0x0010 & vub300->system_port_status.port_flags) ? 1 : 0; } else { retval = -EINVAL; - goto error4; + goto err_free_host; } usb_set_intfdata(interface, vub300); INIT_DELAYED_WORK(&vub300->pollwork, vub300_pollwork_thread); @@ -2338,6 +2341,8 @@ static int vub300_probe(struct usb_interface *interface, return 0; error6: timer_delete_sync(&vub300->inactivity_timer); +err_free_host: + mmc_free_host(mmc); /* * and hence also frees vub300 * which is contained at the end of struct mmc @@ -2365,8 +2370,8 @@ static void vub300_disconnect(struct usb_interface *interface) usb_set_intfdata(interface, NULL); /* prevent more I/O from starting */ vub300->interface = NULL; - kref_put(&vub300->kref, vub300_delete); mmc_remove_host(mmc); + kref_put(&vub300->kref, vub300_delete); pr_info("USB vub300 remote SDIO host controller[%d]" " now disconnected", ifnum); return; diff --git a/drivers/most/core.c b/drivers/most/core.c index da319d108ea1df..40d63e38fef544 100644 --- a/drivers/most/core.c +++ b/drivers/most/core.c @@ -1282,19 +1282,28 @@ int most_register_interface(struct most_interface *iface) int id; struct most_channel *c; - if (!iface || !iface->enqueue || !iface->configure || - !iface->poison_channel || (iface->num_channels > MAX_CHANNELS)) + if (!iface) return -EINVAL; + device_initialize(iface->dev); + + if (!iface->enqueue || !iface->configure || !iface->poison_channel || + (iface->num_channels > MAX_CHANNELS)) { + put_device(iface->dev); + return -EINVAL; + } + id = ida_alloc(&mdev_id, GFP_KERNEL); if (id < 0) { dev_err(iface->dev, "Failed to allocate device ID\n"); + put_device(iface->dev); return id; } iface->p = kzalloc(sizeof(*iface->p), GFP_KERNEL); if (!iface->p) { ida_free(&mdev_id, id); + put_device(iface->dev); return -ENOMEM; } @@ -1304,7 +1313,7 @@ int most_register_interface(struct most_interface *iface) iface->dev->bus = &mostbus; iface->dev->groups = interface_attr_groups; dev_set_drvdata(iface->dev, iface); - if (device_register(iface->dev)) { + if (device_add(iface->dev)) { dev_err(iface->dev, "Failed to register interface device\n"); kfree(iface->p); put_device(iface->dev); diff --git a/drivers/mtd/devices/mtd_intel_dg.c b/drivers/mtd/devices/mtd_intel_dg.c index 2bab30dcd35fd0..7f751c48a76d4c 100644 --- a/drivers/mtd/devices/mtd_intel_dg.c +++ b/drivers/mtd/devices/mtd_intel_dg.c @@ -770,6 +770,7 @@ static int intel_dg_mtd_probe(struct auxiliary_device *aux_dev, kref_init(&nvm->refcnt); mutex_init(&nvm->lock); + nvm->nregions = nregions; for (n = 0, i = 0; i < INTEL_DG_NVM_REGIONS; i++) { if (!invm->regions[i].name) @@ -777,13 +778,15 @@ static int intel_dg_mtd_probe(struct auxiliary_device *aux_dev, char *name = kasprintf(GFP_KERNEL, "%s.%s", dev_name(&aux_dev->dev), invm->regions[i].name); - if (!name) - continue; + if (!name) { + ret = -ENOMEM; + goto err; + } + nvm->regions[n].name = name; nvm->regions[n].id = i; n++; } - nvm->nregions = n; /* in case where kasprintf fail */ ret = devm_pm_runtime_enable(device); if (ret < 0) { diff --git a/drivers/mtd/nand/raw/brcmnand/brcmnand.c b/drivers/mtd/nand/raw/brcmnand/brcmnand.c index 835653bdd5abcc..8f4d001377a1c3 100644 --- a/drivers/mtd/nand/raw/brcmnand/brcmnand.c +++ b/drivers/mtd/nand/raw/brcmnand/brcmnand.c @@ -2350,14 +2350,12 @@ static int brcmnand_write(struct mtd_info *mtd, struct nand_chip *chip, for (i = 0; i < ctrl->max_oob; i += 4) oob_reg_write(ctrl, i, 0xffffffff); - if (mtd->oops_panic_write) + if (mtd->oops_panic_write) { /* switch to interrupt polling and PIO mode */ disable_ctrl_irqs(ctrl); - - if (use_dma(ctrl) && (has_edu(ctrl) || !oob) && flash_dma_buf_ok(buf)) { + } else if (use_dma(ctrl) && (has_edu(ctrl) || !oob) && flash_dma_buf_ok(buf)) { if (ctrl->dma_trans(host, addr, (u32 *)buf, oob, mtd->writesize, CMD_PROGRAM_PAGE)) - ret = -EIO; goto out; diff --git a/drivers/mtd/nand/raw/cadence-nand-controller.c b/drivers/mtd/nand/raw/cadence-nand-controller.c index 5f037753f78c82..d53b35a8b3cb24 100644 --- a/drivers/mtd/nand/raw/cadence-nand-controller.c +++ b/drivers/mtd/nand/raw/cadence-nand-controller.c @@ -1066,7 +1066,7 @@ static int cadence_nand_cdma_send(struct cdns_nand_ctrl *cdns_ctrl, } /* Send SDMA command and wait for finish. */ -static u32 +static int cadence_nand_cdma_send_and_wait(struct cdns_nand_ctrl *cdns_ctrl, u8 thread) { @@ -3133,7 +3133,7 @@ static int cadence_nand_init(struct cdns_nand_ctrl *cdns_ctrl) sizeof(*cdns_ctrl->cdma_desc), &cdns_ctrl->dma_cdma_desc, GFP_KERNEL); - if (!cdns_ctrl->dma_cdma_desc) + if (!cdns_ctrl->cdma_desc) return -ENOMEM; cdns_ctrl->buf_size = SZ_16K; diff --git a/drivers/mtd/nand/raw/nand_base.c b/drivers/mtd/nand/raw/nand_base.c index f2322de93ab41a..19e3bbf42931d6 100644 --- a/drivers/mtd/nand/raw/nand_base.c +++ b/drivers/mtd/nand/raw/nand_base.c @@ -4737,11 +4737,16 @@ static void nand_shutdown(struct mtd_info *mtd) static int nand_lock(struct mtd_info *mtd, loff_t ofs, uint64_t len) { struct nand_chip *chip = mtd_to_nand(mtd); + int ret; if (!chip->ops.lock_area) return -ENOTSUPP; - return chip->ops.lock_area(chip, ofs, len); + nand_get_device(chip); + ret = chip->ops.lock_area(chip, ofs, len); + nand_release_device(chip); + + return ret; } /** @@ -4753,11 +4758,16 @@ static int nand_lock(struct mtd_info *mtd, loff_t ofs, uint64_t len) static int nand_unlock(struct mtd_info *mtd, loff_t ofs, uint64_t len) { struct nand_chip *chip = mtd_to_nand(mtd); + int ret; if (!chip->ops.unlock_area) return -ENOTSUPP; - return chip->ops.unlock_area(chip, ofs, len); + nand_get_device(chip); + ret = chip->ops.unlock_area(chip, ofs, len); + nand_release_device(chip); + + return ret; } /* Set default functions */ diff --git a/drivers/mtd/nand/raw/pl35x-nand-controller.c b/drivers/mtd/nand/raw/pl35x-nand-controller.c index 11bd90e3f18cb0..50d4305729f4ad 100644 --- a/drivers/mtd/nand/raw/pl35x-nand-controller.c +++ b/drivers/mtd/nand/raw/pl35x-nand-controller.c @@ -862,6 +862,9 @@ static int pl35x_nfc_setup_interface(struct nand_chip *chip, int cs, PL35X_SMC_NAND_TAR_CYCLES(tmgs.t_ar) | PL35X_SMC_NAND_TRR_CYCLES(tmgs.t_rr); + writel(plnand->timings, nfc->conf_regs + PL35X_SMC_CYCLES); + pl35x_smc_update_regs(nfc); + return 0; } @@ -976,6 +979,7 @@ static int pl35x_nand_attach_chip(struct nand_chip *chip) fallthrough; case NAND_ECC_ENGINE_TYPE_NONE: case NAND_ECC_ENGINE_TYPE_SOFT: + chip->ecc.write_page_raw = nand_monolithic_write_page_raw; break; case NAND_ECC_ENGINE_TYPE_ON_HOST: ret = pl35x_nand_init_hw_ecc_controller(nfc, chip); diff --git a/drivers/mtd/nand/spi/core.c b/drivers/mtd/nand/spi/core.c index d207286572d878..9540fd04156c7a 100644 --- a/drivers/mtd/nand/spi/core.c +++ b/drivers/mtd/nand/spi/core.c @@ -859,6 +859,14 @@ static void spinand_cont_read_init(struct spinand_device *spinand) (engine_type == NAND_ECC_ENGINE_TYPE_ON_DIE || engine_type == NAND_ECC_ENGINE_TYPE_NONE)) { spinand->cont_read_possible = true; + + /* + * Ensure continuous read is disabled on probe. + * Some devices retain this state across soft reset, + * which leaves the OOB area inaccessible and results + * in false positive returns from spinand_isbad(). + */ + spinand_cont_read_enable(spinand, false); } } diff --git a/drivers/mtd/parsers/ofpart_core.c b/drivers/mtd/parsers/ofpart_core.c index abfa687989182f..09961c6f39496e 100644 --- a/drivers/mtd/parsers/ofpart_core.c +++ b/drivers/mtd/parsers/ofpart_core.c @@ -77,6 +77,7 @@ static int parse_fixed_partitions(struct mtd_info *master, of_id = of_match_node(parse_ofpart_match_table, ofpart_node); if (dedicated && !of_id) { /* The 'partitions' subnode might be used by another parser */ + of_node_put(ofpart_node); return 0; } @@ -91,12 +92,18 @@ static int parse_fixed_partitions(struct mtd_info *master, nr_parts++; } - if (nr_parts == 0) + if (nr_parts == 0) { + if (dedicated) + of_node_put(ofpart_node); return 0; + } parts = kcalloc(nr_parts, sizeof(*parts), GFP_KERNEL); - if (!parts) + if (!parts) { + if (dedicated) + of_node_put(ofpart_node); return -ENOMEM; + } i = 0; for_each_child_of_node(ofpart_node, pp) { @@ -175,6 +182,9 @@ static int parse_fixed_partitions(struct mtd_info *master, if (quirks && quirks->post_parse) quirks->post_parse(master, parts, nr_parts); + if (dedicated) + of_node_put(ofpart_node); + *pparts = parts; return nr_parts; @@ -183,6 +193,8 @@ static int parse_fixed_partitions(struct mtd_info *master, master->name, pp, mtd_node); ret = -EINVAL; ofpart_none: + if (dedicated) + of_node_put(ofpart_node); of_node_put(pp); kfree(parts); return ret; diff --git a/drivers/mtd/parsers/redboot.c b/drivers/mtd/parsers/redboot.c index 3b55b676ca6b9c..c06ba7a2a34b4e 100644 --- a/drivers/mtd/parsers/redboot.c +++ b/drivers/mtd/parsers/redboot.c @@ -270,9 +270,9 @@ static int parse_redboot_partitions(struct mtd_info *master, strcpy(names, fl->img->name); #ifdef CONFIG_MTD_REDBOOT_PARTS_READONLY - if (!memcmp(names, "RedBoot", 8) || - !memcmp(names, "RedBoot config", 15) || - !memcmp(names, "FIS directory", 14)) { + if (!strcmp(names, "RedBoot") || + !strcmp(names, "RedBoot config") || + !strcmp(names, "FIS directory")) { parts[i].mask_flags = MTD_WRITEABLE; } #endif diff --git a/drivers/mtd/parsers/tplink_safeloader.c b/drivers/mtd/parsers/tplink_safeloader.c index e358a029dc70cf..4fcaf92d22e4fe 100644 --- a/drivers/mtd/parsers/tplink_safeloader.c +++ b/drivers/mtd/parsers/tplink_safeloader.c @@ -116,6 +116,7 @@ static int mtd_parser_tplink_safeloader_parse(struct mtd_info *mtd, return idx; err_free: + kfree(buf); for (idx -= 1; idx >= 0; idx--) kfree(parts[idx].name); err_free_parts: diff --git a/drivers/mtd/spi-nor/core.c b/drivers/mtd/spi-nor/core.c index d3f8a78efd3bf9..1f2e312feec787 100644 --- a/drivers/mtd/spi-nor/core.c +++ b/drivers/mtd/spi-nor/core.c @@ -2466,7 +2466,7 @@ spi_nor_spimem_adjust_hwcaps(struct spi_nor *nor, u32 *hwcaps) spi_nor_spimem_setup_op(nor, &op, nor->reg_proto); - if (spi_nor_spimem_check_op(nor, &op)) + if (!spi_mem_supports_op(nor->spimem, &op)) nor->flags |= SNOR_F_NO_READ_CR; } } diff --git a/drivers/mux/Kconfig b/drivers/mux/Kconfig index c68132e38138ef..281d3bad07448f 100644 --- a/drivers/mux/Kconfig +++ b/drivers/mux/Kconfig @@ -31,6 +31,19 @@ config MUX_ADGS1408 To compile the driver as a module, choose M here: the module will be called mux-adgs1408. +config MUX_APPLE_DPXBAR + tristate "Apple Silicon Display Crossbar" + depends on ARCH_APPLE + help + Apple Silicon Display Crossbar multiplexer. + + This drivers adds support for the display crossbar used to route + display controller streams to the three different modes + (DP AltMode, USB4 Tunnel #0/#1) of the Type-C ports. + + To compile this driver as a module, chose M here: the module will be + called mux-apple-display-crossbar. + config MUX_GPIO tristate "GPIO-controlled Multiplexer" depends on GPIOLIB || COMPILE_TEST diff --git a/drivers/mux/Makefile b/drivers/mux/Makefile index 6e9fa47daf5663..7b5b3325068010 100644 --- a/drivers/mux/Makefile +++ b/drivers/mux/Makefile @@ -8,9 +8,11 @@ mux-adg792a-objs := adg792a.o mux-adgs1408-objs := adgs1408.o mux-gpio-objs := gpio.o mux-mmio-objs := mmio.o +mux-apple-display-crossbar-objs := apple-display-crossbar.o obj-$(CONFIG_MULTIPLEXER) += mux-core.o obj-$(CONFIG_MUX_ADG792A) += mux-adg792a.o obj-$(CONFIG_MUX_ADGS1408) += mux-adgs1408.o +obj-$(CONFIG_MUX_APPLE_DPXBAR) += mux-apple-display-crossbar.o obj-$(CONFIG_MUX_GPIO) += mux-gpio.o obj-$(CONFIG_MUX_MMIO) += mux-mmio.o diff --git a/drivers/mux/apple-display-crossbar.c b/drivers/mux/apple-display-crossbar.c new file mode 100644 index 00000000000000..9b17371d92c3ba --- /dev/null +++ b/drivers/mux/apple-display-crossbar.c @@ -0,0 +1,463 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Apple Silicon Display Crossbar multiplexer driver + * + * Copyright (C) Asahi Linux Contributors + * + * Author: Sven Peter + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* + * T602x register interface is cleary different so most of the names below are + * probably wrong. + */ + +#define T602X_FIFO_WR_DPTX_CLK_EN 0x000 +#define T602X_FIFO_WR_N_CLK_EN 0x004 +#define T602X_FIFO_WR_UNK_EN 0x008 +#define T602X_REG_00C 0x00c +#define T602X_REG_014 0x014 +#define T602X_REG_018 0x018 +#define T602X_REG_01C 0x01c +#define T602X_FIFO_RD_PCLK2_EN 0x024 +#define T602X_FIFO_RD_N_CLK_EN 0x028 +#define T602X_FIFO_RD_UNK_EN 0x02c +#define T602X_REG_030 0x030 +#define T602X_REG_034 0x034 + +#define T602X_REG_804_STAT 0x804 // status of 0x004 +#define T602X_REG_810_STAT 0x810 // status of 0x014 +#define T602X_REG_81C_STAT 0x81c // status of 0x024 + +/* + * T8013, T600x, T8112 dp crossbar registers. + */ + +#define FIFO_WR_DPTX_CLK_EN 0x000 +#define FIFO_WR_N_CLK_EN 0x004 +#define FIFO_WR_UNK_EN 0x008 +#define FIFO_RD_PCLK1_EN 0x020 +#define FIFO_RD_PCLK2_EN 0x024 +#define FIFO_RD_N_CLK_EN 0x028 +#define FIFO_RD_UNK_EN 0x02c + +#define OUT_PCLK1_EN 0x040 +#define OUT_PCLK2_EN 0x044 +#define OUT_N_CLK_EN 0x048 +#define OUT_UNK_EN 0x04c + +#define CROSSBAR_DISPEXT_EN 0x050 +#define CROSSBAR_MUX_CTRL 0x060 +#define CROSSBAR_MUX_CTRL_DPPHY_SELECT0 GENMASK(23, 20) +#define CROSSBAR_MUX_CTRL_DPIN1_SELECT0 GENMASK(19, 16) +#define CROSSBAR_MUX_CTRL_DPIN0_SELECT0 GENMASK(15, 12) +#define CROSSBAR_MUX_CTRL_DPPHY_SELECT1 GENMASK(11, 8) +#define CROSSBAR_MUX_CTRL_DPIN1_SELECT1 GENMASK(7, 4) +#define CROSSBAR_MUX_CTRL_DPIN0_SELECT1 GENMASK(3, 0) +#define CROSSBAR_ATC_EN 0x070 + +#define FIFO_WR_DPTX_CLK_EN_STAT 0x800 +#define FIFO_WR_N_CLK_EN_STAT 0x804 +#define FIFO_RD_PCLK1_EN_STAT 0x820 +#define FIFO_RD_PCLK2_EN_STAT 0x824 +#define FIFO_RD_N_CLK_EN_STAT 0x828 + +#define OUT_PCLK1_EN_STAT 0x840 +#define OUT_PCLK2_EN_STAT 0x844 +#define OUT_N_CLK_EN_STAT 0x848 + +#define UNK_TUNABLE 0xc00 + +#define ATC_DPIN0 BIT(0) +#define ATC_DPIN1 BIT(4) +#define ATC_DPPHY BIT(8) + +enum { MUX_DPPHY = 0, MUX_DPIN0 = 1, MUX_DPIN1 = 2, MUX_MAX = 3 }; +static const char *apple_dpxbar_names[MUX_MAX] = { "dpphy", "dpin0", "dpin1" }; + +struct apple_dpxbar_hw { + unsigned int n_ufp; + u32 tunable; + const struct mux_control_ops *ops; +}; + +struct apple_dpxbar { + struct device *dev; + void __iomem *regs; + int selected_dispext[MUX_MAX]; + spinlock_t lock; +}; + +static inline void dpxbar_mask32(struct apple_dpxbar *xbar, u32 reg, u32 mask, + u32 set) +{ + u32 value = readl(xbar->regs + reg); + value &= ~mask; + value |= set; + writel(value, xbar->regs + reg); +} + +static inline void dpxbar_set32(struct apple_dpxbar *xbar, u32 reg, u32 set) +{ + dpxbar_mask32(xbar, reg, 0, set); +} + +static inline void dpxbar_clear32(struct apple_dpxbar *xbar, u32 reg, u32 clear) +{ + dpxbar_mask32(xbar, reg, clear, 0); +} + +static int apple_dpxbar_set_t602x(struct mux_control *mux, int state) +{ + struct apple_dpxbar *dpxbar = mux_chip_priv(mux->chip); + unsigned int index = mux_control_get_index(mux); + unsigned long flags; + unsigned int mux_state; + unsigned int dispext_bit; + unsigned int dispext_bit_en; + bool enable; + int ret = 0; + + if (state == MUX_IDLE_DISCONNECT) { + /* + * Technically this will select dispext0,0 in the mux control + * register. Practically that doesn't matter since everything + * else is disabled. + */ + mux_state = 0; + enable = false; + } else if (state >= 0 && state < 9) { + dispext_bit = 1 << state; + dispext_bit_en = 1 << (2 * state); + mux_state = state; + enable = true; + } else { + return -EINVAL; + } + + spin_lock_irqsave(&dpxbar->lock, flags); + + /* ensure the selected dispext isn't already used in this crossbar */ + if (enable) { + for (int i = 0; i < MUX_MAX; ++i) { + if (i == index) + continue; + if (dpxbar->selected_dispext[i] == state) { + spin_unlock_irqrestore(&dpxbar->lock, flags); + return -EBUSY; + } + } + } + + if (dpxbar->selected_dispext[index] >= 0) { + u32 prev_dispext_bit = 1 << dpxbar->selected_dispext[index]; + u32 prev_dispext_bit_en = 1 << (2 * dpxbar->selected_dispext[index]); + + dpxbar_clear32(dpxbar, T602X_FIFO_RD_UNK_EN, prev_dispext_bit); + dpxbar_clear32(dpxbar, T602X_FIFO_WR_DPTX_CLK_EN, prev_dispext_bit); + dpxbar_clear32(dpxbar, T602X_REG_00C, prev_dispext_bit_en); + + dpxbar_clear32(dpxbar, T602X_REG_01C, 0x100); + + dpxbar_clear32(dpxbar, T602X_FIFO_WR_UNK_EN, prev_dispext_bit); + dpxbar_clear32(dpxbar, T602X_REG_018, prev_dispext_bit_en); + + dpxbar_clear32(dpxbar, T602X_FIFO_RD_N_CLK_EN, 0x100); + + dpxbar_set32(dpxbar, T602X_FIFO_WR_N_CLK_EN, prev_dispext_bit); + dpxbar_set32(dpxbar, T602X_REG_014, 0x4); + + dpxbar_set32(dpxbar, FIFO_RD_PCLK1_EN, 0x100); + + dpxbar->selected_dispext[index] = -1; + } + + if (enable) { + dpxbar_set32(dpxbar, T602X_REG_030, state << 20); + dpxbar_set32(dpxbar, T602X_REG_030, state << 8); + udelay(10); + + dpxbar_clear32(dpxbar, T602X_FIFO_WR_N_CLK_EN, dispext_bit); + dpxbar_clear32(dpxbar, T602X_REG_014, 0x4); + + dpxbar_clear32(dpxbar, T602X_FIFO_RD_PCLK2_EN, 0x100); + + dpxbar_set32(dpxbar, T602X_FIFO_WR_UNK_EN, dispext_bit); + dpxbar_set32(dpxbar, T602X_REG_018, dispext_bit_en); + + dpxbar_set32(dpxbar, T602X_FIFO_RD_N_CLK_EN, 0x100); + dpxbar_set32(dpxbar, T602X_FIFO_WR_DPTX_CLK_EN, dispext_bit); + dpxbar_set32(dpxbar, T602X_REG_00C, dispext_bit); + + dpxbar_set32(dpxbar, T602X_REG_01C, 0x100); + dpxbar_set32(dpxbar, T602X_REG_034, 0x100); + + dpxbar_set32(dpxbar, T602X_FIFO_RD_UNK_EN, dispext_bit); + + dpxbar->selected_dispext[index] = state; + } + + spin_unlock_irqrestore(&dpxbar->lock, flags); + + if (enable) + dev_info(dpxbar->dev, "Switched %s to dispext%u,%u\n", + apple_dpxbar_names[index], mux_state >> 1, + mux_state & 1); + else + dev_info(dpxbar->dev, "Switched %s to disconnected state\n", + apple_dpxbar_names[index]); + + return ret; +} + +static int apple_dpxbar_set(struct mux_control *mux, int state) +{ + struct apple_dpxbar *dpxbar = mux_chip_priv(mux->chip); + unsigned int index = mux_control_get_index(mux); + unsigned long flags; + unsigned int mux_state; + unsigned int dispext_bit; + unsigned int dispext_bit_en; + unsigned int atc_bit; + bool enable; + int ret = 0; + u32 mux_mask, mux_set; + + if (state == MUX_IDLE_DISCONNECT) { + /* + * Technically this will select dispext0,0 in the mux control + * register. Practically that doesn't matter since everything + * else is disabled. + */ + mux_state = 0; + enable = false; + } else if (state >= 0 && state < 9) { + dispext_bit = 1 << state; + dispext_bit_en = 1 << (2 * state); + mux_state = state; + enable = true; + } else { + return -EINVAL; + } + + switch (index) { + case MUX_DPPHY: + mux_mask = CROSSBAR_MUX_CTRL_DPPHY_SELECT0 | + CROSSBAR_MUX_CTRL_DPPHY_SELECT1; + mux_set = + FIELD_PREP(CROSSBAR_MUX_CTRL_DPPHY_SELECT0, mux_state) | + FIELD_PREP(CROSSBAR_MUX_CTRL_DPPHY_SELECT1, mux_state); + atc_bit = ATC_DPPHY; + break; + case MUX_DPIN0: + mux_mask = CROSSBAR_MUX_CTRL_DPIN0_SELECT0 | + CROSSBAR_MUX_CTRL_DPIN0_SELECT1; + mux_set = + FIELD_PREP(CROSSBAR_MUX_CTRL_DPIN0_SELECT0, mux_state) | + FIELD_PREP(CROSSBAR_MUX_CTRL_DPIN0_SELECT1, mux_state); + atc_bit = ATC_DPIN0; + break; + case MUX_DPIN1: + mux_mask = CROSSBAR_MUX_CTRL_DPIN1_SELECT0 | + CROSSBAR_MUX_CTRL_DPIN1_SELECT1; + mux_set = + FIELD_PREP(CROSSBAR_MUX_CTRL_DPIN1_SELECT0, mux_state) | + FIELD_PREP(CROSSBAR_MUX_CTRL_DPIN1_SELECT1, mux_state); + atc_bit = ATC_DPIN1; + break; + default: + return -EINVAL; + } + + spin_lock_irqsave(&dpxbar->lock, flags); + + /* ensure the selected dispext isn't already used in this crossbar */ + if (enable) { + for (int i = 0; i < MUX_MAX; ++i) { + if (i == index) + continue; + if (dpxbar->selected_dispext[i] == state) { + spin_unlock_irqrestore(&dpxbar->lock, flags); + return -EBUSY; + } + } + } + + dpxbar_set32(dpxbar, OUT_N_CLK_EN, atc_bit); + dpxbar_clear32(dpxbar, OUT_UNK_EN, atc_bit); + dpxbar_clear32(dpxbar, OUT_PCLK1_EN, atc_bit); + dpxbar_clear32(dpxbar, CROSSBAR_ATC_EN, atc_bit); + + if (dpxbar->selected_dispext[index] >= 0) { + u32 prev_dispext_bit = 1 << dpxbar->selected_dispext[index]; + u32 prev_dispext_bit_en = 1 << (2 * dpxbar->selected_dispext[index]); + + dpxbar_set32(dpxbar, FIFO_WR_N_CLK_EN, prev_dispext_bit); + dpxbar_set32(dpxbar, FIFO_RD_N_CLK_EN, prev_dispext_bit); + dpxbar_clear32(dpxbar, FIFO_WR_UNK_EN, prev_dispext_bit); + dpxbar_clear32(dpxbar, FIFO_RD_UNK_EN, prev_dispext_bit_en); + dpxbar_clear32(dpxbar, FIFO_WR_DPTX_CLK_EN, prev_dispext_bit); + dpxbar_clear32(dpxbar, FIFO_RD_PCLK1_EN, prev_dispext_bit); + dpxbar_clear32(dpxbar, CROSSBAR_DISPEXT_EN, prev_dispext_bit); + + dpxbar->selected_dispext[index] = -1; + } + + dpxbar_mask32(dpxbar, CROSSBAR_MUX_CTRL, mux_mask, mux_set); + + if (enable) { + dpxbar_clear32(dpxbar, FIFO_WR_N_CLK_EN, dispext_bit); + dpxbar_clear32(dpxbar, FIFO_RD_N_CLK_EN, dispext_bit); + dpxbar_clear32(dpxbar, OUT_N_CLK_EN, atc_bit); + dpxbar_set32(dpxbar, FIFO_WR_UNK_EN, dispext_bit); + dpxbar_set32(dpxbar, FIFO_RD_UNK_EN, dispext_bit_en); + dpxbar_set32(dpxbar, OUT_UNK_EN, atc_bit); + dpxbar_set32(dpxbar, FIFO_WR_DPTX_CLK_EN, dispext_bit); + dpxbar_set32(dpxbar, FIFO_RD_PCLK1_EN, dispext_bit); + dpxbar_set32(dpxbar, OUT_PCLK1_EN, atc_bit); + dpxbar_set32(dpxbar, CROSSBAR_ATC_EN, atc_bit); + dpxbar_set32(dpxbar, CROSSBAR_DISPEXT_EN, dispext_bit); + + /* + * Work around some HW quirk: + * Without toggling the RD_PCLK enable here the connection + * doesn't come up. Testing has shown that a delay of about + * 5 usec is required which is doubled here to be on the + * safe side. + */ + dpxbar_clear32(dpxbar, FIFO_RD_PCLK1_EN, dispext_bit); + udelay(10); + dpxbar_set32(dpxbar, FIFO_RD_PCLK1_EN, dispext_bit); + + dpxbar->selected_dispext[index] = state; + } + + spin_unlock_irqrestore(&dpxbar->lock, flags); + + if (enable) + dev_info(dpxbar->dev, "Switched %s to dispext%u,%u\n", + apple_dpxbar_names[index], mux_state >> 1, + mux_state & 1); + else + dev_info(dpxbar->dev, "Switched %s to disconnected state\n", + apple_dpxbar_names[index]); + + return ret; +} + +static const struct mux_control_ops apple_dpxbar_ops = { + .set = apple_dpxbar_set, +}; + +static const struct mux_control_ops apple_dpxbar_t602x_ops = { + .set = apple_dpxbar_set_t602x, +}; + +static int apple_dpxbar_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct mux_chip *mux_chip; + struct apple_dpxbar *dpxbar; + const struct apple_dpxbar_hw *hw; + int ret; + + hw = of_device_get_match_data(dev); + mux_chip = devm_mux_chip_alloc(dev, MUX_MAX, sizeof(*dpxbar)); + if (IS_ERR(mux_chip)) + return PTR_ERR(mux_chip); + + dpxbar = mux_chip_priv(mux_chip); + mux_chip->ops = hw->ops; + spin_lock_init(&dpxbar->lock); + + dpxbar->dev = dev; + dpxbar->regs = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(dpxbar->regs)) + return PTR_ERR(dpxbar->regs); + + if (!of_device_is_compatible(dev->of_node, "apple,t6020-display-crossbar")) { + readl(dpxbar->regs + UNK_TUNABLE); + writel(hw->tunable, dpxbar->regs + UNK_TUNABLE); + readl(dpxbar->regs + UNK_TUNABLE); + } + + for (unsigned int i = 0; i < MUX_MAX; ++i) { + mux_chip->mux[i].states = hw->n_ufp; + mux_chip->mux[i].idle_state = MUX_IDLE_DISCONNECT; + dpxbar->selected_dispext[i] = -1; + } + + ret = devm_mux_chip_register(dev, mux_chip); + if (ret < 0) + return ret; + + return 0; +} + +static const struct apple_dpxbar_hw apple_dpxbar_hw_t8103 = { + .n_ufp = 2, + .tunable = 0, + .ops = &apple_dpxbar_ops, +}; + +static const struct apple_dpxbar_hw apple_dpxbar_hw_t8112 = { + .n_ufp = 4, + .tunable = 4278196325, + .ops = &apple_dpxbar_ops, +}; + +static const struct apple_dpxbar_hw apple_dpxbar_hw_t6000 = { + .n_ufp = 9, + .tunable = 5, + .ops = &apple_dpxbar_ops, +}; + +static const struct apple_dpxbar_hw apple_dpxbar_hw_t6020 = { + .n_ufp = 9, + .ops = &apple_dpxbar_t602x_ops, +}; + +static const struct of_device_id apple_dpxbar_ids[] = { + { + .compatible = "apple,t8103-display-crossbar", + .data = &apple_dpxbar_hw_t8103, + }, + { + .compatible = "apple,t8112-display-crossbar", + .data = &apple_dpxbar_hw_t8112, + }, + { + .compatible = "apple,t6000-display-crossbar", + .data = &apple_dpxbar_hw_t6000, + }, + { + .compatible = "apple,t6020-display-crossbar", + .data = &apple_dpxbar_hw_t6020, + }, + {} +}; +MODULE_DEVICE_TABLE(of, apple_dpxbar_ids); + +static struct platform_driver apple_dpxbar_driver = { + .driver = { + .name = "apple-display-crossbar", + .of_match_table = apple_dpxbar_ids, + }, + .probe = apple_dpxbar_probe, +}; +module_platform_driver(apple_dpxbar_driver); + +MODULE_DESCRIPTION("Apple Silicon display crossbar multiplexer driver"); +MODULE_AUTHOR("Sven Peter "); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/mux/mmio.c b/drivers/mux/mmio.c index 3409af1ffb80f6..0611ef28bb69ab 100644 --- a/drivers/mux/mmio.c +++ b/drivers/mux/mmio.c @@ -72,7 +72,7 @@ static int mux_mmio_probe(struct platform_device *pdev) if (IS_ERR(base)) regmap = ERR_PTR(-ENODEV); else - regmap = regmap_init_mmio(dev, base, &mux_mmio_regmap_cfg); + regmap = devm_regmap_init_mmio(dev, base, &mux_mmio_regmap_cfg); /* Fallback to checking the parent node on "real" errors. */ if (IS_ERR(regmap) && regmap != ERR_PTR(-EPROBE_DEFER)) { regmap = dev_get_regmap(dev->parent, NULL); diff --git a/drivers/net/arcnet/com20020-pci.c b/drivers/net/arcnet/com20020-pci.c index 0472bcdff13072..b5729d6c0b47ca 100644 --- a/drivers/net/arcnet/com20020-pci.c +++ b/drivers/net/arcnet/com20020-pci.c @@ -115,6 +115,8 @@ static const struct attribute_group com20020_state_group = { .attrs = com20020_state_attrs, }; +static struct com20020_pci_card_info card_info_2p5mbit; + static void com20020pci_remove(struct pci_dev *pdev); static int com20020pci_probe(struct pci_dev *pdev, @@ -140,7 +142,7 @@ static int com20020pci_probe(struct pci_dev *pdev, ci = (struct com20020_pci_card_info *)id->driver_data; if (!ci) - return -EINVAL; + ci = &card_info_2p5mbit; priv->ci = ci; mm = &ci->misc_map; @@ -347,6 +349,18 @@ static struct com20020_pci_card_info card_info_5mbit = { .flags = ARC_IS_5MBIT, }; +static struct com20020_pci_card_info card_info_2p5mbit = { + .name = "ARC-PCI", + .devcount = 1, + .chan_map_tbl = { + { + .bar = 2, + .offset = 0x00, + .size = 0x08, + }, + }, +}; + static struct com20020_pci_card_info card_info_sohard = { .name = "SOHARD SH ARC-PCI", .devcount = 1, diff --git a/drivers/net/bonding/bond_debugfs.c b/drivers/net/bonding/bond_debugfs.c index 8adbec7c5084aa..8967b65f6d8408 100644 --- a/drivers/net/bonding/bond_debugfs.c +++ b/drivers/net/bonding/bond_debugfs.c @@ -34,11 +34,17 @@ static int bond_debug_rlb_hash_show(struct seq_file *m, void *v) for (; hash_index != RLB_NULL_INDEX; hash_index = client_info->used_next) { client_info = &(bond_info->rx_hashtbl[hash_index]); - seq_printf(m, "%-15pI4 %-15pI4 %-17pM %s\n", - &client_info->ip_src, - &client_info->ip_dst, - &client_info->mac_dst, - client_info->slave->dev->name); + if (client_info->slave) + seq_printf(m, "%-15pI4 %-15pI4 %-17pM %s\n", + &client_info->ip_src, + &client_info->ip_dst, + &client_info->mac_dst, + client_info->slave->dev->name); + else + seq_printf(m, "%-15pI4 %-15pI4 %-17pM (none)\n", + &client_info->ip_src, + &client_info->ip_dst, + &client_info->mac_dst); } spin_unlock_bh(&bond->mode_lock); diff --git a/drivers/net/bonding/bond_main.c b/drivers/net/bonding/bond_main.c index 45bd2bb102ffdf..1d84e348f2cc7d 100644 --- a/drivers/net/bonding/bond_main.c +++ b/drivers/net/bonding/bond_main.c @@ -324,7 +324,7 @@ static bool bond_sk_check(struct bonding *bond) } } -bool bond_xdp_check(struct bonding *bond, int mode) +bool __bond_xdp_check(int mode, int xmit_policy) { switch (mode) { case BOND_MODE_ROUNDROBIN: @@ -335,7 +335,7 @@ bool bond_xdp_check(struct bonding *bond, int mode) /* vlan+srcmac is not supported with XDP as in most cases the 802.1q * payload is not in the packet due to hardware offload. */ - if (bond->params.xmit_policy != BOND_XMIT_POLICY_VLAN_SRCMAC) + if (xmit_policy != BOND_XMIT_POLICY_VLAN_SRCMAC) return true; fallthrough; default: @@ -343,6 +343,11 @@ bool bond_xdp_check(struct bonding *bond, int mode) } } +bool bond_xdp_check(struct bonding *bond, int mode) +{ + return __bond_xdp_check(mode, bond->params.xmit_policy); +} + /*---------------------------------- VLAN -----------------------------------*/ /* In the following 2 functions, bond_vlan_rx_add_vid and bond_vlan_rx_kill_vid, @@ -791,26 +796,29 @@ static int bond_update_speed_duplex(struct slave *slave) struct ethtool_link_ksettings ecmd; int res; - slave->speed = SPEED_UNKNOWN; - slave->duplex = DUPLEX_UNKNOWN; - res = __ethtool_get_link_ksettings(slave_dev, &ecmd); if (res < 0) - return 1; + goto speed_duplex_unknown; if (ecmd.base.speed == 0 || ecmd.base.speed == ((__u32)-1)) - return 1; + goto speed_duplex_unknown; switch (ecmd.base.duplex) { case DUPLEX_FULL: case DUPLEX_HALF: break; default: - return 1; + goto speed_duplex_unknown; } slave->speed = ecmd.base.speed; slave->duplex = ecmd.base.duplex; return 0; + +speed_duplex_unknown: + slave->speed = SPEED_UNKNOWN; + slave->duplex = DUPLEX_UNKNOWN; + + return 1; } const char *bond_slave_link_status(s8 link) @@ -1468,6 +1476,52 @@ static netdev_features_t bond_fix_features(struct net_device *dev, return features; } +static int bond_header_create(struct sk_buff *skb, struct net_device *bond_dev, + unsigned short type, const void *daddr, + const void *saddr, unsigned int len) +{ + struct bonding *bond = netdev_priv(bond_dev); + const struct header_ops *slave_ops; + struct slave *slave; + int ret = 0; + + rcu_read_lock(); + slave = rcu_dereference(bond->curr_active_slave); + if (slave) { + slave_ops = READ_ONCE(slave->dev->header_ops); + if (slave_ops && slave_ops->create) + ret = slave_ops->create(skb, slave->dev, + type, daddr, saddr, len); + } + rcu_read_unlock(); + return ret; +} + +static int bond_header_parse(const struct sk_buff *skb, + const struct net_device *dev, + unsigned char *haddr) +{ + struct bonding *bond = netdev_priv(dev); + const struct header_ops *slave_ops; + struct slave *slave; + int ret = 0; + + rcu_read_lock(); + slave = rcu_dereference(bond->curr_active_slave); + if (slave) { + slave_ops = READ_ONCE(slave->dev->header_ops); + if (slave_ops && slave_ops->parse) + ret = slave_ops->parse(skb, slave->dev, haddr); + } + rcu_read_unlock(); + return ret; +} + +static const struct header_ops bond_header_ops = { + .create = bond_header_create, + .parse = bond_header_parse, +}; + static void bond_setup_by_slave(struct net_device *bond_dev, struct net_device *slave_dev) { @@ -1475,7 +1529,8 @@ static void bond_setup_by_slave(struct net_device *bond_dev, dev_close(bond_dev); - bond_dev->header_ops = slave_dev->header_ops; + bond_dev->header_ops = slave_dev->header_ops ? + &bond_header_ops : NULL; bond_dev->type = slave_dev->type; bond_dev->hard_header_len = slave_dev->hard_header_len; @@ -2761,8 +2816,14 @@ static void bond_miimon_commit(struct bonding *bond) continue; + case BOND_LINK_FAIL: + case BOND_LINK_BACK: + slave_dbg(bond->dev, slave->dev, "link_new_state %d on slave\n", + slave->link_new_state); + continue; + default: - slave_err(bond->dev, slave->dev, "invalid new link %d on slave\n", + slave_err(bond->dev, slave->dev, "invalid link_new_state %d on slave\n", slave->link_new_state); bond_propose_link_state(slave, BOND_LINK_NOCHANGE); @@ -3343,7 +3404,7 @@ int bond_rcv_validate(const struct sk_buff *skb, struct bonding *bond, } else if (is_arp) { return bond_arp_rcv(skb, bond, slave); #if IS_ENABLED(CONFIG_IPV6) - } else if (is_ipv6) { + } else if (is_ipv6 && likely(ipv6_mod_enabled())) { return bond_na_rcv(skb, bond, slave); #endif } else { @@ -4311,9 +4372,13 @@ static int bond_close(struct net_device *bond_dev) bond_work_cancel_all(bond); bond->send_peer_notif = 0; + WRITE_ONCE(bond->recv_probe, NULL); + + /* Wait for any in-flight RX handlers */ + synchronize_net(); + if (bond_is_lb(bond)) bond_alb_deinitialize(bond); - bond->recv_probe = NULL; if (BOND_MODE(bond) == BOND_MODE_8023AD && bond->params.broadcast_neighbor) @@ -5029,13 +5094,18 @@ static void bond_set_slave_arr(struct bonding *bond, { struct bond_up_slave *usable, *all; - usable = rtnl_dereference(bond->usable_slaves); - rcu_assign_pointer(bond->usable_slaves, usable_slaves); - kfree_rcu(usable, rcu); - all = rtnl_dereference(bond->all_slaves); rcu_assign_pointer(bond->all_slaves, all_slaves); kfree_rcu(all, rcu); + + if (BOND_MODE(bond) == BOND_MODE_BROADCAST) { + kfree_rcu(usable_slaves, rcu); + return; + } + + usable = rtnl_dereference(bond->usable_slaves); + rcu_assign_pointer(bond->usable_slaves, usable_slaves); + kfree_rcu(usable, rcu); } static void bond_reset_slave_arr(struct bonding *bond) @@ -5230,7 +5300,7 @@ static netdev_tx_t bond_xmit_broadcast(struct sk_buff *skb, if (!(bond_slave_is_up(slave) && slave->link == BOND_LINK_UP)) continue; - if (bond_is_last_slave(bond, slave)) { + if (i + 1 == slaves_count) { skb2 = skb; skb_used = true; } else { diff --git a/drivers/net/bonding/bond_options.c b/drivers/net/bonding/bond_options.c index f1c6e9d8f61671..adc216df43459e 100644 --- a/drivers/net/bonding/bond_options.c +++ b/drivers/net/bonding/bond_options.c @@ -1574,6 +1574,8 @@ static int bond_option_fail_over_mac_set(struct bonding *bond, static int bond_option_xmit_hash_policy_set(struct bonding *bond, const struct bond_opt_value *newval) { + if (bond->xdp_prog && !__bond_xdp_check(BOND_MODE(bond), newval->value)) + return -EOPNOTSUPP; netdev_dbg(bond->dev, "Setting xmit hash policy to %s (%llu)\n", newval->string, newval->value); bond->params.xmit_policy = newval->value; diff --git a/drivers/net/caif/caif_serial.c b/drivers/net/caif/caif_serial.c index c398ac42eae905..1873d8287bb9be 100644 --- a/drivers/net/caif/caif_serial.c +++ b/drivers/net/caif/caif_serial.c @@ -284,6 +284,7 @@ static void ser_release(struct work_struct *work) { struct list_head list; struct ser_device *ser, *tmp; + struct tty_struct *tty; spin_lock(&ser_lock); list_replace_init(&ser_release_list, &list); @@ -292,9 +293,12 @@ static void ser_release(struct work_struct *work) if (!list_empty(&list)) { rtnl_lock(); list_for_each_entry_safe(ser, tmp, &list, node) { + tty = ser->tty; dev_close(ser->dev); unregister_netdevice(ser->dev); debugfs_deinit(ser); + tty_kref_put(tty->link); + tty_kref_put(tty); } rtnl_unlock(); } @@ -328,6 +332,7 @@ static int ldisc_open(struct tty_struct *tty) ser = netdev_priv(dev); ser->tty = tty_kref_get(tty); + tty_kref_get(tty->link); ser->dev = dev; debugfs_init(ser, tty); tty->receive_room = 4096; @@ -336,6 +341,7 @@ static int ldisc_open(struct tty_struct *tty) rtnl_lock(); result = register_netdevice(dev); if (result) { + tty_kref_put(tty->link); tty_kref_put(tty); rtnl_unlock(); free_netdev(dev); @@ -355,8 +361,6 @@ static void ldisc_close(struct tty_struct *tty) { struct ser_device *ser = tty->disc_data; - tty_kref_put(ser->tty); - spin_lock(&ser_lock); list_move(&ser->node, &ser_release_list); spin_unlock(&ser_lock); diff --git a/drivers/net/can/dev/calc_bittiming.c b/drivers/net/can/dev/calc_bittiming.c index cc4022241553fa..42498e9d3f38db 100644 --- a/drivers/net/can/dev/calc_bittiming.c +++ b/drivers/net/can/dev/calc_bittiming.c @@ -8,7 +8,7 @@ #include #include -#define CAN_CALC_MAX_ERROR 50 /* in one-tenth of a percent */ +#define CAN_CALC_MAX_ERROR 500 /* max error 5% */ /* CiA recommended sample points for Non Return to Zero encoding. */ static int can_calc_sample_point_nrz(const struct can_bittiming *bt) diff --git a/drivers/net/can/dev/netlink.c b/drivers/net/can/dev/netlink.c index 0498198a469658..766d455950f55d 100644 --- a/drivers/net/can/dev/netlink.c +++ b/drivers/net/can/dev/netlink.c @@ -601,7 +601,9 @@ static int can_changelink(struct net_device *dev, struct nlattr *tb[], /* We need synchronization with dev->stop() */ ASSERT_RTNL(); - can_ctrlmode_changelink(dev, data, extack); + err = can_ctrlmode_changelink(dev, data, extack); + if (err) + return err; if (data[IFLA_CAN_BITTIMING]) { struct can_bittiming bt; diff --git a/drivers/net/can/dummy_can.c b/drivers/net/can/dummy_can.c index 41953655e3d3c9..cd23de488edce2 100644 --- a/drivers/net/can/dummy_can.c +++ b/drivers/net/can/dummy_can.c @@ -241,6 +241,7 @@ static int __init dummy_can_init(void) dev->netdev_ops = &dummy_can_netdev_ops; dev->ethtool_ops = &dummy_can_ethtool_ops; + dev->flags |= IFF_ECHO; /* enable echo handling */ priv = netdev_priv(dev); priv->can.bittiming_const = &dummy_can_bittiming_const; priv->can.bitrate_max = 20 * MEGA /* BPS */; diff --git a/drivers/net/can/spi/hi311x.c b/drivers/net/can/spi/hi311x.c index e00d3dbc4cf436..91b1fa970f8fb9 100644 --- a/drivers/net/can/spi/hi311x.c +++ b/drivers/net/can/spi/hi311x.c @@ -755,7 +755,9 @@ static int hi3110_open(struct net_device *net) return ret; mutex_lock(&priv->hi3110_lock); - hi3110_power_enable(priv->transceiver, 1); + ret = hi3110_power_enable(priv->transceiver, 1); + if (ret) + goto out_close_candev; priv->force_quit = 0; priv->tx_skb = NULL; @@ -790,6 +792,7 @@ static int hi3110_open(struct net_device *net) hi3110_hw_sleep(spi); out_close: hi3110_power_enable(priv->transceiver, 0); + out_close_candev: close_candev(net); mutex_unlock(&priv->hi3110_lock); return ret; diff --git a/drivers/net/can/spi/mcp251x.c b/drivers/net/can/spi/mcp251x.c index fa97adf25b734a..0d0190ae094a1b 100644 --- a/drivers/net/can/spi/mcp251x.c +++ b/drivers/net/can/spi/mcp251x.c @@ -1214,6 +1214,7 @@ static int mcp251x_open(struct net_device *net) { struct mcp251x_priv *priv = netdev_priv(net); struct spi_device *spi = priv->spi; + bool release_irq = false; unsigned long flags = 0; int ret; @@ -1224,7 +1225,11 @@ static int mcp251x_open(struct net_device *net) } mutex_lock(&priv->mcp_lock); - mcp251x_power_enable(priv->transceiver, 1); + ret = mcp251x_power_enable(priv->transceiver, 1); + if (ret) { + dev_err(&spi->dev, "failed to enable transceiver power: %pe\n", ERR_PTR(ret)); + goto out_close_candev; + } priv->force_quit = 0; priv->tx_skb = NULL; @@ -1257,12 +1262,25 @@ static int mcp251x_open(struct net_device *net) return 0; out_free_irq: - free_irq(spi->irq, priv); + /* The IRQ handler might be running, and if so it will be waiting + * for the lock. But free_irq() must wait for the handler to finish + * so calling it here would deadlock. + * + * Setting priv->force_quit will let the handler exit right away + * without any access to the hardware. This make it safe to call + * free_irq() after the lock is released. + */ + priv->force_quit = 1; + release_irq = true; + mcp251x_hw_sleep(spi); out_close: mcp251x_power_enable(priv->transceiver, 0); +out_close_candev: close_candev(net); mutex_unlock(&priv->mcp_lock); + if (release_irq) + free_irq(spi->irq, priv); return ret; } @@ -1503,11 +1521,25 @@ static int __maybe_unused mcp251x_can_resume(struct device *dev) { struct spi_device *spi = to_spi_device(dev); struct mcp251x_priv *priv = spi_get_drvdata(spi); + int ret = 0; - if (priv->after_suspend & AFTER_SUSPEND_POWER) - mcp251x_power_enable(priv->power, 1); - if (priv->after_suspend & AFTER_SUSPEND_UP) - mcp251x_power_enable(priv->transceiver, 1); + if (priv->after_suspend & AFTER_SUSPEND_POWER) { + ret = mcp251x_power_enable(priv->power, 1); + if (ret) { + dev_err(dev, "failed to restore power: %pe\n", ERR_PTR(ret)); + return ret; + } + } + + if (priv->after_suspend & AFTER_SUSPEND_UP) { + ret = mcp251x_power_enable(priv->transceiver, 1); + if (ret) { + dev_err(dev, "failed to restore transceiver power: %pe\n", ERR_PTR(ret)); + if (priv->after_suspend & AFTER_SUSPEND_POWER) + mcp251x_power_enable(priv->power, 0); + return ret; + } + } if (priv->after_suspend & (AFTER_SUSPEND_POWER | AFTER_SUSPEND_UP)) queue_work(priv->wq, &priv->restart_work); diff --git a/drivers/net/can/usb/ems_usb.c b/drivers/net/can/usb/ems_usb.c index 4c219a5b139bba..9b25dda7c18382 100644 --- a/drivers/net/can/usb/ems_usb.c +++ b/drivers/net/can/usb/ems_usb.c @@ -445,6 +445,11 @@ static void ems_usb_read_bulk_callback(struct urb *urb) start = CPC_HEADER_SIZE; while (msg_count) { + if (start + CPC_MSG_HEADER_LEN > urb->actual_length) { + netdev_err(netdev, "format error\n"); + break; + } + msg = (struct ems_cpc_msg *)&ibuf[start]; switch (msg->type) { @@ -474,7 +479,7 @@ static void ems_usb_read_bulk_callback(struct urb *urb) start += CPC_MSG_HEADER_LEN + msg->length; msg_count--; - if (start > urb->transfer_buffer_length) { + if (start > urb->actual_length) { netdev_err(netdev, "format error\n"); break; } diff --git a/drivers/net/can/usb/etas_es58x/es58x_core.c b/drivers/net/can/usb/etas_es58x/es58x_core.c index 2d248deb69dc1e..b259f61098083c 100644 --- a/drivers/net/can/usb/etas_es58x/es58x_core.c +++ b/drivers/net/can/usb/etas_es58x/es58x_core.c @@ -1461,12 +1461,18 @@ static void es58x_read_bulk_callback(struct urb *urb) } resubmit_urb: + usb_anchor_urb(urb, &es58x_dev->rx_urbs); ret = usb_submit_urb(urb, GFP_ATOMIC); + if (!ret) + return; + + usb_unanchor_urb(urb); + if (ret == -ENODEV) { for (i = 0; i < es58x_dev->num_can_ch; i++) if (es58x_dev->netdev[i]) netif_device_detach(es58x_dev->netdev[i]); - } else if (ret) + } else dev_err_ratelimited(dev, "Failed resubmitting read bulk urb: %pe\n", ERR_PTR(ret)); diff --git a/drivers/net/can/usb/f81604.c b/drivers/net/can/usb/f81604.c index efe61ece79ea25..ea70ddf325d323 100644 --- a/drivers/net/can/usb/f81604.c +++ b/drivers/net/can/usb/f81604.c @@ -413,6 +413,7 @@ static void f81604_read_bulk_callback(struct urb *urb) { struct f81604_can_frame *frame = urb->transfer_buffer; struct net_device *netdev = urb->context; + struct f81604_port_priv *priv = netdev_priv(netdev); int ret; if (!netif_device_present(netdev)) @@ -445,10 +446,15 @@ static void f81604_read_bulk_callback(struct urb *urb) f81604_process_rx_packet(netdev, frame); resubmit_urb: + usb_anchor_urb(urb, &priv->urbs_anchor); ret = usb_submit_urb(urb, GFP_ATOMIC); + if (!ret) + return; + usb_unanchor_urb(urb); + if (ret == -ENODEV) netif_device_detach(netdev); - else if (ret) + else netdev_err(netdev, "%s: failed to resubmit read bulk urb: %pe\n", __func__, ERR_PTR(ret)); @@ -620,6 +626,12 @@ static void f81604_read_int_callback(struct urb *urb) netdev_info(netdev, "%s: Int URB aborted: %pe\n", __func__, ERR_PTR(urb->status)); + if (urb->actual_length < sizeof(*data)) { + netdev_warn(netdev, "%s: short int URB: %u < %zu\n", + __func__, urb->actual_length, sizeof(*data)); + goto resubmit_urb; + } + switch (urb->status) { case 0: /* success */ break; @@ -646,10 +658,15 @@ static void f81604_read_int_callback(struct urb *urb) f81604_handle_tx(priv, data); resubmit_urb: + usb_anchor_urb(urb, &priv->urbs_anchor); ret = usb_submit_urb(urb, GFP_ATOMIC); + if (!ret) + return; + usb_unanchor_urb(urb); + if (ret == -ENODEV) netif_device_detach(netdev); - else if (ret) + else netdev_err(netdev, "%s: failed to resubmit int urb: %pe\n", __func__, ERR_PTR(ret)); } @@ -874,9 +891,27 @@ static void f81604_write_bulk_callback(struct urb *urb) if (!netif_device_present(netdev)) return; - if (urb->status) - netdev_info(netdev, "%s: Tx URB error: %pe\n", __func__, - ERR_PTR(urb->status)); + if (!urb->status) + return; + + switch (urb->status) { + case -ENOENT: + case -ECONNRESET: + case -ESHUTDOWN: + return; + default: + break; + } + + if (net_ratelimit()) + netdev_err(netdev, "%s: Tx URB error: %pe\n", __func__, + ERR_PTR(urb->status)); + + can_free_echo_skb(netdev, 0, NULL); + netdev->stats.tx_dropped++; + netdev->stats.tx_errors++; + + netif_wake_queue(netdev); } static void f81604_clear_reg_work(struct work_struct *work) diff --git a/drivers/net/can/usb/gs_usb.c b/drivers/net/can/usb/gs_usb.c index d8b2dd74b3a1a4..6faa877d33aec1 100644 --- a/drivers/net/can/usb/gs_usb.c +++ b/drivers/net/can/usb/gs_usb.c @@ -772,9 +772,8 @@ static void gs_usb_receive_bulk_callback(struct urb *urb) } } -static int gs_usb_set_bittiming(struct net_device *netdev) +static int gs_usb_set_bittiming(struct gs_can *dev) { - struct gs_can *dev = netdev_priv(netdev); struct can_bittiming *bt = &dev->can.bittiming; struct gs_device_bittiming dbt = { .prop_seg = cpu_to_le32(bt->prop_seg), @@ -791,9 +790,8 @@ static int gs_usb_set_bittiming(struct net_device *netdev) GFP_KERNEL); } -static int gs_usb_set_data_bittiming(struct net_device *netdev) +static int gs_usb_set_data_bittiming(struct gs_can *dev) { - struct gs_can *dev = netdev_priv(netdev); struct can_bittiming *bt = &dev->can.fd.data_bittiming; struct gs_device_bittiming dbt = { .prop_seg = cpu_to_le32(bt->prop_seg), @@ -1057,6 +1055,20 @@ static int gs_can_open(struct net_device *netdev) if (dev->feature & GS_CAN_FEATURE_HW_TIMESTAMP) flags |= GS_CAN_MODE_HW_TIMESTAMP; + rc = gs_usb_set_bittiming(dev); + if (rc) { + netdev_err(netdev, "failed to set bittiming: %pe\n", ERR_PTR(rc)); + goto out_usb_kill_anchored_urbs; + } + + if (ctrlmode & CAN_CTRLMODE_FD) { + rc = gs_usb_set_data_bittiming(dev); + if (rc) { + netdev_err(netdev, "failed to set data bittiming: %pe\n", ERR_PTR(rc)); + goto out_usb_kill_anchored_urbs; + } + } + /* finally start device */ dev->can.state = CAN_STATE_ERROR_ACTIVE; dm.flags = cpu_to_le32(flags); @@ -1370,7 +1382,6 @@ static struct gs_can *gs_make_candev(unsigned int channel, dev->can.state = CAN_STATE_STOPPED; dev->can.clock.freq = le32_to_cpu(bt_const.fclk_can); dev->can.bittiming_const = &dev->bt_const; - dev->can.do_set_bittiming = gs_usb_set_bittiming; dev->can.ctrlmode_supported = CAN_CTRLMODE_CC_LEN8_DLC; @@ -1394,7 +1405,6 @@ static struct gs_can *gs_make_candev(unsigned int channel, * GS_CAN_FEATURE_BT_CONST_EXT is set. */ dev->can.fd.data_bittiming_const = &dev->bt_const; - dev->can.fd.do_set_data_bittiming = gs_usb_set_data_bittiming; } if (feature & GS_CAN_FEATURE_TERMINATION) { diff --git a/drivers/net/can/usb/ucan.c b/drivers/net/can/usb/ucan.c index de61d9da99e359..87ddeff0937fb1 100644 --- a/drivers/net/can/usb/ucan.c +++ b/drivers/net/can/usb/ucan.c @@ -749,7 +749,7 @@ static void ucan_read_bulk_callback(struct urb *urb) len = le16_to_cpu(m->len); /* check sanity (length of content) */ - if (urb->actual_length - pos < len) { + if ((len == 0) || (urb->actual_length - pos < len)) { netdev_warn(up->netdev, "invalid message (short; no data; l:%d)\n", urb->actual_length); diff --git a/drivers/net/dsa/bcm_sf2.c b/drivers/net/dsa/bcm_sf2.c index 960685596093b6..de3efa3ce9a75f 100644 --- a/drivers/net/dsa/bcm_sf2.c +++ b/drivers/net/dsa/bcm_sf2.c @@ -980,15 +980,19 @@ static int bcm_sf2_sw_resume(struct dsa_switch *ds) ret = bcm_sf2_sw_rst(priv); if (ret) { pr_err("%s: failed to software reset switch\n", __func__); + if (!priv->wol_ports_mask) + clk_disable_unprepare(priv->clk); return ret; } bcm_sf2_crossbar_setup(priv); ret = bcm_sf2_cfp_resume(ds); - if (ret) + if (ret) { + if (!priv->wol_ports_mask) + clk_disable_unprepare(priv->clk); return ret; - + } if (priv->hw_params.num_gphy == 1) bcm_sf2_gphy_enable_set(ds, true); diff --git a/drivers/net/dsa/microchip/ksz_ptp.c b/drivers/net/dsa/microchip/ksz_ptp.c index 997e4a76d0a684..318ab38ac49b29 100644 --- a/drivers/net/dsa/microchip/ksz_ptp.c +++ b/drivers/net/dsa/microchip/ksz_ptp.c @@ -1095,6 +1095,7 @@ static int ksz_ptp_msg_irq_setup(struct ksz_port *port, u8 n) const struct ksz_dev_ops *ops = port->ksz_dev->dev_ops; struct ksz_irq *ptpirq = &port->ptpirq; struct ksz_ptp_irq *ptpmsg_irq; + int ret; ptpmsg_irq = &port->ptpmsg_irq[n]; ptpmsg_irq->num = irq_create_mapping(ptpirq->domain, n); @@ -1106,9 +1107,13 @@ static int ksz_ptp_msg_irq_setup(struct ksz_port *port, u8 n) strscpy(ptpmsg_irq->name, name[n]); - return request_threaded_irq(ptpmsg_irq->num, NULL, - ksz_ptp_msg_thread_fn, IRQF_ONESHOT, - ptpmsg_irq->name, ptpmsg_irq); + ret = request_threaded_irq(ptpmsg_irq->num, NULL, + ksz_ptp_msg_thread_fn, IRQF_ONESHOT, + ptpmsg_irq->name, ptpmsg_irq); + if (ret) + irq_dispose_mapping(ptpmsg_irq->num); + + return ret; } int ksz_ptp_irq_setup(struct dsa_switch *ds, u8 p) diff --git a/drivers/net/dsa/realtek/rtl8365mb.c b/drivers/net/dsa/realtek/rtl8365mb.c index c575e164368c84..31fa94dac627d6 100644 --- a/drivers/net/dsa/realtek/rtl8365mb.c +++ b/drivers/net/dsa/realtek/rtl8365mb.c @@ -769,7 +769,7 @@ static int rtl8365mb_phy_ocp_write(struct realtek_priv *priv, int phy, out: rtl83xx_unlock(priv); - return 0; + return ret; } static int rtl8365mb_phy_read(struct realtek_priv *priv, int phy, int regnum) @@ -1480,8 +1480,7 @@ static void rtl8365mb_stats_update(struct realtek_priv *priv, int port) stats->rx_packets = cnt[RTL8365MB_MIB_ifInUcastPkts] + cnt[RTL8365MB_MIB_ifInMulticastPkts] + - cnt[RTL8365MB_MIB_ifInBroadcastPkts] - - cnt[RTL8365MB_MIB_ifOutDiscards]; + cnt[RTL8365MB_MIB_ifInBroadcastPkts]; stats->tx_packets = cnt[RTL8365MB_MIB_ifOutUcastPkts] + cnt[RTL8365MB_MIB_ifOutMulticastPkts] + diff --git a/drivers/net/dsa/realtek/rtl8366rb-leds.c b/drivers/net/dsa/realtek/rtl8366rb-leds.c index 99c890681ae607..509ffd3f8db5cb 100644 --- a/drivers/net/dsa/realtek/rtl8366rb-leds.c +++ b/drivers/net/dsa/realtek/rtl8366rb-leds.c @@ -12,11 +12,11 @@ static inline u32 rtl8366rb_led_group_port_mask(u8 led_group, u8 port) case 0: return FIELD_PREP(RTL8366RB_LED_0_X_CTRL_MASK, BIT(port)); case 1: - return FIELD_PREP(RTL8366RB_LED_0_X_CTRL_MASK, BIT(port)); + return FIELD_PREP(RTL8366RB_LED_X_1_CTRL_MASK, BIT(port)); case 2: - return FIELD_PREP(RTL8366RB_LED_0_X_CTRL_MASK, BIT(port)); + return FIELD_PREP(RTL8366RB_LED_2_X_CTRL_MASK, BIT(port)); case 3: - return FIELD_PREP(RTL8366RB_LED_0_X_CTRL_MASK, BIT(port)); + return FIELD_PREP(RTL8366RB_LED_X_3_CTRL_MASK, BIT(port)); default: return 0; } diff --git a/drivers/net/ethernet/airoha/airoha_eth.c b/drivers/net/ethernet/airoha/airoha_eth.c index 315d97036ac1d6..fee5b2eddebb01 100644 --- a/drivers/net/ethernet/airoha/airoha_eth.c +++ b/drivers/net/ethernet/airoha/airoha_eth.c @@ -697,9 +697,8 @@ static int airoha_qdma_rx_process(struct airoha_queue *q, int budget) if (q->skb) { dev_kfree_skb(q->skb); q->skb = NULL; - } else { - page_pool_put_full_page(q->page_pool, page, true); } + page_pool_put_full_page(q->page_pool, page, true); } airoha_qdma_fill_rx_queue(q); @@ -794,18 +793,34 @@ static int airoha_qdma_init_rx_queue(struct airoha_queue *q, static void airoha_qdma_cleanup_rx_queue(struct airoha_queue *q) { - struct airoha_eth *eth = q->qdma->eth; + struct airoha_qdma *qdma = q->qdma; + struct airoha_eth *eth = qdma->eth; + int qid = q - &qdma->q_rx[0]; while (q->queued) { struct airoha_queue_entry *e = &q->entry[q->tail]; + struct airoha_qdma_desc *desc = &q->desc[q->tail]; struct page *page = virt_to_head_page(e->buf); dma_sync_single_for_cpu(eth->dev, e->dma_addr, e->dma_len, page_pool_get_dma_dir(q->page_pool)); page_pool_put_full_page(q->page_pool, page, false); + /* Reset DMA descriptor */ + WRITE_ONCE(desc->ctrl, 0); + WRITE_ONCE(desc->addr, 0); + WRITE_ONCE(desc->data, 0); + WRITE_ONCE(desc->msg0, 0); + WRITE_ONCE(desc->msg1, 0); + WRITE_ONCE(desc->msg2, 0); + WRITE_ONCE(desc->msg3, 0); + q->tail = (q->tail + 1) % q->ndesc; q->queued--; } + + q->head = q->tail; + airoha_qdma_rmw(qdma, REG_RX_DMA_IDX(qid), RX_RING_DMA_IDX_MASK, + FIELD_PREP(RX_RING_DMA_IDX_MASK, q->tail)); } static int airoha_qdma_init_rx(struct airoha_qdma *qdma) @@ -3080,7 +3095,6 @@ static void airoha_remove(struct platform_device *pdev) if (!port) continue; - airoha_dev_stop(port->dev); unregister_netdev(port->dev); airoha_metadata_dst_free(port); } diff --git a/drivers/net/ethernet/airoha/airoha_ppe.c b/drivers/net/ethernet/airoha/airoha_ppe.c index 2221bafaf7c9fe..36e4f328c6e819 100644 --- a/drivers/net/ethernet/airoha/airoha_ppe.c +++ b/drivers/net/ethernet/airoha/airoha_ppe.c @@ -227,7 +227,9 @@ static int airoha_ppe_get_wdma_info(struct net_device *dev, const u8 *addr, if (!dev) return -ENODEV; + rcu_read_lock(); err = dev_fill_forward_path(dev, addr, &stack); + rcu_read_unlock(); if (err) return err; diff --git a/drivers/net/ethernet/altera/altera_tse_main.c b/drivers/net/ethernet/altera/altera_tse_main.c index ca55c5fd11dfdf..7011558e56fd2b 100644 --- a/drivers/net/ethernet/altera/altera_tse_main.c +++ b/drivers/net/ethernet/altera/altera_tse_main.c @@ -572,6 +572,7 @@ static netdev_tx_t tse_start_xmit(struct sk_buff *skb, struct net_device *dev) DMA_TO_DEVICE); if (dma_mapping_error(priv->device, dma_addr)) { netdev_err(priv->dev, "%s: DMA mapping error\n", __func__); + dev_kfree_skb_any(skb); ret = NETDEV_TX_OK; goto out; } diff --git a/drivers/net/ethernet/amd/Kconfig b/drivers/net/ethernet/amd/Kconfig index d54dca3074eb66..45e8d698781c15 100644 --- a/drivers/net/ethernet/amd/Kconfig +++ b/drivers/net/ethernet/amd/Kconfig @@ -165,7 +165,7 @@ config AMD_XGBE select CRC32 select PHYLIB select AMD_XGBE_HAVE_ECC if X86 - select NET_SELFTESTS + imply NET_SELFTESTS help This driver supports the AMD 10GbE Ethernet device found on an AMD SoC. diff --git a/drivers/net/ethernet/amd/xgbe/xgbe-common.h b/drivers/net/ethernet/amd/xgbe/xgbe-common.h index 62b01de93db49a..826c5caa70d715 100644 --- a/drivers/net/ethernet/amd/xgbe/xgbe-common.h +++ b/drivers/net/ethernet/amd/xgbe/xgbe-common.h @@ -431,7 +431,7 @@ #define MAC_SSIR_SSINC_INDEX 16 #define MAC_SSIR_SSINC_WIDTH 8 #define MAC_TCR_SS_INDEX 29 -#define MAC_TCR_SS_WIDTH 2 +#define MAC_TCR_SS_WIDTH 3 #define MAC_TCR_TE_INDEX 0 #define MAC_TCR_TE_WIDTH 1 #define MAC_TCR_VNE_INDEX 24 diff --git a/drivers/net/ethernet/amd/xgbe/xgbe-drv.c b/drivers/net/ethernet/amd/xgbe/xgbe-drv.c index b5a60a0488967d..6de12a0e06553e 100644 --- a/drivers/net/ethernet/amd/xgbe/xgbe-drv.c +++ b/drivers/net/ethernet/amd/xgbe/xgbe-drv.c @@ -1120,7 +1120,6 @@ int xgbe_powerdown(struct net_device *netdev, unsigned int caller) { struct xgbe_prv_data *pdata = netdev_priv(netdev); struct xgbe_hw_if *hw_if = &pdata->hw_if; - unsigned long flags; DBGPR("-->xgbe_powerdown\n"); @@ -1131,8 +1130,6 @@ int xgbe_powerdown(struct net_device *netdev, unsigned int caller) return -EINVAL; } - spin_lock_irqsave(&pdata->lock, flags); - if (caller == XGMAC_DRIVER_CONTEXT) netif_device_detach(netdev); @@ -1148,8 +1145,6 @@ int xgbe_powerdown(struct net_device *netdev, unsigned int caller) pdata->power_down = 1; - spin_unlock_irqrestore(&pdata->lock, flags); - DBGPR("<--xgbe_powerdown\n"); return 0; @@ -1159,7 +1154,6 @@ int xgbe_powerup(struct net_device *netdev, unsigned int caller) { struct xgbe_prv_data *pdata = netdev_priv(netdev); struct xgbe_hw_if *hw_if = &pdata->hw_if; - unsigned long flags; DBGPR("-->xgbe_powerup\n"); @@ -1170,8 +1164,6 @@ int xgbe_powerup(struct net_device *netdev, unsigned int caller) return -EINVAL; } - spin_lock_irqsave(&pdata->lock, flags); - pdata->power_down = 0; xgbe_napi_enable(pdata, 0); @@ -1186,8 +1178,6 @@ int xgbe_powerup(struct net_device *netdev, unsigned int caller) xgbe_start_timers(pdata); - spin_unlock_irqrestore(&pdata->lock, flags); - DBGPR("<--xgbe_powerup\n"); return 0; @@ -1281,20 +1271,25 @@ static int xgbe_start(struct xgbe_prv_data *pdata) if (ret) goto err_napi; + /* Reset the phy settings */ + ret = xgbe_phy_reset(pdata); + if (ret) + goto err_irqs; + + /* Start the phy */ ret = phy_if->phy_start(pdata); if (ret) goto err_irqs; hw_if->enable_tx(pdata); hw_if->enable_rx(pdata); + /* Synchronize flag with hardware state after enabling TX/RX. + * This prevents stale state after device restart cycles. + */ + pdata->data_path_stopped = false; udp_tunnel_nic_reset_ntf(netdev); - /* Reset the phy settings */ - ret = xgbe_phy_reset(pdata); - if (ret) - goto err_txrx; - netif_tx_start_all_queues(netdev); xgbe_start_timers(pdata); @@ -1304,10 +1299,6 @@ static int xgbe_start(struct xgbe_prv_data *pdata) return 0; -err_txrx: - hw_if->disable_rx(pdata); - hw_if->disable_tx(pdata); - err_irqs: xgbe_free_irqs(pdata); diff --git a/drivers/net/ethernet/amd/xgbe/xgbe-main.c b/drivers/net/ethernet/amd/xgbe/xgbe-main.c index d1f0419edb2349..7d45ea22a02e29 100644 --- a/drivers/net/ethernet/amd/xgbe/xgbe-main.c +++ b/drivers/net/ethernet/amd/xgbe/xgbe-main.c @@ -76,7 +76,6 @@ struct xgbe_prv_data *xgbe_alloc_pdata(struct device *dev) pdata->netdev = netdev; pdata->dev = dev; - spin_lock_init(&pdata->lock); spin_lock_init(&pdata->xpcs_lock); mutex_init(&pdata->rss_mutex); spin_lock_init(&pdata->tstamp_lock); diff --git a/drivers/net/ethernet/amd/xgbe/xgbe-phy-v2.c b/drivers/net/ethernet/amd/xgbe/xgbe-phy-v2.c index c63ddb12237eaa..b8cf6ccfe64148 100644 --- a/drivers/net/ethernet/amd/xgbe/xgbe-phy-v2.c +++ b/drivers/net/ethernet/amd/xgbe/xgbe-phy-v2.c @@ -1942,7 +1942,7 @@ static void xgbe_set_rx_adap_mode(struct xgbe_prv_data *pdata, static void xgbe_rx_adaptation(struct xgbe_prv_data *pdata) { struct xgbe_phy_data *phy_data = pdata->phy_data; - unsigned int reg; + int reg; /* step 2: force PCS to send RX_ADAPT Req to PHY */ XMDIO_WRITE_BITS(pdata, MDIO_MMD_PMAPMD, MDIO_PMA_RX_EQ_CTRL4, @@ -1964,11 +1964,20 @@ static void xgbe_rx_adaptation(struct xgbe_prv_data *pdata) /* Step 4: Check for Block lock */ - /* Link status is latched low, so read once to clear - * and then read again to get current state - */ - reg = XMDIO_READ(pdata, MDIO_MMD_PCS, MDIO_STAT1); reg = XMDIO_READ(pdata, MDIO_MMD_PCS, MDIO_STAT1); + if (reg < 0) + goto set_mode; + + /* Link status is latched low so that momentary link drops + * can be detected. If link was already down read again + * to get the latest state. + */ + if (!pdata->phy.link && !(reg & MDIO_STAT1_LSTATUS)) { + reg = XMDIO_READ(pdata, MDIO_MMD_PCS, MDIO_STAT1); + if (reg < 0) + goto set_mode; + } + if (reg & MDIO_STAT1_LSTATUS) { /* If the block lock is found, update the helpers * and declare the link up @@ -2008,6 +2017,48 @@ static void xgbe_phy_rx_adaptation(struct xgbe_prv_data *pdata) xgbe_rx_adaptation(pdata); } +/* + * xgbe_phy_stop_data_path - Stop TX/RX to prevent packet corruption + * @pdata: driver private data + * + * This function stops the data path (TX and RX) to prevent packet + * corruption during critical PHY operations like RX adaptation. + * Must be called before initiating RX adaptation when link goes down. + */ +static void xgbe_phy_stop_data_path(struct xgbe_prv_data *pdata) +{ + if (pdata->data_path_stopped) + return; + + /* Stop TX/RX to prevent packet corruption during RX adaptation */ + pdata->hw_if.disable_tx(pdata); + pdata->hw_if.disable_rx(pdata); + pdata->data_path_stopped = true; + + netif_dbg(pdata, link, pdata->netdev, + "stopping data path for RX adaptation\n"); +} + +/* + * xgbe_phy_start_data_path - Re-enable TX/RX after RX adaptation + * @pdata: driver private data + * + * This function re-enables the data path (TX and RX) after RX adaptation + * has completed successfully. Only called when link is confirmed up. + */ +static void xgbe_phy_start_data_path(struct xgbe_prv_data *pdata) +{ + if (!pdata->data_path_stopped) + return; + + pdata->hw_if.enable_rx(pdata); + pdata->hw_if.enable_tx(pdata); + pdata->data_path_stopped = false; + + netif_dbg(pdata, link, pdata->netdev, + "restarting data path after RX adaptation\n"); +} + static void xgbe_phy_rx_reset(struct xgbe_prv_data *pdata) { int reg; @@ -2801,13 +2852,27 @@ static int xgbe_phy_link_status(struct xgbe_prv_data *pdata, int *an_restart) if (pdata->en_rx_adap) { /* if the link is available and adaptation is done, * declare link up + * + * Note: When link is up and adaptation is done, we can + * safely re-enable the data path if it was stopped + * for adaptation. */ - if ((reg & MDIO_STAT1_LSTATUS) && pdata->rx_adapt_done) + if ((reg & MDIO_STAT1_LSTATUS) && pdata->rx_adapt_done) { + xgbe_phy_start_data_path(pdata); return 1; + } /* If either link is not available or adaptation is not done, * retrigger the adaptation logic. (if the mode is not set, * then issue mailbox command first) */ + + /* CRITICAL: Stop data path BEFORE triggering RX adaptation + * to prevent CRC errors from packets corrupted during + * the adaptation process. This is especially important + * when AN is OFF in 10G KR mode. + */ + xgbe_phy_stop_data_path(pdata); + if (pdata->mode_set) { xgbe_phy_rx_adaptation(pdata); } else { @@ -2815,8 +2880,11 @@ static int xgbe_phy_link_status(struct xgbe_prv_data *pdata, int *an_restart) xgbe_phy_set_mode(pdata, phy_data->cur_mode); } - if (pdata->rx_adapt_done) + if (pdata->rx_adapt_done) { + /* Adaptation complete, safe to re-enable data path */ + xgbe_phy_start_data_path(pdata); return 1; + } } else if (reg & MDIO_STAT1_LSTATUS) return 1; diff --git a/drivers/net/ethernet/amd/xgbe/xgbe.h b/drivers/net/ethernet/amd/xgbe/xgbe.h index 03ef0f5484830a..3bc748c7cb24df 100644 --- a/drivers/net/ethernet/amd/xgbe/xgbe.h +++ b/drivers/net/ethernet/amd/xgbe/xgbe.h @@ -1003,9 +1003,6 @@ struct xgbe_prv_data { unsigned int pp3; unsigned int pp4; - /* Overall device lock */ - spinlock_t lock; - /* XPCS indirect addressing lock */ spinlock_t xpcs_lock; unsigned int xpcs_window_def_reg; @@ -1245,6 +1242,10 @@ struct xgbe_prv_data { bool en_rx_adap; int rx_adapt_retries; bool rx_adapt_done; + /* Flag to track if data path (TX/RX) was stopped for RX adaptation. + * This prevents packet corruption during the adaptation window. + */ + bool data_path_stopped; bool mode_set; bool sph; }; diff --git a/drivers/net/ethernet/arc/emac_main.c b/drivers/net/ethernet/arc/emac_main.c index 8283aeee35fb6d..dde4046cbf010e 100644 --- a/drivers/net/ethernet/arc/emac_main.c +++ b/drivers/net/ethernet/arc/emac_main.c @@ -934,6 +934,17 @@ int arc_emac_probe(struct net_device *ndev, int interface) /* Set poll rate so that it polls every 1 ms */ arc_reg_set(priv, R_POLLRATE, clock_frequency / 1000000); + /* + * Put the device into a known quiescent state before requesting + * the IRQ. Clear only EMAC interrupt status bits here; leave the + * MDIO completion bit alone and avoid writing TXPL_MASK, which is + * used to force TX polling rather than acknowledge interrupts. + */ + arc_reg_set(priv, R_ENABLE, 0); + arc_reg_set(priv, R_STATUS, RXINT_MASK | TXINT_MASK | ERR_MASK | + TXCH_MASK | MSER_MASK | RXCR_MASK | + RXFR_MASK | RXFL_MASK); + ndev->irq = irq; dev_info(dev, "IRQ is %d\n", ndev->irq); diff --git a/drivers/net/ethernet/broadcom/Kconfig b/drivers/net/ethernet/broadcom/Kconfig index cd7dddeb91dd67..9787c1857e13bf 100644 --- a/drivers/net/ethernet/broadcom/Kconfig +++ b/drivers/net/ethernet/broadcom/Kconfig @@ -25,7 +25,7 @@ config B44 select SSB select MII select PHYLIB - select FIXED_PHY if BCM47XX + select FIXED_PHY help If you have a network (Ethernet) controller of this type, say Y or M here. diff --git a/drivers/net/ethernet/broadcom/asp2/bcmasp.c b/drivers/net/ethernet/broadcom/asp2/bcmasp.c index 014340f33345a0..1fdf0822c8a022 100644 --- a/drivers/net/ethernet/broadcom/asp2/bcmasp.c +++ b/drivers/net/ethernet/broadcom/asp2/bcmasp.c @@ -1157,12 +1157,6 @@ void bcmasp_enable_wol(struct bcmasp_intf *intf, bool en) } } -static void bcmasp_wol_irq_destroy(struct bcmasp_priv *priv) -{ - if (priv->wol_irq > 0) - free_irq(priv->wol_irq, priv); -} - static void bcmasp_eee_fixup(struct bcmasp_intf *intf, bool en) { u32 reg, phy_lpi_overwrite; @@ -1260,7 +1254,7 @@ static int bcmasp_probe(struct platform_device *pdev) if (priv->irq <= 0) return -EINVAL; - priv->clk = devm_clk_get_optional_enabled(dev, "sw_asp"); + priv->clk = devm_clk_get_optional(dev, "sw_asp"); if (IS_ERR(priv->clk)) return dev_err_probe(dev, PTR_ERR(priv->clk), "failed to request clock\n"); @@ -1288,6 +1282,10 @@ static int bcmasp_probe(struct platform_device *pdev) bcmasp_set_pdata(priv, pdata); + ret = clk_prepare_enable(priv->clk); + if (ret) + return dev_err_probe(dev, ret, "failed to start clock\n"); + /* Enable all clocks to ensure successful probing */ bcmasp_core_clock_set(priv, ASP_CTRL_CLOCK_CTRL_ASP_ALL_DISABLE, 0); @@ -1299,8 +1297,10 @@ static int bcmasp_probe(struct platform_device *pdev) ret = devm_request_irq(&pdev->dev, priv->irq, bcmasp_isr, 0, pdev->name, priv); - if (ret) - return dev_err_probe(dev, ret, "failed to request ASP interrupt: %d", ret); + if (ret) { + dev_err(dev, "Failed to request ASP interrupt: %d", ret); + goto err_clock_disable; + } /* Register mdio child nodes */ of_platform_populate(dev->of_node, bcmasp_mdio_of_match, NULL, dev); @@ -1312,20 +1312,27 @@ static int bcmasp_probe(struct platform_device *pdev) priv->mda_filters = devm_kcalloc(dev, priv->num_mda_filters, sizeof(*priv->mda_filters), GFP_KERNEL); - if (!priv->mda_filters) - return -ENOMEM; + if (!priv->mda_filters) { + ret = -ENOMEM; + goto err_clock_disable; + } priv->net_filters = devm_kcalloc(dev, priv->num_net_filters, sizeof(*priv->net_filters), GFP_KERNEL); - if (!priv->net_filters) - return -ENOMEM; + if (!priv->net_filters) { + ret = -ENOMEM; + goto err_clock_disable; + } bcmasp_core_init_filters(priv); + bcmasp_init_wol(priv); + ports_node = of_find_node_by_name(dev->of_node, "ethernet-ports"); if (!ports_node) { dev_warn(dev, "No ports found\n"); - return -EINVAL; + ret = -EINVAL; + goto err_clock_disable; } i = 0; @@ -1333,43 +1340,43 @@ static int bcmasp_probe(struct platform_device *pdev) intf = bcmasp_interface_create(priv, intf_node, i); if (!intf) { dev_err(dev, "Cannot create eth interface %d\n", i); - bcmasp_remove_intfs(priv); - ret = -ENOMEM; - goto of_put_exit; + of_node_put(ports_node); + ret = -EINVAL; + goto err_cleanup; } list_add_tail(&intf->list, &priv->intfs); i++; } - - /* Check and enable WoL */ - bcmasp_init_wol(priv); + of_node_put(ports_node); /* Drop the clock reference count now and let ndo_open()/ndo_close() * manage it for us from now on. */ bcmasp_core_clock_set(priv, 0, ASP_CTRL_CLOCK_CTRL_ASP_ALL_DISABLE); - clk_disable_unprepare(priv->clk); - /* Now do the registration of the network ports which will take care * of managing the clock properly. */ list_for_each_entry(intf, &priv->intfs, list) { ret = register_netdev(intf->ndev); if (ret) { - netdev_err(intf->ndev, - "failed to register net_device: %d\n", ret); - bcmasp_wol_irq_destroy(priv); - bcmasp_remove_intfs(priv); - goto of_put_exit; + dev_err(dev, "failed to register net_device: %d\n", ret); + goto err_cleanup; } count++; } + clk_disable_unprepare(priv->clk); + dev_info(dev, "Initialized %d port(s)\n", count); -of_put_exit: - of_node_put(ports_node); + return ret; + +err_cleanup: + bcmasp_remove_intfs(priv); +err_clock_disable: + clk_disable_unprepare(priv->clk); + return ret; } @@ -1380,7 +1387,6 @@ static void bcmasp_remove(struct platform_device *pdev) if (!priv) return; - bcmasp_wol_irq_destroy(priv); bcmasp_remove_intfs(priv); } diff --git a/drivers/net/ethernet/broadcom/bnge/bnge_auxr.c b/drivers/net/ethernet/broadcom/bnge/bnge_auxr.c index d64592b64e17c1..940ffffd11d86b 100644 --- a/drivers/net/ethernet/broadcom/bnge/bnge_auxr.c +++ b/drivers/net/ethernet/broadcom/bnge/bnge_auxr.c @@ -194,6 +194,7 @@ void bnge_rdma_aux_device_add(struct bnge_dev *bd) dev_warn(bd->dev, "Failed to add auxiliary device for ROCE\n"); auxiliary_device_uninit(aux_dev); bd->flags &= ~BNGE_EN_ROCE; + return; } bd->auxr_dev->net = bd->netdev; diff --git a/drivers/net/ethernet/broadcom/bnge/bnge_hwrm_lib.c b/drivers/net/ethernet/broadcom/bnge/bnge_hwrm_lib.c index 198f49b40dbf00..2994f10446a63c 100644 --- a/drivers/net/ethernet/broadcom/bnge/bnge_hwrm_lib.c +++ b/drivers/net/ethernet/broadcom/bnge/bnge_hwrm_lib.c @@ -442,7 +442,7 @@ __bnge_hwrm_reserve_pf_rings(struct bnge_dev *bd, struct bnge_hw_rings *hwr) struct hwrm_func_cfg_input *req; u32 enables = 0; - if (bnge_hwrm_req_init(bd, req, HWRM_FUNC_QCFG)) + if (bnge_hwrm_req_init(bd, req, HWRM_FUNC_CFG)) return NULL; req->fid = cpu_to_le16(0xffff); diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.c b/drivers/net/ethernet/broadcom/bnxt/bnxt.c index 8419d1eb4035dc..d8c42349ded184 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.c @@ -2927,6 +2927,8 @@ static int bnxt_async_event_process(struct bnxt *bp, u16 type = (u16)BNXT_EVENT_BUF_PRODUCER_TYPE(data1); u32 offset = BNXT_EVENT_BUF_PRODUCER_OFFSET(data2); + if (type >= ARRAY_SIZE(bp->bs_trace)) + goto async_event_process_exit; bnxt_bs_trace_check_wrap(&bp->bs_trace[type], offset); goto async_event_process_exit; } @@ -6212,6 +6214,9 @@ int bnxt_hwrm_cfa_ntuple_filter_free(struct bnxt *bp, int rc; set_bit(BNXT_FLTR_FW_DELETED, &fltr->base.state); + if (!test_bit(BNXT_STATE_OPEN, &bp->state)) + return 0; + rc = hwrm_req_init(bp, req, HWRM_CFA_NTUPLE_FILTER_FREE); if (rc) return rc; @@ -7997,6 +8002,8 @@ static int __bnxt_reserve_rings(struct bnxt *bp) ulp_msix = bnxt_get_avail_msix(bp, bp->ulp_num_msix_want); if (!ulp_msix) bnxt_set_ulp_stat_ctxs(bp, 0); + else + bnxt_set_dflt_ulp_stat_ctxs(bp); if (ulp_msix > bp->ulp_num_msix_want) ulp_msix = bp->ulp_num_msix_want; @@ -8618,7 +8625,7 @@ static int bnxt_hwrm_func_backing_store_qcaps_v2(struct bnxt *bp) struct hwrm_func_backing_store_qcaps_v2_output *resp; struct hwrm_func_backing_store_qcaps_v2_input *req; struct bnxt_ctx_mem_info *ctx = bp->ctx; - u16 type; + u16 type, next_type = 0; int rc; rc = hwrm_req_init(bp, req, HWRM_FUNC_BACKING_STORE_QCAPS_V2); @@ -8634,7 +8641,7 @@ static int bnxt_hwrm_func_backing_store_qcaps_v2(struct bnxt *bp) resp = hwrm_req_hold(bp, req); - for (type = 0; type < BNXT_CTX_V2_MAX; ) { + for (type = 0; type < BNXT_CTX_V2_MAX; type = next_type) { struct bnxt_ctx_mem_type *ctxm = &ctx->ctx_arr[type]; u8 init_val, init_off, i; u32 max_entries; @@ -8647,7 +8654,7 @@ static int bnxt_hwrm_func_backing_store_qcaps_v2(struct bnxt *bp) if (rc) goto ctx_done; flags = le32_to_cpu(resp->flags); - type = le16_to_cpu(resp->next_valid_type); + next_type = le16_to_cpu(resp->next_valid_type); if (!(flags & BNXT_CTX_MEM_TYPE_VALID)) { bnxt_free_one_ctx_mem(bp, ctxm, true); continue; @@ -8662,7 +8669,7 @@ static int bnxt_hwrm_func_backing_store_qcaps_v2(struct bnxt *bp) else continue; } - ctxm->type = le16_to_cpu(resp->type); + ctxm->type = type; ctxm->entry_size = entry_size; ctxm->flags = flags; ctxm->instance_bmap = le32_to_cpu(resp->instance_bit_map); @@ -10827,12 +10834,10 @@ void bnxt_del_one_rss_ctx(struct bnxt *bp, struct bnxt_rss_ctx *rss_ctx, struct bnxt_ntuple_filter *ntp_fltr; int i; - if (netif_running(bp->dev)) { - bnxt_hwrm_vnic_free_one(bp, &rss_ctx->vnic); - for (i = 0; i < BNXT_MAX_CTX_PER_VNIC; i++) { - if (vnic->fw_rss_cos_lb_ctx[i] != INVALID_HW_RING_ID) - bnxt_hwrm_vnic_ctx_free_one(bp, vnic, i); - } + bnxt_hwrm_vnic_free_one(bp, &rss_ctx->vnic); + for (i = 0; i < BNXT_MAX_CTX_PER_VNIC; i++) { + if (vnic->fw_rss_cos_lb_ctx[i] != INVALID_HW_RING_ID) + bnxt_hwrm_vnic_ctx_free_one(bp, vnic, i); } if (!all) return; @@ -12914,6 +12919,21 @@ static int bnxt_tx_nr_rings_per_tc(struct bnxt *bp) return bp->num_tc ? bp->tx_nr_rings / bp->num_tc : bp->tx_nr_rings; } +static void bnxt_set_xdp_tx_rings(struct bnxt *bp) +{ + bp->tx_nr_rings_xdp = bp->tx_nr_rings_per_tc; + bp->tx_nr_rings += bp->tx_nr_rings_xdp; +} + +static void bnxt_adj_tx_rings(struct bnxt *bp) +{ + /* Make adjustments if reserved TX rings are less than requested */ + bp->tx_nr_rings -= bp->tx_nr_rings_xdp; + bp->tx_nr_rings_per_tc = bnxt_tx_nr_rings_per_tc(bp); + if (bp->tx_nr_rings_xdp) + bnxt_set_xdp_tx_rings(bp); +} + static int __bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init) { int rc = 0; @@ -12931,13 +12951,7 @@ static int __bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init) if (rc) return rc; - /* Make adjustments if reserved TX rings are less than requested */ - bp->tx_nr_rings -= bp->tx_nr_rings_xdp; - bp->tx_nr_rings_per_tc = bnxt_tx_nr_rings_per_tc(bp); - if (bp->tx_nr_rings_xdp) { - bp->tx_nr_rings_xdp = bp->tx_nr_rings_per_tc; - bp->tx_nr_rings += bp->tx_nr_rings_xdp; - } + bnxt_adj_tx_rings(bp); rc = bnxt_alloc_mem(bp, irq_re_init); if (rc) { netdev_err(bp->dev, "bnxt_alloc_mem err: %x\n", rc); @@ -15374,11 +15388,19 @@ static int bnxt_change_mtu(struct net_device *dev, int new_mtu) return 0; } +void bnxt_set_cp_rings(struct bnxt *bp, bool sh) +{ + int tx_cp = bnxt_num_tx_to_cp(bp, bp->tx_nr_rings); + + bp->cp_nr_rings = sh ? max_t(int, tx_cp, bp->rx_nr_rings) : + tx_cp + bp->rx_nr_rings; +} + int bnxt_setup_mq_tc(struct net_device *dev, u8 tc) { struct bnxt *bp = netdev_priv(dev); bool sh = false; - int rc, tx_cp; + int rc; if (tc > bp->max_tc) { netdev_err(dev, "Too many traffic classes requested: %d. Max supported is %d.\n", @@ -15411,9 +15433,7 @@ int bnxt_setup_mq_tc(struct net_device *dev, u8 tc) bp->num_tc = 0; } bp->tx_nr_rings += bp->tx_nr_rings_xdp; - tx_cp = bnxt_num_tx_to_cp(bp, bp->tx_nr_rings); - bp->cp_nr_rings = sh ? max_t(int, tx_cp, bp->rx_nr_rings) : - tx_cp + bp->rx_nr_rings; + bnxt_set_cp_rings(bp, sh); if (netif_running(bp->dev)) return bnxt_open_nic(bp, true, false); @@ -16418,6 +16438,19 @@ static void bnxt_trim_dflt_sh_rings(struct bnxt *bp) bp->tx_nr_rings = bnxt_tx_nr_rings(bp); } +static void bnxt_adj_dflt_rings(struct bnxt *bp, bool sh) +{ + if (sh) + bnxt_trim_dflt_sh_rings(bp); + else + bp->cp_nr_rings = bp->tx_nr_rings_per_tc + bp->rx_nr_rings; + bp->tx_nr_rings = bnxt_tx_nr_rings(bp); + if (sh && READ_ONCE(bp->xdp_prog)) { + bnxt_set_xdp_tx_rings(bp); + bnxt_set_cp_rings(bp, true); + } +} + static int bnxt_set_dflt_rings(struct bnxt *bp, bool sh) { int dflt_rings, max_rx_rings, max_tx_rings, rc; @@ -16443,11 +16476,8 @@ static int bnxt_set_dflt_rings(struct bnxt *bp, bool sh) return rc; bp->rx_nr_rings = min_t(int, dflt_rings, max_rx_rings); bp->tx_nr_rings_per_tc = min_t(int, dflt_rings, max_tx_rings); - if (sh) - bnxt_trim_dflt_sh_rings(bp); - else - bp->cp_nr_rings = bp->tx_nr_rings_per_tc + bp->rx_nr_rings; - bp->tx_nr_rings = bnxt_tx_nr_rings(bp); + + bnxt_adj_dflt_rings(bp, sh); avail_msix = bnxt_get_max_func_irqs(bp) - bp->cp_nr_rings; if (avail_msix >= BNXT_MIN_ROCE_CP_RINGS) { @@ -16460,16 +16490,17 @@ static int bnxt_set_dflt_rings(struct bnxt *bp, bool sh) rc = __bnxt_reserve_rings(bp); if (rc && rc != -ENODEV) netdev_warn(bp->dev, "Unable to reserve tx rings\n"); - bp->tx_nr_rings_per_tc = bnxt_tx_nr_rings_per_tc(bp); + + bnxt_adj_tx_rings(bp); if (sh) - bnxt_trim_dflt_sh_rings(bp); + bnxt_adj_dflt_rings(bp, true); /* Rings may have been trimmed, re-reserve the trimmed rings. */ if (bnxt_need_reserve_rings(bp)) { rc = __bnxt_reserve_rings(bp); if (rc && rc != -ENODEV) netdev_warn(bp->dev, "2nd rings reservation failed.\n"); - bp->tx_nr_rings_per_tc = bnxt_tx_nr_rings_per_tc(bp); + bnxt_adj_tx_rings(bp); } if (BNXT_CHIP_TYPE_NITRO_A0(bp)) { bp->rx_nr_rings++; @@ -16503,7 +16534,7 @@ static int bnxt_init_dflt_ring_mode(struct bnxt *bp) if (rc) goto init_dflt_ring_err; - bp->tx_nr_rings_per_tc = bnxt_tx_nr_rings_per_tc(bp); + bnxt_adj_tx_rings(bp); bnxt_set_dflt_rfs(bp); diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.h b/drivers/net/ethernet/broadcom/bnxt/bnxt.h index f88e7769a838a1..9413818788c4e5 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt.h +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.h @@ -2135,7 +2135,7 @@ enum board_idx { }; #define BNXT_TRACE_BUF_MAGIC_BYTE ((u8)0xbc) -#define BNXT_TRACE_MAX 11 +#define BNXT_TRACE_MAX (DBG_LOG_BUFFER_FLUSH_REQ_TYPE_ERR_QPC_TRACE + 1) struct bnxt_bs_trace_info { u8 *magic_byte; @@ -2971,6 +2971,7 @@ int bnxt_check_rings(struct bnxt *bp, int tx, int rx, bool sh, int tcs, int tx_xdp); int bnxt_fw_init_one(struct bnxt *bp); bool bnxt_hwrm_reset_permitted(struct bnxt *bp); +void bnxt_set_cp_rings(struct bnxt *bp, bool sh); int bnxt_setup_mq_tc(struct net_device *dev, u8 tc); struct bnxt_ntuple_filter *bnxt_lookup_ntp_filter_from_idx(struct bnxt *bp, struct bnxt_ntuple_filter *fltr, u32 idx); diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c index 068e191ede19e6..34d9264d51950f 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c @@ -945,7 +945,6 @@ static int bnxt_set_channels(struct net_device *dev, bool sh = false; int tx_xdp = 0; int rc = 0; - int tx_cp; if (channel->other_count) return -EINVAL; @@ -979,8 +978,8 @@ static int bnxt_set_channels(struct net_device *dev, if (bnxt_get_nr_rss_ctxs(bp, req_rx_rings) != bnxt_get_nr_rss_ctxs(bp, bp->rx_nr_rings) && - netif_is_rxfh_configured(dev)) { - netdev_warn(dev, "RSS table size change required, RSS table entries must be default to proceed\n"); + (netif_is_rxfh_configured(dev) || bp->num_rss_ctx)) { + netdev_warn(dev, "RSS table size change required, RSS table entries must be default (with no additional RSS contexts present) to proceed\n"); return -EINVAL; } @@ -1013,9 +1012,7 @@ static int bnxt_set_channels(struct net_device *dev, if (tcs > 1) bp->tx_nr_rings = bp->tx_nr_rings_per_tc * tcs + tx_xdp; - tx_cp = bnxt_num_tx_to_cp(bp, bp->tx_nr_rings); - bp->cp_nr_rings = sh ? max_t(int, tx_cp, bp->rx_nr_rings) : - tx_cp + bp->rx_nr_rings; + bnxt_set_cp_rings(bp, sh); /* After changing number of rx channels, update NTUPLE feature. */ netdev_update_features(dev); @@ -1346,16 +1343,17 @@ static int bnxt_add_ntuple_cls_rule(struct bnxt *bp, struct bnxt_l2_filter *l2_fltr; struct bnxt_flow_masks *fmasks; struct flow_keys *fkeys; - u32 idx, ring; + u32 idx; int rc; - u8 vf; if (!bp->vnic_info) return -EAGAIN; - vf = ethtool_get_flow_spec_ring_vf(fs->ring_cookie); - ring = ethtool_get_flow_spec_ring(fs->ring_cookie); - if ((fs->flow_type & (FLOW_MAC_EXT | FLOW_EXT)) || vf) + if (fs->flow_type & (FLOW_MAC_EXT | FLOW_EXT)) + return -EOPNOTSUPP; + + if (fs->ring_cookie != RX_CLS_FLOW_DISC && + ethtool_get_flow_spec_ring_vf(fs->ring_cookie)) return -EOPNOTSUPP; if (flow_type == IP_USER_FLOW) { @@ -1481,7 +1479,7 @@ static int bnxt_add_ntuple_cls_rule(struct bnxt *bp, if (fs->ring_cookie == RX_CLS_FLOW_DISC) new_fltr->base.flags |= BNXT_ACT_DROP; else - new_fltr->base.rxq = ring; + new_fltr->base.rxq = ethtool_get_flow_spec_ring(fs->ring_cookie); __set_bit(BNXT_FLTR_VALID, &new_fltr->base.state); rc = bnxt_insert_ntp_filter(bp, new_fltr, idx); if (!rc) { diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c index c94a391b1ba5b2..06f35a61c17748 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c @@ -384,7 +384,7 @@ int bnxt_xdp_xmit(struct net_device *dev, int num_frames, static int bnxt_xdp_set(struct bnxt *bp, struct bpf_prog *prog) { struct net_device *dev = bp->dev; - int tx_xdp = 0, tx_cp, rc, tc; + int tx_xdp = 0, rc, tc; struct bpf_prog *old; netdev_assert_locked(dev); @@ -431,8 +431,7 @@ static int bnxt_xdp_set(struct bnxt *bp, struct bpf_prog *prog) } bp->tx_nr_rings_xdp = tx_xdp; bp->tx_nr_rings = bp->tx_nr_rings_per_tc * tc + tx_xdp; - tx_cp = bnxt_num_tx_to_cp(bp, bp->tx_nr_rings); - bp->cp_nr_rings = max_t(int, tx_cp, bp->rx_nr_rings); + bnxt_set_cp_rings(bp, true); bnxt_set_tpa_flags(bp); bnxt_set_ring_params(bp); diff --git a/drivers/net/ethernet/broadcom/genet/bcmgenet.c b/drivers/net/ethernet/broadcom/genet/bcmgenet.c index 05512aa10c209c..1c2fdaca14f9b9 100644 --- a/drivers/net/ethernet/broadcom/genet/bcmgenet.c +++ b/drivers/net/ethernet/broadcom/genet/bcmgenet.c @@ -1342,8 +1342,7 @@ static void bcmgenet_get_ethtool_stats(struct net_device *dev, } } -void bcmgenet_eee_enable_set(struct net_device *dev, bool enable, - bool tx_lpi_enabled) +void bcmgenet_eee_enable_set(struct net_device *dev, bool enable) { struct bcmgenet_priv *priv = netdev_priv(dev); u32 off = priv->hw_params->tbuf_offset + TBUF_ENERGY_CTRL; @@ -1363,7 +1362,7 @@ void bcmgenet_eee_enable_set(struct net_device *dev, bool enable, /* Enable EEE and switch to a 27Mhz clock automatically */ reg = bcmgenet_readl(priv->base + off); - if (tx_lpi_enabled) + if (enable) reg |= TBUF_EEE_EN | TBUF_PM_EN; else reg &= ~(TBUF_EEE_EN | TBUF_PM_EN); @@ -1382,14 +1381,12 @@ void bcmgenet_eee_enable_set(struct net_device *dev, bool enable, priv->clk_eee_enabled = false; } - priv->eee.eee_enabled = enable; - priv->eee.tx_lpi_enabled = tx_lpi_enabled; } static int bcmgenet_get_eee(struct net_device *dev, struct ethtool_keee *e) { struct bcmgenet_priv *priv = netdev_priv(dev); - struct ethtool_keee *p = &priv->eee; + int ret; if (GENET_IS_V1(priv)) return -EOPNOTSUPP; @@ -1397,17 +1394,21 @@ static int bcmgenet_get_eee(struct net_device *dev, struct ethtool_keee *e) if (!dev->phydev) return -ENODEV; - e->tx_lpi_enabled = p->tx_lpi_enabled; + ret = phy_ethtool_get_eee(dev->phydev, e); + if (ret) + return ret; + + /* tx_lpi_timer is maintained by the MAC hardware register; the + * PHY-level eee_cfg timer is not set for GENET. + */ e->tx_lpi_timer = bcmgenet_umac_readl(priv, UMAC_EEE_LPI_TIMER); - return phy_ethtool_get_eee(dev->phydev, e); + return 0; } static int bcmgenet_set_eee(struct net_device *dev, struct ethtool_keee *e) { struct bcmgenet_priv *priv = netdev_priv(dev); - struct ethtool_keee *p = &priv->eee; - bool active; if (GENET_IS_V1(priv)) return -EOPNOTSUPP; @@ -1415,15 +1416,7 @@ static int bcmgenet_set_eee(struct net_device *dev, struct ethtool_keee *e) if (!dev->phydev) return -ENODEV; - p->eee_enabled = e->eee_enabled; - - if (!p->eee_enabled) { - bcmgenet_eee_enable_set(dev, false, false); - } else { - active = phy_init_eee(dev->phydev, false) >= 0; - bcmgenet_umac_writel(priv, e->tx_lpi_timer, UMAC_EEE_LPI_TIMER); - bcmgenet_eee_enable_set(dev, active, e->tx_lpi_enabled); - } + bcmgenet_umac_writel(priv, e->tx_lpi_timer, UMAC_EEE_LPI_TIMER); return phy_ethtool_set_eee(dev->phydev, e); } diff --git a/drivers/net/ethernet/broadcom/genet/bcmgenet.h b/drivers/net/ethernet/broadcom/genet/bcmgenet.h index 5ec3979779ece6..9e4110c7fdf6f9 100644 --- a/drivers/net/ethernet/broadcom/genet/bcmgenet.h +++ b/drivers/net/ethernet/broadcom/genet/bcmgenet.h @@ -665,8 +665,6 @@ struct bcmgenet_priv { u8 sopass[SOPASS_MAX]; struct bcmgenet_mib_counters mib; - - struct ethtool_keee eee; }; static inline bool bcmgenet_has_40bits(struct bcmgenet_priv *priv) @@ -749,7 +747,6 @@ int bcmgenet_wol_power_down_cfg(struct bcmgenet_priv *priv, int bcmgenet_wol_power_up_cfg(struct bcmgenet_priv *priv, enum bcmgenet_power_mode mode); -void bcmgenet_eee_enable_set(struct net_device *dev, bool enable, - bool tx_lpi_enabled); +void bcmgenet_eee_enable_set(struct net_device *dev, bool enable); #endif /* __BCMGENET_H__ */ diff --git a/drivers/net/ethernet/broadcom/genet/bcmgenet_wol.c b/drivers/net/ethernet/broadcom/genet/bcmgenet_wol.c index 8fb55128829806..96d5d4f7f51fe8 100644 --- a/drivers/net/ethernet/broadcom/genet/bcmgenet_wol.c +++ b/drivers/net/ethernet/broadcom/genet/bcmgenet_wol.c @@ -123,7 +123,7 @@ static int bcmgenet_poll_wol_status(struct bcmgenet_priv *priv) while (!(bcmgenet_rbuf_readl(priv, RBUF_STATUS) & RBUF_STATUS_WOL)) { retries++; - if (retries > 5) { + if (retries > 50) { netdev_crit(dev, "polling wol mode timeout\n"); return -ETIMEDOUT; } diff --git a/drivers/net/ethernet/broadcom/genet/bcmmii.c b/drivers/net/ethernet/broadcom/genet/bcmmii.c index 38f854b94a799e..a4e0d5a6826875 100644 --- a/drivers/net/ethernet/broadcom/genet/bcmmii.c +++ b/drivers/net/ethernet/broadcom/genet/bcmmii.c @@ -29,7 +29,6 @@ static void bcmgenet_mac_config(struct net_device *dev) struct bcmgenet_priv *priv = netdev_priv(dev); struct phy_device *phydev = dev->phydev; u32 reg, cmd_bits = 0; - bool active; /* speed */ if (phydev->speed == SPEED_1000) @@ -90,10 +89,6 @@ static void bcmgenet_mac_config(struct net_device *dev) bcmgenet_umac_writel(priv, reg, UMAC_CMD); spin_unlock_bh(&priv->reg_lock); - active = phy_init_eee(phydev, 0) >= 0; - bcmgenet_eee_enable_set(dev, - priv->eee.eee_enabled && active, - priv->eee.tx_lpi_enabled); } /* setup netdev link state when PHY link status change and @@ -113,6 +108,8 @@ void bcmgenet_mii_setup(struct net_device *dev) bcmgenet_ext_writel(priv, reg, EXT_RGMII_OOB_CTRL); } + bcmgenet_eee_enable_set(dev, phydev->enable_tx_lpi); + phy_print_status(phydev); } @@ -412,6 +409,9 @@ int bcmgenet_mii_probe(struct net_device *dev) /* Indicate that the MAC is responsible for PHY PM */ dev->phydev->mac_managed_pm = true; + if (!GENET_IS_V1(priv)) + phy_support_eee(dev->phydev); + return 0; } diff --git a/drivers/net/ethernet/broadcom/tg3.c b/drivers/net/ethernet/broadcom/tg3.c index 75f66587983d73..1a59a2e53d865a 100644 --- a/drivers/net/ethernet/broadcom/tg3.c +++ b/drivers/net/ethernet/broadcom/tg3.c @@ -12300,7 +12300,7 @@ static int tg3_get_link_ksettings(struct net_device *dev, ethtool_convert_legacy_u32_to_link_mode(cmd->link_modes.advertising, advertising); - if (netif_running(dev) && tp->link_up) { + if (netif_running(dev) && netif_carrier_ok(dev)) { cmd->base.speed = tp->link_config.active_speed; cmd->base.duplex = tp->link_config.active_duplex; ethtool_convert_legacy_u32_to_link_mode( @@ -17030,6 +17030,13 @@ static int tg3_get_invariants(struct tg3 *tp, const struct pci_device_id *ent) return err; } +static int tg3_is_default_mac_address(u8 *addr) +{ + static const u8 default_mac_address[ETH_ALEN] = { 0x00, 0x10, 0x18, 0x00, 0x00, 0x00 }; + + return ether_addr_equal(default_mac_address, addr); +} + static int tg3_get_device_address(struct tg3 *tp, u8 *addr) { u32 hi, lo, mac_offset; @@ -17103,6 +17110,10 @@ static int tg3_get_device_address(struct tg3 *tp, u8 *addr) if (!is_valid_ether_addr(addr)) return -EINVAL; + + if (tg3_is_default_mac_address(addr)) + return device_get_mac_address(&tp->pdev->dev, addr); + return 0; } diff --git a/drivers/net/ethernet/cadence/macb_main.c b/drivers/net/ethernet/cadence/macb_main.c index 6511ecd5856bdb..ba794f2325f101 100644 --- a/drivers/net/ethernet/cadence/macb_main.c +++ b/drivers/net/ethernet/cadence/macb_main.c @@ -36,6 +36,7 @@ #include #include #include +#include #include #include "macb.h" @@ -668,6 +669,97 @@ static void macb_mac_link_down(struct phylink_config *config, unsigned int mode, netif_tx_stop_all_queues(ndev); } +/* Use juggling algorithm to left rotate tx ring and tx skb array */ +static void gem_shuffle_tx_one_ring(struct macb_queue *queue) +{ + unsigned int head, tail, count, ring_size, desc_size; + struct macb_tx_skb tx_skb, *skb_curr, *skb_next; + struct macb_dma_desc *desc_curr, *desc_next; + unsigned int i, cycles, shift, curr, next; + struct macb *bp = queue->bp; + unsigned char desc[24]; + unsigned long flags; + + desc_size = macb_dma_desc_get_size(bp); + + if (WARN_ON_ONCE(desc_size > ARRAY_SIZE(desc))) + return; + + spin_lock_irqsave(&queue->tx_ptr_lock, flags); + head = queue->tx_head; + tail = queue->tx_tail; + ring_size = bp->tx_ring_size; + count = CIRC_CNT(head, tail, ring_size); + + if (!(tail % ring_size)) + goto unlock; + + if (!count) { + queue->tx_head = 0; + queue->tx_tail = 0; + goto unlock; + } + + shift = tail % ring_size; + cycles = gcd(ring_size, shift); + + for (i = 0; i < cycles; i++) { + memcpy(&desc, macb_tx_desc(queue, i), desc_size); + memcpy(&tx_skb, macb_tx_skb(queue, i), + sizeof(struct macb_tx_skb)); + + curr = i; + next = (curr + shift) % ring_size; + + while (next != i) { + desc_curr = macb_tx_desc(queue, curr); + desc_next = macb_tx_desc(queue, next); + + memcpy(desc_curr, desc_next, desc_size); + + if (next == ring_size - 1) + desc_curr->ctrl &= ~MACB_BIT(TX_WRAP); + if (curr == ring_size - 1) + desc_curr->ctrl |= MACB_BIT(TX_WRAP); + + skb_curr = macb_tx_skb(queue, curr); + skb_next = macb_tx_skb(queue, next); + memcpy(skb_curr, skb_next, sizeof(struct macb_tx_skb)); + + curr = next; + next = (curr + shift) % ring_size; + } + + desc_curr = macb_tx_desc(queue, curr); + memcpy(desc_curr, &desc, desc_size); + if (i == ring_size - 1) + desc_curr->ctrl &= ~MACB_BIT(TX_WRAP); + if (curr == ring_size - 1) + desc_curr->ctrl |= MACB_BIT(TX_WRAP); + memcpy(macb_tx_skb(queue, curr), &tx_skb, + sizeof(struct macb_tx_skb)); + } + + queue->tx_head = count; + queue->tx_tail = 0; + + /* Make descriptor updates visible to hardware */ + wmb(); + +unlock: + spin_unlock_irqrestore(&queue->tx_ptr_lock, flags); +} + +/* Rotate the queue so that the tail is at index 0 */ +static void gem_shuffle_tx_rings(struct macb *bp) +{ + struct macb_queue *queue; + int q; + + for (q = 0, queue = bp->queues; q < bp->num_queues; q++, queue++) + gem_shuffle_tx_one_ring(queue); +} + static void macb_mac_link_up(struct phylink_config *config, struct phy_device *phy, unsigned int mode, phy_interface_t interface, @@ -705,14 +797,10 @@ static void macb_mac_link_up(struct phylink_config *config, if (rx_pause) ctrl |= MACB_BIT(PAE); - /* Initialize rings & buffers as clearing MACB_BIT(TE) in link down - * cleared the pipeline and control registers. - */ - macb_init_buffers(bp); - - for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) + for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { queue_writel(queue, IER, bp->rx_intr_mask | MACB_TX_INT_FLAGS | MACB_BIT(HRESP)); + } } macb_or_gem_writel(bp, NCFGR, ctrl); @@ -723,8 +811,10 @@ static void macb_mac_link_up(struct phylink_config *config, spin_unlock_irqrestore(&bp->lock, flags); - if (!(bp->caps & MACB_CAPS_MACB_IS_EMAC)) + if (!(bp->caps & MACB_CAPS_MACB_IS_EMAC)) { macb_set_tx_clk(bp, speed); + gem_shuffle_tx_rings(bp); + } /* Enable Rx and Tx; Enable PTP unicast */ ctrl = macb_readl(bp, NCR); @@ -981,7 +1071,7 @@ static void macb_tx_unmap(struct macb *bp, struct macb_tx_skb *tx_skb, int budge } if (tx_skb->skb) { - napi_consume_skb(tx_skb->skb, budget); + dev_consume_skb_any(tx_skb->skb); tx_skb->skb = NULL; } } @@ -2579,6 +2669,14 @@ static void macb_init_tieoff(struct macb *bp) desc->ctrl = 0; } +static void gem_init_rx_ring(struct macb_queue *queue) +{ + queue->rx_tail = 0; + queue->rx_prepared_head = 0; + + gem_rx_refill(queue); +} + static void gem_init_rings(struct macb *bp) { struct macb_queue *queue; @@ -2596,10 +2694,7 @@ static void gem_init_rings(struct macb *bp) queue->tx_head = 0; queue->tx_tail = 0; - queue->rx_tail = 0; - queue->rx_prepared_head = 0; - - gem_rx_refill(queue); + gem_init_rx_ring(queue); } macb_init_tieoff(bp); @@ -2954,6 +3049,7 @@ static int macb_open(struct net_device *dev) } bp->macbgem_ops.mog_init_rings(bp); + macb_init_buffers(bp); for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { napi_enable(&queue->napi_rx); @@ -3128,7 +3224,7 @@ static void gem_get_ethtool_stats(struct net_device *dev, spin_lock_irq(&bp->stats_lock); gem_update_stats(bp); memcpy(data, &bp->ethtool_stats, sizeof(u64) - * (GEM_STATS_LEN + QUEUE_STATS_LEN * MACB_MAX_QUEUES)); + * (GEM_STATS_LEN + QUEUE_STATS_LEN * bp->num_queues)); spin_unlock_irq(&bp->stats_lock); } @@ -3883,6 +3979,9 @@ static int gem_set_rxnfc(struct net_device *netdev, struct ethtool_rxnfc *cmd) struct macb *bp = netdev_priv(netdev); int ret; + if (!(netdev->hw_features & NETIF_F_NTUPLE)) + return -EOPNOTSUPP; + switch (cmd->cmd) { case ETHTOOL_SRXCLSRLINS: if ((cmd->fs.location >= bp->max_tuples) @@ -5674,9 +5773,9 @@ static int __maybe_unused macb_suspend(struct device *dev) struct macb_queue *queue; struct in_device *idev; unsigned long flags; + u32 tmp, ifa_local; unsigned int q; int err; - u32 tmp; if (!device_may_wakeup(&bp->dev->dev)) phy_exit(bp->phy); @@ -5685,14 +5784,21 @@ static int __maybe_unused macb_suspend(struct device *dev) return 0; if (bp->wol & MACB_WOL_ENABLED) { - /* Check for IP address in WOL ARP mode */ - idev = __in_dev_get_rcu(bp->dev); - if (idev) - ifa = rcu_dereference(idev->ifa_list); - if ((bp->wolopts & WAKE_ARP) && !ifa) { - netdev_err(netdev, "IP address not assigned as required by WoL walk ARP\n"); - return -EOPNOTSUPP; + if (bp->wolopts & WAKE_ARP) { + /* Check for IP address in WOL ARP mode */ + rcu_read_lock(); + idev = __in_dev_get_rcu(bp->dev); + if (idev) + ifa = rcu_dereference(idev->ifa_list); + if (!ifa) { + rcu_read_unlock(); + netdev_err(netdev, "IP address not assigned as required by WoL walk ARP\n"); + return -EOPNOTSUPP; + } + ifa_local = be32_to_cpu(ifa->ifa_local); + rcu_read_unlock(); } + spin_lock_irqsave(&bp->lock, flags); /* Disable Tx and Rx engines before disabling the queues, @@ -5731,8 +5837,9 @@ static int __maybe_unused macb_suspend(struct device *dev) if (bp->wolopts & WAKE_ARP) { tmp |= MACB_BIT(ARP); /* write IP address into register */ - tmp |= MACB_BFEXT(IP, be32_to_cpu(ifa->ifa_local)); + tmp |= MACB_BFEXT(IP, ifa_local); } + spin_unlock_irqrestore(&bp->lock, flags); /* Change interrupt handler and * Enable WoL IRQ on queue 0 @@ -5745,11 +5852,12 @@ static int __maybe_unused macb_suspend(struct device *dev) dev_err(dev, "Unable to request IRQ %d (error %d)\n", bp->queues[0].irq, err); - spin_unlock_irqrestore(&bp->lock, flags); return err; } + spin_lock_irqsave(&bp->lock, flags); queue_writel(bp->queues, IER, GEM_BIT(WOL)); gem_writel(bp, WOL, tmp); + spin_unlock_irqrestore(&bp->lock, flags); } else { err = devm_request_irq(dev, bp->queues[0].irq, macb_wol_interrupt, IRQF_SHARED, netdev->name, bp->queues); @@ -5757,13 +5865,13 @@ static int __maybe_unused macb_suspend(struct device *dev) dev_err(dev, "Unable to request IRQ %d (error %d)\n", bp->queues[0].irq, err); - spin_unlock_irqrestore(&bp->lock, flags); return err; } + spin_lock_irqsave(&bp->lock, flags); queue_writel(bp->queues, IER, MACB_BIT(WOL)); macb_writel(bp, WOL, tmp); + spin_unlock_irqrestore(&bp->lock, flags); } - spin_unlock_irqrestore(&bp->lock, flags); enable_irq_wake(bp->queues[0].irq); } @@ -5830,6 +5938,8 @@ static int __maybe_unused macb_resume(struct device *dev) queue_readl(bp->queues, ISR); if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) queue_writel(bp->queues, ISR, -1); + spin_unlock_irqrestore(&bp->lock, flags); + /* Replace interrupt handler on queue 0 */ devm_free_irq(dev, bp->queues[0].irq, bp->queues); err = devm_request_irq(dev, bp->queues[0].irq, macb_interrupt, @@ -5838,10 +5948,8 @@ static int __maybe_unused macb_resume(struct device *dev) dev_err(dev, "Unable to request IRQ %d (error %d)\n", bp->queues[0].irq, err); - spin_unlock_irqrestore(&bp->lock, flags); return err; } - spin_unlock_irqrestore(&bp->lock, flags); disable_irq_wake(bp->queues[0].irq); @@ -5853,8 +5961,18 @@ static int __maybe_unused macb_resume(struct device *dev) rtnl_unlock(); } + if (!(bp->caps & MACB_CAPS_MACB_IS_EMAC)) + macb_init_buffers(bp); + for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { + if (!(bp->caps & MACB_CAPS_MACB_IS_EMAC)) { + if (macb_is_gem(bp)) + gem_init_rx_ring(queue); + else + macb_init_rx_ring(queue); + } + napi_enable(&queue->napi_rx); napi_enable(&queue->napi_tx); } diff --git a/drivers/net/ethernet/cadence/macb_pci.c b/drivers/net/ethernet/cadence/macb_pci.c index fc4f5aee6ab3f6..b79dec17e6b099 100644 --- a/drivers/net/ethernet/cadence/macb_pci.c +++ b/drivers/net/ethernet/cadence/macb_pci.c @@ -96,10 +96,10 @@ static int macb_probe(struct pci_dev *pdev, const struct pci_device_id *id) return 0; err_plat_dev_register: - clk_unregister(plat_data.hclk); + clk_unregister_fixed_rate(plat_data.hclk); err_hclk_register: - clk_unregister(plat_data.pclk); + clk_unregister_fixed_rate(plat_data.pclk); err_pclk_register: return err; @@ -109,10 +109,12 @@ static void macb_remove(struct pci_dev *pdev) { struct platform_device *plat_dev = pci_get_drvdata(pdev); struct macb_platform_data *plat_data = dev_get_platdata(&plat_dev->dev); + struct clk *pclk = plat_data->pclk; + struct clk *hclk = plat_data->hclk; - clk_unregister(plat_data->pclk); - clk_unregister(plat_data->hclk); platform_device_unregister(plat_dev); + clk_unregister_fixed_rate(pclk); + clk_unregister_fixed_rate(hclk); } static const struct pci_device_id dev_id_table[] = { diff --git a/drivers/net/ethernet/cadence/macb_ptp.c b/drivers/net/ethernet/cadence/macb_ptp.c index c9e77819196e17..d91f7b1aa39caa 100644 --- a/drivers/net/ethernet/cadence/macb_ptp.c +++ b/drivers/net/ethernet/cadence/macb_ptp.c @@ -357,8 +357,10 @@ void gem_ptp_remove(struct net_device *ndev) { struct macb *bp = netdev_priv(ndev); - if (bp->ptp_clock) + if (bp->ptp_clock) { ptp_clock_unregister(bp->ptp_clock); + bp->ptp_clock = NULL; + } gem_ptp_clear_timer(bp); diff --git a/drivers/net/ethernet/ec_bhf.c b/drivers/net/ethernet/ec_bhf.c index 67275aa4f65b2f..0c86cbb0313c30 100644 --- a/drivers/net/ethernet/ec_bhf.c +++ b/drivers/net/ethernet/ec_bhf.c @@ -423,7 +423,7 @@ static int ec_bhf_open(struct net_device *net_dev) error_rx_free: dma_free_coherent(dev, priv->rx_buf.alloc_len, priv->rx_buf.alloc, - priv->rx_buf.alloc_len); + priv->rx_buf.alloc_phys); out: return err; } diff --git a/drivers/net/ethernet/faraday/ftgmac100.c b/drivers/net/ethernet/faraday/ftgmac100.c index a863f78412104b..06ee7af7fdb9aa 100644 --- a/drivers/net/ethernet/faraday/ftgmac100.c +++ b/drivers/net/ethernet/faraday/ftgmac100.c @@ -964,19 +964,19 @@ static int ftgmac100_alloc_rings(struct ftgmac100 *priv) priv->tx_skbs = kcalloc(MAX_TX_QUEUE_ENTRIES, sizeof(void *), GFP_KERNEL); if (!priv->tx_skbs) - return -ENOMEM; + goto err_free_rx_skbs; /* Allocate descriptors */ priv->rxdes = dma_alloc_coherent(priv->dev, MAX_RX_QUEUE_ENTRIES * sizeof(struct ftgmac100_rxdes), &priv->rxdes_dma, GFP_KERNEL); if (!priv->rxdes) - return -ENOMEM; + goto err_free_tx_skbs; priv->txdes = dma_alloc_coherent(priv->dev, MAX_TX_QUEUE_ENTRIES * sizeof(struct ftgmac100_txdes), &priv->txdes_dma, GFP_KERNEL); if (!priv->txdes) - return -ENOMEM; + goto err_free_rxdes; /* Allocate scratch packet buffer */ priv->rx_scratch = dma_alloc_coherent(priv->dev, @@ -984,9 +984,29 @@ static int ftgmac100_alloc_rings(struct ftgmac100 *priv) &priv->rx_scratch_dma, GFP_KERNEL); if (!priv->rx_scratch) - return -ENOMEM; + goto err_free_txdes; return 0; + +err_free_txdes: + dma_free_coherent(priv->dev, + MAX_TX_QUEUE_ENTRIES * + sizeof(struct ftgmac100_txdes), + priv->txdes, priv->txdes_dma); + priv->txdes = NULL; +err_free_rxdes: + dma_free_coherent(priv->dev, + MAX_RX_QUEUE_ENTRIES * + sizeof(struct ftgmac100_rxdes), + priv->rxdes, priv->rxdes_dma); + priv->rxdes = NULL; +err_free_tx_skbs: + kfree(priv->tx_skbs); + priv->tx_skbs = NULL; +err_free_rx_skbs: + kfree(priv->rx_skbs); + priv->rx_skbs = NULL; + return -ENOMEM; } static void ftgmac100_init_rings(struct ftgmac100 *priv) diff --git a/drivers/net/ethernet/freescale/Kconfig b/drivers/net/ethernet/freescale/Kconfig index e2a591cf9601f2..11edbb46a1180d 100644 --- a/drivers/net/ethernet/freescale/Kconfig +++ b/drivers/net/ethernet/freescale/Kconfig @@ -28,7 +28,7 @@ config FEC depends on PTP_1588_CLOCK_OPTIONAL select CRC32 select PHYLIB - select FIXED_PHY if M5272 + select FIXED_PHY select PAGE_POOL imply PAGE_POOL_STATS imply NET_SELFTESTS diff --git a/drivers/net/ethernet/freescale/dpaa2/dpaa2-switch.c b/drivers/net/ethernet/freescale/dpaa2/dpaa2-switch.c index 66240c340492ce..e212a014c8d414 100644 --- a/drivers/net/ethernet/freescale/dpaa2/dpaa2-switch.c +++ b/drivers/net/ethernet/freescale/dpaa2/dpaa2-switch.c @@ -1533,7 +1533,7 @@ static irqreturn_t dpaa2_switch_irq0_handler_thread(int irq_num, void *arg) if_id = (status & 0xFFFF0000) >> 16; if (if_id >= ethsw->sw_attr.num_ifs) { dev_err(dev, "Invalid if_id %d in IRQ status\n", if_id); - goto out; + goto out_clear; } port_priv = ethsw->ports[if_id]; @@ -1553,6 +1553,7 @@ static irqreturn_t dpaa2_switch_irq0_handler_thread(int irq_num, void *arg) dpaa2_switch_port_connect_mac(port_priv); } +out_clear: err = dpsw_clear_irq_status(ethsw->mc_io, 0, ethsw->dpsw_handle, DPSW_IRQ_INDEX_IF, status); if (err) @@ -3034,6 +3035,13 @@ static int dpaa2_switch_init(struct fsl_mc_device *sw_dev) goto err_close; } + if (ethsw->sw_attr.num_ifs >= DPSW_MAX_IF) { + dev_err(dev, "DPSW num_ifs %u exceeds max %u\n", + ethsw->sw_attr.num_ifs, DPSW_MAX_IF); + err = -EINVAL; + goto err_close; + } + err = dpsw_get_api_version(ethsw->mc_io, 0, ðsw->major, ðsw->minor); diff --git a/drivers/net/ethernet/freescale/enetc/enetc.c b/drivers/net/ethernet/freescale/enetc/enetc.c index e380a4f3985560..8ec96f39e12631 100644 --- a/drivers/net/ethernet/freescale/enetc/enetc.c +++ b/drivers/net/ethernet/freescale/enetc/enetc.c @@ -2579,6 +2579,7 @@ EXPORT_SYMBOL_GPL(enetc_free_si_resources); static void enetc_setup_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring) { + struct enetc_si *si = container_of(hw, struct enetc_si, hw); int idx = tx_ring->index; u32 tbmr; @@ -2592,10 +2593,20 @@ static void enetc_setup_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring) enetc_txbdr_wr(hw, idx, ENETC_TBLENR, ENETC_RTBLENR_LEN(tx_ring->bd_count)); - /* clearing PI/CI registers for Tx not supported, adjust sw indexes */ + /* For ENETC v1, clearing PI/CI registers for Tx not supported, + * adjust sw indexes + */ tx_ring->next_to_use = enetc_txbdr_rd(hw, idx, ENETC_TBPIR); tx_ring->next_to_clean = enetc_txbdr_rd(hw, idx, ENETC_TBCIR); + if (tx_ring->next_to_use != tx_ring->next_to_clean && + !is_enetc_rev1(si)) { + tx_ring->next_to_use = 0; + tx_ring->next_to_clean = 0; + enetc_txbdr_wr(hw, idx, ENETC_TBPIR, 0); + enetc_txbdr_wr(hw, idx, ENETC_TBCIR, 0); + } + /* enable Tx ints by setting pkt thr to 1 */ enetc_txbdr_wr(hw, idx, ENETC_TBICR0, ENETC_TBICR0_ICEN | 0x1); @@ -3468,7 +3479,7 @@ static int enetc_int_vector_init(struct enetc_ndev_priv *priv, int i, priv->rx_ring[i] = bdr; err = __xdp_rxq_info_reg(&bdr->xdp.rxq, priv->ndev, i, 0, - ENETC_RXB_DMA_SIZE_XDP); + ENETC_RXB_TRUESIZE); if (err) goto free_vector; diff --git a/drivers/net/ethernet/freescale/enetc/enetc4_hw.h b/drivers/net/ethernet/freescale/enetc/enetc4_hw.h index 3ed0f7a0276798..719c88ceb801a1 100644 --- a/drivers/net/ethernet/freescale/enetc/enetc4_hw.h +++ b/drivers/net/ethernet/freescale/enetc/enetc4_hw.h @@ -134,6 +134,12 @@ /* Port operational register */ #define ENETC4_POR 0x4100 +#define POR_TXDIS BIT(0) +#define POR_RXDIS BIT(1) + +/* Port status register */ +#define ENETC4_PSR 0x4104 +#define PSR_RX_BUSY BIT(1) /* Port traffic class a transmit maximum SDU register */ #define ENETC4_PTCTMSDUR(a) ((a) * 0x20 + 0x4208) @@ -173,6 +179,11 @@ /* Port internal MDIO base address, use to access PCS */ #define ENETC4_PM_IMDIO_BASE 0x5030 +/* Port MAC 0/1 Interrupt Event Register */ +#define ENETC4_PM_IEVENT(mac) (0x5040 + (mac) * 0x400) +#define PM_IEVENT_TX_EMPTY BIT(5) +#define PM_IEVENT_RX_EMPTY BIT(6) + /* Port MAC 0/1 Pause Quanta Register */ #define ENETC4_PM_PAUSE_QUANTA(mac) (0x5054 + (mac) * 0x400) diff --git a/drivers/net/ethernet/freescale/enetc/enetc4_pf.c b/drivers/net/ethernet/freescale/enetc/enetc4_pf.c index 5850540634b0cd..993c27e342266d 100644 --- a/drivers/net/ethernet/freescale/enetc/enetc4_pf.c +++ b/drivers/net/ethernet/freescale/enetc/enetc4_pf.c @@ -444,20 +444,11 @@ static void enetc4_set_trx_frame_size(struct enetc_pf *pf) enetc4_pf_reset_tc_msdu(&si->hw); } -static void enetc4_enable_trx(struct enetc_pf *pf) -{ - struct enetc_hw *hw = &pf->si->hw; - - /* Enable port transmit/receive */ - enetc_port_wr(hw, ENETC4_POR, 0); -} - static void enetc4_configure_port(struct enetc_pf *pf) { enetc4_configure_port_si(pf); enetc4_set_trx_frame_size(pf); enetc_set_default_rss_key(pf); - enetc4_enable_trx(pf); } static int enetc4_init_ntmp_user(struct enetc_si *si) @@ -801,15 +792,112 @@ static void enetc4_set_tx_pause(struct enetc_pf *pf, int num_rxbdr, bool tx_paus enetc_port_wr(hw, ENETC4_PPAUOFFTR, pause_off_thresh); } -static void enetc4_enable_mac(struct enetc_pf *pf, bool en) +static void enetc4_mac_wait_tx_empty(struct enetc_si *si, int mac) +{ + u32 val; + + if (read_poll_timeout(enetc_port_rd, val, + val & PM_IEVENT_TX_EMPTY, + 100, 10000, false, &si->hw, + ENETC4_PM_IEVENT(mac))) + dev_warn(&si->pdev->dev, + "MAC %d TX is not empty\n", mac); +} + +static void enetc4_mac_tx_graceful_stop(struct enetc_pf *pf) +{ + struct enetc_hw *hw = &pf->si->hw; + struct enetc_si *si = pf->si; + u32 val; + + val = enetc_port_rd(hw, ENETC4_POR); + val |= POR_TXDIS; + enetc_port_wr(hw, ENETC4_POR, val); + + if (enetc_is_pseudo_mac(si)) + return; + + enetc4_mac_wait_tx_empty(si, 0); + if (si->hw_features & ENETC_SI_F_QBU) + enetc4_mac_wait_tx_empty(si, 1); + + val = enetc_port_mac_rd(si, ENETC4_PM_CMD_CFG(0)); + val &= ~PM_CMD_CFG_TX_EN; + enetc_port_mac_wr(si, ENETC4_PM_CMD_CFG(0), val); +} + +static void enetc4_mac_tx_enable(struct enetc_pf *pf) { + struct enetc_hw *hw = &pf->si->hw; struct enetc_si *si = pf->si; u32 val; val = enetc_port_mac_rd(si, ENETC4_PM_CMD_CFG(0)); - val &= ~(PM_CMD_CFG_TX_EN | PM_CMD_CFG_RX_EN); - val |= en ? (PM_CMD_CFG_TX_EN | PM_CMD_CFG_RX_EN) : 0; + val |= PM_CMD_CFG_TX_EN; + enetc_port_mac_wr(si, ENETC4_PM_CMD_CFG(0), val); + val = enetc_port_rd(hw, ENETC4_POR); + val &= ~POR_TXDIS; + enetc_port_wr(hw, ENETC4_POR, val); +} + +static void enetc4_mac_wait_rx_empty(struct enetc_si *si, int mac) +{ + u32 val; + + if (read_poll_timeout(enetc_port_rd, val, + val & PM_IEVENT_RX_EMPTY, + 100, 10000, false, &si->hw, + ENETC4_PM_IEVENT(mac))) + dev_warn(&si->pdev->dev, + "MAC %d RX is not empty\n", mac); +} + +static void enetc4_mac_rx_graceful_stop(struct enetc_pf *pf) +{ + struct enetc_hw *hw = &pf->si->hw; + struct enetc_si *si = pf->si; + u32 val; + + if (enetc_is_pseudo_mac(si)) + goto check_rx_busy; + + if (si->hw_features & ENETC_SI_F_QBU) { + val = enetc_port_rd(hw, ENETC4_PM_CMD_CFG(1)); + val &= ~PM_CMD_CFG_RX_EN; + enetc_port_wr(hw, ENETC4_PM_CMD_CFG(1), val); + enetc4_mac_wait_rx_empty(si, 1); + } + + val = enetc_port_rd(hw, ENETC4_PM_CMD_CFG(0)); + val &= ~PM_CMD_CFG_RX_EN; + enetc_port_wr(hw, ENETC4_PM_CMD_CFG(0), val); + enetc4_mac_wait_rx_empty(si, 0); + +check_rx_busy: + if (read_poll_timeout(enetc_port_rd, val, + !(val & PSR_RX_BUSY), + 100, 10000, false, hw, + ENETC4_PSR)) + dev_warn(&si->pdev->dev, "Port RX busy\n"); + + val = enetc_port_rd(hw, ENETC4_POR); + val |= POR_RXDIS; + enetc_port_wr(hw, ENETC4_POR, val); +} + +static void enetc4_mac_rx_enable(struct enetc_pf *pf) +{ + struct enetc_hw *hw = &pf->si->hw; + struct enetc_si *si = pf->si; + u32 val; + + val = enetc_port_rd(hw, ENETC4_POR); + val &= ~POR_RXDIS; + enetc_port_wr(hw, ENETC4_POR, val); + + val = enetc_port_mac_rd(si, ENETC4_PM_CMD_CFG(0)); + val |= PM_CMD_CFG_RX_EN; enetc_port_mac_wr(si, ENETC4_PM_CMD_CFG(0), val); } @@ -853,7 +941,8 @@ static void enetc4_pl_mac_link_up(struct phylink_config *config, enetc4_set_hd_flow_control(pf, hd_fc); enetc4_set_tx_pause(pf, priv->num_rx_rings, tx_pause); enetc4_set_rx_pause(pf, rx_pause); - enetc4_enable_mac(pf, true); + enetc4_mac_tx_enable(pf); + enetc4_mac_rx_enable(pf); } static void enetc4_pl_mac_link_down(struct phylink_config *config, @@ -862,7 +951,8 @@ static void enetc4_pl_mac_link_down(struct phylink_config *config, { struct enetc_pf *pf = phylink_to_enetc_pf(config); - enetc4_enable_mac(pf, false); + enetc4_mac_rx_graceful_stop(pf); + enetc4_mac_tx_graceful_stop(pf); } static const struct phylink_mac_ops enetc_pl_mac_ops = { diff --git a/drivers/net/ethernet/freescale/enetc/enetc_ethtool.c b/drivers/net/ethernet/freescale/enetc/enetc_ethtool.c index fed89d4f1e1dc4..7c17acaf7a380b 100644 --- a/drivers/net/ethernet/freescale/enetc/enetc_ethtool.c +++ b/drivers/net/ethernet/freescale/enetc/enetc_ethtool.c @@ -795,9 +795,17 @@ static int enetc_set_rxfh(struct net_device *ndev, struct enetc_si *si = priv->si; int err = 0; + if (rxfh->hfunc != ETH_RSS_HASH_NO_CHANGE && + rxfh->hfunc != ETH_RSS_HASH_TOP) + return -EOPNOTSUPP; + /* set hash key, if PF */ - if (rxfh->key && enetc_si_is_pf(si)) + if (rxfh->key) { + if (!enetc_si_is_pf(si)) + return -EOPNOTSUPP; + enetc_set_rss_key(si, rxfh->key); + } /* set RSS table */ if (rxfh->indir) @@ -813,6 +821,8 @@ static void enetc_get_ringparam(struct net_device *ndev, { struct enetc_ndev_priv *priv = netdev_priv(ndev); + ring->rx_max_pending = priv->rx_bd_count; + ring->tx_max_pending = priv->tx_bd_count; ring->rx_pending = priv->rx_bd_count; ring->tx_pending = priv->tx_bd_count; diff --git a/drivers/net/ethernet/freescale/enetc/netc_blk_ctrl.c b/drivers/net/ethernet/freescale/enetc/netc_blk_ctrl.c index 7fd39f89529014..92a0f824dae7a2 100644 --- a/drivers/net/ethernet/freescale/enetc/netc_blk_ctrl.c +++ b/drivers/net/ethernet/freescale/enetc/netc_blk_ctrl.c @@ -333,11 +333,13 @@ static int netc_get_phy_addr(struct device_node *np) mdio_node = of_get_child_by_name(np, "mdio"); if (!mdio_node) - return 0; + return -ENODEV; phy_node = of_get_next_child(mdio_node, NULL); - if (!phy_node) + if (!phy_node) { + err = -ENODEV; goto of_put_mdio_node; + } err = of_property_read_u32(phy_node, "reg", &addr); if (err) @@ -423,6 +425,9 @@ static int imx95_enetc_mdio_phyaddr_config(struct platform_device *pdev) addr = netc_get_phy_addr(gchild); if (addr < 0) { + if (addr == -ENODEV) + continue; + dev_err(dev, "Failed to get PHY address\n"); return addr; } @@ -433,12 +438,6 @@ static int imx95_enetc_mdio_phyaddr_config(struct platform_device *pdev) return -EINVAL; } - /* The default value of LaBCR[MDIO_PHYAD_PRTAD ] is - * 0, so no need to set the register. - */ - if (!addr) - continue; - switch (bus_devfn) { case IMX95_ENETC0_BUS_DEVFN: netc_reg_write(priv->ierb, IERB_LBCR(0), @@ -578,16 +577,13 @@ static int imx94_enetc_mdio_phyaddr_config(struct netc_blk_ctrl *priv, addr = netc_get_phy_addr(np); if (addr < 0) { + if (addr == -ENODEV) + return 0; + dev_err(dev, "Failed to get PHY address\n"); return addr; } - /* The default value of LaBCR[MDIO_PHYAD_PRTAD] is 0, - * so no need to set the register. - */ - if (!addr) - return 0; - if (phy_mask & BIT(addr)) { dev_err(dev, "Find same PHY address in EMDIO and ENETC node\n"); diff --git a/drivers/net/ethernet/freescale/fec_ptp.c b/drivers/net/ethernet/freescale/fec_ptp.c index 4b7bad9a485df2..56801c2009d59a 100644 --- a/drivers/net/ethernet/freescale/fec_ptp.c +++ b/drivers/net/ethernet/freescale/fec_ptp.c @@ -545,9 +545,6 @@ static int fec_ptp_enable(struct ptp_clock_info *ptp, if (rq->perout.flags) return -EOPNOTSUPP; - if (rq->perout.index != fep->pps_channel) - return -EOPNOTSUPP; - period.tv_sec = rq->perout.period.sec; period.tv_nsec = rq->perout.period.nsec; period_ns = timespec64_to_ns(&period); diff --git a/drivers/net/ethernet/google/gve/gve_tx_dqo.c b/drivers/net/ethernet/google/gve/gve_tx_dqo.c index 40b89b3e5a318a..e5e33966458628 100644 --- a/drivers/net/ethernet/google/gve/gve_tx_dqo.c +++ b/drivers/net/ethernet/google/gve/gve_tx_dqo.c @@ -167,6 +167,25 @@ gve_free_pending_packet(struct gve_tx_ring *tx, } } +static void gve_unmap_packet(struct device *dev, + struct gve_tx_pending_packet_dqo *pkt) +{ + int i; + + if (!pkt->num_bufs) + return; + + /* SKB linear portion is guaranteed to be mapped */ + dma_unmap_single(dev, dma_unmap_addr(pkt, dma[0]), + dma_unmap_len(pkt, len[0]), DMA_TO_DEVICE); + for (i = 1; i < pkt->num_bufs; i++) { + netmem_dma_unmap_page_attrs(dev, dma_unmap_addr(pkt, dma[i]), + dma_unmap_len(pkt, len[i]), + DMA_TO_DEVICE, 0); + } + pkt->num_bufs = 0; +} + /* gve_tx_free_desc - Cleans up all pending tx requests and buffers. */ static void gve_tx_clean_pending_packets(struct gve_tx_ring *tx) @@ -176,21 +195,12 @@ static void gve_tx_clean_pending_packets(struct gve_tx_ring *tx) for (i = 0; i < tx->dqo.num_pending_packets; i++) { struct gve_tx_pending_packet_dqo *cur_state = &tx->dqo.pending_packets[i]; - int j; - - for (j = 0; j < cur_state->num_bufs; j++) { - if (j == 0) { - dma_unmap_single(tx->dev, - dma_unmap_addr(cur_state, dma[j]), - dma_unmap_len(cur_state, len[j]), - DMA_TO_DEVICE); - } else { - dma_unmap_page(tx->dev, - dma_unmap_addr(cur_state, dma[j]), - dma_unmap_len(cur_state, len[j]), - DMA_TO_DEVICE); - } - } + + if (tx->dqo.qpl) + gve_free_tx_qpl_bufs(tx, cur_state); + else + gve_unmap_packet(tx->dev, cur_state); + if (cur_state->skb) { dev_consume_skb_any(cur_state->skb); cur_state->skb = NULL; @@ -1160,22 +1170,6 @@ static void remove_from_list(struct gve_tx_ring *tx, } } -static void gve_unmap_packet(struct device *dev, - struct gve_tx_pending_packet_dqo *pkt) -{ - int i; - - /* SKB linear portion is guaranteed to be mapped */ - dma_unmap_single(dev, dma_unmap_addr(pkt, dma[0]), - dma_unmap_len(pkt, len[0]), DMA_TO_DEVICE); - for (i = 1; i < pkt->num_bufs; i++) { - netmem_dma_unmap_page_attrs(dev, dma_unmap_addr(pkt, dma[i]), - dma_unmap_len(pkt, len[i]), - DMA_TO_DEVICE, 0); - } - pkt->num_bufs = 0; -} - /* Completion types and expected behavior: * No Miss compl + Packet compl = Packet completed normally. * Miss compl + Re-inject compl = Packet completed normally. diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c b/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c index 7a9573dcab741b..e879b04e21b0cd 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c @@ -1048,13 +1048,13 @@ static void hns3_init_tx_spare_buffer(struct hns3_enet_ring *ring) int order; if (!alloc_size) - return; + goto not_init; order = get_order(alloc_size); if (order > MAX_PAGE_ORDER) { if (net_ratelimit()) dev_warn(ring_to_dev(ring), "failed to allocate tx spare buffer, exceed to max order\n"); - return; + goto not_init; } tx_spare = devm_kzalloc(ring_to_dev(ring), sizeof(*tx_spare), @@ -1092,6 +1092,13 @@ static void hns3_init_tx_spare_buffer(struct hns3_enet_ring *ring) devm_kfree(ring_to_dev(ring), tx_spare); devm_kzalloc_error: ring->tqp->handle->kinfo.tx_spare_buf_size = 0; +not_init: + /* When driver init or reset_init, the ring->tx_spare is always NULL; + * but when called from hns3_set_ringparam, it's usually not NULL, and + * will be restored if hns3_init_all_ring() failed. So it's safe to set + * ring->tx_spare to NULL here. + */ + ring->tx_spare = NULL; } /* Use hns3_tx_spare_space() to make sure there is enough buffer diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_cmd.h b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_cmd.h index 416e02e7b995fc..bc333d8710ac1a 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_cmd.h +++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_cmd.h @@ -727,8 +727,8 @@ struct hclge_fd_tcam_config_3_cmd { #define HCLGE_FD_AD_DROP_B 0 #define HCLGE_FD_AD_DIRECT_QID_B 1 -#define HCLGE_FD_AD_QID_S 2 -#define HCLGE_FD_AD_QID_M GENMASK(11, 2) +#define HCLGE_FD_AD_QID_L_S 2 +#define HCLGE_FD_AD_QID_L_M GENMASK(11, 2) #define HCLGE_FD_AD_USE_COUNTER_B 12 #define HCLGE_FD_AD_COUNTER_NUM_S 13 #define HCLGE_FD_AD_COUNTER_NUM_M GENMASK(19, 13) @@ -741,6 +741,7 @@ struct hclge_fd_tcam_config_3_cmd { #define HCLGE_FD_AD_TC_OVRD_B 16 #define HCLGE_FD_AD_TC_SIZE_S 17 #define HCLGE_FD_AD_TC_SIZE_M GENMASK(20, 17) +#define HCLGE_FD_AD_QID_H_B 21 struct hclge_fd_ad_config_cmd { u8 stage; diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c index b8e2aa19f9e614..a90f1a91f9973f 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c @@ -5679,11 +5679,13 @@ static int hclge_fd_ad_config(struct hclge_dev *hdev, u8 stage, int loc, hnae3_set_field(ad_data, HCLGE_FD_AD_TC_SIZE_M, HCLGE_FD_AD_TC_SIZE_S, (u32)action->tc_size); } + hnae3_set_bit(ad_data, HCLGE_FD_AD_QID_H_B, + action->queue_id >= HCLGE_TQP_MAX_SIZE_DEV_V2 ? 1 : 0); ad_data <<= 32; hnae3_set_bit(ad_data, HCLGE_FD_AD_DROP_B, action->drop_packet); hnae3_set_bit(ad_data, HCLGE_FD_AD_DIRECT_QID_B, action->forward_to_direct_queue); - hnae3_set_field(ad_data, HCLGE_FD_AD_QID_M, HCLGE_FD_AD_QID_S, + hnae3_set_field(ad_data, HCLGE_FD_AD_QID_L_M, HCLGE_FD_AD_QID_L_S, action->queue_id); hnae3_set_bit(ad_data, HCLGE_FD_AD_USE_COUNTER_B, action->use_counter); hnae3_set_field(ad_data, HCLGE_FD_AD_COUNTER_NUM_M, diff --git a/drivers/net/ethernet/intel/e1000/e1000_ethtool.c b/drivers/net/ethernet/intel/e1000/e1000_ethtool.c index 726365c567ef33..75d0bfa7530b44 100644 --- a/drivers/net/ethernet/intel/e1000/e1000_ethtool.c +++ b/drivers/net/ethernet/intel/e1000/e1000_ethtool.c @@ -496,14 +496,19 @@ static int e1000_set_eeprom(struct net_device *netdev, */ ret_val = e1000_read_eeprom(hw, first_word, 1, &eeprom_buff[0]); + if (ret_val) + goto out; + ptr++; } - if (((eeprom->offset + eeprom->len) & 1) && (ret_val == 0)) { + if ((eeprom->offset + eeprom->len) & 1) { /* need read/modify/write of last changed EEPROM word * only the first byte of the word is being modified */ ret_val = e1000_read_eeprom(hw, last_word, 1, &eeprom_buff[last_word - first_word]); + if (ret_val) + goto out; } /* Device's eeprom is always little-endian, word addressable */ @@ -522,6 +527,7 @@ static int e1000_set_eeprom(struct net_device *netdev, if ((ret_val == 0) && (first_word <= EEPROM_CHECKSUM_REG)) e1000_update_eeprom_checksum(hw); +out: kfree(eeprom_buff); return ret_val; } diff --git a/drivers/net/ethernet/intel/e1000/e1000_main.c b/drivers/net/ethernet/intel/e1000/e1000_main.c index 7f078ec9c14c5f..15160427c8b308 100644 --- a/drivers/net/ethernet/intel/e1000/e1000_main.c +++ b/drivers/net/ethernet/intel/e1000/e1000_main.c @@ -2952,8 +2952,6 @@ static int e1000_tx_map(struct e1000_adapter *adapter, dma_error: dev_err(&pdev->dev, "TX DMA map failed\n"); buffer_info->dma = 0; - if (count) - count--; while (count--) { if (i == 0) diff --git a/drivers/net/ethernet/intel/e1000e/defines.h b/drivers/net/ethernet/intel/e1000e/defines.h index ba331899d1861b..d4a1041e456dcd 100644 --- a/drivers/net/ethernet/intel/e1000e/defines.h +++ b/drivers/net/ethernet/intel/e1000e/defines.h @@ -33,6 +33,7 @@ /* Extended Device Control */ #define E1000_CTRL_EXT_LPCD 0x00000004 /* LCD Power Cycle Done */ +#define E1000_CTRL_EXT_DPG_EN 0x00000008 /* Dynamic Power Gating Enable */ #define E1000_CTRL_EXT_SDP3_DATA 0x00000080 /* Value of SW Definable Pin 3 */ #define E1000_CTRL_EXT_FORCE_SMBUS 0x00000800 /* Force SMBus mode */ #define E1000_CTRL_EXT_EE_RST 0x00002000 /* Reinitialize from EEPROM */ diff --git a/drivers/net/ethernet/intel/e1000e/ich8lan.c b/drivers/net/ethernet/intel/e1000e/ich8lan.c index 0ff8688ac3b84c..2dceb5548a786b 100644 --- a/drivers/net/ethernet/intel/e1000e/ich8lan.c +++ b/drivers/net/ethernet/intel/e1000e/ich8lan.c @@ -4932,6 +4932,15 @@ static s32 e1000_reset_hw_ich8lan(struct e1000_hw *hw) reg |= E1000_KABGTXD_BGSQLBIAS; ew32(KABGTXD, reg); + /* The hardware reset value of the DPG_EN bit is 1. + * Clear DPG_EN to prevent unexpected autonomous power gating. + */ + if (hw->mac.type >= e1000_pch_ptp) { + reg = er32(CTRL_EXT); + reg &= ~E1000_CTRL_EXT_DPG_EN; + ew32(CTRL_EXT, reg); + } + return 0; } diff --git a/drivers/net/ethernet/intel/e1000e/netdev.c b/drivers/net/ethernet/intel/e1000e/netdev.c index ddbe2f7d811210..6bcb57609d16a6 100644 --- a/drivers/net/ethernet/intel/e1000e/netdev.c +++ b/drivers/net/ethernet/intel/e1000e/netdev.c @@ -5654,8 +5654,6 @@ static int e1000_tx_map(struct e1000_ring *tx_ring, struct sk_buff *skb, dma_error: dev_err(&pdev->dev, "Tx DMA map failed\n"); buffer_info->dma = 0; - if (count) - count--; while (count--) { if (i == 0) diff --git a/drivers/net/ethernet/intel/i40e/i40e_main.c b/drivers/net/ethernet/intel/i40e/i40e_main.c index d3bc3207054f90..598739220dfb90 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_main.c +++ b/drivers/net/ethernet/intel/i40e/i40e_main.c @@ -75,7 +75,13 @@ static const struct pci_device_id i40e_pci_tbl[] = { {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T4), 0}, {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T_BC), 0}, {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_SFP), 0}, - {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_B), 0}, + /* + * This ID conflicts with ipw2200, but the devices can be differentiated + * because i40e devices use PCI_CLASS_NETWORK_ETHERNET and ipw2200 + * devices use PCI_CLASS_NETWORK_OTHER. + */ + {PCI_DEVICE(PCI_VENDOR_ID_INTEL, I40E_DEV_ID_10G_B), + PCI_CLASS_NETWORK_ETHERNET << 8, 0xffff00, 0}, {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_X722), 0}, {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_X722), 0}, {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_X722), 0}, @@ -3563,6 +3569,7 @@ static int i40e_configure_rx_ring(struct i40e_ring *ring) u16 pf_q = vsi->base_queue + ring->queue_index; struct i40e_hw *hw = &vsi->back->hw; struct i40e_hmc_obj_rxq rx_ctx; + u32 xdp_frame_sz; int err = 0; bool ok; @@ -3572,49 +3579,47 @@ static int i40e_configure_rx_ring(struct i40e_ring *ring) memset(&rx_ctx, 0, sizeof(rx_ctx)); ring->rx_buf_len = vsi->rx_buf_len; + xdp_frame_sz = i40e_rx_pg_size(ring) / 2; /* XDP RX-queue info only needed for RX rings exposed to XDP */ if (ring->vsi->type != I40E_VSI_MAIN) goto skip; - if (!xdp_rxq_info_is_reg(&ring->xdp_rxq)) { - err = __xdp_rxq_info_reg(&ring->xdp_rxq, ring->netdev, - ring->queue_index, - ring->q_vector->napi.napi_id, - ring->rx_buf_len); - if (err) - return err; - } - ring->xsk_pool = i40e_xsk_pool(ring); if (ring->xsk_pool) { - xdp_rxq_info_unreg(&ring->xdp_rxq); + xdp_frame_sz = xsk_pool_get_rx_frag_step(ring->xsk_pool); ring->rx_buf_len = xsk_pool_get_rx_frame_size(ring->xsk_pool); err = __xdp_rxq_info_reg(&ring->xdp_rxq, ring->netdev, ring->queue_index, ring->q_vector->napi.napi_id, - ring->rx_buf_len); + xdp_frame_sz); if (err) return err; err = xdp_rxq_info_reg_mem_model(&ring->xdp_rxq, MEM_TYPE_XSK_BUFF_POOL, NULL); if (err) - return err; + goto unreg_xdp; dev_info(&vsi->back->pdev->dev, "Registered XDP mem model MEM_TYPE_XSK_BUFF_POOL on Rx ring %d\n", ring->queue_index); } else { + err = __xdp_rxq_info_reg(&ring->xdp_rxq, ring->netdev, + ring->queue_index, + ring->q_vector->napi.napi_id, + xdp_frame_sz); + if (err) + return err; err = xdp_rxq_info_reg_mem_model(&ring->xdp_rxq, MEM_TYPE_PAGE_SHARED, NULL); if (err) - return err; + goto unreg_xdp; } skip: - xdp_init_buff(&ring->xdp, i40e_rx_pg_size(ring) / 2, &ring->xdp_rxq); + xdp_init_buff(&ring->xdp, xdp_frame_sz, &ring->xdp_rxq); rx_ctx.dbuff = DIV_ROUND_UP(ring->rx_buf_len, BIT_ULL(I40E_RXQ_CTX_DBUFF_SHIFT)); @@ -3648,7 +3653,8 @@ static int i40e_configure_rx_ring(struct i40e_ring *ring) dev_info(&vsi->back->pdev->dev, "Failed to clear LAN Rx queue context on Rx ring %d (pf_q %d), error: %d\n", ring->queue_index, pf_q, err); - return -ENOMEM; + err = -ENOMEM; + goto unreg_xdp; } /* set the context in the HMC */ @@ -3657,7 +3663,8 @@ static int i40e_configure_rx_ring(struct i40e_ring *ring) dev_info(&vsi->back->pdev->dev, "Failed to set LAN Rx queue context on Rx ring %d (pf_q %d), error: %d\n", ring->queue_index, pf_q, err); - return -ENOMEM; + err = -ENOMEM; + goto unreg_xdp; } /* configure Rx buffer alignment */ @@ -3665,7 +3672,8 @@ static int i40e_configure_rx_ring(struct i40e_ring *ring) if (I40E_2K_TOO_SMALL_WITH_PADDING) { dev_info(&vsi->back->pdev->dev, "2k Rx buffer is too small to fit standard MTU and skb_shared_info\n"); - return -EOPNOTSUPP; + err = -EOPNOTSUPP; + goto unreg_xdp; } clear_ring_build_skb_enabled(ring); } else { @@ -3695,6 +3703,11 @@ static int i40e_configure_rx_ring(struct i40e_ring *ring) } return 0; +unreg_xdp: + if (ring->vsi->type == I40E_VSI_MAIN) + xdp_rxq_info_unreg(&ring->xdp_rxq); + + return err; } /** diff --git a/drivers/net/ethernet/intel/i40e/i40e_trace.h b/drivers/net/ethernet/intel/i40e/i40e_trace.h index 759f3d1c4c8f00..dde0ccd789ed16 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_trace.h +++ b/drivers/net/ethernet/intel/i40e/i40e_trace.h @@ -88,7 +88,7 @@ TRACE_EVENT(i40e_napi_poll, __entry->rx_clean_complete = rx_clean_complete; __entry->tx_clean_complete = tx_clean_complete; __entry->irq_num = q->irq_num; - __entry->curr_cpu = get_cpu(); + __entry->curr_cpu = smp_processor_id(); __assign_str(qname); __assign_str(dev_name); __assign_bitmask(irq_affinity, cpumask_bits(&q->affinity_mask), diff --git a/drivers/net/ethernet/intel/i40e/i40e_txrx.c b/drivers/net/ethernet/intel/i40e/i40e_txrx.c index cc0b9efc2637a4..816179c7e2712e 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_txrx.c +++ b/drivers/net/ethernet/intel/i40e/i40e_txrx.c @@ -1470,6 +1470,9 @@ void i40e_clean_rx_ring(struct i40e_ring *rx_ring) if (!rx_ring->rx_bi) return; + if (xdp_rxq_info_is_reg(&rx_ring->xdp_rxq)) + xdp_rxq_info_unreg(&rx_ring->xdp_rxq); + if (rx_ring->xsk_pool) { i40e_xsk_clean_rx_ring(rx_ring); goto skip_free; @@ -1527,8 +1530,6 @@ void i40e_clean_rx_ring(struct i40e_ring *rx_ring) void i40e_free_rx_resources(struct i40e_ring *rx_ring) { i40e_clean_rx_ring(rx_ring); - if (rx_ring->vsi->type == I40E_VSI_MAIN) - xdp_rxq_info_unreg(&rx_ring->xdp_rxq); rx_ring->xdp_prog = NULL; kfree(rx_ring->rx_bi); rx_ring->rx_bi = NULL; diff --git a/drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.c b/drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.c index 1fa877b52f618e..5a383ed09f7905 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.c +++ b/drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.c @@ -3833,10 +3833,10 @@ static int i40e_vc_del_cloud_filter(struct i40e_vf *vf, u8 *msg) cfilter.n_proto = ETH_P_IP; if (mask.dst_ip[0] & tcf.dst_ip[0]) memcpy(&cfilter.ip.v4.dst_ip, tcf.dst_ip, - ARRAY_SIZE(tcf.dst_ip)); - else if (mask.src_ip[0] & tcf.dst_ip[0]) + sizeof(cfilter.ip.v4.dst_ip)); + else if (mask.src_ip[0] & tcf.src_ip[0]) memcpy(&cfilter.ip.v4.src_ip, tcf.src_ip, - ARRAY_SIZE(tcf.dst_ip)); + sizeof(cfilter.ip.v4.src_ip)); break; case VIRTCHNL_TCP_V6_FLOW: cfilter.n_proto = ETH_P_IPV6; @@ -3891,7 +3891,7 @@ static int i40e_vc_del_cloud_filter(struct i40e_vf *vf, u8 *msg) /* for ipv6, mask is set for all sixteen bytes (4 words) */ if (cfilter.n_proto == ETH_P_IPV6 && mask.dst_ip[3]) if (memcmp(&cfilter.ip.v6.dst_ip6, &cf->ip.v6.dst_ip6, - sizeof(cfilter.ip.v6.src_ip6))) + sizeof(cfilter.ip.v6.dst_ip6))) continue; if (mask.vlan_id) if (cfilter.vlan_id != cf->vlan_id) @@ -3979,10 +3979,10 @@ static int i40e_vc_add_cloud_filter(struct i40e_vf *vf, u8 *msg) cfilter->n_proto = ETH_P_IP; if (mask.dst_ip[0] & tcf.dst_ip[0]) memcpy(&cfilter->ip.v4.dst_ip, tcf.dst_ip, - ARRAY_SIZE(tcf.dst_ip)); - else if (mask.src_ip[0] & tcf.dst_ip[0]) + sizeof(cfilter->ip.v4.dst_ip)); + else if (mask.src_ip[0] & tcf.src_ip[0]) memcpy(&cfilter->ip.v4.src_ip, tcf.src_ip, - ARRAY_SIZE(tcf.dst_ip)); + sizeof(cfilter->ip.v4.src_ip)); break; case VIRTCHNL_TCP_V6_FLOW: cfilter->n_proto = ETH_P_IPV6; diff --git a/drivers/net/ethernet/intel/iavf/iavf.h b/drivers/net/ethernet/intel/iavf/iavf.h index a87e0c6d4017ad..e9fb0a0919e376 100644 --- a/drivers/net/ethernet/intel/iavf/iavf.h +++ b/drivers/net/ethernet/intel/iavf/iavf.h @@ -260,7 +260,6 @@ struct iavf_adapter { struct work_struct adminq_task; struct work_struct finish_config; wait_queue_head_t down_waitqueue; - wait_queue_head_t reset_waitqueue; wait_queue_head_t vc_waitqueue; struct iavf_q_vector *q_vectors; struct list_head vlan_filter_list; @@ -626,5 +625,5 @@ void iavf_add_adv_rss_cfg(struct iavf_adapter *adapter); void iavf_del_adv_rss_cfg(struct iavf_adapter *adapter); struct iavf_mac_filter *iavf_add_filter(struct iavf_adapter *adapter, const u8 *macaddr); -int iavf_wait_for_reset(struct iavf_adapter *adapter); +void iavf_reset_step(struct iavf_adapter *adapter); #endif /* _IAVF_H_ */ diff --git a/drivers/net/ethernet/intel/iavf/iavf_ethtool.c b/drivers/net/ethernet/intel/iavf/iavf_ethtool.c index 2cc21289a70779..98bec3afc2006a 100644 --- a/drivers/net/ethernet/intel/iavf/iavf_ethtool.c +++ b/drivers/net/ethernet/intel/iavf/iavf_ethtool.c @@ -313,14 +313,13 @@ static int iavf_get_sset_count(struct net_device *netdev, int sset) { /* Report the maximum number queues, even if not every queue is * currently configured. Since allocation of queues is in pairs, - * use netdev->real_num_tx_queues * 2. The real_num_tx_queues is set - * at device creation and never changes. + * use netdev->num_tx_queues * 2. The num_tx_queues is set at + * device creation and never changes. */ if (sset == ETH_SS_STATS) return IAVF_STATS_LEN + - (IAVF_QUEUE_STATS_LEN * 2 * - netdev->real_num_tx_queues); + (IAVF_QUEUE_STATS_LEN * 2 * netdev->num_tx_queues); else return -EINVAL; } @@ -345,19 +344,19 @@ static void iavf_get_ethtool_stats(struct net_device *netdev, iavf_add_ethtool_stats(&data, adapter, iavf_gstrings_stats); rcu_read_lock(); - /* As num_active_queues describe both tx and rx queues, we can use - * it to iterate over rings' stats. + /* Use num_tx_queues to report stats for the maximum number of queues. + * Queues beyond num_active_queues will report zero. */ - for (i = 0; i < adapter->num_active_queues; i++) { - struct iavf_ring *ring; + for (i = 0; i < netdev->num_tx_queues; i++) { + struct iavf_ring *tx_ring = NULL, *rx_ring = NULL; - /* Tx rings stats */ - ring = &adapter->tx_rings[i]; - iavf_add_queue_stats(&data, ring); + if (i < adapter->num_active_queues) { + tx_ring = &adapter->tx_rings[i]; + rx_ring = &adapter->rx_rings[i]; + } - /* Rx rings stats */ - ring = &adapter->rx_rings[i]; - iavf_add_queue_stats(&data, ring); + iavf_add_queue_stats(&data, tx_ring); + iavf_add_queue_stats(&data, rx_ring); } rcu_read_unlock(); } @@ -376,9 +375,9 @@ static void iavf_get_stat_strings(struct net_device *netdev, u8 *data) iavf_add_stat_strings(&data, iavf_gstrings_stats); /* Queues are always allocated in pairs, so we just use - * real_num_tx_queues for both Tx and Rx queues. + * num_tx_queues for both Tx and Rx queues. */ - for (i = 0; i < netdev->real_num_tx_queues; i++) { + for (i = 0; i < netdev->num_tx_queues; i++) { iavf_add_stat_strings(&data, iavf_gstrings_queue_stats, "tx", i); iavf_add_stat_strings(&data, iavf_gstrings_queue_stats, @@ -492,7 +491,6 @@ static int iavf_set_ringparam(struct net_device *netdev, { struct iavf_adapter *adapter = netdev_priv(netdev); u32 new_rx_count, new_tx_count; - int ret = 0; if ((ring->rx_mini_pending) || (ring->rx_jumbo_pending)) return -EINVAL; @@ -537,13 +535,11 @@ static int iavf_set_ringparam(struct net_device *netdev, } if (netif_running(netdev)) { - iavf_schedule_reset(adapter, IAVF_FLAG_RESET_NEEDED); - ret = iavf_wait_for_reset(adapter); - if (ret) - netdev_warn(netdev, "Changing ring parameters timeout or interrupted waiting for reset"); + adapter->flags |= IAVF_FLAG_RESET_NEEDED; + iavf_reset_step(adapter); } - return ret; + return 0; } /** @@ -1723,7 +1719,6 @@ static int iavf_set_channels(struct net_device *netdev, { struct iavf_adapter *adapter = netdev_priv(netdev); u32 num_req = ch->combined_count; - int ret = 0; if ((adapter->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADQ) && adapter->num_tc) { @@ -1745,13 +1740,10 @@ static int iavf_set_channels(struct net_device *netdev, adapter->num_req_queues = num_req; adapter->flags |= IAVF_FLAG_REINIT_ITR_NEEDED; - iavf_schedule_reset(adapter, IAVF_FLAG_RESET_NEEDED); - - ret = iavf_wait_for_reset(adapter); - if (ret) - netdev_warn(netdev, "Changing channel count timeout or interrupted waiting for reset"); + adapter->flags |= IAVF_FLAG_RESET_NEEDED; + iavf_reset_step(adapter); - return ret; + return 0; } /** diff --git a/drivers/net/ethernet/intel/iavf/iavf_main.c b/drivers/net/ethernet/intel/iavf/iavf_main.c index 4b0fc8f354bc90..0a72d419782e57 100644 --- a/drivers/net/ethernet/intel/iavf/iavf_main.c +++ b/drivers/net/ethernet/intel/iavf/iavf_main.c @@ -185,31 +185,6 @@ static bool iavf_is_reset_in_progress(struct iavf_adapter *adapter) return false; } -/** - * iavf_wait_for_reset - Wait for reset to finish. - * @adapter: board private structure - * - * Returns 0 if reset finished successfully, negative on timeout or interrupt. - */ -int iavf_wait_for_reset(struct iavf_adapter *adapter) -{ - int ret = wait_event_interruptible_timeout(adapter->reset_waitqueue, - !iavf_is_reset_in_progress(adapter), - msecs_to_jiffies(5000)); - - /* If ret < 0 then it means wait was interrupted. - * If ret == 0 then it means we got a timeout while waiting - * for reset to finish. - * If ret > 0 it means reset has finished. - */ - if (ret > 0) - return 0; - else if (ret < 0) - return -EINTR; - else - return -EBUSY; -} - /** * iavf_allocate_dma_mem_d - OS specific memory alloc for shared code * @hw: pointer to the HW structure @@ -782,10 +757,13 @@ iavf_vlan_filter *iavf_add_vlan(struct iavf_adapter *adapter, adapter->num_vlan_filters++; iavf_schedule_aq_request(adapter, IAVF_FLAG_AQ_ADD_VLAN_FILTER); } else if (f->state == IAVF_VLAN_REMOVE) { - /* IAVF_VLAN_REMOVE means that VLAN wasn't yet removed. - * We can safely only change the state here. + /* Re-add the filter since we cannot tell whether the + * pending delete has already been processed by the PF. + * A duplicate add is harmless. */ - f->state = IAVF_VLAN_ACTIVE; + f->state = IAVF_VLAN_ADD; + iavf_schedule_aq_request(adapter, + IAVF_FLAG_AQ_ADD_VLAN_FILTER); } clearout: @@ -2797,7 +2775,22 @@ static void iavf_init_config_adapter(struct iavf_adapter *adapter) netdev->watchdog_timeo = 5 * HZ; netdev->min_mtu = ETH_MIN_MTU; - netdev->max_mtu = LIBIE_MAX_MTU; + + /* PF/VF API: vf_res->max_mtu is max frame size (not MTU). + * Convert to MTU. + */ + if (!adapter->vf_res->max_mtu) { + netdev->max_mtu = LIBIE_MAX_MTU; + } else if (adapter->vf_res->max_mtu < LIBETH_RX_LL_LEN + ETH_MIN_MTU || + adapter->vf_res->max_mtu > + LIBETH_RX_LL_LEN + LIBIE_MAX_MTU) { + netdev_warn_once(adapter->netdev, + "invalid max frame size %d from PF, using default MTU %d", + adapter->vf_res->max_mtu, LIBIE_MAX_MTU); + netdev->max_mtu = LIBIE_MAX_MTU; + } else { + netdev->max_mtu = adapter->vf_res->max_mtu - LIBETH_RX_LL_LEN; + } if (!is_valid_ether_addr(adapter->hw.mac.addr)) { dev_info(&pdev->dev, "Invalid MAC address %pM, using random\n", @@ -3025,6 +3018,8 @@ static void iavf_disable_vf(struct iavf_adapter *adapter) adapter->flags |= IAVF_FLAG_PF_COMMS_FAILED; + iavf_ptp_release(adapter); + /* We don't use netif_running() because it may be true prior to * ndo_open() returning, so we can't assume it means all our open * tasks have finished, since we're not holding the rtnl_lock here. @@ -3100,18 +3095,16 @@ static void iavf_reconfig_qs_bw(struct iavf_adapter *adapter) } /** - * iavf_reset_task - Call-back task to handle hardware reset - * @work: pointer to work_struct + * iavf_reset_step - Perform the VF reset sequence + * @adapter: board private structure * - * During reset we need to shut down and reinitialize the admin queue - * before we can use it to communicate with the PF again. We also clear - * and reinit the rings because that context is lost as well. - **/ -static void iavf_reset_task(struct work_struct *work) + * Requests a reset from PF, polls for completion, and reconfigures + * the driver. Caller must hold the netdev instance lock. + * + * This can sleep for several seconds while polling HW registers. + */ +void iavf_reset_step(struct iavf_adapter *adapter) { - struct iavf_adapter *adapter = container_of(work, - struct iavf_adapter, - reset_task); struct virtchnl_vf_resource *vfres = adapter->vf_res; struct net_device *netdev = adapter->netdev; struct iavf_hw *hw = &adapter->hw; @@ -3122,7 +3115,7 @@ static void iavf_reset_task(struct work_struct *work) int i = 0, err; bool running; - netdev_lock(netdev); + netdev_assert_locked(netdev); iavf_misc_irq_disable(adapter); if (adapter->flags & IAVF_FLAG_RESET_NEEDED) { @@ -3167,7 +3160,6 @@ static void iavf_reset_task(struct work_struct *work) dev_err(&adapter->pdev->dev, "Reset never finished (%x)\n", reg_val); iavf_disable_vf(adapter); - netdev_unlock(netdev); return; /* Do not attempt to reinit. It's dead, Jim. */ } @@ -3179,7 +3171,6 @@ static void iavf_reset_task(struct work_struct *work) iavf_startup(adapter); queue_delayed_work(adapter->wq, &adapter->watchdog_task, msecs_to_jiffies(30)); - netdev_unlock(netdev); return; } @@ -3200,6 +3191,8 @@ static void iavf_reset_task(struct work_struct *work) iavf_change_state(adapter, __IAVF_RESETTING); adapter->flags &= ~IAVF_FLAG_RESET_PENDING; + iavf_ptp_release(adapter); + /* free the Tx/Rx rings and descriptors, might be better to just * re-use them sometime in the future */ @@ -3320,9 +3313,6 @@ static void iavf_reset_task(struct work_struct *work) adapter->flags &= ~IAVF_FLAG_REINIT_ITR_NEEDED; - wake_up(&adapter->reset_waitqueue); - netdev_unlock(netdev); - return; reset_err: if (running) { @@ -3331,10 +3321,21 @@ static void iavf_reset_task(struct work_struct *work) } iavf_disable_vf(adapter); - netdev_unlock(netdev); dev_err(&adapter->pdev->dev, "failed to allocate resources during reinit\n"); } +static void iavf_reset_task(struct work_struct *work) +{ + struct iavf_adapter *adapter = container_of(work, + struct iavf_adapter, + reset_task); + struct net_device *netdev = adapter->netdev; + + netdev_lock(netdev); + iavf_reset_step(adapter); + netdev_unlock(netdev); +} + /** * iavf_adminq_task - worker thread to clean the admin queue * @work: pointer to work_struct containing our data @@ -4600,22 +4601,17 @@ static int iavf_close(struct net_device *netdev) static int iavf_change_mtu(struct net_device *netdev, int new_mtu) { struct iavf_adapter *adapter = netdev_priv(netdev); - int ret = 0; netdev_dbg(netdev, "changing MTU from %d to %d\n", netdev->mtu, new_mtu); WRITE_ONCE(netdev->mtu, new_mtu); if (netif_running(netdev)) { - iavf_schedule_reset(adapter, IAVF_FLAG_RESET_NEEDED); - ret = iavf_wait_for_reset(adapter); - if (ret < 0) - netdev_warn(netdev, "MTU change interrupted waiting for reset"); - else if (ret) - netdev_warn(netdev, "MTU change timed out waiting for reset"); + adapter->flags |= IAVF_FLAG_RESET_NEEDED; + iavf_reset_step(adapter); } - return ret; + return 0; } /** @@ -5420,9 +5416,6 @@ static int iavf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) /* Setup the wait queue for indicating transition to down status */ init_waitqueue_head(&adapter->down_waitqueue); - /* Setup the wait queue for indicating transition to running state */ - init_waitqueue_head(&adapter->reset_waitqueue); - /* Setup the wait queue for indicating virtchannel events */ init_waitqueue_head(&adapter->vc_waitqueue); diff --git a/drivers/net/ethernet/intel/iavf/iavf_virtchnl.c b/drivers/net/ethernet/intel/iavf/iavf_virtchnl.c index 88156082a41da6..a52c100dcbc56d 100644 --- a/drivers/net/ethernet/intel/iavf/iavf_virtchnl.c +++ b/drivers/net/ethernet/intel/iavf/iavf_virtchnl.c @@ -2736,7 +2736,6 @@ void iavf_virtchnl_completion(struct iavf_adapter *adapter, case VIRTCHNL_OP_ENABLE_QUEUES: /* enable transmits */ iavf_irq_enable(adapter, true); - wake_up(&adapter->reset_waitqueue); adapter->flags &= ~IAVF_FLAG_QUEUES_DISABLED; break; case VIRTCHNL_OP_DISABLE_QUEUES: diff --git a/drivers/net/ethernet/intel/ice/devlink/devlink.c b/drivers/net/ethernet/intel/ice/devlink/devlink.c index 2ef39cc70c21d3..7de749d3f04794 100644 --- a/drivers/net/ethernet/intel/ice/devlink/devlink.c +++ b/drivers/net/ethernet/intel/ice/devlink/devlink.c @@ -1360,7 +1360,7 @@ ice_devlink_enable_roce_get(struct devlink *devlink, u32 id, cdev = pf->cdev_info; if (!cdev) - return -ENODEV; + return -EOPNOTSUPP; ctx->val.vbool = !!(cdev->rdma_protocol & IIDC_RDMA_PROTOCOL_ROCEV2); @@ -1427,7 +1427,7 @@ ice_devlink_enable_iw_get(struct devlink *devlink, u32 id, cdev = pf->cdev_info; if (!cdev) - return -ENODEV; + return -EOPNOTSUPP; ctx->val.vbool = !!(cdev->rdma_protocol & IIDC_RDMA_PROTOCOL_IWARP); diff --git a/drivers/net/ethernet/intel/ice/ice.h b/drivers/net/ethernet/intel/ice/ice.h index 00f75d87c73f9a..15a7fcd888b262 100644 --- a/drivers/net/ethernet/intel/ice/ice.h +++ b/drivers/net/ethernet/intel/ice/ice.h @@ -990,6 +990,7 @@ int ice_schedule_reset(struct ice_pf *pf, enum ice_reset_req reset); void ice_print_link_msg(struct ice_vsi *vsi, bool isup); int ice_plug_aux_dev(struct ice_pf *pf); void ice_unplug_aux_dev(struct ice_pf *pf); +void ice_rdma_finalize_setup(struct ice_pf *pf); int ice_init_rdma(struct ice_pf *pf); void ice_deinit_rdma(struct ice_pf *pf); bool ice_is_wol_supported(struct ice_hw *hw); diff --git a/drivers/net/ethernet/intel/ice/ice_base.c b/drivers/net/ethernet/intel/ice/ice_base.c index eadb1e3d12b3a8..5a6da2d501213f 100644 --- a/drivers/net/ethernet/intel/ice/ice_base.c +++ b/drivers/net/ethernet/intel/ice/ice_base.c @@ -124,6 +124,8 @@ static int ice_vsi_alloc_q_vector(struct ice_vsi *vsi, u16 v_idx) if (vsi->type == ICE_VSI_VF) { ice_calc_vf_reg_idx(vsi->vf, q_vector); goto out; + } else if (vsi->type == ICE_VSI_LB) { + goto skip_alloc; } else if (vsi->type == ICE_VSI_CTRL && vsi->vf) { struct ice_vsi *ctrl_vsi = ice_get_vf_ctrl_vsi(pf, vsi); @@ -659,33 +661,22 @@ static int ice_vsi_cfg_rxq(struct ice_rx_ring *ring) { struct device *dev = ice_pf_to_dev(ring->vsi->back); u32 num_bufs = ICE_DESC_UNUSED(ring); - u32 rx_buf_len; int err; - if (ring->vsi->type == ICE_VSI_PF || ring->vsi->type == ICE_VSI_SF) { - if (!xdp_rxq_info_is_reg(&ring->xdp_rxq)) { - err = __xdp_rxq_info_reg(&ring->xdp_rxq, ring->netdev, - ring->q_index, - ring->q_vector->napi.napi_id, - ring->rx_buf_len); - if (err) - return err; - } - + if (ring->vsi->type == ICE_VSI_PF || ring->vsi->type == ICE_VSI_SF || + ring->vsi->type == ICE_VSI_LB) { ice_rx_xsk_pool(ring); err = ice_realloc_rx_xdp_bufs(ring, ring->xsk_pool); if (err) return err; if (ring->xsk_pool) { - xdp_rxq_info_unreg(&ring->xdp_rxq); - - rx_buf_len = - xsk_pool_get_rx_frame_size(ring->xsk_pool); + u32 frag_size = + xsk_pool_get_rx_frag_step(ring->xsk_pool); err = __xdp_rxq_info_reg(&ring->xdp_rxq, ring->netdev, ring->q_index, ring->q_vector->napi.napi_id, - rx_buf_len); + frag_size); if (err) return err; err = xdp_rxq_info_reg_mem_model(&ring->xdp_rxq, @@ -702,14 +693,13 @@ static int ice_vsi_cfg_rxq(struct ice_rx_ring *ring) if (err) return err; - if (!xdp_rxq_info_is_reg(&ring->xdp_rxq)) { - err = __xdp_rxq_info_reg(&ring->xdp_rxq, ring->netdev, - ring->q_index, - ring->q_vector->napi.napi_id, - ring->rx_buf_len); - if (err) - goto err_destroy_fq; - } + err = __xdp_rxq_info_reg(&ring->xdp_rxq, ring->netdev, + ring->q_index, + ring->q_vector->napi.napi_id, + ring->truesize); + if (err) + goto err_destroy_fq; + xdp_rxq_info_attach_page_pool(&ring->xdp_rxq, ring->pp); } diff --git a/drivers/net/ethernet/intel/ice/ice_common.c b/drivers/net/ethernet/intel/ice/ice_common.c index 785bf5cc1b25ec..26eb8e05498b38 100644 --- a/drivers/net/ethernet/intel/ice/ice_common.c +++ b/drivers/net/ethernet/intel/ice/ice_common.c @@ -1854,6 +1854,7 @@ static bool ice_should_retry_sq_send_cmd(u16 opcode) case ice_aqc_opc_lldp_stop: case ice_aqc_opc_lldp_start: case ice_aqc_opc_lldp_filter_ctrl: + case ice_aqc_opc_sff_eeprom: return true; } @@ -1879,6 +1880,7 @@ ice_sq_send_cmd_retry(struct ice_hw *hw, struct ice_ctl_q_info *cq, { struct libie_aq_desc desc_cpy; bool is_cmd_for_retry; + u8 *buf_cpy = NULL; u8 idx = 0; u16 opcode; int status; @@ -1888,8 +1890,11 @@ ice_sq_send_cmd_retry(struct ice_hw *hw, struct ice_ctl_q_info *cq, memset(&desc_cpy, 0, sizeof(desc_cpy)); if (is_cmd_for_retry) { - /* All retryable cmds are direct, without buf. */ - WARN_ON(buf); + if (buf) { + buf_cpy = kmemdup(buf, buf_size, GFP_KERNEL); + if (!buf_cpy) + return -ENOMEM; + } memcpy(&desc_cpy, desc, sizeof(desc_cpy)); } @@ -1901,12 +1906,14 @@ ice_sq_send_cmd_retry(struct ice_hw *hw, struct ice_ctl_q_info *cq, hw->adminq.sq_last_status != LIBIE_AQ_RC_EBUSY) break; + if (buf_cpy) + memcpy(buf, buf_cpy, buf_size); memcpy(desc, &desc_cpy, sizeof(desc_cpy)); - msleep(ICE_SQ_SEND_DELAY_TIME_MS); } while (++idx < ICE_SQ_SEND_MAX_EXECUTE); + kfree(buf_cpy); return status; } @@ -6429,7 +6436,7 @@ int ice_lldp_fltr_add_remove(struct ice_hw *hw, struct ice_vsi *vsi, bool add) struct ice_aqc_lldp_filter_ctrl *cmd; struct libie_aq_desc desc; - if (vsi->type != ICE_VSI_PF || !ice_fw_supports_lldp_fltr_ctrl(hw)) + if (!ice_fw_supports_lldp_fltr_ctrl(hw)) return -EOPNOTSUPP; cmd = libie_aq_raw(&desc); diff --git a/drivers/net/ethernet/intel/ice/ice_ethtool.c b/drivers/net/ethernet/intel/ice/ice_ethtool.c index 3565a5d96c6d18..3125dc1b27654f 100644 --- a/drivers/net/ethernet/intel/ice/ice_ethtool.c +++ b/drivers/net/ethernet/intel/ice/ice_ethtool.c @@ -1289,6 +1289,10 @@ static u64 ice_loopback_test(struct net_device *netdev) test_vsi->netdev = netdev; tx_ring = test_vsi->tx_rings[0]; rx_ring = test_vsi->rx_rings[0]; + /* Dummy q_vector and napi. Fill the minimum required for + * ice_rxq_pp_create(). + */ + rx_ring->q_vector->napi.dev = netdev; if (ice_lbtest_prepare_rings(test_vsi)) { ret = 2; @@ -1926,6 +1930,17 @@ __ice_get_ethtool_stats(struct net_device *netdev, int i = 0; char *p; + if (ice_is_port_repr_netdev(netdev)) { + ice_update_eth_stats(vsi); + + for (j = 0; j < ICE_VSI_STATS_LEN; j++) { + p = (char *)vsi + ice_gstrings_vsi_stats[j].stat_offset; + data[i++] = (ice_gstrings_vsi_stats[j].sizeof_stat == + sizeof(u64)) ? *(u64 *)p : *(u32 *)p; + } + return; + } + ice_update_pf_stats(pf); ice_update_vsi_stats(vsi); @@ -1935,9 +1950,6 @@ __ice_get_ethtool_stats(struct net_device *netdev, sizeof(u64)) ? *(u64 *)p : *(u32 *)p; } - if (ice_is_port_repr_netdev(netdev)) - return; - /* populate per queue stats */ rcu_read_lock(); @@ -3318,7 +3330,7 @@ ice_set_ringparam(struct net_device *netdev, struct ethtool_ringparam *ring, rx_rings = kcalloc(vsi->num_rxq, sizeof(*rx_rings), GFP_KERNEL); if (!rx_rings) { err = -ENOMEM; - goto done; + goto free_xdp; } ice_for_each_rxq(vsi, i) { @@ -3328,6 +3340,7 @@ ice_set_ringparam(struct net_device *netdev, struct ethtool_ringparam *ring, rx_rings[i].cached_phctime = pf->ptp.cached_phc_time; rx_rings[i].desc = NULL; rx_rings[i].xdp_buf = NULL; + rx_rings[i].xdp_rxq = (struct xdp_rxq_info){ }; /* this is to allow wr32 to have something to write to * during early allocation of Rx buffers @@ -3345,7 +3358,7 @@ ice_set_ringparam(struct net_device *netdev, struct ethtool_ringparam *ring, } kfree(rx_rings); err = -ENOMEM; - goto free_tx; + goto free_xdp; } } @@ -3398,6 +3411,13 @@ ice_set_ringparam(struct net_device *netdev, struct ethtool_ringparam *ring, } goto done; +free_xdp: + if (xdp_rings) { + ice_for_each_xdp_txq(vsi, i) + ice_free_tx_ring(&xdp_rings[i]); + kfree(xdp_rings); + } + free_tx: /* error cleanup if the Rx allocations failed after getting Tx */ if (tx_rings) { @@ -4496,7 +4516,7 @@ ice_get_module_eeprom(struct net_device *netdev, u8 addr = ICE_I2C_EEPROM_DEV_ADDR; struct ice_hw *hw = &pf->hw; bool is_sfp = false; - unsigned int i, j; + unsigned int i; u16 offset = 0; u8 page = 0; int status; @@ -4538,26 +4558,19 @@ ice_get_module_eeprom(struct net_device *netdev, if (page == 0 || !(data[0x2] & 0x4)) { u32 copy_len; - /* If i2c bus is busy due to slow page change or - * link management access, call can fail. This is normal. - * So we retry this a few times. - */ - for (j = 0; j < 4; j++) { - status = ice_aq_sff_eeprom(hw, 0, addr, offset, page, - !is_sfp, value, - SFF_READ_BLOCK_SIZE, - 0, NULL); - netdev_dbg(netdev, "SFF %02X %02X %02X %X = %02X%02X%02X%02X.%02X%02X%02X%02X (%X)\n", - addr, offset, page, is_sfp, - value[0], value[1], value[2], value[3], - value[4], value[5], value[6], value[7], - status); - if (status) { - usleep_range(1500, 2500); - memset(value, 0, SFF_READ_BLOCK_SIZE); - continue; - } - break; + status = ice_aq_sff_eeprom(hw, 0, addr, offset, page, + !is_sfp, value, + SFF_READ_BLOCK_SIZE, + 0, NULL); + netdev_dbg(netdev, "SFF %02X %02X %02X %X = %02X%02X%02X%02X.%02X%02X%02X%02X (%pe)\n", + addr, offset, page, is_sfp, + value[0], value[1], value[2], value[3], + value[4], value[5], value[6], value[7], + ERR_PTR(status)); + if (status) { + netdev_err(netdev, "%s: error reading module EEPROM: status %pe\n", + __func__, ERR_PTR(status)); + return status; } /* Make sure we have enough room for the new block */ diff --git a/drivers/net/ethernet/intel/ice/ice_idc.c b/drivers/net/ethernet/intel/ice/ice_idc.c index 420d45c2558b62..ded029aa71d7d0 100644 --- a/drivers/net/ethernet/intel/ice/ice_idc.c +++ b/drivers/net/ethernet/intel/ice/ice_idc.c @@ -360,6 +360,39 @@ void ice_unplug_aux_dev(struct ice_pf *pf) auxiliary_device_uninit(adev); } +/** + * ice_rdma_finalize_setup - Complete RDMA setup after VSI is ready + * @pf: ptr to ice_pf + * + * Sets VSI-dependent information and plugs aux device. + * Must be called after ice_init_rdma(), ice_vsi_rebuild(), and + * ice_dcb_rebuild() complete. + */ +void ice_rdma_finalize_setup(struct ice_pf *pf) +{ + struct device *dev = ice_pf_to_dev(pf); + struct iidc_rdma_priv_dev_info *privd; + int ret; + + if (!ice_is_rdma_ena(pf) || !pf->cdev_info) + return; + + privd = pf->cdev_info->iidc_priv; + if (!privd || !pf->vsi || !pf->vsi[0] || !pf->vsi[0]->netdev) + return; + + /* Assign VSI info now that VSI is valid */ + privd->netdev = pf->vsi[0]->netdev; + privd->vport_id = pf->vsi[0]->vsi_num; + + /* Update QoS info after DCB has been rebuilt */ + ice_setup_dcb_qos_info(pf, &privd->qos_info); + + ret = ice_plug_aux_dev(pf); + if (ret) + dev_warn(dev, "Failed to plug RDMA aux device: %d\n", ret); +} + /** * ice_init_rdma - initializes PF for RDMA use * @pf: ptr to ice_pf @@ -398,22 +431,14 @@ int ice_init_rdma(struct ice_pf *pf) } cdev->iidc_priv = privd; - privd->netdev = pf->vsi[0]->netdev; privd->hw_addr = (u8 __iomem *)pf->hw.hw_addr; cdev->pdev = pf->pdev; - privd->vport_id = pf->vsi[0]->vsi_num; pf->cdev_info->rdma_protocol |= IIDC_RDMA_PROTOCOL_ROCEV2; - ice_setup_dcb_qos_info(pf, &privd->qos_info); - ret = ice_plug_aux_dev(pf); - if (ret) - goto err_plug_aux_dev; + return 0; -err_plug_aux_dev: - pf->cdev_info->adev = NULL; - xa_erase(&ice_aux_id, pf->aux_idx); err_alloc_xa: kfree(privd); err_privd_alloc: @@ -432,7 +457,6 @@ void ice_deinit_rdma(struct ice_pf *pf) if (!ice_is_rdma_ena(pf)) return; - ice_unplug_aux_dev(pf); xa_erase(&ice_aux_id, pf->aux_idx); kfree(pf->cdev_info->iidc_priv); kfree(pf->cdev_info); diff --git a/drivers/net/ethernet/intel/ice/ice_lib.c b/drivers/net/ethernet/intel/ice/ice_lib.c index d47af94f31a991..bad67e4dc044f3 100644 --- a/drivers/net/ethernet/intel/ice/ice_lib.c +++ b/drivers/net/ethernet/intel/ice/ice_lib.c @@ -107,10 +107,6 @@ static int ice_vsi_alloc_arrays(struct ice_vsi *vsi) if (!vsi->rxq_map) goto err_rxq_map; - /* There is no need to allocate q_vectors for a loopback VSI. */ - if (vsi->type == ICE_VSI_LB) - return 0; - /* allocate memory for q_vector pointers */ vsi->q_vectors = devm_kcalloc(dev, vsi->num_q_vectors, sizeof(*vsi->q_vectors), GFP_KERNEL); @@ -239,6 +235,8 @@ static void ice_vsi_set_num_qs(struct ice_vsi *vsi) case ICE_VSI_LB: vsi->alloc_txq = 1; vsi->alloc_rxq = 1; + /* A dummy q_vector, no actual IRQ. */ + vsi->num_q_vectors = 1; break; default: dev_warn(ice_pf_to_dev(pf), "Unknown VSI type %d\n", vsi_type); @@ -2424,14 +2422,21 @@ static int ice_vsi_cfg_def(struct ice_vsi *vsi) } break; case ICE_VSI_LB: - ret = ice_vsi_alloc_rings(vsi); + ret = ice_vsi_alloc_q_vectors(vsi); if (ret) goto unroll_vsi_init; + ret = ice_vsi_alloc_rings(vsi); + if (ret) + goto unroll_alloc_q_vector; + ret = ice_vsi_alloc_ring_stats(vsi); if (ret) goto unroll_vector_base; + /* Simply map the dummy q_vector to the only rx_ring */ + vsi->rx_rings[0]->q_vector = vsi->q_vectors[0]; + break; default: /* clean up the resources and exit */ diff --git a/drivers/net/ethernet/intel/ice/ice_main.c b/drivers/net/ethernet/intel/ice/ice_main.c index d04605d3e61af1..dddf1ae31952db 100644 --- a/drivers/net/ethernet/intel/ice/ice_main.c +++ b/drivers/net/ethernet/intel/ice/ice_main.c @@ -5138,6 +5138,9 @@ int ice_load(struct ice_pf *pf) if (err) goto err_init_rdma; + /* Finalize RDMA: VSI already created, assign info and plug device */ + ice_rdma_finalize_setup(pf); + ice_service_task_restart(pf); clear_bit(ICE_DOWN, pf->state); @@ -5169,6 +5172,7 @@ void ice_unload(struct ice_pf *pf) devl_assert_locked(priv_to_devlink(pf)); + ice_unplug_aux_dev(pf); ice_deinit_rdma(pf); ice_deinit_features(pf); ice_tc_indir_block_unregister(vsi); @@ -5595,6 +5599,7 @@ static int ice_suspend(struct device *dev) */ disabled = ice_service_task_stop(pf); + ice_unplug_aux_dev(pf); ice_deinit_rdma(pf); /* Already suspended?, then there is nothing to do */ @@ -7803,7 +7808,7 @@ static void ice_rebuild(struct ice_pf *pf, enum ice_reset_req reset_type) ice_health_clear(pf); - ice_plug_aux_dev(pf); + ice_rdma_finalize_setup(pf); if (ice_is_feature_supported(pf, ICE_F_SRIOV_LAG)) ice_lag_rebuild(pf); diff --git a/drivers/net/ethernet/intel/ice/ice_ptp.c b/drivers/net/ethernet/intel/ice/ice_ptp.c index 27268300147625..082313023024cc 100644 --- a/drivers/net/ethernet/intel/ice/ice_ptp.c +++ b/drivers/net/ethernet/intel/ice/ice_ptp.c @@ -3048,7 +3048,13 @@ static int ice_ptp_setup_pf(struct ice_pf *pf) struct ice_ptp *ctrl_ptp = ice_get_ctrl_ptp(pf); struct ice_ptp *ptp = &pf->ptp; - if (WARN_ON(!ctrl_ptp) || pf->hw.mac_type == ICE_MAC_UNKNOWN) + if (!ctrl_ptp) { + dev_info(ice_pf_to_dev(pf), + "PTP unavailable: no controlling PF\n"); + return -EOPNOTSUPP; + } + + if (pf->hw.mac_type == ICE_MAC_UNKNOWN) return -ENODEV; INIT_LIST_HEAD(&ptp->port.list_node); diff --git a/drivers/net/ethernet/intel/ice/ice_repr.c b/drivers/net/ethernet/intel/ice/ice_repr.c index cb08746556a670..f1e82ba155cff2 100644 --- a/drivers/net/ethernet/intel/ice/ice_repr.c +++ b/drivers/net/ethernet/intel/ice/ice_repr.c @@ -2,6 +2,7 @@ /* Copyright (C) 2019-2021, Intel Corporation. */ #include "ice.h" +#include "ice_lib.h" #include "ice_eswitch.h" #include "devlink/devlink.h" #include "devlink/port.h" @@ -67,7 +68,7 @@ ice_repr_get_stats64(struct net_device *netdev, struct rtnl_link_stats64 *stats) return; vsi = repr->src_vsi; - ice_update_vsi_stats(vsi); + ice_update_eth_stats(vsi); eth_stats = &vsi->eth_stats; stats->tx_packets = eth_stats->tx_unicast + eth_stats->tx_broadcast + @@ -315,7 +316,7 @@ ice_repr_reg_netdev(struct net_device *netdev, const struct net_device_ops *ops) static int ice_repr_ready_vf(struct ice_repr *repr) { - return !ice_check_vf_ready_for_cfg(repr->vf); + return ice_check_vf_ready_for_cfg(repr->vf); } static int ice_repr_ready_sf(struct ice_repr *repr) diff --git a/drivers/net/ethernet/intel/ice/ice_txrx.c b/drivers/net/ethernet/intel/ice/ice_txrx.c index ad76768a42323f..f47b96ceb9a475 100644 --- a/drivers/net/ethernet/intel/ice/ice_txrx.c +++ b/drivers/net/ethernet/intel/ice/ice_txrx.c @@ -560,7 +560,9 @@ void ice_clean_rx_ring(struct ice_rx_ring *rx_ring) i = 0; } - if (rx_ring->vsi->type == ICE_VSI_PF && + if ((rx_ring->vsi->type == ICE_VSI_PF || + rx_ring->vsi->type == ICE_VSI_SF || + rx_ring->vsi->type == ICE_VSI_LB) && xdp_rxq_info_is_reg(&rx_ring->xdp_rxq)) { xdp_rxq_info_detach_mem_model(&rx_ring->xdp_rxq); xdp_rxq_info_unreg(&rx_ring->xdp_rxq); diff --git a/drivers/net/ethernet/intel/ice/ice_xsk.c b/drivers/net/ethernet/intel/ice/ice_xsk.c index 989ff1fd91103e..102631398af3cd 100644 --- a/drivers/net/ethernet/intel/ice/ice_xsk.c +++ b/drivers/net/ethernet/intel/ice/ice_xsk.c @@ -900,6 +900,9 @@ void ice_xsk_clean_rx_ring(struct ice_rx_ring *rx_ring) u16 ntc = rx_ring->next_to_clean; u16 ntu = rx_ring->next_to_use; + if (xdp_rxq_info_is_reg(&rx_ring->xdp_rxq)) + xdp_rxq_info_unreg(&rx_ring->xdp_rxq); + while (ntc != ntu) { struct xdp_buff *xdp = *ice_xdp_buf(rx_ring, ntc); diff --git a/drivers/net/ethernet/intel/idpf/idpf_ethtool.c b/drivers/net/ethernet/intel/idpf/idpf_ethtool.c index 2efa3c08aba5c7..49cefb973f4dac 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_ethtool.c +++ b/drivers/net/ethernet/intel/idpf/idpf_ethtool.c @@ -307,9 +307,6 @@ static int idpf_del_flow_steer(struct net_device *netdev, vport_config = vport->adapter->vport_config[np->vport_idx]; user_config = &vport_config->user_config; - if (!idpf_sideband_action_ena(vport, fsp)) - return -EOPNOTSUPP; - rule = kzalloc(struct_size(rule, rule_info, 1), GFP_KERNEL); if (!rule) return -ENOMEM; diff --git a/drivers/net/ethernet/intel/idpf/idpf_txrx.c b/drivers/net/ethernet/intel/idpf/idpf_txrx.c index f58f616d87fc4e..d365564831b0bf 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_txrx.c +++ b/drivers/net/ethernet/intel/idpf/idpf_txrx.c @@ -2326,7 +2326,7 @@ void idpf_wait_for_sw_marker_completion(const struct idpf_tx_queue *txq) do { struct idpf_splitq_4b_tx_compl_desc *tx_desc; - struct idpf_tx_queue *target; + struct idpf_tx_queue *target = NULL; u32 ctype_gen, id; tx_desc = flow ? &complq->comp[ntc].common : @@ -2346,14 +2346,14 @@ void idpf_wait_for_sw_marker_completion(const struct idpf_tx_queue *txq) target = complq->txq_grp->txqs[id]; idpf_queue_clear(SW_MARKER, target); - if (target == txq) - break; next: if (unlikely(++ntc == complq->desc_count)) { ntc = 0; gen_flag = !gen_flag; } + if (target == txq) + break; } while (time_before(jiffies, timeout)); idpf_queue_assign(GEN_CHK, complq, gen_flag); @@ -4038,7 +4038,7 @@ static int idpf_vport_intr_req_irq(struct idpf_vport *vport) continue; name = kasprintf(GFP_KERNEL, "%s-%s-%s-%d", drv_name, if_name, - vec_name, vidx); + vec_name, vector); err = request_irq(irq_num, idpf_vport_intr_clean_queues, 0, name, q_vector); diff --git a/drivers/net/ethernet/intel/idpf/idpf_virtchnl.c b/drivers/net/ethernet/intel/idpf/idpf_virtchnl.c index cb702eac86c804..3c8ceff0e669f1 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_virtchnl.c +++ b/drivers/net/ethernet/intel/idpf/idpf_virtchnl.c @@ -284,26 +284,21 @@ int idpf_send_mb_msg(struct idpf_adapter *adapter, u32 op, return err; } -/* API for virtchnl "transaction" support ("xn" for short). - * - * We are reusing the completion lock to serialize the accesses to the - * transaction state for simplicity, but it could be its own separate synchro - * as well. For now, this API is only used from within a workqueue context; - * raw_spin_lock() is enough. - */ +/* API for virtchnl "transaction" support ("xn" for short). */ + /** * idpf_vc_xn_lock - Request exclusive access to vc transaction * @xn: struct idpf_vc_xn* to access */ #define idpf_vc_xn_lock(xn) \ - raw_spin_lock(&(xn)->completed.wait.lock) + spin_lock(&(xn)->lock) /** * idpf_vc_xn_unlock - Release exclusive access to vc transaction * @xn: struct idpf_vc_xn* to access */ #define idpf_vc_xn_unlock(xn) \ - raw_spin_unlock(&(xn)->completed.wait.lock) + spin_unlock(&(xn)->lock) /** * idpf_vc_xn_release_bufs - Release reference to reply buffer(s) and @@ -335,6 +330,7 @@ static void idpf_vc_xn_init(struct idpf_vc_xn_manager *vcxn_mngr) xn->state = IDPF_VC_XN_IDLE; xn->idx = i; idpf_vc_xn_release_bufs(xn); + spin_lock_init(&xn->lock); init_completion(&xn->completed); } @@ -403,7 +399,9 @@ static void idpf_vc_xn_push_free(struct idpf_vc_xn_manager *vcxn_mngr, struct idpf_vc_xn *xn) { idpf_vc_xn_release_bufs(xn); + spin_lock_bh(&vcxn_mngr->xn_bm_lock); set_bit(xn->idx, vcxn_mngr->free_xn_bm); + spin_unlock_bh(&vcxn_mngr->xn_bm_lock); } /** @@ -614,6 +612,10 @@ idpf_vc_xn_forward_reply(struct idpf_adapter *adapter, err = -ENXIO; goto out_unlock; case IDPF_VC_XN_ASYNC: + /* Set reply_sz from the actual payload so that async_handler + * can evaluate the response. + */ + xn->reply_sz = ctlq_msg->data_len; err = idpf_vc_xn_forward_async(adapter, xn, ctlq_msg); idpf_vc_xn_unlock(xn); return err; diff --git a/drivers/net/ethernet/intel/idpf/idpf_virtchnl.h b/drivers/net/ethernet/intel/idpf/idpf_virtchnl.h index eac3d15daa42f8..0a50a08eb9af76 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_virtchnl.h +++ b/drivers/net/ethernet/intel/idpf/idpf_virtchnl.h @@ -42,8 +42,8 @@ typedef int (*async_vc_cb) (struct idpf_adapter *, struct idpf_vc_xn *, * struct idpf_vc_xn - Data structure representing virtchnl transactions * @completed: virtchnl event loop uses that to signal when a reply is * available, uses kernel completion API - * @state: virtchnl event loop stores the data below, protected by the - * completion's lock. + * @lock: protects the transaction state fields below + * @state: virtchnl event loop stores the data below, protected by @lock * @reply_sz: Original size of reply, may be > reply_buf.iov_len; it will be * truncated on its way to the receiver thread according to * reply_buf.iov_len. @@ -58,6 +58,7 @@ typedef int (*async_vc_cb) (struct idpf_adapter *, struct idpf_vc_xn *, */ struct idpf_vc_xn { struct completion completed; + spinlock_t lock; enum idpf_vc_xn_state state; size_t reply_sz; struct kvec reply; diff --git a/drivers/net/ethernet/intel/igb/igb_main.c b/drivers/net/ethernet/intel/igb/igb_main.c index dbea37269d2cdb..33c6d61ef5188e 100644 --- a/drivers/net/ethernet/intel/igb/igb_main.c +++ b/drivers/net/ethernet/intel/igb/igb_main.c @@ -2203,9 +2203,8 @@ void igb_down(struct igb_adapter *adapter) for (i = 0; i < adapter->num_q_vectors; i++) { if (adapter->q_vector[i]) { - napi_synchronize(&adapter->q_vector[i]->napi); - igb_set_queue_napi(adapter, i, NULL); napi_disable(&adapter->q_vector[i]->napi); + igb_set_queue_napi(adapter, i, NULL); } } diff --git a/drivers/net/ethernet/intel/igb/igb_xsk.c b/drivers/net/ethernet/intel/igb/igb_xsk.c index 30ce5fbb5b776d..ce4a7b58cad2ff 100644 --- a/drivers/net/ethernet/intel/igb/igb_xsk.c +++ b/drivers/net/ethernet/intel/igb/igb_xsk.c @@ -524,6 +524,16 @@ bool igb_xmit_zc(struct igb_ring *tx_ring, struct xsk_buff_pool *xsk_pool) return nb_pkts < budget; } +static u32 igb_sw_irq_prep(struct igb_q_vector *q_vector) +{ + u32 eics = 0; + + if (!napi_if_scheduled_mark_missed(&q_vector->napi)) + eics = q_vector->eims_value; + + return eics; +} + int igb_xsk_wakeup(struct net_device *dev, u32 qid, u32 flags) { struct igb_adapter *adapter = netdev_priv(dev); @@ -542,20 +552,32 @@ int igb_xsk_wakeup(struct net_device *dev, u32 qid, u32 flags) ring = adapter->tx_ring[qid]; - if (test_bit(IGB_RING_FLAG_TX_DISABLED, &ring->flags)) - return -ENETDOWN; - if (!READ_ONCE(ring->xsk_pool)) return -EINVAL; - if (!napi_if_scheduled_mark_missed(&ring->q_vector->napi)) { + if (flags & XDP_WAKEUP_TX) { + if (test_bit(IGB_RING_FLAG_TX_DISABLED, &ring->flags)) + return -ENETDOWN; + + eics |= igb_sw_irq_prep(ring->q_vector); + } + + if (flags & XDP_WAKEUP_RX) { + /* If IGB_FLAG_QUEUE_PAIRS is active, the q_vector + * and NAPI is shared between RX and TX. + * If NAPI is already running it would be marked as missed + * from the TX path, making this RX call a NOP + */ + ring = adapter->rx_ring[qid]; + eics |= igb_sw_irq_prep(ring->q_vector); + } + + if (eics) { /* Cause software interrupt */ - if (adapter->flags & IGB_FLAG_HAS_MSIX) { - eics |= ring->q_vector->eims_value; + if (adapter->flags & IGB_FLAG_HAS_MSIX) wr32(E1000_EICS, eics); - } else { + else wr32(E1000_ICS, E1000_ICS_RXDMT0); - } } return 0; diff --git a/drivers/net/ethernet/intel/igc/igc.h b/drivers/net/ethernet/intel/igc/igc.h index a427f05814c1ae..17236813965d33 100644 --- a/drivers/net/ethernet/intel/igc/igc.h +++ b/drivers/net/ethernet/intel/igc/igc.h @@ -781,6 +781,8 @@ int igc_ptp_hwtstamp_set(struct net_device *netdev, struct kernel_hwtstamp_config *config, struct netlink_ext_ack *extack); void igc_ptp_tx_hang(struct igc_adapter *adapter); +void igc_ptp_clear_xsk_tx_tstamp_queue(struct igc_adapter *adapter, + u16 queue_id); void igc_ptp_read(struct igc_adapter *adapter, struct timespec64 *ts); void igc_ptp_tx_tstamp_event(struct igc_adapter *adapter); diff --git a/drivers/net/ethernet/intel/igc/igc_main.c b/drivers/net/ethernet/intel/igc/igc_main.c index 89a321a344d263..b1ca2079e5cf36 100644 --- a/drivers/net/ethernet/intel/igc/igc_main.c +++ b/drivers/net/ethernet/intel/igc/igc_main.c @@ -264,6 +264,13 @@ static void igc_clean_tx_ring(struct igc_ring *tx_ring) /* reset next_to_use and next_to_clean */ tx_ring->next_to_use = 0; tx_ring->next_to_clean = 0; + + /* Clear any lingering XSK TX timestamp requests */ + if (test_bit(IGC_RING_FLAG_TX_HWTSTAMP, &tx_ring->flags)) { + struct igc_adapter *adapter = netdev_priv(tx_ring->netdev); + + igc_ptp_clear_xsk_tx_tstamp_queue(adapter, tx_ring->queue_index); + } } /** @@ -1730,11 +1737,8 @@ static netdev_tx_t igc_xmit_frame(struct sk_buff *skb, /* The minimum packet size with TCTL.PSP set is 17 so pad the skb * in order to meet this minimum size requirement. */ - if (skb->len < 17) { - if (skb_padto(skb, 17)) - return NETDEV_TX_OK; - skb->len = 17; - } + if (skb_put_padto(skb, 17)) + return NETDEV_TX_OK; return igc_xmit_frame_ring(skb, igc_tx_queue_mapping(adapter, skb)); } @@ -6908,28 +6912,29 @@ static int igc_xdp_xmit(struct net_device *dev, int num_frames, return nxmit; } -static void igc_trigger_rxtxq_interrupt(struct igc_adapter *adapter, - struct igc_q_vector *q_vector) +static u32 igc_sw_irq_prep(struct igc_q_vector *q_vector) { - struct igc_hw *hw = &adapter->hw; u32 eics = 0; - eics |= q_vector->eims_value; - wr32(IGC_EICS, eics); + if (!napi_if_scheduled_mark_missed(&q_vector->napi)) + eics = q_vector->eims_value; + + return eics; } int igc_xsk_wakeup(struct net_device *dev, u32 queue_id, u32 flags) { struct igc_adapter *adapter = netdev_priv(dev); - struct igc_q_vector *q_vector; + struct igc_hw *hw = &adapter->hw; struct igc_ring *ring; + u32 eics = 0; if (test_bit(__IGC_DOWN, &adapter->state)) return -ENETDOWN; if (!igc_xdp_is_enabled(adapter)) return -ENXIO; - + /* Check if queue_id is valid. Tx and Rx queue numbers are always same */ if (queue_id >= adapter->num_rx_queues) return -EINVAL; @@ -6938,9 +6943,22 @@ int igc_xsk_wakeup(struct net_device *dev, u32 queue_id, u32 flags) if (!ring->xsk_pool) return -ENXIO; - q_vector = adapter->q_vector[queue_id]; - if (!napi_if_scheduled_mark_missed(&q_vector->napi)) - igc_trigger_rxtxq_interrupt(adapter, q_vector); + if (flags & XDP_WAKEUP_RX) + eics |= igc_sw_irq_prep(ring->q_vector); + + if (flags & XDP_WAKEUP_TX) { + /* If IGC_FLAG_QUEUE_PAIRS is active, the q_vector + * and NAPI is shared between RX and TX. + * If NAPI is already running it would be marked as missed + * from the RX path, making this TX call a NOP + */ + ring = adapter->tx_ring[queue_id]; + eics |= igc_sw_irq_prep(ring->q_vector); + } + + if (eics) + /* Cause software interrupt */ + wr32(IGC_EICS, eics); return 0; } diff --git a/drivers/net/ethernet/intel/igc/igc_ptp.c b/drivers/net/ethernet/intel/igc/igc_ptp.c index 7aae83c108fd76..3d6b2264164af8 100644 --- a/drivers/net/ethernet/intel/igc/igc_ptp.c +++ b/drivers/net/ethernet/intel/igc/igc_ptp.c @@ -550,7 +550,8 @@ static void igc_ptp_free_tx_buffer(struct igc_adapter *adapter, tstamp->buffer_type = 0; /* Trigger txrx interrupt for transmit completion */ - igc_xsk_wakeup(adapter->netdev, tstamp->xsk_queue_index, 0); + igc_xsk_wakeup(adapter->netdev, tstamp->xsk_queue_index, + XDP_WAKEUP_TX); return; } @@ -576,6 +577,39 @@ static void igc_ptp_clear_tx_tstamp(struct igc_adapter *adapter) spin_unlock_irqrestore(&adapter->ptp_tx_lock, flags); } +/** + * igc_ptp_clear_xsk_tx_tstamp_queue - Clear pending XSK TX timestamps for a queue + * @adapter: Board private structure + * @queue_id: TX queue index to clear timestamps for + * + * Iterates over all TX timestamp registers and releases any pending + * timestamp requests associated with the given TX queue. This is + * called when an XDP pool is being disabled to ensure no stale + * timestamp references remain. + */ +void igc_ptp_clear_xsk_tx_tstamp_queue(struct igc_adapter *adapter, u16 queue_id) +{ + unsigned long flags; + int i; + + spin_lock_irqsave(&adapter->ptp_tx_lock, flags); + + for (i = 0; i < IGC_MAX_TX_TSTAMP_REGS; i++) { + struct igc_tx_timestamp_request *tstamp = &adapter->tx_tstamp[i]; + + if (tstamp->buffer_type != IGC_TX_BUFFER_TYPE_XSK) + continue; + if (tstamp->xsk_queue_index != queue_id) + continue; + if (!tstamp->xsk_tx_buffer) + continue; + + igc_ptp_free_tx_buffer(adapter, tstamp); + } + + spin_unlock_irqrestore(&adapter->ptp_tx_lock, flags); +} + static void igc_ptp_disable_tx_timestamp(struct igc_adapter *adapter) { struct igc_hw *hw = &adapter->hw; diff --git a/drivers/net/ethernet/intel/ixgbe/devlink/devlink.c b/drivers/net/ethernet/intel/ixgbe/devlink/devlink.c index d227f4d2a2d17a..f32e640ef4ac0a 100644 --- a/drivers/net/ethernet/intel/ixgbe/devlink/devlink.c +++ b/drivers/net/ethernet/intel/ixgbe/devlink/devlink.c @@ -474,7 +474,7 @@ static int ixgbe_devlink_reload_empr_finish(struct devlink *devlink, adapter->flags2 &= ~(IXGBE_FLAG2_API_MISMATCH | IXGBE_FLAG2_FW_ROLLBACK); - return 0; + return ixgbe_refresh_fw_version(adapter); } static const struct devlink_ops ixgbe_devlink_ops = { diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe.h b/drivers/net/ethernet/intel/ixgbe/ixgbe.h index dce4936708eb44..047f04045585a1 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe.h +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe.h @@ -973,7 +973,7 @@ int ixgbe_init_interrupt_scheme(struct ixgbe_adapter *adapter); bool ixgbe_wol_supported(struct ixgbe_adapter *adapter, u16 device_id, u16 subdevice_id); void ixgbe_set_fw_version_e610(struct ixgbe_adapter *adapter); -void ixgbe_refresh_fw_version(struct ixgbe_adapter *adapter); +int ixgbe_refresh_fw_version(struct ixgbe_adapter *adapter); #ifdef CONFIG_PCI_IOV void ixgbe_full_sync_mac_table(struct ixgbe_adapter *adapter); #endif diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c index 2ad81f687a844a..d82c51f673ec86 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c @@ -1153,12 +1153,17 @@ static int ixgbe_set_eeprom(struct net_device *netdev, return ret_val; } -void ixgbe_refresh_fw_version(struct ixgbe_adapter *adapter) +int ixgbe_refresh_fw_version(struct ixgbe_adapter *adapter) { struct ixgbe_hw *hw = &adapter->hw; + int err; + + err = ixgbe_get_flash_data(hw); + if (err) + return err; - ixgbe_get_flash_data(hw); ixgbe_set_fw_version_e610(adapter); + return 0; } static void ixgbe_get_drvinfo(struct net_device *netdev, @@ -1166,10 +1171,6 @@ static void ixgbe_get_drvinfo(struct net_device *netdev, { struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev); - /* need to refresh info for e610 in case fw reloads in runtime */ - if (adapter->hw.mac.type == ixgbe_mac_e610) - ixgbe_refresh_fw_version(adapter); - strscpy(drvinfo->driver, ixgbe_driver_name, sizeof(drvinfo->driver)); strscpy(drvinfo->fw_version, adapter->eeprom_id, diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c index c58051e4350be2..60eadef423ca78 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c @@ -6289,6 +6289,16 @@ void ixgbe_reinit_locked(struct ixgbe_adapter *adapter) if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) msleep(2000); ixgbe_up(adapter); + + /* E610 has no FW event to notify all PFs of an EMPR reset, so + * refresh the FW version here to pick up any new FW version after + * a hardware reset (e.g. EMPR triggered by another PF's devlink + * reload). ixgbe_refresh_fw_version() updates both hw->flash and + * adapter->eeprom_id so ethtool -i reports the correct string. + */ + if (adapter->hw.mac.type == ixgbe_mac_e610) + (void)ixgbe_refresh_fw_version(adapter); + clear_bit(__IXGBE_RESETTING, &adapter->state); } diff --git a/drivers/net/ethernet/intel/ixgbevf/vf.c b/drivers/net/ethernet/intel/ixgbevf/vf.c index 74d320879513c0..f6df86d124b9e8 100644 --- a/drivers/net/ethernet/intel/ixgbevf/vf.c +++ b/drivers/net/ethernet/intel/ixgbevf/vf.c @@ -709,6 +709,12 @@ static int ixgbevf_negotiate_features_vf(struct ixgbe_hw *hw, u32 *pf_features) return err; } +static int ixgbevf_hv_negotiate_features_vf(struct ixgbe_hw *hw, + u32 *pf_features) +{ + return -EOPNOTSUPP; +} + /** * ixgbevf_set_vfta_vf - Set/Unset VLAN filter table address * @hw: pointer to the HW structure @@ -852,7 +858,8 @@ static s32 ixgbevf_check_mac_link_vf(struct ixgbe_hw *hw, if (!mac->get_link_status) goto out; - if (hw->mac.type == ixgbe_mac_e610_vf) { + if (hw->mac.type == ixgbe_mac_e610_vf && + hw->api_version >= ixgbe_mbox_api_16) { ret_val = ixgbevf_get_pf_link_state(hw, speed, link_up); if (ret_val) goto out; @@ -1141,6 +1148,7 @@ static const struct ixgbe_mac_operations ixgbevf_hv_mac_ops = { .setup_link = ixgbevf_setup_mac_link_vf, .check_link = ixgbevf_hv_check_mac_link_vf, .negotiate_api_version = ixgbevf_hv_negotiate_api_version_vf, + .negotiate_features = ixgbevf_hv_negotiate_features_vf, .set_rar = ixgbevf_hv_set_rar_vf, .update_mc_addr_list = ixgbevf_hv_update_mc_addr_list_vf, .update_xcast_mode = ixgbevf_hv_update_xcast_mode, diff --git a/drivers/net/ethernet/intel/libie/fwlog.c b/drivers/net/ethernet/intel/libie/fwlog.c index f39cc11cb7c56b..3b32986c2978a8 100644 --- a/drivers/net/ethernet/intel/libie/fwlog.c +++ b/drivers/net/ethernet/intel/libie/fwlog.c @@ -433,17 +433,21 @@ libie_debugfs_module_write(struct file *filp, const char __user *buf, module = libie_find_module_by_dentry(fwlog->debugfs_modules, dentry); if (module < 0) { dev_info(dev, "unknown module\n"); - return -EINVAL; + count = -EINVAL; + goto free_cmd_buf; } cnt = sscanf(cmd_buf, "%s", user_val); - if (cnt != 1) - return -EINVAL; + if (cnt != 1) { + count = -EINVAL; + goto free_cmd_buf; + } log_level = sysfs_match_string(libie_fwlog_level_string, user_val); if (log_level < 0) { dev_info(dev, "unknown log level '%s'\n", user_val); - return -EINVAL; + count = -EINVAL; + goto free_cmd_buf; } if (module != LIBIE_AQC_FW_LOG_ID_MAX) { @@ -458,6 +462,9 @@ libie_debugfs_module_write(struct file *filp, const char __user *buf, fwlog->cfg.module_entries[i].log_level = log_level; } +free_cmd_buf: + kfree(cmd_buf); + return count; } @@ -515,23 +522,31 @@ libie_debugfs_nr_messages_write(struct file *filp, const char __user *buf, return PTR_ERR(cmd_buf); ret = sscanf(cmd_buf, "%s", user_val); - if (ret != 1) - return -EINVAL; + if (ret != 1) { + count = -EINVAL; + goto free_cmd_buf; + } ret = kstrtos16(user_val, 0, &nr_messages); - if (ret) - return ret; + if (ret) { + count = ret; + goto free_cmd_buf; + } if (nr_messages < LIBIE_AQC_FW_LOG_MIN_RESOLUTION || nr_messages > LIBIE_AQC_FW_LOG_MAX_RESOLUTION) { dev_err(dev, "Invalid FW log number of messages %d, value must be between %d - %d\n", nr_messages, LIBIE_AQC_FW_LOG_MIN_RESOLUTION, LIBIE_AQC_FW_LOG_MAX_RESOLUTION); - return -EINVAL; + count = -EINVAL; + goto free_cmd_buf; } fwlog->cfg.log_resolution = nr_messages; +free_cmd_buf: + kfree(cmd_buf); + return count; } @@ -588,8 +603,10 @@ libie_debugfs_enable_write(struct file *filp, const char __user *buf, return PTR_ERR(cmd_buf); ret = sscanf(cmd_buf, "%s", user_val); - if (ret != 1) - return -EINVAL; + if (ret != 1) { + ret = -EINVAL; + goto free_cmd_buf; + } ret = kstrtobool(user_val, &enable); if (ret) @@ -624,6 +641,8 @@ libie_debugfs_enable_write(struct file *filp, const char __user *buf, */ if (WARN_ON(ret != (ssize_t)count && ret >= 0)) ret = -EIO; +free_cmd_buf: + kfree(cmd_buf); return ret; } @@ -682,8 +701,10 @@ libie_debugfs_log_size_write(struct file *filp, const char __user *buf, return PTR_ERR(cmd_buf); ret = sscanf(cmd_buf, "%s", user_val); - if (ret != 1) - return -EINVAL; + if (ret != 1) { + ret = -EINVAL; + goto free_cmd_buf; + } index = sysfs_match_string(libie_fwlog_log_size, user_val); if (index < 0) { @@ -712,6 +733,8 @@ libie_debugfs_log_size_write(struct file *filp, const char __user *buf, */ if (WARN_ON(ret != (ssize_t)count && ret >= 0)) ret = -EIO; +free_cmd_buf: + kfree(cmd_buf); return ret; } @@ -1051,6 +1074,10 @@ void libie_fwlog_deinit(struct libie_fwlog *fwlog) { int status; + /* if FW logging isn't supported it means no configuration was done */ + if (!libie_fwlog_supported(fwlog)) + return; + /* make sure FW logging is disabled to not put the FW in a weird state * for the next driver load */ diff --git a/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c b/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c index 33426fded919ae..789e14bb1377a3 100644 --- a/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c +++ b/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c @@ -5018,7 +5018,7 @@ static int mvpp2_bm_switch_buffers(struct mvpp2 *priv, bool percpu) if (priv->percpu_pools) numbufs = port->nrxqs * 2; - if (change_percpu) + if (change_percpu && priv->global_tx_fc) mvpp2_bm_pool_update_priv_fc(priv, false); for (i = 0; i < numbufs; i++) @@ -5043,7 +5043,7 @@ static int mvpp2_bm_switch_buffers(struct mvpp2 *priv, bool percpu) mvpp2_open(port->dev); } - if (change_percpu) + if (change_percpu && priv->global_tx_fc) mvpp2_bm_pool_update_priv_fc(priv, true); return 0; diff --git a/drivers/net/ethernet/marvell/octeon_ep/octep_cn9k_pf.c b/drivers/net/ethernet/marvell/octeon_ep/octep_cn9k_pf.c index b5805969404fa1..01e82d0b6b2cdd 100644 --- a/drivers/net/ethernet/marvell/octeon_ep/octep_cn9k_pf.c +++ b/drivers/net/ethernet/marvell/octeon_ep/octep_cn9k_pf.c @@ -307,7 +307,7 @@ static void octep_setup_iq_regs_cn93_pf(struct octep_device *oct, int iq_no) } /* Setup registers for a hardware Rx Queue */ -static void octep_setup_oq_regs_cn93_pf(struct octep_device *oct, int oq_no) +static int octep_setup_oq_regs_cn93_pf(struct octep_device *oct, int oq_no) { u64 reg_val; u64 oq_ctl = 0ULL; @@ -355,6 +355,7 @@ static void octep_setup_oq_regs_cn93_pf(struct octep_device *oct, int oq_no) reg_val = ((u64)time_threshold << 32) | CFG_GET_OQ_INTR_PKT(oct->conf); octep_write_csr64(oct, CN93_SDP_R_OUT_INT_LEVELS(oq_no), reg_val); + return 0; } /* Setup registers for a PF mailbox */ @@ -696,14 +697,26 @@ static void octep_enable_interrupts_cn93_pf(struct octep_device *oct) /* Disable all interrupts */ static void octep_disable_interrupts_cn93_pf(struct octep_device *oct) { - u64 intr_mask = 0ULL; + u64 reg_val, intr_mask = 0ULL; int srn, num_rings, i; srn = CFG_GET_PORTS_PF_SRN(oct->conf); num_rings = CFG_GET_PORTS_ACTIVE_IO_RINGS(oct->conf); - for (i = 0; i < num_rings; i++) - intr_mask |= (0x1ULL << (srn + i)); + for (i = 0; i < num_rings; i++) { + intr_mask |= BIT_ULL(srn + i); + reg_val = octep_read_csr64(oct, + CN93_SDP_R_IN_INT_LEVELS(srn + i)); + reg_val &= ~CN93_INT_ENA_BIT; + octep_write_csr64(oct, + CN93_SDP_R_IN_INT_LEVELS(srn + i), reg_val); + + reg_val = octep_read_csr64(oct, + CN93_SDP_R_OUT_INT_LEVELS(srn + i)); + reg_val &= ~CN93_INT_ENA_BIT; + octep_write_csr64(oct, + CN93_SDP_R_OUT_INT_LEVELS(srn + i), reg_val); + } octep_write_csr64(oct, CN93_SDP_EPF_IRERR_RINT_ENA_W1C, intr_mask); octep_write_csr64(oct, CN93_SDP_EPF_ORERR_RINT_ENA_W1C, intr_mask); diff --git a/drivers/net/ethernet/marvell/octeon_ep/octep_cnxk_pf.c b/drivers/net/ethernet/marvell/octeon_ep/octep_cnxk_pf.c index 5de0b5ecbc5fd1..09a3f1d0645b85 100644 --- a/drivers/net/ethernet/marvell/octeon_ep/octep_cnxk_pf.c +++ b/drivers/net/ethernet/marvell/octeon_ep/octep_cnxk_pf.c @@ -8,6 +8,7 @@ #include #include #include +#include #include "octep_config.h" #include "octep_main.h" @@ -327,12 +328,14 @@ static void octep_setup_iq_regs_cnxk_pf(struct octep_device *oct, int iq_no) } /* Setup registers for a hardware Rx Queue */ -static void octep_setup_oq_regs_cnxk_pf(struct octep_device *oct, int oq_no) +static int octep_setup_oq_regs_cnxk_pf(struct octep_device *oct, int oq_no) { - u64 reg_val; - u64 oq_ctl = 0ULL; - u32 time_threshold = 0; struct octep_oq *oq = oct->oq[oq_no]; + unsigned long t_out_jiffies; + u32 time_threshold = 0; + u64 oq_ctl = 0ULL; + u64 reg_ba_val; + u64 reg_val; oq_no += CFG_GET_PORTS_PF_SRN(oct->conf); reg_val = octep_read_csr64(oct, CNXK_SDP_R_OUT_CONTROL(oq_no)); @@ -343,6 +346,36 @@ static void octep_setup_oq_regs_cnxk_pf(struct octep_device *oct, int oq_no) reg_val = octep_read_csr64(oct, CNXK_SDP_R_OUT_CONTROL(oq_no)); } while (!(reg_val & CNXK_R_OUT_CTL_IDLE)); } + octep_write_csr64(oct, CNXK_SDP_R_OUT_WMARK(oq_no), oq->max_count); + /* Wait for WMARK to get applied */ + usleep_range(10, 15); + + octep_write_csr64(oct, CNXK_SDP_R_OUT_SLIST_BADDR(oq_no), + oq->desc_ring_dma); + octep_write_csr64(oct, CNXK_SDP_R_OUT_SLIST_RSIZE(oq_no), + oq->max_count); + reg_ba_val = octep_read_csr64(oct, CNXK_SDP_R_OUT_SLIST_BADDR(oq_no)); + + if (reg_ba_val != oq->desc_ring_dma) { + t_out_jiffies = jiffies + 10 * HZ; + do { + if (reg_ba_val == ULLONG_MAX) + return -EFAULT; + octep_write_csr64(oct, + CNXK_SDP_R_OUT_SLIST_BADDR(oq_no), + oq->desc_ring_dma); + octep_write_csr64(oct, + CNXK_SDP_R_OUT_SLIST_RSIZE(oq_no), + oq->max_count); + reg_ba_val = + octep_read_csr64(oct, + CNXK_SDP_R_OUT_SLIST_BADDR(oq_no)); + } while ((reg_ba_val != oq->desc_ring_dma) && + time_before(jiffies, t_out_jiffies)); + + if (reg_ba_val != oq->desc_ring_dma) + return -EAGAIN; + } reg_val &= ~(CNXK_R_OUT_CTL_IMODE); reg_val &= ~(CNXK_R_OUT_CTL_ROR_P); @@ -356,10 +389,6 @@ static void octep_setup_oq_regs_cnxk_pf(struct octep_device *oct, int oq_no) reg_val |= (CNXK_R_OUT_CTL_ES_P); octep_write_csr64(oct, CNXK_SDP_R_OUT_CONTROL(oq_no), reg_val); - octep_write_csr64(oct, CNXK_SDP_R_OUT_SLIST_BADDR(oq_no), - oq->desc_ring_dma); - octep_write_csr64(oct, CNXK_SDP_R_OUT_SLIST_RSIZE(oq_no), - oq->max_count); oq_ctl = octep_read_csr64(oct, CNXK_SDP_R_OUT_CONTROL(oq_no)); @@ -385,6 +414,7 @@ static void octep_setup_oq_regs_cnxk_pf(struct octep_device *oct, int oq_no) reg_val &= ~0xFFFFFFFFULL; reg_val |= CFG_GET_OQ_WMARK(oct->conf); octep_write_csr64(oct, CNXK_SDP_R_OUT_WMARK(oq_no), reg_val); + return 0; } /* Setup registers for a PF mailbox */ @@ -720,14 +750,26 @@ static void octep_enable_interrupts_cnxk_pf(struct octep_device *oct) /* Disable all interrupts */ static void octep_disable_interrupts_cnxk_pf(struct octep_device *oct) { - u64 intr_mask = 0ULL; + u64 reg_val, intr_mask = 0ULL; int srn, num_rings, i; srn = CFG_GET_PORTS_PF_SRN(oct->conf); num_rings = CFG_GET_PORTS_ACTIVE_IO_RINGS(oct->conf); - for (i = 0; i < num_rings; i++) - intr_mask |= (0x1ULL << (srn + i)); + for (i = 0; i < num_rings; i++) { + intr_mask |= BIT_ULL(srn + i); + reg_val = octep_read_csr64(oct, + CNXK_SDP_R_IN_INT_LEVELS(srn + i)); + reg_val &= ~CNXK_INT_ENA_BIT; + octep_write_csr64(oct, + CNXK_SDP_R_IN_INT_LEVELS(srn + i), reg_val); + + reg_val = octep_read_csr64(oct, + CNXK_SDP_R_OUT_INT_LEVELS(srn + i)); + reg_val &= ~CNXK_INT_ENA_BIT; + octep_write_csr64(oct, + CNXK_SDP_R_OUT_INT_LEVELS(srn + i), reg_val); + } octep_write_csr64(oct, CNXK_SDP_EPF_IRERR_RINT_ENA_W1C, intr_mask); octep_write_csr64(oct, CNXK_SDP_EPF_ORERR_RINT_ENA_W1C, intr_mask); diff --git a/drivers/net/ethernet/marvell/octeon_ep/octep_main.c b/drivers/net/ethernet/marvell/octeon_ep/octep_main.c index 57db7ea2f5be9c..16f52d4b11e91e 100644 --- a/drivers/net/ethernet/marvell/octeon_ep/octep_main.c +++ b/drivers/net/ethernet/marvell/octeon_ep/octep_main.c @@ -555,28 +555,43 @@ static void octep_clean_irqs(struct octep_device *oct) } /** - * octep_enable_ioq_irq() - Enable MSI-x interrupt of a Tx/Rx queue. + * octep_update_pkt() - Update IQ/OQ IN/OUT_CNT registers. * * @iq: Octeon Tx queue data structure. * @oq: Octeon Rx queue data structure. */ -static void octep_enable_ioq_irq(struct octep_iq *iq, struct octep_oq *oq) +static void octep_update_pkt(struct octep_iq *iq, struct octep_oq *oq) { - u32 pkts_pend = oq->pkts_pending; + u32 pkts_pend = READ_ONCE(oq->pkts_pending); + u32 last_pkt_count = READ_ONCE(oq->last_pkt_count); + u32 pkts_processed = READ_ONCE(iq->pkts_processed); + u32 pkt_in_done = READ_ONCE(iq->pkt_in_done); netdev_dbg(iq->netdev, "enabling intr for Q-%u\n", iq->q_no); - if (iq->pkts_processed) { - writel(iq->pkts_processed, iq->inst_cnt_reg); - iq->pkt_in_done -= iq->pkts_processed; - iq->pkts_processed = 0; + if (pkts_processed) { + writel(pkts_processed, iq->inst_cnt_reg); + readl(iq->inst_cnt_reg); + WRITE_ONCE(iq->pkt_in_done, (pkt_in_done - pkts_processed)); + WRITE_ONCE(iq->pkts_processed, 0); } - if (oq->last_pkt_count - pkts_pend) { - writel(oq->last_pkt_count - pkts_pend, oq->pkts_sent_reg); - oq->last_pkt_count = pkts_pend; + if (last_pkt_count - pkts_pend) { + writel(last_pkt_count - pkts_pend, oq->pkts_sent_reg); + readl(oq->pkts_sent_reg); + WRITE_ONCE(oq->last_pkt_count, pkts_pend); } /* Flush the previous wrties before writing to RESEND bit */ - wmb(); + smp_wmb(); +} + +/** + * octep_enable_ioq_irq() - Enable MSI-x interrupt of a Tx/Rx queue. + * + * @iq: Octeon Tx queue data structure. + * @oq: Octeon Rx queue data structure. + */ +static void octep_enable_ioq_irq(struct octep_iq *iq, struct octep_oq *oq) +{ writeq(1UL << OCTEP_OQ_INTR_RESEND_BIT, oq->pkts_sent_reg); writeq(1UL << OCTEP_IQ_INTR_RESEND_BIT, iq->inst_cnt_reg); } @@ -602,7 +617,8 @@ static int octep_napi_poll(struct napi_struct *napi, int budget) if (tx_pending || rx_done >= budget) return budget; - napi_complete(napi); + octep_update_pkt(ioq_vector->iq, ioq_vector->oq); + napi_complete_done(napi, rx_done); octep_enable_ioq_irq(ioq_vector->iq, ioq_vector->oq); return rx_done; } diff --git a/drivers/net/ethernet/marvell/octeon_ep/octep_main.h b/drivers/net/ethernet/marvell/octeon_ep/octep_main.h index 81ac4267811c81..35d0ff289a70df 100644 --- a/drivers/net/ethernet/marvell/octeon_ep/octep_main.h +++ b/drivers/net/ethernet/marvell/octeon_ep/octep_main.h @@ -77,7 +77,7 @@ struct octep_pci_win_regs { struct octep_hw_ops { void (*setup_iq_regs)(struct octep_device *oct, int q); - void (*setup_oq_regs)(struct octep_device *oct, int q); + int (*setup_oq_regs)(struct octep_device *oct, int q); void (*setup_mbox_regs)(struct octep_device *oct, int mbox); irqreturn_t (*mbox_intr_handler)(void *ioq_vector); diff --git a/drivers/net/ethernet/marvell/octeon_ep/octep_regs_cn9k_pf.h b/drivers/net/ethernet/marvell/octeon_ep/octep_regs_cn9k_pf.h index ca473502d7a02a..95f1dfff90cce4 100644 --- a/drivers/net/ethernet/marvell/octeon_ep/octep_regs_cn9k_pf.h +++ b/drivers/net/ethernet/marvell/octeon_ep/octep_regs_cn9k_pf.h @@ -386,5 +386,6 @@ #define CN93_PEM_BAR4_INDEX 7 #define CN93_PEM_BAR4_INDEX_SIZE 0x400000ULL #define CN93_PEM_BAR4_INDEX_OFFSET (CN93_PEM_BAR4_INDEX * CN93_PEM_BAR4_INDEX_SIZE) +#define CN93_INT_ENA_BIT BIT_ULL(62) #endif /* _OCTEP_REGS_CN9K_PF_H_ */ diff --git a/drivers/net/ethernet/marvell/octeon_ep/octep_regs_cnxk_pf.h b/drivers/net/ethernet/marvell/octeon_ep/octep_regs_cnxk_pf.h index e637d7c8224d49..4d172a552f80c5 100644 --- a/drivers/net/ethernet/marvell/octeon_ep/octep_regs_cnxk_pf.h +++ b/drivers/net/ethernet/marvell/octeon_ep/octep_regs_cnxk_pf.h @@ -412,5 +412,6 @@ #define CNXK_PEM_BAR4_INDEX 7 #define CNXK_PEM_BAR4_INDEX_SIZE 0x400000ULL #define CNXK_PEM_BAR4_INDEX_OFFSET (CNXK_PEM_BAR4_INDEX * CNXK_PEM_BAR4_INDEX_SIZE) +#define CNXK_INT_ENA_BIT BIT_ULL(62) #endif /* _OCTEP_REGS_CNXK_PF_H_ */ diff --git a/drivers/net/ethernet/marvell/octeon_ep/octep_rx.c b/drivers/net/ethernet/marvell/octeon_ep/octep_rx.c index 82b6b19e76b47a..74de19166488f0 100644 --- a/drivers/net/ethernet/marvell/octeon_ep/octep_rx.c +++ b/drivers/net/ethernet/marvell/octeon_ep/octep_rx.c @@ -12,6 +12,8 @@ #include "octep_config.h" #include "octep_main.h" +static void octep_oq_free_ring_buffers(struct octep_oq *oq); + static void octep_oq_reset_indices(struct octep_oq *oq) { oq->host_read_idx = 0; @@ -170,11 +172,15 @@ static int octep_setup_oq(struct octep_device *oct, int q_no) goto oq_fill_buff_err; octep_oq_reset_indices(oq); - oct->hw_ops.setup_oq_regs(oct, q_no); + if (oct->hw_ops.setup_oq_regs(oct, q_no)) + goto oq_setup_err; + oct->num_oqs++; return 0; +oq_setup_err: + octep_oq_free_ring_buffers(oq); oq_fill_buff_err: vfree(oq->buff_info); oq->buff_info = NULL; @@ -318,10 +324,16 @@ static int octep_oq_check_hw_for_pkts(struct octep_device *oct, struct octep_oq *oq) { u32 pkt_count, new_pkts; + u32 last_pkt_count, pkts_pending; pkt_count = readl(oq->pkts_sent_reg); - new_pkts = pkt_count - oq->last_pkt_count; + last_pkt_count = READ_ONCE(oq->last_pkt_count); + new_pkts = pkt_count - last_pkt_count; + if (pkt_count < last_pkt_count) { + dev_err(oq->dev, "OQ-%u pkt_count(%u) < oq->last_pkt_count(%u)\n", + oq->q_no, pkt_count, last_pkt_count); + } /* Clear the hardware packets counter register if the rx queue is * being processed continuously with-in a single interrupt and * reached half its max value. @@ -332,8 +344,9 @@ static int octep_oq_check_hw_for_pkts(struct octep_device *oct, pkt_count = readl(oq->pkts_sent_reg); new_pkts += pkt_count; } - oq->last_pkt_count = pkt_count; - oq->pkts_pending += new_pkts; + WRITE_ONCE(oq->last_pkt_count, pkt_count); + pkts_pending = READ_ONCE(oq->pkts_pending); + WRITE_ONCE(oq->pkts_pending, (pkts_pending + new_pkts)); return new_pkts; } @@ -408,7 +421,7 @@ static int __octep_oq_process_rx(struct octep_device *oct, u16 rx_ol_flags; u32 read_idx; - read_idx = oq->host_read_idx; + read_idx = READ_ONCE(oq->host_read_idx); rx_bytes = 0; desc_used = 0; for (pkt = 0; pkt < pkts_to_process; pkt++) { @@ -493,7 +506,7 @@ static int __octep_oq_process_rx(struct octep_device *oct, napi_gro_receive(oq->napi, skb); } - oq->host_read_idx = read_idx; + WRITE_ONCE(oq->host_read_idx, read_idx); oq->refill_count += desc_used; oq->stats->packets += pkt; oq->stats->bytes += rx_bytes; @@ -516,22 +529,26 @@ int octep_oq_process_rx(struct octep_oq *oq, int budget) { u32 pkts_available, pkts_processed, total_pkts_processed; struct octep_device *oct = oq->octep_dev; + u32 pkts_pending; pkts_available = 0; pkts_processed = 0; total_pkts_processed = 0; while (total_pkts_processed < budget) { /* update pending count only when current one exhausted */ - if (oq->pkts_pending == 0) + pkts_pending = READ_ONCE(oq->pkts_pending); + if (pkts_pending == 0) octep_oq_check_hw_for_pkts(oct, oq); + pkts_pending = READ_ONCE(oq->pkts_pending); pkts_available = min(budget - total_pkts_processed, - oq->pkts_pending); + pkts_pending); if (!pkts_available) break; pkts_processed = __octep_oq_process_rx(oct, oq, pkts_available); - oq->pkts_pending -= pkts_processed; + pkts_pending = READ_ONCE(oq->pkts_pending); + WRITE_ONCE(oq->pkts_pending, (pkts_pending - pkts_processed)); total_pkts_processed += pkts_processed; } diff --git a/drivers/net/ethernet/marvell/octeon_ep_vf/octep_vf_cn9k.c b/drivers/net/ethernet/marvell/octeon_ep_vf/octep_vf_cn9k.c index 88937fce75f140..4c769b27c27892 100644 --- a/drivers/net/ethernet/marvell/octeon_ep_vf/octep_vf_cn9k.c +++ b/drivers/net/ethernet/marvell/octeon_ep_vf/octep_vf_cn9k.c @@ -196,7 +196,7 @@ static void octep_vf_setup_iq_regs_cn93(struct octep_vf_device *oct, int iq_no) } /* Setup registers for a hardware Rx Queue */ -static void octep_vf_setup_oq_regs_cn93(struct octep_vf_device *oct, int oq_no) +static int octep_vf_setup_oq_regs_cn93(struct octep_vf_device *oct, int oq_no) { struct octep_vf_oq *oq = oct->oq[oq_no]; u32 time_threshold = 0; @@ -239,6 +239,7 @@ static void octep_vf_setup_oq_regs_cn93(struct octep_vf_device *oct, int oq_no) time_threshold = CFG_GET_OQ_INTR_TIME(oct->conf); reg_val = ((u64)time_threshold << 32) | CFG_GET_OQ_INTR_PKT(oct->conf); octep_vf_write_csr64(oct, CN93_VF_SDP_R_OUT_INT_LEVELS(oq_no), reg_val); + return 0; } /* Setup registers for a VF mailbox */ diff --git a/drivers/net/ethernet/marvell/octeon_ep_vf/octep_vf_cnxk.c b/drivers/net/ethernet/marvell/octeon_ep_vf/octep_vf_cnxk.c index 1f79dfad42c626..a968b93a67943b 100644 --- a/drivers/net/ethernet/marvell/octeon_ep_vf/octep_vf_cnxk.c +++ b/drivers/net/ethernet/marvell/octeon_ep_vf/octep_vf_cnxk.c @@ -199,11 +199,13 @@ static void octep_vf_setup_iq_regs_cnxk(struct octep_vf_device *oct, int iq_no) } /* Setup registers for a hardware Rx Queue */ -static void octep_vf_setup_oq_regs_cnxk(struct octep_vf_device *oct, int oq_no) +static int octep_vf_setup_oq_regs_cnxk(struct octep_vf_device *oct, int oq_no) { struct octep_vf_oq *oq = oct->oq[oq_no]; + unsigned long t_out_jiffies; u32 time_threshold = 0; u64 oq_ctl = ULL(0); + u64 reg_ba_val; u64 reg_val; reg_val = octep_vf_read_csr64(oct, CNXK_VF_SDP_R_OUT_CONTROL(oq_no)); @@ -214,6 +216,38 @@ static void octep_vf_setup_oq_regs_cnxk(struct octep_vf_device *oct, int oq_no) reg_val = octep_vf_read_csr64(oct, CNXK_VF_SDP_R_OUT_CONTROL(oq_no)); } while (!(reg_val & CNXK_VF_R_OUT_CTL_IDLE)); } + octep_vf_write_csr64(oct, CNXK_VF_SDP_R_OUT_WMARK(oq_no), + oq->max_count); + /* Wait for WMARK to get applied */ + usleep_range(10, 15); + + octep_vf_write_csr64(oct, CNXK_VF_SDP_R_OUT_SLIST_BADDR(oq_no), + oq->desc_ring_dma); + octep_vf_write_csr64(oct, CNXK_VF_SDP_R_OUT_SLIST_RSIZE(oq_no), + oq->max_count); + reg_ba_val = octep_vf_read_csr64(oct, + CNXK_VF_SDP_R_OUT_SLIST_BADDR(oq_no)); + if (reg_ba_val != oq->desc_ring_dma) { + t_out_jiffies = jiffies + 10 * HZ; + do { + if (reg_ba_val == ULLONG_MAX) + return -EFAULT; + octep_vf_write_csr64(oct, + CNXK_VF_SDP_R_OUT_SLIST_BADDR + (oq_no), oq->desc_ring_dma); + octep_vf_write_csr64(oct, + CNXK_VF_SDP_R_OUT_SLIST_RSIZE + (oq_no), oq->max_count); + reg_ba_val = + octep_vf_read_csr64(oct, + CNXK_VF_SDP_R_OUT_SLIST_BADDR + (oq_no)); + } while ((reg_ba_val != oq->desc_ring_dma) && + time_before(jiffies, t_out_jiffies)); + + if (reg_ba_val != oq->desc_ring_dma) + return -EAGAIN; + } reg_val &= ~(CNXK_VF_R_OUT_CTL_IMODE); reg_val &= ~(CNXK_VF_R_OUT_CTL_ROR_P); @@ -227,8 +261,6 @@ static void octep_vf_setup_oq_regs_cnxk(struct octep_vf_device *oct, int oq_no) reg_val |= (CNXK_VF_R_OUT_CTL_ES_P); octep_vf_write_csr64(oct, CNXK_VF_SDP_R_OUT_CONTROL(oq_no), reg_val); - octep_vf_write_csr64(oct, CNXK_VF_SDP_R_OUT_SLIST_BADDR(oq_no), oq->desc_ring_dma); - octep_vf_write_csr64(oct, CNXK_VF_SDP_R_OUT_SLIST_RSIZE(oq_no), oq->max_count); oq_ctl = octep_vf_read_csr64(oct, CNXK_VF_SDP_R_OUT_CONTROL(oq_no)); /* Clear the ISIZE and BSIZE (22-0) */ @@ -250,6 +282,7 @@ static void octep_vf_setup_oq_regs_cnxk(struct octep_vf_device *oct, int oq_no) reg_val &= ~GENMASK_ULL(31, 0); reg_val |= CFG_GET_OQ_WMARK(oct->conf); octep_vf_write_csr64(oct, CNXK_VF_SDP_R_OUT_WMARK(oq_no), reg_val); + return 0; } /* Setup registers for a VF mailbox */ diff --git a/drivers/net/ethernet/marvell/octeon_ep_vf/octep_vf_main.c b/drivers/net/ethernet/marvell/octeon_ep_vf/octep_vf_main.c index 1d9760b4b8f471..a3c359124887ee 100644 --- a/drivers/net/ethernet/marvell/octeon_ep_vf/octep_vf_main.c +++ b/drivers/net/ethernet/marvell/octeon_ep_vf/octep_vf_main.c @@ -286,28 +286,45 @@ static void octep_vf_clean_irqs(struct octep_vf_device *oct) } /** - * octep_vf_enable_ioq_irq() - Enable MSI-x interrupt of a Tx/Rx queue. + * octep_vf_update_pkt() - Update IQ/OQ IN/OUT_CNT registers. * * @iq: Octeon Tx queue data structure. * @oq: Octeon Rx queue data structure. */ -static void octep_vf_enable_ioq_irq(struct octep_vf_iq *iq, struct octep_vf_oq *oq) + +static void octep_vf_update_pkt(struct octep_vf_iq *iq, struct octep_vf_oq *oq) { - u32 pkts_pend = oq->pkts_pending; + u32 pkts_pend = READ_ONCE(oq->pkts_pending); + u32 last_pkt_count = READ_ONCE(oq->last_pkt_count); + u32 pkts_processed = READ_ONCE(iq->pkts_processed); + u32 pkt_in_done = READ_ONCE(iq->pkt_in_done); netdev_dbg(iq->netdev, "enabling intr for Q-%u\n", iq->q_no); - if (iq->pkts_processed) { - writel(iq->pkts_processed, iq->inst_cnt_reg); - iq->pkt_in_done -= iq->pkts_processed; - iq->pkts_processed = 0; + if (pkts_processed) { + writel(pkts_processed, iq->inst_cnt_reg); + readl(iq->inst_cnt_reg); + WRITE_ONCE(iq->pkt_in_done, (pkt_in_done - pkts_processed)); + WRITE_ONCE(iq->pkts_processed, 0); } - if (oq->last_pkt_count - pkts_pend) { - writel(oq->last_pkt_count - pkts_pend, oq->pkts_sent_reg); - oq->last_pkt_count = pkts_pend; + if (last_pkt_count - pkts_pend) { + writel(last_pkt_count - pkts_pend, oq->pkts_sent_reg); + readl(oq->pkts_sent_reg); + WRITE_ONCE(oq->last_pkt_count, pkts_pend); } /* Flush the previous wrties before writing to RESEND bit */ smp_wmb(); +} + +/** + * octep_vf_enable_ioq_irq() - Enable MSI-x interrupt of a Tx/Rx queue. + * + * @iq: Octeon Tx queue data structure. + * @oq: Octeon Rx queue data structure. + */ +static void octep_vf_enable_ioq_irq(struct octep_vf_iq *iq, + struct octep_vf_oq *oq) +{ writeq(1UL << OCTEP_VF_OQ_INTR_RESEND_BIT, oq->pkts_sent_reg); writeq(1UL << OCTEP_VF_IQ_INTR_RESEND_BIT, iq->inst_cnt_reg); } @@ -333,6 +350,7 @@ static int octep_vf_napi_poll(struct napi_struct *napi, int budget) if (tx_pending || rx_done >= budget) return budget; + octep_vf_update_pkt(ioq_vector->iq, ioq_vector->oq); if (likely(napi_complete_done(napi, rx_done))) octep_vf_enable_ioq_irq(ioq_vector->iq, ioq_vector->oq); diff --git a/drivers/net/ethernet/marvell/octeon_ep_vf/octep_vf_main.h b/drivers/net/ethernet/marvell/octeon_ep_vf/octep_vf_main.h index b9f13506f46205..c74cd2369e90d4 100644 --- a/drivers/net/ethernet/marvell/octeon_ep_vf/octep_vf_main.h +++ b/drivers/net/ethernet/marvell/octeon_ep_vf/octep_vf_main.h @@ -55,7 +55,7 @@ struct octep_vf_mmio { struct octep_vf_hw_ops { void (*setup_iq_regs)(struct octep_vf_device *oct, int q); - void (*setup_oq_regs)(struct octep_vf_device *oct, int q); + int (*setup_oq_regs)(struct octep_vf_device *oct, int q); void (*setup_mbox_regs)(struct octep_vf_device *oct, int mbox); irqreturn_t (*non_ioq_intr_handler)(void *ioq_vector); diff --git a/drivers/net/ethernet/marvell/octeon_ep_vf/octep_vf_rx.c b/drivers/net/ethernet/marvell/octeon_ep_vf/octep_vf_rx.c index d70c8be3cfc40b..b579d5b545c46d 100644 --- a/drivers/net/ethernet/marvell/octeon_ep_vf/octep_vf_rx.c +++ b/drivers/net/ethernet/marvell/octeon_ep_vf/octep_vf_rx.c @@ -12,6 +12,8 @@ #include "octep_vf_config.h" #include "octep_vf_main.h" +static void octep_vf_oq_free_ring_buffers(struct octep_vf_oq *oq); + static void octep_vf_oq_reset_indices(struct octep_vf_oq *oq) { oq->host_read_idx = 0; @@ -171,11 +173,15 @@ static int octep_vf_setup_oq(struct octep_vf_device *oct, int q_no) goto oq_fill_buff_err; octep_vf_oq_reset_indices(oq); - oct->hw_ops.setup_oq_regs(oct, q_no); + if (oct->hw_ops.setup_oq_regs(oct, q_no)) + goto oq_setup_err; + oct->num_oqs++; return 0; +oq_setup_err: + octep_vf_oq_free_ring_buffers(oq); oq_fill_buff_err: vfree(oq->buff_info); oq->buff_info = NULL; @@ -319,9 +325,16 @@ static int octep_vf_oq_check_hw_for_pkts(struct octep_vf_device *oct, struct octep_vf_oq *oq) { u32 pkt_count, new_pkts; + u32 last_pkt_count, pkts_pending; pkt_count = readl(oq->pkts_sent_reg); - new_pkts = pkt_count - oq->last_pkt_count; + last_pkt_count = READ_ONCE(oq->last_pkt_count); + new_pkts = pkt_count - last_pkt_count; + + if (pkt_count < last_pkt_count) { + dev_err(oq->dev, "OQ-%u pkt_count(%u) < oq->last_pkt_count(%u)\n", + oq->q_no, pkt_count, last_pkt_count); + } /* Clear the hardware packets counter register if the rx queue is * being processed continuously with-in a single interrupt and @@ -333,8 +346,9 @@ static int octep_vf_oq_check_hw_for_pkts(struct octep_vf_device *oct, pkt_count = readl(oq->pkts_sent_reg); new_pkts += pkt_count; } - oq->last_pkt_count = pkt_count; - oq->pkts_pending += new_pkts; + WRITE_ONCE(oq->last_pkt_count, pkt_count); + pkts_pending = READ_ONCE(oq->pkts_pending); + WRITE_ONCE(oq->pkts_pending, (pkts_pending + new_pkts)); return new_pkts; } @@ -363,7 +377,7 @@ static int __octep_vf_oq_process_rx(struct octep_vf_device *oct, struct sk_buff *skb; u32 read_idx; - read_idx = oq->host_read_idx; + read_idx = READ_ONCE(oq->host_read_idx); rx_bytes = 0; desc_used = 0; for (pkt = 0; pkt < pkts_to_process; pkt++) { @@ -457,7 +471,7 @@ static int __octep_vf_oq_process_rx(struct octep_vf_device *oct, napi_gro_receive(oq->napi, skb); } - oq->host_read_idx = read_idx; + WRITE_ONCE(oq->host_read_idx, read_idx); oq->refill_count += desc_used; oq->stats->packets += pkt; oq->stats->bytes += rx_bytes; @@ -480,22 +494,26 @@ int octep_vf_oq_process_rx(struct octep_vf_oq *oq, int budget) { u32 pkts_available, pkts_processed, total_pkts_processed; struct octep_vf_device *oct = oq->octep_vf_dev; + u32 pkts_pending; pkts_available = 0; pkts_processed = 0; total_pkts_processed = 0; while (total_pkts_processed < budget) { /* update pending count only when current one exhausted */ - if (oq->pkts_pending == 0) + pkts_pending = READ_ONCE(oq->pkts_pending); + if (pkts_pending == 0) octep_vf_oq_check_hw_for_pkts(oct, oq); + pkts_pending = READ_ONCE(oq->pkts_pending); pkts_available = min(budget - total_pkts_processed, - oq->pkts_pending); + pkts_pending); if (!pkts_available) break; pkts_processed = __octep_vf_oq_process_rx(oct, oq, pkts_available); - oq->pkts_pending -= pkts_processed; + pkts_pending = READ_ONCE(oq->pkts_pending); + WRITE_ONCE(oq->pkts_pending, (pkts_pending - pkts_processed)); total_pkts_processed += pkts_processed; } diff --git a/drivers/net/ethernet/marvell/octeontx2/af/cgx.c b/drivers/net/ethernet/marvell/octeontx2/af/cgx.c index 42044cd810b1fa..fd4792e432bf09 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/cgx.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/cgx.c @@ -1823,6 +1823,8 @@ static int cgx_lmac_exit(struct cgx *cgx) cgx->mac_ops->mac_pause_frm_config(cgx, lmac->lmac_id, false); cgx_configure_interrupt(cgx, lmac, lmac->lmac_id, true); kfree(lmac->mac_to_index_bmap.bmap); + rvu_free_bitmap(&lmac->rx_fc_pfvf_bmap); + rvu_free_bitmap(&lmac->tx_fc_pfvf_bmap); kfree(lmac->name); kfree(lmac); } diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu.c index 747fbdf2a908f1..8530df8b3fdaf7 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu.c @@ -3632,11 +3632,22 @@ static void rvu_remove(struct pci_dev *pdev) devm_kfree(&pdev->dev, rvu); } +static void rvu_shutdown(struct pci_dev *pdev) +{ + struct rvu *rvu = pci_get_drvdata(pdev); + + if (!rvu) + return; + + rvu_clear_rvum_blk_revid(rvu); +} + static struct pci_driver rvu_driver = { .name = DRV_NAME, .id_table = rvu_id_table, .probe = rvu_probe, .remove = rvu_remove, + .shutdown = rvu_shutdown, }; static int __init rvu_init_module(void) diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_devlink.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu_devlink.c index 0f9953eaf1b09f..2a715872e9edf9 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_devlink.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_devlink.c @@ -327,10 +327,10 @@ static int rvu_nix_report_show(struct devlink_fmsg *fmsg, void *ctx, rvu_report_pair_end(fmsg); break; case NIX_AF_RVU_RAS: - intr_val = nix_event_context->nix_af_rvu_err; + intr_val = nix_event_context->nix_af_rvu_ras; rvu_report_pair_start(fmsg, "NIX_AF_RAS"); devlink_fmsg_u64_pair_put(fmsg, "\tNIX RAS Interrupt Reg ", - nix_event_context->nix_af_rvu_err); + nix_event_context->nix_af_rvu_ras); devlink_fmsg_string_put(fmsg, "\n\tPoison Data on:"); if (intr_val & BIT_ULL(34)) devlink_fmsg_string_put(fmsg, "\n\tNIX_AQ_INST_S"); @@ -475,7 +475,7 @@ static int rvu_hw_nix_ras_recover(struct devlink_health_reporter *reporter, if (blkaddr < 0) return blkaddr; - if (nix_event_ctx->nix_af_rvu_int) + if (nix_event_ctx->nix_af_rvu_ras) rvu_write64(rvu, blkaddr, NIX_AF_RAS_ENA_W1S, ~0ULL); return 0; diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_nix.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu_nix.c index 2f485a930edd17..49f7ff5eddfc8c 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_nix.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_nix.c @@ -4938,12 +4938,18 @@ static int rvu_nix_block_init(struct rvu *rvu, struct nix_hw *nix_hw) /* Set chan/link to backpressure TL3 instead of TL2 */ rvu_write64(rvu, blkaddr, NIX_AF_PSE_CHANNEL_LEVEL, 0x01); - /* Disable SQ manager's sticky mode operation (set TM6 = 0) + /* Disable SQ manager's sticky mode operation (set TM6 = 0, TM11 = 0) * This sticky mode is known to cause SQ stalls when multiple - * SQs are mapped to same SMQ and transmitting pkts at a time. + * SQs are mapped to same SMQ and transmitting pkts simultaneously. + * NIX PSE may deadlock when there are any sticky to non-sticky + * transmission. Hence disable it (TM5 = 0). */ cfg = rvu_read64(rvu, blkaddr, NIX_AF_SQM_DBG_CTL_STATUS); - cfg &= ~BIT_ULL(15); + cfg &= ~(BIT_ULL(15) | BIT_ULL(14) | BIT_ULL(23)); + /* NIX may drop credits when condition clocks are turned off. + * Hence enable control flow clk (set TM9 = 1). + */ + cfg |= BIT_ULL(21); rvu_write64(rvu, blkaddr, NIX_AF_SQM_DBG_CTL_STATUS, cfg); ltdefs = rvu->kpu.lt_def; diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc.c index c7c70429eb6c10..8658cb2143dfc7 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc.c @@ -1042,32 +1042,35 @@ void rvu_npc_update_flowkey_alg_idx(struct rvu *rvu, u16 pcifunc, int nixlf, rvu_write64(rvu, blkaddr, NPC_AF_MCAMEX_BANKX_ACTION(index, bank), *(u64 *)&action); - /* update the VF flow rule action with the VF default entry action */ - if (mcam_index < 0) - npc_update_vf_flow_entry(rvu, mcam, blkaddr, pcifunc, - *(u64 *)&action); - /* update the action change in default rule */ pfvf = rvu_get_pfvf(rvu, pcifunc); if (pfvf->def_ucast_rule) pfvf->def_ucast_rule->rx_action = action; - index = npc_get_nixlf_mcam_index(mcam, pcifunc, - nixlf, NIXLF_PROMISC_ENTRY); + if (mcam_index < 0) { + /* update the VF flow rule action with the VF default + * entry action + */ + npc_update_vf_flow_entry(rvu, mcam, blkaddr, pcifunc, + *(u64 *)&action); - /* If PF's promiscuous entry is enabled, - * Set RSS action for that entry as well - */ - npc_update_rx_action_with_alg_idx(rvu, action, pfvf, index, blkaddr, - alg_idx); + index = npc_get_nixlf_mcam_index(mcam, pcifunc, + nixlf, NIXLF_PROMISC_ENTRY); - index = npc_get_nixlf_mcam_index(mcam, pcifunc, - nixlf, NIXLF_ALLMULTI_ENTRY); - /* If PF's allmulti entry is enabled, - * Set RSS action for that entry as well - */ - npc_update_rx_action_with_alg_idx(rvu, action, pfvf, index, blkaddr, - alg_idx); + /* If PF's promiscuous entry is enabled, + * Set RSS action for that entry as well + */ + npc_update_rx_action_with_alg_idx(rvu, action, pfvf, index, + blkaddr, alg_idx); + + index = npc_get_nixlf_mcam_index(mcam, pcifunc, + nixlf, NIXLF_ALLMULTI_ENTRY); + /* If PF's allmulti entry is enabled, + * Set RSS action for that entry as well + */ + npc_update_rx_action_with_alg_idx(rvu, action, pfvf, index, + blkaddr, alg_idx); + } } void npc_enadis_default_mce_entry(struct rvu *rvu, u16 pcifunc, diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c index 6b2d8559f0eb1b..444bb67494ab70 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c @@ -3315,6 +3315,7 @@ static int otx2_probe(struct pci_dev *pdev, const struct pci_device_id *id) err_sriov_cleannup: otx2_sriov_vfcfg_cleanup(pf); err_pf_sriov_init: + otx2_unregister_dl(pf); otx2_shutdown_tc(pf); err_mcam_flow_del: otx2_mcam_flow_del(pf); diff --git a/drivers/net/ethernet/marvell/skge.c b/drivers/net/ethernet/marvell/skge.c index 05349a0b2db1c1..cf4e26d337bb55 100644 --- a/drivers/net/ethernet/marvell/skge.c +++ b/drivers/net/ethernet/marvell/skge.c @@ -78,7 +78,6 @@ static const struct pci_device_id skge_id_table[] = { { PCI_DEVICE(PCI_VENDOR_ID_SYSKONNECT, 0x4320) }, /* SK-98xx V2.0 */ { PCI_DEVICE(PCI_VENDOR_ID_DLINK, 0x4b01) }, /* D-Link DGE-530T (rev.B) */ { PCI_DEVICE(PCI_VENDOR_ID_DLINK, 0x4c00) }, /* D-Link DGE-530T */ - { PCI_DEVICE(PCI_VENDOR_ID_DLINK, 0x4302) }, /* D-Link DGE-530T Rev C1 */ { PCI_DEVICE(PCI_VENDOR_ID_MARVELL, 0x4320) }, /* Marvell Yukon 88E8001/8003/8010 */ { PCI_DEVICE(PCI_VENDOR_ID_MARVELL, 0x5005) }, /* Belkin */ { PCI_DEVICE(PCI_VENDOR_ID_CNET, 0x434E) }, /* CNet PowerG-2000 */ diff --git a/drivers/net/ethernet/mediatek/mtk_eth_soc.c b/drivers/net/ethernet/mediatek/mtk_eth_soc.c index e68997a29191b2..8d3e15bc867d2d 100644 --- a/drivers/net/ethernet/mediatek/mtk_eth_soc.c +++ b/drivers/net/ethernet/mediatek/mtk_eth_soc.c @@ -3749,12 +3749,21 @@ static int mtk_xdp_setup(struct net_device *dev, struct bpf_prog *prog, mtk_stop(dev); old_prog = rcu_replace_pointer(eth->prog, prog, lockdep_rtnl_is_held()); + + if (netif_running(dev) && need_update) { + int err; + + err = mtk_open(dev); + if (err) { + rcu_assign_pointer(eth->prog, old_prog); + + return err; + } + } + if (old_prog) bpf_prog_put(old_prog); - if (netif_running(dev) && need_update) - return mtk_open(dev); - return 0; } diff --git a/drivers/net/ethernet/mediatek/mtk_ppe_offload.c b/drivers/net/ethernet/mediatek/mtk_ppe_offload.c index e9bd3274198379..4894d4f187f701 100644 --- a/drivers/net/ethernet/mediatek/mtk_ppe_offload.c +++ b/drivers/net/ethernet/mediatek/mtk_ppe_offload.c @@ -244,6 +244,25 @@ mtk_flow_set_output_device(struct mtk_eth *eth, struct mtk_foe_entry *foe, return 0; } +static bool +mtk_flow_is_valid_idev(const struct mtk_eth *eth, const struct net_device *idev) +{ + size_t i; + + if (!idev) + return false; + + for (i = 0; i < ARRAY_SIZE(eth->netdev); i++) { + if (!eth->netdev[i]) + continue; + + if (idev->netdev_ops == eth->netdev[i]->netdev_ops) + return true; + } + + return false; +} + static int mtk_flow_offload_replace(struct mtk_eth *eth, struct flow_cls_offload *f, int ppe_index) @@ -270,7 +289,7 @@ mtk_flow_offload_replace(struct mtk_eth *eth, struct flow_cls_offload *f, flow_rule_match_meta(rule, &match); if (mtk_is_netsys_v2_or_greater(eth)) { idev = __dev_get_by_index(&init_net, match.key->ingress_ifindex); - if (idev && idev->netdev_ops == eth->netdev[0]->netdev_ops) { + if (mtk_flow_is_valid_idev(eth, idev)) { struct mtk_mac *mac = netdev_priv(idev); if (WARN_ON(mac->ppe_idx >= eth->soc->ppe_num)) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/devlink.c b/drivers/net/ethernet/mellanox/mlx5/core/devlink.c index ea77fbd98396a2..055ee020c56f4a 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/devlink.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/devlink.c @@ -107,9 +107,7 @@ mlx5_devlink_info_get(struct devlink *devlink, struct devlink_info_req *req, if (err) return err; - err = mlx5_fw_version_query(dev, &running_fw, &stored_fw); - if (err) - return err; + mlx5_fw_version_query(dev, &running_fw, &stored_fw); snprintf(version_str, sizeof(version_str), "%d.%d.%04d", mlx5_fw_ver_major(running_fw), mlx5_fw_ver_minor(running_fw), diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en.h b/drivers/net/ethernet/mellanox/mlx5/core/en.h index ff4ab4691baf0b..a06d08576fd4b8 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en.h @@ -179,7 +179,8 @@ static inline u16 mlx5_min_rx_wqes(int wq_type, u32 wq_size) } /* Use this function to get max num channels (rxqs/txqs) only to create netdev */ -static inline int mlx5e_get_max_num_channels(struct mlx5_core_dev *mdev) +static inline unsigned int +mlx5e_get_max_num_channels(struct mlx5_core_dev *mdev) { return is_kdump_kernel() ? MLX5E_MIN_NUM_CHANNELS : diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c b/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c index 424f8a2728a3ef..74660e7fe6748b 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c @@ -457,22 +457,8 @@ static void mlx5e_ptpsq_unhealthy_work(struct work_struct *work) { struct mlx5e_ptpsq *ptpsq = container_of(work, struct mlx5e_ptpsq, report_unhealthy_work); - struct mlx5e_txqsq *sq = &ptpsq->txqsq; - - /* Recovering the PTP SQ means re-enabling NAPI, which requires the - * netdev instance lock. However, SQ closing has to wait for this work - * task to finish while also holding the same lock. So either get the - * lock or find that the SQ is no longer enabled and thus this work is - * not relevant anymore. - */ - while (!netdev_trylock(sq->netdev)) { - if (!test_bit(MLX5E_SQ_STATE_ENABLED, &sq->state)) - return; - msleep(20); - } mlx5e_reporter_tx_ptpsq_unhealthy(ptpsq); - netdev_unlock(sq->netdev); } static int mlx5e_ptp_open_txqsq(struct mlx5e_ptp *c, u32 tisn, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_rx.c index 0686fbdd5a0599..6efb626b55062b 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_rx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_rx.c @@ -1,6 +1,8 @@ // SPDX-License-Identifier: GPL-2.0 // Copyright (c) 2019 Mellanox Technologies. +#include + #include "health.h" #include "params.h" #include "txrx.h" @@ -177,6 +179,16 @@ static int mlx5e_rx_reporter_timeout_recover(void *ctx) rq = ctx; priv = rq->priv; + /* Acquire netdev instance lock to synchronize with channel close and + * reopen flows. Either successfully obtain the lock, or detect that + * channels are closing for another reason, making this work no longer + * necessary. + */ + while (!netdev_trylock(rq->netdev)) { + if (!test_bit(MLX5E_STATE_CHANNELS_ACTIVE, &rq->priv->state)) + return 0; + msleep(20); + } mutex_lock(&priv->state_lock); eq = rq->cq.mcq.eq; @@ -186,6 +198,7 @@ static int mlx5e_rx_reporter_timeout_recover(void *ctx) clear_bit(MLX5E_SQ_STATE_ENABLED, &rq->icosq->state); mutex_unlock(&priv->state_lock); + netdev_unlock(rq->netdev); return err; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_tx.c b/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_tx.c index 9e2cf191ed3086..d6ace2b6fc1df4 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_tx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_tx.c @@ -1,6 +1,8 @@ /* SPDX-License-Identifier: GPL-2.0 */ /* Copyright (c) 2019 Mellanox Technologies. */ +#include + #include "health.h" #include "en/ptp.h" #include "en/devlink.h" @@ -44,7 +46,6 @@ static void mlx5e_reset_txqsq_cc_pc(struct mlx5e_txqsq *sq) "SQ 0x%x: cc (0x%x) != pc (0x%x)\n", sq->sqn, sq->cc, sq->pc); sq->cc = 0; - sq->dma_fifo_cc = 0; sq->pc = 0; } @@ -78,6 +79,18 @@ static int mlx5e_tx_reporter_err_cqe_recover(void *ctx) if (!test_bit(MLX5E_SQ_STATE_RECOVERING, &sq->state)) return 0; + /* Recovering queues means re-enabling NAPI, which requires the netdev + * instance lock. However, SQ closing flows have to wait for work tasks + * to finish while also holding the netdev instance lock. So either get + * the lock or find that the SQ is no longer enabled and thus this work + * is not relevant anymore. + */ + while (!netdev_trylock(dev)) { + if (!test_bit(MLX5E_SQ_STATE_ENABLED, &sq->state)) + return 0; + msleep(20); + } + err = mlx5_core_query_sq_state(mdev, sq->sqn, &state); if (err) { netdev_err(dev, "Failed to query SQ 0x%x state. err = %d\n", @@ -113,9 +126,11 @@ static int mlx5e_tx_reporter_err_cqe_recover(void *ctx) else mlx5e_trigger_napi_sched(sq->cq.napi); + netdev_unlock(dev); return 0; out: clear_bit(MLX5E_SQ_STATE_RECOVERING, &sq->state); + netdev_unlock(dev); return err; } @@ -136,10 +151,24 @@ static int mlx5e_tx_reporter_timeout_recover(void *ctx) sq = to_ctx->sq; eq = sq->cq.mcq.eq; priv = sq->priv; + + /* Recovering the TX queues implies re-enabling NAPI, which requires + * the netdev instance lock. + * However, channel closing flows have to wait for this work to finish + * while holding the same lock. So either get the lock or find that + * channels are being closed for other reason and this work is not + * relevant anymore. + */ + while (!netdev_trylock(sq->netdev)) { + if (!test_bit(MLX5E_STATE_CHANNELS_ACTIVE, &priv->state)) + return 0; + msleep(20); + } + err = mlx5e_health_channel_eq_recover(sq->netdev, eq, sq->cq.ch_stats); if (!err) { to_ctx->status = 0; /* this sq recovered */ - return err; + goto out; } mutex_lock(&priv->state_lock); @@ -147,7 +176,7 @@ static int mlx5e_tx_reporter_timeout_recover(void *ctx) mutex_unlock(&priv->state_lock); if (!err) { to_ctx->status = 1; /* all channels recovered */ - return err; + goto out; } to_ctx->status = err; @@ -155,7 +184,8 @@ static int mlx5e_tx_reporter_timeout_recover(void *ctx) netdev_err(priv->netdev, "mlx5e_safe_reopen_channels failed recovering from a tx_timeout, err(%d).\n", err); - +out: + netdev_unlock(sq->netdev); return err; } @@ -172,10 +202,22 @@ static int mlx5e_tx_reporter_ptpsq_unhealthy_recover(void *ctx) return 0; priv = ptpsq->txqsq.priv; + netdev = priv->netdev; + + /* Recovering the PTP SQ means re-enabling NAPI, which requires the + * netdev instance lock. However, SQ closing has to wait for this work + * task to finish while also holding the same lock. So either get the + * lock or find that the SQ is no longer enabled and thus this work is + * not relevant anymore. + */ + while (!netdev_trylock(netdev)) { + if (!test_bit(MLX5E_SQ_STATE_ENABLED, &ptpsq->txqsq.state)) + return 0; + msleep(20); + } mutex_lock(&priv->state_lock); chs = &priv->channels; - netdev = priv->netdev; carrier_ok = netif_carrier_ok(netdev); netif_carrier_off(netdev); @@ -192,6 +234,7 @@ static int mlx5e_tx_reporter_ptpsq_unhealthy_recover(void *ctx) netif_carrier_on(netdev); mutex_unlock(&priv->state_lock); + netdev_unlock(netdev); return err; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/meter.c b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/meter.c index 7819fb2972802f..d5d9146efca673 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/meter.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/meter.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB // Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +#include #include #include "lib/aso.h" #include "en/tc/post_act.h" @@ -115,7 +116,6 @@ mlx5e_tc_meter_modify(struct mlx5_core_dev *mdev, struct mlx5e_flow_meters *flow_meters; u8 cir_man, cir_exp, cbs_man, cbs_exp; struct mlx5_aso_wqe *aso_wqe; - unsigned long expires; struct mlx5_aso *aso; u64 rate, burst; u8 ds_cnt; @@ -187,12 +187,8 @@ mlx5e_tc_meter_modify(struct mlx5_core_dev *mdev, mlx5_aso_post_wqe(aso, true, &aso_wqe->ctrl); /* With newer FW, the wait for the first ASO WQE is more than 2us, put the wait 10ms. */ - expires = jiffies + msecs_to_jiffies(10); - do { - err = mlx5_aso_poll_cq(aso, true); - if (err) - usleep_range(2, 10); - } while (err && time_is_after_jiffies(expires)); + read_poll_timeout(mlx5_aso_poll_cq, err, !err, 10, 10 * USEC_PER_MSEC, + false, aso, true); mutex_unlock(&flow_meters->aso_lock); return err; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c index 9c7064187ed0ff..f03507a522b4fb 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c @@ -259,7 +259,6 @@ static void mlx5e_ipsec_init_limits(struct mlx5e_ipsec_sa_entry *sa_entry, static void mlx5e_ipsec_init_macs(struct mlx5e_ipsec_sa_entry *sa_entry, struct mlx5_accel_esp_xfrm_attrs *attrs) { - struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry); struct mlx5e_ipsec_addr *addrs = &attrs->addrs; struct net_device *netdev = sa_entry->dev; struct xfrm_state *x = sa_entry->x; @@ -276,7 +275,7 @@ static void mlx5e_ipsec_init_macs(struct mlx5e_ipsec_sa_entry *sa_entry, attrs->type != XFRM_DEV_OFFLOAD_PACKET) return; - mlx5_query_mac_address(mdev, addr); + ether_addr_copy(addr, netdev->dev_addr); switch (attrs->dir) { case XFRM_DEV_OFFLOAD_IN: src = attrs->dmac; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h index f8eaaf37963b11..abcbd38db9dbbe 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h @@ -287,6 +287,7 @@ struct mlx5e_ipsec_sa_entry { struct mlx5e_ipsec_dwork *dwork; struct mlx5e_ipsec_limits limits; u32 rx_mapped_id; + u8 ctx[MLX5_ST_SZ_BYTES(ipsec_aso)]; }; struct mlx5_accel_pol_xfrm_attrs { diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c index feef86fff4bfda..91cfabc4503255 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c @@ -2912,7 +2912,7 @@ void mlx5e_ipsec_disable_events(struct mlx5e_priv *priv) goto out; peer_priv = mlx5_devcom_get_next_peer_data(priv->devcom, &tmp); - if (peer_priv) + if (peer_priv && peer_priv->ipsec) complete_all(&peer_priv->ipsec->comp); mlx5_devcom_for_each_peer_end(priv->devcom); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_offload.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_offload.c index ef7322d381af65..e0611fa8279710 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_offload.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_offload.c @@ -310,10 +310,11 @@ static void mlx5e_ipsec_aso_update(struct mlx5e_ipsec_sa_entry *sa_entry, mlx5e_ipsec_aso_query(sa_entry, data); } -static void mlx5e_ipsec_update_esn_state(struct mlx5e_ipsec_sa_entry *sa_entry, - u32 mode_param) +static void +mlx5e_ipsec_update_esn_state(struct mlx5e_ipsec_sa_entry *sa_entry, + u32 mode_param, + struct mlx5_accel_esp_xfrm_attrs *attrs) { - struct mlx5_accel_esp_xfrm_attrs attrs = {}; struct mlx5_wqe_aso_ctrl_seg data = {}; if (mode_param < MLX5E_IPSEC_ESN_SCOPE_MID) { @@ -323,18 +324,7 @@ static void mlx5e_ipsec_update_esn_state(struct mlx5e_ipsec_sa_entry *sa_entry, sa_entry->esn_state.overlap = 1; } - mlx5e_ipsec_build_accel_xfrm_attrs(sa_entry, &attrs); - - /* It is safe to execute the modify below unlocked since the only flows - * that could affect this HW object, are create, destroy and this work. - * - * Creation flow can't co-exist with this modify work, the destruction - * flow would cancel this work, and this work is a single entity that - * can't conflict with it self. - */ - spin_unlock_bh(&sa_entry->x->lock); - mlx5_accel_esp_modify_xfrm(sa_entry, &attrs); - spin_lock_bh(&sa_entry->x->lock); + mlx5e_ipsec_build_accel_xfrm_attrs(sa_entry, attrs); data.data_offset_condition_operand = MLX5_IPSEC_ASO_REMOVE_FLOW_PKT_CNT_OFFSET; @@ -370,20 +360,18 @@ static void mlx5e_ipsec_aso_update_soft(struct mlx5e_ipsec_sa_entry *sa_entry, static void mlx5e_ipsec_handle_limits(struct mlx5e_ipsec_sa_entry *sa_entry) { struct mlx5_accel_esp_xfrm_attrs *attrs = &sa_entry->attrs; - struct mlx5e_ipsec *ipsec = sa_entry->ipsec; - struct mlx5e_ipsec_aso *aso = ipsec->aso; bool soft_arm, hard_arm; u64 hard_cnt; lockdep_assert_held(&sa_entry->x->lock); - soft_arm = !MLX5_GET(ipsec_aso, aso->ctx, soft_lft_arm); - hard_arm = !MLX5_GET(ipsec_aso, aso->ctx, hard_lft_arm); + soft_arm = !MLX5_GET(ipsec_aso, sa_entry->ctx, soft_lft_arm); + hard_arm = !MLX5_GET(ipsec_aso, sa_entry->ctx, hard_lft_arm); if (!soft_arm && !hard_arm) /* It is not lifetime event */ return; - hard_cnt = MLX5_GET(ipsec_aso, aso->ctx, remove_flow_pkt_cnt); + hard_cnt = MLX5_GET(ipsec_aso, sa_entry->ctx, remove_flow_pkt_cnt); if (!hard_cnt || hard_arm) { /* It is possible to see packet counter equal to zero without * hard limit event armed. Such situation can be if packet @@ -453,11 +441,11 @@ static void mlx5e_ipsec_handle_event(struct work_struct *_work) struct mlx5e_ipsec_work *work = container_of(_work, struct mlx5e_ipsec_work, work); struct mlx5e_ipsec_sa_entry *sa_entry = work->data; + struct mlx5_accel_esp_xfrm_attrs tmp = {}; struct mlx5_accel_esp_xfrm_attrs *attrs; - struct mlx5e_ipsec_aso *aso; + bool need_modify = false; int ret; - aso = sa_entry->ipsec->aso; attrs = &sa_entry->attrs; spin_lock_bh(&sa_entry->x->lock); @@ -465,18 +453,22 @@ static void mlx5e_ipsec_handle_event(struct work_struct *_work) if (ret) goto unlock; + if (attrs->lft.soft_packet_limit != XFRM_INF) + mlx5e_ipsec_handle_limits(sa_entry); + if (attrs->replay_esn.trigger && - !MLX5_GET(ipsec_aso, aso->ctx, esn_event_arm)) { - u32 mode_param = MLX5_GET(ipsec_aso, aso->ctx, mode_parameter); + !MLX5_GET(ipsec_aso, sa_entry->ctx, esn_event_arm)) { + u32 mode_param = MLX5_GET(ipsec_aso, sa_entry->ctx, + mode_parameter); - mlx5e_ipsec_update_esn_state(sa_entry, mode_param); + mlx5e_ipsec_update_esn_state(sa_entry, mode_param, &tmp); + need_modify = true; } - if (attrs->lft.soft_packet_limit != XFRM_INF) - mlx5e_ipsec_handle_limits(sa_entry); - unlock: spin_unlock_bh(&sa_entry->x->lock); + if (need_modify) + mlx5_accel_esp_modify_xfrm(sa_entry, &tmp); kfree(work); } @@ -629,6 +621,8 @@ int mlx5e_ipsec_aso_query(struct mlx5e_ipsec_sa_entry *sa_entry, /* We are in atomic context */ udelay(10); } while (ret && time_is_after_jiffies(expires)); + if (!ret) + memcpy(sa_entry->ctx, aso->ctx, MLX5_ST_SZ_BYTES(ipsec_aso)); spin_unlock_bh(&aso->lock); return ret; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/macsec.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/macsec.c index 528b04d4de416b..90b3bc5f9166fd 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/macsec.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/macsec.c @@ -5,6 +5,7 @@ #include #include #include +#include #include "en.h" #include "lib/aso.h" @@ -1385,7 +1386,8 @@ static int macsec_aso_set_arm_event(struct mlx5_core_dev *mdev, struct mlx5e_mac MLX5_ACCESS_ASO_OPC_MOD_MACSEC); macsec_aso_build_ctrl(aso, &aso_wqe->aso_ctrl, in); mlx5_aso_post_wqe(maso, false, &aso_wqe->ctrl); - err = mlx5_aso_poll_cq(maso, false); + read_poll_timeout(mlx5_aso_poll_cq, err, !err, 10, 10 * USEC_PER_MSEC, + false, maso, false); mutex_unlock(&aso->aso_lock); return err; @@ -1397,7 +1399,6 @@ static int macsec_aso_query(struct mlx5_core_dev *mdev, struct mlx5e_macsec *mac struct mlx5e_macsec_aso *aso; struct mlx5_aso_wqe *aso_wqe; struct mlx5_aso *maso; - unsigned long expires; int err; aso = &macsec->aso; @@ -1411,12 +1412,8 @@ static int macsec_aso_query(struct mlx5_core_dev *mdev, struct mlx5e_macsec *mac macsec_aso_build_wqe_ctrl_seg(aso, &aso_wqe->aso_ctrl, NULL); mlx5_aso_post_wqe(maso, false, &aso_wqe->ctrl); - expires = jiffies + msecs_to_jiffies(10); - do { - err = mlx5_aso_poll_cq(maso, false); - if (err) - usleep_range(2, 10); - } while (err && time_is_after_jiffies(expires)); + read_poll_timeout(mlx5_aso_poll_cq, err, !err, 10, 10 * USEC_PER_MSEC, + false, maso, false); if (err) goto err_out; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c index 4b2963bbe7ff45..e15e6fb4cd8ead 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c @@ -688,19 +688,7 @@ static void mlx5e_rq_timeout_work(struct work_struct *timeout_work) struct mlx5e_rq, rx_timeout_work); - /* Acquire netdev instance lock to synchronize with channel close and - * reopen flows. Either successfully obtain the lock, or detect that - * channels are closing for another reason, making this work no longer - * necessary. - */ - while (!netdev_trylock(rq->netdev)) { - if (!test_bit(MLX5E_STATE_CHANNELS_ACTIVE, &rq->priv->state)) - return; - msleep(20); - } - mlx5e_reporter_rx_timeout(rq); - netdev_unlock(rq->netdev); } static int mlx5e_alloc_mpwqe_rq_drop_page(struct mlx5e_rq *rq) @@ -1997,20 +1985,7 @@ void mlx5e_tx_err_cqe_work(struct work_struct *recover_work) struct mlx5e_txqsq *sq = container_of(recover_work, struct mlx5e_txqsq, recover_work); - /* Recovering queues means re-enabling NAPI, which requires the netdev - * instance lock. However, SQ closing flows have to wait for work tasks - * to finish while also holding the netdev instance lock. So either get - * the lock or find that the SQ is no longer enabled and thus this work - * is not relevant anymore. - */ - while (!netdev_trylock(sq->netdev)) { - if (!test_bit(MLX5E_SQ_STATE_ENABLED, &sq->state)) - return; - msleep(20); - } - mlx5e_reporter_tx_err_cqe(sq); - netdev_unlock(sq->netdev); } static struct dim_cq_moder mlx5e_get_def_tx_moderation(u8 cq_period_mode) @@ -5121,19 +5096,6 @@ static void mlx5e_tx_timeout_work(struct work_struct *work) struct net_device *netdev = priv->netdev; int i; - /* Recovering the TX queues implies re-enabling NAPI, which requires - * the netdev instance lock. - * However, channel closing flows have to wait for this work to finish - * while holding the same lock. So either get the lock or find that - * channels are being closed for other reason and this work is not - * relevant anymore. - */ - while (!netdev_trylock(netdev)) { - if (!test_bit(MLX5E_STATE_CHANNELS_ACTIVE, &priv->state)) - return; - msleep(20); - } - for (i = 0; i < netdev->real_num_tx_queues; i++) { struct netdev_queue *dev_queue = netdev_get_tx_queue(netdev, i); @@ -5146,8 +5108,6 @@ static void mlx5e_tx_timeout_work(struct work_struct *work) /* break if tried to reopened channels */ break; } - - netdev_unlock(netdev); } static void mlx5e_tx_timeout(struct net_device *dev, unsigned int txqueue) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c index 1f6930c7743784..3000286bf29c86 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c @@ -1759,6 +1759,7 @@ mlx5e_skb_from_cqe_nonlinear(struct mlx5e_rq *rq, struct mlx5e_wqe_frag_info *wi struct skb_shared_info *sinfo; u32 frag_consumed_bytes; struct bpf_prog *prog; + u8 nr_frags_free = 0; struct sk_buff *skb; dma_addr_t addr; u32 truesize; @@ -1801,15 +1802,13 @@ mlx5e_skb_from_cqe_nonlinear(struct mlx5e_rq *rq, struct mlx5e_wqe_frag_info *wi prog = rcu_dereference(rq->xdp_prog); if (prog) { - u8 nr_frags_free, old_nr_frags = sinfo->nr_frags; + u8 old_nr_frags = sinfo->nr_frags; if (mlx5e_xdp_handle(rq, prog, mxbuf)) { if (__test_and_clear_bit(MLX5E_RQ_FLAG_XDP_XMIT, rq->flags)) { struct mlx5e_wqe_frag_info *pwi; - wi -= old_nr_frags - sinfo->nr_frags; - for (pwi = head_wi; pwi < wi; pwi++) pwi->frag_page->frags++; } @@ -1817,10 +1816,8 @@ mlx5e_skb_from_cqe_nonlinear(struct mlx5e_rq *rq, struct mlx5e_wqe_frag_info *wi } nr_frags_free = old_nr_frags - sinfo->nr_frags; - if (unlikely(nr_frags_free)) { - wi -= nr_frags_free; + if (unlikely(nr_frags_free)) truesize -= nr_frags_free * frag_info->frag_stride; - } } skb = mlx5e_build_linear_skb( @@ -1836,7 +1833,7 @@ mlx5e_skb_from_cqe_nonlinear(struct mlx5e_rq *rq, struct mlx5e_wqe_frag_info *wi if (xdp_buff_has_frags(&mxbuf->xdp)) { /* sinfo->nr_frags is reset by build_skb, calculate again. */ - xdp_update_skb_frags_info(skb, wi - head_wi - 1, + xdp_update_skb_frags_info(skb, wi - head_wi - nr_frags_free - 1, sinfo->xdp_frags_size, truesize, xdp_buff_get_skb_flags(&mxbuf->xdp)); @@ -2118,14 +2115,13 @@ mlx5e_skb_from_cqe_mpwrq_nonlinear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *w if (prog) { u8 nr_frags_free, old_nr_frags = sinfo->nr_frags; + u8 new_nr_frags; u32 len; if (mlx5e_xdp_handle(rq, prog, mxbuf)) { if (__test_and_clear_bit(MLX5E_RQ_FLAG_XDP_XMIT, rq->flags)) { struct mlx5e_frag_page *pfp; - frag_page -= old_nr_frags - sinfo->nr_frags; - for (pfp = head_page; pfp < frag_page; pfp++) pfp->frags++; @@ -2136,13 +2132,12 @@ mlx5e_skb_from_cqe_mpwrq_nonlinear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *w return NULL; /* page/packet was consumed by XDP */ } - nr_frags_free = old_nr_frags - sinfo->nr_frags; - if (unlikely(nr_frags_free)) { - frag_page -= nr_frags_free; + new_nr_frags = sinfo->nr_frags; + nr_frags_free = old_nr_frags - new_nr_frags; + if (unlikely(nr_frags_free)) truesize -= (nr_frags_free - 1) * PAGE_SIZE + ALIGN(pg_consumed_bytes, BIT(rq->mpwqe.log_stride_sz)); - } len = mxbuf->xdp.data_end - mxbuf->xdp.data; @@ -2164,7 +2159,7 @@ mlx5e_skb_from_cqe_mpwrq_nonlinear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *w struct mlx5e_frag_page *pagep; /* sinfo->nr_frags is reset by build_skb, calculate again. */ - xdp_update_skb_frags_info(skb, frag_page - head_page, + xdp_update_skb_frags_info(skb, new_nr_frags, sinfo->xdp_frags_size, truesize, xdp_buff_get_skb_flags(&mxbuf->xdp)); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/esw/qos.c b/drivers/net/ethernet/mellanox/mlx5/core/esw/qos.c index 4278bcb04c72ed..2e11574b3a81f8 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/esw/qos.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/esw/qos.c @@ -1490,24 +1490,24 @@ static int esw_qos_node_enable_tc_arbitration(struct mlx5_esw_sched_node *node, return err; } -static u32 mlx5_esw_qos_lag_link_speed_get_locked(struct mlx5_core_dev *mdev) +static u32 mlx5_esw_qos_lag_link_speed_get(struct mlx5_core_dev *mdev, + bool take_rtnl) { struct ethtool_link_ksettings lksettings; struct net_device *slave, *master; u32 speed = SPEED_UNKNOWN; - /* Lock ensures a stable reference to master and slave netdevice - * while port speed of master is queried. - */ - ASSERT_RTNL(); - slave = mlx5_uplink_netdev_get(mdev); if (!slave) goto out; + if (take_rtnl) + rtnl_lock(); master = netdev_master_upper_dev_get(slave); if (master && !__ethtool_get_link_ksettings(master, &lksettings)) speed = lksettings.base.speed; + if (take_rtnl) + rtnl_unlock(); out: mlx5_uplink_netdev_put(mdev, slave); @@ -1515,20 +1515,15 @@ static u32 mlx5_esw_qos_lag_link_speed_get_locked(struct mlx5_core_dev *mdev) } static int mlx5_esw_qos_max_link_speed_get(struct mlx5_core_dev *mdev, u32 *link_speed_max, - bool hold_rtnl_lock, struct netlink_ext_ack *extack) + bool take_rtnl, + struct netlink_ext_ack *extack) { int err; if (!mlx5_lag_is_active(mdev)) goto skip_lag; - if (hold_rtnl_lock) - rtnl_lock(); - - *link_speed_max = mlx5_esw_qos_lag_link_speed_get_locked(mdev); - - if (hold_rtnl_lock) - rtnl_unlock(); + *link_speed_max = mlx5_esw_qos_lag_link_speed_get(mdev, take_rtnl); if (*link_speed_max != (u32)SPEED_UNKNOWN) return 0; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c index 4b7a1ce7f406b1..32bf93e4ffcec7 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c @@ -1072,10 +1072,11 @@ static void mlx5_eswitch_event_handler_register(struct mlx5_eswitch *esw) static void mlx5_eswitch_event_handler_unregister(struct mlx5_eswitch *esw) { - if (esw->mode == MLX5_ESWITCH_OFFLOADS && mlx5_eswitch_is_funcs_handler(esw->dev)) + if (esw->mode == MLX5_ESWITCH_OFFLOADS && + mlx5_eswitch_is_funcs_handler(esw->dev)) { mlx5_eq_notifier_unregister(esw->dev, &esw->esw_funcs.nb); - - flush_workqueue(esw->work_queue); + atomic_inc(&esw->esw_funcs.generation); + } } static void mlx5_eswitch_clear_vf_vports_info(struct mlx5_eswitch *esw) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h index 714ad28e8445b4..0276609a617c08 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h @@ -334,10 +334,12 @@ struct esw_mc_addr { /* SRIOV only */ struct mlx5_host_work { struct work_struct work; struct mlx5_eswitch *esw; + int work_gen; }; struct mlx5_esw_functions { struct mlx5_nb nb; + atomic_t generation; bool host_funcs_disabled; u16 num_vfs; u16 num_ec_vfs; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c index 02b7e474586d9c..31e4eb6bd685bd 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c @@ -1241,21 +1241,17 @@ static int esw_add_fdb_peer_miss_rules(struct mlx5_eswitch *esw, flows[peer_vport->index] = flow; } - if (mlx5_esw_host_functions_enabled(esw->dev)) { - mlx5_esw_for_each_vf_vport(peer_esw, i, peer_vport, - mlx5_core_max_vfs(peer_dev)) { - esw_set_peer_miss_rule_source_port(esw, peer_esw, - spec, - peer_vport->vport); - - flow = mlx5_add_flow_rules(mlx5_eswitch_get_slow_fdb(esw), - spec, &flow_act, &dest, 1); - if (IS_ERR(flow)) { - err = PTR_ERR(flow); - goto add_vf_flow_err; - } - flows[peer_vport->index] = flow; + mlx5_esw_for_each_vf_vport(peer_esw, i, peer_vport, + mlx5_core_max_vfs(peer_dev)) { + esw_set_peer_miss_rule_source_port(esw, peer_esw, spec, + peer_vport->vport); + flow = mlx5_add_flow_rules(mlx5_eswitch_get_slow_fdb(esw), + spec, &flow_act, &dest, 1); + if (IS_ERR(flow)) { + err = PTR_ERR(flow); + goto add_vf_flow_err; } + flows[peer_vport->index] = flow; } if (mlx5_core_ec_sriov_enabled(peer_dev)) { @@ -1347,7 +1343,8 @@ static void esw_del_fdb_peer_miss_rules(struct mlx5_eswitch *esw, mlx5_del_flow_rules(flows[peer_vport->index]); } - if (mlx5_core_is_ecpf_esw_manager(peer_dev)) { + if (mlx5_core_is_ecpf_esw_manager(peer_dev) && + mlx5_esw_host_functions_enabled(peer_dev)) { peer_vport = mlx5_eswitch_get_vport(peer_esw, MLX5_VPORT_PF); mlx5_del_flow_rules(flows[peer_vport->index]); } @@ -3582,22 +3579,28 @@ static void esw_offloads_steering_cleanup(struct mlx5_eswitch *esw) } static void -esw_vfs_changed_event_handler(struct mlx5_eswitch *esw, const u32 *out) +esw_vfs_changed_event_handler(struct mlx5_eswitch *esw, int work_gen, + const u32 *out) { struct devlink *devlink; bool host_pf_disabled; u16 new_num_vfs; + devlink = priv_to_devlink(esw->dev); + devl_lock(devlink); + + /* Stale work from one or more mode changes ago. Bail out. */ + if (work_gen != atomic_read(&esw->esw_funcs.generation)) + goto unlock; + new_num_vfs = MLX5_GET(query_esw_functions_out, out, host_params_context.host_num_of_vfs); host_pf_disabled = MLX5_GET(query_esw_functions_out, out, host_params_context.host_pf_disabled); if (new_num_vfs == esw->esw_funcs.num_vfs || host_pf_disabled) - return; + goto unlock; - devlink = priv_to_devlink(esw->dev); - devl_lock(devlink); /* Number of VFs can only change from "0 to x" or "x to 0". */ if (esw->esw_funcs.num_vfs > 0) { mlx5_eswitch_unload_vf_vports(esw, esw->esw_funcs.num_vfs); @@ -3612,6 +3615,7 @@ esw_vfs_changed_event_handler(struct mlx5_eswitch *esw, const u32 *out) } } esw->esw_funcs.num_vfs = new_num_vfs; +unlock: devl_unlock(devlink); } @@ -3628,7 +3632,7 @@ static void esw_functions_changed_event_handler(struct work_struct *work) if (IS_ERR(out)) goto out; - esw_vfs_changed_event_handler(esw, out); + esw_vfs_changed_event_handler(esw, host_work->work_gen, out); kvfree(out); out: kfree(host_work); @@ -3648,6 +3652,7 @@ int mlx5_esw_funcs_changed_handler(struct notifier_block *nb, unsigned long type esw = container_of(esw_funcs, struct mlx5_eswitch, esw_funcs); host_work->esw = esw; + host_work->work_gen = atomic_read(&esw_funcs->generation); INIT_WORK(&host_work->work, esw_functions_changed_event_handler); queue_work(esw->work_queue, &host_work->work); @@ -3756,6 +3761,8 @@ int esw_offloads_enable(struct mlx5_eswitch *esw) return 0; err_vports: + /* rollback to legacy, indicates don't unregister the uplink netdev */ + esw->dev->priv.flags |= MLX5_PRIV_FLAGS_SWITCH_LEGACY; mlx5_esw_offloads_rep_unload(esw, MLX5_VPORT_UPLINK); err_uplink: esw_offloads_steering_cleanup(esw); @@ -4068,6 +4075,8 @@ int mlx5_devlink_eswitch_mode_set(struct devlink *devlink, u16 mode, if (mlx5_mode == MLX5_ESWITCH_LEGACY) esw->dev->priv.flags |= MLX5_PRIV_FLAGS_SWITCH_LEGACY; + if (mlx5_mode == MLX5_ESWITCH_OFFLOADS) + esw->dev->priv.flags &= ~MLX5_PRIV_FLAGS_SWITCH_LEGACY; mlx5_eswitch_disable_locked(esw); if (mlx5_mode == MLX5_ESWITCH_OFFLOADS) { if (mlx5_devlink_trap_get_num_active(esw->dev)) { diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fw.c b/drivers/net/ethernet/mellanox/mlx5/core/fw.c index eeb4437975f20a..c1f220e5fe1857 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/fw.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/fw.c @@ -822,48 +822,63 @@ mlx5_fw_image_pending(struct mlx5_core_dev *dev, return 0; } -int mlx5_fw_version_query(struct mlx5_core_dev *dev, - u32 *running_ver, u32 *pending_ver) +void mlx5_fw_version_query(struct mlx5_core_dev *dev, + u32 *running_ver, u32 *pending_ver) { u32 reg_mcqi_version[MLX5_ST_SZ_DW(mcqi_version)] = {}; bool pending_version_exists; int component_index; int err; + *running_ver = 0; + *pending_ver = 0; + if (!MLX5_CAP_GEN(dev, mcam_reg) || !MLX5_CAP_MCAM_REG(dev, mcqi) || !MLX5_CAP_MCAM_REG(dev, mcqs)) { mlx5_core_warn(dev, "fw query isn't supported by the FW\n"); - return -EOPNOTSUPP; + return; } component_index = mlx5_get_boot_img_component_index(dev); - if (component_index < 0) - return component_index; + if (component_index < 0) { + mlx5_core_warn(dev, "fw query failed to find boot img component index, err %d\n", + component_index); + return; + } + *running_ver = U32_MAX; /* indicate failure */ err = mlx5_reg_mcqi_version_query(dev, component_index, MCQI_FW_RUNNING_VERSION, reg_mcqi_version); - if (err) - return err; - - *running_ver = MLX5_GET(mcqi_version, reg_mcqi_version, version); - + if (!err) + *running_ver = MLX5_GET(mcqi_version, reg_mcqi_version, + version); + else + mlx5_core_warn(dev, "failed to query running version, err %d\n", + err); + + *pending_ver = U32_MAX; /* indicate failure */ err = mlx5_fw_image_pending(dev, component_index, &pending_version_exists); - if (err) - return err; + if (err) { + mlx5_core_warn(dev, "failed to query pending image, err %d\n", + err); + return; + } if (!pending_version_exists) { *pending_ver = 0; - return 0; + return; } err = mlx5_reg_mcqi_version_query(dev, component_index, MCQI_FW_STORED_VERSION, reg_mcqi_version); - if (err) - return err; - - *pending_ver = MLX5_GET(mcqi_version, reg_mcqi_version, version); - - return 0; + if (!err) + *pending_ver = MLX5_GET(mcqi_version, reg_mcqi_version, + version); + else + mlx5_core_warn(dev, "failed to query pending version, err %d\n", + err); + + return; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lag/debugfs.c b/drivers/net/ethernet/mellanox/mlx5/core/lag/debugfs.c index 62b6faa4276aad..b8d5f6a44d26ab 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lag/debugfs.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lag/debugfs.c @@ -160,8 +160,11 @@ DEFINE_SHOW_ATTRIBUTE(members); void mlx5_ldev_add_debugfs(struct mlx5_core_dev *dev) { + struct mlx5_lag *ldev = mlx5_lag_dev(dev); struct dentry *dbg; + if (!ldev) + return; dbg = debugfs_create_dir("lag", mlx5_debugfs_get_dev_root(dev)); dev->priv.dbg.lag_debugfs = dbg; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c b/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c index a459a30f36cae6..73659a0463cdeb 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c @@ -1654,8 +1654,12 @@ void mlx5_lag_disable_change(struct mlx5_core_dev *dev) mutex_lock(&ldev->lock); ldev->mode_changes_in_progress++; - if (__mlx5_lag_is_active(ldev)) - mlx5_disable_lag(ldev); + if (__mlx5_lag_is_active(ldev)) { + if (ldev->mode == MLX5_LAG_MODE_MPESW) + mlx5_lag_disable_mpesw(ldev); + else + mlx5_disable_lag(ldev); + } mutex_unlock(&ldev->lock); mlx5_devcom_comp_unlock(dev->priv.hca_devcom_comp); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lag/mpesw.c b/drivers/net/ethernet/mellanox/mlx5/core/lag/mpesw.c index 2d86af8f0d9b81..c217998604fdbc 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lag/mpesw.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lag/mpesw.c @@ -65,7 +65,7 @@ static int mlx5_mpesw_metadata_set(struct mlx5_lag *ldev) return err; } -static int enable_mpesw(struct mlx5_lag *ldev) +static int mlx5_lag_enable_mpesw(struct mlx5_lag *ldev) { struct mlx5_core_dev *dev0; int err; @@ -124,7 +124,7 @@ static int enable_mpesw(struct mlx5_lag *ldev) return err; } -static void disable_mpesw(struct mlx5_lag *ldev) +void mlx5_lag_disable_mpesw(struct mlx5_lag *ldev) { if (ldev->mode == MLX5_LAG_MODE_MPESW) { mlx5_mpesw_metadata_cleanup(ldev); @@ -150,9 +150,9 @@ static void mlx5_mpesw_work(struct work_struct *work) } if (mpesww->op == MLX5_MPESW_OP_ENABLE) - mpesww->result = enable_mpesw(ldev); + mpesww->result = mlx5_lag_enable_mpesw(ldev); else if (mpesww->op == MLX5_MPESW_OP_DISABLE) - disable_mpesw(ldev); + mlx5_lag_disable_mpesw(ldev); unlock: mutex_unlock(&ldev->lock); mlx5_devcom_comp_unlock(devcom); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lag/mpesw.h b/drivers/net/ethernet/mellanox/mlx5/core/lag/mpesw.h index 02520f27a033c7..46de93ed790ded 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lag/mpesw.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/lag/mpesw.h @@ -31,5 +31,10 @@ int mlx5_lag_mpesw_do_mirred(struct mlx5_core_dev *mdev, bool mlx5_lag_is_mpesw(struct mlx5_core_dev *dev); void mlx5_lag_mpesw_disable(struct mlx5_core_dev *dev); int mlx5_lag_mpesw_enable(struct mlx5_core_dev *dev); +#ifdef CONFIG_MLX5_ESWITCH +void mlx5_lag_disable_mpesw(struct mlx5_lag *ldev); +#else +static inline void mlx5_lag_disable_mpesw(struct mlx5_lag *ldev) {} +#endif /* CONFIG_MLX5_ESWITCH */ #endif /* __MLX5_LAG_MPESW_H__ */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/main.c b/drivers/net/ethernet/mellanox/mlx5/core/main.c index 55b4e0cceae2f5..b62eddad64ab0b 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/main.c @@ -2267,6 +2267,7 @@ static const struct pci_device_id mlx5_core_pci_table[] = { { PCI_VDEVICE(MELLANOX, 0x1023) }, /* ConnectX-8 */ { PCI_VDEVICE(MELLANOX, 0x1025) }, /* ConnectX-9 */ { PCI_VDEVICE(MELLANOX, 0x1027) }, /* ConnectX-10 */ + { PCI_VDEVICE(MELLANOX, 0x2101) }, /* ConnectX-10 NVLink-C2C */ { PCI_VDEVICE(MELLANOX, 0xa2d2) }, /* BlueField integrated ConnectX-5 network controller */ { PCI_VDEVICE(MELLANOX, 0xa2d3), MLX5_PCI_DEV_IS_VF}, /* BlueField integrated ConnectX-5 network controller VF */ { PCI_VDEVICE(MELLANOX, 0xa2d6) }, /* BlueField-2 integrated ConnectX-6 Dx network controller */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h b/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h index f2d74382fb85d9..c048839f07d6d4 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h @@ -392,8 +392,8 @@ int mlx5_port_max_linkspeed(struct mlx5_core_dev *mdev, u32 *speed); int mlx5_firmware_flash(struct mlx5_core_dev *dev, const struct firmware *fw, struct netlink_ext_ack *extack); -int mlx5_fw_version_query(struct mlx5_core_dev *dev, - u32 *running_ver, u32 *stored_ver); +void mlx5_fw_version_query(struct mlx5_core_dev *dev, u32 *running_ver, + u32 *stored_ver); #ifdef CONFIG_MLX5_CORE_EN int mlx5e_init(void); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/sriov.c b/drivers/net/ethernet/mellanox/mlx5/core/sriov.c index a2fc937d54617c..172862a70c70d2 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/sriov.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/sriov.c @@ -193,7 +193,9 @@ static int mlx5_sriov_enable(struct pci_dev *pdev, int num_vfs) err = pci_enable_sriov(pdev, num_vfs); if (err) { mlx5_core_warn(dev, "pci_enable_sriov failed : %d\n", err); + devl_lock(devlink); mlx5_device_disable_sriov(dev, num_vfs, true, true); + devl_unlock(devlink); } return err; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/sws/dr_dbg.c b/drivers/net/ethernet/mellanox/mlx5/core/steering/sws/dr_dbg.c index 030a5776c93740..a4c19af1775f1b 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/steering/sws/dr_dbg.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/sws/dr_dbg.c @@ -1050,8 +1050,8 @@ static int dr_dump_domain_all(struct seq_file *file, struct mlx5dr_domain *dmn) struct mlx5dr_table *tbl; int ret; - mutex_lock(&dmn->dump_info.dbg_mutex); mlx5dr_domain_lock(dmn); + mutex_lock(&dmn->dump_info.dbg_mutex); ret = dr_dump_domain(file, dmn); if (ret < 0) @@ -1064,8 +1064,8 @@ static int dr_dump_domain_all(struct seq_file *file, struct mlx5dr_domain *dmn) } unlock_mutex: - mlx5dr_domain_unlock(dmn); mutex_unlock(&dmn->dump_info.dbg_mutex); + mlx5dr_domain_unlock(dmn); return ret; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/wc.c b/drivers/net/ethernet/mellanox/mlx5/core/wc.c index 815a7c97d6b09e..04d03be1bb775d 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/wc.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/wc.c @@ -2,6 +2,7 @@ // Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. #include +#include #include #include "lib/clock.h" #include "mlx5_core.h" @@ -15,7 +16,7 @@ #define TEST_WC_NUM_WQES 255 #define TEST_WC_LOG_CQ_SZ (order_base_2(TEST_WC_NUM_WQES)) #define TEST_WC_SQ_LOG_WQ_SZ TEST_WC_LOG_CQ_SZ -#define TEST_WC_POLLING_MAX_TIME_JIFFIES msecs_to_jiffies(100) +#define TEST_WC_POLLING_MAX_TIME_USEC (100 * USEC_PER_MSEC) struct mlx5_wc_cq { /* data path - accessed per cqe */ @@ -359,7 +360,6 @@ static int mlx5_wc_poll_cq(struct mlx5_wc_sq *sq) static void mlx5_core_test_wc(struct mlx5_core_dev *mdev) { unsigned int offset = 0; - unsigned long expires; struct mlx5_wc_sq *sq; int i, err; @@ -389,13 +389,9 @@ static void mlx5_core_test_wc(struct mlx5_core_dev *mdev) mlx5_wc_post_nop(sq, &offset, true); - expires = jiffies + TEST_WC_POLLING_MAX_TIME_JIFFIES; - do { - err = mlx5_wc_poll_cq(sq); - if (err) - usleep_range(2, 10); - } while (mdev->wc_state == MLX5_WC_STATE_UNINITIALIZED && - time_is_after_jiffies(expires)); + poll_timeout_us(mlx5_wc_poll_cq(sq), + mdev->wc_state != MLX5_WC_STATE_UNINITIALIZED, 10, + TEST_WC_POLLING_MAX_TIME_USEC, false); mlx5_wc_destroy_sq(sq); diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_ethtool.c b/drivers/net/ethernet/meta/fbnic/fbnic_ethtool.c index 693ebdf3870552..5edc28ba295531 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_ethtool.c +++ b/drivers/net/ethernet/meta/fbnic/fbnic_ethtool.c @@ -1142,6 +1142,9 @@ static int fbnic_set_cls_rule_ins(struct fbnic_net *fbn, return -EINVAL; } + dest |= FIELD_PREP(FBNIC_RPC_ACT_TBL0_DMA_HINT, + FBNIC_RCD_HDR_AL_DMA_HINT_L4); + /* Write action table values */ act_tcam->dest = dest; act_tcam->rss_en_mask = fbnic_flow_hash_2_rss_en_mask(fbn, hash_idx); diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_fw_log.c b/drivers/net/ethernet/meta/fbnic/fbnic_fw_log.c index 85a883dba385fe..d8a9a7d7c2375f 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_fw_log.c +++ b/drivers/net/ethernet/meta/fbnic/fbnic_fw_log.c @@ -51,8 +51,6 @@ int fbnic_fw_log_init(struct fbnic_dev *fbd) log->data_start = data; log->data_end = data + FBNIC_FW_LOG_SIZE; - fbnic_fw_log_enable(fbd, true); - return 0; } @@ -63,7 +61,6 @@ void fbnic_fw_log_free(struct fbnic_dev *fbd) if (!fbnic_fw_log_ready(fbd)) return; - fbnic_fw_log_disable(fbd); INIT_LIST_HEAD(&log->entries); log->size = 0; vfree(log->data_start); diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_netdev.c b/drivers/net/ethernet/meta/fbnic/fbnic_netdev.c index 81c9d5c9a4b2c0..b4b396ca9bce3c 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_netdev.c +++ b/drivers/net/ethernet/meta/fbnic/fbnic_netdev.c @@ -262,6 +262,23 @@ static int fbnic_set_mac(struct net_device *netdev, void *p) return 0; } +static int fbnic_change_mtu(struct net_device *dev, int new_mtu) +{ + struct fbnic_net *fbn = netdev_priv(dev); + + if (fbnic_check_split_frames(fbn->xdp_prog, new_mtu, fbn->hds_thresh)) { + dev_err(&dev->dev, + "MTU %d is larger than HDS threshold %d in XDP mode\n", + new_mtu, fbn->hds_thresh); + + return -EINVAL; + } + + WRITE_ONCE(dev->mtu, new_mtu); + + return 0; +} + void fbnic_clear_rx_mode(struct fbnic_dev *fbd) { struct net_device *netdev = fbd->netdev; @@ -533,6 +550,7 @@ static const struct net_device_ops fbnic_netdev_ops = { .ndo_start_xmit = fbnic_xmit_frame, .ndo_features_check = fbnic_features_check, .ndo_set_mac_address = fbnic_set_mac, + .ndo_change_mtu = fbnic_change_mtu, .ndo_set_rx_mode = fbnic_set_rx_mode, .ndo_get_stats64 = fbnic_get_stats64, .ndo_bpf = fbnic_bpf, @@ -787,6 +805,8 @@ struct net_device *fbnic_netdev_alloc(struct fbnic_dev *fbd) netdev->hw_enc_features |= netdev->features; netdev->features |= NETIF_F_NTUPLE; + netdev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_RX_SG; + netdev->min_mtu = IPV6_MIN_MTU; netdev->max_mtu = FBNIC_MAX_JUMBO_FRAME_SIZE - ETH_HLEN; diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_pci.c b/drivers/net/ethernet/meta/fbnic/fbnic_pci.c index 9240673c7533d2..e92187bc1c0fa4 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_pci.c +++ b/drivers/net/ethernet/meta/fbnic/fbnic_pci.c @@ -307,11 +307,17 @@ static int fbnic_probe(struct pci_dev *pdev, const struct pci_device_id *ent) goto free_irqs; } + err = fbnic_fw_log_init(fbd); + if (err) + dev_warn(fbd->dev, + "Unable to initialize firmware log buffer: %d\n", + err); + err = fbnic_fw_request_mbx(fbd); if (err) { dev_err(&pdev->dev, "Firmware mailbox initialization failure\n"); - goto free_irqs; + goto free_fw_log; } /* Send the request to enable the FW logging to host. Note if this @@ -319,11 +325,7 @@ static int fbnic_probe(struct pci_dev *pdev, const struct pci_device_id *ent) * possible the FW is just too old to support the logging and needs * to be updated. */ - err = fbnic_fw_log_init(fbd); - if (err) - dev_warn(fbd->dev, - "Unable to initialize firmware log buffer: %d\n", - err); + fbnic_fw_log_enable(fbd, true); fbnic_devlink_register(fbd); fbnic_devlink_otp_check(fbd, "error detected during probe"); @@ -370,6 +372,8 @@ static int fbnic_probe(struct pci_dev *pdev, const struct pci_device_id *ent) * firmware updates for fixes. */ return 0; +free_fw_log: + fbnic_fw_log_free(fbd); free_irqs: fbnic_free_irqs(fbd); err_destroy_health: @@ -404,8 +408,9 @@ static void fbnic_remove(struct pci_dev *pdev) fbnic_hwmon_unregister(fbd); fbnic_dbg_fbd_exit(fbd); fbnic_devlink_unregister(fbd); - fbnic_fw_log_free(fbd); + fbnic_fw_log_disable(fbd); fbnic_fw_free_mbx(fbd); + fbnic_fw_log_free(fbd); fbnic_free_irqs(fbd); fbnic_devlink_health_destroy(fbd); diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_rpc.c b/drivers/net/ethernet/meta/fbnic/fbnic_rpc.c index 7f31e890031c06..42a186db43ea93 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_rpc.c +++ b/drivers/net/ethernet/meta/fbnic/fbnic_rpc.c @@ -338,9 +338,8 @@ void fbnic_rss_reinit(struct fbnic_dev *fbd, struct fbnic_net *fbn) else if (tstamp_mask & (1u << flow_type)) dest |= FBNIC_RPC_ACT_TBL0_TS_ENA; - if (act1_value[flow_type] & FBNIC_RPC_TCAM_ACT1_L4_VALID) - dest |= FIELD_PREP(FBNIC_RPC_ACT_TBL0_DMA_HINT, - FBNIC_RCD_HDR_AL_DMA_HINT_L4); + dest |= FIELD_PREP(FBNIC_RPC_ACT_TBL0_DMA_HINT, + FBNIC_RCD_HDR_AL_DMA_HINT_L4); rss_en_mask = fbnic_flow_hash_2_rss_en_mask(fbn, flow_type); diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_txrx.c b/drivers/net/ethernet/meta/fbnic/fbnic_txrx.c index 13d508ce637f10..cb0be88427f880 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_txrx.c +++ b/drivers/net/ethernet/meta/fbnic/fbnic_txrx.c @@ -927,7 +927,7 @@ static void fbnic_fill_bdq(struct fbnic_ring *bdq) /* Force DMA writes to flush before writing to tail */ dma_wmb(); - writel(i, bdq->doorbell); + writel(i * FBNIC_BD_FRAG_COUNT, bdq->doorbell); } } @@ -2548,7 +2548,7 @@ static void fbnic_enable_bdq(struct fbnic_ring *hpq, struct fbnic_ring *ppq) hpq->tail = 0; hpq->head = 0; - log_size = fls(hpq->size_mask); + log_size = fls(hpq->size_mask) + ilog2(FBNIC_BD_FRAG_COUNT); /* Store descriptor ring address and size */ fbnic_ring_wr32(hpq, FBNIC_QUEUE_BDQ_HPQ_BAL, lower_32_bits(hpq->dma)); @@ -2560,7 +2560,7 @@ static void fbnic_enable_bdq(struct fbnic_ring *hpq, struct fbnic_ring *ppq) if (!ppq->size_mask) goto write_ctl; - log_size = fls(ppq->size_mask); + log_size = fls(ppq->size_mask) + ilog2(FBNIC_BD_FRAG_COUNT); /* Add enabling of PPQ to BDQ control */ bdq_ctl |= FBNIC_QUEUE_BDQ_CTL_PPQ_ENABLE; @@ -2575,7 +2575,8 @@ static void fbnic_enable_bdq(struct fbnic_ring *hpq, struct fbnic_ring *ppq) } static void fbnic_config_drop_mode_rcq(struct fbnic_napi_vector *nv, - struct fbnic_ring *rcq, bool tx_pause) + struct fbnic_ring *rcq, bool tx_pause, + bool hdr_split) { struct fbnic_net *fbn = netdev_priv(nv->napi.dev); u32 drop_mode, rcq_ctl; @@ -2588,22 +2589,26 @@ static void fbnic_config_drop_mode_rcq(struct fbnic_napi_vector *nv, /* Specify packet layout */ rcq_ctl = FIELD_PREP(FBNIC_QUEUE_RDE_CTL0_DROP_MODE_MASK, drop_mode) | FIELD_PREP(FBNIC_QUEUE_RDE_CTL0_MIN_HROOM_MASK, FBNIC_RX_HROOM) | - FIELD_PREP(FBNIC_QUEUE_RDE_CTL0_MIN_TROOM_MASK, FBNIC_RX_TROOM); + FIELD_PREP(FBNIC_QUEUE_RDE_CTL0_MIN_TROOM_MASK, FBNIC_RX_TROOM) | + FIELD_PREP(FBNIC_QUEUE_RDE_CTL0_EN_HDR_SPLIT, hdr_split); fbnic_ring_wr32(rcq, FBNIC_QUEUE_RDE_CTL0, rcq_ctl); } -void fbnic_config_drop_mode(struct fbnic_net *fbn, bool tx_pause) +void fbnic_config_drop_mode(struct fbnic_net *fbn, bool txp) { + bool hds; int i, t; + hds = fbn->hds_thresh < FBNIC_HDR_BYTES_MIN; + for (i = 0; i < fbn->num_napi; i++) { struct fbnic_napi_vector *nv = fbn->napi[i]; for (t = 0; t < nv->rxt_count; t++) { struct fbnic_q_triad *qt = &nv->qt[nv->txt_count + t]; - fbnic_config_drop_mode_rcq(nv, &qt->cmpl, tx_pause); + fbnic_config_drop_mode_rcq(nv, &qt->cmpl, txp, hds); } } } @@ -2654,20 +2659,18 @@ static void fbnic_enable_rcq(struct fbnic_napi_vector *nv, { struct fbnic_net *fbn = netdev_priv(nv->napi.dev); u32 log_size = fls(rcq->size_mask); - u32 hds_thresh = fbn->hds_thresh; u32 rcq_ctl = 0; - - fbnic_config_drop_mode_rcq(nv, rcq, fbn->tx_pause); + bool hdr_split; + u32 hds_thresh; /* Force lower bound on MAX_HEADER_BYTES. Below this, all frames should * be split at L4. It would also result in the frames being split at * L2/L3 depending on the frame size. */ - if (fbn->hds_thresh < FBNIC_HDR_BYTES_MIN) { - rcq_ctl = FBNIC_QUEUE_RDE_CTL0_EN_HDR_SPLIT; - hds_thresh = FBNIC_HDR_BYTES_MIN; - } + hdr_split = fbn->hds_thresh < FBNIC_HDR_BYTES_MIN; + fbnic_config_drop_mode_rcq(nv, rcq, fbn->tx_pause, hdr_split); + hds_thresh = max(fbn->hds_thresh, FBNIC_HDR_BYTES_MIN); rcq_ctl |= FIELD_PREP(FBNIC_QUEUE_RDE_CTL1_PADLEN_MASK, FBNIC_RX_PAD) | FIELD_PREP(FBNIC_QUEUE_RDE_CTL1_MAX_HDR_MASK, hds_thresh) | FIELD_PREP(FBNIC_QUEUE_RDE_CTL1_PAYLD_OFF_MASK, diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_txrx.h b/drivers/net/ethernet/meta/fbnic/fbnic_txrx.h index 27776e844e29bf..f2ee2cbf3486b0 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_txrx.h +++ b/drivers/net/ethernet/meta/fbnic/fbnic_txrx.h @@ -38,7 +38,7 @@ struct fbnic_net; #define FBNIC_MAX_XDPQS 128u /* These apply to TWQs, TCQ, RCQ */ -#define FBNIC_QUEUE_SIZE_MIN 16u +#define FBNIC_QUEUE_SIZE_MIN 64u #define FBNIC_QUEUE_SIZE_MAX SZ_64K #define FBNIC_TXQ_SIZE_DEFAULT 1024 @@ -66,7 +66,7 @@ struct fbnic_net; (4096 - FBNIC_RX_HROOM - FBNIC_RX_TROOM - FBNIC_RX_PAD) #define FBNIC_HDS_THRESH_DEFAULT \ (1536 - FBNIC_RX_PAD) -#define FBNIC_HDR_BYTES_MIN 128 +#define FBNIC_HDR_BYTES_MIN 256 struct fbnic_pkt_buff { struct xdp_buff buff; diff --git a/drivers/net/ethernet/microchip/lan743x_main.c b/drivers/net/ethernet/microchip/lan743x_main.c index e4c542fc6c2b87..09d255e78f6cd1 100644 --- a/drivers/net/ethernet/microchip/lan743x_main.c +++ b/drivers/net/ethernet/microchip/lan743x_main.c @@ -3054,6 +3054,11 @@ static void lan743x_phylink_mac_link_up(struct phylink_config *config, else if (speed == SPEED_100) mac_cr |= MAC_CR_CFG_L_; + if (duplex == DUPLEX_FULL) + mac_cr |= MAC_CR_DPX_; + else + mac_cr &= ~MAC_CR_DPX_; + lan743x_csr_write(adapter, MAC_CR, mac_cr); lan743x_ptp_update_latency(adapter, speed); diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_fdma.c b/drivers/net/ethernet/microchip/lan966x/lan966x_fdma.c index 50267071810429..646f3d65274e3c 100644 --- a/drivers/net/ethernet/microchip/lan966x/lan966x_fdma.c +++ b/drivers/net/ethernet/microchip/lan966x/lan966x_fdma.c @@ -91,6 +91,8 @@ static int lan966x_fdma_rx_alloc_page_pool(struct lan966x_rx *rx) pp_params.dma_dir = DMA_BIDIRECTIONAL; rx->page_pool = page_pool_create(&pp_params); + if (unlikely(IS_ERR(rx->page_pool))) + return PTR_ERR(rx->page_pool); for (int i = 0; i < lan966x->num_phys_ports; ++i) { struct lan966x_port *port; @@ -117,8 +119,10 @@ static int lan966x_fdma_rx_alloc(struct lan966x_rx *rx) return PTR_ERR(rx->page_pool); err = fdma_alloc_coherent(lan966x->dev, fdma); - if (err) + if (err) { + page_pool_destroy(rx->page_pool); return err; + } fdma_dcbs_init(fdma, FDMA_DCB_INFO_DATAL(fdma->db_size), FDMA_DCB_STATUS_INTR); @@ -809,9 +813,15 @@ static int lan966x_qsys_sw_status(struct lan966x *lan966x) static int lan966x_fdma_reload(struct lan966x *lan966x, int new_mtu) { + struct page *(*old_pages)[FDMA_RX_DCB_MAX_DBS]; struct page_pool *page_pool; struct fdma fdma_rx_old; - int err; + int err, i, j; + + old_pages = kmemdup(lan966x->rx.page, sizeof(lan966x->rx.page), + GFP_KERNEL); + if (!old_pages) + return -ENOMEM; /* Store these for later to free them */ memcpy(&fdma_rx_old, &lan966x->rx.fdma, sizeof(struct fdma)); @@ -822,7 +832,6 @@ static int lan966x_fdma_reload(struct lan966x *lan966x, int new_mtu) lan966x_fdma_stop_netdev(lan966x); lan966x_fdma_rx_disable(&lan966x->rx); - lan966x_fdma_rx_free_pages(&lan966x->rx); lan966x->rx.page_order = round_up(new_mtu, PAGE_SIZE) / PAGE_SIZE - 1; lan966x->rx.max_mtu = new_mtu; err = lan966x_fdma_rx_alloc(&lan966x->rx); @@ -830,6 +839,11 @@ static int lan966x_fdma_reload(struct lan966x *lan966x, int new_mtu) goto restore; lan966x_fdma_rx_start(&lan966x->rx); + for (i = 0; i < fdma_rx_old.n_dcbs; ++i) + for (j = 0; j < fdma_rx_old.n_dbs; ++j) + page_pool_put_full_page(page_pool, + old_pages[i][j], false); + fdma_free_coherent(lan966x->dev, &fdma_rx_old); page_pool_destroy(page_pool); @@ -837,12 +851,17 @@ static int lan966x_fdma_reload(struct lan966x *lan966x, int new_mtu) lan966x_fdma_wakeup_netdev(lan966x); napi_enable(&lan966x->napi); - return err; + kfree(old_pages); + return 0; restore: lan966x->rx.page_pool = page_pool; memcpy(&lan966x->rx.fdma, &fdma_rx_old, sizeof(struct fdma)); lan966x_fdma_rx_start(&lan966x->rx); + lan966x_fdma_wakeup_netdev(lan966x); + napi_enable(&lan966x->napi); + + kfree(old_pages); return err; } @@ -956,6 +975,7 @@ int lan966x_fdma_init(struct lan966x *lan966x) err = lan966x_fdma_tx_alloc(&lan966x->tx); if (err) { fdma_free_coherent(lan966x->dev, &lan966x->rx.fdma); + page_pool_destroy(lan966x->rx.page_pool); return err; } diff --git a/drivers/net/ethernet/microchip/sparx5/sparx5_ptp.c b/drivers/net/ethernet/microchip/sparx5/sparx5_ptp.c index 2f168700f63c11..8b2e07821a9505 100644 --- a/drivers/net/ethernet/microchip/sparx5/sparx5_ptp.c +++ b/drivers/net/ethernet/microchip/sparx5/sparx5_ptp.c @@ -576,7 +576,7 @@ static int sparx5_ptp_adjtime(struct ptp_clock_info *ptp, s64 delta) static struct ptp_clock_info sparx5_ptp_clock_info = { .owner = THIS_MODULE, .name = "sparx5 ptp", - .max_adj = 200000, + .max_adj = 10000000, .gettime64 = sparx5_ptp_gettime64, .settime64 = sparx5_ptp_settime64, .adjtime = sparx5_ptp_adjtime, diff --git a/drivers/net/ethernet/microchip/sparx5/sparx5_qos.h b/drivers/net/ethernet/microchip/sparx5/sparx5_qos.h index 1231a80335d7b1..04f76f1e23f60a 100644 --- a/drivers/net/ethernet/microchip/sparx5/sparx5_qos.h +++ b/drivers/net/ethernet/microchip/sparx5/sparx5_qos.h @@ -35,7 +35,7 @@ #define SPX5_SE_BURST_UNIT 4096 /* Dwrr */ -#define SPX5_DWRR_COST_MAX 63 +#define SPX5_DWRR_COST_MAX 31 struct sparx5_shaper { u32 mode; diff --git a/drivers/net/ethernet/microsoft/mana/gdma_main.c b/drivers/net/ethernet/microsoft/mana/gdma_main.c index 0055c231acf6d5..cbea0ea242c269 100644 --- a/drivers/net/ethernet/microsoft/mana/gdma_main.c +++ b/drivers/net/ethernet/microsoft/mana/gdma_main.c @@ -1934,6 +1934,7 @@ static int mana_gd_setup(struct pci_dev *pdev) mana_gd_remove_irqs(pdev); free_workqueue: destroy_workqueue(gc->service_wq); + gc->service_wq = NULL; dev_err(&pdev->dev, "%s failed (error %d)\n", __func__, err); return err; } @@ -1946,7 +1947,10 @@ static void mana_gd_cleanup(struct pci_dev *pdev) mana_gd_remove_irqs(pdev); - destroy_workqueue(gc->service_wq); + if (gc->service_wq) { + destroy_workqueue(gc->service_wq); + gc->service_wq = NULL; + } dev_dbg(&pdev->dev, "mana gdma cleanup successful\n"); } diff --git a/drivers/net/ethernet/microsoft/mana/hw_channel.c b/drivers/net/ethernet/microsoft/mana/hw_channel.c index aa4e2731e2ba7a..840c6b8957c904 100644 --- a/drivers/net/ethernet/microsoft/mana/hw_channel.c +++ b/drivers/net/ethernet/microsoft/mana/hw_channel.c @@ -814,9 +814,6 @@ void mana_hwc_destroy_channel(struct gdma_context *gc) gc->max_num_cqs = 0; } - kfree(hwc->caller_ctx); - hwc->caller_ctx = NULL; - if (hwc->txq) mana_hwc_destroy_wq(hwc, hwc->txq); @@ -826,6 +823,9 @@ void mana_hwc_destroy_channel(struct gdma_context *gc) if (hwc->cq) mana_hwc_destroy_cq(hwc->gdma_dev->gdma_context, hwc->cq); + kfree(hwc->caller_ctx); + hwc->caller_ctx = NULL; + mana_gd_free_res_map(&hwc->inflight_msg_res); hwc->num_inflight_msg = 0; diff --git a/drivers/net/ethernet/microsoft/mana/mana_en.c b/drivers/net/ethernet/microsoft/mana/mana_en.c index 1ad154f9db1adc..215ce3443adea0 100644 --- a/drivers/net/ethernet/microsoft/mana/mana_en.c +++ b/drivers/net/ethernet/microsoft/mana/mana_en.c @@ -737,6 +737,13 @@ static void mana_get_rxbuf_cfg(struct mana_port_context *apc, } *frag_count = 1; + + /* In the single-buffer path, napi_build_skb() must see the + * actual backing allocation size so skb->truesize reflects + * the full page (or higher-order page), not just the usable + * packet area. + */ + *alloc_size = PAGE_SIZE << get_order(*alloc_size); return; } @@ -1725,8 +1732,14 @@ static void mana_poll_tx_cq(struct mana_cq *cq) ndev = txq->ndev; apc = netdev_priv(ndev); + /* Limit CQEs polled to 4 wraparounds of the CQ to ensure the + * doorbell can be rung in time for the hardware's requirement + * of at least one doorbell ring every 8 wraparounds. + */ comp_read = mana_gd_poll_cq(cq->gdma_cq, completions, - CQE_POLLING_BUFFER); + min((cq->gdma_cq->queue_size / + COMP_ENTRY_SIZE) * 4, + CQE_POLLING_BUFFER)); if (comp_read < 1) return; @@ -2111,7 +2124,14 @@ static void mana_poll_rx_cq(struct mana_cq *cq) struct mana_rxq *rxq = cq->rxq; int comp_read, i; - comp_read = mana_gd_poll_cq(cq->gdma_cq, comp, CQE_POLLING_BUFFER); + /* Limit CQEs polled to 4 wraparounds of the CQ to ensure the + * doorbell can be rung in time for the hardware's requirement + * of at least one doorbell ring every 8 wraparounds. + */ + comp_read = mana_gd_poll_cq(cq->gdma_cq, comp, + min((cq->gdma_cq->queue_size / + COMP_ENTRY_SIZE) * 4, + CQE_POLLING_BUFFER)); WARN_ON_ONCE(comp_read > CQE_POLLING_BUFFER); rxq->xdp_flush = false; @@ -2156,11 +2176,11 @@ static int mana_cq_handler(void *context, struct gdma_queue *gdma_queue) mana_gd_ring_cq(gdma_queue, SET_ARM_BIT); cq->work_done_since_doorbell = 0; napi_complete_done(&cq->napi, w); - } else if (cq->work_done_since_doorbell > - cq->gdma_cq->queue_size / COMP_ENTRY_SIZE * 4) { + } else if (cq->work_done_since_doorbell >= + (cq->gdma_cq->queue_size / COMP_ENTRY_SIZE) * 4) { /* MANA hardware requires at least one doorbell ring every 8 * wraparounds of CQ even if there is no need to arm the CQ. - * This driver rings the doorbell as soon as we have exceeded + * This driver rings the doorbell as soon as it has processed * 4 wraparounds. */ mana_gd_ring_cq(gdma_queue, 0); @@ -3363,6 +3383,7 @@ static int add_adev(struct gdma_dev *gd, const char *name) struct auxiliary_device *adev; struct mana_adev *madev; int ret; + int id; madev = kzalloc(sizeof(*madev), GFP_KERNEL); if (!madev) @@ -3372,7 +3393,8 @@ static int add_adev(struct gdma_dev *gd, const char *name) ret = mana_adev_idx_alloc(); if (ret < 0) goto idx_fail; - adev->id = ret; + id = ret; + adev->id = id; adev->name = name; adev->dev.parent = gd->gdma_context->dev; @@ -3398,7 +3420,7 @@ static int add_adev(struct gdma_dev *gd, const char *name) auxiliary_device_uninit(adev); init_fail: - mana_adev_idx_free(adev->id); + mana_adev_idx_free(id); idx_fail: kfree(madev); @@ -3690,7 +3712,9 @@ void mana_rdma_remove(struct gdma_dev *gd) } WRITE_ONCE(gd->rdma_teardown, true); - flush_workqueue(gc->service_wq); + + if (gc->service_wq) + flush_workqueue(gc->service_wq); if (gd->adev) remove_adev(gd); diff --git a/drivers/net/ethernet/mscc/ocelot_net.c b/drivers/net/ethernet/mscc/ocelot_net.c index 469784d3a1a67b..1b82693204640d 100644 --- a/drivers/net/ethernet/mscc/ocelot_net.c +++ b/drivers/net/ethernet/mscc/ocelot_net.c @@ -551,44 +551,81 @@ static int ocelot_port_stop(struct net_device *dev) return 0; } -static netdev_tx_t ocelot_port_xmit(struct sk_buff *skb, struct net_device *dev) +static bool ocelot_xmit_timestamp(struct ocelot *ocelot, int port, + struct sk_buff *skb, u32 *rew_op) { - struct ocelot_port_private *priv = netdev_priv(dev); - struct ocelot_port *ocelot_port = &priv->port; - struct ocelot *ocelot = ocelot_port->ocelot; - int port = priv->port.index; - u32 rew_op = 0; - - if (!static_branch_unlikely(&ocelot_fdma_enabled) && - !ocelot_can_inject(ocelot, 0)) - return NETDEV_TX_BUSY; - - /* Check if timestamping is needed */ if (ocelot->ptp && (skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) { struct sk_buff *clone = NULL; if (ocelot_port_txtstamp_request(ocelot, port, skb, &clone)) { kfree_skb(skb); - return NETDEV_TX_OK; + return false; } if (clone) OCELOT_SKB_CB(skb)->clone = clone; - rew_op = ocelot_ptp_rew_op(skb); + *rew_op = ocelot_ptp_rew_op(skb); } - if (static_branch_unlikely(&ocelot_fdma_enabled)) { - ocelot_fdma_inject_frame(ocelot, port, rew_op, skb, dev); - } else { - ocelot_port_inject_frame(ocelot, port, 0, rew_op, skb); + return true; +} + +static netdev_tx_t ocelot_port_xmit_fdma(struct sk_buff *skb, + struct net_device *dev) +{ + struct ocelot_port_private *priv = netdev_priv(dev); + struct ocelot_port *ocelot_port = &priv->port; + struct ocelot *ocelot = ocelot_port->ocelot; + int port = priv->port.index; + u32 rew_op = 0; + + if (!ocelot_xmit_timestamp(ocelot, port, skb, &rew_op)) + return NETDEV_TX_OK; + + ocelot_fdma_inject_frame(ocelot, port, rew_op, skb, dev); + + return NETDEV_TX_OK; +} - consume_skb(skb); +static netdev_tx_t ocelot_port_xmit_inj(struct sk_buff *skb, + struct net_device *dev) +{ + struct ocelot_port_private *priv = netdev_priv(dev); + struct ocelot_port *ocelot_port = &priv->port; + struct ocelot *ocelot = ocelot_port->ocelot; + int port = priv->port.index; + u32 rew_op = 0; + + ocelot_lock_inj_grp(ocelot, 0); + + if (!ocelot_can_inject(ocelot, 0)) { + ocelot_unlock_inj_grp(ocelot, 0); + return NETDEV_TX_BUSY; } + if (!ocelot_xmit_timestamp(ocelot, port, skb, &rew_op)) { + ocelot_unlock_inj_grp(ocelot, 0); + return NETDEV_TX_OK; + } + + ocelot_port_inject_frame(ocelot, port, 0, rew_op, skb); + + ocelot_unlock_inj_grp(ocelot, 0); + + consume_skb(skb); + return NETDEV_TX_OK; } +static netdev_tx_t ocelot_port_xmit(struct sk_buff *skb, struct net_device *dev) +{ + if (static_branch_unlikely(&ocelot_fdma_enabled)) + return ocelot_port_xmit_fdma(skb, dev); + + return ocelot_port_xmit_inj(skb, dev); +} + enum ocelot_action_type { OCELOT_MACT_LEARN, OCELOT_MACT_FORGET, diff --git a/drivers/net/ethernet/myricom/myri10ge/myri10ge.c b/drivers/net/ethernet/myricom/myri10ge/myri10ge.c index 7be30a8df26858..2f0cdbd4e2ac9e 100644 --- a/drivers/net/ethernet/myricom/myri10ge/myri10ge.c +++ b/drivers/net/ethernet/myricom/myri10ge/myri10ge.c @@ -688,6 +688,9 @@ static int myri10ge_get_firmware_capabilities(struct myri10ge_priv *mgp) /* probe for IPv6 TSO support */ mgp->features = NETIF_F_SG | NETIF_F_HW_CSUM | NETIF_F_TSO; + cmd.data0 = 0, + cmd.data1 = 0, + cmd.data2 = 0, status = myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_MAX_TSO6_HDR_SIZE, &cmd, 0); if (status == 0) { @@ -806,6 +809,7 @@ static int myri10ge_update_mac_address(struct myri10ge_priv *mgp, | (addr[2] << 8) | addr[3]); cmd.data1 = ((addr[4] << 8) | (addr[5])); + cmd.data2 = 0; status = myri10ge_send_cmd(mgp, MXGEFW_SET_MAC_ADDRESS, &cmd, 0); return status; @@ -817,6 +821,9 @@ static int myri10ge_change_pause(struct myri10ge_priv *mgp, int pause) int status, ctl; ctl = pause ? MXGEFW_ENABLE_FLOW_CONTROL : MXGEFW_DISABLE_FLOW_CONTROL; + cmd.data0 = 0, + cmd.data1 = 0, + cmd.data2 = 0, status = myri10ge_send_cmd(mgp, ctl, &cmd, 0); if (status) { @@ -834,6 +841,9 @@ myri10ge_change_promisc(struct myri10ge_priv *mgp, int promisc, int atomic) int status, ctl; ctl = promisc ? MXGEFW_ENABLE_PROMISC : MXGEFW_DISABLE_PROMISC; + cmd.data0 = 0; + cmd.data1 = 0; + cmd.data2 = 0; status = myri10ge_send_cmd(mgp, ctl, &cmd, atomic); if (status) netdev_err(mgp->dev, "Failed to set promisc mode\n"); @@ -1946,6 +1956,8 @@ static int myri10ge_allocate_rings(struct myri10ge_slice_state *ss) /* get ring sizes */ slice = ss - mgp->ss; cmd.data0 = slice; + cmd.data1 = 0; + cmd.data2 = 0; status = myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_SEND_RING_SIZE, &cmd, 0); tx_ring_size = cmd.data0; cmd.data0 = slice; @@ -2238,12 +2250,16 @@ static int myri10ge_get_txrx(struct myri10ge_priv *mgp, int slice) status = 0; if (slice == 0 || (mgp->dev->real_num_tx_queues > 1)) { cmd.data0 = slice; + cmd.data1 = 0; + cmd.data2 = 0; status = myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_SEND_OFFSET, &cmd, 0); ss->tx.lanai = (struct mcp_kreq_ether_send __iomem *) (mgp->sram + cmd.data0); } cmd.data0 = slice; + cmd.data1 = 0; + cmd.data2 = 0; status |= myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_SMALL_RX_OFFSET, &cmd, 0); ss->rx_small.lanai = (struct mcp_kreq_ether_recv __iomem *) @@ -2312,6 +2328,7 @@ static int myri10ge_open(struct net_device *dev) if (mgp->num_slices > 1) { cmd.data0 = mgp->num_slices; cmd.data1 = MXGEFW_SLICE_INTR_MODE_ONE_PER_SLICE; + cmd.data2 = 0; if (mgp->dev->real_num_tx_queues > 1) cmd.data1 |= MXGEFW_SLICE_ENABLE_MULTIPLE_TX_QUEUES; status = myri10ge_send_cmd(mgp, MXGEFW_CMD_ENABLE_RSS_QUEUES, @@ -2414,6 +2431,8 @@ static int myri10ge_open(struct net_device *dev) /* now give firmware buffers sizes, and MTU */ cmd.data0 = dev->mtu + ETH_HLEN + VLAN_HLEN; + cmd.data1 = 0; + cmd.data2 = 0; status = myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_MTU, &cmd, 0); cmd.data0 = mgp->small_bytes; status |= @@ -2472,7 +2491,6 @@ static int myri10ge_open(struct net_device *dev) static int myri10ge_close(struct net_device *dev) { struct myri10ge_priv *mgp = netdev_priv(dev); - struct myri10ge_cmd cmd; int status, old_down_cnt; int i; @@ -2491,8 +2509,13 @@ static int myri10ge_close(struct net_device *dev) netif_tx_stop_all_queues(dev); if (mgp->rebooted == 0) { + struct myri10ge_cmd cmd; + old_down_cnt = mgp->down_cnt; mb(); + cmd.data0 = 0; + cmd.data1 = 0; + cmd.data2 = 0; status = myri10ge_send_cmd(mgp, MXGEFW_CMD_ETHERNET_DOWN, &cmd, 0); if (status) @@ -2956,6 +2979,9 @@ static void myri10ge_set_multicast_list(struct net_device *dev) /* Disable multicast filtering */ + cmd.data0 = 0; + cmd.data1 = 0; + cmd.data2 = 0; err = myri10ge_send_cmd(mgp, MXGEFW_ENABLE_ALLMULTI, &cmd, 1); if (err != 0) { netdev_err(dev, "Failed MXGEFW_ENABLE_ALLMULTI, error status: %d\n", diff --git a/drivers/net/ethernet/pensando/ionic/ionic_ethtool.c b/drivers/net/ethernet/pensando/ionic/ionic_ethtool.c index 2d9efadb5d2ae1..347b0aff100b9f 100644 --- a/drivers/net/ethernet/pensando/ionic/ionic_ethtool.c +++ b/drivers/net/ethernet/pensando/ionic/ionic_ethtool.c @@ -263,9 +263,10 @@ static int ionic_get_link_ksettings(struct net_device *netdev, /* This means there's no module plugged in */ break; default: - dev_info(lif->ionic->dev, "unknown xcvr type pid=%d / 0x%x\n", - idev->port_info->status.xcvr.pid, - idev->port_info->status.xcvr.pid); + dev_dbg_ratelimited(lif->ionic->dev, + "unknown xcvr type pid=%d / 0x%x\n", + idev->port_info->status.xcvr.pid, + idev->port_info->status.xcvr.pid); break; } diff --git a/drivers/net/ethernet/pensando/ionic/ionic_lif.c b/drivers/net/ethernet/pensando/ionic/ionic_lif.c index 058eea86e141c4..38a827203a2f7e 100644 --- a/drivers/net/ethernet/pensando/ionic/ionic_lif.c +++ b/drivers/net/ethernet/pensando/ionic/ionic_lif.c @@ -1719,13 +1719,18 @@ static int ionic_set_mac_address(struct net_device *netdev, void *sa) if (ether_addr_equal(netdev->dev_addr, mac)) return 0; - err = ionic_program_mac(lif, mac); - if (err < 0) - return err; + /* Only program macs for virtual functions to avoid losing the permanent + * Mac across warm reset/reboot. + */ + if (lif->ionic->pdev->is_virtfn) { + err = ionic_program_mac(lif, mac); + if (err < 0) + return err; - if (err > 0) - netdev_dbg(netdev, "%s: SET and GET ATTR Mac are not equal-due to old FW running\n", - __func__); + if (err > 0) + netdev_dbg(netdev, "%s: SET and GET ATTR Mac are not equal-due to old FW running\n", + __func__); + } err = eth_prepare_mac_addr_change(netdev, addr); if (err) diff --git a/drivers/net/ethernet/qualcomm/qca_uart.c b/drivers/net/ethernet/qualcomm/qca_uart.c index 37efb1ea9fcd9c..847a5f928e4134 100644 --- a/drivers/net/ethernet/qualcomm/qca_uart.c +++ b/drivers/net/ethernet/qualcomm/qca_uart.c @@ -100,7 +100,7 @@ qca_tty_receive(struct serdev_device *serdev, const u8 *data, size_t count) if (!qca->rx_skb) { netdev_dbg(netdev, "recv: out of RX resources\n"); n_stats->rx_errors++; - return i; + return i + 1; } } } diff --git a/drivers/net/ethernet/renesas/rswitch_l2.c b/drivers/net/ethernet/renesas/rswitch_l2.c index 4a69ec77d69c60..9433cd8adced97 100644 --- a/drivers/net/ethernet/renesas/rswitch_l2.c +++ b/drivers/net/ethernet/renesas/rswitch_l2.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 /* Renesas Ethernet Switch device driver * - * Copyright (C) 2025 Renesas Electronics Corporation + * Copyright (C) 2025 - 2026 Renesas Electronics Corporation */ #include @@ -60,6 +60,7 @@ static void rswitch_update_l2_hw_learning(struct rswitch_private *priv) static void rswitch_update_l2_hw_forwarding(struct rswitch_private *priv) { struct rswitch_device *rdev; + bool new_forwarding_offload; unsigned int fwd_mask; /* calculate fwd_mask with zeroes in bits corresponding to ports that @@ -73,8 +74,9 @@ static void rswitch_update_l2_hw_forwarding(struct rswitch_private *priv) } rswitch_for_all_ports(priv, rdev) { - if ((rdev_for_l2_offload(rdev) && rdev->forwarding_requested) || - rdev->forwarding_offloaded) { + new_forwarding_offload = (rdev_for_l2_offload(rdev) && rdev->forwarding_requested); + + if (new_forwarding_offload || rdev->forwarding_offloaded) { /* Update allowed offload destinations even for ports * with L2 offload enabled earlier. * @@ -84,13 +86,10 @@ static void rswitch_update_l2_hw_forwarding(struct rswitch_private *priv) priv->addr + FWPC2(rdev->port)); } - if (rdev_for_l2_offload(rdev) && - rdev->forwarding_requested && - !rdev->forwarding_offloaded) { + if (new_forwarding_offload && !rdev->forwarding_offloaded) rswitch_change_l2_hw_offloading(rdev, true, false); - } else if (rdev->forwarding_offloaded) { + else if (!new_forwarding_offload && rdev->forwarding_offloaded) rswitch_change_l2_hw_offloading(rdev, false, false); - } } } diff --git a/drivers/net/ethernet/spacemit/k1_emac.c b/drivers/net/ethernet/spacemit/k1_emac.c index b49c4708bf9eb1..d64ca7bbda9ea4 100644 --- a/drivers/net/ethernet/spacemit/k1_emac.c +++ b/drivers/net/ethernet/spacemit/k1_emac.c @@ -582,7 +582,9 @@ static void emac_alloc_rx_desc_buffers(struct emac_priv *priv) DMA_FROM_DEVICE); if (dma_mapping_error(&priv->pdev->dev, rx_buf->dma_addr)) { dev_err_ratelimited(&ndev->dev, "Mapping skb failed\n"); - goto err_free_skb; + dev_kfree_skb_any(skb); + rx_buf->skb = NULL; + break; } rx_desc_addr = &((struct emac_desc *)rx_ring->desc_addr)[i]; @@ -607,10 +609,6 @@ static void emac_alloc_rx_desc_buffers(struct emac_priv *priv) rx_ring->head = i; return; - -err_free_skb: - dev_kfree_skb_any(skb); - rx_buf->skb = NULL; } /* Returns number of packets received */ @@ -752,7 +750,7 @@ static void emac_tx_mem_map(struct emac_priv *priv, struct sk_buff *skb) struct emac_desc tx_desc, *tx_desc_addr; struct device *dev = &priv->pdev->dev; struct emac_tx_desc_buffer *tx_buf; - u32 head, old_head, frag_num, f; + u32 head, old_head, frag_num, f, i; bool buf_idx; frag_num = skb_shinfo(skb)->nr_frags; @@ -820,6 +818,15 @@ static void emac_tx_mem_map(struct emac_priv *priv, struct sk_buff *skb) err_free_skb: dev_dstats_tx_dropped(priv->ndev); + + i = old_head; + while (i != head) { + emac_free_tx_buf(priv, i); + + if (++i == tx_ring->total_cnt) + i = 0; + } + dev_kfree_skb_any(skb); } diff --git a/drivers/net/ethernet/stmicro/stmmac/chain_mode.c b/drivers/net/ethernet/stmicro/stmmac/chain_mode.c index 120a009c999299..37f9417c7c0e04 100644 --- a/drivers/net/ethernet/stmicro/stmmac/chain_mode.c +++ b/drivers/net/ethernet/stmicro/stmmac/chain_mode.c @@ -20,7 +20,7 @@ static int jumbo_frm(struct stmmac_tx_queue *tx_q, struct sk_buff *skb, unsigned int nopaged_len = skb_headlen(skb); struct stmmac_priv *priv = tx_q->priv_data; unsigned int entry = tx_q->cur_tx; - unsigned int bmax, des2; + unsigned int bmax, buf_len, des2; unsigned int i = 1, len; struct dma_desc *desc; @@ -31,17 +31,18 @@ static int jumbo_frm(struct stmmac_tx_queue *tx_q, struct sk_buff *skb, else bmax = BUF_SIZE_2KiB; - len = nopaged_len - bmax; + buf_len = min_t(unsigned int, nopaged_len, bmax); + len = nopaged_len - buf_len; des2 = dma_map_single(priv->device, skb->data, - bmax, DMA_TO_DEVICE); + buf_len, DMA_TO_DEVICE); desc->des2 = cpu_to_le32(des2); if (dma_mapping_error(priv->device, des2)) return -1; tx_q->tx_skbuff_dma[entry].buf = des2; - tx_q->tx_skbuff_dma[entry].len = bmax; + tx_q->tx_skbuff_dma[entry].len = buf_len; /* do not close the descriptor and do not set own bit */ - stmmac_prepare_tx_desc(priv, desc, 1, bmax, csum, STMMAC_CHAIN_MODE, + stmmac_prepare_tx_desc(priv, desc, 1, buf_len, csum, STMMAC_CHAIN_MODE, 0, false, skb->len); while (len != 0) { diff --git a/drivers/net/ethernet/stmicro/stmmac/common.h b/drivers/net/ethernet/stmicro/stmmac/common.h index 49df46be366997..9ebaddffa5b25f 100644 --- a/drivers/net/ethernet/stmicro/stmmac/common.h +++ b/drivers/net/ethernet/stmicro/stmmac/common.h @@ -390,7 +390,6 @@ enum request_irq_err { REQ_IRQ_ERR_SFTY, REQ_IRQ_ERR_SFTY_UE, REQ_IRQ_ERR_SFTY_CE, - REQ_IRQ_ERR_LPI, REQ_IRQ_ERR_WOL, REQ_IRQ_ERR_MAC, REQ_IRQ_ERR_NO, diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-intel.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-intel.c index aad1be1ec4c11c..92d77b0c2f54b6 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-intel.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-intel.c @@ -719,7 +719,6 @@ static int intel_mgbe_common_data(struct pci_dev *pdev, /* Setup MSI vector offset specific to Intel mGbE controller */ plat->msi_mac_vec = 29; - plat->msi_lpi_vec = 28; plat->msi_sfty_ce_vec = 27; plat->msi_sfty_ue_vec = 26; plat->msi_rx_base_vec = 0; @@ -1177,8 +1176,6 @@ static int stmmac_config_multi_msi(struct pci_dev *pdev, res->irq = pci_irq_vector(pdev, plat->msi_mac_vec); if (plat->msi_wol_vec < STMMAC_MSI_VEC_MAX) res->wol_irq = pci_irq_vector(pdev, plat->msi_wol_vec); - if (plat->msi_lpi_vec < STMMAC_MSI_VEC_MAX) - res->lpi_irq = pci_irq_vector(pdev, plat->msi_lpi_vec); if (plat->msi_sfty_ce_vec < STMMAC_MSI_VEC_MAX) res->sfty_ce_irq = pci_irq_vector(pdev, plat->msi_sfty_ce_vec); if (plat->msi_sfty_ue_vec < STMMAC_MSI_VEC_MAX) @@ -1294,7 +1291,6 @@ static int intel_eth_pci_probe(struct pci_dev *pdev, */ plat->msi_mac_vec = STMMAC_MSI_VEC_MAX; plat->msi_wol_vec = STMMAC_MSI_VEC_MAX; - plat->msi_lpi_vec = STMMAC_MSI_VEC_MAX; plat->msi_sfty_ce_vec = STMMAC_MSI_VEC_MAX; plat->msi_sfty_ue_vec = STMMAC_MSI_VEC_MAX; plat->msi_rx_base_vec = STMMAC_MSI_VEC_MAX; diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-loongson.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-loongson.c index 107a7c84ace803..a5203101268ba5 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-loongson.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-loongson.c @@ -91,8 +91,8 @@ static void loongson_default_data(struct pci_dev *pdev, /* Get bus_id, this can be overwritten later */ plat->bus_id = pci_dev_id(pdev); - /* clk_csr_i = 20-35MHz & MDC = clk_csr_i/16 */ - plat->clk_csr = STMMAC_CSR_20_35M; + /* clk_csr_i = 100-150MHz & MDC = clk_csr_i/62 */ + plat->clk_csr = STMMAC_CSR_100_150M; plat->core_type = DWMAC_CORE_GMAC; plat->force_sf_dma_mode = 1; @@ -443,13 +443,6 @@ static int loongson_dwmac_dt_config(struct pci_dev *pdev, res->wol_irq = res->irq; } - res->lpi_irq = of_irq_get_byname(np, "eth_lpi"); - if (res->lpi_irq < 0) { - dev_err(&pdev->dev, "IRQ eth_lpi not found\n"); - ret = -ENODEV; - goto err_put_node; - } - ret = device_get_phy_mode(&pdev->dev); if (ret < 0) { dev_err(&pdev->dev, "phy_mode not found\n"); diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-tegra.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-tegra.c index d765acbe375481..21a0a11fc0118b 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-tegra.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-tegra.c @@ -9,7 +9,7 @@ #include "stmmac_platform.h" static const char *const mgbe_clks[] = { - "rx-pcs", "tx", "tx-pcs", "mac-divider", "mac", "mgbe", "ptp-ref", "mac" + "rx-pcs", "tx", "tx-pcs", "mac-divider", "mac", "mgbe", "ptp_ref", "mac" }; struct tegra_mgbe { @@ -215,6 +215,7 @@ static int tegra_mgbe_probe(struct platform_device *pdev) { struct plat_stmmacenet_data *plat; struct stmmac_resources res; + bool use_legacy_ptp = false; struct tegra_mgbe *mgbe; int irq, err, i; u32 value; @@ -257,9 +258,23 @@ static int tegra_mgbe_probe(struct platform_device *pdev) if (!mgbe->clks) return -ENOMEM; - for (i = 0; i < ARRAY_SIZE(mgbe_clks); i++) + /* Older device-trees use 'ptp-ref' rather than 'ptp_ref'. + * Fall back when the legacy name is present. + */ + if (of_property_match_string(pdev->dev.of_node, "clock-names", + "ptp-ref") >= 0) + use_legacy_ptp = true; + + for (i = 0; i < ARRAY_SIZE(mgbe_clks); i++) { mgbe->clks[i].id = mgbe_clks[i]; + if (use_legacy_ptp && !strcmp(mgbe_clks[i], "ptp_ref")) { + dev_warn(mgbe->dev, + "Device-tree update needed for PTP clock!\n"); + mgbe->clks[i].id = "ptp-ref"; + } + } + err = devm_clk_bulk_get(mgbe->dev, ARRAY_SIZE(mgbe_clks), mgbe->clks); if (err < 0) return err; diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac.h b/drivers/net/ethernet/stmicro/stmmac/stmmac.h index 012b0a477255df..78a18384e5d007 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac.h +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac.h @@ -31,7 +31,6 @@ struct stmmac_resources { void __iomem *addr; u8 mac[ETH_ALEN]; int wol_irq; - int lpi_irq; int irq; int sfty_irq; int sfty_ce_irq; @@ -297,7 +296,6 @@ struct stmmac_priv { int wol_irq; u32 gmii_address_bus_config; struct timer_list eee_ctrl_timer; - int lpi_irq; u32 tx_lpi_timer; bool tx_lpi_clk_stop; bool eee_enabled; @@ -324,6 +322,7 @@ struct stmmac_priv { void __iomem *ptpaddr; void __iomem *estaddr; unsigned long active_vlans[BITS_TO_LONGS(VLAN_N_VID)]; + unsigned int num_double_vlans; int sfty_irq; int sfty_ce_irq; int sfty_ue_irq; diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c index a379221b96a348..cfe47a8e2c6d59 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c @@ -140,6 +140,7 @@ static void stmmac_tx_timer_arm(struct stmmac_priv *priv, u32 queue); static void stmmac_flush_tx_descriptors(struct stmmac_priv *priv, int queue); static void stmmac_set_dma_operation_mode(struct stmmac_priv *priv, u32 txmode, u32 rxmode, u32 chan); +static void stmmac_vlan_restore(struct stmmac_priv *priv); #ifdef CONFIG_DEBUG_FS static const struct net_device_ops stmmac_netdev_ops; @@ -3712,10 +3713,6 @@ static void stmmac_free_irq(struct net_device *dev, free_irq(priv->sfty_ce_irq, dev); fallthrough; case REQ_IRQ_ERR_SFTY_CE: - if (priv->lpi_irq > 0 && priv->lpi_irq != dev->irq) - free_irq(priv->lpi_irq, dev); - fallthrough; - case REQ_IRQ_ERR_LPI: if (priv->wol_irq > 0 && priv->wol_irq != dev->irq) free_irq(priv->wol_irq, dev); fallthrough; @@ -3773,24 +3770,6 @@ static int stmmac_request_irq_multi_msi(struct net_device *dev) } } - /* Request the LPI IRQ in case of another line - * is used for LPI - */ - if (priv->lpi_irq > 0 && priv->lpi_irq != dev->irq) { - int_name = priv->int_name_lpi; - sprintf(int_name, "%s:%s", dev->name, "lpi"); - ret = request_irq(priv->lpi_irq, - stmmac_mac_interrupt, - 0, int_name, dev); - if (unlikely(ret < 0)) { - netdev_err(priv->dev, - "%s: alloc lpi MSI %d (error: %d)\n", - __func__, priv->lpi_irq, ret); - irq_err = REQ_IRQ_ERR_LPI; - goto irq_error; - } - } - /* Request the common Safety Feature Correctible/Uncorrectible * Error line in case of another line is used */ @@ -3930,19 +3909,6 @@ static int stmmac_request_irq_single(struct net_device *dev) } } - /* Request the IRQ lines */ - if (priv->lpi_irq > 0 && priv->lpi_irq != dev->irq) { - ret = request_irq(priv->lpi_irq, stmmac_interrupt, - IRQF_SHARED, dev->name, dev); - if (unlikely(ret < 0)) { - netdev_err(priv->dev, - "%s: ERROR: allocating the LPI IRQ %d (%d)\n", - __func__, priv->lpi_irq, ret); - irq_err = REQ_IRQ_ERR_LPI; - goto irq_error; - } - } - /* Request the common Safety Feature Correctible/Uncorrectible * Error line in case of another line is used */ @@ -4099,6 +4065,8 @@ static int __stmmac_open(struct net_device *dev, phylink_start(priv->phylink); + stmmac_vlan_restore(priv); + ret = stmmac_request_irq(dev); if (ret) goto irq_error; @@ -5023,13 +4991,27 @@ static unsigned int stmmac_rx_buf2_len(struct stmmac_priv *priv, if (!priv->sph_active) return 0; - /* Not last descriptor */ - if (status & rx_not_ls) + /* For GMAC4, when split header is enabled, in some rare cases, the + * hardware does not fill buf2 of the first descriptor with payload. + * Thus we cannot assume buf2 is always fully filled if it is not + * the last descriptor. Otherwise, the length of buf2 of the second + * descriptor will be calculated wrong and cause an oops. + * + * If this is the last descriptor, 'plen' is the length of the + * received packet that was transferred to system memory. + * Otherwise, it is the accumulated number of bytes that have been + * transferred for the current packet. + * + * Thus 'plen - len' always gives the correct length of buf2. + */ + + /* Not GMAC4 and not last descriptor */ + if (priv->plat->core_type != DWMAC_CORE_GMAC4 && (status & rx_not_ls)) return priv->dma_conf.dma_buf_sz; + /* GMAC4 or last descriptor */ plen = stmmac_get_rx_frame_len(priv, p, coe); - /* Last descriptor */ return plen - len; } @@ -6739,6 +6721,9 @@ static int stmmac_vlan_update(struct stmmac_priv *priv, bool is_double) hash = 0; } + if (!netif_running(priv->dev)) + return 0; + return stmmac_update_vlan_hash(priv, priv->hw, hash, pmatch, is_double); } @@ -6748,6 +6733,7 @@ static int stmmac_vlan_update(struct stmmac_priv *priv, bool is_double) static int stmmac_vlan_rx_add_vid(struct net_device *ndev, __be16 proto, u16 vid) { struct stmmac_priv *priv = netdev_priv(ndev); + unsigned int num_double_vlans; bool is_double = false; int ret; @@ -6759,7 +6745,8 @@ static int stmmac_vlan_rx_add_vid(struct net_device *ndev, __be16 proto, u16 vid is_double = true; set_bit(vid, priv->active_vlans); - ret = stmmac_vlan_update(priv, is_double); + num_double_vlans = priv->num_double_vlans + is_double; + ret = stmmac_vlan_update(priv, num_double_vlans); if (ret) { clear_bit(vid, priv->active_vlans); goto err_pm_put; @@ -6767,9 +6754,15 @@ static int stmmac_vlan_rx_add_vid(struct net_device *ndev, __be16 proto, u16 vid if (priv->hw->num_vlan) { ret = stmmac_add_hw_vlan_rx_fltr(priv, ndev, priv->hw, proto, vid); - if (ret) + if (ret) { + clear_bit(vid, priv->active_vlans); + stmmac_vlan_update(priv, priv->num_double_vlans); goto err_pm_put; + } } + + priv->num_double_vlans = num_double_vlans; + err_pm_put: pm_runtime_put(priv->device); @@ -6782,6 +6775,7 @@ static int stmmac_vlan_rx_add_vid(struct net_device *ndev, __be16 proto, u16 vid static int stmmac_vlan_rx_kill_vid(struct net_device *ndev, __be16 proto, u16 vid) { struct stmmac_priv *priv = netdev_priv(ndev); + unsigned int num_double_vlans; bool is_double = false; int ret; @@ -6793,14 +6787,23 @@ static int stmmac_vlan_rx_kill_vid(struct net_device *ndev, __be16 proto, u16 vi is_double = true; clear_bit(vid, priv->active_vlans); + num_double_vlans = priv->num_double_vlans - is_double; + ret = stmmac_vlan_update(priv, num_double_vlans); + if (ret) { + set_bit(vid, priv->active_vlans); + goto del_vlan_error; + } if (priv->hw->num_vlan) { ret = stmmac_del_hw_vlan_rx_fltr(priv, ndev, priv->hw, proto, vid); - if (ret) + if (ret) { + set_bit(vid, priv->active_vlans); + stmmac_vlan_update(priv, priv->num_double_vlans); goto del_vlan_error; + } } - ret = stmmac_vlan_update(priv, is_double); + priv->num_double_vlans = num_double_vlans; del_vlan_error: pm_runtime_put(priv->device); @@ -6808,6 +6811,17 @@ static int stmmac_vlan_rx_kill_vid(struct net_device *ndev, __be16 proto, u16 vi return ret; } +static void stmmac_vlan_restore(struct stmmac_priv *priv) +{ + if (!(priv->dev->features & NETIF_F_VLAN_FEATURES)) + return; + + if (priv->hw->num_vlan) + stmmac_restore_hw_vlan_rx_fltr(priv, priv->dev, priv->hw); + + stmmac_vlan_update(priv, priv->num_double_vlans); +} + static int stmmac_bpf(struct net_device *dev, struct netdev_bpf *bpf) { struct stmmac_priv *priv = netdev_priv(dev); @@ -7695,7 +7709,6 @@ static int __stmmac_dvr_probe(struct device *device, priv->dev->irq = res->irq; priv->wol_irq = res->wol_irq; - priv->lpi_irq = res->lpi_irq; priv->sfty_irq = res->sfty_irq; priv->sfty_ce_irq = res->sfty_ce_irq; priv->sfty_ue_irq = res->sfty_ue_irq; @@ -8201,10 +8214,10 @@ int stmmac_resume(struct device *dev) stmmac_init_coalesce(priv); phylink_rx_clk_stop_block(priv->phylink); stmmac_set_rx_mode(ndev); - - stmmac_restore_hw_vlan_rx_fltr(priv, ndev, priv->hw); phylink_rx_clk_stop_unblock(priv->phylink); + stmmac_vlan_restore(priv); + stmmac_enable_all_queues(priv); stmmac_enable_all_dma_irq(priv); diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_platform.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_platform.c index 8979a50b550707..5c9fd91a1db9d6 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_platform.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_platform.c @@ -725,14 +725,6 @@ int stmmac_get_platform_resources(struct platform_device *pdev, stmmac_res->wol_irq = stmmac_res->irq; } - stmmac_res->lpi_irq = - platform_get_irq_byname_optional(pdev, "eth_lpi"); - if (stmmac_res->lpi_irq < 0) { - if (stmmac_res->lpi_irq == -EPROBE_DEFER) - return -EPROBE_DEFER; - dev_info(&pdev->dev, "IRQ eth_lpi not found\n"); - } - stmmac_res->sfty_irq = platform_get_irq_byname_optional(pdev, "sfty"); if (stmmac_res->sfty_irq < 0) { diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_vlan.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_vlan.c index b18404dd5a8beb..e24efe3bfedbec 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_vlan.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_vlan.c @@ -76,7 +76,9 @@ static int vlan_add_hw_rx_fltr(struct net_device *dev, } hw->vlan_filter[0] = vid; - vlan_write_single(dev, vid); + + if (netif_running(dev)) + vlan_write_single(dev, vid); return 0; } @@ -97,12 +99,15 @@ static int vlan_add_hw_rx_fltr(struct net_device *dev, return -EPERM; } - ret = vlan_write_filter(dev, hw, index, val); + if (netif_running(dev)) { + ret = vlan_write_filter(dev, hw, index, val); + if (ret) + return ret; + } - if (!ret) - hw->vlan_filter[index] = val; + hw->vlan_filter[index] = val; - return ret; + return 0; } static int vlan_del_hw_rx_fltr(struct net_device *dev, @@ -115,7 +120,9 @@ static int vlan_del_hw_rx_fltr(struct net_device *dev, if (hw->num_vlan == 1) { if ((hw->vlan_filter[0] & VLAN_TAG_VID) == vid) { hw->vlan_filter[0] = 0; - vlan_write_single(dev, 0); + + if (netif_running(dev)) + vlan_write_single(dev, 0); } return 0; } @@ -124,25 +131,23 @@ static int vlan_del_hw_rx_fltr(struct net_device *dev, for (i = 0; i < hw->num_vlan; i++) { if ((hw->vlan_filter[i] & VLAN_TAG_DATA_VEN) && ((hw->vlan_filter[i] & VLAN_TAG_DATA_VID) == vid)) { - ret = vlan_write_filter(dev, hw, i, 0); - if (!ret) - hw->vlan_filter[i] = 0; - else - return ret; + if (netif_running(dev)) { + ret = vlan_write_filter(dev, hw, i, 0); + if (ret) + return ret; + } + + hw->vlan_filter[i] = 0; } } - return ret; + return 0; } static void vlan_restore_hw_rx_fltr(struct net_device *dev, struct mac_device_info *hw) { - void __iomem *ioaddr = hw->pcsr; - u32 value; - u32 hash; - u32 val; int i; /* Single Rx VLAN Filter */ @@ -152,19 +157,8 @@ static void vlan_restore_hw_rx_fltr(struct net_device *dev, } /* Extended Rx VLAN Filter Enable */ - for (i = 0; i < hw->num_vlan; i++) { - if (hw->vlan_filter[i] & VLAN_TAG_DATA_VEN) { - val = hw->vlan_filter[i]; - vlan_write_filter(dev, hw, i, val); - } - } - - hash = readl(ioaddr + VLAN_HASH_TABLE); - if (hash & VLAN_VLHT) { - value = readl(ioaddr + VLAN_TAG); - value |= VLAN_VTHM; - writel(value, ioaddr + VLAN_TAG); - } + for (i = 0; i < hw->num_vlan; i++) + vlan_write_filter(dev, hw, i, hw->vlan_filter[i]); } static void vlan_update_hash(struct mac_device_info *hw, u32 hash, @@ -183,6 +177,10 @@ static void vlan_update_hash(struct mac_device_info *hw, u32 hash, value |= VLAN_EDVLP; value |= VLAN_ESVL; value |= VLAN_DOVLTC; + } else { + value &= ~VLAN_EDVLP; + value &= ~VLAN_ESVL; + value &= ~VLAN_DOVLTC; } writel(value, ioaddr + VLAN_TAG); @@ -193,6 +191,10 @@ static void vlan_update_hash(struct mac_device_info *hw, u32 hash, value |= VLAN_EDVLP; value |= VLAN_ESVL; value |= VLAN_DOVLTC; + } else { + value &= ~VLAN_EDVLP; + value &= ~VLAN_ESVL; + value &= ~VLAN_DOVLTC; } writel(value | perfect_match, ioaddr + VLAN_TAG); diff --git a/drivers/net/ethernet/sun/sunhme.c b/drivers/net/ethernet/sun/sunhme.c index 48f0a96c0e9e3a..6669980829980d 100644 --- a/drivers/net/ethernet/sun/sunhme.c +++ b/drivers/net/ethernet/sun/sunhme.c @@ -2551,6 +2551,9 @@ static int happy_meal_sbus_probe_one(struct platform_device *op, int is_qfe) goto err_out_clear_quattro; } + /* BIGMAC may have bogus sizes */ + if ((op->resource[3].end - op->resource[3].start) >= BMAC_REG_SIZE) + op->resource[3].end = op->resource[3].start + BMAC_REG_SIZE - 1; hp->bigmacregs = devm_platform_ioremap_resource(op, 3); if (IS_ERR(hp->bigmacregs)) { dev_err(&op->dev, "Cannot map BIGMAC registers.\n"); diff --git a/drivers/net/ethernet/ti/Kconfig b/drivers/net/ethernet/ti/Kconfig index fe5b2926d8ab06..c60b04921c62ca 100644 --- a/drivers/net/ethernet/ti/Kconfig +++ b/drivers/net/ethernet/ti/Kconfig @@ -192,6 +192,7 @@ config TI_ICSSG_PRUETH depends on NET_SWITCHDEV depends on ARCH_K3 && OF && TI_K3_UDMA_GLUE_LAYER depends on PTP_1588_CLOCK_OPTIONAL + depends on HSR || !HSR help Support dual Gigabit Ethernet ports over the ICSSG PRU Subsystem. This subsystem is available starting with the AM65 platform. diff --git a/drivers/net/ethernet/ti/am65-cpsw-nuss.c b/drivers/net/ethernet/ti/am65-cpsw-nuss.c index 5924db6be3feaf..265ce5479915f8 100644 --- a/drivers/net/ethernet/ti/am65-cpsw-nuss.c +++ b/drivers/net/ethernet/ti/am65-cpsw-nuss.c @@ -391,7 +391,7 @@ static void am65_cpsw_nuss_ndo_slave_set_rx_mode(struct net_device *ndev) cpsw_ale_set_allmulti(common->ale, ndev->flags & IFF_ALLMULTI, port->port_id); - port_mask = ALE_PORT_HOST; + port_mask = BIT(port->port_id) | ALE_PORT_HOST; /* Clear all mcast from ALE */ cpsw_ale_flush_multicast(common->ale, port_mask, -1); @@ -1351,7 +1351,7 @@ static int am65_cpsw_nuss_rx_packets(struct am65_cpsw_rx_flow *flow, ndev_priv = netdev_priv(ndev); am65_cpsw_nuss_set_offload_fwd_mark(skb, ndev_priv->offload_fwd_mark); skb_put(skb, pkt_len); - if (port->rx_ts_enabled) + if (port->rx_ts_filter) am65_cpts_rx_timestamp(common->cpts, skb); skb_mark_for_recycle(skb); skb->protocol = eth_type_trans(skb, ndev); @@ -1811,11 +1811,14 @@ static int am65_cpsw_nuss_hwtstamp_set(struct net_device *ndev, switch (cfg->rx_filter) { case HWTSTAMP_FILTER_NONE: - port->rx_ts_enabled = false; + port->rx_ts_filter = HWTSTAMP_FILTER_NONE; break; case HWTSTAMP_FILTER_PTP_V1_L4_EVENT: case HWTSTAMP_FILTER_PTP_V1_L4_SYNC: case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ: + port->rx_ts_filter = HWTSTAMP_FILTER_PTP_V1_L4_EVENT; + cfg->rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_EVENT; + break; case HWTSTAMP_FILTER_PTP_V2_L4_EVENT: case HWTSTAMP_FILTER_PTP_V2_L4_SYNC: case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ: @@ -1825,8 +1828,8 @@ static int am65_cpsw_nuss_hwtstamp_set(struct net_device *ndev, case HWTSTAMP_FILTER_PTP_V2_EVENT: case HWTSTAMP_FILTER_PTP_V2_SYNC: case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ: - port->rx_ts_enabled = true; - cfg->rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT | HWTSTAMP_FILTER_PTP_V1_L4_EVENT; + port->rx_ts_filter = HWTSTAMP_FILTER_PTP_V2_EVENT; + cfg->rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT; break; case HWTSTAMP_FILTER_ALL: case HWTSTAMP_FILTER_SOME: @@ -1863,7 +1866,7 @@ static int am65_cpsw_nuss_hwtstamp_set(struct net_device *ndev, ts_ctrl |= AM65_CPSW_TS_TX_ANX_ALL_EN | AM65_CPSW_PN_TS_CTL_TX_VLAN_LT1_EN; - if (port->rx_ts_enabled) + if (port->rx_ts_filter) ts_ctrl |= AM65_CPSW_TS_RX_ANX_ALL_EN | AM65_CPSW_PN_TS_CTL_RX_VLAN_LT1_EN; @@ -1888,8 +1891,7 @@ static int am65_cpsw_nuss_hwtstamp_get(struct net_device *ndev, cfg->flags = 0; cfg->tx_type = port->tx_ts_enabled ? HWTSTAMP_TX_ON : HWTSTAMP_TX_OFF; - cfg->rx_filter = port->rx_ts_enabled ? HWTSTAMP_FILTER_PTP_V2_EVENT | - HWTSTAMP_FILTER_PTP_V1_L4_EVENT : HWTSTAMP_FILTER_NONE; + cfg->rx_filter = port->rx_ts_filter; return 0; } diff --git a/drivers/net/ethernet/ti/am65-cpsw-nuss.h b/drivers/net/ethernet/ti/am65-cpsw-nuss.h index 917c37e4e89bd9..7750448e474685 100644 --- a/drivers/net/ethernet/ti/am65-cpsw-nuss.h +++ b/drivers/net/ethernet/ti/am65-cpsw-nuss.h @@ -52,7 +52,7 @@ struct am65_cpsw_port { bool disabled; struct am65_cpsw_slave_data slave; bool tx_ts_enabled; - bool rx_ts_enabled; + enum hwtstamp_rx_filters rx_ts_filter; struct am65_cpsw_qos qos; struct devlink_port devlink_port; struct bpf_prog *xdp_prog; diff --git a/drivers/net/ethernet/ti/cpsw_ale.c b/drivers/net/ethernet/ti/cpsw_ale.c index fbe35af615a6f0..9632ad3741de16 100644 --- a/drivers/net/ethernet/ti/cpsw_ale.c +++ b/drivers/net/ethernet/ti/cpsw_ale.c @@ -455,14 +455,13 @@ static void cpsw_ale_flush_mcast(struct cpsw_ale *ale, u32 *ale_entry, ale->port_mask_bits); if ((mask & port_mask) == 0) return; /* ports dont intersect, not interested */ - mask &= ~port_mask; + mask &= (~port_mask | ALE_PORT_HOST); - /* free if only remaining port is host port */ - if (mask) + if (mask == 0x0 || mask == ALE_PORT_HOST) + cpsw_ale_set_entry_type(ale_entry, ALE_TYPE_FREE); + else cpsw_ale_set_port_mask(ale_entry, mask, ale->port_mask_bits); - else - cpsw_ale_set_entry_type(ale_entry, ALE_TYPE_FREE); } int cpsw_ale_flush_multicast(struct cpsw_ale *ale, int port_mask, int vid) diff --git a/drivers/net/ethernet/ti/cpsw_new.c b/drivers/net/ethernet/ti/cpsw_new.c index 21af0a10626aaf..7f42f58a4b031f 100644 --- a/drivers/net/ethernet/ti/cpsw_new.c +++ b/drivers/net/ethernet/ti/cpsw_new.c @@ -1472,7 +1472,7 @@ static void cpsw_unregister_ports(struct cpsw_common *cpsw) for (i = 0; i < cpsw->data.slaves; i++) { ndev = cpsw->slaves[i].ndev; - if (!ndev) + if (!ndev || ndev->reg_state != NETREG_REGISTERED) continue; priv = netdev_priv(ndev); @@ -1494,7 +1494,6 @@ static int cpsw_register_ports(struct cpsw_common *cpsw) if (ret) { dev_err(cpsw->dev, "cpsw: err registering net device%d\n", i); - cpsw->slaves[i].ndev = NULL; break; } } @@ -2003,7 +2002,7 @@ static int cpsw_probe(struct platform_device *pdev) /* setup netdevs */ ret = cpsw_create_ports(cpsw); if (ret) - goto clean_unregister_netdev; + goto clean_cpts; /* Grab RX and TX IRQs. Note that we also have RX_THRESHOLD and * MISC IRQs which are always kept disabled with this driver so @@ -2017,14 +2016,14 @@ static int cpsw_probe(struct platform_device *pdev) 0, dev_name(dev), cpsw); if (ret < 0) { dev_err(dev, "error attaching irq (%d)\n", ret); - goto clean_unregister_netdev; + goto clean_cpts; } ret = devm_request_irq(dev, cpsw->irqs_table[1], cpsw_tx_interrupt, 0, dev_name(dev), cpsw); if (ret < 0) { dev_err(dev, "error attaching irq (%d)\n", ret); - goto clean_unregister_netdev; + goto clean_cpts; } if (!cpsw->cpts) @@ -2034,7 +2033,7 @@ static int cpsw_probe(struct platform_device *pdev) 0, dev_name(&pdev->dev), cpsw); if (ret < 0) { dev_err(dev, "error attaching misc irq (%d)\n", ret); - goto clean_unregister_netdev; + goto clean_cpts; } /* Enable misc CPTS evnt_pend IRQ */ @@ -2043,7 +2042,7 @@ static int cpsw_probe(struct platform_device *pdev) skip_cpts: ret = cpsw_register_notifiers(cpsw); if (ret) - goto clean_unregister_netdev; + goto clean_cpts; ret = cpsw_register_devlink(cpsw); if (ret) @@ -2065,8 +2064,6 @@ static int cpsw_probe(struct platform_device *pdev) clean_unregister_notifiers: cpsw_unregister_notifiers(cpsw); -clean_unregister_netdev: - cpsw_unregister_ports(cpsw); clean_cpts: cpts_release(cpsw->cpts); cpdma_ctlr_destroy(cpsw->dma); diff --git a/drivers/net/ethernet/ti/icssg/icssg_common.c b/drivers/net/ethernet/ti/icssg/icssg_common.c index 090aa74d3ce725..a74a41ad59c828 100644 --- a/drivers/net/ethernet/ti/icssg/icssg_common.c +++ b/drivers/net/ethernet/ti/icssg/icssg_common.c @@ -902,6 +902,7 @@ static void emac_dispatch_skb_zc(struct prueth_emac *emac, struct xdp_buff *xdp, skb_reserve(skb, headroom); skb_put(skb, pkt_len); + skb_copy_to_linear_data(skb, xdp->data, pkt_len); skb->dev = ndev; /* RX HW timestamp */ @@ -912,7 +913,6 @@ static void emac_dispatch_skb_zc(struct prueth_emac *emac, struct xdp_buff *xdp, skb->offload_fwd_mark = emac->offload_fwd_mark; skb->protocol = eth_type_trans(skb, ndev); - skb_mark_for_recycle(skb); napi_gro_receive(&emac->napi_rx, skb); ndev->stats.rx_bytes += pkt_len; ndev->stats.rx_packets++; @@ -962,7 +962,6 @@ static int emac_rx_packet_zc(struct prueth_emac *emac, u32 flow_id, pkt_len -= 4; cppi5_desc_get_tags_ids(&desc_rx->hdr, &port_id, NULL); psdata = cppi5_hdesc_get_psdata(desc_rx); - k3_cppi_desc_pool_free(rx_chn->desc_pool, desc_rx); count++; xsk_buff_set_size(xdp, pkt_len); xsk_buff_dma_sync_for_cpu(xdp); @@ -988,6 +987,7 @@ static int emac_rx_packet_zc(struct prueth_emac *emac, u32 flow_id, emac_dispatch_skb_zc(emac, xdp, psdata); xsk_buff_free(xdp); } + k3_cppi_desc_pool_free(rx_chn->desc_pool, desc_rx); } if (xdp_status & ICSSG_XDP_REDIR) @@ -1057,7 +1057,6 @@ static int emac_rx_packet(struct prueth_emac *emac, u32 flow_id, u32 *xdp_state) /* firmware adds 4 CRC bytes, strip them */ pkt_len -= 4; cppi5_desc_get_tags_ids(&desc_rx->hdr, &port_id, NULL); - k3_cppi_desc_pool_free(rx_chn->desc_pool, desc_rx); /* if allocation fails we drop the packet but push the * descriptor back to the ring with old page to prevent a stall @@ -1075,6 +1074,11 @@ static int emac_rx_packet(struct prueth_emac *emac, u32 flow_id, u32 *xdp_state) xdp_prepare_buff(&xdp, pa, PRUETH_HEADROOM, pkt_len, false); *xdp_state = emac_run_xdp(emac, &xdp, &pkt_len); + if (*xdp_state == ICSSG_XDP_CONSUMED) { + page_pool_recycle_direct(pool, page); + goto requeue; + } + if (*xdp_state != ICSSG_XDP_PASS) goto requeue; headroom = xdp.data - xdp.data_hard_start; @@ -1110,6 +1114,7 @@ static int emac_rx_packet(struct prueth_emac *emac, u32 flow_id, u32 *xdp_state) ndev->stats.rx_packets++; requeue: + k3_cppi_desc_pool_free(rx_chn->desc_pool, desc_rx); /* queue another RX DMA */ ret = prueth_dma_rx_push_mapped(emac, &emac->rx_chns, new_page, PRUETH_MAX_PKT_SIZE); diff --git a/drivers/net/ethernet/ti/icssg/icssg_prueth.c b/drivers/net/ethernet/ti/icssg/icssg_prueth.c index f65041662173c1..2c6e161225f6ae 100644 --- a/drivers/net/ethernet/ti/icssg/icssg_prueth.c +++ b/drivers/net/ethernet/ti/icssg/icssg_prueth.c @@ -273,6 +273,14 @@ static int prueth_emac_common_start(struct prueth *prueth) if (ret) goto disable_class; + /* Reset link state to force reconfiguration in + * emac_adjust_link(). Without this, if the link was already up + * before restart, emac_adjust_link() won't detect any state + * change and will skip critical configuration like writing + * speed to firmware. + */ + emac->link = 0; + mutex_lock(&emac->ndev->phydev->lock); emac_adjust_link(emac->ndev); mutex_unlock(&emac->ndev->phydev->lock); diff --git a/drivers/net/ethernet/wangxun/txgbe/txgbe_type.h b/drivers/net/ethernet/wangxun/txgbe/txgbe_type.h index 82433e9cb0e33e..6b05f32b4a0109 100644 --- a/drivers/net/ethernet/wangxun/txgbe/txgbe_type.h +++ b/drivers/net/ethernet/wangxun/txgbe/txgbe_type.h @@ -424,10 +424,10 @@ struct txgbe_nodes { char i2c_name[32]; char sfp_name[32]; char phylink_name[32]; - struct property_entry gpio_props[1]; - struct property_entry i2c_props[3]; - struct property_entry sfp_props[8]; - struct property_entry phylink_props[2]; + struct property_entry gpio_props[2]; + struct property_entry i2c_props[4]; + struct property_entry sfp_props[9]; + struct property_entry phylink_props[3]; struct software_node_ref_args i2c_ref[1]; struct software_node_ref_args gpio0_ref[1]; struct software_node_ref_args gpio1_ref[1]; diff --git a/drivers/net/ethernet/xilinx/xilinx_axienet.h b/drivers/net/ethernet/xilinx/xilinx_axienet.h index 5ff742103beb99..fcd3aaef27fc32 100644 --- a/drivers/net/ethernet/xilinx/xilinx_axienet.h +++ b/drivers/net/ethernet/xilinx/xilinx_axienet.h @@ -105,7 +105,7 @@ #define XAXIDMA_BD_HAS_DRE_MASK 0xF00 /* Whether has DRE mask */ #define XAXIDMA_BD_WORDLEN_MASK 0xFF /* Whether has DRE mask */ -#define XAXIDMA_BD_CTRL_LENGTH_MASK 0x007FFFFF /* Requested len */ +#define XAXIDMA_BD_CTRL_LENGTH_MASK GENMASK(25, 0) /* Requested len */ #define XAXIDMA_BD_CTRL_TXSOF_MASK 0x08000000 /* First tx packet */ #define XAXIDMA_BD_CTRL_TXEOF_MASK 0x04000000 /* Last tx packet */ #define XAXIDMA_BD_CTRL_ALL_MASK 0x0C000000 /* All control bits */ @@ -130,7 +130,7 @@ #define XAXIDMA_BD_CTRL_TXEOF_MASK 0x04000000 /* Last tx packet */ #define XAXIDMA_BD_CTRL_ALL_MASK 0x0C000000 /* All control bits */ -#define XAXIDMA_BD_STS_ACTUAL_LEN_MASK 0x007FFFFF /* Actual len */ +#define XAXIDMA_BD_STS_ACTUAL_LEN_MASK GENMASK(25, 0) /* Actual len */ #define XAXIDMA_BD_STS_COMPLETE_MASK 0x80000000 /* Completed */ #define XAXIDMA_BD_STS_DEC_ERR_MASK 0x40000000 /* Decode error */ #define XAXIDMA_BD_STS_SLV_ERR_MASK 0x20000000 /* Slave error */ diff --git a/drivers/net/ethernet/xilinx/xilinx_axienet_main.c b/drivers/net/ethernet/xilinx/xilinx_axienet_main.c index 284031fb2e2c7f..eefe54ce668521 100644 --- a/drivers/net/ethernet/xilinx/xilinx_axienet_main.c +++ b/drivers/net/ethernet/xilinx/xilinx_axienet_main.c @@ -770,8 +770,8 @@ static int axienet_device_reset(struct net_device *ndev) * @first_bd: Index of first descriptor to clean up * @nr_bds: Max number of descriptors to clean up * @force: Whether to clean descriptors even if not complete - * @sizep: Pointer to a u32 filled with the total sum of all bytes - * in all cleaned-up descriptors. Ignored if NULL. + * @sizep: Pointer to a u32 accumulating the total byte count of + * completed packets (using skb->len). Ignored if NULL. * @budget: NAPI budget (use 0 when not called from NAPI poll) * * Would either be called after a successful transmit operation, or after @@ -805,6 +805,8 @@ static int axienet_free_tx_chain(struct axienet_local *lp, u32 first_bd, DMA_TO_DEVICE); if (cur_p->skb && (status & XAXIDMA_BD_STS_COMPLETE_MASK)) { + if (sizep) + *sizep += cur_p->skb->len; napi_consume_skb(cur_p->skb, budget); packets++; } @@ -818,9 +820,6 @@ static int axienet_free_tx_chain(struct axienet_local *lp, u32 first_bd, wmb(); cur_p->cntrl = 0; cur_p->status = 0; - - if (sizep) - *sizep += status & XAXIDMA_BD_STS_ACTUAL_LEN_MASK; } if (!force) { diff --git a/drivers/net/ethernet/xscale/ixp4xx_eth.c b/drivers/net/ethernet/xscale/ixp4xx_eth.c index e1e7f65553e761..b0faa0f1780d09 100644 --- a/drivers/net/ethernet/xscale/ixp4xx_eth.c +++ b/drivers/net/ethernet/xscale/ixp4xx_eth.c @@ -403,15 +403,12 @@ static int ixp4xx_hwtstamp_set(struct net_device *netdev, int ret; int ch; - if (!cpu_is_ixp46x()) - return -EOPNOTSUPP; - if (!netif_running(netdev)) return -EINVAL; ret = ixp46x_ptp_find(&port->timesync_regs, &port->phc_index); if (ret) - return ret; + return -EOPNOTSUPP; ch = PORT2CHANNEL(port); regs = port->timesync_regs; diff --git a/drivers/net/ethernet/xscale/ptp_ixp46x.c b/drivers/net/ethernet/xscale/ptp_ixp46x.c index 94203eb46e6b02..93c64db22a696c 100644 --- a/drivers/net/ethernet/xscale/ptp_ixp46x.c +++ b/drivers/net/ethernet/xscale/ptp_ixp46x.c @@ -232,6 +232,9 @@ static struct ixp_clock ixp_clock; int ixp46x_ptp_find(struct ixp46x_ts_regs *__iomem *regs, int *phc_index) { + if (!cpu_is_ixp46x()) + return -ENODEV; + *regs = ixp_clock.regs; *phc_index = ptp_clock_index(ixp_clock.ptp_clock); diff --git a/drivers/net/ipa/reg/gsi_reg-v5.0.c b/drivers/net/ipa/reg/gsi_reg-v5.0.c index 36d1e65df71bb1..6c4a7fbe4de949 100644 --- a/drivers/net/ipa/reg/gsi_reg-v5.0.c +++ b/drivers/net/ipa/reg/gsi_reg-v5.0.c @@ -30,7 +30,7 @@ REG_STRIDE_FIELDS(CH_C_CNTXT_0, ch_c_cntxt_0, static const u32 reg_ch_c_cntxt_1_fmask[] = { [CH_R_LENGTH] = GENMASK(23, 0), - [ERINDEX] = GENMASK(31, 24), + [CH_ERINDEX] = GENMASK(31, 24), }; REG_STRIDE_FIELDS(CH_C_CNTXT_1, ch_c_cntxt_1, @@ -156,9 +156,10 @@ REG_FIELDS(EV_CH_CMD, ev_ch_cmd, 0x00025010 + 0x12000 * GSI_EE_AP); static const u32 reg_generic_cmd_fmask[] = { [GENERIC_OPCODE] = GENMASK(4, 0), - [GENERIC_CHID] = GENMASK(9, 5), - [GENERIC_EE] = GENMASK(13, 10), - /* Bits 14-31 reserved */ + [GENERIC_CHID] = GENMASK(12, 5), + [GENERIC_EE] = GENMASK(16, 13), + /* Bits 17-23 reserved */ + [GENERIC_PARAMS] = GENMASK(31, 24), }; REG_FIELDS(GENERIC_CMD, generic_cmd, 0x00025018 + 0x12000 * GSI_EE_AP); diff --git a/drivers/net/macvlan.c b/drivers/net/macvlan.c index c509228be84d1b..4433b8e95b6acc 100644 --- a/drivers/net/macvlan.c +++ b/drivers/net/macvlan.c @@ -1572,6 +1572,11 @@ int macvlan_common_newlink(struct net_device *dev, if (create) macvlan_port_destroy(port->dev); } + /* @dev might have been made visible before an error was detected. + * Make sure to observe an RCU grace period before our caller + * (rtnl_newlink()) frees it. + */ + synchronize_net(); return err; } EXPORT_SYMBOL_GPL(macvlan_common_newlink); diff --git a/drivers/net/mctp/mctp-i2c.c b/drivers/net/mctp/mctp-i2c.c index f782d93f826efc..f138b0251313ee 100644 --- a/drivers/net/mctp/mctp-i2c.c +++ b/drivers/net/mctp/mctp-i2c.c @@ -242,6 +242,12 @@ static int mctp_i2c_slave_cb(struct i2c_client *client, return 0; switch (event) { + case I2C_SLAVE_READ_REQUESTED: + case I2C_SLAVE_READ_PROCESSED: + /* MCTP I2C transport only uses writes */ + midev->rx_pos = 0; + *val = 0xff; + break; case I2C_SLAVE_WRITE_RECEIVED: if (midev->rx_pos < MCTP_I2C_BUFSZ) { midev->rx_buffer[midev->rx_pos] = *val; @@ -279,6 +285,9 @@ static int mctp_i2c_recv(struct mctp_i2c_dev *midev) size_t recvlen; int status; + if (midev->rx_pos == 0) + return 0; + /* + 1 for the PEC */ if (midev->rx_pos < MCTP_I2C_MINLEN + 1) { ndev->stats.rx_length_errors++; @@ -334,6 +343,7 @@ static int mctp_i2c_recv(struct mctp_i2c_dev *midev) } else { status = NET_RX_DROP; spin_unlock_irqrestore(&midev->lock, flags); + kfree_skb(skb); } if (status == NET_RX_SUCCESS) { diff --git a/drivers/net/mctp/mctp-usb.c b/drivers/net/mctp/mctp-usb.c index ef860cfc629f65..3b5dff14417747 100644 --- a/drivers/net/mctp/mctp-usb.c +++ b/drivers/net/mctp/mctp-usb.c @@ -329,7 +329,7 @@ static int mctp_usb_probe(struct usb_interface *intf, SET_NETDEV_DEV(netdev, &intf->dev); dev = netdev_priv(netdev); dev->netdev = netdev; - dev->usbdev = usb_get_dev(interface_to_usbdev(intf)); + dev->usbdev = interface_to_usbdev(intf); dev->intf = intf; usb_set_intfdata(intf, dev); @@ -365,7 +365,6 @@ static void mctp_usb_disconnect(struct usb_interface *intf) mctp_unregister_netdev(dev->netdev); usb_free_urb(dev->tx_urb); usb_free_urb(dev->rx_urb); - usb_put_dev(dev->usbdev); free_netdev(dev->netdev); } diff --git a/drivers/net/mdio/mdio-realtek-rtl9300.c b/drivers/net/mdio/mdio-realtek-rtl9300.c index 405a07075dd115..8d5fb014ca06c4 100644 --- a/drivers/net/mdio/mdio-realtek-rtl9300.c +++ b/drivers/net/mdio/mdio-realtek-rtl9300.c @@ -466,7 +466,6 @@ static int rtl9300_mdiobus_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; struct rtl9300_mdio_priv *priv; - struct fwnode_handle *child; int err; priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); @@ -487,7 +486,7 @@ static int rtl9300_mdiobus_probe(struct platform_device *pdev) if (err) return err; - device_for_each_child_node(dev, child) { + device_for_each_child_node_scoped(dev, child) { err = rtl9300_mdiobus_probe_one(dev, priv, child); if (err) return err; diff --git a/drivers/net/netconsole.c b/drivers/net/netconsole.c index 9cb4dfc242f5f7..eb282b295da8b0 100644 --- a/drivers/net/netconsole.c +++ b/drivers/net/netconsole.c @@ -508,7 +508,7 @@ static ssize_t sysdata_release_enabled_show(struct config_item *item, bool release_enabled; mutex_lock(&dynamic_netconsole_mutex); - release_enabled = !!(nt->sysdata_fields & SYSDATA_TASKNAME); + release_enabled = !!(nt->sysdata_fields & SYSDATA_RELEASE); mutex_unlock(&dynamic_netconsole_mutex); return sysfs_emit(buf, "%d\n", release_enabled); @@ -1524,7 +1524,8 @@ static void send_msg_no_fragmentation(struct netconsole_target *nt, if (release_len) { release = init_utsname()->release; - scnprintf(nt->buf, MAX_PRINT_CHUNK, "%s,%s", release, msg); + scnprintf(nt->buf, MAX_PRINT_CHUNK, "%s,%.*s", release, + msg_len, msg); msg_len += release_len; } else { memcpy(nt->buf, msg, msg_len); diff --git a/drivers/net/netdevsim/netdev.c b/drivers/net/netdevsim/netdev.c index 6927c1962277aa..62223ad2d63f93 100644 --- a/drivers/net/netdevsim/netdev.c +++ b/drivers/net/netdevsim/netdev.c @@ -109,8 +109,11 @@ static int nsim_forward_skb(struct net_device *tx_dev, int ret; ret = __dev_forward_skb(rx_dev, skb); - if (ret) + if (ret) { + if (psp_ext) + __skb_ext_put(psp_ext); return ret; + } nsim_psp_handle_ext(skb, psp_ext); diff --git a/drivers/net/ovpn/io.c b/drivers/net/ovpn/io.c index 3e9e7f8444b344..955c9a37e1f8db 100644 --- a/drivers/net/ovpn/io.c +++ b/drivers/net/ovpn/io.c @@ -355,6 +355,7 @@ netdev_tx_t ovpn_net_xmit(struct sk_buff *skb, struct net_device *dev) struct ovpn_priv *ovpn = netdev_priv(dev); struct sk_buff *segments, *curr, *next; struct sk_buff_head skb_list; + unsigned int tx_bytes = 0; struct ovpn_peer *peer; __be16 proto; int ret; @@ -365,7 +366,27 @@ netdev_tx_t ovpn_net_xmit(struct sk_buff *skb, struct net_device *dev) /* verify IP header size in network packet */ proto = ovpn_ip_check_protocol(skb); if (unlikely(!proto || skb->protocol != proto)) - goto drop; + goto drop_no_peer; + + /* retrieve peer serving the destination IP of this packet */ + peer = ovpn_peer_get_by_dst(ovpn, skb); + if (unlikely(!peer)) { + switch (skb->protocol) { + case htons(ETH_P_IP): + net_dbg_ratelimited("%s: no peer to send data to dst=%pI4\n", + netdev_name(ovpn->dev), + &ip_hdr(skb)->daddr); + break; + case htons(ETH_P_IPV6): + net_dbg_ratelimited("%s: no peer to send data to dst=%pI6c\n", + netdev_name(ovpn->dev), + &ipv6_hdr(skb)->daddr); + break; + } + goto drop_no_peer; + } + /* dst was needed for peer selection - it can now be dropped */ + skb_dst_drop(skb); if (skb_is_gso(skb)) { segments = skb_gso_segment(skb, 0); @@ -394,36 +415,28 @@ netdev_tx_t ovpn_net_xmit(struct sk_buff *skb, struct net_device *dev) continue; } + /* only count what we actually send */ + tx_bytes += curr->len; __skb_queue_tail(&skb_list, curr); } - skb_list.prev->next = NULL; - /* retrieve peer serving the destination IP of this packet */ - peer = ovpn_peer_get_by_dst(ovpn, skb); - if (unlikely(!peer)) { - switch (skb->protocol) { - case htons(ETH_P_IP): - net_dbg_ratelimited("%s: no peer to send data to dst=%pI4\n", - netdev_name(ovpn->dev), - &ip_hdr(skb)->daddr); - break; - case htons(ETH_P_IPV6): - net_dbg_ratelimited("%s: no peer to send data to dst=%pI6c\n", - netdev_name(ovpn->dev), - &ipv6_hdr(skb)->daddr); - break; - } - goto drop; + /* no segments survived: don't jump to 'drop' because we already + * incremented the counter for each failure in the loop + */ + if (unlikely(skb_queue_empty(&skb_list))) { + ovpn_peer_put(peer); + return NETDEV_TX_OK; } - /* dst was needed for peer selection - it can now be dropped */ - skb_dst_drop(skb); + skb_list.prev->next = NULL; - ovpn_peer_stats_increment_tx(&peer->vpn_stats, skb->len); + ovpn_peer_stats_increment_tx(&peer->vpn_stats, tx_bytes); ovpn_send(ovpn, skb_list.next, peer); return NETDEV_TX_OK; drop: + ovpn_peer_put(peer); +drop_no_peer: dev_dstats_tx_dropped(ovpn->dev); skb_tx_error(skb); kfree_skb_list(skb); diff --git a/drivers/net/ovpn/socket.c b/drivers/net/ovpn/socket.c index 9750871ab65cee..448cee3b3f9fa2 100644 --- a/drivers/net/ovpn/socket.c +++ b/drivers/net/ovpn/socket.c @@ -200,6 +200,22 @@ struct ovpn_socket *ovpn_socket_new(struct socket *sock, struct ovpn_peer *peer) ovpn_sock->sk = sk; kref_init(&ovpn_sock->refcount); + /* TCP sockets are per-peer, therefore they are linked to their unique + * peer + */ + if (sk->sk_protocol == IPPROTO_TCP) { + INIT_WORK(&ovpn_sock->tcp_tx_work, ovpn_tcp_tx_work); + ovpn_sock->peer = peer; + ovpn_peer_hold(peer); + } else if (sk->sk_protocol == IPPROTO_UDP) { + /* in UDP we only link the ovpn instance since the socket is + * shared among multiple peers + */ + ovpn_sock->ovpn = peer->ovpn; + netdev_hold(peer->ovpn->dev, &ovpn_sock->dev_tracker, + GFP_KERNEL); + } + /* the newly created ovpn_socket is holding reference to sk, * therefore we increase its refcounter. * @@ -212,29 +228,16 @@ struct ovpn_socket *ovpn_socket_new(struct socket *sock, struct ovpn_peer *peer) ret = ovpn_socket_attach(ovpn_sock, sock, peer); if (ret < 0) { + if (sk->sk_protocol == IPPROTO_TCP) + ovpn_peer_put(peer); + else if (sk->sk_protocol == IPPROTO_UDP) + netdev_put(peer->ovpn->dev, &ovpn_sock->dev_tracker); + sock_put(sk); kfree(ovpn_sock); ovpn_sock = ERR_PTR(ret); - goto sock_release; - } - - /* TCP sockets are per-peer, therefore they are linked to their unique - * peer - */ - if (sk->sk_protocol == IPPROTO_TCP) { - INIT_WORK(&ovpn_sock->tcp_tx_work, ovpn_tcp_tx_work); - ovpn_sock->peer = peer; - ovpn_peer_hold(peer); - } else if (sk->sk_protocol == IPPROTO_UDP) { - /* in UDP we only link the ovpn instance since the socket is - * shared among multiple peers - */ - ovpn_sock->ovpn = peer->ovpn; - netdev_hold(peer->ovpn->dev, &ovpn_sock->dev_tracker, - GFP_KERNEL); } - rcu_assign_sk_user_data(sk, ovpn_sock); sock_release: release_sock(sk); return ovpn_sock; diff --git a/drivers/net/ovpn/tcp.c b/drivers/net/ovpn/tcp.c index 0d7f30360d8746..5499c1572f3e25 100644 --- a/drivers/net/ovpn/tcp.c +++ b/drivers/net/ovpn/tcp.c @@ -70,37 +70,56 @@ static void ovpn_tcp_to_userspace(struct ovpn_peer *peer, struct sock *sk, peer->tcp.sk_cb.sk_data_ready(sk); } -static void ovpn_tcp_rcv(struct strparser *strp, struct sk_buff *skb) +static struct sk_buff *ovpn_tcp_skb_packet(const struct ovpn_peer *peer, + struct sk_buff *orig_skb, + const int pkt_len, const int pkt_off) { - struct ovpn_peer *peer = container_of(strp, struct ovpn_peer, tcp.strp); - struct strp_msg *msg = strp_msg(skb); - size_t pkt_len = msg->full_len - 2; - size_t off = msg->offset + 2; - u8 opcode; + struct sk_buff *ovpn_skb; + int err; - /* ensure skb->data points to the beginning of the openvpn packet */ - if (!pskb_pull(skb, off)) { - net_warn_ratelimited("%s: packet too small for peer %u\n", - netdev_name(peer->ovpn->dev), peer->id); + /* create a new skb with only the content of the current packet */ + ovpn_skb = netdev_alloc_skb(peer->ovpn->dev, pkt_len); + if (unlikely(!ovpn_skb)) goto err; - } - /* strparser does not trim the skb for us, therefore we do it now */ - if (pskb_trim(skb, pkt_len) != 0) { - net_warn_ratelimited("%s: trimming skb failed for peer %u\n", + skb_copy_header(ovpn_skb, orig_skb); + err = skb_copy_bits(orig_skb, pkt_off, skb_put(ovpn_skb, pkt_len), + pkt_len); + if (unlikely(err)) { + net_warn_ratelimited("%s: skb_copy_bits failed for peer %u\n", netdev_name(peer->ovpn->dev), peer->id); + kfree_skb(ovpn_skb); goto err; } - /* we need the first 4 bytes of data to be accessible + consume_skb(orig_skb); + return ovpn_skb; +err: + kfree_skb(orig_skb); + return NULL; +} + +static void ovpn_tcp_rcv(struct strparser *strp, struct sk_buff *skb) +{ + struct ovpn_peer *peer = container_of(strp, struct ovpn_peer, tcp.strp); + struct strp_msg *msg = strp_msg(skb); + int pkt_len = msg->full_len - 2; + u8 opcode; + + /* we need at least 4 bytes of data in the packet * to extract the opcode and the key ID later on */ - if (!pskb_may_pull(skb, OVPN_OPCODE_SIZE)) { + if (unlikely(pkt_len < OVPN_OPCODE_SIZE)) { net_warn_ratelimited("%s: packet too small to fetch opcode for peer %u\n", netdev_name(peer->ovpn->dev), peer->id); goto err; } + /* extract the packet into a new skb */ + skb = ovpn_tcp_skb_packet(peer, skb, pkt_len, msg->offset + 2); + if (unlikely(!skb)) + goto err; + /* DATA_V2 packets are handled in kernel, the rest goes to user space */ opcode = ovpn_opcode_from_skb(skb, 0); if (unlikely(opcode != OVPN_DATA_V2)) { @@ -113,7 +132,7 @@ static void ovpn_tcp_rcv(struct strparser *strp, struct sk_buff *skb) /* The packet size header must be there when sending the packet * to userspace, therefore we put it back */ - skb_push(skb, 2); + *(__be16 *)__skb_push(skb, sizeof(u16)) = htons(pkt_len); ovpn_tcp_to_userspace(peer, strp->sk, skb); return; } @@ -199,7 +218,19 @@ void ovpn_tcp_socket_detach(struct ovpn_socket *ovpn_sock) sk->sk_data_ready = peer->tcp.sk_cb.sk_data_ready; sk->sk_write_space = peer->tcp.sk_cb.sk_write_space; sk->sk_prot = peer->tcp.sk_cb.prot; - sk->sk_socket->ops = peer->tcp.sk_cb.ops; + + /* tcp_close() may race this function and could set + * sk->sk_socket to NULL. It does so by invoking + * sock_orphan(), which holds sk_callback_lock before + * doing the assignment. + * + * For this reason we acquire the same lock to avoid + * sk_socket to disappear under our feet + */ + write_lock_bh(&sk->sk_callback_lock); + if (sk->sk_socket) + sk->sk_socket->ops = peer->tcp.sk_cb.ops; + write_unlock_bh(&sk->sk_callback_lock); rcu_assign_sk_user_data(sk, NULL); } @@ -487,6 +518,7 @@ int ovpn_tcp_socket_attach(struct ovpn_socket *ovpn_sock, /* make sure no pre-existing encapsulation handler exists */ if (ovpn_sock->sk->sk_user_data) return -EBUSY; + rcu_assign_sk_user_data(ovpn_sock->sk, ovpn_sock); /* only a fully connected socket is expected. Connection should be * handled in userspace @@ -495,13 +527,14 @@ int ovpn_tcp_socket_attach(struct ovpn_socket *ovpn_sock, net_err_ratelimited("%s: provided TCP socket is not in ESTABLISHED state: %d\n", netdev_name(peer->ovpn->dev), ovpn_sock->sk->sk_state); - return -EINVAL; + ret = -EINVAL; + goto err; } ret = strp_init(&peer->tcp.strp, ovpn_sock->sk, &cb); if (ret < 0) { DEBUG_NET_WARN_ON_ONCE(1); - return ret; + goto err; } INIT_WORK(&peer->tcp.defer_del_work, ovpn_tcp_peer_del_work); @@ -536,6 +569,9 @@ int ovpn_tcp_socket_attach(struct ovpn_socket *ovpn_sock, strp_check_rcv(&peer->tcp.strp); return 0; +err: + rcu_assign_sk_user_data(ovpn_sock->sk, NULL); + return ret; } static void ovpn_tcp_close(struct sock *sk, long timeout) diff --git a/drivers/net/ovpn/udp.c b/drivers/net/ovpn/udp.c index d6a0f7a0b75d74..272b535ecaad4c 100644 --- a/drivers/net/ovpn/udp.c +++ b/drivers/net/ovpn/udp.c @@ -386,6 +386,7 @@ int ovpn_udp_socket_attach(struct ovpn_socket *ovpn_sock, struct socket *sock, struct ovpn_priv *ovpn) { struct udp_tunnel_sock_cfg cfg = { + .sk_user_data = ovpn_sock, .encap_type = UDP_ENCAP_OVPNINUDP, .encap_rcv = ovpn_udp_encap_recv, .encap_destroy = ovpn_udp_encap_destroy, diff --git a/drivers/net/phy/phy_device.c b/drivers/net/phy/phy_device.c index 81984d4ebb7cb3..a1ed7ed938ac56 100644 --- a/drivers/net/phy/phy_device.c +++ b/drivers/net/phy/phy_device.c @@ -1763,8 +1763,6 @@ int phy_attach_direct(struct net_device *dev, struct phy_device *phydev, goto error; phy_resume(phydev); - if (!phydev->is_on_sfp_module) - phy_led_triggers_register(phydev); /** * If the external phy used by current mac interface is managed by @@ -1879,9 +1877,6 @@ void phy_detach(struct phy_device *phydev) phydev->phy_link_change = NULL; phydev->phylink = NULL; - if (!phydev->is_on_sfp_module) - phy_led_triggers_unregister(phydev); - if (phydev->mdio.dev.driver) module_put(phydev->mdio.dev.driver->owner); @@ -3512,16 +3507,27 @@ static int phy_probe(struct device *dev) /* Set the state to READY by default */ phydev->state = PHY_READY; + /* Register the PHY LED triggers */ + if (!phydev->is_on_sfp_module) + phy_led_triggers_register(phydev); + /* Get the LEDs from the device tree, and instantiate standard * LEDs for them. */ - if (IS_ENABLED(CONFIG_PHYLIB_LEDS) && !phy_driver_is_genphy(phydev)) + if (IS_ENABLED(CONFIG_PHYLIB_LEDS) && !phy_driver_is_genphy(phydev)) { err = of_phy_leds(phydev); + if (err) + goto out; + } + + return 0; out: + if (!phydev->is_on_sfp_module) + phy_led_triggers_unregister(phydev); + /* Re-assert the reset signal on error */ - if (err) - phy_device_reset(phydev, 1); + phy_device_reset(phydev, 1); return err; } @@ -3535,6 +3541,9 @@ static int phy_remove(struct device *dev) if (IS_ENABLED(CONFIG_PHYLIB_LEDS) && !phy_driver_is_genphy(phydev)) phy_leds_unregister(phydev); + if (!phydev->is_on_sfp_module) + phy_led_triggers_unregister(phydev); + phydev->state = PHY_DOWN; sfp_bus_del_upstream(phydev->sfp_bus); diff --git a/drivers/net/phy/qcom/qca807x.c b/drivers/net/phy/qcom/qca807x.c index 1be8295a95cb5c..511cde345e089f 100644 --- a/drivers/net/phy/qcom/qca807x.c +++ b/drivers/net/phy/qcom/qca807x.c @@ -375,7 +375,7 @@ static int qca807x_gpio_get(struct gpio_chip *gc, unsigned int offset) reg = QCA807X_MMD7_LED_FORCE_CTRL(offset); val = phy_read_mmd(priv->phy, MDIO_MMD_AN, reg); - return FIELD_GET(QCA807X_GPIO_FORCE_MODE_MASK, val); + return !!FIELD_GET(QCA807X_GPIO_FORCE_MODE_MASK, val); } static int qca807x_gpio_set(struct gpio_chip *gc, unsigned int offset, int value) diff --git a/drivers/net/phy/sfp.c b/drivers/net/phy/sfp.c index 3e023723887c4b..c62e3f364ea73b 100644 --- a/drivers/net/phy/sfp.c +++ b/drivers/net/phy/sfp.c @@ -367,6 +367,12 @@ static void sfp_fixup_ignore_tx_fault(struct sfp *sfp) sfp->state_ignore_mask |= SFP_F_TX_FAULT; } +static void sfp_fixup_ignore_tx_fault_and_los(struct sfp *sfp) +{ + sfp_fixup_ignore_tx_fault(sfp); + sfp_fixup_ignore_los(sfp); +} + static void sfp_fixup_ignore_hw(struct sfp *sfp, unsigned int mask) { sfp->state_hw_mask &= ~mask; @@ -474,11 +480,16 @@ static void sfp_quirk_ubnt_uf_instant(const struct sfp_eeprom_id *id, { /* Ubiquiti U-Fiber Instant module claims that support all transceiver * types including 10G Ethernet which is not truth. So clear all claimed - * modes and set only one mode which module supports: 1000baseX_Full. + * modes and set only one mode which module supports: 1000baseX_Full, + * along with the Autoneg and pause bits. */ linkmode_zero(caps->link_modes); linkmode_set_bit(ETHTOOL_LINK_MODE_1000baseX_Full_BIT, caps->link_modes); + linkmode_set_bit(ETHTOOL_LINK_MODE_Autoneg_BIT, caps->link_modes); + linkmode_set_bit(ETHTOOL_LINK_MODE_Pause_BIT, caps->link_modes); + linkmode_set_bit(ETHTOOL_LINK_MODE_Asym_Pause_BIT, caps->link_modes); + phy_interface_zero(caps->interfaces); __set_bit(PHY_INTERFACE_MODE_1000BASEX, caps->interfaces); } @@ -530,11 +541,31 @@ static const struct sfp_quirk sfp_quirks[] = { // Huawei MA5671A can operate at 2500base-X, but report 1.2GBd NRZ in // their EEPROM SFP_QUIRK("HUAWEI", "MA5671A", sfp_quirk_2500basex, + sfp_fixup_ignore_tx_fault_and_los), + + // Hisense LXT-010S-H is a GPON ONT SFP (sold as LEOX LXT-010S-H) that + // can operate at 2500base-X, but reports 1000BASE-LX / 1300MBd in its + // EEPROM + SFP_QUIRK("Hisense-Leox", "LXT-010S-H", sfp_quirk_2500basex, + sfp_fixup_ignore_tx_fault), + + // Hisense ZNID-GPON-2311NA can operate at 2500base-X, but reports + // 1000BASE-LX / 1300MBd in its EEPROM + SFP_QUIRK("Hisense", "ZNID-GPON-2311NA", sfp_quirk_2500basex, + sfp_fixup_ignore_tx_fault), + + // HSGQ HSGQ-XPON-Stick can operate at 2500base-X, but reports + // 1000BASE-LX / 1300MBd in its EEPROM + SFP_QUIRK("HSGQ", "HSGQ-XPON-Stick", sfp_quirk_2500basex, sfp_fixup_ignore_tx_fault), - // Lantech 8330-262D-E can operate at 2500base-X, but incorrectly report - // 2500MBd NRZ in their EEPROM + // Lantech 8330-262D-E and 8330-265D can operate at 2500base-X, but + // incorrectly report 2500MBd NRZ in their EEPROM. + // Some 8330-265D modules have inverted LOS, while all of them report + // normal LOS in EEPROM. Therefore we need to ignore LOS entirely. SFP_QUIRK_S("Lantech", "8330-262D-E", sfp_quirk_2500basex), + SFP_QUIRK("Lantech", "8330-265D", sfp_quirk_2500basex, + sfp_fixup_ignore_los), SFP_QUIRK_S("UBNT", "UF-INSTANT", sfp_quirk_ubnt_uf_instant), diff --git a/drivers/net/team/team_core.c b/drivers/net/team/team_core.c index c08a5c1bd6e4d2..98772d749f2bf5 100644 --- a/drivers/net/team/team_core.c +++ b/drivers/net/team/team_core.c @@ -1292,7 +1292,7 @@ static int team_port_add(struct team *team, struct net_device *port_dev, static void __team_port_change_port_removed(struct team_port *port); -static int team_port_del(struct team *team, struct net_device *port_dev) +static int team_port_del(struct team *team, struct net_device *port_dev, bool unregister) { struct net_device *dev = team->dev; struct team_port *port; @@ -1330,7 +1330,13 @@ static int team_port_del(struct team *team, struct net_device *port_dev) __team_port_change_port_removed(port); team_port_set_orig_dev_addr(port); - dev_set_mtu(port_dev, port->orig.mtu); + if (unregister) { + netdev_lock_ops(port_dev); + __netif_set_mtu(port_dev, port->orig.mtu); + netdev_unlock_ops(port_dev); + } else { + dev_set_mtu(port_dev, port->orig.mtu); + } kfree_rcu(port, rcu); netdev_info(dev, "Port device %s removed\n", portname); netdev_compute_master_upper_features(team->dev, true); @@ -1634,7 +1640,7 @@ static void team_uninit(struct net_device *dev) ASSERT_RTNL(); list_for_each_entry_safe(port, tmp, &team->port_list, list) - team_port_del(team, port->dev); + team_port_del(team, port->dev, false); __team_change_mode(team, NULL); /* cleanup */ __team_options_unregister(team, team_options, ARRAY_SIZE(team_options)); @@ -1933,7 +1939,16 @@ static int team_del_slave(struct net_device *dev, struct net_device *port_dev) ASSERT_RTNL(); - return team_port_del(team, port_dev); + return team_port_del(team, port_dev, false); +} + +static int team_del_slave_on_unregister(struct net_device *dev, struct net_device *port_dev) +{ + struct team *team = netdev_priv(dev); + + ASSERT_RTNL(); + + return team_port_del(team, port_dev, true); } static netdev_features_t team_fix_features(struct net_device *dev, @@ -2045,6 +2060,68 @@ static const struct ethtool_ops team_ethtool_ops = { * rt netlink interface ***********************/ +/* For tx path we need a linkup && enabled port and for parse any port + * suffices. + */ +static struct team_port *team_header_port_get_rcu(struct team *team, + bool txable) +{ + struct team_port *port; + + list_for_each_entry_rcu(port, &team->port_list, list) { + if (!txable || team_port_txable(port)) + return port; + } + + return NULL; +} + +static int team_header_create(struct sk_buff *skb, struct net_device *team_dev, + unsigned short type, const void *daddr, + const void *saddr, unsigned int len) +{ + struct team *team = netdev_priv(team_dev); + const struct header_ops *port_ops; + struct team_port *port; + int ret = 0; + + rcu_read_lock(); + port = team_header_port_get_rcu(team, true); + if (port) { + port_ops = READ_ONCE(port->dev->header_ops); + if (port_ops && port_ops->create) + ret = port_ops->create(skb, port->dev, + type, daddr, saddr, len); + } + rcu_read_unlock(); + return ret; +} + +static int team_header_parse(const struct sk_buff *skb, + const struct net_device *team_dev, + unsigned char *haddr) +{ + struct team *team = netdev_priv(team_dev); + const struct header_ops *port_ops; + struct team_port *port; + int ret = 0; + + rcu_read_lock(); + port = team_header_port_get_rcu(team, false); + if (port) { + port_ops = READ_ONCE(port->dev->header_ops); + if (port_ops && port_ops->parse) + ret = port_ops->parse(skb, port->dev, haddr); + } + rcu_read_unlock(); + return ret; +} + +static const struct header_ops team_header_ops = { + .create = team_header_create, + .parse = team_header_parse, +}; + static void team_setup_by_port(struct net_device *dev, struct net_device *port_dev) { @@ -2053,7 +2130,8 @@ static void team_setup_by_port(struct net_device *dev, if (port_dev->type == ARPHRD_ETHER) dev->header_ops = team->header_ops_cache; else - dev->header_ops = port_dev->header_ops; + dev->header_ops = port_dev->header_ops ? + &team_header_ops : NULL; dev->type = port_dev->type; dev->hard_header_len = port_dev->hard_header_len; dev->needed_headroom = port_dev->needed_headroom; @@ -2926,7 +3004,7 @@ static int team_device_event(struct notifier_block *unused, !!netif_oper_up(port->dev)); break; case NETDEV_UNREGISTER: - team_del_slave(port->team->dev, dev); + team_del_slave_on_unregister(port->team->dev, dev); break; case NETDEV_FEAT_CHANGE: if (!port->team->notifier_ctx) { @@ -2999,3 +3077,4 @@ MODULE_LICENSE("GPL v2"); MODULE_AUTHOR("Jiri Pirko "); MODULE_DESCRIPTION("Ethernet team device driver"); MODULE_ALIAS_RTNL_LINK(DRV_NAME); +MODULE_IMPORT_NS("NETDEV_INTERNAL"); diff --git a/drivers/net/tun_vnet.h b/drivers/net/tun_vnet.h index a5f93b6c4482c3..fa5cab9d3e55c5 100644 --- a/drivers/net/tun_vnet.h +++ b/drivers/net/tun_vnet.h @@ -244,7 +244,7 @@ tun_vnet_hdr_tnl_from_skb(unsigned int flags, if (virtio_net_hdr_tnl_from_skb(skb, tnl_hdr, has_tnl_offload, tun_vnet_is_little_endian(flags), - vlan_hlen, true)) { + vlan_hlen, true, false)) { struct virtio_net_hdr_v1 *hdr = &tnl_hdr->hash_hdr.hdr; struct skb_shared_info *sinfo = skb_shinfo(skb); diff --git a/drivers/net/usb/Kconfig b/drivers/net/usb/Kconfig index 856e648d804e02..da0f6a138f4fc7 100644 --- a/drivers/net/usb/Kconfig +++ b/drivers/net/usb/Kconfig @@ -319,7 +319,6 @@ config USB_NET_DM9601 config USB_NET_SR9700 tristate "CoreChip-sz SR9700 based USB 1.1 10/100 ethernet devices" depends on USB_USBNET - select CRC32 help This option adds support for CoreChip-sz SR9700 based USB 1.1 10/100 Ethernet adapters. diff --git a/drivers/net/usb/aqc111.c b/drivers/net/usb/aqc111.c index 9201ee10a13f78..d316aa66dbc236 100644 --- a/drivers/net/usb/aqc111.c +++ b/drivers/net/usb/aqc111.c @@ -1400,14 +1400,14 @@ static int aqc111_suspend(struct usb_interface *intf, pm_message_t message) aqc111_write16_cmd_nopm(dev, AQ_ACCESS_MAC, SFR_MEDIUM_STATUS_MODE, 2, ®16); - aqc111_write_cmd(dev, AQ_WOL_CFG, 0, 0, - WOL_CFG_SIZE, &wol_cfg); - aqc111_write32_cmd(dev, AQ_PHY_OPS, 0, 0, - &aqc111_data->phy_cfg); + aqc111_write_cmd_nopm(dev, AQ_WOL_CFG, 0, 0, + WOL_CFG_SIZE, &wol_cfg); + aqc111_write32_cmd_nopm(dev, AQ_PHY_OPS, 0, 0, + &aqc111_data->phy_cfg); } else { aqc111_data->phy_cfg |= AQ_LOW_POWER; - aqc111_write32_cmd(dev, AQ_PHY_OPS, 0, 0, - &aqc111_data->phy_cfg); + aqc111_write32_cmd_nopm(dev, AQ_PHY_OPS, 0, 0, + &aqc111_data->phy_cfg); /* Disable RX path */ aqc111_read16_cmd_nopm(dev, AQ_ACCESS_MAC, diff --git a/drivers/net/usb/catc.c b/drivers/net/usb/catc.c index 6759388692f8e9..3c824340ffb068 100644 --- a/drivers/net/usb/catc.c +++ b/drivers/net/usb/catc.c @@ -64,6 +64,16 @@ static const char driver_name[] = "catc"; #define CTRL_QUEUE 16 /* Max control requests in flight (power of two) */ #define RX_PKT_SZ 1600 /* Max size of receive packet for F5U011 */ +/* + * USB endpoints. + */ + +enum catc_usb_ep { + CATC_USB_EP_CONTROL = 0, + CATC_USB_EP_BULK = 1, + CATC_USB_EP_INT_IN = 2, +}; + /* * Control requests. */ @@ -772,6 +782,13 @@ static int catc_probe(struct usb_interface *intf, const struct usb_device_id *id u8 broadcast[ETH_ALEN]; u8 *macbuf; int pktsz, ret = -ENOMEM; + static const u8 bulk_ep_addr[] = { + CATC_USB_EP_BULK | USB_DIR_OUT, + CATC_USB_EP_BULK | USB_DIR_IN, + 0}; + static const u8 int_ep_addr[] = { + CATC_USB_EP_INT_IN | USB_DIR_IN, + 0}; macbuf = kmalloc(ETH_ALEN, GFP_KERNEL); if (!macbuf) @@ -784,6 +801,14 @@ static int catc_probe(struct usb_interface *intf, const struct usb_device_id *id goto fail_mem; } + /* Verify that all required endpoints are present */ + if (!usb_check_bulk_endpoints(intf, bulk_ep_addr) || + !usb_check_int_endpoints(intf, int_ep_addr)) { + dev_err(dev, "Missing or invalid endpoints\n"); + ret = -ENODEV; + goto fail_mem; + } + netdev = alloc_etherdev(sizeof(struct catc)); if (!netdev) goto fail_mem; @@ -828,14 +853,14 @@ static int catc_probe(struct usb_interface *intf, const struct usb_device_id *id usb_fill_control_urb(catc->ctrl_urb, usbdev, usb_sndctrlpipe(usbdev, 0), NULL, NULL, 0, catc_ctrl_done, catc); - usb_fill_bulk_urb(catc->tx_urb, usbdev, usb_sndbulkpipe(usbdev, 1), - NULL, 0, catc_tx_done, catc); + usb_fill_bulk_urb(catc->tx_urb, usbdev, usb_sndbulkpipe(usbdev, CATC_USB_EP_BULK), + NULL, 0, catc_tx_done, catc); - usb_fill_bulk_urb(catc->rx_urb, usbdev, usb_rcvbulkpipe(usbdev, 1), - catc->rx_buf, pktsz, catc_rx_done, catc); + usb_fill_bulk_urb(catc->rx_urb, usbdev, usb_rcvbulkpipe(usbdev, CATC_USB_EP_BULK), + catc->rx_buf, pktsz, catc_rx_done, catc); - usb_fill_int_urb(catc->irq_urb, usbdev, usb_rcvintpipe(usbdev, 2), - catc->irq_buf, 2, catc_irq_done, catc, 1); + usb_fill_int_urb(catc->irq_urb, usbdev, usb_rcvintpipe(usbdev, CATC_USB_EP_INT_IN), + catc->irq_buf, 2, catc_irq_done, catc, 1); if (!catc->is_f5u011) { u32 *buf; diff --git a/drivers/net/usb/cdc-phonet.c b/drivers/net/usb/cdc-phonet.c index ad5121e9cf5d6f..165650ecef649f 100644 --- a/drivers/net/usb/cdc-phonet.c +++ b/drivers/net/usb/cdc-phonet.c @@ -157,11 +157,16 @@ static void rx_complete(struct urb *req) PAGE_SIZE); page = NULL; } - } else { + } else if (skb_shinfo(skb)->nr_frags < MAX_SKB_FRAGS) { skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page, 0, req->actual_length, PAGE_SIZE); page = NULL; + } else { + dev_kfree_skb_any(skb); + pnd->rx_skb = NULL; + skb = NULL; + dev->stats.rx_length_errors++; } if (req->actual_length < PAGE_SIZE) pnd->rx_skb = NULL; /* Last fragment */ diff --git a/drivers/net/usb/cdc_ncm.c b/drivers/net/usb/cdc_ncm.c index 5d123df0a866b8..81d7e99fc0f09f 100644 --- a/drivers/net/usb/cdc_ncm.c +++ b/drivers/net/usb/cdc_ncm.c @@ -1656,6 +1656,7 @@ int cdc_ncm_rx_verify_ndp16(struct sk_buff *skb_in, int ndpoffset) struct usbnet *dev = netdev_priv(skb_in->dev); struct usb_cdc_ncm_ndp16 *ndp16; int ret = -EINVAL; + size_t ndp_len; if ((ndpoffset + sizeof(struct usb_cdc_ncm_ndp16)) > skb_in->len) { netif_dbg(dev, rx_err, dev->net, "invalid NDP offset <%u>\n", @@ -1675,8 +1676,8 @@ int cdc_ncm_rx_verify_ndp16(struct sk_buff *skb_in, int ndpoffset) sizeof(struct usb_cdc_ncm_dpe16)); ret--; /* we process NDP entries except for the last one */ - if ((sizeof(struct usb_cdc_ncm_ndp16) + - ret * (sizeof(struct usb_cdc_ncm_dpe16))) > skb_in->len) { + ndp_len = struct_size_t(struct usb_cdc_ncm_ndp16, dpe16, ret); + if (ndpoffset + ndp_len > skb_in->len) { netif_dbg(dev, rx_err, dev->net, "Invalid nframes = %d\n", ret); ret = -EINVAL; } @@ -1692,6 +1693,7 @@ int cdc_ncm_rx_verify_ndp32(struct sk_buff *skb_in, int ndpoffset) struct usbnet *dev = netdev_priv(skb_in->dev); struct usb_cdc_ncm_ndp32 *ndp32; int ret = -EINVAL; + size_t ndp_len; if ((ndpoffset + sizeof(struct usb_cdc_ncm_ndp32)) > skb_in->len) { netif_dbg(dev, rx_err, dev->net, "invalid NDP offset <%u>\n", @@ -1711,8 +1713,8 @@ int cdc_ncm_rx_verify_ndp32(struct sk_buff *skb_in, int ndpoffset) sizeof(struct usb_cdc_ncm_dpe32)); ret--; /* we process NDP entries except for the last one */ - if ((sizeof(struct usb_cdc_ncm_ndp32) + - ret * (sizeof(struct usb_cdc_ncm_dpe32))) > skb_in->len) { + ndp_len = struct_size_t(struct usb_cdc_ncm_ndp32, dpe32, ret); + if (ndpoffset + ndp_len > skb_in->len) { netif_dbg(dev, rx_err, dev->net, "Invalid nframes = %d\n", ret); ret = -EINVAL; } diff --git a/drivers/net/usb/kalmia.c b/drivers/net/usb/kalmia.c index 613fc6910f1481..ee9c48f7f68f95 100644 --- a/drivers/net/usb/kalmia.c +++ b/drivers/net/usb/kalmia.c @@ -132,11 +132,18 @@ kalmia_bind(struct usbnet *dev, struct usb_interface *intf) { int status; u8 ethernet_addr[ETH_ALEN]; + static const u8 ep_addr[] = { + 1 | USB_DIR_IN, + 2 | USB_DIR_OUT, + 0}; /* Don't bind to AT command interface */ if (intf->cur_altsetting->desc.bInterfaceClass != USB_CLASS_VENDOR_SPEC) return -EINVAL; + if (!usb_check_bulk_endpoints(intf, ep_addr)) + return -ENODEV; + dev->in = usb_rcvbulkpipe(dev->udev, 0x81 & USB_ENDPOINT_NUMBER_MASK); dev->out = usb_sndbulkpipe(dev->udev, 0x02 & USB_ENDPOINT_NUMBER_MASK); dev->status = NULL; diff --git a/drivers/net/usb/kaweth.c b/drivers/net/usb/kaweth.c index c9efb7df892ec5..cb2472b59e1046 100644 --- a/drivers/net/usb/kaweth.c +++ b/drivers/net/usb/kaweth.c @@ -765,7 +765,6 @@ static void kaweth_set_rx_mode(struct net_device *net) netdev_dbg(net, "Setting Rx mode to %d\n", packet_filter_bitmap); - netif_stop_queue(net); if (net->flags & IFF_PROMISC) { packet_filter_bitmap |= KAWETH_PACKET_FILTER_PROMISCUOUS; @@ -775,7 +774,6 @@ static void kaweth_set_rx_mode(struct net_device *net) } kaweth->packet_filter_bitmap = packet_filter_bitmap; - netif_wake_queue(net); } /**************************************************************** @@ -885,6 +883,13 @@ static int kaweth_probe( const eth_addr_t bcast_addr = { 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF }; int result = 0; int rv = -EIO; + static const u8 bulk_ep_addr[] = { + 1 | USB_DIR_IN, + 2 | USB_DIR_OUT, + 0}; + static const u8 int_ep_addr[] = { + 3 | USB_DIR_IN, + 0}; dev_dbg(dev, "Kawasaki Device Probe (Device number:%d): 0x%4.4x:0x%4.4x:0x%4.4x\n", @@ -898,6 +903,12 @@ static int kaweth_probe( (int)udev->descriptor.bLength, (int)udev->descriptor.bDescriptorType); + if (!usb_check_bulk_endpoints(intf, bulk_ep_addr) || + !usb_check_int_endpoints(intf, int_ep_addr)) { + dev_err(dev, "couldn't find required endpoints\n"); + return -ENODEV; + } + netdev = alloc_etherdev(sizeof(*kaweth)); if (!netdev) return -ENOMEM; diff --git a/drivers/net/usb/lan78xx.c b/drivers/net/usb/lan78xx.c index 00397a80739342..858a442d6996a0 100644 --- a/drivers/net/usb/lan78xx.c +++ b/drivers/net/usb/lan78xx.c @@ -2094,8 +2094,6 @@ static int lan78xx_mdio_init(struct lan78xx_net *dev) dev->mdiobus->phy_mask = ~(1 << 1); break; case ID_REV_CHIP_ID_7801_: - /* scan thru PHYAD[2..0] */ - dev->mdiobus->phy_mask = ~(0xFF); break; } @@ -3121,6 +3119,10 @@ static int lan78xx_init_ltm(struct lan78xx_net *dev) int ret; u32 buf; + /* LAN7850 is USB 2.0 and does not support LTM */ + if (dev->chipid == ID_REV_CHIP_ID_7850_) + return 0; + ret = lan78xx_read_reg(dev, USB_CFG1, &buf); if (ret < 0) goto init_ltm_failed; @@ -3831,6 +3833,7 @@ static void lan78xx_rx_csum_offload(struct lan78xx_net *dev, */ if (!(dev->net->features & NETIF_F_RXCSUM) || unlikely(rx_cmd_a & RX_CMD_A_ICSM_) || + unlikely(rx_cmd_a & RX_CMD_A_CSE_MASK_) || ((rx_cmd_a & RX_CMD_A_FVTG_) && !(dev->net->features & NETIF_F_HW_VLAN_CTAG_RX))) { skb->ip_summed = CHECKSUM_NONE; @@ -3903,7 +3906,8 @@ static int lan78xx_rx(struct lan78xx_net *dev, struct sk_buff *skb, return 0; } - if (unlikely(rx_cmd_a & RX_CMD_A_RED_)) { + if (unlikely(rx_cmd_a & RX_CMD_A_RED_) && + (rx_cmd_a & RX_CMD_A_RX_HARD_ERRS_MASK_)) { netif_dbg(dev, rx_err, dev->net, "Error rx_cmd_a=0x%08x", rx_cmd_a); } else { @@ -4178,7 +4182,7 @@ static struct skb_data *lan78xx_tx_buf_fill(struct lan78xx_net *dev, } tx_data += len; - entry->length += len; + entry->length += max_t(unsigned int, len, ETH_ZLEN); entry->num_of_packet += skb_shinfo(skb)->gso_segs ?: 1; dev_kfree_skb_any(skb); @@ -4548,8 +4552,6 @@ static void lan78xx_disconnect(struct usb_interface *intf) phylink_disconnect_phy(dev->phylink); rtnl_unlock(); - netif_napi_del(&dev->napi); - unregister_netdev(net); timer_shutdown_sync(&dev->stat_monitor); diff --git a/drivers/net/usb/lan78xx.h b/drivers/net/usb/lan78xx.h index 968e5e5faee0a7..17a934acff3d64 100644 --- a/drivers/net/usb/lan78xx.h +++ b/drivers/net/usb/lan78xx.h @@ -74,6 +74,9 @@ #define RX_CMD_A_ICSM_ (0x00004000) #define RX_CMD_A_LEN_MASK_ (0x00003FFF) +#define RX_CMD_A_RX_HARD_ERRS_MASK_ \ + (RX_CMD_A_RX_ERRS_MASK_ & ~RX_CMD_A_CSE_MASK_) + /* Rx Command B */ #define RX_CMD_B_CSUM_SHIFT_ (16) #define RX_CMD_B_CSUM_MASK_ (0xFFFF0000) diff --git a/drivers/net/usb/pegasus.c b/drivers/net/usb/pegasus.c index c514483134f05f..475b066081c7fe 100644 --- a/drivers/net/usb/pegasus.c +++ b/drivers/net/usb/pegasus.c @@ -31,6 +31,17 @@ static const char driver_name[] = "pegasus"; BMSR_100FULL | BMSR_ANEGCAPABLE) #define CARRIER_CHECK_DELAY (2 * HZ) +/* + * USB endpoints. + */ + +enum pegasus_usb_ep { + PEGASUS_USB_EP_CONTROL = 0, + PEGASUS_USB_EP_BULK_IN = 1, + PEGASUS_USB_EP_BULK_OUT = 2, + PEGASUS_USB_EP_INT_IN = 3, +}; + static bool loopback; static bool mii_mode; static char *devid; @@ -545,7 +556,7 @@ static void read_bulk_callback(struct urb *urb) goto tl_sched; goon: usb_fill_bulk_urb(pegasus->rx_urb, pegasus->usb, - usb_rcvbulkpipe(pegasus->usb, 1), + usb_rcvbulkpipe(pegasus->usb, PEGASUS_USB_EP_BULK_IN), pegasus->rx_skb->data, PEGASUS_MTU, read_bulk_callback, pegasus); rx_status = usb_submit_urb(pegasus->rx_urb, GFP_ATOMIC); @@ -585,7 +596,7 @@ static void rx_fixup(struct tasklet_struct *t) return; } usb_fill_bulk_urb(pegasus->rx_urb, pegasus->usb, - usb_rcvbulkpipe(pegasus->usb, 1), + usb_rcvbulkpipe(pegasus->usb, PEGASUS_USB_EP_BULK_IN), pegasus->rx_skb->data, PEGASUS_MTU, read_bulk_callback, pegasus); try_again: @@ -713,7 +724,7 @@ static netdev_tx_t pegasus_start_xmit(struct sk_buff *skb, ((__le16 *) pegasus->tx_buff)[0] = cpu_to_le16(l16); skb_copy_from_linear_data(skb, pegasus->tx_buff + 2, skb->len); usb_fill_bulk_urb(pegasus->tx_urb, pegasus->usb, - usb_sndbulkpipe(pegasus->usb, 2), + usb_sndbulkpipe(pegasus->usb, PEGASUS_USB_EP_BULK_OUT), pegasus->tx_buff, count, write_bulk_callback, pegasus); if ((res = usb_submit_urb(pegasus->tx_urb, GFP_ATOMIC))) { @@ -804,8 +815,19 @@ static void unlink_all_urbs(pegasus_t *pegasus) static int alloc_urbs(pegasus_t *pegasus) { + static const u8 bulk_ep_addr[] = { + 1 | USB_DIR_IN, + 2 | USB_DIR_OUT, + 0}; + static const u8 int_ep_addr[] = { + 3 | USB_DIR_IN, + 0}; int res = -ENOMEM; + if (!usb_check_bulk_endpoints(pegasus->intf, bulk_ep_addr) || + !usb_check_int_endpoints(pegasus->intf, int_ep_addr)) + return -ENODEV; + pegasus->rx_urb = usb_alloc_urb(0, GFP_KERNEL); if (!pegasus->rx_urb) { return res; @@ -840,7 +862,7 @@ static int pegasus_open(struct net_device *net) set_registers(pegasus, EthID, 6, net->dev_addr); usb_fill_bulk_urb(pegasus->rx_urb, pegasus->usb, - usb_rcvbulkpipe(pegasus->usb, 1), + usb_rcvbulkpipe(pegasus->usb, PEGASUS_USB_EP_BULK_IN), pegasus->rx_skb->data, PEGASUS_MTU, read_bulk_callback, pegasus); if ((res = usb_submit_urb(pegasus->rx_urb, GFP_KERNEL))) { @@ -851,7 +873,7 @@ static int pegasus_open(struct net_device *net) } usb_fill_int_urb(pegasus->intr_urb, pegasus->usb, - usb_rcvintpipe(pegasus->usb, 3), + usb_rcvintpipe(pegasus->usb, PEGASUS_USB_EP_INT_IN), pegasus->intr_buff, sizeof(pegasus->intr_buff), intr_callback, pegasus, pegasus->intr_interval); if ((res = usb_submit_urb(pegasus->intr_urb, GFP_KERNEL))) { @@ -1136,16 +1158,31 @@ static int pegasus_probe(struct usb_interface *intf, pegasus_t *pegasus; int dev_index = id - pegasus_ids; int res = -ENOMEM; + static const u8 bulk_ep_addr[] = { + PEGASUS_USB_EP_BULK_IN | USB_DIR_IN, + PEGASUS_USB_EP_BULK_OUT | USB_DIR_OUT, + 0}; + static const u8 int_ep_addr[] = { + PEGASUS_USB_EP_INT_IN | USB_DIR_IN, + 0}; if (pegasus_blacklisted(dev)) return -ENODEV; + /* Verify that all required endpoints are present */ + if (!usb_check_bulk_endpoints(intf, bulk_ep_addr) || + !usb_check_int_endpoints(intf, int_ep_addr)) { + dev_err(&intf->dev, "Missing or invalid endpoints\n"); + return -ENODEV; + } + net = alloc_etherdev(sizeof(struct pegasus)); if (!net) goto out; pegasus = netdev_priv(net); pegasus->dev_index = dev_index; + pegasus->intf = intf; res = alloc_urbs(pegasus); if (res < 0) { @@ -1157,7 +1194,6 @@ static int pegasus_probe(struct usb_interface *intf, INIT_DELAYED_WORK(&pegasus->carrier_check, check_carrier); - pegasus->intf = intf; pegasus->usb = dev; pegasus->net = net; diff --git a/drivers/net/usb/qmi_wwan.c b/drivers/net/usb/qmi_wwan.c index 3a4985b582cb12..05acac10cd2bad 100644 --- a/drivers/net/usb/qmi_wwan.c +++ b/drivers/net/usb/qmi_wwan.c @@ -928,7 +928,7 @@ static int qmi_wwan_resume(struct usb_interface *intf) static const struct driver_info qmi_wwan_info = { .description = "WWAN/QMI device", - .flags = FLAG_WWAN | FLAG_SEND_ZLP, + .flags = FLAG_WWAN | FLAG_NOMAXMTU | FLAG_SEND_ZLP, .bind = qmi_wwan_bind, .unbind = qmi_wwan_unbind, .manage_power = qmi_wwan_manage_power, @@ -937,7 +937,7 @@ static const struct driver_info qmi_wwan_info = { static const struct driver_info qmi_wwan_info_quirk_dtr = { .description = "WWAN/QMI device", - .flags = FLAG_WWAN | FLAG_SEND_ZLP, + .flags = FLAG_WWAN | FLAG_NOMAXMTU | FLAG_SEND_ZLP, .bind = qmi_wwan_bind, .unbind = qmi_wwan_unbind, .manage_power = qmi_wwan_manage_power, diff --git a/drivers/net/usb/r8152.c b/drivers/net/usb/r8152.c index 2f3baa5f6e9c94..9eda892beb1f86 100644 --- a/drivers/net/usb/r8152.c +++ b/drivers/net/usb/r8152.c @@ -2449,6 +2449,8 @@ static int r8152_tx_agg_fill(struct r8152 *tp, struct tx_agg *agg) ret = usb_submit_urb(agg->urb, GFP_ATOMIC); if (ret < 0) usb_autopm_put_interface_async(tp->intf); + else + netif_trans_update(tp->netdev); out_tx_fill: return ret; @@ -10060,6 +10062,7 @@ static const struct usb_device_id rtl8152_table[] = { { USB_DEVICE(VENDOR_ID_DLINK, 0xb301) }, { USB_DEVICE(VENDOR_ID_DELL, 0xb097) }, { USB_DEVICE(VENDOR_ID_ASUS, 0x1976) }, + { USB_DEVICE(VENDOR_ID_TRENDNET, 0xe02b) }, {} }; diff --git a/drivers/net/usb/sr9700.c b/drivers/net/usb/sr9700.c index 820c4c50697921..a5d364fbc36391 100644 --- a/drivers/net/usb/sr9700.c +++ b/drivers/net/usb/sr9700.c @@ -18,7 +18,6 @@ #include #include #include -#include #include #include "sr9700.h" @@ -265,31 +264,15 @@ static const struct ethtool_ops sr9700_ethtool_ops = { static void sr9700_set_multicast(struct net_device *netdev) { struct usbnet *dev = netdev_priv(netdev); - /* We use the 20 byte dev->data for our 8 byte filter buffer - * to avoid allocating memory that is tricky to free later - */ - u8 *hashes = (u8 *)&dev->data; /* rx_ctl setting : enable, disable_long, disable_crc */ u8 rx_ctl = RCR_RXEN | RCR_DIS_CRC | RCR_DIS_LONG; - memset(hashes, 0x00, SR_MCAST_SIZE); - /* broadcast address */ - hashes[SR_MCAST_SIZE - 1] |= SR_MCAST_ADDR_FLAG; - if (netdev->flags & IFF_PROMISC) { + if (netdev->flags & IFF_PROMISC) rx_ctl |= RCR_PRMSC; - } else if (netdev->flags & IFF_ALLMULTI || - netdev_mc_count(netdev) > SR_MCAST_MAX) { - rx_ctl |= RCR_RUNT; - } else if (!netdev_mc_empty(netdev)) { - struct netdev_hw_addr *ha; - - netdev_for_each_mc_addr(ha, netdev) { - u32 crc = ether_crc(ETH_ALEN, ha->addr) >> 26; - hashes[crc >> 3] |= 1 << (crc & 0x7); - } - } + else if (netdev->flags & IFF_ALLMULTI || !netdev_mc_empty(netdev)) + /* The chip has no multicast filter */ + rx_ctl |= RCR_ALL; - sr_write_async(dev, SR_MAR, SR_MCAST_SIZE, hashes); sr_write_reg_async(dev, SR_RCR, rx_ctl); } diff --git a/drivers/net/usb/sr9700.h b/drivers/net/usb/sr9700.h index ea2b4de621c867..c479908f7d823d 100644 --- a/drivers/net/usb/sr9700.h +++ b/drivers/net/usb/sr9700.h @@ -104,9 +104,7 @@ #define WCR_LINKEN (1 << 5) /* Physical Address Reg */ #define SR_PAR 0x10 /* 0x10 ~ 0x15 6 bytes for PAR */ -/* Multicast Address Reg */ -#define SR_MAR 0x16 /* 0x16 ~ 0x1D 8 bytes for MAR */ -/* 0x1e unused */ +/* 0x16 --> 0x1E unused */ /* Phy Reset Reg */ #define SR_PRR 0x1F #define PRR_PHY_RST (1 << 0) @@ -161,9 +159,6 @@ /* parameters */ #define SR_SHARE_TIMEOUT 1000 #define SR_EEPROM_LEN 256 -#define SR_MCAST_SIZE 8 -#define SR_MCAST_ADDR_FLAG 0x80 -#define SR_MCAST_MAX 64 #define SR_TX_OVERHEAD 2 /* 2bytes header */ #define SR_RX_OVERHEAD 7 /* 3bytes header + 4crc tail */ diff --git a/drivers/net/usb/usbnet.c b/drivers/net/usb/usbnet.c index 9280ef544bbb90..f9f7506d9427e7 100644 --- a/drivers/net/usb/usbnet.c +++ b/drivers/net/usb/usbnet.c @@ -1821,11 +1821,12 @@ usbnet_probe(struct usb_interface *udev, const struct usb_device_id *prod) if ((dev->driver_info->flags & FLAG_NOARP) != 0) net->flags |= IFF_NOARP; - if (net->max_mtu > (dev->hard_mtu - net->hard_header_len)) + if ((dev->driver_info->flags & FLAG_NOMAXMTU) == 0 && + net->max_mtu > (dev->hard_mtu - net->hard_header_len)) net->max_mtu = dev->hard_mtu - net->hard_header_len; - if (net->mtu > net->max_mtu) - net->mtu = net->max_mtu; + if (net->mtu > (dev->hard_mtu - net->hard_header_len)) + net->mtu = dev->hard_mtu - net->hard_header_len; } else if (!info->in || !info->out) status = usbnet_get_endpoints(dev, udev); diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c index db88dcaefb20b1..dd068a4d256d5a 100644 --- a/drivers/net/virtio_net.c +++ b/drivers/net/virtio_net.c @@ -381,8 +381,6 @@ struct receive_queue { struct xdp_buff **xsk_buffs; }; -#define VIRTIO_NET_RSS_MAX_KEY_SIZE 40 - /* Control VQ buffers: protected by the rtnl lock */ struct control_buf { struct virtio_net_ctrl_hdr hdr; @@ -486,7 +484,7 @@ struct virtnet_info { /* Must be last as it ends in a flexible-array member. */ TRAILING_OVERLAP(struct virtio_net_rss_config_trailer, rss_trailer, hash_key_data, - u8 rss_hash_key_data[VIRTIO_NET_RSS_MAX_KEY_SIZE]; + u8 rss_hash_key_data[NETDEV_RSS_KEY_LEN]; ); }; static_assert(offsetof(struct virtnet_info, rss_trailer.hash_key_data) == @@ -3267,8 +3265,12 @@ static int xmit_skb(struct send_queue *sq, struct sk_buff *skb, bool orphan) struct virtio_net_hdr_v1_hash_tunnel *hdr; int num_sg; unsigned hdr_len = vi->hdr_len; + bool feature_hdrlen; bool can_push; + feature_hdrlen = virtio_has_feature(vi->vdev, + VIRTIO_NET_F_GUEST_HDRLEN); + pr_debug("%s: xmit %p %pM\n", vi->dev->name, skb, dest); /* Make sure it's safe to cast between formats */ @@ -3288,7 +3290,7 @@ static int xmit_skb(struct send_queue *sq, struct sk_buff *skb, bool orphan) if (virtio_net_hdr_tnl_from_skb(skb, hdr, vi->tx_tnl, virtio_is_little_endian(vi->vdev), 0, - false)) + false, feature_hdrlen)) return -EPROTO; if (vi->mergeable_rx_bufs) @@ -3351,6 +3353,7 @@ static netdev_tx_t start_xmit(struct sk_buff *skb, struct net_device *dev) /* Don't wait up for transmitted skbs to be freed. */ if (!use_napi) { skb_orphan(skb); + skb_dst_drop(skb); nf_reset_ct(skb); } @@ -6703,6 +6706,7 @@ static int virtnet_probe(struct virtio_device *vdev) struct virtnet_info *vi; u16 max_queue_pairs; int mtu = 0; + u16 key_sz; /* Find if host supports multiqueue/rss virtio_net device */ max_queue_pairs = 1; @@ -6837,14 +6841,13 @@ static int virtnet_probe(struct virtio_device *vdev) } if (vi->has_rss || vi->has_rss_hash_report) { - vi->rss_key_size = - virtio_cread8(vdev, offsetof(struct virtio_net_config, rss_max_key_size)); - if (vi->rss_key_size > VIRTIO_NET_RSS_MAX_KEY_SIZE) { - dev_err(&vdev->dev, "rss_max_key_size=%u exceeds the limit %u.\n", - vi->rss_key_size, VIRTIO_NET_RSS_MAX_KEY_SIZE); - err = -EINVAL; - goto free; - } + key_sz = virtio_cread8(vdev, offsetof(struct virtio_net_config, rss_max_key_size)); + + vi->rss_key_size = min_t(u16, key_sz, NETDEV_RSS_KEY_LEN); + if (key_sz > vi->rss_key_size) + dev_warn(&vdev->dev, + "rss_max_key_size=%u exceeds driver limit %u, clamping\n", + key_sz, vi->rss_key_size); vi->rss_hash_types_supported = virtio_cread32(vdev, offsetof(struct virtio_net_config, supported_hash_types)); diff --git a/drivers/net/vxlan/vxlan_core.c b/drivers/net/vxlan/vxlan_core.c index e957aa12a8a44a..d2d0e0bd43716c 100644 --- a/drivers/net/vxlan/vxlan_core.c +++ b/drivers/net/vxlan/vxlan_core.c @@ -1965,12 +1965,14 @@ static struct sk_buff *vxlan_na_create(struct sk_buff *request, ns_olen = request->len - skb_network_offset(request) - sizeof(struct ipv6hdr) - sizeof(*ns); for (i = 0; i < ns_olen-1; i += (ns->opt[i+1]<<3)) { - if (!ns->opt[i + 1]) { + if (!ns->opt[i + 1] || i + (ns->opt[i + 1] << 3) > ns_olen) { kfree_skb(reply); return NULL; } if (ns->opt[i] == ND_OPT_SOURCE_LL_ADDR) { - daddr = ns->opt + i + sizeof(struct nd_opt_hdr); + if ((ns->opt[i + 1] << 3) >= + sizeof(struct nd_opt_hdr) + ETH_ALEN) + daddr = ns->opt + i + sizeof(struct nd_opt_hdr); break; } } @@ -2130,6 +2132,11 @@ static bool route_shortcircuit(struct net_device *dev, struct sk_buff *skb) { struct ipv6hdr *pip6; + /* check if nd_tbl is not initiliazed due to + * ipv6.disable=1 set during boot + */ + if (!ipv6_stub->nd_tbl) + return false; if (!pskb_may_pull(skb, sizeof(struct ipv6hdr))) return false; pip6 = ipv6_hdr(skb); diff --git a/drivers/net/wan/farsync.c b/drivers/net/wan/farsync.c index 5b01642ca44e05..6b2d1e63855e8b 100644 --- a/drivers/net/wan/farsync.c +++ b/drivers/net/wan/farsync.c @@ -2550,6 +2550,8 @@ fst_remove_one(struct pci_dev *pdev) fst_disable_intr(card); free_irq(card->irq, card); + tasklet_kill(&fst_tx_task); + tasklet_kill(&fst_int_task); iounmap(card->ctlmem); iounmap(card->mem); diff --git a/drivers/net/wan/fsl_ucc_hdlc.c b/drivers/net/wan/fsl_ucc_hdlc.c index f999798a561271..dff84731343cc6 100644 --- a/drivers/net/wan/fsl_ucc_hdlc.c +++ b/drivers/net/wan/fsl_ucc_hdlc.c @@ -790,18 +790,14 @@ static void uhdlc_memclean(struct ucc_hdlc_private *priv) if (priv->rx_buffer) { dma_free_coherent(priv->dev, - RX_BD_RING_LEN * MAX_RX_BUF_LENGTH, + (RX_BD_RING_LEN + TX_BD_RING_LEN) * MAX_RX_BUF_LENGTH, priv->rx_buffer, priv->dma_rx_addr); priv->rx_buffer = NULL; priv->dma_rx_addr = 0; - } - if (priv->tx_buffer) { - dma_free_coherent(priv->dev, - TX_BD_RING_LEN * MAX_RX_BUF_LENGTH, - priv->tx_buffer, priv->dma_tx_addr); priv->tx_buffer = NULL; priv->dma_tx_addr = 0; + } } diff --git a/drivers/net/wan/lapbether.c b/drivers/net/wan/lapbether.c index f357a7ac70ac47..9861c99ea56c4e 100644 --- a/drivers/net/wan/lapbether.c +++ b/drivers/net/wan/lapbether.c @@ -446,33 +446,36 @@ static void lapbeth_free_device(struct lapbethdev *lapbeth) static int lapbeth_device_event(struct notifier_block *this, unsigned long event, void *ptr) { - struct lapbethdev *lapbeth; struct net_device *dev = netdev_notifier_info_to_dev(ptr); + struct lapbethdev *lapbeth; if (dev_net(dev) != &init_net) return NOTIFY_DONE; - if (!dev_is_ethdev(dev) && !lapbeth_get_x25_dev(dev)) + lapbeth = lapbeth_get_x25_dev(dev); + if (!dev_is_ethdev(dev) && !lapbeth) return NOTIFY_DONE; switch (event) { case NETDEV_UP: /* New ethernet device -> new LAPB interface */ - if (!lapbeth_get_x25_dev(dev)) + if (!lapbeth) lapbeth_new_device(dev); break; case NETDEV_GOING_DOWN: /* ethernet device closes -> close LAPB interface */ - lapbeth = lapbeth_get_x25_dev(dev); if (lapbeth) dev_close(lapbeth->axdev); break; case NETDEV_UNREGISTER: /* ethernet device disappears -> remove LAPB interface */ - lapbeth = lapbeth_get_x25_dev(dev); if (lapbeth) lapbeth_free_device(lapbeth); break; + case NETDEV_PRE_TYPE_CHANGE: + /* Our underlying device type must not change. */ + if (lapbeth) + return NOTIFY_BAD; } return NOTIFY_DONE; diff --git a/drivers/net/wireguard/device.c b/drivers/net/wireguard/device.c index 46a71ec36af870..67b07ee2d66003 100644 --- a/drivers/net/wireguard/device.c +++ b/drivers/net/wireguard/device.c @@ -411,12 +411,11 @@ static struct rtnl_link_ops link_ops __read_mostly = { .newlink = wg_newlink, }; -static void wg_netns_pre_exit(struct net *net) +static void __net_exit wg_netns_exit_rtnl(struct net *net, struct list_head *dev_kill_list) { struct wg_device *wg; struct wg_peer *peer; - rtnl_lock(); list_for_each_entry(wg, &device_list, device_list) { if (rcu_access_pointer(wg->creating_net) == net) { pr_debug("%s: Creating namespace exiting\n", wg->dev->name); @@ -429,11 +428,10 @@ static void wg_netns_pre_exit(struct net *net) mutex_unlock(&wg->device_update_lock); } } - rtnl_unlock(); } -static struct pernet_operations pernet_ops = { - .pre_exit = wg_netns_pre_exit +static struct pernet_operations pernet_ops __read_mostly = { + .exit_rtnl = wg_netns_exit_rtnl }; int __init wg_device_init(void) diff --git a/drivers/net/wireless/ath/ath10k/sdio.c b/drivers/net/wireless/ath/ath10k/sdio.c index c06d50db40b818..00d0556dafefdb 100644 --- a/drivers/net/wireless/ath/ath10k/sdio.c +++ b/drivers/net/wireless/ath/ath10k/sdio.c @@ -2487,7 +2487,11 @@ void ath10k_sdio_fw_crashed_dump(struct ath10k *ar) if (fast_dump) ath10k_bmi_start(ar); + mutex_lock(&ar->dump_mutex); + + spin_lock_bh(&ar->data_lock); ar->stats.fw_crash_counter++; + spin_unlock_bh(&ar->data_lock); ath10k_sdio_disable_intrs(ar); @@ -2505,6 +2509,8 @@ void ath10k_sdio_fw_crashed_dump(struct ath10k *ar) ath10k_sdio_enable_intrs(ar); + mutex_unlock(&ar->dump_mutex); + ath10k_core_start_recovery(ar); } diff --git a/drivers/net/wireless/ath/ath10k/wmi.c b/drivers/net/wireless/ath/ath10k/wmi.c index b4aad6604d6d9d..ce22141e5efd9e 100644 --- a/drivers/net/wireless/ath/ath10k/wmi.c +++ b/drivers/net/wireless/ath/ath10k/wmi.c @@ -5289,8 +5289,6 @@ ath10k_wmi_event_peer_sta_ps_state_chg(struct ath10k *ar, struct sk_buff *skb) struct ath10k_sta *arsta; u8 peer_addr[ETH_ALEN]; - lockdep_assert_held(&ar->data_lock); - ev = (struct wmi_peer_sta_ps_state_chg_event *)skb->data; ether_addr_copy(peer_addr, ev->peer_macaddr.addr); @@ -5305,7 +5303,9 @@ ath10k_wmi_event_peer_sta_ps_state_chg(struct ath10k *ar, struct sk_buff *skb) } arsta = (struct ath10k_sta *)sta->drv_priv; + spin_lock_bh(&ar->data_lock); arsta->peer_ps_state = __le32_to_cpu(ev->peer_ps_state); + spin_unlock_bh(&ar->data_lock); exit: rcu_read_unlock(); diff --git a/drivers/net/wireless/ath/ath11k/core.c b/drivers/net/wireless/ath/ath11k/core.c index 812686173ac8a9..78a1b0edd8b453 100644 --- a/drivers/net/wireless/ath/ath11k/core.c +++ b/drivers/net/wireless/ath/ath11k/core.c @@ -994,9 +994,64 @@ static const struct dmi_system_id ath11k_pm_quirk_table[] = { DMI_MATCH(DMI_PRODUCT_NAME, "21F9"), }, }, + { + .driver_data = (void *)ATH11K_PM_WOW, + .matches = { /* Z13 G1 */ + DMI_MATCH(DMI_BOARD_VENDOR, "LENOVO"), + DMI_MATCH(DMI_PRODUCT_NAME, "21D2"), + }, + }, + { + .driver_data = (void *)ATH11K_PM_WOW, + .matches = { /* Z13 G1 */ + DMI_MATCH(DMI_BOARD_VENDOR, "LENOVO"), + DMI_MATCH(DMI_PRODUCT_NAME, "21D3"), + }, + }, + { + .driver_data = (void *)ATH11K_PM_WOW, + .matches = { /* Z16 G1 */ + DMI_MATCH(DMI_BOARD_VENDOR, "LENOVO"), + DMI_MATCH(DMI_PRODUCT_NAME, "21D4"), + }, + }, + { + .driver_data = (void *)ATH11K_PM_WOW, + .matches = { /* Z16 G1 */ + DMI_MATCH(DMI_BOARD_VENDOR, "LENOVO"), + DMI_MATCH(DMI_PRODUCT_NAME, "21D5"), + }, + }, {} }; +static const struct __ath11k_core_usecase_firmware_table { + u32 hw_rev; + const char *compatible; + const char *firmware_name; +} ath11k_core_usecase_firmware_table[] = { + { ATH11K_HW_WCN6855_HW21, "qcom,lemans-evk", "nfa765"}, + { ATH11K_HW_WCN6855_HW21, "qcom,monaco-evk", "nfa765"}, + { ATH11K_HW_WCN6855_HW21, "qcom,hamoa-iot-evk", "nfa765"}, + { /* Sentinel */ } +}; + +const char *ath11k_core_get_usecase_firmware(struct ath11k_base *ab) +{ + const struct __ath11k_core_usecase_firmware_table *entry = NULL; + + entry = ath11k_core_usecase_firmware_table; + while (entry->compatible) { + if (ab->hw_rev == entry->hw_rev && + of_machine_is_compatible(entry->compatible)) + return entry->firmware_name; + entry++; + } + + return NULL; +} +EXPORT_SYMBOL(ath11k_core_get_usecase_firmware); + void ath11k_fw_stats_pdevs_free(struct list_head *head) { struct ath11k_fw_stats_pdev *i, *tmp; diff --git a/drivers/net/wireless/ath/ath11k/core.h b/drivers/net/wireless/ath/ath11k/core.h index e8780b05ce11e3..834988dad591c7 100644 --- a/drivers/net/wireless/ath/ath11k/core.h +++ b/drivers/net/wireless/ath/ath11k/core.h @@ -1275,6 +1275,7 @@ bool ath11k_core_coldboot_cal_support(struct ath11k_base *ab); const struct firmware *ath11k_core_firmware_request(struct ath11k_base *ab, const char *filename); +const char *ath11k_core_get_usecase_firmware(struct ath11k_base *ab); static inline const char *ath11k_scan_state_str(enum ath11k_scan_state state) { @@ -1329,6 +1330,9 @@ static inline void ath11k_core_create_firmware_path(struct ath11k_base *ab, of_property_read_string(ab->dev->of_node, "firmware-name", &fw_name); + if (!fw_name) + fw_name = ath11k_core_get_usecase_firmware(ab); + if (fw_name && strncmp(filename, "board", 5)) snprintf(buf, buf_len, "%s/%s/%s/%s", ATH11K_FW_DIR, ab->hw_params.fw.dir, fw_name, filename); diff --git a/drivers/net/wireless/ath/ath11k/dp_rx.c b/drivers/net/wireless/ath/ath11k/dp_rx.c index b9e976ddcbbf68..44eea682c297b6 100644 --- a/drivers/net/wireless/ath/ath11k/dp_rx.c +++ b/drivers/net/wireless/ath/ath11k/dp_rx.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (c) 2018-2019 The Linux Foundation. All rights reserved. - * Copyright (c) 2021-2025 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */ #include @@ -1110,9 +1110,8 @@ int ath11k_dp_rx_ampdu_stop(struct ath11k *ar, struct ath11k_base *ab = ar->ab; struct ath11k_peer *peer; struct ath11k_sta *arsta = ath11k_sta_to_arsta(params->sta); + struct dp_rx_tid *rx_tid; int vdev_id = arsta->arvif->vdev_id; - dma_addr_t paddr; - bool active; int ret; spin_lock_bh(&ab->base_lock); @@ -1124,15 +1123,14 @@ int ath11k_dp_rx_ampdu_stop(struct ath11k *ar, return -ENOENT; } - paddr = peer->rx_tid[params->tid].paddr; - active = peer->rx_tid[params->tid].active; + rx_tid = &peer->rx_tid[params->tid]; - if (!active) { + if (!rx_tid->active) { spin_unlock_bh(&ab->base_lock); return 0; } - ret = ath11k_peer_rx_tid_reo_update(ar, peer, peer->rx_tid, 1, 0, false); + ret = ath11k_peer_rx_tid_reo_update(ar, peer, rx_tid, 1, 0, false); spin_unlock_bh(&ab->base_lock); if (ret) { ath11k_warn(ab, "failed to update reo for rx tid %d: %d\n", @@ -1141,7 +1139,8 @@ int ath11k_dp_rx_ampdu_stop(struct ath11k *ar, } ret = ath11k_wmi_peer_rx_reorder_queue_setup(ar, vdev_id, - params->sta->addr, paddr, + params->sta->addr, + rx_tid->paddr, params->tid, 1, 1); if (ret) ath11k_warn(ab, "failed to send wmi to delete rx tid %d\n", diff --git a/drivers/net/wireless/ath/ath11k/mhi.c b/drivers/net/wireless/ath/ath11k/mhi.c index acd76e9392d311..d2c44f7f9b6226 100644 --- a/drivers/net/wireless/ath/ath11k/mhi.c +++ b/drivers/net/wireless/ath/ath11k/mhi.c @@ -34,7 +34,6 @@ static const struct mhi_channel_config ath11k_mhi_channels_qca6390[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, }, { .num = 21, @@ -48,7 +47,6 @@ static const struct mhi_channel_config ath11k_mhi_channels_qca6390[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = true, }, }; @@ -99,7 +97,6 @@ static const struct mhi_channel_config ath11k_mhi_channels_qcn9074[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, }, { .num = 21, @@ -113,7 +110,6 @@ static const struct mhi_channel_config ath11k_mhi_channels_qcn9074[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = true, }, }; diff --git a/drivers/net/wireless/ath/ath11k/reg.c b/drivers/net/wireless/ath/ath11k/reg.c index d62a2014315a08..49b79648752cf6 100644 --- a/drivers/net/wireless/ath/ath11k/reg.c +++ b/drivers/net/wireless/ath/ath11k/reg.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (c) 2018-2019 The Linux Foundation. All rights reserved. - * Copyright (c) 2021-2025 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */ #include @@ -926,8 +926,11 @@ int ath11k_reg_handle_chan_list(struct ath11k_base *ab, */ if (ab->default_regd[pdev_idx] && !ab->new_regd[pdev_idx] && !memcmp((char *)ab->default_regd[pdev_idx]->alpha2, - (char *)reg_info->alpha2, 2)) - goto retfail; + (char *)reg_info->alpha2, 2) && + power_type == IEEE80211_REG_UNSET_AP) { + ath11k_reg_reset_info(reg_info); + return 0; + } /* Intersect new rules with default regd if a new country setting was * requested, i.e a default regd was already set during initialization diff --git a/drivers/net/wireless/ath/ath12k/debugfs_htt_stats.c b/drivers/net/wireless/ath/ath12k/debugfs_htt_stats.c index 48b010a1b7566d..4f749d473d0e1f 100644 --- a/drivers/net/wireless/ath/ath12k/debugfs_htt_stats.c +++ b/drivers/net/wireless/ath/ath12k/debugfs_htt_stats.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved. - * Copyright (c) 2021-2025 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */ #include @@ -29,8 +29,10 @@ print_array_to_buf_index(u8 *buf, u32 offset, const char *header, u32 stats_inde " %u:%u,", stats_index++, le32_to_cpu(array[i])); } /* To overwrite the last trailing comma */ - index--; - *(buf + offset + index) = '\0'; + if (array_len > 0) { + index--; + *(buf + offset + index) = '\0'; + } if (footer) { index += scnprintf(buf + offset + index, diff --git a/drivers/net/wireless/ath/ath12k/mac.c b/drivers/net/wireless/ath/ath12k/mac.c index e0e49f782bf8df..63684ff9332d4f 100644 --- a/drivers/net/wireless/ath/ath12k/mac.c +++ b/drivers/net/wireless/ath/ath12k/mac.c @@ -4175,8 +4175,10 @@ ath12k_mac_op_change_vif_links(struct ieee80211_hw *hw, if (WARN_ON(!arvif)) return -EINVAL; - if (!arvif->is_created) + if (!arvif->is_created) { + ath12k_mac_unassign_link_vif(arvif); continue; + } if (WARN_ON(!arvif->ar)) return -EINVAL; diff --git a/drivers/net/wireless/ath/ath12k/mhi.c b/drivers/net/wireless/ath/ath12k/mhi.c index 08f44baf182a5e..2dbdb95ae7bea6 100644 --- a/drivers/net/wireless/ath/ath12k/mhi.c +++ b/drivers/net/wireless/ath/ath12k/mhi.c @@ -31,7 +31,6 @@ static const struct mhi_channel_config ath12k_mhi_channels_qcn9274[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, }, { .num = 21, @@ -45,7 +44,6 @@ static const struct mhi_channel_config ath12k_mhi_channels_qcn9274[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = true, }, }; @@ -96,7 +94,6 @@ static const struct mhi_channel_config ath12k_mhi_channels_wcn7850[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, }, { .num = 21, @@ -110,7 +107,6 @@ static const struct mhi_channel_config ath12k_mhi_channels_wcn7850[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = true, }, }; diff --git a/drivers/net/wireless/ath/ath12k/wmi.c b/drivers/net/wireless/ath/ath12k/wmi.c index 3ce5fcb0e46004..1613492b383501 100644 --- a/drivers/net/wireless/ath/ath12k/wmi.c +++ b/drivers/net/wireless/ath/ath12k/wmi.c @@ -496,6 +496,7 @@ ath12k_pull_mac_phy_cap_svc_ready_ext(struct ath12k_wmi_pdev *wmi_handle, struct ath12k_band_cap *cap_band; struct ath12k_pdev_cap *pdev_cap = &pdev->cap; struct ath12k_fw_pdev *fw_pdev; + u32 supported_bands; u32 phy_map; u32 hw_idx, phy_idx = 0; int i; @@ -519,14 +520,19 @@ ath12k_pull_mac_phy_cap_svc_ready_ext(struct ath12k_wmi_pdev *wmi_handle, return -EINVAL; mac_caps = wmi_mac_phy_caps + phy_idx; + supported_bands = le32_to_cpu(mac_caps->supported_bands); + + if (!(supported_bands & WMI_HOST_WLAN_2GHZ_CAP) && + !(supported_bands & WMI_HOST_WLAN_5GHZ_CAP)) + return -EINVAL; pdev->pdev_id = ath12k_wmi_mac_phy_get_pdev_id(mac_caps); pdev->hw_link_id = ath12k_wmi_mac_phy_get_hw_link_id(mac_caps); - pdev_cap->supported_bands |= le32_to_cpu(mac_caps->supported_bands); + pdev_cap->supported_bands |= supported_bands; pdev_cap->ampdu_density = le32_to_cpu(mac_caps->ampdu_density); fw_pdev = &ab->fw_pdev[ab->fw_pdev_count]; - fw_pdev->supported_bands = le32_to_cpu(mac_caps->supported_bands); + fw_pdev->supported_bands = supported_bands; fw_pdev->pdev_id = ath12k_wmi_mac_phy_get_pdev_id(mac_caps); fw_pdev->phy_id = le32_to_cpu(mac_caps->phy_id); ab->fw_pdev_count++; @@ -535,10 +541,12 @@ ath12k_pull_mac_phy_cap_svc_ready_ext(struct ath12k_wmi_pdev *wmi_handle, * band to band for a single radio, need to see how this should be * handled. */ - if (le32_to_cpu(mac_caps->supported_bands) & WMI_HOST_WLAN_2GHZ_CAP) { + if (supported_bands & WMI_HOST_WLAN_2GHZ_CAP) { pdev_cap->tx_chain_mask = le32_to_cpu(mac_caps->tx_chain_mask_2g); pdev_cap->rx_chain_mask = le32_to_cpu(mac_caps->rx_chain_mask_2g); - } else if (le32_to_cpu(mac_caps->supported_bands) & WMI_HOST_WLAN_5GHZ_CAP) { + } + + if (supported_bands & WMI_HOST_WLAN_5GHZ_CAP) { pdev_cap->vht_cap = le32_to_cpu(mac_caps->vht_cap_info_5g); pdev_cap->vht_mcs = le32_to_cpu(mac_caps->vht_supp_mcs_5g); pdev_cap->he_mcs = le32_to_cpu(mac_caps->he_supp_mcs_5g); @@ -548,8 +556,6 @@ ath12k_pull_mac_phy_cap_svc_ready_ext(struct ath12k_wmi_pdev *wmi_handle, WMI_NSS_RATIO_EN_DIS_GET(mac_caps->nss_ratio); pdev_cap->nss_ratio_info = WMI_NSS_RATIO_INFO_GET(mac_caps->nss_ratio); - } else { - return -EINVAL; } /* tx/rx chainmask reported from fw depends on the actual hw chains used, @@ -565,7 +571,7 @@ ath12k_pull_mac_phy_cap_svc_ready_ext(struct ath12k_wmi_pdev *wmi_handle, pdev_cap->rx_chain_mask_shift = find_first_bit((unsigned long *)&pdev_cap->rx_chain_mask, 32); - if (le32_to_cpu(mac_caps->supported_bands) & WMI_HOST_WLAN_2GHZ_CAP) { + if (supported_bands & WMI_HOST_WLAN_2GHZ_CAP) { cap_band = &pdev_cap->band[NL80211_BAND_2GHZ]; cap_band->phy_id = le32_to_cpu(mac_caps->phy_id); cap_band->max_bw_supported = le32_to_cpu(mac_caps->max_bw_supported_2g); @@ -585,7 +591,7 @@ ath12k_pull_mac_phy_cap_svc_ready_ext(struct ath12k_wmi_pdev *wmi_handle, le32_to_cpu(mac_caps->he_ppet2g.ppet16_ppet8_ru3_ru0[i]); } - if (le32_to_cpu(mac_caps->supported_bands) & WMI_HOST_WLAN_5GHZ_CAP) { + if (supported_bands & WMI_HOST_WLAN_5GHZ_CAP) { cap_band = &pdev_cap->band[NL80211_BAND_5GHZ]; cap_band->phy_id = le32_to_cpu(mac_caps->phy_id); cap_band->max_bw_supported = @@ -4545,7 +4551,7 @@ static int ath12k_wmi_hw_mode_caps(struct ath12k_base *soc, pref = soc->wmi_ab.preferred_hw_mode; - if (ath12k_hw_mode_pri_map[mode] < ath12k_hw_mode_pri_map[pref]) { + if (ath12k_hw_mode_pri_map[mode] <= ath12k_hw_mode_pri_map[pref]) { svc_rdy_ext->pref_hw_mode_caps = *hw_mode_caps; soc->wmi_ab.preferred_hw_mode = mode; } diff --git a/drivers/net/wireless/ath/ath12k/wow.c b/drivers/net/wireless/ath/ath12k/wow.c index e8481626f19404..c78aa95d497911 100644 --- a/drivers/net/wireless/ath/ath12k/wow.c +++ b/drivers/net/wireless/ath/ath12k/wow.c @@ -135,6 +135,9 @@ static int ath12k_wow_cleanup(struct ath12k *ar) lockdep_assert_wiphy(ath12k_ar_to_hw(ar)->wiphy); list_for_each_entry(arvif, &ar->arvifs, list) { + if (arvif != &arvif->ahvif->deflink) + continue; + ret = ath12k_wow_vif_cleanup(arvif); if (ret) { ath12k_warn(ar->ab, "failed to clean wow wakeups on vdev %i: %d\n", @@ -479,8 +482,12 @@ static int ath12k_wow_set_wakeups(struct ath12k *ar, lockdep_assert_wiphy(ath12k_ar_to_hw(ar)->wiphy); list_for_each_entry(arvif, &ar->arvifs, list) { + if (arvif != &arvif->ahvif->deflink) + continue; + if (ath12k_wow_is_p2p_vdev(arvif->ahvif)) continue; + ret = ath12k_wow_vif_set_wakeups(arvif, wowlan); if (ret) { ath12k_warn(ar->ab, "failed to set wow wakeups on vdev %i: %d\n", @@ -538,6 +545,9 @@ static int ath12k_wow_nlo_cleanup(struct ath12k *ar) lockdep_assert_wiphy(ath12k_ar_to_hw(ar)->wiphy); list_for_each_entry(arvif, &ar->arvifs, list) { + if (arvif != &arvif->ahvif->deflink) + continue; + if (ath12k_wow_is_p2p_vdev(arvif->ahvif)) continue; @@ -745,6 +755,9 @@ static int ath12k_wow_arp_ns_offload(struct ath12k *ar, bool enable) list_for_each_entry(arvif, &ar->arvifs, list) { ahvif = arvif->ahvif; + if (arvif != &ahvif->deflink) + continue; + if (ahvif->vdev_type != WMI_VDEV_TYPE_STA) continue; @@ -776,6 +789,9 @@ static int ath12k_gtk_rekey_offload(struct ath12k *ar, bool enable) lockdep_assert_wiphy(ath12k_ar_to_hw(ar)->wiphy); list_for_each_entry(arvif, &ar->arvifs, list) { + if (arvif != &arvif->ahvif->deflink) + continue; + if (arvif->ahvif->vdev_type != WMI_VDEV_TYPE_STA || !arvif->is_up || !arvif->rekey_data.enable_offload) diff --git a/drivers/net/wireless/ath/ath9k/Kconfig b/drivers/net/wireless/ath/ath9k/Kconfig index 0c47be06c153be..47d570a5ca6a1c 100644 --- a/drivers/net/wireless/ath/ath9k/Kconfig +++ b/drivers/net/wireless/ath/ath9k/Kconfig @@ -47,7 +47,7 @@ config ATH9K_PCI config ATH9K_AHB bool "Atheros ath9k AHB bus support" - depends on ATH9K + depends on ATH9K && OF default n help This option enables the AHB bus support in ath9k. diff --git a/drivers/net/wireless/ath/ath9k/channel.c b/drivers/net/wireless/ath/ath9k/channel.c index 121e51ce1bc0ea..8b27d8cc086ab7 100644 --- a/drivers/net/wireless/ath/ath9k/channel.c +++ b/drivers/net/wireless/ath/ath9k/channel.c @@ -1006,7 +1006,7 @@ static void ath_scan_send_probe(struct ath_softc *sc, skb_set_queue_mapping(skb, IEEE80211_AC_VO); if (!ieee80211_tx_prepare_skb(sc->hw, vif, skb, band, NULL)) - goto error; + return; txctl.txq = sc->tx.txq_map[IEEE80211_AC_VO]; if (ath_tx_start(sc->hw, skb, &txctl)) @@ -1119,10 +1119,8 @@ ath_chanctx_send_vif_ps_frame(struct ath_softc *sc, struct ath_vif *avp, skb->priority = 7; skb_set_queue_mapping(skb, IEEE80211_AC_VO); - if (!ieee80211_tx_prepare_skb(sc->hw, vif, skb, band, &sta)) { - dev_kfree_skb_any(skb); + if (!ieee80211_tx_prepare_skb(sc->hw, vif, skb, band, &sta)) return false; - } break; default: return false; diff --git a/drivers/net/wireless/ath/ath9k/common-debug.h b/drivers/net/wireless/ath/ath9k/common-debug.h index 2938b5b96b0749..97948af97682b9 100644 --- a/drivers/net/wireless/ath/ath9k/common-debug.h +++ b/drivers/net/wireless/ath/ath9k/common-debug.h @@ -19,14 +19,14 @@ /** * struct ath_rx_stats - RX Statistics * @rx_pkts_all: No. of total frames received, including ones that - may have had errors. + * may have had errors. * @rx_bytes_all: No. of total bytes received, including ones that - may have had errors. + * may have had errors. * @crc_err: No. of frames with incorrect CRC value * @decrypt_crc_err: No. of frames whose CRC check failed after - decryption process completed + * decryption process completed * @phy_err: No. of frames whose reception failed because the PHY - encountered an error + * encountered an error * @mic_err: No. of frames with incorrect TKIP MIC verification failure * @pre_delim_crc_err: Pre-Frame delimiter CRC error detections * @post_delim_crc_err: Post-Frame delimiter CRC error detections diff --git a/drivers/net/wireless/ath/ath9k/debug.h b/drivers/net/wireless/ath/ath9k/debug.h index cb3e75969875af..804e2a0a0c20f8 100644 --- a/drivers/net/wireless/ath/ath9k/debug.h +++ b/drivers/net/wireless/ath/ath9k/debug.h @@ -142,11 +142,12 @@ struct ath_interrupt_stats { /** * struct ath_tx_stats - Statistics about TX * @tx_pkts_all: No. of total frames transmitted, including ones that - may have had errors. + * may have had errors. * @tx_bytes_all: No. of total bytes transmitted, including ones that - may have had errors. + * may have had errors. * @queued: Total MPDUs (non-aggr) queued * @completed: Total MPDUs (non-aggr) completed + * @xretries: Total MPDUs with xretries * @a_aggr: Total no. of aggregates queued * @a_queued_hw: Total AMPDUs queued to hardware * @a_completed: Total AMPDUs completed @@ -154,14 +155,14 @@ struct ath_interrupt_stats { * @a_xretries: No. of AMPDUs dropped due to xretries * @txerr_filtered: No. of frames with TXERR_FILT flag set. * @fifo_underrun: FIFO underrun occurrences - Valid only for: - - non-aggregate condition. - - first packet of aggregate. + * Valid only for: + * - non-aggregate condition. + * - first packet of aggregate. * @xtxop: No. of frames filtered because of TXOP limit * @timer_exp: Transmit timer expiry * @desc_cfg_err: Descriptor configuration errors - * @data_urn: TX data underrun errors - * @delim_urn: TX delimiter underrun errors + * @data_underrun: TX data underrun errors + * @delim_underrun: TX delimiter underrun errors * @puttxbuf: Number of times hardware was given txbuf to write. * @txstart: Number of times hardware was told to start tx. * @txprocdesc: Number of times tx descriptor was processed diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/Makefile b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/Makefile index e5ca0f51182271..6fd805023500be 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/Makefile +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/Makefile @@ -25,7 +25,11 @@ brcmfmac-objs += \ btcoex.o \ vendor.o \ pno.o \ - xtlv.o + join_param.o \ + scan_param.o \ + xtlv.o \ + interface_create.o + brcmfmac-$(CONFIG_BRCMFMAC_PROTO_BCDC) += \ bcdc.o \ fwsignal.o diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c index 6a3f187320fc41..13952dfeb3e30c 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c @@ -951,11 +951,10 @@ int brcmf_sdiod_probe(struct brcmf_sdio_dev *sdiodev) goto out; /* try to attach to the target device */ - sdiodev->bus = brcmf_sdio_probe(sdiodev); - if (IS_ERR(sdiodev->bus)) { - ret = PTR_ERR(sdiodev->bus); + ret = brcmf_sdio_probe(sdiodev); + if (ret) goto out; - } + brcmf_sdiod_host_fixup(sdiodev->func2->card->host); out: if (ret) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bus.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bus.h index fe31051a9e11b1..5efd7f6d757a4c 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bus.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bus.h @@ -107,6 +107,7 @@ struct brcmf_bus_ops { void (*debugfs_create)(struct device *dev); int (*reset)(struct device *dev); void (*remove)(struct device *dev); + void (*d2h_mb_rx)(struct device *dev, u32 data); }; @@ -286,6 +287,15 @@ static inline void brcmf_bus_remove(struct brcmf_bus *bus) bus->ops->remove(bus->dev); } +static inline +void brcmf_bus_d2h_mb_rx(struct brcmf_bus *bus, u32 data) +{ + if (!bus->ops->d2h_mb_rx) + return; + + return bus->ops->d2h_mb_rx(bus->dev, data); +} + /* * interface functions from common layer */ diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index bb96b87b2a6e56..a87965c5d1226a 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -32,7 +32,11 @@ #include "vendor.h" #include "bus.h" #include "common.h" +#include "feature.h" #include "fwvid.h" +#include "xtlv.h" +#include "ratespec.h" +#include "interface_create.h" #define BRCMF_SCAN_IE_LEN_MAX 2048 @@ -64,6 +68,8 @@ #define RSN_CAP_MFPR_MASK BIT(6) #define RSN_CAP_MFPC_MASK BIT(7) #define RSN_PMKID_COUNT_LEN 2 +#define DPP_AKM_SUITE_TYPE 2 +#define WLAN_AKM_SUITE_DPP SUITE(WLAN_OUI_WFA, DPP_AKM_SUITE_TYPE) #define VNDR_IE_CMD_LEN 4 /* length of the set command * string :"add", "del" (+ NUL) @@ -76,10 +82,6 @@ #define DOT11_BCN_PRB_FIXED_LEN 12 /* beacon/probe fixed length */ -#define BRCMF_SCAN_JOIN_ACTIVE_DWELL_TIME_MS 320 -#define BRCMF_SCAN_JOIN_PASSIVE_DWELL_TIME_MS 400 -#define BRCMF_SCAN_JOIN_PROBE_INTERVAL_MS 20 - #define BRCMF_SCAN_CHANNEL_TIME 40 #define BRCMF_SCAN_UNASSOC_TIME 40 #define BRCMF_SCAN_PASSIVE_TIME 120 @@ -98,9 +100,6 @@ #define PKT_TOKEN_IDX 15 #define IDLE_TOKEN_IDX 12 -#define BRCMF_ASSOC_PARAMS_FIXED_SIZE \ - (sizeof(struct brcmf_assoc_params_le) - sizeof(u16)) - #define BRCMF_MAX_CHANSPEC_LIST \ (BRCMF_DCMD_MEDLEN / sizeof(__le32) - 1) @@ -124,6 +123,13 @@ struct cca_msrmnt_query { u32 time_req; }; +/* algo bit vector */ +#define KEY_ALGO_MASK(_algo) (1 << (_algo)) + +/* start enum value for BSS properties */ +#define WL_WSEC_INFO_BSS_BASE 0x0100 +#define WL_WSEC_INFO_BSS_ALGOS (WL_WSEC_INFO_BSS_BASE + 6) + static bool check_vif_up(struct brcmf_cfg80211_vif *vif) { if (!test_bit(BRCMF_VIF_STATUS_READY, &vif->sme_state)) { @@ -178,6 +184,15 @@ static struct ieee80211_rate __wl_rates[] = { .max_power = 30, \ } +#define CHAN6G(_channel) { \ + .band = NL80211_BAND_6GHZ, \ + .center_freq = ((_channel == 2) ? 5935 : 5950 + (5 * (_channel))), \ + .hw_value = (_channel), \ + .max_antenna_gain = 0, \ + .max_power = 30, \ +} + + static struct ieee80211_channel __wl_2ghz_channels[] = { CHAN2G(1, 2412), CHAN2G(2, 2417), CHAN2G(3, 2422), CHAN2G(4, 2427), CHAN2G(5, 2432), CHAN2G(6, 2437), CHAN2G(7, 2442), CHAN2G(8, 2447), @@ -194,6 +209,23 @@ static struct ieee80211_channel __wl_5ghz_channels[] = { CHAN5G(153), CHAN5G(157), CHAN5G(161), CHAN5G(165) }; +static struct ieee80211_channel __wl_6ghz_channels[] = { + CHAN6G(1), CHAN6G(2), CHAN6G(5), CHAN6G(9), CHAN6G(13), + CHAN6G(17), CHAN6G(21), CHAN6G(25), CHAN6G(29), CHAN6G(33), + CHAN6G(37), CHAN6G(41), CHAN6G(45), CHAN6G(49), CHAN6G(53), + CHAN6G(57), CHAN6G(61), CHAN6G(65), CHAN6G(69), CHAN6G(73), + CHAN6G(77), CHAN6G(81), CHAN6G(85), CHAN6G(89), CHAN6G(93), + CHAN6G(97), CHAN6G(101), CHAN6G(105), CHAN6G(109), CHAN6G(113), + CHAN6G(117), CHAN6G(121), CHAN6G(125), CHAN6G(129), CHAN6G(133), + CHAN6G(137), CHAN6G(141), CHAN6G(145), CHAN6G(149), CHAN6G(153), + CHAN6G(157), CHAN6G(161), CHAN6G(165), CHAN6G(169), CHAN6G(173), + CHAN6G(177), CHAN6G(181), CHAN6G(185), CHAN6G(189), CHAN6G(193), + CHAN6G(197), CHAN6G(201), CHAN6G(205), CHAN6G(209), CHAN6G(213), + CHAN6G(217), CHAN6G(221), CHAN6G(225), CHAN6G(229), CHAN6G(233), +}; + +struct ieee80211_sband_iftype_data sdata[NUM_NL80211_BANDS]; + /* Band templates duplicated per wiphy. The channel info * above is added to the band during setup. */ @@ -209,6 +241,12 @@ static const struct ieee80211_supported_band __wl_band_5ghz = { .n_bitrates = wl_a_rates_size, }; +static const struct ieee80211_supported_band __wl_band_6ghz = { + .band = NL80211_BAND_6GHZ, + .bitrates = wl_a_rates, + .n_bitrates = wl_a_rates_size, +}; + /* This is to override regulatory domains defined in cfg80211 module (reg.c) * By default world regulatory domain defined in reg.c puts the flags * NL80211_RRF_NO_IR for 5GHz channels (for * 36..48 and 149..165). @@ -217,35 +255,43 @@ static const struct ieee80211_supported_band __wl_band_5ghz = { * domain are to be done here. */ static const struct ieee80211_regdomain brcmf_regdom = { - .n_reg_rules = 4, + .n_reg_rules = 5, .alpha2 = "99", .reg_rules = { /* IEEE 802.11b/g, channels 1..11 */ - REG_RULE(2412-10, 2472+10, 40, 6, 20, 0), + REG_RULE(2412 - 10, 2472 + 10, 40, 6, 20, 0), /* If any */ /* IEEE 802.11 channel 14 - Only JP enables * this and for 802.11b only */ - REG_RULE(2484-10, 2484+10, 20, 6, 20, 0), + REG_RULE(2484 - 10, 2484 + 10, 20, 6, 20, 0), /* IEEE 802.11a, channel 36..64 */ - REG_RULE(5150-10, 5350+10, 160, 6, 20, 0), + REG_RULE(5150 - 10, 5350 + 10, 160, 6, 20, 0), /* IEEE 802.11a, channel 100..165 */ - REG_RULE(5470-10, 5850+10, 160, 6, 20, 0), } + REG_RULE(5470 - 10, 5850 + 10, 160, 6, 20, 0), + /* IEEE 802.11ax, 6E */ + REG_RULE(5935 - 10, 7115 + 10, 160, 6, 20, 0), } }; /* Note: brcmf_cipher_suites is an array of int defining which cipher suites * are supported. A pointer to this array and the number of entries is passed * on to upper layers. AES_CMAC defines whether or not the driver supports MFP. - * So the cipher suite AES_CMAC has to be the last one in the array, and when - * device does not support MFP then the number of suites will be decreased by 1 + * MFP support includes a few other suites, so if MFP is not supported, + * then the number of suites will be decreased by 4 */ static const u32 brcmf_cipher_suites[] = { WLAN_CIPHER_SUITE_WEP40, WLAN_CIPHER_SUITE_WEP104, WLAN_CIPHER_SUITE_TKIP, WLAN_CIPHER_SUITE_CCMP, - /* Keep as last entry: */ - WLAN_CIPHER_SUITE_AES_CMAC + WLAN_CIPHER_SUITE_CCMP_256, + WLAN_CIPHER_SUITE_GCMP, + WLAN_CIPHER_SUITE_GCMP_256, + /* Keep as last 4 entries: */ + WLAN_CIPHER_SUITE_AES_CMAC, + WLAN_CIPHER_SUITE_BIP_CMAC_256, + WLAN_CIPHER_SUITE_BIP_GMAC_128, + WLAN_CIPHER_SUITE_BIP_GMAC_256 }; /* Vendor specific ie. id = 221, oui and type defines exact ie */ @@ -267,48 +313,6 @@ struct parsed_vndr_ies { struct parsed_vndr_ie_info ie_info[VNDR_IE_PARSE_LIMIT]; }; -#define WL_INTERFACE_CREATE_VER_1 1 -#define WL_INTERFACE_CREATE_VER_2 2 -#define WL_INTERFACE_CREATE_VER_3 3 -#define WL_INTERFACE_CREATE_VER_MAX WL_INTERFACE_CREATE_VER_3 - -#define WL_INTERFACE_MAC_DONT_USE 0x0 -#define WL_INTERFACE_MAC_USE 0x2 - -#define WL_INTERFACE_CREATE_STA 0x0 -#define WL_INTERFACE_CREATE_AP 0x1 - -struct wl_interface_create_v1 { - u16 ver; /* structure version */ - u32 flags; /* flags for operation */ - u8 mac_addr[ETH_ALEN]; /* MAC address */ - u32 wlc_index; /* optional for wlc index */ -}; - -struct wl_interface_create_v2 { - u16 ver; /* structure version */ - u8 pad1[2]; - u32 flags; /* flags for operation */ - u8 mac_addr[ETH_ALEN]; /* MAC address */ - u8 iftype; /* type of interface created */ - u8 pad2; - u32 wlc_index; /* optional for wlc index */ -}; - -struct wl_interface_create_v3 { - u16 ver; /* structure version */ - u16 len; /* length of structure + data */ - u16 fixed_len; /* length of structure */ - u8 iftype; /* type of interface created */ - u8 wlc_index; /* optional for wlc index */ - u32 flags; /* flags for operation */ - u8 mac_addr[ETH_ALEN]; /* MAC address */ - u8 bssid[ETH_ALEN]; /* optional for BSSID */ - u8 if_index; /* interface index request */ - u8 pad[3]; - u8 data[]; /* Optional for specific data */ -}; - static u8 nl80211_band_to_fwil(enum nl80211_band band) { switch (band) { @@ -316,6 +320,8 @@ static u8 nl80211_band_to_fwil(enum nl80211_band band) return WLC_BAND_2G; case NL80211_BAND_5GHZ: return WLC_BAND_5G; + case NL80211_BAND_6GHZ: + return WLC_BAND_6G; default: WARN_ON(1); break; @@ -323,8 +329,25 @@ static u8 nl80211_band_to_fwil(enum nl80211_band band) return 0; } -static u16 chandef_to_chanspec(struct brcmu_d11inf *d11inf, - struct cfg80211_chan_def *ch) +static int nl80211_band_to_chanspec_band(enum nl80211_band band) +{ + switch (band) { + case NL80211_BAND_2GHZ: + return BRCMU_CHAN_BAND_2G; + case NL80211_BAND_5GHZ: + return BRCMU_CHAN_BAND_5G; + case NL80211_BAND_6GHZ: + return BRCMU_CHAN_BAND_6G; + case NL80211_BAND_60GHZ: + default: + WARN_ON_ONCE(1); + // Choose a safe default + return BRCMU_CHAN_BAND_2G; + } +} + +u16 chandef_to_chanspec(struct brcmu_d11inf *d11inf, + struct cfg80211_chan_def *ch) { struct brcmu_chan ch_inf; s32 primary_offset; @@ -382,17 +405,7 @@ static u16 chandef_to_chanspec(struct brcmu_d11inf *d11inf, default: WARN_ON_ONCE(1); } - switch (ch->chan->band) { - case NL80211_BAND_2GHZ: - ch_inf.band = BRCMU_CHAN_BAND_2G; - break; - case NL80211_BAND_5GHZ: - ch_inf.band = BRCMU_CHAN_BAND_5G; - break; - case NL80211_BAND_60GHZ: - default: - WARN_ON_ONCE(1); - } + ch_inf.band = nl80211_band_to_chanspec_band(ch->chan->band); d11inf->encchspec(&ch_inf); brcmf_dbg(TRACE, "chanspec: 0x%x\n", ch_inf.chspec); @@ -404,6 +417,7 @@ u16 channel_to_chanspec(struct brcmu_d11inf *d11inf, { struct brcmu_chan ch_inf; + ch_inf.band = nl80211_band_to_chanspec_band(ch->band); ch_inf.chnum = ieee80211_frequency_to_channel(ch->center_freq); ch_inf.bw = BRCMU_CHAN_BW_20; d11inf->encchspec(&ch_inf); @@ -581,231 +595,6 @@ brcmf_cfg80211_update_proto_addr_mode(struct wireless_dev *wdev) ADDR_INDIRECT); } -static int brcmf_get_first_free_bsscfgidx(struct brcmf_pub *drvr) -{ - int bsscfgidx; - - for (bsscfgidx = 0; bsscfgidx < BRCMF_MAX_IFS; bsscfgidx++) { - /* bsscfgidx 1 is reserved for legacy P2P */ - if (bsscfgidx == 1) - continue; - if (!drvr->iflist[bsscfgidx]) - return bsscfgidx; - } - - return -ENOMEM; -} - -static void brcmf_set_vif_sta_macaddr(struct brcmf_if *ifp, u8 *mac_addr) -{ - u8 mac_idx = ifp->drvr->sta_mac_idx; - - /* set difference MAC address with locally administered bit */ - memcpy(mac_addr, ifp->mac_addr, ETH_ALEN); - mac_addr[0] |= 0x02; - mac_addr[3] ^= mac_idx ? 0xC0 : 0xA0; - mac_idx++; - mac_idx = mac_idx % 2; - ifp->drvr->sta_mac_idx = mac_idx; -} - -static int brcmf_cfg80211_request_sta_if(struct brcmf_if *ifp, u8 *macaddr) -{ - struct wl_interface_create_v1 iface_v1; - struct wl_interface_create_v2 iface_v2; - struct wl_interface_create_v3 iface_v3; - u32 iface_create_ver; - int err; - - /* interface_create version 1 */ - memset(&iface_v1, 0, sizeof(iface_v1)); - iface_v1.ver = WL_INTERFACE_CREATE_VER_1; - iface_v1.flags = WL_INTERFACE_CREATE_STA | - WL_INTERFACE_MAC_USE; - if (!is_zero_ether_addr(macaddr)) - memcpy(iface_v1.mac_addr, macaddr, ETH_ALEN); - else - brcmf_set_vif_sta_macaddr(ifp, iface_v1.mac_addr); - - err = brcmf_fil_iovar_data_get(ifp, "interface_create", - &iface_v1, - sizeof(iface_v1)); - if (err) { - brcmf_info("failed to create interface(v1), err=%d\n", - err); - } else { - brcmf_dbg(INFO, "interface created(v1)\n"); - return 0; - } - - /* interface_create version 2 */ - memset(&iface_v2, 0, sizeof(iface_v2)); - iface_v2.ver = WL_INTERFACE_CREATE_VER_2; - iface_v2.flags = WL_INTERFACE_MAC_USE; - iface_v2.iftype = WL_INTERFACE_CREATE_STA; - if (!is_zero_ether_addr(macaddr)) - memcpy(iface_v2.mac_addr, macaddr, ETH_ALEN); - else - brcmf_set_vif_sta_macaddr(ifp, iface_v2.mac_addr); - - err = brcmf_fil_iovar_data_get(ifp, "interface_create", - &iface_v2, - sizeof(iface_v2)); - if (err) { - brcmf_info("failed to create interface(v2), err=%d\n", - err); - } else { - brcmf_dbg(INFO, "interface created(v2)\n"); - return 0; - } - - /* interface_create version 3+ */ - /* get supported version from firmware side */ - iface_create_ver = 0; - err = brcmf_fil_bsscfg_int_query(ifp, "interface_create", - &iface_create_ver); - if (err) { - brcmf_err("fail to get supported version, err=%d\n", err); - return -EOPNOTSUPP; - } - - switch (iface_create_ver) { - case WL_INTERFACE_CREATE_VER_3: - memset(&iface_v3, 0, sizeof(iface_v3)); - iface_v3.ver = WL_INTERFACE_CREATE_VER_3; - iface_v3.flags = WL_INTERFACE_MAC_USE; - iface_v3.iftype = WL_INTERFACE_CREATE_STA; - if (!is_zero_ether_addr(macaddr)) - memcpy(iface_v3.mac_addr, macaddr, ETH_ALEN); - else - brcmf_set_vif_sta_macaddr(ifp, iface_v3.mac_addr); - - err = brcmf_fil_iovar_data_get(ifp, "interface_create", - &iface_v3, - sizeof(iface_v3)); - - if (!err) - brcmf_dbg(INFO, "interface created(v3)\n"); - break; - default: - brcmf_err("not support interface create(v%d)\n", - iface_create_ver); - err = -EOPNOTSUPP; - break; - } - - if (err) { - brcmf_info("station interface creation failed (%d)\n", - err); - return -EIO; - } - - return 0; -} - -static int brcmf_cfg80211_request_ap_if(struct brcmf_if *ifp) -{ - struct wl_interface_create_v1 iface_v1; - struct wl_interface_create_v2 iface_v2; - struct wl_interface_create_v3 iface_v3; - u32 iface_create_ver; - struct brcmf_pub *drvr = ifp->drvr; - struct brcmf_mbss_ssid_le mbss_ssid_le; - int bsscfgidx; - int err; - - /* interface_create version 1 */ - memset(&iface_v1, 0, sizeof(iface_v1)); - iface_v1.ver = WL_INTERFACE_CREATE_VER_1; - iface_v1.flags = WL_INTERFACE_CREATE_AP | - WL_INTERFACE_MAC_USE; - - brcmf_set_vif_sta_macaddr(ifp, iface_v1.mac_addr); - - err = brcmf_fil_iovar_data_get(ifp, "interface_create", - &iface_v1, - sizeof(iface_v1)); - if (err) { - brcmf_info("failed to create interface(v1), err=%d\n", - err); - } else { - brcmf_dbg(INFO, "interface created(v1)\n"); - return 0; - } - - /* interface_create version 2 */ - memset(&iface_v2, 0, sizeof(iface_v2)); - iface_v2.ver = WL_INTERFACE_CREATE_VER_2; - iface_v2.flags = WL_INTERFACE_MAC_USE; - iface_v2.iftype = WL_INTERFACE_CREATE_AP; - - brcmf_set_vif_sta_macaddr(ifp, iface_v2.mac_addr); - - err = brcmf_fil_iovar_data_get(ifp, "interface_create", - &iface_v2, - sizeof(iface_v2)); - if (err) { - brcmf_info("failed to create interface(v2), err=%d\n", - err); - } else { - brcmf_dbg(INFO, "interface created(v2)\n"); - return 0; - } - - /* interface_create version 3+ */ - /* get supported version from firmware side */ - iface_create_ver = 0; - err = brcmf_fil_bsscfg_int_query(ifp, "interface_create", - &iface_create_ver); - if (err) { - brcmf_err("fail to get supported version, err=%d\n", err); - return -EOPNOTSUPP; - } - - switch (iface_create_ver) { - case WL_INTERFACE_CREATE_VER_3: - memset(&iface_v3, 0, sizeof(iface_v3)); - iface_v3.ver = WL_INTERFACE_CREATE_VER_3; - iface_v3.flags = WL_INTERFACE_MAC_USE; - iface_v3.iftype = WL_INTERFACE_CREATE_AP; - brcmf_set_vif_sta_macaddr(ifp, iface_v3.mac_addr); - - err = brcmf_fil_iovar_data_get(ifp, "interface_create", - &iface_v3, - sizeof(iface_v3)); - - if (!err) - brcmf_dbg(INFO, "interface created(v3)\n"); - break; - default: - brcmf_err("not support interface create(v%d)\n", - iface_create_ver); - err = -EOPNOTSUPP; - break; - } - - if (err) { - brcmf_info("Does not support interface_create (%d)\n", - err); - memset(&mbss_ssid_le, 0, sizeof(mbss_ssid_le)); - bsscfgidx = brcmf_get_first_free_bsscfgidx(ifp->drvr); - if (bsscfgidx < 0) - return bsscfgidx; - - mbss_ssid_le.bsscfgidx = cpu_to_le32(bsscfgidx); - mbss_ssid_le.SSID_len = cpu_to_le32(5); - sprintf(mbss_ssid_le.SSID, "ssid%d", bsscfgidx); - - err = brcmf_fil_bsscfg_data_set(ifp, "bsscfg:ssid", &mbss_ssid_le, - sizeof(mbss_ssid_le)); - - if (err < 0) - bphy_err(drvr, "setting ssid failed %d\n", err); - } - - return err; -} - /** * brcmf_apsta_add_vif() - create a new AP or STA virtual interface * @@ -1058,134 +847,11 @@ bool brcmf_is_apmode_operating(struct wiphy *wiphy) return ret; } -static void brcmf_scan_params_v2_to_v1(struct brcmf_scan_params_v2_le *params_v2_le, - struct brcmf_scan_params_le *params_le) -{ - size_t params_size; - u32 ch; - int n_channels, n_ssids; - - memcpy(¶ms_le->ssid_le, ¶ms_v2_le->ssid_le, - sizeof(params_le->ssid_le)); - memcpy(¶ms_le->bssid, ¶ms_v2_le->bssid, - sizeof(params_le->bssid)); - - params_le->bss_type = params_v2_le->bss_type; - params_le->scan_type = le32_to_cpu(params_v2_le->scan_type); - params_le->nprobes = params_v2_le->nprobes; - params_le->active_time = params_v2_le->active_time; - params_le->passive_time = params_v2_le->passive_time; - params_le->home_time = params_v2_le->home_time; - params_le->channel_num = params_v2_le->channel_num; - - ch = le32_to_cpu(params_v2_le->channel_num); - n_channels = ch & BRCMF_SCAN_PARAMS_COUNT_MASK; - n_ssids = ch >> BRCMF_SCAN_PARAMS_NSSID_SHIFT; - - params_size = sizeof(u16) * n_channels; - if (n_ssids > 0) { - params_size = roundup(params_size, sizeof(u32)); - params_size += sizeof(struct brcmf_ssid_le) * n_ssids; - } - - memcpy(¶ms_le->channel_list[0], - ¶ms_v2_le->channel_list[0], params_size); -} - -static void brcmf_escan_prep(struct brcmf_cfg80211_info *cfg, - struct brcmf_scan_params_v2_le *params_le, - struct cfg80211_scan_request *request) -{ - u32 n_ssids; - u32 n_channels; - s32 i; - s32 offset; - u16 chanspec; - char *ptr; - int length; - struct brcmf_ssid_le ssid_le; - - eth_broadcast_addr(params_le->bssid); - - length = BRCMF_SCAN_PARAMS_V2_FIXED_SIZE; - - params_le->version = cpu_to_le16(BRCMF_SCAN_PARAMS_VERSION_V2); - params_le->bss_type = DOT11_BSSTYPE_ANY; - params_le->scan_type = cpu_to_le32(BRCMF_SCANTYPE_ACTIVE); - params_le->channel_num = 0; - params_le->nprobes = cpu_to_le32(-1); - params_le->active_time = cpu_to_le32(-1); - params_le->passive_time = cpu_to_le32(-1); - params_le->home_time = cpu_to_le32(-1); - memset(¶ms_le->ssid_le, 0, sizeof(params_le->ssid_le)); - - /* Scan abort */ - if (!request) { - length += sizeof(u16); - params_le->channel_num = cpu_to_le32(1); - params_le->channel_list[0] = cpu_to_le16(-1); - params_le->length = cpu_to_le16(length); - return; - } - - n_ssids = request->n_ssids; - n_channels = request->n_channels; - - /* Copy channel array if applicable */ - brcmf_dbg(SCAN, "### List of channelspecs to scan ### %d\n", - n_channels); - if (n_channels > 0) { - length += roundup(sizeof(u16) * n_channels, sizeof(u32)); - for (i = 0; i < n_channels; i++) { - chanspec = channel_to_chanspec(&cfg->d11inf, - request->channels[i]); - brcmf_dbg(SCAN, "Chan : %d, Channel spec: %x\n", - request->channels[i]->hw_value, chanspec); - params_le->channel_list[i] = cpu_to_le16(chanspec); - } - } else { - brcmf_dbg(SCAN, "Scanning all channels\n"); - } - - /* Copy ssid array if applicable */ - brcmf_dbg(SCAN, "### List of SSIDs to scan ### %d\n", n_ssids); - if (n_ssids > 0) { - offset = offsetof(struct brcmf_scan_params_v2_le, channel_list) + - n_channels * sizeof(u16); - offset = roundup(offset, sizeof(u32)); - length += sizeof(ssid_le) * n_ssids; - ptr = (char *)params_le + offset; - for (i = 0; i < n_ssids; i++) { - memset(&ssid_le, 0, sizeof(ssid_le)); - ssid_le.SSID_len = - cpu_to_le32(request->ssids[i].ssid_len); - memcpy(ssid_le.SSID, request->ssids[i].ssid, - request->ssids[i].ssid_len); - if (!ssid_le.SSID_len) - brcmf_dbg(SCAN, "%d: Broadcast scan\n", i); - else - brcmf_dbg(SCAN, "%d: scan for %.32s size=%d\n", - i, ssid_le.SSID, ssid_le.SSID_len); - memcpy(ptr, &ssid_le, sizeof(ssid_le)); - ptr += sizeof(ssid_le); - } - } else { - brcmf_dbg(SCAN, "Performing passive scan\n"); - params_le->scan_type = cpu_to_le32(BRCMF_SCANTYPE_PASSIVE); - } - params_le->length = cpu_to_le16(length); - /* Adding mask to channel numbers */ - params_le->channel_num = - cpu_to_le32((n_ssids << BRCMF_SCAN_PARAMS_NSSID_SHIFT) | - (n_channels & BRCMF_SCAN_PARAMS_COUNT_MASK)); -} - s32 brcmf_notify_escan_complete(struct brcmf_cfg80211_info *cfg, struct brcmf_if *ifp, bool aborted, bool fw_abort) { struct brcmf_pub *drvr = cfg->pub; - struct brcmf_scan_params_v2_le params_v2_le; struct cfg80211_scan_request *scan_request; u64 reqid; u32 bucket; @@ -1201,25 +867,16 @@ s32 brcmf_notify_escan_complete(struct brcmf_cfg80211_info *cfg, timer_delete_sync(&cfg->escan_timeout); if (fw_abort) { + u32 len; + void *data = drvr->scan_param_handler.get_struct_for_request(cfg, &len, NULL); + if (!data){ + bphy_err(drvr, "Scan abort failed to prepare abort struct\n"); + return 0; + } /* Do a scan abort to stop the driver's scan engine */ brcmf_dbg(SCAN, "ABORT scan in firmware\n"); - - brcmf_escan_prep(cfg, ¶ms_v2_le, NULL); - - /* E-Scan (or anyother type) can be aborted by SCAN */ - if (brcmf_feat_is_enabled(ifp, BRCMF_FEAT_SCAN_V2)) { - err = brcmf_fil_cmd_data_set(ifp, BRCMF_C_SCAN, - ¶ms_v2_le, - sizeof(params_v2_le)); - } else { - struct brcmf_scan_params_le params_le; - - brcmf_scan_params_v2_to_v1(¶ms_v2_le, ¶ms_le); - err = brcmf_fil_cmd_data_set(ifp, BRCMF_C_SCAN, - ¶ms_le, - sizeof(params_le)); - } - + err = brcmf_fil_cmd_data_set(ifp, BRCMF_C_SCAN, data, len); + kfree(data); if (err) bphy_err(drvr, "Scan abort failed\n"); } @@ -1443,19 +1100,24 @@ brcmf_run_escan(struct brcmf_cfg80211_info *cfg, struct brcmf_if *ifp, struct cfg80211_scan_request *request) { struct brcmf_pub *drvr = cfg->pub; - s32 params_size = BRCMF_SCAN_PARAMS_V2_FIXED_SIZE + - offsetof(struct brcmf_escan_params_le, params_v2_le); + u32 struct_size = 0; + void *prepped_params = NULL; + u32 params_size = 0; struct brcmf_escan_params_le *params; s32 err = 0; brcmf_dbg(SCAN, "E-SCAN START\n"); - if (request != NULL) { - /* Allocate space for populating ssids in struct */ - params_size += sizeof(u32) * ((request->n_channels + 1) / 2); - - /* Allocate space for populating ssids in struct */ - params_size += sizeof(struct brcmf_ssid_le) * request->n_ssids; + prepped_params = drvr->scan_param_handler.get_struct_for_request(cfg, &struct_size, request); + if (!prepped_params) { + err = -EINVAL; + goto exit; + } + params_size = struct_size + + offsetof(struct brcmf_escan_params_le, params_v4_le); + if (!params_size) { + err = -EINVAL; + goto exit; } params = kzalloc(params_size, GFP_KERNEL); @@ -1463,27 +1125,14 @@ brcmf_run_escan(struct brcmf_cfg80211_info *cfg, struct brcmf_if *ifp, err = -ENOMEM; goto exit; } - BUG_ON(params_size + sizeof("escan") >= BRCMF_DCMD_MEDLEN); - brcmf_escan_prep(cfg, ¶ms->params_v2_le, request); - - params->version = cpu_to_le32(BRCMF_ESCAN_REQ_VERSION_V2); - - if (!brcmf_feat_is_enabled(ifp, BRCMF_FEAT_SCAN_V2)) { - struct brcmf_escan_params_le *params_v1; - - params_size -= BRCMF_SCAN_PARAMS_V2_FIXED_SIZE; - params_size += BRCMF_SCAN_PARAMS_FIXED_SIZE; - params_v1 = kzalloc(params_size, GFP_KERNEL); - if (!params_v1) { - err = -ENOMEM; - goto exit_params; - } - params_v1->version = cpu_to_le32(BRCMF_ESCAN_REQ_VERSION); - brcmf_scan_params_v2_to_v1(¶ms->params_v2_le, ¶ms_v1->params_le); - kfree(params); - params = params_v1; - } + /* Copy into the largest part */ + unsafe_memcpy( + ¶ms->params_v4_le, prepped_params, struct_size, + /* A composite flex-array that is at least as large as the memcpy due to the allocation above */); + /* We can now free the original prepped parameters */ + kfree(prepped_params); + params->version = cpu_to_le32(drvr->scan_param_handler.version); params->action = cpu_to_le16(WL_ESCAN_ACTION_START); params->sync_id = cpu_to_le16(0x1234); @@ -1495,7 +1144,6 @@ brcmf_run_escan(struct brcmf_cfg80211_info *cfg, struct brcmf_if *ifp, bphy_err(drvr, "error (%d)\n", err); } -exit_params: kfree(params); exit: return err; @@ -1779,21 +1427,19 @@ static void brcmf_link_down(struct brcmf_cfg80211_vif *vif, u16 reason, brcmf_dbg(TRACE, "Exit\n"); } -static s32 -brcmf_cfg80211_join_ibss(struct wiphy *wiphy, struct net_device *ndev, - struct cfg80211_ibss_params *params) +static s32 brcmf_cfg80211_join_ibss(struct wiphy *wiphy, + struct net_device *ndev, + struct cfg80211_ibss_params *params) { struct brcmf_cfg80211_info *cfg = wiphy_to_cfg(wiphy); struct brcmf_if *ifp = netdev_priv(ndev); struct brcmf_cfg80211_profile *profile = &ifp->vif->profile; struct brcmf_pub *drvr = cfg->pub; - struct brcmf_join_params join_params; - size_t join_params_size = 0; - s32 err = 0; + void *join_params; + u32 join_params_size = 0; s32 wsec = 0; s32 bcnprd; - u16 chanspec; - u32 ssid_len; + s32 err = 0; brcmf_dbg(TRACE, "Enter\n"); if (!check_vif_up(ifp->vif)) @@ -1867,58 +1513,39 @@ brcmf_cfg80211_join_ibss(struct wiphy *wiphy, struct net_device *ndev, goto done; } - /* Configure required join parameter */ - memset(&join_params, 0, sizeof(struct brcmf_join_params)); - - /* SSID */ - ssid_len = min_t(u32, params->ssid_len, IEEE80211_MAX_SSID_LEN); - memcpy(join_params.ssid_le.SSID, params->ssid, ssid_len); - join_params.ssid_le.SSID_len = cpu_to_le32(ssid_len); - join_params_size = sizeof(join_params.ssid_le); - - /* BSSID */ if (params->bssid) { - memcpy(join_params.params_le.bssid, params->bssid, ETH_ALEN); - join_params_size += BRCMF_ASSOC_PARAMS_FIXED_SIZE; memcpy(profile->bssid, params->bssid, ETH_ALEN); } else { - eth_broadcast_addr(join_params.params_le.bssid); eth_zero_addr(profile->bssid); } - /* Channel */ + cfg->ibss_starter = false; + cfg->channel = 0; if (params->chandef.chan) { - u32 target_channel; + u16 chanspec; + cfg->channel = ieee80211_frequency_to_channel( + params->chandef.chan->center_freq); + /* adding chanspec */ + chanspec = chandef_to_chanspec(&cfg->d11inf, ¶ms->chandef); - cfg->channel = - ieee80211_frequency_to_channel( - params->chandef.chan->center_freq); - if (params->channel_fixed) { - /* adding chanspec */ - chanspec = chandef_to_chanspec(&cfg->d11inf, - ¶ms->chandef); - join_params.params_le.chanspec_list[0] = - cpu_to_le16(chanspec); - join_params.params_le.chanspec_num = cpu_to_le32(1); - join_params_size += sizeof(join_params.params_le); - } - - /* set channel for starter */ - target_channel = cfg->channel; - err = brcmf_fil_cmd_int_set(ifp, BRCMF_C_SET_CHANNEL, - target_channel); + /* set chanspec */ + err = brcmf_fil_iovar_int_set(ifp, "chanspec", chanspec); if (err) { - bphy_err(drvr, "WLC_SET_CHANNEL failed (%d)\n", err); + bphy_err(drvr, "Setting chanspec failed (%d)\n", err); goto done; } - } else - cfg->channel = 0; - - cfg->ibss_starter = false; - + } - err = brcmf_fil_cmd_data_set(ifp, BRCMF_C_SET_SSID, - &join_params, join_params_size); + join_params = drvr->join_param_handler.get_struct_for_ibss( + cfg, &join_params_size, params); + if (!join_params) { + bphy_err(drvr, "Converting join params failed\n"); + goto done; + } + err = brcmf_fil_cmd_data_set(ifp, BRCMF_C_SET_SSID, join_params, + join_params_size); + /* Free params no matter what */ + kfree(join_params); if (err) { bphy_err(drvr, "WLC_SET_SSID failed (%d)\n", err); goto done; @@ -1969,6 +1596,10 @@ static s32 brcmf_set_wpa_version(struct net_device *ndev, if (drvr->bus_if->fwvid == BRCMF_FWVENDOR_CYW && sme->crypto.akm_suites[0] == WLAN_AKM_SUITE_SAE) val = WPA3_AUTH_SAE_PSK; + else if (sme->crypto.akm_suites[0] == WLAN_AKM_SUITE_SAE) + val = WPA3_AUTH_SAE_PSK; + else if (sme->crypto.akm_suites[0] == WLAN_AKM_SUITE_OWE) + val = WPA3_AUTH_OWE; else val = WPA2_AUTH_PSK | WPA2_AUTH_UNSPECIFIED; } else if (sme->crypto.wpa_versions & NL80211_WPA_VERSION_3) { @@ -2026,6 +1657,48 @@ static s32 brcmf_set_auth_type(struct net_device *ndev, return err; } +static s32 brcmf_set_wsec_info_algos(struct brcmf_if *ifp, u32 algos, u32 mask) +{ + struct brcmf_pub *drvr = ifp->drvr; + s32 err = 0; + struct brcmf_wsec_info *wsec_info; + struct brcmf_xtlv *wsec_info_tlv; + u16 tlv_data_len; + u8 tlv_data[8]; + u32 param_len; + u8 *buf; + + brcmf_dbg(TRACE, "Enter\n"); + + buf = kzalloc(sizeof(struct brcmf_wsec_info) + sizeof(tlv_data), + GFP_KERNEL); + if (!buf) { + bphy_err(drvr, "unable to allocate.\n"); + return -ENOMEM; + } + wsec_info = (struct brcmf_wsec_info *)buf; + wsec_info->version = BRCMF_WSEC_INFO_VER; + wsec_info_tlv = + (struct brcmf_xtlv *)(buf + + offsetof(struct brcmf_wsec_info, tlvs)); + wsec_info->num_tlvs++; + tlv_data_len = sizeof(tlv_data); + memcpy(tlv_data, &algos, sizeof(algos)); + memcpy(tlv_data + sizeof(algos), &mask, sizeof(mask)); + brcmf_xtlv_pack_header(wsec_info_tlv, WL_WSEC_INFO_BSS_ALGOS, + tlv_data_len, tlv_data, 0); + + param_len = offsetof(struct brcmf_wsec_info, tlvs) + + offsetof(struct brcmf_wsec_info_tlv, data) + tlv_data_len; + + err = brcmf_fil_bsscfg_data_set(ifp, "wsec_info", buf, param_len); + if (err) + brcmf_err("set wsec_info_error:%d\n", err); + + kfree(buf); + return err; +} + static s32 brcmf_set_wsec_mode(struct net_device *ndev, struct cfg80211_connect_params *sme) @@ -2038,6 +1711,8 @@ brcmf_set_wsec_mode(struct net_device *ndev, s32 gval = 0; s32 wsec; s32 err = 0; + u32 algos = 0; + u32 mask = 0; if (sme->crypto.n_ciphers_pairwise) { switch (sme->crypto.ciphers_pairwise[0]) { @@ -2054,6 +1729,15 @@ brcmf_set_wsec_mode(struct net_device *ndev, case WLAN_CIPHER_SUITE_AES_CMAC: pval = AES_ENABLED; break; + case WLAN_CIPHER_SUITE_GCMP_256: + if (!brcmf_feat_is_enabled(ifp, BRCMF_FEAT_GCMP)) { + brcmf_err("This chip does not support GCMP\n"); + return -EOPNOTSUPP; + } + pval = AES_ENABLED; + algos = KEY_ALGO_MASK(CRYPTO_ALGO_AES_GCM256); + mask = algos | KEY_ALGO_MASK(CRYPTO_ALGO_AES_CCM); + break; default: bphy_err(drvr, "invalid cipher pairwise (%d)\n", sme->crypto.ciphers_pairwise[0]); @@ -2075,6 +1759,15 @@ brcmf_set_wsec_mode(struct net_device *ndev, case WLAN_CIPHER_SUITE_AES_CMAC: gval = AES_ENABLED; break; + case WLAN_CIPHER_SUITE_GCMP_256: + if (!brcmf_feat_is_enabled(ifp, BRCMF_FEAT_GCMP)) { + brcmf_err("This chip does not support GCMP\n"); + return -EOPNOTSUPP; + } + gval = AES_ENABLED; + algos = KEY_ALGO_MASK(CRYPTO_ALGO_AES_GCM256); + mask = algos | KEY_ALGO_MASK(CRYPTO_ALGO_AES_CCM); + break; default: bphy_err(drvr, "invalid cipher group (%d)\n", sme->crypto.cipher_group); @@ -2083,6 +1776,7 @@ brcmf_set_wsec_mode(struct net_device *ndev, } brcmf_dbg(CONN, "pval (%d) gval (%d)\n", pval, gval); + brcmf_dbg(CONN, "algos (0x%x) mask (0x%x)\n", algos, mask); /* In case of privacy, but no security and WPS then simulate */ /* setting AES. WPS-2.0 allows no security */ if (brcmf_find_wpsie(sme->ie, sme->ie_len) && !pval && !gval && @@ -2095,6 +1789,15 @@ brcmf_set_wsec_mode(struct net_device *ndev, bphy_err(drvr, "error (%d)\n", err); return err; } + if (brcmf_feat_is_enabled(ifp, BRCMF_FEAT_GCMP)) { + brcmf_dbg(CONN, "set_wsec_info algos (0x%x) mask (0x%x)\n", + algos, mask); + err = brcmf_set_wsec_info_algos(ifp, algos, mask); + if (err) { + brcmf_err("set wsec_info error (%d)\n", err); + return err; + } + } sec = &profile->sec; sec->cipher_pairwise = sme->crypto.ciphers_pairwise[0]; @@ -2118,9 +1821,13 @@ brcmf_set_key_mgmt(struct net_device *ndev, struct cfg80211_connect_params *sme) u16 rsn_cap; u32 mfp; u16 count; + s32 okc_enable; + u16 pmkid_count; + const u8 *group_mgmt_cs = NULL; profile->use_fwsup = BRCMF_PROFILE_FWSUP_NONE; profile->is_ft = false; + profile->is_okc = false; if (!sme->crypto.n_akm_suites) return 0; @@ -2137,13 +1844,15 @@ brcmf_set_key_mgmt(struct net_device *ndev, struct cfg80211_connect_params *sme) val = WPA_AUTH_UNSPECIFIED; if (sme->want_1x) profile->use_fwsup = BRCMF_PROFILE_FWSUP_1X; + else + profile->use_fwsup = BRCMF_PROFILE_FWSUP_ROAM; break; case WLAN_AKM_SUITE_PSK: val = WPA_AUTH_PSK; break; default: - bphy_err(drvr, "invalid akm suite (%d)\n", - sme->crypto.akm_suites[0]); + bphy_err(drvr, "invalid cipher group (%d)\n", + sme->crypto.cipher_group); return -EINVAL; } } else if (val & (WPA2_AUTH_PSK | WPA2_AUTH_UNSPECIFIED)) { @@ -2152,11 +1861,15 @@ brcmf_set_key_mgmt(struct net_device *ndev, struct cfg80211_connect_params *sme) val = WPA2_AUTH_UNSPECIFIED; if (sme->want_1x) profile->use_fwsup = BRCMF_PROFILE_FWSUP_1X; + else + profile->use_fwsup = BRCMF_PROFILE_FWSUP_ROAM; break; case WLAN_AKM_SUITE_8021X_SHA256: val = WPA2_AUTH_1X_SHA256; if (sme->want_1x) profile->use_fwsup = BRCMF_PROFILE_FWSUP_1X; + else + profile->use_fwsup = BRCMF_PROFILE_FWSUP_ROAM; break; case WLAN_AKM_SUITE_PSK_SHA256: val = WPA2_AUTH_PSK_SHA256; @@ -2169,14 +1882,35 @@ brcmf_set_key_mgmt(struct net_device *ndev, struct cfg80211_connect_params *sme) profile->is_ft = true; if (sme->want_1x) profile->use_fwsup = BRCMF_PROFILE_FWSUP_1X; + else + profile->use_fwsup = BRCMF_PROFILE_FWSUP_ROAM; break; case WLAN_AKM_SUITE_FT_PSK: val = WPA2_AUTH_PSK | WPA2_AUTH_FT; profile->is_ft = true; + if (brcmf_feat_is_enabled(ifp, BRCMF_FEAT_FWSUP)) + profile->use_fwsup = BRCMF_PROFILE_FWSUP_PSK; + else + profile->use_fwsup = BRCMF_PROFILE_FWSUP_ROAM; + break; + case WLAN_AKM_SUITE_DPP: + val = WFA_AUTH_DPP; + profile->use_fwsup = BRCMF_PROFILE_FWSUP_NONE; + break; + case WLAN_AKM_SUITE_OWE: + val = WPA3_AUTH_OWE; + profile->use_fwsup = BRCMF_PROFILE_FWSUP_ROAM; + break; + case WLAN_AKM_SUITE_8021X_SUITE_B_192: + val = WPA3_AUTH_1X_SUITE_B_SHA384; + if (sme->want_1x) + profile->use_fwsup = BRCMF_PROFILE_FWSUP_1X; + else + profile->use_fwsup = BRCMF_PROFILE_FWSUP_ROAM; break; default: - bphy_err(drvr, "invalid akm suite (%d)\n", - sme->crypto.akm_suites[0]); + bphy_err(drvr, "invalid cipher group (%d)\n", + sme->crypto.cipher_group); return -EINVAL; } } else if (val & WPA3_AUTH_SAE_PSK) { @@ -2189,17 +1923,38 @@ brcmf_set_key_mgmt(struct net_device *ndev, struct cfg80211_connect_params *sme) profile->is_ft = true; break; default: - bphy_err(drvr, "invalid akm suite (%d)\n", - sme->crypto.akm_suites[0]); + bphy_err(drvr, "invalid cipher group (%d)\n", + sme->crypto.cipher_group); return -EINVAL; } if (sme->crypto.sae_pwd) { profile->use_fwsup = BRCMF_PROFILE_FWSUP_SAE; } } - - if (profile->use_fwsup == BRCMF_PROFILE_FWSUP_1X) + if ((profile->use_fwsup == BRCMF_PROFILE_FWSUP_1X) || + (profile->use_fwsup == BRCMF_PROFILE_FWSUP_ROAM)) { brcmf_dbg(INFO, "using 1X offload\n"); + err = brcmf_fil_bsscfg_int_get(netdev_priv(ndev), "okc_enable", + &okc_enable); + if (err) { + bphy_err(drvr, "get okc_enable failed (%d)\n", err); + } else { + brcmf_dbg(INFO, "get okc_enable (%d)\n", okc_enable); + profile->is_okc = okc_enable; + } + } else if (profile->use_fwsup != BRCMF_PROFILE_FWSUP_SAE && + (val == WPA3_AUTH_SAE_PSK)) { + brcmf_dbg(INFO, "not using SAE offload\n"); + err = brcmf_fil_bsscfg_int_get(netdev_priv(ndev), "okc_enable", + &okc_enable); + if (err) { + bphy_err(drvr, "get okc_enable failed (%d)\n", err); + } else { + brcmf_dbg(INFO, "get okc_enable (%d)\n", okc_enable); + profile->is_okc = okc_enable; + } + } + if (profile->use_fwsup == BRCMF_PROFILE_FWSUP_SAE) brcmf_dbg(INFO, "using SAE offload\n"); @@ -2235,14 +1990,47 @@ brcmf_set_key_mgmt(struct net_device *ndev, struct cfg80211_connect_params *sme) mfp = BRCMF_MFP_REQUIRED; else if (rsn_cap & RSN_CAP_MFPC_MASK) mfp = BRCMF_MFP_CAPABLE; + /* In case of dpp, very low tput is observed if MFPC is set in + * firmmare. Firmware needs to ensure that MFPC is not set when + * MFPR was requested from fmac. However since this change being + * specific to DPP, fmac needs to set wpa_auth prior to mfp, so + * that firmware can use this info to prevent MFPC being set in + * case of dpp. + */ + if (val == WFA_AUTH_DPP) { + brcmf_dbg(CONN, "setting wpa_auth to 0x%0x\n", val); + err = brcmf_fil_bsscfg_int_set(netdev_priv(ndev), "wpa_auth", + val); + if (err) { + bphy_err(drvr, "could not set wpa_auth (%d)\n", err); + return err; + } + } + brcmf_fil_bsscfg_int_set(netdev_priv(ndev), "mfp", mfp); + offset += RSN_CAP_LEN; + if (mfp && (ie_len - offset >= RSN_PMKID_COUNT_LEN)) { + pmkid_count = ie[offset] + (ie[offset + 1] << 8); + offset += RSN_PMKID_COUNT_LEN + (pmkid_count * WLAN_PMKID_LEN); + if (ie_len - offset >= WPA_IE_MIN_OUI_LEN) { + group_mgmt_cs = &ie[offset]; + if (memcmp(group_mgmt_cs, RSN_OUI, TLV_OUI_LEN) == 0) { + brcmf_fil_bsscfg_data_set(ifp, "bip", + (void *)group_mgmt_cs, + WPA_IE_MIN_OUI_LEN); + } + } + } skip_mfp_config: - brcmf_dbg(CONN, "setting wpa_auth to 0x%0x\n", val); - err = brcmf_fil_bsscfg_int_set(netdev_priv(ndev), "wpa_auth", val); - if (err) { - bphy_err(drvr, "could not set wpa_auth (%d)\n", err); - return err; + if (val != WFA_AUTH_DPP) { + brcmf_dbg(CONN, "setting wpa_auth to 0x%0x\n", val); + err = brcmf_fil_bsscfg_int_set(netdev_priv(ndev), "wpa_auth", + val); + if (err) { + bphy_err(drvr, "could not set wpa_auth (%d)\n", err); + return err; + } } return err; @@ -2375,52 +2163,51 @@ static void brcmf_set_join_pref(struct brcmf_if *ifp, static s32 brcmf_cfg80211_connect(struct wiphy *wiphy, struct net_device *ndev, - struct cfg80211_connect_params *sme) + struct cfg80211_connect_params *params) { struct brcmf_cfg80211_info *cfg = wiphy_to_cfg(wiphy); struct brcmf_if *ifp = netdev_priv(ndev); struct brcmf_cfg80211_profile *profile = &ifp->vif->profile; - struct ieee80211_channel *chan = sme->channel; + struct ieee80211_channel *chan = params->channel; struct brcmf_pub *drvr = ifp->drvr; - struct brcmf_join_params join_params; - size_t join_params_size; + void *join_params; + u32 join_params_size; + void *fallback_join_params; + u32 fallback_join_params_size; const struct brcmf_tlv *rsn_ie; const struct brcmf_vs_tlv *wpa_ie; const void *ie; u32 ie_len; - struct brcmf_ext_join_params_le *ext_join_params; - u16 chanspec; s32 err = 0; - u32 ssid_len; brcmf_dbg(TRACE, "Enter\n"); if (!check_vif_up(ifp->vif)) return -EIO; - if (!sme->ssid) { + if (!params->ssid) { bphy_err(drvr, "Invalid ssid\n"); return -EOPNOTSUPP; } - if (sme->channel_hint) - chan = sme->channel_hint; + if (params->channel_hint) + chan = params->channel_hint; - if (sme->bssid_hint) - sme->bssid = sme->bssid_hint; + if (params->bssid_hint) + params->bssid = params->bssid_hint; if (ifp->vif == cfg->p2p.bss_idx[P2PAPI_BSSCFG_PRIMARY].vif) { /* A normal (non P2P) connection request setup. */ ie = NULL; ie_len = 0; /* find the WPA_IE */ - wpa_ie = brcmf_find_wpaie((u8 *)sme->ie, sme->ie_len); + wpa_ie = brcmf_find_wpaie((u8 *)params->ie, params->ie_len); if (wpa_ie) { ie = wpa_ie; ie_len = wpa_ie->len + TLV_HDR_LEN; } else { /* find the RSN_IE */ - rsn_ie = brcmf_parse_tlvs((const u8 *)sme->ie, - sme->ie_len, + rsn_ie = brcmf_parse_tlvs((const u8 *)params->ie, + params->ie_len, WLAN_EID_RSN); if (rsn_ie) { ie = rsn_ie; @@ -2431,7 +2218,7 @@ brcmf_cfg80211_connect(struct wiphy *wiphy, struct net_device *ndev, } err = brcmf_vif_set_mgmt_ie(ifp->vif, BRCMF_VNDR_IE_ASSOCREQ_FLAG, - sme->ie, sme->ie_len); + params->ie, params->ie_len); if (err) bphy_err(drvr, "Set Assoc REQ IE Failed\n"); else @@ -2442,166 +2229,129 @@ brcmf_cfg80211_connect(struct wiphy *wiphy, struct net_device *ndev, if (chan) { cfg->channel = ieee80211_frequency_to_channel(chan->center_freq); - chanspec = channel_to_chanspec(&cfg->d11inf, chan); - brcmf_dbg(CONN, "channel=%d, center_req=%d, chanspec=0x%04x\n", - cfg->channel, chan->center_freq, chanspec); + brcmf_dbg(CONN, "channel=%d, center_req=%d\n", + cfg->channel, chan->center_freq); } else { cfg->channel = 0; - chanspec = 0; } - brcmf_dbg(INFO, "ie (%p), ie_len (%zd)\n", sme->ie, sme->ie_len); + brcmf_dbg(INFO, "ie (%p), ie_len (%zd)\n", params->ie, params->ie_len); - err = brcmf_set_wpa_version(ndev, sme); + err = brcmf_set_wpa_version(ndev, params); if (err) { bphy_err(drvr, "wl_set_wpa_version failed (%d)\n", err); goto done; } - sme->auth_type = brcmf_war_auth_type(ifp, sme->auth_type); - err = brcmf_set_auth_type(ndev, sme); + params->auth_type = brcmf_war_auth_type(ifp, params->auth_type); + err = brcmf_set_auth_type(ndev, params); if (err) { bphy_err(drvr, "wl_set_auth_type failed (%d)\n", err); goto done; } - err = brcmf_set_wsec_mode(ndev, sme); + err = brcmf_set_wsec_mode(ndev, params); if (err) { bphy_err(drvr, "wl_set_set_cipher failed (%d)\n", err); goto done; } - err = brcmf_set_key_mgmt(ndev, sme); + err = brcmf_set_key_mgmt(ndev, params); if (err) { bphy_err(drvr, "wl_set_key_mgmt failed (%d)\n", err); goto done; } - err = brcmf_set_sharedkey(ndev, sme); - if (err) { - bphy_err(drvr, "brcmf_set_sharedkey failed (%d)\n", err); - goto done; - } - - if (sme->crypto.psk && - profile->use_fwsup != BRCMF_PROFILE_FWSUP_SAE) { - if (WARN_ON(profile->use_fwsup != BRCMF_PROFILE_FWSUP_NONE)) { - err = -EINVAL; - goto done; - } - brcmf_dbg(INFO, "using PSK offload\n"); - profile->use_fwsup = BRCMF_PROFILE_FWSUP_PSK; - } - - if (profile->use_fwsup != BRCMF_PROFILE_FWSUP_NONE) { - /* enable firmware supplicant for this interface */ - err = brcmf_fil_iovar_int_set(ifp, "sup_wpa", 1); - if (err < 0) { - bphy_err(drvr, "failed to enable fw supplicant\n"); - goto done; - } - } - - if (profile->use_fwsup == BRCMF_PROFILE_FWSUP_PSK) - err = brcmf_set_pmk(ifp, sme->crypto.psk, - BRCMF_WSEC_MAX_PSK_LEN); - else if (profile->use_fwsup == BRCMF_PROFILE_FWSUP_SAE) { - /* clean up user-space RSNE */ - err = brcmf_fil_iovar_data_set(ifp, "wpaie", NULL, 0); - if (err) { - bphy_err(drvr, "failed to clean up user-space RSNE\n"); - goto done; - } - err = brcmf_fwvid_set_sae_password(ifp, &sme->crypto); - if (!err && sme->crypto.psk) - err = brcmf_set_pmk(ifp, sme->crypto.psk, - BRCMF_WSEC_MAX_PSK_LEN); - } - if (err) - goto done; - - /* Join with specific BSSID and cached SSID - * If SSID is zero join based on BSSID only - */ - join_params_size = offsetof(struct brcmf_ext_join_params_le, assoc_le) + - offsetof(struct brcmf_assoc_params_le, chanspec_list); - if (cfg->channel) - join_params_size += sizeof(u16); - ext_join_params = kzalloc(sizeof(*ext_join_params), GFP_KERNEL); - if (ext_join_params == NULL) { - err = -ENOMEM; + err = brcmf_set_sharedkey(ndev, params); + if (err) { + bphy_err(drvr, "brcmf_set_sharedkey failed (%d)\n", err); goto done; } - ssid_len = min_t(u32, sme->ssid_len, IEEE80211_MAX_SSID_LEN); - ext_join_params->ssid_le.SSID_len = cpu_to_le32(ssid_len); - memcpy(&ext_join_params->ssid_le.SSID, sme->ssid, ssid_len); - if (ssid_len < IEEE80211_MAX_SSID_LEN) - brcmf_dbg(CONN, "SSID \"%s\", len (%d)\n", - ext_join_params->ssid_le.SSID, ssid_len); + if (brcmf_feat_is_enabled(ifp, BRCMF_FEAT_FWSUP)) { + if (params->crypto.psk) { + if ((profile->use_fwsup != BRCMF_PROFILE_FWSUP_SAE) && + (profile->use_fwsup != BRCMF_PROFILE_FWSUP_PSK)) { + if (WARN_ON(profile->use_fwsup != + BRCMF_PROFILE_FWSUP_NONE)) { + err = -EINVAL; + goto done; + } + brcmf_dbg(INFO, "using PSK offload\n"); + profile->use_fwsup = BRCMF_PROFILE_FWSUP_PSK; + } + } - /* Set up join scan parameters */ - ext_join_params->scan_le.scan_type = -1; - ext_join_params->scan_le.home_time = cpu_to_le32(-1); + if (profile->use_fwsup != BRCMF_PROFILE_FWSUP_NONE) { + /* enable firmware supplicant for this interface */ + err = brcmf_fil_iovar_int_set(ifp, "sup_wpa", 1); + if (err < 0) { + bphy_err(drvr, + "failed to enable fw supplicant\n"); + goto done; + } + } else { + err = brcmf_fil_iovar_int_set(ifp, "sup_wpa", 0); + } - if (sme->bssid) - memcpy(&ext_join_params->assoc_le.bssid, sme->bssid, ETH_ALEN); - else - eth_broadcast_addr(ext_join_params->assoc_le.bssid); + if ((profile->use_fwsup == BRCMF_PROFILE_FWSUP_PSK) && + params->crypto.psk) + err = brcmf_set_pmk(ifp, params->crypto.psk, + BRCMF_WSEC_MAX_PSK_LEN); + else if (profile->use_fwsup == BRCMF_PROFILE_FWSUP_SAE) { + /* clean up user-space RSNE */ + if (brcmf_fil_iovar_data_set(ifp, "wpaie", NULL, 0)) { + bphy_err( + drvr, + "failed to clean up user-space RSNE\n"); + goto done; + } + err = brcmf_fwvid_set_sae_password(ifp, ¶ms->crypto); + if (!err && params->crypto.psk) + err = brcmf_set_pmk(ifp, params->crypto.psk, + BRCMF_WSEC_MAX_PSK_LEN); + } + if (err) + goto done; + } + brcmf_set_join_pref(ifp, ¶ms->bss_select); + if (params->ssid_len < IEEE80211_MAX_SSID_LEN) + brcmf_dbg(CONN, "SSID \"%s\", len (%zu)\n", params->ssid, + params->ssid_len); + join_params = drvr->join_param_handler.get_struct_for_connect( + cfg, &join_params_size, params); - if (cfg->channel) { - ext_join_params->assoc_le.chanspec_num = cpu_to_le32(1); + if (join_params) { + err = brcmf_fil_bsscfg_data_set(ifp, "join", join_params, + join_params_size); - ext_join_params->assoc_le.chanspec_list[0] = - cpu_to_le16(chanspec); - /* Increase dwell time to receive probe response or detect - * beacon from target AP at a noisy air only during connect - * command. - */ - ext_join_params->scan_le.active_time = - cpu_to_le32(BRCMF_SCAN_JOIN_ACTIVE_DWELL_TIME_MS); - ext_join_params->scan_le.passive_time = - cpu_to_le32(BRCMF_SCAN_JOIN_PASSIVE_DWELL_TIME_MS); - /* To sync with presence period of VSDB GO send probe request - * more frequently. Probe request will be stopped when it gets - * probe response from target AP/GO. - */ - ext_join_params->scan_le.nprobes = - cpu_to_le32(BRCMF_SCAN_JOIN_ACTIVE_DWELL_TIME_MS / - BRCMF_SCAN_JOIN_PROBE_INTERVAL_MS); - } else { - ext_join_params->scan_le.active_time = cpu_to_le32(-1); - ext_join_params->scan_le.passive_time = cpu_to_le32(-1); - ext_join_params->scan_le.nprobes = cpu_to_le32(-1); + /* We only free the join parameters if we were successful. + * Otherwise they are used to extract the fallback, below */ + if (!err) { + kfree(join_params); + /* This is it. join command worked, we are done */ + goto done; + } + /* For versions >= 1, this should have worked, so report the error */ + if (drvr->join_param_handler.version >= 1) { + bphy_err(drvr, "Failed to use join iovar to join: %d\n", + err); + } } - brcmf_set_join_pref(ifp, &sme->bss_select); - - err = brcmf_fil_bsscfg_data_set(ifp, "join", ext_join_params, - join_params_size); - kfree(ext_join_params); - if (!err) - /* This is it. join command worked, we are done */ + /* Fallback to using WLC_SET_SSID approach, which just uses join_params parts of the structure */ + fallback_join_params = drvr->join_param_handler.get_join_from_ext_join( + join_params, &fallback_join_params_size); + if (!fallback_join_params) { + bphy_err(drvr, "Unable to generate fallback join params\n"); + kfree(join_params); goto done; - - /* join command failed, fallback to set ssid */ - memset(&join_params, 0, sizeof(join_params)); - join_params_size = sizeof(join_params.ssid_le); - - memcpy(&join_params.ssid_le.SSID, sme->ssid, ssid_len); - join_params.ssid_le.SSID_len = cpu_to_le32(ssid_len); - - if (sme->bssid) - memcpy(join_params.params_le.bssid, sme->bssid, ETH_ALEN); - else - eth_broadcast_addr(join_params.params_le.bssid); - - if (cfg->channel) { - join_params.params_le.chanspec_list[0] = cpu_to_le16(chanspec); - join_params.params_le.chanspec_num = cpu_to_le32(1); - join_params_size += sizeof(join_params.params_le); } err = brcmf_fil_cmd_data_set(ifp, BRCMF_C_SET_SSID, - &join_params, join_params_size); + fallback_join_params, + fallback_join_params_size); + + kfree(join_params); + kfree(fallback_join_params); if (err) bphy_err(drvr, "BRCMF_C_SET_SSID failed (%d)\n", err); @@ -2807,6 +2557,8 @@ brcmf_cfg80211_add_key(struct wiphy *wiphy, struct net_device *ndev, s32 val; s32 wsec; s32 err; + u32 algos = 0; + u32 mask = 0; u8 keybuf[8]; bool ext_key; @@ -2890,6 +2642,30 @@ brcmf_cfg80211_add_key(struct wiphy *wiphy, struct net_device *ndev, val = AES_ENABLED; brcmf_dbg(CONN, "WLAN_CIPHER_SUITE_CCMP\n"); break; + case WLAN_CIPHER_SUITE_GCMP_256: + if (!brcmf_feat_is_enabled(ifp, BRCMF_FEAT_GCMP)) { + brcmf_err("the low layer not support GCMP\n"); + err = -EOPNOTSUPP; + goto done; + } + key->algo = CRYPTO_ALGO_AES_GCM256; + val = AES_ENABLED; + brcmf_dbg(CONN, "WLAN_CIPHER_SUITE_GCMP_256\n"); + algos = KEY_ALGO_MASK(CRYPTO_ALGO_AES_GCM256); + mask = algos | KEY_ALGO_MASK(CRYPTO_ALGO_AES_CCM); + break; + case WLAN_CIPHER_SUITE_BIP_GMAC_256: + if (!brcmf_feat_is_enabled(ifp, BRCMF_FEAT_GCMP)) { + brcmf_err("the low layer not support GCMP\n"); + err = -EOPNOTSUPP; + goto done; + } + key->algo = CRYPTO_ALGO_BIP_GMAC256; + val = AES_ENABLED; + algos = KEY_ALGO_MASK(CRYPTO_ALGO_BIP_GMAC256); + mask = algos | KEY_ALGO_MASK(CRYPTO_ALGO_AES_CCM); + brcmf_dbg(CONN, "WLAN_CIPHER_SUITE_BIP_GMAC_256\n"); + break; default: bphy_err(drvr, "Invalid cipher (0x%x)\n", params->cipher); err = -EINVAL; @@ -2911,6 +2687,17 @@ brcmf_cfg80211_add_key(struct wiphy *wiphy, struct net_device *ndev, bphy_err(drvr, "set wsec error (%d)\n", err); goto done; } + if (brcmf_feat_is_enabled(ifp, BRCMF_FEAT_GCMP)) { + brcmf_dbg(CONN, + "set_wsdec_info algos (0x%x) mask (0x%x)\n", + algos, mask); + err = brcmf_set_wsec_info_algos(ifp, algos, mask); + if (err) { + brcmf_err("set wsec_info error (%d)\n", err); + return err; + } + } + done: brcmf_dbg(TRACE, "Exit\n"); @@ -3131,6 +2918,70 @@ brcmf_cfg80211_get_station_ibss(struct brcmf_if *ifp, return 0; } +static void brcmf_convert_ratespec_to_rateinfo(u32 ratespec, + struct rate_info *rateinfo) +{ + /* First extract the bandwidth info */ + switch (ratespec & BRCMF_RSPEC_BW_MASK) { + case BRCMF_RSPEC_BW_20MHZ: + rateinfo->bw = RATE_INFO_BW_20; + break; + case BRCMF_RSPEC_BW_40MHZ: + rateinfo->bw = RATE_INFO_BW_40; + break; + case BRCMF_RSPEC_BW_80MHZ: + rateinfo->bw = RATE_INFO_BW_80; + break; + case BRCMF_RSPEC_BW_160MHZ: + rateinfo->bw = RATE_INFO_BW_160; + break; + case BRCMF_RSPEC_BW_320MHZ: + rateinfo->bw = RATE_INFO_BW_320; + break; + default: + /* Fill in nothing */ + break; + } + if (BRCMF_RSPEC_ISHT(ratespec)) { + rateinfo->flags |= RATE_INFO_FLAGS_MCS; + rateinfo->mcs = ratespec & BRCMF_RSPEC_HT_MCS_MASK; + } else if (BRCMF_RSPEC_ISVHT(ratespec)) { + rateinfo->flags |= RATE_INFO_FLAGS_VHT_MCS; + rateinfo->mcs = ratespec & BRCMF_RSPEC_VHT_MCS_MASK; + rateinfo->nss = (ratespec & BRCMF_RSPEC_VHT_NSS_MASK) >> + BRCMF_RSPEC_VHT_NSS_SHIFT; + } else if (BRCMF_RSPEC_ISHE(ratespec)) { + u32 ltf_gi = BRCMF_RSPEC_HE_LTF_GI(ratespec); + + rateinfo->flags |= RATE_INFO_FLAGS_HE_MCS; + rateinfo->mcs = ratespec & BRCMF_RSPEC_HE_MCS_MASK; + rateinfo->nss = (ratespec & BRCMF_RSPEC_HE_NSS_MASK) >> + BRCMF_RSPEC_HE_NSS_SHIFT; + rateinfo->he_dcm = BRCMF_RSPEC_HE_DCM(ratespec); + if (HE_IS_GI_0_8us(ltf_gi)) { + rateinfo->he_gi = NL80211_RATE_INFO_HE_GI_0_8; + } else if (HE_IS_GI_1_6us(ltf_gi)) { + rateinfo->he_gi = NL80211_RATE_INFO_HE_GI_1_6; + } else if (HE_IS_GI_3_2us(ltf_gi)) { + rateinfo->he_gi = NL80211_RATE_INFO_HE_GI_3_2; + } + } else if (BRCMF_RSPEC_ISEHT(ratespec)) { + u32 ltf_gi = BRCMF_RSPEC_EHT_LTF_GI(ratespec); + + rateinfo->flags |= RATE_INFO_FLAGS_EHT_MCS; + rateinfo->mcs = ratespec & BRCMF_RSPEC_EHT_MCS_MASK; + rateinfo->nss = (ratespec & BRCMF_RSPEC_EHT_NSS_MASK) >> + BRCMF_RSPEC_EHT_NSS_SHIFT; + if (EHT_IS_GI_0_8us(ltf_gi)) { + rateinfo->eht_gi = NL80211_RATE_INFO_EHT_GI_0_8; + } else if (EHT_IS_GI_1_6us(ltf_gi)) { + rateinfo->eht_gi = NL80211_RATE_INFO_EHT_GI_1_6; + } else if (EHT_IS_GI_3_2us(ltf_gi)) { + rateinfo->eht_gi = NL80211_RATE_INFO_EHT_GI_3_2; + } + } +} + static s32 brcmf_cfg80211_get_station(struct wiphy *wiphy, struct net_device *ndev, const u8 *mac, struct station_info *sinfo) @@ -3148,6 +2999,8 @@ brcmf_cfg80211_get_station(struct wiphy *wiphy, struct net_device *ndev, s32 count_rssi = 0; int rssi; u32 i; + u16 struct_ver; + u16 info_len; brcmf_dbg(TRACE, "Enter, MAC %pM\n", mac); if (!check_vif_up(ifp->vif)) @@ -3171,7 +3024,9 @@ brcmf_cfg80211_get_station(struct wiphy *wiphy, struct net_device *ndev, goto done; } } - brcmf_dbg(TRACE, "version %d\n", le16_to_cpu(sta_info_le.ver)); + info_len = le16_to_cpu(sta_info_le.len); + struct_ver = le16_to_cpu(sta_info_le.ver); + brcmf_dbg(TRACE, "version %d\n", struct_ver); sinfo->filled = BIT_ULL(NL80211_STA_INFO_INACTIVE_TIME); sinfo->inactive_time = le32_to_cpu(sta_info_le.idle) * 1000; sta_flags = le32_to_cpu(sta_info_le.flags); @@ -3205,12 +3060,13 @@ brcmf_cfg80211_get_station(struct wiphy *wiphy, struct net_device *ndev, sinfo->rxrate.legacy = le32_to_cpu(sta_info_le.rx_rate) / 100; } - if (le16_to_cpu(sta_info_le.ver) >= 4) { + if (struct_ver >= 4) { sinfo->filled |= BIT_ULL(NL80211_STA_INFO_TX_BYTES); sinfo->tx_bytes = le64_to_cpu(sta_info_le.tx_tot_bytes); sinfo->filled |= BIT_ULL(NL80211_STA_INFO_RX_BYTES); sinfo->rx_bytes = le64_to_cpu(sta_info_le.rx_tot_bytes); } + for (i = 0; i < BRCMF_ANT_MAX; i++) { if (sta_info_le.rssi[i] == 0 || sta_info_le.rx_lastpkt_rssi[i] == 0) @@ -3249,6 +3105,25 @@ brcmf_cfg80211_get_station(struct wiphy *wiphy, struct net_device *ndev, } } } + /* Some version 7 structs have ratespecs from the last packet. */ + if (struct_ver >= 7) { + if (info_len >= sizeof(sta_info_le)) { + brcmf_convert_ratespec_to_rateinfo( + le32_to_cpu(sta_info_le.v7.tx_rspec), + &sinfo->txrate); + brcmf_convert_ratespec_to_rateinfo( + le32_to_cpu(sta_info_le.v7.rx_rspec), + &sinfo->rxrate); + } else { + /* We didn't get the fields we were expecting, fallback to nrate */ + u32 nrate = 0; + err = brcmf_fil_iovar_int_get(ifp, "nrate", &nrate); + if (!err) { + brcmf_convert_ratespec_to_rateinfo( + nrate, &sinfo->txrate); + } + } + } done: brcmf_dbg(TRACE, "Exit\n"); return err; @@ -3349,6 +3224,7 @@ static s32 brcmf_inform_single_bss(struct brcmf_cfg80211_info *cfg, struct cfg80211_bss *bss; enum nl80211_band band; struct brcmu_chan ch; + u16 chanspec; u16 channel; u32 freq; u16 notify_capability; @@ -3362,20 +3238,41 @@ static s32 brcmf_inform_single_bss(struct brcmf_cfg80211_info *cfg, return -EINVAL; } + chanspec = le16_to_cpu(bi->chanspec); if (!bi->ctl_ch) { - ch.chspec = le16_to_cpu(bi->chanspec); + ch.chspec = chanspec; cfg->d11inf.decchspec(&ch); bi->ctl_ch = ch.control_ch_num; } channel = bi->ctl_ch; - if (channel <= CH_MAX_2G_CHANNEL) - band = NL80211_BAND_2GHZ; - else + if (CHSPEC_IS6G(chanspec)) + band = NL80211_BAND_6GHZ; + else if (CHSPEC_IS5G(chanspec)) band = NL80211_BAND_5GHZ; + else + band = NL80211_BAND_2GHZ; freq = ieee80211_channel_to_frequency(channel, band); + if (!freq) { + brcmf_err("Invalid frequency %d returned for channel %d, band %d. chanspec was %04x\n", + freq, channel, band, bi->chanspec); + + /* We ignore this BSS ID rather than try to continue on. + * Otherwise we will cause an OOPs because our frequency is 0. + * The main case this occurs is some new frequency band + * we have not seen before, and if we return an error, + * we will cause the scan to fail. It seems better to + * report the error, skip this BSS, and move on. + */ + return 0; + } bss_data.chan = ieee80211_get_channel(wiphy, freq); + if (!bss_data.chan) { + brcmf_err("Could not convert frequency into channel for channel %d, band %d, chanspec was %04x\n", + channel, band, bi->chanspec); + return 0; + } bss_data.boottime_ns = ktime_to_ns(ktime_get_boottime()); notify_capability = le16_to_cpu(bi->capability); @@ -3424,8 +3321,9 @@ static s32 brcmf_inform_bss(struct brcmf_cfg80211_info *cfg) bss_list = (struct brcmf_scan_results *)cfg->escan_info.escan_buf; if (bss_list->count != 0 && - bss_list->version != BRCMF_BSS_INFO_VERSION) { - bphy_err(drvr, "Version %d != WL_BSS_INFO_VERSION\n", + (bss_list->version < BRCMF_BSS_INFO_MIN_VERSION || + bss_list->version > BRCMF_BSS_INFO_MAX_VERSION)) { + bphy_err(drvr, "BSS info version %d unsupported\n", bss_list->version); return -EOPNOTSUPP; } @@ -3463,7 +3361,7 @@ static s32 brcmf_inform_ibss(struct brcmf_cfg80211_info *cfg, buf = kzalloc(WL_BSS_INFO_MAX, GFP_KERNEL); if (buf == NULL) { err = -ENOMEM; - goto CleanUp; + goto cleanup; } *(__le32 *)buf = cpu_to_le32(WL_BSS_INFO_MAX); @@ -3472,7 +3370,7 @@ static s32 brcmf_inform_ibss(struct brcmf_cfg80211_info *cfg, buf, WL_BSS_INFO_MAX); if (err) { bphy_err(drvr, "WLC_GET_BSS_INFO failed: %d\n", err); - goto CleanUp; + goto cleanup; } bi = (struct brcmf_bss_info_le *)(buf + 4); @@ -3482,10 +3380,18 @@ static s32 brcmf_inform_ibss(struct brcmf_cfg80211_info *cfg, if (ch.band == BRCMU_CHAN_BAND_2G) band = wiphy->bands[NL80211_BAND_2GHZ]; - else + else if (ch.band == BRCMU_CHAN_BAND_5G) band = wiphy->bands[NL80211_BAND_5GHZ]; + else + band = wiphy->bands[NL80211_BAND_6GHZ]; freq = ieee80211_channel_to_frequency(ch.control_ch_num, band->band); + if (freq == 0) { + brcmf_err("Invalid frequency %d returned for channel %d, band %d. chanspec was %04x\n", + freq, ch.control_ch_num, ch.band, bi->chanspec); + goto cleanup; + } + cfg->channel = freq; notify_channel = ieee80211_get_channel(wiphy, freq); @@ -3508,12 +3414,12 @@ static s32 brcmf_inform_ibss(struct brcmf_cfg80211_info *cfg, if (!bss) { err = -ENOMEM; - goto CleanUp; + goto cleanup; } cfg80211_put_bss(wiphy, bss); -CleanUp: +cleanup: kfree(buf); @@ -3765,17 +3671,11 @@ brcmf_alloc_internal_escan_request(struct wiphy *wiphy, u32 n_netinfo) { } static int brcmf_internal_escan_add_info(struct cfg80211_scan_request *req, - u8 *ssid, u8 ssid_len, u8 channel) + u8 *ssid, u8 ssid_len, u8 channel, enum nl80211_band band) { struct ieee80211_channel *chan; - enum nl80211_band band; int freq, i; - if (channel <= CH_MAX_2G_CHANNEL) - band = NL80211_BAND_2GHZ; - else - band = NL80211_BAND_5GHZ; - freq = ieee80211_channel_to_frequency(channel, band); if (!freq) return -EINVAL; @@ -3831,53 +3731,30 @@ static int brcmf_start_internal_escan(struct brcmf_if *ifp, u32 fwmap, return 0; } -static struct brcmf_pno_net_info_le * -brcmf_get_netinfo_array(struct brcmf_pno_scanresults_le *pfn_v1) -{ - struct brcmf_pno_scanresults_v2_le *pfn_v2; - struct brcmf_pno_net_info_le *netinfo; - - switch (pfn_v1->version) { - default: - WARN_ON(1); - fallthrough; - case cpu_to_le32(1): - netinfo = (struct brcmf_pno_net_info_le *)(pfn_v1 + 1); - break; - case cpu_to_le32(2): - pfn_v2 = (struct brcmf_pno_scanresults_v2_le *)pfn_v1; - netinfo = (struct brcmf_pno_net_info_le *)(pfn_v2 + 1); - break; - } - - return netinfo; -} - /* PFN result doesn't have all the info which are required by the supplicant * (For e.g IEs) Do a target Escan so that sched scan results are reported * via wl_inform_single_bss in the required format. Escan does require the * scan request in the form of cfg80211_scan_request. For timebeing, create * cfg80211_scan_request one out of the received PNO event. */ -static s32 -brcmf_notify_sched_scan_results(struct brcmf_if *ifp, - const struct brcmf_event_msg *e, void *data) +static s32 brcmf_notify_sched_scan_results(struct brcmf_if *ifp, + const struct brcmf_event_msg *e, + void *data) { struct brcmf_pub *drvr = ifp->drvr; struct brcmf_cfg80211_info *cfg = drvr->config; - struct brcmf_pno_net_info_le *netinfo, *netinfo_start; struct cfg80211_scan_request *request = NULL; struct wiphy *wiphy = cfg_to_wiphy(cfg); int i, err = 0; - struct brcmf_pno_scanresults_le *pfn_result; u32 bucket_map; u32 result_count; u32 status; - u32 datalen; + u32 min_data_len; brcmf_dbg(SCAN, "Enter\n"); + min_data_len = drvr->pno_handler.get_min_data_len(); - if (e->datalen < (sizeof(*pfn_result) + sizeof(*netinfo))) { + if (e->datalen < min_data_len) { brcmf_dbg(SCAN, "Event data too small. Ignore\n"); return 0; } @@ -3887,9 +3764,8 @@ brcmf_notify_sched_scan_results(struct brcmf_if *ifp, return 0; } - pfn_result = (struct brcmf_pno_scanresults_le *)data; - result_count = le32_to_cpu(pfn_result->count); - status = le32_to_cpu(pfn_result->status); + result_count = drvr->pno_handler.get_result_count(data); + status = drvr->pno_handler.get_result_status(data); /* PFN event is limited to fit 512 bytes so we may get * multiple NET_FOUND events. For now place a warning here. @@ -3900,38 +3776,33 @@ brcmf_notify_sched_scan_results(struct brcmf_if *ifp, bphy_err(drvr, "FALSE PNO Event. (pfn_count == 0)\n"); goto out_err; } - - netinfo_start = brcmf_get_netinfo_array(pfn_result); - datalen = e->datalen - ((void *)netinfo_start - (void *)pfn_result); - if (datalen < result_count * sizeof(*netinfo)) { - bphy_err(drvr, "insufficient event data\n"); + err = drvr->pno_handler.validate_pfn_results(data, e->datalen); + if (err) { + bphy_err(drvr, "Invalid escan results (%d)", err); goto out_err; } - - request = brcmf_alloc_internal_escan_request(wiphy, - result_count); + request = brcmf_alloc_internal_escan_request(wiphy, result_count); if (!request) { err = -ENOMEM; goto out_err; } - bucket_map = 0; for (i = 0; i < result_count; i++) { - netinfo = &netinfo_start[i]; - - if (netinfo->SSID_len > IEEE80211_MAX_SSID_LEN) - netinfo->SSID_len = IEEE80211_MAX_SSID_LEN; - brcmf_dbg(SCAN, "SSID:%.32s Channel:%d\n", - netinfo->SSID, netinfo->channel); - bucket_map |= brcmf_pno_get_bucket_map(cfg->pno, netinfo); - err = brcmf_internal_escan_add_info(request, - netinfo->SSID, - netinfo->SSID_len, - netinfo->channel); + u8 channel; + enum nl80211_band band; + u8 ssid[IEEE80211_MAX_SSID_LEN]; + u8 ssid_len; + + drvr->pno_handler.get_result_info(data, i, &ssid, &ssid_len, + &channel, &band); + brcmf_dbg(SCAN, "SSID:%.32s Channel:%d Band:%d\n", ssid, + channel, band); + bucket_map |= drvr->pno_handler.get_bucket_map(data, i, cfg->pno); + err = brcmf_internal_escan_add_info(request, ssid, ssid_len, + channel, band); if (err) goto out_err; } - if (!bucket_map) goto free_req; @@ -4034,48 +3905,50 @@ static s32 brcmf_config_wowl_pattern(struct brcmf_if *ifp, u8 cmd[4], return ret; } -static s32 -brcmf_wowl_nd_results(struct brcmf_if *ifp, const struct brcmf_event_msg *e, - void *data) +static s32 brcmf_wowl_nd_results(struct brcmf_if *ifp, + const struct brcmf_event_msg *e, void *data) { struct brcmf_pub *drvr = ifp->drvr; struct brcmf_cfg80211_info *cfg = drvr->config; - struct brcmf_pno_scanresults_le *pfn_result; - struct brcmf_pno_net_info_le *netinfo; + u32 min_data_len; + u8 channel; + enum nl80211_band band; + u8 ssid[IEEE80211_MAX_SSID_LEN]; + u8 ssid_len; + u32 result_count; brcmf_dbg(SCAN, "Enter\n"); - if (e->datalen < (sizeof(*pfn_result) + sizeof(*netinfo))) { + min_data_len = drvr->pno_handler.get_min_data_len(); + + if (e->datalen < min_data_len) { brcmf_dbg(SCAN, "Event data too small. Ignore\n"); return 0; } - pfn_result = (struct brcmf_pno_scanresults_le *)data; if (e->event_code == BRCMF_E_PFN_NET_LOST) { brcmf_dbg(SCAN, "PFN NET LOST event. Ignore\n"); return 0; } - if (le32_to_cpu(pfn_result->count) < 1) { + result_count = drvr->pno_handler.get_result_count(data); + if (result_count < 1) { bphy_err(drvr, "Invalid result count, expected 1 (%d)\n", - le32_to_cpu(pfn_result->count)); + result_count); return -EINVAL; } - netinfo = brcmf_get_netinfo_array(pfn_result); - if (netinfo->SSID_len > IEEE80211_MAX_SSID_LEN) - netinfo->SSID_len = IEEE80211_MAX_SSID_LEN; - memcpy(cfg->wowl.nd->ssid.ssid, netinfo->SSID, netinfo->SSID_len); - cfg->wowl.nd->ssid.ssid_len = netinfo->SSID_len; + drvr->pno_handler.get_result_info(data, 0, &ssid, &ssid_len, &channel, + &band); + memcpy(cfg->wowl.nd->ssid.ssid, ssid, ssid_len); + cfg->wowl.nd->ssid.ssid_len = ssid_len; cfg->wowl.nd->n_channels = 1; cfg->wowl.nd->channels[0] = - ieee80211_channel_to_frequency(netinfo->channel, - netinfo->channel <= CH_MAX_2G_CHANNEL ? - NL80211_BAND_2GHZ : NL80211_BAND_5GHZ); + ieee80211_channel_to_frequency(channel, band); + cfg->wowl.nd_info->n_matches = 1; cfg->wowl.nd_info->matches[0] = cfg->wowl.nd; - /* Inform (the resume task) that the net detect information was recvd */ cfg->wowl.nd_data_completed = true; wake_up(&cfg->wowl.nd_data_wait); @@ -5149,6 +5022,25 @@ brcmf_cfg80211_start_ap(struct wiphy *wiphy, struct net_device *ndev, settings->inactivity_timeout); dev_role = ifp->vif->wdev.iftype; mbss = ifp->vif->mbss; + /* Bring firmware into correct state for AP mode*/ + if (dev_role == NL80211_IFTYPE_AP) { + brcmf_dbg(TRACE, "set AP mode\n"); + err = brcmf_fil_cmd_int_set(ifp, BRCMF_C_SET_AP, 1); + if (err < 0) { + bphy_err(drvr, "setting AP mode failed %d\n", + err); + goto exit; + } + + bss_enable.bsscfgidx = cpu_to_le32(ifp->bsscfgidx); + bss_enable.enable = cpu_to_le32(WLC_AP_IOV_OP_MANUAL_AP_BSSCFG_CREATE); + err = brcmf_fil_iovar_data_set(ifp, "bss", &bss_enable, + sizeof(bss_enable)); + if (err < 0) { + bphy_err(drvr, "AP role set error, %d\n", err); + goto exit; + } + } /* store current 11d setting */ if (brcmf_fil_cmd_int_get(ifp, BRCMF_C_GET_REGULATORY, @@ -5742,6 +5634,9 @@ static int brcmf_cfg80211_get_channel(struct wiphy *wiphy, case BRCMU_CHAN_BAND_5G: band = NL80211_BAND_5GHZ; break; + case BRCMU_CHAN_BAND_6G: + band = NL80211_BAND_6GHZ; + break; } switch (ch.bw) { @@ -5763,9 +5658,19 @@ static int brcmf_cfg80211_get_channel(struct wiphy *wiphy, } freq = ieee80211_channel_to_frequency(ch.control_ch_num, band); + if (freq == 0) { + brcmf_err("Invalid frequency %d returned for channel %d, band %d. chanspec was %04x\n", + freq, ch.control_ch_num, ch.band, chanspec); + return -EINVAL; + } chandef->chan = ieee80211_get_channel(wiphy, freq); chandef->width = width; chandef->center_freq1 = ieee80211_channel_to_frequency(ch.chnum, band); + if (chandef->center_freq1 == 0) { + brcmf_err("Invalid frequency %d returned for channel %d, band %d. chanspec was %04x\n", + freq, ch.chnum, ch.band, chanspec); + return -EINVAL; + } chandef->center_freq2 = 0; return 0; @@ -5930,17 +5835,29 @@ static int brcmf_cfg80211_set_pmk(struct wiphy *wiphy, struct net_device *dev, const struct cfg80211_pmk_conf *conf) { struct brcmf_if *ifp; - + struct brcmf_pub *drvr; + int ret; brcmf_dbg(TRACE, "enter\n"); /* expect using firmware supplicant for 1X */ ifp = netdev_priv(dev); - if (WARN_ON(ifp->vif->profile.use_fwsup != BRCMF_PROFILE_FWSUP_1X)) + drvr = ifp->drvr; + if (WARN_ON((ifp->vif->profile.use_fwsup != BRCMF_PROFILE_FWSUP_1X) && + (ifp->vif->profile.use_fwsup != BRCMF_PROFILE_FWSUP_ROAM) && + (ifp->vif->profile.is_ft != true) && + (ifp->vif->profile.is_okc != true))) return -EINVAL; if (conf->pmk_len > BRCMF_WSEC_MAX_PSK_LEN) return -ERANGE; + if (ifp->vif->profile.is_okc) { + ret = brcmf_fil_iovar_data_set(ifp, "okc_info_pmk", conf->pmk, + conf->pmk_len); + if (ret < 0) + bphy_err(drvr, "okc_info_pmk iovar failed: ret=%d\n", + ret); + } return brcmf_set_pmk(ifp, conf->pmk, conf->pmk_len); } @@ -6399,6 +6316,46 @@ static s32 brcmf_get_assoc_ies(struct brcmf_cfg80211_info *cfg, return err; } +static bool brcmf_has_pmkid(const u8 *parse, u32 len) +{ + const struct brcmf_tlv *rsn_ie; + const u8 *ie; + u32 ie_len; + u32 offset; + u16 count; + + rsn_ie = brcmf_parse_tlvs(parse, len, WLAN_EID_RSN); + if (!rsn_ie) + goto done; + ie = (const u8 *)rsn_ie; + ie_len = rsn_ie->len + TLV_HDR_LEN; + /* Skip group data cipher suite */ + offset = TLV_HDR_LEN + WPA_IE_VERSION_LEN + WPA_IE_MIN_OUI_LEN; + if (offset + WPA_IE_SUITE_COUNT_LEN >= ie_len) + goto done; + /* Skip pairwise cipher suite(s) */ + count = ie[offset] + (ie[offset + 1] << 8); + offset += WPA_IE_SUITE_COUNT_LEN + (count * WPA_IE_MIN_OUI_LEN); + if (offset + WPA_IE_SUITE_COUNT_LEN >= ie_len) + goto done; + /* Skip auth key management suite(s) */ + count = ie[offset] + (ie[offset + 1] << 8); + offset += WPA_IE_SUITE_COUNT_LEN + (count * WPA_IE_MIN_OUI_LEN); + if (offset + RSN_CAP_LEN >= ie_len) + goto done; + /* Skip rsn capabilities */ + offset += RSN_CAP_LEN; + if (offset + RSN_PMKID_COUNT_LEN > ie_len) + goto done; + /* Extract PMKID count */ + count = ie[offset] + (ie[offset + 1] << 8); + if (count) + return true; + +done: + return false; +} + static s32 brcmf_bss_roaming_done(struct brcmf_cfg80211_info *cfg, struct net_device *ndev, @@ -6443,10 +6400,17 @@ brcmf_bss_roaming_done(struct brcmf_cfg80211_info *cfg, if (ch.band == BRCMU_CHAN_BAND_2G) band = wiphy->bands[NL80211_BAND_2GHZ]; - else + else if (ch.band == BRCMU_CHAN_BAND_5G) band = wiphy->bands[NL80211_BAND_5GHZ]; + else + band = wiphy->bands[NL80211_BAND_6GHZ]; freq = ieee80211_channel_to_frequency(ch.control_ch_num, band->band); + if (freq == 0) { + brcmf_err("Invalid frequency %d returned for channel %d, band %d. chanspec was %04x\n", + freq, ch.control_ch_num, ch.band, bi->chanspec); + goto done; + } notify_channel = ieee80211_get_channel(wiphy, freq); done: @@ -6462,11 +6426,16 @@ brcmf_bss_roaming_done(struct brcmf_cfg80211_info *cfg, cfg80211_roamed(ndev, &roam_info, GFP_KERNEL); brcmf_dbg(CONN, "Report roaming result\n"); - if (profile->use_fwsup == BRCMF_PROFILE_FWSUP_1X && profile->is_ft) { - cfg80211_port_authorized(ndev, profile->bssid, NULL, 0, GFP_KERNEL); + if (((profile->use_fwsup == BRCMF_PROFILE_FWSUP_1X || + profile->use_fwsup == BRCMF_PROFILE_FWSUP_ROAM) && + (brcmf_has_pmkid(roam_info.req_ie, roam_info.req_ie_len) || + profile->is_ft || profile->is_okc))) { + cfg80211_port_authorized(ndev, profile->bssid, NULL, 0, + GFP_KERNEL); brcmf_dbg(CONN, "Report port authorized\n"); } + clear_bit(BRCMF_VIF_STATUS_CONNECTING, &ifp->vif->sme_state); set_bit(BRCMF_VIF_STATUS_CONNECTED, &ifp->vif->sme_state); brcmf_dbg(TRACE, "Exit\n"); return err; @@ -6925,8 +6894,6 @@ static s32 brcmf_dongle_roam(struct brcmf_if *ifp) if (err) bphy_err(drvr, "WLC_SET_ROAM_TRIGGER error (%d)\n", err); - roam_delta[0] = cpu_to_le32(WL_ROAM_DELTA); - roam_delta[1] = cpu_to_le32(BRCM_BAND_ALL); err = brcmf_fil_cmd_data_set(ifp, BRCMF_C_SET_ROAM_DELTA, (void *)roam_delta, sizeof(roam_delta)); if (err) @@ -7019,15 +6986,34 @@ static int brcmf_construct_chaninfo(struct brcmf_cfg80211_info *cfg, goto fail_pbuf; } + /* Changing regulatory domain may change power limits upwards. + * To ensure that we correctly set the new band info, copy the original + * info first. + */ band = wiphy->bands[NL80211_BAND_2GHZ]; - if (band) + if (band) { + memcpy(band->channels, &__wl_2ghz_channels, + sizeof(__wl_2ghz_channels)); + band->n_channels = ARRAY_SIZE(__wl_2ghz_channels); for (i = 0; i < band->n_channels; i++) band->channels[i].flags = IEEE80211_CHAN_DISABLED; + } band = wiphy->bands[NL80211_BAND_5GHZ]; - if (band) + if (band) { + memcpy(band->channels, &__wl_5ghz_channels, + sizeof(__wl_5ghz_channels)); + band->n_channels = ARRAY_SIZE(__wl_5ghz_channels); for (i = 0; i < band->n_channels; i++) band->channels[i].flags = IEEE80211_CHAN_DISABLED; - + } + band = wiphy->bands[NL80211_BAND_6GHZ]; + if (band) { + memcpy(band->channels, &__wl_6ghz_channels, + sizeof(__wl_6ghz_channels)); + band->n_channels = ARRAY_SIZE(__wl_6ghz_channels); + for (i = 0; i < band->n_channels; i++) + band->channels[i].flags = IEEE80211_CHAN_DISABLED; + } total = le32_to_cpu(list->count); if (total > BRCMF_MAX_CHANSPEC_LIST) { bphy_err(drvr, "Invalid count of channel Spec. (%u)\n", @@ -7044,6 +7030,8 @@ static int brcmf_construct_chaninfo(struct brcmf_cfg80211_info *cfg, band = wiphy->bands[NL80211_BAND_2GHZ]; } else if (ch.band == BRCMU_CHAN_BAND_5G) { band = wiphy->bands[NL80211_BAND_5GHZ]; + } else if (ch.band == BRCMU_CHAN_BAND_6G) { + band = wiphy->bands[NL80211_BAND_6GHZ]; } else { bphy_err(drvr, "Invalid channel Spec. 0x%x.\n", ch.chspec); @@ -7065,6 +7053,7 @@ static int brcmf_construct_chaninfo(struct brcmf_cfg80211_info *cfg, break; } } + if (!channel) { /* It seems firmware supports some channel we never * considered. Something new in IEEE standard? @@ -7137,17 +7126,25 @@ static int brcmf_enable_bw40_2g(struct brcmf_cfg80211_info *cfg) struct brcmu_chan ch; u32 num_chan; int i, j; + s32 updown; /* verify support for bw_cap command */ - val = WLC_BAND_5G; + val = WLC_BAND_2G; err = brcmf_fil_iovar_int_query(ifp, "bw_cap", &val); - + brcmf_dbg(INFO, "Check bw_cap support:%d\n", err); if (!err) { + /* Setting the bw_cap is DOWN restricted. */ + updown = 0; + brcmf_fil_cmd_data_set(ifp, BRCMF_C_DOWN, &updown, sizeof(s32)); /* only set 2G bandwidth using bw_cap command */ band_bwcap.band = cpu_to_le32(WLC_BAND_2G); band_bwcap.bw_cap = cpu_to_le32(WLC_BW_CAP_40MHZ); err = brcmf_fil_iovar_data_set(ifp, "bw_cap", &band_bwcap, sizeof(band_bwcap)); + brcmf_dbg(INFO, "set bw_cap support:%d\n", err); + brcmf_c_set_joinpref_default(ifp); + updown = 1; + brcmf_fil_cmd_data_set(ifp, BRCMF_C_UP, &updown, sizeof(s32)); } else { brcmf_dbg(INFO, "fallback to mimo_bw_cap\n"); val = WLC_N_BW_40ALL; @@ -7209,7 +7206,7 @@ static int brcmf_enable_bw40_2g(struct brcmf_cfg80211_info *cfg) return err; } -static void brcmf_get_bwcap(struct brcmf_if *ifp, u32 bw_cap[]) +static void brcmf_get_bwcap(struct brcmf_if *ifp, u32 bw_cap[4], bool has_6g) { struct brcmf_pub *drvr = ifp->drvr; u32 band, mimo_bwcap; @@ -7217,17 +7214,29 @@ static void brcmf_get_bwcap(struct brcmf_if *ifp, u32 bw_cap[]) band = WLC_BAND_2G; err = brcmf_fil_iovar_int_query(ifp, "bw_cap", &band); - if (!err) { - bw_cap[NL80211_BAND_2GHZ] = band; - band = WLC_BAND_5G; - err = brcmf_fil_iovar_int_query(ifp, "bw_cap", &band); - if (!err) { - bw_cap[NL80211_BAND_5GHZ] = band; - return; - } - WARN_ON(1); + if (err) + goto fallback; + bw_cap[NL80211_BAND_2GHZ] = band; + band = WLC_BAND_5G; + err |= brcmf_fil_iovar_int_query(ifp, "bw_cap", &band); + if (err) + goto fallback; + bw_cap[NL80211_BAND_5GHZ] = band; + if (!has_6g) return; - } + band = WLC_BAND_6G; + err |= brcmf_fil_iovar_int_query(ifp, "bw_cap", &band); + /* Prior to the introduction of 6g, this function only + * did fallback in the case of 2g and 5g -failing. + * As mimo_bwcap does not have 6g bwcap info anyway, + * we keep that behavior. + */ + if (err) + return; + bw_cap[NL80211_BAND_6GHZ] = band; + return; +fallback: + brcmf_dbg(INFO, "fallback to mimo_bw_cap info\n"); err = brcmf_fil_iovar_int_get(ifp, "mimo_bw_cap", &mimo_bwcap); if (err) @@ -7251,8 +7260,11 @@ static void brcmf_get_bwcap(struct brcmf_if *ifp, u32 bw_cap[]) } static void brcmf_update_ht_cap(struct ieee80211_supported_band *band, - u32 bw_cap[2], u32 nchain) + u32 bw_cap[4], u32 nrxchain) { + /* Not supported in 6G band */ + if (band->band == NL80211_BAND_6GHZ) + return; band->ht_cap.ht_supported = true; if (bw_cap[band->band] & WLC_BW_40MHZ_BIT) { band->ht_cap.cap |= IEEE80211_HT_CAP_SGI_40; @@ -7262,32 +7274,49 @@ static void brcmf_update_ht_cap(struct ieee80211_supported_band *band, band->ht_cap.cap |= IEEE80211_HT_CAP_DSSSCCK40; band->ht_cap.ampdu_factor = IEEE80211_HT_MAX_AMPDU_64K; band->ht_cap.ampdu_density = IEEE80211_HT_MPDU_DENSITY_16; - memset(band->ht_cap.mcs.rx_mask, 0xff, nchain); + memset(band->ht_cap.mcs.rx_mask, 0xff, nrxchain); band->ht_cap.mcs.tx_params = IEEE80211_HT_MCS_TX_DEFINED; } -static __le16 brcmf_get_mcs_map(u32 nchain, enum ieee80211_vht_mcs_support supp) +static __le16 brcmf_get_mcs_map(u32 nstreams, + enum ieee80211_vht_mcs_support supp) { u16 mcs_map; int i; - for (i = 0, mcs_map = 0xFFFF; i < nchain; i++) + for (i = 0, mcs_map = 0xFFFF; i < nstreams; i++) mcs_map = (mcs_map << 2) | supp; return cpu_to_le16(mcs_map); } static void brcmf_update_vht_cap(struct ieee80211_supported_band *band, - u32 bw_cap[2], u32 nchain, u32 txstreams, - u32 txbf_bfe_cap, u32 txbf_bfr_cap) + u32 bw_cap[4], u32 txstreams, u32 rxstreams, + u32 txbf_bfe_cap, u32 txbf_bfr_cap, + u32 ldpc_cap, u32 stbc_rx, u32 stbc_tx) { __le16 mcs_map; - /* not allowed in 2.4G band */ - if (band->band == NL80211_BAND_2GHZ) + /* not allowed in 2.4G or 6G band */ + if (band->band == NL80211_BAND_2GHZ || band->band == NL80211_BAND_6GHZ) return; band->vht_cap.vht_supported = true; + band->vht_cap.vht_mcs.tx_highest = cpu_to_le16(433 * txstreams); + band->vht_cap.vht_mcs.rx_highest = cpu_to_le16(433 * rxstreams); + + band->vht_cap.cap |= IEEE80211_VHT_CAP_RX_ANTENNA_PATTERN | + IEEE80211_VHT_CAP_TX_ANTENNA_PATTERN; + + if (ldpc_cap) + band->vht_cap.cap |= IEEE80211_VHT_CAP_RXLDPC; + if (stbc_tx) + band->vht_cap.cap |= IEEE80211_VHT_CAP_TXSTBC; + + if (stbc_rx) + band->vht_cap.cap |= + (stbc_rx << IEEE80211_VHT_CAP_RXSTBC_SHIFT); + /* 80MHz is mandatory */ band->vht_cap.cap |= IEEE80211_VHT_CAP_SHORT_GI_80; if (bw_cap[band->band] & WLC_BW_160MHZ_BIT) { @@ -7295,8 +7324,10 @@ static void brcmf_update_vht_cap(struct ieee80211_supported_band *band, band->vht_cap.cap |= IEEE80211_VHT_CAP_SHORT_GI_160; } /* all support 256-QAM */ - mcs_map = brcmf_get_mcs_map(nchain, IEEE80211_VHT_MCS_SUPPORT_0_9); + mcs_map = brcmf_get_mcs_map(rxstreams, IEEE80211_VHT_MCS_SUPPORT_0_9); band->vht_cap.vht_mcs.rx_mcs_map = mcs_map; + mcs_map = brcmf_get_mcs_map(txstreams, IEEE80211_VHT_MCS_SUPPORT_0_9); + band->vht_cap.vht_mcs.tx_mcs_map = mcs_map; /* Beamforming support information */ @@ -7312,11 +7343,129 @@ static void brcmf_update_vht_cap(struct ieee80211_supported_band *band, if ((txbf_bfe_cap || txbf_bfr_cap) && (txstreams > 1)) { band->vht_cap.cap |= (2 << IEEE80211_VHT_CAP_BEAMFORMEE_STS_SHIFT); - band->vht_cap.cap |= ((txstreams - 1) << - IEEE80211_VHT_CAP_SOUNDING_DIMENSIONS_SHIFT); + band->vht_cap.cap |= + ((txstreams - 1) + << IEEE80211_VHT_CAP_SOUNDING_DIMENSIONS_SHIFT); band->vht_cap.cap |= IEEE80211_VHT_CAP_VHT_LINK_ADAPTATION_VHT_MRQ_MFB; } + /* AMPDU length limit, support max 1MB (2 ^ (13 + 7)) */ + band->vht_cap.cap |= + (7 << IEEE80211_VHT_CAP_MAX_A_MPDU_LENGTH_EXPONENT_SHIFT); +} + +static void brcmf_update_he_cap(struct ieee80211_supported_band *band, + struct ieee80211_sband_iftype_data *data) +{ + int idx = 1; + struct ieee80211_sta_he_cap *he_cap = &data->he_cap; + struct ieee80211_he_cap_elem *he_cap_elem = &he_cap->he_cap_elem; + struct ieee80211_he_mcs_nss_supp *he_mcs = &he_cap->he_mcs_nss_supp; + struct ieee80211_he_6ghz_capa *he_6ghz_capa = &data->he_6ghz_capa; + + if (!data) { + brcmf_err("failed to allocate sdata\n"); + return; + } + + data->types_mask = BIT(NL80211_IFTYPE_STATION); + he_cap->has_he = true; + + /* HE MAC Capabilities Information */ + he_cap_elem->mac_cap_info[0] = IEEE80211_HE_MAC_CAP0_HTC_HE | + IEEE80211_HE_MAC_CAP0_TWT_REQ | + IEEE80211_HE_MAC_CAP0_TWT_RES; + + he_cap_elem->mac_cap_info[1] = + IEEE80211_HE_MAC_CAP1_TF_MAC_PAD_DUR_8US | + IEEE80211_HE_MAC_CAP1_TF_MAC_PAD_DUR_16US; + + he_cap_elem->mac_cap_info[2] = IEEE80211_HE_MAC_CAP2_BSR | + IEEE80211_HE_MAC_CAP2_BCAST_TWT; + + he_cap_elem->mac_cap_info[3] = + IEEE80211_HE_MAC_CAP3_OMI_CONTROL | + IEEE80211_HE_MAC_CAP3_MAX_AMPDU_LEN_EXP_EXT_1 | + IEEE80211_HE_MAC_CAP3_FLEX_TWT_SCHED; + + he_cap_elem->mac_cap_info[4] = IEEE80211_HE_MAC_CAP4_AMSDU_IN_AMPDU; + + /* HE PHY Capabilities Information */ + he_cap_elem->phy_cap_info[0] = + IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_40MHZ_IN_2G | + IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_40MHZ_80MHZ_IN_5G | + IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_160MHZ_IN_5G; + ; + + he_cap_elem->phy_cap_info[1] = + IEEE80211_HE_PHY_CAP1_LDPC_CODING_IN_PAYLOAD; + + he_cap_elem->phy_cap_info[2] = + IEEE80211_HE_PHY_CAP2_NDP_4x_LTF_AND_3_2US | + IEEE80211_HE_PHY_CAP2_UL_MU_FULL_MU_MIMO | + IEEE80211_HE_PHY_CAP2_UL_MU_PARTIAL_MU_MIMO; + + he_cap_elem->phy_cap_info[3] = + IEEE80211_HE_PHY_CAP3_DCM_MAX_CONST_TX_QPSK | + IEEE80211_HE_PHY_CAP3_DCM_MAX_TX_NSS_2 | + IEEE80211_HE_PHY_CAP3_DCM_MAX_CONST_RX_16_QAM | + IEEE80211_HE_PHY_CAP3_SU_BEAMFORMER; + + he_cap_elem->phy_cap_info[4] = + IEEE80211_HE_PHY_CAP4_SU_BEAMFORMEE | + IEEE80211_HE_PHY_CAP4_BEAMFORMEE_MAX_STS_UNDER_80MHZ_MASK | + IEEE80211_HE_PHY_CAP4_BEAMFORMEE_MAX_STS_ABOVE_80MHZ_4 | + IEEE80211_HE_PHY_CAP4_BEAMFORMEE_MAX_STS_UNDER_80MHZ_8; + + he_cap_elem->phy_cap_info[5] = + IEEE80211_HE_PHY_CAP5_NG16_SU_FEEDBACK | + IEEE80211_HE_PHY_CAP5_NG16_MU_FEEDBACK | + IEEE80211_HE_PHY_CAP5_BEAMFORMEE_NUM_SND_DIM_UNDER_80MHZ_2; + + he_cap_elem->phy_cap_info[6] = + IEEE80211_HE_PHY_CAP6_CODEBOOK_SIZE_42_SU | + IEEE80211_HE_PHY_CAP6_CODEBOOK_SIZE_75_MU | + IEEE80211_HE_PHY_CAP6_TRIG_SU_BEAMFORMING_FB | + IEEE80211_HE_PHY_CAP6_TRIG_MU_BEAMFORMING_PARTIAL_BW_FB | + IEEE80211_HE_PHY_CAP6_TRIG_CQI_FB | + IEEE80211_HE_PHY_CAP6_PARTIAL_BW_EXT_RANGE | + IEEE80211_HE_PHY_CAP6_PPE_THRESHOLD_PRESENT; + + he_cap_elem->phy_cap_info[7] = + IEEE80211_HE_PHY_CAP7_HE_SU_MU_PPDU_4XLTF_AND_08_US_GI | + IEEE80211_HE_PHY_CAP7_MAX_NC_1; + + he_cap_elem->phy_cap_info[8] = + IEEE80211_HE_PHY_CAP8_HE_ER_SU_PPDU_4XLTF_AND_08_US_GI | + IEEE80211_HE_PHY_CAP8_20MHZ_IN_40MHZ_HE_PPDU_IN_2G | + IEEE80211_HE_PHY_CAP8_20MHZ_IN_160MHZ_HE_PPDU | + IEEE80211_HE_PHY_CAP8_80MHZ_IN_160MHZ_HE_PPDU; + + he_cap_elem->phy_cap_info[9] = + IEEE80211_HE_PHY_CAP9_TX_1024_QAM_LESS_THAN_242_TONE_RU | + IEEE80211_HE_PHY_CAP9_RX_1024_QAM_LESS_THAN_242_TONE_RU | + IEEE80211_HE_PHY_CAP9_RX_FULL_BW_SU_USING_MU_WITH_COMP_SIGB | + IEEE80211_HE_PHY_CAP9_RX_FULL_BW_SU_USING_MU_WITH_NON_COMP_SIGB; + + /* HE Supported MCS and NSS Set */ + he_mcs->rx_mcs_80 = cpu_to_le16(0xfffa); + he_mcs->tx_mcs_80 = cpu_to_le16(0xfffa); + he_mcs->rx_mcs_160 = cpu_to_le16(0xfffa); + he_mcs->tx_mcs_160 = cpu_to_le16(0xfffa); + /* HE 6 GHz band capabilities */ + if (band->band == NL80211_BAND_6GHZ) { + u16 capa = 0; + + capa = FIELD_PREP(IEEE80211_HE_6GHZ_CAP_MIN_MPDU_START, + IEEE80211_HT_MPDU_DENSITY_8) | + FIELD_PREP(IEEE80211_HE_6GHZ_CAP_MAX_AMPDU_LEN_EXP, + IEEE80211_VHT_MAX_AMPDU_1024K) | + FIELD_PREP(IEEE80211_HE_6GHZ_CAP_MAX_MPDU_LEN, + IEEE80211_VHT_CAP_MAX_MPDU_LENGTH_11454); + he_6ghz_capa->capa = cpu_to_le16(capa); + } + band->n_iftype_data = idx; + band->iftype_data = data; } static int brcmf_setup_wiphybands(struct brcmf_cfg80211_info *cfg) @@ -7326,26 +7475,49 @@ static int brcmf_setup_wiphybands(struct brcmf_cfg80211_info *cfg) struct wiphy *wiphy = cfg_to_wiphy(cfg); u32 nmode; u32 vhtmode = 0; - u32 bw_cap[2] = { WLC_BW_20MHZ_BIT, WLC_BW_20MHZ_BIT }; + /* 2GHZ, 5GHZ, 60GHZ, 6GHZ */ + u32 bw_cap[4] = { 0, 0, 0, 0 }; u32 rxchain; - u32 nchain; + u32 txchain; + u32 nrxchain; + u32 ntxchain; int err; s32 i; struct ieee80211_supported_band *band; u32 txstreams = 0; + u32 rxstreams = 0; u32 txbf_bfe_cap = 0; u32 txbf_bfr_cap = 0; + u8 he_enable; + struct brcmf_he_defcap he_cap; + u32 ldpc_cap = 0; + u32 stbc_rx = 0; + u32 stbc_tx = 0; (void)brcmf_fil_iovar_int_get(ifp, "vhtmode", &vhtmode); + (void)brcmf_fil_iovar_int_get(ifp, "ldpc_cap", &ldpc_cap); + (void)brcmf_fil_iovar_int_get(ifp, "stbc_rx", &stbc_rx); + (void)brcmf_fil_iovar_int_get(ifp, "stbc_tx", &stbc_tx); + err = brcmf_fil_xtlv_int8_get(ifp, "he", BRCMF_HE_CMD_ENABLE, + &he_enable); + if (!err && he_enable) { + brcmf_fil_xtlv_data_get(ifp, "he", BRCMF_HE_CMD_DEFCAP, &he_cap, + sizeof(he_cap)); + brcmf_dbg_hex_dump(BRCMF_INFO_ON(), he_cap.mac_cap, 6, + "default HE mac cap\n"); + brcmf_dbg_hex_dump(BRCMF_INFO_ON(), he_cap.phy_cap, 11, + "default HE phy cap\n"); + } err = brcmf_fil_iovar_int_get(ifp, "nmode", &nmode); if (err) { bphy_err(drvr, "nmode error (%d)\n", err); - } else { - brcmf_get_bwcap(ifp, bw_cap); } - brcmf_dbg(INFO, "nmode=%d, vhtmode=%d, bw_cap=(%d, %d)\n", + brcmf_get_bwcap(ifp, bw_cap, he_enable != 0); + brcmf_dbg(INFO, + "nmode=%d, vhtmode=%d, bw_cap=(%d, %d, %d), he_enable=%d\n", nmode, vhtmode, bw_cap[NL80211_BAND_2GHZ], - bw_cap[NL80211_BAND_5GHZ]); + bw_cap[NL80211_BAND_5GHZ], bw_cap[NL80211_BAND_6GHZ], + he_enable); err = brcmf_fil_iovar_int_get(ifp, "rxchain", &rxchain); if (err) { @@ -7355,12 +7527,31 @@ static int brcmf_setup_wiphybands(struct brcmf_cfg80211_info *cfg) else bphy_err(drvr, "rxchain error (%d)\n", err); - nchain = 1; + nrxchain = 1; + rxchain = 1; } else { - for (nchain = 0; rxchain; nchain++) + for (nrxchain = 0; rxchain; nrxchain++) rxchain = rxchain & (rxchain - 1); } - brcmf_dbg(INFO, "nchain=%d\n", nchain); + brcmf_dbg(INFO, "nrxchain=%d\n", nrxchain); + err = brcmf_fil_iovar_int_get(ifp, "txchain", &txchain); + if (err) { + /* rxchain unsupported by firmware of older chips */ + if (err == -EBADE) + bphy_info_once(drvr, "rxchain unsupported\n"); + else + bphy_err(drvr, "rxchain error (%d)\n", err); + + ntxchain = 1; + txchain = 1; + } else { + for (ntxchain = 0; txchain; ntxchain++) + txchain = txchain & (txchain - 1); + } + brcmf_dbg(INFO, "ntxchain=%d\n", ntxchain); + + wiphy->available_antennas_rx = nrxchain; + wiphy->available_antennas_tx = ntxchain; err = brcmf_construct_chaninfo(cfg, bw_cap); if (err) { @@ -7369,6 +7560,7 @@ static int brcmf_setup_wiphybands(struct brcmf_cfg80211_info *cfg) } if (vhtmode) { + (void)brcmf_fil_iovar_int_get(ifp, "rxstreams", &rxstreams); (void)brcmf_fil_iovar_int_get(ifp, "txstreams", &txstreams); (void)brcmf_fil_iovar_int_get(ifp, "txbf_bfe_cap", &txbf_bfe_cap); @@ -7382,10 +7574,13 @@ static int brcmf_setup_wiphybands(struct brcmf_cfg80211_info *cfg) continue; if (nmode) - brcmf_update_ht_cap(band, bw_cap, nchain); + brcmf_update_ht_cap(band, bw_cap, nrxchain); if (vhtmode) - brcmf_update_vht_cap(band, bw_cap, nchain, txstreams, - txbf_bfe_cap, txbf_bfr_cap); + brcmf_update_vht_cap(band, bw_cap, txstreams, rxstreams, + txbf_bfe_cap, txbf_bfr_cap, + ldpc_cap, stbc_rx, stbc_tx); + if (he_enable) + brcmf_update_he_cap(band, &sdata[band->band]); } return 0; @@ -7640,7 +7835,7 @@ static int brcmf_setup_wiphy(struct wiphy *wiphy, struct brcmf_if *ifp) struct ieee80211_supported_band *band; u16 max_interfaces = 0; bool gscan; - __le32 bandlist[3]; + __le32 bandlist[16]; u32 n_bands; int err, i; @@ -7708,6 +7903,13 @@ static int brcmf_setup_wiphy(struct wiphy *wiphy, struct brcmf_if *ifp) } if (brcmf_feat_is_enabled(ifp, BRCMF_FEAT_SAE_EXT)) wiphy->features |= NL80211_FEATURE_SAE; + + /* High accuracy and low power scans are always supported. */ + wiphy_ext_feature_set(wiphy, NL80211_EXT_FEATURE_HIGH_ACCURACY_SCAN); + wiphy_ext_feature_set(wiphy, NL80211_EXT_FEATURE_LOW_POWER_SCAN); + wiphy_ext_feature_set(wiphy, NL80211_EXT_FEATURE_LOW_SPAN_SCAN); + wiphy->features |= NL80211_FEATURE_LOW_PRIORITY_SCAN; + wiphy->mgmt_stypes = brcmf_txrx_stypes; wiphy->max_remain_on_channel_duration = 5000; if (brcmf_feat_is_enabled(ifp, BRCMF_FEAT_PNO)) { @@ -7763,12 +7965,27 @@ static int brcmf_setup_wiphy(struct wiphy *wiphy, struct brcmf_if *ifp) band->n_channels = ARRAY_SIZE(__wl_5ghz_channels); wiphy->bands[NL80211_BAND_5GHZ] = band; } - } + if (bandlist[i] == cpu_to_le32(WLC_BAND_6G)) { + band = kmemdup(&__wl_band_6ghz, sizeof(__wl_band_6ghz), + GFP_KERNEL); + if (!band) + return -ENOMEM; + + band->channels = kmemdup(&__wl_6ghz_channels, + sizeof(__wl_6ghz_channels), + GFP_KERNEL); + if (!band->channels) { + kfree(band); + return -ENOMEM; + } + band->n_channels = ARRAY_SIZE(__wl_6ghz_channels); + wiphy->bands[NL80211_BAND_6GHZ] = band; + } + } if (wiphy->bands[NL80211_BAND_5GHZ] && brcmf_feat_is_enabled(ifp, BRCMF_FEAT_DOT11H)) - wiphy_ext_feature_set(wiphy, - NL80211_EXT_FEATURE_DFS_OFFLOAD); + wiphy_ext_feature_set(wiphy, NL80211_EXT_FEATURE_DFS_OFFLOAD); wiphy_ext_feature_set(wiphy, NL80211_EXT_FEATURE_CQM_RSSI_LIST); @@ -8267,9 +8484,17 @@ static void brcmf_cfg80211_reg_notifier(struct wiphy *wiphy, } err = brcmf_translate_country_code(ifp->drvr, req->alpha2, &ccreq); - if (err) - return; - + if (err) { + /* Because we ignore the default country code above, + * we will start out in our custom reg domain, but the chip + * may already be set to the right country. + * As such, we force the bands to be re-set the first + * time we try to set a country for real. + */ + if (err != -EAGAIN || !cfg->force_band_setup) + return; + } + cfg->force_band_setup = false; err = brcmf_fil_iovar_data_set(ifp, "country", &ccreq, sizeof(ccreq)); if (err) { bphy_err(drvr, "Firmware rejected country setting\n"); @@ -8298,6 +8523,10 @@ static void brcmf_free_wiphy(struct wiphy *wiphy) kfree(wiphy->bands[NL80211_BAND_5GHZ]->channels); kfree(wiphy->bands[NL80211_BAND_5GHZ]); } + if (wiphy->bands[NL80211_BAND_6GHZ]) { + kfree(wiphy->bands[NL80211_BAND_6GHZ]->channels); + kfree(wiphy->bands[NL80211_BAND_6GHZ]); + } #if IS_ENABLED(CONFIG_PM) if (wiphy->wowlan != &brcmf_wowlan_support) kfree(wiphy->wowlan); @@ -8332,6 +8561,7 @@ struct brcmf_cfg80211_info *brcmf_cfg80211_attach(struct brcmf_pub *drvr, cfg->pub = drvr; init_vif_event(&cfg->vif_event); INIT_LIST_HEAD(&cfg->vif_list); + cfg->force_band_setup = true; vif = brcmf_alloc_vif(cfg, NL80211_IFTYPE_STATION); if (IS_ERR(vif)) @@ -8389,18 +8619,21 @@ struct brcmf_cfg80211_info *brcmf_cfg80211_attach(struct brcmf_pub *drvr, if (brcmf_feat_is_enabled(ifp, BRCMF_FEAT_DUMP_OBSS)) ops->dump_survey = brcmf_cfg80211_dump_survey; - err = wiphy_register(wiphy); - if (err < 0) { - bphy_err(drvr, "Could not register wiphy device (%d)\n", err); - goto priv_out; - } - + /* We have to configure the bands before we register the wiphy device + * because it requires that band capabilities be correct. + */ err = brcmf_setup_wiphybands(cfg); if (err) { bphy_err(drvr, "Setting wiphy bands failed (%d)\n", err); goto wiphy_unreg_out; } + err = wiphy_register(wiphy); + if (err < 0) { + bphy_err(drvr, "Could not register wiphy device (%d)\n", err); + goto priv_out; + } + /* If cfg80211 didn't disable 40MHz HT CAP in wiphy_register(), * setup 40MHz in 2GHz band and enable OBSS scanning. */ diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h index 273c80f2d483a3..11f651ab9f4228 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h @@ -8,6 +8,7 @@ /* for brcmu_d11inf */ #include +#include #include "core.h" #include "fwil_types.h" @@ -127,7 +128,8 @@ enum brcmf_profile_fwsup { BRCMF_PROFILE_FWSUP_NONE, BRCMF_PROFILE_FWSUP_PSK, BRCMF_PROFILE_FWSUP_1X, - BRCMF_PROFILE_FWSUP_SAE + BRCMF_PROFILE_FWSUP_SAE, + BRCMF_PROFILE_FWSUP_ROAM }; /** @@ -172,6 +174,7 @@ struct brcmf_cfg80211_profile { enum brcmf_profile_fwsup use_fwsup; u16 use_fwauth; bool is_ft; + bool is_okc; }; /** @@ -350,6 +353,7 @@ struct brcmf_cfg80211_wowl { * @dongle_up: indicate whether dongle up or not. * @roam_on: on/off switch for dongle self-roaming. * @scan_tried: indicates if first scan attempted. + * @force_band_setup: indicates if we should force band setup * @dcmd_buf: dcmd buffer. * @extra_buf: mainly to grab assoc information. * @debugfsdir: debugfs folder for this device. @@ -380,6 +384,7 @@ struct brcmf_cfg80211_info { bool pwr_save; bool dongle_up; bool scan_tried; + bool force_band_setup; u8 *dcmd_buf; u8 *extra_buf; struct dentry *debugfsdir; @@ -409,6 +414,22 @@ struct brcmf_tlv { u8 data[]; }; +static inline enum nl80211_band fwil_band_to_nl80211(u16 band) +{ + switch (band) { + case WLC_BAND_2G: + return NL80211_BAND_2GHZ; + case WLC_BAND_5G: + return NL80211_BAND_5GHZ; + case WLC_BAND_6G: + return NL80211_BAND_6GHZ; + default: + WARN_ON(1); + break; + } + return 0; +} + static inline struct wiphy *cfg_to_wiphy(struct brcmf_cfg80211_info *cfg) { return cfg->wiphy; @@ -476,6 +497,8 @@ s32 brcmf_vif_set_mgmt_ie(struct brcmf_cfg80211_vif *vif, s32 pktflag, s32 brcmf_vif_clear_mgmt_ies(struct brcmf_cfg80211_vif *vif); u16 channel_to_chanspec(struct brcmu_d11inf *d11inf, struct ieee80211_channel *ch); +u16 chandef_to_chanspec(struct brcmu_d11inf *d11inf, + struct cfg80211_chan_def *ch); bool brcmf_get_vif_state_any(struct brcmf_cfg80211_info *cfg, unsigned long state); void brcmf_cfg80211_arm_vif_event(struct brcmf_cfg80211_info *cfg, diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c index 4239f2b21e5423..75b6bfa28b6c29 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c @@ -162,6 +162,15 @@ struct sbconfig { #define SRCI_SRBSZ_SHIFT 0 #define SR_BSZ_BASE 14 +#define SYSMEM_SRCI_ROMNB_MASK 0x3e0 +#define SYSMEM_SRCI_ROMNB_SHIFT 5 +#define SYSMEM_SRCI_SRNB_MASK 0x1f +#define SYSMEM_SRCI_SRNB_SHIFT 0 +#define SYSMEM_SRCI_NEW_ROMNB_MASK 0xff000000 +#define SYSMEM_SRCI_NEW_ROMNB_SHIFT 24 +#define SYSMEM_SRCI_NEW_SRNB_MASK 0xff0000 +#define SYSMEM_SRCI_NEW_SRNB_SHIFT 16 + struct sbsocramregs { u32 coreinfo; u32 bwalloc; @@ -436,25 +445,11 @@ static void brcmf_chip_ai_resetcore(struct brcmf_core_priv *core, u32 prereset, { struct brcmf_chip_priv *ci; int count; - struct brcmf_core *d11core2 = NULL; - struct brcmf_core_priv *d11priv2 = NULL; ci = core->chip; - /* special handle two D11 cores reset */ - if (core->pub.id == BCMA_CORE_80211) { - d11core2 = brcmf_chip_get_d11core(&ci->pub, 1); - if (d11core2) { - brcmf_dbg(INFO, "found two d11 cores, reset both\n"); - d11priv2 = container_of(d11core2, - struct brcmf_core_priv, pub); - } - } - /* must disable first to work for arbitrary current core state */ brcmf_chip_ai_coredisable(core, prereset, reset); - if (d11priv2) - brcmf_chip_ai_coredisable(d11priv2, prereset, reset); count = 0; while (ci->ops->read32(ci->ctx, core->wrapbase + BCMA_RESET_CTL) & @@ -466,30 +461,9 @@ static void brcmf_chip_ai_resetcore(struct brcmf_core_priv *core, u32 prereset, usleep_range(40, 60); } - if (d11priv2) { - count = 0; - while (ci->ops->read32(ci->ctx, - d11priv2->wrapbase + BCMA_RESET_CTL) & - BCMA_RESET_CTL_RESET) { - ci->ops->write32(ci->ctx, - d11priv2->wrapbase + BCMA_RESET_CTL, - 0); - count++; - if (count > 50) - break; - usleep_range(40, 60); - } - } - ci->ops->write32(ci->ctx, core->wrapbase + BCMA_IOCTL, postreset | BCMA_IOCTL_CLK); ci->ops->read32(ci->ctx, core->wrapbase + BCMA_IOCTL); - - if (d11priv2) { - ci->ops->write32(ci->ctx, d11priv2->wrapbase + BCMA_IOCTL, - postreset | BCMA_IOCTL_CLK); - ci->ops->read32(ci->ctx, d11priv2->wrapbase + BCMA_IOCTL); - } } char *brcmf_chip_name(u32 id, u32 rev, char *buf, uint len) @@ -659,6 +633,7 @@ static u32 brcmf_chip_sysmem_ramsize(struct brcmf_core_priv *sysmem) u32 memsize = 0; u32 coreinfo; u32 idx; + u32 nrb; u32 nb; u32 banksize; @@ -666,10 +641,16 @@ static u32 brcmf_chip_sysmem_ramsize(struct brcmf_core_priv *sysmem) brcmf_chip_resetcore(&sysmem->pub, 0, 0, 0); coreinfo = brcmf_chip_core_read32(sysmem, SYSMEMREGOFFS(coreinfo)); - nb = (coreinfo & SRCI_SRNB_MASK) >> SRCI_SRNB_SHIFT; + if (sysmem->pub.rev >= 12) { + nrb = (coreinfo & SYSMEM_SRCI_NEW_ROMNB_MASK) >> SYSMEM_SRCI_NEW_ROMNB_SHIFT; + nb = (coreinfo & SYSMEM_SRCI_NEW_SRNB_MASK) >> SYSMEM_SRCI_NEW_SRNB_SHIFT; + } else { + nrb = (coreinfo & SYSMEM_SRCI_ROMNB_MASK) >> SYSMEM_SRCI_ROMNB_SHIFT; + nb = (coreinfo & SYSMEM_SRCI_SRNB_MASK) >> SYSMEM_SRCI_SRNB_SHIFT; + } for (idx = 0; idx < nb; idx++) { - brcmf_chip_socram_banksize(sysmem, idx, &banksize); + brcmf_chip_socram_banksize(sysmem, idx + nrb, &banksize); memsize += banksize; } @@ -731,6 +712,7 @@ static u32 brcmf_chip_tcm_rambase(struct brcmf_chip_priv *ci) case BRCM_CC_4366_CHIP_ID: case BRCM_CC_43664_CHIP_ID: case BRCM_CC_43666_CHIP_ID: + case BRCM_CC_4388_CHIP_ID: return 0x200000; case BRCM_CC_4355_CHIP_ID: case BRCM_CC_4359_CHIP_ID: @@ -1338,14 +1320,15 @@ static inline void brcmf_chip_ca7_set_passive(struct brcmf_chip_priv *chip) { struct brcmf_core *core; + int i; brcmf_chip_disable_arm(chip, BCMA_CORE_ARM_CA7); - core = brcmf_chip_get_core(&chip->pub, BCMA_CORE_80211); - brcmf_chip_resetcore(core, D11_BCMA_IOCTL_PHYRESET | - D11_BCMA_IOCTL_PHYCLOCKEN, - D11_BCMA_IOCTL_PHYCLOCKEN, - D11_BCMA_IOCTL_PHYCLOCKEN); + /* Disable the cores only and let the firmware enable them. */ + for (i = 0; (core = brcmf_chip_get_d11core(&chip->pub, i)); i++) + brcmf_chip_coredisable(core, D11_BCMA_IOCTL_PHYRESET | + D11_BCMA_IOCTL_PHYCLOCKEN, + D11_BCMA_IOCTL_PHYCLOCKEN); } static bool brcmf_chip_ca7_set_active(struct brcmf_chip_priv *chip, u32 rstvec) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c index 688f16c513192d..b80e0b4ad422ab 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c @@ -13,6 +13,7 @@ #include "core.h" #include "bus.h" #include "debug.h" +#include "fweh.h" #include "fwil.h" #include "fwil_types.h" #include "tracepoint.h" @@ -266,7 +267,6 @@ static int brcmf_c_process_cal_blob(struct brcmf_if *ifp) int brcmf_c_preinit_dcmds(struct brcmf_if *ifp) { struct brcmf_pub *drvr = ifp->drvr; - struct brcmf_fweh_info *fweh = drvr->fweh; u8 buf[BRCMF_DCMD_SMLEN]; struct brcmf_bus *bus; struct brcmf_rev_info_le revinfo; @@ -412,27 +412,6 @@ int brcmf_c_preinit_dcmds(struct brcmf_if *ifp) brcmf_c_set_joinpref_default(ifp); - /* Setup event_msgs, enable E_IF */ - err = brcmf_fil_iovar_data_get(ifp, "event_msgs", fweh->event_mask, - fweh->event_mask_len); - if (err) { - bphy_err(drvr, "Get event_msgs error (%d)\n", err); - goto done; - } - /* - * BRCMF_E_IF can safely be used to set the appropriate bit - * in the event_mask as the firmware event code is guaranteed - * to match the value of BRCMF_E_IF because it is old cruft - * that all vendors have. - */ - setbit(fweh->event_mask, BRCMF_E_IF); - err = brcmf_fil_iovar_data_set(ifp, "event_msgs", fweh->event_mask, - fweh->event_mask_len); - if (err) { - bphy_err(drvr, "Set event_msgs error (%d)\n", err); - goto done; - } - /* Setup default scan channel time */ err = brcmf_fil_cmd_int_set(ifp, BRCMF_C_SET_SCAN_CHANNEL_TIME, BRCMF_DEFAULT_SCAN_CHANNEL_TIME); diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c index 862a0336a0b591..3ef91663863da8 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c @@ -1227,7 +1227,14 @@ static int brcmf_bus_started(struct brcmf_pub *drvr, struct cfg80211_ops *ops) if (ret < 0) goto fail; - brcmf_feat_attach(drvr); + ret = brcmf_feat_attach(drvr); + if (ret) + goto fail; + + /* Setup event_msgs, enable E_IF */ + ret = brcmf_fweh_init_events(ifp); + if (ret) + goto fail; ret = brcmf_proto_init_done(drvr); if (ret < 0) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.h index 399b6810e394de..4b52a3aa855de8 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.h @@ -97,6 +97,68 @@ struct brcmf_rev_info { u32 nvramrev; }; +struct brcmf_pno_info; +enum nl80211_band; +/** + * struct pno_struct_handler + */ +struct pno_struct_handler { + u8 version; + int (*pno_config)(struct brcmf_if *ifp, u32 scan_freq, u32 mscan, + u32 bestn); + u32 (*get_min_data_len)(void); + u32 (*get_result_count)(void *data); + u32 (*get_result_status)(void *data); + int (*validate_pfn_results)(void *data, u32 event_datalen); + u32 (*get_bucket_map)(void *data, int idx, struct brcmf_pno_info *pi); + int (*get_result_info)(void *data, int result_idx, + u8 (*ssid)[IEEE80211_MAX_SSID_LEN], u8 *ssid_len, + u8 *channel, enum nl80211_band *band); +}; + +struct cfg80211_scan_request; +struct scan_param_struct_handler { + u8 version; + void *(*get_struct_for_request)(struct brcmf_cfg80211_info *cfg, + u32 *struct_size, + struct cfg80211_scan_request *request); +}; + +struct cfg80211_ibss_params; +struct cfg80211_connect_params; + +/** + * struct join_param_struct_handler - Handler for different join parameter versions + * + * There are a number of different, incompatible structures and interface versions for join/extended join parameters + * We abstract away the actual structures used, so that code does not have to worry about filling in structs properly. + * + * This interface deliberately takes and returns opaque structures. + * + * @version - Interface version the firmware supports/uses + * @get_struct_for_ibss - Return a join parameter structure for a set of IBSS parameters. + * This structure can be used to join the passed BSS. + * @get_struct_for_connect - Return an extended join parameter structure for a set of connect + * parameters. This structure can be used to join the SSID specified in the parameters. + * @get_join_from_ext_join - When an extended join does not work, we fall back to a regular join. + * This function produces a join parameter struture from an extended join one. + */ +struct join_param_struct_handler { + u8 version; + /* This returns a join_param type struct */ + void *(*get_struct_for_ibss)(struct brcmf_cfg80211_info *cfg, + u32 *struct_size, + struct cfg80211_ibss_params *params); + /* This returns an ext_join_param type struct */ + void *(*get_struct_for_connect)(struct brcmf_cfg80211_info *cfg, + u32 *struct_size, + struct cfg80211_connect_params *params); + /* This returns the join param portion of an ext_join_param type struct. + * The memory returned is separately allocated from the passed-in struct. + */ + void *(*get_join_from_ext_join)(void *ext_join_param, u32 *struct_size); +}; + /* Common structure for module and instance linkage */ struct brcmf_pub { /* Linkage ponters */ @@ -145,6 +207,10 @@ struct brcmf_pub { u8 sta_mac_idx; const struct brcmf_fwvid_ops *vops; void *vdata; + u16 cnt_ver; + struct pno_struct_handler pno_handler; + struct scan_param_struct_handler scan_param_handler; + struct join_param_struct_handler join_param_handler; }; /* forward declarations */ diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/debug.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/debug.h index 9bb5f709d41a27..432d93ae8fb854 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/debug.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/debug.h @@ -85,6 +85,7 @@ do { \ #define BRCMF_FIL_ON() (brcmf_msg_level & BRCMF_FIL_VAL) #define BRCMF_FWCON_ON() (brcmf_msg_level & BRCMF_FWCON_VAL) #define BRCMF_SCAN_ON() (brcmf_msg_level & BRCMF_SCAN_VAL) +#define BRCMF_INFO_ON() (brcmf_msg_level & BRCMF_INFO_VAL) #else /* defined(DEBUG) || defined(CONFIG_BRCM_TRACING) */ @@ -104,6 +105,7 @@ do { \ #define BRCMF_FIL_ON() 0 #define BRCMF_FWCON_ON() 0 #define BRCMF_SCAN_ON() 0 +#define BRCMF_INFO_ON() 0 #endif /* defined(DEBUG) || defined(CONFIG_BRCM_TRACING) */ diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.c index 488364ef8ff2a1..d823ced048454a 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.c @@ -16,9 +16,24 @@ #include "fwvid.h" #include "feature.h" #include "common.h" +#include "pno.h" +#include "scan_param.h" +#include "join_param.h" #define BRCMF_FW_UNSUPPORTED 23 +/* MIN branch version supporting join iovar versioning */ +#define MIN_JOINEXT_V1_FW_MAJOR 17u +/* Branch/es supporting join iovar versioning prior to + * MIN_JOINEXT_V1_FW_MAJOR + */ +#define MIN_JOINEXT_V1_BR2_FW_MAJOR 16 +#define MIN_JOINEXT_V1_BR2_FW_MINOR 1 + +#define MIN_JOINEXT_V1_BR1_FW_MAJOR 14 +#define MIN_JOINEXT_V1_BR1_FW_MINOR_2 2 +#define MIN_JOINEXT_V1_BR1_FW_MINOR_4 4 + /* * expand feature list to array of feature strings. */ @@ -45,6 +60,7 @@ static const struct brcmf_feat_fwcap brcmf_fwcap_map[] = { { BRCMF_FEAT_SAE, "sae " }, { BRCMF_FEAT_FWAUTH, "idauth" }, { BRCMF_FEAT_SAE_EXT, "sae_ext" }, + { BRCMF_FEAT_GCMP, "gcmp"} }; #ifdef DEBUG @@ -136,7 +152,7 @@ struct brcmf_feat_wlcfeat { static const struct brcmf_feat_wlcfeat brcmf_feat_wlcfeat_map[] = { { 12, 0, BIT(BRCMF_FEAT_PMKID_V2) }, - { 13, 0, BIT(BRCMF_FEAT_PMKID_V3) }, + { 13, 0, BIT(BRCMF_FEAT_PMKID_V3) } }; static void brcmf_feat_wlc_version_overrides(struct brcmf_pub *drv) @@ -286,9 +302,12 @@ static int brcmf_feat_fwcap_debugfs_read(struct seq_file *seq, void *data) return 0; } -void brcmf_feat_attach(struct brcmf_pub *drvr) +int brcmf_feat_attach(struct brcmf_pub *drvr) { struct brcmf_if *ifp = brcmf_get_ifp(drvr, 0); + struct brcmf_join_version_le join_ver; + struct brcmf_scan_version_le scan_ver; + struct brcmf_pno_param_v3_le pno_params; struct brcmf_pno_macaddr_le pfn_mac; struct brcmf_gscan_config gscan_cfg; u32 wowl_cap; @@ -331,6 +350,7 @@ void brcmf_feat_attach(struct brcmf_pub *drvr) brcmf_feat_iovar_int_get(ifp, BRCMF_FEAT_TDLS, "tdls_enable"); brcmf_feat_iovar_int_get(ifp, BRCMF_FEAT_MFP, "mfp"); brcmf_feat_iovar_int_get(ifp, BRCMF_FEAT_DUMP_OBSS, "dump_obss"); + brcmf_feat_iovar_int_get(ifp, BRCMF_FEAT_EVENT_MSGS_EXT, "event_msgs_ext"); pfn_mac.version = BRCMF_PFN_MACADDR_CFG_VER; err = brcmf_fil_iovar_data_get(ifp, "pfn_macaddr", &pfn_mac, @@ -339,13 +359,71 @@ void brcmf_feat_attach(struct brcmf_pub *drvr) ifp->drvr->feat_flags |= BIT(BRCMF_FEAT_SCAN_RANDOM_MAC); brcmf_feat_iovar_int_get(ifp, BRCMF_FEAT_FWSUP, "sup_wpa"); - brcmf_feat_iovar_int_get(ifp, BRCMF_FEAT_SCAN_V2, "scan_ver"); + + err = brcmf_fil_iovar_data_get(ifp, "join_ver", &join_ver, sizeof(join_ver)); + if (!err) { + u16 ver = le16_to_cpu(join_ver.join_ver_major); + err = brcmf_join_param_setup_for_version(drvr, ver); + } else { + /* Default to version 0, unless it is one of the firmware branches + * that doesn't have a join_ver iovar but are still version 1 */ + u8 version = 0; + struct brcmf_wlc_version_le ver; + err = brcmf_fil_iovar_data_get(ifp, "wlc_ver", &ver, + sizeof(ver)); + if (!err) { + u16 major = le16_to_cpu(ver.wlc_ver_major); + u16 minor = le16_to_cpu(ver.wlc_ver_minor); + if (((major == MIN_JOINEXT_V1_BR1_FW_MAJOR) && + ((minor == MIN_JOINEXT_V1_BR1_FW_MINOR_2) || + (minor == MIN_JOINEXT_V1_BR1_FW_MINOR_4))) || + ((major == MIN_JOINEXT_V1_BR2_FW_MAJOR) && + (minor >= MIN_JOINEXT_V1_BR2_FW_MINOR)) || + (major >= MIN_JOINEXT_V1_FW_MAJOR)) { + version = 1; + } + } + err = brcmf_join_param_setup_for_version(drvr, version); + } + if (err) { + bphy_err(drvr, "Error setting up join structure handler: %d\n", + err); + return err; + } + err = brcmf_fil_iovar_data_get(ifp, "scan_ver", &scan_ver, + sizeof(scan_ver)); + if (!err) { + u16 ver = le16_to_cpu(scan_ver.scan_ver_major); + err = brcmf_scan_param_setup_for_version(drvr, ver); + } else { + /* Default to version 1. */ + err = brcmf_scan_param_setup_for_version(drvr, 1); + } + if (err) { + bphy_err(drvr, "Error setting up scan structure handler: %d\n", + err); + return err; + } + /* See what version of PFN scan is supported*/ + err = brcmf_fil_iovar_data_get(ifp, "pno_set", &pno_params, + sizeof(pno_params)); + if (!err) { + err = brcmf_pno_setup_for_version( + drvr, le16_to_cpu(pno_params.version)); + } else { + /* Default to version 2, supported by all chips we support. */ + err = brcmf_pno_setup_for_version(drvr, 2); + } + if (err) { + bphy_err(drvr, "Error setting up escan structure handler: %d\n", + err); + return err; + } brcmf_feat_wlc_version_overrides(drvr); brcmf_feat_firmware_overrides(drvr); brcmf_fwvid_feat_attach(ifp); - if (drvr->settings->feature_disable) { brcmf_dbg(INFO, "Features: 0x%02x, disable: 0x%02x\n", ifp->drvr->feat_flags, @@ -365,6 +443,7 @@ void brcmf_feat_attach(struct brcmf_pub *drvr) /* no quirks */ break; } + return 0; } void brcmf_feat_debugfs_create(struct brcmf_pub *drvr) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.h index 31f8695ca41765..be271ca0fca588 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.h @@ -30,8 +30,12 @@ * SAE: simultaneous authentication of equals * FWAUTH: Firmware authenticator * DUMP_OBSS: Firmware has capable to dump obss info to support ACS - * SCAN_V2: Version 2 scan params * SAE_EXT: SAE authentication handled by user-space supplicant + * PMKID_V2: Version 2 PMKID + * PMKID_V3: Version 3 PMKID + * EVENT_MSGS_EXT: Event messages extension + * JOIN_V1: Version 1 join struct + * GCMP: GCMP Cipher suite support */ #define BRCMF_FEAT_LIST \ BRCMF_FEAT_DEF(MBSS) \ @@ -56,10 +60,11 @@ BRCMF_FEAT_DEF(SAE) \ BRCMF_FEAT_DEF(FWAUTH) \ BRCMF_FEAT_DEF(DUMP_OBSS) \ - BRCMF_FEAT_DEF(SCAN_V2) \ BRCMF_FEAT_DEF(PMKID_V2) \ BRCMF_FEAT_DEF(PMKID_V3) \ - BRCMF_FEAT_DEF(SAE_EXT) + BRCMF_FEAT_DEF(SAE_EXT) \ + BRCMF_FEAT_DEF(EVENT_MSGS_EXT) \ + BRCMF_FEAT_DEF(GCMP) /* * Quirks: @@ -97,8 +102,10 @@ enum brcmf_feat_quirk { * brcmf_feat_attach() - determine features and quirks. * * @drvr: driver instance. + * + * Return: 0 in case of success, error code otherwise. */ -void brcmf_feat_attach(struct brcmf_pub *drvr); +int brcmf_feat_attach(struct brcmf_pub *drvr); /** * brcmf_feat_debugfs_create() - create debugfs entries. diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fweh.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fweh.c index c2d98ee6652f3a..b3e766601db475 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fweh.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fweh.c @@ -11,8 +11,10 @@ #include "core.h" #include "debug.h" #include "tracepoint.h" +#include "feature.h" #include "fweh.h" #include "fwil.h" +#include "fwil_types.h" #include "proto.h" #include "bus.h" #include "fwvid.h" @@ -153,6 +155,11 @@ static void brcmf_fweh_handle_if_event(struct brcmf_pub *drvr, bphy_err(drvr, "invalid interface index: %u\n", ifevent->ifidx); return; } + if (ifevent->bsscfgidx >= BRCMF_MAX_IFS) { + bphy_err(drvr, "invalid bsscfg index: %u\n", + ifevent->bsscfgidx); + return; + } ifp = drvr->iflist[ifevent->bsscfgidx]; @@ -425,6 +432,67 @@ void brcmf_fweh_unregister(struct brcmf_pub *drvr, drvr->fweh->evt_handler[evt_handler_idx] = NULL; } +/** + * brcmf_fweh_init_events() - initialize event handling. + * + * @ifp: primary interface object. + */ +int brcmf_fweh_init_events(struct brcmf_if *ifp) +{ + struct brcmf_pub *drvr = ifp->drvr; + struct brcmf_eventmsgs_ext_le *eventmsgs; + size_t size = sizeof(*eventmsgs) + drvr->fweh->event_mask_len; + int err; + + eventmsgs = kzalloc(size, GFP_KERNEL); + if(!eventmsgs) + return -ENOMEM; + + eventmsgs->version = EVENTMSGS_VER; + eventmsgs->command = EVENTMSGS_NONE; + eventmsgs->len = drvr->fweh->event_mask_len; + eventmsgs->maxgetsize = drvr->fweh->event_mask_len; + + if (brcmf_feat_is_enabled(ifp, BRCMF_FEAT_EVENT_MSGS_EXT)) + err = brcmf_fil_iovar_data_get(ifp, "event_msgs_ext", + eventmsgs, size); + else + err = brcmf_fil_iovar_data_get(ifp, "event_msgs", + drvr->fweh->event_mask, + drvr->fweh->event_mask_len); + + if (err) { + bphy_err(drvr, "Get event_msgs error (%d)\n", err); + kfree(eventmsgs); + return err; + } + + brcmf_dbg(EVENT, "Event mask len: driver=%d fw=%d\n", + drvr->fweh->event_mask_len, eventmsgs->len); + + /* want to handle IF event as well */ + brcmf_dbg(EVENT, "enable event IF\n"); + setbit(eventmsgs->mask, BRCMF_E_IF); + + eventmsgs->version = EVENTMSGS_VER; + eventmsgs->command = EVENTMSGS_SET_MASK; + eventmsgs->len = drvr->fweh->event_mask_len; + + if (brcmf_feat_is_enabled(ifp, BRCMF_FEAT_EVENT_MSGS_EXT)) + err = brcmf_fil_iovar_data_set(ifp, "event_msgs_ext", + eventmsgs, size); + else + err = brcmf_fil_iovar_data_set(ifp, "event_msgs", + drvr->fweh->event_mask, + drvr->fweh->event_mask_len); + + if (err) + bphy_err(drvr, "Set event_msgs error (%d)\n", err); + + kfree(eventmsgs); + return err; +} + /** * brcmf_fweh_activate_events() - enables firmware events registered. * @@ -432,32 +500,47 @@ void brcmf_fweh_unregister(struct brcmf_pub *drvr, */ int brcmf_fweh_activate_events(struct brcmf_if *ifp) { - struct brcmf_fweh_info *fweh = ifp->drvr->fweh; - enum brcmf_fweh_event_code code; + struct brcmf_pub *drvr = ifp->drvr; + struct brcmf_eventmsgs_ext_le *eventmsgs; + size_t size = sizeof(*eventmsgs) + drvr->fweh->event_mask_len; int i, err; - memset(fweh->event_mask, 0, fweh->event_mask_len); - for (i = 0; i < fweh->num_event_codes; i++) { - if (fweh->evt_handler[i]) { - brcmf_fweh_map_fwevt_code(fweh, i, &code); + eventmsgs = kzalloc(size, GFP_KERNEL); + if(!eventmsgs) + return -ENOMEM; + + for (i = 0; i < drvr->fweh->num_event_codes; i++) { + if (drvr->fweh->evt_handler[i]) { brcmf_dbg(EVENT, "enable event %s\n", - brcmf_fweh_event_name(code)); - setbit(fweh->event_mask, i); + brcmf_fweh_event_name(i)); + setbit(eventmsgs->mask, i); } } /* want to handle IF event as well */ brcmf_dbg(EVENT, "enable event IF\n"); - setbit(fweh->event_mask, BRCMF_E_IF); + setbit(eventmsgs->mask, BRCMF_E_IF); + + eventmsgs->version = EVENTMSGS_VER; + eventmsgs->command = EVENTMSGS_SET_MASK; + eventmsgs->len = drvr->fweh->event_mask_len; /* allow per-vendor method to activate firmware events */ if (!brcmf_fwvid_activate_events(ifp)) return 0; - err = brcmf_fil_iovar_data_set(ifp, "event_msgs", fweh->event_mask, - fweh->event_mask_len); + if (brcmf_feat_is_enabled(ifp, BRCMF_FEAT_EVENT_MSGS_EXT)) + err = brcmf_fil_iovar_data_set(ifp, "event_msgs_ext", + eventmsgs, size); + else + err = brcmf_fil_iovar_data_set(ifp, "event_msgs", + drvr->fweh->event_mask, + drvr->fweh->event_mask_len); + if (err) - bphy_err(fweh->drvr, "Set event_msgs error (%d)\n", err); + bphy_err(drvr, "Set event_msgs error (%d)\n", err); + + kfree(eventmsgs); return err; } diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fweh.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fweh.h index e327dd58d29c95..53c4b58e6323cc 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fweh.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fweh.h @@ -356,6 +356,7 @@ int brcmf_fweh_register(struct brcmf_pub *drvr, enum brcmf_fweh_event_code code, void *data)); void brcmf_fweh_unregister(struct brcmf_pub *drvr, enum brcmf_fweh_event_code code); +int brcmf_fweh_init_events(struct brcmf_if *ifp); int brcmf_fweh_activate_events(struct brcmf_if *ifp); void brcmf_fweh_process_event(struct brcmf_pub *drvr, struct brcmf_event *event_packet, diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h index e74a23e11830c1..7b8f809cdc412d 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h @@ -18,7 +18,8 @@ #define BRCMF_ARP_OL_HOST_AUTO_REPLY 0x00000004 #define BRCMF_ARP_OL_PEER_AUTO_REPLY 0x00000008 -#define BRCMF_BSS_INFO_VERSION 109 /* curr ver of brcmf_bss_info_le struct */ +#define BRCMF_BSS_INFO_MIN_VERSION 109 /* min ver of brcmf_bss_info_le struct */ +#define BRCMF_BSS_INFO_MAX_VERSION 112 /* max ver of brcmf_bss_info_le struct */ #define BRCMF_BSS_RSSI_ON_CHANNEL 0x0004 #define BRCMF_STA_BRCM 0x00000001 /* Running a Broadcom driver */ @@ -46,12 +47,10 @@ #define BRCMF_STA_DWDS_CAP 0x01000000 /* DWDS CAP */ #define BRCMF_STA_DWDS 0x02000000 /* DWDS active */ -/* size of brcmf_scan_params not including variable length array */ -#define BRCMF_SCAN_PARAMS_FIXED_SIZE 64 -#define BRCMF_SCAN_PARAMS_V2_FIXED_SIZE 72 - /* version of brcmf_scan_params structure */ #define BRCMF_SCAN_PARAMS_VERSION_V2 2 +#define BRCMF_SCAN_PARAMS_VERSION_V3 3 +#define BRCMF_SCAN_PARAMS_VERSION_V4 4 /* masks for channel and ssid count */ #define BRCMF_SCAN_PARAMS_COUNT_MASK 0x0000ffff @@ -62,16 +61,26 @@ #define BRCMF_SCANTYPE_ACTIVE 0 #define BRCMF_SCANTYPE_PASSIVE 1 +/* Additional scanning flags */ +#define BRCMF_SCANFLAGS_LOW_PRIO 0x2 +#define BRCMF_SCANFLAGS_LOW_POWER 0x1000 +#define BRCMF_SCANFLAGS_HIGH_ACCURACY 0x2000 +#define BRCMF_SCANFLAGS_LOW_SPAN 0x4000 + +/* scan ssid_type flags */ +#define BRCMF_SCANSSID_INC_RNR 0x02 /* Include RNR channels*/ + #define BRCMF_WSEC_MAX_PSK_LEN 32 #define BRCMF_WSEC_PASSPHRASE BIT(0) -#define BRCMF_WSEC_MAX_SAE_PASSWORD_LEN 128 +#define BRCMF_WSEC_MAX_SAE_PASSWORD_LEN 256 /* primary (ie tx) key */ #define BRCMF_PRIMARY_KEY (1 << 1) #define DOT11_BSSTYPE_ANY 2 #define BRCMF_ESCAN_REQ_VERSION 1 #define BRCMF_ESCAN_REQ_VERSION_V2 2 +#define BRCMF_ESCAN_REQ_VERSION_V3 3 #define BRCMF_MAXRATES_IN_SET 16 /* max # of rates in rateset */ @@ -320,29 +329,57 @@ struct brcmf_bss_info_le { __le16 beacon_period; /* units are Kusec */ __le16 capability; /* Capability information */ u8 SSID_len; - u8 SSID[32]; + u8 SSID[IEEE80211_MAX_SSID_LEN]; + u8 bcnflags; /* additional flags w.r.t. beacon */ struct { __le32 count; /* # rates in this set */ u8 rates[16]; /* rates in 500kbps units w/hi bit set if basic */ } rateset; /* supported rates */ __le16 chanspec; /* chanspec for bss */ __le16 atim_window; /* units are Kusec */ - u8 dtim_period; /* DTIM period */ + u8 dtim_period; /* DTIM period */ + u8 accessnet; /* from beacon interwork IE (if bcnflags) */ __le16 RSSI; /* receive signal strength (in dBm) */ s8 phy_noise; /* noise (in dBm) */ u8 n_cap; /* BSS is 802.11N Capable */ + u8 he_cap; /* BSS is he capable */ + u8 load; /* BSS Load from QBSS load IE if available */ /* 802.11N BSS Capabilities (based on HT_CAP_*): */ __le32 nbss_cap; u8 ctl_ch; /* 802.11N BSS control channel number */ - __le32 reserved32[1]; /* Reserved for expansion of BSS properties */ + u8 reserved1[3]; /* Reserved for expansion of BSS properties */ + __le16 vht_rxmcsmap; /* VHT rx mcs map (802.11ac IE, VHT_CAP_MCS_MAP_*) */ + __le16 vht_txmcsmap; /* VHT tx mcs map (802.11ac IE, VHT_CAP_MCS_MAP_*) */ u8 flags; /* flags */ - u8 reserved[3]; /* Reserved for expansion of BSS properties */ + u8 vht_cap; /* BSS is vht capable */ + u8 reserved2[2]; /* Reserved for expansion of BSS properties */ u8 basic_mcs[BRCMF_MCSSET_LEN]; /* 802.11N BSS required MCS set */ __le16 ie_offset; /* offset at which IEs start, from beginning */ + u8 reserved3[2]; /* Reserved for expansion of BSS properties */ __le32 ie_length; /* byte length of Information Elements */ __le16 SNR; /* average SNR of during frame reception */ + __le16 vht_mcsmap; /**< STA's Associated vhtmcsmap */ + __le16 vht_mcsmap_prop; /**< STA's Associated prop vhtmcsmap */ + __le16 vht_txmcsmap_prop; /**< prop VHT tx mcs prop */ + __le32 he_mcsmap; /**< STA's Associated hemcsmap */ + __le32 he_rxmcsmap; /**< HE rx mcs map (802.11ax IE, HE_CAP_MCS_MAP_*) */ + __le32 he_txmcsmap; /**< HE tx mcs map (802.11ax IE, HE_CAP_MCS_MAP_*) */ + __le32 timestamp[2]; /* Beacon Timestamp for FAKEAP req */ + /* V112 fields follow */ + u8 eht_cap; /* BSS is EHT capable */ + u8 reserved4[3]; /* Reserved for expansion of BSS properties */ + /* by the spec. it is maximum 16 streams hence all mcs code for all nss may not fit + * in a 32 bit mcs nss map but since this field only reflects the common mcs nss map + * between that of the peer and our device so it's probably ok to make it 32 bit and + * allow only a limited number of nss e.g. upto 8 of them in the map given the fact + * that our device probably won't exceed 4 streams anyway... + */ + __le32 eht_mcsmap; /* STA's associated EHT mcs code map */ + /* FIXME: change the following mcs code map to uint32 if all mcs+nss can fit in */ + u8 eht_rxmcsmap[6]; /* EHT rx mcs code map */ + u8 eht_txmcsmap[6]; /* EHT tx mcs code map */ /* Add new fields here */ /* variable length Information Elements */ }; @@ -366,23 +403,23 @@ struct brcmf_ssid8_le { }; struct brcmf_scan_params_le { - struct brcmf_ssid_le ssid_le; /* default: {0, ""} */ - u8 bssid[ETH_ALEN]; /* default: bcast */ - s8 bss_type; /* default: any, + struct brcmf_ssid_le ssid_le; /* default: {0, ""} */ + u8 bssid[ETH_ALEN]; /* default: bcast */ + s8 bss_type; /* default: any, * DOT11_BSSTYPE_ANY/INFRASTRUCTURE/INDEPENDENT */ - u8 scan_type; /* flags, 0 use default */ - __le32 nprobes; /* -1 use default, number of probes per channel */ - __le32 active_time; /* -1 use default, dwell time per channel for + u8 scan_type; /* flags, 0 use default */ + __le32 nprobes; /* -1 use default, number of probes per channel */ + __le32 active_time; /* -1 use default, dwell time per channel for * active scanning */ - __le32 passive_time; /* -1 use default, dwell time per channel + __le32 passive_time; /* -1 use default, dwell time per channel * for passive scanning */ __le32 home_time; /* -1 use default, dwell time for the * home channel between channel scans */ - __le32 channel_num; /* count of channels and ssids that follow + __le32 channel_num; /* count of channels and ssids that follow * * low half is count of channels in * channel_list, 0 means default (use all @@ -398,56 +435,125 @@ struct brcmf_scan_params_le { * fixed parameter portion is assumed, otherwise * ssid in the fixed portion is ignored */ - union { - __le16 padding; /* Reserve space for at least 1 entry for abort - * which uses an on stack brcmf_scan_params_le - */ - DECLARE_FLEX_ARRAY(__le16, channel_list); /* chanspecs */ - }; + __le16 channel_list[]; /* chanspecs */ }; struct brcmf_scan_params_v2_le { - __le16 version; /* structure version */ - __le16 length; /* structure length */ - struct brcmf_ssid_le ssid_le; /* default: {0, ""} */ - u8 bssid[ETH_ALEN]; /* default: bcast */ - s8 bss_type; /* default: any, - * DOT11_BSSTYPE_ANY/INFRASTRUCTURE/INDEPENDENT - */ - u8 pad; - __le32 scan_type; /* flags, 0 use default */ - __le32 nprobes; /* -1 use default, number of probes per channel */ - __le32 active_time; /* -1 use default, dwell time per channel for - * active scanning - */ - __le32 passive_time; /* -1 use default, dwell time per channel - * for passive scanning - */ - __le32 home_time; /* -1 use default, dwell time for the - * home channel between channel scans - */ - __le32 channel_num; /* count of channels and ssids that follow - * - * low half is count of channels in - * channel_list, 0 means default (use all - * available channels) - * - * high half is entries in struct brcmf_ssid - * array that follows channel_list, aligned for - * s32 (4 bytes) meaning an odd channel count - * implies a 2-byte pad between end of - * channel_list and first ssid - * - * if ssid count is zero, single ssid in the - * fixed parameter portion is assumed, otherwise - * ssid in the fixed portion is ignored - */ - union { - __le16 padding; /* Reserve space for at least 1 entry for abort - * which uses an on stack brcmf_scan_params_v2_le - */ - DECLARE_FLEX_ARRAY(__le16, channel_list); /* chanspecs */ - }; + __le16 version; /* structure version */ + __le16 length; /* structure length */ + struct brcmf_ssid_le ssid_le; /* default: {0, ""} */ + u8 bssid[ETH_ALEN]; /* default: bcast */ + s8 bss_type; /* default: any, + * DOT11_BSSTYPE_ANY/INFRASTRUCTURE/INDEPENDENT + */ + u8 PAD; + __le32 scan_type; /* flags, 0 use default */ + __le32 nprobes; /* -1 use default, number of probes per channel */ + __le32 active_time; /* -1 use default, dwell time per channel for + * active scanning + */ + __le32 passive_time; /* -1 use default, dwell time per channel + * for passive scanning + */ + __le32 home_time; /* -1 use default, dwell time for the + * home channel between channel scans + */ + __le32 channel_num; /* count of channels and ssids that follow + * + * low half is count of channels in + * channel_list, 0 means default (use all + * available channels) + * + * high half is entries in struct brcmf_ssid + * array that follows channel_list, aligned for + * s32 (4 bytes) meaning an odd channel count + * implies a 2-byte pad between end of + * channel_list and first ssid + * + * if ssid count is zero, single ssid in the + * fixed parameter portion is assumed, otherwise + * ssid in the fixed portion is ignored + */ + __le16 channel_list[]; /* chanspecs */ +}; + +struct brcmf_scan_params_v3_le { + __le16 version; /* structure version */ + __le16 length; /* structure length */ + struct brcmf_ssid_le ssid_le; /* default: {0, ""} */ + u8 bssid[ETH_ALEN]; /* default: bcast */ + s8 bss_type; /* default: any, + * DOT11_BSSTYPE_ANY/INFRASTRUCTURE/INDEPENDENT + */ + u8 ssid_type; /* short vs regular SSID */ + __le32 scan_type; /* flags, 0 use default */ + __le32 nprobes; /* -1 use default, number of probes per channel */ + __le32 active_time; /* -1 use default, dwell time per channel for + * active scanning + */ + __le32 passive_time; /* -1 use default, dwell time per channel + * for passive scanning + */ + __le32 home_time; /* -1 use default, dwell time for the + * home channel between channel scans + */ + __le32 channel_num; /* count of channels and ssids that follow + * + * low half is count of channels in + * channel_list, 0 means default (use all + * available channels) + * + * high half is entries in struct brcmf_ssid + * array that follows channel_list, aligned for + * s32 (4 bytes) meaning an odd channel count + * implies a 2-byte pad between end of + * channel_list and first ssid + * + * if ssid count is zero, single ssid in the + * fixed parameter portion is assumed, otherwise + * ssid in the fixed portion is ignored + */ + __le16 channel_list[]; /* chanspecs */ +}; + +struct brcmf_scan_params_v4_le { + __le16 version; /* structure version */ + __le16 length; /* structure length */ + struct brcmf_ssid_le ssid_le; /* default: {0, ""} */ + u8 bssid[ETH_ALEN]; /* default: bcast */ + s8 bss_type; /* default: any, + * DOT11_BSSTYPE_ANY/INFRASTRUCTURE/INDEPENDENT + */ + u8 ssid_type; /* short vs regular SSID */ + __le32 scan_type; /* flags, 0 use default */ + __le32 scan_type_ext; /* ext flags, 0 use default */ + __le32 nprobes; /* -1 use default, number of probes per channel */ + __le32 active_time; /* -1 use default, dwell time per channel for + * active scanning + */ + __le32 passive_time; /* -1 use default, dwell time per channel + * for passive scanning + */ + __le32 home_time; /* -1 use default, dwell time for the + * home channel between channel scans + */ + __le32 channel_num; /* count of channels and ssids that follow + * + * low half is count of channels in + * channel_list, 0 means default (use all + * available channels) + * + * high half is entries in struct brcmf_ssid + * array that follows channel_list, aligned for + * s32 (4 bytes) meaning an odd channel count + * implies a 2-byte pad between end of + * channel_list and first ssid + * + * if ssid count is zero, single ssid in the + * fixed parameter portion is assumed, otherwise + * ssid in the fixed portion is ignored + */ + __le16 channel_list[]; /* chanspecs */ }; struct brcmf_scan_results { @@ -464,6 +570,8 @@ struct brcmf_escan_params_le { union { struct brcmf_scan_params_le params_le; struct brcmf_scan_params_v2_le params_v2_le; + struct brcmf_scan_params_v3_le params_v3_le; + struct brcmf_scan_params_v4_le params_v4_le; }; }; @@ -482,11 +590,67 @@ struct brcmf_escan_result_le { struct brcmf_assoc_params_le { /* 00:00:00:00:00:00: broadcast scan */ u8 bssid[ETH_ALEN]; + /* 0: use chanspec_num, and the single bssid, + * otherwise count of chanspecs in chanspec_list + * AND paired bssids following chanspec_list + * also, chanspec_num has to be set to zero + * for bssid list to be used + */ + __le16 bssid_cnt; + /* 0: all available channels, otherwise count of chanspecs in + * chanspec_list */ + __le32 chanspec_num; + /* list of chanspecs */ + __le16 chanspec_list[]; +}; + +struct brcmf_assoc_params_v1_le { + __le16 version; + __le16 flags; + /* 00:00:00:00:00:00: broadcast scan */ + u8 bssid[ETH_ALEN]; + /* 0: use chanspec_num, and the single bssid, + * otherwise count of chanspecs in chanspec_list + * AND paired bssids following chanspec_list + * also, chanspec_num has to be set to zero + * for bssid list to be used + */ + __le16 bssid_cnt; /* 0: all available channels, otherwise count of chanspecs in * chanspec_list */ __le32 chanspec_num; /* list of chanspecs */ - __le16 chanspec_list[1]; + __le16 chanspec_list[]; +}; + +/* ML assoc and scan params */ +struct brcmf_ml_assoc_scan_params_v1_le { + /* whether to follow strictly ordered assoc ? */ + u8 ml_assoc_mode; + /* to identify whether ml scan needs to be triggered */ + u8 ml_scan_mode; + u8 pad[2]; +}; + +struct brcmf_assoc_params_v2_le { + __le16 version; + __le16 flags; + /* 00:00:00:00:00:00: broadcast scan */ + u8 bssid[ETH_ALEN]; + /* 0: use chanspec_num, and the single bssid, + * otherwise count of chanspecs in chanspec_list + * AND paired bssids following chanspec_list + * also, chanspec_num has to be set to zero + * for bssid list to be used + */ + __le16 bssid_cnt; + /* Multilink association and scan params */ + struct brcmf_ml_assoc_scan_params_v1_le ml_assoc_scan_params; + /* 0: all available channels, otherwise count of chanspecs in + * chanspec_list */ + __le32 chanspec_num; + /* list of chanspecs */ + __le16 chanspec_list[]; }; /** @@ -511,9 +675,19 @@ struct brcmf_join_params { struct brcmf_assoc_params_le params_le; }; +struct brcmf_join_params_v1 { + struct brcmf_ssid_le ssid_le; + struct brcmf_assoc_params_v1_le params_le; +}; +struct brcmf_join_params_v2 { + struct brcmf_ssid_le ssid_le; + struct brcmf_assoc_params_v2_le params_le; +}; + /* scan params for extended join */ struct brcmf_join_scan_params_le { u8 scan_type; /* 0 use default, active or passive scan */ + u8 PAD[3]; __le32 nprobes; /* -1 use default, nr of probes per channel */ __le32 active_time; /* -1 use default, dwell time per channel for * active scanning @@ -526,6 +700,23 @@ struct brcmf_join_scan_params_le { */ }; +/* scan params for extended join */ +struct brcmf_join_scan_params_v1_le { + u8 scan_type; /* 0 use default, active or passive scan */ + u8 ml_scan_mode; /* 0 scan ML channels in RNR, 1 scan only provided channels */ + u8 PAD[2]; + __le32 nprobes; /* -1 use default, nr of probes per channel */ + __le32 active_time; /* -1 use default, dwell time per channel for + * active scanning + */ + __le32 passive_time; /* -1 use default, dwell time per channel + * for passive scanning + */ + __le32 home_time; /* -1 use default, dwell time for the home + * channel between channel scans + */ +}; + /* extended join params */ struct brcmf_ext_join_params_le { struct brcmf_ssid_le ssid_le; /* {0, ""}: wildcard scan */ @@ -533,6 +724,24 @@ struct brcmf_ext_join_params_le { struct brcmf_assoc_params_le assoc_le; }; +/* extended join params */ +struct brcmf_ext_join_params_v1_le { + __le16 version; + u16 pad; + struct brcmf_ssid_le ssid_le; /* {0, ""}: wildcard scan */ + struct brcmf_join_scan_params_le scan_le; + struct brcmf_assoc_params_v1_le assoc_le; +}; + +/* extended join params v2 */ +struct brcmf_ext_join_params_v2_le { + __le16 version; + u16 pad; + struct brcmf_ssid_le ssid_le; /* {0, ""}: wildcard scan */ + struct brcmf_join_scan_params_v1_le scan_le; + struct brcmf_assoc_params_v2_le assoc_le; +}; + struct brcmf_wsec_key { u32 index; /* key index */ u32 len; /* key length */ @@ -580,11 +789,15 @@ struct brcmf_wsec_key_le { * @key_len: number of octets in key material. * @flags: key handling qualifiers. * @key: PMK key material. + * @opt_len: optional field length + * @opt_tlvs: optional fields in TLV format */ struct brcmf_wsec_pmk_le { __le16 key_len; __le16 flags; u8 key[BRCMF_WSEC_MAX_SAE_PASSWORD_LEN]; + __le16 opt_len; + u8 opt_tlvs[]; }; /** @@ -611,13 +824,17 @@ struct brcmf_channel_info_le { __le32 scan_channel; }; +#define BRCMF_MAX_ASSOC_OUI_NUM 6 +#define BRCMF_ASSOC_OUI_LEN 3 struct brcmf_sta_info_le { __le16 ver; /* version of this struct */ __le16 len; /* length in bytes of this structure */ __le16 cap; /* sta's advertised capabilities */ + u16 PAD; __le32 flags; /* flags defined below */ __le32 idle; /* time since data pkt rx'd from sta */ u8 ea[ETH_ALEN]; /* Station address */ + u16 PAD2; __le32 count; /* # rates in this set */ u8 rates[BRCMF_MAXRATES_IN_SET]; /* rates in 500kbps units */ /* w/hi bit set if basic */ @@ -649,6 +866,7 @@ struct brcmf_sta_info_le { __le16 aid; /* association ID */ __le16 ht_capabilities; /* advertised ht caps */ __le16 vht_flags; /* converted vht flags */ + u16 PAD3; __le32 tx_pkts_retry_cnt; /* # of frames where a retry was * exhausted. */ @@ -701,6 +919,13 @@ struct brcmf_sta_info_le { __le32 tx_rspec; /* Rate of last successful tx frame */ __le32 rx_rspec; /* Rate of last successful rx frame */ __le32 wnm_cap; /* wnm capabilities */ + __le16 he_flags; /* converted he flags */ + u16 PAD; + struct { + u8 count; + u8 oui[BRCMF_MAX_ASSOC_OUI_NUM][BRCMF_ASSOC_OUI_LEN]; + } vendor_oui; + u8 link_bw; } v7; }; }; @@ -833,6 +1058,30 @@ struct brcmf_wlc_version_le { __le16 wlc_ver_minor; }; +/** + * struct brcmf_join_version_le - join interface version + */ +struct brcmf_join_version_le { + __le16 version; /**< version of the structure */ + __le16 length; /**< length of the entire structure */ + + /* join interface version numbers */ + __le16 join_ver_major; /**< join interface major version number */ + u8 pad[2]; +}; +#define BRCMF_JOIN_VERSION_VERSION 1 + +/** + * struct brcmf_scan_version_le - scan interface version + */ +struct brcmf_scan_version_le { + __le16 version; + __le16 length; + __le16 scan_ver_major; +}; + +#define BRCMF_SCAN_VERSION_VERSION 1 + /** * struct brcmf_assoclist_le - request assoc list. * @@ -1009,6 +1258,46 @@ struct brcmf_pno_param_le { __le32 slow_freq; }; +/** + * struct brcmf_pno_param_le - PNO scan configuration parameters + * + * @version: PNO parameters version. + * @length: Length of PNO structure + * @scan_freq: scan frequency. + * @lost_network_timeout: #sec. to declare discovered network as lost. + * @flags: Bit field to control features of PFN such as sort criteria auto + * enable switch and background scan. + * @rssi_margin: Margin to avoid jitter for choosing a PFN based on RSSI sort + * criteria. + * @bestn: number of best networks in each scan. + * @mscan: number of scans recorded. + * @repeat: minimum number of scan intervals before scan frequency changes + * in adaptive scan. + * @exp: exponent of 2 for maximum scan interval. + * @slow_freq: slow scan period. + * @min_bound: min bound for scan time randomization + * @max_bound: max bound for scan time randomization + * @pfn_lp_scan_disable: unused + * @pfn_lp_scan_cnt: allow interleaving lp scan with hp scan + */ +struct brcmf_pno_param_v3_le { + __le16 version; + __le16 length; + __le32 scan_freq; + __le32 lost_network_timeout; + __le16 flags; + __le16 rssi_margin; + u8 bestn; + u8 mscan; + u8 repeat; + u8 exp; + __le32 slow_freq; + u8 min_bound; + u8 max_bound; + u8 pfn_lp_scan_disable; + u8 pfn_lp_scan_cnt; +}; + /** * struct brcmf_pno_config_le - PNO channel configuration. * @@ -1062,6 +1351,28 @@ struct brcmf_pno_net_info_le { __le16 timestamp; }; +/** + * struct brcmf_pno_net_info_v3_le - information per found network. + * + * @bssid: BSS network identifier. + * @chanspec: channel spec. + * @SSID_len: length of ssid. + * @SSID: ssid characters. + * @flags: flags + * @RSSI: receive signal strength (in dBm). + * @timestamp: age in seconds. + */ +struct brcmf_pno_net_info_v3_le { + u8 bssid[6]; + u16 chanspec; + u8 SSID_len; + u8 padding; + u16 flags; + u8 SSID[32]; + __le16 RSSI; + __le16 timestamp; +}; + /** * struct brcmf_pno_scanresults_le - result returned in PNO NET FOUND event. * @@ -1082,6 +1393,14 @@ struct brcmf_pno_scanresults_v2_le { __le32 scan_ch_bucket; }; +/* V2 and V3 structs are the same */ +struct brcmf_pno_scanresults_v3_le { + __le32 version; + __le32 status; + __le32 count; + __le32 scan_ch_bucket; +}; + /** * struct brcmf_pno_macaddr_le - to configure PNO macaddr randomization. * @@ -1236,4 +1555,141 @@ struct brcmf_mkeep_alive_pkt_le { u8 data[]; } __packed; +enum event_msgs_ext_command { + EVENTMSGS_NONE = 0, + EVENTMSGS_SET_BIT = 1, + EVENTMSGS_RESET_BIT = 2, + EVENTMSGS_SET_MASK = 3 +}; + +#define EVENTMSGS_VER 1 + +/** + * struct brcmf_eventmsgs_ext_le - new event message mask commands + * + * @version: EVENTMSGS_VER + * @command: one of enum event_msgs_ext_command + * @len: for set, the mask size from the application to the firmware. + * for get, the actual firmware mask size. + * @maxgetsize: for get, the max size that the application can read from + * the firmware. + */ +struct brcmf_eventmsgs_ext_le { + u8 version; + u8 command; + u8 len; + u8 maxgetsize; + u8 mask[]; +}; + +/* version of the brcmf_wl_wsec_info structure */ +#define BRCMF_WSEC_INFO_VER 1 + +/* tlv used to return wl_wsec_info properties */ +struct brcmf_wsec_info_tlv { + u16 type; + u16 len; /* data length */ + u8 data[1]; /* data follows */ +}; + +/* input/output data type for wsec_info iovar */ +struct brcmf_wsec_info { + u8 version; /* structure version */ + u8 pad[2]; + u8 num_tlvs; + struct brcmf_wsec_info_tlv tlvs[1]; /* tlv data follows */ +}; + +/* HE top level command IDs */ +enum { + BRCMF_HE_CMD_ENABLE = 0, + BRCMF_HE_CMD_FEATURES = 1, + BRCMF_HE_CMD_SR = 2, + BRCMF_HE_CMD_TESTBED = 3, + BRCMF_HE_CMD_BSR_SUPPORT = 4, + BRCMF_HE_CMD_BSSCOLOR = 5, + BRCMF_HE_CMD_PARTIAL_BSSCOLOR = 6, + BRCMF_HE_CMD_CAP = 7, + BRCMF_HE_CMD_OMI = 8, + BRCMF_HE_CMD_RANGE_EXT = 9, + BRCMF_HE_CMD_RTSDURTHRESH = 10, + BRCMF_HE_CMD_PEDURATION = 11, + BRCMF_HE_CMD_MUEDCA = 12, + BRCMF_HE_CMD_DYNFRAG = 13, + BRCMF_HE_CMD_PPET = 14, + BRCMF_HE_CMD_HTC = 15, + BRCMF_HE_CMD_AXMODE = 16, + BRCMF_HE_CMD_FRAGTX = 17, + BRCMF_HE_CMD_DEFCAP = 18, +}; + +#define BRCMF_HE_VER_1 1 + +struct brcmf_he_bsscolor { + u8 color; /* 1..63, on get returns currently in use color */ + u8 disabled; /* 0/1, 0 means disabled is false, so coloring is enabled */ + u8 switch_count; /* 0, immediate programming, 1 .. 255 beacon count down */ + u8 PAD; +}; + +struct brcmf_he_omi { + u8 peer[ETH_ALEN]; /* leave it all 0s' for non-AP */ + u8 rx_nss; /* 0..7 */ + u8 channel_width; /* 0:20, 1:40, 2:80, 3:160 */ + u8 ul_mu_disable; /* 0|1 */ + u8 tx_nsts; /* 0..7 */ + u8 er_su_disable; /* 0|1 */ + u8 dl_mumimo_resound; /* 0|1 */ + u8 ul_mu_data_disable; /* 0|1 */ + u8 tx_override; /* 0, only used for testbed AP */ + u8 PAD[2]; +}; + +struct brcmf_he_edca_v1 { + u8 aci_aifsn; + u8 ecw_min_max; + u8 muedca_timer; + u8 PAD; +}; + +#define BRCMF_AC_COUNT 4 +struct brcmf_he_muedca_v1 { + /* structure control */ + __le16 version; /* structure version */ + __le16 length; /* data length (starting after this field) */ + struct brcmf_he_edca_v1 ac_param_ap[BRCMF_AC_COUNT]; + struct brcmf_he_edca_v1 ac_param_sta[BRCMF_AC_COUNT]; +}; + +#define BRCMF_HE_SR_VER_1 1 + +#define SRC_PSR_DIS 0x01 +#define SRC_NON_SRG_OBSS_PD_SR_DIS 0x02 +#define SRC_NON_SRG_OFFSET_PRESENT 0x04 +#define SRC_SRG_INFORMATION_PRESENT 0x08 +#define SRC_HESIGA_SPATIAL_REUSE_VALUE15_ALLOWED 0x10 + +#define HE_SR_SRG_INFO_LEN 18 + +struct brcmf_he_sr_v1 { + /* structure control */ + __le16 version; /* structure version */ + __le16 length; /* data length (starting after this field) */ + u8 enabled; + u8 src; /* SR control, see above defines. */ + u8 non_srg_offset; /* Non-SRG Offset */ + u8 srg[HE_SR_SRG_INFO_LEN]; /* SRG Information */ +}; + +#define BRCMF_HE_DEFCAP_VER_1 1 + +struct brcmf_he_defcap { + __le16 version; /* structure version */ + __le16 length; /* data length (starting after this field) */ + u8 bsscfg_type; + u8 bsscfg_subtype; + u8 mac_cap[6]; + u8 phy_cap[11]; +}; + #endif /* FWIL_TYPES_H_ */ diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/interface_create.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/interface_create.c new file mode 100644 index 00000000000000..1f40ff8d632c25 --- /dev/null +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/interface_create.c @@ -0,0 +1,270 @@ +// SPDX-License-Identifier: ISC +/* + * Copyright (c) 2023 Daniel Berlin + */ + +/* This file handles firmware-side interface creation */ + +#include +#include +#include +#include +#include +#include + +#include +#include +#include "cfg80211.h" +#include "debug.h" +#include "fwil.h" +#include "proto.h" +#include "bus.h" +#include "common.h" +#include "interface_create.h" + +#define BRCMF_INTERFACE_CREATE_VER_1 1 +#define BRCMF_INTERFACE_CREATE_VER_2 2 +#define BRCMF_INTERFACE_CREATE_VER_3 3 +#define BRCMF_INTERFACE_CREATE_VER_MAX BRCMF_INTERFACE_CREATE_VER_3 + +/* These sets of flags specify whether to use various fields in the interface create structures */ + +/* This is only used with version 0 or 1 */ +#define BRCMF_INTERFACE_CREATE_STA (0 << 0) +#define BRCMF_INTERFACE_CREATE_AP (1 << 0) + +#define BRCMF_INTERFACE_MAC_DONT_USE (0 << 1) +#define BRCMF_INTERFACE_MAC_USE (1 << 1) + +#define BRCMF_INTERFACE_WLC_INDEX_DONT_USE (0 << 2) +#define BRCMF_INTERFACE_WLC_INDEX_USE (1 << 2) + +#define BRCMF_INTERFACE_IF_INDEX_DONT_USE (0 << 3) +#define BRCMF_INTERFACE_IF_INDEX_USE (1 << 3) + +#define BRCMF_INTERFACE_BSSID_DONT_USE (0 << 4) +#define BRCMF_INTERFACE_BSSID_USE (1 << 4) + +/* + * From revision >= 2 Bit 0 of flags field will not be used for STA or AP interface creation. + * "iftype" field shall be used for identifying the interface type. + */ +enum brcmf_interface_type { + BRCMF_INTERFACE_TYPE_STA = 0, + BRCMF_INTERFACE_TYPE_AP = 1, + /* The missing number here is deliberate */ + BRCMF_INTERFACE_TYPE_NAN = 3, + BRCMF_INTERFACE_TYPE_P2P_GO = 4, + BRCMF_INTERFACE_TYPE_P2P_GC = 5, + BRCMF_INTERFACE_TYPE_P2P_DISC = 6, + BRCMF_INTERFACE_TYPE_IBSS = 7, + BRCMF_INTERFACE_TYPE_MESH = 8 +}; + + +/* All sources treat these structures as being host endian. + * However, firmware treats it as little endian, so we do as well */ + +struct brcmf_interface_create_v1 { + __le16 ver; /* structure version */ + u8 pad1[2]; + __le32 flags; /* flags for operation */ + u8 mac_addr[ETH_ALEN]; /* MAC address */ + u8 pad2[2]; + __le32 wlc_index; /* optional for wlc index */ +}; + +struct brcmf_interface_create_v2 { + __le16 ver; /* structure version */ + u8 pad1[2]; + __le32 flags; /* flags for operation */ + u8 mac_addr[ETH_ALEN]; /* MAC address */ + u8 iftype; /* type of interface created */ + u8 pad2; + u32 wlc_index; /* optional for wlc index */ +}; + +struct brcmf_interface_create_v3 { + __le16 ver; /* structure version */ + __le16 len; /* length of structure + data */ + __le16 fixed_len; /* length of structure */ + u8 iftype; /* type of interface created */ + u8 wlc_index; /* optional for wlc index */ + __le32 flags; /* flags for operation */ + u8 mac_addr[ETH_ALEN]; /* MAC address */ + u8 bssid[ETH_ALEN]; /* optional for BSSID */ + u8 if_index; /* interface index request */ + u8 pad[3]; + u8 data[]; /* Optional for specific data */ +}; + +static int brcmf_get_first_free_bsscfgidx(struct brcmf_pub *drvr) +{ + int bsscfgidx; + + for (bsscfgidx = 0; bsscfgidx < BRCMF_MAX_IFS; bsscfgidx++) { + /* bsscfgidx 1 is reserved for legacy P2P */ + if (bsscfgidx == 1) + continue; + if (!drvr->iflist[bsscfgidx]) + return bsscfgidx; + } + + return -ENOMEM; +} + +static void brcmf_set_vif_sta_macaddr(struct brcmf_if *ifp, u8 *mac_addr) +{ + u8 mac_idx = ifp->drvr->sta_mac_idx; + + /* set difference MAC address with locally administered bit */ + memcpy(mac_addr, ifp->mac_addr, ETH_ALEN); + mac_addr[0] |= 0x02; + mac_addr[3] ^= mac_idx ? 0xC0 : 0xA0; + mac_idx++; + mac_idx = mac_idx % 2; + ifp->drvr->sta_mac_idx = mac_idx; +} + +static int brcmf_cfg80211_request_if_internal(struct brcmf_if *ifp, u32 version, + enum brcmf_interface_type if_type, + u8 *macaddr) +{ + switch (version) { + case BRCMF_INTERFACE_CREATE_VER_1: { + struct brcmf_interface_create_v1 iface_v1 = {}; + u32 flags = if_type; + + iface_v1.ver = cpu_to_le16(BRCMF_INTERFACE_CREATE_VER_1); + if (macaddr) { + flags |= BRCMF_INTERFACE_MAC_USE; + if (!is_zero_ether_addr(macaddr)) + memcpy(iface_v1.mac_addr, macaddr, ETH_ALEN); + else + brcmf_set_vif_sta_macaddr(ifp, + iface_v1.mac_addr); + } + iface_v1.flags = cpu_to_le32(flags); + return brcmf_fil_iovar_data_get(ifp, "interface_create", + &iface_v1, sizeof(iface_v1)); + } + case BRCMF_INTERFACE_CREATE_VER_2: { + struct brcmf_interface_create_v2 iface_v2 = {}; + u32 flags = 0; + + iface_v2.ver = cpu_to_le16(BRCMF_INTERFACE_CREATE_VER_2); + iface_v2.iftype = if_type; + if (macaddr) { + flags = BRCMF_INTERFACE_MAC_USE; + if (!is_zero_ether_addr(macaddr)) + memcpy(iface_v2.mac_addr, macaddr, ETH_ALEN); + else + brcmf_set_vif_sta_macaddr(ifp, + iface_v2.mac_addr); + } + iface_v2.flags = cpu_to_le32(flags); + return brcmf_fil_iovar_data_get(ifp, "interface_create", + &iface_v2, sizeof(iface_v2)); + } + case BRCMF_INTERFACE_CREATE_VER_3: { + struct brcmf_interface_create_v3 iface_v3 = {}; + u32 flags = 0; + + iface_v3.ver = cpu_to_le16(BRCMF_INTERFACE_CREATE_VER_3); + iface_v3.iftype = if_type; + iface_v3.len = cpu_to_le16(sizeof(iface_v3)); + iface_v3.fixed_len = cpu_to_le16(sizeof(iface_v3)); + if (macaddr) { + flags = BRCMF_INTERFACE_MAC_USE; + if (!is_zero_ether_addr(macaddr)) + memcpy(iface_v3.mac_addr, macaddr, ETH_ALEN); + else + brcmf_set_vif_sta_macaddr(ifp, + iface_v3.mac_addr); + } + iface_v3.flags = cpu_to_le32(flags); + return brcmf_fil_iovar_data_get(ifp, "interface_create", + &iface_v3, sizeof(iface_v3)); + } + default: + bphy_err(ifp->drvr, "Unknown interface create version:%d\n", + version); + return -EINVAL; + } +} +static int brcmf_cfg80211_request_if(struct brcmf_if *ifp, + enum brcmf_interface_type if_type, + u8 *macaddr) +{ + s32 err; + u32 iface_create_ver; + + /* Query the creation version, see if the firmware knows */ + iface_create_ver = 0; + err = brcmf_fil_bsscfg_int_query(ifp, "interface_create", + &iface_create_ver); + if (!err) { + err = brcmf_cfg80211_request_if_internal(ifp, iface_create_ver, + if_type, macaddr); + if (!err) { + brcmf_info("interface created (version %d)\n", + iface_create_ver); + } else { + bphy_err(ifp->drvr, + "failed to create interface (version %d):%d\n", + iface_create_ver, err); + } + return err; + } + /* Either version one or version two */ + err = brcmf_cfg80211_request_if_internal( + ifp, if_type, BRCMF_INTERFACE_CREATE_VER_2, macaddr); + if (!err) { + brcmf_info("interface created (version 2)\n"); + return 0; + } + err = brcmf_cfg80211_request_if_internal( + ifp, if_type, BRCMF_INTERFACE_CREATE_VER_1, macaddr); + if (!err) { + brcmf_info("interface created (version 1)\n"); + return 0; + } + bphy_err(ifp->drvr, + "interface creation failed, tried query, v2, v1: %d\n", err); + return -EINVAL; +} + +int brcmf_cfg80211_request_sta_if(struct brcmf_if *ifp, u8 *macaddr) +{ + return brcmf_cfg80211_request_if(ifp, BRCMF_INTERFACE_TYPE_STA, + macaddr); +} + +int brcmf_cfg80211_request_ap_if(struct brcmf_if *ifp) +{ + int err; + + err = brcmf_cfg80211_request_if(ifp, BRCMF_INTERFACE_TYPE_AP, NULL); + if (err) { + struct brcmf_mbss_ssid_le mbss_ssid_le; + int bsscfgidx; + + brcmf_info("Does not support interface_create (%d)\n", err); + memset(&mbss_ssid_le, 0, sizeof(mbss_ssid_le)); + bsscfgidx = brcmf_get_first_free_bsscfgidx(ifp->drvr); + if (bsscfgidx < 0) + return bsscfgidx; + + mbss_ssid_le.bsscfgidx = cpu_to_le32(bsscfgidx); + mbss_ssid_le.SSID_len = cpu_to_le32(5); + sprintf(mbss_ssid_le.SSID, "ssid%d", bsscfgidx); + + err = brcmf_fil_bsscfg_data_set(ifp, "bsscfg:ssid", + &mbss_ssid_le, + sizeof(mbss_ssid_le)); + + if (err < 0) + bphy_err(ifp->drvr, "setting ssid failed %d\n", err); + } + return err; +} diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/interface_create.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/interface_create.h new file mode 100644 index 00000000000000..669fa1508b67f6 --- /dev/null +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/interface_create.h @@ -0,0 +1,13 @@ +// SPDX-License-Identifier: ISC +/* + * Copyright (c) 2023 Daniel Berlin + */ + +#ifndef _BRCMF_INTERFACE_CREATE_H_ +#define _BRCMF_INTERFACE_CREATE_H_ +#include "core.h" + +int brcmf_cfg80211_request_sta_if(struct brcmf_if *ifp, u8 *macaddr); +int brcmf_cfg80211_request_ap_if(struct brcmf_if *ifp); + +#endif /* _BRCMF_INTERFACE_CREATE_H_ */ diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/join_param.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/join_param.c new file mode 100644 index 00000000000000..4f026571c7e7eb --- /dev/null +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/join_param.c @@ -0,0 +1,288 @@ +// SPDX-License-Identifier: ISC +/* + * Copyright (c) 2023 Daniel Berlin + */ +#include +#include + +#include "core.h" +#include "debug.h" +#include "fwil_types.h" +#include "cfg80211.h" +#include "join_param.h" + +/* These defaults are the same as found in the DHD drivers, and represent + * reasonable defaults for various scan dwell and probe times. */ +#define BRCMF_SCAN_JOIN_ACTIVE_DWELL_TIME_MS 320 +#define BRCMF_SCAN_JOIN_PASSIVE_DWELL_TIME_MS 400 +#define BRCMF_SCAN_JOIN_PROBE_INTERVAL_MS 20 + +/* Most of the actual structure fields we fill in are the same for various versions + * However, due to various incompatible changes and variants, the fields are not always + * in the same place. + * This makes for code duplication, so we try to commonize setting fields where it makes sense. + */ + +static void brcmf_joinscan_set_ssid(struct brcmf_ssid_le *ssid_le, + const u8 *ssid, u32 ssid_len) +{ + ssid_len = min_t(u32, ssid_len, IEEE80211_MAX_SSID_LEN); + ssid_le->SSID_len = cpu_to_le32(ssid_len); + memcpy(ssid_le->SSID, ssid, ssid_len); +} + +static void brcmf_joinscan_set_bssid(u8 out_bssid[6], const u8 *in_bssid) +{ + if (in_bssid) { + memcpy(out_bssid, in_bssid, ETH_ALEN); + } else { + eth_broadcast_addr(out_bssid); + } +} + +/* Create a single channel chanspec list from a wireless stack channel */ +static void brcmf_joinscan_set_single_chanspec_from_channel( + struct brcmf_cfg80211_info *cfg, struct ieee80211_channel *chan, + __le32 *chanspec_count, __le16 (*chanspec_list)[]) +{ + u16 chanspec = channel_to_chanspec(&cfg->d11inf, chan); + *chanspec_count = cpu_to_le32(1); + (*chanspec_list)[0] = cpu_to_le16(chanspec); +} + +/* Create a single channel chanspec list from a wireless stack chandef */ +static void brcmf_joinscan_set_single_chanspec_from_chandef( + struct brcmf_cfg80211_info *cfg, struct cfg80211_chan_def *chandef, + __le32 *chanspec_count, __le16 (*chanspec_list)[]) +{ + u16 chanspec = chandef_to_chanspec(&cfg->d11inf, chandef); + *chanspec_count = cpu_to_le32(1); + (*chanspec_list)[0] = cpu_to_le16(chanspec); +} + +static void *brcmf_get_struct_for_ibss_v0(struct brcmf_cfg80211_info *cfg, + u32 *struct_size, + struct cfg80211_ibss_params *params) +{ + struct brcmf_join_params *join_params; + + u32 join_params_size = struct_size(join_params, params_le.chanspec_list, + params->chandef.chan != NULL); + + *struct_size = join_params_size; + join_params = kzalloc(join_params_size, GFP_KERNEL); + if (!join_params) { + bphy_err(cfg, "Unable to allocate memory for join params\n"); + return NULL; + } + brcmf_joinscan_set_ssid(&join_params->ssid_le, params->ssid, + params->ssid_len); + brcmf_joinscan_set_bssid(join_params->params_le.bssid, params->bssid); + /* Channel */ + if (cfg->channel) { + brcmf_joinscan_set_single_chanspec_from_chandef( + cfg, ¶ms->chandef, + &join_params->params_le.chanspec_num, + &join_params->params_le.chanspec_list); + } + return join_params; +} + +static void * +brcmf_get_prepped_struct_for_ibss_v1(struct brcmf_cfg80211_info *cfg, + u32 *struct_size, + struct cfg80211_ibss_params *params) +{ + struct brcmf_join_params_v1 *join_params; + u32 join_params_size = struct_size(join_params, params_le.chanspec_list, + params->chandef.chan != NULL); + + *struct_size = join_params_size; + join_params = kzalloc(join_params_size, GFP_KERNEL); + if (!join_params) { + bphy_err(cfg, "Unable to allocate memory for join params\n"); + return NULL; + } + join_params->params_le.version = cpu_to_le16(1); + brcmf_joinscan_set_ssid(&join_params->ssid_le, params->ssid, + params->ssid_len); + brcmf_joinscan_set_bssid(join_params->params_le.bssid, params->bssid); + /* Channel */ + if (cfg->channel) { + brcmf_joinscan_set_single_chanspec_from_chandef( + cfg, ¶ms->chandef, + &join_params->params_le.chanspec_num, + &join_params->params_le.chanspec_list); + } + return join_params; +} + +static void +brcmf_joinscan_set_common_v0v1_params(struct brcmf_join_scan_params_le *scan_le, + bool have_channel) +{ + /* Set up join scan parameters */ + scan_le->scan_type = 0; + scan_le->home_time = cpu_to_le32(-1); + + if (have_channel) { + /* Increase dwell time to receive probe response or detect + * beacon from target AP at a noisy air only during connect + * command. + */ + scan_le->active_time = + cpu_to_le32(BRCMF_SCAN_JOIN_ACTIVE_DWELL_TIME_MS); + scan_le->passive_time = + cpu_to_le32(BRCMF_SCAN_JOIN_PASSIVE_DWELL_TIME_MS); + /* To sync with presence period of VSDB GO send probe request + * more frequently. Probe request will be stopped when it gets + * probe response from target AP/GO. + */ + scan_le->nprobes = + cpu_to_le32(BRCMF_SCAN_JOIN_ACTIVE_DWELL_TIME_MS / + BRCMF_SCAN_JOIN_PROBE_INTERVAL_MS); + } else { + scan_le->active_time = cpu_to_le32(-1); + scan_le->passive_time = cpu_to_le32(-1); + scan_le->nprobes = cpu_to_le32(-1); + } +} +static void * +brcmf_get_struct_for_connect_v0(struct brcmf_cfg80211_info *cfg, + u32 *struct_size, + struct cfg80211_connect_params *params) +{ + struct brcmf_ext_join_params_le *ext_v0; + u32 join_params_size = + struct_size(ext_v0, assoc_le.chanspec_list, cfg->channel != 0); + + *struct_size = join_params_size; + ext_v0 = kzalloc(join_params_size, GFP_KERNEL); + if (!ext_v0) { + bphy_err( + cfg, + "Could not allocate memory for extended join parameters\n"); + return NULL; + } + brcmf_joinscan_set_ssid(&ext_v0->ssid_le, params->ssid, + params->ssid_len); + brcmf_joinscan_set_common_v0v1_params(&ext_v0->scan_le, + cfg->channel != 0); + brcmf_joinscan_set_bssid(ext_v0->assoc_le.bssid, params->bssid); + if (cfg->channel) { + struct ieee80211_channel *chan = params->channel_hint ? + params->channel_hint : + params->channel; + brcmf_joinscan_set_single_chanspec_from_channel( + cfg, chan, &ext_v0->assoc_le.chanspec_num, + &ext_v0->assoc_le.chanspec_list); + } + return ext_v0; +} + +static void * +brcmf_get_struct_for_connect_v1(struct brcmf_cfg80211_info *cfg, + u32 *struct_size, + struct cfg80211_connect_params *params) +{ + struct brcmf_ext_join_params_v1_le *ext_v1; + u32 join_params_size = + struct_size(ext_v1, assoc_le.chanspec_list, cfg->channel != 0); + + *struct_size = join_params_size; + ext_v1 = kzalloc(join_params_size, GFP_KERNEL); + if (!ext_v1) { + bphy_err( + cfg, + "Could not allocate memory for extended join parameters\n"); + return NULL; + } + ext_v1->version = cpu_to_le16(1); + ext_v1->assoc_le.version = cpu_to_le16(1); + brcmf_joinscan_set_ssid(&ext_v1->ssid_le, params->ssid, + params->ssid_len); + brcmf_joinscan_set_common_v0v1_params(&ext_v1->scan_le, + cfg->channel != 0); + brcmf_joinscan_set_bssid(ext_v1->assoc_le.bssid, params->bssid); + if (cfg->channel) { + struct ieee80211_channel *chan = params->channel_hint ? + params->channel_hint : + params->channel; + brcmf_joinscan_set_single_chanspec_from_channel( + cfg, chan, &ext_v1->assoc_le.chanspec_num, + &ext_v1->assoc_le.chanspec_list); + } + return ext_v1; +} + +static void *brcmf_get_join_from_ext_join_v0(void *ext_join, u32 *struct_size) +{ + struct brcmf_ext_join_params_le *ext_join_v0 = + (struct brcmf_ext_join_params_le *)ext_join; + u32 chanspec_num = le32_to_cpu(ext_join_v0->assoc_le.chanspec_num); + struct brcmf_join_params *join_params; + u32 join_params_size = + struct_size(join_params, params_le.chanspec_list, chanspec_num); + u32 assoc_size = struct_size_t(struct brcmf_assoc_params_le, + chanspec_list, chanspec_num); + + *struct_size = join_params_size; + join_params = kzalloc(join_params_size, GFP_KERNEL); + if (!join_params) { + return NULL; + } + memcpy(&join_params->ssid_le, &ext_join_v0->ssid_le, + sizeof(ext_join_v0->ssid_le)); + memcpy(&join_params->params_le, &ext_join_v0->assoc_le, assoc_size); + + return join_params; +} + +static void *brcmf_get_join_from_ext_join_v1(void *ext_join, u32 *struct_size) +{ + struct brcmf_ext_join_params_v1_le *ext_join_v1 = + (struct brcmf_ext_join_params_v1_le *)ext_join; + u32 chanspec_num = le32_to_cpu(ext_join_v1->assoc_le.chanspec_num); + struct brcmf_join_params_v1 *join_params; + u32 join_params_size = + struct_size(join_params, params_le.chanspec_list, chanspec_num); + u32 assoc_size = struct_size_t(struct brcmf_assoc_params_le, + chanspec_list, chanspec_num); + + *struct_size = join_params_size; + join_params = kzalloc(join_params_size, GFP_KERNEL); + if (!join_params) { + return NULL; + } + memcpy(&join_params->ssid_le, &ext_join_v1->ssid_le, + sizeof(ext_join_v1->ssid_le)); + memcpy(&join_params->params_le, &ext_join_v1->assoc_le, assoc_size); + + return join_params; +} + +int brcmf_join_param_setup_for_version(struct brcmf_pub *drvr, u8 version) +{ + drvr->join_param_handler.version = version; + switch (version) { + case 0: + drvr->join_param_handler.get_struct_for_ibss = + brcmf_get_struct_for_ibss_v0; + drvr->join_param_handler.get_struct_for_connect = + brcmf_get_struct_for_connect_v0; + drvr->join_param_handler.get_join_from_ext_join = + brcmf_get_join_from_ext_join_v0; + break; + case 1: + drvr->join_param_handler.get_struct_for_ibss = + brcmf_get_prepped_struct_for_ibss_v1; + drvr->join_param_handler.get_struct_for_connect = + brcmf_get_struct_for_connect_v1; + drvr->join_param_handler.get_join_from_ext_join = + brcmf_get_join_from_ext_join_v1; + break; + default: + return -EINVAL; + } + return 0; +} diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/join_param.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/join_param.h new file mode 100644 index 00000000000000..f549fe2a740823 --- /dev/null +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/join_param.h @@ -0,0 +1,22 @@ +// SPDX-License-Identifier: ISC +/* + * Copyright (c) 2023 Daniel Berlin + */ + +#ifndef _BRCMF_JOIN_PARAM_H +#define _BRCMF_JOIN_PARAM_H + +struct brcmf_pub; + +/** + * brcmf_join_param_setup_for_version() - Setup the driver to handle join structures + * + * There are a number of different structures and interface versions for join/extended join parameters + * This sets up the driver to handle a particular interface version. + * + * @drvr Driver structure to setup + * @ver Interface version + * Return: %0 if okay, error code otherwise + */ +int brcmf_join_param_setup_for_version(struct brcmf_pub *drvr, u8 ver); +#endif /* _BRCMF_JOIN_PARAM_H */ diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c index 45fbcbdc7d9e4b..0e41d618486d39 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c @@ -47,9 +47,35 @@ #define MSGBUF_TYPE_RX_CMPLT 0x12 #define MSGBUF_TYPE_LPBK_DMAXFER 0x13 #define MSGBUF_TYPE_LPBK_DMAXFER_CMPLT 0x14 +#define MSGBUF_TYPE_FLOW_RING_RESUME 0x15 +#define MSGBUF_TYPE_FLOW_RING_RESUME_CMPLT 0x16 +#define MSGBUF_TYPE_FLOW_RING_SUSPEND 0x17 +#define MSGBUF_TYPE_FLOW_RING_SUSPEND_CMPLT 0x18 +#define MSGBUF_TYPE_INFO_BUF_POST 0x19 +#define MSGBUF_TYPE_INFO_BUF_CMPLT 0x1A +#define MSGBUF_TYPE_H2D_RING_CREATE 0x1B +#define MSGBUF_TYPE_D2H_RING_CREATE 0x1C +#define MSGBUF_TYPE_H2D_RING_CREATE_CMPLT 0x1D +#define MSGBUF_TYPE_D2H_RING_CREATE_CMPLT 0x1E +#define MSGBUF_TYPE_H2D_RING_CONFIG 0x1F +#define MSGBUF_TYPE_D2H_RING_CONFIG 0x20 +#define MSGBUF_TYPE_H2D_RING_CONFIG_CMPLT 0x21 +#define MSGBUF_TYPE_D2H_RING_CONFIG_CMPLT 0x22 +#define MSGBUF_TYPE_H2D_MAILBOX_DATA 0x23 +#define MSGBUF_TYPE_D2H_MAILBOX_DATA 0x24 +#define MSGBUF_TYPE_TIMSTAMP_BUFPOST 0x25 +#define MSGBUF_TYPE_HOSTTIMSTAMP 0x26 +#define MSGBUF_TYPE_HOSTTIMSTAMP_CMPLT 0x27 +#define MSGBUF_TYPE_FIRMWARE_TIMESTAMP 0x28 +#define MSGBUF_TYPE_SNAPSHOT_UPLOAD 0x29 +#define MSGBUF_TYPE_SNAPSHOT_CMPLT 0x2A +#define MSGBUF_TYPE_H2D_RING_DELETE 0x2B +#define MSGBUF_TYPE_D2H_RING_DELETE 0x2C +#define MSGBUF_TYPE_H2D_RING_DELETE_CMPLT 0x2D +#define MSGBUF_TYPE_D2H_RING_DELETE_CMPLT 0x2E #define NR_TX_PKTIDS 2048 -#define NR_RX_PKTIDS 1024 +#define NR_RX_PKTIDS 2048 #define BRCMF_IOCTL_REQ_PKTID 0xFFFE @@ -218,6 +244,19 @@ struct msgbuf_flowring_flush_resp { __le32 rsvd0[3]; }; +struct msgbuf_h2d_mailbox_data { + struct msgbuf_common_hdr msg; + __le32 data; + __le32 rsvd0[7]; +}; + +struct msgbuf_d2h_mailbox_data { + struct msgbuf_common_hdr msg; + struct msgbuf_completion_hdr compl_hdr; + __le32 data; + __le32 rsvd0[2]; +}; + struct brcmf_msgbuf_work_item { struct list_head queue; u32 flowid; @@ -1285,6 +1324,16 @@ brcmf_msgbuf_process_flow_ring_delete_response(struct brcmf_msgbuf *msgbuf, } +static void brcmf_msgbuf_process_d2h_mailbox_data(struct brcmf_msgbuf *msgbuf, + void *buf) +{ + struct msgbuf_d2h_mailbox_data *d2h_mb_data = buf; + struct brcmf_pub *drvr = msgbuf->drvr; + + brcmf_bus_d2h_mb_rx(drvr->bus_if, le32_to_cpu(d2h_mb_data->data)); +} + + static void brcmf_msgbuf_process_msgtype(struct brcmf_msgbuf *msgbuf, void *buf) { struct brcmf_pub *drvr = msgbuf->drvr; @@ -1327,6 +1376,10 @@ static void brcmf_msgbuf_process_msgtype(struct brcmf_msgbuf *msgbuf, void *buf) brcmf_dbg(MSGBUF, "MSGBUF_TYPE_RX_CMPLT\n"); brcmf_msgbuf_process_rx_complete(msgbuf, buf); break; + case MSGBUF_TYPE_D2H_MAILBOX_DATA: + brcmf_dbg(MSGBUF, "MSGBUF_TYPE_D2H_MAILBOX_DATA\n"); + brcmf_msgbuf_process_d2h_mailbox_data(msgbuf, buf); + break; default: bphy_err(drvr, "Unsupported msgtype %d\n", msg->msgtype); break; @@ -1465,6 +1518,38 @@ void brcmf_msgbuf_delete_flowring(struct brcmf_pub *drvr, u16 flowid) } } + +int brcmf_msgbuf_h2d_mb_write(struct brcmf_pub *drvr, u32 data) +{ + struct brcmf_msgbuf *msgbuf = (struct brcmf_msgbuf *)drvr->proto->pd; + struct brcmf_commonring *commonring; + struct msgbuf_h2d_mailbox_data *request; + void *ret_ptr; + int err; + + commonring = msgbuf->commonrings[BRCMF_H2D_MSGRING_CONTROL_SUBMIT]; + brcmf_commonring_lock(commonring); + ret_ptr = brcmf_commonring_reserve_for_write(commonring); + if (!ret_ptr) { + bphy_err(drvr, "Failed to reserve space in commonring\n"); + brcmf_commonring_unlock(commonring); + return -ENOMEM; + } + + request = (struct msgbuf_h2d_mailbox_data *)ret_ptr; + request->msg.msgtype = MSGBUF_TYPE_H2D_MAILBOX_DATA; + request->msg.ifidx = -1; + request->msg.flags = 0; + request->msg.request_id = 0; + request->data = data; + + err = brcmf_commonring_write_complete(commonring); + brcmf_commonring_unlock(commonring); + + return err; +} + + #ifdef DEBUG static int brcmf_msgbuf_stats_read(struct seq_file *seq, void *data) { diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.h index 6a849f4a94dd7f..0ed48cf13d93cf 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.h @@ -8,10 +8,10 @@ #ifdef CONFIG_BRCMFMAC_PROTO_MSGBUF #define BRCMF_H2D_MSGRING_CONTROL_SUBMIT_MAX_ITEM 64 -#define BRCMF_H2D_MSGRING_RXPOST_SUBMIT_MAX_ITEM 1024 +#define BRCMF_H2D_MSGRING_RXPOST_SUBMIT_MAX_ITEM 2048 #define BRCMF_D2H_MSGRING_CONTROL_COMPLETE_MAX_ITEM 64 #define BRCMF_D2H_MSGRING_TX_COMPLETE_MAX_ITEM 1024 -#define BRCMF_D2H_MSGRING_RX_COMPLETE_MAX_ITEM 1024 +#define BRCMF_D2H_MSGRING_RX_COMPLETE_MAX_ITEM 2048 #define BRCMF_H2D_TXFLOWRING_MAX_ITEM 512 #define BRCMF_H2D_MSGRING_CONTROL_SUBMIT_ITEMSIZE 40 @@ -32,6 +32,7 @@ int brcmf_proto_msgbuf_rx_trigger(struct device *dev); void brcmf_msgbuf_delete_flowring(struct brcmf_pub *drvr, u16 flowid); int brcmf_proto_msgbuf_attach(struct brcmf_pub *drvr); void brcmf_proto_msgbuf_detach(struct brcmf_pub *drvr); +int brcmf_msgbuf_h2d_mb_write(struct brcmf_pub *drvr, u32 data); #else static inline int brcmf_proto_msgbuf_attach(struct brcmf_pub *drvr) { diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c index e1752a513c733d..06d2933162b26f 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c @@ -1784,8 +1784,8 @@ bool brcmf_p2p_send_action_frame(struct brcmf_if *ifp, /* do not configure anything. it will be */ /* sent with a default configuration */ } else { - bphy_err(drvr, "Unknown Frame: category 0x%x, action 0x%x\n", - category, action); + bphy_info_once(drvr, "Unknown Frame: category 0x%x, action 0x%x\n", + category, action); return false; } diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c index 6327f4eca50078..9f11d555ade453 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c @@ -71,6 +71,8 @@ BRCMF_FW_CLM_DEF(4377B3, "brcmfmac4377b3-pcie"); BRCMF_FW_CLM_DEF(4378B1, "brcmfmac4378b1-pcie"); BRCMF_FW_CLM_DEF(4378B3, "brcmfmac4378b3-pcie"); BRCMF_FW_CLM_DEF(4387C2, "brcmfmac4387c2-pcie"); +BRCMF_FW_CLM_DEF(4388B0, "brcmfmac4388b0-pcie"); +BRCMF_FW_CLM_DEF(4388C0, "brcmfmac4388c0-pcie"); BRCMF_FW_CLM_DEF(54591, "brcmfmac54591-pcie"); /* firmware config files */ @@ -112,6 +114,8 @@ static const struct brcmf_firmware_mapping brcmf_pcie_fwnames[] = { BRCMF_FW_ENTRY(BRCM_CC_4378_CHIP_ID, 0x0000000F, 4378B1), /* revision ID 3 */ BRCMF_FW_ENTRY(BRCM_CC_4378_CHIP_ID, 0xFFFFFFE0, 4378B3), /* revision ID 5 */ BRCMF_FW_ENTRY(BRCM_CC_4387_CHIP_ID, 0xFFFFFFFF, 4387C2), /* revision ID 7 */ + BRCMF_FW_ENTRY(BRCM_CC_4388_CHIP_ID, 0x0000000F, 4388B0), + BRCMF_FW_ENTRY(BRCM_CC_4388_CHIP_ID, 0xFFFFFFF0, 4388C0), /* revision ID 4 */ }; #define BRCMF_PCIE_FW_UP_TIMEOUT 5000 /* msec */ @@ -219,11 +223,64 @@ static const struct brcmf_firmware_mapping brcmf_pcie_fwnames[] = { #define BRCMF_PCIE_SHARED_VERSION_MASK 0x00FF #define BRCMF_PCIE_SHARED_DMA_INDEX 0x10000 #define BRCMF_PCIE_SHARED_DMA_2B_IDX 0x100000 +#define BRCMF_PCIE_SHARED_USE_MAILBOX 0x2000000 +#define BRCMF_PCIE_SHARED_TIMESTAMP_DB0 0x8000000 #define BRCMF_PCIE_SHARED_HOSTRDY_DB1 0x10000000 +#define BRCMF_PCIE_SHARED_NO_OOB_DW 0x20000000 +#define BRCMF_PCIE_SHARED_INBAND_DS 0x40000000 +#define BRCMF_PCIE_SHARED_DAR 0x80000000 + +#define BRCMF_PCIE_SHARED2_EXTENDED_TRAP_DATA 0x1 +#define BRCMF_PCIE_SHARED2_TXSTATUS_METADATA 0x2 +#define BRCMF_PCIE_SHARED2_BT_LOGGING 0x4 +#define BRCMF_PCIE_SHARED2_SNAPSHOT_UPLOAD 0x8 +#define BRCMF_PCIE_SHARED2_SUBMIT_COUNT_WAR 0x10 +#define BRCMF_PCIE_SHARED2_FAST_DELETE_RING 0x20 +#define BRCMF_PCIE_SHARED2_EVTBUF_MAX_MASK 0xC0 +#define BRCMF_PCIE_SHARED2_PKT_TX_STATUS 0x100 +#define BRCMF_PCIE_SHARED2_FW_SMALL_MEMDUMP 0x200 +#define BRCMF_PCIE_SHARED2_FW_HC_ON_TRAP 0x400 +#define BRCMF_PCIE_SHARED2_HSCB 0x800 +#define BRCMF_PCIE_SHARED2_EDL_RING 0x1000 +#define BRCMF_PCIE_SHARED2_DEBUG_BUF_DEST 0x2000 +#define BRCMF_PCIE_SHARED2_PCIE_ENUM_RESET_FLR 0x4000 +#define BRCMF_PCIE_SHARED2_PKT_TIMESTAMP 0x8000 +#define BRCMF_PCIE_SHARED2_HP2P 0x10000 +#define BRCMF_PCIE_SHARED2_HWA 0x20000 +#define BRCMF_PCIE_SHARED2_TRAP_ON_HOST_DB7 0x40000 +#define BRCMF_PCIE_SHARED2_DURATION_SCALE 0x100000 +#define BRCMF_PCIE_SHARED2_D2H_D11_TX_STATUS 0x40000000 +#define BRCMF_PCIE_SHARED2_H2D_D11_TX_STATUS 0x80000000 #define BRCMF_PCIE_FLAGS_HTOD_SPLIT 0x4000 #define BRCMF_PCIE_FLAGS_DTOH_SPLIT 0x8000 +#define BRCMF_HOSTCAP_PCIEAPI_VERSION_MASK 0x000000FF +#define BRCMF_HOSTCAP_H2D_VALID_PHASE 0x00000100 +#define BRCMF_HOSTCAP_H2D_ENABLE_TRAP_ON_BADPHASE 0x00000200 +#define BRCMF_HOSTCAP_H2D_ENABLE_HOSTRDY 0x400 +#define BRCMF_HOSTCAP_DB0_TIMESTAMP 0x800 +#define BRCMF_HOSTCAP_DS_NO_OOB_DW 0x1000 +#define BRCMF_HOSTCAP_DS_INBAND_DW 0x2000 +#define BRCMF_HOSTCAP_H2D_IDMA 0x4000 +#define BRCMF_HOSTCAP_H2D_IFRM 0x8000 +#define BRCMF_HOSTCAP_H2D_DAR 0x10000 +#define BRCMF_HOSTCAP_EXTENDED_TRAP_DATA 0x20000 +#define BRCMF_HOSTCAP_TXSTATUS_METADATA 0x40000 +#define BRCMF_HOSTCAP_BT_LOGGING 0x80000 +#define BRCMF_HOSTCAP_SNAPSHOT_UPLOAD 0x100000 +#define BRCMF_HOSTCAP_FAST_DELETE_RING 0x200000 +#define BRCMF_HOSTCAP_PKT_TXSTATUS 0x400000 +#define BRCMF_HOSTCAP_UR_FW_NO_TRAP 0x800000 +#define BRCMF_HOSTCAP_HSCB 0x2000000 +#define BRCMF_HOSTCAP_EXT_TRAP_DBGBUF 0x4000000 +#define BRCMF_HOSTCAP_EDL_RING 0x10000000 +#define BRCMF_HOSTCAP_PKT_TIMESTAMP 0x20000000 +#define BRCMF_HOSTCAP_PKT_HP2P 0x40000000 +#define BRCMF_HOSTCAP_HWA 0x80000000 +#define BRCMF_HOSTCAP2_DURATION_SCALE_MASK 0x3F + +#define BRCMF_SHARED_FLAGS_OFFSET 0 #define BRCMF_SHARED_MAX_RXBUFPOST_OFFSET 34 #define BRCMF_SHARED_RING_BASE_OFFSET 52 #define BRCMF_SHARED_RX_DATAOFFSET_OFFSET 36 @@ -235,6 +292,11 @@ static const struct brcmf_firmware_mapping brcmf_pcie_fwnames[] = { #define BRCMF_SHARED_DMA_SCRATCH_ADDR_OFFSET 56 #define BRCMF_SHARED_DMA_RINGUPD_LEN_OFFSET 64 #define BRCMF_SHARED_DMA_RINGUPD_ADDR_OFFSET 68 +#define BRCMF_SHARED_FLAGS2_OFFSET 80 +#define BRCMF_SHARED_HOST_CAP_OFFSET 84 +#define BRCMF_SHARED_FLAGS3_OFFSET 108 +#define BRCMF_SHARED_HOST_CAP2_OFFSET 112 +#define BRCMF_SHARED_HOST_CAP3_OFFSET 116 #define BRCMF_RING_H2D_RING_COUNT_OFFSET 0 #define BRCMF_RING_D2H_RING_COUNT_OFFSET 1 @@ -280,6 +342,7 @@ static const struct brcmf_firmware_mapping brcmf_pcie_fwnames[] = { #define BRCMF_PCIE_CFGREG_PML1_SUB_CTRL1 0x248 #define BRCMF_PCIE_CFGREG_REG_BAR2_CONFIG 0x4E0 #define BRCMF_PCIE_CFGREG_REG_BAR3_CONFIG 0x4F4 +#define BRCMF_PCIE_CFGREG_TLCNTRL_5 0x814 #define BRCMF_PCIE_LINK_STATUS_CTRL_ASPM_ENAB 3 /* Magic number at a magic location to find RAM size */ @@ -299,6 +362,8 @@ struct brcmf_pcie_console { struct brcmf_pcie_shared_info { u32 tcm_base_address; u32 flags; + u32 flags2; + u32 flags3; struct brcmf_pcie_ringbuf *commonrings[BRCMF_NROF_COMMON_MSGRINGS]; struct brcmf_pcie_ringbuf *flowrings; u16 max_rxbufpost; @@ -315,6 +380,7 @@ struct brcmf_pcie_shared_info { void *ringupd; dma_addr_t ringupd_dmahandle; u8 version; + bool mb_via_ctl; }; #define BRCMF_OTP_MAX_PARAM_LEN 16 @@ -331,6 +397,7 @@ struct brcmf_pciedev_info { bool in_irq; struct pci_dev *pdev; char fw_name[BRCMF_FW_NAME_LEN]; + char sig_name[BRCMF_FW_NAME_LEN]; char nvram_name[BRCMF_FW_NAME_LEN]; char clm_name[BRCMF_FW_NAME_LEN]; char txcap_name[BRCMF_FW_NAME_LEN]; @@ -339,14 +406,16 @@ struct brcmf_pciedev_info { const struct brcmf_pcie_reginfo *reginfo; void __iomem *regs; void __iomem *tcm; - u32 ram_base; - u32 ram_size; + u32 fw_size; + bool skip_reset_vector; struct brcmf_chip *ci; u32 coreid; struct brcmf_pcie_shared_info shared; wait_queue_head_t mbdata_resp_wait; bool mbdata_completed; bool irq_allocated; + bool irq_ready; + bool have_msi; bool wowl_enabled; u8 dma_idx_sz; void *idxbuf; @@ -433,8 +502,6 @@ struct brcmf_pcie_reginfo { u32 intmask; u32 mailboxint; u32 mailboxmask; - u32 h2d_mailbox_0; - u32 h2d_mailbox_1; u32 int_d2h_db; u32 int_fn0; }; @@ -443,8 +510,6 @@ static const struct brcmf_pcie_reginfo brcmf_reginfo_default = { .intmask = BRCMF_PCIE_PCIE2REG_INTMASK, .mailboxint = BRCMF_PCIE_PCIE2REG_MAILBOXINT, .mailboxmask = BRCMF_PCIE_PCIE2REG_MAILBOXMASK, - .h2d_mailbox_0 = BRCMF_PCIE_PCIE2REG_H2D_MAILBOX_0, - .h2d_mailbox_1 = BRCMF_PCIE_PCIE2REG_H2D_MAILBOX_1, .int_d2h_db = BRCMF_PCIE_MB_INT_D2H_DB, .int_fn0 = BRCMF_PCIE_MB_INT_FN0, }; @@ -453,8 +518,6 @@ static const struct brcmf_pcie_reginfo brcmf_reginfo_64 = { .intmask = BRCMF_PCIE_64_PCIE2REG_INTMASK, .mailboxint = BRCMF_PCIE_64_PCIE2REG_MAILBOXINT, .mailboxmask = BRCMF_PCIE_64_PCIE2REG_MAILBOXMASK, - .h2d_mailbox_0 = BRCMF_PCIE_64_PCIE2REG_H2D_MAILBOX_0, - .h2d_mailbox_1 = BRCMF_PCIE_64_PCIE2REG_H2D_MAILBOX_1, .int_d2h_db = BRCMF_PCIE_64_MB_INT_D2H_DB, .int_fn0 = 0, }; @@ -493,6 +556,19 @@ brcmf_pcie_write_reg32(struct brcmf_pciedev_info *devinfo, u32 reg_offset, iowrite32(value, address); } +static u32 +brcmf_pcie_read_pcie32(struct brcmf_pciedev_info *devinfo, u32 reg_offset) +{ + return brcmf_pcie_read_reg32(devinfo, 0x2000 + reg_offset); +} + + +static void +brcmf_pcie_write_pcie32(struct brcmf_pciedev_info *devinfo, u32 reg_offset, + u32 value) +{ + brcmf_pcie_write_reg32(devinfo, 0x2000 + reg_offset, value); +} static u8 brcmf_pcie_read_tcm8(struct brcmf_pciedev_info *devinfo, u32 mem_offset) @@ -686,8 +762,30 @@ static void brcmf_pcie_reset_device(struct brcmf_pciedev_info *devinfo) /* Watchdog reset */ brcmf_pcie_select_core(devinfo, BCMA_CORE_CHIPCOMMON); - WRITECC32(devinfo, watchdog, 4); - msleep(100); + core = brcmf_chip_get_chipcommon(devinfo->ci); + + if (core->rev >= 65) { + u32 mask = CC_WD_SSRESET_PCIE_F0_EN; + + core = brcmf_chip_get_core(devinfo->ci, BCMA_CORE_PCIE2); + if (core->rev < 66) + mask |= CC_WD_SSRESET_PCIE_ALL_FN_EN; + + val = READCC32(devinfo, watchdog); + val &= ~CC_WD_ENABLE_MASK; + val |= mask; + WRITECC32(devinfo, watchdog, val); + val &= ~CC_WD_COUNTER_MASK; + val |= 4; + WRITECC32(devinfo, watchdog, val); + msleep(10); + val = READCC32(devinfo, intstatus); + val |= mask; + WRITECC32(devinfo, intstatus, val); + } else { + WRITECC32(devinfo, watchdog, 4); + msleep(100); + } /* Restore ASPM */ brcmf_pcie_select_core(devinfo, BCMA_CORE_PCIE2); @@ -697,14 +795,14 @@ static void brcmf_pcie_reset_device(struct brcmf_pciedev_info *devinfo) core = brcmf_chip_get_core(devinfo->ci, BCMA_CORE_PCIE2); if (core->rev <= 13) { for (i = 0; i < ARRAY_SIZE(cfg_offset); i++) { - brcmf_pcie_write_reg32(devinfo, + brcmf_pcie_write_pcie32(devinfo, BRCMF_PCIE_PCIE2REG_CONFIGADDR, cfg_offset[i]); - val = brcmf_pcie_read_reg32(devinfo, + val = brcmf_pcie_read_pcie32(devinfo, BRCMF_PCIE_PCIE2REG_CONFIGDATA); brcmf_dbg(PCIE, "config offset 0x%04x, value 0x%04x\n", cfg_offset[i], val); - brcmf_pcie_write_reg32(devinfo, + brcmf_pcie_write_pcie32(devinfo, BRCMF_PCIE_PCIE2REG_CONFIGDATA, val); } @@ -718,9 +816,9 @@ static void brcmf_pcie_attach(struct brcmf_pciedev_info *devinfo) /* BAR1 window may not be sized properly */ brcmf_pcie_select_core(devinfo, BCMA_CORE_PCIE2); - brcmf_pcie_write_reg32(devinfo, BRCMF_PCIE_PCIE2REG_CONFIGADDR, 0x4e0); - config = brcmf_pcie_read_reg32(devinfo, BRCMF_PCIE_PCIE2REG_CONFIGDATA); - brcmf_pcie_write_reg32(devinfo, BRCMF_PCIE_PCIE2REG_CONFIGDATA, config); + brcmf_pcie_write_pcie32(devinfo, BRCMF_PCIE_PCIE2REG_CONFIGADDR, 0x4e0); + config = brcmf_pcie_read_pcie32(devinfo, BRCMF_PCIE_PCIE2REG_CONFIGDATA); + brcmf_pcie_write_pcie32(devinfo, BRCMF_PCIE_PCIE2REG_CONFIGDATA, config); device_wakeup_enable(&devinfo->pdev->dev); } @@ -739,6 +837,21 @@ static int brcmf_pcie_enter_download_state(struct brcmf_pciedev_info *devinfo) brcmf_pcie_write_reg32(devinfo, BRCMF_PCIE_ARMCR4REG_BANKPDA, 0); } + + /* Ensure all IRQs are masked so the firmware doesn't get + * a hostready notification too early. + */ + + brcmf_pcie_write_pcie32(devinfo, devinfo->reginfo->mailboxmask, 0); + brcmf_pcie_write_pcie32(devinfo, devinfo->reginfo->mailboxint, + 0xffffffff); + + pci_write_config_dword(devinfo->pdev, BRCMF_PCIE_REG_INTMASK, 0); + + brcmf_pcie_write_pcie32(devinfo, BRCMF_PCIE_PCIE2REG_CONFIGADDR, + BRCMF_PCIE_CFGREG_TLCNTRL_5); + brcmf_pcie_write_pcie32(devinfo, BRCMF_PCIE_PCIE2REG_CONFIGDATA, + 0xffffffff); return 0; } @@ -769,6 +882,19 @@ brcmf_pcie_send_mb_data(struct brcmf_pciedev_info *devinfo, u32 htod_mb_data) u32 i; shared = &devinfo->shared; + + if (shared->mb_via_ctl) { + struct pci_dev *pdev = devinfo->pdev; + struct brcmf_bus *bus = dev_get_drvdata(&pdev->dev); + int ret; + + ret = brcmf_msgbuf_h2d_mb_write(bus->drvr, htod_mb_data); + if (ret < 0) + brcmf_err(bus, "Failed to send H2D mailbox data (%d)\n", + ret); + return ret; + } + addr = shared->htod_mb_data_addr; cur_htod_mb_data = brcmf_pcie_read_tcm32(devinfo, addr); @@ -796,8 +922,29 @@ brcmf_pcie_send_mb_data(struct brcmf_pciedev_info *devinfo, u32 htod_mb_data) return 0; } +static void brcmf_pcie_handle_mb_data(struct brcmf_pciedev_info *devinfo, u32 data) +{ + brcmf_dbg(PCIE, "D2H_MB_DATA: 0x%04x\n", data); + if (data & BRCMF_D2H_DEV_DS_ENTER_REQ) { + brcmf_dbg(PCIE, "D2H_MB_DATA: DEEP SLEEP REQ\n"); + brcmf_pcie_send_mb_data(devinfo, BRCMF_H2D_HOST_DS_ACK); + brcmf_dbg(PCIE, "D2H_MB_DATA: sent DEEP SLEEP ACK\n"); + } + if (data & BRCMF_D2H_DEV_DS_EXIT_NOTE) + brcmf_dbg(PCIE, "D2H_MB_DATA: DEEP SLEEP EXIT\n"); + if (data & BRCMF_D2H_DEV_D3_ACK) { + brcmf_dbg(PCIE, "D2H_MB_DATA: D3 ACK\n"); + devinfo->mbdata_completed = true; + wake_up(&devinfo->mbdata_resp_wait); + } + if (data & BRCMF_D2H_DEV_FWHALT) { + brcmf_dbg(PCIE, "D2H_MB_DATA: FW HALT\n"); + brcmf_fw_crashed(&devinfo->pdev->dev); + } +} -static void brcmf_pcie_handle_mb_data(struct brcmf_pciedev_info *devinfo) + +static void brcmf_pcie_poll_mb_data(struct brcmf_pciedev_info *devinfo) { struct brcmf_pcie_shared_info *shared; u32 addr; @@ -812,23 +959,16 @@ static void brcmf_pcie_handle_mb_data(struct brcmf_pciedev_info *devinfo) brcmf_pcie_write_tcm32(devinfo, addr, 0); - brcmf_dbg(PCIE, "D2H_MB_DATA: 0x%04x\n", dtoh_mb_data); - if (dtoh_mb_data & BRCMF_D2H_DEV_DS_ENTER_REQ) { - brcmf_dbg(PCIE, "D2H_MB_DATA: DEEP SLEEP REQ\n"); - brcmf_pcie_send_mb_data(devinfo, BRCMF_H2D_HOST_DS_ACK); - brcmf_dbg(PCIE, "D2H_MB_DATA: sent DEEP SLEEP ACK\n"); - } - if (dtoh_mb_data & BRCMF_D2H_DEV_DS_EXIT_NOTE) - brcmf_dbg(PCIE, "D2H_MB_DATA: DEEP SLEEP EXIT\n"); - if (dtoh_mb_data & BRCMF_D2H_DEV_D3_ACK) { - brcmf_dbg(PCIE, "D2H_MB_DATA: D3 ACK\n"); - devinfo->mbdata_completed = true; - wake_up(&devinfo->mbdata_resp_wait); - } - if (dtoh_mb_data & BRCMF_D2H_DEV_FWHALT) { - brcmf_dbg(PCIE, "D2H_MB_DATA: FW HALT\n"); - brcmf_fw_crashed(&devinfo->pdev->dev); - } + brcmf_pcie_handle_mb_data(devinfo, dtoh_mb_data); +} + + +static void brcmf_pcie_d2h_mb_rx(struct device *dev, u32 data) +{ + struct brcmf_bus *bus_if = dev_get_drvdata(dev); + struct brcmf_pciedev *buspub = bus_if->bus_priv.pcie; + + brcmf_pcie_handle_mb_data(buspub->devinfo, data); } @@ -907,33 +1047,45 @@ static void brcmf_pcie_bus_console_read(struct brcmf_pciedev_info *devinfo, static void brcmf_pcie_intr_disable(struct brcmf_pciedev_info *devinfo) { - brcmf_pcie_write_reg32(devinfo, devinfo->reginfo->mailboxmask, 0); + brcmf_pcie_write_pcie32(devinfo, devinfo->reginfo->mailboxmask, 0); + + devinfo->irq_ready = false; } static void brcmf_pcie_intr_enable(struct brcmf_pciedev_info *devinfo) { - brcmf_pcie_write_reg32(devinfo, devinfo->reginfo->mailboxmask, + brcmf_pcie_write_pcie32(devinfo, devinfo->reginfo->mailboxmask, devinfo->reginfo->int_d2h_db | devinfo->reginfo->int_fn0); + + devinfo->irq_ready = true; } static void brcmf_pcie_hostready(struct brcmf_pciedev_info *devinfo) { - if (devinfo->shared.flags & BRCMF_PCIE_SHARED_HOSTRDY_DB1) - brcmf_pcie_write_reg32(devinfo, - devinfo->reginfo->h2d_mailbox_1, 1); + if (devinfo->shared.flags & BRCMF_PCIE_SHARED_HOSTRDY_DB1) { + if (devinfo->shared.flags & BRCMF_PCIE_SHARED_DAR) + brcmf_pcie_write_pcie32(devinfo, BRCMF_PCIE_64_PCIE2REG_H2D_MAILBOX_1, 1); + else + brcmf_pcie_write_pcie32(devinfo, BRCMF_PCIE_PCIE2REG_H2D_MAILBOX_1, 1); + } } static irqreturn_t brcmf_pcie_quick_check_isr(int irq, void *arg) { struct brcmf_pciedev_info *devinfo = (struct brcmf_pciedev_info *)arg; - if (brcmf_pcie_read_reg32(devinfo, devinfo->reginfo->mailboxint)) { + if (brcmf_pcie_read_pcie32(devinfo, devinfo->reginfo->mailboxint)) { brcmf_pcie_intr_disable(devinfo); brcmf_dbg(PCIE, "Enter\n"); return IRQ_WAKE_THREAD; } + + /* mailboxint is cleared by the firmware in MSI mode */ + if (devinfo->have_msi) + return IRQ_WAKE_THREAD; + return IRQ_NONE; } @@ -944,19 +1096,19 @@ static irqreturn_t brcmf_pcie_isr_thread(int irq, void *arg) u32 status; devinfo->in_irq = true; - status = brcmf_pcie_read_reg32(devinfo, devinfo->reginfo->mailboxint); + status = brcmf_pcie_read_pcie32(devinfo, devinfo->reginfo->mailboxint); brcmf_dbg(PCIE, "Enter %x\n", status); if (status) { - brcmf_pcie_write_reg32(devinfo, devinfo->reginfo->mailboxint, + brcmf_pcie_write_pcie32(devinfo, devinfo->reginfo->mailboxint, status); if (status & devinfo->reginfo->int_fn0) - brcmf_pcie_handle_mb_data(devinfo); - if (status & devinfo->reginfo->int_d2h_db) { - if (devinfo->state == BRCMFMAC_PCIE_STATE_UP) - brcmf_proto_msgbuf_rx_trigger( - &devinfo->pdev->dev); - } + brcmf_pcie_poll_mb_data(devinfo); } + if (devinfo->have_msi || status & devinfo->reginfo->int_d2h_db) { + if (devinfo->state == BRCMFMAC_PCIE_STATE_UP && devinfo->irq_ready) + brcmf_proto_msgbuf_rx_trigger(&devinfo->pdev->dev); + } + brcmf_pcie_bus_console_read(devinfo, false); if (devinfo->state == BRCMFMAC_PCIE_STATE_UP) brcmf_pcie_intr_enable(devinfo); @@ -974,7 +1126,10 @@ static int brcmf_pcie_request_irq(struct brcmf_pciedev_info *devinfo) brcmf_dbg(PCIE, "Enter\n"); - pci_enable_msi(pdev); + devinfo->have_msi = pci_enable_msi(pdev) >= 0; + if (devinfo->have_msi) + brcmf_dbg(PCIE, "MSI enabled\n"); + if (request_threaded_irq(pdev->irq, brcmf_pcie_quick_check_isr, brcmf_pcie_isr_thread, IRQF_SHARED, "brcmf_pcie_intr", devinfo)) { @@ -1010,8 +1165,8 @@ static void brcmf_pcie_release_irq(struct brcmf_pciedev_info *devinfo) if (devinfo->in_irq) brcmf_err(bus, "Still in IRQ (processing) !!!\n"); - status = brcmf_pcie_read_reg32(devinfo, devinfo->reginfo->mailboxint); - brcmf_pcie_write_reg32(devinfo, devinfo->reginfo->mailboxint, status); + status = brcmf_pcie_read_pcie32(devinfo, devinfo->reginfo->mailboxint); + brcmf_pcie_write_pcie32(devinfo, devinfo->reginfo->mailboxint, status); devinfo->irq_allocated = false; } @@ -1063,7 +1218,10 @@ static int brcmf_pcie_ring_mb_ring_bell(void *ctx) brcmf_dbg(PCIE, "RING !\n"); /* Any arbitrary value will do, lets use 1 */ - brcmf_pcie_write_reg32(devinfo, devinfo->reginfo->h2d_mailbox_0, 1); + if (devinfo->shared.flags & BRCMF_PCIE_SHARED_DAR) + brcmf_pcie_write_pcie32(devinfo, BRCMF_PCIE_64_PCIE2REG_H2D_MAILBOX_0, 1); + else + brcmf_pcie_write_pcie32(devinfo, BRCMF_PCIE_PCIE2REG_H2D_MAILBOX_0, 1); return 0; } @@ -1589,6 +1747,7 @@ static const struct brcmf_bus_ops brcmf_pcie_bus_ops = { .get_blob = brcmf_pcie_get_blob, .reset = brcmf_pcie_reset, .debugfs_create = brcmf_pcie_debugfs_create, + .d2h_mb_rx = brcmf_pcie_d2h_mb_rx, }; @@ -1620,12 +1779,16 @@ brcmf_pcie_init_share_ram_info(struct brcmf_pciedev_info *devinfo, { struct brcmf_bus *bus = dev_get_drvdata(&devinfo->pdev->dev); struct brcmf_pcie_shared_info *shared; + u32 host_cap; + u32 host_cap2; u32 addr; shared = &devinfo->shared; shared->tcm_base_address = sharedram_addr; - shared->flags = brcmf_pcie_read_tcm32(devinfo, sharedram_addr); + shared->flags = brcmf_pcie_read_tcm32(devinfo, sharedram_addr + + BRCMF_SHARED_FLAGS_OFFSET); + shared->version = (u8)(shared->flags & BRCMF_PCIE_SHARED_VERSION_MASK); brcmf_dbg(PCIE, "PCIe protocol version %d\n", shared->version); if ((shared->version > BRCMF_PCIE_MAX_SHARED_VERSION) || @@ -1666,29 +1829,223 @@ brcmf_pcie_init_share_ram_info(struct brcmf_pciedev_info *devinfo, brcmf_pcie_bus_console_init(devinfo); brcmf_pcie_bus_console_read(devinfo, false); + /* Features added in revision 6 follow */ + if (shared->version < 6) + return 0; + + shared->flags2 = brcmf_pcie_read_tcm32(devinfo, sharedram_addr + + BRCMF_SHARED_FLAGS2_OFFSET); + shared->flags3 = brcmf_pcie_read_tcm32(devinfo, sharedram_addr + + BRCMF_SHARED_FLAGS3_OFFSET); + + /* Check which mailbox mechanism to use */ + if (!(shared->flags & BRCMF_PCIE_SHARED_USE_MAILBOX)) + shared->mb_via_ctl = true; + + /* Update host support flags */ + host_cap = shared->version; + host_cap2 = 0; + + if (shared->flags & BRCMF_PCIE_SHARED_HOSTRDY_DB1) + host_cap |= BRCMF_HOSTCAP_H2D_ENABLE_HOSTRDY; + + if (shared->flags & BRCMF_PCIE_SHARED_DAR) + host_cap |= BRCMF_HOSTCAP_H2D_DAR; + + /* Disable DS: this is not currently properly supported */ + host_cap |= BRCMF_HOSTCAP_DS_NO_OOB_DW; + + brcmf_pcie_write_tcm32(devinfo, sharedram_addr + + BRCMF_SHARED_HOST_CAP_OFFSET, host_cap); + brcmf_pcie_write_tcm32(devinfo, sharedram_addr + + BRCMF_SHARED_HOST_CAP2_OFFSET, host_cap2); + return 0; } -struct brcmf_random_seed_footer { +struct brcmf_rtlv_footer { __le32 length; __le32 magic; }; +/** struct brcmf_fw_memmap_region - start/end of memory regions for chip + */ +struct brcmf_fw_memmap_region { + u32 start; + u32 end; +}; + +/** struct brcmf_fw_memmap + * + * @reset_vec - Reset vector - read only + * @int_vec - copied from ram, jumps here on success + * @rom - bootloader at rom start + * @mmap - struct/memory map written by host + * @vstatus - verification status + * @fw - firmware + * @sig - firwmare signature + * @heap - region for heap allocations + * @stack - region for stack allocations + * @prng - PRNG data, may be 0 length + * @nvram - NVRAM data + */ +struct brcmf_fw_memmap { + struct brcmf_fw_memmap_region reset_vec; + struct brcmf_fw_memmap_region int_vec; + struct brcmf_fw_memmap_region rom; + struct brcmf_fw_memmap_region mmap; + struct brcmf_fw_memmap_region vstatus; + struct brcmf_fw_memmap_region fw; + struct brcmf_fw_memmap_region sig; + struct brcmf_fw_memmap_region heap; + struct brcmf_fw_memmap_region stack; + struct brcmf_fw_memmap_region prng; + struct brcmf_fw_memmap_region nvram; +}; + +#define BRCMF_BL_HEAP_START_GAP 0x1000 +#define BRCMF_BL_HEAP_SIZE 0x10000 #define BRCMF_RANDOM_SEED_MAGIC 0xfeedc0de #define BRCMF_RANDOM_SEED_LENGTH 0x100 +#define BRCMF_FW_SIG_MAGIC 0xfeedfe51 +#define BRCMF_NVRAM_SIG_MAGIC 0xfeedfe52 +#define BRCMF_MEMMAP_MAGIC 0xfeedfe53 +#define BRCMF_VSTATUS_MAGIC 0xfeedfe54 +#define BRCMF_VSTATUS_SIZE 0x28 +#define BRCMF_END_MAGIC 0xfeed0e2d -static noinline_for_stack void -brcmf_pcie_provide_random_bytes(struct brcmf_pciedev_info *devinfo, u32 address) +static int brcmf_alloc_rtlv(struct brcmf_pciedev_info *devinfo, u32 *address, u32 type, u32 length) { + struct brcmf_bus *bus = dev_get_drvdata(&devinfo->pdev->dev); + u32 fw_top = devinfo->ci->rambase + devinfo->fw_size; + u32 ram_start = ALIGN(fw_top + BRCMF_BL_HEAP_START_GAP, 4); + u32 ram_end = ram_start + BRCMF_BL_HEAP_SIZE; + u32 start_addr; + struct brcmf_rtlv_footer footer = { + .magic = type, + }; + + length = ALIGN(length, 4); + start_addr = *address - length - sizeof(struct brcmf_rtlv_footer); + + if (length > 0xffff || start_addr > *address || start_addr < ram_end) { + brcmf_err(bus, "failed to allocate 0x%x bytes for rTLV type 0x%x\n", + length, type); + return -ENOMEM; + } + + /* Random seed does not use the length check code */ + if (type == BRCMF_RANDOM_SEED_MAGIC) + footer.length = length; + else + footer.length = length | ((length ^ 0xffff) << 16); + + memcpy_toio(devinfo->tcm + *address - sizeof(struct brcmf_rtlv_footer), + &footer, sizeof(struct brcmf_rtlv_footer)); + + *address = start_addr; + + return 0; +} + +static noinline_for_stack int +brcmf_pcie_add_random_seed(struct brcmf_pciedev_info *devinfo, u32 *address) +{ + int err; u8 randbuf[BRCMF_RANDOM_SEED_LENGTH]; + err = brcmf_alloc_rtlv(devinfo, address, + BRCMF_RANDOM_SEED_MAGIC, BRCMF_RANDOM_SEED_LENGTH); + if (err) + return err; + + /* Some Apple chips/firmwares expect a buffer of random + * data to be present before NVRAM + */ + brcmf_dbg(PCIE, "Download random seed\n"); + get_random_bytes(randbuf, BRCMF_RANDOM_SEED_LENGTH); - memcpy_toio(devinfo->tcm + address, randbuf, BRCMF_RANDOM_SEED_LENGTH); + memcpy_toio(devinfo->tcm + *address, randbuf, BRCMF_RANDOM_SEED_LENGTH); + + return 0; +} + +static int brcmf_pcie_add_signature(struct brcmf_pciedev_info *devinfo, + u32 *address, const struct firmware *fwsig) +{ + int err; + struct brcmf_fw_memmap memmap; + + brcmf_dbg(PCIE, "Download firmware signature\n"); + + memset(&memmap, 0, sizeof(memmap)); + + memmap.sig.end = *address; + err = brcmf_alloc_rtlv(devinfo, address, BRCMF_FW_SIG_MAGIC, fwsig->size); + if (err) + return err; + memmap.sig.start = *address; + + memmap.vstatus.end = *address; + err = brcmf_alloc_rtlv(devinfo, address, BRCMF_VSTATUS_MAGIC, BRCMF_VSTATUS_SIZE); + if (err) + return err; + memmap.vstatus.start = *address; + + err = brcmf_alloc_rtlv(devinfo, address, BRCMF_MEMMAP_MAGIC, sizeof(memmap)); + if (err) + return err; + + memmap.fw.start = devinfo->ci->rambase; + memmap.fw.end = memmap.fw.start + devinfo->fw_size; + memmap.heap.start = ALIGN(memmap.fw.end + BRCMF_BL_HEAP_START_GAP, 4); + memmap.heap.end = memmap.heap.start + BRCMF_BL_HEAP_SIZE; + + if (memmap.heap.end > *address) + return -ENOMEM; + + memcpy_toio(devinfo->tcm + memmap.sig.start, fwsig->data, fwsig->size); + memset_io(devinfo->tcm + memmap.vstatus.start, 0, BRCMF_VSTATUS_SIZE); + memcpy_toio(devinfo->tcm + *address, &memmap, sizeof(memmap)); + + err = brcmf_alloc_rtlv(devinfo, address, BRCMF_END_MAGIC, 0); + if (err) + return err; + + devinfo->skip_reset_vector = true; + + return 0; +} + +static int brcmf_pcie_populate_footers(struct brcmf_pciedev_info *devinfo, + u32 *address, const struct firmware *fwsig) +{ + int err; + + /* We only do this for Apple firmwares. If any other + * production firmwares are found to need this, the condition + * needs to be adjusted. + */ + if (!devinfo->fwseed) + return 0; + + err = brcmf_pcie_add_random_seed(devinfo, address); + if (err) + return err; + + if (fwsig) { + err = brcmf_pcie_add_signature(devinfo, address, fwsig); + if (err) + return err; + } + + return 0; } static int brcmf_pcie_download_fw_nvram(struct brcmf_pciedev_info *devinfo, - const struct firmware *fw, void *nvram, - u32 nvram_len) + const struct firmware *fw, + const struct firmware *fwsig, + void *nvram, u32 nvram_len) { struct brcmf_bus *bus = dev_get_drvdata(&devinfo->pdev->dev); u32 sharedram_addr; @@ -1708,6 +2065,7 @@ static int brcmf_pcie_download_fw_nvram(struct brcmf_pciedev_info *devinfo, (void *)fw->data, fw->size); resetintr = get_unaligned_le32(fw->data); + devinfo->fw_size = fw->size; release_firmware(fw); /* reset last 4 bytes of RAM address. to be used for shared @@ -1715,37 +2073,31 @@ static int brcmf_pcie_download_fw_nvram(struct brcmf_pciedev_info *devinfo, */ brcmf_pcie_write_ram32(devinfo, devinfo->ci->ramsize - 4, 0); + address = devinfo->ci->rambase + devinfo->ci->ramsize; + if (nvram) { brcmf_dbg(PCIE, "Download NVRAM %s\n", devinfo->nvram_name); - address = devinfo->ci->rambase + devinfo->ci->ramsize - - nvram_len; + address -= nvram_len; memcpy_toio(devinfo->tcm + address, nvram, nvram_len); brcmf_fw_nvram_free(nvram); - if (devinfo->fwseed) { - size_t rand_len = BRCMF_RANDOM_SEED_LENGTH; - struct brcmf_random_seed_footer footer = { - .length = cpu_to_le32(rand_len), - .magic = cpu_to_le32(BRCMF_RANDOM_SEED_MAGIC), - }; - - /* Some chips/firmwares expect a buffer of random - * data to be present before NVRAM - */ - brcmf_dbg(PCIE, "Download random seed\n"); - - address -= sizeof(footer); - memcpy_toio(devinfo->tcm + address, &footer, - sizeof(footer)); - - address -= rand_len; - brcmf_pcie_provide_random_bytes(devinfo, address); - } + err = brcmf_pcie_populate_footers(devinfo, &address, fwsig); + if (err) + brcmf_err(bus, "failed to populate firmware footers err=%d\n", err); } else { brcmf_dbg(PCIE, "No matching NVRAM file found %s\n", devinfo->nvram_name); } + release_firmware(fwsig); + + /* Clear free TCM. This isn't really necessary, but it + * makes debugging memory dumps a lot easier since we + * don't get a bunch of junk filling up the free space. + */ + memset_io(devinfo->tcm + devinfo->ci->rambase + devinfo->fw_size, + 0, address - devinfo->fw_size - devinfo->ci->rambase); + sharedram_addr_written = brcmf_pcie_read_ram32(devinfo, devinfo->ci->ramsize - 4); @@ -1889,9 +2241,9 @@ static int brcmf_pcie_buscore_reset(void *ctx, struct brcmf_chip *chip) else reg = BRCMF_PCIE_PCIE2REG_MAILBOXINT; - val = brcmf_pcie_read_reg32(devinfo, reg); + val = brcmf_pcie_read_pcie32(devinfo, reg); if (val != 0xffffffff) - brcmf_pcie_write_reg32(devinfo, reg, val); + brcmf_pcie_write_pcie32(devinfo, reg, val); return 0; } @@ -1902,7 +2254,8 @@ static void brcmf_pcie_buscore_activate(void *ctx, struct brcmf_chip *chip, { struct brcmf_pciedev_info *devinfo = (struct brcmf_pciedev_info *)ctx; - brcmf_pcie_write_tcm32(devinfo, 0, rstvec); + if (!devinfo->skip_reset_vector) + brcmf_pcie_write_tcm32(devinfo, 0, rstvec); } @@ -2073,6 +2426,11 @@ static int brcmf_pcie_read_otp(struct brcmf_pciedev_info *devinfo) base = 0x113c; words = 0x170; break; + case BRCM_CC_4388_CHIP_ID: + coreid = BCMA_CORE_GCI; + base = 0x115c; + words = 0x150; + break; default: /* OTP not supported on this chip */ return 0; @@ -2131,11 +2489,12 @@ static int brcmf_pcie_read_otp(struct brcmf_pciedev_info *devinfo) #define BRCMF_PCIE_FW_NVRAM 1 #define BRCMF_PCIE_FW_CLM 2 #define BRCMF_PCIE_FW_TXCAP 3 +#define BRCMF_PCIE_FW_SIG 4 static void brcmf_pcie_setup(struct device *dev, int ret, struct brcmf_fw_request *fwreq) { - const struct firmware *fw; + const struct firmware *fw, *fwsig; void *nvram; struct brcmf_bus *bus; struct brcmf_pciedev *pcie_bus_dev; @@ -2154,6 +2513,7 @@ static void brcmf_pcie_setup(struct device *dev, int ret, brcmf_pcie_attach(devinfo); fw = fwreq->items[BRCMF_PCIE_FW_CODE].binary; + fwsig = fwreq->items[BRCMF_PCIE_FW_SIG].binary; nvram = fwreq->items[BRCMF_PCIE_FW_NVRAM].nv_data.data; nvram_len = fwreq->items[BRCMF_PCIE_FW_NVRAM].nv_data.len; devinfo->clm_fw = fwreq->items[BRCMF_PCIE_FW_CLM].binary; @@ -2164,6 +2524,7 @@ static void brcmf_pcie_setup(struct device *dev, int ret, if (ret) { brcmf_err(bus, "Failed to get RAM info\n"); release_firmware(fw); + release_firmware(fwsig); brcmf_fw_nvram_free(nvram); goto fail; } @@ -2175,7 +2536,15 @@ static void brcmf_pcie_setup(struct device *dev, int ret, */ brcmf_pcie_adjust_ramsize(devinfo, (u8 *)fw->data, fw->size); - ret = brcmf_pcie_download_fw_nvram(devinfo, fw, nvram, nvram_len); + /* Newer firmwares will signal firmware boot via MSI, so make sure we + * initialize that upfront. + */ + brcmf_pcie_select_core(devinfo, BCMA_CORE_PCIE2); + ret = brcmf_pcie_request_irq(devinfo); + if (ret) + goto fail; + + ret = brcmf_pcie_download_fw_nvram(devinfo, fw, fwsig, nvram, nvram_len); if (ret) goto fail; @@ -2190,9 +2559,6 @@ static void brcmf_pcie_setup(struct device *dev, int ret, goto fail; brcmf_pcie_select_core(devinfo, BCMA_CORE_PCIE2); - ret = brcmf_pcie_request_irq(devinfo); - if (ret) - goto fail; /* hook the commonrings in the bus structure. */ for (i = 0; i < BRCMF_NROF_COMMON_MSGRINGS; i++) @@ -2240,6 +2606,7 @@ brcmf_pcie_prepare_fw_request(struct brcmf_pciedev_info *devinfo) { ".txt", devinfo->nvram_name }, { ".clm_blob", devinfo->clm_name }, { ".txcap_blob", devinfo->txcap_name }, + { ".sig", devinfo->sig_name }, }; fwreq = brcmf_fw_alloc_request(devinfo->ci->chip, devinfo->ci->chiprev, @@ -2250,6 +2617,8 @@ brcmf_pcie_prepare_fw_request(struct brcmf_pciedev_info *devinfo) return NULL; fwreq->items[BRCMF_PCIE_FW_CODE].type = BRCMF_FW_TYPE_BINARY; + fwreq->items[BRCMF_PCIE_FW_SIG].type = BRCMF_FW_TYPE_BINARY; + fwreq->items[BRCMF_PCIE_FW_SIG].flags = BRCMF_FW_REQF_OPTIONAL; fwreq->items[BRCMF_PCIE_FW_NVRAM].type = BRCMF_FW_TYPE_NVRAM; fwreq->items[BRCMF_PCIE_FW_NVRAM].flags = BRCMF_FW_REQF_OPTIONAL; fwreq->items[BRCMF_PCIE_FW_CLM].type = BRCMF_FW_TYPE_BINARY; @@ -2668,12 +3037,13 @@ static int brcmf_pcie_pm_leave_D3(struct device *dev) brcmf_dbg(PCIE, "Enter, dev=%p, bus=%p\n", dev, bus); /* Check if device is still up and running, if so we are ready */ - if (brcmf_pcie_read_reg32(devinfo, devinfo->reginfo->intmask) != 0) { + if (brcmf_pcie_read_pcie32(devinfo, devinfo->reginfo->intmask) != 0) { brcmf_dbg(PCIE, "Try to wakeup device....\n"); + /* Set the device up, so we can write the MB data message in ring mode */ + devinfo->state = BRCMFMAC_PCIE_STATE_UP; if (brcmf_pcie_send_mb_data(devinfo, BRCMF_H2D_HOST_D0_INFORM)) goto cleanup; brcmf_dbg(PCIE, "Hot resume, continue....\n"); - devinfo->state = BRCMFMAC_PCIE_STATE_UP; brcmf_pcie_select_core(devinfo, BCMA_CORE_PCIE2); brcmf_bus_change_state(bus, BRCMF_BUS_UP); brcmf_pcie_intr_enable(devinfo); @@ -2683,6 +3053,7 @@ static int brcmf_pcie_pm_leave_D3(struct device *dev) } cleanup: + devinfo->state = BRCMFMAC_PCIE_STATE_DOWN; brcmf_chip_detach(devinfo->ci); devinfo->ci = NULL; pdev = devinfo->pdev; @@ -2750,6 +3121,7 @@ static const struct pci_device_id brcmf_pcie_devid_table[] = { BRCMF_PCIE_DEVICE(BRCM_PCIE_4377_DEVICE_ID, WCC_SEED), BRCMF_PCIE_DEVICE(BRCM_PCIE_4378_DEVICE_ID, WCC_SEED), BRCMF_PCIE_DEVICE(BRCM_PCIE_4387_DEVICE_ID, WCC_SEED), + BRCMF_PCIE_DEVICE(BRCM_PCIE_4388_DEVICE_ID, WCC_SEED), BRCMF_PCIE_DEVICE(BRCM_PCIE_43752_DEVICE_ID, WCC_SEED), BRCMF_PCIE_DEVICE(CY_PCIE_54591_DEVICE_ID, CYW), { /* end: all zeroes */ } diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pno.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pno.c index 05f66ab13bed6d..dbeeaef75b165a 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pno.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pno.c @@ -12,8 +12,10 @@ #include "fwil_types.h" #include "cfg80211.h" #include "pno.h" +#include "feature.h" -#define BRCMF_PNO_VERSION 2 +#define BRCMF_PNO_VERSION_2 2 +#define BRCMF_PNO_VERSION_3 3 #define BRCMF_PNO_REPEAT 4 #define BRCMF_PNO_FREQ_EXPO_MAX 3 #define BRCMF_PNO_IMMEDIATE_SCAN_BIT 3 @@ -99,8 +101,62 @@ static int brcmf_pno_channel_config(struct brcmf_if *ifp, return brcmf_fil_iovar_data_set(ifp, "pfn_cfg", cfg, sizeof(*cfg)); } -static int brcmf_pno_config(struct brcmf_if *ifp, u32 scan_freq, - u32 mscan, u32 bestn) +static int brcmf_pno_config_v3(struct brcmf_if *ifp, u32 scan_freq, u32 mscan, + u32 bestn) +{ + struct brcmf_pub *drvr = ifp->drvr; + struct brcmf_pno_param_v3_le pfn_param; + u16 flags; + u32 pfnmem; + s32 err; + + memset(&pfn_param, 0, sizeof(pfn_param)); + pfn_param.version = cpu_to_le16(BRCMF_PNO_VERSION_3); + pfn_param.length = cpu_to_le16(sizeof(struct brcmf_pno_param_v3_le)); + + /* set extra pno params */ + flags = BIT(BRCMF_PNO_IMMEDIATE_SCAN_BIT) | + BIT(BRCMF_PNO_ENABLE_ADAPTSCAN_BIT); + pfn_param.repeat = BRCMF_PNO_REPEAT; + pfn_param.exp = BRCMF_PNO_FREQ_EXPO_MAX; + + /* set up pno scan fr */ + pfn_param.scan_freq = cpu_to_le32(scan_freq); + + if (mscan) { + pfnmem = bestn; + + /* set bestn in firmware */ + err = brcmf_fil_iovar_int_set(ifp, "pfnmem", pfnmem); + if (err < 0) { + bphy_err(drvr, "failed to set pfnmem\n"); + goto exit; + } + /* get max mscan which the firmware supports */ + err = brcmf_fil_iovar_int_get(ifp, "pfnmem", &pfnmem); + if (err < 0) { + bphy_err(drvr, "failed to get pfnmem\n"); + goto exit; + } + mscan = min_t(u32, mscan, pfnmem); + pfn_param.mscan = mscan; + pfn_param.bestn = bestn; + flags |= BIT(BRCMF_PNO_ENABLE_BD_SCAN_BIT); + brcmf_dbg(INFO, "mscan=%d, bestn=%d\n", mscan, bestn); + } + + pfn_param.flags = cpu_to_le16(flags); + err = brcmf_fil_iovar_data_set(ifp, "pfn_set", &pfn_param, + sizeof(pfn_param)); + if (err) + bphy_err(drvr, "pfn_set failed, err=%d\n", err); + +exit: + return err; +} + +static int brcmf_pno_config_v2(struct brcmf_if *ifp, u32 scan_freq, u32 mscan, + u32 bestn) { struct brcmf_pub *drvr = ifp->drvr; struct brcmf_pno_param_le pfn_param; @@ -109,7 +165,7 @@ static int brcmf_pno_config(struct brcmf_if *ifp, u32 scan_freq, s32 err; memset(&pfn_param, 0, sizeof(pfn_param)); - pfn_param.version = cpu_to_le32(BRCMF_PNO_VERSION); + pfn_param.version = cpu_to_le32(BRCMF_PNO_VERSION_2); /* set extra pno params */ flags = BIT(BRCMF_PNO_IMMEDIATE_SCAN_BIT) | @@ -152,6 +208,12 @@ static int brcmf_pno_config(struct brcmf_if *ifp, u32 scan_freq, return err; } +static int brcmf_pno_config(struct brcmf_if *ifp, u32 scan_freq, u32 mscan, + u32 bestn) +{ + return ifp->drvr->pno_handler.pno_config(ifp, scan_freq, mscan, bestn); +} + static int brcmf_pno_set_random(struct brcmf_if *ifp, struct brcmf_pno_info *pi) { struct brcmf_pub *drvr = ifp->drvr; @@ -275,7 +337,7 @@ static int brcmf_pno_get_bucket_channels(struct cfg80211_sched_scan_request *r, { u32 n_chan = le32_to_cpu(pno_cfg->channel_num); u16 chan; - int i, err = 0; + int i, err; for (i = 0; i < r->n_channels; i++) { if (n_chan >= BRCMF_NUMCHANNELS) { @@ -562,9 +624,82 @@ u64 brcmf_pno_find_reqid_by_bucket(struct brcmf_pno_info *pi, u32 bucket) return reqid; } -u32 brcmf_pno_get_bucket_map(struct brcmf_pno_info *pi, - struct brcmf_pno_net_info_le *ni) + +static struct brcmf_pno_net_info_le * +brcmf_get_netinfo_array(void *pfn_v1_data) +{ + struct brcmf_pno_scanresults_le *pfn_v1 = + (struct brcmf_pno_scanresults_le *)pfn_v1_data; + struct brcmf_pno_scanresults_v2_le *pfn_v2; + struct brcmf_pno_net_info_le *netinfo = NULL; + + switch (pfn_v1->version) { + default: + WARN_ON(1); + fallthrough; + case cpu_to_le32(1): + netinfo = (struct brcmf_pno_net_info_le *)(pfn_v1 + 1); + break; + case cpu_to_le32(2): + pfn_v2 = (struct brcmf_pno_scanresults_v2_le *)pfn_v1; + netinfo = (struct brcmf_pno_net_info_le *)(pfn_v2 + 1); + break; + case cpu_to_le32(3): + brcmf_err("Need to use brcmf_get_netinfo_v3_array\n"); + break; + } + + return netinfo; +} + +static struct brcmf_pno_net_info_v3_le * +brcmf_get_netinfo_v3_array(void*pfn_v3_data) +{ + struct brcmf_pno_scanresults_v3_le *pfn_v3 = + (struct brcmf_pno_scanresults_v3_le *)pfn_v3_data; + return (struct brcmf_pno_net_info_v3_le *) (pfn_v3 + 1); +} + +static u32 brcmf_pno_get_bucket_map(void *data, int idx, struct brcmf_pno_info *pi) +{ + + struct brcmf_pno_net_info_le *netinfo_start = + brcmf_get_netinfo_array(data); + struct brcmf_pno_net_info_le *ni = &netinfo_start[idx]; + struct cfg80211_sched_scan_request *req; + struct cfg80211_match_set *ms; + u32 bucket_map = 0; + int i, j; + + mutex_lock(&pi->req_lock); + for (i = 0; i < pi->n_reqs; i++) { + req = pi->reqs[i]; + + if (!req->n_match_sets) + continue; + for (j = 0; j < req->n_match_sets; j++) { + ms = &req->match_sets[j]; + if (ms->ssid.ssid_len == ni->SSID_len && + !memcmp(ms->ssid.ssid, ni->SSID, ni->SSID_len)) { + bucket_map |= BIT(i); + break; + } + if (is_valid_ether_addr(ms->bssid) && + !memcmp(ms->bssid, ni->bssid, ETH_ALEN)) { + bucket_map |= BIT(i); + break; + } + } + } + mutex_unlock(&pi->req_lock); + return bucket_map; +} + +static u32 brcmf_pno_get_bucket_map_v3(void *data, int idx, struct brcmf_pno_info *pi) { + struct brcmf_pno_net_info_v3_le *netinfo_v3_start = + brcmf_get_netinfo_v3_array(data); + struct brcmf_pno_net_info_v3_le *ni = &netinfo_v3_start[idx]; struct cfg80211_sched_scan_request *req; struct cfg80211_match_set *ms; u32 bucket_map = 0; @@ -593,3 +728,148 @@ u32 brcmf_pno_get_bucket_map(struct brcmf_pno_info *pi, mutex_unlock(&pi->req_lock); return bucket_map; } + +static u32 brcmf_pno_min_data_len(void) +{ + return sizeof(struct brcmf_pno_scanresults_le) + + sizeof(struct brcmf_pno_net_info_le); +} +static u32 brcmf_pno_min_data_len_v3(void) +{ + return sizeof(struct brcmf_pno_scanresults_v3_le) + + sizeof(struct brcmf_pno_net_info_v3_le); +} + +static int brcmf_pno_validate_pfn_results_v3(void *data, u32 eventlen) +{ + struct brcmf_pno_scanresults_v3_le *scanresult = + (struct brcmf_pno_scanresults_v3_le *)data; + struct brcmf_pno_net_info_v3_le *netinfo_v3_start = + brcmf_get_netinfo_v3_array(scanresult); + u32 datalen; + + if (!netinfo_v3_start) { + brcmf_err("did not get netinfo_v3 data\n"); + return -EINVAL; + } + datalen = eventlen - ((void *)netinfo_v3_start - (void *)data); + if (datalen < le32_to_cpu(scanresult->count) * sizeof(struct brcmf_pno_net_info_v3_le)) { + brcmf_err("insufficient event data\n"); + return -EINVAL; + } + return 0; +} + +static int brcmf_pno_validate_pfn_results(void *data, u32 eventlen) +{ + struct brcmf_pno_scanresults_le *scanresult = + (struct brcmf_pno_scanresults_le *)data; + struct brcmf_pno_net_info_le *netinfo_start = + brcmf_get_netinfo_array(scanresult); + u32 datalen; + + if (!netinfo_start) { + brcmf_err("did not get netinfo data\n"); + return -EINVAL; + } + datalen = eventlen - ((void *)netinfo_start - (void *)data); + if (datalen < le32_to_cpu(scanresult->count) * sizeof(struct brcmf_pno_net_info_le)) { + brcmf_err("insufficient event data\n"); + return -EINVAL; + } + return 0; +} + +static int brcmf_pno_get_result_info(void *data, int result_idx, + u8 (*ssid)[IEEE80211_MAX_SSID_LEN], + u8 *ssid_len, u8 *channel, + enum nl80211_band *band) +{ + struct brcmf_pno_scanresults_le *scanresult = + (struct brcmf_pno_scanresults_le *)data; + struct brcmf_pno_net_info_le *netinfo_start = + brcmf_get_netinfo_array(scanresult); + struct brcmf_pno_net_info_le *netinfo = &netinfo_start[result_idx]; + + *channel = netinfo->channel; + *band = netinfo->channel <= CH_MAX_2G_CHANNEL ? NL80211_BAND_2GHZ : + NL80211_BAND_5GHZ; + *ssid_len = netinfo->SSID_len; + if (netinfo->SSID_len > IEEE80211_MAX_SSID_LEN) + *ssid_len = IEEE80211_MAX_SSID_LEN; + memcpy(ssid, netinfo->SSID, *ssid_len); + + return 0; +} + +static int brcmf_pno_get_result_info_v3(void *data, int result_idx, + u8 (*ssid)[IEEE80211_MAX_SSID_LEN], + u8 *ssid_len, u8 *channel, + enum nl80211_band *band) +{ + struct brcmf_pno_scanresults_v3_le *scanresult = + (struct brcmf_pno_scanresults_v3_le *)data; + struct brcmf_pno_net_info_v3_le *netinfo_v3_start = + brcmf_get_netinfo_v3_array(scanresult); + struct brcmf_pno_net_info_v3_le *netinfo_v3 = + &netinfo_v3_start[result_idx]; + + *channel = CHSPEC_CHANNEL(netinfo_v3->chanspec); + *band = fwil_band_to_nl80211(CHSPEC_BAND(netinfo_v3->chanspec)); + *ssid_len = netinfo_v3->SSID_len; + if (netinfo_v3->SSID_len > IEEE80211_MAX_SSID_LEN) + *ssid_len = IEEE80211_MAX_SSID_LEN; + memcpy(ssid, netinfo_v3->SSID, *ssid_len); + + return 0; +} + +/* The count and status fields are in the same place for v1/2/3 */ +static u32 brcmf_pno_get_result_count_v123(void *data) +{ + struct brcmf_pno_scanresults_le *results = + (struct brcmf_pno_scanresults_le *)data; + return le32_to_cpu(results->count); +} +static u32 brcmf_pno_get_result_status_v123(void *data) +{ + struct brcmf_pno_scanresults_le *results = + (struct brcmf_pno_scanresults_le *)data; + return le32_to_cpu(results->status); +} + +int brcmf_pno_setup_for_version(struct brcmf_pub *drvr, u8 vers) +{ + /* The first supported version by this driver was version 2. + * The v2 functions handle version one structures if handed to them, + * but the config was always set to interface version 2. */ + switch (vers) { + case BRCMF_PNO_VERSION_2: { + drvr->pno_handler.version = BRCMF_PNO_VERSION_2; + drvr->pno_handler.pno_config = brcmf_pno_config_v2; + drvr->pno_handler.get_result_count = brcmf_pno_get_result_count_v123; + drvr->pno_handler.get_result_status = brcmf_pno_get_result_status_v123; + drvr->pno_handler.get_bucket_map = brcmf_pno_get_bucket_map; + drvr->pno_handler.get_min_data_len = brcmf_pno_min_data_len; + drvr->pno_handler.get_result_info = brcmf_pno_get_result_info; + drvr->pno_handler.validate_pfn_results = + brcmf_pno_validate_pfn_results; + break; + } + case BRCMF_PNO_VERSION_3: { + drvr->pno_handler.version = BRCMF_PNO_VERSION_3; + drvr->pno_handler.pno_config = brcmf_pno_config_v3; + drvr->pno_handler.get_result_count = brcmf_pno_get_result_count_v123; + drvr->pno_handler.get_result_status = brcmf_pno_get_result_status_v123; + drvr->pno_handler.get_bucket_map = brcmf_pno_get_bucket_map_v3; + drvr->pno_handler.get_min_data_len = brcmf_pno_min_data_len_v3; + drvr->pno_handler.get_result_info = brcmf_pno_get_result_info_v3; + drvr->pno_handler.validate_pfn_results = + brcmf_pno_validate_pfn_results_v3; + break; + } + default: + return -EINVAL; + } + return 0; +} diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pno.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pno.h index 25d406019ac340..0163c762f5385a 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pno.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pno.h @@ -61,12 +61,12 @@ void brcmf_pno_detach(struct brcmf_cfg80211_info *cfg); u64 brcmf_pno_find_reqid_by_bucket(struct brcmf_pno_info *pi, u32 bucket); /** - * brcmf_pno_get_bucket_map - determine bucket map for given netinfo. + * brcmf_pno_setup_for_version - setup our PNO handler for whatever version structures + * are supported by the chip * - * @pi: pno instance used. - * @netinfo: netinfo to compare with bucket configuration. + * @cfg: CFG to fill in. + * @vers: Version to use */ -u32 brcmf_pno_get_bucket_map(struct brcmf_pno_info *pi, - struct brcmf_pno_net_info_le *netinfo); +int brcmf_pno_setup_for_version(struct brcmf_pub *drvr, u8 vers); #endif /* _BRCMF_PNO_H */ diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/ratespec.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/ratespec.h new file mode 100644 index 00000000000000..37e722daab14d4 --- /dev/null +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/ratespec.h @@ -0,0 +1,252 @@ +// SPDX-License-Identifier: ISC +/* + * Copyright (c) 2023 Daniel Berlin + */ + +#ifndef BRCMFMAC_RATESPEC_H +#define BRCMFMAC_RATESPEC_H +/* Rate spec. definitions */ +/* for BRCMF_RSPEC_ENCODING field >= BRCMF_RSPEC_ENCODING_HE, backward compatible */ + +/**< Legacy rate or MCS or MCS + NSS */ +#define BRCMF_RSPEC_RATE_MASK 0x000000FFu +/**< Tx chain expansion beyond Nsts */ +#define BRCMF_RSPEC_TXEXP_MASK 0x00000300u +#define BRCMF_RSPEC_TXEXP_SHIFT 8u +/* EHT GI indices */ +#define BRCMF_RSPEC_EHT_GI_MASK 0x00000C00u +#define BRCMF_RSPEC_EHT_GI_SHIFT 10u +/* HE GI indices */ +#define BRCMF_RSPEC_HE_GI_MASK 0x00000C00u +#define BRCMF_RSPEC_HE_GI_SHIFT 10u +/**< Range extension mask */ +#define BRCMF_RSPEC_ER_MASK 0x0000C000u +#define BRCMF_RSPEC_ER_SHIFT 14u +/**< Range extension tone config */ +#define BRCMF_RSPEC_ER_TONE_MASK 0x00004000u +#define BRCMF_RSPEC_ER_TONE_SHIFT 14u +/**< Range extension enable */ +#define BRCMF_RSPEC_ER_ENAB_MASK 0x00008000u +#define BRCMF_RSPEC_ER_ENAB_SHIFT 15u +/**< Bandwidth */ +#define BRCMF_RSPEC_BW_MASK 0x00070000u +#define BRCMF_RSPEC_BW_SHIFT 16u +/**< Dual Carrier Modulation */ +#define BRCMF_RSPEC_DCM 0x00080000u +#define BRCMF_RSPEC_DCM_SHIFT 19u +/**< STBC expansion, Nsts = 2 * Nss */ +#define BRCMF_RSPEC_STBC 0x00100000u +#define BRCMF_RSPEC_TXBF 0x00200000u +#define BRCMF_RSPEC_LDPC 0x00400000u +/* HT/VHT SGI indication */ +#define BRCMF_RSPEC_SGI 0x00800000u +/**< DSSS short preable - Encoding 0 */ +#define BRCMF_RSPEC_SHORT_PREAMBLE 0x00800000u +/**< Encoding of RSPEC_RATE field */ +#define BRCMF_RSPEC_ENCODING_MASK 0x07000000u +#define BRCMF_RSPEC_ENCODING_SHIFT 24u +#define BRCMF_RSPEC_OVERRIDE_RATE 0x40000000u /**< override rate only */ +#define BRCMF_RSPEC_OVERRIDE_MODE 0x80000000u /**< override both rate & mode */ + +/* ======== RSPEC_EHT_GI|RSPEC_SGI fields for EHT ======== */ +/* 11be Draft 0.4 Table 36-35:Common field for non-OFDMA transmission. + * Table 36-32 Common field for OFDMA transmission + */ +#define BRCMF_RSPEC_EHT_LTF_GI(rspec) \ + (((rspec) & BRCMF_RSPEC_EHT_GI_MASK) >> BRCMF_RSPEC_EHT_GI_SHIFT) +#define BRCMF_RSPEC_EHT_2x_LTF_GI_0_8us (0x0u) +#define BRCMF_RSPEC_EHT_2x_LTF_GI_1_6us (0x1u) +#define BRCMF_RSPEC_EHT_4x_LTF_GI_0_8us (0x2u) +#define BRCMF_RSPEC_EHT_4x_LTF_GI_3_2us (0x3u) +#define WL_EHT_GI_TO_RSPEC(gi) \ + ((u32)(((gi) << BRCMF_RSPEC_EHT_GI_SHIFT) & \ + BRCMF_RSPEC_EHT_GI_MASK)) +#define WL_EHT_GI_TO_RSPEC_SET(rspec, gi) \ + ((rspec & (~BRCMF_RSPEC_EHT_GI_MASK)) | WL_EHT_GI_TO_RSPEC(gi)) + +/* Macros for EHT LTF and GI */ +#define EHT_IS_2X_LTF(gi) \ + (((gi) == BRCMF_RSPEC_EHT_2x_LTF_GI_0_8us) || \ + ((gi) == BRCMF_RSPEC_EHT_2x_LTF_GI_1_6us)) +#define EHT_IS_4X_LTF(gi) \ + (((gi) == BRCMF_RSPEC_EHT_4x_LTF_GI_0_8us) || \ + ((gi) == BRCMF_RSPEC_EHT_4x_LTF_GI_3_2us)) + +#define EHT_IS_GI_0_8us(gi) \ + (((gi) == BRCMF_RSPEC_EHT_2x_LTF_GI_0_8us) || \ + ((gi) == BRCMF_RSPEC_EHT_4x_LTF_GI_0_8us)) +#define EHT_IS_GI_1_6us(gi) ((gi) == BRCMF_RSPEC_EHT_2x_LTF_GI_1_6us) +#define EHT_IS_GI_3_2us(gi) ((gi) == BRCMF_RSPEC_EHT_4x_LTF_GI_3_2us) + +/* ======== RSPEC_HE_GI|RSPEC_SGI fields for HE ======== */ + +/* GI for HE */ +#define BRCMF_RSPEC_HE_LTF_GI(rspec) \ + (((rspec) & BRCMF_RSPEC_HE_GI_MASK) >> BRCMF_RSPEC_HE_GI_SHIFT) +#define BRCMF_RSPEC_HE_1x_LTF_GI_0_8us (0x0u) +#define BRCMF_RSPEC_HE_2x_LTF_GI_0_8us (0x1u) +#define BRCMF_RSPEC_HE_2x_LTF_GI_1_6us (0x2u) +#define BRCMF_RSPEC_HE_4x_LTF_GI_3_2us (0x3u) +#define BRCMF_RSPEC_ISHEGI(rspec) \ + (RSPEC_HE_LTF_GI(rspec) > BRCMF_RSPEC_HE_1x_LTF_GI_0_8us) +#define HE_GI_TO_RSPEC(gi) \ + (((u32)(gi) << BRCMF_RSPEC_HE_GI_SHIFT) & BRCMF_RSPEC_HE_GI_MASK) +#define HE_GI_TO_RSPEC_SET(rspec, gi) \ + ((rspec & (~BRCMF_RSPEC_HE_GI_MASK)) | HE_GI_TO_RSPEC(gi)) + +/* Macros for HE LTF and GI */ +#define HE_IS_1X_LTF(gi) ((gi) == BRCMF_RSPEC_HE_1x_LTF_GI_0_8us) +#define HE_IS_2X_LTF(gi) \ + (((gi) == BRCMF_RSPEC_HE_2x_LTF_GI_0_8us) || \ + ((gi) == BRCMF_RSPEC_HE_2x_LTF_GI_1_6us)) +#define HE_IS_4X_LTF(gi) ((gi) == BRCMF_RSPEC_HE_4x_LTF_GI_3_2us) + +#define HE_IS_GI_0_8us(gi) \ + (((gi) == BRCMF_RSPEC_HE_1x_LTF_GI_0_8us) || \ + ((gi) == BRCMF_RSPEC_HE_2x_LTF_GI_0_8us)) +#define HE_IS_GI_1_6us(gi) ((gi) == BRCMF_RSPEC_HE_2x_LTF_GI_1_6us) +#define HE_IS_GI_3_2us(gi) ((gi) == BRCMF_RSPEC_HE_4x_LTF_GI_3_2us) + +/* RSPEC Macros for extracting and using HE-ER and DCM */ +#define BRCMF_RSPEC_HE_DCM(rspec) \ + (((rspec) & BRCMF_RSPEC_DCM) >> BRCMF_RSPEC_DCM_SHIFT) +#define BRCMF_RSPEC_HE_ER(rspec) \ + (((rspec) & BRCMF_RSPEC_ER_MASK) >> BRCMF_RSPEC_ER_SHIFT) +#define BRCMF_RSPEC_HE_ER_ENAB(rspec) \ + (((rspec) & BRCMF_RSPEC_ER_ENAB_MASK) >> BRCMF_RSPEC_ER_ENAB_SHIFT) +#define BRCMF_RSPEC_HE_ER_TONE(rspec) \ + (((rspec) & BRCMF_RSPEC_ER_TONE_MASK) >> BRCMF_RSPEC_ER_TONE_SHIFT) +/* ======== RSPEC_RATE field ======== */ + +/* Encoding 0 - legacy rate */ +/* DSSS, CCK, and OFDM rates in [500kbps] units */ +#define BRCMF_RSPEC_LEGACY_RATE_MASK 0x0000007F +#define WLC_RATE_1M 2 +#define WLC_RATE_2M 4 +#define WLC_RATE_5M5 11 +#define WLC_RATE_11M 22 +#define WLC_RATE_6M 12 +#define WLC_RATE_9M 18 +#define WLC_RATE_12M 24 +#define WLC_RATE_18M 36 +#define WLC_RATE_24M 48 +#define WLC_RATE_36M 72 +#define WLC_RATE_48M 96 +#define WLC_RATE_54M 108 + +/* Encoding 1 - HT MCS */ +/**< HT MCS value mask in rspec */ +#define BRCMF_RSPEC_HT_MCS_MASK 0x0000007F + +/* Encoding >= 2 */ +/* NSS & MCS values mask in rspec */ +#define BRCMF_RSPEC_NSS_MCS_MASK 0x000000FF +/* mimo MCS value mask in rspec */ +#define BRCMF_RSPEC_MCS_MASK 0x0000000F +/* mimo NSS value mask in rspec */ +#define BRCMF_RSPEC_NSS_MASK 0x000000F0 +/* mimo NSS value shift in rspec */ +#define BRCMF_RSPEC_NSS_SHIFT 4 + +/* Encoding 2 - VHT MCS + NSS */ +/**< VHT MCS value mask in rspec */ +#define BRCMF_RSPEC_VHT_MCS_MASK BRCMF_RSPEC_MCS_MASK +/**< VHT Nss value mask in rspec */ +#define BRCMF_RSPEC_VHT_NSS_MASK BRCMF_RSPEC_NSS_MASK +/**< VHT Nss value shift in rspec */ +#define BRCMF_RSPEC_VHT_NSS_SHIFT BRCMF_RSPEC_NSS_SHIFT + +/* Encoding 3 - HE MCS + NSS */ +/**< HE MCS value mask in rspec */ +#define BRCMF_RSPEC_HE_MCS_MASK BRCMF_RSPEC_MCS_MASK +/**< HE Nss value mask in rspec */ +#define BRCMF_RSPEC_HE_NSS_MASK BRCMF_RSPEC_NSS_MASK +/**< HE Nss value shift in rpsec */ +#define BRCMF_RSPEC_HE_NSS_SHIFT BRCMF_RSPEC_NSS_SHIFT + +#define BRCMF_RSPEC_HE_NSS_UNSPECIFIED 0xf + +/* Encoding 4 - EHT MCS + NSS */ +/**< EHT MCS value mask in rspec */ +#define BRCMF_RSPEC_EHT_MCS_MASK BRCMF_RSPEC_MCS_MASK +/**< EHT Nss value mask in rspec */ +#define BRCMF_RSPEC_EHT_NSS_MASK BRCMF_RSPEC_NSS_MASK +/**< EHT Nss value shift in rpsec */ +#define BRCMF_RSPEC_EHT_NSS_SHIFT BRCMF_RSPEC_NSS_SHIFT + +/* ======== RSPEC_BW field ======== */ + +#define BRCMF_RSPEC_BW_UNSPECIFIED 0u +#define BRCMF_RSPEC_BW_20MHZ 0x00010000u +#define BRCMF_RSPEC_BW_40MHZ 0x00020000u +#define BRCMF_RSPEC_BW_80MHZ 0x00030000u +#define BRCMF_RSPEC_BW_160MHZ 0x00040000u +#define BRCMF_RSPEC_BW_320MHZ 0x00060000u + +/* ======== RSPEC_ENCODING field ======== */ + +/* NOTE: Assuming the rate field is always NSS+MCS starting from VHT encoding! + * Modify/fix RSPEC_ISNSSMCS() macro if above condition changes any time. + */ +/**< Legacy rate is stored in RSPEC_RATE */ +#define BRCMF_RSPEC_ENCODE_RATE 0x00000000u +/**< HT MCS is stored in RSPEC_RATE */ +#define BRCMF_RSPEC_ENCODE_HT 0x01000000u +/**< VHT MCS and NSS are stored in RSPEC_RATE */ +#define BRCMF_RSPEC_ENCODE_VHT 0x02000000u +/**< HE MCS and NSS are stored in RSPEC_RATE */ +#define BRCMF_RSPEC_ENCODE_HE 0x03000000u +/**< EHT MCS and NSS are stored in RSPEC_RATE */ +#define BRCMF_RSPEC_ENCODE_EHT 0x04000000u + +/** + * =============================== + * Handy macros to parse rate spec + * =============================== + */ +#define BRCMF_RSPEC_BW(rspec) ((rspec) & BRCMF_RSPEC_BW_MASK) +#define BRCMF_RSPEC_IS20MHZ(rspec) (RSPEC_BW(rspec) == BRCMF_RSPEC_BW_20MHZ) +#define BRCMF_RSPEC_IS40MHZ(rspec) (RSPEC_BW(rspec) == BRCMF_RSPEC_BW_40MHZ) +#define BRCMF_RSPEC_IS80MHZ(rspec) (RSPEC_BW(rspec) == BRCMF_RSPEC_BW_80MHZ) +#define BRCMF_RSPEC_IS160MHZ(rspec) (RSPEC_BW(rspec) == BRCMF_RSPEC_BW_160MHZ) +#if defined(WL_BW320MHZ) +#define BRCMF_RSPEC_IS320MHZ(rspec) (RSPEC_BW(rspec) == BRCMF_RSPEC_BW_320MHZ) +#else +#define BRCMF_RSPEC_IS320MHZ(rspec) (FALSE) +#endif /* WL_BW320MHZ */ + +#define BRCMF_RSPEC_BW_GE(rspec, rspec_bw) (RSPEC_BW(rspec) >= rspec_bw) +#define BRCMF_RSPEC_BW_LE(rspec, rspec_bw) (RSPEC_BW(rspec) <= rspec_bw) +#define BRCMF_RSPEC_BW_GT(rspec, rspec_bw) (!RSPEC_BW_LE(rspec, rspec_bw)) +#define BRCMF_RSPEC_BW_LT(rspec, rspec_bw) (!RSPEC_BW_GE(rspec, rspec_bw)) + +#define BRCMF_RSPEC_ISSGI(rspec) (((rspec) & BRCMF_RSPEC_SGI) != 0) +#define BRCMF_RSPEC_ISLDPC(rspec) (((rspec) & BRCMF_RSPEC_LDPC) != 0) +#define BRCMF_RSPEC_ISSTBC(rspec) (((rspec) & BRCMF_RSPEC_STBC) != 0) +#define BRCMF_RSPEC_ISTXBF(rspec) (((rspec) & BRCMF_RSPEC_TXBF) != 0) + +#define BRCMF_RSPEC_TXEXP(rspec) \ + (((rspec) & BRCMF_RSPEC_TXEXP_MASK) >> BRCMF_RSPEC_TXEXP_SHIFT) + +#define BRCMF_RSPEC_ENCODE(rspec) \ + (((rspec) & BRCMF_RSPEC_ENCODING_MASK) >> BRCMF_RSPEC_ENCODING_SHIFT) +#define BRCMF_RSPEC_ISLEGACY(rspec) \ + (((rspec) & BRCMF_RSPEC_ENCODING_MASK) == BRCMF_RSPEC_ENCODE_RATE) + +#define BRCMF_RSPEC_ISHT(rspec) \ + (((rspec) & BRCMF_RSPEC_ENCODING_MASK) == BRCMF_RSPEC_ENCODE_HT) +#define BRCMF_RSPEC_ISVHT(rspec) \ + (((rspec) & BRCMF_RSPEC_ENCODING_MASK) == BRCMF_RSPEC_ENCODE_VHT) +#define BRCMF_RSPEC_ISHE(rspec) \ + (((rspec) & BRCMF_RSPEC_ENCODING_MASK) == BRCMF_RSPEC_ENCODE_HE) +#define BRCMF_RSPEC_ISEHT(rspec) \ + (((rspec) & BRCMF_RSPEC_ENCODING_MASK) == BRCMF_RSPEC_ENCODE_EHT) + +/* fast check if rate field is NSS+MCS format (starting from VHT ratespec) */ +#define BRCMF_RSPEC_ISVHTEXT(rspec) \ + (((rspec) & BRCMF_RSPEC_ENCODING_MASK) >= BRCMF_RSPEC_ENCODE_VHT) +/* fast check if rate field is NSS+MCS format (starting from HE ratespec) */ +#define BRCMF_RSPEC_ISHEEXT(rspec) \ + (((rspec) & BRCMF_RSPEC_ENCODING_MASK) >= BRCMF_RSPEC_ENCODE_HE) + +#endif /* BRCMFMAC_RATESPEC_H */ diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/scan_param.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/scan_param.c new file mode 100644 index 00000000000000..4f634509d25256 --- /dev/null +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/scan_param.c @@ -0,0 +1,446 @@ +// SPDX-License-Identifier: ISC +/* + * Copyright (c) 2023 Daniel Berlin + */ +#include +#include + +#include "core.h" +#include "debug.h" +#include "fwil_types.h" +#include "cfg80211.h" +#include "scan_param.h" + +static void brcmf_scan_param_set_defaults(u8 (*bssid)[ETH_ALEN], s8 *bss_type, __le32 *channel_num, + __le32 *nprobes, __le32 *active_time, + __le32 *passive_time, + __le32 *home_time) +{ + eth_broadcast_addr(*bssid); + *bss_type = DOT11_BSSTYPE_ANY; + *channel_num = 0; + *nprobes = cpu_to_le32(-1); + *active_time = cpu_to_le32(-1); + *passive_time = cpu_to_le32(-1); + *home_time = cpu_to_le32(-1); +} + +static void brcmf_scan_param_copy_chanspecs( + struct brcmf_cfg80211_info *cfg, __le16 (*dest_channels)[], + struct ieee80211_channel **in_channels, u32 n_channels) +{ + int i; + for (i = 0; i < n_channels; i++) { + u32 chanspec = + channel_to_chanspec(&cfg->d11inf, in_channels[i]); + brcmf_dbg(SCAN, "Chan : %d, Channel spec: %x\n", + in_channels[i]->hw_value, chanspec); + (*dest_channels)[i] = cpu_to_le16(chanspec); + } +} + +static void brcmf_scan_param_copy_ssids(char *dest_ssids, + struct cfg80211_ssid *in_ssids, + u32 n_ssids) +{ + int i; + for (i = 0; i < n_ssids; i++) { + struct brcmf_ssid_le ssid_le; + memset(&ssid_le, 0, sizeof(ssid_le)); + ssid_le.SSID_len = cpu_to_le32(in_ssids[i].ssid_len); + memcpy(ssid_le.SSID, in_ssids[i].ssid, in_ssids[i].ssid_len); + if (!ssid_le.SSID_len) + brcmf_dbg(SCAN, "%d: Broadcast scan\n", i); + else + brcmf_dbg(SCAN, "%d: scan for %.32s size=%d\n", i, + ssid_le.SSID, ssid_le.SSID_len); + memcpy(dest_ssids, &ssid_le, sizeof(ssid_le)); + dest_ssids += sizeof(ssid_le); + } +} + +/* The scan parameter structures have an array of SSID's that appears at the end in some cases. + * In these cases, the chan list is really the lower half of a pair, the upper half is a ssid number, + * and then after all of that there is an array of SSIDs */ +static u32 +brcmf_scan_param_tail_size(const struct cfg80211_scan_request *request, + u32 params_size) +{ + if (request != NULL) { + /* Allocate space for populating ssid upper half in struct */ + params_size += sizeof(u32) * ((request->n_channels + 1) / 2); + /* Allocate space for populating ssids in struct */ + params_size += sizeof(struct brcmf_ssid_le) * request->n_ssids; + } else { + params_size += sizeof(u16); + } + return params_size; +} + +static u32 brcmf_nl80211_scan_flags_to_scan_flags(u32 nl80211_flags) +{ + u32 scan_flags = 0; + if (nl80211_flags & NL80211_SCAN_FLAG_LOW_SPAN) { + scan_flags |= BRCMF_SCANFLAGS_LOW_SPAN; + brcmf_dbg(SCAN, "requested low span scan\n"); + } + if (nl80211_flags & NL80211_SCAN_FLAG_HIGH_ACCURACY) { + scan_flags |= BRCMF_SCANFLAGS_HIGH_ACCURACY; + brcmf_dbg(SCAN, "requested high accuracy scan\n"); + } + if (nl80211_flags & NL80211_SCAN_FLAG_LOW_POWER) { + scan_flags |= BRCMF_SCANFLAGS_LOW_POWER; + brcmf_dbg(SCAN, "requested low power scan\n"); + } + if (nl80211_flags & NL80211_SCAN_FLAG_LOW_PRIORITY) { + scan_flags |= BRCMF_SCANFLAGS_LOW_PRIO; + brcmf_dbg(SCAN, "requested low priority scan\n"); + } + return scan_flags; +} + +static void * +brcmf_scan_param_get_prepped_struct_v1(struct brcmf_cfg80211_info *cfg, + u32 *struct_size, + struct cfg80211_scan_request *request) +{ + u32 n_ssids; + u32 n_channels; + u32 params_size = sizeof(struct brcmf_scan_params_le); + u32 length; + struct brcmf_scan_params_le *params_le = NULL; + u8 scan_type = BRCMF_SCANTYPE_ACTIVE; + + length = offsetof(struct brcmf_scan_params_le, channel_list); + params_size = brcmf_scan_param_tail_size(request, params_size); + params_le = kzalloc(params_size, GFP_KERNEL); + if (!params_le) { + bphy_err(cfg, "Could not allocate scan params\n"); + return NULL; + } + brcmf_scan_param_set_defaults(¶ms_le->bssid, + ¶ms_le->bss_type, ¶ms_le->channel_num, + ¶ms_le->nprobes, ¶ms_le->active_time, + ¶ms_le->passive_time, ¶ms_le->home_time); + + /* Scan abort */ + if (!request) { + params_le->channel_num = cpu_to_le32(1); + params_le->channel_list[0] = cpu_to_le16(-1); + goto done; + } + + n_ssids = request->n_ssids; + n_channels = request->n_channels; + + /* Copy channel array if applicable */ + brcmf_dbg(SCAN, "### List of channelspecs to scan ### %d\n", + n_channels); + if (n_channels > 0) { + length += roundup(sizeof(u16) * n_channels, sizeof(u32)); + brcmf_scan_param_copy_chanspecs(cfg, ¶ms_le->channel_list, + request->channels, n_channels); + } else { + brcmf_dbg(SCAN, "Scanning all channels\n"); + } + + /* Copy ssid array if applicable */ + brcmf_dbg(SCAN, "### List of SSIDs to scan ### %d\n", n_ssids); + if (n_ssids > 0) { + s32 offset; + char *ptr; + + offset = + offsetof(struct brcmf_scan_params_le, channel_list) + + n_channels * sizeof(u16); + offset = roundup(offset, sizeof(u32)); + length += sizeof(struct brcmf_ssid_le) * n_ssids; + ptr = (char *)params_le + offset; + brcmf_scan_param_copy_ssids(ptr, request->ssids, n_ssids); + } else { + brcmf_dbg(SCAN, "Performing passive scan\n"); + scan_type = BRCMF_SCANTYPE_PASSIVE; + } + scan_type |= brcmf_nl80211_scan_flags_to_scan_flags(request->flags); + params_le->scan_type =scan_type; + /* Adding mask to channel numbers */ + params_le->channel_num = + cpu_to_le32((n_ssids << BRCMF_SCAN_PARAMS_NSSID_SHIFT) | + (n_channels & BRCMF_SCAN_PARAMS_COUNT_MASK)); +done: + *struct_size = length; + return params_le; +} + +static void * +brcmf_scan_param_get_prepped_struct_v2(struct brcmf_cfg80211_info *cfg, + u32 *struct_size, + struct cfg80211_scan_request *request) +{ + u32 n_ssids; + u32 n_channels; + u32 params_size = sizeof(struct brcmf_scan_params_v2_le); + u32 length; + struct brcmf_scan_params_v2_le *params_le = NULL; + u32 scan_type = BRCMF_SCANTYPE_ACTIVE; + + length = offsetof(struct brcmf_scan_params_v2_le, channel_list); + params_size = brcmf_scan_param_tail_size(request, params_size); + params_le = kzalloc(params_size, GFP_KERNEL); + if (!params_le) { + bphy_err(cfg, "Could not allocate scan params\n"); + return NULL; + } + params_le->version = cpu_to_le16(BRCMF_SCAN_PARAMS_VERSION_V2); + brcmf_scan_param_set_defaults(¶ms_le->bssid, + ¶ms_le->bss_type, ¶ms_le->channel_num, + ¶ms_le->nprobes, ¶ms_le->active_time, + ¶ms_le->passive_time, ¶ms_le->home_time); + + /* Scan abort */ + if (!request) { + length += sizeof(u16); + params_le->channel_num = cpu_to_le32(1); + params_le->channel_list[0] = cpu_to_le16(-1); + params_le->length = cpu_to_le16(length); + goto done; + } + + n_ssids = request->n_ssids; + n_channels = request->n_channels; + + /* Copy channel array if applicable */ + brcmf_dbg(SCAN, "### List of channelspecs to scan ### %d\n", + n_channels); + if (n_channels > 0) { + length += roundup(sizeof(u16) * n_channels, sizeof(u32)); + brcmf_scan_param_copy_chanspecs(cfg, ¶ms_le->channel_list, + request->channels, n_channels); + } else { + brcmf_dbg(SCAN, "Scanning all channels\n"); + } + + /* Copy ssid array if applicable */ + brcmf_dbg(SCAN, "### List of SSIDs to scan ### %d\n", n_ssids); + if (n_ssids > 0) { + s32 offset; + char *ptr; + + offset = + offsetof(struct brcmf_scan_params_v2_le, channel_list) + + n_channels * sizeof(u16); + offset = roundup(offset, sizeof(u32)); + length += sizeof(struct brcmf_ssid_le) * n_ssids; + ptr = (char *)params_le + offset; + brcmf_scan_param_copy_ssids(ptr, request->ssids, n_ssids); + + } else { + brcmf_dbg(SCAN, "Performing passive scan\n"); + scan_type = BRCMF_SCANTYPE_PASSIVE; + } + scan_type |= brcmf_nl80211_scan_flags_to_scan_flags(request->flags); + params_le->scan_type = cpu_to_le32(scan_type); + params_le->length = cpu_to_le16(length); + /* Adding mask to channel numbers */ + params_le->channel_num = + cpu_to_le32((n_ssids << BRCMF_SCAN_PARAMS_NSSID_SHIFT) | + (n_channels & BRCMF_SCAN_PARAMS_COUNT_MASK)); +done: + *struct_size = length; + return params_le; +} + +static void * +brcmf_scan_param_get_prepped_struct_v3(struct brcmf_cfg80211_info *cfg, + u32 *struct_size, + struct cfg80211_scan_request *request) +{ + u32 n_ssids; + u32 n_channels; + u32 params_size = sizeof(struct brcmf_scan_params_v3_le); + u32 length; + struct brcmf_scan_params_v3_le *params_le = NULL; + u32 scan_type = BRCMF_SCANTYPE_ACTIVE; + + length = offsetof(struct brcmf_scan_params_v3_le, channel_list); + params_size = brcmf_scan_param_tail_size(request, params_size); + params_le = kzalloc(params_size, GFP_KERNEL); + if (!params_le) { + bphy_err(cfg, "Could not allocate scan params\n"); + return NULL; + } + + params_le->version = cpu_to_le16(BRCMF_SCAN_PARAMS_VERSION_V3); + params_le->ssid_type = 0; + brcmf_scan_param_set_defaults(¶ms_le->bssid, + ¶ms_le->bss_type, ¶ms_le->channel_num, + ¶ms_le->nprobes, ¶ms_le->active_time, + ¶ms_le->passive_time, ¶ms_le->home_time); + + /* Scan abort */ + if (!request) { + length += sizeof(u16); + params_le->channel_num = cpu_to_le32(1); + params_le->channel_list[0] = cpu_to_le16(-1); + params_le->length = cpu_to_le16(length); + goto done; + } + + n_ssids = request->n_ssids; + n_channels = request->n_channels; + + /* Copy channel array if applicable */ + brcmf_dbg(SCAN, "### List of channelspecs to scan ### %d\n", + n_channels); + if (n_channels > 0) { + length += roundup(sizeof(u16) * n_channels, sizeof(u32)); + brcmf_scan_param_copy_chanspecs(cfg, ¶ms_le->channel_list, + request->channels, n_channels); + + } else { + brcmf_dbg(SCAN, "Scanning all channels\n"); + } + + /* Copy ssid array if applicable */ + brcmf_dbg(SCAN, "### List of SSIDs to scan ### %d\n", n_ssids); + if (n_ssids > 0) { + s32 offset; + char *ptr; + + offset = + offsetof(struct brcmf_scan_params_v3_le, channel_list) + + n_channels * sizeof(u16); + offset = roundup(offset, sizeof(u32)); + length += sizeof(struct brcmf_ssid_le) * n_ssids; + ptr = (char *)params_le + offset; + brcmf_scan_param_copy_ssids(ptr, request->ssids, n_ssids); + + } else { + brcmf_dbg(SCAN, "Performing passive scan\n"); + scan_type = BRCMF_SCANTYPE_PASSIVE; + } + scan_type |= brcmf_nl80211_scan_flags_to_scan_flags(request->flags); + params_le->scan_type = cpu_to_le32(scan_type); + params_le->length = cpu_to_le16(length); + params_le->channel_num = + cpu_to_le32((n_ssids << BRCMF_SCAN_PARAMS_NSSID_SHIFT) | + (n_channels & BRCMF_SCAN_PARAMS_COUNT_MASK)); + + /* Include RNR results if requested */ + if (request->flags & NL80211_SCAN_FLAG_COLOCATED_6GHZ) { + params_le->ssid_type |= BRCMF_SCANSSID_INC_RNR; + } + /* Adding mask to channel numbers */ +done: + *struct_size = length; + return params_le; +} + +static void * +brcmf_scan_param_get_prepped_struct_v4(struct brcmf_cfg80211_info *cfg, + u32 *struct_size, + struct cfg80211_scan_request *request) +{ + u32 n_ssids; + u32 n_channels; + u32 params_size = sizeof(struct brcmf_scan_params_v4_le); + u32 length; + struct brcmf_scan_params_v4_le *params_le = NULL; + u32 scan_type = BRCMF_SCANTYPE_ACTIVE; + + length = offsetof(struct brcmf_scan_params_v4_le, channel_list); + params_size = brcmf_scan_param_tail_size(request, params_size); + params_le = kzalloc(params_size, GFP_KERNEL); + if (!params_le) { + bphy_err(cfg, "Could not allocate scan params\n"); + return NULL; + } + params_le->version = cpu_to_le16(BRCMF_SCAN_PARAMS_VERSION_V4); + params_le->ssid_type = 0; + brcmf_scan_param_set_defaults(¶ms_le->bssid, + ¶ms_le->bss_type, ¶ms_le->channel_num, + ¶ms_le->nprobes, ¶ms_le->active_time, + ¶ms_le->passive_time, ¶ms_le->home_time); + + /* Scan abort */ + if (!request) { + length += sizeof(u16); + params_le->channel_num = cpu_to_le32(1); + params_le->channel_list[0] = cpu_to_le16(-1); + params_le->length = cpu_to_le16(length); + goto done; + } + + n_ssids = request->n_ssids; + n_channels = request->n_channels; + + /* Copy channel array if applicable */ + brcmf_dbg(SCAN, "### List of channelspecs to scan ### %d\n", + n_channels); + if (n_channels > 0) { + length += roundup(sizeof(u16) * n_channels, sizeof(u32)); + brcmf_scan_param_copy_chanspecs(cfg, ¶ms_le->channel_list, + request->channels, n_channels); + } else { + brcmf_dbg(SCAN, "Scanning all channels\n"); + } + + /* Copy ssid array if applicable */ + brcmf_dbg(SCAN, "### List of SSIDs to scan ### %d\n", n_ssids); + if (n_ssids > 0) { + s32 offset; + char *ptr; + + offset = + offsetof(struct brcmf_scan_params_v4_le, channel_list) + + n_channels * sizeof(u16); + offset = roundup(offset, sizeof(u32)); + length += sizeof(struct brcmf_ssid_le) * n_ssids; + ptr = (char *)params_le + offset; + brcmf_scan_param_copy_ssids(ptr, request->ssids, n_ssids); + } else { + brcmf_dbg(SCAN, "Performing passive scan\n"); + scan_type = BRCMF_SCANTYPE_PASSIVE; + } + scan_type |= brcmf_nl80211_scan_flags_to_scan_flags(request->flags); + params_le->scan_type = cpu_to_le32(scan_type); + params_le->length = cpu_to_le16(length); + /* Adding mask to channel numbers */ + params_le->channel_num = + cpu_to_le32((n_ssids << BRCMF_SCAN_PARAMS_NSSID_SHIFT) | + (n_channels & BRCMF_SCAN_PARAMS_COUNT_MASK)); + /* Include RNR results if requested */ + if (request->flags & NL80211_SCAN_FLAG_COLOCATED_6GHZ) { + params_le->ssid_type |= BRCMF_SCANSSID_INC_RNR; + } +done: + *struct_size = length; + return params_le; +} + +int brcmf_scan_param_setup_for_version(struct brcmf_pub *drvr, u8 version) +{ + drvr->scan_param_handler.version = version; + switch (version) { + case 1: { + drvr->scan_param_handler.get_struct_for_request = + brcmf_scan_param_get_prepped_struct_v1; + } break; + case 2: { + drvr->scan_param_handler.get_struct_for_request = + brcmf_scan_param_get_prepped_struct_v2; + } break; + case 3: { + drvr->scan_param_handler.get_struct_for_request = + brcmf_scan_param_get_prepped_struct_v3; + } break; + case 4: { + drvr->scan_param_handler.get_struct_for_request = + brcmf_scan_param_get_prepped_struct_v4; + + } break; + default: + return -EINVAL; + } + return 0; +} diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/scan_param.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/scan_param.h new file mode 100644 index 00000000000000..577de083c6e3cd --- /dev/null +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/scan_param.h @@ -0,0 +1,22 @@ +// SPDX-License-Identifier: ISC +/* + * Copyright (c) 2023 Daniel Berlin + */ + +#ifndef _BRCMF_SCAN_PARAM_H +#define _BRCMF_SCAN_PARAM_H + +struct brcmf_pub; + +/** + * brcmf_scan_param_setup_for_version() - Setup the driver to handle join structures + * + * There are a number of different structures and interface versions for scanning info + * This sets up the driver to handle a particular interface version. + * + * @drvr Driver structure to setup + * @ver Interface version + * Return: %0 if okay, error code otherwise + */ +int brcmf_scan_param_setup_for_version(struct brcmf_pub *, u8 ver); +#endif /* _BRCMF_SCAN_PARAM_H */ diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c index 8cf9d7e7c3f70c..4e6ed02c159134 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c @@ -4445,7 +4445,7 @@ brcmf_sdio_prepare_fw_request(struct brcmf_sdio *bus) return fwreq; } -struct brcmf_sdio *brcmf_sdio_probe(struct brcmf_sdio_dev *sdiodev) +int brcmf_sdio_probe(struct brcmf_sdio_dev *sdiodev) { int ret; struct brcmf_sdio *bus; @@ -4551,11 +4551,12 @@ struct brcmf_sdio *brcmf_sdio_probe(struct brcmf_sdio_dev *sdiodev) goto fail; } - return bus; + return 0; fail: brcmf_sdio_remove(bus); - return ERR_PTR(ret); + sdiodev->bus = NULL; + return ret; } /* Detach and free everything */ diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.h index 0d18ed15b4032a..80180d5c6c879a 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.h @@ -358,7 +358,7 @@ void brcmf_sdiod_freezer_uncount(struct brcmf_sdio_dev *sdiodev); int brcmf_sdiod_probe(struct brcmf_sdio_dev *sdiodev); int brcmf_sdiod_remove(struct brcmf_sdio_dev *sdiodev); -struct brcmf_sdio *brcmf_sdio_probe(struct brcmf_sdio_dev *sdiodev); +int brcmf_sdio_probe(struct brcmf_sdio_dev *sdiodev); void brcmf_sdio_remove(struct brcmf_sdio *bus); void brcmf_sdio_isr(struct brcmf_sdio *bus, bool in_isr); diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmsmac/dma.c b/drivers/net/wireless/broadcom/brcm80211/brcmsmac/dma.c index c739bf7463b316..13d0d6b68238d7 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmsmac/dma.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmsmac/dma.c @@ -483,7 +483,7 @@ static void *dma_ringalloc(struct dma_info *di, u32 boundary, uint size, if (((desc_strtaddr + size - 1) & boundary) != (desc_strtaddr & boundary)) { *alignbits = dma_align_sizetobits(size); - dma_free_coherent(di->dmadev, size, va, *descpa); + dma_free_coherent(di->dmadev, *alloced, va, *descpa); va = dma_alloc_consistent(di, size, *alignbits, alloced, descpa); } diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmutil/d11.c b/drivers/net/wireless/broadcom/brcm80211/brcmutil/d11.c index 1e2b1e487eb76e..faf7eeeeb2d57e 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmutil/d11.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmutil/d11.c @@ -87,10 +87,20 @@ static void brcmu_d11ac_encchspec(struct brcmu_chan *ch) 0, d11ac_bw(ch->bw)); ch->chspec &= ~BRCMU_CHSPEC_D11AC_BND_MASK; - if (ch->chnum <= CH_MAX_2G_CHANNEL) - ch->chspec |= BRCMU_CHSPEC_D11AC_BND_2G; - else + switch (ch->band) { + case BRCMU_CHAN_BAND_6G: + ch->chspec |= BRCMU_CHSPEC_D11AC_BND_6G; + break; + case BRCMU_CHAN_BAND_5G: ch->chspec |= BRCMU_CHSPEC_D11AC_BND_5G; + break; + case BRCMU_CHAN_BAND_2G: + ch->chspec |= BRCMU_CHSPEC_D11AC_BND_2G; + break; + default: + WARN_ONCE(1, "Invalid band 0x%04x\n", ch->band); + break; + } } static void brcmu_d11n_decchspec(struct brcmu_chan *ch) @@ -117,7 +127,9 @@ static void brcmu_d11n_decchspec(struct brcmu_chan *ch) } break; default: - WARN_ONCE(1, "Invalid chanspec 0x%04x\n", ch->chspec); + WARN_ONCE(1, + "Invalid chanspec - unknown 11n bandwidth 0x%04x\n", + ch->chspec); break; } @@ -129,7 +141,8 @@ static void brcmu_d11n_decchspec(struct brcmu_chan *ch) ch->band = BRCMU_CHAN_BAND_2G; break; default: - WARN_ONCE(1, "Invalid chanspec 0x%04x\n", ch->chspec); + WARN_ONCE(1, "Invalid chanspec - unknown 11n band 0x%04x\n", + ch->chspec); break; } } @@ -156,7 +169,9 @@ static void brcmu_d11ac_decchspec(struct brcmu_chan *ch) ch->sb = BRCMU_CHAN_SB_U; ch->control_ch_num += CH_10MHZ_APART; } else { - WARN_ONCE(1, "Invalid chanspec 0x%04x\n", ch->chspec); + WARN_ONCE(1, + "Invalid chanspec - unknown 11ac channel distance 0x%04x\n", + ch->chspec); } break; case BRCMU_CHSPEC_D11AC_BW_80: @@ -177,7 +192,9 @@ static void brcmu_d11ac_decchspec(struct brcmu_chan *ch) ch->control_ch_num += CH_30MHZ_APART; break; default: - WARN_ONCE(1, "Invalid chanspec 0x%04x\n", ch->chspec); + WARN_ONCE(1, + "Invalid chanspec - unknown 11ac channel distance 0x%04x\n", + ch->chspec); break; } break; @@ -211,17 +228,24 @@ static void brcmu_d11ac_decchspec(struct brcmu_chan *ch) ch->control_ch_num += CH_70MHZ_APART; break; default: - WARN_ONCE(1, "Invalid chanspec 0x%04x\n", ch->chspec); + WARN_ONCE(1, + "Invalid chanspec - unknown 11ac channel distance 0x%04x\n", + ch->chspec); break; } break; case BRCMU_CHSPEC_D11AC_BW_8080: default: - WARN_ONCE(1, "Invalid chanspec 0x%04x\n", ch->chspec); + WARN_ONCE(1, + "Invalid chanspec - unknown 11ac channel bandwidth 0x%04x\n", + ch->chspec); break; } switch (ch->chspec & BRCMU_CHSPEC_D11AC_BND_MASK) { + case BRCMU_CHSPEC_D11AC_BND_6G: + ch->band = BRCMU_CHAN_BAND_6G; + break; case BRCMU_CHSPEC_D11AC_BND_5G: ch->band = BRCMU_CHAN_BAND_5G; break; @@ -229,7 +253,9 @@ static void brcmu_d11ac_decchspec(struct brcmu_chan *ch) ch->band = BRCMU_CHAN_BAND_2G; break; default: - WARN_ONCE(1, "Invalid chanspec 0x%04x\n", ch->chspec); + WARN_ONCE(1, + "Invalid chanspec - unknown 11ac channel band 0x%04x\n", + ch->chspec); break; } } diff --git a/drivers/net/wireless/broadcom/brcm80211/include/brcm_hw_ids.h b/drivers/net/wireless/broadcom/brcm80211/include/brcm_hw_ids.h index df3b67ba4db290..f749337a06942e 100644 --- a/drivers/net/wireless/broadcom/brcm80211/include/brcm_hw_ids.h +++ b/drivers/net/wireless/broadcom/brcm80211/include/brcm_hw_ids.h @@ -57,6 +57,7 @@ #define BRCM_CC_4377_CHIP_ID 0x4377 #define BRCM_CC_4378_CHIP_ID 0x4378 #define BRCM_CC_4387_CHIP_ID 0x4387 +#define BRCM_CC_4388_CHIP_ID 0x4388 #define CY_CC_4373_CHIP_ID 0x4373 #define CY_CC_43012_CHIP_ID 43012 #define CY_CC_43439_CHIP_ID 43439 @@ -99,6 +100,7 @@ #define BRCM_PCIE_4377_DEVICE_ID 0x4488 #define BRCM_PCIE_4378_DEVICE_ID 0x4425 #define BRCM_PCIE_4387_DEVICE_ID 0x4433 +#define BRCM_PCIE_4388_DEVICE_ID 0x4434 #define CY_PCIE_54591_DEVICE_ID 0x4417 /* brcmsmac IDs */ diff --git a/drivers/net/wireless/broadcom/brcm80211/include/brcmu_d11.h b/drivers/net/wireless/broadcom/brcm80211/include/brcmu_d11.h index f6344023855c36..bb48b744206223 100644 --- a/drivers/net/wireless/broadcom/brcm80211/include/brcmu_d11.h +++ b/drivers/net/wireless/broadcom/brcm80211/include/brcmu_d11.h @@ -69,24 +69,44 @@ #define BRCMU_CHSPEC_D11AC_SB_UU BRCMU_CHSPEC_D11AC_SB_LUU #define BRCMU_CHSPEC_D11AC_SB_L BRCMU_CHSPEC_D11AC_SB_LLL #define BRCMU_CHSPEC_D11AC_SB_U BRCMU_CHSPEC_D11AC_SB_LLU +/* channel sideband indication for frequency >= 240MHz */ +#define BRCMU_CHSPEC_D11AC_320_SB_MASK 0x0780 +#define BRCMU_CHSPEC_D11AC_320_SB_SHIFT 7 +#define BRCMU_CHSPEC_D11AC_SB_LLLL 0x0000 +#define BRCMU_CHSPEC_D11AC_SB_LLLU 0x0080 +#define BRCMU_CHSPEC_D11AC_SB_LLUL 0x0100 +#define BRCMU_CHSPEC_D11AC_SB_LLUU 0x0180 +#define BRCMU_CHSPEC_D11AC_SB_LULL 0x0200 +#define BRCMU_CHSPEC_D11AC_SB_LULU 0x0280 +#define BRCMU_CHSPEC_D11AC_SB_LUUL 0x0300 +#define BRCMU_CHSPEC_D11AC_SB_LUUU 0x0380 +#define BRCMU_CHSPEC_D11AC_SB_ULLL 0x0400 +#define BRCMU_CHSPEC_D11AC_SB_ULLU 0x0480 +#define BRCMU_CHSPEC_D11AC_SB_ULUL 0x0500 +#define BRCMU_CHSPEC_D11AC_SB_ULUU 0x0580 +#define BRCMU_CHSPEC_D11AC_SB_UULL 0x0600 +#define BRCMU_CHSPEC_D11AC_SB_UULU 0x0680 +#define BRCMU_CHSPEC_D11AC_SB_UUUL 0x0700 +#define BRCMU_CHSPEC_D11AC_SB_UUUU 0x0780 #define BRCMU_CHSPEC_D11AC_BW_MASK 0x3800 #define BRCMU_CHSPEC_D11AC_BW_SHIFT 11 -#define BRCMU_CHSPEC_D11AC_BW_5 0x0000 -#define BRCMU_CHSPEC_D11AC_BW_10 0x0800 -#define BRCMU_CHSPEC_D11AC_BW_20 0x1000 -#define BRCMU_CHSPEC_D11AC_BW_40 0x1800 -#define BRCMU_CHSPEC_D11AC_BW_80 0x2000 -#define BRCMU_CHSPEC_D11AC_BW_160 0x2800 -#define BRCMU_CHSPEC_D11AC_BW_8080 0x3000 -#define BRCMU_CHSPEC_D11AC_BND_MASK 0xc000 -#define BRCMU_CHSPEC_D11AC_BND_SHIFT 14 -#define BRCMU_CHSPEC_D11AC_BND_2G 0x0000 -#define BRCMU_CHSPEC_D11AC_BND_3G 0x4000 -#define BRCMU_CHSPEC_D11AC_BND_4G 0x8000 -#define BRCMU_CHSPEC_D11AC_BND_5G 0xc000 +#define BRCMU_CHSPEC_D11AC_BW_10 0x0800 +#define BRCMU_CHSPEC_D11AC_BW_20 0x1000 +#define BRCMU_CHSPEC_D11AC_BW_40 0x1800 +#define BRCMU_CHSPEC_D11AC_BW_80 0x2000 +#define BRCMU_CHSPEC_D11AC_BW_160 0x2800 +#define BRCMU_CHSPEC_D11AC_BW_320 0x0000 +#define BRCMU_CHSPEC_D11AC_BW_8080 0x3000 +#define BRCMU_CHSPEC_D11AC_BND_MASK 0xc000 +#define BRCMU_CHSPEC_D11AC_BND_SHIFT 14 +#define BRCMU_CHSPEC_D11AC_BND_2G 0x0000 +#define BRCMU_CHSPEC_D11AC_BND_4G 0x8000 +#define BRCMU_CHSPEC_D11AC_BND_5G 0xc000 +#define BRCMU_CHSPEC_D11AC_BND_6G 0x4000 #define BRCMU_CHAN_BAND_2G 0 #define BRCMU_CHAN_BAND_5G 1 +#define BRCMU_CHAN_BAND_6G 2 enum brcmu_chan_bw { BRCMU_CHAN_BW_20, diff --git a/drivers/net/wireless/broadcom/brcm80211/include/brcmu_wifi.h b/drivers/net/wireless/broadcom/brcm80211/include/brcmu_wifi.h index 7552bdb91991ce..ef042beeb586f9 100644 --- a/drivers/net/wireless/broadcom/brcm80211/include/brcmu_wifi.h +++ b/drivers/net/wireless/broadcom/brcm80211/include/brcmu_wifi.h @@ -31,6 +31,7 @@ /* bandstate array indices */ #define BAND_2G_INDEX 0 /* wlc->bandstate[x] index */ #define BAND_5G_INDEX 1 /* wlc->bandstate[x] index */ +#define BAND_6G_INDEX 2 /* wlc->bandstate[x] index */ /* * max # supported channels. The max channel no is 216, this is that + 1 @@ -48,17 +49,22 @@ #define WL_CHANSPEC_CTL_SB_UPPER 0x0200 #define WL_CHANSPEC_CTL_SB_NONE 0x0300 -#define WL_CHANSPEC_BW_MASK 0x0C00 -#define WL_CHANSPEC_BW_SHIFT 10 +#define WL_CHANSPEC_BW_MASK 0x3800 +#define WL_CHANSPEC_BW_SHIFT 11 #define WL_CHANSPEC_BW_10 0x0400 #define WL_CHANSPEC_BW_20 0x0800 #define WL_CHANSPEC_BW_40 0x0C00 #define WL_CHANSPEC_BW_80 0x2000 - -#define WL_CHANSPEC_BAND_MASK 0xf000 -#define WL_CHANSPEC_BAND_SHIFT 12 -#define WL_CHANSPEC_BAND_5G 0x1000 -#define WL_CHANSPEC_BAND_2G 0x2000 +#define WL_CHANSPEC_BW_160 0x2800 +#define WL_CHANSPEC_BW_8080 0x3000 +#define WL_CHANSPEC_BW_320 0x0000 + +#define WL_CHANSPEC_BAND_MASK 0xc000 +#define WL_CHANSPEC_BAND_SHIFT 14 +#define WL_CHANSPEC_BAND_2G 0x0000 +#define WL_CHANSPEC_BAND_4G 0x8000 +#define WL_CHANSPEC_BAND_5G 0xc000 +#define WL_CHANSPEC_BAND_6G 0x4000 #define INVCHANSPEC 255 #define WL_CHAN_VALID_HW (1 << 0) /* valid with current HW */ @@ -93,6 +99,9 @@ #define WLC_BAND_5G 1 /* 5 Ghz */ #define WLC_BAND_2G 2 /* 2.4 Ghz */ #define WLC_BAND_ALL 3 /* all bands */ +#define WLC_BAND_6G 4 /* 6 Ghz */ + +#define WLC_AP_IOV_OP_MANUAL_AP_BSSCFG_CREATE 2 #define CHSPEC_CHANNEL(chspec) ((u8)((chspec) & WL_CHANSPEC_CHAN_MASK)) #define CHSPEC_BAND(chspec) ((chspec) & WL_CHANSPEC_BAND_MASK) @@ -112,6 +121,12 @@ #define CHSPEC_IS80(chspec) \ (((chspec) & WL_CHANSPEC_BW_MASK) == WL_CHANSPEC_BW_80) +#define CHSPEC_IS160(chspec) \ + (((chspec) & WL_CHANSPEC_BW_MASK) == WL_CHANSPEC_BW_160) + +#define CHSPEC_IS6G(chspec) \ + (((chspec) & WL_CHANSPEC_BAND_MASK) == WL_CHANSPEC_BAND_6G) + #define CHSPEC_IS5G(chspec) \ (((chspec) & WL_CHANSPEC_BAND_MASK) == WL_CHANSPEC_BAND_5G) @@ -200,6 +215,13 @@ static inline bool ac_bitmap_tst(u8 bitmap, int prec) #define CRYPTO_ALGO_AES_RESERVED1 5 #define CRYPTO_ALGO_AES_RESERVED2 6 #define CRYPTO_ALGO_NALG 7 +#define CRYPTO_ALGO_AES_GCM 14 /* 128 bit GCM */ +#define CRYPTO_ALGO_AES_CCM256 15 /* 256 bit CCM */ +#define CRYPTO_ALGO_AES_GCM256 16 /* 256 bit GCM */ +#define CRYPTO_ALGO_BIP_CMAC256 17 /* 256 bit BIP CMAC */ +#define CRYPTO_ALGO_BIP_GMAC 18 /* 128 bit BIP GMAC */ +#define CRYPTO_ALGO_BIP_GMAC256 19 /* 256 bit BIP GMAC */ + /* wireless security bitvec */ @@ -232,6 +254,13 @@ static inline bool ac_bitmap_tst(u8 bitmap, int prec) #define WPA2_AUTH_PSK_SHA256 0x8000 /* PSK with SHA256 key derivation */ #define WPA3_AUTH_SAE_PSK 0x40000 /* SAE with 4-way handshake */ +#define WPA3_AUTH_OWE 0x100000 /* OWE */ +#define WFA_AUTH_DPP 0x200000 /* WFA DPP AUTH */ +#define WPA3_AUTH_1X_SUITE_B_SHA384 0x400000 /* Suite B-192 SHA384 */ + + +#define WFA_OUI "\x50\x6F\x9A" /* WFA OUI */ +#define DPP_VER 0x1A /* WFA DPP v1.0 */ #define DOT11_DEFAULT_RTS_LEN 2347 #define DOT11_DEFAULT_FRAG_LEN 2346 diff --git a/drivers/net/wireless/broadcom/brcm80211/include/chipcommon.h b/drivers/net/wireless/broadcom/brcm80211/include/chipcommon.h index 0340bba968688f..5c3b8fb41194ae 100644 --- a/drivers/net/wireless/broadcom/brcm80211/include/chipcommon.h +++ b/drivers/net/wireless/broadcom/brcm80211/include/chipcommon.h @@ -302,6 +302,14 @@ struct chipcregs { #define PMU_RCTL_LOGIC_DISABLE_MASK (1 << 27) +/* watchdog */ +#define CC_WD_SSRESET_PCIE_F0_EN 0x10000000 +#define CC_WD_SSRESET_PCIE_F1_EN 0x20000000 +#define CC_WD_SSRESET_PCIE_F2_EN 0x40000000 +#define CC_WD_SSRESET_PCIE_ALL_FN_EN 0x80000000 +#define CC_WD_COUNTER_MASK 0x0fffffff +#define CC_WD_ENABLE_MASK 0xf0000000 + /* * Maximum delay for the PMU state transition in us. * This is an upper bound intended for spinwaits etc. diff --git a/drivers/net/wireless/intel/ipw2x00/ipw2200.c b/drivers/net/wireless/intel/ipw2x00/ipw2200.c index 09035a77e775f5..b0e769da941569 100644 --- a/drivers/net/wireless/intel/ipw2x00/ipw2200.c +++ b/drivers/net/wireless/intel/ipw2x00/ipw2200.c @@ -11387,7 +11387,13 @@ static const struct pci_device_id card_ids[] = { {PCI_VENDOR_ID_INTEL, 0x1043, 0x8086, 0x2754, 0, 0, 0}, {PCI_VENDOR_ID_INTEL, 0x1043, 0x8086, 0x2761, 0, 0, 0}, {PCI_VENDOR_ID_INTEL, 0x1043, 0x8086, 0x2762, 0, 0, 0}, - {PCI_VDEVICE(INTEL, 0x104f), 0}, + /* + * This ID conflicts with i40e, but the devices can be differentiated + * because i40e devices use PCI_CLASS_NETWORK_ETHERNET and ipw2200 + * devices use PCI_CLASS_NETWORK_OTHER. + */ + {PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x104f), + PCI_CLASS_NETWORK_OTHER << 8, 0xffff00, 0}, {PCI_VDEVICE(INTEL, 0x4220), 0}, /* BG */ {PCI_VDEVICE(INTEL, 0x4221), 0}, /* BG */ {PCI_VDEVICE(INTEL, 0x4223), 0}, /* ABG */ diff --git a/drivers/net/wireless/intel/iwlegacy/3945-mac.c b/drivers/net/wireless/intel/iwlegacy/3945-mac.c index 104748fcdc33e9..54991f31c52c56 100644 --- a/drivers/net/wireless/intel/iwlegacy/3945-mac.c +++ b/drivers/net/wireless/intel/iwlegacy/3945-mac.c @@ -3224,7 +3224,9 @@ il3945_store_measurement(struct device *d, struct device_attribute *attr, D_INFO("Invoking measurement of type %d on " "channel %d (for '%s')\n", type, params.channel, buf); + mutex_lock(&il->mutex); il3945_get_measurement(il, ¶ms, type); + mutex_unlock(&il->mutex); return count; } diff --git a/drivers/net/wireless/intel/iwlegacy/4965-mac.c b/drivers/net/wireless/intel/iwlegacy/4965-mac.c index 3588dec75ebdd9..57fa866efd9f87 100644 --- a/drivers/net/wireless/intel/iwlegacy/4965-mac.c +++ b/drivers/net/wireless/intel/iwlegacy/4965-mac.c @@ -4606,7 +4606,9 @@ il4965_store_tx_power(struct device *d, struct device_attribute *attr, if (ret) IL_INFO("%s is not in decimal form.\n", buf); else { + mutex_lock(&il->mutex); ret = il_set_tx_power(il, val, false); + mutex_unlock(&il->mutex); if (ret) IL_ERR("failed setting tx power (0x%08x).\n", ret); else diff --git a/drivers/net/wireless/intel/iwlwifi/fw/api/commands.h b/drivers/net/wireless/intel/iwlwifi/fw/api/commands.h index 8d64a271bb9452..36159a76991674 100644 --- a/drivers/net/wireless/intel/iwlwifi/fw/api/commands.h +++ b/drivers/net/wireless/intel/iwlwifi/fw/api/commands.h @@ -296,6 +296,11 @@ enum iwl_legacy_cmds { */ SCAN_OFFLOAD_UPDATE_PROFILES_CMD = 0x6E, + /** + * @SCAN_START_NOTIFICATION_UMAC: uses &struct iwl_umac_scan_start + */ + SCAN_START_NOTIFICATION_UMAC = 0xb2, + /** * @MATCH_FOUND_NOTIFICATION: scan match found */ diff --git a/drivers/net/wireless/intel/iwlwifi/fw/api/scan.h b/drivers/net/wireless/intel/iwlwifi/fw/api/scan.h index 60f0a4924ddfb0..46fcc32608e34e 100644 --- a/drivers/net/wireless/intel/iwlwifi/fw/api/scan.h +++ b/drivers/net/wireless/intel/iwlwifi/fw/api/scan.h @@ -1156,6 +1156,16 @@ enum iwl_umac_scan_abort_status { IWL_UMAC_SCAN_ABORT_STATUS_NOT_FOUND, }; +/** + * struct iwl_umac_scan_start - scan start notification + * @uid: scan id, &enum iwl_umac_scan_uid_offsets + * @reserved: for future use + */ +struct iwl_umac_scan_start { + __le32 uid; + __le32 reserved; +} __packed; /* SCAN_START_UMAC_API_S_VER_1 */ + /** * struct iwl_umac_scan_complete - scan complete notification * @uid: scan id, &enum iwl_umac_scan_uid_offsets diff --git a/drivers/net/wireless/intel/iwlwifi/fw/smem.c b/drivers/net/wireless/intel/iwlwifi/fw/smem.c index 90fd69b4860c1a..344ddde85b1897 100644 --- a/drivers/net/wireless/intel/iwlwifi/fw/smem.c +++ b/drivers/net/wireless/intel/iwlwifi/fw/smem.c @@ -6,6 +6,7 @@ */ #include "iwl-drv.h" #include "runtime.h" +#include "dbg.h" #include "fw/api/commands.h" static void iwl_parse_shared_mem_22000(struct iwl_fw_runtime *fwrt, @@ -17,7 +18,9 @@ static void iwl_parse_shared_mem_22000(struct iwl_fw_runtime *fwrt, u8 api_ver = iwl_fw_lookup_notif_ver(fwrt->fw, SYSTEM_GROUP, SHARED_MEM_CFG_CMD, 0); - if (WARN_ON(lmac_num > ARRAY_SIZE(mem_cfg->lmac_smem))) + /* Note: notification has 3 entries, but we only expect 2 */ + if (IWL_FW_CHECK(fwrt, lmac_num > ARRAY_SIZE(fwrt->smem_cfg.lmac), + "FW advertises %d LMACs\n", lmac_num)) return; fwrt->smem_cfg.num_lmacs = lmac_num; @@ -26,7 +29,8 @@ static void iwl_parse_shared_mem_22000(struct iwl_fw_runtime *fwrt, fwrt->smem_cfg.rxfifo2_size = le32_to_cpu(mem_cfg->rxfifo2_size); if (api_ver >= 4 && - !WARN_ON_ONCE(iwl_rx_packet_payload_len(pkt) < sizeof(*mem_cfg))) { + !IWL_FW_CHECK(fwrt, iwl_rx_packet_payload_len(pkt) < sizeof(*mem_cfg), + "bad shared mem notification size\n")) { fwrt->smem_cfg.rxfifo2_control_size = le32_to_cpu(mem_cfg->rxfifo2_control_size); } diff --git a/drivers/net/wireless/intel/iwlwifi/mld/iface.c b/drivers/net/wireless/intel/iwlwifi/mld/iface.c index f15d1f5d1bf593..a770ee5e0e73a6 100644 --- a/drivers/net/wireless/intel/iwlwifi/mld/iface.c +++ b/drivers/net/wireless/intel/iwlwifi/mld/iface.c @@ -111,14 +111,75 @@ static bool iwl_mld_is_nic_ack_enabled(struct iwl_mld *mld, IEEE80211_HE_MAC_CAP2_ACK_EN); } -static void iwl_mld_set_he_support(struct iwl_mld *mld, - struct ieee80211_vif *vif, - struct iwl_mac_config_cmd *cmd) +struct iwl_mld_mac_wifi_gen_sta_iter_data { + struct ieee80211_vif *vif; + struct iwl_mac_wifi_gen_support *support; +}; + +static void iwl_mld_mac_wifi_gen_sta_iter(void *_data, + struct ieee80211_sta *sta) { - if (vif->type == NL80211_IFTYPE_AP) - cmd->wifi_gen.he_ap_support = 1; - else - cmd->wifi_gen.he_support = 1; + struct iwl_mld_sta *mld_sta = iwl_mld_sta_from_mac80211(sta); + struct iwl_mld_mac_wifi_gen_sta_iter_data *data = _data; + struct ieee80211_link_sta *link_sta; + unsigned int link_id; + + if (mld_sta->vif != data->vif) + return; + + for_each_sta_active_link(data->vif, sta, link_sta, link_id) { + if (link_sta->he_cap.has_he) + data->support->he_support = 1; + if (link_sta->eht_cap.has_eht) + data->support->eht_support = 1; + } +} + +static void iwl_mld_set_wifi_gen(struct iwl_mld *mld, + struct ieee80211_vif *vif, + struct iwl_mac_wifi_gen_support *support) +{ + struct iwl_mld_mac_wifi_gen_sta_iter_data sta_iter_data = { + .vif = vif, + .support = support, + }; + struct ieee80211_bss_conf *link_conf; + unsigned int link_id; + + switch (vif->type) { + case NL80211_IFTYPE_MONITOR: + /* for sniffer, set to HW capabilities */ + support->he_support = 1; + support->eht_support = mld->trans->cfg->eht_supported; + break; + case NL80211_IFTYPE_AP: + /* for AP set according to the link configs */ + for_each_vif_active_link(vif, link_conf, link_id) { + support->he_ap_support |= link_conf->he_support; + support->eht_support |= link_conf->eht_support; + } + break; + default: + /* + * If we have MLO enabled, then the firmware needs to enable + * address translation for the station(s) we add. That depends + * on having EHT enabled in firmware, which in turn depends on + * mac80211 in the iteration below. + * However, mac80211 doesn't enable capabilities on the AP STA + * until it has parsed the association response successfully, + * so set EHT (and HE as a pre-requisite for EHT) when the vif + * is an MLD. + */ + if (ieee80211_vif_is_mld(vif)) { + support->he_support = 1; + support->eht_support = 1; + } + + ieee80211_iterate_stations_mtx(mld->hw, + iwl_mld_mac_wifi_gen_sta_iter, + &sta_iter_data); + break; + } } /* fill the common part for all interface types */ @@ -128,8 +189,6 @@ static void iwl_mld_mac_cmd_fill_common(struct iwl_mld *mld, u32 action) { struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif); - struct ieee80211_bss_conf *link_conf; - unsigned int link_id; lockdep_assert_wiphy(mld->wiphy); @@ -147,29 +206,7 @@ static void iwl_mld_mac_cmd_fill_common(struct iwl_mld *mld, cmd->nic_not_ack_enabled = cpu_to_le32(!iwl_mld_is_nic_ack_enabled(mld, vif)); - /* If we have MLO enabled, then the firmware needs to enable - * address translation for the station(s) we add. That depends - * on having EHT enabled in firmware, which in turn depends on - * mac80211 in the code below. - * However, mac80211 doesn't enable HE/EHT until it has parsed - * the association response successfully, so just skip all that - * and enable both when we have MLO. - */ - if (ieee80211_vif_is_mld(vif)) { - iwl_mld_set_he_support(mld, vif, cmd); - cmd->wifi_gen.eht_support = 1; - return; - } - - for_each_vif_active_link(vif, link_conf, link_id) { - if (!link_conf->he_support) - continue; - - iwl_mld_set_he_support(mld, vif, cmd); - - /* EHT, if supported, was already set above */ - break; - } + iwl_mld_set_wifi_gen(mld, vif, &cmd->wifi_gen); } static void iwl_mld_fill_mac_cmd_sta(struct iwl_mld *mld, diff --git a/drivers/net/wireless/intel/iwlwifi/mld/mac80211.c b/drivers/net/wireless/intel/iwlwifi/mld/mac80211.c index cd0dce8de85690..77793da147b732 100644 --- a/drivers/net/wireless/intel/iwlwifi/mld/mac80211.c +++ b/drivers/net/wireless/intel/iwlwifi/mld/mac80211.c @@ -984,7 +984,9 @@ int iwl_mld_assign_vif_chanctx(struct ieee80211_hw *hw, { struct iwl_mld *mld = IWL_MAC80211_GET_MLD(hw); struct iwl_mld_link *mld_link = iwl_mld_link_from_mac80211(link); - unsigned int n_active = iwl_mld_count_active_links(mld, vif); + struct iwl_mld_link *temp_mld_link; + struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif); + u16 final_active_links = 0; int ret; lockdep_assert_wiphy(mld->wiphy); @@ -992,10 +994,7 @@ int iwl_mld_assign_vif_chanctx(struct ieee80211_hw *hw, if (WARN_ON(!mld_link)) return -EINVAL; - /* if the assigned one was not counted yet, count it now */ if (!rcu_access_pointer(mld_link->chan_ctx)) { - n_active++; - /* Track addition of non-BSS link */ if (ieee80211_vif_type_p2p(vif) != NL80211_IFTYPE_STATION) { ret = iwl_mld_emlsr_check_non_bss_block(mld, 1); @@ -1016,17 +1015,25 @@ int iwl_mld_assign_vif_chanctx(struct ieee80211_hw *hw, rcu_assign_pointer(mld_link->chan_ctx, ctx); - if (n_active > 1) { - struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif); + /* We cannot rely on vif->active_links at this stage as it contains + * both the removed links and the newly added links. + * Therefore, we create our own bitmap of the final active links, + * which does not include the removed links. + */ + for_each_mld_vif_valid_link(mld_vif, temp_mld_link) { + if (rcu_access_pointer(temp_mld_link->chan_ctx)) + final_active_links |= BIT(link_id); + } + if (hweight16(final_active_links) > 1) { /* Indicate to mac80211 that EML is enabled */ vif->driver_flags |= IEEE80211_VIF_EML_ACTIVE; mld_vif->emlsr.last_entry_ts = jiffies; - if (vif->active_links & BIT(mld_vif->emlsr.selected_links)) + if (final_active_links == mld_vif->emlsr.selected_links) mld_vif->emlsr.primary = mld_vif->emlsr.selected_primary; else - mld_vif->emlsr.primary = __ffs(vif->active_links); + mld_vif->emlsr.primary = __ffs(final_active_links); iwl_dbg_tlv_time_point(&mld->fwrt, IWL_FW_INI_TIME_ESR_LINK_UP, NULL); @@ -1683,6 +1690,16 @@ static int iwl_mld_move_sta_state_up(struct iwl_mld *mld, if (vif->type == NL80211_IFTYPE_STATION) iwl_mld_link_set_2mhz_block(mld, vif, sta); + + if (sta->tdls) { + /* + * update MAC since wifi generation flags may change, + * we also update MAC on association to the AP via the + * vif assoc change + */ + iwl_mld_mac_fw_action(mld, vif, FW_CTXT_ACTION_MODIFY); + } + /* Now the link_sta's capabilities are set, update the FW */ iwl_mld_config_tlc(mld, vif, sta); @@ -1792,6 +1809,15 @@ static int iwl_mld_move_sta_state_down(struct iwl_mld *mld, /* just removed last TDLS STA, so enable PM */ iwl_mld_update_mac_power(mld, vif, false); } + + if (sta->tdls) { + /* + * update MAC since wifi generation flags may change, + * we also update MAC on disassociation to the AP via + * the vif assoc change + */ + iwl_mld_mac_fw_action(mld, vif, FW_CTXT_ACTION_MODIFY); + } } else { return -EINVAL; } diff --git a/drivers/net/wireless/intel/iwlwifi/mld/mld.c b/drivers/net/wireless/intel/iwlwifi/mld/mld.c index 8a4c96385640bf..18691871bdacc0 100644 --- a/drivers/net/wireless/intel/iwlwifi/mld/mld.c +++ b/drivers/net/wireless/intel/iwlwifi/mld/mld.c @@ -171,6 +171,7 @@ static const struct iwl_hcmd_names iwl_mld_legacy_names[] = { HCMD_NAME(MISSED_BEACONS_NOTIFICATION), HCMD_NAME(MAC_PM_POWER_TABLE), HCMD_NAME(MFUART_LOAD_NOTIFICATION), + HCMD_NAME(SCAN_START_NOTIFICATION_UMAC), HCMD_NAME(RSS_CONFIG_CMD), HCMD_NAME(SCAN_ITERATION_COMPLETE_UMAC), HCMD_NAME(REPLY_RX_MPDU_CMD), diff --git a/drivers/net/wireless/intel/iwlwifi/mld/mlo.c b/drivers/net/wireless/intel/iwlwifi/mld/mlo.c index c6b151f269216b..47d36f0149efbc 100644 --- a/drivers/net/wireless/intel/iwlwifi/mld/mlo.c +++ b/drivers/net/wireless/intel/iwlwifi/mld/mlo.c @@ -725,7 +725,7 @@ iwl_mld_set_link_sel_data(struct iwl_mld *mld, /* Ignore any BSS that was not seen in the last MLO scan */ if (ktime_before(link_conf->bss->ts_boottime, - mld->scan.last_mlo_scan_time)) + mld->scan.last_mlo_scan_start_time)) continue; data[n_data].link_id = link_id; @@ -844,9 +844,9 @@ iwl_mld_emlsr_pair_state(struct ieee80211_vif *vif, if (c_low->chan->center_freq > c_high->chan->center_freq) swap(c_low, c_high); - c_low_upper_edge = c_low->chan->center_freq + + c_low_upper_edge = c_low->center_freq1 + cfg80211_chandef_get_width(c_low) / 2; - c_high_lower_edge = c_high->chan->center_freq - + c_high_lower_edge = c_high->center_freq1 - cfg80211_chandef_get_width(c_high) / 2; if (a->chandef->chan->band == NL80211_BAND_5GHZ && @@ -931,7 +931,7 @@ static void _iwl_mld_select_links(struct iwl_mld *mld, if (!mld_vif->authorized || hweight16(usable_links) <= 1) return; - if (WARN(ktime_before(mld->scan.last_mlo_scan_time, + if (WARN(ktime_before(mld->scan.last_mlo_scan_start_time, ktime_sub_ns(ktime_get_boottime_ns(), 5ULL * NSEC_PER_SEC)), "Last MLO scan was too long ago, can't select links\n")) diff --git a/drivers/net/wireless/intel/iwlwifi/mld/notif.c b/drivers/net/wireless/intel/iwlwifi/mld/notif.c index 4cf3920b005fe7..ca90a2f905262c 100644 --- a/drivers/net/wireless/intel/iwlwifi/mld/notif.c +++ b/drivers/net/wireless/intel/iwlwifi/mld/notif.c @@ -284,6 +284,8 @@ static void iwl_mld_handle_beacon_notification(struct iwl_mld *mld, * at least enough bytes to cover the structure listed in the CMD_VER_ENTRY. */ +CMD_VERSIONS(scan_start_notif, + CMD_VER_ENTRY(1, iwl_umac_scan_start)) CMD_VERSIONS(scan_complete_notif, CMD_VER_ENTRY(1, iwl_umac_scan_complete)) CMD_VERSIONS(scan_iter_complete_notif, @@ -355,6 +357,7 @@ DEFINE_SIMPLE_CANCELLATION(datapath_monitor, iwl_datapath_monitor_notif, link_id) DEFINE_SIMPLE_CANCELLATION(roc, iwl_roc_notif, activity) DEFINE_SIMPLE_CANCELLATION(scan_complete, iwl_umac_scan_complete, uid) +DEFINE_SIMPLE_CANCELLATION(scan_start, iwl_umac_scan_start, uid) DEFINE_SIMPLE_CANCELLATION(probe_resp_data, iwl_probe_resp_data_notif, mac_id) DEFINE_SIMPLE_CANCELLATION(uapsd_misbehaving_ap, iwl_uapsd_misbehaving_ap_notif, @@ -397,6 +400,8 @@ const struct iwl_rx_handler iwl_mld_rx_handlers[] = { RX_HANDLER_SYNC) RX_HANDLER_NO_OBJECT(LEGACY_GROUP, BA_NOTIF, compressed_ba_notif, RX_HANDLER_SYNC) + RX_HANDLER_OF_SCAN(LEGACY_GROUP, SCAN_START_NOTIFICATION_UMAC, + scan_start_notif) RX_HANDLER_OF_SCAN(LEGACY_GROUP, SCAN_COMPLETE_UMAC, scan_complete_notif) RX_HANDLER_NO_OBJECT(LEGACY_GROUP, SCAN_ITERATION_COMPLETE_UMAC, diff --git a/drivers/net/wireless/intel/iwlwifi/mld/scan.c b/drivers/net/wireless/intel/iwlwifi/mld/scan.c index fd1022ddc9122b..76ac6fd5f9ff36 100644 --- a/drivers/net/wireless/intel/iwlwifi/mld/scan.c +++ b/drivers/net/wireless/intel/iwlwifi/mld/scan.c @@ -473,6 +473,9 @@ iwl_mld_scan_get_cmd_gen_flags(struct iwl_mld *mld, params->flags & NL80211_SCAN_FLAG_COLOCATED_6GHZ) flags |= IWL_UMAC_SCAN_GEN_FLAGS_V2_TRIGGER_UHB_SCAN; + if (scan_status == IWL_MLD_SCAN_INT_MLO) + flags |= IWL_UMAC_SCAN_GEN_FLAGS_V2_NTF_START; + if (params->enable_6ghz_passive) flags |= IWL_UMAC_SCAN_GEN_FLAGS_V2_6GHZ_PASSIVE_SCAN; @@ -1817,9 +1820,6 @@ static void iwl_mld_int_mlo_scan_start(struct iwl_mld *mld, ret = _iwl_mld_single_scan_start(mld, vif, req, &ies, IWL_MLD_SCAN_INT_MLO); - if (!ret) - mld->scan.last_mlo_scan_time = ktime_get_boottime_ns(); - IWL_DEBUG_SCAN(mld, "Internal MLO scan: ret=%d\n", ret); } @@ -1904,6 +1904,30 @@ void iwl_mld_handle_match_found_notif(struct iwl_mld *mld, ieee80211_sched_scan_results(mld->hw); } +void iwl_mld_handle_scan_start_notif(struct iwl_mld *mld, + struct iwl_rx_packet *pkt) +{ + struct iwl_umac_scan_complete *notif = (void *)pkt->data; + u32 uid = le32_to_cpu(notif->uid); + + if (IWL_FW_CHECK(mld, uid >= ARRAY_SIZE(mld->scan.uid_status), + "FW reports out-of-range scan UID %d\n", uid)) + return; + + if (IWL_FW_CHECK(mld, !(mld->scan.uid_status[uid] & mld->scan.status), + "FW reports scan UID %d we didn't trigger\n", uid)) + return; + + IWL_DEBUG_SCAN(mld, "Scan started: uid=%u type=%u\n", uid, + mld->scan.uid_status[uid]); + if (IWL_FW_CHECK(mld, mld->scan.uid_status[uid] != IWL_MLD_SCAN_INT_MLO, + "FW reports scan start notification %d we didn't trigger\n", + mld->scan.uid_status[uid])) + return; + + mld->scan.last_mlo_scan_start_time = ktime_get_boottime_ns(); +} + void iwl_mld_handle_scan_complete_notif(struct iwl_mld *mld, struct iwl_rx_packet *pkt) { diff --git a/drivers/net/wireless/intel/iwlwifi/mld/scan.h b/drivers/net/wireless/intel/iwlwifi/mld/scan.h index 69110f0cfc8e2a..de5620e7f463b5 100644 --- a/drivers/net/wireless/intel/iwlwifi/mld/scan.h +++ b/drivers/net/wireless/intel/iwlwifi/mld/scan.h @@ -27,6 +27,9 @@ int iwl_mld_sched_scan_start(struct iwl_mld *mld, void iwl_mld_handle_match_found_notif(struct iwl_mld *mld, struct iwl_rx_packet *pkt); +void iwl_mld_handle_scan_start_notif(struct iwl_mld *mld, + struct iwl_rx_packet *pkt); + void iwl_mld_handle_scan_complete_notif(struct iwl_mld *mld, struct iwl_rx_packet *pkt); @@ -114,8 +117,8 @@ enum iwl_mld_traffic_load { * in jiffies. * @last_start_time_jiffies: stores the last start time in jiffies * (interface up/reset/resume). - * @last_mlo_scan_time: start time of the last MLO scan in nanoseconds since - * boot. + * @last_mlo_scan_start_time: start time of the last MLO scan in nanoseconds + * since boot. */ struct iwl_mld_scan { /* Add here fields that need clean up on restart */ @@ -136,7 +139,7 @@ struct iwl_mld_scan { void *cmd; unsigned long last_6ghz_passive_jiffies; unsigned long last_start_time_jiffies; - u64 last_mlo_scan_time; + u64 last_mlo_scan_start_time; }; /** diff --git a/drivers/net/wireless/intel/iwlwifi/mld/tx.c b/drivers/net/wireless/intel/iwlwifi/mld/tx.c index 3b4b575aadaa5d..e3fb4fc4f452e2 100644 --- a/drivers/net/wireless/intel/iwlwifi/mld/tx.c +++ b/drivers/net/wireless/intel/iwlwifi/mld/tx.c @@ -345,6 +345,11 @@ u8 iwl_mld_get_lowest_rate(struct iwl_mld *mld, iwl_mld_get_basic_rates_and_band(mld, vif, info, &basic_rates, &band); + if (band >= NUM_NL80211_BANDS) { + WARN_ON(vif->type != NL80211_IFTYPE_NAN); + return IWL_FIRST_OFDM_RATE; + } + sband = mld->hw->wiphy->bands[band]; for_each_set_bit(i, &basic_rates, BITS_PER_LONG) { u16 hw = sband->bitrates[i].hw_value; diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/d3.c b/drivers/net/wireless/intel/iwlwifi/mvm/d3.c index af1a45845999bb..11afe373961f3d 100644 --- a/drivers/net/wireless/intel/iwlwifi/mvm/d3.c +++ b/drivers/net/wireless/intel/iwlwifi/mvm/d3.c @@ -2834,7 +2834,7 @@ static void iwl_mvm_nd_match_info_handler(struct iwl_mvm *mvm, if (IS_ERR_OR_NULL(vif)) return; - if (len < sizeof(struct iwl_scan_offload_match_info)) { + if (len < sizeof(struct iwl_scan_offload_match_info) + matches_len) { IWL_ERR(mvm, "Invalid scan match info notification\n"); return; } diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/fw.c b/drivers/net/wireless/intel/iwlwifi/mvm/fw.c index edae13755ee61f..b9c9ee30272eca 100644 --- a/drivers/net/wireless/intel/iwlwifi/mvm/fw.c +++ b/drivers/net/wireless/intel/iwlwifi/mvm/fw.c @@ -470,7 +470,8 @@ static void iwl_mvm_uats_init(struct iwl_mvm *mvm) .dataflags[0] = IWL_HCMD_DFL_NOCOPY, }; - if (mvm->trans->mac_cfg->device_family < IWL_DEVICE_FAMILY_AX210) { + if (mvm->trans->mac_cfg->device_family < IWL_DEVICE_FAMILY_AX210 || + !mvm->trans->cfg->uhb_supported) { IWL_DEBUG_RADIO(mvm, "UATS feature is not supported\n"); return; } diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/mac-ctxt.c b/drivers/net/wireless/intel/iwlwifi/mvm/mac-ctxt.c index 867807abde6643..49ffc4ecee855a 100644 --- a/drivers/net/wireless/intel/iwlwifi/mvm/mac-ctxt.c +++ b/drivers/net/wireless/intel/iwlwifi/mvm/mac-ctxt.c @@ -1761,6 +1761,20 @@ void iwl_mvm_probe_resp_data_notif(struct iwl_mvm *mvm, mvmvif = iwl_mvm_vif_from_mac80211(vif); + /* + * len_low should be 2 + n*13 (where n is the number of descriptors. + * 13 is the size of a NoA descriptor). We can have either one or two + * descriptors. + */ + if (IWL_FW_CHECK(mvm, notif->noa_active && + notif->noa_attr.len_low != 2 + + sizeof(struct ieee80211_p2p_noa_desc) && + notif->noa_attr.len_low != 2 + + sizeof(struct ieee80211_p2p_noa_desc) * 2, + "Invalid noa_attr.len_low (%d)\n", + notif->noa_attr.len_low)) + return; + new_data = kzalloc(sizeof(*new_data), GFP_KERNEL); if (!new_data) return; diff --git a/drivers/net/wireless/marvell/libertas/if_usb.c b/drivers/net/wireless/marvell/libertas/if_usb.c index b3c4040257a670..924ab93b7b6714 100644 --- a/drivers/net/wireless/marvell/libertas/if_usb.c +++ b/drivers/net/wireless/marvell/libertas/if_usb.c @@ -426,6 +426,8 @@ static int usb_tx_block(struct if_usb_card *cardp, uint8_t *payload, uint16_t nb goto tx_ret; } + usb_kill_urb(cardp->tx_urb); + usb_fill_bulk_urb(cardp->tx_urb, cardp->udev, usb_sndbulkpipe(cardp->udev, cardp->ep_out), diff --git a/drivers/net/wireless/marvell/libertas/main.c b/drivers/net/wireless/marvell/libertas/main.c index d44e02c6fe3852..dd97f1b61f4d1b 100644 --- a/drivers/net/wireless/marvell/libertas/main.c +++ b/drivers/net/wireless/marvell/libertas/main.c @@ -799,8 +799,8 @@ static void lbs_free_adapter(struct lbs_private *priv) { lbs_free_cmd_buffer(priv); kfifo_free(&priv->event_fifo); - timer_delete(&priv->command_timer); - timer_delete(&priv->tx_lockup_timer); + timer_delete_sync(&priv->command_timer); + timer_delete_sync(&priv->tx_lockup_timer); } static const struct net_device_ops lbs_netdev_ops = { diff --git a/drivers/net/wireless/mediatek/mt76/mt76_connac_mac.c b/drivers/net/wireless/mediatek/mt76/mt76_connac_mac.c index 3304b5971be09e..b41ca1410da922 100644 --- a/drivers/net/wireless/mediatek/mt76/mt76_connac_mac.c +++ b/drivers/net/wireless/mediatek/mt76/mt76_connac_mac.c @@ -413,6 +413,7 @@ mt76_connac2_mac_write_txwi_80211(struct mt76_dev *dev, __le32 *txwi, u32 val; if (ieee80211_is_action(fc) && + skb->len >= IEEE80211_MIN_ACTION_SIZE + 1 + 1 + 2 && mgmt->u.action.category == WLAN_CATEGORY_BACK && mgmt->u.action.u.addba_req.action_code == WLAN_ACTION_ADDBA_REQ) { u16 capab = le16_to_cpu(mgmt->u.action.u.addba_req.capab); diff --git a/drivers/net/wireless/mediatek/mt76/mt7925/mac.c b/drivers/net/wireless/mediatek/mt76/mt7925/mac.c index 871b67101976a7..0d94359004233e 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7925/mac.c +++ b/drivers/net/wireless/mediatek/mt76/mt7925/mac.c @@ -668,6 +668,7 @@ mt7925_mac_write_txwi_80211(struct mt76_dev *dev, __le32 *txwi, u32 val; if (ieee80211_is_action(fc) && + skb->len >= IEEE80211_MIN_ACTION_SIZE + 1 && mgmt->u.action.category == WLAN_CATEGORY_BACK && mgmt->u.action.u.addba_req.action_code == WLAN_ACTION_ADDBA_REQ) tid = MT_TX_ADDBA; diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/mac.c b/drivers/net/wireless/mediatek/mt76/mt7996/mac.c index 2560e2f46e89a4..d4f3ee943b4727 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7996/mac.c +++ b/drivers/net/wireless/mediatek/mt76/mt7996/mac.c @@ -800,6 +800,7 @@ mt7996_mac_write_txwi_80211(struct mt7996_dev *dev, __le32 *txwi, u32 val; if (ieee80211_is_action(fc) && + skb->len >= IEEE80211_MIN_ACTION_SIZE + 1 && mgmt->u.action.category == WLAN_CATEGORY_BACK && mgmt->u.action.u.addba_req.action_code == WLAN_ACTION_ADDBA_REQ) { if (is_mt7990(&dev->mt76)) diff --git a/drivers/net/wireless/mediatek/mt76/scan.c b/drivers/net/wireless/mediatek/mt76/scan.c index ff9176cdee3dee..63b0447e55c15b 100644 --- a/drivers/net/wireless/mediatek/mt76/scan.c +++ b/drivers/net/wireless/mediatek/mt76/scan.c @@ -63,10 +63,8 @@ mt76_scan_send_probe(struct mt76_dev *dev, struct cfg80211_ssid *ssid) rcu_read_lock(); - if (!ieee80211_tx_prepare_skb(phy->hw, vif, skb, band, NULL)) { - ieee80211_free_txskb(phy->hw, skb); + if (!ieee80211_tx_prepare_skb(phy->hw, vif, skb, band, NULL)) goto out; - } info = IEEE80211_SKB_CB(skb); if (req->no_cck) diff --git a/drivers/net/wireless/microchip/wilc1000/hif.c b/drivers/net/wireless/microchip/wilc1000/hif.c index a229c6cab3322d..d795ca05729abe 100644 --- a/drivers/net/wireless/microchip/wilc1000/hif.c +++ b/drivers/net/wireless/microchip/wilc1000/hif.c @@ -163,7 +163,7 @@ int wilc_scan(struct wilc_vif *vif, u8 scan_source, u32 index = 0; u32 i, scan_timeout; u8 *buffer; - u8 valuesize = 0; + u32 valuesize = 0; u8 *search_ssid_vals = NULL; const u8 ch_list_len = request->n_channels; struct host_if_drv *hif_drv = vif->hif_drv; diff --git a/drivers/net/wireless/ralink/rt2x00/rt2x00usb.c b/drivers/net/wireless/ralink/rt2x00/rt2x00usb.c index a6d50149e0c3eb..1275bd8232d213 100644 --- a/drivers/net/wireless/ralink/rt2x00/rt2x00usb.c +++ b/drivers/net/wireless/ralink/rt2x00/rt2x00usb.c @@ -828,7 +828,7 @@ int rt2x00usb_probe(struct usb_interface *usb_intf, if (retval) goto exit_free_device; - rt2x00dev->anchor = devm_kmalloc(&usb_dev->dev, + rt2x00dev->anchor = devm_kmalloc(&usb_intf->dev, sizeof(struct usb_anchor), GFP_KERNEL); if (!rt2x00dev->anchor) { diff --git a/drivers/net/wireless/realtek/rtl8xxxu/core.c b/drivers/net/wireless/realtek/rtl8xxxu/core.c index c06ad064f37c65..f9a527f6a17535 100644 --- a/drivers/net/wireless/realtek/rtl8xxxu/core.c +++ b/drivers/net/wireless/realtek/rtl8xxxu/core.c @@ -7826,6 +7826,7 @@ static int rtl8xxxu_probe(struct usb_interface *interface, goto err_set_intfdata; hw->vif_data_size = sizeof(struct rtl8xxxu_vif); + hw->sta_data_size = sizeof(struct rtl8xxxu_sta_info); hw->wiphy->max_scan_ssids = 1; hw->wiphy->max_scan_ie_len = IEEE80211_MAX_DATA_LEN; diff --git a/drivers/net/wireless/realtek/rtw88/main.c b/drivers/net/wireless/realtek/rtw88/main.c index fa0ed39cb1992a..dde2ea6a00e06c 100644 --- a/drivers/net/wireless/realtek/rtw88/main.c +++ b/drivers/net/wireless/realtek/rtw88/main.c @@ -730,10 +730,10 @@ void rtw_set_rx_freq_band(struct rtw_rx_pkt_stat *pkt_stat, u8 channel) } EXPORT_SYMBOL(rtw_set_rx_freq_band); -void rtw_set_dtim_period(struct rtw_dev *rtwdev, int dtim_period) +void rtw_set_dtim_period(struct rtw_dev *rtwdev, u8 dtim_period) { rtw_write32_set(rtwdev, REG_TCR, BIT_TCR_UPDATE_TIMIE); - rtw_write8(rtwdev, REG_DTIM_COUNTER_ROOT, dtim_period - 1); + rtw_write8(rtwdev, REG_DTIM_COUNTER_ROOT, dtim_period ? dtim_period - 1 : 0); } void rtw_update_channel(struct rtw_dev *rtwdev, u8 center_channel, @@ -1658,14 +1658,41 @@ static u16 rtw_get_max_scan_ie_len(struct rtw_dev *rtwdev) return len; } +static struct ieee80211_supported_band * +rtw_sband_dup(struct rtw_dev *rtwdev, + const struct ieee80211_supported_band *sband) +{ + struct ieee80211_supported_band *dup; + + dup = devm_kmemdup(rtwdev->dev, sband, sizeof(*sband), GFP_KERNEL); + if (!dup) + return NULL; + + dup->channels = devm_kmemdup_array(rtwdev->dev, sband->channels, + sband->n_channels, + sizeof(*sband->channels), + GFP_KERNEL); + if (!dup->channels) + return NULL; + + dup->bitrates = devm_kmemdup_array(rtwdev->dev, sband->bitrates, + sband->n_bitrates, + sizeof(*sband->bitrates), + GFP_KERNEL); + if (!dup->bitrates) + return NULL; + + return dup; +} + static void rtw_set_supported_band(struct ieee80211_hw *hw, const struct rtw_chip_info *chip) { - struct rtw_dev *rtwdev = hw->priv; struct ieee80211_supported_band *sband; + struct rtw_dev *rtwdev = hw->priv; if (chip->band & RTW_BAND_2G) { - sband = kmemdup(&rtw_band_2ghz, sizeof(*sband), GFP_KERNEL); + sband = rtw_sband_dup(rtwdev, &rtw_band_2ghz); if (!sband) goto err_out; if (chip->ht_supported) @@ -1674,7 +1701,7 @@ static void rtw_set_supported_band(struct ieee80211_hw *hw, } if (chip->band & RTW_BAND_5G) { - sband = kmemdup(&rtw_band_5ghz, sizeof(*sband), GFP_KERNEL); + sband = rtw_sband_dup(rtwdev, &rtw_band_5ghz); if (!sband) goto err_out; if (chip->ht_supported) @@ -1690,13 +1717,6 @@ static void rtw_set_supported_band(struct ieee80211_hw *hw, rtw_err(rtwdev, "failed to set supported band\n"); } -static void rtw_unset_supported_band(struct ieee80211_hw *hw, - const struct rtw_chip_info *chip) -{ - kfree(hw->wiphy->bands[NL80211_BAND_2GHZ]); - kfree(hw->wiphy->bands[NL80211_BAND_5GHZ]); -} - static void rtw_vif_smps_iter(void *data, u8 *mac, struct ieee80211_vif *vif) { @@ -2320,10 +2340,7 @@ EXPORT_SYMBOL(rtw_register_hw); void rtw_unregister_hw(struct rtw_dev *rtwdev, struct ieee80211_hw *hw) { - const struct rtw_chip_info *chip = rtwdev->chip; - ieee80211_unregister_hw(hw); - rtw_unset_supported_band(hw, chip); rtw_debugfs_deinit(rtwdev); rtw_led_deinit(rtwdev); } @@ -2444,10 +2461,10 @@ void rtw_core_enable_beacon(struct rtw_dev *rtwdev, bool enable) if (enable) { rtw_write32_set(rtwdev, REG_BCN_CTRL, BIT_EN_BCN_FUNCTION); - rtw_write32_clr(rtwdev, REG_TXPAUSE, BIT_HIGH_QUEUE); + rtw_write8_clr(rtwdev, REG_TXPAUSE, BIT_HIGH_QUEUE); } else { rtw_write32_clr(rtwdev, REG_BCN_CTRL, BIT_EN_BCN_FUNCTION); - rtw_write32_set(rtwdev, REG_TXPAUSE, BIT_HIGH_QUEUE); + rtw_write8_set(rtwdev, REG_TXPAUSE, BIT_HIGH_QUEUE); } } diff --git a/drivers/net/wireless/realtek/rtw88/main.h b/drivers/net/wireless/realtek/rtw88/main.h index 43ed6d6b42919e..1ab70214ce36eb 100644 --- a/drivers/net/wireless/realtek/rtw88/main.h +++ b/drivers/net/wireless/realtek/rtw88/main.h @@ -2226,7 +2226,7 @@ enum nl80211_band rtw_hw_to_nl80211_band(enum rtw_supported_band hw_band) } void rtw_set_rx_freq_band(struct rtw_rx_pkt_stat *pkt_stat, u8 channel); -void rtw_set_dtim_period(struct rtw_dev *rtwdev, int dtim_period); +void rtw_set_dtim_period(struct rtw_dev *rtwdev, u8 dtim_period); void rtw_get_channel_params(struct cfg80211_chan_def *chandef, struct rtw_channel_params *ch_param); bool check_hw_ready(struct rtw_dev *rtwdev, u32 addr, u32 mask, u32 target); diff --git a/drivers/net/wireless/realtek/rtw88/rtw8821cu.c b/drivers/net/wireless/realtek/rtw88/rtw8821cu.c index 7a0fffc359e255..8cd09d66655db6 100644 --- a/drivers/net/wireless/realtek/rtw88/rtw8821cu.c +++ b/drivers/net/wireless/realtek/rtw88/rtw8821cu.c @@ -37,6 +37,8 @@ static const struct usb_device_id rtw_8821cu_id_table[] = { .driver_info = (kernel_ulong_t)&(rtw8821c_hw_spec) }, /* Edimax */ { USB_DEVICE_AND_INTERFACE_INFO(0x7392, 0xd811, 0xff, 0xff, 0xff), .driver_info = (kernel_ulong_t)&(rtw8821c_hw_spec) }, /* Edimax */ + { USB_DEVICE_AND_INTERFACE_INFO(0x2c4e, 0x0105, 0xff, 0xff, 0xff), + .driver_info = (kernel_ulong_t)&(rtw8821c_hw_spec) }, /* Mercusys */ {}, }; MODULE_DEVICE_TABLE(usb, rtw_8821cu_id_table); diff --git a/drivers/net/wireless/realtek/rtw88/rtw8822b.c b/drivers/net/wireless/realtek/rtw88/rtw8822b.c index 89b6485b229a82..4d88cc2f414851 100644 --- a/drivers/net/wireless/realtek/rtw88/rtw8822b.c +++ b/drivers/net/wireless/realtek/rtw88/rtw8822b.c @@ -1005,7 +1005,8 @@ static int rtw8822b_set_antenna(struct rtw_dev *rtwdev, hal->antenna_tx = antenna_tx; hal->antenna_rx = antenna_rx; - rtw8822b_config_trx_mode(rtwdev, antenna_tx, antenna_rx, false); + if (test_bit(RTW_FLAG_POWERON, rtwdev->flags)) + rtw8822b_config_trx_mode(rtwdev, antenna_tx, antenna_rx, false); return 0; } diff --git a/drivers/net/wireless/realtek/rtw88/usb.c b/drivers/net/wireless/realtek/rtw88/usb.c index 3b5126ffc81a1c..6e841a11c752bb 100644 --- a/drivers/net/wireless/realtek/rtw88/usb.c +++ b/drivers/net/wireless/realtek/rtw88/usb.c @@ -1040,7 +1040,7 @@ static int rtw_usb_intf_init(struct rtw_dev *rtwdev, struct usb_interface *intf) { struct rtw_usb *rtwusb = rtw_get_usb_priv(rtwdev); - struct usb_device *udev = usb_get_dev(interface_to_usbdev(intf)); + struct usb_device *udev = interface_to_usbdev(intf); int ret; rtwusb->udev = udev; @@ -1066,7 +1066,6 @@ static void rtw_usb_intf_deinit(struct rtw_dev *rtwdev, { struct rtw_usb *rtwusb = rtw_get_usb_priv(rtwdev); - usb_put_dev(rtwusb->udev); kfree(rtwusb->usb_data); usb_set_intfdata(intf, NULL); } diff --git a/drivers/net/wireless/realtek/rtw89/chan.c b/drivers/net/wireless/realtek/rtw89/chan.c index 86f1b39a967fea..8fe6a7ef738f7d 100644 --- a/drivers/net/wireless/realtek/rtw89/chan.c +++ b/drivers/net/wireless/realtek/rtw89/chan.c @@ -2608,17 +2608,20 @@ bool rtw89_mcc_detect_go_bcn(struct rtw89_dev *rtwdev, static void rtw89_mcc_detect_connection(struct rtw89_dev *rtwdev, struct rtw89_mcc_role *role) { + struct rtw89_vif_link *rtwvif_link = role->rtwvif_link; struct ieee80211_vif *vif; bool start_detect; int ret; ret = rtw89_core_send_nullfunc(rtwdev, role->rtwvif_link, true, false, RTW89_MCC_PROBE_TIMEOUT); - if (ret) + if (ret && + READ_ONCE(rtwvif_link->sync_bcn_tsf) == rtwvif_link->last_sync_bcn_tsf) role->probe_count++; else role->probe_count = 0; + rtwvif_link->last_sync_bcn_tsf = READ_ONCE(rtwvif_link->sync_bcn_tsf); if (role->probe_count < RTW89_MCC_PROBE_MAX_TRIES) return; diff --git a/drivers/net/wireless/realtek/rtw89/core.c b/drivers/net/wireless/realtek/rtw89/core.c index 0824940c91aee1..4b86a7c4fe3295 100644 --- a/drivers/net/wireless/realtek/rtw89/core.c +++ b/drivers/net/wireless/realtek/rtw89/core.c @@ -1207,7 +1207,7 @@ rtw89_core_tx_update_desc_info(struct rtw89_dev *rtwdev, if (addr_cam->valid && desc_info->mlo) upd_wlan_hdr = true; - if (rtw89_is_tx_rpt_skb(rtwdev, tx_req->skb)) + if (info->flags & IEEE80211_TX_CTL_REQ_TX_STATUS || tx_req->with_wait) rtw89_tx_rpt_init(rtwdev, tx_req); is_bmc = (is_broadcast_ether_addr(hdr->addr1) || @@ -1342,13 +1342,15 @@ static int rtw89_core_tx_write_link(struct rtw89_dev *rtwdev, tx_req.rtwvif_link = rtwvif_link; tx_req.rtwsta_link = rtwsta_link; tx_req.desc_info.sw_mld = sw_mld; - rcu_assign_pointer(skb_data->wait, wait); + tx_req.with_wait = !!wait; rtw89_traffic_stats_accu(rtwdev, rtwvif, skb, true, true); rtw89_wow_parse_akm(rtwdev, skb); rtw89_core_tx_update_desc_info(rtwdev, &tx_req); rtw89_core_tx_wake(rtwdev, &tx_req); + rcu_assign_pointer(skb_data->wait, wait); + ret = rtw89_hci_tx_write(rtwdev, &tx_req); if (ret) { rtw89_err(rtwdev, "failed to transmit skb to HCI\n"); @@ -2785,7 +2787,7 @@ static void rtw89_core_bcn_track_assoc(struct rtw89_dev *rtwdev, rcu_read_lock(); bss_conf = rtw89_vif_rcu_dereference_link(rtwvif_link, true); - beacon_int = bss_conf->beacon_int; + beacon_int = bss_conf->beacon_int ?: 100; dtim = bss_conf->dtim_period; rcu_read_unlock(); @@ -2815,9 +2817,7 @@ static void rtw89_core_bcn_track_reset(struct rtw89_dev *rtwdev) memset(&rtwdev->bcn_track, 0, sizeof(rtwdev->bcn_track)); } -static void rtw89_vif_rx_bcn_stat(struct rtw89_dev *rtwdev, - struct ieee80211_bss_conf *bss_conf, - struct sk_buff *skb) +static void rtw89_vif_rx_bcn_stat(struct rtw89_dev *rtwdev, struct sk_buff *skb) { #define RTW89_APPEND_TSF_2GHZ 384 #define RTW89_APPEND_TSF_5GHZ 52 @@ -2826,7 +2826,7 @@ static void rtw89_vif_rx_bcn_stat(struct rtw89_dev *rtwdev, struct ieee80211_rx_status *rx_status = IEEE80211_SKB_RXCB(skb); struct rtw89_beacon_stat *bcn_stat = &rtwdev->phystat.bcn_stat; struct rtw89_beacon_track_info *bcn_track = &rtwdev->bcn_track; - u32 bcn_intvl_us = ieee80211_tu_to_usec(bss_conf->beacon_int); + u32 bcn_intvl_us = ieee80211_tu_to_usec(bcn_track->beacon_int); u64 tsf = le64_to_cpu(mgmt->u.beacon.timestamp); u8 wp, num = bcn_stat->num; u16 append; @@ -2834,6 +2834,10 @@ static void rtw89_vif_rx_bcn_stat(struct rtw89_dev *rtwdev, if (!RTW89_CHK_FW_FEATURE(BEACON_TRACKING, &rtwdev->fw)) return; + /* Skip if not yet associated */ + if (!bcn_intvl_us) + return; + switch (rx_status->band) { default: case NL80211_BAND_2GHZ: @@ -2921,7 +2925,7 @@ static void rtw89_vif_rx_stats_iter(void *data, u8 *mac, pkt_stat->beacon_rate = desc_info->data_rate; pkt_stat->beacon_len = skb->len; - rtw89_vif_rx_bcn_stat(rtwdev, bss_conf, skb); + rtw89_vif_rx_bcn_stat(rtwdev, skb); } if (!ether_addr_equal(bss_conf->addr, hdr->addr1)) @@ -5234,7 +5238,7 @@ static void rtw89_init_eht_cap(struct rtw89_dev *rtwdev, u8 val, val_mcs13; int sts = 8; - if (chip->chip_gen == RTW89_CHIP_AX) + if (chip->chip_gen == RTW89_CHIP_AX || hal->no_eht) return; if (hal->no_mcs_12_13) diff --git a/drivers/net/wireless/realtek/rtw89/core.h b/drivers/net/wireless/realtek/rtw89/core.h index a9cb47ea0b9352..a032a20d4c23b9 100644 --- a/drivers/net/wireless/realtek/rtw89/core.h +++ b/drivers/net/wireless/realtek/rtw89/core.h @@ -1211,6 +1211,8 @@ struct rtw89_core_tx_request { struct rtw89_vif_link *rtwvif_link; struct rtw89_sta_link *rtwsta_link; struct rtw89_tx_desc_info desc_info; + + bool with_wait; }; struct rtw89_txq { @@ -5037,6 +5039,7 @@ struct rtw89_hal { bool support_cckpd; bool support_igi; bool no_mcs_12_13; + bool no_eht; atomic_t roc_chanctx_idx; u8 roc_link_index; diff --git a/drivers/net/wireless/realtek/rtw89/debug.c b/drivers/net/wireless/realtek/rtw89/debug.c index 1264c2f82600bf..987eef8170f2bd 100644 --- a/drivers/net/wireless/realtek/rtw89/debug.c +++ b/drivers/net/wireless/realtek/rtw89/debug.c @@ -825,10 +825,6 @@ static ssize_t __print_txpwr_map(struct rtw89_dev *rtwdev, char *buf, size_t buf s8 *bufp, tmp; int ret; - bufp = vzalloc(map->addr_to - map->addr_from + 4); - if (!bufp) - return -ENOMEM; - if (path_num == 1) max_valid_addr = map->addr_to_1ss; else @@ -837,6 +833,10 @@ static ssize_t __print_txpwr_map(struct rtw89_dev *rtwdev, char *buf, size_t buf if (max_valid_addr == 0) return -EOPNOTSUPP; + bufp = vzalloc(map->addr_to - map->addr_from + 4); + if (!bufp) + return -ENOMEM; + for (addr = map->addr_from; addr <= max_valid_addr; addr += 4) { ret = rtw89_mac_txpwr_read32(rtwdev, RTW89_PHY_0, addr, &val); if (ret) diff --git a/drivers/net/wireless/realtek/rtw89/fw.c b/drivers/net/wireless/realtek/rtw89/fw.c index 7b9d9989e51706..2f68a04cc028f4 100644 --- a/drivers/net/wireless/realtek/rtw89/fw.c +++ b/drivers/net/wireless/realtek/rtw89/fw.c @@ -8114,6 +8114,7 @@ int rtw89_hw_scan_start(struct rtw89_dev *rtwdev, struct cfg80211_scan_request *req = &scan_req->req; const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, rtwvif_link->chanctx_idx); + struct ieee80211_vif *vif = rtwvif_link_to_vif(rtwvif_link); struct rtw89_vif *rtwvif = rtwvif_link->rtwvif; struct rtw89_chanctx_pause_parm pause_parm = { .rsn = RTW89_CHANCTX_PAUSE_REASON_HW_SCAN, @@ -8142,6 +8143,8 @@ int rtw89_hw_scan_start(struct rtw89_dev *rtwdev, if (req->flags & NL80211_SCAN_FLAG_RANDOM_ADDR) get_random_mask_addr(mac_addr, req->mac_addr, req->mac_addr_mask); + else if (ieee80211_vif_is_mld(vif)) + ether_addr_copy(mac_addr, vif->addr); else ether_addr_copy(mac_addr, rtwvif_link->mac_addr); diff --git a/drivers/net/wireless/realtek/rtw89/fw.h b/drivers/net/wireless/realtek/rtw89/fw.h index cedb4a47a769cf..ba7c332911310f 100644 --- a/drivers/net/wireless/realtek/rtw89/fw.h +++ b/drivers/net/wireless/realtek/rtw89/fw.h @@ -42,6 +42,10 @@ struct rtw89_c2hreg_phycap { #define RTW89_C2HREG_PHYCAP_W0_BW GENMASK(31, 24) #define RTW89_C2HREG_PHYCAP_W1_TX_NSS GENMASK(7, 0) #define RTW89_C2HREG_PHYCAP_W1_PROT GENMASK(15, 8) +#define RTW89_C2HREG_PHYCAP_W1_PROT_11N 1 +#define RTW89_C2HREG_PHYCAP_W1_PROT_11AC 2 +#define RTW89_C2HREG_PHYCAP_W1_PROT_11AX 3 +#define RTW89_C2HREG_PHYCAP_W1_PROT_11BE 4 #define RTW89_C2HREG_PHYCAP_W1_NIC GENMASK(23, 16) #define RTW89_C2HREG_PHYCAP_W1_WL_FUNC GENMASK(31, 24) #define RTW89_C2HREG_PHYCAP_W2_HW_TYPE GENMASK(7, 0) diff --git a/drivers/net/wireless/realtek/rtw89/mac.c b/drivers/net/wireless/realtek/rtw89/mac.c index d78fbe73e36577..fbce71cd5a05cf 100644 --- a/drivers/net/wireless/realtek/rtw89/mac.c +++ b/drivers/net/wireless/realtek/rtw89/mac.c @@ -3061,6 +3061,7 @@ static int rtw89_mac_setup_phycap_part0(struct rtw89_dev *rtwdev) struct rtw89_efuse *efuse = &rtwdev->efuse; struct rtw89_mac_c2h_info c2h_info = {}; struct rtw89_hal *hal = &rtwdev->hal; + u8 protocol; u8 tx_nss; u8 rx_nss; u8 tx_ant; @@ -3108,6 +3109,10 @@ static int rtw89_mac_setup_phycap_part0(struct rtw89_dev *rtwdev) rtw89_debug(rtwdev, RTW89_DBG_FW, "TX path diversity=%d\n", hal->tx_path_diversity); rtw89_debug(rtwdev, RTW89_DBG_FW, "Antenna diversity=%d\n", hal->ant_diversity); + protocol = u32_get_bits(phycap->w1, RTW89_C2HREG_PHYCAP_W1_PROT); + if (protocol < RTW89_C2HREG_PHYCAP_W1_PROT_11BE) + hal->no_eht = true; + return 0; } @@ -4341,6 +4346,7 @@ static void rtw89_mac_bcn_drop(struct rtw89_dev *rtwdev, #define BCN_HOLD_DEF 200 #define BCN_MASK_DEF 0 #define TBTT_ERLY_DEF 5 +#define TBTT_AGG_DEF 1 #define BCN_SET_UNIT 32 #define BCN_ERLY_SET_DLY (10 * 2) @@ -4644,6 +4650,16 @@ static void rtw89_mac_port_cfg_tbtt_early(struct rtw89_dev *rtwdev, B_AX_TBTTERLY_MASK, TBTT_ERLY_DEF); } +static void rtw89_mac_port_cfg_tbtt_agg(struct rtw89_dev *rtwdev, + struct rtw89_vif_link *rtwvif_link) +{ + const struct rtw89_mac_gen_def *mac = rtwdev->chip->mac_def; + const struct rtw89_port_reg *p = mac->port_base; + + rtw89_write16_port_mask(rtwdev, rtwvif_link, p->tbtt_agg, + B_AX_TBTT_AGG_NUM_MASK, TBTT_AGG_DEF); +} + static void rtw89_mac_port_cfg_bss_color(struct rtw89_dev *rtwdev, struct rtw89_vif_link *rtwvif_link) { @@ -4904,6 +4920,7 @@ int rtw89_mac_port_update(struct rtw89_dev *rtwdev, struct rtw89_vif_link *rtwvi rtw89_mac_port_cfg_bcn_hold_time(rtwdev, rtwvif_link); rtw89_mac_port_cfg_bcn_mask_area(rtwdev, rtwvif_link); rtw89_mac_port_cfg_tbtt_early(rtwdev, rtwvif_link); + rtw89_mac_port_cfg_tbtt_agg(rtwdev, rtwvif_link); rtw89_mac_port_cfg_bss_color(rtwdev, rtwvif_link); rtw89_mac_port_cfg_mbssid(rtwdev, rtwvif_link); rtw89_mac_port_cfg_func_en(rtwdev, rtwvif_link, true); @@ -7184,6 +7201,7 @@ const struct rtw89_mac_gen_def rtw89_mac_gen_ax = { .check_mac_en = rtw89_mac_check_mac_en_ax, .sys_init = sys_init_ax, .trx_init = trx_init_ax, + .err_imr_ctrl = err_imr_ctrl_ax, .hci_func_en = rtw89_mac_hci_func_en_ax, .dmac_func_pre_en = rtw89_mac_dmac_func_pre_en_ax, .dle_func_en = dle_func_en_ax, diff --git a/drivers/net/wireless/realtek/rtw89/mac.h b/drivers/net/wireless/realtek/rtw89/mac.h index 0007229d675378..a4ed1c545609ea 100644 --- a/drivers/net/wireless/realtek/rtw89/mac.h +++ b/drivers/net/wireless/realtek/rtw89/mac.h @@ -1019,6 +1019,7 @@ struct rtw89_mac_gen_def { enum rtw89_mac_hwmod_sel sel); int (*sys_init)(struct rtw89_dev *rtwdev); int (*trx_init)(struct rtw89_dev *rtwdev); + void (*err_imr_ctrl)(struct rtw89_dev *rtwdev, bool en); void (*hci_func_en)(struct rtw89_dev *rtwdev); void (*dmac_func_pre_en)(struct rtw89_dev *rtwdev); void (*dle_func_en)(struct rtw89_dev *rtwdev, bool enable); diff --git a/drivers/net/wireless/realtek/rtw89/mac80211.c b/drivers/net/wireless/realtek/rtw89/mac80211.c index f39ca1c2ed100f..d08eac3d99266f 100644 --- a/drivers/net/wireless/realtek/rtw89/mac80211.c +++ b/drivers/net/wireless/realtek/rtw89/mac80211.c @@ -127,6 +127,7 @@ static int __rtw89_ops_add_iface_link(struct rtw89_dev *rtwdev, rtwvif_link->reg_6ghz_power = RTW89_REG_6GHZ_POWER_DFLT; rtwvif_link->rand_tsf_done = false; rtwvif_link->detect_bcn_count = 0; + rtwvif_link->last_sync_bcn_tsf = 0; rcu_read_lock(); diff --git a/drivers/net/wireless/realtek/rtw89/mac_be.c b/drivers/net/wireless/realtek/rtw89/mac_be.c index 556e5f98e8d419..dee5ff71b75fe0 100644 --- a/drivers/net/wireless/realtek/rtw89/mac_be.c +++ b/drivers/net/wireless/realtek/rtw89/mac_be.c @@ -1175,7 +1175,7 @@ static int resp_pktctl_init_be(struct rtw89_dev *rtwdev, u8 mac_idx) reg = rtw89_mac_reg_by_idx(rtwdev, R_BE_RESP_CSI_RESERVED_PAGE, mac_idx); rtw89_write32_mask(rtwdev, reg, B_BE_CSI_RESERVED_START_PAGE_MASK, qt_cfg.pktid); - rtw89_write32_mask(rtwdev, reg, B_BE_CSI_RESERVED_PAGE_NUM_MASK, qt_cfg.pg_num); + rtw89_write32_mask(rtwdev, reg, B_BE_CSI_RESERVED_PAGE_NUM_MASK, qt_cfg.pg_num + 1); return 0; } @@ -2601,6 +2601,7 @@ const struct rtw89_mac_gen_def rtw89_mac_gen_be = { .check_mac_en = rtw89_mac_check_mac_en_be, .sys_init = sys_init_be, .trx_init = trx_init_be, + .err_imr_ctrl = err_imr_ctrl_be, .hci_func_en = rtw89_mac_hci_func_en_be, .dmac_func_pre_en = rtw89_mac_dmac_func_pre_en_be, .dle_func_en = dle_func_en_be, diff --git a/drivers/net/wireless/realtek/rtw89/pci.c b/drivers/net/wireless/realtek/rtw89/pci.c index a66fcdb0293b64..fb4469a76bc03b 100644 --- a/drivers/net/wireless/realtek/rtw89/pci.c +++ b/drivers/net/wireless/realtek/rtw89/pci.c @@ -604,8 +604,15 @@ static void rtw89_pci_release_rpp(struct rtw89_dev *rtwdev, void *rpp) info->parse_rpp(rtwdev, rpp, &rpp_info); - if (rpp_info.txch == RTW89_TXCH_CH12) { - rtw89_warn(rtwdev, "should no fwcmd release report\n"); + if (unlikely(rpp_info.txch >= RTW89_TXCH_NUM || + info->tx_dma_ch_mask & BIT(rpp_info.txch))) { + rtw89_warn(rtwdev, "should no release report on txch %d\n", + rpp_info.txch); + return; + } + + if (unlikely(rpp_info.seq >= RTW89_PCI_TXWD_NUM_MAX)) { + rtw89_warn(rtwdev, "invalid seq %d\n", rpp_info.seq); return; } @@ -4598,6 +4605,7 @@ static int __maybe_unused rtw89_pci_resume(struct device *dev) rtw89_write32_clr(rtwdev, R_AX_PCIE_PS_CTRL_V1, B_AX_SEL_REQ_ENTR_L1); } + rtw89_pci_hci_ldo(rtwdev); rtw89_pci_l2_hci_ldo(rtwdev); rtw89_pci_basic_cfg(rtwdev, true); diff --git a/drivers/net/wireless/realtek/rtw89/regd.c b/drivers/net/wireless/realtek/rtw89/regd.c index 209d84909f8855..c3425ed44732e2 100644 --- a/drivers/net/wireless/realtek/rtw89/regd.c +++ b/drivers/net/wireless/realtek/rtw89/regd.c @@ -1142,6 +1142,7 @@ static int rtw89_reg_6ghz_power_recalc(struct rtw89_dev *rtwdev, } } else { rtwvif_link->reg_6ghz_power = RTW89_REG_6GHZ_POWER_DFLT; + dflt = true; } rcu_read_unlock(); diff --git a/drivers/net/wireless/realtek/rtw89/rtw8852au.c b/drivers/net/wireless/realtek/rtw89/rtw8852au.c index ca782469c455dc..ccdbcc178c2a4c 100644 --- a/drivers/net/wireless/realtek/rtw89/rtw8852au.c +++ b/drivers/net/wireless/realtek/rtw89/rtw8852au.c @@ -52,6 +52,8 @@ static const struct usb_device_id rtw_8852au_id_table[] = { .driver_info = (kernel_ulong_t)&rtw89_8852au_info }, { USB_DEVICE_AND_INTERFACE_INFO(0x2001, 0x3321, 0xff, 0xff, 0xff), .driver_info = (kernel_ulong_t)&rtw89_8852au_info }, + { USB_DEVICE_AND_INTERFACE_INFO(0x2001, 0x3323, 0xff, 0xff, 0xff), + .driver_info = (kernel_ulong_t)&rtw89_8852au_info }, { USB_DEVICE_AND_INTERFACE_INFO(0x2001, 0x332c, 0xff, 0xff, 0xff), .driver_info = (kernel_ulong_t)&rtw89_8852au_info }, { USB_DEVICE_AND_INTERFACE_INFO(0x2357, 0x013f, 0xff, 0xff, 0xff), @@ -60,6 +62,8 @@ static const struct usb_device_id rtw_8852au_id_table[] = { .driver_info = (kernel_ulong_t)&rtw89_8852au_info }, { USB_DEVICE_AND_INTERFACE_INFO(0x2357, 0x0141, 0xff, 0xff, 0xff), .driver_info = (kernel_ulong_t)&rtw89_8852au_info }, + { USB_DEVICE_AND_INTERFACE_INFO(0x3625, 0x010d, 0xff, 0xff, 0xff), + .driver_info = (kernel_ulong_t)&rtw89_8852au_info }, { USB_DEVICE_AND_INTERFACE_INFO(0x3625, 0x010f, 0xff, 0xff, 0xff), .driver_info = (kernel_ulong_t)&rtw89_8852au_info }, {}, diff --git a/drivers/net/wireless/realtek/rtw89/rtw8852bu.c b/drivers/net/wireless/realtek/rtw89/rtw8852bu.c index 980d17ef68d0a9..84cd3ec971f98b 100644 --- a/drivers/net/wireless/realtek/rtw89/rtw8852bu.c +++ b/drivers/net/wireless/realtek/rtw89/rtw8852bu.c @@ -54,6 +54,8 @@ static const struct usb_device_id rtw_8852bu_id_table[] = { .driver_info = (kernel_ulong_t)&rtw89_8852bu_info }, { USB_DEVICE_AND_INTERFACE_INFO(0x0db0, 0x6931, 0xff, 0xff, 0xff), .driver_info = (kernel_ulong_t)&rtw89_8852bu_info }, + { USB_DEVICE_AND_INTERFACE_INFO(0x0db0, 0xf0c8, 0xff, 0xff, 0xff), + .driver_info = (kernel_ulong_t)&rtw89_8852bu_info }, { USB_DEVICE_AND_INTERFACE_INFO(0x2001, 0x3327, 0xff, 0xff, 0xff), .driver_info = (kernel_ulong_t)&rtw89_8852bu_info }, { USB_DEVICE_AND_INTERFACE_INFO(0x3574, 0x6121, 0xff, 0xff, 0xff), diff --git a/drivers/net/wireless/realtek/rtw89/rtw8852cu.c b/drivers/net/wireless/realtek/rtw89/rtw8852cu.c index 2708b523ca1417..3b9825c92a0d90 100644 --- a/drivers/net/wireless/realtek/rtw89/rtw8852cu.c +++ b/drivers/net/wireless/realtek/rtw89/rtw8852cu.c @@ -46,6 +46,8 @@ static const struct usb_device_id rtw_8852cu_id_table[] = { .driver_info = (kernel_ulong_t)&rtw89_8852cu_info }, { USB_DEVICE_AND_INTERFACE_INFO(0x0db0, 0x991d, 0xff, 0xff, 0xff), .driver_info = (kernel_ulong_t)&rtw89_8852cu_info }, + { USB_DEVICE_AND_INTERFACE_INFO(0x28de, 0x2432, 0xff, 0xff, 0xff), + .driver_info = (kernel_ulong_t)&rtw89_8852cu_info }, { USB_DEVICE_AND_INTERFACE_INFO(0x35b2, 0x0502, 0xff, 0xff, 0xff), .driver_info = (kernel_ulong_t)&rtw89_8852cu_info }, { USB_DEVICE_AND_INTERFACE_INFO(0x35bc, 0x0101, 0xff, 0xff, 0xff), diff --git a/drivers/net/wireless/realtek/rtw89/rtw8922a.c b/drivers/net/wireless/realtek/rtw89/rtw8922a.c index 4437279c554b09..52da0fa02da01e 100644 --- a/drivers/net/wireless/realtek/rtw89/rtw8922a.c +++ b/drivers/net/wireless/realtek/rtw89/rtw8922a.c @@ -636,16 +636,30 @@ static int rtw8922a_read_efuse_rf(struct rtw89_dev *rtwdev, u8 *log_map) static int rtw8922a_read_efuse(struct rtw89_dev *rtwdev, u8 *log_map, enum rtw89_efuse_block block) { + struct rtw89_efuse *efuse = &rtwdev->efuse; + int ret; + switch (block) { case RTW89_EFUSE_BLOCK_HCI_DIG_PCIE_SDIO: - return rtw8922a_read_efuse_pci_sdio(rtwdev, log_map); + ret = rtw8922a_read_efuse_pci_sdio(rtwdev, log_map); + break; case RTW89_EFUSE_BLOCK_HCI_DIG_USB: - return rtw8922a_read_efuse_usb(rtwdev, log_map); + ret = rtw8922a_read_efuse_usb(rtwdev, log_map); + break; case RTW89_EFUSE_BLOCK_RF: - return rtw8922a_read_efuse_rf(rtwdev, log_map); + ret = rtw8922a_read_efuse_rf(rtwdev, log_map); + break; default: - return 0; + ret = 0; + break; + } + + if (!ret && is_zero_ether_addr(efuse->addr)) { + rtw89_info(rtwdev, "efuse mac address is zero, using random mac\n"); + eth_random_addr(efuse->addr); } + + return ret; } #define THM_TRIM_POSITIVE_MASK BIT(6) @@ -1756,6 +1770,32 @@ static int rtw8922a_ctrl_rx_path_tmac(struct rtw89_dev *rtwdev, } #define DIGITAL_PWR_COMP_REG_NUM 22 +static const u32 rtw8922a_digital_pwr_comp_2g_s0_val[][DIGITAL_PWR_COMP_REG_NUM] = { + {0x012C0064, 0x04B00258, 0x00432710, 0x019000A7, 0x06400320, + 0x0D05091D, 0x14D50FA0, 0x00000000, 0x01010000, 0x00000101, + 0x01010101, 0x02020201, 0x02010000, 0x03030202, 0x00000303, + 0x03020101, 0x06060504, 0x01010000, 0x06050403, 0x01000606, + 0x05040202, 0x07070706}, + {0x012C0064, 0x04B00258, 0x00432710, 0x019000A7, 0x06400320, + 0x0D05091D, 0x14D50FA0, 0x00000000, 0x01010100, 0x00000101, + 0x01000000, 0x01010101, 0x01010000, 0x02020202, 0x00000404, + 0x03020101, 0x04040303, 0x02010000, 0x03030303, 0x00000505, + 0x03030201, 0x05050303}, +}; + +static const u32 rtw8922a_digital_pwr_comp_2g_s1_val[][DIGITAL_PWR_COMP_REG_NUM] = { + {0x012C0064, 0x04B00258, 0x00432710, 0x019000A7, 0x06400320, + 0x0D05091D, 0x14D50FA0, 0x01010000, 0x01010101, 0x00000101, + 0x01010100, 0x01010101, 0x01010000, 0x02020202, 0x01000202, + 0x02020101, 0x03030202, 0x02010000, 0x05040403, 0x01000606, + 0x05040302, 0x07070605}, + {0x012C0064, 0x04B00258, 0x00432710, 0x019000A7, 0x06400320, + 0x0D05091D, 0x14D50FA0, 0x00000000, 0x01010100, 0x00000101, + 0x01010000, 0x02020201, 0x02010100, 0x03030202, 0x01000404, + 0x04030201, 0x05050404, 0x01010100, 0x04030303, 0x01000505, + 0x03030101, 0x05050404}, +}; + static const u32 rtw8922a_digital_pwr_comp_val[][DIGITAL_PWR_COMP_REG_NUM] = { {0x012C0096, 0x044C02BC, 0x00322710, 0x015E0096, 0x03C8028A, 0x0BB80708, 0x17701194, 0x02020100, 0x03030303, 0x01000303, @@ -1770,7 +1810,7 @@ static const u32 rtw8922a_digital_pwr_comp_val[][DIGITAL_PWR_COMP_REG_NUM] = { }; static void rtw8922a_set_digital_pwr_comp(struct rtw89_dev *rtwdev, - bool enable, u8 nss, + u8 band, u8 nss, enum rtw89_rf_path path) { static const u32 ltpc_t0[2] = {R_BE_LTPC_T0_PATH0, R_BE_LTPC_T0_PATH1}; @@ -1778,14 +1818,25 @@ static void rtw8922a_set_digital_pwr_comp(struct rtw89_dev *rtwdev, u32 addr, val; u32 i; - if (nss == 1) - digital_pwr_comp = rtw8922a_digital_pwr_comp_val[0]; - else - digital_pwr_comp = rtw8922a_digital_pwr_comp_val[1]; + if (nss == 1) { + if (band == RTW89_BAND_2G) + digital_pwr_comp = path == RF_PATH_A ? + rtw8922a_digital_pwr_comp_2g_s0_val[0] : + rtw8922a_digital_pwr_comp_2g_s1_val[0]; + else + digital_pwr_comp = rtw8922a_digital_pwr_comp_val[0]; + } else { + if (band == RTW89_BAND_2G) + digital_pwr_comp = path == RF_PATH_A ? + rtw8922a_digital_pwr_comp_2g_s0_val[1] : + rtw8922a_digital_pwr_comp_2g_s1_val[1]; + else + digital_pwr_comp = rtw8922a_digital_pwr_comp_val[1]; + } addr = ltpc_t0[path]; for (i = 0; i < DIGITAL_PWR_COMP_REG_NUM; i++, addr += 4) { - val = enable ? digital_pwr_comp[i] : 0; + val = digital_pwr_comp[i]; rtw89_phy_write32(rtwdev, addr, val); } } @@ -1794,7 +1845,7 @@ static void rtw8922a_digital_pwr_comp(struct rtw89_dev *rtwdev, enum rtw89_phy_idx phy_idx) { const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_CHANCTX_0); - bool enable = chan->band_type != RTW89_BAND_2G; + u8 band = chan->band_type; u8 path; if (rtwdev->mlo_dbcc_mode == MLO_1_PLUS_1_1RF) { @@ -1802,10 +1853,10 @@ static void rtw8922a_digital_pwr_comp(struct rtw89_dev *rtwdev, path = RF_PATH_A; else path = RF_PATH_B; - rtw8922a_set_digital_pwr_comp(rtwdev, enable, 1, path); + rtw8922a_set_digital_pwr_comp(rtwdev, band, 1, path); } else { - rtw8922a_set_digital_pwr_comp(rtwdev, enable, 2, RF_PATH_A); - rtw8922a_set_digital_pwr_comp(rtwdev, enable, 2, RF_PATH_B); + rtw8922a_set_digital_pwr_comp(rtwdev, band, 2, RF_PATH_A); + rtw8922a_set_digital_pwr_comp(rtwdev, band, 2, RF_PATH_B); } } diff --git a/drivers/net/wireless/realtek/rtw89/ser.c b/drivers/net/wireless/realtek/rtw89/ser.c index f99e179f7ff9fe..7fdc69578da317 100644 --- a/drivers/net/wireless/realtek/rtw89/ser.c +++ b/drivers/net/wireless/realtek/rtw89/ser.c @@ -431,6 +431,14 @@ static void hal_send_m4_event(struct rtw89_ser *ser) rtw89_mac_set_err_status(rtwdev, MAC_AX_ERR_L1_RCVY_EN); } +static void hal_enable_err_imr(struct rtw89_ser *ser) +{ + struct rtw89_dev *rtwdev = container_of(ser, struct rtw89_dev, ser); + const struct rtw89_mac_gen_def *mac = rtwdev->chip->mac_def; + + mac->err_imr_ctrl(rtwdev, true); +} + /* state handler */ static void ser_idle_st_hdl(struct rtw89_ser *ser, u8 evt) { @@ -552,6 +560,8 @@ static void ser_do_hci_st_hdl(struct rtw89_ser *ser, u8 evt) break; case SER_EV_MAC_RESET_DONE: + hal_enable_err_imr(ser); + ser_state_goto(ser, SER_IDLE_ST); break; diff --git a/drivers/net/wireless/realtek/rtw89/wow.c b/drivers/net/wireless/realtek/rtw89/wow.c index 46aba4cb2ee9e5..534966b4d9c430 100644 --- a/drivers/net/wireless/realtek/rtw89/wow.c +++ b/drivers/net/wireless/realtek/rtw89/wow.c @@ -809,6 +809,10 @@ static void rtw89_wow_show_wakeup_reason(struct rtw89_dev *rtwdev) reason = rtw89_read8(rtwdev, wow_reason_reg); switch (reason) { + case RTW89_WOW_RSN_RX_DISASSOC: + wakeup.disconnect = true; + rtw89_debug(rtwdev, RTW89_DBG_WOW, "WOW: Rx disassoc\n"); + break; case RTW89_WOW_RSN_RX_DEAUTH: wakeup.disconnect = true; rtw89_debug(rtwdev, RTW89_DBG_WOW, "WOW: Rx deauth\n"); diff --git a/drivers/net/wireless/realtek/rtw89/wow.h b/drivers/net/wireless/realtek/rtw89/wow.h index d2ba6cebc2a6b0..71e07f482174f1 100644 --- a/drivers/net/wireless/realtek/rtw89/wow.h +++ b/drivers/net/wireless/realtek/rtw89/wow.h @@ -33,6 +33,7 @@ enum rtw89_wake_reason { RTW89_WOW_RSN_RX_PTK_REKEY = 0x1, RTW89_WOW_RSN_RX_GTK_REKEY = 0x2, + RTW89_WOW_RSN_RX_DISASSOC = 0x4, RTW89_WOW_RSN_RX_DEAUTH = 0x8, RTW89_WOW_RSN_DISCONNECT = 0x10, RTW89_WOW_RSN_RX_MAGIC_PKT = 0x21, diff --git a/drivers/net/wireless/rsi/rsi_91x_mac80211.c b/drivers/net/wireless/rsi/rsi_91x_mac80211.c index 8c8e074a3a7056..c7ae8031436ae1 100644 --- a/drivers/net/wireless/rsi/rsi_91x_mac80211.c +++ b/drivers/net/wireless/rsi/rsi_91x_mac80211.c @@ -668,7 +668,7 @@ static int rsi_mac80211_config(struct ieee80211_hw *hw, struct rsi_hw *adapter = hw->priv; struct rsi_common *common = adapter->priv; struct ieee80211_conf *conf = &hw->conf; - int status = -EOPNOTSUPP; + int status = 0; mutex_lock(&common->mutex); diff --git a/drivers/net/wireless/st/cw1200/pm.c b/drivers/net/wireless/st/cw1200/pm.c index 2002e3f9fe45b9..b656afe65db072 100644 --- a/drivers/net/wireless/st/cw1200/pm.c +++ b/drivers/net/wireless/st/cw1200/pm.c @@ -264,12 +264,14 @@ int cw1200_wow_suspend(struct ieee80211_hw *hw, struct cfg80211_wowlan *wowlan) wiphy_err(priv->hw->wiphy, "PM request failed: %d. WoW is disabled.\n", ret); cw1200_wow_resume(hw); + mutex_unlock(&priv->conf_mutex); return -EBUSY; } /* Force resume if event is coming from the device. */ if (atomic_read(&priv->bh_rx)) { cw1200_wow_resume(hw); + mutex_unlock(&priv->conf_mutex); return -EAGAIN; } diff --git a/drivers/net/wireless/ti/wl1251/tx.c b/drivers/net/wireless/ti/wl1251/tx.c index adb4840b048932..c264d83e71d9c8 100644 --- a/drivers/net/wireless/ti/wl1251/tx.c +++ b/drivers/net/wireless/ti/wl1251/tx.c @@ -402,12 +402,14 @@ static void wl1251_tx_packet_cb(struct wl1251 *wl, int hdrlen; u8 *frame; - skb = wl->tx_frames[result->id]; - if (skb == NULL) { - wl1251_error("SKB for packet %d is NULL", result->id); + if (unlikely(result->id >= ARRAY_SIZE(wl->tx_frames) || + wl->tx_frames[result->id] == NULL)) { + wl1251_error("invalid packet id %u", result->id); return; } + skb = wl->tx_frames[result->id]; + info = IEEE80211_SKB_CB(skb); if (!(info->flags & IEEE80211_TX_CTL_NO_ACK) && diff --git a/drivers/net/wireless/ti/wlcore/main.c b/drivers/net/wireless/ti/wlcore/main.c index 12f0167d7380ee..1f6b906594930f 100644 --- a/drivers/net/wireless/ti/wlcore/main.c +++ b/drivers/net/wireless/ti/wlcore/main.c @@ -1875,6 +1875,8 @@ static int __maybe_unused wl1271_op_resume(struct ieee80211_hw *hw) wl->wow_enabled); WARN_ON(!wl->wow_enabled); + mutex_lock(&wl->mutex); + ret = pm_runtime_force_resume(wl->dev); if (ret < 0) { wl1271_error("ELP wakeup failure!"); @@ -1891,8 +1893,6 @@ static int __maybe_unused wl1271_op_resume(struct ieee80211_hw *hw) run_irq_work = true; spin_unlock_irqrestore(&wl->wl_lock, flags); - mutex_lock(&wl->mutex); - /* test the recovery flag before calling any SDIO functions */ pending_recovery = test_bit(WL1271_FLAG_RECOVERY_IN_PROGRESS, &wl->flags); diff --git a/drivers/net/wireless/ti/wlcore/tx.c b/drivers/net/wireless/ti/wlcore/tx.c index 6241866d39df6d..75cfbcfb7626dc 100644 --- a/drivers/net/wireless/ti/wlcore/tx.c +++ b/drivers/net/wireless/ti/wlcore/tx.c @@ -210,7 +210,7 @@ static int wl1271_tx_allocate(struct wl1271 *wl, struct wl12xx_vif *wlvif, if (skb_headroom(skb) < (total_len - skb->len) && pskb_expand_head(skb, (total_len - skb->len), 0, GFP_ATOMIC)) { wl1271_free_tx_id(wl, id); - return -EAGAIN; + return -ENOMEM; } desc = skb_push(skb, total_len - skb->len); diff --git a/drivers/net/wireless/virtual/mac80211_hwsim.c b/drivers/net/wireless/virtual/mac80211_hwsim.c index 79cc63272134d4..cfbd0c50be1c9d 100644 --- a/drivers/net/wireless/virtual/mac80211_hwsim.c +++ b/drivers/net/wireless/virtual/mac80211_hwsim.c @@ -3021,7 +3021,6 @@ static void hw_scan_work(struct work_struct *work) hwsim->tmp_chan->band, NULL)) { rcu_read_unlock(); - kfree_skb(probe); continue; } diff --git a/drivers/net/wireless/virtual/virt_wifi.c b/drivers/net/wireless/virtual/virt_wifi.c index 4eae89376feb55..cd6b66242bff22 100644 --- a/drivers/net/wireless/virtual/virt_wifi.c +++ b/drivers/net/wireless/virtual/virt_wifi.c @@ -557,7 +557,6 @@ static int virt_wifi_newlink(struct net_device *dev, eth_hw_addr_inherit(dev, priv->lowerdev); netif_stacked_transfer_operstate(priv->lowerdev, dev); - SET_NETDEV_DEV(dev, &priv->lowerdev->dev); dev->ieee80211_ptr = kzalloc(sizeof(*dev->ieee80211_ptr), GFP_KERNEL); if (!dev->ieee80211_ptr) { diff --git a/drivers/net/wwan/mhi_wwan_mbim.c b/drivers/net/wwan/mhi_wwan_mbim.c index f8bc9a39bfa307..1d7e3ad900c128 100644 --- a/drivers/net/wwan/mhi_wwan_mbim.c +++ b/drivers/net/wwan/mhi_wwan_mbim.c @@ -98,7 +98,8 @@ static struct mhi_mbim_link *mhi_mbim_get_link_rcu(struct mhi_mbim_context *mbim static int mhi_mbim_get_link_mux_id(struct mhi_controller *cntrl) { if (strcmp(cntrl->name, "foxconn-dw5934e") == 0 || - strcmp(cntrl->name, "foxconn-t99w640") == 0) + strcmp(cntrl->name, "foxconn-t99w640") == 0 || + strcmp(cntrl->name, "foxconn-t99w760") == 0) return WDS_BIND_MUX_DATA_PORT_MUX_ID; return 0; diff --git a/drivers/net/xen-netback/xenbus.c b/drivers/net/xen-netback/xenbus.c index a78a25b872409a..61b547aab286a2 100644 --- a/drivers/net/xen-netback/xenbus.c +++ b/drivers/net/xen-netback/xenbus.c @@ -735,10 +735,11 @@ static void connect(struct backend_info *be) */ requested_num_queues = xenbus_read_unsigned(dev->otherend, "multi-queue-num-queues", 1); - if (requested_num_queues > xenvif_max_queues) { + if (requested_num_queues > xenvif_max_queues || + requested_num_queues == 0) { /* buggy or malicious guest */ xenbus_dev_fatal(dev, -EINVAL, - "guest requested %u queues, exceeding the maximum of %u.", + "guest requested %u queues, but valid range is 1 - %u.", requested_num_queues, xenvif_max_queues); return; } diff --git a/drivers/nfc/nxp-nci/i2c.c b/drivers/nfc/nxp-nci/i2c.c index 049662ffdf9729..b3d34433bd14a0 100644 --- a/drivers/nfc/nxp-nci/i2c.c +++ b/drivers/nfc/nxp-nci/i2c.c @@ -47,8 +47,8 @@ static int nxp_nci_i2c_set_mode(void *phy_id, { struct nxp_nci_i2c_phy *phy = (struct nxp_nci_i2c_phy *) phy_id; - gpiod_set_value(phy->gpiod_fw, (mode == NXP_NCI_MODE_FW) ? 1 : 0); - gpiod_set_value(phy->gpiod_en, (mode != NXP_NCI_MODE_COLD) ? 1 : 0); + gpiod_set_value_cansleep(phy->gpiod_fw, (mode == NXP_NCI_MODE_FW) ? 1 : 0); + gpiod_set_value_cansleep(phy->gpiod_en, (mode != NXP_NCI_MODE_COLD) ? 1 : 0); usleep_range(10000, 15000); if (mode == NXP_NCI_MODE_COLD) @@ -305,7 +305,7 @@ static int nxp_nci_i2c_probe(struct i2c_client *client) r = request_threaded_irq(client->irq, NULL, nxp_nci_i2c_irq_thread_fn, - IRQF_TRIGGER_RISING | IRQF_ONESHOT, + IRQF_ONESHOT, NXP_NCI_I2C_DRIVER_NAME, phy); if (r < 0) nfc_err(&client->dev, "Unable to register IRQ handler\n"); diff --git a/drivers/nfc/pn533/uart.c b/drivers/nfc/pn533/uart.c index a081bce61c29f6..3aacfc79fa72f7 100644 --- a/drivers/nfc/pn533/uart.c +++ b/drivers/nfc/pn533/uart.c @@ -211,14 +211,22 @@ static size_t pn532_receive_buf(struct serdev_device *serdev, timer_delete(&dev->cmd_timeout); for (i = 0; i < count; i++) { + if (!dev->recv_skb) { + dev->recv_skb = alloc_skb(PN532_UART_SKB_BUFF_LEN, + GFP_KERNEL); + if (!dev->recv_skb) + return i; + } + + if (unlikely(!skb_tailroom(dev->recv_skb))) + skb_trim(dev->recv_skb, 0); + skb_put_u8(dev->recv_skb, *data++); if (!pn532_uart_rx_is_frame(dev->recv_skb)) continue; pn533_recv_frame(dev->priv, dev->recv_skb, 0); - dev->recv_skb = alloc_skb(PN532_UART_SKB_BUFF_LEN, GFP_KERNEL); - if (!dev->recv_skb) - return 0; + dev->recv_skb = NULL; } return i; diff --git a/drivers/nfc/pn533/usb.c b/drivers/nfc/pn533/usb.c index 018a80674f06ed..0f12f86ebb0234 100644 --- a/drivers/nfc/pn533/usb.c +++ b/drivers/nfc/pn533/usb.c @@ -628,6 +628,7 @@ static void pn533_usb_disconnect(struct usb_interface *interface) usb_free_urb(phy->out_urb); usb_free_urb(phy->ack_urb); kfree(phy->ack_buffer); + usb_put_dev(phy->udev); nfc_info(&interface->dev, "NXP PN533 NFC device disconnected\n"); } diff --git a/drivers/nfc/s3fwrn5/uart.c b/drivers/nfc/s3fwrn5/uart.c index 9c09c10c2a4640..4ee481bd7e9654 100644 --- a/drivers/nfc/s3fwrn5/uart.c +++ b/drivers/nfc/s3fwrn5/uart.c @@ -58,6 +58,12 @@ static size_t s3fwrn82_uart_read(struct serdev_device *serdev, size_t i; for (i = 0; i < count; i++) { + if (!phy->recv_skb) { + phy->recv_skb = alloc_skb(NCI_SKB_BUFF_LEN, GFP_KERNEL); + if (!phy->recv_skb) + return i; + } + skb_put_u8(phy->recv_skb, *data++); if (phy->recv_skb->len < S3FWRN82_NCI_HEADER) @@ -69,9 +75,7 @@ static size_t s3fwrn82_uart_read(struct serdev_device *serdev, s3fwrn5_recv_frame(phy->common.ndev, phy->recv_skb, phy->common.mode); - phy->recv_skb = alloc_skb(NCI_SKB_BUFF_LEN, GFP_KERNEL); - if (!phy->recv_skb) - return 0; + phy->recv_skb = NULL; } return i; diff --git a/drivers/ntb/hw/mscc/ntb_hw_switchtec.c b/drivers/ntb/hw/mscc/ntb_hw_switchtec.c index f851397b65d6e5..0536521fa6cccc 100644 --- a/drivers/ntb/hw/mscc/ntb_hw_switchtec.c +++ b/drivers/ntb/hw/mscc/ntb_hw_switchtec.c @@ -1202,7 +1202,8 @@ static void switchtec_ntb_init_mw(struct switchtec_ntb *sndev) sndev->mmio_self_ctrl); sndev->nr_lut_mw = ioread16(&sndev->mmio_self_ctrl->lut_table_entries); - sndev->nr_lut_mw = rounddown_pow_of_two(sndev->nr_lut_mw); + if (sndev->nr_lut_mw) + sndev->nr_lut_mw = rounddown_pow_of_two(sndev->nr_lut_mw); dev_dbg(&sndev->stdev->dev, "MWs: %d direct, %d lut\n", sndev->nr_direct_mw, sndev->nr_lut_mw); @@ -1212,7 +1213,8 @@ static void switchtec_ntb_init_mw(struct switchtec_ntb *sndev) sndev->peer_nr_lut_mw = ioread16(&sndev->mmio_peer_ctrl->lut_table_entries); - sndev->peer_nr_lut_mw = rounddown_pow_of_two(sndev->peer_nr_lut_mw); + if (sndev->peer_nr_lut_mw) + sndev->peer_nr_lut_mw = rounddown_pow_of_two(sndev->peer_nr_lut_mw); dev_dbg(&sndev->stdev->dev, "Peer MWs: %d direct, %d lut\n", sndev->peer_nr_direct_mw, sndev->peer_nr_lut_mw); @@ -1314,6 +1316,12 @@ static void switchtec_ntb_init_shared(struct switchtec_ntb *sndev) for (i = 0; i < sndev->nr_lut_mw; i++) { int idx = sndev->nr_direct_mw + i; + if (idx >= MAX_MWS) { + dev_err(&sndev->stdev->dev, + "Total number of MW cannot be bigger than %d", MAX_MWS); + break; + } + sndev->self_shared->mw_sizes[idx] = LUT_SIZE; } } diff --git a/drivers/ntb/ntb_transport.c b/drivers/ntb/ntb_transport.c index 71d4bb25f7fdd1..2cee3c1729c35e 100644 --- a/drivers/ntb/ntb_transport.c +++ b/drivers/ntb/ntb_transport.c @@ -1236,9 +1236,9 @@ static int ntb_transport_init_queue(struct ntb_transport_ctx *nt, qp->tx_max_entry = tx_size / qp->tx_max_frame; if (nt->debugfs_node_dir) { - char debugfs_name[4]; + char debugfs_name[8]; - snprintf(debugfs_name, 4, "qp%d", qp_num); + snprintf(debugfs_name, sizeof(debugfs_name), "qp%d", qp_num); qp->debugfs_dir = debugfs_create_dir(debugfs_name, nt->debugfs_node_dir); @@ -1810,12 +1810,13 @@ static void ntb_tx_copy_callback(void *data, static void ntb_memcpy_tx(struct ntb_queue_entry *entry, void __iomem *offset) { -#ifdef ARCH_HAS_NOCACHE_UACCESS +#ifdef copy_to_nontemporal /* * Using non-temporal mov to improve performance on non-cached - * writes, even though we aren't actually copying from user space. + * writes. This only works if __iomem is strictly memory-like, + * but that is the case on x86-64 */ - __copy_from_user_inatomic_nocache(offset, entry->buf, entry->len); + copy_to_nontemporal(offset, entry->buf, entry->len); #else memcpy_toio(offset, entry->buf, entry->len); #endif diff --git a/drivers/nvdimm/bus.c b/drivers/nvdimm/bus.c index 87178a53ff9c2b..8eb205459d4c06 100644 --- a/drivers/nvdimm/bus.c +++ b/drivers/nvdimm/bus.c @@ -486,14 +486,15 @@ EXPORT_SYMBOL_GPL(nd_synchronize); static void nd_async_device_register(void *d, async_cookie_t cookie) { struct device *dev = d; + struct device *parent = dev->parent; if (device_add(dev) != 0) { dev_err(dev, "%s: failed\n", __func__); put_device(dev); } put_device(dev); - if (dev->parent) - put_device(dev->parent); + if (parent) + put_device(parent); } static void nd_async_device_unregister(void *d, async_cookie_t cookie) diff --git a/drivers/nvdimm/nd_virtio.c b/drivers/nvdimm/nd_virtio.c index c3f07be4aa22ad..af82385be7c6aa 100644 --- a/drivers/nvdimm/nd_virtio.c +++ b/drivers/nvdimm/nd_virtio.c @@ -44,6 +44,8 @@ static int virtio_pmem_flush(struct nd_region *nd_region) unsigned long flags; int err, err1; + guard(mutex)(&vpmem->flush_lock); + /* * Don't bother to submit the request to the device if the device is * not activated. @@ -53,7 +55,6 @@ static int virtio_pmem_flush(struct nd_region *nd_region) return -EIO; } - might_sleep(); req_data = kmalloc(sizeof(*req_data), GFP_KERNEL); if (!req_data) return -ENOMEM; diff --git a/drivers/nvdimm/virtio_pmem.c b/drivers/nvdimm/virtio_pmem.c index 2396d19ce54969..77b1966619059c 100644 --- a/drivers/nvdimm/virtio_pmem.c +++ b/drivers/nvdimm/virtio_pmem.c @@ -64,6 +64,7 @@ static int virtio_pmem_probe(struct virtio_device *vdev) goto out_err; } + mutex_init(&vpmem->flush_lock); vpmem->vdev = vdev; vdev->priv = vpmem; err = init_vq(vpmem); diff --git a/drivers/nvdimm/virtio_pmem.h b/drivers/nvdimm/virtio_pmem.h index 0dddefe594c46a..f72cf17f9518fb 100644 --- a/drivers/nvdimm/virtio_pmem.h +++ b/drivers/nvdimm/virtio_pmem.h @@ -13,6 +13,7 @@ #include #include #include +#include #include struct virtio_pmem_request { @@ -35,6 +36,9 @@ struct virtio_pmem { /* Virtio pmem request queue */ struct virtqueue *req_vq; + /* Serialize flush requests to the device. */ + struct mutex flush_lock; + /* nvdimm bus registers virtio pmem device */ struct nvdimm_bus *nvdimm_bus; struct nvdimm_bus_descriptor nd_desc; diff --git a/drivers/nvme/host/apple.c b/drivers/nvme/host/apple.c index ed61b97fde59f7..e4085e6ac8236d 100644 --- a/drivers/nvme/host/apple.c +++ b/drivers/nvme/host/apple.c @@ -203,8 +203,20 @@ struct apple_nvme { int irq; spinlock_t lock; + + /* + * Delayed cache flush handling state + */ + struct nvme_ns *flush_ns; + unsigned long flush_interval; + unsigned long last_flush; + struct delayed_work flush_dwork; }; +unsigned int flush_interval = 1000; +module_param(flush_interval, uint, 0644); +MODULE_PARM_DESC(flush_interval, "Grace period in msecs between flushes"); + static_assert(sizeof(struct nvme_command) == 64); static_assert(sizeof(struct apple_nvmmu_tcb) == 128); @@ -762,6 +774,26 @@ static int apple_nvme_remove_sq(struct apple_nvme *anv) return nvme_submit_sync_cmd(anv->ctrl.admin_q, &c, NULL, 0); } +static bool apple_nvme_delayed_flush(struct apple_nvme *anv, struct nvme_ns *ns, + struct request *req) +{ + if (!anv->flush_interval || req_op(req) != REQ_OP_FLUSH) + return false; + if (delayed_work_pending(&anv->flush_dwork)) + return true; + if (time_before(jiffies, anv->last_flush + anv->flush_interval)) { + kblockd_mod_delayed_work_on(WORK_CPU_UNBOUND, &anv->flush_dwork, + anv->flush_interval); + if (WARN_ON_ONCE(anv->flush_ns && anv->flush_ns != ns)) + goto out; + anv->flush_ns = ns; + return true; + } +out: + anv->last_flush = jiffies; + return false; +} + static blk_status_t apple_nvme_queue_rq(struct blk_mq_hw_ctx *hctx, const struct blk_mq_queue_data *bd) { @@ -798,6 +830,11 @@ static blk_status_t apple_nvme_queue_rq(struct blk_mq_hw_ctx *hctx, nvme_start_request(req); + if (apple_nvme_delayed_flush(anv, ns, req)) { + blk_mq_complete_request(req); + return BLK_STS_OK; + } + if (anv->hw->has_lsq_nvmmu) apple_nvme_submit_cmd_t8103(q, cmnd); else @@ -1453,6 +1490,28 @@ static void devm_apple_nvme_mempool_destroy(void *data) mempool_destroy(data); } +static void apple_nvme_flush_work(struct work_struct *work) +{ + struct nvme_command c = { }; + struct apple_nvme *anv; + struct nvme_ns *ns; + int err; + + anv = container_of(work, struct apple_nvme, flush_dwork.work); + ns = anv->flush_ns; + if (WARN_ON_ONCE(!ns)) + return; + + c.common.opcode = nvme_cmd_flush; + c.common.nsid = cpu_to_le32(anv->flush_ns->head->ns_id); + err = nvme_submit_sync_cmd(ns->queue, &c, NULL, 0); + if (err) { + dev_err(anv->dev, "Deferred flush failed: %d\n", err); + } else { + anv->last_flush = jiffies; + } +} + static struct apple_nvme *apple_nvme_alloc(struct platform_device *pdev) { struct device *dev = &pdev->dev; @@ -1621,6 +1680,14 @@ static int apple_nvme_probe(struct platform_device *pdev) goto out_uninit_ctrl; } + if (flush_interval) { + anv->flush_interval = msecs_to_jiffies(flush_interval); + anv->flush_ns = NULL; + anv->last_flush = jiffies - anv->flush_interval; + } + + INIT_DELAYED_WORK(&anv->flush_dwork, apple_nvme_flush_work); + nvme_reset_ctrl(&anv->ctrl); async_schedule(apple_nvme_async_probe, anv); @@ -1658,6 +1725,7 @@ static void apple_nvme_shutdown(struct platform_device *pdev) { struct apple_nvme *anv = platform_get_drvdata(pdev); + flush_delayed_work(&anv->flush_dwork); apple_nvme_disable(anv, true); if (apple_rtkit_is_running(anv->rtk)) { apple_rtkit_shutdown(anv->rtk); diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c index 7bf228df6001f1..3fdcd73b954680 100644 --- a/drivers/nvme/host/core.c +++ b/drivers/nvme/host/core.c @@ -4864,6 +4864,13 @@ int nvme_alloc_admin_tag_set(struct nvme_ctrl *ctrl, struct blk_mq_tag_set *set, if (ret) return ret; + /* + * If a previous admin queue exists (e.g., from before a reset), + * put it now before allocating a new one to avoid orphaning it. + */ + if (ctrl->admin_q) + blk_put_queue(ctrl->admin_q); + ctrl->admin_q = blk_mq_alloc_queue(set, &lim, NULL); if (IS_ERR(ctrl->admin_q)) { ret = PTR_ERR(ctrl->admin_q); diff --git a/drivers/nvme/host/fabrics.c b/drivers/nvme/host/fabrics.c index 55a8afd2efd503..d37cb140d83232 100644 --- a/drivers/nvme/host/fabrics.c +++ b/drivers/nvme/host/fabrics.c @@ -1290,8 +1290,8 @@ void nvmf_free_options(struct nvmf_ctrl_options *opts) kfree(opts->subsysnqn); kfree(opts->host_traddr); kfree(opts->host_iface); - kfree(opts->dhchap_secret); - kfree(opts->dhchap_ctrl_secret); + kfree_sensitive(opts->dhchap_secret); + kfree_sensitive(opts->dhchap_ctrl_secret); kfree(opts); } EXPORT_SYMBOL_GPL(nvmf_free_options); diff --git a/drivers/nvme/host/multipath.c b/drivers/nvme/host/multipath.c index 174027d1cc198e..5e41fbaf5f46ad 100644 --- a/drivers/nvme/host/multipath.c +++ b/drivers/nvme/host/multipath.c @@ -1310,13 +1310,11 @@ void nvme_mpath_remove_disk(struct nvme_ns_head *head) if (!list_empty(&head->list)) goto out; - if (head->delayed_removal_secs) { - /* - * Ensure that no one could remove this module while the head - * remove work is pending. - */ - if (!try_module_get(THIS_MODULE)) - goto out; + /* + * Ensure that no one could remove this module while the head + * remove work is pending. + */ + if (head->delayed_removal_secs && try_module_get(THIS_MODULE)) { mod_delayed_work(nvme_wq, &head->remove_work, head->delayed_removal_secs * HZ); } else { diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c index d86f2565a92cac..f6d4f5910bdbc4 100644 --- a/drivers/nvme/host/pci.c +++ b/drivers/nvme/host/pci.c @@ -400,7 +400,7 @@ static void nvme_dbbuf_set(struct nvme_dev *dev) /* Free memory and continue on */ nvme_dbbuf_dma_free(dev); - for (i = 1; i <= dev->online_queues; i++) + for (i = 1; i < dev->online_queues; i++) nvme_dbbuf_free(&dev->queues[i]); } } @@ -1484,14 +1484,16 @@ static irqreturn_t nvme_irq_check(int irq, void *data) static void nvme_poll_irqdisable(struct nvme_queue *nvmeq) { struct pci_dev *pdev = to_pci_dev(nvmeq->dev->dev); + int irq; WARN_ON_ONCE(test_bit(NVMEQ_POLLED, &nvmeq->flags)); - disable_irq(pci_irq_vector(pdev, nvmeq->cq_vector)); + irq = pci_irq_vector(pdev, nvmeq->cq_vector); + disable_irq(irq); spin_lock(&nvmeq->cq_poll_lock); nvme_poll_cq(nvmeq, NULL); spin_unlock(&nvmeq->cq_poll_lock); - enable_irq(pci_irq_vector(pdev, nvmeq->cq_vector)); + enable_irq(irq); } static int nvme_poll(struct blk_mq_hw_ctx *hctx, struct io_comp_batch *iob) @@ -1499,7 +1501,8 @@ static int nvme_poll(struct blk_mq_hw_ctx *hctx, struct io_comp_batch *iob) struct nvme_queue *nvmeq = hctx->driver_data; bool found; - if (!nvme_cqe_pending(nvmeq)) + if (!test_bit(NVMEQ_POLLED, &nvmeq->flags) || + !nvme_cqe_pending(nvmeq)) return 0; spin_lock(&nvmeq->cq_poll_lock); @@ -2776,7 +2779,13 @@ static int nvme_setup_io_queues(struct nvme_dev *dev) dev->nr_write_queues = write_queues; dev->nr_poll_queues = poll_queues; - nr_io_queues = dev->nr_allocated_queues - 1; + /* + * The initial number of allocated queue slots may be too large if the + * user reduced the special queue parameters. Cap the value to the + * number we need for this round. + */ + nr_io_queues = min(nvme_max_io_queues(dev), + dev->nr_allocated_queues - 1); result = nvme_set_queue_count(&dev->ctrl, &nr_io_queues); if (result < 0) return result; diff --git a/drivers/nvme/host/pr.c b/drivers/nvme/host/pr.c index ad2ecc2f49a972..fe7dbe26481584 100644 --- a/drivers/nvme/host/pr.c +++ b/drivers/nvme/host/pr.c @@ -242,7 +242,7 @@ static int nvme_pr_read_keys(struct block_device *bdev, if (rse_len > U32_MAX) return -EINVAL; - rse = kzalloc(rse_len, GFP_KERNEL); + rse = kvzalloc(rse_len, GFP_KERNEL); if (!rse) return -ENOMEM; @@ -267,7 +267,7 @@ static int nvme_pr_read_keys(struct block_device *bdev, } free_rse: - kfree(rse); + kvfree(rse); return ret; } diff --git a/drivers/nvme/target/admin-cmd.c b/drivers/nvme/target/admin-cmd.c index 3da31bb1183eb7..100d1466ff8416 100644 --- a/drivers/nvme/target/admin-cmd.c +++ b/drivers/nvme/target/admin-cmd.c @@ -1586,7 +1586,7 @@ void nvmet_execute_async_event(struct nvmet_req *req) ctrl->async_event_cmds[ctrl->nr_async_event_cmds++] = req; mutex_unlock(&ctrl->lock); - queue_work(nvmet_wq, &ctrl->async_event_work); + queue_work(nvmet_aen_wq, &ctrl->async_event_work); } void nvmet_execute_keep_alive(struct nvmet_req *req) diff --git a/drivers/nvme/target/core.c b/drivers/nvme/target/core.c index cc88e5a28c8a9d..5075f7123358af 100644 --- a/drivers/nvme/target/core.c +++ b/drivers/nvme/target/core.c @@ -26,6 +26,8 @@ static DEFINE_IDA(cntlid_ida); struct workqueue_struct *nvmet_wq; EXPORT_SYMBOL_GPL(nvmet_wq); +struct workqueue_struct *nvmet_aen_wq; +EXPORT_SYMBOL_GPL(nvmet_aen_wq); /* * This read/write semaphore is used to synchronize access to configuration @@ -205,7 +207,7 @@ void nvmet_add_async_event(struct nvmet_ctrl *ctrl, u8 event_type, list_add_tail(&aen->entry, &ctrl->async_events); mutex_unlock(&ctrl->lock); - queue_work(nvmet_wq, &ctrl->async_event_work); + queue_work(nvmet_aen_wq, &ctrl->async_event_work); } static void nvmet_add_to_changed_ns_log(struct nvmet_ctrl *ctrl, __le32 nsid) @@ -1958,9 +1960,14 @@ static int __init nvmet_init(void) if (!nvmet_wq) goto out_free_buffered_work_queue; + nvmet_aen_wq = alloc_workqueue("nvmet-aen-wq", + WQ_MEM_RECLAIM | WQ_UNBOUND, 0); + if (!nvmet_aen_wq) + goto out_free_nvmet_work_queue; + error = nvmet_init_debugfs(); if (error) - goto out_free_nvmet_work_queue; + goto out_free_nvmet_aen_work_queue; error = nvmet_init_discovery(); if (error) @@ -1976,6 +1983,8 @@ static int __init nvmet_init(void) nvmet_exit_discovery(); out_exit_debugfs: nvmet_exit_debugfs(); +out_free_nvmet_aen_work_queue: + destroy_workqueue(nvmet_aen_wq); out_free_nvmet_work_queue: destroy_workqueue(nvmet_wq); out_free_buffered_work_queue: @@ -1993,6 +2002,7 @@ static void __exit nvmet_exit(void) nvmet_exit_discovery(); nvmet_exit_debugfs(); ida_destroy(&cntlid_ida); + destroy_workqueue(nvmet_aen_wq); destroy_workqueue(nvmet_wq); destroy_workqueue(buffered_io_wq); destroy_workqueue(zbd_wq); diff --git a/drivers/nvme/target/fcloop.c b/drivers/nvme/target/fcloop.c index c30e9a3e014fb1..38bd2db3d6bbe7 100644 --- a/drivers/nvme/target/fcloop.c +++ b/drivers/nvme/target/fcloop.c @@ -491,6 +491,7 @@ fcloop_t2h_xmt_ls_rsp(struct nvme_fc_local_port *localport, struct fcloop_rport *rport = remoteport->private; struct nvmet_fc_target_port *targetport = rport->targetport; struct fcloop_tport *tport; + int ret = 0; if (!targetport) { /* @@ -500,12 +501,18 @@ fcloop_t2h_xmt_ls_rsp(struct nvme_fc_local_port *localport, * We end up here from delete association exchange: * nvmet_fc_xmt_disconnect_assoc sends an async request. * - * Return success because this is what LLDDs do; silently - * drop the response. + * Return success when remoteport is still online because this + * is what LLDDs do and silently drop the response. Otherwise, + * return with error to signal upper layer to perform the lsrsp + * resource cleanup. */ - lsrsp->done(lsrsp); + if (remoteport->port_state == FC_OBJSTATE_ONLINE) + lsrsp->done(lsrsp); + else + ret = -ENODEV; + kmem_cache_free(lsreq_cache, tls_req); - return 0; + return ret; } memcpy(lsreq->rspaddr, lsrsp->rspbuf, diff --git a/drivers/nvme/target/nvmet.h b/drivers/nvme/target/nvmet.h index b664b584fdc8e6..319d6a5e9cf053 100644 --- a/drivers/nvme/target/nvmet.h +++ b/drivers/nvme/target/nvmet.h @@ -501,6 +501,7 @@ extern struct kmem_cache *nvmet_bvec_cache; extern struct workqueue_struct *buffered_io_wq; extern struct workqueue_struct *zbd_wq; extern struct workqueue_struct *nvmet_wq; +extern struct workqueue_struct *nvmet_aen_wq; static inline void nvmet_set_result(struct nvmet_req *req, u32 result) { diff --git a/drivers/nvme/target/rdma.c b/drivers/nvme/target/rdma.c index 9c12b2361a6d7a..03843236496711 100644 --- a/drivers/nvme/target/rdma.c +++ b/drivers/nvme/target/rdma.c @@ -2088,6 +2088,7 @@ static void nvmet_rdma_remove_one(struct ib_device *ib_device, void *client_data mutex_unlock(&nvmet_rdma_queue_mutex); flush_workqueue(nvmet_wq); + flush_workqueue(nvmet_aen_wq); } static struct ib_client nvmet_rdma_ib_client = { diff --git a/drivers/nvmem/Kconfig b/drivers/nvmem/Kconfig index bf47a982cf6290..74ddbd0f79b0ec 100644 --- a/drivers/nvmem/Kconfig +++ b/drivers/nvmem/Kconfig @@ -30,7 +30,7 @@ source "drivers/nvmem/layouts/Kconfig" config NVMEM_AN8855_EFUSE tristate "Airoha AN8855 eFuse support" - depends on MFD_AIROHA_AN8855 || COMPILE_TEST + depends on COMPILE_TEST help Say y here to enable support for reading eFuses on Airoha AN8855 Switch. These are e.g. used to store factory programmed diff --git a/drivers/nvmem/core.c b/drivers/nvmem/core.c index 387c88c5525954..07cd766c809063 100644 --- a/drivers/nvmem/core.c +++ b/drivers/nvmem/core.c @@ -831,6 +831,7 @@ static int nvmem_add_cells_from_dt(struct nvmem_device *nvmem, struct device_nod kfree(info.name); if (ret) { of_node_put(child); + of_node_put(info.np); return ret; } } @@ -1618,12 +1619,14 @@ static void nvmem_shift_read_buffer_in_place(struct nvmem_cell_entry *cell, void *p = *b++ >> bit_offset; /* setup rest of the bytes if any */ - for (i = 1; i < cell->bytes; i++) { + for (i = 1; i < (cell->bytes - bytes_offset); i++) { /* Get bits from next byte and shift them towards msb */ *p++ |= *b << (BITS_PER_BYTE - bit_offset); *p = *b++ >> bit_offset; } + /* point to end of the buffer unused bits will be cleared */ + p = buf + cell->bytes - 1; } else if (p != b) { memmove(p, b, cell->bytes - bytes_offset); p += cell->bytes - 1; diff --git a/drivers/nvmem/imx-ocotp-ele.c b/drivers/nvmem/imx-ocotp-ele.c index 7cf7e809a8f51a..a0d2985c6d0304 100644 --- a/drivers/nvmem/imx-ocotp-ele.c +++ b/drivers/nvmem/imx-ocotp-ele.c @@ -131,6 +131,7 @@ static int imx_ocotp_cell_pp(void *context, const char *id, int index, static void imx_ocotp_fixup_dt_cell_info(struct nvmem_device *nvmem, struct nvmem_cell_info *cell) { + cell->raw_len = round_up(cell->bytes, 4); cell->read_post_process = imx_ocotp_cell_pp; } diff --git a/drivers/nvmem/imx-ocotp.c b/drivers/nvmem/imx-ocotp.c index 7bf7656d4f9631..108d78d7f6cba6 100644 --- a/drivers/nvmem/imx-ocotp.c +++ b/drivers/nvmem/imx-ocotp.c @@ -589,6 +589,7 @@ MODULE_DEVICE_TABLE(of, imx_ocotp_dt_ids); static void imx_ocotp_fixup_dt_cell_info(struct nvmem_device *nvmem, struct nvmem_cell_info *cell) { + cell->raw_len = round_up(cell->bytes, 4); cell->read_post_process = imx_ocotp_cell_pp; } diff --git a/drivers/nvmem/zynqmp_nvmem.c b/drivers/nvmem/zynqmp_nvmem.c index 7da717d6c7faf3..d297ff150dc024 100644 --- a/drivers/nvmem/zynqmp_nvmem.c +++ b/drivers/nvmem/zynqmp_nvmem.c @@ -66,7 +66,7 @@ static int zynqmp_efuse_access(void *context, unsigned int offset, dma_addr_t dma_buf; size_t words = bytes / WORD_INBYTES; int ret; - int value; + unsigned int value; char *data; if (bytes % WORD_INBYTES != 0) { @@ -80,7 +80,7 @@ static int zynqmp_efuse_access(void *context, unsigned int offset, } if (pufflag == 1 && flag == EFUSE_WRITE) { - memcpy(&value, val, bytes); + memcpy(&value, val, sizeof(value)); if ((offset == EFUSE_PUF_START_OFFSET || offset == EFUSE_PUF_MID_OFFSET) && value & P_USER_0_64_UPPER_MASK) { @@ -100,7 +100,7 @@ static int zynqmp_efuse_access(void *context, unsigned int offset, if (!efuse) return -ENOMEM; - data = dma_alloc_coherent(dev, sizeof(bytes), + data = dma_alloc_coherent(dev, bytes, &dma_buf, GFP_KERNEL); if (!data) { ret = -ENOMEM; @@ -134,7 +134,7 @@ static int zynqmp_efuse_access(void *context, unsigned int offset, if (flag == EFUSE_READ) memcpy(val, data, bytes); efuse_access_err: - dma_free_coherent(dev, sizeof(bytes), + dma_free_coherent(dev, bytes, data, dma_buf); efuse_data_fail: dma_free_coherent(dev, sizeof(struct xilinx_efuse), diff --git a/drivers/of/kexec.c b/drivers/of/kexec.c index 1ee2d31816aebe..c4cf3552c01838 100644 --- a/drivers/of/kexec.c +++ b/drivers/of/kexec.c @@ -128,7 +128,6 @@ int __init ima_get_kexec_buffer(void **addr, size_t *size) { int ret, len; unsigned long tmp_addr; - unsigned long start_pfn, end_pfn; size_t tmp_size; const void *prop; @@ -144,17 +143,9 @@ int __init ima_get_kexec_buffer(void **addr, size_t *size) if (!tmp_size) return -ENOENT; - /* - * Calculate the PFNs for the buffer and ensure - * they are with in addressable memory. - */ - start_pfn = PHYS_PFN(tmp_addr); - end_pfn = PHYS_PFN(tmp_addr + tmp_size - 1); - if (!page_is_ram(start_pfn) || !page_is_ram(end_pfn)) { - pr_warn("IMA buffer at 0x%lx, size = 0x%zx beyond memory\n", - tmp_addr, tmp_size); - return -EINVAL; - } + ret = ima_validate_range(tmp_addr, tmp_size); + if (ret) + return ret; *addr = __va(tmp_addr); *size = tmp_size; diff --git a/drivers/of/unittest.c b/drivers/of/unittest.c index 3b773aaf9d0503..9c184e93f50c67 100644 --- a/drivers/of/unittest.c +++ b/drivers/of/unittest.c @@ -804,11 +804,13 @@ static void __init of_unittest_property_copy(void) new = __of_prop_dup(&p1, GFP_KERNEL); unittest(new && propcmp(&p1, new), "empty property didn't copy correctly\n"); - __of_prop_free(new); + if (new) + __of_prop_free(new); new = __of_prop_dup(&p2, GFP_KERNEL); unittest(new && propcmp(&p2, new), "non-empty property didn't copy correctly\n"); - __of_prop_free(new); + if (new) + __of_prop_free(new); #endif } diff --git a/drivers/opp/core.c b/drivers/opp/core.c index dbebb8c829bcb4..ae43c656f108c2 100644 --- a/drivers/opp/core.c +++ b/drivers/opp/core.c @@ -241,7 +241,7 @@ unsigned int dev_pm_opp_get_level(struct dev_pm_opp *opp) { if (IS_ERR_OR_NULL(opp) || !opp->available) { pr_err("%s: Invalid parameters\n", __func__); - return 0; + return U32_MAX; } return opp->level; diff --git a/drivers/pci/bus.c b/drivers/pci/bus.c index 4383a36fd6ca09..41e5c45e38b5ec 100644 --- a/drivers/pci/bus.c +++ b/drivers/pci/bus.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include @@ -379,6 +380,13 @@ void pci_bus_add_device(struct pci_dev *dev) put_device(&pdev->dev); } + /* + * Enable runtime PM, which potentially allows the device to + * suspend immediately, only after the PCI state has been + * configured completely. + */ + pm_runtime_enable(&dev->dev); + if (!dn || of_device_is_available(dn)) pci_dev_allow_binding(dev); diff --git a/drivers/pci/controller/Kconfig b/drivers/pci/controller/Kconfig index c254d2b8bf17b0..316d80f743a5e2 100644 --- a/drivers/pci/controller/Kconfig +++ b/drivers/pci/controller/Kconfig @@ -45,6 +45,7 @@ config PCIE_APPLE depends on ARCH_APPLE || COMPILE_TEST depends on OF depends on PCI_MSI + depends on PAGE_SIZE_16KB || COMPILE_TEST select PCI_HOST_COMMON select IRQ_MSI_LIB help diff --git a/drivers/pci/controller/cadence/pci-j721e.c b/drivers/pci/controller/cadence/pci-j721e.c index ecd1b031240063..6f2501479c7019 100644 --- a/drivers/pci/controller/cadence/pci-j721e.c +++ b/drivers/pci/controller/cadence/pci-j721e.c @@ -620,9 +620,11 @@ static int j721e_pcie_probe(struct platform_device *pdev) gpiod_set_value_cansleep(pcie->reset_gpio, 1); } - ret = cdns_pcie_host_setup(rc); - if (ret < 0) - goto err_pcie_setup; + if (IS_ENABLED(CONFIG_PCI_J721E_HOST)) { + ret = cdns_pcie_host_setup(rc); + if (ret < 0) + goto err_pcie_setup; + } break; case PCI_MODE_EP: @@ -632,9 +634,11 @@ static int j721e_pcie_probe(struct platform_device *pdev) goto err_get_sync; } - ret = cdns_pcie_ep_setup(ep); - if (ret < 0) - goto err_pcie_setup; + if (IS_ENABLED(CONFIG_PCI_J721E_EP)) { + ret = cdns_pcie_ep_setup(ep); + if (ret < 0) + goto err_pcie_setup; + } break; } @@ -659,10 +663,11 @@ static void j721e_pcie_remove(struct platform_device *pdev) struct cdns_pcie_ep *ep; struct cdns_pcie_rc *rc; - if (pcie->mode == PCI_MODE_RC) { + if (IS_ENABLED(CONFIG_PCI_J721E_HOST) && + pcie->mode == PCI_MODE_RC) { rc = container_of(cdns_pcie, struct cdns_pcie_rc, pcie); cdns_pcie_host_disable(rc); - } else { + } else if (IS_ENABLED(CONFIG_PCI_J721E_EP)) { ep = container_of(cdns_pcie, struct cdns_pcie_ep, pcie); cdns_pcie_ep_disable(ep); } @@ -728,10 +733,12 @@ static int j721e_pcie_resume_noirq(struct device *dev) gpiod_set_value_cansleep(pcie->reset_gpio, 1); } - ret = cdns_pcie_host_link_setup(rc); - if (ret < 0) { - clk_disable_unprepare(pcie->refclk); - return ret; + if (IS_ENABLED(CONFIG_PCI_J721E_HOST)) { + ret = cdns_pcie_host_link_setup(rc); + if (ret < 0) { + clk_disable_unprepare(pcie->refclk); + return ret; + } } /* @@ -741,10 +748,12 @@ static int j721e_pcie_resume_noirq(struct device *dev) for (enum cdns_pcie_rp_bar bar = RP_BAR0; bar <= RP_NO_BAR; bar++) rc->avail_ib_bar[bar] = true; - ret = cdns_pcie_host_init(rc); - if (ret) { - clk_disable_unprepare(pcie->refclk); - return ret; + if (IS_ENABLED(CONFIG_PCI_J721E_HOST)) { + ret = cdns_pcie_host_init(rc); + if (ret) { + clk_disable_unprepare(pcie->refclk); + return ret; + } } } diff --git a/drivers/pci/controller/cadence/pcie-cadence-host-common.c b/drivers/pci/controller/cadence/pcie-cadence-host-common.c index 15415d7f35ee9c..2b0211870f02ab 100644 --- a/drivers/pci/controller/cadence/pcie-cadence-host-common.c +++ b/drivers/pci/controller/cadence/pcie-cadence-host-common.c @@ -173,11 +173,21 @@ int cdns_pcie_host_dma_ranges_cmp(void *priv, const struct list_head *a, const struct list_head *b) { struct resource_entry *entry1, *entry2; + u64 size1, size2; entry1 = container_of(a, struct resource_entry, node); entry2 = container_of(b, struct resource_entry, node); - return resource_size(entry2->res) - resource_size(entry1->res); + size1 = resource_size(entry1->res); + size2 = resource_size(entry2->res); + + if (size1 > size2) + return -1; + + if (size1 < size2) + return 1; + + return 0; } EXPORT_SYMBOL_GPL(cdns_pcie_host_dma_ranges_cmp); diff --git a/drivers/pci/controller/cadence/pcie-cadence.c b/drivers/pci/controller/cadence/pcie-cadence.c index e6f1a4ac0fb7ab..a1eada56edba76 100644 --- a/drivers/pci/controller/cadence/pcie-cadence.c +++ b/drivers/pci/controller/cadence/pcie-cadence.c @@ -13,13 +13,13 @@ u8 cdns_pcie_find_capability(struct cdns_pcie *pcie, u8 cap) { return PCI_FIND_NEXT_CAP(cdns_pcie_read_cfg, PCI_CAPABILITY_LIST, - cap, pcie); + cap, NULL, pcie); } EXPORT_SYMBOL_GPL(cdns_pcie_find_capability); u16 cdns_pcie_find_ext_capability(struct cdns_pcie *pcie, u8 cap) { - return PCI_FIND_NEXT_EXT_CAP(cdns_pcie_read_cfg, 0, cap, pcie); + return PCI_FIND_NEXT_EXT_CAP(cdns_pcie_read_cfg, 0, cap, NULL, pcie); } EXPORT_SYMBOL_GPL(cdns_pcie_find_ext_capability); diff --git a/drivers/pci/controller/dwc/pci-dra7xx.c b/drivers/pci/controller/dwc/pci-dra7xx.c index 01cfd9aeb0b818..d5d26229063f2a 100644 --- a/drivers/pci/controller/dwc/pci-dra7xx.c +++ b/drivers/pci/controller/dwc/pci-dra7xx.c @@ -424,6 +424,7 @@ static int dra7xx_pcie_raise_irq(struct dw_pcie_ep *ep, u8 func_no, } static const struct pci_epc_features dra7xx_pcie_epc_features = { + DWC_EPC_COMMON_FEATURES, .linkup_notifier = true, .msi_capable = true, }; diff --git a/drivers/pci/controller/dwc/pci-imx6.c b/drivers/pci/controller/dwc/pci-imx6.c index 4668fc9648bff6..c6dfbd57880eab 100644 --- a/drivers/pci/controller/dwc/pci-imx6.c +++ b/drivers/pci/controller/dwc/pci-imx6.c @@ -52,6 +52,8 @@ #define IMX95_PCIE_REF_CLKEN BIT(23) #define IMX95_PCIE_PHY_CR_PARA_SEL BIT(9) #define IMX95_PCIE_SS_RW_REG_1 0xf4 +#define IMX95_PCIE_CLKREQ_OVERRIDE_EN BIT(8) +#define IMX95_PCIE_CLKREQ_OVERRIDE_VAL BIT(9) #define IMX95_PCIE_SYS_AUX_PWR_DET BIT(31) #define IMX95_PE0_GEN_CTRL_1 0x1050 @@ -114,6 +116,7 @@ enum imx_pcie_variants { #define IMX_PCIE_FLAG_BROKEN_SUSPEND BIT(9) #define IMX_PCIE_FLAG_HAS_LUT BIT(10) #define IMX_PCIE_FLAG_8GT_ECN_ERR051586 BIT(11) +#define IMX_PCIE_FLAG_SKIP_L23_READY BIT(12) #define imx_check_flag(pci, val) (pci->drvdata->flags & val) @@ -706,6 +709,22 @@ static int imx7d_pcie_enable_ref_clk(struct imx_pcie *imx_pcie, bool enable) return 0; } +static void imx95_pcie_clkreq_override(struct imx_pcie *imx_pcie, bool enable) +{ + regmap_update_bits(imx_pcie->iomuxc_gpr, IMX95_PCIE_SS_RW_REG_1, + IMX95_PCIE_CLKREQ_OVERRIDE_EN, + enable ? IMX95_PCIE_CLKREQ_OVERRIDE_EN : 0); + regmap_update_bits(imx_pcie->iomuxc_gpr, IMX95_PCIE_SS_RW_REG_1, + IMX95_PCIE_CLKREQ_OVERRIDE_VAL, + enable ? IMX95_PCIE_CLKREQ_OVERRIDE_VAL : 0); +} + +static int imx95_pcie_enable_ref_clk(struct imx_pcie *imx_pcie, bool enable) +{ + imx95_pcie_clkreq_override(imx_pcie, enable); + return 0; +} + static int imx_pcie_clk_enable(struct imx_pcie *imx_pcie) { struct dw_pcie *pci = imx_pcie->pci; @@ -1387,6 +1406,7 @@ static int imx_pcie_ep_raise_irq(struct dw_pcie_ep *ep, u8 func_no, } static const struct pci_epc_features imx8m_pcie_epc_features = { + DWC_EPC_COMMON_FEATURES, .msi_capable = true, .bar[BAR_1] = { .type = BAR_RESERVED, }, .bar[BAR_3] = { .type = BAR_RESERVED, }, @@ -1396,6 +1416,7 @@ static const struct pci_epc_features imx8m_pcie_epc_features = { }; static const struct pci_epc_features imx8q_pcie_epc_features = { + DWC_EPC_COMMON_FEATURES, .msi_capable = true, .bar[BAR_1] = { .type = BAR_RESERVED, }, .bar[BAR_3] = { .type = BAR_RESERVED, }, @@ -1416,6 +1437,7 @@ static const struct pci_epc_features imx8q_pcie_epc_features = { * BAR5 | Enable | 32-bit | 64 KB | Programmable Size */ static const struct pci_epc_features imx95_pcie_epc_features = { + DWC_EPC_COMMON_FEATURES, .msi_capable = true, .bar[BAR_1] = { .type = BAR_FIXED, .fixed_size = SZ_64K, }, .align = SZ_4K, @@ -1777,6 +1799,8 @@ static int imx_pcie_probe(struct platform_device *pdev) */ imx_pcie_add_lut_by_rid(imx_pcie, 0); } else { + if (imx_check_flag(imx_pcie, IMX_PCIE_FLAG_SKIP_L23_READY)) + pci->pp.skip_l23_ready = true; pci->pp.use_atu_msg = true; ret = dw_pcie_host_init(&pci->pp); if (ret < 0) @@ -1838,6 +1862,7 @@ static const struct imx_pcie_drvdata drvdata[] = { .variant = IMX6QP, .flags = IMX_PCIE_FLAG_IMX_PHY | IMX_PCIE_FLAG_SPEED_CHANGE_WORKAROUND | + IMX_PCIE_FLAG_SKIP_L23_READY | IMX_PCIE_FLAG_SUPPORTS_SUSPEND, .dbi_length = 0x200, .gpr = "fsl,imx6q-iomuxc-gpr", @@ -1854,6 +1879,7 @@ static const struct imx_pcie_drvdata drvdata[] = { .variant = IMX7D, .flags = IMX_PCIE_FLAG_SUPPORTS_SUSPEND | IMX_PCIE_FLAG_HAS_APP_RESET | + IMX_PCIE_FLAG_SKIP_L23_READY | IMX_PCIE_FLAG_HAS_PHY_RESET, .gpr = "fsl,imx7d-iomuxc-gpr", .mode_off[0] = IOMUXC_GPR12, @@ -1913,6 +1939,7 @@ static const struct imx_pcie_drvdata drvdata[] = { .core_reset = imx95_pcie_core_reset, .init_phy = imx95_pcie_init_phy, .wait_pll_lock = imx95_pcie_wait_for_phy_pll_lock, + .enable_ref_clk = imx95_pcie_enable_ref_clk, }, [IMX8MQ_EP] = { .variant = IMX8MQ_EP, @@ -1969,6 +1996,7 @@ static const struct imx_pcie_drvdata drvdata[] = { .core_reset = imx95_pcie_core_reset, .wait_pll_lock = imx95_pcie_wait_for_phy_pll_lock, .epc_features = &imx95_pcie_epc_features, + .enable_ref_clk = imx95_pcie_enable_ref_clk, .mode = DW_PCIE_EP_TYPE, }, }; diff --git a/drivers/pci/controller/dwc/pci-keystone.c b/drivers/pci/controller/dwc/pci-keystone.c index f86d9111f863f1..20fa4dadb82afe 100644 --- a/drivers/pci/controller/dwc/pci-keystone.c +++ b/drivers/pci/controller/dwc/pci-keystone.c @@ -930,6 +930,7 @@ static int ks_pcie_am654_raise_irq(struct dw_pcie_ep *ep, u8 func_no, } static const struct pci_epc_features ks_pcie_am654_epc_features = { + DWC_EPC_COMMON_FEATURES, .msi_capable = true, .msix_capable = true, .bar[BAR_0] = { .type = BAR_RESERVED, }, diff --git a/drivers/pci/controller/dwc/pcie-artpec6.c b/drivers/pci/controller/dwc/pcie-artpec6.c index f4a136ee2daf38..e994b75986c34a 100644 --- a/drivers/pci/controller/dwc/pcie-artpec6.c +++ b/drivers/pci/controller/dwc/pcie-artpec6.c @@ -370,6 +370,7 @@ static int artpec6_pcie_raise_irq(struct dw_pcie_ep *ep, u8 func_no, } static const struct pci_epc_features artpec6_pcie_epc_features = { + DWC_EPC_COMMON_FEATURES, .msi_capable = true, }; diff --git a/drivers/pci/controller/dwc/pcie-designware-debugfs.c b/drivers/pci/controller/dwc/pcie-designware-debugfs.c index 0fbf86c0b97e0c..df98fee69892bb 100644 --- a/drivers/pci/controller/dwc/pcie-designware-debugfs.c +++ b/drivers/pci/controller/dwc/pcie-designware-debugfs.c @@ -485,6 +485,8 @@ static const char *ltssm_status_string(enum dw_pcie_ltssm ltssm) DW_PCIE_LTSSM_NAME(DW_PCIE_LTSSM_RCVRY_EQ1); DW_PCIE_LTSSM_NAME(DW_PCIE_LTSSM_RCVRY_EQ2); DW_PCIE_LTSSM_NAME(DW_PCIE_LTSSM_RCVRY_EQ3); + DW_PCIE_LTSSM_NAME(DW_PCIE_LTSSM_L1_1); + DW_PCIE_LTSSM_NAME(DW_PCIE_LTSSM_L1_2); default: str = "DW_PCIE_LTSSM_UNKNOWN"; break; diff --git a/drivers/pci/controller/dwc/pcie-designware-ep.c b/drivers/pci/controller/dwc/pcie-designware-ep.c index 19571ac2b96171..7ebb01fa5076fe 100644 --- a/drivers/pci/controller/dwc/pcie-designware-ep.c +++ b/drivers/pci/controller/dwc/pcie-designware-ep.c @@ -72,47 +72,15 @@ EXPORT_SYMBOL_GPL(dw_pcie_ep_reset_bar); static u8 dw_pcie_ep_find_capability(struct dw_pcie_ep *ep, u8 func_no, u8 cap) { return PCI_FIND_NEXT_CAP(dw_pcie_ep_read_cfg, PCI_CAPABILITY_LIST, - cap, ep, func_no); + cap, NULL, ep, func_no); } -/** - * dw_pcie_ep_hide_ext_capability - Hide a capability from the linked list - * @pci: DWC PCI device - * @prev_cap: Capability preceding the capability that should be hidden - * @cap: Capability that should be hidden - * - * Return: 0 if success, errno otherwise. - */ -int dw_pcie_ep_hide_ext_capability(struct dw_pcie *pci, u8 prev_cap, u8 cap) +static u16 dw_pcie_ep_find_ext_capability(struct dw_pcie_ep *ep, + u8 func_no, u8 cap) { - u16 prev_cap_offset, cap_offset; - u32 prev_cap_header, cap_header; - - prev_cap_offset = dw_pcie_find_ext_capability(pci, prev_cap); - if (!prev_cap_offset) - return -EINVAL; - - prev_cap_header = dw_pcie_readl_dbi(pci, prev_cap_offset); - cap_offset = PCI_EXT_CAP_NEXT(prev_cap_header); - cap_header = dw_pcie_readl_dbi(pci, cap_offset); - - /* cap must immediately follow prev_cap. */ - if (PCI_EXT_CAP_ID(cap_header) != cap) - return -EINVAL; - - /* Clear next ptr. */ - prev_cap_header &= ~GENMASK(31, 20); - - /* Set next ptr to next ptr of cap. */ - prev_cap_header |= cap_header & GENMASK(31, 20); - - dw_pcie_dbi_ro_wr_en(pci); - dw_pcie_writel_dbi(pci, prev_cap_offset, prev_cap_header); - dw_pcie_dbi_ro_wr_dis(pci); - - return 0; + return PCI_FIND_NEXT_EXT_CAP(dw_pcie_ep_read_cfg, 0, + cap, NULL, ep, func_no); } -EXPORT_SYMBOL_GPL(dw_pcie_ep_hide_ext_capability); static int dw_pcie_ep_write_header(struct pci_epc *epc, u8 func_no, u8 vfunc_no, struct pci_epf_header *hdr) @@ -139,18 +107,23 @@ static int dw_pcie_ep_write_header(struct pci_epc *epc, u8 func_no, u8 vfunc_no, return 0; } -static int dw_pcie_ep_inbound_atu(struct dw_pcie_ep *ep, u8 func_no, int type, - dma_addr_t parent_bus_addr, enum pci_barno bar, - size_t size) +/* BAR Match Mode inbound iATU mapping */ +static int dw_pcie_ep_ib_atu_bar(struct dw_pcie_ep *ep, u8 func_no, int type, + dma_addr_t parent_bus_addr, enum pci_barno bar, + size_t size) { int ret; u32 free_win; struct dw_pcie *pci = to_dw_pcie_from_ep(ep); + struct dw_pcie_ep_func *ep_func = dw_pcie_ep_get_func_from_ep(ep, func_no); - if (!ep->bar_to_atu[bar]) + if (!ep_func) + return -EINVAL; + + if (!ep_func->bar_to_atu[bar]) free_win = find_first_zero_bit(ep->ib_window_map, pci->num_ib_windows); else - free_win = ep->bar_to_atu[bar] - 1; + free_win = ep_func->bar_to_atu[bar] - 1; if (free_win >= pci->num_ib_windows) { dev_err(pci->dev, "No free inbound window\n"); @@ -168,12 +141,190 @@ static int dw_pcie_ep_inbound_atu(struct dw_pcie_ep *ep, u8 func_no, int type, * Always increment free_win before assignment, since value 0 is used to identify * unallocated mapping. */ - ep->bar_to_atu[bar] = free_win + 1; + ep_func->bar_to_atu[bar] = free_win + 1; set_bit(free_win, ep->ib_window_map); return 0; } +static void dw_pcie_ep_clear_ib_maps(struct dw_pcie_ep *ep, u8 func_no, enum pci_barno bar) +{ + struct dw_pcie_ep_func *ep_func = dw_pcie_ep_get_func_from_ep(ep, func_no); + struct dw_pcie *pci = to_dw_pcie_from_ep(ep); + struct device *dev = pci->dev; + unsigned int i, num; + u32 atu_index; + u32 *indexes; + + if (!ep_func) + return; + + /* Tear down the BAR Match Mode mapping, if any. */ + if (ep_func->bar_to_atu[bar]) { + atu_index = ep_func->bar_to_atu[bar] - 1; + dw_pcie_disable_atu(pci, PCIE_ATU_REGION_DIR_IB, atu_index); + clear_bit(atu_index, ep->ib_window_map); + ep_func->bar_to_atu[bar] = 0; + } + + /* Tear down all Address Match Mode mappings, if any. */ + indexes = ep_func->ib_atu_indexes[bar]; + num = ep_func->num_ib_atu_indexes[bar]; + ep_func->ib_atu_indexes[bar] = NULL; + ep_func->num_ib_atu_indexes[bar] = 0; + if (!indexes) + return; + for (i = 0; i < num; i++) { + dw_pcie_disable_atu(pci, PCIE_ATU_REGION_DIR_IB, indexes[i]); + clear_bit(indexes[i], ep->ib_window_map); + } + devm_kfree(dev, indexes); +} + +static u64 dw_pcie_ep_read_bar_assigned(struct dw_pcie_ep *ep, u8 func_no, + enum pci_barno bar, int flags) +{ + u32 reg = PCI_BASE_ADDRESS_0 + (4 * bar); + u32 lo, hi; + u64 addr; + + lo = dw_pcie_ep_readl_dbi(ep, func_no, reg); + + if (flags & PCI_BASE_ADDRESS_SPACE) + return lo & PCI_BASE_ADDRESS_IO_MASK; + + addr = lo & PCI_BASE_ADDRESS_MEM_MASK; + if (!(flags & PCI_BASE_ADDRESS_MEM_TYPE_64)) + return addr; + + hi = dw_pcie_ep_readl_dbi(ep, func_no, reg + 4); + return addr | ((u64)hi << 32); +} + +static int dw_pcie_ep_validate_submap(struct dw_pcie_ep *ep, + const struct pci_epf_bar_submap *submap, + unsigned int num_submap, size_t bar_size) +{ + struct dw_pcie *pci = to_dw_pcie_from_ep(ep); + u32 align = pci->region_align; + size_t off = 0; + unsigned int i; + size_t size; + + if (!align || !IS_ALIGNED(bar_size, align)) + return -EINVAL; + + /* + * The submap array order defines the BAR layout (submap[0] starts + * at offset 0 and each entry immediately follows the previous + * one). Here, validate that it forms a strict, gapless + * decomposition of the BAR: + * - each entry has a non-zero size + * - sizes, implicit offsets and phys_addr are aligned to + * pci->region_align + * - each entry lies within the BAR range + * - the entries exactly cover the whole BAR + * + * Note: dw_pcie_prog_inbound_atu() also checks alignment for the + * PCI address and the target phys_addr, but validating up-front + * avoids partially programming iATU windows in vain. + */ + for (i = 0; i < num_submap; i++) { + size = submap[i].size; + + if (!size) + return -EINVAL; + + if (!IS_ALIGNED(size, align) || !IS_ALIGNED(off, align)) + return -EINVAL; + + if (!IS_ALIGNED(submap[i].phys_addr, align)) + return -EINVAL; + + if (off > bar_size || size > bar_size - off) + return -EINVAL; + + off += size; + } + if (off != bar_size) + return -EINVAL; + + return 0; +} + +/* Address Match Mode inbound iATU mapping */ +static int dw_pcie_ep_ib_atu_addr(struct dw_pcie_ep *ep, u8 func_no, int type, + const struct pci_epf_bar *epf_bar) +{ + struct dw_pcie_ep_func *ep_func = dw_pcie_ep_get_func_from_ep(ep, func_no); + const struct pci_epf_bar_submap *submap = epf_bar->submap; + struct dw_pcie *pci = to_dw_pcie_from_ep(ep); + enum pci_barno bar = epf_bar->barno; + struct device *dev = pci->dev; + u64 pci_addr, parent_bus_addr; + u64 size, base, off = 0; + int free_win, ret; + unsigned int i; + u32 *indexes; + + if (!ep_func || !epf_bar->num_submap || !submap || !epf_bar->size) + return -EINVAL; + + ret = dw_pcie_ep_validate_submap(ep, submap, epf_bar->num_submap, + epf_bar->size); + if (ret) + return ret; + + base = dw_pcie_ep_read_bar_assigned(ep, func_no, bar, epf_bar->flags); + if (!base) { + dev_err(dev, + "BAR%u not assigned, cannot set up sub-range mappings\n", + bar); + return -EINVAL; + } + + indexes = devm_kcalloc(dev, epf_bar->num_submap, sizeof(*indexes), + GFP_KERNEL); + if (!indexes) + return -ENOMEM; + + ep_func->ib_atu_indexes[bar] = indexes; + ep_func->num_ib_atu_indexes[bar] = 0; + + for (i = 0; i < epf_bar->num_submap; i++) { + size = submap[i].size; + parent_bus_addr = submap[i].phys_addr; + + if (off > (~0ULL) - base) { + ret = -EINVAL; + goto err; + } + + pci_addr = base + off; + off += size; + + free_win = find_first_zero_bit(ep->ib_window_map, + pci->num_ib_windows); + if (free_win >= pci->num_ib_windows) { + ret = -ENOSPC; + goto err; + } + + ret = dw_pcie_prog_inbound_atu(pci, free_win, type, + parent_bus_addr, pci_addr, size); + if (ret) + goto err; + + set_bit(free_win, ep->ib_window_map); + indexes[i] = free_win; + ep_func->num_ib_atu_indexes[bar] = i + 1; + } + return 0; +err: + dw_pcie_ep_clear_ib_maps(ep, func_no, bar); + return ret; +} + static int dw_pcie_ep_outbound_atu(struct dw_pcie_ep *ep, struct dw_pcie_ob_atu_cfg *atu) { @@ -204,35 +355,34 @@ static void dw_pcie_ep_clear_bar(struct pci_epc *epc, u8 func_no, u8 vfunc_no, struct dw_pcie_ep *ep = epc_get_drvdata(epc); struct dw_pcie *pci = to_dw_pcie_from_ep(ep); enum pci_barno bar = epf_bar->barno; - u32 atu_index = ep->bar_to_atu[bar] - 1; + struct dw_pcie_ep_func *ep_func = dw_pcie_ep_get_func_from_ep(ep, func_no); - if (!ep->bar_to_atu[bar]) + if (!ep_func || !ep_func->epf_bar[bar]) return; __dw_pcie_ep_reset_bar(pci, func_no, bar, epf_bar->flags); - dw_pcie_disable_atu(pci, PCIE_ATU_REGION_DIR_IB, atu_index); - clear_bit(atu_index, ep->ib_window_map); - ep->epf_bar[bar] = NULL; - ep->bar_to_atu[bar] = 0; + dw_pcie_ep_clear_ib_maps(ep, func_no, bar); + + ep_func->epf_bar[bar] = NULL; } -static unsigned int dw_pcie_ep_get_rebar_offset(struct dw_pcie *pci, +static unsigned int dw_pcie_ep_get_rebar_offset(struct dw_pcie_ep *ep, u8 func_no, enum pci_barno bar) { u32 reg, bar_index; unsigned int offset, nbars; int i; - offset = dw_pcie_find_ext_capability(pci, PCI_EXT_CAP_ID_REBAR); + offset = dw_pcie_ep_find_ext_capability(ep, func_no, PCI_EXT_CAP_ID_REBAR); if (!offset) return offset; - reg = dw_pcie_readl_dbi(pci, offset + PCI_REBAR_CTRL); + reg = dw_pcie_ep_readl_dbi(ep, func_no, offset + PCI_REBAR_CTRL); nbars = FIELD_GET(PCI_REBAR_CTRL_NBAR_MASK, reg); for (i = 0; i < nbars; i++, offset += PCI_REBAR_CTRL) { - reg = dw_pcie_readl_dbi(pci, offset + PCI_REBAR_CTRL); + reg = dw_pcie_ep_readl_dbi(ep, func_no, offset + PCI_REBAR_CTRL); bar_index = FIELD_GET(PCI_REBAR_CTRL_BAR_IDX, reg); if (bar_index == bar) return offset; @@ -253,7 +403,7 @@ static int dw_pcie_ep_set_bar_resizable(struct dw_pcie_ep *ep, u8 func_no, u32 rebar_cap, rebar_ctrl; int ret; - rebar_offset = dw_pcie_ep_get_rebar_offset(pci, bar); + rebar_offset = dw_pcie_ep_get_rebar_offset(ep, func_no, bar); if (!rebar_offset) return -EINVAL; @@ -283,16 +433,16 @@ static int dw_pcie_ep_set_bar_resizable(struct dw_pcie_ep *ep, u8 func_no, * 1 MB to 128 TB. Bits 31:16 in PCI_REBAR_CTRL define "supported sizes" * bits for sizes 256 TB to 8 EB. Disallow sizes 256 TB to 8 EB. */ - rebar_ctrl = dw_pcie_readl_dbi(pci, rebar_offset + PCI_REBAR_CTRL); + rebar_ctrl = dw_pcie_ep_readl_dbi(ep, func_no, rebar_offset + PCI_REBAR_CTRL); rebar_ctrl &= ~GENMASK(31, 16); - dw_pcie_writel_dbi(pci, rebar_offset + PCI_REBAR_CTRL, rebar_ctrl); + dw_pcie_ep_writel_dbi(ep, func_no, rebar_offset + PCI_REBAR_CTRL, rebar_ctrl); /* * The "selected size" (bits 13:8) in PCI_REBAR_CTRL are automatically * updated when writing PCI_REBAR_CAP, see "Figure 3-26 Resizable BAR * Example for 32-bit Memory BAR0" in DWC EP databook 5.96a. */ - dw_pcie_writel_dbi(pci, rebar_offset + PCI_REBAR_CAP, rebar_cap); + dw_pcie_ep_writel_dbi(ep, func_no, rebar_offset + PCI_REBAR_CAP, rebar_cap); dw_pcie_dbi_ro_wr_dis(pci); @@ -341,12 +491,16 @@ static int dw_pcie_ep_set_bar(struct pci_epc *epc, u8 func_no, u8 vfunc_no, { struct dw_pcie_ep *ep = epc_get_drvdata(epc); struct dw_pcie *pci = to_dw_pcie_from_ep(ep); + struct dw_pcie_ep_func *ep_func = dw_pcie_ep_get_func_from_ep(ep, func_no); enum pci_barno bar = epf_bar->barno; size_t size = epf_bar->size; enum pci_epc_bar_type bar_type; int flags = epf_bar->flags; int ret, type; + if (!ep_func) + return -EINVAL; + /* * DWC does not allow BAR pairs to overlap, e.g. you cannot combine BARs * 1 and 2 to form a 64-bit BAR. @@ -360,21 +514,46 @@ static int dw_pcie_ep_set_bar(struct pci_epc *epc, u8 func_no, u8 vfunc_no, * calling clear_bar() would clear the BAR's PCI address assigned by the * host). */ - if (ep->epf_bar[bar]) { + if (ep_func->epf_bar[bar]) { /* * We can only dynamically change a BAR if the new BAR size and * BAR flags do not differ from the existing configuration. + * + * Note: this safety check only works when the caller uses + * a new struct pci_epf_bar in the second set_bar() call. + * If the same instance is updated in place and passed in, + * we cannot reliably detect invalid barno/size/flags + * changes here. */ - if (ep->epf_bar[bar]->barno != bar || - ep->epf_bar[bar]->size != size || - ep->epf_bar[bar]->flags != flags) + if (ep_func->epf_bar[bar]->barno != bar || + ep_func->epf_bar[bar]->size != size || + ep_func->epf_bar[bar]->flags != flags) return -EINVAL; + /* + * When dynamically changing a BAR, tear down any existing + * mappings before re-programming. This is redundant when + * both the old and new mappings are BAR Match Mode, but + * required to handle in-place updates and match-mode + * changes reliably. + */ + dw_pcie_ep_clear_ib_maps(ep, func_no, bar); + /* * When dynamically changing a BAR, skip writing the BAR reg, as * that would clear the BAR's PCI address assigned by the host. */ goto config_atu; + } else { + /* + * Subrange mapping is an update-only operation. The BAR + * must have been configured once without submaps so that + * subsequent set_bar() calls can update inbound mappings + * without touching the BAR register (and clobbering the + * host-assigned address). + */ + if (epf_bar->num_submap) + return -EINVAL; } bar_type = dw_pcie_ep_get_bar_type(ep, bar); @@ -408,12 +587,16 @@ static int dw_pcie_ep_set_bar(struct pci_epc *epc, u8 func_no, u8 vfunc_no, else type = PCIE_ATU_TYPE_IO; - ret = dw_pcie_ep_inbound_atu(ep, func_no, type, epf_bar->phys_addr, bar, - size); + if (epf_bar->num_submap) + ret = dw_pcie_ep_ib_atu_addr(ep, func_no, type, epf_bar); + else + ret = dw_pcie_ep_ib_atu_bar(ep, func_no, type, + epf_bar->phys_addr, bar, size); + if (ret) return ret; - ep->epf_bar[bar] = epf_bar; + ep_func->epf_bar[bar] = epf_bar; return 0; } @@ -601,6 +784,16 @@ static void dw_pcie_ep_stop(struct pci_epc *epc) struct dw_pcie_ep *ep = epc_get_drvdata(epc); struct dw_pcie *pci = to_dw_pcie_from_ep(ep); + /* + * Tear down the dedicated outbound window used for MSI + * generation. This avoids leaking an iATU window across + * endpoint stop/start cycles. + */ + if (ep->msi_iatu_mapped) { + dw_pcie_ep_unmap_addr(epc, 0, 0, ep->msi_mem_phys); + ep->msi_iatu_mapped = false; + } + dw_pcie_stop_link(pci); } @@ -702,14 +895,41 @@ int dw_pcie_ep_raise_msi_irq(struct dw_pcie_ep *ep, u8 func_no, msg_addr = ((u64)msg_addr_upper) << 32 | msg_addr_lower; msg_addr = dw_pcie_ep_align_addr(epc, msg_addr, &map_size, &offset); - ret = dw_pcie_ep_map_addr(epc, func_no, 0, ep->msi_mem_phys, msg_addr, - map_size); - if (ret) - return ret; - writel(msg_data | (interrupt_num - 1), ep->msi_mem + offset); + /* + * Program the outbound iATU once and keep it enabled. + * + * The spec warns that updating iATU registers while there are + * operations in flight on the AXI bridge interface is not + * supported, so we avoid reprogramming the region on every MSI, + * specifically unmapping immediately after writel(). + */ + if (ep->msi_iatu_mapped && (ep->msi_msg_addr != msg_addr || + ep->msi_map_size != map_size)) { + /* + * The host changed the MSI target address or the required + * mapping size changed. Reprogramming the iATU when there are + * operations in flight is unsafe on this controller. However, + * there is no unified way to check if we have operations in + * flight, thus we don't know if we should WARN() or not. + */ + dw_pcie_ep_unmap_addr(epc, func_no, 0, ep->msi_mem_phys); + ep->msi_iatu_mapped = false; + } - dw_pcie_ep_unmap_addr(epc, func_no, 0, ep->msi_mem_phys); + if (!ep->msi_iatu_mapped) { + ret = dw_pcie_ep_map_addr(epc, func_no, 0, + ep->msi_mem_phys, msg_addr, + map_size); + if (ret) + return ret; + + ep->msi_iatu_mapped = true; + ep->msi_msg_addr = msg_addr; + ep->msi_map_size = map_size; + } + + writel(msg_data | (interrupt_num - 1), ep->msi_mem + offset); return 0; } @@ -775,7 +995,7 @@ int dw_pcie_ep_raise_msix_irq(struct dw_pcie_ep *ep, u8 func_no, bir = FIELD_GET(PCI_MSIX_TABLE_BIR, tbl_offset); tbl_offset &= PCI_MSIX_TABLE_OFFSET; - msix_tbl = ep->epf_bar[bir]->addr + tbl_offset; + msix_tbl = ep_func->epf_bar[bir]->addr + tbl_offset; msg_addr = msix_tbl[(interrupt_num - 1)].msg_addr; msg_data = msix_tbl[(interrupt_num - 1)].msg_data; vec_ctrl = msix_tbl[(interrupt_num - 1)].vector_ctrl; @@ -793,6 +1013,9 @@ int dw_pcie_ep_raise_msix_irq(struct dw_pcie_ep *ep, u8 func_no, writel(msg_data, ep->msi_mem + offset); + /* flush posted write before unmap */ + readl(ep->msi_mem + offset); + dw_pcie_ep_unmap_addr(epc, func_no, 0, ep->msi_mem_phys); return 0; @@ -836,20 +1059,20 @@ void dw_pcie_ep_deinit(struct dw_pcie_ep *ep) } EXPORT_SYMBOL_GPL(dw_pcie_ep_deinit); -static void dw_pcie_ep_init_non_sticky_registers(struct dw_pcie *pci) +static void dw_pcie_ep_init_rebar_registers(struct dw_pcie_ep *ep, u8 func_no) { - struct dw_pcie_ep *ep = &pci->ep; - unsigned int offset; - unsigned int nbars; + struct dw_pcie_ep_func *ep_func = dw_pcie_ep_get_func_from_ep(ep, func_no); + unsigned int offset, nbars; enum pci_barno bar; u32 reg, i, val; - offset = dw_pcie_find_ext_capability(pci, PCI_EXT_CAP_ID_REBAR); + if (!ep_func) + return; - dw_pcie_dbi_ro_wr_en(pci); + offset = dw_pcie_ep_find_ext_capability(ep, func_no, PCI_EXT_CAP_ID_REBAR); if (offset) { - reg = dw_pcie_readl_dbi(pci, offset + PCI_REBAR_CTRL); + reg = dw_pcie_ep_readl_dbi(ep, func_no, offset + PCI_REBAR_CTRL); nbars = FIELD_GET(PCI_REBAR_CTRL_NBAR_MASK, reg); /* @@ -870,16 +1093,28 @@ static void dw_pcie_ep_init_non_sticky_registers(struct dw_pcie *pci) * the controller when RESBAR_CAP_REG is written, which * is why RESBAR_CAP_REG is written here. */ - val = dw_pcie_readl_dbi(pci, offset + PCI_REBAR_CTRL); + val = dw_pcie_ep_readl_dbi(ep, func_no, offset + PCI_REBAR_CTRL); bar = FIELD_GET(PCI_REBAR_CTRL_BAR_IDX, val); - if (ep->epf_bar[bar]) - pci_epc_bar_size_to_rebar_cap(ep->epf_bar[bar]->size, &val); + if (ep_func->epf_bar[bar]) + pci_epc_bar_size_to_rebar_cap(ep_func->epf_bar[bar]->size, &val); else val = BIT(4); - dw_pcie_writel_dbi(pci, offset + PCI_REBAR_CAP, val); + dw_pcie_ep_writel_dbi(ep, func_no, offset + PCI_REBAR_CAP, val); } } +} + +static void dw_pcie_ep_init_non_sticky_registers(struct dw_pcie *pci) +{ + struct dw_pcie_ep *ep = &pci->ep; + u8 funcs = ep->epc->max_functions; + u8 func_no; + + dw_pcie_dbi_ro_wr_en(pci); + + for (func_no = 0; func_no < funcs; func_no++) + dw_pcie_ep_init_rebar_registers(ep, func_no); dw_pcie_setup(pci); dw_pcie_dbi_ro_wr_dis(pci); @@ -1087,6 +1322,9 @@ int dw_pcie_ep_init(struct dw_pcie_ep *ep) struct device *dev = pci->dev; INIT_LIST_HEAD(&ep->func_list); + ep->msi_iatu_mapped = false; + ep->msi_msg_addr = 0; + ep->msi_map_size = 0; epc = devm_pci_epc_create(dev, &epc_ops); if (IS_ERR(epc)) { diff --git a/drivers/pci/controller/dwc/pcie-designware-host.c b/drivers/pci/controller/dwc/pcie-designware-host.c index 372207c33a857b..53125b97530b66 100644 --- a/drivers/pci/controller/dwc/pcie-designware-host.c +++ b/drivers/pci/controller/dwc/pcie-designware-host.c @@ -665,14 +665,8 @@ int dw_pcie_host_init(struct dw_pcie_rp *pp) goto err_remove_edma; } - /* - * Note: Skip the link up delay only when a Link Up IRQ is present. - * If there is no Link Up IRQ, we should not bypass the delay - * because that would require users to manually rescan for devices. - */ - if (!pp->use_linkup_irq) - /* Ignore errors, the link may come up later */ - dw_pcie_wait_for_link(pci); + /* Ignore errors, the link may come up later */ + dw_pcie_wait_for_link(pci); ret = pci_host_probe(bridge); if (ret) @@ -952,7 +946,14 @@ static int dw_pcie_iatu_setup(struct dw_pcie_rp *pp) dev_warn(pci->dev, "Ranges exceed outbound iATU size (%d)\n", pci->num_ob_windows); - pp->msg_atu_index = i; + if (pp->use_atu_msg) { + if (pci->num_ob_windows > ++i) { + pp->msg_atu_index = i; + } else { + dev_err(pci->dev, "Cannot add outbound window for MSG TLP\n"); + return -ENOMEM; + } + } i = 0; resource_list_for_each_entry(entry, &pp->bridge->dma_ranges) { @@ -1158,8 +1159,11 @@ static int dw_pcie_pme_turn_off(struct dw_pcie *pci) int dw_pcie_suspend_noirq(struct dw_pcie *pci) { u8 offset = dw_pcie_find_capability(pci, PCI_CAP_ID_EXP); + int ret = 0; u32 val; - int ret; + + if (!dw_pcie_link_up(pci)) + goto stop_link; /* * If L1SS is supported, then do not put the link into L2 as some @@ -1176,6 +1180,16 @@ int dw_pcie_suspend_noirq(struct dw_pcie *pci) return ret; } + /* + * Some SoCs do not support reading the LTSSM register after + * PME_Turn_Off broadcast. For those SoCs, skip waiting for L2/L3 Ready + * state and wait 10ms as recommended in PCIe spec r6.0, sec 5.3.3.2.1. + */ + if (pci->pp.skip_l23_ready) { + mdelay(PCIE_PME_TO_L2_TIMEOUT_US/1000); + goto stop_link; + } + ret = read_poll_timeout(dw_pcie_get_ltssm, val, val == DW_PCIE_LTSSM_L2_IDLE || val <= DW_PCIE_LTSSM_DETECT_WAIT, @@ -1194,6 +1208,7 @@ int dw_pcie_suspend_noirq(struct dw_pcie *pci) */ udelay(1); +stop_link: dw_pcie_stop_link(pci); if (pci->pp.ops->deinit) pci->pp.ops->deinit(&pci->pp); diff --git a/drivers/pci/controller/dwc/pcie-designware-plat.c b/drivers/pci/controller/dwc/pcie-designware-plat.c index 12f41886c65d1e..8530746ec5cbb6 100644 --- a/drivers/pci/controller/dwc/pcie-designware-plat.c +++ b/drivers/pci/controller/dwc/pcie-designware-plat.c @@ -61,6 +61,7 @@ static int dw_plat_pcie_ep_raise_irq(struct dw_pcie_ep *ep, u8 func_no, } static const struct pci_epc_features dw_plat_pcie_epc_features = { + DWC_EPC_COMMON_FEATURES, .msi_capable = true, .msix_capable = true, }; diff --git a/drivers/pci/controller/dwc/pcie-designware.c b/drivers/pci/controller/dwc/pcie-designware.c index 75fc8b767fccfe..345365ea97c74c 100644 --- a/drivers/pci/controller/dwc/pcie-designware.c +++ b/drivers/pci/controller/dwc/pcie-designware.c @@ -226,16 +226,69 @@ void dw_pcie_version_detect(struct dw_pcie *pci) u8 dw_pcie_find_capability(struct dw_pcie *pci, u8 cap) { return PCI_FIND_NEXT_CAP(dw_pcie_read_cfg, PCI_CAPABILITY_LIST, cap, - pci); + NULL, pci); } EXPORT_SYMBOL_GPL(dw_pcie_find_capability); u16 dw_pcie_find_ext_capability(struct dw_pcie *pci, u8 cap) { - return PCI_FIND_NEXT_EXT_CAP(dw_pcie_read_cfg, 0, cap, pci); + return PCI_FIND_NEXT_EXT_CAP(dw_pcie_read_cfg, 0, cap, NULL, pci); } EXPORT_SYMBOL_GPL(dw_pcie_find_ext_capability); +void dw_pcie_remove_capability(struct dw_pcie *pci, u8 cap) +{ + u8 cap_pos, pre_pos, next_pos; + u16 reg; + + cap_pos = PCI_FIND_NEXT_CAP(dw_pcie_read_cfg, PCI_CAPABILITY_LIST, cap, + &pre_pos, pci); + if (!cap_pos) + return; + + reg = dw_pcie_readw_dbi(pci, cap_pos); + next_pos = (reg & 0xff00) >> 8; + + dw_pcie_dbi_ro_wr_en(pci); + if (pre_pos == PCI_CAPABILITY_LIST) + dw_pcie_writeb_dbi(pci, PCI_CAPABILITY_LIST, next_pos); + else + dw_pcie_writeb_dbi(pci, pre_pos + 1, next_pos); + dw_pcie_dbi_ro_wr_dis(pci); +} +EXPORT_SYMBOL_GPL(dw_pcie_remove_capability); + +void dw_pcie_remove_ext_capability(struct dw_pcie *pci, u8 cap) +{ + int cap_pos, next_pos, pre_pos; + u32 pre_header, header; + + cap_pos = PCI_FIND_NEXT_EXT_CAP(dw_pcie_read_cfg, 0, cap, &pre_pos, pci); + if (!cap_pos) + return; + + header = dw_pcie_readl_dbi(pci, cap_pos); + /* + * If the first cap at offset PCI_CFG_SPACE_SIZE is removed, + * only set it's capid to zero as it cannot be skipped. + */ + if (cap_pos == PCI_CFG_SPACE_SIZE) { + dw_pcie_dbi_ro_wr_en(pci); + dw_pcie_writel_dbi(pci, cap_pos, header & 0xffff0000); + dw_pcie_dbi_ro_wr_dis(pci); + return; + } + + pre_header = dw_pcie_readl_dbi(pci, pre_pos); + next_pos = PCI_EXT_CAP_NEXT(header); + + dw_pcie_dbi_ro_wr_en(pci); + dw_pcie_writel_dbi(pci, pre_pos, + (pre_header & 0xfffff) | (next_pos << 20)); + dw_pcie_dbi_ro_wr_dis(pci); +} +EXPORT_SYMBOL_GPL(dw_pcie_remove_ext_capability); + static u16 __dw_pcie_find_vsec_capability(struct dw_pcie *pci, u16 vendor_id, u16 vsec_id) { @@ -246,7 +299,7 @@ static u16 __dw_pcie_find_vsec_capability(struct dw_pcie *pci, u16 vendor_id, return 0; while ((vsec = PCI_FIND_NEXT_EXT_CAP(dw_pcie_read_cfg, vsec, - PCI_EXT_CAP_ID_VNDR, pci))) { + PCI_EXT_CAP_ID_VNDR, NULL, pci))) { header = dw_pcie_readl_dbi(pci, vsec + PCI_VNDR_HEADER); if (PCI_VNDR_HEADER_ID(header) == vsec_id) return vsec; diff --git a/drivers/pci/controller/dwc/pcie-designware.h b/drivers/pci/controller/dwc/pcie-designware.h index 31685951a08045..ca3ff1fefab5d6 100644 --- a/drivers/pci/controller/dwc/pcie-designware.h +++ b/drivers/pci/controller/dwc/pcie-designware.h @@ -305,6 +305,10 @@ /* Default eDMA LLP memory size */ #define DMA_LLP_MEM_SIZE PAGE_SIZE +/* Common struct pci_epc_feature bits among DWC EP glue drivers */ +#define DWC_EPC_COMMON_FEATURES .dynamic_inbound_mapping = true, \ + .subrange_mapping = true + struct dw_pcie; struct dw_pcie_rp; struct dw_pcie_ep; @@ -388,6 +392,10 @@ enum dw_pcie_ltssm { DW_PCIE_LTSSM_RCVRY_EQ2 = 0x22, DW_PCIE_LTSSM_RCVRY_EQ3 = 0x23, + /* Vendor glue drivers provide pseudo L1 substates from get_ltssm() */ + DW_PCIE_LTSSM_L1_1 = 0x141, + DW_PCIE_LTSSM_L1_2 = 0x142, + DW_PCIE_LTSSM_UNKNOWN = 0xFFFFFFFF, }; @@ -434,11 +442,11 @@ struct dw_pcie_rp { bool use_atu_msg; int msg_atu_index; struct resource *msg_res; - bool use_linkup_irq; struct pci_eq_presets presets; struct pci_config_window *cfg; bool ecam_enabled; bool native_ecam; + bool skip_l23_ready; }; struct dw_pcie_ep_ops { @@ -463,6 +471,12 @@ struct dw_pcie_ep_func { u8 func_no; u8 msi_cap; /* MSI capability offset */ u8 msix_cap; /* MSI-X capability offset */ + u8 bar_to_atu[PCI_STD_NUM_BARS]; + struct pci_epf_bar *epf_bar[PCI_STD_NUM_BARS]; + + /* Only for Address Match Mode inbound iATU */ + u32 *ib_atu_indexes[PCI_STD_NUM_BARS]; + unsigned int num_ib_atu_indexes[PCI_STD_NUM_BARS]; }; struct dw_pcie_ep { @@ -472,13 +486,16 @@ struct dw_pcie_ep { phys_addr_t phys_base; size_t addr_size; size_t page_size; - u8 bar_to_atu[PCI_STD_NUM_BARS]; phys_addr_t *outbound_addr; unsigned long *ib_window_map; unsigned long *ob_window_map; void __iomem *msi_mem; phys_addr_t msi_mem_phys; - struct pci_epf_bar *epf_bar[PCI_STD_NUM_BARS]; + + /* MSI outbound iATU state */ + bool msi_iatu_mapped; + u64 msi_msg_addr; + size_t msi_map_size; }; struct dw_pcie_ops { @@ -562,6 +579,8 @@ void dw_pcie_version_detect(struct dw_pcie *pci); u8 dw_pcie_find_capability(struct dw_pcie *pci, u8 cap); u16 dw_pcie_find_ext_capability(struct dw_pcie *pci, u8 cap); +void dw_pcie_remove_capability(struct dw_pcie *pci, u8 cap); +void dw_pcie_remove_ext_capability(struct dw_pcie *pci, u8 cap); u16 dw_pcie_find_rasdes_capability(struct dw_pcie *pci); u16 dw_pcie_find_ptm_capability(struct dw_pcie *pci); @@ -899,7 +918,6 @@ int dw_pcie_ep_raise_msix_irq(struct dw_pcie_ep *ep, u8 func_no, int dw_pcie_ep_raise_msix_irq_doorbell(struct dw_pcie_ep *ep, u8 func_no, u16 interrupt_num); void dw_pcie_ep_reset_bar(struct dw_pcie *pci, enum pci_barno bar); -int dw_pcie_ep_hide_ext_capability(struct dw_pcie *pci, u8 prev_cap, u8 cap); struct dw_pcie_ep_func * dw_pcie_ep_get_func_from_ep(struct dw_pcie_ep *ep, u8 func_no); #else @@ -957,12 +975,6 @@ static inline void dw_pcie_ep_reset_bar(struct dw_pcie *pci, enum pci_barno bar) { } -static inline int dw_pcie_ep_hide_ext_capability(struct dw_pcie *pci, - u8 prev_cap, u8 cap) -{ - return 0; -} - static inline struct dw_pcie_ep_func * dw_pcie_ep_get_func_from_ep(struct dw_pcie_ep *ep, u8 func_no) { diff --git a/drivers/pci/controller/dwc/pcie-dw-rockchip.c b/drivers/pci/controller/dwc/pcie-dw-rockchip.c index f8605fe61a415a..5b17da63151d52 100644 --- a/drivers/pci/controller/dwc/pcie-dw-rockchip.c +++ b/drivers/pci/controller/dwc/pcie-dw-rockchip.c @@ -68,6 +68,11 @@ #define PCIE_CLKREQ_NOT_READY FIELD_PREP_WM16(BIT(0), 0) #define PCIE_CLKREQ_PULL_DOWN FIELD_PREP_WM16(GENMASK(13, 12), 1) +/* RASDES TBA information */ +#define PCIE_CLIENT_CDM_RASDES_TBA_INFO_CMN 0x154 +#define PCIE_CLIENT_CDM_RASDES_TBA_L1_1 BIT(4) +#define PCIE_CLIENT_CDM_RASDES_TBA_L1_2 BIT(5) + /* Hot Reset Control Register */ #define PCIE_CLIENT_HOT_RESET_CTRL 0x180 #define PCIE_LTSSM_APP_DLY2_EN BIT(1) @@ -80,6 +85,8 @@ #define PCIE_LINKUP_MASK GENMASK(17, 16) #define PCIE_LTSSM_STATUS_MASK GENMASK(5, 0) +#define PCIE_TYPE0_HDR_DBI2_OFFSET 0x100000 + struct rockchip_pcie { struct dw_pcie pci; void __iomem *apb_base; @@ -181,11 +188,26 @@ static int rockchip_pcie_init_irq_domain(struct rockchip_pcie *rockchip) return 0; } -static u32 rockchip_pcie_get_ltssm(struct rockchip_pcie *rockchip) +static u32 rockchip_pcie_get_ltssm_reg(struct rockchip_pcie *rockchip) { return rockchip_pcie_readl_apb(rockchip, PCIE_CLIENT_LTSSM_STATUS); } +static enum dw_pcie_ltssm rockchip_pcie_get_ltssm(struct dw_pcie *pci) +{ + struct rockchip_pcie *rockchip = to_rockchip_pcie(pci); + u32 val = rockchip_pcie_readl_apb(rockchip, + PCIE_CLIENT_CDM_RASDES_TBA_INFO_CMN); + + if (val & PCIE_CLIENT_CDM_RASDES_TBA_L1_1) + return DW_PCIE_LTSSM_L1_1; + + if (val & PCIE_CLIENT_CDM_RASDES_TBA_L1_2) + return DW_PCIE_LTSSM_L1_2; + + return rockchip_pcie_get_ltssm_reg(rockchip) & PCIE_LTSSM_STATUS_MASK; +} + static void rockchip_pcie_enable_ltssm(struct rockchip_pcie *rockchip) { rockchip_pcie_writel_apb(rockchip, PCIE_CLIENT_ENABLE_LTSSM, @@ -201,7 +223,7 @@ static void rockchip_pcie_disable_ltssm(struct rockchip_pcie *rockchip) static bool rockchip_pcie_link_up(struct dw_pcie *pci) { struct rockchip_pcie *rockchip = to_rockchip_pcie(pci); - u32 val = rockchip_pcie_get_ltssm(rockchip); + u32 val = rockchip_pcie_get_ltssm_reg(rockchip); return FIELD_GET(PCIE_LINKUP_MASK, val) == PCIE_LINKUP; } @@ -292,6 +314,8 @@ static int rockchip_pcie_host_init(struct dw_pcie_rp *pp) if (irq < 0) return irq; + pci->dbi_base2 = pci->dbi_base + PCIE_TYPE0_HDR_DBI2_OFFSET; + ret = rockchip_pcie_init_irq_domain(rockchip); if (ret < 0) dev_err(dev, "failed to init irq domain\n"); @@ -302,6 +326,10 @@ static int rockchip_pcie_host_init(struct dw_pcie_rp *pp) rockchip_pcie_configure_l1ss(pci); rockchip_pcie_enable_l0s(pci); + /* Disable Root Ports BAR0 and BAR1 as they report bogus size */ + dw_pcie_writel_dbi2(pci, PCI_BASE_ADDRESS_0, 0x0); + dw_pcie_writel_dbi2(pci, PCI_BASE_ADDRESS_1, 0x0); + return 0; } @@ -327,9 +355,7 @@ static void rockchip_pcie_ep_hide_broken_ats_cap_rk3588(struct dw_pcie_ep *ep) if (!of_device_is_compatible(dev->of_node, "rockchip,rk3588-pcie-ep")) return; - if (dw_pcie_ep_hide_ext_capability(pci, PCI_EXT_CAP_ID_SECPCI, - PCI_EXT_CAP_ID_ATS)) - dev_err(dev, "failed to hide ATS capability\n"); + dw_pcie_remove_ext_capability(pci, PCI_EXT_CAP_ID_ATS); } static void rockchip_pcie_ep_init(struct dw_pcie_ep *ep) @@ -364,6 +390,7 @@ static int rockchip_pcie_raise_irq(struct dw_pcie_ep *ep, u8 func_no, } static const struct pci_epc_features rockchip_pcie_epc_features_rk3568 = { + DWC_EPC_COMMON_FEATURES, .linkup_notifier = true, .msi_capable = true, .msix_capable = true, @@ -384,6 +411,7 @@ static const struct pci_epc_features rockchip_pcie_epc_features_rk3568 = { * BARs) would be overwritten, resulting in (all other BARs) no longer working. */ static const struct pci_epc_features rockchip_pcie_epc_features_rk3588 = { + DWC_EPC_COMMON_FEATURES, .linkup_notifier = true, .msi_capable = true, .msix_capable = true, @@ -485,36 +513,9 @@ static const struct dw_pcie_ops dw_pcie_ops = { .link_up = rockchip_pcie_link_up, .start_link = rockchip_pcie_start_link, .stop_link = rockchip_pcie_stop_link, + .get_ltssm = rockchip_pcie_get_ltssm, }; -static irqreturn_t rockchip_pcie_rc_sys_irq_thread(int irq, void *arg) -{ - struct rockchip_pcie *rockchip = arg; - struct dw_pcie *pci = &rockchip->pci; - struct dw_pcie_rp *pp = &pci->pp; - struct device *dev = pci->dev; - u32 reg; - - reg = rockchip_pcie_readl_apb(rockchip, PCIE_CLIENT_INTR_STATUS_MISC); - rockchip_pcie_writel_apb(rockchip, reg, PCIE_CLIENT_INTR_STATUS_MISC); - - dev_dbg(dev, "PCIE_CLIENT_INTR_STATUS_MISC: %#x\n", reg); - dev_dbg(dev, "LTSSM_STATUS: %#x\n", rockchip_pcie_get_ltssm(rockchip)); - - if (reg & PCIE_RDLH_LINK_UP_CHGED) { - if (rockchip_pcie_link_up(pci)) { - msleep(PCIE_RESET_CONFIG_WAIT_MS); - dev_dbg(dev, "Received Link up event. Starting enumeration!\n"); - /* Rescan the bus to enumerate endpoint devices */ - pci_lock_rescan_remove(); - pci_rescan_bus(pp->bridge->bus); - pci_unlock_rescan_remove(); - } - } - - return IRQ_HANDLED; -} - static irqreturn_t rockchip_pcie_ep_sys_irq_thread(int irq, void *arg) { struct rockchip_pcie *rockchip = arg; @@ -526,7 +527,7 @@ static irqreturn_t rockchip_pcie_ep_sys_irq_thread(int irq, void *arg) rockchip_pcie_writel_apb(rockchip, reg, PCIE_CLIENT_INTR_STATUS_MISC); dev_dbg(dev, "PCIE_CLIENT_INTR_STATUS_MISC: %#x\n", reg); - dev_dbg(dev, "LTSSM_STATUS: %#x\n", rockchip_pcie_get_ltssm(rockchip)); + dev_dbg(dev, "LTSSM_STATUS: %#x\n", rockchip_pcie_get_ltssm_reg(rockchip)); if (reg & PCIE_LINK_REQ_RST_NOT_INT) { dev_dbg(dev, "hot reset or link-down reset\n"); @@ -547,29 +548,14 @@ static irqreturn_t rockchip_pcie_ep_sys_irq_thread(int irq, void *arg) return IRQ_HANDLED; } -static int rockchip_pcie_configure_rc(struct platform_device *pdev, - struct rockchip_pcie *rockchip) +static int rockchip_pcie_configure_rc(struct rockchip_pcie *rockchip) { - struct device *dev = &pdev->dev; struct dw_pcie_rp *pp; - int irq, ret; u32 val; if (!IS_ENABLED(CONFIG_PCIE_ROCKCHIP_DW_HOST)) return -ENODEV; - irq = platform_get_irq_byname(pdev, "sys"); - if (irq < 0) - return irq; - - ret = devm_request_threaded_irq(dev, irq, NULL, - rockchip_pcie_rc_sys_irq_thread, - IRQF_ONESHOT, "pcie-sys-rc", rockchip); - if (ret) { - dev_err(dev, "failed to request PCIe sys IRQ\n"); - return ret; - } - /* LTSSM enable control mode */ val = FIELD_PREP_WM16(PCIE_LTSSM_ENABLE_ENHANCE, 1); rockchip_pcie_writel_apb(rockchip, val, PCIE_CLIENT_HOT_RESET_CTRL); @@ -580,19 +566,8 @@ static int rockchip_pcie_configure_rc(struct platform_device *pdev, pp = &rockchip->pci.pp; pp->ops = &rockchip_pcie_host_ops; - pp->use_linkup_irq = true; - - ret = dw_pcie_host_init(pp); - if (ret) { - dev_err(dev, "failed to initialize host\n"); - return ret; - } - /* unmask DLL up/down indicator */ - val = FIELD_PREP_WM16(PCIE_RDLH_LINK_UP_CHGED, 0); - rockchip_pcie_writel_apb(rockchip, val, PCIE_CLIENT_INTR_MASK_MISC); - - return ret; + return dw_pcie_host_init(pp); } static int rockchip_pcie_configure_ep(struct platform_device *pdev, @@ -711,7 +686,7 @@ static int rockchip_pcie_probe(struct platform_device *pdev) switch (data->mode) { case DW_PCIE_RC_TYPE: - ret = rockchip_pcie_configure_rc(pdev, rockchip); + ret = rockchip_pcie_configure_rc(rockchip); if (ret) goto deinit_clk; break; diff --git a/drivers/pci/controller/dwc/pcie-keembay.c b/drivers/pci/controller/dwc/pcie-keembay.c index 60e74ac782af32..2666a9c3d67e71 100644 --- a/drivers/pci/controller/dwc/pcie-keembay.c +++ b/drivers/pci/controller/dwc/pcie-keembay.c @@ -309,6 +309,7 @@ static int keembay_pcie_ep_raise_irq(struct dw_pcie_ep *ep, u8 func_no, } static const struct pci_epc_features keembay_pcie_epc_features = { + DWC_EPC_COMMON_FEATURES, .msi_capable = true, .msix_capable = true, .bar[BAR_0] = { .only_64bit = true, }, diff --git a/drivers/pci/controller/dwc/pcie-nxp-s32g.c b/drivers/pci/controller/dwc/pcie-nxp-s32g.c index 47745749f75c36..b3ec38099fa386 100644 --- a/drivers/pci/controller/dwc/pcie-nxp-s32g.c +++ b/drivers/pci/controller/dwc/pcie-nxp-s32g.c @@ -282,12 +282,12 @@ static int s32g_pcie_parse_ports(struct device *dev, struct s32g_pcie *s32g_pp) ret = s32g_pcie_parse_port(s32g_pp, of_port); if (ret) - goto err_port; + break; } -err_port: - list_for_each_entry_safe(port, tmp, &s32g_pp->ports, list) - list_del(&port->list); + if (ret) + list_for_each_entry_safe(port, tmp, &s32g_pp->ports, list) + list_del(&port->list); return ret; } diff --git a/drivers/pci/controller/dwc/pcie-qcom-ep.c b/drivers/pci/controller/dwc/pcie-qcom-ep.c index f1bc0ac81a928b..5e990c7a5879f4 100644 --- a/drivers/pci/controller/dwc/pcie-qcom-ep.c +++ b/drivers/pci/controller/dwc/pcie-qcom-ep.c @@ -820,6 +820,7 @@ static void qcom_pcie_ep_init_debugfs(struct qcom_pcie_ep *pcie_ep) } static const struct pci_epc_features qcom_pcie_epc_features = { + DWC_EPC_COMMON_FEATURES, .linkup_notifier = true, .msi_capable = true, .align = SZ_4K, diff --git a/drivers/pci/controller/dwc/pcie-qcom.c b/drivers/pci/controller/dwc/pcie-qcom.c index 5a318487b2b3f6..cf1cc7279c10d9 100644 --- a/drivers/pci/controller/dwc/pcie-qcom.c +++ b/drivers/pci/controller/dwc/pcie-qcom.c @@ -55,9 +55,6 @@ #define PARF_AXI_MSTR_WR_ADDR_HALT_V2 0x1a8 #define PARF_Q2A_FLUSH 0x1ac #define PARF_LTSSM 0x1b0 -#define PARF_INT_ALL_STATUS 0x224 -#define PARF_INT_ALL_CLEAR 0x228 -#define PARF_INT_ALL_MASK 0x22c #define PARF_SID_OFFSET 0x234 #define PARF_BDF_TRANSLATE_CFG 0x24c #define PARF_DBI_BASE_ADDR_V2 0x350 @@ -134,10 +131,6 @@ /* PARF_LTSSM register fields */ #define LTSSM_EN BIT(8) -/* PARF_INT_ALL_{STATUS/CLEAR/MASK} register fields */ -#define PARF_INT_ALL_LINK_UP BIT(13) -#define PARF_INT_MSI_DEV_0_7 GENMASK(30, 23) - /* PARF_NO_SNOOP_OVERRIDE register fields */ #define WR_NO_SNOOP_OVERRIDE_EN BIT(1) #define RD_NO_SNOOP_OVERRIDE_EN BIT(3) @@ -1635,32 +1628,6 @@ static void qcom_pcie_init_debugfs(struct qcom_pcie *pcie) qcom_pcie_link_transition_count); } -static irqreturn_t qcom_pcie_global_irq_thread(int irq, void *data) -{ - struct qcom_pcie *pcie = data; - struct dw_pcie_rp *pp = &pcie->pci->pp; - struct device *dev = pcie->pci->dev; - u32 status = readl_relaxed(pcie->parf + PARF_INT_ALL_STATUS); - - writel_relaxed(status, pcie->parf + PARF_INT_ALL_CLEAR); - - if (FIELD_GET(PARF_INT_ALL_LINK_UP, status)) { - msleep(PCIE_RESET_CONFIG_WAIT_MS); - dev_dbg(dev, "Received Link up event. Starting enumeration!\n"); - /* Rescan the bus to enumerate endpoint devices */ - pci_lock_rescan_remove(); - pci_rescan_bus(pp->bridge->bus); - pci_unlock_rescan_remove(); - - qcom_pcie_icc_opp_update(pcie); - } else { - dev_WARN_ONCE(dev, 1, "Received unknown event. INT_STATUS: 0x%08x\n", - status); - } - - return IRQ_HANDLED; -} - static void qcom_pci_free_msi(void *ptr) { struct dw_pcie_rp *pp = (struct dw_pcie_rp *)ptr; @@ -1805,8 +1772,7 @@ static int qcom_pcie_probe(struct platform_device *pdev) struct dw_pcie_rp *pp; struct resource *res; struct dw_pcie *pci; - int ret, irq; - char *name; + int ret; pcie_cfg = of_device_get_match_data(dev); if (!pcie_cfg) { @@ -1957,37 +1923,12 @@ static int qcom_pcie_probe(struct platform_device *pdev) platform_set_drvdata(pdev, pcie); - irq = platform_get_irq_byname_optional(pdev, "global"); - if (irq > 0) - pp->use_linkup_irq = true; - ret = dw_pcie_host_init(pp); if (ret) { dev_err(dev, "cannot initialize host\n"); goto err_phy_exit; } - name = devm_kasprintf(dev, GFP_KERNEL, "qcom_pcie_global_irq%d", - pci_domain_nr(pp->bridge->bus)); - if (!name) { - ret = -ENOMEM; - goto err_host_deinit; - } - - if (irq > 0) { - ret = devm_request_threaded_irq(&pdev->dev, irq, NULL, - qcom_pcie_global_irq_thread, - IRQF_ONESHOT, name, pcie); - if (ret) { - dev_err_probe(&pdev->dev, ret, - "Failed to request Global IRQ\n"); - goto err_host_deinit; - } - - writel_relaxed(PARF_INT_ALL_LINK_UP | PARF_INT_MSI_DEV_0_7, - pcie->parf + PARF_INT_ALL_MASK); - } - qcom_pcie_icc_opp_update(pcie); if (pcie->mhi) @@ -1995,8 +1936,6 @@ static int qcom_pcie_probe(struct platform_device *pdev) return 0; -err_host_deinit: - dw_pcie_host_deinit(pp); err_phy_exit: list_for_each_entry_safe(port, tmp, &pcie->ports, list) { phy_exit(port->phy); diff --git a/drivers/pci/controller/dwc/pcie-rcar-gen4.c b/drivers/pci/controller/dwc/pcie-rcar-gen4.c index 80778917d2ddd3..a6912e85e4ddc3 100644 --- a/drivers/pci/controller/dwc/pcie-rcar-gen4.c +++ b/drivers/pci/controller/dwc/pcie-rcar-gen4.c @@ -420,6 +420,7 @@ static int rcar_gen4_pcie_ep_raise_irq(struct dw_pcie_ep *ep, u8 func_no, } static const struct pci_epc_features rcar_gen4_pcie_epc_features = { + DWC_EPC_COMMON_FEATURES, .msi_capable = true, .bar[BAR_1] = { .type = BAR_RESERVED, }, .bar[BAR_3] = { .type = BAR_RESERVED, }, diff --git a/drivers/pci/controller/dwc/pcie-sophgo.c b/drivers/pci/controller/dwc/pcie-sophgo.c index ad4baaa34ffa1d..044088898819e5 100644 --- a/drivers/pci/controller/dwc/pcie-sophgo.c +++ b/drivers/pci/controller/dwc/pcie-sophgo.c @@ -161,6 +161,22 @@ static void sophgo_pcie_msi_enable(struct dw_pcie_rp *pp) raw_spin_unlock_irqrestore(&pp->lock, flags); } +static void sophgo_pcie_disable_l0s_l1(struct dw_pcie_rp *pp) +{ + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + u32 offset, val; + + offset = dw_pcie_find_capability(pci, PCI_CAP_ID_EXP); + + dw_pcie_dbi_ro_wr_en(pci); + + val = dw_pcie_readl_dbi(pci, PCI_EXP_LNKCAP + offset); + val &= ~(PCI_EXP_LNKCAP_ASPM_L0S | PCI_EXP_LNKCAP_ASPM_L1); + dw_pcie_writel_dbi(pci, PCI_EXP_LNKCAP + offset, val); + + dw_pcie_dbi_ro_wr_dis(pci); +} + static int sophgo_pcie_host_init(struct dw_pcie_rp *pp) { int irq; @@ -171,6 +187,8 @@ static int sophgo_pcie_host_init(struct dw_pcie_rp *pp) irq_set_chained_handler_and_data(irq, sophgo_pcie_intx_handler, pp); + sophgo_pcie_disable_l0s_l1(pp); + sophgo_pcie_msi_enable(pp); return 0; diff --git a/drivers/pci/controller/dwc/pcie-stm32-ep.c b/drivers/pci/controller/dwc/pcie-stm32-ep.c index 2cecf32d2b0f36..c1944b40ce02f5 100644 --- a/drivers/pci/controller/dwc/pcie-stm32-ep.c +++ b/drivers/pci/controller/dwc/pcie-stm32-ep.c @@ -70,6 +70,7 @@ static int stm32_pcie_raise_irq(struct dw_pcie_ep *ep, u8 func_no, } static const struct pci_epc_features stm32_pcie_epc_features = { + DWC_EPC_COMMON_FEATURES, .msi_capable = true, .align = SZ_64K, }; diff --git a/drivers/pci/controller/dwc/pcie-tegra194.c b/drivers/pci/controller/dwc/pcie-tegra194.c index 0ddeef70726dda..06571d806ab31f 100644 --- a/drivers/pci/controller/dwc/pcie-tegra194.c +++ b/drivers/pci/controller/dwc/pcie-tegra194.c @@ -1988,6 +1988,7 @@ static int tegra_pcie_ep_raise_irq(struct dw_pcie_ep *ep, u8 func_no, } static const struct pci_epc_features tegra_pcie_epc_features = { + DWC_EPC_COMMON_FEATURES, .linkup_notifier = true, .msi_capable = true, .bar[BAR_0] = { .type = BAR_FIXED, .fixed_size = SZ_1M, diff --git a/drivers/pci/controller/dwc/pcie-uniphier-ep.c b/drivers/pci/controller/dwc/pcie-uniphier-ep.c index d6e73811216e2e..d52753060970fa 100644 --- a/drivers/pci/controller/dwc/pcie-uniphier-ep.c +++ b/drivers/pci/controller/dwc/pcie-uniphier-ep.c @@ -420,6 +420,7 @@ static const struct uniphier_pcie_ep_soc_data uniphier_pro5_data = { .init = uniphier_pcie_pro5_init_ep, .wait = NULL, .features = { + DWC_EPC_COMMON_FEATURES, .linkup_notifier = false, .msi_capable = true, .msix_capable = false, @@ -438,6 +439,7 @@ static const struct uniphier_pcie_ep_soc_data uniphier_nx1_data = { .init = uniphier_pcie_nx1_init_ep, .wait = uniphier_pcie_nx1_wait_ep, .features = { + DWC_EPC_COMMON_FEATURES, .linkup_notifier = false, .msi_capable = true, .msix_capable = false, diff --git a/drivers/pci/controller/pci-hyperv.c b/drivers/pci/controller/pci-hyperv.c index 1e237d3538f9c4..7f1c1a2e5c69d8 100644 --- a/drivers/pci/controller/pci-hyperv.c +++ b/drivers/pci/controller/pci-hyperv.c @@ -2486,6 +2486,14 @@ static void hv_pci_assign_numa_node(struct hv_pcibus_device *hbus) if (!hv_dev) continue; + /* + * If the Hyper-V host doesn't provide a NUMA node for the + * device, default to node 0. With NUMA_NO_NODE the kernel + * may spread work across NUMA nodes, which degrades + * performance on Hyper-V. + */ + set_dev_node(&dev->dev, 0); + if (hv_dev->desc.flags & HV_PCI_DEVICE_FLAG_NUMA_AFFINITY && hv_dev->desc.virtual_numa_node < num_possible_nodes()) /* @@ -3781,7 +3789,7 @@ static int hv_pci_probe(struct hv_device *hdev, hbus->bridge->domain_nr); if (!hbus->wq) { ret = -ENOMEM; - goto free_dom; + goto free_bus; } hdev->channel->next_request_id_callback = vmbus_next_request_id; @@ -3877,8 +3885,6 @@ static int hv_pci_probe(struct hv_device *hdev, vmbus_close(hdev->channel); destroy_wq: destroy_workqueue(hbus->wq); -free_dom: - pci_bus_release_emul_domain_nr(hbus->bridge->domain_nr); free_bus: kfree(hbus); return ret; diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index 2d92fc79f6ddfa..d962724750840d 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -33,6 +33,10 @@ #include "pci-host-common.h" +static int link_up_timeout = 500; +module_param(link_up_timeout, int, 0644); +MODULE_PARM_DESC(link_up_timeout, "PCIe link training timeout in milliseconds"); + /* T8103 (original M1) and related SoCs */ #define CORE_RC_PHYIF_CTL 0x00024 #define CORE_RC_PHYIF_CTL_RUN BIT(0) @@ -550,22 +554,105 @@ static u32 apple_pcie_rid2sid_write(struct apple_pcie_port *port, return readl_relaxed(port_rid2sid_addr(port, idx)); } +static int apple_pcie_setup_link(struct apple_pcie *pcie, + struct apple_pcie_port *port, + struct device_node *np) +{ +#define MAX_AUX_PERST 3 + struct gpio_desc *aux_reset[MAX_AUX_PERST] = { NULL }; + u32 num_aux_resets = 0; + struct gpio_desc *reset, *pwren = NULL; + u32 stat; + int ret; + + /* + * Assert PERST# and configure the pin as output. + * The Aquantia AQC113 10GB nic used desktop macs is sensitive to + * deasserting it without prior clock setup. + * Observed on M1 Max/Ultra Mac Studios under m1n1's hypervisor. + */ + reset = devm_fwnode_gpiod_get(pcie->dev, of_fwnode_handle(np), "reset", + GPIOD_OUT_HIGH, "PERST#"); + if (IS_ERR(reset)) + return PTR_ERR(reset); + // HACK: use additional "reset-gpios" until pci-pwrctrl gains PERST# support. + for (u32 idx = 0; idx < MAX_AUX_PERST; idx++) { + aux_reset[idx] = devm_fwnode_gpiod_get_index(pcie->dev, + of_fwnode_handle(np), + "reset", idx + 1, + GPIOD_OUT_HIGH, + "PERST#"); + if (IS_ERR(aux_reset[idx])) { + if (PTR_ERR(aux_reset[idx]) == -ENOENT) + break; + else + return PTR_ERR(aux_reset[idx]); + } + num_aux_resets++; + } + dev_info(pcie->dev, "Using %u auxiliary PERST#\n", num_aux_resets); + + pwren = devm_fwnode_gpiod_get(pcie->dev, of_fwnode_handle(np), "pwren", + GPIOD_ASIS, "PWREN"); + if (IS_ERR(pwren)) { + if (PTR_ERR(pwren) == -ENOENT) + pwren = NULL; + else + return PTR_ERR(pwren); + } + + rmw_set(PORT_APPCLK_EN, port->base + PORT_APPCLK); + + /* Assert PERST# before setting up the clock */ + gpiod_set_value_cansleep(reset, 1); + for (u32 idx = 0; idx < num_aux_resets; idx++) + gpiod_set_value_cansleep(aux_reset[idx], 1); + + /* Power on the device if required */ + gpiod_set_value_cansleep(pwren, 1); + + ret = apple_pcie_setup_refclk(pcie, port); + if (ret < 0) + return ret; + + /* + * The minimal Tperst-clk value is 100us (PCIe CEM r5.0, 2.9.2) + * If powering up, the minimal Tpvperl is 100ms + */ + if (pwren) + msleep(100); + else + usleep_range(100, 200); + + /* Deassert PERST# */ + rmw_set(PORT_PERST_OFF, port->base + pcie->hw->port_perst); + gpiod_set_value_cansleep(reset, 0); + for (u32 idx = 0; idx < num_aux_resets; idx++) + gpiod_set_value_cansleep(aux_reset[idx], 0); + + /* Wait for 100ms after PERST# deassertion (PCIe r5.0, 6.6.1) */ + msleep(100); + + ret = readl_relaxed_poll_timeout(port->base + PORT_STATUS, stat, + stat & PORT_STATUS_READY, 100, 250000); + if (ret < 0) { + dev_err(pcie->dev, "port %pOF ready wait timeout\n", np); + return ret; + } + + return 0; +} + static int apple_pcie_setup_port(struct apple_pcie *pcie, struct device_node *np) { struct platform_device *platform = to_platform_device(pcie->dev); struct apple_pcie_port *port; - struct gpio_desc *reset; struct resource *res; char name[16]; - u32 stat, idx; + u32 link_stat, idx; int ret, i; - reset = devm_fwnode_gpiod_get(pcie->dev, of_fwnode_handle(np), "reset", - GPIOD_OUT_LOW, "PERST#"); - if (IS_ERR(reset)) - return PTR_ERR(reset); - port = devm_kzalloc(pcie->dev, sizeof(*port), GFP_KERNEL); if (!port) return -ENOMEM; @@ -601,30 +688,12 @@ static int apple_pcie_setup_port(struct apple_pcie *pcie, else port->phy = pcie->base + CORE_PHY_DEFAULT_BASE(port->idx); - rmw_set(PORT_APPCLK_EN, port->base + PORT_APPCLK); - - /* Assert PERST# before setting up the clock */ - gpiod_set_value_cansleep(reset, 1); - - ret = apple_pcie_setup_refclk(pcie, port); - if (ret < 0) - return ret; - - /* The minimal Tperst-clk value is 100us (PCIe CEM r5.0, 2.9.2) */ - usleep_range(100, 200); - - /* Deassert PERST# */ - rmw_set(PORT_PERST_OFF, port->base + pcie->hw->port_perst); - gpiod_set_value_cansleep(reset, 0); - - /* Wait for 100ms after PERST# deassertion (PCIe r5.0, 6.6.1) */ - msleep(100); - - ret = readl_relaxed_poll_timeout(port->base + PORT_STATUS, stat, - stat & PORT_STATUS_READY, 100, 250000); - if (ret < 0) { - dev_err(pcie->dev, "port %pOF ready wait timeout\n", np); - return ret; + /* link might be already brought up by u-boot, skip setup then */ + link_stat = readl_relaxed(port->base + PORT_LINKSTS); + if (!(link_stat & PORT_LINKSTS_UP)) { + ret = apple_pcie_setup_link(pcie, port, np); + if (ret) + return ret; } if (pcie->hw->port_refclk) @@ -658,10 +727,21 @@ static int apple_pcie_setup_port(struct apple_pcie *pcie, ret = apple_pcie_port_register_irqs(port); WARN_ON(ret); - writel_relaxed(PORT_LTSSMCTL_START, port->base + PORT_LTSSMCTL); + link_stat = readl_relaxed(port->base + PORT_LINKSTS); + if (!(link_stat & PORT_LINKSTS_UP)) { + unsigned long timeout, left; + /* start link training */ + writel_relaxed(PORT_LTSSMCTL_START, port->base + PORT_LTSSMCTL); - if (!wait_for_completion_timeout(&pcie->event, HZ / 10)) - dev_warn(pcie->dev, "%pOF link didn't come up\n", np); + timeout = link_up_timeout * HZ / 1000; + left = wait_for_completion_timeout(&pcie->event, timeout); + if (!left) + dev_warn(pcie->dev, "%pOF link didn't come up\n", np); + else + dev_info(pcie->dev, "%pOF link up after %ldms\n", np, + (timeout - left) * 1000 / HZ); + + } return 0; } @@ -845,13 +925,47 @@ static const struct pci_ecam_ops apple_pcie_cfg_ecam_ops = { } }; +static int apple_pcie_probe_port(struct device_node *np) +{ + struct gpio_desc *gd; + + /* check whether the GPPIO pin exists but leave it as is */ + gd = fwnode_gpiod_get_index(of_fwnode_handle(np), "reset", 0, + GPIOD_ASIS, "PERST#"); + if (IS_ERR(gd)) + return PTR_ERR(gd); + + gpiod_put(gd); + + gd = fwnode_gpiod_get_index(of_fwnode_handle(np), "pwren", 0, + GPIOD_ASIS, "PWREN"); + if (IS_ERR(gd)) { + if (PTR_ERR(gd) != -ENOENT) + return PTR_ERR(gd); + } else { + gpiod_put(gd); + } + + return 0; +} + static int apple_pcie_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; struct pci_host_bridge *bridge; + struct device_node *of_port; struct apple_pcie *pcie; int ret; + /* Check for probe dependencies for all ports first */ + for_each_available_child_of_node(dev->of_node, of_port) { + ret = apple_pcie_probe_port(of_port); + if (ret) { + of_node_put(of_port); + return dev_err_probe(dev, ret, "Port %pOF probe fail\n", of_port); + } + } + bridge = devm_pci_alloc_host_bridge(dev, sizeof(*pcie)); if (!bridge) return -ENOMEM; diff --git a/drivers/pci/controller/pcie-mediatek.c b/drivers/pci/controller/pcie-mediatek.c index 4b78b6528f9fd3..5defa5cc4c2bdc 100644 --- a/drivers/pci/controller/pcie-mediatek.c +++ b/drivers/pci/controller/pcie-mediatek.c @@ -585,8 +585,10 @@ static int mtk_pcie_init_irq_domain(struct mtk_pcie_port *port, if (IS_ENABLED(CONFIG_PCI_MSI)) { ret = mtk_pcie_allocate_msi_domains(port); - if (ret) + if (ret) { + irq_domain_remove(port->irq_domain); return ret; + } } return 0; diff --git a/drivers/pci/controller/pcie-rzg3s-host.c b/drivers/pci/controller/pcie-rzg3s-host.c index 83ec66a7082361..1bea8360f89444 100644 --- a/drivers/pci/controller/pcie-rzg3s-host.c +++ b/drivers/pci/controller/pcie-rzg3s-host.c @@ -439,28 +439,9 @@ static void __iomem *rzg3s_pcie_root_map_bus(struct pci_bus *bus, return host->pcie + where; } -/* Serialized by 'pci_lock' */ -static int rzg3s_pcie_root_write(struct pci_bus *bus, unsigned int devfn, - int where, int size, u32 val) -{ - struct rzg3s_pcie_host *host = bus->sysdata; - int ret; - - /* Enable access control to the CFGU */ - writel_relaxed(RZG3S_PCI_PERM_CFG_HWINIT_EN, - host->axi + RZG3S_PCI_PERM); - - ret = pci_generic_config_write(bus, devfn, where, size, val); - - /* Disable access control to the CFGU */ - writel_relaxed(0, host->axi + RZG3S_PCI_PERM); - - return ret; -} - static struct pci_ops rzg3s_pcie_root_ops = { .read = pci_generic_config_read, - .write = rzg3s_pcie_root_write, + .write = pci_generic_config_write, .map_bus = rzg3s_pcie_root_map_bus, }; @@ -1065,14 +1046,14 @@ static int rzg3s_pcie_config_init(struct rzg3s_pcie_host *host) writel_relaxed(0xffffffff, host->pcie + RZG3S_PCI_CFG_BARMSK00L); writel_relaxed(0xffffffff, host->pcie + RZG3S_PCI_CFG_BARMSK00U); + /* Disable access control to the CFGU */ + writel_relaxed(0, host->axi + RZG3S_PCI_PERM); + /* Update bus info */ writeb_relaxed(primary_bus, host->pcie + PCI_PRIMARY_BUS); writeb_relaxed(secondary_bus, host->pcie + PCI_SECONDARY_BUS); writeb_relaxed(subordinate_bus, host->pcie + PCI_SUBORDINATE_BUS); - /* Disable access control to the CFGU */ - writel_relaxed(0, host->axi + RZG3S_PCI_PERM); - return 0; } @@ -1162,7 +1143,8 @@ static int rzg3s_pcie_resets_prepare_and_get(struct rzg3s_pcie_host *host) static int rzg3s_pcie_host_parse_port(struct rzg3s_pcie_host *host) { - struct device_node *of_port = of_get_next_child(host->dev->of_node, NULL); + struct device_node *of_port __free(device_node) = + of_get_next_child(host->dev->of_node, NULL); struct rzg3s_pcie_port *port = &host->port; int ret; diff --git a/drivers/pci/controller/pcie-xilinx.c b/drivers/pci/controller/pcie-xilinx.c index 937ea6ae1ac486..4aa139abac16e8 100644 --- a/drivers/pci/controller/pcie-xilinx.c +++ b/drivers/pci/controller/pcie-xilinx.c @@ -302,9 +302,10 @@ static int xilinx_allocate_msi_domains(struct xilinx_pcie *pcie) return 0; } -static void xilinx_free_msi_domains(struct xilinx_pcie *pcie) +static void xilinx_free_irq_domains(struct xilinx_pcie *pcie) { irq_domain_remove(pcie->msi_domain); + irq_domain_remove(pcie->leg_domain); } /* INTx Functions */ @@ -480,8 +481,10 @@ static int xilinx_pcie_init_irq_domain(struct xilinx_pcie *pcie) phys_addr_t pa = ALIGN_DOWN(virt_to_phys(pcie), SZ_4K); ret = xilinx_allocate_msi_domains(pcie); - if (ret) + if (ret) { + irq_domain_remove(pcie->leg_domain); return ret; + } pcie_write(pcie, upper_32_bits(pa), XILINX_PCIE_REG_MSIBASE1); pcie_write(pcie, lower_32_bits(pa), XILINX_PCIE_REG_MSIBASE2); @@ -600,7 +603,7 @@ static int xilinx_pcie_probe(struct platform_device *pdev) err = pci_host_probe(bridge); if (err) - xilinx_free_msi_domains(pcie); + xilinx_free_irq_domains(pcie); return err; } diff --git a/drivers/pci/endpoint/functions/pci-epf-mhi.c b/drivers/pci/endpoint/functions/pci-epf-mhi.c index 6643a88c7a0ce3..27de533f05716a 100644 --- a/drivers/pci/endpoint/functions/pci-epf-mhi.c +++ b/drivers/pci/endpoint/functions/pci-epf-mhi.c @@ -686,7 +686,7 @@ static int pci_epf_mhi_dma_init(struct pci_epf_mhi *epf_mhi) goto err_release_tx; } - epf_mhi->dma_wq = alloc_workqueue("pci_epf_mhi_dma_wq", 0, 0); + epf_mhi->dma_wq = alloc_workqueue("pci_epf_mhi_dma_wq", WQ_PERCPU, 0); if (!epf_mhi->dma_wq) { ret = -ENOMEM; goto err_release_rx; diff --git a/drivers/pci/endpoint/functions/pci-epf-ntb.c b/drivers/pci/endpoint/functions/pci-epf-ntb.c index e01a98e74d2111..a3a588e522e715 100644 --- a/drivers/pci/endpoint/functions/pci-epf-ntb.c +++ b/drivers/pci/endpoint/functions/pci-epf-ntb.c @@ -2124,8 +2124,13 @@ static int __init epf_ntb_init(void) { int ret; - kpcintb_workqueue = alloc_workqueue("kpcintb", WQ_MEM_RECLAIM | - WQ_HIGHPRI, 0); + kpcintb_workqueue = alloc_workqueue("kpcintb", + WQ_MEM_RECLAIM | WQ_HIGHPRI | WQ_PERCPU, 0); + if (!kpcintb_workqueue) { + pr_err("Failed to allocate kpcintb workqueue\n"); + return -ENOMEM; + } + ret = pci_epf_register_driver(&epf_ntb_driver); if (ret) { destroy_workqueue(kpcintb_workqueue); diff --git a/drivers/pci/endpoint/functions/pci-epf-test.c b/drivers/pci/endpoint/functions/pci-epf-test.c index debd235253c5ba..62804120cd79a7 100644 --- a/drivers/pci/endpoint/functions/pci-epf-test.c +++ b/drivers/pci/endpoint/functions/pci-epf-test.c @@ -1188,7 +1188,7 @@ static int __init pci_epf_test_init(void) int ret; kpcitest_workqueue = alloc_workqueue("kpcitest", - WQ_MEM_RECLAIM | WQ_HIGHPRI, 0); + WQ_MEM_RECLAIM | WQ_HIGHPRI | WQ_PERCPU, 0); if (!kpcitest_workqueue) { pr_err("Failed to allocate the kpcitest work queue\n"); return -ENOMEM; diff --git a/drivers/pci/endpoint/functions/pci-epf-vntb.c b/drivers/pci/endpoint/functions/pci-epf-vntb.c index 3ecc5059f92b31..65f5bbf28480de 100644 --- a/drivers/pci/endpoint/functions/pci-epf-vntb.c +++ b/drivers/pci/endpoint/functions/pci-epf-vntb.c @@ -763,19 +763,6 @@ static void epf_ntb_mw_bar_clear(struct epf_ntb *ntb, int num_mws) } } -/** - * epf_ntb_epc_destroy() - Cleanup NTB EPC interface - * @ntb: NTB device that facilitates communication between HOST and VHOST - * - * Wrapper for epf_ntb_epc_destroy_interface() to cleanup all the NTB interfaces - */ -static void epf_ntb_epc_destroy(struct epf_ntb *ntb) -{ - pci_epc_remove_epf(ntb->epf->epc, ntb->epf, 0); - pci_epc_put(ntb->epf->epc); -} - - /** * epf_ntb_is_bar_used() - Check if a bar is used in the ntb configuration * @ntb: NTB device that facilitates communication between HOST and VHOST @@ -955,6 +942,7 @@ static int epf_ntb_epc_init(struct epf_ntb *ntb) */ static void epf_ntb_epc_cleanup(struct epf_ntb *ntb) { + disable_delayed_work_sync(&ntb->cmd_handler); epf_ntb_mw_bar_clear(ntb, ntb->num_mws); epf_ntb_db_bar_clear(ntb); epf_ntb_config_sspad_bar_clear(ntb); @@ -1525,7 +1513,7 @@ static int epf_ntb_bind(struct pci_epf *epf) ret = epf_ntb_init_epc_bar(ntb); if (ret) { dev_err(dev, "Failed to create NTB EPC\n"); - goto err_bar_init; + return ret; } ret = epf_ntb_config_spad_bar_alloc(ntb); @@ -1565,9 +1553,6 @@ static int epf_ntb_bind(struct pci_epf *epf) err_bar_alloc: epf_ntb_config_spad_bar_free(ntb); -err_bar_init: - epf_ntb_epc_destroy(ntb); - return ret; } @@ -1583,7 +1568,6 @@ static void epf_ntb_unbind(struct pci_epf *epf) epf_ntb_epc_cleanup(ntb); epf_ntb_config_spad_bar_free(ntb); - epf_ntb_epc_destroy(ntb); pci_unregister_driver(&vntb_pci_driver); } @@ -1651,8 +1635,13 @@ static int __init epf_ntb_init(void) { int ret; - kpcintb_workqueue = alloc_workqueue("kpcintb", WQ_MEM_RECLAIM | - WQ_HIGHPRI, 0); + kpcintb_workqueue = alloc_workqueue("kpcintb", + WQ_MEM_RECLAIM | WQ_HIGHPRI | WQ_PERCPU, 0); + if (!kpcintb_workqueue) { + pr_err("Failed to allocate kpcintb workqueue\n"); + return -ENOMEM; + } + ret = pci_epf_register_driver(&epf_ntb_driver); if (ret) { destroy_workqueue(kpcintb_workqueue); diff --git a/drivers/pci/endpoint/pci-ep-cfs.c b/drivers/pci/endpoint/pci-ep-cfs.c index ef50c82e647f4d..8b392a8363bb19 100644 --- a/drivers/pci/endpoint/pci-ep-cfs.c +++ b/drivers/pci/endpoint/pci-ep-cfs.c @@ -23,7 +23,6 @@ struct pci_epf_group { struct config_group group; struct config_group primary_epc_group; struct config_group secondary_epc_group; - struct delayed_work cfs_work; struct pci_epf *epf; int index; }; @@ -69,8 +68,8 @@ static int pci_secondary_epc_epf_link(struct config_item *epf_item, return 0; } -static void pci_secondary_epc_epf_unlink(struct config_item *epc_item, - struct config_item *epf_item) +static void pci_secondary_epc_epf_unlink(struct config_item *epf_item, + struct config_item *epc_item) { struct pci_epf_group *epf_group = to_pci_epf_group(epf_item->ci_parent); struct pci_epc_group *epc_group = to_pci_epc_group(epc_item); @@ -103,7 +102,7 @@ static struct config_group secondary_epc_group = &epf_group->secondary_epc_group; config_group_init_type_name(secondary_epc_group, "secondary", &pci_secondary_epc_type); - configfs_register_group(&epf_group->group, secondary_epc_group); + configfs_add_default_group(secondary_epc_group, &epf_group->group); return secondary_epc_group; } @@ -133,8 +132,8 @@ static int pci_primary_epc_epf_link(struct config_item *epf_item, return 0; } -static void pci_primary_epc_epf_unlink(struct config_item *epc_item, - struct config_item *epf_item) +static void pci_primary_epc_epf_unlink(struct config_item *epf_item, + struct config_item *epc_item) { struct pci_epf_group *epf_group = to_pci_epf_group(epf_item->ci_parent); struct pci_epc_group *epc_group = to_pci_epc_group(epc_item); @@ -166,7 +165,7 @@ static struct config_group config_group_init_type_name(primary_epc_group, "primary", &pci_primary_epc_type); - configfs_register_group(&epf_group->group, primary_epc_group); + configfs_add_default_group(primary_epc_group, &epf_group->group); return primary_epc_group; } @@ -570,15 +569,13 @@ static void pci_ep_cfs_add_type_group(struct pci_epf_group *epf_group) return; } - configfs_register_group(&epf_group->group, group); + configfs_add_default_group(group, &epf_group->group); } -static void pci_epf_cfs_work(struct work_struct *work) +static void pci_epf_cfs_add_sub_groups(struct pci_epf_group *epf_group) { - struct pci_epf_group *epf_group; struct config_group *group; - epf_group = container_of(work, struct pci_epf_group, cfs_work.work); group = pci_ep_cfs_add_primary_group(epf_group); if (IS_ERR(group)) { pr_err("failed to create 'primary' EPC interface\n"); @@ -637,9 +634,7 @@ static struct config_group *pci_epf_make(struct config_group *group, kfree(epf_name); - INIT_DELAYED_WORK(&epf_group->cfs_work, pci_epf_cfs_work); - queue_delayed_work(system_wq, &epf_group->cfs_work, - msecs_to_jiffies(1)); + pci_epf_cfs_add_sub_groups(epf_group); return &epf_group->group; diff --git a/drivers/pci/endpoint/pci-epc-core.c b/drivers/pci/endpoint/pci-epc-core.c index ca7f19cc973a43..068155819c575a 100644 --- a/drivers/pci/endpoint/pci-epc-core.c +++ b/drivers/pci/endpoint/pci-epc-core.c @@ -596,6 +596,14 @@ int pci_epc_set_bar(struct pci_epc *epc, u8 func_no, u8 vfunc_no, if (!epc_features) return -EINVAL; + if (epf_bar->num_submap && !epf_bar->submap) + return -EINVAL; + + if (epf_bar->num_submap && + !(epc_features->dynamic_inbound_mapping && + epc_features->subrange_mapping)) + return -EINVAL; + if (epc_features->bar[bar].type == BAR_RESIZABLE && (epf_bar->size < SZ_1M || (u64)epf_bar->size > (SZ_128G * 1024))) return -EINVAL; diff --git a/drivers/pci/hotplug/pnv_php.c b/drivers/pci/hotplug/pnv_php.c index c5345bff9a5538..35f1758126c68b 100644 --- a/drivers/pci/hotplug/pnv_php.c +++ b/drivers/pci/hotplug/pnv_php.c @@ -802,7 +802,7 @@ static struct pnv_php_slot *pnv_php_alloc_slot(struct device_node *dn) } /* Allocate workqueue for this slot's interrupt handling */ - php_slot->wq = alloc_workqueue("pciehp-%s", 0, 0, php_slot->name); + php_slot->wq = alloc_workqueue("pciehp-%s", WQ_PERCPU, 0, php_slot->name); if (!php_slot->wq) { SLOT_WARN(php_slot, "Cannot alloc workqueue\n"); kfree(php_slot->name); diff --git a/drivers/pci/hotplug/shpchp_core.c b/drivers/pci/hotplug/shpchp_core.c index 0c341453afc605..56308515ecbaf3 100644 --- a/drivers/pci/hotplug/shpchp_core.c +++ b/drivers/pci/hotplug/shpchp_core.c @@ -80,7 +80,8 @@ static int init_slots(struct controller *ctrl) slot->device = ctrl->slot_device_offset + i; slot->number = ctrl->first_slot + (ctrl->slot_num_inc * i); - slot->wq = alloc_workqueue("shpchp-%d", 0, 0, slot->number); + slot->wq = alloc_workqueue("shpchp-%d", WQ_PERCPU, 0, + slot->number); if (!slot->wq) { retval = -ENOMEM; goto error_slot; diff --git a/drivers/pci/iov.c b/drivers/pci/iov.c index 00784a60ba80bb..4a659c34935e11 100644 --- a/drivers/pci/iov.c +++ b/drivers/pci/iov.c @@ -495,7 +495,9 @@ static ssize_t sriov_numvfs_store(struct device *dev, if (num_vfs == 0) { /* disable VFs */ + pci_lock_rescan_remove(); ret = pdev->driver->sriov_configure(pdev, 0); + pci_unlock_rescan_remove(); goto exit; } @@ -507,7 +509,9 @@ static ssize_t sriov_numvfs_store(struct device *dev, goto exit; } + pci_lock_rescan_remove(); ret = pdev->driver->sriov_configure(pdev, num_vfs); + pci_unlock_rescan_remove(); if (ret < 0) goto exit; @@ -629,18 +633,15 @@ static int sriov_add_vfs(struct pci_dev *dev, u16 num_vfs) if (dev->no_vf_scan) return 0; - pci_lock_rescan_remove(); for (i = 0; i < num_vfs; i++) { rc = pci_iov_add_virtfn(dev, i); if (rc) goto failed; } - pci_unlock_rescan_remove(); return 0; failed: while (i--) pci_iov_remove_virtfn(dev, i); - pci_unlock_rescan_remove(); return rc; } @@ -765,10 +766,8 @@ static void sriov_del_vfs(struct pci_dev *dev) struct pci_sriov *iov = dev->sriov; int i; - pci_lock_rescan_remove(); for (i = 0; i < iov->num_VFs; i++) pci_iov_remove_virtfn(dev, i); - pci_unlock_rescan_remove(); } static void sriov_disable(struct pci_dev *dev) diff --git a/drivers/pci/msi/msi.c b/drivers/pci/msi/msi.c index 34d664139f48fc..e010ecd9f90dde 100644 --- a/drivers/pci/msi/msi.c +++ b/drivers/pci/msi/msi.c @@ -737,7 +737,7 @@ static int msix_capability_init(struct pci_dev *dev, struct msix_entry *entries, ret = msix_setup_interrupts(dev, entries, nvec, affd); if (ret) - goto out_disable; + goto out_unmap; /* Disable INTX */ pci_intx_for_msi(dev, 0); @@ -758,6 +758,8 @@ static int msix_capability_init(struct pci_dev *dev, struct msix_entry *entries, pcibios_free_irq(dev); return 0; +out_unmap: + iounmap(dev->msix_base); out_disable: dev->msix_enabled = 0; pci_msix_clear_and_set_ctrl(dev, PCI_MSIX_FLAGS_MASKALL | PCI_MSIX_FLAGS_ENABLE, 0); diff --git a/drivers/pci/p2pdma.c b/drivers/pci/p2pdma.c index 4a2fc7ab42c349..79a414fd6623bd 100644 --- a/drivers/pci/p2pdma.c +++ b/drivers/pci/p2pdma.c @@ -147,11 +147,19 @@ static int p2pmem_alloc_mmap(struct file *filp, struct kobject *kobj, * we have just allocated the page no one else should be * using it. */ - VM_WARN_ON_ONCE_PAGE(!page_ref_count(page), page); + VM_WARN_ON_ONCE_PAGE(page_ref_count(page), page); set_page_count(page, 1); ret = vm_insert_page(vma, vaddr, page); if (ret) { gen_pool_free(p2pdma->pool, (uintptr_t)kaddr, len); + + /* + * Reset the page count. We don't use put_page() + * because we don't want to trigger the + * p2pdma_folio_free() path. + */ + set_page_count(page, 0); + percpu_ref_put(ref); return ret; } percpu_ref_get(ref); diff --git a/drivers/pci/pci-acpi.c b/drivers/pci/pci-acpi.c index 9369377725fa03..0162acfb57896b 100644 --- a/drivers/pci/pci-acpi.c +++ b/drivers/pci/pci-acpi.c @@ -271,21 +271,6 @@ static acpi_status decode_type1_hpx_record(union acpi_object *record, return AE_OK; } -static bool pcie_root_rcb_set(struct pci_dev *dev) -{ - struct pci_dev *rp = pcie_find_root_port(dev); - u16 lnkctl; - - if (!rp) - return false; - - pcie_capability_read_word(rp, PCI_EXP_LNKCTL, &lnkctl); - if (lnkctl & PCI_EXP_LNKCTL_RCB) - return true; - - return false; -} - /* _HPX PCI Express Setting Record (Type 2) */ struct hpx_type2 { u32 revision; @@ -311,6 +296,7 @@ static void program_hpx_type2(struct pci_dev *dev, struct hpx_type2 *hpx) { int pos; u32 reg32; + const struct pci_host_bridge *host; if (!hpx) return; @@ -318,6 +304,15 @@ static void program_hpx_type2(struct pci_dev *dev, struct hpx_type2 *hpx) if (!pci_is_pcie(dev)) return; + host = pci_find_host_bridge(dev->bus); + + /* + * Only do the _HPX Type 2 programming if OS owns PCIe native + * hotplug but not AER. + */ + if (!host->native_pcie_hotplug || host->native_aer) + return; + if (hpx->revision > 1) { pci_warn(dev, "PCIe settings rev %d not supported\n", hpx->revision); @@ -325,33 +320,27 @@ static void program_hpx_type2(struct pci_dev *dev, struct hpx_type2 *hpx) } /* - * Don't allow _HPX to change MPS or MRRS settings. We manage - * those to make sure they're consistent with the rest of the - * platform. + * We only allow _HPX to program DEVCTL bits related to AER, namely + * PCI_EXP_DEVCTL_CERE, PCI_EXP_DEVCTL_NFERE, PCI_EXP_DEVCTL_FERE, + * and PCI_EXP_DEVCTL_URRE. + * + * The rest of DEVCTL is managed by the OS to make sure it's + * consistent with the rest of the platform. */ - hpx->pci_exp_devctl_and |= PCI_EXP_DEVCTL_PAYLOAD | - PCI_EXP_DEVCTL_READRQ; - hpx->pci_exp_devctl_or &= ~(PCI_EXP_DEVCTL_PAYLOAD | - PCI_EXP_DEVCTL_READRQ); + hpx->pci_exp_devctl_and |= ~PCI_EXP_AER_FLAGS; + hpx->pci_exp_devctl_or &= PCI_EXP_AER_FLAGS; /* Initialize Device Control Register */ pcie_capability_clear_and_set_word(dev, PCI_EXP_DEVCTL, ~hpx->pci_exp_devctl_and, hpx->pci_exp_devctl_or); - /* Initialize Link Control Register */ + /* Log if _HPX attempts to modify Link Control Register */ if (pcie_cap_has_lnkctl(dev)) { - - /* - * If the Root Port supports Read Completion Boundary of - * 128, set RCB to 128. Otherwise, clear it. - */ - hpx->pci_exp_lnkctl_and |= PCI_EXP_LNKCTL_RCB; - hpx->pci_exp_lnkctl_or &= ~PCI_EXP_LNKCTL_RCB; - if (pcie_root_rcb_set(dev)) - hpx->pci_exp_lnkctl_or |= PCI_EXP_LNKCTL_RCB; - - pcie_capability_clear_and_set_word(dev, PCI_EXP_LNKCTL, - ~hpx->pci_exp_lnkctl_and, hpx->pci_exp_lnkctl_or); + if (hpx->pci_exp_lnkctl_and != 0xffff || + hpx->pci_exp_lnkctl_or != 0) + pci_info(dev, "_HPX attempts Link Control setting (AND %#06x OR %#06x)\n", + hpx->pci_exp_lnkctl_and, + hpx->pci_exp_lnkctl_or); } /* Find Advanced Error Reporting Enhanced Capability */ diff --git a/drivers/pci/pci-driver.c b/drivers/pci/pci-driver.c index 7c2d9d59625868..301a9418e38e05 100644 --- a/drivers/pci/pci-driver.c +++ b/drivers/pci/pci-driver.c @@ -1650,6 +1650,14 @@ static int pci_dma_configure(struct device *dev) ret = acpi_dma_configure(dev, acpi_get_dma_attr(adev)); } + /* + * Attempt to enable ACS regardless of capability because some Root + * Ports (e.g. those quirked with *_intel_pch_acs_*) do not have + * the standard ACS capability but still support ACS via those + * quirks. + */ + pci_enable_acs(to_pci_dev(dev)); + pci_put_host_bridge_device(bridge); /* @drv may not be valid when we're called from the IOMMU layer */ diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c index 13dbb405dc31f8..8500b862f4f2e4 100644 --- a/drivers/pci/pci.c +++ b/drivers/pci/pci.c @@ -426,7 +426,7 @@ static int pci_dev_str_match(struct pci_dev *dev, const char *p, static u8 __pci_find_next_cap(struct pci_bus *bus, unsigned int devfn, u8 pos, int cap) { - return PCI_FIND_NEXT_CAP(pci_bus_read_config, pos, cap, bus, devfn); + return PCI_FIND_NEXT_CAP(pci_bus_read_config, pos, cap, NULL, bus, devfn); } u8 pci_find_next_capability(struct pci_dev *dev, u8 pos, int cap) @@ -531,7 +531,7 @@ u16 pci_find_next_ext_capability(struct pci_dev *dev, u16 start, int cap) return 0; return PCI_FIND_NEXT_EXT_CAP(pci_bus_read_config, start, cap, - dev->bus, dev->devfn); + NULL, dev->bus, dev->devfn); } EXPORT_SYMBOL_GPL(pci_find_next_ext_capability); @@ -600,7 +600,7 @@ static u8 __pci_find_next_ht_cap(struct pci_dev *dev, u8 pos, int ht_cap) mask = HT_5BIT_CAP_MASK; pos = PCI_FIND_NEXT_CAP(pci_bus_read_config, pos, - PCI_CAP_ID_HT, dev->bus, dev->devfn); + PCI_CAP_ID_HT, NULL, dev->bus, dev->devfn); while (pos) { rc = pci_read_config_byte(dev, pos + 3, &cap); if (rc != PCIBIOS_SUCCESSFUL) @@ -611,7 +611,7 @@ static u8 __pci_find_next_ht_cap(struct pci_dev *dev, u8 pos, int ht_cap) pos = PCI_FIND_NEXT_CAP(pci_bus_read_config, pos + PCI_CAP_LIST_NEXT, - PCI_CAP_ID_HT, dev->bus, + PCI_CAP_ID_HT, NULL, dev->bus, dev->devfn); } @@ -1015,7 +1015,7 @@ static void pci_std_enable_acs(struct pci_dev *dev, struct pci_acs *caps) * pci_enable_acs - enable ACS if hardware support it * @dev: the PCI device */ -static void pci_enable_acs(struct pci_dev *dev) +void pci_enable_acs(struct pci_dev *dev) { struct pci_acs caps; bool enable_acs = false; @@ -1488,6 +1488,9 @@ static int pci_set_low_power_state(struct pci_dev *dev, pci_power_t state, bool || (state == PCI_D2 && !dev->d2_support)) return -EIO; + if (dev->current_state == state) + return 0; + pci_read_config_word(dev, dev->pm_cap + PCI_PM_CTRL, &pmcsr); if (PCI_POSSIBLE_ERROR(pmcsr)) { pci_err(dev, "Unable to change power state from %s to %s, device inaccessible\n", @@ -3196,8 +3199,14 @@ void pci_pm_init(struct pci_dev *dev) poweron: pci_pm_power_up_and_verify_state(dev); pm_runtime_forbid(&dev->dev); + + /* + * Runtime PM will be enabled for the device when it has been fully + * configured, but since its parent and suppliers may suspend in + * the meantime, prevent them from doing so by changing the + * device's runtime PM status to "active". + */ pm_runtime_set_active(&dev->dev); - pm_runtime_enable(&dev->dev); } static unsigned long pci_ea_flags(struct pci_dev *dev, u8 prop) @@ -3648,14 +3657,6 @@ bool pci_acs_path_enabled(struct pci_dev *start, void pci_acs_init(struct pci_dev *dev) { dev->acs_cap = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ACS); - - /* - * Attempt to enable ACS regardless of capability because some Root - * Ports (e.g. those quirked with *_intel_pch_acs_*) do not have - * the standard ACS capability but still support ACS via those - * quirks. - */ - pci_enable_acs(dev); } /** @@ -5290,10 +5291,9 @@ static int pci_bus_trylock(struct pci_bus *bus) /* Do any devices on or below this slot prevent a bus reset? */ static bool pci_slot_resettable(struct pci_slot *slot) { - struct pci_dev *dev; + struct pci_dev *dev, *bridge = slot->bus->self; - if (slot->bus->self && - (slot->bus->self->dev_flags & PCI_DEV_FLAGS_NO_BUS_RESET)) + if (bridge && (bridge->dev_flags & PCI_DEV_FLAGS_NO_BUS_RESET)) return false; list_for_each_entry(dev, &slot->bus->devices, bus_list) { @@ -5310,7 +5310,10 @@ static bool pci_slot_resettable(struct pci_slot *slot) /* Lock devices from the top of the tree down */ static void pci_slot_lock(struct pci_slot *slot) { - struct pci_dev *dev; + struct pci_dev *dev, *bridge = slot->bus->self; + + if (bridge) + pci_dev_lock(bridge); list_for_each_entry(dev, &slot->bus->devices, bus_list) { if (!dev->slot || dev->slot != slot) @@ -5325,7 +5328,7 @@ static void pci_slot_lock(struct pci_slot *slot) /* Unlock devices from the bottom of the tree up */ static void pci_slot_unlock(struct pci_slot *slot) { - struct pci_dev *dev; + struct pci_dev *dev, *bridge = slot->bus->self; list_for_each_entry(dev, &slot->bus->devices, bus_list) { if (!dev->slot || dev->slot != slot) @@ -5335,21 +5338,25 @@ static void pci_slot_unlock(struct pci_slot *slot) else pci_dev_unlock(dev); } + + if (bridge) + pci_dev_unlock(bridge); } /* Return 1 on successful lock, 0 on contention */ static int pci_slot_trylock(struct pci_slot *slot) { - struct pci_dev *dev; + struct pci_dev *dev, *bridge = slot->bus->self; + + if (bridge && !pci_dev_trylock(bridge)) + return 0; list_for_each_entry(dev, &slot->bus->devices, bus_list) { if (!dev->slot || dev->slot != slot) continue; if (dev->subordinate) { - if (!pci_bus_trylock(dev->subordinate)) { - pci_dev_unlock(dev); + if (!pci_bus_trylock(dev->subordinate)) goto unlock; - } } else if (!pci_dev_trylock(dev)) goto unlock; } @@ -5365,6 +5372,9 @@ static int pci_slot_trylock(struct pci_slot *slot) else pci_dev_unlock(dev); } + + if (bridge) + pci_dev_unlock(bridge); return 0; } @@ -6591,7 +6601,7 @@ static void of_pci_bus_release_domain_nr(struct device *parent, int domain_nr) return; /* Release domain from IDA where it was allocated. */ - if (of_get_pci_domain_nr(parent->of_node) == domain_nr) + if (parent && of_get_pci_domain_nr(parent->of_node) == domain_nr) ida_free(&pci_domain_nr_static_ida, domain_nr); else ida_free(&pci_domain_nr_dynamic_ida, domain_nr); diff --git a/drivers/pci/pci.h b/drivers/pci/pci.h index 0e67014aa0013a..ecc67fbb159c4a 100644 --- a/drivers/pci/pci.h +++ b/drivers/pci/pci.h @@ -88,6 +88,9 @@ struct pcie_tlp_log; #define PCI_BUS_BRIDGE_MEM_WINDOW 1 #define PCI_BUS_BRIDGE_PREF_MEM_WINDOW 2 +#define PCI_EXP_AER_FLAGS (PCI_EXP_DEVCTL_CERE | PCI_EXP_DEVCTL_NFERE | \ + PCI_EXP_DEVCTL_FERE | PCI_EXP_DEVCTL_URRE) + extern const unsigned char pcie_link_speed[]; extern bool pci_early_dump; @@ -103,17 +106,21 @@ bool pcie_cap_has_rtctl(const struct pci_dev *dev); * @read_cfg: Function pointer for reading PCI config space * @start: Starting position to begin search * @cap: Capability ID to find + * @prev_ptr: Pointer to store position of preceding capability (optional) * @args: Arguments to pass to read_cfg function * - * Search the capability list in PCI config space to find @cap. + * Search the capability list in PCI config space to find @cap. If + * found, update *prev_ptr with the position of the preceding capability + * (if prev_ptr != NULL) * Implements TTL (time-to-live) protection against infinite loops. * * Return: Position of the capability if found, 0 otherwise. */ -#define PCI_FIND_NEXT_CAP(read_cfg, start, cap, args...) \ +#define PCI_FIND_NEXT_CAP(read_cfg, start, cap, prev_ptr, args...) \ ({ \ int __ttl = PCI_FIND_CAP_TTL; \ - u8 __id, __found_pos = 0; \ + u8 __id, __found_pos = 0; \ + u8 __prev_pos = (start); \ u8 __pos = (start); \ u16 __ent; \ \ @@ -132,9 +139,12 @@ bool pcie_cap_has_rtctl(const struct pci_dev *dev); \ if (__id == (cap)) { \ __found_pos = __pos; \ + if (prev_ptr != NULL) \ + *(u8 *)prev_ptr = __prev_pos; \ break; \ } \ \ + __prev_pos = __pos; \ __pos = FIELD_GET(PCI_CAP_LIST_NEXT_MASK, __ent); \ } \ __found_pos; \ @@ -146,21 +156,26 @@ bool pcie_cap_has_rtctl(const struct pci_dev *dev); * @read_cfg: Function pointer for reading PCI config space * @start: Starting position to begin search (0 for initial search) * @cap: Extended capability ID to find + * @prev_ptr: Pointer to store position of preceding capability (optional) * @args: Arguments to pass to read_cfg function * * Search the extended capability list in PCI config space to find @cap. + * If found, update *prev_ptr with the position of the preceding capability + * (if prev_ptr != NULL) * Implements TTL protection against infinite loops using a calculated * maximum search count. * * Return: Position of the capability if found, 0 otherwise. */ -#define PCI_FIND_NEXT_EXT_CAP(read_cfg, start, cap, args...) \ +#define PCI_FIND_NEXT_EXT_CAP(read_cfg, start, cap, prev_ptr, args...) \ ({ \ u16 __pos = (start) ?: PCI_CFG_SPACE_SIZE; \ u16 __found_pos = 0; \ + u16 __prev_pos; \ int __ttl, __ret; \ u32 __header; \ \ + __prev_pos = __pos; \ __ttl = (PCI_CFG_SPACE_EXP_SIZE - PCI_CFG_SPACE_SIZE) / 8; \ while (__ttl-- > 0 && __pos >= PCI_CFG_SPACE_SIZE) { \ __ret = read_cfg##_dword(args, __pos, &__header); \ @@ -172,9 +187,12 @@ bool pcie_cap_has_rtctl(const struct pci_dev *dev); \ if (PCI_EXT_CAP_ID(__header) == (cap) && __pos != start) {\ __found_pos = __pos; \ + if (prev_ptr != NULL) \ + *(u16 *)prev_ptr = __prev_pos; \ break; \ } \ \ + __prev_pos = __pos; \ __pos = PCI_EXT_CAP_NEXT(__header); \ } \ __found_pos; \ @@ -939,6 +957,7 @@ static inline resource_size_t pci_resource_alignment(struct pci_dev *dev, } void pci_acs_init(struct pci_dev *dev); +void pci_enable_acs(struct pci_dev *dev); #ifdef CONFIG_PCI_QUIRKS int pci_dev_specific_acs_enabled(struct pci_dev *dev, u16 acs_flags); int pci_dev_specific_enable_acs(struct pci_dev *dev); diff --git a/drivers/pci/pcie/aer.c b/drivers/pci/pcie/aer.c index e0bcaa896803c9..73cb6d587202ca 100644 --- a/drivers/pci/pcie/aer.c +++ b/drivers/pci/pcie/aer.c @@ -239,9 +239,6 @@ void pcie_ecrc_get_policy(char *str) } #endif /* CONFIG_PCIE_ECRC */ -#define PCI_EXP_AER_FLAGS (PCI_EXP_DEVCTL_CERE | PCI_EXP_DEVCTL_NFERE | \ - PCI_EXP_DEVCTL_FERE | PCI_EXP_DEVCTL_URRE) - int pcie_aer_is_native(struct pci_dev *dev) { struct pci_host_bridge *host = pci_find_host_bridge(dev->bus); @@ -1608,6 +1605,20 @@ static void aer_disable_irq(struct pci_dev *pdev) pci_write_config_dword(pdev, aer + PCI_ERR_ROOT_COMMAND, reg32); } +static int clear_status_iter(struct pci_dev *dev, void *data) +{ + u16 devctl; + + /* Skip if pci_enable_pcie_error_reporting() hasn't been called yet */ + pcie_capability_read_word(dev, PCI_EXP_DEVCTL, &devctl); + if (!(devctl & PCI_EXP_AER_FLAGS)) + return 0; + + pci_aer_clear_status(dev); + pcie_clear_device_status(dev); + return 0; +} + /** * aer_enable_rootport - enable Root Port's interrupts when receiving messages * @rpc: pointer to a Root Port data structure @@ -1629,9 +1640,19 @@ static void aer_enable_rootport(struct aer_rpc *rpc) pcie_capability_clear_word(pdev, PCI_EXP_RTCTL, SYSTEM_ERROR_INTR_ON_MESG_MASK); - /* Clear error status */ + /* Clear error status of this Root Port or RCEC */ pci_read_config_dword(pdev, aer + PCI_ERR_ROOT_STATUS, ®32); pci_write_config_dword(pdev, aer + PCI_ERR_ROOT_STATUS, reg32); + + /* Clear error status of agents reporting to this Root Port or RCEC */ + if (reg32 & AER_ERR_STATUS_MASK) { + if (pci_pcie_type(pdev) == PCI_EXP_TYPE_RC_EC) + pcie_walk_rcec(pdev, clear_status_iter, NULL); + else if (pdev->subordinate) + pci_walk_bus(pdev->subordinate, clear_status_iter, + NULL); + } + pci_read_config_dword(pdev, aer + PCI_ERR_COR_STATUS, ®32); pci_write_config_dword(pdev, aer + PCI_ERR_COR_STATUS, reg32); pci_read_config_dword(pdev, aer + PCI_ERR_UNCOR_STATUS, ®32); diff --git a/drivers/pci/pcie/bwctrl.c b/drivers/pci/pcie/bwctrl.c index 36f939f23d34e8..4ae92c9f912a8b 100644 --- a/drivers/pci/pcie/bwctrl.c +++ b/drivers/pci/pcie/bwctrl.c @@ -250,6 +250,9 @@ static int pcie_bwnotif_probe(struct pcie_device *srv) struct pci_dev *port = srv->port; int ret; + if (port->no_bw_notif) + return -ENODEV; + /* Can happen if we run out of bus numbers during enumeration. */ if (!port->subordinate) return -ENODEV; diff --git a/drivers/pci/pcie/portdrv.c b/drivers/pci/pcie/portdrv.c index 38a41ccf79b9a3..a0991da4821363 100644 --- a/drivers/pci/pcie/portdrv.c +++ b/drivers/pci/pcie/portdrv.c @@ -557,10 +557,10 @@ static int pcie_port_remove_service(struct device *dev) pciedev = to_pcie_device(dev); driver = to_service_driver(dev->driver); - if (driver && driver->remove) { + if (driver && driver->remove) driver->remove(pciedev); - put_device(dev); - } + + put_device(dev); return 0; } diff --git a/drivers/pci/pcie/ptm.c b/drivers/pci/pcie/ptm.c index ed0f9691e7d16e..c7c61869bc9cf8 100644 --- a/drivers/pci/pcie/ptm.c +++ b/drivers/pci/pcie/ptm.c @@ -542,8 +542,10 @@ struct pci_ptm_debugfs *pcie_ptm_create_debugfs(struct device *dev, void *pdata, return NULL; dirname = devm_kasprintf(dev, GFP_KERNEL, "pcie_ptm_%s", dev_name(dev)); - if (!dirname) + if (!dirname) { + kfree(ptm_debugfs); return NULL; + } ptm_debugfs->debugfs = debugfs_create_dir(dirname, NULL); ptm_debugfs->pdata = pdata; @@ -574,6 +576,7 @@ void pcie_ptm_destroy_debugfs(struct pci_ptm_debugfs *ptm_debugfs) mutex_destroy(&ptm_debugfs->lock); debugfs_remove_recursive(ptm_debugfs->debugfs); + kfree(ptm_debugfs); } EXPORT_SYMBOL_GPL(pcie_ptm_destroy_debugfs); #endif diff --git a/drivers/pci/probe.c b/drivers/pci/probe.c index 41183aed8f5d94..cd1cd044aaf9d7 100644 --- a/drivers/pci/probe.c +++ b/drivers/pci/probe.c @@ -287,8 +287,7 @@ int __pci_read_base(struct pci_dev *dev, enum pci_bar_type type, if ((sizeof(pci_bus_addr_t) < 8 || sizeof(resource_size_t) < 8) && sz64 > 0x100000000ULL) { res->flags |= IORESOURCE_UNSET | IORESOURCE_DISABLED; - res->start = 0; - res->end = 0; + resource_set_range(res, 0, 0); pci_err(dev, "%s: can't handle BAR larger than 4GB (size %#010llx)\n", res_name, (unsigned long long)sz64); goto out; @@ -297,8 +296,7 @@ int __pci_read_base(struct pci_dev *dev, enum pci_bar_type type, if ((sizeof(pci_bus_addr_t) < 8) && l) { /* Above 32-bit boundary; try to reallocate */ res->flags |= IORESOURCE_UNSET; - res->start = 0; - res->end = sz64 - 1; + resource_set_range(res, 0, sz64); pci_info(dev, "%s: can't handle BAR above 4GB (bus address %#010llx)\n", res_name, (unsigned long long)l64); goto out; @@ -2270,7 +2268,8 @@ int pci_configure_extended_tags(struct pci_dev *dev, void *ign) u16 ctl; int ret; - if (!pci_is_pcie(dev)) + /* PCI_EXP_DEVCTL_EXT_TAG is RsvdP in VFs */ + if (!pci_is_pcie(dev) || dev->is_virtfn) return 0; ret = pcie_capability_read_dword(dev, PCI_EXP_DEVCAP, &cap); @@ -2410,6 +2409,37 @@ static void pci_configure_serr(struct pci_dev *dev) } } +static void pci_configure_rcb(struct pci_dev *dev) +{ + struct pci_dev *rp; + u16 rp_lnkctl; + + /* + * Per PCIe r7.0, sec 7.5.3.7, RCB is only meaningful in Root Ports + * (where it is read-only), Endpoints, and Bridges. It may only be + * set for Endpoints and Bridges if it is set in the Root Port. For + * Endpoints, it is 'RsvdP' for Virtual Functions. + */ + if (!pci_is_pcie(dev) || + pci_pcie_type(dev) == PCI_EXP_TYPE_ROOT_PORT || + pci_pcie_type(dev) == PCI_EXP_TYPE_UPSTREAM || + pci_pcie_type(dev) == PCI_EXP_TYPE_DOWNSTREAM || + pci_pcie_type(dev) == PCI_EXP_TYPE_RC_EC || + dev->is_virtfn) + return; + + /* Root Port often not visible to virtualized guests */ + rp = pcie_find_root_port(dev); + if (!rp) + return; + + pcie_capability_read_word(rp, PCI_EXP_LNKCTL, &rp_lnkctl); + pcie_capability_clear_and_set_word(dev, PCI_EXP_LNKCTL, + PCI_EXP_LNKCTL_RCB, + (rp_lnkctl & PCI_EXP_LNKCTL_RCB) ? + PCI_EXP_LNKCTL_RCB : 0); +} + static void pci_configure_device(struct pci_dev *dev) { pci_configure_mps(dev); @@ -2419,6 +2449,7 @@ static void pci_configure_device(struct pci_dev *dev) pci_configure_aspm_l1ss(dev); pci_configure_eetlp_prefix(dev); pci_configure_serr(dev); + pci_configure_rcb(dev); pci_acpi_program_hp_params(dev); } diff --git a/drivers/pci/pwrctrl/pci-pwrctrl-tc9563.c b/drivers/pci/pwrctrl/pci-pwrctrl-tc9563.c index ec423432ac6550..0a63add84d095b 100644 --- a/drivers/pci/pwrctrl/pci-pwrctrl-tc9563.c +++ b/drivers/pci/pwrctrl/pci-pwrctrl-tc9563.c @@ -533,7 +533,7 @@ static int tc9563_pwrctrl_probe(struct platform_device *pdev) ctx->client = i2c_new_dummy_device(ctx->adapter, addr); if (IS_ERR(ctx->client)) { dev_err(dev, "Failed to create I2C client\n"); - i2c_put_adapter(ctx->adapter); + put_device(&ctx->adapter->dev); return PTR_ERR(ctx->client); } @@ -613,7 +613,7 @@ static int tc9563_pwrctrl_probe(struct platform_device *pdev) tc9563_pwrctrl_power_off(ctx); remove_i2c: i2c_unregister_device(ctx->client); - i2c_put_adapter(ctx->adapter); + put_device(&ctx->adapter->dev); return ret; } @@ -623,7 +623,7 @@ static void tc9563_pwrctrl_remove(struct platform_device *pdev) tc9563_pwrctrl_power_off(ctx); i2c_unregister_device(ctx->client); - i2c_put_adapter(ctx->adapter); + put_device(&ctx->adapter->dev); } static const struct of_device_id tc9563_pwrctrl_of_match[] = { diff --git a/drivers/pci/quirks.c b/drivers/pci/quirks.c index 280cd50d693bd2..fd86f72f54aefc 100644 --- a/drivers/pci/quirks.c +++ b/drivers/pci/quirks.c @@ -1359,6 +1359,16 @@ static void quirk_transparent_bridge(struct pci_dev *dev) DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82380FB, quirk_transparent_bridge); DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_TOSHIBA, 0x605, quirk_transparent_bridge); +/* + * Enabling Link Bandwidth Management Interrupts (BW notifications) can cause + * boot hangs on P45. + */ +static void quirk_p45_bw_notifications(struct pci_dev *dev) +{ + dev->no_bw_notif = 1; +} +DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e21, quirk_p45_bw_notifications); + /* * Common misconfiguration of the MediaGX/Geode PCI master that will reduce * PCI bandwidth from 70MB/s to 25MB/s. See the GXM/GXLV/GX1 datasheets @@ -3748,6 +3758,14 @@ static void quirk_no_bus_reset(struct pci_dev *dev) dev->dev_flags |= PCI_DEV_FLAGS_NO_BUS_RESET; } +/* + * After asserting Secondary Bus Reset to downstream devices via a GB10 + * Root Port, the link may not retrain correctly. + * https://lore.kernel.org/r/20251113084441.2124737-1-Johnny-CC.Chang@mediatek.com + */ +DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_NVIDIA, 0x22CE, quirk_no_bus_reset); +DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_NVIDIA, 0x22D0, quirk_no_bus_reset); + /* * Some NVIDIA GPU devices do not work with bus reset, SBR needs to be * prevented for those affected devices. @@ -3791,6 +3809,16 @@ DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_CAVIUM, 0xa100, quirk_no_bus_reset); */ DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_TI, 0xb005, quirk_no_bus_reset); +/* + * Reports from users making use of PCI device assignment with ASM1164 + * controllers indicate an issue with bus reset where the device fails to + * retrain. The issue appears more common in configurations with multiple + * controllers. The device does indicate PM reset support (NoSoftRst-), + * therefore this still leaves a viable reset method. + * https://forum.proxmox.com/threads/problems-with-pcie-passthrough-with-two-identical-devices.149003/ + */ +DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_ASMEDIA, 0x1164, quirk_no_bus_reset); + static void quirk_no_pm_reset(struct pci_dev *dev) { /* @@ -5107,6 +5135,10 @@ static const struct pci_dev_acs_enabled { { PCI_VENDOR_ID_QCOM, 0x0401, pci_quirk_qcom_rp_acs }, /* QCOM SA8775P root port */ { PCI_VENDOR_ID_QCOM, 0x0115, pci_quirk_qcom_rp_acs }, + /* QCOM Hamoa root port */ + { PCI_VENDOR_ID_QCOM, 0x0111, pci_quirk_qcom_rp_acs }, + /* QCOM Glymur root port */ + { PCI_VENDOR_ID_QCOM, 0x0120, pci_quirk_qcom_rp_acs }, /* HXT SD4800 root ports. The ACS design is same as QCOM QDF2xxx */ { PCI_VENDOR_ID_HXT, 0x0401, pci_quirk_qcom_rp_acs }, /* Intel PCH root ports */ @@ -5581,6 +5613,7 @@ static void quirk_no_ext_tags(struct pci_dev *pdev) pci_walk_bus(bridge->bus, pci_configure_extended_tags, NULL); } DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_3WARE, 0x1004, quirk_no_ext_tags); +DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_3WARE, 0x1005, quirk_no_ext_tags); DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_SERVERWORKS, 0x0132, quirk_no_ext_tags); DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_SERVERWORKS, 0x0140, quirk_no_ext_tags); DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_SERVERWORKS, 0x0141, quirk_no_ext_tags); @@ -6188,6 +6221,10 @@ DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_PERICOM, 0x2303, pci_fixup_pericom_acs_store_forward); DECLARE_PCI_FIXUP_RESUME(PCI_VENDOR_ID_PERICOM, 0x2303, pci_fixup_pericom_acs_store_forward); +DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_PERICOM, 0xb404, + pci_fixup_pericom_acs_store_forward); +DECLARE_PCI_FIXUP_RESUME(PCI_VENDOR_ID_PERICOM, 0xb404, + pci_fixup_pericom_acs_store_forward); static void nvidia_ion_ahci_fixup(struct pci_dev *pdev) { diff --git a/drivers/pci/setup-bus.c b/drivers/pci/setup-bus.c index a61d38777cdc4a..8311b06fadcc65 100644 --- a/drivers/pci/setup-bus.c +++ b/drivers/pci/setup-bus.c @@ -14,6 +14,7 @@ * tighter packing. Prefetchable range support. */ +#include #include #include #include @@ -225,14 +226,21 @@ static struct resource *pbus_select_window_for_type(struct pci_bus *bus, switch (iores_type) { case IORESOURCE_IO: - return pci_bus_resource_n(bus, PCI_BUS_BRIDGE_IO_WINDOW); + win = pci_bus_resource_n(bus, PCI_BUS_BRIDGE_IO_WINDOW); + if (win && (win->flags & IORESOURCE_IO)) + return win; + return NULL; case IORESOURCE_MEM: mmio = pci_bus_resource_n(bus, PCI_BUS_BRIDGE_MEM_WINDOW); mmio_pref = pci_bus_resource_n(bus, PCI_BUS_BRIDGE_PREF_MEM_WINDOW); - if (!(type & IORESOURCE_PREFETCH) || - !(mmio_pref->flags & IORESOURCE_MEM)) + if (mmio && !(mmio->flags & IORESOURCE_MEM)) + mmio = NULL; + if (mmio_pref && !(mmio_pref->flags & IORESOURCE_MEM)) + mmio_pref = NULL; + + if (!(type & IORESOURCE_PREFETCH) || !mmio_pref) return mmio; if ((type & IORESOURCE_MEM_64) || @@ -456,7 +464,7 @@ static void reassign_resources_sorted(struct list_head *realloc_head, "%s %pR: ignoring failure in optional allocation\n", res_name, res); } - } else if (add_size > 0) { + } else if (add_size > 0 || !IS_ALIGNED(res->start, align)) { res->flags |= add_res->flags & (IORESOURCE_STARTALIGN|IORESOURCE_SIZEALIGN); if (pci_reassign_resource(dev, idx, add_size, align)) @@ -1070,16 +1078,13 @@ static resource_size_t calculate_memsize(resource_size_t size, resource_size_t min_size, resource_size_t add_size, resource_size_t children_add_size, - resource_size_t old_size, resource_size_t align) { if (size < min_size) size = min_size; - if (old_size == 1) - old_size = 0; size = max(size, add_size) + children_add_size; - return ALIGN(max(size, old_size), align); + return ALIGN(size, align); } resource_size_t __weak pcibios_window_alignment(struct pci_bus *bus, @@ -1227,66 +1232,43 @@ static inline resource_size_t calculate_mem_align(resource_size_t *aligns, return min_align; } -/** - * pbus_upstream_space_available - Check no upstream resource limits allocation - * @bus: The bus - * @res: The resource to help select the correct bridge window - * @size: The size required from the bridge window - * @align: Required alignment for the resource - * - * Check that @size can fit inside the upstream bridge resources that are - * already assigned. Select the upstream bridge window based on the type of - * @res. - * - * Return: %true if enough space is available on all assigned upstream +/* + * Calculate bridge window head alignment that leaves no gaps in between * resources. */ -static bool pbus_upstream_space_available(struct pci_bus *bus, - struct resource *res, - resource_size_t size, - resource_size_t align) -{ - struct resource_constraint constraint = { - .max = RESOURCE_SIZE_MAX, - .align = align, - }; - struct pci_bus *downstream = bus; - - while ((bus = bus->parent)) { - if (pci_is_root_bus(bus)) - break; +static resource_size_t calculate_head_align(resource_size_t *aligns, + int max_order) +{ + resource_size_t head_align = 1; + resource_size_t remainder = 0; + int order; - res = pbus_select_window(bus, res); - if (!res) - return false; - if (!res->parent) - continue; + /* Take the largest alignment as the starting point. */ + head_align <<= max_order + __ffs(SZ_1M); - if (resource_size(res) >= size) { - struct resource gap = {}; + for (order = max_order - 1; order >= 0; order--) { + resource_size_t align1 = 1; - if (find_resource_space(res, &gap, size, &constraint) == 0) { - gap.flags = res->flags; - pci_dbg(bus->self, - "Assigned bridge window %pR to %pR free space at %pR\n", - res, &bus->busn_res, &gap); - return true; - } - } + align1 <<= order + __ffs(SZ_1M); - if (bus->self) { - pci_info(bus->self, - "Assigned bridge window %pR to %pR cannot fit 0x%llx required for %s bridging to %pR\n", - res, &bus->busn_res, - (unsigned long long)size, - pci_name(downstream->self), - &downstream->busn_res); - } + /* + * Account smaller resources with alignment < max_order that + * could be used to fill head room if alignment less than + * max_order is used. + */ + remainder += aligns[order]; - return false; + /* + * Test if head fill is enough to satisfy the alignment of + * the larger resources after reducing the alignment. + */ + while ((head_align > align1) && (remainder >= head_align / 2)) { + head_align /= 2; + remainder -= head_align; + } } - return true; + return head_align; } /** @@ -1314,14 +1296,12 @@ static void pbus_size_mem(struct pci_bus *bus, unsigned long type, { struct pci_dev *dev; resource_size_t min_align, win_align, align, size, size0, size1 = 0; - resource_size_t aligns[28]; /* Alignments from 1MB to 128TB */ + resource_size_t aligns[28] = {}; /* Alignments from 1MB to 128TB */ int order, max_order; struct resource *b_res = pbus_select_window_for_type(bus, type); resource_size_t children_add_size = 0; resource_size_t children_add_align = 0; resource_size_t add_align = 0; - resource_size_t relaxed_align; - resource_size_t old_size; if (!b_res) return; @@ -1330,7 +1310,6 @@ static void pbus_size_mem(struct pci_bus *bus, unsigned long type, if (b_res->parent) return; - memset(aligns, 0, sizeof(aligns)); max_order = 0; size = 0; @@ -1375,12 +1354,8 @@ static void pbus_size_mem(struct pci_bus *bus, unsigned long type, continue; } size += max(r_size, align); - /* - * Exclude ranges with size > align from calculation of - * the alignment. - */ - if (r_size <= align) - aligns[order] += align; + + aligns[order] += align; if (order > max_order) max_order = order; @@ -1392,44 +1367,20 @@ static void pbus_size_mem(struct pci_bus *bus, unsigned long type, } } - old_size = resource_size(b_res); win_align = window_alignment(bus, b_res->flags); - min_align = calculate_mem_align(aligns, max_order); + min_align = calculate_head_align(aligns, max_order); min_align = max(min_align, win_align); - size0 = calculate_memsize(size, min_size, 0, 0, old_size, min_align); + size0 = calculate_memsize(size, min_size, 0, 0, win_align); if (size0) { resource_set_range(b_res, min_align, size0); b_res->flags &= ~IORESOURCE_DISABLED; } - if (bus->self && size0 && - !pbus_upstream_space_available(bus, b_res, size0, min_align)) { - relaxed_align = 1ULL << (max_order + __ffs(SZ_1M)); - relaxed_align = max(relaxed_align, win_align); - min_align = min(min_align, relaxed_align); - size0 = calculate_memsize(size, min_size, 0, 0, old_size, win_align); - resource_set_range(b_res, min_align, size0); - pci_info(bus->self, "bridge window %pR to %pR requires relaxed alignment rules\n", - b_res, &bus->busn_res); - } - if (realloc_head && (add_size > 0 || children_add_size > 0)) { add_align = max(min_align, add_align); size1 = calculate_memsize(size, min_size, add_size, children_add_size, - old_size, add_align); - - if (bus->self && size1 && - !pbus_upstream_space_available(bus, b_res, size1, add_align)) { - relaxed_align = 1ULL << (max_order + __ffs(SZ_1M)); - relaxed_align = max(relaxed_align, win_align); - min_align = min(min_align, relaxed_align); - size1 = calculate_memsize(size, min_size, add_size, children_add_size, - old_size, win_align); - pci_info(bus->self, - "bridge window %pR to %pR requires relaxed alignment rules\n", - b_res, &bus->busn_res); - } + win_align); } if (!size0 && !size1) { @@ -1442,12 +1393,13 @@ static void pbus_size_mem(struct pci_bus *bus, unsigned long type, resource_set_range(b_res, min_align, size0); b_res->flags |= IORESOURCE_STARTALIGN; - if (bus->self && size1 > size0 && realloc_head) { + if (bus->self && realloc_head && (size1 > size0 || add_align > min_align)) { b_res->flags &= ~IORESOURCE_DISABLED; - add_to_list(realloc_head, bus->self, b_res, size1-size0, add_align); + add_size = size1 > size0 ? size1 - size0 : 0; + add_to_list(realloc_head, bus->self, b_res, add_size, add_align); pci_info(bus->self, "bridge window %pR to %pR add_size %llx add_align %llx\n", b_res, &bus->busn_res, - (unsigned long long) (size1 - size0), + (unsigned long long) add_size, (unsigned long long) add_align); } } @@ -1733,6 +1685,8 @@ static void pci_claim_bridge_resources(struct pci_dev *dev) if (!r->flags || r->parent) continue; + if (r->flags & IORESOURCE_DISABLED) + continue; pci_claim_bridge_resource(dev, i); } diff --git a/drivers/perf/arm-cmn.c b/drivers/perf/arm-cmn.c index 23245352a3fc0a..4fbafc4b798436 100644 --- a/drivers/perf/arm-cmn.c +++ b/drivers/perf/arm-cmn.c @@ -210,6 +210,7 @@ enum cmn_model { enum cmn_part { PART_CMN600 = 0x434, PART_CMN650 = 0x436, + PART_CMN600AE = 0x438, PART_CMN700 = 0x43c, PART_CI700 = 0x43a, PART_CMN_S3 = 0x43e, @@ -2266,6 +2267,9 @@ static int arm_cmn_discover(struct arm_cmn *cmn, unsigned int rgn_offset) reg = readq_relaxed(cfg_region + CMN_CFGM_PERIPH_ID_01); part = FIELD_GET(CMN_CFGM_PID0_PART_0, reg); part |= FIELD_GET(CMN_CFGM_PID1_PART_1, reg) << 8; + /* 600AE is close enough that it's not really worth more complexity */ + if (part == PART_CMN600AE) + part = PART_CMN600; if (cmn->part && cmn->part != part) dev_warn(cmn->dev, "Firmware binding mismatch: expected part number 0x%x, found 0x%x\n", @@ -2418,6 +2422,15 @@ static int arm_cmn_discover(struct arm_cmn *cmn, unsigned int rgn_offset) arm_cmn_init_node_info(cmn, reg & CMN_CHILD_NODE_ADDR, dn); dn->portid_bits = xp->portid_bits; dn->deviceid_bits = xp->deviceid_bits; + /* + * Logical IDs are assigned from 0 per node type, so as + * soon as we see one bigger than expected, we can assume + * there are more than we can cope with. + */ + if (dn->logid > CMN_MAX_NODES_PER_EVENT) { + dev_err(cmn->dev, "Node ID invalid for supported CMN versions: %d\n", dn->logid); + return -ENODEV; + } switch (dn->type) { case CMN_TYPE_DTC: @@ -2467,7 +2480,7 @@ static int arm_cmn_discover(struct arm_cmn *cmn, unsigned int rgn_offset) break; /* Something has gone horribly wrong */ default: - dev_err(cmn->dev, "invalid device node type: 0x%x\n", dn->type); + dev_err(cmn->dev, "Device node type invalid for supported CMN versions: 0x%x\n", dn->type); return -ENODEV; } } @@ -2495,6 +2508,10 @@ static int arm_cmn_discover(struct arm_cmn *cmn, unsigned int rgn_offset) cmn->mesh_x = cmn->num_xps; cmn->mesh_y = cmn->num_xps / cmn->mesh_x; + if (max(cmn->mesh_x, cmn->mesh_y) > CMN_MAX_DIMENSION) { + dev_err(cmn->dev, "Mesh size invalid for supported CMN versions: %dx%d\n", cmn->mesh_x, cmn->mesh_y); + return -ENODEV; + } /* 1x1 config plays havoc with XP event encodings */ if (cmn->num_xps == 1) dev_warn(cmn->dev, "1x1 config not fully supported, translate XP events manually\n"); diff --git a/drivers/perf/arm_spe_pmu.c b/drivers/perf/arm_spe_pmu.c index 4801115f2b5405..5410fb7428d0e4 100644 --- a/drivers/perf/arm_spe_pmu.c +++ b/drivers/perf/arm_spe_pmu.c @@ -106,6 +106,8 @@ struct arm_spe_pmu { /* Keep track of our dynamic hotplug state */ static enum cpuhp_state arm_spe_pmu_online; +static void arm_spe_pmu_stop(struct perf_event *event, int flags); + enum arm_spe_pmu_buf_fault_action { SPE_PMU_BUF_FAULT_ACT_SPURIOUS, SPE_PMU_BUF_FAULT_ACT_FATAL, @@ -607,8 +609,8 @@ static u64 arm_spe_pmu_next_off(struct perf_output_handle *handle) return limit; } -static void arm_spe_perf_aux_output_begin(struct perf_output_handle *handle, - struct perf_event *event) +static int arm_spe_perf_aux_output_begin(struct perf_output_handle *handle, + struct perf_event *event) { u64 base, limit; struct arm_spe_pmu_buf *buf; @@ -622,7 +624,6 @@ static void arm_spe_perf_aux_output_begin(struct perf_output_handle *handle, /* Start a new aux session */ buf = perf_aux_output_begin(handle, event); if (!buf) { - event->hw.state |= PERF_HES_STOPPED; /* * We still need to clear the limit pointer, since the * profiler might only be disabled by virtue of a fault. @@ -642,6 +643,7 @@ static void arm_spe_perf_aux_output_begin(struct perf_output_handle *handle, out_write_limit: write_sysreg_s(limit, SYS_PMBLIMITR_EL1); + return (limit & PMBLIMITR_EL1_E) ? 0 : -EIO; } static void arm_spe_perf_aux_output_end(struct perf_output_handle *handle) @@ -781,7 +783,10 @@ static irqreturn_t arm_spe_pmu_irq_handler(int irq, void *dev) * when we get to it. */ if (!(handle->aux_flags & PERF_AUX_FLAG_TRUNCATED)) { - arm_spe_perf_aux_output_begin(handle, event); + if (arm_spe_perf_aux_output_begin(handle, event)) { + arm_spe_pmu_stop(event, PERF_EF_UPDATE); + break; + } isb(); } break; @@ -880,9 +885,10 @@ static void arm_spe_pmu_start(struct perf_event *event, int flags) struct perf_output_handle *handle = this_cpu_ptr(spe_pmu->handle); hwc->state = 0; - arm_spe_perf_aux_output_begin(handle, event); - if (hwc->state) + if (arm_spe_perf_aux_output_begin(handle, event)) { + arm_spe_pmu_stop(event, 0); return; + } reg = arm_spe_event_to_pmsfcr(event); write_sysreg_s(reg, SYS_PMSFCR_EL1); diff --git a/drivers/perf/cxl_pmu.c b/drivers/perf/cxl_pmu.c index d094030220bf25..68a54d97d2a8af 100644 --- a/drivers/perf/cxl_pmu.c +++ b/drivers/perf/cxl_pmu.c @@ -877,7 +877,7 @@ static int cxl_pmu_probe(struct device *dev) if (!irq_name) return -ENOMEM; - rc = devm_request_irq(dev, irq, cxl_pmu_irq, IRQF_SHARED | IRQF_ONESHOT, + rc = devm_request_irq(dev, irq, cxl_pmu_irq, IRQF_SHARED | IRQF_NO_THREAD, irq_name, info); if (rc) return rc; diff --git a/drivers/phy/Kconfig b/drivers/phy/Kconfig index 678dd0452f0aa0..2fb5e3b313effc 100644 --- a/drivers/phy/Kconfig +++ b/drivers/phy/Kconfig @@ -103,6 +103,7 @@ config PHY_NXP_PTN3222 source "drivers/phy/allwinner/Kconfig" source "drivers/phy/amlogic/Kconfig" +source "drivers/phy/apple/Kconfig" source "drivers/phy/broadcom/Kconfig" source "drivers/phy/cadence/Kconfig" source "drivers/phy/freescale/Kconfig" diff --git a/drivers/phy/Makefile b/drivers/phy/Makefile index bfb27fb5a49428..eb84c3328d1062 100644 --- a/drivers/phy/Makefile +++ b/drivers/phy/Makefile @@ -15,6 +15,7 @@ obj-$(CONFIG_PHY_AIROHA_PCIE) += phy-airoha-pcie.o obj-$(CONFIG_PHY_NXP_PTN3222) += phy-nxp-ptn3222.o obj-y += allwinner/ \ amlogic/ \ + apple/ \ broadcom/ \ cadence/ \ freescale/ \ diff --git a/drivers/phy/apple/Kconfig b/drivers/phy/apple/Kconfig new file mode 100644 index 00000000000000..8409b67f6d1ecb --- /dev/null +++ b/drivers/phy/apple/Kconfig @@ -0,0 +1,23 @@ +# SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause +config PHY_APPLE_ATC + tristate "Apple Type-C PHY" + depends on (ARM64 && ARCH_APPLE) || (COMPILE_TEST && !GENERIC_ATOMIC64) + depends on TYPEC + select GENERIC_PHY + select APPLE_TUNABLE + help + Enable this to add support for the Apple Type-C PHY found in + Apple Silicon M-series SoCs. This PHY supports USB2, + USB3, USB4, Thunderbolt, and DisplayPort. + + If M is selected the module will be called 'phy-apple-atc'. + +config PHY_APPLE_DPTX + tristate "Apple DPTX PHY" + depends on ARCH_APPLE || COMPILE_TEST + select GENERIC_PHY + help + Enable this to add support for the Apple DPTX PHY found on Apple SoCs + such as the M2. + This driver provides support for DisplayPort and is used on the + Mac mini (M2 and M2 Pro, 2023). diff --git a/drivers/phy/apple/Makefile b/drivers/phy/apple/Makefile new file mode 100644 index 00000000000000..b9e7bf3e4ac170 --- /dev/null +++ b/drivers/phy/apple/Makefile @@ -0,0 +1,7 @@ +# SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause + +obj-$(CONFIG_PHY_APPLE_ATC) += phy-apple-atc.o +phy-apple-atc-y := atc.o + +obj-$(CONFIG_PHY_APPLE_DPTX) += phy-apple-dptx.o +phy-apple-dptx-y += dptx.o diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c new file mode 100644 index 00000000000000..64d0c3dba1cbb9 --- /dev/null +++ b/drivers/phy/apple/atc.c @@ -0,0 +1,2297 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause +/* + * Apple Type-C PHY driver + * + * The Apple Type-C PHY (ATCPHY) is a combined PHY for USB 2.0, USB 3.x, + * USB4/Thunderbolt, and DisplayPort connectivity via Type-C ports found in + * Apple Silicon SoCs. + * + * The PHY handles muxing between these different protocols and also provides the + * reset controller for the attached DWC3 USB controller. + * + * No documentation for this PHY is available and its operation has been + * reverse engineered by observing the XNU's MMIO access using a thin hypervisor + * and correlating register access to XNU's very verbose debug output. Most + * register names comes from this debug output as well. + * + * In order to correctly setup the high speed lanes for the various modes + * calibration values copied from Apple's firmware by our bootloader m1n1 are + * required. Without these only USB2 operation is possible. + * + * Copyright (C) The Asahi Linux Contributors + * Author: Sven Peter + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define AUSPLL_FSM_CTRL 0x1014 + +#define AUSPLL_APB_CMD_OVERRIDE 0x2000 +#define AUSPLL_APB_CMD_OVERRIDE_REQ BIT(0) +#define AUSPLL_APB_CMD_OVERRIDE_ACK BIT(1) +#define AUSPLL_APB_CMD_OVERRIDE_UNK28 BIT(28) +#define AUSPLL_APB_CMD_OVERRIDE_CMD GENMASK(27, 3) + +#define AUSPLL_FREQ_DESC_A 0x2080 +#define AUSPLL_FD_FREQ_COUNT_TARGET GENMASK(9, 0) +#define AUSPLL_FD_FBDIVN_HALF BIT(10) +#define AUSPLL_FD_REV_DIVN GENMASK(13, 11) +#define AUSPLL_FD_KI_MAN GENMASK(17, 14) +#define AUSPLL_FD_KI_EXP GENMASK(21, 18) +#define AUSPLL_FD_KP_MAN GENMASK(25, 22) +#define AUSPLL_FD_KP_EXP GENMASK(29, 26) +#define AUSPLL_FD_KPKI_SCALE_HBW GENMASK(31, 30) + +#define AUSPLL_FREQ_DESC_B 0x2084 +#define AUSPLL_FD_FBDIVN_FRAC_DEN GENMASK(13, 0) +#define AUSPLL_FD_FBDIVN_FRAC_NUM GENMASK(27, 14) + +#define AUSPLL_FREQ_DESC_C 0x2088 +#define AUSPLL_FD_SDM_SSC_STEP GENMASK(7, 0) +#define AUSPLL_FD_SDM_SSC_EN BIT(8) +#define AUSPLL_FD_PCLK_DIV_SEL GENMASK(13, 9) +#define AUSPLL_FD_LFSDM_DIV GENMASK(15, 14) +#define AUSPLL_FD_LFCLK_CTRL GENMASK(19, 16) +#define AUSPLL_FD_VCLK_OP_DIVN GENMASK(21, 20) +#define AUSPLL_FD_VCLK_PRE_DIVN BIT(22) + +#define AUSPLL_DCO_EFUSE_SPARE 0x222c +#define AUSPLL_RODCO_ENCAP_EFUSE GENMASK(10, 9) +#define AUSPLL_RODCO_BIAS_ADJUST_EFUSE GENMASK(14, 12) + +#define AUSPLL_FRACN_CAN 0x22a4 +#define AUSPLL_DLL_START_CAPCODE GENMASK(18, 17) + +#define AUSPLL_CLKOUT_MASTER 0x2200 +#define AUSPLL_CLKOUT_MASTER_PCLK_DRVR_EN BIT(2) +#define AUSPLL_CLKOUT_MASTER_PCLK2_DRVR_EN BIT(4) +#define AUSPLL_CLKOUT_MASTER_REFBUFCLK_DRVR_EN BIT(6) + +#define AUSPLL_CLKOUT_DIV 0x2208 +#define AUSPLL_CLKOUT_PLLA_REFBUFCLK_DI GENMASK(20, 16) + +#define AUSPLL_BGR 0x2214 +#define AUSPLL_BGR_CTRL_AVAIL BIT(0) + +#define AUSPLL_CLKOUT_DTC_VREG 0x2220 +#define AUSPLL_DTC_VREG_ADJUST GENMASK(16, 14) +#define AUSPLL_DTC_VREG_BYPASS BIT(7) + +#define AUSPLL_FREQ_CFG 0x2224 +#define AUSPLL_FREQ_REFCLK GENMASK(1, 0) + +#define AUS_COMMON_SHIM_BLK_VREG 0x0a04 +#define AUS_VREG_TRIM GENMASK(6, 2) + +#define AUS_UNK_A20 0x0a20 +#define AUS_UNK_A20_TX_CAL_CODE GENMASK(23, 20) + +#define ACIOPHY_CMN_SHM_STS_REG0 0x0a74 +#define ACIOPHY_CMN_SHM_STS_REG0_CMD_READY BIT(0) + +#define CIO3PLL_CLK_CTRL 0x2a00 +#define CIO3PLL_CLK_PCLK_EN BIT(1) +#define CIO3PLL_CLK_REFCLK_EN BIT(5) + +#define CIO3PLL_DCO_NCTRL 0x2a38 +#define CIO3PLL_DCO_COARSEBIN_EFUSE0 GENMASK(6, 0) +#define CIO3PLL_DCO_COARSEBIN_EFUSE1 GENMASK(23, 17) + +#define CIO3PLL_FRACN_CAN 0x2aa4 +#define CIO3PLL_DLL_CAL_START_CAPCODE GENMASK(18, 17) + +#define CIO3PLL_DTC_VREG 0x2a20 +#define CIO3PLL_DTC_VREG_ADJUST GENMASK(16, 14) + +#define ACIOPHY_CFG0 0x08 +#define ACIOPHY_CFG0_COMMON_BIG_OV BIT(1) +#define ACIOPHY_CFG0_COMMON_SMALL_OV BIT(3) +#define ACIOPHY_CFG0_COMMON_CLAMP_OV BIT(5) +#define ACIOPHY_CFG0_RX_SMALL_OV GENMASK(9, 8) +#define ACIOPHY_CFG0_RX_BIG_OV GENMASK(13, 12) +#define ACIOPHY_CFG0_RX_CLAMP_OV GENMASK(17, 16) + +#define ACIOPHY_CROSSBAR 0x4c +#define ACIOPHY_CROSSBAR_PROTOCOL GENMASK(4, 0) +#define ACIOPHY_CROSSBAR_PROTOCOL_USB4 0x0 +#define ACIOPHY_CROSSBAR_PROTOCOL_USB4_SWAPPED 0x1 +#define ACIOPHY_CROSSBAR_PROTOCOL_USB3 0xa +#define ACIOPHY_CROSSBAR_PROTOCOL_USB3_SWAPPED 0xb +#define ACIOPHY_CROSSBAR_PROTOCOL_USB3_DP 0x10 +#define ACIOPHY_CROSSBAR_PROTOCOL_USB3_DP_SWAPPED 0x11 +#define ACIOPHY_CROSSBAR_PROTOCOL_DP 0x14 +#define ACIOPHY_CROSSBAR_DP_SINGLE_PMA GENMASK(16, 5) +#define ACIOPHY_CROSSBAR_DP_SINGLE_PMA_NONE 0x0000 +#define ACIOPHY_CROSSBAR_DP_SINGLE_PMA_UNK100 0x100 +#define ACIOPHY_CROSSBAR_DP_SINGLE_PMA_UNK008 0x008 +#define ACIOPHY_CROSSBAR_DP_BOTH_PMA BIT(17) + +#define ACIOPHY_LANE_MODE 0x48 +#define ACIOPHY_LANE_MODE_RX0 GENMASK(2, 0) +#define ACIOPHY_LANE_MODE_TX0 GENMASK(5, 3) +#define ACIOPHY_LANE_MODE_RX1 GENMASK(8, 6) +#define ACIOPHY_LANE_MODE_TX1 GENMASK(11, 9) + +enum atcphy_lane_mode { + ACIOPHY_LANE_MODE_USB4 = 0, + ACIOPHY_LANE_MODE_USB3 = 1, + ACIOPHY_LANE_MODE_DP = 2, + ACIOPHY_LANE_MODE_OFF = 3, +}; + +#define ACIOPHY_TOP_BIST_CIOPHY_CFG1 0x84 +#define ACIOPHY_TOP_BIST_CIOPHY_CFG1_CLK_EN BIT(27) +#define ACIOPHY_TOP_BIST_CIOPHY_CFG1_BIST_EN BIT(28) + +#define ACIOPHY_TOP_BIST_OV_CFG 0x8c +#define ACIOPHY_TOP_BIST_OV_CFG_LN0_RESET_N_OV BIT(13) +#define ACIOPHY_TOP_BIST_OV_CFG_LN0_PWR_DOWN_OV BIT(25) + +#define ACIOPHY_TOP_BIST_READ_CTRL 0x90 +#define ACIOPHY_TOP_BIST_READ_CTRL_LN0_PHY_STATUS_RE BIT(2) + +#define ACIOPHY_TOP_PHY_STAT 0x9c +#define ACIOPHY_TOP_PHY_STAT_LN0_UNK0 BIT(0) +#define ACIOPHY_TOP_PHY_STAT_LN0_UNK23 BIT(23) + +#define ACIOPHY_TOP_BIST_PHY_CFG0 0xa8 +#define ACIOPHY_TOP_BIST_PHY_CFG0_LN0_RESET_N BIT(0) + +#define ACIOPHY_TOP_BIST_PHY_CFG1 0xac +#define ACIOPHY_TOP_BIST_PHY_CFG1_LN0_PWR_DOWN GENMASK(13, 10) + +#define ACIOPHY_SLEEP_CTRL 0x1b0 +#define ACIOPHY_SLEEP_CTRL_TX_BIG_OV GENMASK(3, 2) +#define ACIOPHY_SLEEP_CTRL_TX_SMALL_OV GENMASK(7, 6) +#define ACIOPHY_SLEEP_CTRL_TX_CLAMP_OV GENMASK(11, 10) + +#define ACIOPHY_PLL_PCTL_FSM_CTRL1 0x1014 +#define ACIOPHY_PLL_APB_REQ_OV_SEL GENMASK(21, 13) +#define ACIOPHY_PLL_COMMON_CTRL 0x1028 +#define ACIOPHY_PLL_WAIT_FOR_CMN_READY_BEFORE_RESET_EXIT BIT(24) + +#define ATCPHY_POWER_CTRL 0x20000 +#define ATCPHY_POWER_STAT 0x20004 +#define ATCPHY_POWER_SLEEP_SMALL BIT(0) +#define ATCPHY_POWER_SLEEP_BIG BIT(1) +#define ATCPHY_POWER_CLAMP_EN BIT(2) +#define ATCPHY_POWER_APB_RESET_N BIT(3) +#define ATCPHY_POWER_PHY_RESET_N BIT(4) + +#define ATCPHY_MISC 0x20008 +#define ATCPHY_MISC_RESET_N BIT(0) +#define ATCPHY_MISC_LANE_SWAP BIT(2) + +#define ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0 0x7000 +#define DP_PMA_BYTECLK_RESET BIT(0) +#define DP_MAC_DIV20_CLK_SEL BIT(1) +#define DPTXPHY_PMA_LANE_RESET_N BIT(2) +#define DPTXPHY_PMA_LANE_RESET_N_OV BIT(3) +#define DPTX_PCLK1_SELECT GENMASK(6, 4) +#define DPTX_PCLK2_SELECT GENMASK(9, 7) +#define DPRX_PCLK_SELECT GENMASK(12, 10) +#define DPTX_PCLK1_ENABLE BIT(13) +#define DPTX_PCLK2_ENABLE BIT(14) +#define DPRX_PCLK_ENABLE BIT(15) + +#define ACIOPHY_DP_PCLK_STAT 0x7044 +#define ACIOPHY_AUSPLL_LOCK BIT(3) + +#define LN0_AUSPMA_RX_TOP 0x9000 +#define LN0_AUSPMA_RX_EQ 0xA000 +#define LN0_AUSPMA_RX_SHM 0xB000 +#define LN0_AUSPMA_TX_TOP 0xC000 +#define LN0_AUSPMA_TX_SHM 0xD000 + +#define LN1_AUSPMA_RX_TOP 0x10000 +#define LN1_AUSPMA_RX_EQ 0x11000 +#define LN1_AUSPMA_RX_SHM 0x12000 +#define LN1_AUSPMA_TX_TOP 0x13000 +#define LN1_AUSPMA_TX_SHM 0x14000 + +#define LN_AUSPMA_RX_TOP_PMAFSM 0x0010 +#define LN_AUSPMA_RX_TOP_PMAFSM_PCS_OV BIT(0) +#define LN_AUSPMA_RX_TOP_PMAFSM_PCS_REQ BIT(9) + +#define LN_AUSPMA_RX_TOP_TJ_CFG_RX_TXMODE 0x00F0 +#define LN_RX_TXMODE BIT(0) + +#define LN_AUSPMA_RX_SHM_TJ_RXA_CTLE_CTRL0 0x00 +#define LN_TX_CLK_EN BIT(20) +#define LN_TX_CLK_EN_OV BIT(21) + +#define LN_AUSPMA_RX_SHM_TJ_RXA_AFE_CTRL1 0x04 +#define LN_RX_DIV20_RESET_N_OV BIT(29) +#define LN_RX_DIV20_RESET_N BIT(30) + +#define LN_AUSPMA_RX_SHM_TJ_RXA_UNK_CTRL2 0x08 +#define LN_AUSPMA_RX_SHM_TJ_RXA_UNK_CTRL3 0x0C +#define LN_AUSPMA_RX_SHM_TJ_RXA_UNK_CTRL4 0x10 +#define LN_AUSPMA_RX_SHM_TJ_RXA_UNK_CTRL5 0x14 +#define LN_AUSPMA_RX_SHM_TJ_RXA_UNK_CTRL6 0x18 +#define LN_AUSPMA_RX_SHM_TJ_RXA_UNK_CTRL7 0x1C +#define LN_AUSPMA_RX_SHM_TJ_RXA_UNK_CTRL8 0x20 +#define LN_AUSPMA_RX_SHM_TJ_RXA_UNK_CTRL9 0x24 +#define LN_AUSPMA_RX_SHM_TJ_RXA_DFE_CTRL10 0x28 +#define LN_DTVREG_ADJUST GENMASK(31, 27) + +#define LN_AUSPMA_RX_SHM_TJ_RXA_DFE_CTRL11 0x2C +#define LN_DTVREG_BIG_EN BIT(23) +#define LN_DTVREG_BIG_EN_OV BIT(24) +#define LN_DTVREG_SML_EN BIT(25) +#define LN_DTVREG_SML_EN_OV BIT(26) + +#define LN_AUSPMA_RX_SHM_TJ_RXA_DFE_CTRL12 0x30 +#define LN_TX_BYTECLK_RESET_SYNC_CLR BIT(22) +#define LN_TX_BYTECLK_RESET_SYNC_CLR_OV BIT(23) +#define LN_TX_BYTECLK_RESET_SYNC_EN BIT(24) +#define LN_TX_BYTECLK_RESET_SYNC_EN_OV BIT(25) +#define LN_TX_HRCLK_SEL BIT(28) +#define LN_TX_HRCLK_SEL_OV BIT(29) +#define LN_TX_PBIAS_EN BIT(30) +#define LN_TX_PBIAS_EN_OV BIT(31) + +#define LN_AUSPMA_RX_SHM_TJ_RXA_DFE_CTRL13 0x34 +#define LN_TX_PRE_EN BIT(0) +#define LN_TX_PRE_EN_OV BIT(1) +#define LN_TX_PST1_EN BIT(2) +#define LN_TX_PST1_EN_OV BIT(3) +#define LN_DTVREG_ADJUST_OV BIT(15) + +#define LN_AUSPMA_RX_SHM_TJ_UNK_CTRL14A 0x38 +#define LN_AUSPMA_RX_SHM_TJ_UNK_CTRL14B 0x3C +#define LN_AUSPMA_RX_SHM_TJ_UNK_CTRL15A 0x40 +#define LN_AUSPMA_RX_SHM_TJ_UNK_CTRL15B 0x44 +#define LN_AUSPMA_RX_SHM_TJ_RXA_SAVOS_CTRL16 0x48 +#define LN_RXTERM_EN BIT(21) +#define LN_RXTERM_EN_OV BIT(22) +#define LN_RXTERM_PULLUP_LEAK_EN BIT(23) +#define LN_RXTERM_PULLUP_LEAK_EN_OV BIT(24) +#define LN_TX_CAL_CODE GENMASK(29, 25) +#define LN_TX_CAL_CODE_OV BIT(30) + +#define LN_AUSPMA_RX_SHM_TJ_RXA_TX_CTRL17 0x4C +#define LN_TX_MARGIN GENMASK(19, 15) +#define LN_TX_MARGIN_OV BIT(20) +#define LN_TX_MARGIN_LSB BIT(21) +#define LN_TX_MARGIN_LSB_OV BIT(22) +#define LN_TX_MARGIN_P1 GENMASK(26, 23) +#define LN_TX_MARGIN_P1_OV BIT(27) +#define LN_TX_MARGIN_P1_LSB GENMASK(29, 28) +#define LN_TX_MARGIN_P1_LSB_OV BIT(30) + +#define LN_AUSPMA_RX_SHM_TJ_RXA_TX_CTRL18 0x50 +#define LN_TX_P1_CODE GENMASK(3, 0) +#define LN_TX_P1_CODE_OV BIT(4) +#define LN_TX_P1_LSB_CODE GENMASK(6, 5) +#define LN_TX_P1_LSB_CODE_OV BIT(7) +#define LN_TX_MARGIN_PRE GENMASK(10, 8) +#define LN_TX_MARGIN_PRE_OV BIT(11) +#define LN_TX_MARGIN_PRE_LSB GENMASK(13, 12) +#define LN_TX_MARGIN_PRE_LSB_OV BIT(14) +#define LN_TX_PRE_LSB_CODE GENMASK(16, 15) +#define LN_TX_PRE_LSB_CODE_OV BIT(17) +#define LN_TX_PRE_CODE GENMASK(21, 18) +#define LN_TX_PRE_CODE_OV BIT(22) + +#define LN_AUSPMA_RX_SHM_TJ_RXA_TERM_CTRL19 0x54 +#define LN_TX_TEST_EN BIT(21) +#define LN_TX_TEST_EN_OV BIT(22) +#define LN_TX_EN BIT(23) +#define LN_TX_EN_OV BIT(24) +#define LN_TX_CLK_DLY_CTRL_TAPGEN GENMASK(27, 25) +#define LN_TX_CLK_DIV2_EN BIT(28) +#define LN_TX_CLK_DIV2_EN_OV BIT(29) +#define LN_TX_CLK_DIV2_RST BIT(30) +#define LN_TX_CLK_DIV2_RST_OV BIT(31) + +#define LN_AUSPMA_RX_SHM_TJ_RXA_UNK_CTRL20 0x58 +#define LN_AUSPMA_RX_SHM_TJ_RXA_UNK_CTRL21 0x5C +#define LN_AUSPMA_RX_SHM_TJ_RXA_VREF_CTRL22 0x60 +#define LN_VREF_ADJUST_GRAY GENMASK(11, 7) +#define LN_VREF_ADJUST_GRAY_OV BIT(12) +#define LN_VREF_BIAS_SEL GENMASK(14, 13) +#define LN_VREF_BIAS_SEL_OV BIT(15) +#define LN_VREF_BOOST_EN BIT(16) +#define LN_VREF_BOOST_EN_OV BIT(17) +#define LN_VREF_EN BIT(18) +#define LN_VREF_EN_OV BIT(19) +#define LN_VREF_LPBKIN_DATA GENMASK(29, 28) +#define LN_VREF_TEST_RXLPBKDT_EN BIT(30) +#define LN_VREF_TEST_RXLPBKDT_EN_OV BIT(31) + +#define LN_AUSPMA_TX_SHM_TXA_CFG_MAIN_REG0 0x00 +#define LN_BYTECLK_RESET_SYNC_EN_OV BIT(2) +#define LN_BYTECLK_RESET_SYNC_EN BIT(3) +#define LN_BYTECLK_RESET_SYNC_CLR_OV BIT(4) +#define LN_BYTECLK_RESET_SYNC_CLR BIT(5) +#define LN_BYTECLK_RESET_SYNC_SEL_OV BIT(6) + +#define LN_AUSPMA_TX_SHM_TXA_CFG_MAIN_REG1 0x04 +#define LN_TXA_DIV2_EN_OV BIT(8) +#define LN_TXA_DIV2_EN BIT(9) +#define LN_TXA_DIV2_RESET_OV BIT(10) +#define LN_TXA_DIV2_RESET BIT(11) +#define LN_TXA_CLK_EN_OV BIT(22) +#define LN_TXA_CLK_EN BIT(23) + +#define LN_AUSPMA_TX_SHM_TXA_IMP_REG0 0x08 +#define LN_TXA_CAL_CTRL_OV BIT(0) +#define LN_TXA_CAL_CTRL GENMASK(18, 1) +#define LN_TXA_CAL_CTRL_BASE_OV BIT(19) +#define LN_TXA_CAL_CTRL_BASE GENMASK(23, 20) +#define LN_TXA_HIZ_OV BIT(29) +#define LN_TXA_HIZ BIT(30) + +#define LN_AUSPMA_TX_SHM_TXA_IMP_REG1 0x0C +#define LN_AUSPMA_TX_SHM_TXA_IMP_REG2 0x10 +#define LN_TXA_MARGIN_OV BIT(0) +#define LN_TXA_MARGIN GENMASK(18, 1) +#define LN_TXA_MARGIN_2R_OV BIT(19) +#define LN_TXA_MARGIN_2R BIT(20) + +#define LN_AUSPMA_TX_SHM_TXA_IMP_REG3 0x14 +#define LN_TXA_MARGIN_POST_OV BIT(0) +#define LN_TXA_MARGIN_POST GENMASK(10, 1) +#define LN_TXA_MARGIN_POST_2R_OV BIT(11) +#define LN_TXA_MARGIN_POST_2R BIT(12) +#define LN_TXA_MARGIN_POST_4R_OV BIT(13) +#define LN_TXA_MARGIN_POST_4R BIT(14) +#define LN_TXA_MARGIN_PRE_OV BIT(15) +#define LN_TXA_MARGIN_PRE GENMASK(21, 16) +#define LN_TXA_MARGIN_PRE_2R_OV BIT(22) +#define LN_TXA_MARGIN_PRE_2R BIT(23) +#define LN_TXA_MARGIN_PRE_4R_OV BIT(24) +#define LN_TXA_MARGIN_PRE_4R BIT(25) + +#define LN_AUSPMA_TX_SHM_TXA_UNK_REG0 0x18 +#define LN_AUSPMA_TX_SHM_TXA_UNK_REG1 0x1C +#define LN_AUSPMA_TX_SHM_TXA_UNK_REG2 0x20 + +#define LN_AUSPMA_TX_SHM_TXA_LDOCLK 0x24 +#define LN_LDOCLK_BYPASS_SML_OV BIT(8) +#define LN_LDOCLK_BYPASS_SML BIT(9) +#define LN_LDOCLK_BYPASS_BIG_OV BIT(10) +#define LN_LDOCLK_BYPASS_BIG BIT(11) +#define LN_LDOCLK_EN_SML_OV BIT(12) +#define LN_LDOCLK_EN_SML BIT(13) +#define LN_LDOCLK_EN_BIG_OV BIT(14) +#define LN_LDOCLK_EN_BIG BIT(15) + +/* LPDPTX registers */ +#define LPDPTX_AUX_CFG_BLK_AUX_CTRL 0x0000 +#define LPDPTX_BLK_AUX_CTRL_PWRDN BIT(4) +#define LPDPTX_BLK_AUX_RXOFFSET GENMASK(25, 22) + +#define LPDPTX_AUX_CFG_BLK_AUX_LDO_CTRL 0x0008 + +#define LPDPTX_AUX_CFG_BLK_AUX_MARGIN 0x000c +#define LPDPTX_MARGIN_RCAL_RXOFFSET_EN BIT(5) +#define LPDPTX_AUX_MARGIN_RCAL_TXSWING GENMASK(10, 6) + +#define LPDPTX_AUX_SHM_CFG_BLK_AUX_CTRL_REG0 0x0204 +#define LPDPTX_CFG_PMA_AUX_SEL_LF_DATA BIT(15) + +#define LPDPTX_AUX_SHM_CFG_BLK_AUX_CTRL_REG1 0x0208 +#define LPDPTX_CFG_PMA_PHYS_ADJ GENMASK(22, 20) +#define LPDPTX_CFG_PMA_PHYS_ADJ_OV BIT(19) + +#define LPDPTX_AUX_CONTROL 0x4000 +#define LPDPTX_AUX_PWN_DOWN 0x10 +#define LPDPTX_AUX_CLAMP_EN 0x04 +#define LPDPTX_SLEEP_B_BIG_IN 0x02 +#define LPDPTX_SLEEP_B_SML_IN 0x01 +#define LPDPTX_TXTERM_CODEMSB 0x400 +#define LPDPTX_TXTERM_CODE GENMASK(9, 5) + +/* pipehandler registers */ +#define PIPEHANDLER_OVERRIDE 0x00 +#define PIPEHANDLER_OVERRIDE_RXVALID BIT(0) +#define PIPEHANDLER_OVERRIDE_RXDETECT BIT(2) + +#define PIPEHANDLER_OVERRIDE_VALUES 0x04 +#define PIPEHANDLER_OVERRIDE_VAL_RXDETECT0 BIT(1) +#define PIPEHANDLER_OVERRIDE_VAL_RXDETECT1 BIT(2) +#define PIPEHANDLER_OVERRIDE_VAL_PHY_STATUS BIT(4) + +#define PIPEHANDLER_MUX_CTRL 0x0c +#define PIPEHANDLER_MUX_CTRL_CLK GENMASK(5, 3) +#define PIPEHANDLER_MUX_CTRL_DATA GENMASK(2, 0) +#define PIPEHANDLER_MUX_CTRL_CLK_OFF 0 +#define PIPEHANDLER_MUX_CTRL_CLK_USB3 1 +#define PIPEHANDLER_MUX_CTRL_CLK_USB4 2 +#define PIPEHANDLER_MUX_CTRL_CLK_DUMMY 4 + +#define PIPEHANDLER_MUX_CTRL_DATA_USB3 0 +#define PIPEHANDLER_MUX_CTRL_DATA_USB4 1 +#define PIPEHANDLER_MUX_CTRL_DATA_DUMMY 2 + +#define PIPEHANDLER_LOCK_REQ 0x10 +#define PIPEHANDLER_LOCK_ACK 0x14 +#define PIPEHANDLER_LOCK_EN BIT(0) + +#define PIPEHANDLER_AON_GEN 0x1C +#define PIPEHANDLER_AON_GEN_DWC3_FORCE_CLAMP_EN BIT(4) +#define PIPEHANDLER_AON_GEN_DWC3_RESET_N BIT(0) + +#define PIPEHANDLER_NONSELECTED_OVERRIDE 0x20 +#define PIPEHANDLER_NATIVE_RESET BIT(12) +#define PIPEHANDLER_DUMMY_PHY_EN BIT(15) +#define PIPEHANDLER_NATIVE_POWER_DOWN GENMASK(3, 0) + +#define PIPEHANDLER_LOCK_ACK_TIMEOUT_US 1000 + +/* USB2 PHY regs */ +#define USB2PHY_USBCTL 0x00 +#define USB2PHY_USBCTL_RUN 2 +#define USB2PHY_USBCTL_ISOLATION 4 + +#define USB2PHY_CTL 0x04 +#define USB2PHY_CTL_RESET BIT(0) +#define USB2PHY_CTL_PORT_RESET BIT(1) +#define USB2PHY_CTL_APB_RESET_N BIT(2) +#define USB2PHY_CTL_SIDDQ BIT(3) + +#define USB2PHY_SIG 0x08 +#define USB2PHY_SIG_VBUSDET_FORCE_VAL BIT(0) +#define USB2PHY_SIG_VBUSDET_FORCE_EN BIT(1) +#define USB2PHY_SIG_VBUSVLDEXT_FORCE_VAL BIT(2) +#define USB2PHY_SIG_VBUSVLDEXT_FORCE_EN BIT(3) +#define USB2PHY_SIG_HOST (7 << 12) + +#define USB2PHY_MISCTUNE 0x1c +#define USB2PHY_MISCTUNE_APBCLK_GATE_OFF BIT(29) +#define USB2PHY_MISCTUNE_REFCLK_GATE_OFF BIT(30) + +enum atcphy_dp_link_rate { + ATCPHY_DP_LINK_RATE_RBR, + ATCPHY_DP_LINK_RATE_HBR, + ATCPHY_DP_LINK_RATE_HBR2, + ATCPHY_DP_LINK_RATE_HBR3, +}; + +/** + * enum atcphy_pipehandler_state - States of the PIPE mux interface ("pipehandler") + * @ATCPHY_PIPEHANDLER_STATE_DUMMY: "Dummy PHY" (disables USB3, USB2 only) + * @ATCPHY_PIPEHANDLER_STATE_USB3: USB3 directly connected to the Type-C port + * @ATCPHY_PIPEHANDLER_STATE_USB4: USB3 tunneled via USB4/Thunderbolt + * + * DWC3's USB3 PIPE interface is connected to a multiplexer inside this PHY + * which can switch between a dummy state (which effectively disables any USB3 + * support and falls back to USB2 only operation via the separate ULPI interface), + * a USB3 state (for regular USB3 or USB3+DisplayPort operation) and a USB4 state + * (for USB3 tunneled via USB4/Thunderbolt). + */ +enum atcphy_pipehandler_state { + ATCPHY_PIPEHANDLER_STATE_DUMMY, + ATCPHY_PIPEHANDLER_STATE_USB3, + ATCPHY_PIPEHANDLER_STATE_USB4, +}; + +/** + * enum atcphy_mode - Operating modes of the PHY + * @APPLE_ATCPHY_MODE_OFF: all PHYs powered off + * @APPLE_ATCPHY_MODE_USB2: Nothing on the four SS lanes (i.e. USB2 only on D-/+) + * @APPLE_ATCPHY_MODE_USB3: USB3 on two lanes, nothing on the other two + * @APPLE_ATCPHY_MODE_USB3_DP: USB3 on two lanes and DisplayPort on the other two + * @APPLE_ATCPHY_MODE_TBT: Thunderbolt on all lanes + * @APPLE_ATCPHY_MODE_USB4: USB4 on all lanes + * @APPLE_ATCPHY_MODE_DP: DisplayPort on all lanes + */ +enum atcphy_mode { + APPLE_ATCPHY_MODE_OFF, + APPLE_ATCPHY_MODE_USB2, + APPLE_ATCPHY_MODE_USB3, + APPLE_ATCPHY_MODE_USB3_DP, + APPLE_ATCPHY_MODE_TBT, + APPLE_ATCPHY_MODE_USB4, + APPLE_ATCPHY_MODE_DP, +}; + +enum atcphy_lane { + APPLE_ATCPHY_LANE_0, + APPLE_ATCPHY_LANE_1, +}; + +/* Link rate configuration, field names are taken from XNU debug output or register names */ +struct atcphy_dp_link_rate_configuration { + u16 freqinit_count_target; + u16 fbdivn_frac_den; + u16 fbdivn_frac_num; + u16 pclk_div_sel; + u8 lfclk_ctrl; + u8 vclk_op_divn; + bool plla_clkout_vreg_bypass; + bool txa_ldoclk_bypass; + bool txa_div2_en; +}; + +/* Crossbar and lane configuration */ +struct atcphy_mode_configuration { + u32 crossbar; + u32 crossbar_dp_single_pma; + bool crossbar_dp_both_pma; + enum atcphy_lane_mode lane_mode[2]; + bool dp_lane[2]; + bool set_swap; +}; + +/** + * struct apple_atcphy - Apple Type-C PHY device struct + * @np: Device node pointer + * @dev: Device pointer + * @tunables: Firmware-provided tunable parameters + * @tunables.axi2af: AXI to AF interface tunables + * @tunables.common: Common tunables for all lanes + * @tunables.lane_usb3: USB3 lane-specific tunables + * @tunables.lane_dp: DisplayPort lane-specific tunables + * @tunables.lane_usb4: USB4 lane-specific tunables + * @mode: Current PHY operating mode + * @swap_lanes: True if lanes must be swapped due to cable orientation + * @dp_link_rate: DisplayPort link rate + * @pipehandler_up: True if the PIPE mux ("pipehandler") is set to USB3 or USB4 mode + * @regs: Memory-mapped registers + * @regs.core: Core registers + * @regs.axi2af: AXI to Apple Fabric interface registers + * @regs.usb2phy: USB2 PHY registers + * @regs.pipehandler: USB3 PIPE interface ("pipehandler") registers + * @regs.lpdptx: DisplayPort registers + * @res: Resources for memory-mapped registers, used to verify that tunables aren't out of bounds + * @res.core: Core register resource + * @res.axi2af: AXI to Apple Fabric interface resource + * @phys: PHY instances + * @phys.usb2: USB2 PHY instance + * @phys.usb3: USB3 PHY instance + * @phys.dp: DisplayPort PHY instance + * @phy_provider: PHY provider instance + * @rcdev: Reset controller device + * @sw: Type-C switch instance + * @mux: Type-C mux instance + * @lock: Mutex for synchronizing register access across PHY, Type-C switch/mux and reset controller + */ +struct apple_atcphy { + struct device_node *np; + struct device *dev; + + struct { + struct apple_tunable *axi2af; + struct apple_tunable *common[2]; + struct apple_tunable *lane_usb3[2]; + struct apple_tunable *lane_dp[2]; + struct apple_tunable *lane_usb4[2]; + } tunables; + + enum atcphy_mode mode; + bool swap_lanes; + int dp_link_rate; + bool pipehandler_up; + + struct { + void __iomem *core; + void __iomem *axi2af; + void __iomem *usb2phy; + void __iomem *pipehandler; + void __iomem *lpdptx; + } regs; + + struct { + struct resource *core; + struct resource *axi2af; + } res; + + struct { + struct phy *usb2; + struct phy *usb3; + struct phy *dp; + } phys; + struct phy_provider *phy_provider; + + struct reset_controller_dev rcdev; + + struct typec_switch *sw; + struct typec_mux *mux; + + struct mutex lock; +}; + +static const struct { + const struct atcphy_mode_configuration normal; + const struct atcphy_mode_configuration swapped; + bool enable_dp_aux; + enum atcphy_pipehandler_state pipehandler_state; +} atcphy_modes[] = { + [APPLE_ATCPHY_MODE_OFF] = { + .normal = { + .crossbar = ACIOPHY_CROSSBAR_PROTOCOL_USB3, + .crossbar_dp_single_pma = ACIOPHY_CROSSBAR_DP_SINGLE_PMA_NONE, + .crossbar_dp_both_pma = false, + .lane_mode = {ACIOPHY_LANE_MODE_OFF, ACIOPHY_LANE_MODE_OFF}, + .dp_lane = {false, false}, + .set_swap = false, + }, + .swapped = { + .crossbar = ACIOPHY_CROSSBAR_PROTOCOL_USB3_SWAPPED, + .crossbar_dp_single_pma = ACIOPHY_CROSSBAR_DP_SINGLE_PMA_NONE, + .crossbar_dp_both_pma = false, + .lane_mode = {ACIOPHY_LANE_MODE_OFF, ACIOPHY_LANE_MODE_OFF}, + .dp_lane = {false, false}, + .set_swap = false, /* doesn't matter since the SS lanes are off */ + }, + .enable_dp_aux = false, + .pipehandler_state = ATCPHY_PIPEHANDLER_STATE_DUMMY, + }, + [APPLE_ATCPHY_MODE_USB2] = { + .normal = { + .crossbar = ACIOPHY_CROSSBAR_PROTOCOL_USB3, + .crossbar_dp_single_pma = ACIOPHY_CROSSBAR_DP_SINGLE_PMA_NONE, + .crossbar_dp_both_pma = false, + .lane_mode = {ACIOPHY_LANE_MODE_OFF, ACIOPHY_LANE_MODE_OFF}, + .dp_lane = {false, false}, + .set_swap = false, + }, + .swapped = { + .crossbar = ACIOPHY_CROSSBAR_PROTOCOL_USB3_SWAPPED, + .crossbar_dp_single_pma = ACIOPHY_CROSSBAR_DP_SINGLE_PMA_NONE, + .crossbar_dp_both_pma = false, + .lane_mode = {ACIOPHY_LANE_MODE_OFF, ACIOPHY_LANE_MODE_OFF}, + .dp_lane = {false, false}, + .set_swap = false, /* doesn't matter since the SS lanes are off */ + }, + .enable_dp_aux = false, + .pipehandler_state = ATCPHY_PIPEHANDLER_STATE_DUMMY, + }, + [APPLE_ATCPHY_MODE_USB3] = { + /* + * Setting up the lanes as DP/USB3 is intentional here, USB3/USB3 does not work + * and isn't required since this PHY does not support 20GBps mode anyway. + * The only difference to APPLE_ATCPHY_MODE_USB3_DP is that DP Aux is not enabled. + */ + .normal = { + .crossbar = ACIOPHY_CROSSBAR_PROTOCOL_USB3_DP, + .crossbar_dp_single_pma = ACIOPHY_CROSSBAR_DP_SINGLE_PMA_UNK008, + .crossbar_dp_both_pma = false, + .lane_mode = {ACIOPHY_LANE_MODE_USB3, ACIOPHY_LANE_MODE_DP}, + .dp_lane = {false, true}, + .set_swap = false, + }, + .swapped = { + .crossbar = ACIOPHY_CROSSBAR_PROTOCOL_USB3_DP_SWAPPED, + .crossbar_dp_single_pma = ACIOPHY_CROSSBAR_DP_SINGLE_PMA_UNK008, + .crossbar_dp_both_pma = false, + .lane_mode = {ACIOPHY_LANE_MODE_DP, ACIOPHY_LANE_MODE_USB3}, + .dp_lane = {true, false}, + .set_swap = true, + }, + .enable_dp_aux = false, + .pipehandler_state = ATCPHY_PIPEHANDLER_STATE_USB3, + }, + [APPLE_ATCPHY_MODE_USB3_DP] = { + .normal = { + .crossbar = ACIOPHY_CROSSBAR_PROTOCOL_USB3_DP, + .crossbar_dp_single_pma = ACIOPHY_CROSSBAR_DP_SINGLE_PMA_UNK008, + .crossbar_dp_both_pma = false, + .lane_mode = {ACIOPHY_LANE_MODE_USB3, ACIOPHY_LANE_MODE_DP}, + .dp_lane = {false, true}, + .set_swap = false, + }, + .swapped = { + .crossbar = ACIOPHY_CROSSBAR_PROTOCOL_USB3_DP_SWAPPED, + .crossbar_dp_single_pma = ACIOPHY_CROSSBAR_DP_SINGLE_PMA_UNK008, + .crossbar_dp_both_pma = false, + .lane_mode = {ACIOPHY_LANE_MODE_DP, ACIOPHY_LANE_MODE_USB3}, + .dp_lane = {true, false}, + .set_swap = true, + }, + .enable_dp_aux = true, + .pipehandler_state = ATCPHY_PIPEHANDLER_STATE_USB3, + }, + [APPLE_ATCPHY_MODE_TBT] = { + .normal = { + .crossbar = ACIOPHY_CROSSBAR_PROTOCOL_USB4, + .crossbar_dp_single_pma = ACIOPHY_CROSSBAR_DP_SINGLE_PMA_NONE, + .crossbar_dp_both_pma = false, + .lane_mode = {ACIOPHY_LANE_MODE_USB4, ACIOPHY_LANE_MODE_USB4}, + .dp_lane = {false, false}, + .set_swap = false, + }, + .swapped = { + .crossbar = ACIOPHY_CROSSBAR_PROTOCOL_USB4_SWAPPED, + .crossbar_dp_single_pma = ACIOPHY_CROSSBAR_DP_SINGLE_PMA_NONE, + .crossbar_dp_both_pma = false, + .lane_mode = {ACIOPHY_LANE_MODE_USB4, ACIOPHY_LANE_MODE_USB4}, + .dp_lane = {false, false}, + .set_swap = false, /* intentionally false */ + }, + .enable_dp_aux = false, + .pipehandler_state = ATCPHY_PIPEHANDLER_STATE_DUMMY, + }, + [APPLE_ATCPHY_MODE_USB4] = { + .normal = { + .crossbar = ACIOPHY_CROSSBAR_PROTOCOL_USB4, + .crossbar_dp_single_pma = ACIOPHY_CROSSBAR_DP_SINGLE_PMA_NONE, + .crossbar_dp_both_pma = false, + .lane_mode = {ACIOPHY_LANE_MODE_USB4, ACIOPHY_LANE_MODE_USB4}, + .dp_lane = {false, false}, + .set_swap = false, + }, + .swapped = { + .crossbar = ACIOPHY_CROSSBAR_PROTOCOL_USB4_SWAPPED, + .crossbar_dp_single_pma = ACIOPHY_CROSSBAR_DP_SINGLE_PMA_NONE, + .crossbar_dp_both_pma = false, + .lane_mode = {ACIOPHY_LANE_MODE_USB4, ACIOPHY_LANE_MODE_USB4}, + .dp_lane = {false, false}, + .set_swap = false, /* intentionally false */ + }, + .enable_dp_aux = false, + .pipehandler_state = ATCPHY_PIPEHANDLER_STATE_USB4, + }, + [APPLE_ATCPHY_MODE_DP] = { + .normal = { + .crossbar = ACIOPHY_CROSSBAR_PROTOCOL_DP, + .crossbar_dp_single_pma = ACIOPHY_CROSSBAR_DP_SINGLE_PMA_UNK100, + .crossbar_dp_both_pma = true, + .lane_mode = {ACIOPHY_LANE_MODE_DP, ACIOPHY_LANE_MODE_DP}, + .dp_lane = {true, true}, + .set_swap = false, + }, + .swapped = { + .crossbar = ACIOPHY_CROSSBAR_PROTOCOL_DP, + .crossbar_dp_single_pma = ACIOPHY_CROSSBAR_DP_SINGLE_PMA_UNK008, + .crossbar_dp_both_pma = false, /* intentionally false */ + .lane_mode = {ACIOPHY_LANE_MODE_DP, ACIOPHY_LANE_MODE_DP}, + .dp_lane = {true, true}, + .set_swap = false, /* intentionally false */ + }, + .enable_dp_aux = true, + .pipehandler_state = ATCPHY_PIPEHANDLER_STATE_DUMMY, + }, +}; + +static const struct atcphy_dp_link_rate_configuration dp_lr_config[] = { + [ATCPHY_DP_LINK_RATE_RBR] = { + .freqinit_count_target = 0x21c, + .fbdivn_frac_den = 0x0, + .fbdivn_frac_num = 0x0, + .pclk_div_sel = 0x13, + .lfclk_ctrl = 0x5, + .vclk_op_divn = 0x2, + .plla_clkout_vreg_bypass = true, + .txa_ldoclk_bypass = true, + .txa_div2_en = true, + }, + [ATCPHY_DP_LINK_RATE_HBR] = { + .freqinit_count_target = 0x1c2, + .fbdivn_frac_den = 0x3ffe, + .fbdivn_frac_num = 0x1fff, + .pclk_div_sel = 0x9, + .lfclk_ctrl = 0x5, + .vclk_op_divn = 0x2, + .plla_clkout_vreg_bypass = true, + .txa_ldoclk_bypass = true, + .txa_div2_en = false, + }, + [ATCPHY_DP_LINK_RATE_HBR2] = { + .freqinit_count_target = 0x1c2, + .fbdivn_frac_den = 0x3ffe, + .fbdivn_frac_num = 0x1fff, + .pclk_div_sel = 0x4, + .lfclk_ctrl = 0x5, + .vclk_op_divn = 0x0, + .plla_clkout_vreg_bypass = true, + .txa_ldoclk_bypass = true, + .txa_div2_en = false, + }, + [ATCPHY_DP_LINK_RATE_HBR3] = { + .freqinit_count_target = 0x2a3, + .fbdivn_frac_den = 0x3ffc, + .fbdivn_frac_num = 0x2ffd, + .pclk_div_sel = 0x4, + .lfclk_ctrl = 0x6, + .vclk_op_divn = 0x0, + .plla_clkout_vreg_bypass = false, + .txa_ldoclk_bypass = false, + .txa_div2_en = false, + }, +}; + +static inline void mask32(void __iomem *reg, u32 mask, u32 set) +{ + u32 value = readl(reg); + + value &= ~mask; + value |= set; + writel(value, reg); +} + +static inline void core_mask32(struct apple_atcphy *atcphy, u32 reg, u32 mask, u32 set) +{ + mask32(atcphy->regs.core + reg, mask, set); +} + +static inline void set32(void __iomem *reg, u32 set) +{ + mask32(reg, 0, set); +} + +static inline void core_set32(struct apple_atcphy *atcphy, u32 reg, u32 set) +{ + core_mask32(atcphy, reg, 0, set); +} + +static inline void clear32(void __iomem *reg, u32 clear) +{ + mask32(reg, clear, 0); +} + +static inline void core_clear32(struct apple_atcphy *atcphy, u32 reg, u32 clear) +{ + core_mask32(atcphy, reg, clear, 0); +} + +static const struct atcphy_mode_configuration *atcphy_get_mode_config(struct apple_atcphy *atcphy, + enum atcphy_mode mode) +{ + if (atcphy->swap_lanes) + return &atcphy_modes[mode].swapped; + else + return &atcphy_modes[mode].normal; +} + +static void atcphy_apply_tunables(struct apple_atcphy *atcphy, enum atcphy_mode mode) +{ + const int lane0 = atcphy->swap_lanes ? 1 : 0; + const int lane1 = atcphy->swap_lanes ? 0 : 1; + + apple_tunable_apply(atcphy->regs.core, atcphy->tunables.common[0]); + apple_tunable_apply(atcphy->regs.axi2af, atcphy->tunables.axi2af); + apple_tunable_apply(atcphy->regs.core, atcphy->tunables.common[1]); + + switch (mode) { + /* + * USB 3.2 Gen 2x2 / SuperSpeed 20Gbps is not supported by this hardware and applying USB3 + * tunables to both lanes does not result in a working PHY configuration. Thus, both + * USB3-only and USB3/DP get the same tunable setup here. + */ + case APPLE_ATCPHY_MODE_USB3: + case APPLE_ATCPHY_MODE_USB3_DP: + apple_tunable_apply(atcphy->regs.core, atcphy->tunables.lane_usb3[lane0]); + apple_tunable_apply(atcphy->regs.core, atcphy->tunables.lane_dp[lane1]); + break; + + case APPLE_ATCPHY_MODE_DP: + apple_tunable_apply(atcphy->regs.core, atcphy->tunables.lane_dp[lane0]); + apple_tunable_apply(atcphy->regs.core, atcphy->tunables.lane_dp[lane1]); + break; + + /* + * Even though the various Thunderbolt versions and USB4 are different protocols they need + * the same tunables. The actual protocol-specific setup happens inside the Thunderbolt/USB4 + * native host interface. + */ + case APPLE_ATCPHY_MODE_TBT: + case APPLE_ATCPHY_MODE_USB4: + apple_tunable_apply(atcphy->regs.core, atcphy->tunables.lane_usb4[lane0]); + apple_tunable_apply(atcphy->regs.core, atcphy->tunables.lane_usb4[lane1]); + break; + + case APPLE_ATCPHY_MODE_OFF: + case APPLE_ATCPHY_MODE_USB2: + break; + } +} + +static int atcphy_pipehandler_lock(struct apple_atcphy *atcphy) +{ + int ret; + u32 reg; + + if (readl(atcphy->regs.pipehandler + PIPEHANDLER_LOCK_REQ) & PIPEHANDLER_LOCK_EN) { + dev_warn(atcphy->dev, "Pipehandler already locked\n"); + return 0; + } + + set32(atcphy->regs.pipehandler + PIPEHANDLER_LOCK_REQ, PIPEHANDLER_LOCK_EN); + + ret = readl_poll_timeout(atcphy->regs.pipehandler + PIPEHANDLER_LOCK_ACK, reg, + reg & PIPEHANDLER_LOCK_EN, 10, PIPEHANDLER_LOCK_ACK_TIMEOUT_US); + if (ret) { + clear32(atcphy->regs.pipehandler + PIPEHANDLER_LOCK_REQ, 1); + dev_warn(atcphy->dev, "Pipehandler lock not acked.\n"); + } + + return ret; +} + +static int atcphy_pipehandler_unlock(struct apple_atcphy *atcphy) +{ + int ret; + u32 reg; + + clear32(atcphy->regs.pipehandler + PIPEHANDLER_LOCK_REQ, PIPEHANDLER_LOCK_EN); + ret = readl_poll_timeout(atcphy->regs.pipehandler + PIPEHANDLER_LOCK_ACK, reg, + !(reg & PIPEHANDLER_LOCK_EN), 10, PIPEHANDLER_LOCK_ACK_TIMEOUT_US); + if (ret) + dev_warn(atcphy->dev, "Pipehandler lock release not acked.\n"); + + return ret; +} + +static int atcphy_pipehandler_check(struct apple_atcphy *atcphy) +{ + int ret; + + lockdep_assert_held(&atcphy->lock); + + if (readl(atcphy->regs.pipehandler + PIPEHANDLER_LOCK_ACK) & PIPEHANDLER_LOCK_EN) { + dev_warn(atcphy->dev, "Pipehandler already locked\n"); + + ret = atcphy_pipehandler_unlock(atcphy); + if (ret) { + dev_err(atcphy->dev, "Failed to unlock pipehandler\n"); + return ret; + } + } + + return 0; +} + +static int atcphy_configure_pipehandler_usb3(struct apple_atcphy *atcphy, bool host) +{ + int ret; + u32 reg; + + ret = atcphy_pipehandler_check(atcphy); + if (ret) + return ret; + + /* + * Only host mode requires this unknown BIST sequence to work correctly, possibly due to + * some hardware quirk. Guest mode breaks if we try to apply this sequence. + */ + if (host) { + /* Force disable link detection */ + clear32(atcphy->regs.pipehandler + PIPEHANDLER_OVERRIDE_VALUES, + PIPEHANDLER_OVERRIDE_VAL_RXDETECT0 | PIPEHANDLER_OVERRIDE_VAL_RXDETECT1); + set32(atcphy->regs.pipehandler + PIPEHANDLER_OVERRIDE, + PIPEHANDLER_OVERRIDE_RXVALID); + set32(atcphy->regs.pipehandler + PIPEHANDLER_OVERRIDE, + PIPEHANDLER_OVERRIDE_RXDETECT); + + ret = atcphy_pipehandler_lock(atcphy); + if (ret) { + dev_err(atcphy->dev, "Failed to lock pipehandler"); + return ret; + } + + /* BIST dance */ + core_set32(atcphy, ACIOPHY_TOP_BIST_PHY_CFG0, + ACIOPHY_TOP_BIST_PHY_CFG0_LN0_RESET_N); + core_set32(atcphy, ACIOPHY_TOP_BIST_OV_CFG, ACIOPHY_TOP_BIST_OV_CFG_LN0_RESET_N_OV); + ret = readl_poll_timeout(atcphy->regs.core + ACIOPHY_TOP_PHY_STAT, reg, + !(reg & ACIOPHY_TOP_PHY_STAT_LN0_UNK23), 10, 10000); + if (ret) + dev_warn(atcphy->dev, + "Timed out waiting for ACIOPHY_TOP_PHY_STAT_LN0_UNK23\n"); + + core_set32(atcphy, ACIOPHY_TOP_BIST_READ_CTRL, + ACIOPHY_TOP_BIST_READ_CTRL_LN0_PHY_STATUS_RE); + core_clear32(atcphy, ACIOPHY_TOP_BIST_READ_CTRL, + ACIOPHY_TOP_BIST_READ_CTRL_LN0_PHY_STATUS_RE); + + core_mask32(atcphy, ACIOPHY_TOP_BIST_PHY_CFG1, + ACIOPHY_TOP_BIST_PHY_CFG1_LN0_PWR_DOWN, + FIELD_PREP(ACIOPHY_TOP_BIST_PHY_CFG1_LN0_PWR_DOWN, 3)); + + core_set32(atcphy, ACIOPHY_TOP_BIST_OV_CFG, + ACIOPHY_TOP_BIST_OV_CFG_LN0_PWR_DOWN_OV); + core_set32(atcphy, ACIOPHY_TOP_BIST_CIOPHY_CFG1, + ACIOPHY_TOP_BIST_CIOPHY_CFG1_CLK_EN); + core_set32(atcphy, ACIOPHY_TOP_BIST_CIOPHY_CFG1, + ACIOPHY_TOP_BIST_CIOPHY_CFG1_BIST_EN); + writel(0, atcphy->regs.core + ACIOPHY_TOP_BIST_CIOPHY_CFG1); + + ret = readl_poll_timeout(atcphy->regs.core + ACIOPHY_TOP_PHY_STAT, reg, + (reg & ACIOPHY_TOP_PHY_STAT_LN0_UNK0), 10, 10000); + if (ret) + dev_warn(atcphy->dev, + "timed out waiting for ACIOPHY_TOP_PHY_STAT_LN0_UNK0\n"); + + ret = readl_poll_timeout(atcphy->regs.core + ACIOPHY_TOP_PHY_STAT, reg, + !(reg & ACIOPHY_TOP_PHY_STAT_LN0_UNK23), 10, 10000); + if (ret) + dev_warn(atcphy->dev, + "timed out waiting for ACIOPHY_TOP_PHY_STAT_LN0_UNK23\n"); + + /* Clear reset for non-selected USB3 PHY (?) */ + mask32(atcphy->regs.pipehandler + PIPEHANDLER_NONSELECTED_OVERRIDE, + PIPEHANDLER_NATIVE_POWER_DOWN, FIELD_PREP(PIPEHANDLER_NATIVE_POWER_DOWN, 3)); + clear32(atcphy->regs.pipehandler + PIPEHANDLER_NONSELECTED_OVERRIDE, + PIPEHANDLER_NATIVE_RESET); + + /* More BIST stuff (?) */ + writel(0, atcphy->regs.core + ACIOPHY_TOP_BIST_OV_CFG); + core_set32(atcphy, ACIOPHY_TOP_BIST_CIOPHY_CFG1, + ACIOPHY_TOP_BIST_CIOPHY_CFG1_CLK_EN); + core_set32(atcphy, ACIOPHY_TOP_BIST_CIOPHY_CFG1, + ACIOPHY_TOP_BIST_CIOPHY_CFG1_BIST_EN); + } + + /* Configure PIPE mux to USB3 PHY */ + mask32(atcphy->regs.pipehandler + PIPEHANDLER_MUX_CTRL, PIPEHANDLER_MUX_CTRL_CLK, + FIELD_PREP(PIPEHANDLER_MUX_CTRL_CLK, PIPEHANDLER_MUX_CTRL_CLK_OFF)); + udelay(10); + mask32(atcphy->regs.pipehandler + PIPEHANDLER_MUX_CTRL, PIPEHANDLER_MUX_CTRL_DATA, + FIELD_PREP(PIPEHANDLER_MUX_CTRL_DATA, PIPEHANDLER_MUX_CTRL_DATA_USB3)); + udelay(10); + mask32(atcphy->regs.pipehandler + PIPEHANDLER_MUX_CTRL, PIPEHANDLER_MUX_CTRL_CLK, + FIELD_PREP(PIPEHANDLER_MUX_CTRL_CLK, PIPEHANDLER_MUX_CTRL_CLK_USB3)); + udelay(10); + + /* Remove link detection override */ + clear32(atcphy->regs.pipehandler + PIPEHANDLER_OVERRIDE, PIPEHANDLER_OVERRIDE_RXVALID); + clear32(atcphy->regs.pipehandler + PIPEHANDLER_OVERRIDE, PIPEHANDLER_OVERRIDE_RXDETECT); + + /* Pipehandler was only locked when the BIST sequence was applied for host mode */ + if (host) { + ret = atcphy_pipehandler_unlock(atcphy); + if (ret) + dev_warn(atcphy->dev, "Failed to unlock pipehandler"); + } + + return 0; +} + +static int atcphy_configure_pipehandler_dummy(struct apple_atcphy *atcphy) +{ + int ret; + + ret = atcphy_pipehandler_check(atcphy); + if (ret) + return ret; + + /* Force disable link detection */ + clear32(atcphy->regs.pipehandler + PIPEHANDLER_OVERRIDE_VALUES, + PIPEHANDLER_OVERRIDE_VAL_RXDETECT0 | PIPEHANDLER_OVERRIDE_VAL_RXDETECT1); + set32(atcphy->regs.pipehandler + PIPEHANDLER_OVERRIDE, PIPEHANDLER_OVERRIDE_RXVALID); + set32(atcphy->regs.pipehandler + PIPEHANDLER_OVERRIDE, PIPEHANDLER_OVERRIDE_RXDETECT); + + ret = atcphy_pipehandler_lock(atcphy); + if (ret) + dev_warn(atcphy->dev, "Failed to lock pipehandler"); + + /* Switch to dummy PHY */ + mask32(atcphy->regs.pipehandler + PIPEHANDLER_MUX_CTRL, PIPEHANDLER_MUX_CTRL_CLK, + FIELD_PREP(PIPEHANDLER_MUX_CTRL_CLK, PIPEHANDLER_MUX_CTRL_CLK_OFF)); + udelay(10); + mask32(atcphy->regs.pipehandler + PIPEHANDLER_MUX_CTRL, PIPEHANDLER_MUX_CTRL_DATA, + FIELD_PREP(PIPEHANDLER_MUX_CTRL_DATA, PIPEHANDLER_MUX_CTRL_DATA_DUMMY)); + udelay(10); + mask32(atcphy->regs.pipehandler + PIPEHANDLER_MUX_CTRL, PIPEHANDLER_MUX_CTRL_CLK, + FIELD_PREP(PIPEHANDLER_MUX_CTRL_CLK, PIPEHANDLER_MUX_CTRL_CLK_DUMMY)); + udelay(10); + + ret = atcphy_pipehandler_unlock(atcphy); + if (ret) + dev_warn(atcphy->dev, "Failed to unlock pipehandler"); + + mask32(atcphy->regs.pipehandler + PIPEHANDLER_NONSELECTED_OVERRIDE, + PIPEHANDLER_NATIVE_POWER_DOWN, FIELD_PREP(PIPEHANDLER_NATIVE_POWER_DOWN, 2)); + set32(atcphy->regs.pipehandler + PIPEHANDLER_NONSELECTED_OVERRIDE, + PIPEHANDLER_NATIVE_RESET); + + return 0; +} + +static int atcphy_configure_pipehandler(struct apple_atcphy *atcphy, bool host) +{ + int ret; + + lockdep_assert_held(&atcphy->lock); + + switch (atcphy_modes[atcphy->mode].pipehandler_state) { + case ATCPHY_PIPEHANDLER_STATE_USB3: + ret = atcphy_configure_pipehandler_usb3(atcphy, host); + atcphy->pipehandler_up = true; + break; + case ATCPHY_PIPEHANDLER_STATE_USB4: + dev_warn(atcphy->dev, + "ATCPHY_PIPEHANDLER_STATE_USB4 not implemented; falling back to USB2\n"); + ret = atcphy_configure_pipehandler_dummy(atcphy); + atcphy->pipehandler_up = false; + break; + default: + ret = -EINVAL; + } + + return ret; +} + +static void atcphy_setup_pipehandler(struct apple_atcphy *atcphy) +{ + lockdep_assert_held(&atcphy->lock); + + mask32(atcphy->regs.pipehandler + PIPEHANDLER_MUX_CTRL, PIPEHANDLER_MUX_CTRL_CLK, + FIELD_PREP(PIPEHANDLER_MUX_CTRL_CLK, PIPEHANDLER_MUX_CTRL_CLK_OFF)); + udelay(10); + mask32(atcphy->regs.pipehandler + PIPEHANDLER_MUX_CTRL, PIPEHANDLER_MUX_CTRL_DATA, + FIELD_PREP(PIPEHANDLER_MUX_CTRL_DATA, PIPEHANDLER_MUX_CTRL_DATA_DUMMY)); + udelay(10); + mask32(atcphy->regs.pipehandler + PIPEHANDLER_MUX_CTRL, PIPEHANDLER_MUX_CTRL_CLK, + FIELD_PREP(PIPEHANDLER_MUX_CTRL_CLK, PIPEHANDLER_MUX_CTRL_CLK_DUMMY)); + udelay(10); +} + +static void atcphy_configure_lanes(struct apple_atcphy *atcphy, enum atcphy_mode mode) +{ + const struct atcphy_mode_configuration *mode_cfg = atcphy_get_mode_config(atcphy, mode); + + core_mask32(atcphy, ACIOPHY_LANE_MODE, ACIOPHY_LANE_MODE_RX0, + FIELD_PREP(ACIOPHY_LANE_MODE_RX0, mode_cfg->lane_mode[0])); + core_mask32(atcphy, ACIOPHY_LANE_MODE, ACIOPHY_LANE_MODE_TX0, + FIELD_PREP(ACIOPHY_LANE_MODE_TX0, mode_cfg->lane_mode[0])); + core_mask32(atcphy, ACIOPHY_LANE_MODE, ACIOPHY_LANE_MODE_RX1, + FIELD_PREP(ACIOPHY_LANE_MODE_RX1, mode_cfg->lane_mode[1])); + core_mask32(atcphy, ACIOPHY_LANE_MODE, ACIOPHY_LANE_MODE_TX1, + FIELD_PREP(ACIOPHY_LANE_MODE_TX1, mode_cfg->lane_mode[1])); + core_mask32(atcphy, ACIOPHY_CROSSBAR, ACIOPHY_CROSSBAR_PROTOCOL, + FIELD_PREP(ACIOPHY_CROSSBAR_PROTOCOL, mode_cfg->crossbar)); + + if (mode_cfg->set_swap) + core_set32(atcphy, ATCPHY_MISC, ATCPHY_MISC_LANE_SWAP); + else + core_clear32(atcphy, ATCPHY_MISC, ATCPHY_MISC_LANE_SWAP); + + core_mask32(atcphy, ACIOPHY_CROSSBAR, ACIOPHY_CROSSBAR_DP_SINGLE_PMA, + FIELD_PREP(ACIOPHY_CROSSBAR_DP_SINGLE_PMA, mode_cfg->crossbar_dp_single_pma)); + if (mode_cfg->crossbar_dp_both_pma) + core_set32(atcphy, ACIOPHY_CROSSBAR, ACIOPHY_CROSSBAR_DP_BOTH_PMA); + else + core_clear32(atcphy, ACIOPHY_CROSSBAR, ACIOPHY_CROSSBAR_DP_BOTH_PMA); + + if (mode_cfg->dp_lane[0]) { + core_set32(atcphy, LN0_AUSPMA_RX_TOP + LN_AUSPMA_RX_TOP_PMAFSM, + LN_AUSPMA_RX_TOP_PMAFSM_PCS_OV); + udelay(10); + core_clear32(atcphy, LN0_AUSPMA_RX_TOP + LN_AUSPMA_RX_TOP_PMAFSM, + LN_AUSPMA_RX_TOP_PMAFSM_PCS_REQ); + } else { + core_clear32(atcphy, LN0_AUSPMA_RX_TOP + LN_AUSPMA_RX_TOP_PMAFSM, + LN_AUSPMA_RX_TOP_PMAFSM_PCS_OV); + udelay(10); + } + + if (mode_cfg->dp_lane[1]) { + core_set32(atcphy, LN1_AUSPMA_RX_TOP + LN_AUSPMA_RX_TOP_PMAFSM, + LN_AUSPMA_RX_TOP_PMAFSM_PCS_OV); + udelay(10); + core_clear32(atcphy, LN1_AUSPMA_RX_TOP + LN_AUSPMA_RX_TOP_PMAFSM, + LN_AUSPMA_RX_TOP_PMAFSM_PCS_REQ); + } else { + core_clear32(atcphy, LN1_AUSPMA_RX_TOP + LN_AUSPMA_RX_TOP_PMAFSM, + LN_AUSPMA_RX_TOP_PMAFSM_PCS_OV); + udelay(10); + } +} + +static void atcphy_enable_dp_aux(struct apple_atcphy *atcphy) +{ + core_set32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, DPTXPHY_PMA_LANE_RESET_N); + core_set32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, DPTXPHY_PMA_LANE_RESET_N_OV); + + core_mask32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, DPRX_PCLK_SELECT, + FIELD_PREP(DPRX_PCLK_SELECT, 1)); + core_set32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, DPRX_PCLK_ENABLE); + + core_mask32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, DPTX_PCLK1_SELECT, + FIELD_PREP(DPTX_PCLK1_SELECT, 1)); + core_set32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, DPTX_PCLK1_ENABLE); + + core_mask32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, DPTX_PCLK2_SELECT, + FIELD_PREP(DPTX_PCLK2_SELECT, 1)); + core_set32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, DPTX_PCLK2_ENABLE); + + core_set32(atcphy, ACIOPHY_PLL_COMMON_CTRL, + ACIOPHY_PLL_WAIT_FOR_CMN_READY_BEFORE_RESET_EXIT); + + set32(atcphy->regs.lpdptx + LPDPTX_AUX_CONTROL, LPDPTX_AUX_CLAMP_EN); + set32(atcphy->regs.lpdptx + LPDPTX_AUX_CONTROL, LPDPTX_SLEEP_B_SML_IN); + udelay(10); + set32(atcphy->regs.lpdptx + LPDPTX_AUX_CONTROL, LPDPTX_SLEEP_B_BIG_IN); + udelay(10); + clear32(atcphy->regs.lpdptx + LPDPTX_AUX_CONTROL, LPDPTX_AUX_CLAMP_EN); + clear32(atcphy->regs.lpdptx + LPDPTX_AUX_CONTROL, LPDPTX_AUX_PWN_DOWN); + clear32(atcphy->regs.lpdptx + LPDPTX_AUX_CONTROL, LPDPTX_TXTERM_CODEMSB); + mask32(atcphy->regs.lpdptx + LPDPTX_AUX_CONTROL, LPDPTX_TXTERM_CODE, + FIELD_PREP(LPDPTX_TXTERM_CODE, 0x16)); + + set32(atcphy->regs.lpdptx + LPDPTX_AUX_CFG_BLK_AUX_LDO_CTRL, 0x1c00); + mask32(atcphy->regs.lpdptx + LPDPTX_AUX_SHM_CFG_BLK_AUX_CTRL_REG1, LPDPTX_CFG_PMA_PHYS_ADJ, + FIELD_PREP(LPDPTX_CFG_PMA_PHYS_ADJ, 5)); + set32(atcphy->regs.lpdptx + LPDPTX_AUX_SHM_CFG_BLK_AUX_CTRL_REG1, + LPDPTX_CFG_PMA_PHYS_ADJ_OV); + + clear32(atcphy->regs.lpdptx + LPDPTX_AUX_CFG_BLK_AUX_MARGIN, + LPDPTX_MARGIN_RCAL_RXOFFSET_EN); + + clear32(atcphy->regs.lpdptx + LPDPTX_AUX_CFG_BLK_AUX_CTRL, LPDPTX_BLK_AUX_CTRL_PWRDN); + set32(atcphy->regs.lpdptx + LPDPTX_AUX_SHM_CFG_BLK_AUX_CTRL_REG0, + LPDPTX_CFG_PMA_AUX_SEL_LF_DATA); + mask32(atcphy->regs.lpdptx + LPDPTX_AUX_CFG_BLK_AUX_CTRL, LPDPTX_BLK_AUX_RXOFFSET, + FIELD_PREP(LPDPTX_BLK_AUX_RXOFFSET, 3)); + + mask32(atcphy->regs.lpdptx + LPDPTX_AUX_CFG_BLK_AUX_MARGIN, LPDPTX_AUX_MARGIN_RCAL_TXSWING, + FIELD_PREP(LPDPTX_AUX_MARGIN_RCAL_TXSWING, 12)); + + atcphy->dp_link_rate = -1; +} + +static void atcphy_disable_dp_aux(struct apple_atcphy *atcphy) +{ + set32(atcphy->regs.lpdptx + LPDPTX_AUX_CONTROL, LPDPTX_AUX_PWN_DOWN); + set32(atcphy->regs.lpdptx + LPDPTX_AUX_CFG_BLK_AUX_CTRL, LPDPTX_BLK_AUX_CTRL_PWRDN); + set32(atcphy->regs.lpdptx + LPDPTX_AUX_CONTROL, LPDPTX_AUX_CLAMP_EN); + clear32(atcphy->regs.lpdptx + LPDPTX_AUX_CONTROL, LPDPTX_SLEEP_B_SML_IN); + udelay(10); + clear32(atcphy->regs.lpdptx + LPDPTX_AUX_CONTROL, LPDPTX_SLEEP_B_BIG_IN); + udelay(10); + + core_clear32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, DPTXPHY_PMA_LANE_RESET_N); + core_clear32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, DPRX_PCLK_ENABLE); + core_clear32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, DPTX_PCLK1_ENABLE); + core_clear32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, DPTX_PCLK2_ENABLE); +} + +static int atcphy_dp_configure_lane(struct apple_atcphy *atcphy, enum atcphy_lane lane, + const struct atcphy_dp_link_rate_configuration *cfg) +{ + void __iomem *tx_shm, *rx_shm, *rx_top; + unsigned int tx_cal_code; + + lockdep_assert_held(&atcphy->lock); + + switch (lane) { + case APPLE_ATCPHY_LANE_0: + tx_shm = atcphy->regs.core + LN0_AUSPMA_TX_SHM; + rx_shm = atcphy->regs.core + LN0_AUSPMA_RX_SHM; + rx_top = atcphy->regs.core + LN0_AUSPMA_RX_TOP; + break; + case APPLE_ATCPHY_LANE_1: + tx_shm = atcphy->regs.core + LN1_AUSPMA_TX_SHM; + rx_shm = atcphy->regs.core + LN1_AUSPMA_RX_SHM; + rx_top = atcphy->regs.core + LN1_AUSPMA_RX_TOP; + break; + default: + return -EINVAL; + } + + set32(tx_shm + LN_AUSPMA_TX_SHM_TXA_LDOCLK, LN_LDOCLK_EN_SML); + set32(tx_shm + LN_AUSPMA_TX_SHM_TXA_LDOCLK, LN_LDOCLK_EN_SML_OV); + udelay(10); + + set32(tx_shm + LN_AUSPMA_TX_SHM_TXA_LDOCLK, LN_LDOCLK_EN_BIG); + set32(tx_shm + LN_AUSPMA_TX_SHM_TXA_LDOCLK, LN_LDOCLK_EN_BIG_OV); + udelay(10); + + if (cfg->txa_ldoclk_bypass) { + set32(tx_shm + LN_AUSPMA_TX_SHM_TXA_LDOCLK, LN_LDOCLK_BYPASS_SML); + set32(tx_shm + LN_AUSPMA_TX_SHM_TXA_LDOCLK, LN_LDOCLK_BYPASS_SML_OV); + udelay(10); + + set32(tx_shm + LN_AUSPMA_TX_SHM_TXA_LDOCLK, LN_LDOCLK_BYPASS_BIG); + set32(tx_shm + LN_AUSPMA_TX_SHM_TXA_LDOCLK, LN_LDOCLK_BYPASS_BIG_OV); + udelay(10); + } else { + clear32(tx_shm + LN_AUSPMA_TX_SHM_TXA_LDOCLK, LN_LDOCLK_BYPASS_SML); + clear32(tx_shm + LN_AUSPMA_TX_SHM_TXA_LDOCLK, LN_LDOCLK_BYPASS_SML_OV); + udelay(10); + + clear32(tx_shm + LN_AUSPMA_TX_SHM_TXA_LDOCLK, LN_LDOCLK_BYPASS_BIG); + clear32(tx_shm + LN_AUSPMA_TX_SHM_TXA_LDOCLK, LN_LDOCLK_BYPASS_BIG_OV); + udelay(10); + } + + set32(tx_shm + LN_AUSPMA_TX_SHM_TXA_CFG_MAIN_REG0, LN_BYTECLK_RESET_SYNC_SEL_OV); + set32(tx_shm + LN_AUSPMA_TX_SHM_TXA_CFG_MAIN_REG0, LN_BYTECLK_RESET_SYNC_EN); + set32(tx_shm + LN_AUSPMA_TX_SHM_TXA_CFG_MAIN_REG0, LN_BYTECLK_RESET_SYNC_EN_OV); + clear32(tx_shm + LN_AUSPMA_TX_SHM_TXA_CFG_MAIN_REG0, LN_BYTECLK_RESET_SYNC_CLR); + set32(tx_shm + LN_AUSPMA_TX_SHM_TXA_CFG_MAIN_REG0, LN_BYTECLK_RESET_SYNC_CLR_OV); + + if (cfg->txa_div2_en) + set32(tx_shm + LN_AUSPMA_TX_SHM_TXA_CFG_MAIN_REG1, LN_TXA_DIV2_EN); + else + clear32(tx_shm + LN_AUSPMA_TX_SHM_TXA_CFG_MAIN_REG1, LN_TXA_DIV2_EN); + set32(tx_shm + LN_AUSPMA_TX_SHM_TXA_CFG_MAIN_REG1, LN_TXA_DIV2_EN_OV); + set32(tx_shm + LN_AUSPMA_TX_SHM_TXA_CFG_MAIN_REG1, LN_TXA_CLK_EN); + set32(tx_shm + LN_AUSPMA_TX_SHM_TXA_CFG_MAIN_REG1, LN_TXA_CLK_EN_OV); + clear32(tx_shm + LN_AUSPMA_TX_SHM_TXA_CFG_MAIN_REG1, LN_TXA_DIV2_RESET); + set32(tx_shm + LN_AUSPMA_TX_SHM_TXA_CFG_MAIN_REG1, LN_TXA_DIV2_RESET_OV); + + mask32(tx_shm + LN_AUSPMA_TX_SHM_TXA_IMP_REG0, LN_TXA_CAL_CTRL_BASE, + FIELD_PREP(LN_TXA_CAL_CTRL_BASE, 0xf)); + set32(tx_shm + LN_AUSPMA_TX_SHM_TXA_IMP_REG0, LN_TXA_CAL_CTRL_BASE_OV); + + tx_cal_code = FIELD_GET(AUS_UNK_A20_TX_CAL_CODE, readl(atcphy->regs.core + AUS_UNK_A20)); + mask32(tx_shm + LN_AUSPMA_TX_SHM_TXA_IMP_REG0, LN_TXA_CAL_CTRL, + FIELD_PREP(LN_TXA_CAL_CTRL, (1 << tx_cal_code) - 1)); + set32(tx_shm + LN_AUSPMA_TX_SHM_TXA_IMP_REG0, LN_TXA_CAL_CTRL_OV); + + clear32(tx_shm + LN_AUSPMA_TX_SHM_TXA_IMP_REG2, LN_TXA_MARGIN); + set32(tx_shm + LN_AUSPMA_TX_SHM_TXA_IMP_REG2, LN_TXA_MARGIN_OV); + clear32(tx_shm + LN_AUSPMA_TX_SHM_TXA_IMP_REG2, LN_TXA_MARGIN_2R); + set32(tx_shm + LN_AUSPMA_TX_SHM_TXA_IMP_REG2, LN_TXA_MARGIN_2R_OV); + + clear32(tx_shm + LN_AUSPMA_TX_SHM_TXA_IMP_REG3, LN_TXA_MARGIN_POST); + set32(tx_shm + LN_AUSPMA_TX_SHM_TXA_IMP_REG3, LN_TXA_MARGIN_POST_OV); + clear32(tx_shm + LN_AUSPMA_TX_SHM_TXA_IMP_REG3, LN_TXA_MARGIN_POST_2R); + set32(tx_shm + LN_AUSPMA_TX_SHM_TXA_IMP_REG3, LN_TXA_MARGIN_POST_2R_OV); + clear32(tx_shm + LN_AUSPMA_TX_SHM_TXA_IMP_REG3, LN_TXA_MARGIN_POST_4R); + set32(tx_shm + LN_AUSPMA_TX_SHM_TXA_IMP_REG3, LN_TXA_MARGIN_POST_4R_OV); + clear32(tx_shm + LN_AUSPMA_TX_SHM_TXA_IMP_REG3, LN_TXA_MARGIN_PRE); + set32(tx_shm + LN_AUSPMA_TX_SHM_TXA_IMP_REG3, LN_TXA_MARGIN_PRE_OV); + clear32(tx_shm + LN_AUSPMA_TX_SHM_TXA_IMP_REG3, LN_TXA_MARGIN_PRE_2R); + set32(tx_shm + LN_AUSPMA_TX_SHM_TXA_IMP_REG3, LN_TXA_MARGIN_PRE_2R_OV); + clear32(tx_shm + LN_AUSPMA_TX_SHM_TXA_IMP_REG3, LN_TXA_MARGIN_PRE_4R); + set32(tx_shm + LN_AUSPMA_TX_SHM_TXA_IMP_REG3, LN_TXA_MARGIN_PRE_4R_OV); + + clear32(tx_shm + LN_AUSPMA_TX_SHM_TXA_IMP_REG0, LN_TXA_HIZ); + set32(tx_shm + LN_AUSPMA_TX_SHM_TXA_IMP_REG0, LN_TXA_HIZ_OV); + + clear32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_AFE_CTRL1, LN_RX_DIV20_RESET_N); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_AFE_CTRL1, LN_RX_DIV20_RESET_N_OV); + udelay(10); + + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_AFE_CTRL1, LN_RX_DIV20_RESET_N); + + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_DFE_CTRL12, LN_TX_BYTECLK_RESET_SYNC_EN); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_DFE_CTRL12, LN_TX_BYTECLK_RESET_SYNC_EN_OV); + + mask32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_SAVOS_CTRL16, LN_TX_CAL_CODE, + FIELD_PREP(LN_TX_CAL_CODE, tx_cal_code)); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_SAVOS_CTRL16, LN_TX_CAL_CODE_OV); + + mask32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_TERM_CTRL19, LN_TX_CLK_DLY_CTRL_TAPGEN, + FIELD_PREP(LN_TX_CLK_DLY_CTRL_TAPGEN, 3)); + + clear32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_DFE_CTRL10, LN_DTVREG_ADJUST); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_DFE_CTRL13, LN_DTVREG_ADJUST_OV); + + clear32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_SAVOS_CTRL16, LN_RXTERM_EN); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_SAVOS_CTRL16, LN_RXTERM_EN_OV); + + clear32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_TERM_CTRL19, LN_TX_TEST_EN); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_TERM_CTRL19, LN_TX_TEST_EN_OV); + + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_VREF_CTRL22, LN_VREF_TEST_RXLPBKDT_EN); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_VREF_CTRL22, LN_VREF_TEST_RXLPBKDT_EN_OV); + mask32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_VREF_CTRL22, LN_VREF_LPBKIN_DATA, + FIELD_PREP(LN_VREF_LPBKIN_DATA, 3)); + mask32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_VREF_CTRL22, LN_VREF_BIAS_SEL, + FIELD_PREP(LN_VREF_BIAS_SEL, 2)); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_VREF_CTRL22, LN_VREF_BIAS_SEL_OV); + mask32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_VREF_CTRL22, LN_VREF_ADJUST_GRAY, + FIELD_PREP(LN_VREF_ADJUST_GRAY, 0x18)); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_VREF_CTRL22, LN_VREF_ADJUST_GRAY_OV); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_VREF_CTRL22, LN_VREF_EN); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_VREF_CTRL22, LN_VREF_EN_OV); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_VREF_CTRL22, LN_VREF_BOOST_EN); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_VREF_CTRL22, LN_VREF_BOOST_EN_OV); + udelay(10); + + clear32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_VREF_CTRL22, LN_VREF_BOOST_EN); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_VREF_CTRL22, LN_VREF_BOOST_EN_OV); + udelay(10); + + clear32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_DFE_CTRL13, LN_TX_PRE_EN); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_DFE_CTRL13, LN_TX_PRE_EN_OV); + clear32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_DFE_CTRL13, LN_TX_PST1_EN); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_DFE_CTRL13, LN_TX_PST1_EN_OV); + + clear32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_DFE_CTRL12, LN_TX_PBIAS_EN); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_DFE_CTRL12, LN_TX_PBIAS_EN_OV); + + clear32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_SAVOS_CTRL16, LN_RXTERM_PULLUP_LEAK_EN); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_SAVOS_CTRL16, LN_RXTERM_PULLUP_LEAK_EN_OV); + + set32(rx_top + LN_AUSPMA_RX_TOP_TJ_CFG_RX_TXMODE, LN_RX_TXMODE); + + if (cfg->txa_div2_en) + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_TERM_CTRL19, LN_TX_CLK_DIV2_EN); + else + clear32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_TERM_CTRL19, LN_TX_CLK_DIV2_EN); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_TERM_CTRL19, LN_TX_CLK_DIV2_EN_OV); + + clear32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_TERM_CTRL19, LN_TX_CLK_DIV2_RST); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_TERM_CTRL19, LN_TX_CLK_DIV2_RST_OV); + + clear32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_DFE_CTRL12, LN_TX_HRCLK_SEL); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_DFE_CTRL12, LN_TX_HRCLK_SEL_OV); + + clear32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_TX_CTRL17, LN_TX_MARGIN); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_TX_CTRL17, LN_TX_MARGIN_OV); + clear32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_TX_CTRL17, LN_TX_MARGIN_LSB); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_TX_CTRL17, LN_TX_MARGIN_LSB_OV); + clear32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_TX_CTRL17, LN_TX_MARGIN_P1); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_TX_CTRL17, LN_TX_MARGIN_P1_OV); + clear32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_TX_CTRL17, LN_TX_MARGIN_P1_LSB); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_TX_CTRL17, LN_TX_MARGIN_P1_LSB_OV); + + clear32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_TX_CTRL18, LN_TX_P1_CODE); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_TX_CTRL18, LN_TX_P1_CODE_OV); + clear32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_TX_CTRL18, LN_TX_P1_LSB_CODE); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_TX_CTRL18, LN_TX_P1_LSB_CODE_OV); + clear32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_TX_CTRL18, LN_TX_MARGIN_PRE); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_TX_CTRL18, LN_TX_MARGIN_PRE_OV); + clear32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_TX_CTRL18, LN_TX_MARGIN_PRE_LSB); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_TX_CTRL18, LN_TX_MARGIN_PRE_LSB_OV); + clear32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_TX_CTRL18, LN_TX_PRE_LSB_CODE); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_TX_CTRL18, LN_TX_PRE_LSB_CODE_OV); + clear32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_TX_CTRL18, LN_TX_PRE_CODE); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_TX_CTRL18, LN_TX_PRE_CODE_OV); + + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_DFE_CTRL11, LN_DTVREG_SML_EN); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_DFE_CTRL11, LN_DTVREG_SML_EN_OV); + udelay(10); + + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_DFE_CTRL11, LN_DTVREG_BIG_EN); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_DFE_CTRL11, LN_DTVREG_BIG_EN_OV); + udelay(10); + + mask32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_DFE_CTRL10, LN_DTVREG_ADJUST, + FIELD_PREP(LN_DTVREG_ADJUST, 0xa)); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_DFE_CTRL13, LN_DTVREG_ADJUST_OV); + udelay(10); + + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_TERM_CTRL19, LN_TX_EN); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_TERM_CTRL19, LN_TX_EN_OV); + udelay(10); + + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_CTLE_CTRL0, LN_TX_CLK_EN); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_CTLE_CTRL0, LN_TX_CLK_EN_OV); + + clear32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_DFE_CTRL12, LN_TX_BYTECLK_RESET_SYNC_CLR); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_DFE_CTRL12, LN_TX_BYTECLK_RESET_SYNC_CLR_OV); + + return 0; +} + +static int atcphy_auspll_apb_command(struct apple_atcphy *atcphy, u32 command) +{ + int ret; + u32 reg; + + reg = readl(atcphy->regs.core + AUSPLL_APB_CMD_OVERRIDE); + reg &= ~AUSPLL_APB_CMD_OVERRIDE_CMD; + reg |= FIELD_PREP(AUSPLL_APB_CMD_OVERRIDE_CMD, command); + reg |= AUSPLL_APB_CMD_OVERRIDE_REQ; + reg |= AUSPLL_APB_CMD_OVERRIDE_UNK28; + writel(reg, atcphy->regs.core + AUSPLL_APB_CMD_OVERRIDE); + + ret = readl_poll_timeout(atcphy->regs.core + AUSPLL_APB_CMD_OVERRIDE, reg, + (reg & AUSPLL_APB_CMD_OVERRIDE_ACK), 10, 10000); + if (ret) + dev_warn(atcphy->dev, "AUSPLL APB command was not acked\n"); + + core_clear32(atcphy, AUSPLL_APB_CMD_OVERRIDE, AUSPLL_APB_CMD_OVERRIDE_REQ); + + return 0; +} + +static int atcphy_dp_configure(struct apple_atcphy *atcphy, enum atcphy_dp_link_rate lr) +{ + const struct atcphy_dp_link_rate_configuration *cfg; + const struct atcphy_mode_configuration *mode_cfg; + int ret; + u32 reg; + + guard(mutex)(&atcphy->lock); + mode_cfg = atcphy_get_mode_config(atcphy, atcphy->mode); + cfg = &dp_lr_config[lr]; + + if (atcphy->dp_link_rate == lr) + return 0; + + ret = readl_poll_timeout(atcphy->regs.core + ACIOPHY_CMN_SHM_STS_REG0, reg, + (reg & ACIOPHY_CMN_SHM_STS_REG0_CMD_READY), 10, 10000); + if (ret) { + dev_err(atcphy->dev, "ACIOPHY_CMN_SHM_STS_REG0_CMD_READY not set.\n"); + return ret; + } + + core_clear32(atcphy, AUSPLL_FREQ_CFG, AUSPLL_FREQ_REFCLK); + + core_mask32(atcphy, AUSPLL_FREQ_DESC_A, AUSPLL_FD_FREQ_COUNT_TARGET, + FIELD_PREP(AUSPLL_FD_FREQ_COUNT_TARGET, cfg->freqinit_count_target)); + core_clear32(atcphy, AUSPLL_FREQ_DESC_A, AUSPLL_FD_FBDIVN_HALF); + core_clear32(atcphy, AUSPLL_FREQ_DESC_A, AUSPLL_FD_REV_DIVN); + core_mask32(atcphy, AUSPLL_FREQ_DESC_A, AUSPLL_FD_KI_MAN, FIELD_PREP(AUSPLL_FD_KI_MAN, 8)); + core_mask32(atcphy, AUSPLL_FREQ_DESC_A, AUSPLL_FD_KI_EXP, FIELD_PREP(AUSPLL_FD_KI_EXP, 3)); + core_mask32(atcphy, AUSPLL_FREQ_DESC_A, AUSPLL_FD_KP_MAN, FIELD_PREP(AUSPLL_FD_KP_MAN, 8)); + core_mask32(atcphy, AUSPLL_FREQ_DESC_A, AUSPLL_FD_KP_EXP, FIELD_PREP(AUSPLL_FD_KP_EXP, 7)); + core_clear32(atcphy, AUSPLL_FREQ_DESC_A, AUSPLL_FD_KPKI_SCALE_HBW); + + core_mask32(atcphy, AUSPLL_FREQ_DESC_B, AUSPLL_FD_FBDIVN_FRAC_DEN, + FIELD_PREP(AUSPLL_FD_FBDIVN_FRAC_DEN, cfg->fbdivn_frac_den)); + core_mask32(atcphy, AUSPLL_FREQ_DESC_B, AUSPLL_FD_FBDIVN_FRAC_NUM, + FIELD_PREP(AUSPLL_FD_FBDIVN_FRAC_NUM, cfg->fbdivn_frac_num)); + + core_clear32(atcphy, AUSPLL_FREQ_DESC_C, AUSPLL_FD_SDM_SSC_STEP); + core_clear32(atcphy, AUSPLL_FREQ_DESC_C, AUSPLL_FD_SDM_SSC_EN); + core_mask32(atcphy, AUSPLL_FREQ_DESC_C, AUSPLL_FD_PCLK_DIV_SEL, + FIELD_PREP(AUSPLL_FD_PCLK_DIV_SEL, cfg->pclk_div_sel)); + core_mask32(atcphy, AUSPLL_FREQ_DESC_C, AUSPLL_FD_LFSDM_DIV, + FIELD_PREP(AUSPLL_FD_LFSDM_DIV, 1)); + core_mask32(atcphy, AUSPLL_FREQ_DESC_C, AUSPLL_FD_LFCLK_CTRL, + FIELD_PREP(AUSPLL_FD_LFCLK_CTRL, cfg->lfclk_ctrl)); + core_mask32(atcphy, AUSPLL_FREQ_DESC_C, AUSPLL_FD_VCLK_OP_DIVN, + FIELD_PREP(AUSPLL_FD_VCLK_OP_DIVN, cfg->vclk_op_divn)); + core_set32(atcphy, AUSPLL_FREQ_DESC_C, AUSPLL_FD_VCLK_PRE_DIVN); + + core_mask32(atcphy, AUSPLL_CLKOUT_DIV, AUSPLL_CLKOUT_PLLA_REFBUFCLK_DI, + FIELD_PREP(AUSPLL_CLKOUT_PLLA_REFBUFCLK_DI, 7)); + + if (cfg->plla_clkout_vreg_bypass) + core_set32(atcphy, AUSPLL_CLKOUT_DTC_VREG, AUSPLL_DTC_VREG_BYPASS); + else + core_clear32(atcphy, AUSPLL_CLKOUT_DTC_VREG, AUSPLL_DTC_VREG_BYPASS); + + core_set32(atcphy, AUSPLL_BGR, AUSPLL_BGR_CTRL_AVAIL); + + core_set32(atcphy, AUSPLL_CLKOUT_MASTER, AUSPLL_CLKOUT_MASTER_PCLK_DRVR_EN); + core_set32(atcphy, AUSPLL_CLKOUT_MASTER, AUSPLL_CLKOUT_MASTER_PCLK2_DRVR_EN); + core_set32(atcphy, AUSPLL_CLKOUT_MASTER, AUSPLL_CLKOUT_MASTER_REFBUFCLK_DRVR_EN); + + ret = atcphy_auspll_apb_command(atcphy, 0); + if (ret) + return ret; + + ret = readl_poll_timeout(atcphy->regs.core + ACIOPHY_DP_PCLK_STAT, reg, + (reg & ACIOPHY_AUSPLL_LOCK), 10, 10000); + if (ret) { + dev_err(atcphy->dev, "ACIOPHY_DP_PCLK did not lock.\n"); + return ret; + } + + ret = atcphy_auspll_apb_command(atcphy, 0x2800); + if (ret) + return ret; + + if (mode_cfg->dp_lane[0]) { + ret = atcphy_dp_configure_lane(atcphy, APPLE_ATCPHY_LANE_0, cfg); + if (ret) + return ret; + } + + if (mode_cfg->dp_lane[1]) { + ret = atcphy_dp_configure_lane(atcphy, APPLE_ATCPHY_LANE_1, cfg); + if (ret) + return ret; + } + + core_clear32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, DP_PMA_BYTECLK_RESET); + core_clear32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, DP_MAC_DIV20_CLK_SEL); + + atcphy->dp_link_rate = lr; + return 0; +} + +static void atcphy_usb2_power_off(struct apple_atcphy *atcphy) +{ + /* Disable the PHY, this clears USB2PHY_USBCTL_RUN */ + writel(USB2PHY_USBCTL_ISOLATION, atcphy->regs.usb2phy + USB2PHY_USBCTL); + udelay(10); + + /* Switch the PHY to low power mode */ + set32(atcphy->regs.usb2phy + USB2PHY_CTL, USB2PHY_CTL_SIDDQ); + udelay(10); + + /* Enable all resets */ + set32(atcphy->regs.usb2phy + USB2PHY_CTL, USB2PHY_CTL_PORT_RESET); + udelay(10); + set32(atcphy->regs.usb2phy + USB2PHY_CTL, USB2PHY_CTL_RESET); + udelay(10); + clear32(atcphy->regs.usb2phy + USB2PHY_CTL, USB2PHY_CTL_APB_RESET_N); + udelay(10); + set32(atcphy->regs.usb2phy + USB2PHY_MISCTUNE, USB2PHY_MISCTUNE_APBCLK_GATE_OFF); + set32(atcphy->regs.usb2phy + USB2PHY_MISCTUNE, USB2PHY_MISCTUNE_REFCLK_GATE_OFF); +} + +static int atcphy_power_off(struct apple_atcphy *atcphy) +{ + u32 reg; + int ret; + + atcphy_disable_dp_aux(atcphy); + + /* Enable all reset lines */ + core_clear32(atcphy, ATCPHY_POWER_CTRL, ATCPHY_POWER_PHY_RESET_N); + core_set32(atcphy, ATCPHY_POWER_CTRL, ATCPHY_POWER_CLAMP_EN); + core_clear32(atcphy, ATCPHY_MISC, ATCPHY_MISC_RESET_N | ATCPHY_MISC_LANE_SWAP); + core_clear32(atcphy, ATCPHY_POWER_CTRL, ATCPHY_POWER_APB_RESET_N); + + core_clear32(atcphy, ATCPHY_POWER_CTRL, ATCPHY_POWER_SLEEP_BIG); + ret = readl_poll_timeout(atcphy->regs.core + ATCPHY_POWER_STAT, reg, + !(reg & ATCPHY_POWER_SLEEP_BIG), 10, 1000); + if (ret) { + dev_err(atcphy->dev, "Failed to sleep atcphy \"big\"\n"); + return ret; + } + + core_clear32(atcphy, ATCPHY_POWER_CTRL, ATCPHY_POWER_SLEEP_SMALL); + ret = readl_poll_timeout(atcphy->regs.core + ATCPHY_POWER_STAT, reg, + !(reg & ATCPHY_POWER_SLEEP_SMALL), 10, 1000); + if (ret) { + dev_err(atcphy->dev, "Failed to sleep atcphy \"small\"\n"); + return ret; + } + + return 0; +} + +static void atcphy_usb2_power_on(struct apple_atcphy *atcphy) +{ + set32(atcphy->regs.usb2phy + USB2PHY_SIG, + USB2PHY_SIG_VBUSDET_FORCE_VAL | USB2PHY_SIG_VBUSDET_FORCE_EN | + USB2PHY_SIG_VBUSVLDEXT_FORCE_VAL | USB2PHY_SIG_VBUSVLDEXT_FORCE_EN); + udelay(10); + + /* Take the PHY out of its low power state */ + clear32(atcphy->regs.usb2phy + USB2PHY_CTL, USB2PHY_CTL_SIDDQ); + udelay(10); + + /* Release reset */ + clear32(atcphy->regs.usb2phy + USB2PHY_CTL, USB2PHY_CTL_RESET); + udelay(10); + clear32(atcphy->regs.usb2phy + USB2PHY_CTL, USB2PHY_CTL_PORT_RESET); + udelay(10); + set32(atcphy->regs.usb2phy + USB2PHY_CTL, USB2PHY_CTL_APB_RESET_N); + udelay(10); + clear32(atcphy->regs.usb2phy + USB2PHY_MISCTUNE, USB2PHY_MISCTUNE_APBCLK_GATE_OFF); + clear32(atcphy->regs.usb2phy + USB2PHY_MISCTUNE, USB2PHY_MISCTUNE_REFCLK_GATE_OFF); + + /* Enable the PHY */ + writel(USB2PHY_USBCTL_RUN, atcphy->regs.usb2phy + USB2PHY_USBCTL); +} + +static int atcphy_power_on(struct apple_atcphy *atcphy) +{ + u32 reg; + int ret; + + atcphy_usb2_power_on(atcphy); + + core_set32(atcphy, ATCPHY_MISC, ATCPHY_MISC_RESET_N); + + core_set32(atcphy, ATCPHY_POWER_CTRL, ATCPHY_POWER_SLEEP_SMALL); + ret = readl_poll_timeout(atcphy->regs.core + ATCPHY_POWER_STAT, reg, + reg & ATCPHY_POWER_SLEEP_SMALL, 100, 100000); + if (ret) { + dev_err(atcphy->dev, "failed to wakeup atcphy \"small\"\n"); + return ret; + } + + core_set32(atcphy, ATCPHY_POWER_CTRL, ATCPHY_POWER_SLEEP_BIG); + ret = readl_poll_timeout(atcphy->regs.core + ATCPHY_POWER_STAT, reg, + reg & ATCPHY_POWER_SLEEP_BIG, 100, 100000); + if (ret) { + dev_err(atcphy->dev, "failed to wakeup atcphy \"big\"\n"); + return ret; + } + + core_clear32(atcphy, ATCPHY_POWER_CTRL, ATCPHY_POWER_CLAMP_EN); + core_set32(atcphy, ATCPHY_POWER_CTRL, ATCPHY_POWER_APB_RESET_N); + + return 0; +} + +static int atcphy_configure(struct apple_atcphy *atcphy, enum atcphy_mode mode) +{ + int ret = 0; + + lockdep_assert_held(&atcphy->lock); + + if (mode == APPLE_ATCPHY_MODE_OFF) { + ret = atcphy_power_off(atcphy); + atcphy->mode = mode; + return ret; + } + + ret = atcphy_power_on(atcphy); + if (ret) + return ret; + + atcphy_apply_tunables(atcphy, mode); + + core_set32(atcphy, AUSPLL_FSM_CTRL, 0x1fe000); + core_set32(atcphy, AUSPLL_APB_CMD_OVERRIDE, AUSPLL_APB_CMD_OVERRIDE_UNK28); + + set32(atcphy->regs.core + ACIOPHY_CFG0, ACIOPHY_CFG0_COMMON_SMALL_OV); + udelay(10); + set32(atcphy->regs.core + ACIOPHY_CFG0, ACIOPHY_CFG0_COMMON_BIG_OV); + udelay(10); + set32(atcphy->regs.core + ACIOPHY_CFG0, ACIOPHY_CFG0_COMMON_CLAMP_OV); + udelay(10); + + mask32(atcphy->regs.core + ACIOPHY_SLEEP_CTRL, ACIOPHY_SLEEP_CTRL_TX_SMALL_OV, + FIELD_PREP(ACIOPHY_SLEEP_CTRL_TX_SMALL_OV, 3)); + udelay(10); + mask32(atcphy->regs.core + ACIOPHY_SLEEP_CTRL, ACIOPHY_SLEEP_CTRL_TX_BIG_OV, + FIELD_PREP(ACIOPHY_SLEEP_CTRL_TX_BIG_OV, 3)); + udelay(10); + mask32(atcphy->regs.core + ACIOPHY_SLEEP_CTRL, ACIOPHY_SLEEP_CTRL_TX_CLAMP_OV, + FIELD_PREP(ACIOPHY_SLEEP_CTRL_TX_CLAMP_OV, 3)); + udelay(10); + + mask32(atcphy->regs.core + ACIOPHY_CFG0, ACIOPHY_CFG0_RX_BIG_OV, + FIELD_PREP(ACIOPHY_CFG0_RX_BIG_OV, 3)); + udelay(10); + mask32(atcphy->regs.core + ACIOPHY_CFG0, ACIOPHY_CFG0_RX_SMALL_OV, + FIELD_PREP(ACIOPHY_CFG0_RX_SMALL_OV, 3)); + udelay(10); + mask32(atcphy->regs.core + ACIOPHY_CFG0, ACIOPHY_CFG0_RX_CLAMP_OV, + FIELD_PREP(ACIOPHY_CFG0_RX_CLAMP_OV, 3)); + udelay(10); + + /* Setup AUX channel if DP altmode is requested */ + if (atcphy_modes[mode].enable_dp_aux) + atcphy_enable_dp_aux(atcphy); + + /* Enable clocks and configure lanes */ + core_set32(atcphy, CIO3PLL_CLK_CTRL, CIO3PLL_CLK_PCLK_EN); + core_set32(atcphy, CIO3PLL_CLK_CTRL, CIO3PLL_CLK_REFCLK_EN); + atcphy_configure_lanes(atcphy, mode); + + /* Take the USB3 PHY out of reset */ + core_set32(atcphy, ATCPHY_POWER_CTRL, ATCPHY_POWER_PHY_RESET_N); + + atcphy->mode = mode; + + return 0; +} + +static int atcphy_usb2_set_mode(struct phy *phy, enum phy_mode mode, int submode) +{ + struct apple_atcphy *atcphy = phy_get_drvdata(phy); + + guard(mutex)(&atcphy->lock); + + switch (mode) { + case PHY_MODE_USB_HOST: + set32(atcphy->regs.usb2phy + USB2PHY_SIG, USB2PHY_SIG_HOST); + break; + case PHY_MODE_USB_DEVICE: + clear32(atcphy->regs.usb2phy + USB2PHY_SIG, USB2PHY_SIG_HOST); + break; + default: + return -EINVAL; + } + + return 0; +} + +static const struct phy_ops apple_atc_usb2_phy_ops = { + .owner = THIS_MODULE, + .set_mode = atcphy_usb2_set_mode, +}; + +static int atcphy_usb3_power_off(struct phy *phy) +{ + struct apple_atcphy *atcphy = phy_get_drvdata(phy); + int ret; + + guard(mutex)(&atcphy->lock); + + ret = atcphy_configure_pipehandler_dummy(atcphy); + if (ret) + dev_warn(atcphy->dev, "Failed to switch pipe to dummy: %d", ret); + + atcphy->pipehandler_up = false; + + if (atcphy->mode != APPLE_ATCPHY_MODE_OFF) + atcphy_configure(atcphy, APPLE_ATCPHY_MODE_OFF); + + return 0; +} + +static int atcphy_usb3_set_mode(struct phy *phy, enum phy_mode mode, int submode) +{ + struct apple_atcphy *atcphy = phy_get_drvdata(phy); + + guard(mutex)(&atcphy->lock); + + /* + * We may get multiple calls to set_mode (for host mode e.g. at least one from the dwc3 glue + * driver and then another one from the generic xhci code) but must only configure the + * PIPE handler once. + */ + if (atcphy->pipehandler_up) + return 0; + + switch (mode) { + case PHY_MODE_USB_HOST: + return atcphy_configure_pipehandler(atcphy, true); + case PHY_MODE_USB_DEVICE: + return atcphy_configure_pipehandler(atcphy, false); + default: + return -EINVAL; + } +} + +static const struct phy_ops apple_atc_usb3_phy_ops = { + .owner = THIS_MODULE, + .power_off = atcphy_usb3_power_off, + .set_mode = atcphy_usb3_set_mode, +}; + +static int atcphy_dpphy_set_mode(struct phy *phy, enum phy_mode mode, int submode) +{ + /* Nothing to do here since the setup already happened in mux_set */ + if (mode == PHY_MODE_DP && submode == 0) + return 0; + return -EINVAL; +} + +static int atcphy_dpphy_validate(struct phy *phy, enum phy_mode mode, int submode, + union phy_configure_opts *opts_) +{ + struct phy_configure_opts_dp *opts = &opts_->dp; + struct apple_atcphy *atcphy = phy_get_drvdata(phy); + + if (mode != PHY_MODE_DP) + return -EINVAL; + if (submode != 0) + return -EINVAL; + + switch (atcphy->mode) { + case APPLE_ATCPHY_MODE_USB3_DP: + opts->lanes = 2; + break; + case APPLE_ATCPHY_MODE_DP: + opts->lanes = 4; + break; + default: + opts->lanes = 0; + } + + return 0; +} + +static int atcphy_dpphy_configure(struct phy *phy, union phy_configure_opts *opts_) +{ + struct phy_configure_opts_dp *opts = &opts_->dp; + struct apple_atcphy *atcphy = phy_get_drvdata(phy); + enum atcphy_dp_link_rate link_rate; + + if (opts->set_voltages) + return -EINVAL; + if (opts->set_lanes) + return -EINVAL; + + if (opts->set_rate) { + switch (opts->link_rate) { + case 1620: + link_rate = ATCPHY_DP_LINK_RATE_RBR; + break; + case 2700: + link_rate = ATCPHY_DP_LINK_RATE_HBR; + break; + case 5400: + link_rate = ATCPHY_DP_LINK_RATE_HBR2; + break; + case 8100: + link_rate = ATCPHY_DP_LINK_RATE_HBR3; + break; + case 0: + return 0; + default: + dev_err(atcphy->dev, "Unsupported link rate: %d\n", opts->link_rate); + return -EINVAL; + } + + return atcphy_dp_configure(atcphy, link_rate); + } + + return 0; +} + +static const struct phy_ops apple_atc_dp_phy_ops = { + .owner = THIS_MODULE, + .configure = atcphy_dpphy_configure, + .validate = atcphy_dpphy_validate, + .set_mode = atcphy_dpphy_set_mode, +}; + +static struct phy *atcphy_xlate(struct device *dev, const struct of_phandle_args *args) +{ + struct apple_atcphy *atcphy = dev_get_drvdata(dev); + + switch (args->args[0]) { + case PHY_TYPE_USB2: + return atcphy->phys.usb2; + case PHY_TYPE_USB3: + return atcphy->phys.usb3; + case PHY_TYPE_DP: + return atcphy->phys.dp; + } + return ERR_PTR(-ENODEV); +} + +static int atcphy_probe_phy(struct apple_atcphy *atcphy) +{ + struct { + struct phy **phy; + const struct phy_ops *ops; + } phys[] = { + { &atcphy->phys.usb2, &apple_atc_usb2_phy_ops }, + { &atcphy->phys.usb3, &apple_atc_usb3_phy_ops }, + { &atcphy->phys.dp, &apple_atc_dp_phy_ops }, + }; + + for (int i = 0; i < ARRAY_SIZE(phys); i++) { + *phys[i].phy = devm_phy_create(atcphy->dev, NULL, phys[i].ops); + if (IS_ERR(*phys[i].phy)) + return PTR_ERR(*phys[i].phy); + phy_set_drvdata(*phys[i].phy, atcphy); + } + + atcphy->phy_provider = devm_of_phy_provider_register(atcphy->dev, atcphy_xlate); + if (IS_ERR(atcphy->phy_provider)) + return PTR_ERR(atcphy->phy_provider); + return 0; +} + +static void _atcphy_dwc3_reset_assert(struct apple_atcphy *atcphy) +{ + lockdep_assert_held(&atcphy->lock); + + clear32(atcphy->regs.pipehandler + PIPEHANDLER_AON_GEN, PIPEHANDLER_AON_GEN_DWC3_RESET_N); + set32(atcphy->regs.pipehandler + PIPEHANDLER_AON_GEN, + PIPEHANDLER_AON_GEN_DWC3_FORCE_CLAMP_EN); +} + +static int atcphy_dwc3_reset_assert(struct reset_controller_dev *rcdev, unsigned long id) +{ + struct apple_atcphy *atcphy = container_of(rcdev, struct apple_atcphy, rcdev); + int ret; + + guard(mutex)(&atcphy->lock); + + _atcphy_dwc3_reset_assert(atcphy); + + if (atcphy->pipehandler_up) { + ret = atcphy_configure_pipehandler_dummy(atcphy); + if (ret) + dev_warn(atcphy->dev, "Failed to switch PIPE to dummy: %d\n", ret); + else + atcphy->pipehandler_up = false; + } + + atcphy_usb2_power_off(atcphy); + + return 0; +} + +static int atcphy_dwc3_reset_deassert(struct reset_controller_dev *rcdev, unsigned long id) +{ + struct apple_atcphy *atcphy = container_of(rcdev, struct apple_atcphy, rcdev); + + guard(mutex)(&atcphy->lock); + + clear32(atcphy->regs.pipehandler + PIPEHANDLER_AON_GEN, + PIPEHANDLER_AON_GEN_DWC3_FORCE_CLAMP_EN); + set32(atcphy->regs.pipehandler + PIPEHANDLER_AON_GEN, PIPEHANDLER_AON_GEN_DWC3_RESET_N); + + return 0; +} + +const struct reset_control_ops atcphy_dwc3_reset_ops = { + .assert = atcphy_dwc3_reset_assert, + .deassert = atcphy_dwc3_reset_deassert, +}; + +static int atcphy_reset_xlate(struct reset_controller_dev *rcdev, + const struct of_phandle_args *reset_spec) +{ + return 0; +} + +static int atcphy_probe_rcdev(struct apple_atcphy *atcphy) +{ + atcphy->rcdev.owner = THIS_MODULE; + atcphy->rcdev.nr_resets = 1; + atcphy->rcdev.ops = &atcphy_dwc3_reset_ops; + atcphy->rcdev.of_node = atcphy->dev->of_node; + atcphy->rcdev.of_reset_n_cells = 0; + atcphy->rcdev.of_xlate = atcphy_reset_xlate; + + return devm_reset_controller_register(atcphy->dev, &atcphy->rcdev); +} + +static int atcphy_sw_set(struct typec_switch_dev *sw, enum typec_orientation orientation) +{ + struct apple_atcphy *atcphy = typec_switch_get_drvdata(sw); + + guard(mutex)(&atcphy->lock); + + switch (orientation) { + case TYPEC_ORIENTATION_NONE: + break; + case TYPEC_ORIENTATION_NORMAL: + atcphy->swap_lanes = false; + break; + case TYPEC_ORIENTATION_REVERSE: + atcphy->swap_lanes = true; + break; + } + + return 0; +} + +static int atcphy_probe_switch(struct apple_atcphy *atcphy) +{ + struct typec_switch_desc sw_desc = { + .drvdata = atcphy, + .fwnode = atcphy->dev->fwnode, + .set = atcphy_sw_set, + }; + + return PTR_ERR_OR_ZERO(typec_switch_register(atcphy->dev, &sw_desc)); +} + +static int atcphy_mux_set(struct typec_mux_dev *mux, struct typec_mux_state *state) +{ + struct apple_atcphy *atcphy = typec_mux_get_drvdata(mux); + enum atcphy_mode target_mode; + + guard(mutex)(&atcphy->lock); + + if (state->mode == TYPEC_STATE_SAFE) { + target_mode = APPLE_ATCPHY_MODE_OFF; + } else if (state->mode == TYPEC_STATE_USB) { + target_mode = APPLE_ATCPHY_MODE_USB3; + } else if (!state->alt && state->mode == TYPEC_MODE_USB4) { + struct enter_usb_data *data = state->data; + u32 eudo_usb_mode = FIELD_GET(EUDO_USB_MODE_MASK, data->eudo); + + switch (eudo_usb_mode) { + case EUDO_USB_MODE_USB2: + target_mode = APPLE_ATCPHY_MODE_USB2; + break; + case EUDO_USB_MODE_USB3: + target_mode = APPLE_ATCPHY_MODE_USB3; + break; + case EUDO_USB_MODE_USB4: + target_mode = APPLE_ATCPHY_MODE_USB4; + break; + default: + dev_warn(atcphy->dev, "Unsupported EUDO USB mode: 0x%x.\n", eudo_usb_mode); + target_mode = APPLE_ATCPHY_MODE_OFF; + } + } else if (state->alt && state->alt->svid == USB_TYPEC_TBT_SID) { + target_mode = APPLE_ATCPHY_MODE_TBT; + } else if (state->alt && state->alt->svid == USB_TYPEC_DP_SID) { + switch (state->mode) { + case TYPEC_DP_STATE_C: + case TYPEC_DP_STATE_E: + target_mode = APPLE_ATCPHY_MODE_DP; + break; + case TYPEC_DP_STATE_D: + target_mode = APPLE_ATCPHY_MODE_USB3_DP; + break; + default: + dev_err(atcphy->dev, + "Unsupported DP pin assignment: 0x%lx, your connected device will not work.\n", + state->mode); + target_mode = APPLE_ATCPHY_MODE_OFF; + } + } else if (state->alt) { + dev_err(atcphy->dev, + "Unknown alternate mode SVID: 0x%x, your connected device will not work.\n", + state->alt->svid); + target_mode = APPLE_ATCPHY_MODE_OFF; + } else { + dev_err(atcphy->dev, "Unknown mode: 0x%lx, your connected device will not work.\n", + state->mode); + target_mode = APPLE_ATCPHY_MODE_OFF; + } + + if (atcphy->mode == target_mode) + return 0; + + /* + * If the pipehandler is still/already up here there's a bug somewhere so make sure to + * complain loudly. We can still try to switch modes and hope for the best though, + * in the worst case the hardware will fall back to USB2-only. + */ + WARN_ON_ONCE(atcphy->pipehandler_up); + return atcphy_configure(atcphy, target_mode); +} + +static int atcphy_probe_mux(struct apple_atcphy *atcphy) +{ + struct typec_mux_desc mux_desc = { + .drvdata = atcphy, + .fwnode = atcphy->dev->fwnode, + .set = atcphy_mux_set, + }; + + return PTR_ERR_OR_ZERO(typec_mux_register(atcphy->dev, &mux_desc)); +} + +static int atcphy_load_tunables(struct apple_atcphy *atcphy) +{ + struct { + const char *dt_name; + struct apple_tunable **tunable; + struct resource *res; + } tunables[] = { + { "apple,tunable-axi2af", &atcphy->tunables.axi2af, atcphy->res.axi2af }, + { "apple,tunable-common-a", &atcphy->tunables.common[0], atcphy->res.core }, + { "apple,tunable-common-b", &atcphy->tunables.common[1], atcphy->res.core }, + { "apple,tunable-lane0-usb", &atcphy->tunables.lane_usb3[0], atcphy->res.core }, + { "apple,tunable-lane1-usb", &atcphy->tunables.lane_usb3[1], atcphy->res.core }, + { "apple,tunable-lane0-cio", &atcphy->tunables.lane_usb4[0], atcphy->res.core }, + { "apple,tunable-lane1-cio", &atcphy->tunables.lane_usb4[1], atcphy->res.core }, + { "apple,tunable-lane0-dp", &atcphy->tunables.lane_dp[0], atcphy->res.core }, + { "apple,tunable-lane1-dp", &atcphy->tunables.lane_dp[1], atcphy->res.core }, + }; + + for (int i = 0; i < ARRAY_SIZE(tunables); i++) { + *tunables[i].tunable = devm_apple_tunable_parse( + atcphy->dev, atcphy->np, tunables[i].dt_name, tunables[i].res); + if (IS_ERR(*tunables[i].tunable)) { + dev_err(atcphy->dev, "Failed to read tunable %s: %ld\n", + tunables[i].dt_name, PTR_ERR(*tunables[i].tunable)); + return PTR_ERR(*tunables[i].tunable); + } + } + + return 0; +} + +static int atcphy_map_resources(struct platform_device *pdev, struct apple_atcphy *atcphy) +{ + struct { + const char *name; + void __iomem **addr; + struct resource **res; + } resources[] = { + { "core", &atcphy->regs.core, &atcphy->res.core }, + { "lpdptx", &atcphy->regs.lpdptx, NULL }, + { "axi2af", &atcphy->regs.axi2af, &atcphy->res.axi2af }, + { "usb2phy", &atcphy->regs.usb2phy, NULL }, + { "pipehandler", &atcphy->regs.pipehandler, NULL }, + }; + struct resource *res; + void __iomem *addr; + + for (int i = 0; i < ARRAY_SIZE(resources); i++) { + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, resources[i].name); + addr = devm_ioremap_resource(&pdev->dev, res); + if (IS_ERR(addr)) + return dev_err_probe(atcphy->dev, PTR_ERR(addr), + "Unable to map %s regs", resources[i].name); + + *resources[i].addr = addr; + if (resources[i].res) + *resources[i].res = res; + } + + return 0; +} + +static int atcphy_probe_finalize(struct apple_atcphy *atcphy) +{ + int ret; + + guard(mutex)(&atcphy->lock); + + /* Reset dwc3 on probe, let dwc3 (consumer) deassert it */ + _atcphy_dwc3_reset_assert(atcphy); + + /* Reset atcphy to clear any state potentially left by the bootloader */ + atcphy_usb2_power_off(atcphy); + atcphy_power_off(atcphy); + atcphy_setup_pipehandler(atcphy); + + ret = atcphy_probe_rcdev(atcphy); + if (ret) + return dev_err_probe(atcphy->dev, ret, "Probing rcdev failed"); + ret = atcphy_probe_mux(atcphy); + if (ret) + return dev_err_probe(atcphy->dev, ret, "Probing mux failed"); + ret = atcphy_probe_switch(atcphy); + if (ret) + return dev_err_probe(atcphy->dev, ret, "Probing switch failed"); + ret = atcphy_probe_phy(atcphy); + if (ret) + return dev_err_probe(atcphy->dev, ret, "Probing phy failed"); + + return 0; +} + +static int atcphy_probe(struct platform_device *pdev) +{ + struct apple_atcphy *atcphy; + struct device *dev = &pdev->dev; + int ret; + + atcphy = devm_kzalloc(&pdev->dev, sizeof(*atcphy), GFP_KERNEL); + if (!atcphy) + return -ENOMEM; + + atcphy->dev = dev; + atcphy->np = dev->of_node; + mutex_init(&atcphy->lock); + platform_set_drvdata(pdev, atcphy); + + ret = atcphy_map_resources(pdev, atcphy); + if (ret) + return ret; + ret = atcphy_load_tunables(atcphy); + if (ret) + return ret; + + atcphy->mode = APPLE_ATCPHY_MODE_OFF; + atcphy->pipehandler_up = false; + + return atcphy_probe_finalize(atcphy); +} + +static const struct of_device_id atcphy_match[] = { + { .compatible = "apple,t8103-atcphy" }, + {}, +}; +MODULE_DEVICE_TABLE(of, atcphy_match); + +static struct platform_driver atcphy_driver = { + .driver = { + .name = "phy-apple-atc", + .of_match_table = atcphy_match, + }, + .probe = atcphy_probe, +}; +module_platform_driver(atcphy_driver); + +MODULE_AUTHOR("Sven Peter "); +MODULE_DESCRIPTION("Apple Type-C PHY driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/phy/apple/dptx.c b/drivers/phy/apple/dptx.c new file mode 100644 index 00000000000000..f0df2d40a18023 --- /dev/null +++ b/drivers/phy/apple/dptx.c @@ -0,0 +1,690 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause +/* + * Apple dptx PHY driver + * + * Copyright (C) The Asahi Linux Contributors + * Author: Janne Grunau + * + * based on drivers/phy/apple/atc.c + * + * Copyright (C) The Asahi Linux Contributors + * Author: Sven Peter + */ + +#include "dptx.h" + +#include +#include "linux/of.h" +#include +#include +#include +#include +#include +#include +#include + +#define DPTX_MAX_LANES 4 +#define DPTX_LANE0_OFFSET 0x5000 +#define DPTX_LANE_STRIDE 0x1000 +#define DPTX_LANE_END (DPTX_LANE0_OFFSET + DPTX_MAX_LANES * DPTX_LANE_STRIDE) + +enum apple_dptx_type { + DPTX_PHY_T8112, + DPTX_PHY_T6020, +}; + +struct apple_dptx_phy_hw { + enum apple_dptx_type type; +}; + +struct apple_dptx_phy { + struct device *dev; + + struct apple_dptx_phy_hw hw; + + int dp_link_rate; + + struct { + void __iomem *core; + void __iomem *dptx; + } regs; + + struct phy *phy_dp; + struct phy_provider *phy_provider; + + struct mutex lock; + + // TODO: m1n1 port things to clean up + u32 active_lanes; +}; + + +static inline void mask32(void __iomem *reg, u32 mask, u32 set) +{ + u32 value = readl(reg); + value &= ~mask; + value |= set; + writel(value, reg); +} + +static inline void set32(void __iomem *reg, u32 set) +{ + mask32(reg, 0, set); +} + +static inline void clear32(void __iomem *reg, u32 clear) +{ + mask32(reg, clear, 0); +} + + +static int dptx_phy_set_active_lane_count(struct apple_dptx_phy *phy, u32 num_lanes) +{ + u32 l, ctrl; + + dev_dbg(phy->dev, "set_active_lane_count(%u)\n", num_lanes); + + if (num_lanes == 3 || num_lanes > DPTX_MAX_LANES) + return -1; + + ctrl = readl(phy->regs.dptx + 0x4000); + writel(ctrl, phy->regs.dptx + 0x4000); + + for (l = 0; l < num_lanes; l++) { + u64 offset = 0x5000 + 0x1000 * l; + readl(phy->regs.dptx + offset); + writel(0x100, phy->regs.dptx + offset); + } + for (; l < DPTX_MAX_LANES; l++) { + u64 offset = 0x5000 + 0x1000 * l; + readl(phy->regs.dptx + offset); + writel(0x300, phy->regs.dptx + offset); + } + for (l = 0; l < num_lanes; l++) { + u64 offset = 0x5000 + 0x1000 * l; + readl(phy->regs.dptx + offset); + writel(0x0, phy->regs.dptx + offset); + } + for (; l < DPTX_MAX_LANES; l++) { + u64 offset = 0x5000 + 0x1000 * l; + readl(phy->regs.dptx + offset); + writel(0x300, phy->regs.dptx + offset); + } + + if (num_lanes > 0) { + // clear32(phy->regs.dptx + 0x4000, 0x4000000); + ctrl = readl(phy->regs.dptx + 0x4000); + ctrl &= ~0x4000000; + writel(ctrl, phy->regs.dptx + 0x4000); + } + phy->active_lanes = num_lanes; + + return 0; +} + +static int dptx_phy_activate(struct apple_dptx_phy *phy, u32 dcp_index) +{ + u32 val_2014; + u32 val_4008; + u32 val_4408; + + dev_dbg(phy->dev, "activate(dcp:%u)\n", dcp_index); + + // MMIO: R.4 0x23c500010 (dptx-phy[1], offset 0x10) = 0x0 + // MMIO: W.4 0x23c500010 (dptx-phy[1], offset 0x10) = 0x0 + readl(phy->regs.core + 0x10); + writel(dcp_index, phy->regs.core + 0x10); + + // MMIO: R.4 0x23c500048 (dptx-phy[1], offset 0x48) = 0x444 + // MMIO: W.4 0x23c500048 (dptx-phy[1], offset 0x48) = 0x454 + set32(phy->regs.core + 0x48, 0x010); + // MMIO: R.4 0x23c500048 (dptx-phy[1], offset 0x48) = 0x454 + // MMIO: W.4 0x23c500048 (dptx-phy[1], offset 0x48) = 0x474 + set32(phy->regs.core + 0x48, 0x020); + // MMIO: R.4 0x23c500048 (dptx-phy[1], offset 0x48) = 0x474 + // MMIO: W.4 0x23c500048 (dptx-phy[1], offset 0x48) = 0x434 + clear32(phy->regs.core + 0x48, 0x040); + // MMIO: R.4 0x23c500048 (dptx-phy[1], offset 0x48) = 0x434 + // MMIO: W.4 0x23c500048 (dptx-phy[1], offset 0x48) = 0x534 + set32(phy->regs.core + 0x48, 0x100); + // MMIO: R.4 0x23c500048 (dptx-phy[1], offset 0x48) = 0x534 + // MMIO: W.4 0x23c500048 (dptx-phy[1], offset 0x48) = 0x734 + set32(phy->regs.core + 0x48, 0x200); + // MMIO: R.4 0x23c500048 (dptx-phy[1], offset 0x48) = 0x734 + // MMIO: W.4 0x23c500048 (dptx-phy[1], offset 0x48) = 0x334 + clear32(phy->regs.core + 0x48, 0x400); + // MMIO: R.4 0x23c500048 (dptx-phy[1], offset 0x48) = 0x334 + // MMIO: W.4 0x23c500048 (dptx-phy[1], offset 0x48) = 0x335 + set32(phy->regs.core + 0x48, 0x001); + // MMIO: R.4 0x23c500048 (dptx-phy[1], offset 0x48) = 0x335 + // MMIO: W.4 0x23c500048 (dptx-phy[1], offset 0x48) = 0x337 + set32(phy->regs.core + 0x48, 0x002); + // MMIO: R.4 0x23c500048 (dptx-phy[1], offset 0x48) = 0x337 + // MMIO: W.4 0x23c500048 (dptx-phy[1], offset 0x48) = 0x333 + clear32(phy->regs.core + 0x48, 0x004); + + // MMIO: R.4 0x23c542014 (dptx-phy[0], offset 0x2014) = 0x80a0c + val_2014 = readl(phy->regs.dptx + 0x2014); + // MMIO: W.4 0x23c542014 (dptx-phy[0], offset 0x2014) = 0x300a0c + writel((0x30 << 16) | (val_2014 & 0xffff), phy->regs.dptx + 0x2014); + + // MMIO: R.4 0x23c5420b8 (dptx-phy[0], offset 0x20b8) = 0x644800 + // MMIO: W.4 0x23c5420b8 (dptx-phy[0], offset 0x20b8) = 0x654800 + set32(phy->regs.dptx + 0x20b8, 0x010000); + + // MMIO: R.4 0x23c542220 (dptx-phy[0], offset 0x2220) = 0x11090a2 + // MMIO: W.4 0x23c542220 (dptx-phy[0], offset 0x2220) = 0x11090a0 + clear32(phy->regs.dptx + 0x2220, 0x0000002); + + // MMIO: R.4 0x23c54222c (dptx-phy[0], offset 0x222c) = 0x103003 + // MMIO: W.4 0x23c54222c (dptx-phy[0], offset 0x222c) = 0x103803 + set32(phy->regs.dptx + 0x222c, 0x000800); + // MMIO: R.4 0x23c54222c (dptx-phy[0], offset 0x222c) = 0x103803 + // MMIO: W.4 0x23c54222c (dptx-phy[0], offset 0x222c) = 0x103903 + set32(phy->regs.dptx + 0x222c, 0x000100); + + // MMIO: R.4 0x23c542230 (dptx-phy[0], offset 0x2230) = 0x2308804 + // MMIO: W.4 0x23c542230 (dptx-phy[0], offset 0x2230) = 0x2208804 + clear32(phy->regs.dptx + 0x2230, 0x0100000); + + // MMIO: R.4 0x23c542278 (dptx-phy[0], offset 0x2278) = 0x18300811 + // MMIO: W.4 0x23c542278 (dptx-phy[0], offset 0x2278) = 0x10300811 + clear32(phy->regs.dptx + 0x2278, 0x08000000); + + // MMIO: R.4 0x23c5422a4 (dptx-phy[0], offset 0x22a4) = 0x1044200 + // MMIO: W.4 0x23c5422a4 (dptx-phy[0], offset 0x22a4) = 0x1044201 + set32(phy->regs.dptx + 0x22a4, 0x0000001); + + // MMIO: R.4 0x23c544008 (dptx-phy[0], offset 0x4008) = 0x18030 + val_4008 = readl(phy->regs.dptx + 0x4008); + // MMIO: W.4 0x23c544008 (dptx-phy[0], offset 0x4008) = 0x30030 + writel((0x6 << 15) | (val_4008 & 0x7fff), phy->regs.dptx + 0x4008); + // MMIO: R.4 0x23c544008 (dptx-phy[0], offset 0x4008) = 0x30030 + // MMIO: W.4 0x23c544008 (dptx-phy[0], offset 0x4008) = 0x30010 + clear32(phy->regs.dptx + 0x4008, 0x00020); + + // MMIO: R.4 0x23c54420c (dptx-phy[0], offset 0x420c) = 0x88e3 + // MMIO: W.4 0x23c54420c (dptx-phy[0], offset 0x420c) = 0x88c3 + clear32(phy->regs.dptx + 0x420c, 0x0020); + + // MMIO: R.4 0x23c544600 (dptx-phy[0], offset 0x4600) = 0x0 + // MMIO: W.4 0x23c544600 (dptx-phy[0], offset 0x4600) = 0x8000000 + set32(phy->regs.dptx + 0x4600, 0x8000000); + + // MMIO: R.4 0x23c545040 (dptx-phy[0], offset 0x5040) = 0x21780 + // MMIO: W.4 0x23c545040 (dptx-phy[0], offset 0x5040) = 0x221780 + // MMIO: R.4 0x23c546040 (dptx-phy[0], offset 0x6040) = 0x21780 + // MMIO: W.4 0x23c546040 (dptx-phy[0], offset 0x6040) = 0x221780 + // MMIO: R.4 0x23c547040 (dptx-phy[0], offset 0x7040) = 0x21780 + // MMIO: W.4 0x23c547040 (dptx-phy[0], offset 0x7040) = 0x221780 + // MMIO: R.4 0x23c548040 (dptx-phy[0], offset 0x8040) = 0x21780 + // MMIO: W.4 0x23c548040 (dptx-phy[0], offset 0x8040) = 0x221780 + for (u32 loff = DPTX_LANE0_OFFSET; loff < DPTX_LANE_END; + loff += DPTX_LANE_STRIDE) + set32(phy->regs.dptx + loff + 0x40, 0x200000); + + // MMIO: R.4 0x23c545040 (dptx-phy[0], offset 0x5040) = 0x221780 + // MMIO: W.4 0x23c545040 (dptx-phy[0], offset 0x5040) = 0x2a1780 + // MMIO: R.4 0x23c546040 (dptx-phy[0], offset 0x6040) = 0x221780 + // MMIO: W.4 0x23c546040 (dptx-phy[0], offset 0x6040) = 0x2a1780 + // MMIO: R.4 0x23c547040 (dptx-phy[0], offset 0x7040) = 0x221780 + // MMIO: W.4 0x23c547040 (dptx-phy[0], offset 0x7040) = 0x2a1780 + // MMIO: R.4 0x23c548040 (dptx-phy[0], offset 0x8040) = 0x221780 + // MMIO: W.4 0x23c548040 (dptx-phy[0], offset 0x8040) = 0x2a1780 + for (u32 loff = DPTX_LANE0_OFFSET; loff < DPTX_LANE_END; + loff += DPTX_LANE_STRIDE) + set32(phy->regs.dptx + loff + 0x40, 0x080000); + + // MMIO: R.4 0x23c545244 (dptx-phy[0], offset 0x5244) = 0x18 + // MMIO: W.4 0x23c545244 (dptx-phy[0], offset 0x5244) = 0x8 + // MMIO: R.4 0x23c546244 (dptx-phy[0], offset 0x6244) = 0x18 + // MMIO: W.4 0x23c546244 (dptx-phy[0], offset 0x6244) = 0x8 + // MMIO: R.4 0x23c547244 (dptx-phy[0], offset 0x7244) = 0x18 + // MMIO: W.4 0x23c547244 (dptx-phy[0], offset 0x7244) = 0x8 + // MMIO: R.4 0x23c548244 (dptx-phy[0], offset 0x8244) = 0x18 + // MMIO: W.4 0x23c548244 (dptx-phy[0], offset 0x8244) = 0x8 + for (u32 loff = DPTX_LANE0_OFFSET; loff < DPTX_LANE_END; + loff += DPTX_LANE_STRIDE) + clear32(phy->regs.dptx + loff + 0x244, 0x10); + + // MMIO: R.4 0x23c542214 (dptx-phy[0], offset 0x2214) = 0x1e0 + // MMIO: W.4 0x23c542214 (dptx-phy[0], offset 0x2214) = 0x1e1 + set32(phy->regs.dptx + 0x2214, 0x001); + + // MMIO: R.4 0x23c542224 (dptx-phy[0], offset 0x2224) = 0x20086001 + // MMIO: W.4 0x23c542224 (dptx-phy[0], offset 0x2224) = 0x20086000 + clear32(phy->regs.dptx + 0x2224, 0x00000001); + + // MMIO: R.4 0x23c542200 (dptx-phy[0], offset 0x2200) = 0x2000 + // MMIO: W.4 0x23c542200 (dptx-phy[0], offset 0x2200) = 0x2002 + set32(phy->regs.dptx + 0x2200, 0x0002); + + // MMIO: R.4 0x23c541000 (dptx-phy[0], offset 0x1000) = 0xe0000003 + // MMIO: W.4 0x23c541000 (dptx-phy[0], offset 0x1000) = 0xe0000001 + clear32(phy->regs.dptx + 0x1000, 0x00000002); + + // MMIO: R.4 0x23c544004 (dptx-phy[0], offset 0x4004) = 0x41 + // MMIO: W.4 0x23c544004 (dptx-phy[0], offset 0x4004) = 0x49 + set32(phy->regs.dptx + 0x4004, 0x08); + + /* TODO: no idea what happens here, supposedly setting/clearing some bits */ + // MMIO: R.4 0x23c544404 (dptx-phy[0], offset 0x4404) = 0x555d444 + readl(phy->regs.dptx + 0x4404); + // MMIO: W.4 0x23c544404 (dptx-phy[0], offset 0x4404) = 0x555d444 + writel(0x555d444, phy->regs.dptx + 0x4404); + // MMIO: R.4 0x23c544404 (dptx-phy[0], offset 0x4404) = 0x555d444 + readl(phy->regs.dptx + 0x4404); + // MMIO: W.4 0x23c544404 (dptx-phy[0], offset 0x4404) = 0x555d444 + writel(0x555d444, phy->regs.dptx + 0x4404); + + dptx_phy_set_active_lane_count(phy, 0); + + // MMIO: R.4 0x23c544200 (dptx-phy[0], offset 0x4200) = 0x4002430 + // MMIO: W.4 0x23c544200 (dptx-phy[0], offset 0x4200) = 0x4002420 + clear32(phy->regs.dptx + 0x4200, 0x0000010); + + // MMIO: R.4 0x23c544600 (dptx-phy[0], offset 0x4600) = 0x8000000 + // MMIO: W.4 0x23c544600 (dptx-phy[0], offset 0x4600) = 0x8000000 + clear32(phy->regs.dptx + 0x4600, 0x0000001); + // MMIO: R.4 0x23c544600 (dptx-phy[0], offset 0x4600) = 0x8000000 + // MMIO: W.4 0x23c544600 (dptx-phy[0], offset 0x4600) = 0x8000001 + set32(phy->regs.dptx + 0x4600, 0x0000001); + // MMIO: R.4 0x23c544600 (dptx-phy[0], offset 0x4600) = 0x8000001 + // MMIO: W.4 0x23c544600 (dptx-phy[0], offset 0x4600) = 0x8000003 + set32(phy->regs.dptx + 0x4600, 0x0000002); + // MMIO: R.4 0x23c544600 (dptx-phy[0], offset 0x4600) = 0x8000043 + // MMIO: R.4 0x23c544600 (dptx-phy[0], offset 0x4600) = 0x8000043 + // MMIO: W.4 0x23c544600 (dptx-phy[0], offset 0x4600) = 0x8000041 + /* TODO: read first to check if the previous set(...,0x2) sticked? */ + readl(phy->regs.dptx + 0x4600); + clear32(phy->regs.dptx + 0x4600, 0x0000001); + + // MMIO: R.4 0x23c544408 (dptx-phy[0], offset 0x4408) = 0x482 + // MMIO: W.4 0x23c544408 (dptx-phy[0], offset 0x4408) = 0x482 + /* TODO: probably a set32 of an already set bit */ + val_4408 = readl(phy->regs.dptx + 0x4408); + if (val_4408 != 0x482 && val_4408 != 0x483) + dev_warn( + phy->dev, + "unexpected initial value at regs.dptx offset 0x4408: 0x%03x\n", + val_4408); + writel(val_4408, phy->regs.dptx + 0x4408); + // MMIO: R.4 0x23c544408 (dptx-phy[0], offset 0x4408) = 0x482 + // MMIO: W.4 0x23c544408 (dptx-phy[0], offset 0x4408) = 0x483 + set32(phy->regs.dptx + 0x4408, 0x001); + + return 0; +} + +static int dptx_phy_deactivate(struct apple_dptx_phy *phy) +{ + return 0; +} + +static int dptx_phy_set_link_rate(struct apple_dptx_phy *phy, u32 link_rate) +{ + u32 sts_1008, sts_1014, val_100c, val_20b0, val_20b4; + + dev_dbg(phy->dev, "set_link_rate(%u)\n", link_rate); + + // MMIO: R.4 0x23c544004 (dptx-phy[0], offset 0x4004) = 0x49 + // MMIO: W.4 0x23c544004 (dptx-phy[0], offset 0x4004) = 0x49 + set32(phy->regs.dptx + 0x4004, 0x08); + + // MMIO: R.4 0x23c544000 (dptx-phy[0], offset 0x4000) = 0x41021ac + // MMIO: W.4 0x23c544000 (dptx-phy[0], offset 0x4000) = 0x41021ac + clear32(phy->regs.dptx + 0x4000, 0x0000040); + + // MMIO: R.4 0x23c544004 (dptx-phy[0], offset 0x4004) = 0x49 + // MMIO: W.4 0x23c544004 (dptx-phy[0], offset 0x4004) = 0x41 + clear32(phy->regs.dptx + 0x4004, 0x08); + + // MMIO: R.4 0x23c544000 (dptx-phy[0], offset 0x4000) = 0x41021ac + // MMIO: W.4 0x23c544000 (dptx-phy[0], offset 0x4000) = 0x41021ac + clear32(phy->regs.dptx + 0x4000, 0x2000000); + // MMIO: R.4 0x23c544000 (dptx-phy[0], offset 0x4000) = 0x41021ac + // MMIO: W.4 0x23c544000 (dptx-phy[0], offset 0x4000) = 0x41021ac + set32(phy->regs.dptx + 0x4000, 0x1000000); + + // MMIO: R.4 0x23c542200 (dptx-phy[0], offset 0x2200) = 0x2002 + // MMIO: R.4 0x23c542200 (dptx-phy[0], offset 0x2200) = 0x2002 + // MMIO: W.4 0x23c542200 (dptx-phy[0], offset 0x2200) = 0x2000 + /* TODO: what is this read checking for? */ + readl(phy->regs.dptx + 0x2200); + clear32(phy->regs.dptx + 0x2200, 0x0002); + + // MMIO: R.4 0x23c54100c (dptx-phy[0], offset 0x100c) = 0xf000 + // MMIO: W.4 0x23c54100c (dptx-phy[0], offset 0x100c) = 0xf000 + // MMIO: R.4 0x23c54100c (dptx-phy[0], offset 0x100c) = 0xf000 + // MMIO: W.4 0x23c54100c (dptx-phy[0], offset 0x100c) = 0xf008 + /* TODO: what is the setting/clearing? */ + val_100c = readl(phy->regs.dptx + 0x100c); + writel(val_100c, phy->regs.dptx + 0x100c); + set32(phy->regs.dptx + 0x100c, 0x0008); + + // MMIO: R.4 0x23c541014 (dptx-phy[0], offset 0x1014) = 0x1 + sts_1014 = readl(phy->regs.dptx + 0x1014); + if (sts_1014 != 0x1) + dev_dbg(phy->dev, "unexpected?: dptx[0x1014]: %02x\n", sts_1014); + + // MMIO: R.4 0x23c54100c (dptx-phy[0], offset 0x100c) = 0xf008 + // MMIO: W.4 0x23c54100c (dptx-phy[0], offset 0x100c) = 0xf000 + clear32(phy->regs.dptx + 0x100c, 0x0008); + + // MMIO: R.4 0x23c541008 (dptx-phy[0], offset 0x1008) = 0x1 + sts_1008 = readl(phy->regs.dptx + 0x1008); + if (sts_1008 != 0x1) + dev_dbg(phy->dev, "unexpected?: dptx[0x1008]: %02x\n", sts_1008); + + // MMIO: R.4 0x23c542220 (dptx-phy[0], offset 0x2220) = 0x11090a0 + // MMIO: W.4 0x23c542220 (dptx-phy[0], offset 0x2220) = 0x1109020 + clear32(phy->regs.dptx + 0x2220, 0x0000080); + + // MMIO: R.4 0x23c5420b0 (dptx-phy[0], offset 0x20b0) = 0x1e0e01c2 + // MMIO: W.4 0x23c5420b0 (dptx-phy[0], offset 0x20b0) = 0x1e0e01c2 + val_20b0 = readl(phy->regs.dptx + 0x20b0); + /* TODO: what happens on dptx-phy */ + if (phy->hw.type == DPTX_PHY_T6020) + val_20b0 = (val_20b0 & ~0x3ff) | 0x2a3; + writel(val_20b0, phy->regs.dptx + 0x20b0); + + // MMIO: R.4 0x23c5420b4 (dptx-phy[0], offset 0x20b4) = 0x7fffffe + // MMIO: W.4 0x23c5420b4 (dptx-phy[0], offset 0x20b4) = 0x7fffffe + val_20b4 = readl(phy->regs.dptx + 0x20b4); + /* TODO: what happens on dptx-phy */ + if (phy->hw.type == DPTX_PHY_T6020) + val_20b4 = (val_20b4 | 0x4000000) & ~0x0008000; + writel(val_20b4, phy->regs.dptx + 0x20b4); + + // MMIO: R.4 0x23c5420b4 (dptx-phy[0], offset 0x20b4) = 0x7fffffe + // MMIO: W.4 0x23c5420b4 (dptx-phy[0], offset 0x20b4) = 0x7fffffe + val_20b4 = readl(phy->regs.dptx + 0x20b4); + /* TODO: what happens on dptx-phy */ + if (phy->hw.type == DPTX_PHY_T6020) + val_20b4 = (val_20b4 | 0x0000001) & ~0x0000004; + writel(val_20b4, phy->regs.dptx + 0x20b4); + + // MMIO: R.4 0x23c5420b8 (dptx-phy[0], offset 0x20b8) = 0x654800 + // MMIO: W.4 0x23c5420b8 (dptx-phy[0], offset 0x20b8) = 0x654800 + /* TODO: unclear */ + set32(phy->regs.dptx + 0x20b8, 0); + // MMIO: R.4 0x23c5420b8 (dptx-phy[0], offset 0x20b8) = 0x654800 + // MMIO: W.4 0x23c5420b8 (dptx-phy[0], offset 0x20b8) = 0x654800 + /* TODO: unclear */ + set32(phy->regs.dptx + 0x20b8, 0); + // MMIO: R.4 0x23c5420b8 (dptx-phy[0], offset 0x20b8) = 0x654800 + // MMIO: W.4 0x23c5420b8 (dptx-phy[0], offset 0x20b8) = 0x654800 + /* TODO: unclear */ + if (phy->hw.type == DPTX_PHY_T6020) + set32(phy->regs.dptx + 0x20b8, 0x010000); + else + set32(phy->regs.dptx + 0x20b8, 0); + // MMIO: R.4 0x23c5420b8 (dptx-phy[0], offset 0x20b8) = 0x654800 + // MMIO: W.4 0x23c5420b8 (dptx-phy[0], offset 0x20b8) = 0x454800 + clear32(phy->regs.dptx + 0x20b8, 0x200000); + + // MMIO: R.4 0x23c5420b8 (dptx-phy[0], offset 0x20b8) = 0x454800 + // MMIO: W.4 0x23c5420b8 (dptx-phy[0], offset 0x20b8) = 0x454800 + /* TODO: unclear */ + set32(phy->regs.dptx + 0x20b8, 0); + + // MMIO: R.4 0x23c5000a0 (dptx-phy[1], offset 0xa0) = 0x0 + // MMIO: W.4 0x23c5000a0 (dptx-phy[1], offset 0xa0) = 0x8 + // MMIO: R.4 0x23c5000a0 (dptx-phy[1], offset 0xa0) = 0x8 + // MMIO: W.4 0x23c5000a0 (dptx-phy[1], offset 0xa0) = 0xc + // MMIO: R.4 0x23c5000a0 (dptx-phy[1], offset 0xa0) = 0xc + // MMIO: W.4 0x23c5000a0 (dptx-phy[1], offset 0xa0) = 0x4000c + // MMIO: R.4 0x23c5000a0 (dptx-phy[1], offset 0xa0) = 0x4000c + // MMIO: W.4 0x23c5000a0 (dptx-phy[1], offset 0xa0) = 0xc + // MMIO: R.4 0x23c5000a0 (dptx-phy[1], offset 0xa0) = 0xc + // MMIO: W.4 0x23c5000a0 (dptx-phy[1], offset 0xa0) = 0x8000c + // MMIO: R.4 0x23c5000a0 (dptx-phy[1], offset 0xa0) = 0x8000c + // MMIO: W.4 0x23c5000a0 (dptx-phy[1], offset 0xa0) = 0xc + // MMIO: R.4 0x23c5000a0 (dptx-phy[1], offset 0xa0) = 0xc + // MMIO: W.4 0x23c5000a0 (dptx-phy[1], offset 0xa0) = 0x8 + // MMIO: R.4 0x23c5000a0 (dptx-phy[1], offset 0xa0) = 0x8 + // MMIO: W.4 0x23c5000a0 (dptx-phy[1], offset 0xa0) = 0x0 + set32(phy->regs.core + 0xa0, 0x8); + set32(phy->regs.core + 0xa0, 0x4); + set32(phy->regs.core + 0xa0, 0x40000); + clear32(phy->regs.core + 0xa0, 0x40000); + set32(phy->regs.core + 0xa0, 0x80000); + clear32(phy->regs.core + 0xa0, 0x80000); + clear32(phy->regs.core + 0xa0, 0x4); + clear32(phy->regs.core + 0xa0, 0x8); + + // MMIO: R.4 0x23c542000 (dptx-phy[0], offset 0x2000) = 0x2 + // MMIO: W.4 0x23c542000 (dptx-phy[0], offset 0x2000) = 0x2 + /* TODO: unclear */ + set32(phy->regs.dptx + 0x2000, 0x0); + + // MMIO: R.4 0x23c542018 (dptx-phy[0], offset 0x2018) = 0x0 + // MMIO: W.4 0x23c542018 (dptx-phy[0], offset 0x2018) = 0x0 + clear32(phy->regs.dptx + 0x2018, 0x0); + + // MMIO: R.4 0x23c54100c (dptx-phy[0], offset 0x100c) = 0xf000 + // MMIO: W.4 0x23c54100c (dptx-phy[0], offset 0x100c) = 0xf007 + set32(phy->regs.dptx + 0x100c, 0x0007); + // MMIO: R.4 0x23c54100c (dptx-phy[0], offset 0x100c) = 0xf007 + // MMIO: W.4 0x23c54100c (dptx-phy[0], offset 0x100c) = 0xf00f + set32(phy->regs.dptx + 0x100c, 0x0008); + + // MMIO: R.4 0x23c541014 (dptx-phy[0], offset 0x1014) = 0x38f + sts_1014 = readl(phy->regs.dptx + 0x1014); + if (sts_1014 != 0x38f) + dev_dbg(phy->dev, "unexpected?: dptx[0x1014]: %02x\n", sts_1014); + + // MMIO: R.4 0x23c54100c (dptx-phy[0], offset 0x100c) = 0xf00f + // MMIO: W.4 0x23c54100c (dptx-phy[0], offset 0x100c) = 0xf007 + clear32(phy->regs.dptx + 0x100c, 0x0008); + + // MMIO: R.4 0x23c541008 (dptx-phy[0], offset 0x1008) = 0x9 + sts_1008 = readl(phy->regs.dptx + 0x1008); + if (sts_1008 != 0x9) + dev_dbg(phy->dev, "unexpected?: dptx[0x1008]: %02x\n", sts_1008); + + // MMIO: R.4 0x23c542200 (dptx-phy[0], offset 0x2200) = 0x2000 + // MMIO: W.4 0x23c542200 (dptx-phy[0], offset 0x2200) = 0x2002 + set32(phy->regs.dptx + 0x2200, 0x0002); + + // MMIO: R.4 0x23c545010 (dptx-phy[0], offset 0x5010) = 0x18003000 + // MMIO: W.4 0x23c545010 (dptx-phy[0], offset 0x5010) = 0x18003000 + // MMIO: R.4 0x23c546010 (dptx-phy[0], offset 0x6010) = 0x18003000 + // MMIO: W.4 0x23c546010 (dptx-phy[0], offset 0x6010) = 0x18003000 + // MMIO: R.4 0x23c547010 (dptx-phy[0], offset 0x7010) = 0x18003000 + // MMIO: W.4 0x23c547010 (dptx-phy[0], offset 0x7010) = 0x18003000 + // MMIO: R.4 0x23c548010 (dptx-phy[0], offset 0x8010) = 0x18003000 + // MMIO: W.4 0x23c548010 (dptx-phy[0], offset 0x8010) = 0x18003000 + writel(0x18003000, phy->regs.dptx + 0x8010); + for (u32 loff = DPTX_LANE0_OFFSET; loff < DPTX_LANE_END; loff += DPTX_LANE_STRIDE) { + u32 val_l010 = readl(phy->regs.dptx + loff + 0x10); + writel(val_l010, phy->regs.dptx + loff + 0x10); + } + + // MMIO: R.4 0x23c544000 (dptx-phy[0], offset 0x4000) = 0x41021ac + // MMIO: W.4 0x23c544000 (dptx-phy[0], offset 0x4000) = 0x51021ac + set32(phy->regs.dptx + 0x4000, 0x1000000); + // MMIO: R.4 0x23c544000 (dptx-phy[0], offset 0x4000) = 0x51021ac + // MMIO: W.4 0x23c544000 (dptx-phy[0], offset 0x4000) = 0x71021ac + set32(phy->regs.dptx + 0x4000, 0x2000000); + + // MMIO: R.4 0x23c544004 (dptx-phy[0], offset 0x4004) = 0x41 + // MMIO: W.4 0x23c544004 (dptx-phy[0], offset 0x4004) = 0x49 + set32(phy->regs.dptx + 0x4004, 0x08); + + // MMIO: R.4 0x23c544000 (dptx-phy[0], offset 0x4000) = 0x71021ac + // MMIO: W.4 0x23c544000 (dptx-phy[0], offset 0x4000) = 0x71021ec + set32(phy->regs.dptx + 0x4000, 0x0000040); + + // MMIO: R.4 0x23c544004 (dptx-phy[0], offset 0x4004) = 0x49 + // MMIO: W.4 0x23c544004 (dptx-phy[0], offset 0x4004) = 0x48 + clear32(phy->regs.dptx + 0x4004, 0x01); + + return 0; +} + +static int dptx_phy_set_mode(struct phy *phy, enum phy_mode mode, int submode) +{ + struct apple_dptx_phy *dptx_phy = phy_get_drvdata(phy); + + switch (mode) { + case PHY_MODE_INVALID: + return dptx_phy_deactivate(dptx_phy); + case PHY_MODE_DP: + if (submode < 0 || submode > 5) + return -EINVAL; + return dptx_phy_activate(dptx_phy, submode); + default: + break; + } + + return -EINVAL; +} + +static int dptx_phy_validate(struct phy *phy, enum phy_mode mode, int submode, + union phy_configure_opts *opts_) +{ + struct phy_configure_opts_dp *opts = &opts_->dp; + + if (mode == PHY_MODE_INVALID) { + memset(opts, 0, sizeof(*opts)); + return 0; + } + + if (mode != PHY_MODE_DP) + return -EINVAL; + if (submode < 0 || submode > 5) + return -EINVAL; + + opts->lanes = 4; + opts->link_rate = 8100; + + for (int i = 0; i < 4; ++i) { + opts->voltage[i] = 3; + opts->pre[i] = 3; + } + + return 0; +} + +static int dptx_phy_configure(struct phy *phy, union phy_configure_opts *opts_) +{ + struct phy_configure_opts_dp *opts = &opts_->dp; + struct apple_dptx_phy *dptx_phy = phy_get_drvdata(phy); + enum dptx_phy_link_rate link_rate; + int ret = 0; + + if (opts->set_lanes) { + mutex_lock(&dptx_phy->lock); + ret = dptx_phy_set_active_lane_count(dptx_phy, opts->lanes); + mutex_unlock(&dptx_phy->lock); + } + + if (opts->set_rate) { + switch (opts->link_rate) { + case 1620: + link_rate = DPTX_PHY_LINK_RATE_RBR; + break; + case 2700: + link_rate = DPTX_PHY_LINK_RATE_HBR; + break; + case 5400: + link_rate = DPTX_PHY_LINK_RATE_HBR2; + break; + case 8100: + link_rate = DPTX_PHY_LINK_RATE_HBR3; + break; + case 0: + // TODO: disable! + return 0; + break; + default: + dev_err(dptx_phy->dev, "Unsupported link rate: %d\n", + opts->link_rate); + return -EINVAL; + } + + mutex_lock(&dptx_phy->lock); + ret = dptx_phy_set_link_rate(dptx_phy, link_rate); + mutex_unlock(&dptx_phy->lock); + } + + return ret; +} + +static const struct phy_ops apple_atc_dp_phy_ops = { + .owner = THIS_MODULE, + .configure = dptx_phy_configure, + .validate = dptx_phy_validate, + .set_mode = dptx_phy_set_mode, +}; + +static int dptx_phy_probe(struct platform_device *pdev) +{ + struct apple_dptx_phy *dptx_phy; + struct device *dev = &pdev->dev; + + dptx_phy = devm_kzalloc(dev, sizeof(*dptx_phy), GFP_KERNEL); + if (!dptx_phy) + return -ENOMEM; + + dptx_phy->dev = dev; + dptx_phy->hw = + *(struct apple_dptx_phy_hw *)of_device_get_match_data(dev); + platform_set_drvdata(pdev, dptx_phy); + + mutex_init(&dptx_phy->lock); + + dptx_phy->regs.core = + devm_platform_ioremap_resource_byname(pdev, "core"); + if (IS_ERR(dptx_phy->regs.core)) + return PTR_ERR(dptx_phy->regs.core); + dptx_phy->regs.dptx = + devm_platform_ioremap_resource_byname(pdev, "dptx"); + if (IS_ERR(dptx_phy->regs.dptx)) + return PTR_ERR(dptx_phy->regs.dptx); + + /* create phy */ + dptx_phy->phy_dp = + devm_phy_create(dptx_phy->dev, NULL, &apple_atc_dp_phy_ops); + if (IS_ERR(dptx_phy->phy_dp)) + return PTR_ERR(dptx_phy->phy_dp); + phy_set_drvdata(dptx_phy->phy_dp, dptx_phy); + + dptx_phy->phy_provider = + devm_of_phy_provider_register(dev, of_phy_simple_xlate); + if (IS_ERR(dptx_phy->phy_provider)) + return PTR_ERR(dptx_phy->phy_provider); + + return 0; +} + +static const struct apple_dptx_phy_hw apple_dptx_hw_t6020 = { + .type = DPTX_PHY_T6020, +}; + +static const struct apple_dptx_phy_hw apple_dptx_hw_t8112 = { + .type = DPTX_PHY_T8112, +}; + +static const struct of_device_id dptx_phy_match[] = { + { .compatible = "apple,t6020-dptx-phy", .data = &apple_dptx_hw_t6020 }, + { .compatible = "apple,t8112-dptx-phy", .data = &apple_dptx_hw_t8112 }, + {}, +}; +MODULE_DEVICE_TABLE(of, dptx_phy_match); + +static struct platform_driver dptx_phy_driver = { + .driver = { + .name = "phy-apple-dptx", + .of_match_table = dptx_phy_match, + }, + .probe = dptx_phy_probe, +}; + +module_platform_driver(dptx_phy_driver); + +MODULE_AUTHOR("Janne Grunau "); +MODULE_DESCRIPTION("Apple DP TX PHY driver"); + +MODULE_LICENSE("GPL"); diff --git a/drivers/phy/apple/dptx.h b/drivers/phy/apple/dptx.h new file mode 100644 index 00000000000000..2dd36d753eb357 --- /dev/null +++ b/drivers/phy/apple/dptx.h @@ -0,0 +1,18 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause +/* + * Apple DP TX PHY driver + * + * Copyright (C) The Asahi Linux Contributors + * Author: Janne Grunau + */ + +#ifndef PHY_APPLE_DPTX_H +#define PHY_APPLE_DPTX_H + +enum dptx_phy_link_rate { + DPTX_PHY_LINK_RATE_RBR, + DPTX_PHY_LINK_RATE_HBR, + DPTX_PHY_LINK_RATE_HBR2, + DPTX_PHY_LINK_RATE_HBR3, +}; +#endif /* PHY_APPLE_DPTX_H */ diff --git a/drivers/phy/cadence/phy-cadence-torrent.c b/drivers/phy/cadence/phy-cadence-torrent.c index 37fa4bad6bd72c..877f22177c699c 100644 --- a/drivers/phy/cadence/phy-cadence-torrent.c +++ b/drivers/phy/cadence/phy-cadence-torrent.c @@ -397,6 +397,7 @@ struct cdns_torrent_refclk_driver { struct clk_hw hw; struct regmap_field *cmn_fields[REFCLK_OUT_NUM_CMN_CONFIG]; struct clk_init_data clk_data; + u8 parent_index; }; #define to_cdns_torrent_refclk_driver(_hw) \ @@ -3326,11 +3327,29 @@ static const struct cdns_torrent_vals sgmii_qsgmii_xcvr_diag_ln_vals = { .num_regs = ARRAY_SIZE(sgmii_qsgmii_xcvr_diag_ln_regs), }; +static void cdns_torrent_refclk_driver_suspend(struct cdns_torrent_phy *cdns_phy) +{ + struct clk_hw *hw = cdns_phy->clk_hw_data->hws[CDNS_TORRENT_REFCLK_DRIVER]; + struct cdns_torrent_refclk_driver *refclk_driver = to_cdns_torrent_refclk_driver(hw); + + refclk_driver->parent_index = cdns_torrent_refclk_driver_get_parent(hw); +} + +static int cdns_torrent_refclk_driver_resume(struct cdns_torrent_phy *cdns_phy) +{ + struct clk_hw *hw = cdns_phy->clk_hw_data->hws[CDNS_TORRENT_REFCLK_DRIVER]; + struct cdns_torrent_refclk_driver *refclk_driver = to_cdns_torrent_refclk_driver(hw); + + return cdns_torrent_refclk_driver_set_parent(hw, refclk_driver->parent_index); +} + static int cdns_torrent_phy_suspend_noirq(struct device *dev) { struct cdns_torrent_phy *cdns_phy = dev_get_drvdata(dev); int i; + cdns_torrent_refclk_driver_suspend(cdns_phy); + reset_control_assert(cdns_phy->phy_rst); reset_control_assert(cdns_phy->apb_rst); for (i = 0; i < cdns_phy->nsubnodes; i++) @@ -3352,6 +3371,10 @@ static int cdns_torrent_phy_resume_noirq(struct device *dev) int node = cdns_phy->nsubnodes; int ret, i; + ret = cdns_torrent_refclk_driver_resume(cdns_phy); + if (ret) + return ret; + ret = cdns_torrent_clk(cdns_phy); if (ret) return ret; diff --git a/drivers/phy/freescale/phy-fsl-imx8mq-usb.c b/drivers/phy/freescale/phy-fsl-imx8mq-usb.c index 91b3e62743d3a9..9c340c889c80ce 100644 --- a/drivers/phy/freescale/phy-fsl-imx8mq-usb.c +++ b/drivers/phy/freescale/phy-fsl-imx8mq-usb.c @@ -676,6 +676,8 @@ static int imx8mq_usb_phy_probe(struct platform_device *pdev) if (!imx_phy) return -ENOMEM; + platform_set_drvdata(pdev, imx_phy); + imx_phy->clk = devm_clk_get(dev, "phy"); if (IS_ERR(imx_phy->clk)) { dev_err(dev, "failed to get imx8mq usb phy clock\n"); @@ -730,6 +732,7 @@ static struct platform_driver imx8mq_usb_phy_driver = { .driver = { .name = "imx8mq-usb-phy", .of_match_table = imx8mq_usb_phy_of_match, + .suppress_bind_attrs = true, } }; module_platform_driver(imx8mq_usb_phy_driver); diff --git a/drivers/phy/freescale/phy-fsl-imx8qm-hsio.c b/drivers/phy/freescale/phy-fsl-imx8qm-hsio.c index 977d21d753a59a..279b8ac7822df7 100644 --- a/drivers/phy/freescale/phy-fsl-imx8qm-hsio.c +++ b/drivers/phy/freescale/phy-fsl-imx8qm-hsio.c @@ -251,7 +251,7 @@ static void imx_hsio_configure_clk_pad(struct phy *phy) struct imx_hsio_lane *lane = phy_get_drvdata(phy); struct imx_hsio_priv *priv = lane->priv; - if (strncmp(priv->refclk_pad, "output", 6) == 0) { + if (priv->refclk_pad && strncmp(priv->refclk_pad, "output", 6) == 0) { pll = true; regmap_update_bits(priv->misc, HSIO_CTRL0, HSIO_IOB_A_0_TXOE | HSIO_IOB_A_0_M1M0_MASK, diff --git a/drivers/phy/marvell/phy-mvebu-cp110-utmi.c b/drivers/phy/marvell/phy-mvebu-cp110-utmi.c index 59903f86b13f56..dd3e515a8e8650 100644 --- a/drivers/phy/marvell/phy-mvebu-cp110-utmi.c +++ b/drivers/phy/marvell/phy-mvebu-cp110-utmi.c @@ -338,7 +338,7 @@ static int mvebu_cp110_utmi_phy_probe(struct platform_device *pdev) return -ENOMEM; } - port->dr_mode = of_usb_get_dr_mode_by_phy(child, -1); + port->dr_mode = of_usb_get_dr_mode_by_phy(child, 0); if ((port->dr_mode != USB_DR_MODE_HOST) && (port->dr_mode != USB_DR_MODE_PERIPHERAL)) { dev_err(&pdev->dev, diff --git a/drivers/phy/qualcomm/phy-qcom-edp.c b/drivers/phy/qualcomm/phy-qcom-edp.c index f1b51018683d51..06a08c9ea0f709 100644 --- a/drivers/phy/qualcomm/phy-qcom-edp.c +++ b/drivers/phy/qualcomm/phy-qcom-edp.c @@ -103,7 +103,9 @@ struct qcom_edp { struct phy_configure_opts_dp dp_opts; - struct clk_bulk_data clks[2]; + struct clk_bulk_data *clks; + int num_clks; + struct regulator_bulk_data supplies[2]; bool is_edp; @@ -218,7 +220,7 @@ static int qcom_edp_phy_init(struct phy *phy) if (ret) return ret; - ret = clk_bulk_prepare_enable(ARRAY_SIZE(edp->clks), edp->clks); + ret = clk_bulk_prepare_enable(edp->num_clks, edp->clks); if (ret) goto out_disable_supplies; @@ -885,7 +887,7 @@ static int qcom_edp_phy_exit(struct phy *phy) { struct qcom_edp *edp = phy_get_drvdata(phy); - clk_bulk_disable_unprepare(ARRAY_SIZE(edp->clks), edp->clks); + clk_bulk_disable_unprepare(edp->num_clks, edp->clks); regulator_bulk_disable(ARRAY_SIZE(edp->supplies), edp->supplies); return 0; @@ -1092,11 +1094,9 @@ static int qcom_edp_phy_probe(struct platform_device *pdev) if (IS_ERR(edp->pll)) return PTR_ERR(edp->pll); - edp->clks[0].id = "aux"; - edp->clks[1].id = "cfg_ahb"; - ret = devm_clk_bulk_get(dev, ARRAY_SIZE(edp->clks), edp->clks); - if (ret) - return ret; + edp->num_clks = devm_clk_bulk_get_all(dev, &edp->clks); + if (edp->num_clks < 0) + return dev_err_probe(dev, edp->num_clks, "failed to get clocks\n"); edp->supplies[0].supply = "vdda-phy"; edp->supplies[1].supply = "vdda-pll"; diff --git a/drivers/phy/qualcomm/phy-qcom-qmp-ufs.c b/drivers/phy/qualcomm/phy-qcom-qmp-ufs.c index 8a280433a42b11..dda877561f8ca3 100644 --- a/drivers/phy/qualcomm/phy-qcom-qmp-ufs.c +++ b/drivers/phy/qualcomm/phy-qcom-qmp-ufs.c @@ -928,6 +928,7 @@ static const struct qmp_phy_init_tbl sm8650_ufsphy_pcs[] = { QMP_PHY_INIT_CFG(QPHY_V6_PCS_UFS_MULTI_LANE_CTRL1, 0x02), QMP_PHY_INIT_CFG(QPHY_V6_PCS_UFS_TX_MID_TERM_CTRL1, 0x43), QMP_PHY_INIT_CFG(QPHY_V6_PCS_UFS_PCS_CTRL1, 0xc1), + QMP_PHY_INIT_CFG(QPHY_V6_PCS_UFS_PLL_CNTL, 0x33), QMP_PHY_INIT_CFG(QPHY_V6_PCS_UFS_TX_LARGE_AMP_DRV_LVL, 0x0f), QMP_PHY_INIT_CFG(QPHY_V6_PCS_UFS_RX_SIGDET_CTRL2, 0x68), QMP_PHY_INIT_CFG(QPHY_V6_PCS_UFS_TX_POST_EMP_LVL_S4, 0x0e), @@ -937,13 +938,11 @@ static const struct qmp_phy_init_tbl sm8650_ufsphy_pcs[] = { }; static const struct qmp_phy_init_tbl sm8650_ufsphy_g4_pcs[] = { - QMP_PHY_INIT_CFG(QPHY_V6_PCS_UFS_PLL_CNTL, 0x13), QMP_PHY_INIT_CFG(QPHY_V6_PCS_UFS_TX_HSGEAR_CAPABILITY, 0x04), QMP_PHY_INIT_CFG(QPHY_V6_PCS_UFS_RX_HSGEAR_CAPABILITY, 0x04), }; static const struct qmp_phy_init_tbl sm8650_ufsphy_g5_pcs[] = { - QMP_PHY_INIT_CFG(QPHY_V6_PCS_UFS_PLL_CNTL, 0x33), QMP_PHY_INIT_CFG(QPHY_V6_PCS_UFS_TX_HSGEAR_CAPABILITY, 0x05), QMP_PHY_INIT_CFG(QPHY_V6_PCS_UFS_RX_HSGEAR_CAPABILITY, 0x05), QMP_PHY_INIT_CFG(QPHY_V6_PCS_UFS_RX_HS_G5_SYNC_LENGTH_CAPABILITY, 0x4d), diff --git a/drivers/phy/rockchip/phy-rockchip-samsung-hdptx.c b/drivers/phy/rockchip/phy-rockchip-samsung-hdptx.c index 29de2f7bdae8a3..cafa618d70fdca 100644 --- a/drivers/phy/rockchip/phy-rockchip-samsung-hdptx.c +++ b/drivers/phy/rockchip/phy-rockchip-samsung-hdptx.c @@ -414,6 +414,8 @@ struct rk_hdptx_phy { static const struct ropll_config ropll_tmds_cfg[] = { { 594000000ULL, 124, 124, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 62, 1, 16, 5, 0, 1, 1, 0, 0x20, 0x0c, 1, 0x0e, 0, 0, }, + { 461101250ULL, 97, 97, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 71, 1, 53, 2, 6, + 35, 1, 0, 0x20, 0x0c, 1, 0x0e, 0, 0, }, { 371250000ULL, 155, 155, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 62, 1, 16, 5, 0, 1, 1, 0, 0x20, 0x0c, 1, 0x0e, 0, 0, }, { 297000000ULL, 124, 124, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 62, 1, 16, 5, 0, diff --git a/drivers/phy/ti/phy-j721e-wiz.c b/drivers/phy/ti/phy-j721e-wiz.c index a8b440c6c46bb0..77f18de6fdf62a 100644 --- a/drivers/phy/ti/phy-j721e-wiz.c +++ b/drivers/phy/ti/phy-j721e-wiz.c @@ -393,6 +393,7 @@ struct wiz { struct clk *output_clks[WIZ_MAX_OUTPUT_CLOCKS]; struct clk_onecell_data clk_data; const struct wiz_data *data; + int mux_sel_status[WIZ_MUX_NUM_CLOCKS]; }; static int wiz_reset(struct wiz *wiz) @@ -1424,6 +1425,7 @@ static int wiz_get_lane_phy_types(struct device *dev, struct wiz *wiz) dev_err(dev, "%s: Reading \"reg\" from \"%s\" failed: %d\n", __func__, subnode->name, ret); + of_node_put(serdes); return ret; } of_property_read_u32(subnode, "cdns,num-lanes", &num_lanes); @@ -1438,6 +1440,7 @@ static int wiz_get_lane_phy_types(struct device *dev, struct wiz *wiz) } } + of_node_put(serdes); return 0; } @@ -1654,11 +1657,25 @@ static void wiz_remove(struct platform_device *pdev) pm_runtime_disable(dev); } +static int wiz_suspend_noirq(struct device *dev) +{ + struct wiz *wiz = dev_get_drvdata(dev); + int i; + + for (i = 0; i < WIZ_MUX_NUM_CLOCKS; i++) + regmap_field_read(wiz->mux_sel_field[i], &wiz->mux_sel_status[i]); + + return 0; +} + static int wiz_resume_noirq(struct device *dev) { struct device_node *node = dev->of_node; struct wiz *wiz = dev_get_drvdata(dev); - int ret; + int ret, i; + + for (i = 0; i < WIZ_MUX_NUM_CLOCKS; i++) + regmap_field_write(wiz->mux_sel_field[i], wiz->mux_sel_status[i]); /* Enable supplemental Control override if available */ if (wiz->sup_legacy_clk_override) @@ -1680,7 +1697,7 @@ static int wiz_resume_noirq(struct device *dev) return ret; } -static DEFINE_NOIRQ_DEV_PM_OPS(wiz_pm_ops, NULL, wiz_resume_noirq); +static DEFINE_NOIRQ_DEV_PM_OPS(wiz_pm_ops, wiz_suspend_noirq, wiz_resume_noirq); static struct platform_driver wiz_driver = { .probe = wiz_probe, diff --git a/drivers/pinctrl/cirrus/pinctrl-cs42l43.c b/drivers/pinctrl/cirrus/pinctrl-cs42l43.c index a8f82104a3842e..227c37c360e19a 100644 --- a/drivers/pinctrl/cirrus/pinctrl-cs42l43.c +++ b/drivers/pinctrl/cirrus/pinctrl-cs42l43.c @@ -574,10 +574,9 @@ static int cs42l43_pin_probe(struct platform_device *pdev) if (child) { ret = devm_add_action_or_reset(&pdev->dev, cs42l43_fwnode_put, child); - if (ret) { - fwnode_handle_put(child); + if (ret) return ret; - } + if (!child->dev) child->dev = priv->dev; fwnode = child; diff --git a/drivers/pinctrl/intel/Kconfig b/drivers/pinctrl/intel/Kconfig index 248c2e558ff32b..3ebf0723714573 100644 --- a/drivers/pinctrl/intel/Kconfig +++ b/drivers/pinctrl/intel/Kconfig @@ -52,7 +52,10 @@ config PINCTRL_ALDERLAKE select PINCTRL_INTEL help This pinctrl driver provides an interface that allows configuring - of Intel Alder Lake PCH pins and using them as GPIOs. + PCH pins of the following platforms and using them as GPIOs: + - Alder Lake HX, N, and S + - Raptor Lake HX, E, and S + - Twin Lake config PINCTRL_BROXTON tristate "Intel Broxton pinctrl and GPIO driver" @@ -136,15 +139,17 @@ config PINCTRL_METEORLAKE select PINCTRL_INTEL help This pinctrl driver provides an interface that allows configuring - of Intel Meteor Lake pins and using them as GPIOs. + SoC pins of the following platforms and using them as GPIOs: + - Arrow Lake (all variants) + - Meteor Lake (all variants) config PINCTRL_METEORPOINT tristate "Intel Meteor Point pinctrl and GPIO driver" select PINCTRL_INTEL help - Meteor Point is the PCH of Intel Meteor Lake. This pinctrl driver - provides an interface that allows configuring of PCH pins and - using them as GPIOs. + This pinctrl driver provides an interface that allows configuring + PCH pins of the following platforms and using them as GPIOs: + - Arrow Lake HX and S config PINCTRL_SUNRISEPOINT tristate "Intel Sunrisepoint pinctrl and GPIO driver" @@ -159,7 +164,11 @@ config PINCTRL_TIGERLAKE select PINCTRL_INTEL help This pinctrl driver provides an interface that allows configuring - of Intel Tiger Lake PCH pins and using them as GPIOs. + PCH pins of the following platforms and using them as GPIOs: + - Alder Lake H, P, PS, and U + - Raptor Lake H, P, PS, PX, and U + - Rocket Lake S + - Tiger Lake (all variants) source "drivers/pinctrl/intel/Kconfig.tng" endmenu diff --git a/drivers/pinctrl/intel/pinctrl-intel.c b/drivers/pinctrl/intel/pinctrl-intel.c index cf9db8ac0f42ea..106835b5ee5a52 100644 --- a/drivers/pinctrl/intel/pinctrl-intel.c +++ b/drivers/pinctrl/intel/pinctrl-intel.c @@ -1610,7 +1610,7 @@ int intel_pinctrl_probe(struct platform_device *pdev, value = readl(regs + REVID); if (value == ~0u) return -ENODEV; - if (((value & REVID_MASK) >> REVID_SHIFT) >= 0x94) { + if (((value & REVID_MASK) >> REVID_SHIFT) >= 0x92) { community->features |= PINCTRL_FEATURE_DEBOUNCE; community->features |= PINCTRL_FEATURE_1K_PD; } diff --git a/drivers/pinctrl/mediatek/mtk-eint.c b/drivers/pinctrl/mediatek/mtk-eint.c index c8c5097c11c4d1..2a3c04eedc5f38 100644 --- a/drivers/pinctrl/mediatek/mtk-eint.c +++ b/drivers/pinctrl/mediatek/mtk-eint.c @@ -544,24 +544,32 @@ int mtk_eint_do_init(struct mtk_eint *eint, struct mtk_eint_pin *eint_pin) } } - eint->pin_list = devm_kmalloc(eint->dev, eint->nbase * sizeof(u16 *), GFP_KERNEL); + eint->pin_list = devm_kcalloc(eint->dev, eint->nbase, + sizeof(*eint->pin_list), GFP_KERNEL); if (!eint->pin_list) goto err_pin_list; - eint->wake_mask = devm_kmalloc(eint->dev, eint->nbase * sizeof(u32 *), GFP_KERNEL); + eint->wake_mask = devm_kcalloc(eint->dev, eint->nbase, + sizeof(*eint->wake_mask), GFP_KERNEL); if (!eint->wake_mask) goto err_wake_mask; - eint->cur_mask = devm_kmalloc(eint->dev, eint->nbase * sizeof(u32 *), GFP_KERNEL); + eint->cur_mask = devm_kcalloc(eint->dev, eint->nbase, + sizeof(*eint->cur_mask), GFP_KERNEL); if (!eint->cur_mask) goto err_cur_mask; for (i = 0; i < eint->nbase; i++) { - eint->pin_list[i] = devm_kzalloc(eint->dev, eint->base_pin_num[i] * sizeof(u16), + eint->pin_list[i] = devm_kzalloc(eint->dev, + eint->base_pin_num[i] * sizeof(**eint->pin_list), GFP_KERNEL); port = DIV_ROUND_UP(eint->base_pin_num[i], 32); - eint->wake_mask[i] = devm_kzalloc(eint->dev, port * sizeof(u32), GFP_KERNEL); - eint->cur_mask[i] = devm_kzalloc(eint->dev, port * sizeof(u32), GFP_KERNEL); + eint->wake_mask[i] = devm_kzalloc(eint->dev, + port * sizeof(**eint->wake_mask), + GFP_KERNEL); + eint->cur_mask[i] = devm_kzalloc(eint->dev, + port * sizeof(**eint->cur_mask), + GFP_KERNEL); if (!eint->pin_list[i] || !eint->wake_mask[i] || !eint->cur_mask[i]) goto err_eint; } @@ -597,12 +605,9 @@ int mtk_eint_do_init(struct mtk_eint *eint, struct mtk_eint_pin *eint_pin) err_eint: for (i = 0; i < eint->nbase; i++) { - if (eint->cur_mask[i]) - devm_kfree(eint->dev, eint->cur_mask[i]); - if (eint->wake_mask[i]) - devm_kfree(eint->dev, eint->wake_mask[i]); - if (eint->pin_list[i]) - devm_kfree(eint->dev, eint->pin_list[i]); + devm_kfree(eint->dev, eint->cur_mask[i]); + devm_kfree(eint->dev, eint->wake_mask[i]); + devm_kfree(eint->dev, eint->pin_list[i]); } devm_kfree(eint->dev, eint->cur_mask); err_cur_mask: diff --git a/drivers/pinctrl/mediatek/pinctrl-mtk-common.c b/drivers/pinctrl/mediatek/pinctrl-mtk-common.c index d6a46fe0cda891..3f518dce6d23f1 100644 --- a/drivers/pinctrl/mediatek/pinctrl-mtk-common.c +++ b/drivers/pinctrl/mediatek/pinctrl-mtk-common.c @@ -1135,9 +1135,12 @@ int mtk_pctrl_init(struct platform_device *pdev, goto chip_error; } - ret = mtk_eint_init(pctl, pdev); - if (ret) - goto chip_error; + /* Only initialize EINT if we have EINT pins */ + if (data->eint_hw.ap_num > 0) { + ret = mtk_eint_init(pctl, pdev); + if (ret) + goto chip_error; + } return 0; diff --git a/drivers/pinctrl/meson/pinctrl-amlogic-a4.c b/drivers/pinctrl/meson/pinctrl-amlogic-a4.c index d9e3a8d5932a82..e2293a872dcb7f 100644 --- a/drivers/pinctrl/meson/pinctrl-amlogic-a4.c +++ b/drivers/pinctrl/meson/pinctrl-amlogic-a4.c @@ -24,6 +24,7 @@ #include #include "../core.h" +#include "../pinctrl-utils.h" #include "../pinconf.h" #define gpio_chip_to_bank(chip) \ @@ -672,11 +673,78 @@ static void aml_pin_dbg_show(struct pinctrl_dev *pcdev, struct seq_file *s, seq_printf(s, " %s", dev_name(pcdev->dev)); } +static int aml_dt_node_to_map_pinmux(struct pinctrl_dev *pctldev, + struct device_node *np, + struct pinctrl_map **map, + unsigned int *num_maps) +{ + struct device *dev = pctldev->dev; + unsigned long *configs = NULL; + unsigned int num_configs = 0; + struct property *prop; + unsigned int reserved_maps; + int reserve; + int ret; + + prop = of_find_property(np, "pinmux", NULL); + if (!prop) { + dev_info(dev, "Missing pinmux property\n"); + return -ENOENT; + } + + struct device_node *pnode __free(device_node) = of_get_parent(np); + if (!pnode) { + dev_info(dev, "Missing function node\n"); + return -EINVAL; + } + + reserved_maps = 0; + *map = NULL; + *num_maps = 0; + + ret = pinconf_generic_parse_dt_config(np, pctldev, &configs, + &num_configs); + if (ret < 0) { + dev_err(dev, "%pOF: could not parse node property\n", np); + return ret; + } + + reserve = 1; + if (num_configs) + reserve++; + + ret = pinctrl_utils_reserve_map(pctldev, map, &reserved_maps, + num_maps, reserve); + if (ret < 0) + goto exit; + + ret = pinctrl_utils_add_map_mux(pctldev, map, + &reserved_maps, num_maps, np->name, + pnode->name); + if (ret < 0) + goto exit; + + if (num_configs) { + ret = pinctrl_utils_add_map_configs(pctldev, map, &reserved_maps, + num_maps, np->name, configs, + num_configs, PIN_MAP_TYPE_CONFIGS_GROUP); + if (ret < 0) + goto exit; + } + +exit: + kfree(configs); + if (ret) + pinctrl_utils_free_map(pctldev, *map, *num_maps); + + return ret; +} + static const struct pinctrl_ops aml_pctrl_ops = { .get_groups_count = aml_get_groups_count, .get_group_name = aml_get_group_name, .get_group_pins = aml_get_group_pins, - .dt_node_to_map = pinconf_generic_dt_node_to_map_pinmux, + .dt_node_to_map = aml_dt_node_to_map_pinmux, .dt_free_map = pinconf_generic_dt_free_map, .pin_dbg_show = aml_pin_dbg_show, }; @@ -725,8 +793,9 @@ static u32 aml_bank_pins(struct device_node *np) if (of_parse_phandle_with_fixed_args(np, "gpio-ranges", 3, 0, &of_args)) return 0; - else - return of_args.args[2]; + + of_node_put(of_args.np); + return of_args.args[2]; } static int aml_bank_number(struct device_node *np) @@ -736,8 +805,9 @@ static int aml_bank_number(struct device_node *np) if (of_parse_phandle_with_fixed_args(np, "gpio-ranges", 3, 0, &of_args)) return -EINVAL; - else - return of_args.args[1] >> 8; + + of_node_put(of_args.np); + return of_args.args[1] >> 8; } static unsigned int aml_count_pins(struct device_node *np) @@ -893,7 +963,7 @@ static const struct gpio_chip aml_gpio_template = { .direction_input = aml_gpio_direction_input, .direction_output = aml_gpio_direction_output, .get_direction = aml_gpio_get_direction, - .can_sleep = false, + .can_sleep = true, }; static void init_bank_register_bit(struct aml_pinctrl *info, diff --git a/drivers/pinctrl/pinconf-generic.c b/drivers/pinctrl/pinconf-generic.c index 366775841c6399..2b030bd0e6adca 100644 --- a/drivers/pinctrl/pinconf-generic.c +++ b/drivers/pinctrl/pinconf-generic.c @@ -351,13 +351,13 @@ int pinconf_generic_parse_dt_config(struct device_node *np, ret = parse_dt_cfg(np, dt_params, ARRAY_SIZE(dt_params), cfg, &ncfg); if (ret) - return ret; + goto out; if (pctldev && pctldev->desc->num_custom_params && pctldev->desc->custom_params) { ret = parse_dt_cfg(np, pctldev->desc->custom_params, pctldev->desc->num_custom_params, cfg, &ncfg); if (ret) - return ret; + goto out; } /* no configs found at all */ @@ -385,75 +385,6 @@ int pinconf_generic_parse_dt_config(struct device_node *np, } EXPORT_SYMBOL_GPL(pinconf_generic_parse_dt_config); -int pinconf_generic_dt_node_to_map_pinmux(struct pinctrl_dev *pctldev, - struct device_node *np, - struct pinctrl_map **map, - unsigned int *num_maps) -{ - struct device *dev = pctldev->dev; - struct device_node *pnode; - unsigned long *configs = NULL; - unsigned int num_configs = 0; - struct property *prop; - unsigned int reserved_maps; - int reserve; - int ret; - - prop = of_find_property(np, "pinmux", NULL); - if (!prop) { - dev_info(dev, "Missing pinmux property\n"); - return -ENOENT; - } - - pnode = of_get_parent(np); - if (!pnode) { - dev_info(dev, "Missing function node\n"); - return -EINVAL; - } - - reserved_maps = 0; - *map = NULL; - *num_maps = 0; - - ret = pinconf_generic_parse_dt_config(np, pctldev, &configs, - &num_configs); - if (ret < 0) { - dev_err(dev, "%pOF: could not parse node property\n", np); - return ret; - } - - reserve = 1; - if (num_configs) - reserve++; - - ret = pinctrl_utils_reserve_map(pctldev, map, &reserved_maps, - num_maps, reserve); - if (ret < 0) - goto exit; - - ret = pinctrl_utils_add_map_mux(pctldev, map, - &reserved_maps, num_maps, np->name, - pnode->name); - if (ret < 0) - goto exit; - - if (num_configs) { - ret = pinctrl_utils_add_map_configs(pctldev, map, &reserved_maps, - num_maps, np->name, configs, - num_configs, PIN_MAP_TYPE_CONFIGS_GROUP); - if (ret < 0) - goto exit; - } - -exit: - kfree(configs); - if (ret) - pinctrl_utils_free_map(pctldev, *map, *num_maps); - - return ret; -} -EXPORT_SYMBOL_GPL(pinconf_generic_dt_node_to_map_pinmux); - int pinconf_generic_dt_subnode_to_map(struct pinctrl_dev *pctldev, struct device_node *np, struct pinctrl_map **map, unsigned int *reserved_maps, unsigned int *num_maps, diff --git a/drivers/pinctrl/pinctrl-cy8c95x0.c b/drivers/pinctrl/pinctrl-cy8c95x0.c index a4b04bf6d081f6..5c055d344ac9d5 100644 --- a/drivers/pinctrl/pinctrl-cy8c95x0.c +++ b/drivers/pinctrl/pinctrl-cy8c95x0.c @@ -627,7 +627,7 @@ static int cy8c95x0_write_regs_mask(struct cy8c95x0_pinctrl *chip, int reg, bitmap_scatter(tmask, mask, chip->map, MAX_LINE); bitmap_scatter(tval, val, chip->map, MAX_LINE); - for_each_set_clump8(offset, bits, tmask, chip->tpin) { + for_each_set_clump8(offset, bits, tmask, chip->nport * BANK_SZ) { unsigned int i = offset / 8; write_val = bitmap_get_value8(tval, offset); @@ -655,7 +655,7 @@ static int cy8c95x0_read_regs_mask(struct cy8c95x0_pinctrl *chip, int reg, bitmap_scatter(tmask, mask, chip->map, MAX_LINE); bitmap_scatter(tval, val, chip->map, MAX_LINE); - for_each_set_clump8(offset, bits, tmask, chip->tpin) { + for_each_set_clump8(offset, bits, tmask, chip->nport * BANK_SZ) { unsigned int i = offset / 8; ret = cy8c95x0_regmap_read_bits(chip, reg, i, bits, &read_val); diff --git a/drivers/pinctrl/pinctrl-equilibrium.c b/drivers/pinctrl/pinctrl-equilibrium.c index 2d04829b29c997..ba1c867b7b891a 100644 --- a/drivers/pinctrl/pinctrl-equilibrium.c +++ b/drivers/pinctrl/pinctrl-equilibrium.c @@ -23,7 +23,7 @@ #define PIN_NAME_LEN 10 #define PAD_REG_OFF 0x100 -static void eqbr_gpio_disable_irq(struct irq_data *d) +static void eqbr_irq_mask(struct irq_data *d) { struct gpio_chip *gc = irq_data_get_irq_chip_data(d); struct eqbr_gpio_ctrl *gctrl = gpiochip_get_data(gc); @@ -36,7 +36,7 @@ static void eqbr_gpio_disable_irq(struct irq_data *d) gpiochip_disable_irq(gc, offset); } -static void eqbr_gpio_enable_irq(struct irq_data *d) +static void eqbr_irq_unmask(struct irq_data *d) { struct gpio_chip *gc = irq_data_get_irq_chip_data(d); struct eqbr_gpio_ctrl *gctrl = gpiochip_get_data(gc); @@ -50,7 +50,7 @@ static void eqbr_gpio_enable_irq(struct irq_data *d) raw_spin_unlock_irqrestore(&gctrl->lock, flags); } -static void eqbr_gpio_ack_irq(struct irq_data *d) +static void eqbr_irq_ack(struct irq_data *d) { struct gpio_chip *gc = irq_data_get_irq_chip_data(d); struct eqbr_gpio_ctrl *gctrl = gpiochip_get_data(gc); @@ -62,10 +62,17 @@ static void eqbr_gpio_ack_irq(struct irq_data *d) raw_spin_unlock_irqrestore(&gctrl->lock, flags); } -static void eqbr_gpio_mask_ack_irq(struct irq_data *d) +static void eqbr_irq_mask_ack(struct irq_data *d) { - eqbr_gpio_disable_irq(d); - eqbr_gpio_ack_irq(d); + struct gpio_chip *gc = irq_data_get_irq_chip_data(d); + struct eqbr_gpio_ctrl *gctrl = gpiochip_get_data(gc); + unsigned int offset = irqd_to_hwirq(d); + unsigned long flags; + + raw_spin_lock_irqsave(&gctrl->lock, flags); + writel(BIT(offset), gctrl->membase + GPIO_IRNENCLR); + writel(BIT(offset), gctrl->membase + GPIO_IRNCR); + raw_spin_unlock_irqrestore(&gctrl->lock, flags); } static inline void eqbr_cfg_bit(void __iomem *addr, @@ -92,7 +99,7 @@ static int eqbr_irq_type_cfg(struct gpio_irq_type *type, return 0; } -static int eqbr_gpio_set_irq_type(struct irq_data *d, unsigned int type) +static int eqbr_irq_set_type(struct irq_data *d, unsigned int type) { struct gpio_chip *gc = irq_data_get_irq_chip_data(d); struct eqbr_gpio_ctrl *gctrl = gpiochip_get_data(gc); @@ -166,11 +173,11 @@ static void eqbr_irq_handler(struct irq_desc *desc) static const struct irq_chip eqbr_irq_chip = { .name = "gpio_irq", - .irq_mask = eqbr_gpio_disable_irq, - .irq_unmask = eqbr_gpio_enable_irq, - .irq_ack = eqbr_gpio_ack_irq, - .irq_mask_ack = eqbr_gpio_mask_ack_irq, - .irq_set_type = eqbr_gpio_set_irq_type, + .irq_ack = eqbr_irq_ack, + .irq_mask = eqbr_irq_mask, + .irq_mask_ack = eqbr_irq_mask_ack, + .irq_unmask = eqbr_irq_unmask, + .irq_set_type = eqbr_irq_set_type, .flags = IRQCHIP_IMMUTABLE, GPIOCHIP_IRQ_RESOURCE_HELPERS, }; @@ -846,6 +853,7 @@ static int pinbank_init(struct device_node *np, bank->pin_base = spec.args[1]; bank->nr_pins = spec.args[2]; + of_node_put(spec.np); bank->aval_pinmap = readl(bank->membase + REG_AVAIL); bank->id = id; diff --git a/drivers/pinctrl/pinctrl-k230.c b/drivers/pinctrl/pinctrl-k230.c index d716f23d837f7a..20f7c0f70eb77c 100644 --- a/drivers/pinctrl/pinctrl-k230.c +++ b/drivers/pinctrl/pinctrl-k230.c @@ -65,6 +65,7 @@ struct k230_pmx_func { }; struct k230_pinctrl { + struct device *dev; struct pinctrl_desc pctl; struct pinctrl_dev *pctl_dev; struct regmap *regmap_base; @@ -470,7 +471,7 @@ static int k230_pinctrl_parse_groups(struct device_node *np, struct k230_pinctrl *info, unsigned int index) { - struct device *dev = info->pctl_dev->dev; + struct device *dev = info->dev; const __be32 *list; int size, i, ret; @@ -511,7 +512,7 @@ static int k230_pinctrl_parse_functions(struct device_node *np, struct k230_pinctrl *info, unsigned int index) { - struct device *dev = info->pctl_dev->dev; + struct device *dev = info->dev; struct k230_pmx_func *func; struct k230_pin_group *grp; static unsigned int idx, i; @@ -596,6 +597,8 @@ static int k230_pinctrl_probe(struct platform_device *pdev) if (!info) return -ENOMEM; + info->dev = dev; + pctl = &info->pctl; pctl->name = "k230-pinctrl"; diff --git a/drivers/pinctrl/pinctrl-mcp23s08.c b/drivers/pinctrl/pinctrl-mcp23s08.c index 586f2f67c6177f..b89b3169e8be55 100644 --- a/drivers/pinctrl/pinctrl-mcp23s08.c +++ b/drivers/pinctrl/pinctrl-mcp23s08.c @@ -664,6 +664,15 @@ int mcp23s08_probe_one(struct mcp23s08 *mcp, struct device *dev, if (mcp->irq && mcp->irq_controller) { struct gpio_irq_chip *girq = &mcp->chip.irq; + /* + * Disable all pin interrupts, to prevent the interrupt handler from + * calling nested handlers for any currently-enabled interrupts that + * do not (yet) have an actual handler. + */ + ret = mcp_write(mcp, MCP_GPINTEN, 0); + if (ret < 0) + return dev_err_probe(dev, ret, "can't disable interrupts\n"); + gpio_irq_chip_set_chip(girq, &mcp23s08_irq_chip); /* This will let us handle the parent IRQ in the driver */ girq->parent_handler = NULL; diff --git a/drivers/pinctrl/pinctrl-single.c b/drivers/pinctrl/pinctrl-single.c index 998f23d6c3179e..d85e6c1f632186 100644 --- a/drivers/pinctrl/pinctrl-single.c +++ b/drivers/pinctrl/pinctrl-single.c @@ -1359,6 +1359,7 @@ static int pcs_add_gpio_func(struct device_node *node, struct pcs_device *pcs) } range = devm_kzalloc(pcs->dev, sizeof(*range), GFP_KERNEL); if (!range) { + of_node_put(gpiospec.np); ret = -ENOMEM; break; } @@ -1368,6 +1369,7 @@ static int pcs_add_gpio_func(struct device_node *node, struct pcs_device *pcs) mutex_lock(&pcs->mutex); list_add_tail(&range->node, &pcs->gpiofuncs); mutex_unlock(&pcs->mutex); + of_node_put(gpiospec.np); } return ret; } diff --git a/drivers/pinctrl/qcom/pinctrl-qcs615.c b/drivers/pinctrl/qcom/pinctrl-qcs615.c index 4dfa820d4e77ce..f1c827ddbfbfa5 100644 --- a/drivers/pinctrl/qcom/pinctrl-qcs615.c +++ b/drivers/pinctrl/qcom/pinctrl-qcs615.c @@ -1067,6 +1067,7 @@ static const struct msm_pinctrl_soc_data qcs615_tlmm = { .ntiles = ARRAY_SIZE(qcs615_tiles), .wakeirq_map = qcs615_pdc_map, .nwakeirq_map = ARRAY_SIZE(qcs615_pdc_map), + .wakeirq_dual_edge_errata = true, }; static const struct of_device_id qcs615_tlmm_of_match[] = { diff --git a/drivers/pinctrl/qcom/pinctrl-sm8250-lpass-lpi.c b/drivers/pinctrl/qcom/pinctrl-sm8250-lpass-lpi.c index 64494a86490e2f..c27452eece3e6b 100644 --- a/drivers/pinctrl/qcom/pinctrl-sm8250-lpass-lpi.c +++ b/drivers/pinctrl/qcom/pinctrl-sm8250-lpass-lpi.c @@ -73,7 +73,7 @@ static const char * const i2s1_ws_groups[] = { "gpio7" }; static const char * const i2s1_data_groups[] = { "gpio8", "gpio9" }; static const char * const wsa_swr_clk_groups[] = { "gpio10" }; static const char * const wsa_swr_data_groups[] = { "gpio11" }; -static const char * const i2s2_data_groups[] = { "gpio12", "gpio12" }; +static const char * const i2s2_data_groups[] = { "gpio12", "gpio13" }; static const struct lpi_pingroup sm8250_groups[] = { LPI_PINGROUP(0, 0, swr_tx_clk, qua_mi2s_sclk, _, _), diff --git a/drivers/pinctrl/qcom/pinctrl-spmi-gpio.c b/drivers/pinctrl/qcom/pinctrl-spmi-gpio.c index 83f940fe30b26a..d02d42513ebbca 100644 --- a/drivers/pinctrl/qcom/pinctrl-spmi-gpio.c +++ b/drivers/pinctrl/qcom/pinctrl-spmi-gpio.c @@ -723,6 +723,21 @@ static const struct pinconf_ops pmic_gpio_pinconf_ops = { .pin_config_group_dbg_show = pmic_gpio_config_dbg_show, }; +static int pmic_gpio_get_direction(struct gpio_chip *chip, unsigned pin) +{ + struct pmic_gpio_state *state = gpiochip_get_data(chip); + struct pmic_gpio_pad *pad; + + pad = state->ctrl->desc->pins[pin].drv_data; + + if (!pad->is_enabled || pad->analog_pass || + (!pad->input_enabled && !pad->output_enabled)) + return -EINVAL; + + /* Make sure the state is aligned on what pmic_gpio_get() returns */ + return pad->input_enabled ? GPIO_LINE_DIRECTION_IN : GPIO_LINE_DIRECTION_OUT; +} + static int pmic_gpio_direction_input(struct gpio_chip *chip, unsigned pin) { struct pmic_gpio_state *state = gpiochip_get_data(chip); @@ -801,6 +816,7 @@ static void pmic_gpio_dbg_show(struct seq_file *s, struct gpio_chip *chip) } static const struct gpio_chip pmic_gpio_gpio_template = { + .get_direction = pmic_gpio_get_direction, .direction_input = pmic_gpio_direction_input, .direction_output = pmic_gpio_direction_output, .get = pmic_gpio_get, diff --git a/drivers/pinctrl/renesas/pinctrl-rza1.c b/drivers/pinctrl/renesas/pinctrl-rza1.c index 3cfa4c8be80eaf..d83c7d8ee82c44 100644 --- a/drivers/pinctrl/renesas/pinctrl-rza1.c +++ b/drivers/pinctrl/renesas/pinctrl-rza1.c @@ -589,7 +589,7 @@ static inline unsigned int rza1_get_bit(struct rza1_port *port, { void __iomem *mem = RZA1_ADDR(port->base, reg, port->id); - return ioread16(mem) & BIT(bit); + return !!(ioread16(mem) & BIT(bit)); } /** diff --git a/drivers/pinctrl/renesas/pinctrl-rzt2h.c b/drivers/pinctrl/renesas/pinctrl-rzt2h.c index 4826ff91cd9060..24b90c80f5131f 100644 --- a/drivers/pinctrl/renesas/pinctrl-rzt2h.c +++ b/drivers/pinctrl/renesas/pinctrl-rzt2h.c @@ -51,6 +51,7 @@ #define PFC_MASK GENMASK_ULL(5, 0) #define PFC_PIN_MASK(pin) (PFC_MASK << ((pin) * 8)) +#define PFC_FUNC_INTERRUPT 0 /* * Use 16 lower bits [15:0] for pin identifier @@ -486,6 +487,7 @@ static int rzt2h_gpio_get_direction(struct gpio_chip *chip, unsigned int offset) struct rzt2h_pinctrl *pctrl = gpiochip_get_data(chip); u8 port = RZT2H_PIN_ID_TO_PORT(offset); u8 bit = RZT2H_PIN_ID_TO_PIN(offset); + u64 reg64; u16 reg; int ret; @@ -493,8 +495,25 @@ static int rzt2h_gpio_get_direction(struct gpio_chip *chip, unsigned int offset) if (ret) return ret; - if (rzt2h_pinctrl_readb(pctrl, port, PMC(port)) & BIT(bit)) + guard(spinlock_irqsave)(&pctrl->lock); + + if (rzt2h_pinctrl_readb(pctrl, port, PMC(port)) & BIT(bit)) { + /* + * When a GPIO is being requested as an IRQ, the pinctrl + * framework expects to be able to read the GPIO's direction. + * IRQ function is separate from GPIO, and enabling it takes the + * pin out of GPIO mode. + * At this point, .child_to_parent_hwirq() has already been + * called to enable the IRQ function. + * Default to input direction for IRQ function. + */ + reg64 = rzt2h_pinctrl_readq(pctrl, port, PFC(port)); + reg64 = (reg64 >> (bit * 8)) & PFC_MASK; + if (reg64 == PFC_FUNC_INTERRUPT) + return GPIO_LINE_DIRECTION_IN; + return -EINVAL; + } reg = rzt2h_pinctrl_readw(pctrl, port, PM(port)); reg = (reg >> (bit * 2)) & PM_MASK; @@ -629,6 +648,7 @@ static int rzt2h_gpio_register(struct rzt2h_pinctrl *pctrl) if (ret) return dev_err_probe(dev, ret, "Unable to parse gpio-ranges\n"); + of_node_put(of_args.np); if (of_args.args[0] != 0 || of_args.args[1] != 0 || of_args.args[2] != pctrl->data->n_port_pins) return dev_err_probe(dev, -EINVAL, diff --git a/drivers/pinctrl/stm32/Kconfig b/drivers/pinctrl/stm32/Kconfig index 5f67e1ee66dd96..d6a17152301216 100644 --- a/drivers/pinctrl/stm32/Kconfig +++ b/drivers/pinctrl/stm32/Kconfig @@ -65,6 +65,7 @@ config PINCTRL_STM32_HDP select PINMUX select GENERIC_PINCONF select GPIOLIB + select GPIO_GENERIC help The Hardware Debug Port allows the observation of internal signals. It uses configurable multiplexer to route signals in a dedicated observation register. diff --git a/drivers/platform/chrome/cros_ec_lightbar.c b/drivers/platform/chrome/cros_ec_lightbar.c index 8352e97327911f..3702baff5d4f17 100644 --- a/drivers/platform/chrome/cros_ec_lightbar.c +++ b/drivers/platform/chrome/cros_ec_lightbar.c @@ -126,7 +126,7 @@ static int get_lightbar_version(struct cros_ec_dev *ec, param = (struct ec_params_lightbar *)msg->data; param->cmd = LIGHTBAR_CMD_VERSION; msg->outsize = sizeof(param->cmd); - msg->result = sizeof(resp->version); + msg->insize = sizeof(resp->version); ret = cros_ec_cmd_xfer_status(ec->ec_dev, msg); if (ret < 0 && ret != -EINVAL) { ret = 0; diff --git a/drivers/platform/chrome/cros_typec_switch.c b/drivers/platform/chrome/cros_typec_switch.c index 8d7c34abb0a128..d8a28d4e51a85e 100644 --- a/drivers/platform/chrome/cros_typec_switch.c +++ b/drivers/platform/chrome/cros_typec_switch.c @@ -230,20 +230,20 @@ static int cros_typec_register_switches(struct cros_typec_switch_data *sdata) adev = to_acpi_device_node(fwnode); if (!adev) { - dev_err(fwnode->dev, "Couldn't get ACPI device handle\n"); + dev_err(dev, "Couldn't get ACPI device handle for %pfwP\n", fwnode); ret = -ENODEV; goto err_switch; } ret = acpi_evaluate_integer(adev->handle, "_ADR", NULL, &index); if (ACPI_FAILURE(ret)) { - dev_err(fwnode->dev, "_ADR wasn't evaluated\n"); + dev_err(dev, "_ADR wasn't evaluated for %pfwP\n", fwnode); ret = -ENODATA; goto err_switch; } if (index >= EC_USB_PD_MAX_PORTS) { - dev_err(fwnode->dev, "Invalid port index number: %llu\n", index); + dev_err(dev, "%pfwP: Invalid port index number: %llu\n", fwnode, index); ret = -EINVAL; goto err_switch; } diff --git a/drivers/platform/olpc/olpc-xo175-ec.c b/drivers/platform/olpc/olpc-xo175-ec.c index fa7b3bda688a63..bee271a4fda1a9 100644 --- a/drivers/platform/olpc/olpc-xo175-ec.c +++ b/drivers/platform/olpc/olpc-xo175-ec.c @@ -482,7 +482,7 @@ static int olpc_xo175_ec_cmd(u8 cmd, u8 *inbuf, size_t inlen, u8 *resp, dev_dbg(dev, "CMD %x, %zd bytes expected\n", cmd, resp_len); if (inlen > 5) { - dev_err(dev, "command len %zd too big!\n", resp_len); + dev_err(dev, "command len %zd too big!\n", inlen); return -EOVERFLOW; } diff --git a/drivers/platform/x86/amd/pmc/pmc-quirks.c b/drivers/platform/x86/amd/pmc/pmc-quirks.c index ed285afaf9b0d7..24506e3429430f 100644 --- a/drivers/platform/x86/amd/pmc/pmc-quirks.c +++ b/drivers/platform/x86/amd/pmc/pmc-quirks.c @@ -203,6 +203,15 @@ static const struct dmi_system_id fwbug_list[] = { DMI_MATCH(DMI_PRODUCT_NAME, "82XQ"), } }, + /* https://bugzilla.kernel.org/show_bug.cgi?id=221273 */ + { + .ident = "Thinkpad L14 Gen3", + .driver_data = &quirk_s2idle_bug, + .matches = { + DMI_MATCH(DMI_BOARD_VENDOR, "LENOVO"), + DMI_MATCH(DMI_PRODUCT_NAME, "21C6"), + } + }, /* https://gitlab.freedesktop.org/drm/amd/-/issues/4434 */ { .ident = "Lenovo Yoga 6 13ALC6", diff --git a/drivers/platform/x86/amd/pmf/core.c b/drivers/platform/x86/amd/pmf/core.c index 8fc293c9c5380d..15c27edfb6d855 100644 --- a/drivers/platform/x86/amd/pmf/core.c +++ b/drivers/platform/x86/amd/pmf/core.c @@ -314,6 +314,61 @@ int amd_pmf_init_metrics_table(struct amd_pmf_dev *dev) return 0; } +static int amd_pmf_reinit_ta(struct amd_pmf_dev *pdev) +{ + bool status; + int ret, i; + + for (i = 0; i < ARRAY_SIZE(amd_pmf_ta_uuid); i++) { + ret = amd_pmf_tee_init(pdev, &amd_pmf_ta_uuid[i]); + if (ret) { + dev_err(pdev->dev, "TEE init failed for UUID[%d] ret: %d\n", i, ret); + return ret; + } + + ret = amd_pmf_start_policy_engine(pdev); + dev_dbg(pdev->dev, "start policy engine ret: %d (UUID idx: %d)\n", ret, i); + status = ret == TA_PMF_TYPE_SUCCESS; + if (status) + break; + amd_pmf_tee_deinit(pdev); + } + + return 0; +} + +static int amd_pmf_restore_handler(struct device *dev) +{ + struct amd_pmf_dev *pdev = dev_get_drvdata(dev); + int ret; + + if (pdev->buf) { + ret = amd_pmf_set_dram_addr(pdev, false); + if (ret) + return ret; + } + + if (pdev->smart_pc_enabled) + amd_pmf_reinit_ta(pdev); + + return 0; +} + +static int amd_pmf_freeze_handler(struct device *dev) +{ + struct amd_pmf_dev *pdev = dev_get_drvdata(dev); + + if (!pdev->smart_pc_enabled) + return 0; + + cancel_delayed_work_sync(&pdev->pb_work); + /* Clear all TEE resources */ + amd_pmf_tee_deinit(pdev); + pdev->session_id = 0; + + return 0; +} + static int amd_pmf_suspend_handler(struct device *dev) { struct amd_pmf_dev *pdev = dev_get_drvdata(dev); @@ -347,7 +402,12 @@ static int amd_pmf_resume_handler(struct device *dev) return 0; } -static DEFINE_SIMPLE_DEV_PM_OPS(amd_pmf_pm, amd_pmf_suspend_handler, amd_pmf_resume_handler); +static const struct dev_pm_ops amd_pmf_pm = { + .suspend = amd_pmf_suspend_handler, + .resume = amd_pmf_resume_handler, + .freeze = amd_pmf_freeze_handler, + .restore = amd_pmf_restore_handler, +}; static void amd_pmf_init_features(struct amd_pmf_dev *dev) { diff --git a/drivers/platform/x86/amd/pmf/pmf.h b/drivers/platform/x86/amd/pmf/pmf.h index 9144c8c3bbaf2f..513a6309ce1303 100644 --- a/drivers/platform/x86/amd/pmf/pmf.h +++ b/drivers/platform/x86/amd/pmf/pmf.h @@ -129,6 +129,12 @@ struct cookie_header { typedef void (*apmf_event_handler_t)(acpi_handle handle, u32 event, void *data); +static const uuid_t amd_pmf_ta_uuid[] __used = { UUID_INIT(0xd9b39bf2, 0x66bd, 0x4154, 0xaf, 0xb8, + 0x8a, 0xcc, 0x2b, 0x2b, 0x60, 0xd6), + UUID_INIT(0x6fd93b77, 0x3fb8, 0x524d, 0xb1, 0x2d, + 0xc5, 0x29, 0xb1, 0x3d, 0x85, 0x43), + }; + /* APTS PMF BIOS Interface */ struct amd_pmf_apts_output { u16 table_version; @@ -895,4 +901,8 @@ void amd_pmf_populate_ta_inputs(struct amd_pmf_dev *dev, struct ta_pmf_enact_tab void amd_pmf_dump_ta_inputs(struct amd_pmf_dev *dev, struct ta_pmf_enact_table *in); int amd_pmf_invoke_cmd_enact(struct amd_pmf_dev *dev); +int amd_pmf_tee_init(struct amd_pmf_dev *dev, const uuid_t *uuid); +void amd_pmf_tee_deinit(struct amd_pmf_dev *dev); +int amd_pmf_start_policy_engine(struct amd_pmf_dev *dev); + #endif /* PMF_H */ diff --git a/drivers/platform/x86/amd/pmf/tee-if.c b/drivers/platform/x86/amd/pmf/tee-if.c index 0abce76f89ffe5..95ceb906a5f39a 100644 --- a/drivers/platform/x86/amd/pmf/tee-if.c +++ b/drivers/platform/x86/amd/pmf/tee-if.c @@ -27,12 +27,6 @@ module_param(pb_side_load, bool, 0444); MODULE_PARM_DESC(pb_side_load, "Sideload policy binaries debug policy failures"); #endif -static const uuid_t amd_pmf_ta_uuid[] = { UUID_INIT(0xd9b39bf2, 0x66bd, 0x4154, 0xaf, 0xb8, 0x8a, - 0xcc, 0x2b, 0x2b, 0x60, 0xd6), - UUID_INIT(0x6fd93b77, 0x3fb8, 0x524d, 0xb1, 0x2d, 0xc5, - 0x29, 0xb1, 0x3d, 0x85, 0x43), - }; - static const char *amd_pmf_uevent_as_str(unsigned int state) { switch (state) { @@ -324,7 +318,7 @@ static void amd_pmf_invoke_cmd(struct work_struct *work) schedule_delayed_work(&dev->pb_work, msecs_to_jiffies(pb_actions_ms)); } -static int amd_pmf_start_policy_engine(struct amd_pmf_dev *dev) +int amd_pmf_start_policy_engine(struct amd_pmf_dev *dev) { struct cookie_header *header; int res; @@ -480,7 +474,7 @@ static int amd_pmf_register_input_device(struct amd_pmf_dev *dev) return 0; } -static int amd_pmf_tee_init(struct amd_pmf_dev *dev, const uuid_t *uuid) +int amd_pmf_tee_init(struct amd_pmf_dev *dev, const uuid_t *uuid) { u32 size; int ret; @@ -528,7 +522,7 @@ static int amd_pmf_tee_init(struct amd_pmf_dev *dev, const uuid_t *uuid) return ret; } -static void amd_pmf_tee_deinit(struct amd_pmf_dev *dev) +void amd_pmf_tee_deinit(struct amd_pmf_dev *dev) { if (!dev->tee_ctx) return; diff --git a/drivers/platform/x86/asus-nb-wmi.c b/drivers/platform/x86/asus-nb-wmi.c index a38a65f5c550d0..b4677c5bba5b44 100644 --- a/drivers/platform/x86/asus-nb-wmi.c +++ b/drivers/platform/x86/asus-nb-wmi.c @@ -548,7 +548,7 @@ static const struct dmi_system_id asus_quirks[] = { .callback = dmi_matched, .ident = "ASUS ROG Z13", .matches = { - DMI_MATCH(DMI_SYS_VENDOR, "ASUSTeK COMPUTER INC."), + DMI_MATCH(DMI_SYS_VENDOR, "ASUS"), DMI_MATCH(DMI_PRODUCT_NAME, "ROG Flow Z13"), }, .driver_data = &quirk_asus_z13, diff --git a/drivers/platform/x86/dell/alienware-wmi-wmax.c b/drivers/platform/x86/dell/alienware-wmi-wmax.c index e69b50162bb1b7..d1b4df91401b14 100644 --- a/drivers/platform/x86/dell/alienware-wmi-wmax.c +++ b/drivers/platform/x86/dell/alienware-wmi-wmax.c @@ -175,7 +175,7 @@ static const struct dmi_system_id awcc_dmi_table[] __initconst = { DMI_MATCH(DMI_SYS_VENDOR, "Alienware"), DMI_MATCH(DMI_PRODUCT_NAME, "Alienware m18"), }, - .driver_data = &generic_quirks, + .driver_data = &g_series_quirks, }, { .ident = "Alienware x15", diff --git a/drivers/platform/x86/dell/dell-wmi-base.c b/drivers/platform/x86/dell/dell-wmi-base.c index 28076929d6af5b..907f1da01c8db7 100644 --- a/drivers/platform/x86/dell/dell-wmi-base.c +++ b/drivers/platform/x86/dell/dell-wmi-base.c @@ -80,6 +80,12 @@ static const struct dmi_system_id dell_wmi_smbios_list[] __initconst = { static const struct key_entry dell_wmi_keymap_type_0000[] = { { KE_IGNORE, 0x003a, { KEY_CAPSLOCK } }, + /* Audio mute toggle */ + { KE_KEY, 0x0109, { KEY_MUTE } }, + + /* Mic mute toggle */ + { KE_KEY, 0x0150, { KEY_MICMUTE } }, + /* Meta key lock */ { KE_IGNORE, 0xe000, { KEY_RIGHTMETA } }, diff --git a/drivers/platform/x86/dell/dell-wmi-sysman/passwordattr-interface.c b/drivers/platform/x86/dell/dell-wmi-sysman/passwordattr-interface.c index 86ec962aace9b8..e586f7957946bf 100644 --- a/drivers/platform/x86/dell/dell-wmi-sysman/passwordattr-interface.c +++ b/drivers/platform/x86/dell/dell-wmi-sysman/passwordattr-interface.c @@ -93,7 +93,6 @@ int set_new_password(const char *password_type, const char *new) if (ret < 0) goto out; - print_hex_dump_bytes("set new password data: ", DUMP_PREFIX_NONE, buffer, buffer_size); ret = call_password_interface(wmi_priv.password_attr_wdev, buffer, buffer_size); /* on success copy the new password to current password */ if (!ret) diff --git a/drivers/platform/x86/hp/hp-bioscfg/enum-attributes.c b/drivers/platform/x86/hp/hp-bioscfg/enum-attributes.c index f346aad8e9d895..af4d1920d48809 100644 --- a/drivers/platform/x86/hp/hp-bioscfg/enum-attributes.c +++ b/drivers/platform/x86/hp/hp-bioscfg/enum-attributes.c @@ -94,8 +94,11 @@ int hp_alloc_enumeration_data(void) bioscfg_drv.enumeration_instances_count = hp_get_instance_count(HP_WMI_BIOS_ENUMERATION_GUID); - bioscfg_drv.enumeration_data = kcalloc(bioscfg_drv.enumeration_instances_count, - sizeof(*bioscfg_drv.enumeration_data), GFP_KERNEL); + if (!bioscfg_drv.enumeration_instances_count) + return -EINVAL; + bioscfg_drv.enumeration_data = kvcalloc(bioscfg_drv.enumeration_instances_count, + sizeof(*bioscfg_drv.enumeration_data), GFP_KERNEL); + if (!bioscfg_drv.enumeration_data) { bioscfg_drv.enumeration_instances_count = 0; return -ENOMEM; @@ -444,6 +447,6 @@ void hp_exit_enumeration_attributes(void) } bioscfg_drv.enumeration_instances_count = 0; - kfree(bioscfg_drv.enumeration_data); + kvfree(bioscfg_drv.enumeration_data); bioscfg_drv.enumeration_data = NULL; } diff --git a/drivers/platform/x86/hp/hp-wmi.c b/drivers/platform/x86/hp/hp-wmi.c index f4ea1ea05997b6..7d03903cf221a0 100644 --- a/drivers/platform/x86/hp/hp-wmi.c +++ b/drivers/platform/x86/hp/hp-wmi.c @@ -53,6 +53,66 @@ MODULE_ALIAS("wmi:5FB7F034-2C63-45E9-BE91-3D44E2C707E4"); #define zero_if_sup(tmp) (zero_insize_support?0:sizeof(tmp)) // use when zero insize is required +enum hp_thermal_profile_omen_v0 { + HP_OMEN_V0_THERMAL_PROFILE_DEFAULT = 0x00, + HP_OMEN_V0_THERMAL_PROFILE_PERFORMANCE = 0x01, + HP_OMEN_V0_THERMAL_PROFILE_COOL = 0x02, +}; + +enum hp_thermal_profile_omen_v1 { + HP_OMEN_V1_THERMAL_PROFILE_DEFAULT = 0x30, + HP_OMEN_V1_THERMAL_PROFILE_PERFORMANCE = 0x31, + HP_OMEN_V1_THERMAL_PROFILE_COOL = 0x50, +}; + +enum hp_thermal_profile_omen_flags { + HP_OMEN_EC_FLAGS_TURBO = 0x04, + HP_OMEN_EC_FLAGS_NOTIMER = 0x02, + HP_OMEN_EC_FLAGS_JUSTSET = 0x01, +}; + +enum hp_thermal_profile_victus { + HP_VICTUS_THERMAL_PROFILE_DEFAULT = 0x00, + HP_VICTUS_THERMAL_PROFILE_PERFORMANCE = 0x01, + HP_VICTUS_THERMAL_PROFILE_QUIET = 0x03, +}; + +enum hp_thermal_profile_victus_s { + HP_VICTUS_S_THERMAL_PROFILE_DEFAULT = 0x00, + HP_VICTUS_S_THERMAL_PROFILE_PERFORMANCE = 0x01, +}; + +enum hp_thermal_profile { + HP_THERMAL_PROFILE_PERFORMANCE = 0x00, + HP_THERMAL_PROFILE_DEFAULT = 0x01, + HP_THERMAL_PROFILE_COOL = 0x02, + HP_THERMAL_PROFILE_QUIET = 0x03, +}; + +struct thermal_profile_params { + u8 performance; + u8 balanced; + u8 low_power; +}; + +static const struct thermal_profile_params victus_s_thermal_params = { + .performance = HP_VICTUS_S_THERMAL_PROFILE_PERFORMANCE, + .balanced = HP_VICTUS_S_THERMAL_PROFILE_DEFAULT, + .low_power = HP_VICTUS_S_THERMAL_PROFILE_DEFAULT, +}; + +static const struct thermal_profile_params omen_v1_thermal_params = { + .performance = HP_OMEN_V1_THERMAL_PROFILE_PERFORMANCE, + .balanced = HP_OMEN_V1_THERMAL_PROFILE_DEFAULT, + .low_power = HP_OMEN_V1_THERMAL_PROFILE_DEFAULT, +}; + +/* + * A generic pointer for the currently-active board's thermal profile + * parameters. + */ +static struct thermal_profile_params *active_thermal_profile_params; + /* DMI board names of devices that should use the omen specific path for * thermal profiles. * This was obtained by taking a look in the windows omen command center @@ -73,6 +133,7 @@ static const char * const omen_thermal_profile_boards[] = { "8900", "8901", "8902", "8912", "8917", "8918", "8949", "894A", "89EB", "8A15", "8A42", "8BAD", + "8E41", }; /* DMI Board names of Omen laptops that are specifically set to be thermal @@ -93,18 +154,59 @@ static const char * const omen_timed_thermal_profile_boards[] = { "8BAD", }; -/* DMI Board names of Victus 16-d1xxx laptops */ +/* DMI Board names of Victus 16-d laptops */ static const char * const victus_thermal_profile_boards[] = { + "88F8", "8A25", }; /* DMI Board names of Victus 16-r and Victus 16-s laptops */ -static const char * const victus_s_thermal_profile_boards[] = { - "8BBE", "8BD4", "8BD5", - "8C78", "8C99", "8C9C", - "8D41", +static const struct dmi_system_id victus_s_thermal_profile_boards[] __initconst = { + { + .matches = { DMI_MATCH(DMI_BOARD_NAME, "8BAB") }, + .driver_data = (void *)&omen_v1_thermal_params, + }, + { + .matches = { DMI_MATCH(DMI_BOARD_NAME, "8BBE") }, + .driver_data = (void *)&victus_s_thermal_params, + }, + { + .matches = { DMI_MATCH(DMI_BOARD_NAME, "8BCD") }, + .driver_data = (void *)&omen_v1_thermal_params, + }, + { + .matches = { DMI_MATCH(DMI_BOARD_NAME, "8BD4") }, + .driver_data = (void *)&victus_s_thermal_params, + }, + { + .matches = { DMI_MATCH(DMI_BOARD_NAME, "8BD5") }, + .driver_data = (void *)&victus_s_thermal_params, + }, + { + .matches = { DMI_MATCH(DMI_BOARD_NAME, "8C76") }, + .driver_data = (void *)&omen_v1_thermal_params, + }, + { + .matches = { DMI_MATCH(DMI_BOARD_NAME, "8C78") }, + .driver_data = (void *)&omen_v1_thermal_params, + }, + { + .matches = { DMI_MATCH(DMI_BOARD_NAME, "8C99") }, + .driver_data = (void *)&victus_s_thermal_params, + }, + { + .matches = { DMI_MATCH(DMI_BOARD_NAME, "8C9C") }, + .driver_data = (void *)&victus_s_thermal_params, + }, + { + .matches = { DMI_MATCH(DMI_BOARD_NAME, "8D41") }, + .driver_data = (void *)&victus_s_thermal_params, + }, + {}, }; +static bool is_victus_s_board; + enum hp_wmi_radio { HPWMI_WIFI = 0x0, HPWMI_BLUETOOTH = 0x1, @@ -225,42 +327,6 @@ enum hp_wireless2_bits { HPWMI_POWER_FW_OR_HW = HPWMI_POWER_BIOS | HPWMI_POWER_HARD, }; -enum hp_thermal_profile_omen_v0 { - HP_OMEN_V0_THERMAL_PROFILE_DEFAULT = 0x00, - HP_OMEN_V0_THERMAL_PROFILE_PERFORMANCE = 0x01, - HP_OMEN_V0_THERMAL_PROFILE_COOL = 0x02, -}; - -enum hp_thermal_profile_omen_v1 { - HP_OMEN_V1_THERMAL_PROFILE_DEFAULT = 0x30, - HP_OMEN_V1_THERMAL_PROFILE_PERFORMANCE = 0x31, - HP_OMEN_V1_THERMAL_PROFILE_COOL = 0x50, -}; - -enum hp_thermal_profile_omen_flags { - HP_OMEN_EC_FLAGS_TURBO = 0x04, - HP_OMEN_EC_FLAGS_NOTIMER = 0x02, - HP_OMEN_EC_FLAGS_JUSTSET = 0x01, -}; - -enum hp_thermal_profile_victus { - HP_VICTUS_THERMAL_PROFILE_DEFAULT = 0x00, - HP_VICTUS_THERMAL_PROFILE_PERFORMANCE = 0x01, - HP_VICTUS_THERMAL_PROFILE_QUIET = 0x03, -}; - -enum hp_thermal_profile_victus_s { - HP_VICTUS_S_THERMAL_PROFILE_DEFAULT = 0x00, - HP_VICTUS_S_THERMAL_PROFILE_PERFORMANCE = 0x01, -}; - -enum hp_thermal_profile { - HP_THERMAL_PROFILE_PERFORMANCE = 0x00, - HP_THERMAL_PROFILE_DEFAULT = 0x01, - HP_THERMAL_PROFILE_COOL = 0x02, - HP_THERMAL_PROFILE_QUIET = 0x03, -}; - #define IS_HWBLOCKED(x) ((x & HPWMI_POWER_FW_OR_HW) != HPWMI_POWER_FW_OR_HW) #define IS_SWBLOCKED(x) !(x & HPWMI_POWER_SOFT) @@ -1581,15 +1647,8 @@ static int platform_profile_victus_set_ec(enum platform_profile_option profile) static bool is_victus_s_thermal_profile(void) { - const char *board_name; - - board_name = dmi_get_system_info(DMI_BOARD_NAME); - if (!board_name) - return false; - - return match_string(victus_s_thermal_profile_boards, - ARRAY_SIZE(victus_s_thermal_profile_boards), - board_name) >= 0; + /* Initialised in driver init, hence safe to use here */ + return is_victus_s_board; } static int victus_s_gpu_thermal_profile_get(bool *ctgp_enable, @@ -1672,25 +1731,30 @@ static int victus_s_set_cpu_pl1_pl2(u8 pl1, u8 pl2) static int platform_profile_victus_s_set_ec(enum platform_profile_option profile) { + struct thermal_profile_params *params; bool gpu_ctgp_enable, gpu_ppab_enable; u8 gpu_dstate; /* Test shows 1 = 100%, 2 = 50%, 3 = 25%, 4 = 12.5% */ int err, tp; + params = active_thermal_profile_params; + if (!params) + return -ENODEV; + switch (profile) { case PLATFORM_PROFILE_PERFORMANCE: - tp = HP_VICTUS_S_THERMAL_PROFILE_PERFORMANCE; + tp = params->performance; gpu_ctgp_enable = true; gpu_ppab_enable = true; gpu_dstate = 1; break; case PLATFORM_PROFILE_BALANCED: - tp = HP_VICTUS_S_THERMAL_PROFILE_DEFAULT; + tp = params->balanced; gpu_ctgp_enable = false; gpu_ppab_enable = true; gpu_dstate = 1; break; case PLATFORM_PROFILE_LOW_POWER: - tp = HP_VICTUS_S_THERMAL_PROFILE_DEFAULT; + tp = params->low_power; gpu_ctgp_enable = false; gpu_ppab_enable = false; gpu_dstate = 1; @@ -2227,6 +2291,26 @@ static int hp_wmi_hwmon_init(void) return 0; } +static void __init setup_active_thermal_profile_params(void) +{ + const struct dmi_system_id *id; + + /* + * Currently only victus_s devices use the + * active_thermal_profile_params + */ + id = dmi_first_match(victus_s_thermal_profile_boards); + if (id) { + /* + * Marking this boolean is required to ensure that + * is_victus_s_thermal_profile() behaves like a valid + * wrapper. + */ + is_victus_s_board = true; + active_thermal_profile_params = id->driver_data; + } +} + static int __init hp_wmi_init(void) { int event_capable = wmi_has_guid(HPWMI_EVENT_GUID); @@ -2254,6 +2338,11 @@ static int __init hp_wmi_init(void) goto err_destroy_input; } + /* + * Setup active board's thermal profile parameters before + * starting platform driver probe. + */ + setup_active_thermal_profile_params(); err = platform_driver_probe(&hp_wmi_driver, hp_wmi_bios_setup); if (err) goto err_unregister_device; diff --git a/drivers/platform/x86/intel/hid.c b/drivers/platform/x86/intel/hid.c index 560cc063198e10..c5e80887d0cb01 100644 --- a/drivers/platform/x86/intel/hid.c +++ b/drivers/platform/x86/intel/hid.c @@ -135,6 +135,13 @@ static const struct dmi_system_id button_array_table[] = { DMI_MATCH(DMI_PRODUCT_FAMILY, "ThinkPad X1 Tablet Gen 2"), }, }, + { + .ident = "Lenovo ThinkPad X1 Fold 16 Gen 1", + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "LENOVO"), + DMI_MATCH(DMI_PRODUCT_FAMILY, "ThinkPad X1 Fold 16 Gen 1"), + }, + }, { .ident = "Microsoft Surface Go 3", .matches = { @@ -189,6 +196,12 @@ static const struct dmi_system_id dmi_vgbs_allow_list[] = { DMI_MATCH(DMI_PRODUCT_NAME, "Dell Pro Rugged 12 Tablet RA02260"), }, }, + { + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."), + DMI_MATCH(DMI_PRODUCT_NAME, "Dell 14 Plus 2-in-1 DB04250"), + }, + }, { } }; @@ -419,6 +432,14 @@ static int intel_hid_pl_suspend_handler(struct device *device) return 0; } +static int intel_hid_pl_freeze_handler(struct device *device) +{ + struct intel_hid_priv *priv = dev_get_drvdata(device); + + priv->wakeup_mode = false; + return intel_hid_pl_suspend_handler(device); +} + static int intel_hid_pl_resume_handler(struct device *device) { intel_hid_pm_complete(device); @@ -433,7 +454,7 @@ static int intel_hid_pl_resume_handler(struct device *device) static const struct dev_pm_ops intel_hid_pl_pm_ops = { .prepare = intel_hid_pm_prepare, .complete = intel_hid_pm_complete, - .freeze = intel_hid_pl_suspend_handler, + .freeze = intel_hid_pl_freeze_handler, .thaw = intel_hid_pl_resume_handler, .restore = intel_hid_pl_resume_handler, .suspend = intel_hid_pl_suspend_handler, diff --git a/drivers/platform/x86/intel/int0002_vgpio.c b/drivers/platform/x86/intel/int0002_vgpio.c index 6f5629dc3f8dbc..562e8802564367 100644 --- a/drivers/platform/x86/intel/int0002_vgpio.c +++ b/drivers/platform/x86/intel/int0002_vgpio.c @@ -206,8 +206,8 @@ static int int0002_probe(struct platform_device *pdev) * FIXME: augment this if we managed to pull handling of shared * IRQs into gpiolib. */ - ret = devm_request_irq(dev, irq, int0002_irq, - IRQF_ONESHOT | IRQF_SHARED, "INT0002", chip); + ret = devm_request_irq(dev, irq, int0002_irq, IRQF_SHARED, "INT0002", + chip); if (ret) { dev_err(dev, "Error requesting IRQ %d: %d\n", irq, ret); return ret; diff --git a/drivers/platform/x86/intel/speed_select_if/isst_tpmi_core.c b/drivers/platform/x86/intel/speed_select_if/isst_tpmi_core.c index 34bff2f65a835b..c8de8688daedc6 100644 --- a/drivers/platform/x86/intel/speed_select_if/isst_tpmi_core.c +++ b/drivers/platform/x86/intel/speed_select_if/isst_tpmi_core.c @@ -558,6 +558,9 @@ static bool disable_dynamic_sst_features(void) { u64 value; + if (!static_cpu_has(X86_FEATURE_HWP)) + return true; + rdmsrq(MSR_PM_ENABLE, value); return !(value & 0x1); } @@ -612,6 +615,9 @@ static long isst_if_core_power_state(void __user *argp) return -EINVAL; if (core_power.get_set) { + if (power_domain_info->write_blocked) + return -EPERM; + _write_cp_info("cp_enable", core_power.enable, SST_CP_CONTROL_OFFSET, SST_CP_ENABLE_START, SST_CP_ENABLE_WIDTH, SST_MUL_FACTOR_NONE) _write_cp_info("cp_prio_type", core_power.priority_type, SST_CP_CONTROL_OFFSET, @@ -865,7 +871,7 @@ static int isst_if_get_perf_level(void __user *argp) _read_pp_info("current_level", perf_level.current_level, SST_PP_STATUS_OFFSET, SST_PP_LEVEL_START, SST_PP_LEVEL_WIDTH, SST_MUL_FACTOR_NONE) _read_pp_info("locked", perf_level.locked, SST_PP_STATUS_OFFSET, - SST_PP_LOCK_START, SST_PP_LEVEL_WIDTH, SST_MUL_FACTOR_NONE) + SST_PP_LOCK_START, SST_PP_LOCK_WIDTH, SST_MUL_FACTOR_NONE) _read_pp_info("feature_state", perf_level.feature_state, SST_PP_STATUS_OFFSET, SST_PP_FEATURE_STATE_START, SST_PP_FEATURE_STATE_WIDTH, SST_MUL_FACTOR_NONE) perf_level.enabled = !!(power_domain_info->sst_header.cap_mask & BIT(1)); @@ -1454,6 +1460,8 @@ static int isst_if_get_turbo_freq_info(void __user *argp) SST_MUL_FACTOR_FREQ) } + memset(turbo_freq.bucket_core_counts, 0, sizeof(turbo_freq.bucket_core_counts)); + if (feature_rev >= 2) { bool has_tf_info_8 = false; @@ -1720,55 +1728,67 @@ EXPORT_SYMBOL_NS_GPL(tpmi_sst_dev_remove, "INTEL_TPMI_SST"); void tpmi_sst_dev_suspend(struct auxiliary_device *auxdev) { struct tpmi_sst_struct *tpmi_sst = auxiliary_get_drvdata(auxdev); - struct tpmi_per_power_domain_info *power_domain_info; + struct tpmi_per_power_domain_info *power_domain_info, *pd_info; struct oobmsm_plat_info *plat_info; void __iomem *cp_base; + int num_resources, i; plat_info = tpmi_get_platform_data(auxdev); if (!plat_info) return; power_domain_info = tpmi_sst->power_domain_info[plat_info->partition]; + num_resources = tpmi_sst->number_of_power_domains[plat_info->partition]; - cp_base = power_domain_info->sst_base + power_domain_info->sst_header.cp_offset; - power_domain_info->saved_sst_cp_control = readq(cp_base + SST_CP_CONTROL_OFFSET); - - memcpy_fromio(power_domain_info->saved_clos_configs, cp_base + SST_CLOS_CONFIG_0_OFFSET, - sizeof(power_domain_info->saved_clos_configs)); + for (i = 0; i < num_resources; i++) { + pd_info = &power_domain_info[i]; + if (!pd_info || !pd_info->sst_base) + continue; - memcpy_fromio(power_domain_info->saved_clos_assocs, cp_base + SST_CLOS_ASSOC_0_OFFSET, - sizeof(power_domain_info->saved_clos_assocs)); + cp_base = pd_info->sst_base + pd_info->sst_header.cp_offset; + pd_info->saved_sst_cp_control = readq(cp_base + SST_CP_CONTROL_OFFSET); + memcpy_fromio(pd_info->saved_clos_configs, cp_base + SST_CLOS_CONFIG_0_OFFSET, + sizeof(pd_info->saved_clos_configs)); + memcpy_fromio(pd_info->saved_clos_assocs, cp_base + SST_CLOS_ASSOC_0_OFFSET, + sizeof(pd_info->saved_clos_assocs)); - power_domain_info->saved_pp_control = readq(power_domain_info->sst_base + - power_domain_info->sst_header.pp_offset + - SST_PP_CONTROL_OFFSET); + pd_info->saved_pp_control = readq(pd_info->sst_base + + pd_info->sst_header.pp_offset + + SST_PP_CONTROL_OFFSET); + } } EXPORT_SYMBOL_NS_GPL(tpmi_sst_dev_suspend, "INTEL_TPMI_SST"); void tpmi_sst_dev_resume(struct auxiliary_device *auxdev) { struct tpmi_sst_struct *tpmi_sst = auxiliary_get_drvdata(auxdev); - struct tpmi_per_power_domain_info *power_domain_info; + struct tpmi_per_power_domain_info *power_domain_info, *pd_info; struct oobmsm_plat_info *plat_info; void __iomem *cp_base; + int num_resources, i; plat_info = tpmi_get_platform_data(auxdev); if (!plat_info) return; power_domain_info = tpmi_sst->power_domain_info[plat_info->partition]; + num_resources = tpmi_sst->number_of_power_domains[plat_info->partition]; - cp_base = power_domain_info->sst_base + power_domain_info->sst_header.cp_offset; - writeq(power_domain_info->saved_sst_cp_control, cp_base + SST_CP_CONTROL_OFFSET); - - memcpy_toio(cp_base + SST_CLOS_CONFIG_0_OFFSET, power_domain_info->saved_clos_configs, - sizeof(power_domain_info->saved_clos_configs)); + for (i = 0; i < num_resources; i++) { + pd_info = &power_domain_info[i]; + if (!pd_info || !pd_info->sst_base) + continue; - memcpy_toio(cp_base + SST_CLOS_ASSOC_0_OFFSET, power_domain_info->saved_clos_assocs, - sizeof(power_domain_info->saved_clos_assocs)); + cp_base = pd_info->sst_base + pd_info->sst_header.cp_offset; + writeq(pd_info->saved_sst_cp_control, cp_base + SST_CP_CONTROL_OFFSET); + memcpy_toio(cp_base + SST_CLOS_CONFIG_0_OFFSET, pd_info->saved_clos_configs, + sizeof(pd_info->saved_clos_configs)); + memcpy_toio(cp_base + SST_CLOS_ASSOC_0_OFFSET, pd_info->saved_clos_assocs, + sizeof(pd_info->saved_clos_assocs)); - writeq(power_domain_info->saved_pp_control, power_domain_info->sst_base + - power_domain_info->sst_header.pp_offset + SST_PP_CONTROL_OFFSET); + writeq(pd_info->saved_pp_control, power_domain_info->sst_base + + pd_info->sst_header.pp_offset + SST_PP_CONTROL_OFFSET); + } } EXPORT_SYMBOL_NS_GPL(tpmi_sst_dev_resume, "INTEL_TPMI_SST"); diff --git a/drivers/platform/x86/intel/uncore-frequency/uncore-frequency-tpmi.c b/drivers/platform/x86/intel/uncore-frequency/uncore-frequency-tpmi.c index 1237d957088650..4c7e64db478c02 100644 --- a/drivers/platform/x86/intel/uncore-frequency/uncore-frequency-tpmi.c +++ b/drivers/platform/x86/intel/uncore-frequency/uncore-frequency-tpmi.c @@ -537,6 +537,7 @@ static void set_cdie_id(int domain_id, struct tpmi_uncore_cluster_info *cluster_ #define UNCORE_VERSION_MASK GENMASK_ULL(7, 0) #define UNCORE_LOCAL_FABRIC_CLUSTER_ID_MASK GENMASK_ULL(15, 8) #define UNCORE_CLUSTER_OFF_MASK GENMASK_ULL(7, 0) +#define UNCORE_AUTONOMOUS_UFS_DISABLED BIT(32) #define UNCORE_MAX_CLUSTER_PER_DOMAIN 8 static int uncore_probe(struct auxiliary_device *auxdev, const struct auxiliary_device_id *id) @@ -598,6 +599,7 @@ static int uncore_probe(struct auxiliary_device *auxdev, const struct auxiliary_ for (i = 0; i < num_resources; ++i) { struct tpmi_uncore_power_domain_info *pd_info; + bool auto_ufs_enabled; struct resource *res; u64 cluster_offset; u8 cluster_mask; @@ -647,6 +649,8 @@ static int uncore_probe(struct auxiliary_device *auxdev, const struct auxiliary_ continue; } + auto_ufs_enabled = !(header & UNCORE_AUTONOMOUS_UFS_DISABLED); + /* Find out number of clusters in this resource */ pd_info->cluster_count = hweight8(cluster_mask); @@ -689,7 +693,9 @@ static int uncore_probe(struct auxiliary_device *auxdev, const struct auxiliary_ cluster_info->uncore_root = tpmi_uncore; - if (TPMI_MINOR_VERSION(pd_info->ufs_header_ver) >= UNCORE_ELC_SUPPORTED_VERSION) + if ((TPMI_MINOR_VERSION(pd_info->ufs_header_ver) >= + UNCORE_ELC_SUPPORTED_VERSION) && + auto_ufs_enabled) cluster_info->elc_supported = true; ret = uncore_freq_add_entry(&cluster_info->uncore_data, 0); diff --git a/drivers/platform/x86/lenovo/thinkpad_acpi.c b/drivers/platform/x86/lenovo/thinkpad_acpi.c index cc19fe520ea96e..075543cd0e77e0 100644 --- a/drivers/platform/x86/lenovo/thinkpad_acpi.c +++ b/drivers/platform/x86/lenovo/thinkpad_acpi.c @@ -9525,14 +9525,16 @@ static int tpacpi_battery_get(int what, int battery, int *ret) { switch (what) { case THRESHOLD_START: - if ACPI_FAILURE(tpacpi_battery_acpi_eval(GET_START, ret, battery)) + if (!battery_info.batteries[battery].start_support || + ACPI_FAILURE(tpacpi_battery_acpi_eval(GET_START, ret, battery))) return -ENODEV; /* The value is in the low 8 bits of the response */ *ret = *ret & 0xFF; return 0; case THRESHOLD_STOP: - if ACPI_FAILURE(tpacpi_battery_acpi_eval(GET_STOP, ret, battery)) + if (!battery_info.batteries[battery].stop_support || + ACPI_FAILURE(tpacpi_battery_acpi_eval(GET_STOP, ret, battery))) return -ENODEV; /* Value is in lower 8 bits */ *ret = *ret & 0xFF; diff --git a/drivers/platform/x86/lenovo/wmi-gamezone.c b/drivers/platform/x86/lenovo/wmi-gamezone.c index 381836d29a9646..c7fe7e3c9f1791 100644 --- a/drivers/platform/x86/lenovo/wmi-gamezone.c +++ b/drivers/platform/x86/lenovo/wmi-gamezone.c @@ -31,8 +31,6 @@ #define LWMI_GZ_METHOD_ID_SMARTFAN_SET 44 #define LWMI_GZ_METHOD_ID_SMARTFAN_GET 45 -static BLOCKING_NOTIFIER_HEAD(gz_chain_head); - struct lwmi_gz_priv { enum thermal_mode current_mode; struct notifier_block event_nb; diff --git a/drivers/platform/x86/oxpec.c b/drivers/platform/x86/oxpec.c index 144a454103b93b..6d4a53a2ed603d 100644 --- a/drivers/platform/x86/oxpec.c +++ b/drivers/platform/x86/oxpec.c @@ -11,7 +11,7 @@ * * Copyright (C) 2022 Joaquín I. Aramendía * Copyright (C) 2024 Derek J. Clark - * Copyright (C) 2025 Antheas Kapenekakis + * Copyright (C) 2025-2026 Antheas Kapenekakis */ #include @@ -114,6 +114,13 @@ static const struct dmi_system_id dmi_table[] = { }, .driver_data = (void *)aok_zoe_a1, }, + { + .matches = { + DMI_MATCH(DMI_BOARD_VENDOR, "AOKZOE"), + DMI_EXACT_MATCH(DMI_BOARD_NAME, "AOKZOE A2 Pro"), + }, + .driver_data = (void *)aok_zoe_a1, + }, { .matches = { DMI_MATCH(DMI_BOARD_VENDOR, "AOKZOE"), @@ -142,6 +149,13 @@ static const struct dmi_system_id dmi_table[] = { }, .driver_data = (void *)oxp_2, }, + { + .matches = { + DMI_MATCH(DMI_BOARD_VENDOR, "ONE-NETBOOK"), + DMI_EXACT_MATCH(DMI_BOARD_NAME, "ONEXPLAYER APEX"), + }, + .driver_data = (void *)oxp_fly, + }, { .matches = { DMI_MATCH(DMI_BOARD_VENDOR, "ONE-NETBOOK"), @@ -212,6 +226,13 @@ static const struct dmi_system_id dmi_table[] = { }, .driver_data = (void *)oxp_mini_amd_pro, }, + { + .matches = { + DMI_MATCH(DMI_BOARD_VENDOR, "ONE-NETBOOK"), + DMI_EXACT_MATCH(DMI_BOARD_NAME, "ONEXPLAYER X1z"), + }, + .driver_data = (void *)oxp_x1, + }, { .matches = { DMI_MATCH(DMI_BOARD_VENDOR, "ONE-NETBOOK"), @@ -226,6 +247,13 @@ static const struct dmi_system_id dmi_table[] = { }, .driver_data = (void *)oxp_x1, }, + { + .matches = { + DMI_MATCH(DMI_BOARD_VENDOR, "ONE-NETBOOK"), + DMI_EXACT_MATCH(DMI_BOARD_NAME, "ONEXPLAYER X1Air"), + }, + .driver_data = (void *)oxp_x1, + }, { .matches = { DMI_MATCH(DMI_BOARD_VENDOR, "ONE-NETBOOK"), diff --git a/drivers/platform/x86/touchscreen_dmi.c b/drivers/platform/x86/touchscreen_dmi.c index bdc19cd8d3edf7..d83c387821ea1c 100644 --- a/drivers/platform/x86/touchscreen_dmi.c +++ b/drivers/platform/x86/touchscreen_dmi.c @@ -410,6 +410,16 @@ static const struct ts_dmi_data gdix1002_upside_down_data = { .properties = gdix1001_upside_down_props, }; +static const struct property_entry gdix1001_y_inverted_props[] = { + PROPERTY_ENTRY_BOOL("touchscreen-inverted-y"), + { } +}; + +static const struct ts_dmi_data gdix1001_y_inverted_data = { + .acpi_name = "GDIX1001", + .properties = gdix1001_y_inverted_props, +}; + static const struct property_entry gp_electronic_t701_props[] = { PROPERTY_ENTRY_U32("touchscreen-size-x", 960), PROPERTY_ENTRY_U32("touchscreen-size-y", 640), @@ -1658,6 +1668,14 @@ const struct dmi_system_id touchscreen_dmi_table[] = { DMI_MATCH(DMI_PRODUCT_SKU, "PN20170413488"), }, }, + { + /* SUPI S10 */ + .driver_data = (void *)&gdix1001_y_inverted_data, + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "SUPI"), + DMI_MATCH(DMI_PRODUCT_NAME, "S10"), + }, + }, { /* Techbite Arc 11.6 */ .driver_data = (void *)&techbite_arc_11_6_data, diff --git a/drivers/pmdomain/apple/Kconfig b/drivers/pmdomain/apple/Kconfig index a8973f8057fba7..6e9deb9de0739c 100644 --- a/drivers/pmdomain/apple/Kconfig +++ b/drivers/pmdomain/apple/Kconfig @@ -14,4 +14,10 @@ config APPLE_PMGR_PWRSTATE controls for SoC devices. This driver manages them through the generic power domain framework, and also provides reset support. +config APPLE_PMP_REPORT + bool "Apple PMP report control" + depends on OF + depends on PM + select PM_GENERIC_DOMAINS + endif diff --git a/drivers/pmdomain/apple/Makefile b/drivers/pmdomain/apple/Makefile index 53665af630be22..b098b204d9868a 100644 --- a/drivers/pmdomain/apple/Makefile +++ b/drivers/pmdomain/apple/Makefile @@ -1,2 +1,3 @@ # SPDX-License-Identifier: GPL-2.0-only obj-$(CONFIG_APPLE_PMGR_PWRSTATE) += pmgr-pwrstate.o +obj-$(CONFIG_APPLE_PMP_REPORT) += pmp-report.o diff --git a/drivers/pmdomain/apple/pmgr-pwrstate.c b/drivers/pmdomain/apple/pmgr-pwrstate.c index 82c33cf727a825..a92db8cf88bb90 100644 --- a/drivers/pmdomain/apple/pmgr-pwrstate.c +++ b/drivers/pmdomain/apple/pmgr-pwrstate.c @@ -21,7 +21,8 @@ #define APPLE_PMGR_AUTO_ENABLE BIT(28) #define APPLE_PMGR_PS_AUTO GENMASK(27, 24) #define APPLE_PMGR_PS_MIN GENMASK(19, 16) -#define APPLE_PMGR_PARENT_OFF BIT(11) +#define APPLE_PMGR_PS_RESET BIT(12) +#define APPLE_PMGR_BUSY BIT(11) #define APPLE_PMGR_DEV_DISABLE BIT(10) #define APPLE_PMGR_WAS_CLKGATED BIT(9) #define APPLE_PMGR_WAS_PWRGATED BIT(8) @@ -44,6 +45,9 @@ struct apple_pmgr_ps { struct regmap *regmap; u32 offset; u32 min_state; + bool force_disable; + bool force_reset; + bool externally_clocked; }; #define genpd_to_apple_pmgr_ps(_genpd) container_of(_genpd, struct apple_pmgr_ps, genpd) @@ -53,7 +57,7 @@ static int apple_pmgr_ps_set(struct generic_pm_domain *genpd, u32 pstate, bool a { int ret; struct apple_pmgr_ps *ps = genpd_to_apple_pmgr_ps(genpd); - u32 reg; + u32 reg, cur; ret = regmap_read(ps->regmap, ps->offset, ®); if (ret < 0) @@ -64,24 +68,57 @@ static int apple_pmgr_ps_set(struct generic_pm_domain *genpd, u32 pstate, bool a dev_err(ps->dev, "PS %s: powering off with RESET active\n", genpd->name); - reg &= ~(APPLE_PMGR_AUTO_ENABLE | APPLE_PMGR_FLAGS | APPLE_PMGR_PS_TARGET); + if (pstate != APPLE_PMGR_PS_ACTIVE && (ps->force_disable || ps->force_reset)) { + u32 reg_pre = reg & ~(APPLE_PMGR_AUTO_ENABLE | APPLE_PMGR_FLAGS); + + if (ps->force_disable) + reg_pre |= APPLE_PMGR_DEV_DISABLE; + if (ps->force_reset) + reg_pre |= APPLE_PMGR_PS_RESET; + + regmap_write(ps->regmap, ps->offset, reg_pre); + + ret = regmap_read_poll_timeout_atomic( + ps->regmap, ps->offset, cur, + (cur & (APPLE_PMGR_DEV_DISABLE | APPLE_PMGR_PS_RESET)) == + (reg_pre & (APPLE_PMGR_DEV_DISABLE | APPLE_PMGR_PS_RESET)), 1, + APPLE_PMGR_PS_SET_TIMEOUT); + + if (ret < 0) + dev_err(ps->dev, "PS %s: Failed to set reset/disable bits (now: 0x%x)\n", + genpd->name, reg); + } + + reg &= ~(APPLE_PMGR_DEV_DISABLE | APPLE_PMGR_PS_RESET | + APPLE_PMGR_AUTO_ENABLE | APPLE_PMGR_FLAGS | APPLE_PMGR_PS_TARGET); reg |= FIELD_PREP(APPLE_PMGR_PS_TARGET, pstate); dev_dbg(ps->dev, "PS %s: pwrstate = 0x%x: 0x%x\n", genpd->name, pstate, reg); regmap_write(ps->regmap, ps->offset, reg); - ret = regmap_read_poll_timeout_atomic( - ps->regmap, ps->offset, reg, - (FIELD_GET(APPLE_PMGR_PS_ACTUAL, reg) == pstate), 1, - APPLE_PMGR_PS_SET_TIMEOUT); + if (ps->externally_clocked && pstate == APPLE_PMGR_PS_ACTIVE) { + /* + * If this clock domain requires an external clock, then + * consider the "clock gated" state to be good enough. + */ + ret = regmap_read_poll_timeout_atomic( + ps->regmap, ps->offset, cur, + FIELD_GET(APPLE_PMGR_PS_ACTUAL, cur) >= APPLE_PMGR_PS_CLKGATE, 1, + APPLE_PMGR_PS_SET_TIMEOUT); + } else { + ret = regmap_read_poll_timeout_atomic( + ps->regmap, ps->offset, cur, + FIELD_GET(APPLE_PMGR_PS_ACTUAL, cur) == pstate, 1, + APPLE_PMGR_PS_SET_TIMEOUT); + } + if (ret < 0) dev_err(ps->dev, "PS %s: Failed to reach power state 0x%x (now: 0x%x)\n", genpd->name, pstate, reg); if (auto_enable) { /* Not all devices implement this; this is a no-op where not implemented. */ - reg &= ~APPLE_PMGR_FLAGS; reg |= APPLE_PMGR_AUTO_ENABLE; regmap_write(ps->regmap, ps->offset, reg); } @@ -234,6 +271,15 @@ static int apple_pmgr_ps_probe(struct platform_device *pdev) regmap_update_bits(regmap, ps->offset, APPLE_PMGR_FLAGS | APPLE_PMGR_PS_MIN, FIELD_PREP(APPLE_PMGR_PS_MIN, ps->min_state)); + if (of_property_read_bool(node, "apple,force-disable")) + ps->force_disable = true; + + if (of_property_read_bool(node, "apple,force-reset")) + ps->force_reset = true; + + if (of_property_read_bool(node, "apple,externally-clocked")) + ps->externally_clocked = true; + active = apple_pmgr_ps_is_active(ps); if (of_property_read_bool(node, "apple,always-on")) { ps->genpd.flags |= GENPD_FLAG_ALWAYS_ON; @@ -242,6 +288,8 @@ static int apple_pmgr_ps_probe(struct platform_device *pdev) /* Turn it on so pm_genpd_init does not fail */ active = apple_pmgr_ps_power_on(&ps->genpd) == 0; } + } else if (active) { + ps->genpd.flags |= GENPD_FLAG_DEFER_OFF | GENPD_FLAG_ACTIVE_WAKEUP; } /* Turn on auto-PM if the domain is already on */ diff --git a/drivers/pmdomain/apple/pmp-report.c b/drivers/pmdomain/apple/pmp-report.c new file mode 100644 index 00000000000000..8050ade5ef8d08 --- /dev/null +++ b/drivers/pmdomain/apple/pmp-report.c @@ -0,0 +1,239 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Apple SoC PMP power state reporting driver + * + * Copyright The Asahi Linux Contributors + */ + +#include +#include +#include +#include +#include +#include +#include + +#define PMP_REPORT_READY 0x1 + +struct apple_pmp_report_offsets { + u32 tgt_read; + u32 tgt_write; + u32 actual; + u32 status; +}; + +struct apple_pmp_report { + struct device *dev; + const struct apple_pmp_report_offsets *offsets; + void __iomem *base; + spinlock_t lock; +}; + +static int apple_pmp_report_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct device_node *np = dev->of_node; + struct apple_pmp_report *rep; + int ret; + + rep = devm_kzalloc(dev, sizeof(*rep), GFP_KERNEL); + if (!rep) + return -ENOMEM; + + rep->dev = dev; + rep->base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(rep->base)) + return PTR_ERR(rep->base); + rep->offsets = of_device_get_match_data(dev); + dev_set_drvdata(dev, rep); + ret = of_platform_populate(np, NULL, NULL, dev); + if (ret) + return dev_err_probe(dev, ret, "failed to create child devices\n"); + + return 0; +} + +static const struct apple_pmp_report_offsets apple_pmp_offsets_t600x = { + .tgt_read = 0xf80, + .tgt_write = 0x107c0, + .actual = 0x1000, + .status = 0x10, +}; + +static const struct apple_pmp_report_offsets apple_pmp_offsets_t602x = { + .tgt_read = 0x2000, + .tgt_write = 0x11000, + .actual = 0x2080, + .status = 0x10, +}; + +static const struct apple_pmp_report_offsets apple_pmp_offsets_t8112 = { + .tgt_read = 0xa00, + .tgt_write = 0x10500, + .actual = 0xa40, + .status = 0x10, +}; + +static const struct of_device_id apple_pmp_report_of_match[] = { + { .compatible = "apple,t6000-pmp-v2-report", .data = &apple_pmp_offsets_t600x }, + { .compatible = "apple,t6020-pmp-v2-report", .data = &apple_pmp_offsets_t602x }, + { .compatible = "apple,t8112-pmp-v2-report", .data = &apple_pmp_offsets_t8112 }, + {} +}; + +static struct platform_driver apple_pmp_report_driver = { + .probe = apple_pmp_report_probe, + .driver = { + .name = "apple-pmp-report", + .of_match_table = apple_pmp_report_of_match, + }, +}; + +struct apple_pmp_report_entry { + struct device *dev; + struct generic_pm_domain genpd; + u32 id; +}; + +#define genpd_to_apple_pmp_report_entry(_genpd) \ + container_of(_genpd, struct apple_pmp_report_entry, genpd) + +static int apple_pmp_report_set_state(struct generic_pm_domain *genpd, bool enable) +{ + struct apple_pmp_report_entry *ent = genpd_to_apple_pmp_report_entry(genpd); + struct apple_pmp_report *rep = dev_get_drvdata(ent->dev->parent); + u64 bit_val = 1 << ent->id; + u64 val; + unsigned long flags; + + spin_lock_irqsave(&rep->lock, flags); + val = readq(rep->base + rep->offsets->tgt_read); + val &= ~bit_val; + if (enable) + val |= bit_val; + writeq(val, rep->base + rep->offsets->tgt_write); + spin_unlock_irqrestore(&rep->lock, flags); + val = readq(rep->base + rep->offsets->status); + if ((val & PMP_REPORT_READY) == 0) + return 0; + return readq_poll_timeout_atomic( + rep->base + rep->offsets->actual, + val, + !!(val & bit_val) == !!enable, + 100, + 50000); +} + +static int apple_pmp_report_entry_power_on(struct generic_pm_domain *genpd) +{ + return apple_pmp_report_set_state(genpd, true); +} + +static int apple_pmp_report_entry_power_off(struct generic_pm_domain *genpd) +{ + return apple_pmp_report_set_state(genpd, false); +} + +static int apple_pmp_report_entry_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct device_node *node = dev->of_node; + struct apple_pmp_report_entry *ent; + int ret; + const char *name; + struct of_phandle_iterator it; + + ent = devm_kzalloc(dev, sizeof(*ent), GFP_KERNEL); + if (!ent) + return -ENOMEM; + + ent->dev = dev; + + ret = of_property_read_u32(node, "reg", &ent->id); + if (ret) + return dev_err_probe(dev, ret, "missing reg property\n"); + + ret = of_property_read_string(node, "label", &name); + if (ret < 0) + return dev_err_probe(dev, ret, "missing label property\n"); + + if (of_property_read_bool(node, "apple,always-on")) { + ent->genpd.flags |= GENPD_FLAG_ACTIVE_WAKEUP; + apple_pmp_report_set_state(&ent->genpd, true); + } + + ent->genpd.name = name; + ent->genpd.power_on = apple_pmp_report_entry_power_on; + ent->genpd.power_off = apple_pmp_report_entry_power_off; + + ret = pm_genpd_init(&ent->genpd, NULL, true); + if (ret) + return dev_err_probe(dev, ret, "pm_genpd_init failed\n"); + + ret = of_genpd_add_provider_simple(node, &ent->genpd); + if (ret) + return dev_err_probe(dev, ret, "of_genpd_add_provider_simple failed\n"); + + of_for_each_phandle(&it, ret, node, "power-domains", "#power-domain-cells", -1) { + struct of_phandle_args parent, child; + + parent.np = it.node; + parent.args_count = of_phandle_iterator_args(&it, parent.args, MAX_PHANDLE_ARGS); + child.np = node; + child.args_count = 0; + ret = of_genpd_add_subdomain(&parent, &child); + + if (ret == -EPROBE_DEFER) { + of_node_put(parent.np); + goto err_remove; + } else if (ret < 0) { + dev_err(dev, "failed to add to parent domain: %d (%s -> %s)\n", + ret, it.node->name, node->name); + of_node_put(parent.np); + goto err_remove; + } + } + + pm_genpd_remove_device(dev); + + return 0; +err_remove: + of_genpd_del_provider(node); + pm_genpd_remove(&ent->genpd); + return ret; +} + +static const struct of_device_id apple_pmp_report_entry_of_match[] = { + { .compatible = "apple,t6000-pmp-v2-report-entry" }, + {} +}; + +static struct platform_driver apple_pmp_report_entry_driver = { + .probe = apple_pmp_report_entry_probe, + .driver = { + .name = "apple-pmp-report-entry", + .of_match_table = apple_pmp_report_entry_of_match, + }, +}; + +MODULE_DEVICE_TABLE(of, apple_pmp_report_of_match); +MODULE_DEVICE_TABLE(of, apple_pmp_report_entry_of_match); + +static int __init apple_pmp_report_init(void) +{ + platform_driver_register(&apple_pmp_report_entry_driver); + platform_driver_register(&apple_pmp_report_driver); + return 0; +} + +static void __exit apple_pmp_report_exit(void) +{ + platform_driver_unregister(&apple_pmp_report_entry_driver); + platform_driver_unregister(&apple_pmp_report_driver); +} + +module_init(apple_pmp_report_init); +module_exit(apple_pmp_report_exit); + +MODULE_DESCRIPTION("PMP power state reporting driver for Apple SoCs"); +MODULE_LICENSE("Dual MIT/GPL"); diff --git a/drivers/pmdomain/bcm/bcm2835-power.c b/drivers/pmdomain/bcm/bcm2835-power.c index 1d29addfe03634..eee87a30053258 100644 --- a/drivers/pmdomain/bcm/bcm2835-power.c +++ b/drivers/pmdomain/bcm/bcm2835-power.c @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -153,7 +154,6 @@ struct bcm2835_power { static int bcm2835_asb_control(struct bcm2835_power *power, u32 reg, bool enable) { void __iomem *base = power->asb; - u64 start; u32 val; switch (reg) { @@ -166,8 +166,6 @@ static int bcm2835_asb_control(struct bcm2835_power *power, u32 reg, bool enable break; } - start = ktime_get_ns(); - /* Enable the module's async AXI bridges. */ if (enable) { val = readl(base + reg) & ~ASB_REQ_STOP; @@ -176,11 +174,9 @@ static int bcm2835_asb_control(struct bcm2835_power *power, u32 reg, bool enable } writel(PM_PASSWORD | val, base + reg); - while (!!(readl(base + reg) & ASB_ACK) == enable) { - cpu_relax(); - if (ktime_get_ns() - start >= 1000) - return -ETIMEDOUT; - } + if (readl_poll_timeout_atomic(base + reg, val, + !!(val & ASB_ACK) != enable, 0, 5)) + return -ETIMEDOUT; return 0; } @@ -580,11 +576,11 @@ static int bcm2835_reset_status(struct reset_controller_dev *rcdev, switch (id) { case BCM2835_RESET_V3D: - return !PM_READ(PM_GRAFX & PM_V3DRSTN); + return !(PM_READ(PM_GRAFX) & PM_V3DRSTN); case BCM2835_RESET_H264: - return !PM_READ(PM_IMAGE & PM_H264RSTN); + return !(PM_READ(PM_IMAGE) & PM_H264RSTN); case BCM2835_RESET_ISP: - return !PM_READ(PM_IMAGE & PM_ISPRSTN); + return !(PM_READ(PM_IMAGE) & PM_ISPRSTN); default: return -EINVAL; } diff --git a/drivers/pmdomain/core.c b/drivers/pmdomain/core.c index bf82775f6a6738..1d3493c57559fd 100644 --- a/drivers/pmdomain/core.c +++ b/drivers/pmdomain/core.c @@ -7,6 +7,7 @@ #define pr_fmt(fmt) "PM: " fmt #include +#include #include #include #include @@ -188,6 +189,7 @@ static const struct genpd_lock_ops genpd_raw_spin_ops = { #define genpd_is_dev_name_fw(genpd) (genpd->flags & GENPD_FLAG_DEV_NAME_FW) #define genpd_is_no_sync_state(genpd) (genpd->flags & GENPD_FLAG_NO_SYNC_STATE) #define genpd_is_no_stay_on(genpd) (genpd->flags & GENPD_FLAG_NO_STAY_ON) +#define genpd_is_defer_off(genpd) (genpd->flags & GENPD_FLAG_DEFER_OFF) static inline bool irq_safe_dev_in_sleep_domain(struct device *dev, const struct generic_pm_domain *genpd) @@ -941,6 +943,27 @@ static void genpd_queue_power_off_work(struct generic_pm_domain *genpd) queue_work(pm_wq, &genpd->power_off_work); } +/** + * genpd_must_defer - Check whether the genpd cannot be safely powered off. + * @genpd: PM domain about to be powered down. + * @one_dev_probing: True if we are being called from RPM callbacks on a device that + * is probing, to allow poweroff if that device is the sole remaining consumer probing. + * + * Returns true if the @genpd has the GENPD_FLAG_DEFER_OFF flag and there + * are any consumer devices which either do not exist yet (only represented + * by fwlinks) or whose drivers have not probed yet. + */ +static bool genpd_must_defer(struct generic_pm_domain *genpd, bool one_dev_probing) +{ + if (genpd_is_defer_off(genpd) && genpd->has_provider) { + int absent = fw_devlink_count_absent_consumers(genpd->provider); + + if (absent > (one_dev_probing ? 1 : 0)) + return true; + } + return false; +} + /** * genpd_power_off - Remove power from a given PM domain. * @genpd: PM domain to power down. @@ -954,7 +977,7 @@ static void genpd_queue_power_off_work(struct generic_pm_domain *genpd) * have been powered down, remove power from @genpd. */ static void genpd_power_off(struct generic_pm_domain *genpd, bool one_dev_on, - unsigned int depth) + bool one_dev_probing, unsigned int depth) { struct pm_domain_data *pdd; struct gpd_link *link; @@ -1002,6 +1025,14 @@ static void genpd_power_off(struct generic_pm_domain *genpd, bool one_dev_on, if (not_suspended > 1 || (not_suspended == 1 && !one_dev_on)) return; + /* + * Do not allow PM domain to be powered off if it is marked + * as GENPD_FLAG_DEFER_OFF and there are consumer devices + * which have not probed yet. + */ + if (genpd_must_defer(genpd, one_dev_probing)) + return; + if (genpd->gov && genpd->gov->power_down_ok) { if (!genpd->gov->power_down_ok(&genpd->domain)) return; @@ -1027,7 +1058,7 @@ static void genpd_power_off(struct generic_pm_domain *genpd, bool one_dev_on, list_for_each_entry(link, &genpd->child_links, child_node) { genpd_sd_counter_dec(link->parent); genpd_lock_nested(link->parent, depth + 1); - genpd_power_off(link->parent, false, depth + 1); + genpd_power_off(link->parent, false, false, depth + 1); genpd_unlock(link->parent); } } @@ -1086,7 +1117,7 @@ static int genpd_power_on(struct generic_pm_domain *genpd, unsigned int depth) child_node) { genpd_sd_counter_dec(link->parent); genpd_lock_nested(link->parent, depth + 1); - genpd_power_off(link->parent, false, depth + 1); + genpd_power_off(link->parent, false, false, depth + 1); genpd_unlock(link->parent); } @@ -1153,7 +1184,7 @@ static void genpd_power_off_work_fn(struct work_struct *work) genpd = container_of(work, struct generic_pm_domain, power_off_work); genpd_lock(genpd); - genpd_power_off(genpd, false, 0); + genpd_power_off(genpd, false, false, 0); genpd_unlock(genpd); } @@ -1218,6 +1249,7 @@ static int genpd_runtime_suspend(struct device *dev) struct generic_pm_domain_data *gpd_data = dev_gpd_data(dev); struct gpd_timing_data *td = gpd_data->td; bool runtime_pm = pm_runtime_enabled(dev); + bool probing = dev->links.status != DL_DEV_DRIVER_BOUND; ktime_t time_start = 0; s64 elapsed_ns; int ret; @@ -1272,7 +1304,7 @@ static int genpd_runtime_suspend(struct device *dev) return 0; genpd_lock(genpd); - genpd_power_off(genpd, true, 0); + genpd_power_off(genpd, true, probing, 0); gpd_data->rpm_pstate = genpd_drop_performance_state(dev); genpd_unlock(genpd); @@ -1293,6 +1325,7 @@ static int genpd_runtime_resume(struct device *dev) struct generic_pm_domain_data *gpd_data = dev_gpd_data(dev); struct gpd_timing_data *td = gpd_data->td; bool timed = td && pm_runtime_enabled(dev); + bool probing = dev->links.status != DL_DEV_DRIVER_BOUND; ktime_t time_start = 0; s64 elapsed_ns; int ret; @@ -1350,7 +1383,7 @@ static int genpd_runtime_resume(struct device *dev) err_poweroff: if (!pm_runtime_is_irq_safe(dev) || genpd_is_irq_safe(genpd)) { genpd_lock(genpd); - genpd_power_off(genpd, true, 0); + genpd_power_off(genpd, true, probing, 0); gpd_data->rpm_pstate = genpd_drop_performance_state(dev); genpd_unlock(genpd); } @@ -1418,6 +1451,9 @@ static void genpd_sync_power_off(struct generic_pm_domain *genpd, bool use_lock, || atomic_read(&genpd->sd_count) > 0) return; + if (genpd_must_defer(genpd, false)) + return; + /* Check that the children are in their deepest (powered-off) state. */ list_for_each_entry(link, &genpd->parent_links, parent_node) { struct generic_pm_domain *child = link->child; @@ -2445,6 +2481,12 @@ int pm_genpd_init(struct generic_pm_domain *genpd, return -EINVAL; } + /* Deferred-off power domains should be powered on at initialization. */ + if (genpd_is_defer_off(genpd) && !genpd_status_on(genpd)) { + pr_warn("deferred-off PM domain %s is not on at init\n", genpd->name); + genpd->flags &= ~GENPD_FLAG_DEFER_OFF; + } + /* Multiple states but no governor doesn't make sense. */ if (!gov && genpd->state_count > 1) pr_warn("%s: no governor for states\n", genpd->name); @@ -3511,7 +3553,7 @@ void of_genpd_sync_state(struct device_node *np) if (genpd->provider == of_fwnode_handle(np)) { genpd_lock(genpd); genpd->stay_on = false; - genpd_power_off(genpd, false, 0); + genpd_power_off(genpd, false, false, 0); genpd_unlock(genpd); } } @@ -3539,7 +3581,7 @@ static void genpd_provider_sync_state(struct device *dev) case GENPD_SYNC_STATE_SIMPLE: genpd_lock(genpd); genpd->stay_on = false; - genpd_power_off(genpd, false, 0); + genpd_power_off(genpd, false, false, 0); genpd_unlock(genpd); break; diff --git a/drivers/pmdomain/imx/imx8mp-blk-ctrl.c b/drivers/pmdomain/imx/imx8mp-blk-ctrl.c index 8fc79f9723f07e..3f5b9499d30a0c 100644 --- a/drivers/pmdomain/imx/imx8mp-blk-ctrl.c +++ b/drivers/pmdomain/imx/imx8mp-blk-ctrl.c @@ -352,9 +352,6 @@ static void imx8mp_hdmi_blk_ctrl_power_on(struct imx8mp_blk_ctrl *bc, regmap_set_bits(bc->regmap, HDMI_RTX_RESET_CTL0, BIT(12)); regmap_clear_bits(bc->regmap, HDMI_TX_CONTROL0, BIT(3)); break; - case IMX8MP_HDMIBLK_PD_HDCP: - regmap_set_bits(bc->regmap, HDMI_RTX_CLK_CTL0, BIT(11)); - break; case IMX8MP_HDMIBLK_PD_HRV: regmap_set_bits(bc->regmap, HDMI_RTX_CLK_CTL1, BIT(3) | BIT(4) | BIT(5)); regmap_set_bits(bc->regmap, HDMI_RTX_RESET_CTL0, BIT(15)); @@ -408,9 +405,6 @@ static void imx8mp_hdmi_blk_ctrl_power_off(struct imx8mp_blk_ctrl *bc, regmap_clear_bits(bc->regmap, HDMI_RTX_CLK_CTL0, BIT(7)); regmap_clear_bits(bc->regmap, HDMI_RTX_CLK_CTL1, BIT(22) | BIT(24)); break; - case IMX8MP_HDMIBLK_PD_HDCP: - regmap_clear_bits(bc->regmap, HDMI_RTX_CLK_CTL0, BIT(11)); - break; case IMX8MP_HDMIBLK_PD_HRV: regmap_clear_bits(bc->regmap, HDMI_RTX_RESET_CTL0, BIT(15)); regmap_clear_bits(bc->regmap, HDMI_RTX_CLK_CTL1, BIT(3) | BIT(4) | BIT(5)); @@ -439,7 +433,7 @@ static int imx8mp_hdmi_power_notifier(struct notifier_block *nb, regmap_write(bc->regmap, HDMI_RTX_CLK_CTL0, 0x0); regmap_write(bc->regmap, HDMI_RTX_CLK_CTL1, 0x0); regmap_set_bits(bc->regmap, HDMI_RTX_CLK_CTL0, - BIT(0) | BIT(1) | BIT(10)); + BIT(0) | BIT(1) | BIT(10) | BIT(11)); regmap_set_bits(bc->regmap, HDMI_RTX_RESET_CTL0, BIT(0)); /* diff --git a/drivers/pmdomain/mediatek/mtk-pm-domains.c b/drivers/pmdomain/mediatek/mtk-pm-domains.c index f64f24d520ddd7..e2800aa1bc597f 100644 --- a/drivers/pmdomain/mediatek/mtk-pm-domains.c +++ b/drivers/pmdomain/mediatek/mtk-pm-domains.c @@ -1203,7 +1203,7 @@ static int scpsys_probe(struct platform_device *pdev) scpsys->soc_data = soc; scpsys->pd_data.domains = scpsys->domains; - scpsys->pd_data.num_domains = soc->num_domains; + scpsys->pd_data.num_domains = num_domains; parent = dev->parent; if (!parent) { diff --git a/drivers/pmdomain/rockchip/pm-domains.c b/drivers/pmdomain/rockchip/pm-domains.c index 997e93c12951cf..44d34840ede7a8 100644 --- a/drivers/pmdomain/rockchip/pm-domains.c +++ b/drivers/pmdomain/rockchip/pm-domains.c @@ -1311,7 +1311,7 @@ static const struct rockchip_domain_info rk3576_pm_domains[] = { static const struct rockchip_domain_info rk3588_pm_domains[] = { [RK3588_PD_GPU] = DOMAIN_RK3588("gpu", 0x0, BIT(0), 0, 0x0, 0, BIT(1), 0x0, BIT(0), BIT(0), false, true), [RK3588_PD_NPU] = DOMAIN_RK3588("npu", 0x0, BIT(1), BIT(1), 0x0, 0, 0, 0x0, 0, 0, false, true), - [RK3588_PD_VCODEC] = DOMAIN_RK3588("vcodec", 0x0, BIT(2), BIT(2), 0x0, 0, 0, 0x0, 0, 0, false, false), + [RK3588_PD_VCODEC] = DOMAIN_RK3588("vcodec", 0x0, BIT(2), BIT(2), 0x0, 0, 0, 0x0, 0, 0, false, true), [RK3588_PD_NPUTOP] = DOMAIN_RK3588("nputop", 0x0, BIT(3), 0, 0x0, BIT(11), BIT(2), 0x0, BIT(1), BIT(1), false, false), [RK3588_PD_NPU1] = DOMAIN_RK3588("npu1", 0x0, BIT(4), 0, 0x0, BIT(12), BIT(3), 0x0, BIT(2), BIT(2), false, false), [RK3588_PD_NPU2] = DOMAIN_RK3588("npu2", 0x0, BIT(5), 0, 0x0, BIT(13), BIT(4), 0x0, BIT(3), BIT(3), false, false), diff --git a/drivers/power/reset/macsmc-reboot.c b/drivers/power/reset/macsmc-reboot.c index e9702acdd366b0..94fcbf12fe3b93 100644 --- a/drivers/power/reset/macsmc-reboot.c +++ b/drivers/power/reset/macsmc-reboot.c @@ -205,6 +205,14 @@ static int macsmc_reboot_probe(struct platform_device *pdev) struct macsmc_reboot *reboot; int ret, i; + /* + * MFD will probe this device even without a node in the device tree, + * thus bail out early if the SMC on the current machines does not + * support reboot and has no node in the device tree. + */ + if (!pdev->dev.of_node) + return -ENODEV; + reboot = devm_kzalloc(&pdev->dev, sizeof(*reboot), GFP_KERNEL); if (!reboot) return -ENOMEM; diff --git a/drivers/power/reset/nvmem-reboot-mode.c b/drivers/power/reset/nvmem-reboot-mode.c index 41530b70cfc48c..d260715fccf67f 100644 --- a/drivers/power/reset/nvmem-reboot-mode.c +++ b/drivers/power/reset/nvmem-reboot-mode.c @@ -10,6 +10,7 @@ #include #include #include +#include struct nvmem_reboot_mode { struct reboot_mode_driver reboot; @@ -19,12 +20,22 @@ struct nvmem_reboot_mode { static int nvmem_reboot_mode_write(struct reboot_mode_driver *reboot, unsigned int magic) { - int ret; struct nvmem_reboot_mode *nvmem_rbm; + size_t buf_len; + void *buf; + int ret; nvmem_rbm = container_of(reboot, struct nvmem_reboot_mode, reboot); - ret = nvmem_cell_write(nvmem_rbm->cell, &magic, sizeof(magic)); + buf = nvmem_cell_read(nvmem_rbm->cell, &buf_len); + if (IS_ERR(buf)) + return PTR_ERR(buf); + kfree(buf); + + if (buf_len > sizeof(magic)) + return -EINVAL; + + ret = nvmem_cell_write(nvmem_rbm->cell, &magic, buf_len); if (ret < 0) dev_err(reboot->dev, "update reboot mode bits failed\n"); diff --git a/drivers/power/reset/tdx-ec-poweroff.c b/drivers/power/reset/tdx-ec-poweroff.c index 3302a127fce522..8040aa03d74d4f 100644 --- a/drivers/power/reset/tdx-ec-poweroff.c +++ b/drivers/power/reset/tdx-ec-poweroff.c @@ -8,7 +8,10 @@ */ #include +#include +#include #include +#include #include #include #include @@ -31,6 +34,8 @@ #define EC_REG_MAX 0xD0 +#define EC_CMD_TIMEOUT_MS 1000 + static const struct regmap_range volatile_ranges[] = { regmap_reg_range(EC_CMD_REG, EC_CMD_REG), }; @@ -75,6 +80,13 @@ static int tdx_ec_power_off(struct sys_off_data *data) err = tdx_ec_cmd(regmap, EC_CMD_POWEROFF); + if (err) { + dev_err(data->dev, "Failed to send power off command\n"); + } else { + mdelay(EC_CMD_TIMEOUT_MS); + WARN_ONCE(1, "Unable to power off system\n"); + } + return err ? NOTIFY_BAD : NOTIFY_DONE; } @@ -85,6 +97,13 @@ static int tdx_ec_restart(struct sys_off_data *data) err = tdx_ec_cmd(regmap, EC_CMD_RESET); + if (err) { + dev_err(data->dev, "Failed to send restart command\n"); + } else { + mdelay(EC_CMD_TIMEOUT_MS); + WARN_ONCE(1, "Unable to restart system\n"); + } + return err ? NOTIFY_BAD : NOTIFY_DONE; } diff --git a/drivers/power/sequencing/core.c b/drivers/power/sequencing/core.c index 190564e5598855..1fcf0af7cc0bbc 100644 --- a/drivers/power/sequencing/core.c +++ b/drivers/power/sequencing/core.c @@ -914,8 +914,10 @@ int pwrseq_power_on(struct pwrseq_desc *desc) if (target->post_enable) { ret = target->post_enable(pwrseq); if (ret) { - pwrseq_unit_disable(pwrseq, unit); - desc->powered_on = false; + scoped_guard(mutex, &pwrseq->state_lock) { + pwrseq_unit_disable(pwrseq, unit); + desc->powered_on = false; + } } } diff --git a/drivers/power/supply/Kconfig b/drivers/power/supply/Kconfig index 92f9f7aae92f24..e1018b864d8cd2 100644 --- a/drivers/power/supply/Kconfig +++ b/drivers/power/supply/Kconfig @@ -1132,4 +1132,11 @@ config FUEL_GAUGE_MM8013 the state of charge, temperature, cycle count, actual and design capacity, etc. +config CHARGER_MACSMC + tristate "Apple SMC Charger / Battery support" + depends on MFD_MACSMC + help + Say Y here to enable support for the charger and battery controls on + Apple SMC controllers, as used on Apple Silicon Macs. + endif # POWER_SUPPLY diff --git a/drivers/power/supply/Makefile b/drivers/power/supply/Makefile index 4b79d5abc49a7f..7c09106ab23004 100644 --- a/drivers/power/supply/Makefile +++ b/drivers/power/supply/Makefile @@ -81,6 +81,7 @@ obj-$(CONFIG_CHARGER_GPIO) += gpio-charger.o obj-$(CONFIG_CHARGER_MANAGER) += charger-manager.o obj-$(CONFIG_CHARGER_LT3651) += lt3651-charger.o obj-$(CONFIG_CHARGER_LTC4162L) += ltc4162-l-charger.o +obj-$(CONFIG_CHARGER_MACSMC) += macsmc-power.o obj-$(CONFIG_CHARGER_MAX14577) += max14577_charger.o obj-$(CONFIG_CHARGER_DETECTOR_MAX14656) += max14656_charger_detector.o obj-$(CONFIG_CHARGER_MAX77650) += max77650-charger.o diff --git a/drivers/power/supply/ab8500_charger.c b/drivers/power/supply/ab8500_charger.c index 5f4537766e5b90..1813fbdfa1c1f6 100644 --- a/drivers/power/supply/ab8500_charger.c +++ b/drivers/power/supply/ab8500_charger.c @@ -3466,26 +3466,6 @@ static int ab8500_charger_probe(struct platform_device *pdev) return ret; } - /* Request interrupts */ - for (i = 0; i < ARRAY_SIZE(ab8500_charger_irq); i++) { - irq = platform_get_irq_byname(pdev, ab8500_charger_irq[i].name); - if (irq < 0) - return irq; - - ret = devm_request_threaded_irq(dev, - irq, NULL, ab8500_charger_irq[i].isr, - IRQF_SHARED | IRQF_NO_SUSPEND | IRQF_ONESHOT, - ab8500_charger_irq[i].name, di); - - if (ret != 0) { - dev_err(dev, "failed to request %s IRQ %d: %d\n" - , ab8500_charger_irq[i].name, irq, ret); - return ret; - } - dev_dbg(dev, "Requested %s IRQ %d: %d\n", - ab8500_charger_irq[i].name, irq, ret); - } - /* initialize lock */ spin_lock_init(&di->usb_state.usb_lock); mutex_init(&di->usb_ipt_crnt_lock); @@ -3614,6 +3594,26 @@ static int ab8500_charger_probe(struct platform_device *pdev) return PTR_ERR(di->usb_chg.psy); } + /* Request interrupts */ + for (i = 0; i < ARRAY_SIZE(ab8500_charger_irq); i++) { + irq = platform_get_irq_byname(pdev, ab8500_charger_irq[i].name); + if (irq < 0) + return irq; + + ret = devm_request_threaded_irq(dev, + irq, NULL, ab8500_charger_irq[i].isr, + IRQF_SHARED | IRQF_NO_SUSPEND | IRQF_ONESHOT, + ab8500_charger_irq[i].name, di); + + if (ret != 0) { + dev_err(dev, "failed to request %s IRQ %d: %d\n" + , ab8500_charger_irq[i].name, irq, ret); + return ret; + } + dev_dbg(dev, "Requested %s IRQ %d: %d\n", + ab8500_charger_irq[i].name, irq, ret); + } + /* * Check what battery we have, since we always have the USB * psy, use that as a handle. diff --git a/drivers/power/supply/act8945a_charger.c b/drivers/power/supply/act8945a_charger.c index 3901a02f326a55..9dec4486b1439a 100644 --- a/drivers/power/supply/act8945a_charger.c +++ b/drivers/power/supply/act8945a_charger.c @@ -597,14 +597,6 @@ static int act8945a_charger_probe(struct platform_device *pdev) return irq ?: -ENXIO; } - ret = devm_request_irq(&pdev->dev, irq, act8945a_status_changed, - IRQF_TRIGGER_FALLING, "act8945a_interrupt", - charger); - if (ret) { - dev_err(&pdev->dev, "failed to request nIRQ pin IRQ\n"); - return ret; - } - charger->desc.name = "act8945a-charger"; charger->desc.get_property = act8945a_charger_get_property; charger->desc.properties = act8945a_charger_props; @@ -625,6 +617,14 @@ static int act8945a_charger_probe(struct platform_device *pdev) return PTR_ERR(charger->psy); } + ret = devm_request_irq(&pdev->dev, irq, act8945a_status_changed, + IRQF_TRIGGER_FALLING, "act8945a_interrupt", + charger); + if (ret) { + dev_err(&pdev->dev, "failed to request nIRQ pin IRQ\n"); + return ret; + } + platform_set_drvdata(pdev, charger); INIT_WORK(&charger->work, act8945a_work); diff --git a/drivers/power/supply/bq256xx_charger.c b/drivers/power/supply/bq256xx_charger.c index ae14162f017a9e..d3de4f8b80db1e 100644 --- a/drivers/power/supply/bq256xx_charger.c +++ b/drivers/power/supply/bq256xx_charger.c @@ -1741,6 +1741,12 @@ static int bq256xx_probe(struct i2c_client *client) usb_register_notifier(bq->usb3_phy, &bq->usb_nb); } + ret = bq256xx_power_supply_init(bq, &psy_cfg, dev); + if (ret) { + dev_err(dev, "Failed to register power supply\n"); + return ret; + } + if (client->irq) { ret = devm_request_threaded_irq(dev, client->irq, NULL, bq256xx_irq_handler_thread, @@ -1753,12 +1759,6 @@ static int bq256xx_probe(struct i2c_client *client) } } - ret = bq256xx_power_supply_init(bq, &psy_cfg, dev); - if (ret) { - dev_err(dev, "Failed to register power supply\n"); - return ret; - } - ret = bq256xx_hw_init(bq); if (ret) { dev_err(dev, "Cannot initialize the chip.\n"); diff --git a/drivers/power/supply/bq25980_charger.c b/drivers/power/supply/bq25980_charger.c index 723858d62d1414..73f06f09f134cd 100644 --- a/drivers/power/supply/bq25980_charger.c +++ b/drivers/power/supply/bq25980_charger.c @@ -1241,6 +1241,12 @@ static int bq25980_probe(struct i2c_client *client) return ret; } + ret = bq25980_power_supply_init(bq, dev); + if (ret) { + dev_err(dev, "Failed to register power supply\n"); + return ret; + } + if (client->irq) { ret = devm_request_threaded_irq(dev, client->irq, NULL, bq25980_irq_handler_thread, @@ -1251,12 +1257,6 @@ static int bq25980_probe(struct i2c_client *client) return ret; } - ret = bq25980_power_supply_init(bq, dev); - if (ret) { - dev_err(dev, "Failed to register power supply\n"); - return ret; - } - ret = bq25980_hw_init(bq); if (ret) { dev_err(dev, "Cannot initialize the chip.\n"); diff --git a/drivers/power/supply/bq27xxx_battery.c b/drivers/power/supply/bq27xxx_battery.c index 19445e39651c71..45f0e39b8c2dd4 100644 --- a/drivers/power/supply/bq27xxx_battery.c +++ b/drivers/power/supply/bq27xxx_battery.c @@ -1172,7 +1172,7 @@ static inline int bq27xxx_write(struct bq27xxx_device_info *di, int reg_index, return -EINVAL; if (!di->bus.write) - return -EPERM; + return -EOPNOTSUPP; ret = di->bus.write(di, di->regs[reg_index], value, single); if (ret < 0) @@ -1191,7 +1191,7 @@ static inline int bq27xxx_read_block(struct bq27xxx_device_info *di, int reg_ind return -EINVAL; if (!di->bus.read_bulk) - return -EPERM; + return -EOPNOTSUPP; ret = di->bus.read_bulk(di, di->regs[reg_index], data, len); if (ret < 0) @@ -1210,7 +1210,7 @@ static inline int bq27xxx_write_block(struct bq27xxx_device_info *di, int reg_in return -EINVAL; if (!di->bus.write_bulk) - return -EPERM; + return -EOPNOTSUPP; ret = di->bus.write_bulk(di, di->regs[reg_index], data, len); if (ret < 0) diff --git a/drivers/power/supply/cpcap-battery.c b/drivers/power/supply/cpcap-battery.c index 8106d1edcbc26a..507fdc1c866d56 100644 --- a/drivers/power/supply/cpcap-battery.c +++ b/drivers/power/supply/cpcap-battery.c @@ -1122,10 +1122,6 @@ static int cpcap_battery_probe(struct platform_device *pdev) platform_set_drvdata(pdev, ddata); - error = cpcap_battery_init_interrupts(pdev, ddata); - if (error) - return error; - error = cpcap_battery_init_iio(ddata); if (error) return error; @@ -1142,6 +1138,10 @@ static int cpcap_battery_probe(struct platform_device *pdev) return error; } + error = cpcap_battery_init_interrupts(pdev, ddata); + if (error) + return error; + atomic_set(&ddata->active, 1); error = cpcap_battery_calibrate(ddata); diff --git a/drivers/power/supply/goldfish_battery.c b/drivers/power/supply/goldfish_battery.c index 479195e35d734a..5aa24e4dc4455d 100644 --- a/drivers/power/supply/goldfish_battery.c +++ b/drivers/power/supply/goldfish_battery.c @@ -224,12 +224,6 @@ static int goldfish_battery_probe(struct platform_device *pdev) if (data->irq < 0) return -ENODEV; - ret = devm_request_irq(&pdev->dev, data->irq, - goldfish_battery_interrupt, - IRQF_SHARED, pdev->name, data); - if (ret) - return ret; - psy_cfg.drv_data = data; data->ac = devm_power_supply_register(&pdev->dev, @@ -244,6 +238,12 @@ static int goldfish_battery_probe(struct platform_device *pdev) if (IS_ERR(data->battery)) return PTR_ERR(data->battery); + ret = devm_request_irq(&pdev->dev, data->irq, + goldfish_battery_interrupt, + IRQF_SHARED, pdev->name, data); + if (ret) + return ret; + GOLDFISH_BATTERY_WRITE(data, BATTERY_INT_ENABLE, BATTERY_INT_MASK); return 0; } diff --git a/drivers/power/supply/macsmc-power.c b/drivers/power/supply/macsmc-power.c new file mode 100644 index 00000000000000..dc3ec5ef2b81cf --- /dev/null +++ b/drivers/power/supply/macsmc-power.c @@ -0,0 +1,1053 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Apple SMC Power/Battery Management + * Copyright The Asahi Linux Contributors + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define MAX_STRING_LENGTH 256 + +/* + * This number is not reported anywhere by SMC, but seems to be a good + * conversion factor for charge to energy across machines. We need this + * to convert in the driver, since if we don't userspace will try to do + * the conversion with a randomly guessed voltage and get it wrong. + * + * Ideally there would be a power supply prop to inform userspace of this + * number, but there isn't, only min/max. + */ +#define MACSMC_NOMINAL_CELL_VOLTAGE_MV 3800 + +struct macsmc_power { + struct device *dev; + struct apple_smc *smc; + struct power_supply_desc ac_desc; + struct power_supply_desc batt_desc; + + struct power_supply *batt; + char model_name[MAX_STRING_LENGTH]; + char serial_number[MAX_STRING_LENGTH]; + char mfg_date[MAX_STRING_LENGTH]; + + bool has_chwa; + bool has_chls; + bool has_ch0i; + bool has_ch0c; + bool has_chte; + + u8 num_cells; + int nominal_voltage_mv; + + struct power_supply *ac; + + struct notifier_block nb; + + struct work_struct critical_work; + bool shutdown_started; + + struct delayed_work dbg_log_work; +}; + +static int macsmc_log_power_set(const char *val, const struct kernel_param *kp); + +static const struct kernel_param_ops macsmc_log_power_ops = { + .set = macsmc_log_power_set, + .get = param_get_bool, +}; + +static bool log_power = false; +module_param_cb(log_power, &macsmc_log_power_ops, &log_power, 0644); +MODULE_PARM_DESC(log_power, "Periodically log power consumption for debugging"); + +#define POWER_LOG_INTERVAL (HZ) + +static struct macsmc_power *g_power; + +#define CHNC_BATTERY_FULL BIT(0) +#define CHNC_NO_CHARGER BIT(7) +#define CHNC_NOCHG_CH0C BIT(14) +#define CHNC_NOCHG_CH0B_CH0K BIT(15) +#define CHNC_BATTERY_FULL_2 BIT(18) +#define CHNC_BMS_BUSY BIT(23) +#define CHNC_CHLS_LIMIT BIT(24) +#define CHNC_NOAC_CH0J BIT(53) +#define CHNC_NOAC_CH0I BIT(54) + +#define CH0R_LOWER_FLAGS GENMASK(15, 0) +#define CH0R_NOAC_CH0I BIT(0) +#define CH0R_NOAC_DISCONNECTED BIT(4) +#define CH0R_NOAC_CH0J BIT(5) +#define CH0R_BMS_BUSY BIT(8) +#define CH0R_NOAC_CH0K BIT(9) +#define CH0R_NOAC_CHWA BIT(11) + +#define CH0X_CH0C BIT(0) +#define CH0X_CH0B BIT(1) + +#define ACSt_CAN_BOOT_AP BIT(2) +#define ACSt_CAN_BOOT_IBOOT BIT(1) + +#define CHWA_CHLS_FIXED_START_OFFSET 5 +#define CHLS_MIN_END_THRESHOLD 10 +#define CHLS_FORCE_DISCHARGE 0x100 +#define CHWA_FIXED_END_THRESHOLD 80 +#define CHWA_PROP_WRITE_THRESHOLD 95 + +#define FLT_EXP_BIAS 127 +#define FLT_EXP_MASK GENMASK(30, 23) +#define FLT_MANT_BIAS 23 +#define FLT_MANT_MASK GENMASK(22, 0) +#define FLT_SIGN_MASK BIT(31) +/* + * Many sensors report their data as IEEE-754 floats. No other SMC function uses + * them. + */ +static int apple_smc_read_f32_scaled(struct apple_smc *smc, smc_key key, + int *p, int scale) +{ + u32 fval; + u64 val; + int ret, exp; + + BUILD_BUG_ON(scale <= 0); + + ret = apple_smc_read_u32(smc, key, &fval); + if (ret < 0) + return ret; + + val = ((u64)((fval & FLT_MANT_MASK) | BIT(23))); + exp = ((fval >> 23) & 0xff) - FLT_EXP_BIAS - FLT_MANT_BIAS; + val *= scale; + + if (exp > 63) + val = U64_MAX; + else if (exp < -63) + val = 0; + else if (exp < 0) + val >>= -exp; + else if (exp != 0 && (val & ~((1UL << (64 - exp)) - 1))) /* overflow */ + val = U64_MAX; + else + val <<= exp; + + if (fval & FLT_SIGN_MASK) { + if (val > (-(s64)INT_MIN)) + *p = INT_MIN; + else + *p = -val; + } else { + if (val > INT_MAX) + *p = INT_MAX; + else + *p = val; + } + + return 0; +} + +static void macsmc_do_dbg(struct macsmc_power *power) +{ + int p_in = 0, p_sys = 0, p_3v8 = 0, p_mpmu = 0, p_spmu = 0, p_clvr = 0, p_cpu = 0; + s32 p_bat = 0; + s16 t_full = 0, t_empty = 0; + u8 charge = 0; + + apple_smc_read_f32_scaled(power->smc, SMC_KEY(PDTR), &p_in, 1000); + apple_smc_read_f32_scaled(power->smc, SMC_KEY(PSTR), &p_sys, 1000); + apple_smc_read_f32_scaled(power->smc, SMC_KEY(PMVR), &p_3v8, 1000); + apple_smc_read_f32_scaled(power->smc, SMC_KEY(PHPC), &p_cpu, 1000); + apple_smc_read_f32_scaled(power->smc, SMC_KEY(PSVR), &p_clvr, 1000); + apple_smc_read_f32_scaled(power->smc, SMC_KEY(PPMC), &p_mpmu, 1000); + apple_smc_read_f32_scaled(power->smc, SMC_KEY(PPSC), &p_spmu, 1000); + apple_smc_read_s32(power->smc, SMC_KEY(B0AP), &p_bat); + apple_smc_read_s16(power->smc, SMC_KEY(B0TE), &t_empty); + apple_smc_read_s16(power->smc, SMC_KEY(B0TF), &t_full); + apple_smc_read_u8(power->smc, SMC_KEY(BUIC), &charge); + +#define FD3(x) ((x) / 1000), abs((x) % 1000) + dev_info(power->dev, + "In %2d.%03dW Sys %2d.%03dW 3V8 %2d.%03dW MPMU %2d.%03dW SPMU %2d.%03dW " + "CLVR %2d.%03dW CPU %2d.%03dW Batt %2d.%03dW %d%% T%s %dm\n", + FD3(p_in), FD3(p_sys), FD3(p_3v8), FD3(p_mpmu), FD3(p_spmu), FD3(p_clvr), + FD3(p_cpu), FD3(p_bat), charge, + t_full >= 0 ? "full" : "empty", + t_full >= 0 ? t_full : t_empty); +#undef FD3 +} + +static int macsmc_battery_get_status(struct macsmc_power *power) +{ + u64 nocharge_flags; + u32 nopower_flags; + u16 ac_current; + int charge_limit = 0; + bool limited = false; + bool flag; + int ret; + + /* + * Note: there are fallbacks in case some of these SMC keys disappear in the future + * or are not present on some machines. We treat the absence of the CHCE/CHCC/BSFC/CHSC + * flags as an error, since they are quite fundamental and simple booleans. + */ + + /* + * If power input is inhibited, we are definitely discharging. + * However, if the only reason is the BMS is doing a balancing cycle, + * go ahead and ignore that one to avoid spooking users. + */ + ret = apple_smc_read_u32(power->smc, SMC_KEY(CH0R), &nopower_flags); + if (!ret && (nopower_flags & CH0R_LOWER_FLAGS & ~CH0R_BMS_BUSY)) + return POWER_SUPPLY_STATUS_DISCHARGING; + + /* If no charger is present, we are definitely discharging. */ + ret = apple_smc_read_flag(power->smc, SMC_KEY(CHCE), &flag); + if (ret < 0) + return ret; + if (!flag) + return POWER_SUPPLY_STATUS_DISCHARGING; + + /* If AC is not charge capable, we are definitely discharging. */ + ret = apple_smc_read_flag(power->smc, SMC_KEY(CHCC), &flag); + if (ret < 0) + return ret; + if (!flag) + return POWER_SUPPLY_STATUS_DISCHARGING; + + /* + * If the AC input current limit is tiny or 0, we are discharging no matter + * how much the BMS believes it can charge. + */ + ret = apple_smc_read_u16(power->smc, SMC_KEY(AC-i), &ac_current); + if (!ret && ac_current < 100) + return POWER_SUPPLY_STATUS_DISCHARGING; + + /* If the battery is full, report it as such. */ + ret = apple_smc_read_flag(power->smc, SMC_KEY(BSFC), &flag); + if (ret < 0) + return ret; + if (flag) + return POWER_SUPPLY_STATUS_FULL; + + /* + * If we have charge limits supported and enabled and the SoC is above + * the start threshold, that means we are not charging for that reason + * (if not charging). + */ + if (power->has_chls) { + u16 vu16; + + ret = apple_smc_read_u16(power->smc, SMC_KEY(CHLS), &vu16); + if (ret == sizeof(vu16) && (vu16 & 0xff) >= CHLS_MIN_END_THRESHOLD) + charge_limit = (vu16 & 0xff) - CHWA_CHLS_FIXED_START_OFFSET; + } else if (power->has_chwa) { + ret = apple_smc_read_flag(power->smc, SMC_KEY(CHWA), &flag); + if (ret == 0 && flag) + charge_limit = CHWA_FIXED_END_THRESHOLD - CHWA_CHLS_FIXED_START_OFFSET; + } + + if (charge_limit > 0) { + u8 buic = 0; + + if (apple_smc_read_u8(power->smc, SMC_KEY(BUIC), &buic) >= 0 && + buic >= charge_limit) + limited = true; + } + + /* If there are reasons we aren't charging... */ + ret = apple_smc_read_u64(power->smc, SMC_KEY(CHNC), &nocharge_flags); + if (!ret) { + /* Perhaps the battery is full after all */ + if (nocharge_flags & CHNC_BATTERY_FULL) + return POWER_SUPPLY_STATUS_FULL; + /* + * Or maybe the BMS is just busy doing something, if so call it charging anyway. + * But CHWA limits show up as this, so exclude those. + */ + else if (nocharge_flags == CHNC_BMS_BUSY && !limited) + return POWER_SUPPLY_STATUS_CHARGING; + /* If we have other reasons we aren't charging, say we aren't */ + else if (nocharge_flags) + return POWER_SUPPLY_STATUS_NOT_CHARGING; + /* Else we're either charging or about to charge */ + else + return POWER_SUPPLY_STATUS_CHARGING; + } + + /* As a fallback, use the system charging flag. */ + ret = apple_smc_read_flag(power->smc, SMC_KEY(CHSC), &flag); + if (ret < 0) + return ret; + if (!flag) + return POWER_SUPPLY_STATUS_NOT_CHARGING; + else + return POWER_SUPPLY_STATUS_CHARGING; +} + +static int macsmc_battery_get_charge_behaviour(struct macsmc_power *power) +{ + int ret; + u8 val8; + u8 chte_buf[4]; + + if (power->has_ch0i) { + /* CH0I returns a bitmask like the low byte of CH0R */ + ret = apple_smc_read_u8(power->smc, SMC_KEY(CH0I), &val8); + if (ret) + return ret; + if (val8 & CH0R_NOAC_CH0I) + return POWER_SUPPLY_CHARGE_BEHAVIOUR_FORCE_DISCHARGE; + } + + /* Prefer CHTE available in newer firmwares */ + if (power->has_chte) { + ret = apple_smc_read(power->smc, SMC_KEY(CHTE), chte_buf, 4); + if (ret < 0) + return ret; + + if (chte_buf[0] == 0x01) + return POWER_SUPPLY_CHARGE_BEHAVIOUR_INHIBIT_CHARGE; + + } else if (power->has_ch0c) { + /* CH0C returns a bitmask containing CH0B/CH0C flags */ + ret = apple_smc_read_u8(power->smc, SMC_KEY(CH0C), &val8); + if (ret) + return ret; + if (val8 & CH0X_CH0C) + return POWER_SUPPLY_CHARGE_BEHAVIOUR_INHIBIT_CHARGE; + } + + return POWER_SUPPLY_CHARGE_BEHAVIOUR_AUTO; +} + +static int macsmc_battery_set_charge_behaviour(struct macsmc_power *power, int val) +{ + int ret; + + /* + * apple_smc_write_u32 does weird things with endianess, + * so we write raw bytes to ensure correctness of CHTE + */ + u8 chte_inhibit[4] = {0x01, 0x00, 0x00, 0x00}; + u8 chte_auto[4] = {0x00, 0x00, 0x00, 0x00}; + + /* + * CH0I/CH0C/CHTE are "hard" controls that will allow the battery to run down to 0. + * CH0K/CH0B are "soft" controls that are reset to 0 when SOC drops below 50%; + * we don't expose these yet. + */ + + switch (val) { + case POWER_SUPPLY_CHARGE_BEHAVIOUR_AUTO: + if (power->has_ch0i) { + ret = apple_smc_write_u8(power->smc, SMC_KEY(CH0I), 0); + if (ret) + return ret; + } + + if (power->has_chte) { + ret = apple_smc_write(power->smc, SMC_KEY(CHTE), chte_auto, 4); + if (ret) + return ret; + } else if (power->has_ch0c) { + ret = apple_smc_write_u8(power->smc, SMC_KEY(CH0C), 0); + if (ret) + return ret; + } + break; + + case POWER_SUPPLY_CHARGE_BEHAVIOUR_INHIBIT_CHARGE: + if (power->has_ch0i) { + ret = apple_smc_write_u8(power->smc, SMC_KEY(CH0I), 0); + if (ret) + return ret; + } + + /* Prefer CHTE available in newer firmwares */ + if (power->has_chte) + return apple_smc_write(power->smc, SMC_KEY(CHTE), chte_inhibit, 4); + else if (power->has_ch0c) + return apple_smc_write_u8(power->smc, SMC_KEY(CH0C), 1); + else + return -EINVAL; + break; + + case POWER_SUPPLY_CHARGE_BEHAVIOUR_FORCE_DISCHARGE: + if (!power->has_ch0i) + return -EINVAL; + + /* Prefer CHTE available in newer firmwares */ + if (power->has_chte) { + ret = apple_smc_write(power->smc, SMC_KEY(CHTE), chte_auto, 4); + if (ret) + return ret; + } else if (power->has_ch0c) { + ret = apple_smc_write_u8(power->smc, SMC_KEY(CH0C), 0); + if (ret) + return ret; + } + + return apple_smc_write_u8(power->smc, SMC_KEY(CH0I), 1); + + default: + return -EINVAL; + } + + return 0; +} + +static int macsmc_battery_get_date(const char *s, int *out) +{ + if (!isdigit(s[0]) || !isdigit(s[1])) + return -ENOTSUPP; + + *out = (s[0] - '0') * 10 + s[1] - '0'; + return 0; +} + +static int macsmc_battery_get_capacity_level(struct macsmc_power *power) +{ + bool flag; + u32 val; + int ret; + + /* Check for emergency shutdown condition */ + if (apple_smc_read_u32(power->smc, SMC_KEY(BCF0), &val) >= 0 && val) + return POWER_SUPPLY_CAPACITY_LEVEL_CRITICAL; + + /* Check AC status for whether we could boot in this state */ + if (apple_smc_read_u32(power->smc, SMC_KEY(ACSt), &val) >= 0) { + if (!(val & ACSt_CAN_BOOT_IBOOT)) + return POWER_SUPPLY_CAPACITY_LEVEL_CRITICAL; + + if (!(val & ACSt_CAN_BOOT_AP)) + return POWER_SUPPLY_CAPACITY_LEVEL_LOW; + } + + /* Check battery full flag */ + ret = apple_smc_read_flag(power->smc, SMC_KEY(BSFC), &flag); + if (ret < 0) + return POWER_SUPPLY_CAPACITY_LEVEL_UNKNOWN; + if (flag) + return POWER_SUPPLY_CAPACITY_LEVEL_FULL; + else + return POWER_SUPPLY_CAPACITY_LEVEL_NORMAL; +} + +static int macsmc_battery_get_property(struct power_supply *psy, + enum power_supply_property psp, + union power_supply_propval *val) +{ + struct macsmc_power *power = power_supply_get_drvdata(psy); + int ret = 0; + u8 vu8; + u16 vu16; + s16 vs16; + s32 vs32; + s64 vs64; + bool flag; + + switch (psp) { + case POWER_SUPPLY_PROP_STATUS: + val->intval = macsmc_battery_get_status(power); + ret = val->intval < 0 ? val->intval : 0; + break; + case POWER_SUPPLY_PROP_PRESENT: + val->intval = 1; + break; + case POWER_SUPPLY_PROP_CHARGE_BEHAVIOUR: + val->intval = macsmc_battery_get_charge_behaviour(power); + ret = val->intval < 0 ? val->intval : 0; + break; + case POWER_SUPPLY_PROP_TIME_TO_EMPTY_NOW: + ret = apple_smc_read_u16(power->smc, SMC_KEY(B0TE), &vu16); + val->intval = vu16 == 0xffff ? 0 : vu16 * 60; + break; + case POWER_SUPPLY_PROP_TIME_TO_FULL_NOW: + ret = apple_smc_read_u16(power->smc, SMC_KEY(B0TF), &vu16); + val->intval = vu16 == 0xffff ? 0 : vu16 * 60; + break; + case POWER_SUPPLY_PROP_CAPACITY: + ret = apple_smc_read_u8(power->smc, SMC_KEY(BUIC), &vu8); + val->intval = vu8; + break; + case POWER_SUPPLY_PROP_CAPACITY_LEVEL: + val->intval = macsmc_battery_get_capacity_level(power); + ret = val->intval < 0 ? val->intval : 0; + break; + case POWER_SUPPLY_PROP_VOLTAGE_NOW: + ret = apple_smc_read_u16(power->smc, SMC_KEY(B0AV), &vu16); + val->intval = vu16 * 1000; + break; + case POWER_SUPPLY_PROP_CURRENT_NOW: + ret = apple_smc_read_s16(power->smc, SMC_KEY(B0AC), &vs16); + val->intval = vs16 * 1000; + break; + case POWER_SUPPLY_PROP_POWER_NOW: + ret = apple_smc_read_s32(power->smc, SMC_KEY(B0AP), &vs32); + val->intval = vs32 * 1000; + break; + case POWER_SUPPLY_PROP_VOLTAGE_MIN_DESIGN: + ret = apple_smc_read_u16(power->smc, SMC_KEY(BITV), &vu16); + val->intval = vu16 * 1000; + break; + case POWER_SUPPLY_PROP_VOLTAGE_MAX_DESIGN: + /* + * Battery cell max voltage? BVV* seem to return per-cell voltages, + * BVV[NOP] are probably the max voltages for the 3 cells but we don't + * know what will happen if they ever change the number of cells. + * So go with BVVN and multiply by the cell count (BNCB). + * BVVL seems to be the per-cell limit adjusted dynamically. + * Guess: BVVL = Limit, BVVN = Nominal, and the other cells got filled + * in around nearby letters? + */ + ret = apple_smc_read_u16(power->smc, SMC_KEY(BVVN), &vu16); + val->intval = vu16 * 1000 * power->num_cells; + break; + case POWER_SUPPLY_PROP_VOLTAGE_MIN: + /* Lifetime min */ + ret = apple_smc_read_s16(power->smc, SMC_KEY(BLPM), &vs16); + val->intval = vs16 * 1000; + break; + case POWER_SUPPLY_PROP_VOLTAGE_MAX: + /* Lifetime max */ + ret = apple_smc_read_s16(power->smc, SMC_KEY(BLPX), &vs16); + val->intval = vs16 * 1000; + break; + case POWER_SUPPLY_PROP_CHARGE_TERM_CURRENT: + ret = apple_smc_read_u16(power->smc, SMC_KEY(B0RC), &vu16); + val->intval = vu16 * 1000; + break; + case POWER_SUPPLY_PROP_CONSTANT_CHARGE_CURRENT_MAX: + ret = apple_smc_read_u16(power->smc, SMC_KEY(B0RI), &vu16); + val->intval = vu16 * 1000; + break; + case POWER_SUPPLY_PROP_CONSTANT_CHARGE_VOLTAGE: + ret = apple_smc_read_u16(power->smc, SMC_KEY(B0RV), &vu16); + val->intval = vu16 * 1000; + break; + case POWER_SUPPLY_PROP_CHARGE_FULL_DESIGN: + ret = apple_smc_read_u16(power->smc, SMC_KEY(B0DC), &vu16); + val->intval = vu16 * 1000; + break; + case POWER_SUPPLY_PROP_CHARGE_FULL: + ret = apple_smc_read_u16(power->smc, SMC_KEY(B0FC), &vu16); + val->intval = vu16 * 1000; + break; + case POWER_SUPPLY_PROP_CHARGE_NOW: + ret = apple_smc_read_u16(power->smc, SMC_KEY(B0RM), &vu16); + val->intval = swab16(vu16) * 1000; + break; + case POWER_SUPPLY_PROP_ENERGY_FULL_DESIGN: + ret = apple_smc_read_u16(power->smc, SMC_KEY(B0DC), &vu16); + val->intval = vu16 * power->nominal_voltage_mv; + break; + case POWER_SUPPLY_PROP_ENERGY_FULL: + ret = apple_smc_read_u16(power->smc, SMC_KEY(B0FC), &vu16); + val->intval = vu16 * power->nominal_voltage_mv; + break; + case POWER_SUPPLY_PROP_ENERGY_NOW: + ret = apple_smc_read_u16(power->smc, SMC_KEY(B0RM), &vu16); + val->intval = swab16(vu16) * power->nominal_voltage_mv; + break; + case POWER_SUPPLY_PROP_TEMP: + ret = apple_smc_read_u16(power->smc, SMC_KEY(B0AT), &vu16); + val->intval = vu16 - 2732; + break; + case POWER_SUPPLY_PROP_CHARGE_COUNTER: + ret = apple_smc_read_s64(power->smc, SMC_KEY(BAAC), &vs64); + val->intval = vs64; + break; + case POWER_SUPPLY_PROP_CYCLE_COUNT: + ret = apple_smc_read_u16(power->smc, SMC_KEY(B0CT), &vu16); + val->intval = vu16; + break; + case POWER_SUPPLY_PROP_SCOPE: + val->intval = POWER_SUPPLY_SCOPE_SYSTEM; + break; + case POWER_SUPPLY_PROP_HEALTH: + flag = false; + ret = apple_smc_read_flag(power->smc, SMC_KEY(BBAD), &flag); + val->intval = flag ? POWER_SUPPLY_HEALTH_DEAD : POWER_SUPPLY_HEALTH_GOOD; + break; + case POWER_SUPPLY_PROP_MODEL_NAME: + val->strval = power->model_name; + break; + case POWER_SUPPLY_PROP_SERIAL_NUMBER: + val->strval = power->serial_number; + break; + case POWER_SUPPLY_PROP_MANUFACTURE_YEAR: + ret = macsmc_battery_get_date(&power->mfg_date[0], &val->intval); + val->intval += 2000 - 8; /* -8 is a fixup for a firmware bug... */ + break; + case POWER_SUPPLY_PROP_MANUFACTURE_MONTH: + ret = macsmc_battery_get_date(&power->mfg_date[2], &val->intval); + break; + case POWER_SUPPLY_PROP_MANUFACTURE_DAY: + ret = macsmc_battery_get_date(&power->mfg_date[4], &val->intval); + break; + case POWER_SUPPLY_PROP_CHARGE_CONTROL_START_THRESHOLD: + case POWER_SUPPLY_PROP_CHARGE_CONTROL_END_THRESHOLD: + if (power->has_chls) { + ret = apple_smc_read_u16(power->smc, SMC_KEY(CHLS), &vu16); + val->intval = vu16 & 0xff; + if (val->intval < CHLS_MIN_END_THRESHOLD || val->intval >= 100) + val->intval = 100; + } else if (power->has_chwa) { + flag = false; + ret = apple_smc_read_flag(power->smc, SMC_KEY(CHWA), &flag); + val->intval = flag ? CHWA_FIXED_END_THRESHOLD : 100; + } else { + return -EINVAL; + } + if (psp == POWER_SUPPLY_PROP_CHARGE_CONTROL_START_THRESHOLD && + ret >= 0 && val->intval < 100 && val->intval >= CHLS_MIN_END_THRESHOLD) + val->intval -= CHWA_CHLS_FIXED_START_OFFSET; + break; + default: + return -EINVAL; + } + + return ret; +} + +static int macsmc_battery_set_property(struct power_supply *psy, + enum power_supply_property psp, + const union power_supply_propval *val) +{ + struct macsmc_power *power = power_supply_get_drvdata(psy); + + switch (psp) { + case POWER_SUPPLY_PROP_CHARGE_BEHAVIOUR: + return macsmc_battery_set_charge_behaviour(power, val->intval); + case POWER_SUPPLY_PROP_CHARGE_CONTROL_START_THRESHOLD: + /* + * Ignore, we allow writes so userspace isn't confused but this is + * not configurable independently, it always is end - 5 or 100 depending + * on the end_threshold setting. + */ + return 0; + case POWER_SUPPLY_PROP_CHARGE_CONTROL_END_THRESHOLD: + if (power->has_chls) { + u16 kval = 0; + /* TODO: Make CHLS_FORCE_DISCHARGE configurable */ + if (val->intval < CHLS_MIN_END_THRESHOLD) + kval = CHLS_FORCE_DISCHARGE | CHLS_MIN_END_THRESHOLD; + else if (val->intval < 100) + kval = CHLS_FORCE_DISCHARGE | (val->intval & 0xff); + return apple_smc_write_u16(power->smc, SMC_KEY(CHLS), kval); + } else if (power->has_chwa) { + return apple_smc_write_flag(power->smc, SMC_KEY(CHWA), + val->intval <= CHWA_PROP_WRITE_THRESHOLD); + } else { + return -EINVAL; + } + default: + return -EINVAL; + } +} + +static int macsmc_battery_property_is_writeable(struct power_supply *psy, + enum power_supply_property psp) +{ + struct macsmc_power *power = power_supply_get_drvdata(psy); + + switch (psp) { + case POWER_SUPPLY_PROP_CHARGE_BEHAVIOUR: + return true; + case POWER_SUPPLY_PROP_CHARGE_CONTROL_START_THRESHOLD: + case POWER_SUPPLY_PROP_CHARGE_CONTROL_END_THRESHOLD: + return power->has_chwa || power->has_chls; + default: + return false; + } +} + +static const enum power_supply_property macsmc_battery_props[] = { + POWER_SUPPLY_PROP_STATUS, + POWER_SUPPLY_PROP_PRESENT, + POWER_SUPPLY_PROP_CHARGE_BEHAVIOUR, + POWER_SUPPLY_PROP_TIME_TO_EMPTY_NOW, + POWER_SUPPLY_PROP_TIME_TO_FULL_NOW, + POWER_SUPPLY_PROP_CAPACITY, + POWER_SUPPLY_PROP_CAPACITY_LEVEL, + POWER_SUPPLY_PROP_VOLTAGE_NOW, + POWER_SUPPLY_PROP_CURRENT_NOW, + POWER_SUPPLY_PROP_POWER_NOW, + POWER_SUPPLY_PROP_VOLTAGE_MIN_DESIGN, + POWER_SUPPLY_PROP_VOLTAGE_MAX_DESIGN, + POWER_SUPPLY_PROP_VOLTAGE_MIN, + POWER_SUPPLY_PROP_VOLTAGE_MAX, + POWER_SUPPLY_PROP_CHARGE_TERM_CURRENT, + POWER_SUPPLY_PROP_CONSTANT_CHARGE_CURRENT_MAX, + POWER_SUPPLY_PROP_CONSTANT_CHARGE_VOLTAGE, + POWER_SUPPLY_PROP_CHARGE_FULL_DESIGN, + POWER_SUPPLY_PROP_CHARGE_FULL, + POWER_SUPPLY_PROP_CHARGE_NOW, + POWER_SUPPLY_PROP_ENERGY_FULL_DESIGN, + POWER_SUPPLY_PROP_ENERGY_FULL, + POWER_SUPPLY_PROP_ENERGY_NOW, + POWER_SUPPLY_PROP_TEMP, + POWER_SUPPLY_PROP_CHARGE_COUNTER, + POWER_SUPPLY_PROP_CYCLE_COUNT, + POWER_SUPPLY_PROP_SCOPE, + POWER_SUPPLY_PROP_HEALTH, + POWER_SUPPLY_PROP_MODEL_NAME, + POWER_SUPPLY_PROP_SERIAL_NUMBER, + POWER_SUPPLY_PROP_MANUFACTURE_YEAR, + POWER_SUPPLY_PROP_MANUFACTURE_MONTH, + POWER_SUPPLY_PROP_MANUFACTURE_DAY, + POWER_SUPPLY_PROP_CHARGE_CONTROL_START_THRESHOLD, + POWER_SUPPLY_PROP_CHARGE_CONTROL_END_THRESHOLD +}; + +static const struct power_supply_desc macsmc_battery_desc = { + .name = "macsmc-battery", + .type = POWER_SUPPLY_TYPE_BATTERY, + .get_property = macsmc_battery_get_property, + .set_property = macsmc_battery_set_property, + .property_is_writeable = macsmc_battery_property_is_writeable, + .properties = macsmc_battery_props, + .num_properties = ARRAY_SIZE(macsmc_battery_props), + .charge_behaviours = BIT(POWER_SUPPLY_CHARGE_BEHAVIOUR_AUTO) + | BIT(POWER_SUPPLY_CHARGE_BEHAVIOUR_FORCE_DISCHARGE) + | BIT(POWER_SUPPLY_CHARGE_BEHAVIOUR_INHIBIT_CHARGE), +}; + +static int macsmc_ac_get_property(struct power_supply *psy, + enum power_supply_property psp, + union power_supply_propval *val) +{ + struct macsmc_power *power = power_supply_get_drvdata(psy); + int ret = 0; + u16 vu16; + u32 vu32; + + switch (psp) { + case POWER_SUPPLY_PROP_ONLINE: + ret = apple_smc_read_u32(power->smc, SMC_KEY(CHIS), &vu32); + val->intval = !!vu32; + break; + case POWER_SUPPLY_PROP_VOLTAGE_NOW: + ret = apple_smc_read_u16(power->smc, SMC_KEY(AC-n), &vu16); + val->intval = vu16 * 1000; + break; + case POWER_SUPPLY_PROP_INPUT_CURRENT_LIMIT: + ret = apple_smc_read_u16(power->smc, SMC_KEY(AC-i), &vu16); + val->intval = vu16 * 1000; + break; + case POWER_SUPPLY_PROP_INPUT_POWER_LIMIT: + ret = apple_smc_read_u32(power->smc, SMC_KEY(ACPW), &vu32); + val->intval = vu32 * 1000; + break; + default: + return -EINVAL; + } + + return ret; +} + +static enum power_supply_property macsmc_ac_props[] = { + POWER_SUPPLY_PROP_ONLINE, + POWER_SUPPLY_PROP_INPUT_POWER_LIMIT, + POWER_SUPPLY_PROP_VOLTAGE_NOW, + POWER_SUPPLY_PROP_INPUT_CURRENT_LIMIT, +}; + +static const struct power_supply_desc macsmc_ac_desc = { + .name = "macsmc-ac", + .type = POWER_SUPPLY_TYPE_MAINS, + .get_property = macsmc_ac_get_property, + .properties = macsmc_ac_props, + .num_properties = ARRAY_SIZE(macsmc_ac_props), +}; + +static int macsmc_log_power_set(const char *val, const struct kernel_param *kp) +{ + int ret = param_set_bool(val, kp); + + if (ret < 0) + return ret; + + if (log_power && g_power) + schedule_delayed_work(&g_power->dbg_log_work, 0); + + return 0; +} + +static void macsmc_dbg_work(struct work_struct *wrk) +{ + struct macsmc_power *power = container_of(to_delayed_work(wrk), + struct macsmc_power, dbg_log_work); + + macsmc_do_dbg(power); + + if (log_power) + schedule_delayed_work(&power->dbg_log_work, POWER_LOG_INTERVAL); +} + +static void macsmc_power_critical_work(struct work_struct *wrk) +{ + struct macsmc_power *power = container_of(wrk, struct macsmc_power, critical_work); + int ret; + u32 bcf0; + u16 bitv, b0av; + + /* + * Check if the battery voltage is below the design voltage. If it is, + * we have a few seconds until the machine dies. Explicitly shut down, + * which at least gets the NVMe controller to flush its cache. + */ + if (apple_smc_read_u16(power->smc, SMC_KEY(BITV), &bitv) >= 0 && + apple_smc_read_u16(power->smc, SMC_KEY(B0AV), &b0av) >= 0 && + b0av < bitv) { + dev_crit(power->dev, "Emergency notification: Battery is critical\n"); + if (kernel_can_power_off()) + kernel_power_off(); + else /* Missing macsmc-reboot driver? In this state, this will not boot anyway. */ + kernel_restart("Battery is critical"); + } + + /* This spams once per second, so make sure we only trigger shutdown once. */ + if (power->shutdown_started) + return; + + /* Check for battery empty condition */ + ret = apple_smc_read_u32(power->smc, SMC_KEY(BCF0), &bcf0); + if (ret < 0) { + dev_err(power->dev, + "Emergency notification: Failed to read battery status\n"); + } else if (bcf0 == 0) { + dev_warn(power->dev, "Emergency notification: Battery status is OK?\n"); + return; + } else { + dev_warn(power->dev, "Emergency notification: Battery is empty\n"); + } + + power->shutdown_started = true; + + /* + * Attempt to trigger an orderly shutdown. At this point, we should have a few + * minutes of reserve capacity left, enough to do a clean shutdown. + */ + dev_warn(power->dev, "Shutting down in 10 seconds\n"); + ssleep(10); + + /* + * Don't force it; if this stalls or fails, the last-resort check above will + * trigger a hard shutdown when shutdown is truly imminent. + */ + orderly_poweroff(false); +} + +static int macsmc_power_event(struct notifier_block *nb, unsigned long event, void *data) +{ + struct macsmc_power *power = container_of(nb, struct macsmc_power, nb); + + if ((event & 0xffffff00) == 0x71010100) { + bool charging = (event & 0xff) != 0; + + dev_info(power->dev, "Charging: %d\n", charging); + power_supply_changed(power->batt); + power_supply_changed(power->ac); + + return NOTIFY_OK; + } else if (event == 0x71020000) { + schedule_work(&power->critical_work); + + return NOTIFY_OK; + } else if ((event & 0xffff0000) == 0x71060000) { + u8 changed_port = event >> 8; + u8 cur_port; + + /* Port charging state change? */ + if (apple_smc_read_u8(power->smc, SMC_KEY(AC-W), &cur_port) >= 0) { + dev_info(power->dev, "Port %d state change (charge port: %d)\n", + changed_port + 1, cur_port); + } + + power_supply_changed(power->batt); + power_supply_changed(power->ac); + + return NOTIFY_OK; + } else if ((event & 0xffff0000) == 0x71130000) { + u8 port_index = (event >> 8) & 0xff; + u8 status = event & 0xff; + + if (port_index == 0xff) + dev_info(power->dev, "Connector event: Disconnect (status 0x%02x)\n", + status); + else + dev_info(power->dev, "Connector event: Port %d (status 0x%02x)\n", + port_index + 1, status); + + power_supply_changed(power->batt); + power_supply_changed(power->ac); + + return NOTIFY_OK; + } else if ((event & 0xff000000) == 0x71000000) { + dev_info(power->dev, "Unknown charger event 0x%lx\n", event); + + return NOTIFY_OK; + } else if ((event & 0xffff0000) == 0x72010000) { + /* Button event handled by macsmc-hid, but let's do a debug print */ + if (log_power) + macsmc_do_dbg(power); + + return NOTIFY_OK; + } + + return NOTIFY_DONE; +} + +static int macsmc_power_probe(struct platform_device *pdev) +{ + struct apple_smc *smc = dev_get_drvdata(pdev->dev.parent); + struct power_supply_config psy_cfg = {}; + struct macsmc_power *power; + bool flag; + u8 val8; + u16 vu16; + u32 val32; + int ret; + + power = devm_kzalloc(&pdev->dev, sizeof(*power), GFP_KERNEL); + if (!power) + return -ENOMEM; + + power->dev = &pdev->dev; + power->smc = smc; + power->ac_desc = macsmc_ac_desc; + power->batt_desc = macsmc_battery_desc; + dev_set_drvdata(&pdev->dev, power); + + /* Ignore devices without a charger/battery */ + if (macsmc_battery_get_status(power) <= POWER_SUPPLY_STATUS_UNKNOWN) + return -ENODEV; + + /* Fetch string properties */ + apple_smc_read(smc, SMC_KEY(BMDN), power->model_name, sizeof(power->model_name) - 1); + apple_smc_read(smc, SMC_KEY(BMSN), power->serial_number, sizeof(power->serial_number) - 1); + apple_smc_read(smc, SMC_KEY(BMDT), power->mfg_date, sizeof(power->mfg_date) - 1); + + if (apple_smc_read_u32(power->smc, SMC_KEY(CHTE), &val32) >= 0) + power->has_chte = true; + + if (apple_smc_read_u8(power->smc, SMC_KEY(CH0C), &val8) >= 0) + power->has_ch0c = true; + + if (apple_smc_read_u8(power->smc, SMC_KEY(CH0I), &val8) >= 0) + power->has_ch0i = true; + + /* Turn off the "optimized battery charging" flags, in case macOS left them on */ + if (power->has_chte) + apple_smc_write_u32(power->smc, SMC_KEY(CHTE), 0); + else if (power->has_ch0c) + apple_smc_write_u8(power->smc, SMC_KEY(CH0C), 0); + + if (power->has_ch0i) + apple_smc_write_u8(power->smc, SMC_KEY(CH0I), 0); + + apple_smc_write_u8(power->smc, SMC_KEY(CH0K), 0); + apple_smc_write_u8(power->smc, SMC_KEY(CH0B), 0); + + power->batt_desc.charge_behaviours = BIT(POWER_SUPPLY_CHARGE_BEHAVIOUR_AUTO); + + /* Newer firmwares do not have force discharge, so check if it's supported */ + if (power->has_ch0i) + power->batt_desc.charge_behaviours |= BIT(POWER_SUPPLY_CHARGE_BEHAVIOUR_FORCE_DISCHARGE); + + /* Older firmware uses CH0C, and newer firmware uses CHTE, so check if at least one is present*/ + if (power->has_chte || power->has_ch0c) + power->batt_desc.charge_behaviours |= BIT(POWER_SUPPLY_CHARGE_BEHAVIOUR_INHIBIT_CHARGE); + + /* + * Prefer CHWA as the SMC firmware from iBoot-10151.1.1 is not compatible with + * this CHLS usage. + */ + if (apple_smc_read_flag(power->smc, SMC_KEY(CHWA), &flag) == 0) { + power->has_chwa = true; + } else if (apple_smc_read_u16(power->smc, SMC_KEY(CHLS), &vu16) >= 0) { + power->has_chls = true; + } else { + /* Remove the last 2 properties that control the charge threshold */ + power->batt_desc.num_properties -= 2; + } + + apple_smc_read_u8(power->smc, SMC_KEY(BNCB), &power->num_cells); + power->nominal_voltage_mv = MACSMC_NOMINAL_CELL_VOLTAGE_MV * power->num_cells; + + /* Doing one read of this flag enables critical shutdown notifications */ + apple_smc_read_u32(power->smc, SMC_KEY(BCF0), &val32); + + psy_cfg.drv_data = power; + power->batt = devm_power_supply_register(&pdev->dev, &power->batt_desc, &psy_cfg); + if (IS_ERR(power->batt)) { + dev_err(&pdev->dev, "Failed to register battery\n"); + ret = PTR_ERR(power->batt); + return ret; + } + + /* SMC firmware in macOS 15.4 dropped "AC-i" and "AC-n" (and all keys + * with lower case last letter) without obvious replacement. */ + if (apple_smc_read_u16(power->smc, SMC_KEY(AC-n), &vu16) < 0) + power->ac_desc.num_properties -= 2; + + power->ac = devm_power_supply_register(&pdev->dev, &power->ac_desc, &psy_cfg); + if (IS_ERR(power->ac)) { + dev_err(&pdev->dev, "Failed to register AC adapter\n"); + ret = PTR_ERR(power->ac); + return ret; + } + + power->nb.notifier_call = macsmc_power_event; + blocking_notifier_chain_register(&smc->event_handlers, &power->nb); + + INIT_WORK(&power->critical_work, macsmc_power_critical_work); + INIT_DELAYED_WORK(&power->dbg_log_work, macsmc_dbg_work); + + g_power = power; + + if (log_power) + schedule_delayed_work(&power->dbg_log_work, 0); + + return 0; +} + +static void macsmc_power_remove(struct platform_device *pdev) +{ + struct macsmc_power *power = dev_get_drvdata(&pdev->dev); + + cancel_work(&power->critical_work); + cancel_delayed_work(&power->dbg_log_work); + + g_power = NULL; + + blocking_notifier_chain_unregister(&power->smc->event_handlers, &power->nb); +} + +static struct platform_driver macsmc_power_driver = { + .driver = { + .name = "macsmc-power", + .owner = THIS_MODULE, + }, + .probe = macsmc_power_probe, + .remove = macsmc_power_remove, +}; +module_platform_driver(macsmc_power_driver); + +MODULE_LICENSE("Dual MIT/GPL"); +MODULE_DESCRIPTION("Apple SMC battery and power management driver"); +MODULE_AUTHOR("Hector Martin "); +MODULE_ALIAS("platform:macsmc-power"); diff --git a/drivers/power/supply/pf1550-charger.c b/drivers/power/supply/pf1550-charger.c index 98f1ee8eca3bc8..a457862ef46108 100644 --- a/drivers/power/supply/pf1550-charger.c +++ b/drivers/power/supply/pf1550-charger.c @@ -584,22 +584,6 @@ static int pf1550_charger_probe(struct platform_device *pdev) return dev_err_probe(chg->dev, ret, "failed to add battery sense work\n"); - for (i = 0; i < PF1550_CHARGER_IRQ_NR; i++) { - irq = platform_get_irq(pdev, i); - if (irq < 0) - return irq; - - chg->virqs[i] = irq; - - ret = devm_request_threaded_irq(&pdev->dev, irq, NULL, - pf1550_charger_irq_handler, - IRQF_NO_SUSPEND, - "pf1550-charger", chg); - if (ret) - return dev_err_probe(&pdev->dev, ret, - "failed irq request\n"); - } - psy_cfg.drv_data = chg; chg->charger = devm_power_supply_register(&pdev->dev, @@ -616,6 +600,22 @@ static int pf1550_charger_probe(struct platform_device *pdev) return dev_err_probe(&pdev->dev, PTR_ERR(chg->battery), "failed: power supply register\n"); + for (i = 0; i < PF1550_CHARGER_IRQ_NR; i++) { + irq = platform_get_irq(pdev, i); + if (irq < 0) + return irq; + + chg->virqs[i] = irq; + + ret = devm_request_threaded_irq(&pdev->dev, irq, NULL, + pf1550_charger_irq_handler, + IRQF_NO_SUSPEND, + "pf1550-charger", chg); + if (ret) + return dev_err_probe(&pdev->dev, ret, + "failed irq request\n"); + } + pf1550_dt_parse_dev_info(chg); return pf1550_reg_init(chg); diff --git a/drivers/power/supply/pm8916_bms_vm.c b/drivers/power/supply/pm8916_bms_vm.c index 5120be086e6ffc..de5d571c03e212 100644 --- a/drivers/power/supply/pm8916_bms_vm.c +++ b/drivers/power/supply/pm8916_bms_vm.c @@ -167,15 +167,6 @@ static int pm8916_bms_vm_battery_probe(struct platform_device *pdev) if (ret < 0) return -EINVAL; - irq = platform_get_irq_byname(pdev, "fifo"); - if (irq < 0) - return irq; - - ret = devm_request_threaded_irq(dev, irq, NULL, pm8916_bms_vm_fifo_update_done_irq, - IRQF_ONESHOT, "pm8916_vm_bms", bat); - if (ret) - return ret; - ret = regmap_bulk_read(bat->regmap, bat->reg + PM8916_PERPH_TYPE, &tmp, 2); if (ret) goto comm_error; @@ -220,6 +211,15 @@ static int pm8916_bms_vm_battery_probe(struct platform_device *pdev) if (ret) return dev_err_probe(dev, ret, "Unable to get battery info\n"); + irq = platform_get_irq_byname(pdev, "fifo"); + if (irq < 0) + return irq; + + ret = devm_request_threaded_irq(dev, irq, NULL, pm8916_bms_vm_fifo_update_done_irq, + IRQF_ONESHOT, "pm8916_vm_bms", bat); + if (ret) + return ret; + platform_set_drvdata(pdev, bat); return 0; diff --git a/drivers/power/supply/pm8916_lbc.c b/drivers/power/supply/pm8916_lbc.c index c74b75b1b2676c..6b631012a7959a 100644 --- a/drivers/power/supply/pm8916_lbc.c +++ b/drivers/power/supply/pm8916_lbc.c @@ -274,15 +274,6 @@ static int pm8916_lbc_charger_probe(struct platform_device *pdev) return dev_err_probe(dev, -EINVAL, "Wrong amount of reg values: %d (4 expected)\n", len); - irq = platform_get_irq_byname(pdev, "usb_vbus"); - if (irq < 0) - return irq; - - ret = devm_request_threaded_irq(dev, irq, NULL, pm8916_lbc_charger_state_changed_irq, - IRQF_ONESHOT, "pm8916_lbc", chg); - if (ret) - return ret; - ret = device_property_read_u32_array(dev, "reg", chg->reg, len); if (ret) return ret; @@ -332,6 +323,10 @@ static int pm8916_lbc_charger_probe(struct platform_device *pdev) if (ret) return dev_err_probe(dev, ret, "Unable to get battery info\n"); + irq = platform_get_irq_byname(pdev, "usb_vbus"); + if (irq < 0) + return irq; + chg->edev = devm_extcon_dev_allocate(dev, pm8916_lbc_charger_cable); if (IS_ERR(chg->edev)) return PTR_ERR(chg->edev); @@ -340,6 +335,11 @@ static int pm8916_lbc_charger_probe(struct platform_device *pdev) if (ret < 0) return dev_err_probe(dev, ret, "failed to register extcon device\n"); + ret = devm_request_threaded_irq(dev, irq, NULL, pm8916_lbc_charger_state_changed_irq, + IRQF_ONESHOT, "pm8916_lbc", chg); + if (ret) + return ret; + ret = regmap_read(chg->regmap, chg->reg[LBC_USB] + PM8916_INT_RT_STS, &tmp); if (ret) goto comm_error; diff --git a/drivers/power/supply/qcom_battmgr.c b/drivers/power/supply/qcom_battmgr.c index c8028606bba009..80572ee945b4f8 100644 --- a/drivers/power/supply/qcom_battmgr.c +++ b/drivers/power/supply/qcom_battmgr.c @@ -1240,7 +1240,8 @@ static unsigned int qcom_battmgr_sc8280xp_parse_technology(const char *chemistry if ((!strncmp(chemistry, "LIO", BATTMGR_CHEMISTRY_LEN)) || (!strncmp(chemistry, "OOI", BATTMGR_CHEMISTRY_LEN))) return POWER_SUPPLY_TECHNOLOGY_LION; - if (!strncmp(chemistry, "LIP", BATTMGR_CHEMISTRY_LEN)) + if (!strncmp(chemistry, "LIP", BATTMGR_CHEMISTRY_LEN) || + !strncmp(chemistry, "LiP", BATTMGR_CHEMISTRY_LEN)) return POWER_SUPPLY_TECHNOLOGY_LIPO; pr_err("Unknown battery technology '%s'\n", chemistry); diff --git a/drivers/power/supply/rt9455_charger.c b/drivers/power/supply/rt9455_charger.c index 1ffe7f02932f6d..5130d2395e88fa 100644 --- a/drivers/power/supply/rt9455_charger.c +++ b/drivers/power/supply/rt9455_charger.c @@ -1663,6 +1663,15 @@ static int rt9455_probe(struct i2c_client *client) rt9455_charger_config.supplied_to = rt9455_charger_supplied_to; rt9455_charger_config.num_supplicants = ARRAY_SIZE(rt9455_charger_supplied_to); + + info->charger = devm_power_supply_register(dev, &rt9455_charger_desc, + &rt9455_charger_config); + if (IS_ERR(info->charger)) { + dev_err(dev, "Failed to register charger\n"); + ret = PTR_ERR(info->charger); + goto put_usb_notifier; + } + ret = devm_request_threaded_irq(dev, client->irq, NULL, rt9455_irq_handler_thread, IRQF_TRIGGER_LOW | IRQF_ONESHOT, @@ -1678,14 +1687,6 @@ static int rt9455_probe(struct i2c_client *client) goto put_usb_notifier; } - info->charger = devm_power_supply_register(dev, &rt9455_charger_desc, - &rt9455_charger_config); - if (IS_ERR(info->charger)) { - dev_err(dev, "Failed to register charger\n"); - ret = PTR_ERR(info->charger); - goto put_usb_notifier; - } - return 0; put_usb_notifier: diff --git a/drivers/power/supply/sbs-battery.c b/drivers/power/supply/sbs-battery.c index 943c82ee978f40..43c48196c16741 100644 --- a/drivers/power/supply/sbs-battery.c +++ b/drivers/power/supply/sbs-battery.c @@ -1174,24 +1174,6 @@ static int sbs_probe(struct i2c_client *client) i2c_set_clientdata(client, chip); - if (!chip->gpio_detect) - goto skip_gpio; - - irq = gpiod_to_irq(chip->gpio_detect); - if (irq <= 0) { - dev_warn(&client->dev, "Failed to get gpio as irq: %d\n", irq); - goto skip_gpio; - } - - rc = devm_request_threaded_irq(&client->dev, irq, NULL, sbs_irq, - IRQF_TRIGGER_RISING | IRQF_TRIGGER_FALLING | IRQF_ONESHOT, - dev_name(&client->dev), chip); - if (rc) { - dev_warn(&client->dev, "Failed to request irq: %d\n", rc); - goto skip_gpio; - } - -skip_gpio: /* * Before we register, we might need to make sure we can actually talk * to the battery. @@ -1217,6 +1199,24 @@ static int sbs_probe(struct i2c_client *client) return dev_err_probe(&client->dev, PTR_ERR(chip->power_supply), "Failed to register power supply\n"); + if (!chip->gpio_detect) + goto out; + + irq = gpiod_to_irq(chip->gpio_detect); + if (irq <= 0) { + dev_warn(&client->dev, "Failed to get gpio as irq: %d\n", irq); + goto out; + } + + rc = devm_request_threaded_irq(&client->dev, irq, NULL, sbs_irq, + IRQF_TRIGGER_RISING | IRQF_TRIGGER_FALLING | IRQF_ONESHOT, + dev_name(&client->dev), chip); + if (rc) { + dev_warn(&client->dev, "Failed to request irq: %d\n", rc); + goto out; + } + +out: dev_info(&client->dev, "%s: battery gas gauge device registered\n", client->name); diff --git a/drivers/power/supply/wm97xx_battery.c b/drivers/power/supply/wm97xx_battery.c index b3b0c37a9dd2d5..f00722c88c6fea 100644 --- a/drivers/power/supply/wm97xx_battery.c +++ b/drivers/power/supply/wm97xx_battery.c @@ -178,12 +178,6 @@ static int wm97xx_bat_probe(struct platform_device *dev) "failed to get charge GPIO\n"); if (charge_gpiod) { gpiod_set_consumer_name(charge_gpiod, "BATT CHRG"); - ret = request_irq(gpiod_to_irq(charge_gpiod), - wm97xx_chrg_irq, 0, - "AC Detect", dev); - if (ret) - return dev_err_probe(&dev->dev, ret, - "failed to request GPIO irq\n"); props++; /* POWER_SUPPLY_PROP_STATUS */ } @@ -199,10 +193,8 @@ static int wm97xx_bat_probe(struct platform_device *dev) props++; /* POWER_SUPPLY_PROP_VOLTAGE_MIN */ prop = kcalloc(props, sizeof(*prop), GFP_KERNEL); - if (!prop) { - ret = -ENOMEM; - goto err3; - } + if (!prop) + return -ENOMEM; prop[i++] = POWER_SUPPLY_PROP_PRESENT; if (charge_gpiod) @@ -236,15 +228,27 @@ static int wm97xx_bat_probe(struct platform_device *dev) schedule_work(&bat_work); } else { ret = PTR_ERR(bat_psy); - goto err4; + goto free; + } + + if (charge_gpiod) { + ret = request_irq(gpiod_to_irq(charge_gpiod), wm97xx_chrg_irq, + 0, "AC Detect", dev); + if (ret) { + dev_err_probe(&dev->dev, ret, + "failed to request GPIO irq\n"); + goto unregister; + } } return 0; -err4: + +unregister: + power_supply_unregister(bat_psy); + +free: kfree(prop); -err3: - if (charge_gpiod) - free_irq(gpiod_to_irq(charge_gpiod), dev); + return ret; } diff --git a/drivers/powercap/intel_rapl_common.c b/drivers/powercap/intel_rapl_common.c index 3ff6da3bf4e630..3705d0608a0fba 100644 --- a/drivers/powercap/intel_rapl_common.c +++ b/drivers/powercap/intel_rapl_common.c @@ -254,7 +254,7 @@ static void rapl_init_domains(struct rapl_package *rp); static int rapl_read_data_raw(struct rapl_domain *rd, enum rapl_primitives prim, bool xlate, u64 *data, - bool atomic); + bool pmu_ctx); static int rapl_write_data_raw(struct rapl_domain *rd, enum rapl_primitives prim, unsigned long long value); @@ -832,7 +832,7 @@ prim_fixups(struct rapl_domain *rd, enum rapl_primitives prim) */ static int rapl_read_data_raw(struct rapl_domain *rd, enum rapl_primitives prim, bool xlate, u64 *data, - bool atomic) + bool pmu_ctx) { u64 value; enum rapl_primitives prim_fixed = prim_fixups(rd, prim); @@ -854,7 +854,7 @@ static int rapl_read_data_raw(struct rapl_domain *rd, ra.mask = rpi->mask; - if (rd->rp->priv->read_raw(get_rid(rd->rp), &ra, atomic)) { + if (rd->rp->priv->read_raw(get_rid(rd->rp), &ra, pmu_ctx)) { pr_debug("failed to read reg 0x%llx for %s:%s\n", ra.reg.val, rd->rp->name, rd->name); return -EIO; } diff --git a/drivers/powercap/intel_rapl_msr.c b/drivers/powercap/intel_rapl_msr.c index 9a7e150b3536b9..3d5e7f56d68a15 100644 --- a/drivers/powercap/intel_rapl_msr.c +++ b/drivers/powercap/intel_rapl_msr.c @@ -110,16 +110,14 @@ static int rapl_cpu_down_prep(unsigned int cpu) return 0; } -static int rapl_msr_read_raw(int cpu, struct reg_action *ra, bool atomic) +static int rapl_msr_read_raw(int cpu, struct reg_action *ra, bool pmu_ctx) { /* - * When called from atomic-context (eg PMU event handler) - * perform MSR read directly using rdmsrq(). + * When called from PMU context, perform MSR read directly using + * rdmsrq() without IPI overhead. Package-scoped MSRs are readable + * from any CPU in the package. */ - if (atomic) { - if (unlikely(smp_processor_id() != cpu)) - return -EIO; - + if (pmu_ctx) { rdmsrq(ra->reg.msr, ra->value); goto out; } @@ -162,6 +160,7 @@ static int rapl_msr_write_raw(int cpu, struct reg_action *ra) /* List of verified CPUs. */ static const struct x86_cpu_id pl4_support_ids[] = { + X86_MATCH_VFM(INTEL_ICELAKE_L, NULL), X86_MATCH_VFM(INTEL_TIGERLAKE_L, NULL), X86_MATCH_VFM(INTEL_ALDERLAKE, NULL), X86_MATCH_VFM(INTEL_ALDERLAKE_L, NULL), diff --git a/drivers/powercap/intel_rapl_tpmi.c b/drivers/powercap/intel_rapl_tpmi.c index 0a0b85f4528b1b..0f8abdc592bc19 100644 --- a/drivers/powercap/intel_rapl_tpmi.c +++ b/drivers/powercap/intel_rapl_tpmi.c @@ -157,7 +157,7 @@ static int parse_one_domain(struct tpmi_rapl_package *trp, u32 offset) tpmi_domain_flags = tpmi_domain_header >> 32 & 0xffff; if (tpmi_domain_version == TPMI_VERSION_INVALID) { - pr_warn(FW_BUG "Invalid version\n"); + pr_debug("Invalid version, other instances may be valid\n"); return -ENODEV; } diff --git a/drivers/ptp/ptp_vmclock.c b/drivers/ptp/ptp_vmclock.c index b3a83b03d9c14e..cbbfc494680c70 100644 --- a/drivers/ptp/ptp_vmclock.c +++ b/drivers/ptp/ptp_vmclock.c @@ -591,6 +591,7 @@ static int vmclock_probe(struct platform_device *pdev) static const struct acpi_device_id vmclock_acpi_ids[] = { { "AMZNC10C", 0 }, + { "VMCLOCK", 0 }, {} }; MODULE_DEVICE_TABLE(acpi, vmclock_acpi_ids); diff --git a/drivers/pwm/pwm-tiehrpwm.c b/drivers/pwm/pwm-tiehrpwm.c index 7a86cb090f76f1..2533c95b0ba9d3 100644 --- a/drivers/pwm/pwm-tiehrpwm.c +++ b/drivers/pwm/pwm-tiehrpwm.c @@ -237,8 +237,6 @@ static int ehrpwm_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm, if (period_cycles < 1) period_cycles = 1; - pm_runtime_get_sync(pwmchip_parent(chip)); - /* Update clock prescaler values */ ehrpwm_modify(pc->mmio_base, TBCTL, TBCTL_CLKDIV_MASK, tb_divval); @@ -290,8 +288,6 @@ static int ehrpwm_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm, if (!(duty_cycles > period_cycles)) ehrpwm_write(pc->mmio_base, cmp_reg, duty_cycles); - pm_runtime_put_sync(pwmchip_parent(chip)); - return 0; } @@ -378,6 +374,8 @@ static int ehrpwm_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, int err; bool enabled = pwm->state.enabled; + guard(pm_runtime_active)(pwmchip_parent(chip)); + if (state->polarity != pwm->state.polarity) { if (enabled) { ehrpwm_pwm_disable(chip, pwm); diff --git a/drivers/rapidio/rio-scan.c b/drivers/rapidio/rio-scan.c index c12941f71e2cba..dcd6619a4b0277 100644 --- a/drivers/rapidio/rio-scan.c +++ b/drivers/rapidio/rio-scan.c @@ -854,7 +854,8 @@ static struct rio_net *rio_scan_alloc_net(struct rio_mport *mport, if (idtab == NULL) { pr_err("RIO: failed to allocate destID table\n"); - rio_free_net(net); + kfree(net); + mport->net = NULL; net = NULL; } else { net->enum_data = idtab; diff --git a/drivers/ras/ras.c b/drivers/ras/ras.c index 2a5b5a9fdcb36c..03df3db6233463 100644 --- a/drivers/ras/ras.c +++ b/drivers/ras/ras.c @@ -72,7 +72,11 @@ void log_arm_hw_error(struct cper_sec_proc_arm *err, const u8 sev) ctx_err = (u8 *)ctx_info; for (n = 0; n < err->context_info_num; n++) { - sz = sizeof(struct cper_arm_ctx_info) + ctx_info->size; + sz = sizeof(struct cper_arm_ctx_info); + + if (sz + (long)ctx_info - (long)err >= err->section_length) + sz += ctx_info->size; + ctx_info = (struct cper_arm_ctx_info *)((long)ctx_info + sz); ctx_len += sz; } diff --git a/drivers/regulator/bq257xx-regulator.c b/drivers/regulator/bq257xx-regulator.c index fc1ccede446882..dab8f1ab44503e 100644 --- a/drivers/regulator/bq257xx-regulator.c +++ b/drivers/regulator/bq257xx-regulator.c @@ -115,11 +115,10 @@ static void bq257xx_reg_dt_parse_gpio(struct platform_device *pdev) return; subchild = of_get_child_by_name(child, pdata->desc.of_match); + of_node_put(child); if (!subchild) return; - of_node_put(child); - pdata->otg_en_gpio = devm_fwnode_gpiod_get_index(&pdev->dev, of_fwnode_handle(subchild), "enable", 0, diff --git a/drivers/regulator/core.c b/drivers/regulator/core.c index 4b6182cde859ad..838bbdcdede9af 100644 --- a/drivers/regulator/core.c +++ b/drivers/regulator/core.c @@ -1444,6 +1444,33 @@ static int set_machine_constraints(struct regulator_dev *rdev) int ret = 0; const struct regulator_ops *ops = rdev->desc->ops; + /* + * If there is no mechanism for controlling the regulator then + * flag it as always_on so we don't end up duplicating checks + * for this so much. Note that we could control the state of + * a supply to control the output on a regulator that has no + * direct control. + */ + if (!rdev->ena_pin && !ops->enable) { + if (rdev->supply_name && !rdev->supply) + return -EPROBE_DEFER; + + if (rdev->supply) + rdev->constraints->always_on = + rdev->supply->rdev->constraints->always_on; + else + rdev->constraints->always_on = true; + } + + /* + * If we want to enable this regulator, make sure that we know the + * supplying regulator. + */ + if (rdev->constraints->always_on || rdev->constraints->boot_on) { + if (rdev->supply_name && !rdev->supply) + return -EPROBE_DEFER; + } + ret = machine_constraints_voltage(rdev, rdev->constraints); if (ret != 0) return ret; @@ -1609,37 +1636,15 @@ static int set_machine_constraints(struct regulator_dev *rdev) } } - /* - * If there is no mechanism for controlling the regulator then - * flag it as always_on so we don't end up duplicating checks - * for this so much. Note that we could control the state of - * a supply to control the output on a regulator that has no - * direct control. - */ - if (!rdev->ena_pin && !ops->enable) { - if (rdev->supply_name && !rdev->supply) - return -EPROBE_DEFER; - - if (rdev->supply) - rdev->constraints->always_on = - rdev->supply->rdev->constraints->always_on; - else - rdev->constraints->always_on = true; - } - /* If the constraints say the regulator should be on at this point * and we have control then make sure it is enabled. */ if (rdev->constraints->always_on || rdev->constraints->boot_on) { bool supply_enabled = false; - /* If we want to enable this regulator, make sure that we know - * the supplying regulator. - */ - if (rdev->supply_name && !rdev->supply) - return -EPROBE_DEFER; - - /* If supplying regulator has already been enabled, + /* We have ensured a potential supply has been resolved above. + * + * If supplying regulator has already been enabled, * it's not intended to have use_count increment * when rdev is only boot-on. */ @@ -1944,8 +1949,6 @@ static const struct file_operations constraint_flags_fops = { #endif }; -#define REG_STR_SIZE 64 - static void link_and_create_debugfs(struct regulator *regulator, struct regulator_dev *rdev, struct device *dev) { @@ -1993,15 +1996,7 @@ static struct regulator *create_regulator(struct regulator_dev *rdev, lockdep_assert_held_once(&rdev->mutex.base); if (dev) { - char buf[REG_STR_SIZE]; - int size; - - size = snprintf(buf, REG_STR_SIZE, "%s-%s", - dev->kobj.name, supply_name); - if (size >= REG_STR_SIZE) - return NULL; - - supply_name = kstrdup(buf, GFP_KERNEL); + supply_name = kasprintf(GFP_KERNEL, "%s-%s", dev->kobj.name, supply_name); if (supply_name == NULL) return NULL; } else { @@ -2268,10 +2263,21 @@ static int regulator_resolve_supply(struct regulator_dev *rdev) * under-voltage. */ ret = register_regulator_event_forwarding(rdev); - if (ret < 0) + if (ret < 0) { + struct regulator *supply; + rdev_warn(rdev, "Failed to register event forwarding: %pe\n", ERR_PTR(ret)); + supply = rdev->supply; + rdev->supply = NULL; + + regulator_unlock_two(rdev, supply->rdev, &ww_ctx); + + regulator_put(supply); + goto out; + } + regulator_unlock_two(rdev, r, &ww_ctx); /* rdev->supply was created in set_supply() */ @@ -2285,8 +2291,16 @@ static int regulator_resolve_supply(struct regulator_dev *rdev) if (rdev->use_count) { ret = regulator_enable(rdev->supply); if (ret < 0) { - _regulator_put(rdev->supply); + struct regulator *supply; + + regulator_lock_two(rdev, rdev->supply->rdev, &ww_ctx); + + supply = rdev->supply; rdev->supply = NULL; + + regulator_unlock_two(rdev, supply->rdev, &ww_ctx); + + regulator_put(supply); goto out; } } diff --git a/drivers/regulator/fp9931.c b/drivers/regulator/fp9931.c index 7fbcc6327cc635..abea3b69d8a085 100644 --- a/drivers/regulator/fp9931.c +++ b/drivers/regulator/fp9931.c @@ -144,13 +144,12 @@ static int fp9931_hwmon_read(struct device *dev, enum hwmon_sensor_types type, return ret; ret = regmap_read(data->regmap, FP9931_REG_TMST_VALUE, &val); - if (ret) - return ret; + if (!ret) + *temp = (s8)val * 1000; pm_runtime_put_autosuspend(data->dev); - *temp = (s8)val * 1000; - return 0; + return ret; } static umode_t fp9931_hwmon_is_visible(const void *data, diff --git a/drivers/regulator/mt6363-regulator.c b/drivers/regulator/mt6363-regulator.c index e0fbf92e768517..0aebcbda0a1963 100644 --- a/drivers/regulator/mt6363-regulator.c +++ b/drivers/regulator/mt6363-regulator.c @@ -861,7 +861,7 @@ static int mt6363_regulator_probe(struct platform_device *pdev) struct irq_domain *domain; struct irq_fwspec fwspec; struct spmi_device *sdev; - int i, ret; + int i, ret, val; config.regmap = mt6363_spmi_register_regmap(dev); if (IS_ERR(config.regmap)) @@ -870,6 +870,13 @@ static int mt6363_regulator_probe(struct platform_device *pdev) config.dev = dev; sdev = to_spmi_device(dev->parent); + /* + * The first read may fail if the bootloader sets sleep mode: wake up + * this PMIC with W/R on the SPMI bus and ignore the first result. + * This matches the MT6373 driver behavior. + */ + regmap_read(config.regmap, MT6363_TOP_TRAP, &val); + interrupt_parent = of_irq_find_parent(dev->of_node); if (!interrupt_parent) return dev_err_probe(dev, -EINVAL, "Cannot find IRQ parent\n"); @@ -892,10 +899,8 @@ static int mt6363_regulator_probe(struct platform_device *pdev) "Failed to map IRQ%d\n", info->hwirq); ret = devm_add_action_or_reset(dev, mt6363_irq_remove, &info->virq); - if (ret) { - irq_dispose_mapping(info->hwirq); + if (ret) return ret; - } config.driver_data = info; INIT_DELAYED_WORK(&info->oc_work, mt6363_oc_irq_enable_work); diff --git a/drivers/regulator/pca9450-regulator.c b/drivers/regulator/pca9450-regulator.c index 5fa8682642505a..45d7dc44c2cd08 100644 --- a/drivers/regulator/pca9450-regulator.c +++ b/drivers/regulator/pca9450-regulator.c @@ -1293,6 +1293,7 @@ static int pca9450_i2c_probe(struct i2c_client *i2c) struct regulator_dev *ldo5; struct pca9450 *pca9450; unsigned int device_id, i; + const char *type_name; int ret; pca9450 = devm_kzalloc(&i2c->dev, sizeof(struct pca9450), GFP_KERNEL); @@ -1303,15 +1304,22 @@ static int pca9450_i2c_probe(struct i2c_client *i2c) case PCA9450_TYPE_PCA9450A: regulator_desc = pca9450a_regulators; pca9450->rcnt = ARRAY_SIZE(pca9450a_regulators); + type_name = "pca9450a"; break; case PCA9450_TYPE_PCA9450BC: regulator_desc = pca9450bc_regulators; pca9450->rcnt = ARRAY_SIZE(pca9450bc_regulators); + type_name = "pca9450bc"; break; case PCA9450_TYPE_PCA9451A: + regulator_desc = pca9451a_regulators; + pca9450->rcnt = ARRAY_SIZE(pca9451a_regulators); + type_name = "pca9451a"; + break; case PCA9450_TYPE_PCA9452: regulator_desc = pca9451a_regulators; pca9450->rcnt = ARRAY_SIZE(pca9451a_regulators); + type_name = "pca9452"; break; default: dev_err(&i2c->dev, "Unknown device type"); @@ -1369,7 +1377,7 @@ static int pca9450_i2c_probe(struct i2c_client *i2c) if (pca9450->irq) { ret = devm_request_threaded_irq(pca9450->dev, pca9450->irq, NULL, pca9450_irq_handler, - (IRQF_TRIGGER_FALLING | IRQF_ONESHOT), + (IRQF_TRIGGER_LOW | IRQF_ONESHOT), "pca9450-irq", pca9450); if (ret != 0) return dev_err_probe(pca9450->dev, ret, "Failed to request IRQ: %d\n", @@ -1413,9 +1421,7 @@ static int pca9450_i2c_probe(struct i2c_client *i2c) pca9450_i2c_restart_handler, pca9450)) dev_warn(&i2c->dev, "Failed to register restart handler\n"); - dev_info(&i2c->dev, "%s probed.\n", - type == PCA9450_TYPE_PCA9450A ? "pca9450a" : - (type == PCA9450_TYPE_PCA9451A ? "pca9451a" : "pca9450bc")); + dev_info(&i2c->dev, "%s probed.\n", type_name); return 0; } diff --git a/drivers/regulator/pf9453-regulator.c b/drivers/regulator/pf9453-regulator.c index 779a6fdb0574bf..eed3055d1c1cad 100644 --- a/drivers/regulator/pf9453-regulator.c +++ b/drivers/regulator/pf9453-regulator.c @@ -809,7 +809,7 @@ static int pf9453_i2c_probe(struct i2c_client *i2c) } ret = devm_request_threaded_irq(pf9453->dev, pf9453->irq, NULL, pf9453_irq_handler, - (IRQF_TRIGGER_FALLING | IRQF_ONESHOT), + IRQF_ONESHOT, "pf9453-irq", pf9453); if (ret) return dev_err_probe(pf9453->dev, ret, "Failed to request IRQ: %d\n", pf9453->irq); diff --git a/drivers/remoteproc/imx_dsp_rproc.c b/drivers/remoteproc/imx_dsp_rproc.c index 5130a35214c929..9e4f50e0e822dd 100644 --- a/drivers/remoteproc/imx_dsp_rproc.c +++ b/drivers/remoteproc/imx_dsp_rproc.c @@ -644,6 +644,32 @@ static void imx_dsp_rproc_free_mbox(struct imx_dsp_rproc *priv) mbox_free_channel(priv->rxdb_ch); } +static int imx_dsp_rproc_mem_alloc(struct rproc *rproc, + struct rproc_mem_entry *mem) +{ + struct device *dev = rproc->dev.parent; + void *va; + + va = ioremap_wc(mem->dma, mem->len); + if (!va) { + dev_err(dev, "Unable to map memory region: %pa+%zx\n", + &mem->dma, mem->len); + return -ENOMEM; + } + + mem->va = va; + + return 0; +} + +static int imx_dsp_rproc_mem_release(struct rproc *rproc, + struct rproc_mem_entry *mem) +{ + iounmap(mem->va); + + return 0; +} + /** * imx_dsp_rproc_add_carveout() - request mailbox channels * @priv: private data pointer @@ -659,7 +685,6 @@ static int imx_dsp_rproc_add_carveout(struct imx_dsp_rproc *priv) struct device *dev = rproc->dev.parent; struct device_node *np = dev->of_node; struct rproc_mem_entry *mem; - void __iomem *cpu_addr; int a, i = 0; u64 da; @@ -673,15 +698,10 @@ static int imx_dsp_rproc_add_carveout(struct imx_dsp_rproc *priv) if (imx_dsp_rproc_sys_to_da(priv, att->sa, att->size, &da)) return -EINVAL; - cpu_addr = devm_ioremap_wc(dev, att->sa, att->size); - if (!cpu_addr) { - dev_err(dev, "failed to map memory %p\n", &att->sa); - return -ENOMEM; - } - /* Register memory region */ - mem = rproc_mem_entry_init(dev, (void __force *)cpu_addr, (dma_addr_t)att->sa, - att->size, da, NULL, NULL, "dsp_mem"); + mem = rproc_mem_entry_init(dev, NULL, (dma_addr_t)att->sa, + att->size, da, imx_dsp_rproc_mem_alloc, + imx_dsp_rproc_mem_release, "dsp_mem"); if (mem) rproc_coredump_add_segment(rproc, da, att->size); @@ -709,15 +729,11 @@ static int imx_dsp_rproc_add_carveout(struct imx_dsp_rproc *priv) if (imx_dsp_rproc_sys_to_da(priv, res.start, resource_size(&res), &da)) return -EINVAL; - cpu_addr = devm_ioremap_resource_wc(dev, &res); - if (IS_ERR(cpu_addr)) { - dev_err(dev, "failed to map memory %pR\n", &res); - return PTR_ERR(cpu_addr); - } - /* Register memory region */ - mem = rproc_mem_entry_init(dev, (void __force *)cpu_addr, (dma_addr_t)res.start, - resource_size(&res), da, NULL, NULL, + mem = rproc_mem_entry_init(dev, NULL, (dma_addr_t)res.start, + resource_size(&res), da, + imx_dsp_rproc_mem_alloc, + imx_dsp_rproc_mem_release, "%.*s", strchrnul(res.name, '@') - res.name, res.name); if (!mem) return -ENOMEM; @@ -984,9 +1000,11 @@ static int imx_dsp_rproc_load(struct rproc *rproc, const struct firmware *fw) * Clear buffers after pm rumtime for internal ocram is not * accessible if power and clock are not enabled. */ - list_for_each_entry(carveout, &rproc->carveouts, node) { - if (carveout->va) - memset(carveout->va, 0, carveout->len); + if (rproc->state == RPROC_OFFLINE) { + list_for_each_entry(carveout, &rproc->carveouts, node) { + if (carveout->va) + memset(carveout->va, 0, carveout->len); + } } ret = imx_dsp_rproc_elf_load_segments(rproc, fw); @@ -1242,6 +1260,15 @@ static int imx_dsp_suspend(struct device *dev) if (rproc->state != RPROC_RUNNING) goto out; + /* + * No channel available for sending messages; + * indicates no mailboxes present, so trigger PM runtime suspend + */ + if (!priv->tx_ch) { + dev_dbg(dev, "No initialized mbox tx channel, suspend directly.\n"); + goto out; + } + reinit_completion(&priv->pm_comp); /* Tell DSP that suspend is happening */ diff --git a/drivers/remoteproc/imx_rproc.c b/drivers/remoteproc/imx_rproc.c index 3be8790c14a2cc..d93bf5134d6f07 100644 --- a/drivers/remoteproc/imx_rproc.c +++ b/drivers/remoteproc/imx_rproc.c @@ -609,6 +609,10 @@ imx_rproc_elf_find_loaded_rsc_table(struct rproc *rproc, const struct firmware * { struct imx_rproc *priv = rproc->priv; + /* No resource table in the firmware */ + if (!rproc->table_ptr) + return NULL; + if (priv->rsc_table) return (struct resource_table *)priv->rsc_table; @@ -694,7 +698,7 @@ static int imx_rproc_addr_init(struct imx_rproc *priv, } priv->mem[b].sys_addr = res.start; priv->mem[b].size = resource_size(&res); - if (!strcmp(res.name, "rsc-table")) + if (strstarts(res.name, "rsc-table")) priv->rsc_table = priv->mem[b].cpu_addr; b++; } diff --git a/drivers/remoteproc/mtk_scp.c b/drivers/remoteproc/mtk_scp.c index db8fd045468d9a..b0b65aefc71903 100644 --- a/drivers/remoteproc/mtk_scp.c +++ b/drivers/remoteproc/mtk_scp.c @@ -283,7 +283,7 @@ static irqreturn_t scp_irq_handler(int irq, void *priv) struct mtk_scp *scp = priv; int ret; - ret = clk_prepare_enable(scp->clk); + ret = clk_enable(scp->clk); if (ret) { dev_err(scp->dev, "failed to enable clocks\n"); return IRQ_NONE; @@ -291,7 +291,7 @@ static irqreturn_t scp_irq_handler(int irq, void *priv) scp->data->scp_irq_handler(scp); - clk_disable_unprepare(scp->clk); + clk_disable(scp->clk); return IRQ_HANDLED; } @@ -665,7 +665,7 @@ static int scp_load(struct rproc *rproc, const struct firmware *fw) struct device *dev = scp->dev; int ret; - ret = clk_prepare_enable(scp->clk); + ret = clk_enable(scp->clk); if (ret) { dev_err(dev, "failed to enable clocks\n"); return ret; @@ -680,7 +680,7 @@ static int scp_load(struct rproc *rproc, const struct firmware *fw) ret = scp_elf_load_segments(rproc, fw); leave: - clk_disable_unprepare(scp->clk); + clk_disable(scp->clk); return ret; } @@ -691,14 +691,14 @@ static int scp_parse_fw(struct rproc *rproc, const struct firmware *fw) struct device *dev = scp->dev; int ret; - ret = clk_prepare_enable(scp->clk); + ret = clk_enable(scp->clk); if (ret) { dev_err(dev, "failed to enable clocks\n"); return ret; } ret = scp_ipi_init(scp, fw); - clk_disable_unprepare(scp->clk); + clk_disable(scp->clk); return ret; } @@ -709,7 +709,7 @@ static int scp_start(struct rproc *rproc) struct scp_run *run = &scp->run; int ret; - ret = clk_prepare_enable(scp->clk); + ret = clk_enable(scp->clk); if (ret) { dev_err(dev, "failed to enable clocks\n"); return ret; @@ -734,14 +734,14 @@ static int scp_start(struct rproc *rproc) goto stop; } - clk_disable_unprepare(scp->clk); + clk_disable(scp->clk); dev_info(dev, "SCP is ready. FW version %s\n", run->fw_ver); return 0; stop: scp->data->scp_reset_assert(scp); - clk_disable_unprepare(scp->clk); + clk_disable(scp->clk); return ret; } @@ -909,7 +909,7 @@ static int scp_stop(struct rproc *rproc) struct mtk_scp *scp = rproc->priv; int ret; - ret = clk_prepare_enable(scp->clk); + ret = clk_enable(scp->clk); if (ret) { dev_err(scp->dev, "failed to enable clocks\n"); return ret; @@ -917,12 +917,29 @@ static int scp_stop(struct rproc *rproc) scp->data->scp_reset_assert(scp); scp->data->scp_stop(scp); - clk_disable_unprepare(scp->clk); + clk_disable(scp->clk); return 0; } +static int scp_prepare(struct rproc *rproc) +{ + struct mtk_scp *scp = rproc->priv; + + return clk_prepare(scp->clk); +} + +static int scp_unprepare(struct rproc *rproc) +{ + struct mtk_scp *scp = rproc->priv; + + clk_unprepare(scp->clk); + return 0; +} + static const struct rproc_ops scp_ops = { + .prepare = scp_prepare, + .unprepare = scp_unprepare, .start = scp_start, .stop = scp_stop, .load = scp_load, @@ -1580,12 +1597,51 @@ static const struct of_device_id mtk_scp_of_match[] = { }; MODULE_DEVICE_TABLE(of, mtk_scp_of_match); +static int __maybe_unused scp_suspend(struct device *dev) +{ + struct mtk_scp *scp = dev_get_drvdata(dev); + struct rproc *rproc = scp->rproc; + + /* + * Only unprepare if the SCP is running and holding the clock. + * + * Note: `scp_ops` doesn't implement .attach() callback, hence + * `rproc->state` can never be RPROC_ATTACHED. Otherwise, it + * should also be checked here. + */ + if (rproc->state == RPROC_RUNNING) + clk_unprepare(scp->clk); + return 0; +} + +static int __maybe_unused scp_resume(struct device *dev) +{ + struct mtk_scp *scp = dev_get_drvdata(dev); + struct rproc *rproc = scp->rproc; + + /* + * Only prepare if the SCP was running and holding the clock. + * + * Note: `scp_ops` doesn't implement .attach() callback, hence + * `rproc->state` can never be RPROC_ATTACHED. Otherwise, it + * should also be checked here. + */ + if (rproc->state == RPROC_RUNNING) + return clk_prepare(scp->clk); + return 0; +} + +static const struct dev_pm_ops scp_pm_ops = { + SET_SYSTEM_SLEEP_PM_OPS(scp_suspend, scp_resume) +}; + static struct platform_driver mtk_scp_driver = { .probe = scp_probe, .remove = scp_remove, .driver = { .name = "mtk-scp", .of_match_table = mtk_scp_of_match, + .pm = &scp_pm_ops, }, }; diff --git a/drivers/remoteproc/mtk_scp_ipi.c b/drivers/remoteproc/mtk_scp_ipi.c index c068227e251e7c..7a37e273b3af8d 100644 --- a/drivers/remoteproc/mtk_scp_ipi.c +++ b/drivers/remoteproc/mtk_scp_ipi.c @@ -171,7 +171,7 @@ int scp_ipi_send(struct mtk_scp *scp, u32 id, void *buf, unsigned int len, WARN_ON(len > scp_sizes->ipi_share_buffer_size) || WARN_ON(!buf)) return -EINVAL; - ret = clk_prepare_enable(scp->clk); + ret = clk_enable(scp->clk); if (ret) { dev_err(scp->dev, "failed to enable clock\n"); return ret; @@ -211,7 +211,7 @@ int scp_ipi_send(struct mtk_scp *scp, u32 id, void *buf, unsigned int len, unlock_mutex: mutex_unlock(&scp->send_lock); - clk_disable_unprepare(scp->clk); + clk_disable(scp->clk); return ret; } diff --git a/drivers/remoteproc/qcom_sysmon.c b/drivers/remoteproc/qcom_sysmon.c index 660ac6fc408213..c6cc6e519fe566 100644 --- a/drivers/remoteproc/qcom_sysmon.c +++ b/drivers/remoteproc/qcom_sysmon.c @@ -203,7 +203,7 @@ static const struct qmi_elem_info ssctl_shutdown_resp_ei[] = { }; struct ssctl_subsys_event_req { - u8 subsys_name_len; + u32 subsys_name_len; char subsys_name[SSCTL_SUBSYS_NAME_LENGTH]; u32 event; u8 evt_driven_valid; diff --git a/drivers/remoteproc/qcom_wcnss.c b/drivers/remoteproc/qcom_wcnss.c index ee18bf2e80549e..4add9037dbd5a1 100644 --- a/drivers/remoteproc/qcom_wcnss.c +++ b/drivers/remoteproc/qcom_wcnss.c @@ -537,7 +537,7 @@ static int wcnss_alloc_memory_region(struct qcom_wcnss *wcnss) wcnss->mem_phys = wcnss->mem_reloc = res.start; wcnss->mem_size = resource_size(&res); - wcnss->mem_region = devm_ioremap_resource_wc(wcnss->dev, &res); + wcnss->mem_region = devm_ioremap_wc(wcnss->dev, wcnss->mem_phys, wcnss->mem_size); if (IS_ERR(wcnss->mem_region)) { dev_err(wcnss->dev, "unable to map memory region: %pR\n", &res); return PTR_ERR(wcnss->mem_region); diff --git a/drivers/resctrl/mpam_devices.c b/drivers/resctrl/mpam_devices.c index b495d52918681b..41fe4211718136 100644 --- a/drivers/resctrl/mpam_devices.c +++ b/drivers/resctrl/mpam_devices.c @@ -1428,6 +1428,7 @@ static void mpam_reprogram_ris_partid(struct mpam_msc_ris *ris, u16 partid, static int mpam_restore_mbwu_state(void *_ris) { int i; + u64 val; struct mon_read mwbu_arg; struct mpam_msc_ris *ris = _ris; struct mpam_class *class = ris->vmsc->comp->class; @@ -1437,6 +1438,7 @@ static int mpam_restore_mbwu_state(void *_ris) mwbu_arg.ris = ris; mwbu_arg.ctx = &ris->mbwu_state[i].cfg; mwbu_arg.type = mpam_msmon_choose_counter(class); + mwbu_arg.val = &val; __ris_msmon_read(&mwbu_arg); } diff --git a/drivers/reset/Kconfig b/drivers/reset/Kconfig index 6e5d6deffa7d36..52ee102621eef1 100644 --- a/drivers/reset/Kconfig +++ b/drivers/reset/Kconfig @@ -161,7 +161,7 @@ config RESET_K210 config RESET_K230 tristate "Reset controller driver for Canaan Kendryte K230 SoC" depends on ARCH_CANAAN || COMPILE_TEST - depends on OF + default ARCH_CANAAN help Support for the Canaan Kendryte K230 RISC-V SoC reset controller. Say Y if you want to control reset signals provided by this diff --git a/drivers/reset/core.c b/drivers/reset/core.c index 0135dd0ae20498..58ecde760b6ef0 100644 --- a/drivers/reset/core.c +++ b/drivers/reset/core.c @@ -856,7 +856,6 @@ static int reset_add_gpio_aux_device(struct device *parent, ret = __auxiliary_device_add(adev, "reset"); if (ret) { auxiliary_device_uninit(adev); - kfree(adev); return ret; } diff --git a/drivers/reset/reset-gpio.c b/drivers/reset/reset-gpio.c index e5512b3b596b52..626c4c639c1559 100644 --- a/drivers/reset/reset-gpio.c +++ b/drivers/reset/reset-gpio.c @@ -111,6 +111,7 @@ static struct auxiliary_driver reset_gpio_driver = { .id_table = reset_gpio_ids, .driver = { .name = "reset-gpio", + .suppress_bind_attrs = true, }, }; module_auxiliary_driver(reset_gpio_driver); diff --git a/drivers/rpmsg/rpmsg_core.c b/drivers/rpmsg/rpmsg_core.c index 5d661681a9b6ce..96964745065b1f 100644 --- a/drivers/rpmsg/rpmsg_core.c +++ b/drivers/rpmsg/rpmsg_core.c @@ -352,50 +352,38 @@ field##_show(struct device *dev, \ } \ static DEVICE_ATTR_RO(field); -#define rpmsg_string_attr(field, member) \ -static ssize_t \ -field##_store(struct device *dev, struct device_attribute *attr, \ - const char *buf, size_t sz) \ -{ \ - struct rpmsg_device *rpdev = to_rpmsg_device(dev); \ - const char *old; \ - char *new; \ - \ - new = kstrndup(buf, sz, GFP_KERNEL); \ - if (!new) \ - return -ENOMEM; \ - new[strcspn(new, "\n")] = '\0'; \ - \ - device_lock(dev); \ - old = rpdev->member; \ - if (strlen(new)) { \ - rpdev->member = new; \ - } else { \ - kfree(new); \ - rpdev->member = NULL; \ - } \ - device_unlock(dev); \ - \ - kfree(old); \ - \ - return sz; \ -} \ -static ssize_t \ -field##_show(struct device *dev, \ - struct device_attribute *attr, char *buf) \ -{ \ - struct rpmsg_device *rpdev = to_rpmsg_device(dev); \ - \ - return sprintf(buf, "%s\n", rpdev->member); \ -} \ -static DEVICE_ATTR_RW(field) - /* for more info, see Documentation/ABI/testing/sysfs-bus-rpmsg */ rpmsg_show_attr(name, id.name, "%s\n"); rpmsg_show_attr(src, src, "0x%x\n"); rpmsg_show_attr(dst, dst, "0x%x\n"); rpmsg_show_attr(announce, announce ? "true" : "false", "%s\n"); -rpmsg_string_attr(driver_override, driver_override); + +static ssize_t driver_override_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct rpmsg_device *rpdev = to_rpmsg_device(dev); + int ret; + + ret = driver_set_override(dev, &rpdev->driver_override, buf, count); + if (ret) + return ret; + + return count; +} + +static ssize_t driver_override_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct rpmsg_device *rpdev = to_rpmsg_device(dev); + ssize_t len; + + device_lock(dev); + len = sysfs_emit(buf, "%s\n", rpdev->driver_override); + device_unlock(dev); + return len; +} +static DEVICE_ATTR_RW(driver_override); static ssize_t modalias_show(struct device *dev, struct device_attribute *attr, char *buf) diff --git a/drivers/rtc/Kconfig b/drivers/rtc/Kconfig index 50dc779f7f9830..50ba48609d74ed 100644 --- a/drivers/rtc/Kconfig +++ b/drivers/rtc/Kconfig @@ -418,6 +418,7 @@ config RTC_DRV_SPACEMIT_P1 config RTC_DRV_NVIDIA_VRS10 tristate "NVIDIA VRS10 RTC device" + depends on ARCH_TEGRA || COMPILE_TEST help If you say yes here you will get support for the battery backed RTC device of NVIDIA VRS (Voltage Regulator Specification). The RTC is connected via diff --git a/drivers/rtc/interface.c b/drivers/rtc/interface.c index b8b298efd9a9c3..1906f4884a834c 100644 --- a/drivers/rtc/interface.c +++ b/drivers/rtc/interface.c @@ -457,7 +457,7 @@ static int __rtc_set_alarm(struct rtc_device *rtc, struct rtc_wkalrm *alarm) * are in, we can return -ETIME to signal that the timer has already * expired, which is true in both cases. */ - if ((scheduled - now) <= 1) { + if (!err && (scheduled - now) <= 1) { err = __rtc_read_time(rtc, &tm); if (err) return err; diff --git a/drivers/rtc/rtc-amlogic-a4.c b/drivers/rtc/rtc-amlogic-a4.c index 123fb372fc9fe0..50938c35af36a6 100644 --- a/drivers/rtc/rtc-amlogic-a4.c +++ b/drivers/rtc/rtc-amlogic-a4.c @@ -369,7 +369,7 @@ static int aml_rtc_probe(struct platform_device *pdev) return PTR_ERR(rtc->rtc_dev); ret = devm_request_irq(dev, rtc->irq, aml_rtc_handler, - IRQF_ONESHOT, "aml-rtc alarm", rtc); + 0, "aml-rtc alarm", rtc); if (ret) { dev_err_probe(dev, ret, "IRQ%d request failed, ret = %d\n", rtc->irq, ret); diff --git a/drivers/rtc/rtc-max31335.c b/drivers/rtc/rtc-max31335.c index 23b7bf16b4cd5d..952b455071d68b 100644 --- a/drivers/rtc/rtc-max31335.c +++ b/drivers/rtc/rtc-max31335.c @@ -591,7 +591,7 @@ static struct nvmem_config max31335_nvmem_cfg = { .size = MAX31335_RAM_SIZE, }; -#if IS_REACHABLE(HWMON) +#if IS_REACHABLE(CONFIG_HWMON) static int max31335_read_temp(struct device *dev, enum hwmon_sensor_types type, u32 attr, int channel, long *val) { @@ -672,7 +672,7 @@ static int max31335_clkout_register(struct device *dev) static int max31335_probe(struct i2c_client *client) { struct max31335_data *max31335; -#if IS_REACHABLE(HWMON) +#if IS_REACHABLE(CONFIG_HWMON) struct device *hwmon; #endif const struct chip_desc *match; @@ -727,7 +727,7 @@ static int max31335_probe(struct i2c_client *client) return dev_err_probe(&client->dev, ret, "cannot register rtc nvmem\n"); -#if IS_REACHABLE(HWMON) +#if IS_REACHABLE(CONFIG_HWMON) if (max31335->chip->temp_reg) { hwmon = devm_hwmon_device_register_with_info(&client->dev, client->name, max31335, &max31335_chip_info, NULL); diff --git a/drivers/rtc/rtc-pcf8563.c b/drivers/rtc/rtc-pcf8563.c index 4e61011fb7a967..b281e9489df1d4 100644 --- a/drivers/rtc/rtc-pcf8563.c +++ b/drivers/rtc/rtc-pcf8563.c @@ -424,7 +424,7 @@ static const struct clk_ops pcf8563_clkout_ops = { static struct clk *pcf8563_clkout_register_clk(struct pcf8563 *pcf8563) { - struct device_node *node = pcf8563->rtc->dev.of_node; + struct device_node *node = pcf8563->rtc->dev.parent->of_node; struct clk_init_data init; struct clk *clk; int ret; diff --git a/drivers/rtc/rtc-zynqmp.c b/drivers/rtc/rtc-zynqmp.c index 3baa2b481d9f20..856bc1678e7d31 100644 --- a/drivers/rtc/rtc-zynqmp.c +++ b/drivers/rtc/rtc-zynqmp.c @@ -345,7 +345,10 @@ static int xlnx_rtc_probe(struct platform_device *pdev) &xrtcdev->freq); if (ret) xrtcdev->freq = RTC_CALIB_DEF; + } else { + xrtcdev->freq--; } + ret = readl(xrtcdev->reg_base + RTC_CALIB_RD); if (!ret) writel(xrtcdev->freq, (xrtcdev->reg_base + RTC_CALIB_WR)); diff --git a/drivers/s390/block/dasd_eckd.c b/drivers/s390/block/dasd_eckd.c index b08e900687f393..c0f665a2929d9f 100644 --- a/drivers/s390/block/dasd_eckd.c +++ b/drivers/s390/block/dasd_eckd.c @@ -6135,6 +6135,7 @@ static void copy_pair_set_active(struct dasd_copy_relation *copy, char *new_busi static int dasd_eckd_copy_pair_swap(struct dasd_device *device, char *prim_busid, char *sec_busid) { + struct dasd_eckd_private *prim_priv, *sec_priv; struct dasd_device *primary, *secondary; struct dasd_copy_relation *copy; struct dasd_block *block; @@ -6155,6 +6156,9 @@ static int dasd_eckd_copy_pair_swap(struct dasd_device *device, char *prim_busid if (!secondary) return DASD_COPYPAIRSWAP_SECONDARY; + prim_priv = primary->private; + sec_priv = secondary->private; + /* * usually the device should be quiesced for swap * for paranoia stop device and requeue requests again @@ -6182,6 +6186,18 @@ static int dasd_eckd_copy_pair_swap(struct dasd_device *device, char *prim_busid dev_name(&secondary->cdev->dev), rc); } + if (primary->stopped & DASD_STOPPED_QUIESCE) { + dasd_device_set_stop_bits(secondary, DASD_STOPPED_QUIESCE); + dasd_device_remove_stop_bits(primary, DASD_STOPPED_QUIESCE); + } + + /* + * The secondary device never got through format detection, but since it + * is a copy of the primary device, the format is exactly the same; + * therefore, the detected layout can simply be copied. + */ + sec_priv->uses_cdl = prim_priv->uses_cdl; + /* re-enable device */ dasd_device_remove_stop_bits(primary, DASD_STOPPED_PPRC); dasd_device_remove_stop_bits(secondary, DASD_STOPPED_PPRC); diff --git a/drivers/s390/cio/css.c b/drivers/s390/cio/css.c index 4c85df7a548ef1..ac24e019020e89 100644 --- a/drivers/s390/cio/css.c +++ b/drivers/s390/cio/css.c @@ -235,7 +235,7 @@ struct subchannel *css_alloc_subchannel(struct subchannel_id schid, return sch; err: - kfree(sch); + put_device(&sch->dev); return ERR_PTR(ret); } diff --git a/drivers/s390/crypto/zcrypt_ccamisc.c b/drivers/s390/crypto/zcrypt_ccamisc.c index 573bad1d6d86c7..37a157a1d9691d 100644 --- a/drivers/s390/crypto/zcrypt_ccamisc.c +++ b/drivers/s390/crypto/zcrypt_ccamisc.c @@ -1639,11 +1639,13 @@ int cca_get_info(u16 cardnr, u16 domain, struct cca_info *ci, u32 xflags) memset(ci, 0, sizeof(*ci)); - /* get first info from zcrypt device driver about this apqn */ - rc = zcrypt_device_status_ext(cardnr, domain, &devstat); - if (rc) - return rc; - ci->hwtype = devstat.hwtype; + /* if specific domain given, fetch status and hw info for this apqn */ + if (domain != AUTOSEL_DOM) { + rc = zcrypt_device_status_ext(cardnr, domain, &devstat); + if (rc) + return rc; + ci->hwtype = devstat.hwtype; + } /* * Prep memory for rule array and var array use. diff --git a/drivers/s390/crypto/zcrypt_cex4.c b/drivers/s390/crypto/zcrypt_cex4.c index 6ba7fbddd3f7c6..8aa78f4153363f 100644 --- a/drivers/s390/crypto/zcrypt_cex4.c +++ b/drivers/s390/crypto/zcrypt_cex4.c @@ -84,8 +84,7 @@ static ssize_t cca_serialnr_show(struct device *dev, memset(&ci, 0, sizeof(ci)); - if (ap_domain_index >= 0) - cca_get_info(ac->id, ap_domain_index, &ci, 0); + cca_get_info(ac->id, AUTOSEL_DOM, &ci, 0); return sysfs_emit(buf, "%s\n", ci.serial); } diff --git a/drivers/s390/crypto/zcrypt_msgtype6.c b/drivers/s390/crypto/zcrypt_msgtype6.c index a0dcab5dc4f2f7..23a32221e41a77 100644 --- a/drivers/s390/crypto/zcrypt_msgtype6.c +++ b/drivers/s390/crypto/zcrypt_msgtype6.c @@ -953,6 +953,10 @@ static atomic_t zcrypt_step = ATOMIC_INIT(0); /* * The request distributor calls this function if it picked the CEXxC * device to handle a modexpo request. + * This function assumes that ap_msg has been initialized with + * ap_init_apmsg() and thus a valid buffer with the size of + * ap_msg->bufsize is available within ap_msg. Also the caller has + * to make sure ap_release_apmsg() is always called even on failure. * @zq: pointer to zcrypt_queue structure that identifies the * CEXxC device to the request distributor * @mex: pointer to the modexpo request buffer @@ -964,21 +968,17 @@ static long zcrypt_msgtype6_modexpo(struct zcrypt_queue *zq, struct ap_response_type *resp_type = &ap_msg->response; int rc; - ap_msg->msg = (void *)get_zeroed_page(GFP_KERNEL); - if (!ap_msg->msg) - return -ENOMEM; - ap_msg->bufsize = PAGE_SIZE; ap_msg->receive = zcrypt_msgtype6_receive; ap_msg->psmid = (((unsigned long)current->pid) << 32) + atomic_inc_return(&zcrypt_step); rc = icamex_msg_to_type6mex_msgx(zq, ap_msg, mex); if (rc) - goto out_free; + goto out; resp_type->type = CEXXC_RESPONSE_TYPE_ICA; init_completion(&resp_type->work); rc = ap_queue_message(zq->queue, ap_msg); if (rc) - goto out_free; + goto out; rc = wait_for_completion_interruptible(&resp_type->work); if (rc == 0) { rc = ap_msg->rc; @@ -991,15 +991,17 @@ static long zcrypt_msgtype6_modexpo(struct zcrypt_queue *zq, ap_cancel_message(zq->queue, ap_msg); } -out_free: - free_page((unsigned long)ap_msg->msg); - ap_msg->msg = NULL; +out: return rc; } /* * The request distributor calls this function if it picked the CEXxC * device to handle a modexpo_crt request. + * This function assumes that ap_msg has been initialized with + * ap_init_apmsg() and thus a valid buffer with the size of + * ap_msg->bufsize is available within ap_msg. Also the caller has + * to make sure ap_release_apmsg() is always called even on failure. * @zq: pointer to zcrypt_queue structure that identifies the * CEXxC device to the request distributor * @crt: pointer to the modexpoc_crt request buffer @@ -1011,21 +1013,17 @@ static long zcrypt_msgtype6_modexpo_crt(struct zcrypt_queue *zq, struct ap_response_type *resp_type = &ap_msg->response; int rc; - ap_msg->msg = (void *)get_zeroed_page(GFP_KERNEL); - if (!ap_msg->msg) - return -ENOMEM; - ap_msg->bufsize = PAGE_SIZE; ap_msg->receive = zcrypt_msgtype6_receive; ap_msg->psmid = (((unsigned long)current->pid) << 32) + atomic_inc_return(&zcrypt_step); rc = icacrt_msg_to_type6crt_msgx(zq, ap_msg, crt); if (rc) - goto out_free; + goto out; resp_type->type = CEXXC_RESPONSE_TYPE_ICA; init_completion(&resp_type->work); rc = ap_queue_message(zq->queue, ap_msg); if (rc) - goto out_free; + goto out; rc = wait_for_completion_interruptible(&resp_type->work); if (rc == 0) { rc = ap_msg->rc; @@ -1038,9 +1036,7 @@ static long zcrypt_msgtype6_modexpo_crt(struct zcrypt_queue *zq, ap_cancel_message(zq->queue, ap_msg); } -out_free: - free_page((unsigned long)ap_msg->msg); - ap_msg->msg = NULL; +out: return rc; } diff --git a/drivers/scsi/BusLogic.c b/drivers/scsi/BusLogic.c index a86d780d1ba40c..026c3e617cb1c0 100644 --- a/drivers/scsi/BusLogic.c +++ b/drivers/scsi/BusLogic.c @@ -920,7 +920,8 @@ static int __init blogic_init_fp_probeinfo(struct blogic_adapter *adapter) a particular probe order. */ -static void __init blogic_init_probeinfo_list(struct blogic_adapter *adapter) +static noinline_for_stack void __init +blogic_init_probeinfo_list(struct blogic_adapter *adapter) { /* If a PCI BIOS is present, interrogate it for MultiMaster and @@ -1690,7 +1691,8 @@ static bool __init blogic_rdconfig(struct blogic_adapter *adapter) blogic_reportconfig reports the configuration of Host Adapter. */ -static bool __init blogic_reportconfig(struct blogic_adapter *adapter) +static noinline_for_stack bool __init +blogic_reportconfig(struct blogic_adapter *adapter) { unsigned short alltgt_mask = (1 << adapter->maxdev) - 1; unsigned short sync_ok, fast_ok; diff --git a/drivers/scsi/csiostor/csio_scsi.c b/drivers/scsi/csiostor/csio_scsi.c index 34bde6650fae0f..356a7c577ec3ef 100644 --- a/drivers/scsi/csiostor/csio_scsi.c +++ b/drivers/scsi/csiostor/csio_scsi.c @@ -2074,7 +2074,7 @@ csio_eh_lun_reset_handler(struct scsi_cmnd *cmnd) struct csio_scsi_level_data sld; if (!rn) - goto fail; + goto fail_ret; csio_dbg(hw, "Request to reset LUN:%llu (ssni:0x%x tgtid:%d)\n", cmnd->device->lun, rn->flowid, rn->scsi_id); @@ -2220,6 +2220,7 @@ csio_eh_lun_reset_handler(struct scsi_cmnd *cmnd) csio_put_scsi_ioreq_lock(hw, scsim, ioreq); fail: CSIO_INC_STATS(rn, n_lun_rst_fail); +fail_ret: return FAILED; } diff --git a/drivers/scsi/elx/efct/efct_driver.c b/drivers/scsi/elx/efct/efct_driver.c index 1bd42f7db1773d..528399f725d428 100644 --- a/drivers/scsi/elx/efct/efct_driver.c +++ b/drivers/scsi/elx/efct/efct_driver.c @@ -415,12 +415,6 @@ efct_intr_thread(int irq, void *handle) return IRQ_HANDLED; } -static irqreturn_t -efct_intr_msix(int irq, void *handle) -{ - return IRQ_WAKE_THREAD; -} - static int efct_setup_msix(struct efct *efct, u32 num_intrs) { @@ -450,7 +444,7 @@ efct_setup_msix(struct efct *efct, u32 num_intrs) intr_ctx->index = i; rc = request_threaded_irq(pci_irq_vector(efct->pci, i), - efct_intr_msix, efct_intr_thread, 0, + NULL, efct_intr_thread, IRQF_ONESHOT, EFCT_DRIVER_NAME, intr_ctx); if (rc) { dev_err(&efct->pci->dev, diff --git a/drivers/scsi/hisi_sas/hisi_sas_main.c b/drivers/scsi/hisi_sas/hisi_sas_main.c index 30a9c66126513a..c2b082f1252c3e 100644 --- a/drivers/scsi/hisi_sas/hisi_sas_main.c +++ b/drivers/scsi/hisi_sas/hisi_sas_main.c @@ -2578,7 +2578,7 @@ int hisi_sas_probe(struct platform_device *pdev, shost->transportt = hisi_sas_stt; shost->max_id = HISI_SAS_MAX_DEVICES; shost->max_lun = ~0; - shost->max_channel = 1; + shost->max_channel = 0; shost->max_cmd_len = HISI_SAS_MAX_CDB_LEN; if (hisi_hba->hw->slot_index_alloc) { shost->can_queue = HISI_SAS_MAX_COMMANDS; diff --git a/drivers/scsi/hisi_sas/hisi_sas_v3_hw.c b/drivers/scsi/hisi_sas/hisi_sas_v3_hw.c index 2f9e01717ef389..f69efc6494b8e2 100644 --- a/drivers/scsi/hisi_sas/hisi_sas_v3_hw.c +++ b/drivers/scsi/hisi_sas/hisi_sas_v3_hw.c @@ -4993,7 +4993,7 @@ hisi_sas_v3_probe(struct pci_dev *pdev, const struct pci_device_id *id) shost->transportt = hisi_sas_stt; shost->max_id = HISI_SAS_MAX_DEVICES; shost->max_lun = ~0; - shost->max_channel = 1; + shost->max_channel = 0; shost->max_cmd_len = HISI_SAS_MAX_CDB_LEN; shost->can_queue = HISI_SAS_UNRESERVED_IPTT; shost->cmd_per_lun = HISI_SAS_UNRESERVED_IPTT; diff --git a/drivers/scsi/ibmvscsi/ibmvfc.c b/drivers/scsi/ibmvscsi/ibmvfc.c index 228daffb286dec..8d7c304636d990 100644 --- a/drivers/scsi/ibmvscsi/ibmvfc.c +++ b/drivers/scsi/ibmvscsi/ibmvfc.c @@ -4965,7 +4965,8 @@ static void ibmvfc_discover_targets_done(struct ibmvfc_event *evt) switch (mad_status) { case IBMVFC_MAD_SUCCESS: ibmvfc_dbg(vhost, "Discover Targets succeeded\n"); - vhost->num_targets = be32_to_cpu(rsp->num_written); + vhost->num_targets = min_t(u32, be32_to_cpu(rsp->num_written), + max_targets); ibmvfc_set_host_action(vhost, IBMVFC_HOST_ACTION_ALLOC_TGTS); break; case IBMVFC_MAD_FAILED: diff --git a/drivers/scsi/lpfc/lpfc_init.c b/drivers/scsi/lpfc/lpfc_init.c index b1460b16dd91d9..c6bb45c3d4c4a6 100644 --- a/drivers/scsi/lpfc/lpfc_init.c +++ b/drivers/scsi/lpfc/lpfc_init.c @@ -12034,6 +12034,8 @@ lpfc_sli4_pci_mem_unset(struct lpfc_hba *phba) iounmap(phba->sli4_hba.conf_regs_memmap_p); if (phba->sli4_hba.dpp_regs_memmap_p) iounmap(phba->sli4_hba.dpp_regs_memmap_p); + if (phba->sli4_hba.dpp_regs_memmap_wc_p) + iounmap(phba->sli4_hba.dpp_regs_memmap_wc_p); break; case LPFC_SLI_INTF_IF_TYPE_1: break; diff --git a/drivers/scsi/lpfc/lpfc_sli.c b/drivers/scsi/lpfc/lpfc_sli.c index 73d77cfab5f820..bddfc412b04b5b 100644 --- a/drivers/scsi/lpfc/lpfc_sli.c +++ b/drivers/scsi/lpfc/lpfc_sli.c @@ -15981,6 +15981,32 @@ lpfc_dual_chute_pci_bar_map(struct lpfc_hba *phba, uint16_t pci_barset) return NULL; } +static __maybe_unused void __iomem * +lpfc_dpp_wc_map(struct lpfc_hba *phba, uint8_t dpp_barset) +{ + + /* DPP region is supposed to cover 64-bit BAR2 */ + if (dpp_barset != WQ_PCI_BAR_4_AND_5) { + lpfc_log_msg(phba, KERN_WARNING, LOG_INIT, + "3273 dpp_barset x%x != WQ_PCI_BAR_4_AND_5\n", + dpp_barset); + return NULL; + } + + if (!phba->sli4_hba.dpp_regs_memmap_wc_p) { + void __iomem *dpp_map; + + dpp_map = ioremap_wc(phba->pci_bar2_map, + pci_resource_len(phba->pcidev, + PCI_64BIT_BAR4)); + + if (dpp_map) + phba->sli4_hba.dpp_regs_memmap_wc_p = dpp_map; + } + + return phba->sli4_hba.dpp_regs_memmap_wc_p; +} + /** * lpfc_modify_hba_eq_delay - Modify Delay Multiplier on EQs * @phba: HBA structure that EQs are on. @@ -16944,9 +16970,6 @@ lpfc_wq_create(struct lpfc_hba *phba, struct lpfc_queue *wq, uint8_t dpp_barset; uint32_t dpp_offset; uint8_t wq_create_version; -#ifdef CONFIG_X86 - unsigned long pg_addr; -#endif /* sanity check on queue memory */ if (!wq || !cq) @@ -17132,14 +17155,15 @@ lpfc_wq_create(struct lpfc_hba *phba, struct lpfc_queue *wq, #ifdef CONFIG_X86 /* Enable combined writes for DPP aperture */ - pg_addr = (unsigned long)(wq->dpp_regaddr) & PAGE_MASK; - rc = set_memory_wc(pg_addr, 1); - if (rc) { + bar_memmap_p = lpfc_dpp_wc_map(phba, dpp_barset); + if (!bar_memmap_p) { lpfc_printf_log(phba, KERN_ERR, LOG_INIT, "3272 Cannot setup Combined " "Write on WQ[%d] - disable DPP\n", wq->queue_id); phba->cfg_enable_dpp = 0; + } else { + wq->dpp_regaddr = bar_memmap_p + dpp_offset; } #else phba->cfg_enable_dpp = 0; diff --git a/drivers/scsi/lpfc/lpfc_sli4.h b/drivers/scsi/lpfc/lpfc_sli4.h index fd6dab1578872f..40f313e2769fc5 100644 --- a/drivers/scsi/lpfc/lpfc_sli4.h +++ b/drivers/scsi/lpfc/lpfc_sli4.h @@ -785,6 +785,9 @@ struct lpfc_sli4_hba { void __iomem *dpp_regs_memmap_p; /* Kernel memory mapped address for * dpp registers */ + void __iomem *dpp_regs_memmap_wc_p;/* Kernel memory mapped address for + * dpp registers with write combining + */ union { struct { /* IF Type 0, BAR 0 PCI cfg space reg mem map */ diff --git a/drivers/scsi/mpi3mr/mpi3mr_fw.c b/drivers/scsi/mpi3mr/mpi3mr_fw.c index 8c4bb7169a87c3..4c8d78b840fc99 100644 --- a/drivers/scsi/mpi3mr/mpi3mr_fw.c +++ b/drivers/scsi/mpi3mr/mpi3mr_fw.c @@ -1530,6 +1530,7 @@ static int mpi3mr_bring_ioc_ready(struct mpi3mr_ioc *mrioc) ioc_info(mrioc, "successfully transitioned to %s state\n", mpi3mr_iocstate_name(ioc_state)); + mpi3mr_clear_reset_history(mrioc); return 0; } ioc_status = readl(&mrioc->sysif_regs->ioc_status); @@ -1549,6 +1550,15 @@ static int mpi3mr_bring_ioc_ready(struct mpi3mr_ioc *mrioc) elapsed_time_sec = jiffies_to_msecs(jiffies - start_time)/1000; } while (elapsed_time_sec < mrioc->ready_timeout); + ioc_state = mpi3mr_get_iocstate(mrioc); + if (ioc_state == MRIOC_STATE_READY) { + ioc_info(mrioc, + "successfully transitioned to %s state after %llu seconds\n", + mpi3mr_iocstate_name(ioc_state), elapsed_time_sec); + mpi3mr_clear_reset_history(mrioc); + return 0; + } + out_failed: elapsed_time_sec = jiffies_to_msecs(jiffies - start_time)/1000; if ((retry < 2) && (elapsed_time_sec < (mrioc->ready_timeout - 60))) { @@ -4705,21 +4715,25 @@ void mpi3mr_memset_buffers(struct mpi3mr_ioc *mrioc) } for (i = 0; i < mrioc->num_queues; i++) { - mrioc->op_reply_qinfo[i].qid = 0; - mrioc->op_reply_qinfo[i].ci = 0; - mrioc->op_reply_qinfo[i].num_replies = 0; - mrioc->op_reply_qinfo[i].ephase = 0; - atomic_set(&mrioc->op_reply_qinfo[i].pend_ios, 0); - atomic_set(&mrioc->op_reply_qinfo[i].in_use, 0); - mpi3mr_memset_op_reply_q_buffers(mrioc, i); - - mrioc->req_qinfo[i].ci = 0; - mrioc->req_qinfo[i].pi = 0; - mrioc->req_qinfo[i].num_requests = 0; - mrioc->req_qinfo[i].qid = 0; - mrioc->req_qinfo[i].reply_qid = 0; - spin_lock_init(&mrioc->req_qinfo[i].q_lock); - mpi3mr_memset_op_req_q_buffers(mrioc, i); + if (mrioc->op_reply_qinfo) { + mrioc->op_reply_qinfo[i].qid = 0; + mrioc->op_reply_qinfo[i].ci = 0; + mrioc->op_reply_qinfo[i].num_replies = 0; + mrioc->op_reply_qinfo[i].ephase = 0; + atomic_set(&mrioc->op_reply_qinfo[i].pend_ios, 0); + atomic_set(&mrioc->op_reply_qinfo[i].in_use, 0); + mpi3mr_memset_op_reply_q_buffers(mrioc, i); + } + + if (mrioc->req_qinfo) { + mrioc->req_qinfo[i].ci = 0; + mrioc->req_qinfo[i].pi = 0; + mrioc->req_qinfo[i].num_requests = 0; + mrioc->req_qinfo[i].qid = 0; + mrioc->req_qinfo[i].reply_qid = 0; + spin_lock_init(&mrioc->req_qinfo[i].q_lock); + mpi3mr_memset_op_req_q_buffers(mrioc, i); + } } atomic_set(&mrioc->pend_large_data_sz, 0); diff --git a/drivers/scsi/pm8001/pm8001_sas.c b/drivers/scsi/pm8001/pm8001_sas.c index 6a8d35aea93a55..645524f3fe2d06 100644 --- a/drivers/scsi/pm8001/pm8001_sas.c +++ b/drivers/scsi/pm8001/pm8001_sas.c @@ -525,8 +525,9 @@ int pm8001_queue_command(struct sas_task *task, gfp_t gfp_flags) } else { task->task_done(task); } - rc = -ENODEV; - goto err_out; + spin_unlock_irqrestore(&pm8001_ha->lock, flags); + pm8001_dbg(pm8001_ha, IO, "pm8001_task_exec device gone\n"); + return 0; } ccb = pm8001_ccb_alloc(pm8001_ha, pm8001_dev, task); diff --git a/drivers/scsi/qla2xxx/qla_bsg.c b/drivers/scsi/qla2xxx/qla_bsg.c index ccfc2d26dd3725..0798bfd0372e47 100644 --- a/drivers/scsi/qla2xxx/qla_bsg.c +++ b/drivers/scsi/qla2xxx/qla_bsg.c @@ -1546,8 +1546,9 @@ qla2x00_update_optrom(struct bsg_job *bsg_job) ha->optrom_buffer = NULL; ha->optrom_state = QLA_SWAITING; mutex_unlock(&ha->optrom_mutex); - bsg_job_done(bsg_job, bsg_reply->result, - bsg_reply->reply_payload_rcv_len); + if (!rval) + bsg_job_done(bsg_job, bsg_reply->result, + bsg_reply->reply_payload_rcv_len); return rval; } @@ -2612,8 +2613,9 @@ qla2x00_manage_host_stats(struct bsg_job *bsg_job) sizeof(struct ql_vnd_mng_host_stats_resp)); bsg_reply->result = DID_OK; - bsg_job_done(bsg_job, bsg_reply->result, - bsg_reply->reply_payload_rcv_len); + if (!ret) + bsg_job_done(bsg_job, bsg_reply->result, + bsg_reply->reply_payload_rcv_len); return ret; } @@ -2702,8 +2704,9 @@ qla2x00_get_host_stats(struct bsg_job *bsg_job) bsg_job->reply_payload.sg_cnt, data, response_len); bsg_reply->result = DID_OK; - bsg_job_done(bsg_job, bsg_reply->result, - bsg_reply->reply_payload_rcv_len); + if (!ret) + bsg_job_done(bsg_job, bsg_reply->result, + bsg_reply->reply_payload_rcv_len); kfree(data); host_stat_out: @@ -2802,8 +2805,9 @@ qla2x00_get_tgt_stats(struct bsg_job *bsg_job) bsg_job->reply_payload.sg_cnt, data, response_len); bsg_reply->result = DID_OK; - bsg_job_done(bsg_job, bsg_reply->result, - bsg_reply->reply_payload_rcv_len); + if (!ret) + bsg_job_done(bsg_job, bsg_reply->result, + bsg_reply->reply_payload_rcv_len); tgt_stat_out: kfree(data); @@ -2864,8 +2868,9 @@ qla2x00_manage_host_port(struct bsg_job *bsg_job) bsg_job->reply_payload.sg_cnt, &rsp_data, sizeof(struct ql_vnd_mng_host_port_resp)); bsg_reply->result = DID_OK; - bsg_job_done(bsg_job, bsg_reply->result, - bsg_reply->reply_payload_rcv_len); + if (!ret) + bsg_job_done(bsg_job, bsg_reply->result, + bsg_reply->reply_payload_rcv_len); return ret; } @@ -3240,7 +3245,8 @@ int qla2x00_mailbox_passthru(struct bsg_job *bsg_job) bsg_job->reply_len = sizeof(*bsg_job->reply); bsg_reply->result = DID_OK << 16; - bsg_job_done(bsg_job, bsg_reply->result, bsg_reply->reply_payload_rcv_len); + if (!ret) + bsg_job_done(bsg_job, bsg_reply->result, bsg_reply->reply_payload_rcv_len); kfree(req_data); diff --git a/drivers/scsi/qla2xxx/qla_gs.c b/drivers/scsi/qla2xxx/qla_gs.c index 51c7cea71f9022..880cd73feaca45 100644 --- a/drivers/scsi/qla2xxx/qla_gs.c +++ b/drivers/scsi/qla2xxx/qla_gs.c @@ -3266,9 +3266,6 @@ void qla_fab_scan_finish(scsi_qla_host_t *vha, srb_t *sp) atomic_read(&fcport->state) == FCS_ONLINE) || do_delete) { if (fcport->loop_id != FC_NO_LOOP_ID) { - if (fcport->flags & FCF_FCP2_DEVICE) - continue; - ql_log(ql_log_warn, vha, 0x20f0, "%s %d %8phC post del sess\n", __func__, __LINE__, @@ -3535,8 +3532,8 @@ int qla_fab_async_scan(scsi_qla_host_t *vha, srb_t *sp) if (vha->scan.scan_flags & SF_SCANNING) { spin_unlock_irqrestore(&vha->work_lock, flags); ql_dbg(ql_dbg_disc + ql_dbg_verbose, vha, 0x2012, - "%s: scan active\n", __func__); - return rval; + "%s: scan active for sp:%p\n", __func__, sp); + goto done_free_sp; } vha->scan.scan_flags |= SF_SCANNING; if (!sp) @@ -3701,23 +3698,25 @@ int qla_fab_async_scan(scsi_qla_host_t *vha, srb_t *sp) return rval; done_free_sp: - if (sp->u.iocb_cmd.u.ctarg.req) { - dma_free_coherent(&vha->hw->pdev->dev, - sp->u.iocb_cmd.u.ctarg.req_allocated_size, - sp->u.iocb_cmd.u.ctarg.req, - sp->u.iocb_cmd.u.ctarg.req_dma); - sp->u.iocb_cmd.u.ctarg.req = NULL; - } - if (sp->u.iocb_cmd.u.ctarg.rsp) { - dma_free_coherent(&vha->hw->pdev->dev, - sp->u.iocb_cmd.u.ctarg.rsp_allocated_size, - sp->u.iocb_cmd.u.ctarg.rsp, - sp->u.iocb_cmd.u.ctarg.rsp_dma); - sp->u.iocb_cmd.u.ctarg.rsp = NULL; - } + if (sp) { + if (sp->u.iocb_cmd.u.ctarg.req) { + dma_free_coherent(&vha->hw->pdev->dev, + sp->u.iocb_cmd.u.ctarg.req_allocated_size, + sp->u.iocb_cmd.u.ctarg.req, + sp->u.iocb_cmd.u.ctarg.req_dma); + sp->u.iocb_cmd.u.ctarg.req = NULL; + } + if (sp->u.iocb_cmd.u.ctarg.rsp) { + dma_free_coherent(&vha->hw->pdev->dev, + sp->u.iocb_cmd.u.ctarg.rsp_allocated_size, + sp->u.iocb_cmd.u.ctarg.rsp, + sp->u.iocb_cmd.u.ctarg.rsp_dma); + sp->u.iocb_cmd.u.ctarg.rsp = NULL; + } - /* ref: INIT */ - kref_put(&sp->cmd_kref, qla2x00_sp_release); + /* ref: INIT */ + kref_put(&sp->cmd_kref, qla2x00_sp_release); + } spin_lock_irqsave(&vha->work_lock, flags); vha->scan.scan_flags &= ~SF_SCANNING; diff --git a/drivers/scsi/qla2xxx/qla_init.c b/drivers/scsi/qla2xxx/qla_init.c index d395cbfe6802e9..80d66ad324ba7e 100644 --- a/drivers/scsi/qla2xxx/qla_init.c +++ b/drivers/scsi/qla2xxx/qla_init.c @@ -1859,15 +1859,6 @@ void qla2x00_handle_rscn(scsi_qla_host_t *vha, struct event_arg *ea) case RSCN_PORT_ADDR: fcport = qla2x00_find_fcport_by_nportid(vha, &ea->id, 1); if (fcport) { - if (ql2xfc2target && - fcport->flags & FCF_FCP2_DEVICE && - atomic_read(&fcport->state) == FCS_ONLINE) { - ql_dbg(ql_dbg_disc, vha, 0x2115, - "Delaying session delete for FCP2 portid=%06x %8phC ", - fcport->d_id.b24, fcport->port_name); - return; - } - if (vha->hw->flags.edif_enabled && DBELL_ACTIVE(vha)) { /* * On ipsec start by remote port, Target port @@ -2471,8 +2462,23 @@ qla24xx_handle_plogi_done_event(struct scsi_qla_host *vha, struct event_arg *ea) ea->sp->gen1, fcport->rscn_gen, ea->data[0], ea->data[1], ea->iop[0], ea->iop[1]); - if ((fcport->fw_login_state == DSC_LS_PLOGI_PEND) || - (fcport->fw_login_state == DSC_LS_PRLI_PEND)) { + if (fcport->fw_login_state == DSC_LS_PLOGI_PEND) { + ql_dbg(ql_dbg_disc, vha, 0x20ea, + "%s %d %8phC Remote is trying to login\n", + __func__, __LINE__, fcport->port_name); + /* + * If we get here, there is port thats already logged in, + * but it's state has not moved ahead. Recheck with FW on + * what state it is in and proceed ahead + */ + if (!N2N_TOPO(vha->hw)) { + fcport->fw_login_state = DSC_LS_PRLI_COMP; + qla24xx_post_gpdb_work(vha, fcport, 0); + } + return; + } + + if (fcport->fw_login_state == DSC_LS_PRLI_PEND) { ql_dbg(ql_dbg_disc, vha, 0x20ea, "%s %d %8phC Remote is trying to login\n", __func__, __LINE__, fcport->port_name); diff --git a/drivers/scsi/qla2xxx/qla_iocb.c b/drivers/scsi/qla2xxx/qla_iocb.c index 3224044f177539..0de015de7eb592 100644 --- a/drivers/scsi/qla2xxx/qla_iocb.c +++ b/drivers/scsi/qla2xxx/qla_iocb.c @@ -2751,7 +2751,6 @@ qla24xx_els_dcmd_iocb(scsi_qla_host_t *vha, int els_opcode, if (!elsio->u.els_logo.els_logo_pyld) { /* ref: INIT */ kref_put(&sp->cmd_kref, qla2x00_sp_release); - qla2x00_free_fcport(fcport); return QLA_FUNCTION_FAILED; } @@ -2776,7 +2775,6 @@ qla24xx_els_dcmd_iocb(scsi_qla_host_t *vha, int els_opcode, if (rval != QLA_SUCCESS) { /* ref: INIT */ kref_put(&sp->cmd_kref, qla2x00_sp_release); - qla2x00_free_fcport(fcport); return QLA_FUNCTION_FAILED; } diff --git a/drivers/scsi/qla2xxx/qla_isr.c b/drivers/scsi/qla2xxx/qla_isr.c index a04a5aa0d00572..608d2f36e7b4ff 100644 --- a/drivers/scsi/qla2xxx/qla_isr.c +++ b/drivers/scsi/qla2xxx/qla_isr.c @@ -1676,13 +1676,28 @@ qla2x00_async_event(scsi_qla_host_t *vha, struct rsp_que *rsp, uint16_t *mb) /* Port logout */ fcport = qla2x00_find_fcport_by_loopid(vha, mb[1]); - if (!fcport) + if (!fcport) { + ql_dbg(ql_dbg_async, vha, 0x5011, + "Could not find fcport:%04x %04x %04x\n", + mb[1], mb[2], mb[3]); break; - if (atomic_read(&fcport->state) != FCS_ONLINE) + } + + if (atomic_read(&fcport->state) != FCS_ONLINE) { + ql_dbg(ql_dbg_async, vha, 0x5012, + "Port state is not online State:0x%x \n", + atomic_read(&fcport->state)); + ql_dbg(ql_dbg_async, vha, 0x5012, + "Scheduling session for deletion \n"); + fcport->logout_on_delete = 0; + qlt_schedule_sess_for_deletion(fcport); break; + } + ql_dbg(ql_dbg_async, vha, 0x508a, "Marking port lost loopid=%04x portid=%06x.\n", fcport->loop_id, fcport->d_id.b24); + if (qla_ini_mode_enabled(vha)) { fcport->logout_on_delete = 0; qlt_schedule_sess_for_deletion(fcport); diff --git a/drivers/scsi/qla2xxx/qla_os.c b/drivers/scsi/qla2xxx/qla_os.c index e939bc88e15197..35fb3e5f4e9f32 100644 --- a/drivers/scsi/qla2xxx/qla_os.c +++ b/drivers/scsi/qla2xxx/qla_os.c @@ -1183,7 +1183,8 @@ qla2x00_wait_for_hba_ready(scsi_qla_host_t *vha) while ((qla2x00_reset_active(vha) || ha->dpc_active || ha->flags.mbox_busy) || test_bit(FX00_RESET_RECOVERY, &vha->dpc_flags) || - test_bit(FX00_TARGET_SCAN, &vha->dpc_flags)) { + test_bit(FX00_TARGET_SCAN, &vha->dpc_flags) || + (vha->scan.scan_flags & SF_SCANNING)) { if (test_bit(UNLOADING, &base_vha->dpc_flags)) break; msleep(1000); diff --git a/drivers/scsi/scsi_devinfo.c b/drivers/scsi/scsi_devinfo.c index 78346b2b69c91c..c51146882a1fa9 100644 --- a/drivers/scsi/scsi_devinfo.c +++ b/drivers/scsi/scsi_devinfo.c @@ -190,7 +190,7 @@ static struct { {"IBM", "2076", NULL, BLIST_NO_VPD_SIZE}, {"IBM", "2105", NULL, BLIST_RETRY_HWERROR}, {"iomega", "jaz 1GB", "J.86", BLIST_NOTQ | BLIST_NOLUN}, - {"IOMEGA", "ZIP", NULL, BLIST_NOTQ | BLIST_NOLUN}, + {"IOMEGA", "ZIP", NULL, BLIST_NOTQ | BLIST_NOLUN | BLIST_SKIP_IO_HINTS}, {"IOMEGA", "Io20S *F", NULL, BLIST_KEY}, {"INSITE", "Floptical F*8I", NULL, BLIST_KEY}, {"INSITE", "I325VM", NULL, BLIST_KEY}, diff --git a/drivers/scsi/scsi_scan.c b/drivers/scsi/scsi_scan.c index 7acbfcfc2172eb..c8762323c54d04 100644 --- a/drivers/scsi/scsi_scan.c +++ b/drivers/scsi/scsi_scan.c @@ -360,11 +360,8 @@ static struct scsi_device *scsi_alloc_sdev(struct scsi_target *starget, * default device queue depth to figure out sbitmap shift * since we use this queue depth most of times. */ - if (scsi_realloc_sdev_budget_map(sdev, depth)) { - put_device(&starget->dev); - kfree(sdev); - goto out; - } + if (scsi_realloc_sdev_budget_map(sdev, depth)) + goto out_device_destroy; scsi_change_queue_depth(sdev, depth); diff --git a/drivers/scsi/scsi_transport_sas.c b/drivers/scsi/scsi_transport_sas.c index d69c7c444a3116..081c1680943740 100644 --- a/drivers/scsi/scsi_transport_sas.c +++ b/drivers/scsi/scsi_transport_sas.c @@ -1734,7 +1734,7 @@ static int sas_user_scan(struct Scsi_Host *shost, uint channel, break; default: - if (channel < shost->max_channel) { + if (channel <= shost->max_channel) { res = scsi_scan_host_selected(shost, channel, id, lun, SCSI_SCAN_MANUAL); } else { diff --git a/drivers/scsi/ses.c b/drivers/scsi/ses.c index 2c61624cb4b038..7e1f085ad350a1 100644 --- a/drivers/scsi/ses.c +++ b/drivers/scsi/ses.c @@ -216,7 +216,7 @@ static unsigned char *ses_get_page2_descriptor(struct enclosure_device *edev, unsigned char *type_ptr = ses_dev->page1_types; unsigned char *desc_ptr = ses_dev->page2 + 8; - if (ses_recv_diag(sdev, 2, ses_dev->page2, ses_dev->page2_len) < 0) + if (ses_recv_diag(sdev, 2, ses_dev->page2, ses_dev->page2_len)) return NULL; for (i = 0; i < ses_dev->page1_num_types; i++, type_ptr += 4) { @@ -529,9 +529,8 @@ struct efd { }; static int ses_enclosure_find_by_addr(struct enclosure_device *edev, - void *data) + struct efd *efd) { - struct efd *efd = data; int i; struct ses_component *scomp; @@ -684,7 +683,7 @@ static void ses_match_to_enclosure(struct enclosure_device *edev, if (efd.addr) { efd.dev = &sdev->sdev_gendev; - enclosure_for_each_device(ses_enclosure_find_by_addr, &efd); + ses_enclosure_find_by_addr(edev, &efd); } } diff --git a/drivers/scsi/smartpqi/smartpqi_init.c b/drivers/scsi/smartpqi/smartpqi_init.c index fe549e2b7c9407..c829d9590558df 100644 --- a/drivers/scsi/smartpqi/smartpqi_init.c +++ b/drivers/scsi/smartpqi/smartpqi_init.c @@ -1241,7 +1241,8 @@ static inline int pqi_report_phys_luns(struct pqi_ctrl_info *ctrl_info, void **b dev_err(&ctrl_info->pci_dev->dev, "RPL returned unsupported data format %u\n", rpl_response_format); - return -EINVAL; + rc = -EINVAL; + goto out_free_rpl_list; } else { dev_warn(&ctrl_info->pci_dev->dev, "RPL returned extended format 2 instead of 4\n"); @@ -1253,8 +1254,10 @@ static inline int pqi_report_phys_luns(struct pqi_ctrl_info *ctrl_info, void **b rpl_16byte_wwid_list = kmalloc(struct_size(rpl_16byte_wwid_list, lun_entries, num_physicals), GFP_KERNEL); - if (!rpl_16byte_wwid_list) - return -ENOMEM; + if (!rpl_16byte_wwid_list) { + rc = -ENOMEM; + goto out_free_rpl_list; + } put_unaligned_be32(num_physicals * sizeof(struct report_phys_lun_16byte_wwid), &rpl_16byte_wwid_list->header.list_length); @@ -1275,6 +1278,10 @@ static inline int pqi_report_phys_luns(struct pqi_ctrl_info *ctrl_info, void **b *buffer = rpl_16byte_wwid_list; return 0; + +out_free_rpl_list: + kfree(rpl_list); + return rc; } static inline int pqi_report_logical_luns(struct pqi_ctrl_info *ctrl_info, void **buffer) diff --git a/drivers/scsi/storvsc_drv.c b/drivers/scsi/storvsc_drv.c index b43d876747b76c..68c837146b9ea2 100644 --- a/drivers/scsi/storvsc_drv.c +++ b/drivers/scsi/storvsc_drv.c @@ -1855,8 +1855,9 @@ static int storvsc_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *scmnd) cmd_request->payload_sz = payload_sz; /* Invokes the vsc to start an IO */ - ret = storvsc_do_io(dev, cmd_request, get_cpu()); - put_cpu(); + migrate_disable(); + ret = storvsc_do_io(dev, cmd_request, smp_processor_id()); + migrate_enable(); if (ret) scsi_dma_unmap(scmnd); diff --git a/drivers/slimbus/qcom-ngd-ctrl.c b/drivers/slimbus/qcom-ngd-ctrl.c index ba3d80d12605cd..d2d11f6294b707 100644 --- a/drivers/slimbus/qcom-ngd-ctrl.c +++ b/drivers/slimbus/qcom-ngd-ctrl.c @@ -1539,10 +1539,8 @@ static int of_qcom_slim_ngd_register(struct device *parent, ngd->id = id; ngd->pdev->dev.parent = parent; - ret = driver_set_override(&ngd->pdev->dev, - &ngd->pdev->driver_override, - QCOM_SLIM_NGD_DRV_NAME, - strlen(QCOM_SLIM_NGD_DRV_NAME)); + ret = device_set_driver_override(&ngd->pdev->dev, + QCOM_SLIM_NGD_DRV_NAME); if (ret) { platform_device_put(ngd->pdev); kfree(ngd); diff --git a/drivers/soc/apple/Kconfig b/drivers/soc/apple/Kconfig index ad67368892311b..67284bd6490091 100644 --- a/drivers/soc/apple/Kconfig +++ b/drivers/soc/apple/Kconfig @@ -4,6 +4,15 @@ if ARCH_APPLE || COMPILE_TEST menu "Apple SoC drivers" +config APPLE_DOCKCHANNEL + tristate "Apple DockChannel FIFO" + depends on ARCH_APPLE || COMPILE_TEST + help + DockChannel is a simple FIFO used on Apple SoCs for debug and inter-processor + communications. + + Say 'y' here if you have an Apple SoC. + config APPLE_MAILBOX tristate "Apple SoC mailboxes" depends on PM @@ -16,6 +25,18 @@ config APPLE_MAILBOX Say Y here if you have an Apple SoC. +config APPLE_PMGR_MISC + bool "Apple SoC PMGR miscellaneous support" + depends on PM + help + The PMGR block in Apple SoCs provides high-level power state + controls for SoC devices. This driver manages miscellaneous + power controls. + +config APPLE_TUNABLE + tristate + depends on ARCH_APPLE || COMPILE_TEST + config APPLE_RTKIT tristate "Apple RTKit co-processor IPC protocol" depends on APPLE_MAILBOX @@ -28,6 +49,19 @@ config APPLE_RTKIT Say 'y' here if you have an Apple SoC. +config APPLE_RTKIT_HELPER + tristate "Apple Generic RTKit helper co-processor" + depends on APPLE_RTKIT + depends on ARCH_APPLE || COMPILE_TEST + help + Apple SoCs such as the M1 come with various co-processors running + their proprietary RTKit operating system. This option enables support + for a generic co-processor that does not implement any additional + in-band communications. It can be used for testing purposes, or for + coprocessors such as MTP that communicate over a different interface. + + Say 'y' here if you have an Apple SoC. + config APPLE_SART tristate "Apple SART DMA address filter" depends on ARCH_APPLE || COMPILE_TEST @@ -38,6 +72,56 @@ config APPLE_SART Say 'y' here if you have an Apple SoC. +config RUST_APPLE_MAILBOX + bool + depends on PM + depends on RUST + select APPLE_MAILBOX + +config RUST_APPLE_RTKIT + bool + depends on PM + depends on RUST + select APPLE_RTKIT + +config APPLE_AOP + tristate "Apple \"Always-on\" Processor" + depends on ARCH_APPLE || COMPILE_TEST + depends on PM + depends on RUST + select RUST_APPLE_RTKIT + help + A co-processor persent on certain Apple SoCs controlling accelerometers, + gyros, ambient light sensors and microphones. Is not actually always on. + + Say 'y' here if you have an Apple laptop. + +config APPLE_SEP + tristate "Apple Secure Element Processor" + depends on ARCH_APPLE || COMPILE_TEST + depends on PM + depends on RUST + select RUST_APPLE_RTKIT + select RUST_APPLE_MAILBOX + help + A security co-processor persent on Apple SoCs, controlling transparent + disk encryption, secure boot, HDCP, biometric auth and probably more. + + Say 'y' here if you have an Apple SoC. + +config APPLE_PMP + tristate "Apple Power Management Processor" + depends on ARCH_APPLE || COMPILE_TEST + depends on RUST + select APPLE_PMP_REPORT + select RUST_APPLE_RTKIT + default y if ARCH_APPLE + help + A co-processor present on Apple SoCs, controlling power states of + fabric and other uncore components. + + Say 'y' here if you have an Apple SoC. + endmenu endif diff --git a/drivers/soc/apple/Makefile b/drivers/soc/apple/Makefile index 4d9ab8f3037b71..f9ab2a1ff53f42 100644 --- a/drivers/soc/apple/Makefile +++ b/drivers/soc/apple/Makefile @@ -1,10 +1,27 @@ # SPDX-License-Identifier: GPL-2.0-only +obj-$(CONFIG_APPLE_DOCKCHANNEL) += apple-dockchannel.o +apple-dockchannel-y = dockchannel.o + obj-$(CONFIG_APPLE_MAILBOX) += apple-mailbox.o apple-mailbox-y = mailbox.o +obj-$(CONFIG_APPLE_PMGR_MISC) += apple-pmgr-misc.o + +obj-$(CONFIG_APPLE_TUNABLE) += apple-tunable.o +apple-tunable-y = tunable.o + obj-$(CONFIG_APPLE_RTKIT) += apple-rtkit.o apple-rtkit-y = rtkit.o rtkit-crashlog.o +obj-$(CONFIG_APPLE_RTKIT_HELPER) += apple-rtkit-helper.o +apple-rtkit-helper-y = rtkit-helper.o + obj-$(CONFIG_APPLE_SART) += apple-sart.o apple-sart-y = sart.o + +obj-$(CONFIG_APPLE_AOP) += aop.o + +obj-$(CONFIG_APPLE_SEP) += sep.o + +obj-$(CONFIG_APPLE_PMP) += pmp.o diff --git a/drivers/soc/apple/aop.rs b/drivers/soc/apple/aop.rs new file mode 100644 index 00000000000000..02318c11bd13b7 --- /dev/null +++ b/drivers/soc/apple/aop.rs @@ -0,0 +1,998 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#![recursion_limit = "2048"] + +//! Apple AOP driver +//! +//! Copyright (C) The Asahi Linux Contributors + +use core::{arch::asm, cmp, mem, ptr, slice}; + +use kernel::{ + bindings, c_str, device, + device::Core, + dma::{CoherentAllocation, Device, DmaMask}, + error::from_err_ptr, + io::mem::IoMem, + iosys_map::IoSysMapRef, + module_platform_driver, new_condvar, new_mutex, of, platform, + prelude::*, + soc::apple::aop::{from_fourcc, EPICService, FakehidListener, AOP}, + soc::apple::rtkit, + sync::{Arc, ArcBorrow, CondVar, Mutex}, + types::{ARef, ForeignOwnable}, + workqueue::{self, impl_has_work, new_work, Work, WorkItem}, +}; + +const AOP_MAX_CALLS: usize = 8; +const AOP_MMIO_SIZE: usize = 0x1e0000; +const ASC_MMIO_SIZE: usize = 0x4000; +const BOOTARGS_OFFSET: usize = 0x22c; +const BOOTARGS_SIZE: usize = 0x230; +const CPU_CONTROL: usize = 0x44; +const CPU_RUN: u32 = 0x1 << 4; +const AFK_ENDPOINT_START: u8 = 0x20; +const AFK_ENDPOINT_COUNT: u8 = 0xf; +const AFK_OPC_GET_BUF: u64 = 0x89; +const AFK_OPC_INIT: u64 = 0x80; +const AFK_OPC_INIT_RX: u64 = 0x8b; +const AFK_OPC_INIT_TX: u64 = 0x8a; +const AFK_OPC_INIT_UNK: u64 = 0x8c; +const AFK_OPC_SEND: u64 = 0xa2; +const AFK_OPC_START_ACK: u64 = 0x86; +const AFK_OPC_SHUTDOWN_ACK: u64 = 0xc1; +const AFK_OPC_RECV: u64 = 0x85; +const AFK_MSG_GET_BUF_ACK: u64 = 0xa1 << 48; +const AFK_MSG_INIT: u64 = AFK_OPC_INIT << 48; +const AFK_MSG_INIT_ACK: u64 = 0xa0 << 48; +const AFK_MSG_START: u64 = 0xa3 << 48; +const AFK_MSG_SHUTDOWN: u64 = 0xc0 << 48; +const AFK_RB_BLOCK_STEP: usize = 0x40; +const EPIC_TYPE_NOTIFY: u32 = 0; +const EPIC_CATEGORY_REPORT: u8 = 0x00; +const EPIC_CATEGORY_NOTIFY: u8 = 0x10; +const EPIC_CATEGORY_REPLY: u8 = 0x20; +const EPIC_SUBTYPE_STD_SERVICE: u16 = 0xc0; +const EPIC_SUBTYPE_FAKEHID_REPORT: u16 = 0xc4; +const EPIC_SUBTYPE_RETCODE: u16 = 0x84; +const EPIC_SUBTYPE_RETCODE_PAYLOAD: u16 = 0xa0; +const EPIC_SUBTYPE_STRING: u16 = 0x8a; +const QE_MAGIC1: u32 = from_fourcc(b" POI"); +const QE_MAGIC2: u32 = from_fourcc(b" POA"); + +fn align_up(v: usize, a: usize) -> usize { + (v + a - 1) & !(a - 1) +} + +#[inline(always)] +fn mem_sync() { + unsafe { + asm!("dsb sy"); + } +} + +#[repr(C, packed)] +#[derive(Clone, Copy, Default)] +struct QEHeader { + magic: u32, + size: u32, + channel: u32, + ty: u32, +} + +#[repr(C, packed)] +#[derive(Clone, Copy, Default)] +struct EPICHeader { + version: u8, + seq: u16, + _pad0: u8, + _unk0: u32, + timestamp: u64, + // Subheader + length: u32, + sub_version: u8, + category: u8, + subtype: u16, + tag: u16, + _unk1: u16, + _pad1: u64, + inline_len: u32, +} + +#[repr(C, packed)] +struct EPICServiceAnnounce { + name: [u8; 20], + _unk0: u32, + retcode: u32, + _unk1: u32, + channel: u32, + _unk2: u32, + _unk3: u32, +} + +#[pin_data] +struct FutureValue { + #[pin] + val: Mutex>, + #[pin] + completion: CondVar, +} + +impl FutureValue { + fn pin_init() -> impl PinInit> { + pin_init!( + FutureValue { + val <- new_mutex!(None), + completion <- new_condvar!() + } + ) + } + fn complete(&self, val: T) { + *self.val.lock() = Some(val); + self.completion.notify_all(); + } + fn wait(&self) -> T { + let mut ret_guard = self.val.lock(); + while ret_guard.is_none() { + self.completion.wait(&mut ret_guard); + } + ret_guard.take().unwrap() + } + fn reset(&self) { + *self.val.lock() = None; + } +} + +struct AFKRingBuffer { + offset: usize, + block_size: usize, + buf_size: usize, +} + +struct CallResult { + retcode: u32, + extra_data: Option>, +} + +struct AFKEndpoint { + index: u8, + iomem: Option>, + txbuf: Option, + rxbuf: Option, + seq: u16, + calls: [Option>>; AOP_MAX_CALLS], + call_returns: [Option>; AOP_MAX_CALLS], +} + +unsafe impl Send for AFKEndpoint {} + +impl AFKEndpoint { + fn new(index: u8) -> AFKEndpoint { + AFKEndpoint { + index, + iomem: None, + txbuf: None, + rxbuf: None, + seq: 0, + calls: [const { None }; AOP_MAX_CALLS], + call_returns: [const { None }; AOP_MAX_CALLS], + } + } + + fn start(&self, rtkit: Pin<&mut rtkit::RtKit>) -> Result<()> { + rtkit.send_message(self.index, AFK_MSG_INIT) + } + + fn stop(&self, rtkit: Pin<&mut rtkit::RtKit>) -> Result<()> { + rtkit.send_message(self.index, AFK_MSG_SHUTDOWN) + } + + fn recv_message( + &mut self, + client: ArcBorrow<'_, AopData>, + rtkit: Pin<&mut rtkit::RtKit>, + msg: u64, + ) -> Result<()> { + let opc = msg >> 48; + match opc { + AFK_OPC_INIT => { + rtkit.send_message(self.index, AFK_MSG_INIT_ACK)?; + } + AFK_OPC_GET_BUF => { + self.recv_get_buf(client.dev.clone(), rtkit, msg)?; + } + AFK_OPC_INIT_UNK => {} // no-op + AFK_OPC_START_ACK => {} + AFK_OPC_INIT_RX => { + if self.rxbuf.is_some() { + dev_err!( + client.dev, + "Got InitRX message with existing rxbuf at endpoint {}", + self.index + ); + return Err(EIO); + } + self.rxbuf = Some(self.parse_ring_buf(msg)?); + if self.txbuf.is_some() { + rtkit.send_message(self.index, AFK_MSG_START)?; + } + } + AFK_OPC_INIT_TX => { + if self.txbuf.is_some() { + dev_err!( + client.dev, + "Got InitTX message with existing txbuf at endpoint {}", + self.index + ); + return Err(EIO); + } + self.txbuf = Some(self.parse_ring_buf(msg)?); + if self.rxbuf.is_some() { + rtkit.send_message(self.index, AFK_MSG_START)?; + } + } + AFK_OPC_RECV => { + self.recv_rb(client)?; + } + AFK_OPC_SHUTDOWN_ACK => { + client.shutdown_complete(); + } + _ => dev_err!( + client.dev, + "AFK endpoint {} got unknown message {}", + self.index, + msg + ), + } + Ok(()) + } + + fn parse_ring_buf(&self, msg: u64) -> Result { + let msg = msg as usize; + let size = ((msg >> 16) & 0xFFFF) * AFK_RB_BLOCK_STEP; + let offset = ((msg >> 32) & 0xFFFF) * AFK_RB_BLOCK_STEP; + let buf_size = self.iomem_read32(offset)? as usize; + let block_size = (size - buf_size) / 3; + Ok(AFKRingBuffer { + offset, + block_size, + buf_size, + }) + } + fn iomem_write32(&mut self, off: usize, data: u32) -> Result<()> { + let size = core::mem::size_of::(); + let data = data.to_le_bytes(); + let buf = unsafe { self.iomem.as_mut().ok_or(ENXIO)?.as_slice_mut(off, size)? }; + buf.copy_from_slice(&data); + Ok(()) + } + + fn iomem_read32(&self, off: usize) -> Result { + let size = core::mem::size_of::(); + let buf = unsafe { self.iomem.as_ref().ok_or(ENXIO)?.as_slice(off, size)? }; + Ok(u32::from_le_bytes(buf.try_into().unwrap())) + } + + fn memcpy_from_iomem(&self, off: usize, target: &mut [u8]) -> Result<()> { + // SAFETY: + // as_slice() checks that off and target.len() are whithin iomem's limits. + unsafe { + let src = self + .iomem + .as_ref() + .ok_or(ENXIO)? + .as_slice(off, target.len())?; + target.copy_from_slice(src); + } + Ok(()) + } + + fn memcpy_to_iomem(&mut self, off: usize, src: &[u8]) -> Result<()> { + // SAFETY: + // as_slice_mut() checks that off and src.len() are whithin iomem's limits. + unsafe { + let target = self + .iomem + .as_mut() + .ok_or(ENXIO)? + .as_slice_mut(off, src.len())?; + target.copy_from_slice(src); + } + Ok(()) + } + + fn recv_get_buf( + &mut self, + dev: ARef, + rtkit: Pin<&mut rtkit::RtKit>, + msg: u64, + ) -> Result<()> { + let size = ((msg & 0xFFFF0000) >> 16) as usize * AFK_RB_BLOCK_STEP; + if self.iomem.is_some() { + dev_err!( + dev, + "Got GetBuf message with existing buffer on endpoint {}", + self.index + ); + return Err(EIO); + } + let iomem = dev.while_bound_with(|bound_dev| { + CoherentAllocation::::alloc_coherent(bound_dev, size, GFP_KERNEL) + })?; + rtkit.send_message(self.index, AFK_MSG_GET_BUF_ACK | iomem.dma_handle())?; + self.iomem = Some(iomem); + Ok(()) + } + + fn recv_rb(&mut self, client: ArcBorrow<'_, AopData>) -> Result<()> { + let (buf_offset, block_size, buf_size) = match self.rxbuf.as_ref() { + Some(b) => (b.offset, b.block_size, b.buf_size), + None => { + dev_err!( + client.dev, + "Got Recv message with no rxbuf at endpoint {}", + self.index + ); + return Err(EIO); + } + }; + let mut rptr = self.iomem_read32(buf_offset + block_size)? as usize; + let mut wptr = self.iomem_read32(buf_offset + block_size * 2)?; + mem_sync(); + let base = buf_offset + block_size * 3; + let mut msg_buf = KVec::new(); + const QEH_SIZE: usize = mem::size_of::(); + while wptr as usize != rptr { + let mut qeh_bytes = [0; QEH_SIZE]; + self.memcpy_from_iomem(base + rptr, &mut qeh_bytes)?; + let mut qeh = unsafe { &*(qeh_bytes.as_ptr() as *const QEHeader) }; + if qeh.magic != QE_MAGIC1 && qeh.magic != QE_MAGIC2 { + let magic = qeh.magic; + dev_err!( + client.dev, + "Invalid magic on ep {}, got {:x}", + self.index, + magic + ); + return Err(EIO); + } + if qeh.size as usize > (buf_size - rptr - QEH_SIZE) { + rptr = 0; + self.memcpy_from_iomem(base + rptr, &mut qeh_bytes)?; + qeh = unsafe { &*(qeh_bytes.as_ptr() as *const QEHeader) }; + + if qeh.magic != QE_MAGIC1 && qeh.magic != QE_MAGIC2 { + let magic = qeh.magic; + dev_err!( + client.dev, + "Invalid magic on ep {}, got {:x}", + self.index, + magic + ); + return Err(EIO); + } + } + msg_buf.resize(qeh.size as usize, 0, GFP_KERNEL)?; + self.memcpy_from_iomem(base + rptr + QEH_SIZE, &mut msg_buf)?; + let (hdr_bytes, msg) = msg_buf.split_at(mem::size_of::()); + let header = unsafe { &*(hdr_bytes.as_ptr() as *const EPICHeader) }; + self.handle_ipc(client, qeh, header, msg)?; + rptr = align_up(rptr + QEH_SIZE + qeh.size as usize, block_size) % buf_size; + mem_sync(); + self.iomem_write32(buf_offset + block_size, rptr as u32)?; + wptr = self.iomem_read32(buf_offset + block_size * 2)?; + mem_sync(); + } + Ok(()) + } + fn handle_ipc( + &mut self, + client: ArcBorrow<'_, AopData>, + qhdr: &QEHeader, + ehdr: &EPICHeader, + data: &[u8], + ) -> Result<()> { + let subtype = ehdr.subtype; + if ehdr.category == EPIC_CATEGORY_REPORT { + if subtype == EPIC_SUBTYPE_STD_SERVICE { + let announce = unsafe { &*(data.as_ptr() as *const EPICServiceAnnounce) }; + let chan = announce.channel; + let name_len = announce + .name + .iter() + .position(|x| *x == 0) + .unwrap_or(announce.name.len()); + return Into::>::into(client).register_service( + self, + chan, + &announce.name[..name_len], + ); + } else if subtype == EPIC_SUBTYPE_FAKEHID_REPORT { + return client.process_fakehid_report(self, qhdr.channel, data); + } else { + dev_err!( + client.dev, + "Unexpected EPIC report subtype {:x} on endpoint {}", + subtype, + self.index + ); + return Err(EIO); + } + } else if ehdr.category == EPIC_CATEGORY_REPLY { + if subtype == EPIC_SUBTYPE_RETCODE_PAYLOAD + || subtype == EPIC_SUBTYPE_RETCODE + || subtype == EPIC_SUBTYPE_STRING + { + if data.len() < mem::size_of::() { + dev_err!( + client.dev, + "Retcode data too short on endpoint {}", + self.index + ); + return Err(EIO); + } + let retcode = u32::from_ne_bytes(data[..4].try_into().unwrap()); + let tag = ehdr.tag as usize; + if tag == 0 || tag - 1 > self.calls.len() || self.calls[tag - 1].is_none() { + dev_err!( + client.dev, + "Got a retcode with invalid tag {:?} on endpoint {}", + tag, + self.index + ); + return Err(EIO); + } + let future = self.calls[tag - 1].take().unwrap(); + let extra_data = if let Some(mut ret) = self.call_returns[tag - 1].take() { + let len = cmp::min(data.len() - 4, ret.len()); + ret[..len].copy_from_slice(&data[4..(len + 4)]); + ret.truncate(len); + Some(ret) + } else { + None + }; + future.complete(CallResult { + retcode, + extra_data, + }); + + return Ok(()); + } else { + dev_err!( + client.dev, + "Unexpected EPIC reply subtype {:x} on endpoint {}", + subtype, + self.index + ); + return Err(EIO); + } + } + dev_err!( + client.dev, + "Unexpected EPIC category {:x} on endpoint {}", + ehdr.category, + self.index + ); + Err(EIO) + } + fn send_rb( + &mut self, + client: &AopData, + rtkit: Pin<&mut rtkit::RtKit>, + channel: u32, + ty: u32, + header: &[u8], + data: &[u8], + ) -> Result<()> { + let (buf_offset, block_size, buf_size) = match self.txbuf.as_ref() { + Some(b) => (b.offset, b.block_size, b.buf_size), + None => { + dev_err!( + client.dev, + "Attempting to send message with no txbuf at endpoint {}", + self.index + ); + return Err(EIO); + } + }; + let base = buf_offset + block_size * 3; + mem_sync(); + let rptr = self.iomem_read32(buf_offset + block_size)? as usize; + let mut wptr = self.iomem_read32(buf_offset + block_size * 2)? as usize; + const QEH_SIZE: usize = mem::size_of::(); + if wptr < rptr && wptr + QEH_SIZE >= rptr { + dev_err!(client.dev, "Tx buffer full at endpoint {}", self.index); + return Err(EIO); + } + let payload_len = header.len() + data.len(); + let qeh = QEHeader { + magic: QE_MAGIC1, + size: payload_len as u32, + channel, + ty, + }; + let qeh_bytes = unsafe { + slice::from_raw_parts( + &qeh as *const QEHeader as *const u8, + mem::size_of::(), + ) + }; + self.memcpy_to_iomem(base + wptr, qeh_bytes)?; + if payload_len > buf_size - wptr - QEH_SIZE { + wptr = 0; + self.memcpy_to_iomem(base + wptr, qeh_bytes)?; + } + self.memcpy_to_iomem(base + wptr + QEH_SIZE, header)?; + self.memcpy_to_iomem(base + wptr + QEH_SIZE + header.len(), data)?; + wptr = align_up(wptr + QEH_SIZE + payload_len, block_size) % buf_size; + self.iomem_write32(buf_offset + block_size * 2, wptr as u32)?; + let msg = wptr as u64 | (AFK_OPC_SEND << 48); + rtkit.send_message(self.index, msg) + } + fn epic_notify( + &mut self, + client: &AopData, + rtkit: Pin<&mut rtkit::RtKit>, + channel: u32, + subtype: u16, + data: &[u8], + ret: Option>, + ) -> Result>> { + let mut tag = 0; + for i in 0..self.calls.len() { + if self.calls[i].is_none() { + tag = i + 1; + break; + } + } + if tag == 0 { + dev_err!( + client.dev, + "Too many inflight calls on endpoint {}", + self.index + ); + return Err(EIO); + } + let call = Arc::pin_init(FutureValue::pin_init(), GFP_KERNEL)?; + let hdr = EPICHeader { + version: 2, + seq: self.seq, + length: data.len() as u32, + sub_version: 2, + category: EPIC_CATEGORY_NOTIFY, + subtype, + tag: tag as u16, + ..EPICHeader::default() + }; + self.call_returns[tag - 1] = ret; + self.send_rb( + client, + rtkit, + channel, + EPIC_TYPE_NOTIFY, + unsafe { + slice::from_raw_parts( + &hdr as *const EPICHeader as *const u8, + mem::size_of::(), + ) + }, + data, + )?; + self.seq = self.seq.wrapping_add(1); + self.calls[tag - 1] = Some(call.clone()); + Ok(call) + } +} + +struct ListenerEntry { + svc: EPICService, + listener: Arc, +} + +unsafe impl Send for ListenerEntry {} + +#[pin_data] +struct AopData { + dev: ARef, + #[pin] + rtkit: Mutex>>, + #[pin] + endpoints: [Mutex; AFK_ENDPOINT_COUNT as usize], + #[pin] + ep_shutdown: FutureValue<()>, + #[pin] + hid_listeners: Mutex>, + #[pin] + subdevices: Mutex>, +} + +unsafe impl Send for AopData {} +unsafe impl Sync for AopData {} + +#[pin_data] +struct AopServiceRegisterWork { + name: &'static CStr, + data: Arc, + service: EPICService, + #[pin] + work: Work, +} + +impl_has_work! { + impl HasWork for AopServiceRegisterWork { self.work } +} + +impl AopServiceRegisterWork { + fn new( + name: &'static CStr, + data: Arc, + service: EPICService, + ) -> Result>> { + KBox::pin_init( + pin_init!(AopServiceRegisterWork { + name, data, service, + work <- new_work!("AopServiceRegisterWork::work"), + }), + GFP_KERNEL, + ) + } +} + +impl WorkItem for AopServiceRegisterWork { + type Pointer = Pin>; + + fn run(this: Pin>) { + let fwnode = this + .data + .dev + .fwnode() + .and_then(|x| x.get_child_by_name(this.name)); + let info = bindings::platform_device_info { + parent: this.data.dev.as_raw(), + name: this.name.as_ptr() as *const _, + id: bindings::PLATFORM_DEVID_AUTO, + res: ptr::null_mut(), + num_res: 0, + data: &this.service as *const EPICService as *const _, + size_data: mem::size_of::(), + dma_mask: 0, + fwnode: fwnode.map(|x| x.as_raw()).unwrap_or(ptr::null_mut()), + properties: ptr::null_mut(), + of_node_reused: false, + }; + let pdev = unsafe { from_err_ptr(bindings::platform_device_register_full(&info)) }; + match pdev { + Err(e) => { + dev_err!( + this.data.dev, + "Failed to create device for service {:?}: {:?}", + this.name, + e + ); + } + Ok(pdev) => { + let res = this.data.subdevices.lock().push(pdev, GFP_KERNEL); + if res.is_err() { + dev_err!(this.data.dev, "Failed to store subdevice"); + } + } + } + } +} + +impl AopData { + fn new(dev: &platform::Device) -> Result> { + Arc::pin_init( + pin_init!( + AopData { + dev: dev.as_ref().into(), + rtkit <- new_mutex!(None), + endpoints <- pin_init::pin_init_array_from_fn(|i| { + new_mutex!(AFKEndpoint::new(AFK_ENDPOINT_START + i as u8)) + }), + ep_shutdown <- FutureValue::pin_init(), + hid_listeners <- new_mutex!(KVec::new()), + subdevices <- new_mutex!(KVec::new()), + } + ), + GFP_KERNEL, + ) + } + fn start(&self) -> Result<()> { + { + let mut guard = self.rtkit.lock(); + let mut rtk = guard.as_mut().as_pin_mut().unwrap(); + rtk.as_mut().wake()?; + } + for ep in 0..AFK_ENDPOINT_COUNT { + let rtk_ep_num = AFK_ENDPOINT_START + ep; + let mut guard = self.rtkit.lock(); + let mut rtk = guard.as_mut().as_pin_mut().unwrap(); + if !rtk.as_mut().has_endpoint(rtk_ep_num) { + continue; + } + rtk.as_mut().start_endpoint(rtk_ep_num)?; + let ep_guard = self.endpoints[ep as usize].lock(); + ep_guard.start(rtk.as_mut())?; + } + Ok(()) + } + fn register_service( + self: Arc, + ep: &mut AFKEndpoint, + channel: u32, + name: &[u8], + ) -> Result<()> { + let svc = EPICService { + channel, + endpoint: ep.index, + }; + let dev_name = match name { + b"aop-audio" => c_str!("audio"), + b"las" => c_str!("las"), + b"als" => c_str!("als"), + _ => { + return Ok(()); + } + }; + // probe can call back into us, run it with locks dropped. + let work = AopServiceRegisterWork::new(dev_name, self, svc)?; + workqueue::system().enqueue(work); + Ok(()) + } + + fn process_fakehid_report(&self, ep: &AFKEndpoint, ch: u32, data: &[u8]) -> Result<()> { + let guard = self.hid_listeners.lock(); + for entry in &*guard { + if entry.svc.endpoint == ep.index && entry.svc.channel == ch { + return entry.listener.process_fakehid_report(data); + } + } + Ok(()) + } + + fn shutdown_complete(&self) { + self.ep_shutdown.complete(()); + } + + fn stop(&self) -> Result<()> { + for ep in 0..AFK_ENDPOINT_COUNT { + { + let rtk_ep_num = AFK_ENDPOINT_START + ep; + let mut guard = self.rtkit.lock(); + let mut rtk = guard.as_mut().as_pin_mut().unwrap(); + if !rtk.as_mut().has_endpoint(rtk_ep_num) { + continue; + } + let ep_guard = self.endpoints[ep as usize].lock(); + ep_guard.stop(rtk.as_mut())?; + } + self.ep_shutdown.wait(); + self.ep_shutdown.reset(); + } + Ok(()) + } + + fn patch_bootargs( + &self, + aop_mmio: &IoMem, + patches: &[(u32, u64)], + ) -> Result<()> { + let offset = aop_mmio.read32_relaxed(BOOTARGS_OFFSET) as usize; + let size = aop_mmio.read32_relaxed(BOOTARGS_SIZE) as usize; + let mut arg_bytes = KVec::::from_elem(0, size, GFP_KERNEL)?; + aop_mmio.try_memcpy_fromio(&mut arg_bytes, offset)?; + let mut idx = 0; + while idx < size { + let key = u32::from_le_bytes(arg_bytes[idx..idx + 4].try_into().unwrap()); + let size = u32::from_le_bytes(arg_bytes[idx + 4..idx + 8].try_into().unwrap()) as usize; + idx += 8; + for (k, v) in patches.iter() { + if *k != key { + continue; + } + arg_bytes[idx..idx + size].copy_from_slice(&(*v as u64).to_le_bytes()[..size]); + break; + } + idx += size; + } + aop_mmio.try_memcpy_toio(offset, &arg_bytes) + } + + fn start_cpu(&self, asc_mmio: &IoMem) -> Result<()> { + let val = asc_mmio.read32_relaxed(CPU_CONTROL); + asc_mmio.write32_relaxed(val | CPU_RUN, CPU_CONTROL); + Ok(()) + } +} + +impl AOP for AopData { + fn epic_call(&self, svc: &EPICService, subtype: u16, msg_bytes: &[u8]) -> Result { + let ep_idx = svc.endpoint - AFK_ENDPOINT_START; + let call = { + let mut rtk_guard = self.rtkit.lock(); + let mut rtk = rtk_guard.as_mut().as_pin_mut().unwrap(); + let mut ep_guard = self.endpoints[ep_idx as usize].lock(); + ep_guard.epic_notify(self, rtk.as_mut(), svc.channel, subtype, msg_bytes, None)? + }; + Ok(call.wait().retcode) + } + fn epic_call_ret( + &self, + svc: &EPICService, + subtype: u16, + msg_bytes: &[u8], + ret_len: usize, + ) -> Result<(u32, KVec)> { + let ep_idx = svc.endpoint - AFK_ENDPOINT_START; + let call = { + let mut rtk_guard = self.rtkit.lock(); + let mut rtk = rtk_guard.as_mut().as_pin_mut().unwrap(); + let mut ep_guard = self.endpoints[ep_idx as usize].lock(); + let mut ret_buf = KVec::new(); + ret_buf.resize(ret_len, 0, GFP_KERNEL)?; + ep_guard.epic_notify( + self, + rtk.as_mut(), + svc.channel, + subtype, + msg_bytes, + Some(ret_buf), + )? + }; + let res = call.wait(); + Ok((res.retcode, res.extra_data.unwrap())) + } + fn add_fakehid_listener( + &self, + svc: EPICService, + listener: Arc, + ) -> Result<()> { + let mut guard = self.hid_listeners.lock(); + Ok(guard.push(ListenerEntry { svc, listener }, GFP_KERNEL)?) + } + fn remove_fakehid_listener(&self, svc: &EPICService) -> bool { + let mut guard = self.hid_listeners.lock(); + for i in 0..guard.len() { + if guard[i].svc == *svc { + guard.swap_remove(i); + return true; + } + } + false + } + fn remove(&self) { + if let Err(e) = self.stop() { + dev_err!(self.dev, "Failed to stop AOP {:?}", e); + } + *self.rtkit.lock() = None; + let guard = self.subdevices.lock(); + for pdev in &*guard { + unsafe { + bindings::platform_device_unregister(*pdev); + } + } + } +} + +struct NoBuffer; +impl rtkit::Buffer for NoBuffer { + fn iova(&self) -> Result { + unreachable!() + } + fn buf(&mut self) -> Result> { + unreachable!() + } +} + +#[vtable] +impl rtkit::Operations for AopData { + type Data = Arc; + type Buffer = NoBuffer; + + fn recv_message(data: ::Borrowed<'_>, ep: u8, msg: u64) { + let mut guard = data.rtkit.lock(); + let mut rtk = guard.as_mut().as_pin_mut().unwrap(); + let mut ep_guard = data.endpoints[(ep - AFK_ENDPOINT_START) as usize].lock(); + let ret = ep_guard.recv_message(data, rtk.as_mut(), msg); + if let Err(e) = ret { + dev_err!(data.dev, "Failed to handle rtkit message, error: {:?}", e); + } + } + + fn crashed(data: ::Borrowed<'_>, _crashlog: Option<&[u8]>) { + dev_err!(data.dev, "AOP firmware crashed"); + } +} + +#[repr(transparent)] +struct AopDriver(Arc); + +struct AopHwConfig { + ec0p: u64, + alig: u64, + aopt: u64, +} + +const HW_CFG_T8103: AopHwConfig = AopHwConfig { + ec0p: 0x020000, + aopt: 1, + alig: 128, +}; +const HW_CFG_T8112: AopHwConfig = AopHwConfig { + ec0p: 0x020000, + aopt: 0, + alig: 128, +}; +const HW_CFG_T6000: AopHwConfig = AopHwConfig { + ec0p: 0x020000, + aopt: 0, + alig: 64, +}; +const HW_CFG_T6020: AopHwConfig = AopHwConfig { + ec0p: 0x0100_00000000, + aopt: 0, + alig: 64, +}; + +kernel::of_device_table!( + OF_TABLE, + MODULE_OF_TABLE, + ::IdInfo, + [ + (of::DeviceId::new(c_str!("apple,t8103-aop")), &HW_CFG_T8103), + (of::DeviceId::new(c_str!("apple,t8112-aop")), &HW_CFG_T8112), + (of::DeviceId::new(c_str!("apple,t6000-aop")), &HW_CFG_T6000), + (of::DeviceId::new(c_str!("apple,t6020-aop")), &HW_CFG_T6020), + ] +); + +impl platform::Driver for AopDriver { + type IdInfo = &'static AopHwConfig; + + const OF_ID_TABLE: Option> = Some(&OF_TABLE); + + fn probe( + pdev: &platform::Device, + info: Option<&Self::IdInfo>, + ) -> impl PinInit { + let cfg = info.ok_or(ENODEV)?; + unsafe { pdev.dma_set_mask_and_coherent(DmaMask::new::<42>())? }; + let aop_req = pdev.io_request_by_index(0).ok_or(EINVAL)?; + let aop_mmio = KBox::pin_init(aop_req.iomap_sized::(), GFP_KERNEL)?; + let asc_req = pdev.io_request_by_index(1).ok_or(EINVAL)?; + let asc_mmio = KBox::pin_init(asc_req.iomap_sized::(), GFP_KERNEL)?; + let data = AopData::new(pdev)?; + let aop_mmio = aop_mmio.access(pdev.as_ref())?; + data.patch_bootargs( + aop_mmio, + &[ + (from_fourcc(b"EC0p"), cfg.ec0p), + (from_fourcc(b"nCal"), 0x0), + (from_fourcc(b"alig"), cfg.alig), + (from_fourcc(b"AOPt"), cfg.aopt), + ], + )?; + let rtkit = rtkit::RtKit::::new(pdev.as_ref(), None, 0, data.clone())?; + *data.rtkit.lock() = Some(rtkit); + let asc_mmio = asc_mmio.access(pdev.as_ref())?; + let _ = data.start_cpu(asc_mmio); + data.start()?; + let data = data as Arc; + Ok(Self(data)) + } +} + +impl Drop for AopDriver { + fn drop(&mut self) { + self.0.remove(); + } +} + +unsafe impl Send for AopDriver {} + +module_platform_driver! { + type: AopDriver, + name: "apple_aop", + description: "AOP driver", + license: "Dual MIT/GPL", +} diff --git a/drivers/soc/apple/apple-pmgr-misc.c b/drivers/soc/apple/apple-pmgr-misc.c new file mode 100644 index 00000000000000..e768f34aacc586 --- /dev/null +++ b/drivers/soc/apple/apple-pmgr-misc.c @@ -0,0 +1,158 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Apple SoC PMGR device power state driver + * + * Copyright The Asahi Linux Contributors + */ + +#include +#include +#include +#include +#include +#include +#include + +#define APPLE_CLKGEN_PSTATE 0 +#define APPLE_CLKGEN_PSTATE_DESIRED GENMASK(3, 0) + +#define SYS_DEV_PSTATE_SUSPEND 1 + +enum sys_device { + DEV_FABRIC, + DEV_DCS, + DEV_MAX, +}; + +struct apple_pmgr_sys_device { + void __iomem *base; + u32 active_state; + u32 suspend_state; +}; + +struct apple_pmgr_misc { + struct device *dev; + struct apple_pmgr_sys_device devices[DEV_MAX]; +}; + +static void apple_pmgr_sys_dev_set_pstate(struct apple_pmgr_misc *misc, + enum sys_device dev, bool active) +{ + u32 pstate; + u32 val; + + if (!misc->devices[dev].base) + return; + + if (active) + pstate = misc->devices[dev].active_state; + else + pstate = misc->devices[dev].suspend_state; + + printk("set %d ps to pstate %d\n", dev, pstate); + + val = readl_relaxed(misc->devices[dev].base + APPLE_CLKGEN_PSTATE); + val &= ~APPLE_CLKGEN_PSTATE_DESIRED; + val |= FIELD_PREP(APPLE_CLKGEN_PSTATE_DESIRED, pstate); + writel_relaxed(val, misc->devices[dev].base); +} + +static int __maybe_unused apple_pmgr_misc_suspend_noirq(struct device *dev) +{ + struct apple_pmgr_misc *misc = dev_get_drvdata(dev); + int i; + + for (i = 0; i < DEV_MAX; i++) + apple_pmgr_sys_dev_set_pstate(misc, i, false); + + return 0; +} + +static int __maybe_unused apple_pmgr_misc_resume_noirq(struct device *dev) +{ + struct apple_pmgr_misc *misc = dev_get_drvdata(dev); + int i; + + for (i = 0; i < DEV_MAX; i++) + apple_pmgr_sys_dev_set_pstate(misc, i, true); + + return 0; +} + +static bool apple_pmgr_init_device(struct apple_pmgr_misc *misc, + enum sys_device dev, const char *device_name) +{ + void __iomem *base; + char name[32]; + u32 val; + + snprintf(name, sizeof(name), "%s-ps", device_name); + + base = devm_platform_ioremap_resource_byname( + to_platform_device(misc->dev), name); + if (!base) + return false; + + val = readl_relaxed(base + APPLE_CLKGEN_PSTATE); + + misc->devices[dev].base = base; + misc->devices[dev].active_state = + FIELD_GET(APPLE_CLKGEN_PSTATE_DESIRED, val); + misc->devices[dev].suspend_state = SYS_DEV_PSTATE_SUSPEND; + + snprintf(name, sizeof(name), "apple,%s-min-ps", device_name); + of_property_read_u32(misc->dev->of_node, name, + &misc->devices[dev].suspend_state); + + return true; +} + +static int apple_pmgr_misc_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct apple_pmgr_misc *misc; + int ret = -ENODEV; + + misc = devm_kzalloc(dev, sizeof(*misc), GFP_KERNEL); + if (!misc) + return -ENOMEM; + + misc->dev = dev; + + if (apple_pmgr_init_device(misc, DEV_FABRIC, "fabric")) + ret = 0; + + if (apple_pmgr_init_device(misc, DEV_DCS, "dcs")) + ret = 0; + + platform_set_drvdata(pdev, misc); + + return ret; +} + +static const struct of_device_id apple_pmgr_misc_of_match[] = { + { .compatible = "apple,t6000-pmgr-misc" }, + {} +}; + +MODULE_DEVICE_TABLE(of, apple_pmgr_misc_of_match); + +static const struct dev_pm_ops apple_pmgr_misc_pm_ops = { + SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(apple_pmgr_misc_suspend_noirq, + apple_pmgr_misc_resume_noirq) +}; + +static struct platform_driver apple_pmgr_misc_driver = { + .probe = apple_pmgr_misc_probe, + .driver = { + .name = "apple-pmgr-misc", + .of_match_table = apple_pmgr_misc_of_match, + .pm = pm_ptr(&apple_pmgr_misc_pm_ops), + }, +}; + +MODULE_AUTHOR("Hector Martin "); +MODULE_DESCRIPTION("PMGR misc driver for Apple SoCs"); +MODULE_LICENSE("GPL v2"); + +module_platform_driver(apple_pmgr_misc_driver); diff --git a/drivers/soc/apple/dockchannel.c b/drivers/soc/apple/dockchannel.c new file mode 100644 index 00000000000000..8f67c639da4d20 --- /dev/null +++ b/drivers/soc/apple/dockchannel.c @@ -0,0 +1,405 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Apple DockChannel FIFO driver + * Copyright The Asahi Linux Contributors + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define DOCKCHANNEL_MAX_IRQ 32 + +#define DOCKCHANNEL_TX_TIMEOUT_MS 1000 +#define DOCKCHANNEL_RX_TIMEOUT_MS 1000 + +#define IRQ_MASK 0x0 +#define IRQ_FLAG 0x4 + +#define IRQ_TX BIT(0) +#define IRQ_RX BIT(1) + +#define CONFIG_TX_THRESH 0x0 +#define CONFIG_RX_THRESH 0x4 + +#define DATA_TX8 0x4 +#define DATA_TX16 0x8 +#define DATA_TX24 0xc +#define DATA_TX32 0x10 +#define DATA_TX_FREE 0x14 +#define DATA_RX8 0x1c +#define DATA_RX16 0x20 +#define DATA_RX24 0x24 +#define DATA_RX32 0x28 +#define DATA_RX_COUNT 0x2c + +struct dockchannel { + struct device *dev; + int tx_irq; + int rx_irq; + + void __iomem *config_base; + void __iomem *data_base; + + u32 fifo_size; + bool awaiting; + struct completion tx_comp; + struct completion rx_comp; + + void *cookie; + void (*data_available)(void *cookie, size_t avail); +}; + +struct dockchannel_common { + struct device *dev; + struct irq_domain *domain; + int irq; + + void __iomem *irq_base; +}; + +/* Dockchannel FIFO functions */ + +static irqreturn_t dockchannel_tx_irq(int irq, void *data) +{ + struct dockchannel *dockchannel = data; + + disable_irq_nosync(irq); + complete(&dockchannel->tx_comp); + + return IRQ_HANDLED; +} + +static irqreturn_t dockchannel_rx_irq(int irq, void *data) +{ + struct dockchannel *dockchannel = data; + + disable_irq_nosync(irq); + + if (dockchannel->awaiting) { + return IRQ_WAKE_THREAD; + } else { + complete(&dockchannel->rx_comp); + return IRQ_HANDLED; + } +} + +static irqreturn_t dockchannel_rx_irq_thread(int irq, void *data) +{ + struct dockchannel *dockchannel = data; + size_t avail = readl_relaxed(dockchannel->data_base + DATA_RX_COUNT); + + dockchannel->awaiting = false; + dockchannel->data_available(dockchannel->cookie, avail); + + return IRQ_HANDLED; +} + +int dockchannel_send(struct dockchannel *dockchannel, const void *buf, size_t count) +{ + size_t left = count; + const u8 *p = buf; + + while (left > 0) { + size_t avail = readl_relaxed(dockchannel->data_base + DATA_TX_FREE); + size_t block = min(left, avail); + + if (avail == 0) { + size_t threshold = min((size_t)(dockchannel->fifo_size / 2), left); + + writel_relaxed(threshold, dockchannel->config_base + CONFIG_TX_THRESH); + reinit_completion(&dockchannel->tx_comp); + enable_irq(dockchannel->tx_irq); + + if (!wait_for_completion_timeout(&dockchannel->tx_comp, + msecs_to_jiffies(DOCKCHANNEL_TX_TIMEOUT_MS))) { + disable_irq(dockchannel->tx_irq); + return -ETIMEDOUT; + } + + continue; + } + + while (block >= 4) { + writel_relaxed(get_unaligned_le32(p), dockchannel->data_base + DATA_TX32); + p += 4; + left -= 4; + block -= 4; + } + while (block > 0) { + writeb_relaxed(*p++, dockchannel->data_base + DATA_TX8); + left--; + block--; + } + } + + return count; +} +EXPORT_SYMBOL(dockchannel_send); + +int dockchannel_recv(struct dockchannel *dockchannel, void *buf, size_t count) +{ + size_t left = count; + u8 *p = buf; + + while (left > 0) { + size_t avail = readl_relaxed(dockchannel->data_base + DATA_RX_COUNT); + size_t block = min(left, avail); + + if (avail == 0) { + size_t threshold = min((size_t)(dockchannel->fifo_size / 2), left); + + writel_relaxed(threshold, dockchannel->config_base + CONFIG_RX_THRESH); + reinit_completion(&dockchannel->rx_comp); + enable_irq(dockchannel->rx_irq); + + if (!wait_for_completion_timeout(&dockchannel->rx_comp, + msecs_to_jiffies(DOCKCHANNEL_RX_TIMEOUT_MS))) { + disable_irq(dockchannel->rx_irq); + return -ETIMEDOUT; + } + + continue; + } + + while (block >= 4) { + put_unaligned_le32(readl_relaxed(dockchannel->data_base + DATA_RX32), p); + p += 4; + left -= 4; + block -= 4; + } + while (block > 0) { + *p++ = readl_relaxed(dockchannel->data_base + DATA_RX8) >> 8; + left--; + block--; + } + } + + return count; +} +EXPORT_SYMBOL(dockchannel_recv); + +int dockchannel_await(struct dockchannel *dockchannel, + void (*callback)(void *cookie, size_t avail), + void *cookie, size_t count) +{ + size_t threshold = min((size_t)dockchannel->fifo_size, count); + + if (!count) { + dockchannel->awaiting = false; + disable_irq(dockchannel->rx_irq); + return 0; + } + + dockchannel->data_available = callback; + dockchannel->cookie = cookie; + dockchannel->awaiting = true; + writel_relaxed(threshold, dockchannel->config_base + CONFIG_RX_THRESH); + enable_irq(dockchannel->rx_irq); + + return threshold; +} +EXPORT_SYMBOL(dockchannel_await); + +struct dockchannel *dockchannel_init(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct dockchannel *dockchannel; + int ret; + + dockchannel = devm_kzalloc(dev, sizeof(*dockchannel), GFP_KERNEL); + if (!dockchannel) + return ERR_PTR(-ENOMEM); + + dockchannel->dev = dev; + dockchannel->config_base = devm_platform_ioremap_resource_byname(pdev, "config"); + if (IS_ERR(dockchannel->config_base)) + return (__force void *)dockchannel->config_base; + + dockchannel->data_base = devm_platform_ioremap_resource_byname(pdev, "data"); + if (IS_ERR(dockchannel->data_base)) + return (__force void *)dockchannel->data_base; + + ret = of_property_read_u32(dev->of_node, "apple,fifo-size", &dockchannel->fifo_size); + if (ret) + return ERR_PTR(dev_err_probe(dev, ret, "Missing apple,fifo-size property")); + + init_completion(&dockchannel->tx_comp); + init_completion(&dockchannel->rx_comp); + + dockchannel->tx_irq = platform_get_irq_byname(pdev, "tx"); + if (dockchannel->tx_irq <= 0) { + return ERR_PTR(dev_err_probe(dev, dockchannel->tx_irq, + "Failed to get TX IRQ")); + } + + dockchannel->rx_irq = platform_get_irq_byname(pdev, "rx"); + if (dockchannel->rx_irq <= 0) { + return ERR_PTR(dev_err_probe(dev, dockchannel->rx_irq, + "Failed to get RX IRQ")); + } + + ret = devm_request_irq(dev, dockchannel->tx_irq, dockchannel_tx_irq, IRQF_NO_AUTOEN, + "apple-dockchannel-tx", dockchannel); + if (ret) + return ERR_PTR(dev_err_probe(dev, ret, "Failed to request TX IRQ")); + + ret = devm_request_threaded_irq(dev, dockchannel->rx_irq, dockchannel_rx_irq, + dockchannel_rx_irq_thread, IRQF_NO_AUTOEN, + "apple-dockchannel-rx", dockchannel); + if (ret) + return ERR_PTR(dev_err_probe(dev, ret, "Failed to request RX IRQ")); + + return dockchannel; +} +EXPORT_SYMBOL(dockchannel_init); + + +/* Dockchannel IRQchip */ + +static void dockchannel_irq(struct irq_desc *desc) +{ + unsigned int irq = irq_desc_get_irq(desc); + struct irq_chip *chip = irq_desc_get_chip(desc); + struct dockchannel_common *dcc = irq_get_handler_data(irq); + unsigned long flags = readl_relaxed(dcc->irq_base + IRQ_FLAG); + int bit; + + chained_irq_enter(chip, desc); + + for_each_set_bit(bit, &flags, DOCKCHANNEL_MAX_IRQ) + generic_handle_domain_irq(dcc->domain, bit); + + chained_irq_exit(chip, desc); +} + +static void dockchannel_irq_ack(struct irq_data *data) +{ + struct dockchannel_common *dcc = irq_data_get_irq_chip_data(data); + unsigned int hwirq = data->hwirq; + + writel_relaxed(BIT(hwirq), dcc->irq_base + IRQ_FLAG); +} + +static void dockchannel_irq_mask(struct irq_data *data) +{ + struct dockchannel_common *dcc = irq_data_get_irq_chip_data(data); + unsigned int hwirq = data->hwirq; + u32 val = readl_relaxed(dcc->irq_base + IRQ_MASK); + + writel_relaxed(val & ~BIT(hwirq), dcc->irq_base + IRQ_MASK); +} + +static void dockchannel_irq_unmask(struct irq_data *data) +{ + struct dockchannel_common *dcc = irq_data_get_irq_chip_data(data); + unsigned int hwirq = data->hwirq; + u32 val = readl_relaxed(dcc->irq_base + IRQ_MASK); + + writel_relaxed(val | BIT(hwirq), dcc->irq_base + IRQ_MASK); +} + +static const struct irq_chip dockchannel_irqchip = { + .name = "dockchannel-irqc", + .irq_ack = dockchannel_irq_ack, + .irq_mask = dockchannel_irq_mask, + .irq_unmask = dockchannel_irq_unmask, +}; + +static int dockchannel_irq_domain_map(struct irq_domain *d, unsigned int virq, + irq_hw_number_t hw) +{ + irq_set_chip_data(virq, d->host_data); + irq_set_chip_and_handler(virq, &dockchannel_irqchip, handle_level_irq); + + return 0; +} + +static const struct irq_domain_ops dockchannel_irq_domain_ops = { + .xlate = irq_domain_xlate_twocell, + .map = dockchannel_irq_domain_map, +}; + +static int dockchannel_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct dockchannel_common *dcc; + struct device_node *child; + + dcc = devm_kzalloc(dev, sizeof(*dcc), GFP_KERNEL); + if (!dcc) + return -ENOMEM; + + dcc->dev = dev; + platform_set_drvdata(pdev, dcc); + + dcc->irq_base = devm_platform_ioremap_resource_byname(pdev, "irq"); + if (IS_ERR(dcc->irq_base)) + return PTR_ERR(dcc->irq_base); + + writel_relaxed(0, dcc->irq_base + IRQ_MASK); + writel_relaxed(~0, dcc->irq_base + IRQ_FLAG); + + dcc->domain = irq_domain_add_linear(dev->of_node, DOCKCHANNEL_MAX_IRQ, + &dockchannel_irq_domain_ops, dcc); + if (!dcc->domain) + return -ENOMEM; + + dcc->irq = platform_get_irq(pdev, 0); + if (dcc->irq <= 0) + return dev_err_probe(dev, dcc->irq, "Failed to get IRQ"); + + irq_set_handler_data(dcc->irq, dcc); + irq_set_chained_handler(dcc->irq, dockchannel_irq); + + devm_of_platform_populate(dev); + + return 0; +} + +static void dockchannel_remove(struct platform_device *pdev) +{ + struct dockchannel_common *dcc = platform_get_drvdata(pdev); + int hwirq; + + device_for_each_child(&pdev->dev, NULL, of_platform_device_destroy); + + irq_set_chained_handler_and_data(dcc->irq, NULL, NULL); + + for (hwirq = 0; hwirq < DOCKCHANNEL_MAX_IRQ; hwirq++) + irq_dispose_mapping(irq_find_mapping(dcc->domain, hwirq)); + + irq_domain_remove(dcc->domain); + + writel_relaxed(0, dcc->irq_base + IRQ_MASK); + writel_relaxed(~0, dcc->irq_base + IRQ_FLAG); +} + +static const struct of_device_id dockchannel_of_match[] = { + { .compatible = "apple,dockchannel" }, + {}, +}; +MODULE_DEVICE_TABLE(of, dockchannel_of_match); + +static struct platform_driver dockchannel_driver = { + .driver = { + .name = "dockchannel", + .of_match_table = dockchannel_of_match, + }, + .probe = dockchannel_probe, + .remove = dockchannel_remove, +}; +module_platform_driver(dockchannel_driver); + +MODULE_AUTHOR("Hector Martin "); +MODULE_LICENSE("Dual MIT/GPL"); +MODULE_DESCRIPTION("Apple DockChannel driver"); diff --git a/drivers/soc/apple/mailbox.c b/drivers/soc/apple/mailbox.c index 5c48455185c9ba..330015eea2ebe4 100644 --- a/drivers/soc/apple/mailbox.c +++ b/drivers/soc/apple/mailbox.c @@ -28,9 +28,9 @@ #include #include #include +#include #include #include -#include "mailbox.h" #define APPLE_ASC_MBOX_CONTROL_FULL BIT(16) #define APPLE_ASC_MBOX_CONTROL_EMPTY BIT(17) diff --git a/drivers/soc/apple/pmp.rs b/drivers/soc/apple/pmp.rs new file mode 100644 index 00000000000000..10524b2d19794b --- /dev/null +++ b/drivers/soc/apple/pmp.rs @@ -0,0 +1,431 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#![recursion_limit = "2048"] + +//! Apple PMP driver +//! +//! Copyright (C) The Asahi Linux Contributors + +use core::{ + mem, + slice, // +}; + +use kernel::{ + bindings, + device::{ + self, + Core, // + }, + devres::Devres, + dma::CoherentAllocation, + io::mem::IoMem, + iosys_map::IoSysMapRef, + kvec, + module_platform_driver, + new_mutex, + of, + platform, + prelude::*, + soc::apple::rtkit, + str::CString, + sync::{ + Arc, + Mutex, // + }, + types::{ + ARef, + ForeignOwnable, // + }, // +}; + +const PMP_MMIO_SIZE: usize = 0x80000; +const ASC_MMIO_SIZE: usize = 0x4000; +const BOOTARGS_OFFSET: usize = 0x22c; +const BOOTARGS_SIZE: usize = 0x230; +const CPU_CONTROL: usize = 0x44; +const CPU_RUN: u32 = 0x1 << 4; +const PMP_ENDPOINT: u8 = 0x20; +const OPC_GET_IOVA_TABLE: u64 = 0x10; +const OPC_MALLOC: u64 = 0x12; +const OPC_FREE: u64 = 0x14; +const OPC_SET_BUF: u64 = 0x30; +const OPC_REGISTER_IOREG: u64 = 0x32; +const OPC_SET_IOREG: u64 = 0x34; +const OPC_ACK_MASK: u64 = 0x1; +const OPC_SHIFT: u32 = 48; +const MALLOC_SIZE_MASK: u64 = 0xFFFFFF; +const MSG_IOVA_MASK: u64 = 0xFFFFFFFFFFFF; +const SET_IOREG_INDEX_MASK: u64 = 0xFFFF; +const PIO_VM_BASE: u64 = 0xc0000000; +const PIO_GRANULARITY: u64 = 0x1000000; + +const fn from_fourcc(b: &[u8; 4]) -> u32 { + b[3] as u32 | (b[2] as u32) << 8 | (b[1] as u32) << 16 | (b[0] as u32) << 24 +} + +struct PmpAllocation { + addr: u64, + alloc: CoherentAllocation, +} + +struct PmpState { + iova_table: Option>, + allocs: KVec, + value_buf: Option, + ioreg_entries: KVec, +} + +impl PmpState { + fn new() -> Result { + Ok(PmpState { + iova_table: None, + allocs: KVec::with_capacity(10, GFP_KERNEL)?, + value_buf: None, + ioreg_entries: KVec::with_capacity(340, GFP_KERNEL)?, + }) + } + fn find_alloc(&self, addr: u64) -> Option { + // Due to how pmp manages memory, iterating in reverse will + // usually result in us getting the right one on the first try + for (i, e) in self.allocs.iter().enumerate().rev() { + if e.addr == addr { + return Some(i); + } + } + None + } + fn get_buf(&mut self, addr: u64) -> Option<&mut CoherentAllocation> { + let idx = self.find_alloc(addr)?; + Some(&mut self.allocs[idx].alloc) + } +} + +#[pin_data] +struct PmpData { + dev: ARef, + pmp_mmio: Pin>>>, + asc_mmio: Pin>>>, + #[pin] + rtkit: Mutex>>, + #[pin] + state: Mutex, +} + +impl PmpData { + fn new(dev: &platform::Device) -> Result> { + let pmp_req = dev.io_request_by_name(c"pmp").ok_or(EINVAL)?; + let pmp_mmio = KBox::pin_init(pmp_req.iomap_sized::(), GFP_KERNEL)?; + let asc_req = dev.io_request_by_name(c"asc").ok_or(EINVAL)?; + let asc_mmio = KBox::pin_init(asc_req.iomap_sized::(), GFP_KERNEL)?; + Arc::pin_init( + try_pin_init!( + PmpData { + dev: dev.as_ref().into(), + pmp_mmio, + asc_mmio, + rtkit <- new_mutex!(None), + state <- new_mutex!(PmpState::new()?) + } + ), + GFP_KERNEL, + ) + } + fn start_cpu(&self, dev: &platform::Device) -> Result<()> { + let asc_mmio = self.asc_mmio.access(dev.as_ref())?; + let val = asc_mmio.read32_relaxed(CPU_CONTROL); + asc_mmio.write32_relaxed(val | CPU_RUN, CPU_CONTROL); + Ok(()) + } + fn start(&self) -> Result<()> { + let mut guard = self.rtkit.lock(); + let mut rtk = guard.as_mut().as_pin_mut().unwrap(); + rtk.as_mut().wake()?; + rtk.start_endpoint(PMP_ENDPOINT) + } + fn patch_bootargs(&self, dev: &platform::Device, patches: &[(u32, u32)]) -> Result<()> { + let io = self.pmp_mmio.access(dev.as_ref())?; + let offset = io.read32_relaxed(BOOTARGS_OFFSET) as usize; + let size = io.read32_relaxed(BOOTARGS_SIZE) as usize; + let mut arg_bytes = kvec![0u8; size]?; + io.try_memcpy_fromio(&mut arg_bytes, offset)?; + let mut idx = 0; + while idx < size { + let key = u32::from_le_bytes(arg_bytes[idx..idx + 4].try_into().unwrap()); + let size = u32::from_le_bytes(arg_bytes[idx + 4..idx + 8].try_into().unwrap()) as usize; + idx += 8; + for (k, v) in patches.iter() { + if *k != key { + continue; + } + arg_bytes[idx..idx + size].copy_from_slice(&(*v as u64).to_le_bytes()[..size]); + break; + } + idx += size; + } + io.try_memcpy_toio(offset, &arg_bytes) + } + fn get_iova_table(&self) -> Result { + let mut state = self.state.lock(); + if state.iova_table.is_some() { + dev_err!(self.dev, "Asked for iova table with existing buffer"); + return Err(EIO); + } + let node = self.dev.fwnode().ok_or(EIO)?; + let mut pio_base = PIO_VM_BASE; + let prop_name = c"apple,pio-ranges"; + if !node.property_present(prop_name) { + return Ok((OPC_GET_IOVA_TABLE | OPC_ACK_MASK) << OPC_SHIFT); + } + let n_entries = node.property_count_elem::(prop_name)? / 2; + let ranges = node + .property_read_array_vec::(prop_name, n_entries * 2)? + .required_by(&self.dev)?; + let mut table = self.dev.while_bound_with(|bound_dev| { + CoherentAllocation::alloc_coherent(bound_dev, 512, GFP_KERNEL) + })?; + for i in 0..table.count() { + unsafe { table.write(&[0], i)? }; + } + + let domain = unsafe { bindings::iommu_get_domain_for_dev(self.dev.as_raw()) }; + for i in 0..n_entries { + let host_addr = ranges[i * 2]; + let size = ranges[i * 2 + 1]; + unsafe { + let err = bindings::iommu_map( + domain, + pio_base as usize, + host_addr, + size as usize, + (bindings::IOMMU_READ | bindings::IOMMU_WRITE | bindings::IOMMU_MMIO) as i32, + bindings::GFP_KERNEL, + ); + if err != 0 { + return Err(Error::from_errno(err)); + } + } + unsafe { table.write(&[host_addr, pio_base, size], i * 3)? }; + pio_base += PIO_GRANULARITY; + } + let msg = (OPC_GET_IOVA_TABLE | OPC_ACK_MASK) << OPC_SHIFT | table.dma_handle(); + state.iova_table = Some(table); + Ok(msg) + } + fn malloc(&self, size: u64) -> Result { + let iomem = self.dev.while_bound_with(|bound_dev| { + CoherentAllocation::alloc_coherent(bound_dev, size as usize, GFP_KERNEL) + })?; + let mut state = self.state.lock(); + let addr = iomem.dma_handle(); + let msg = (OPC_MALLOC | OPC_ACK_MASK) << OPC_SHIFT | addr; + state.allocs.push( + PmpAllocation { + addr: addr, + alloc: iomem, + }, + GFP_KERNEL, + )?; + Ok(msg) + } + fn free(&self, addr: u64) -> Result { + let mut state = self.state.lock(); + if let Some(idx) = state.find_alloc(addr) { + state.allocs.swap_remove(idx); + } else { + dev_err!( + self.dev, + "Attempted to free memory that was not allocated {}", + addr + ); + return Err(EIO); + } + let msg = (OPC_FREE | OPC_ACK_MASK) << OPC_SHIFT; + Ok(msg) + } + fn set_buf(&self, addr: u64) -> Result { + let mut state = self.state.lock(); + if state.value_buf.is_some() { + dev_err!(self.dev, "Setting a buffer when one exists"); + return Err(EIO); + } + let ptr_buf = if let Some(s) = state.get_buf(addr) { + s + } else { + dev_err!(self.dev, "Unable to find buffer"); + return Err(EIO); + }; + if ptr_buf.count() < mem::size_of::() { + dev_err!(self.dev, "Buffer too small"); + return Err(EIO); + } + let ptr = unsafe { *(ptr_buf.start_ptr() as *const u64) }; + state.value_buf = Some(ptr); + let msg = (OPC_SET_BUF | OPC_ACK_MASK) << OPC_SHIFT; + Ok(msg) + } + fn register_ioreg(&self, addr: u64) -> Result { + let mut state = self.state.lock(); + let msg_buf = if let Some(s) = state.get_buf(addr) { + s + } else { + dev_err!(self.dev, "Unable to find buffer"); + return Err(EIO); + }; + if msg_buf.count() < 0x44 { + dev_err!(self.dev, "Buffer too small"); + return Err(EIO); + } + let mut size = unsafe { *(msg_buf.start_ptr().offset(0x40) as *const u32) }; + if size == 0 { + let mut name_vec = KVec::with_capacity(0x31, GFP_KERNEL)?; + name_vec + .extend_from_slice( + unsafe { slice::from_raw_parts(msg_buf.start_ptr(), 0x30) }, + GFP_KERNEL, + ) + .unwrap(); + name_vec.push(0, GFP_KERNEL).unwrap(); + let name_str = CStr::from_bytes_until_nul(&name_vec).unwrap(); + let name_str = CString::try_from_fmt(fmt!("apple,tunable-{name_str}"))?; + let node = self.dev.fwnode().ok_or(EIO)?; + if state.value_buf.is_none() { + dev_err!(self.dev, "Value buf not set"); + return Err(EIO); + } + let val_buf_addr = state.value_buf.unwrap(); + let val_buf = if let Some(s) = state.get_buf(val_buf_addr) { + s + } else { + dev_err!(self.dev, "Unable to find value buffer"); + return Err(EIO); + }; + if node.property_present(&name_str) { + let len = node.property_count_elem::(&name_str)?; + let data = node + .property_read_array_vec::(&name_str, len)? + .required_by(&self.dev)?; + unsafe { + slice::from_raw_parts_mut(val_buf.start_ptr_mut(), len).copy_from_slice(&data); + } + size = len as u32; + } else { + dev_info!(self.dev, "unknown property {:?}", name_str); + } + } + state.ioreg_entries.push(size, GFP_KERNEL)?; + let index = state.ioreg_entries.len() as u64; + let msg = (OPC_REGISTER_IOREG | OPC_ACK_MASK) << OPC_SHIFT | (index << 32) | size as u64; + Ok(msg) + } + fn set_ioreg(&self, index: u64) -> Result { + let len = *self + .state + .lock() + .ioreg_entries + .get(index as usize) + .ok_or(EIO)? as u64; + let msg = (OPC_SET_IOREG | OPC_ACK_MASK) << OPC_SHIFT | len; + Ok(msg) + } + fn recv_message(&self, msg: u64) -> Result<()> { + let opc = (msg >> OPC_SHIFT) & 0xFF; + let reply = match opc { + OPC_GET_IOVA_TABLE => self.get_iova_table()?, + OPC_MALLOC => self.malloc(msg & MALLOC_SIZE_MASK)?, + OPC_FREE => self.free(msg & MSG_IOVA_MASK)?, + OPC_SET_BUF => self.set_buf(msg & MSG_IOVA_MASK)?, + OPC_REGISTER_IOREG => self.register_ioreg(msg & MSG_IOVA_MASK)?, + OPC_SET_IOREG => self.set_ioreg(msg & SET_IOREG_INDEX_MASK)?, + _ => { + dev_err!(self.dev, "Got unknown message {}", msg); + return Err(EIO); + } + }; + let mut rtk_guard = self.rtkit.lock(); + let rtk = rtk_guard.as_mut().as_pin_mut().unwrap(); + rtk.send_message(PMP_ENDPOINT, reply)?; + Ok(()) + } +} + +unsafe impl Send for PmpData {} +unsafe impl Sync for PmpData {} + +struct NoBuffer; +impl rtkit::Buffer for NoBuffer { + fn iova(&self) -> Result { + unreachable!() + } + fn buf(&mut self) -> Result> { + unreachable!() + } +} + +#[vtable] +impl rtkit::Operations for PmpData { + type Data = Arc; + type Buffer = NoBuffer; + + fn recv_message(data: ::Borrowed<'_>, _ep: u8, msg: u64) { + let ret = data.recv_message(msg); + if let Err(e) = ret { + dev_err!(data.dev, "Failed to handle rtkit message, error: {:?}", e); + } + } + + fn crashed(data: ::Borrowed<'_>, _crashlog: Option<&[u8]>) { + dev_err!(data.dev, "PMP firmware crashed"); + } +} + +#[allow(dead_code)] +struct PmpDriver(Arc); + +kernel::of_device_table!( + OF_TABLE, + MODULE_OF_TABLE, + (), + [(of::DeviceId::new(c"apple,t6000-pmp-v2"), ())] +); + +impl platform::Driver for PmpDriver { + type IdInfo = (); + + const OF_ID_TABLE: Option> = Some(&OF_TABLE); + + fn probe(pdev: &platform::Device, _info: Option<&()>) -> impl PinInit { + let dev: ARef = pdev.as_ref().into(); + let data = PmpData::new(pdev)?; + let node = dev.fwnode().ok_or(EIO)?; + let dvid = node + .property_read(c"apple,dram-vendor-id") + .required_by(&dev)?; + let bdid = node.property_read(c"apple,board-id").required_by(&dev)?; + match node.property_read(c"apple,dram-capacity").optional() { + Some(dcap) => data.patch_bootargs( + pdev, + &[ + (from_fourcc(b"BDID"), bdid), + (from_fourcc(b"DCAP"), dcap), + (from_fourcc(b"DVID"), dvid), + ], + )?, + None => data.patch_bootargs( + pdev, + &[(from_fourcc(b"BDID"), bdid), (from_fourcc(b"DVID"), dvid)], + )?, + }; + let rtkit = rtkit::RtKit::::new(&dev, None, 0, data.clone())?; + *data.rtkit.lock() = Some(rtkit); + data.start_cpu(pdev)?; + data.start()?; + Ok(PmpDriver(data)) + } +} + +module_platform_driver! { + type: PmpDriver, + name: "apple_pmp", + description: "Apple Power Management Processor", + license: "Dual MIT/GPL", +} diff --git a/drivers/soc/apple/rtkit-helper.c b/drivers/soc/apple/rtkit-helper.c new file mode 100644 index 00000000000000..080d083ed9bd2f --- /dev/null +++ b/drivers/soc/apple/rtkit-helper.c @@ -0,0 +1,151 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Apple Generic RTKit helper coprocessor + * Copyright The Asahi Linux Contributors + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#define APPLE_ASC_CPU_CONTROL 0x44 +#define APPLE_ASC_CPU_CONTROL_RUN BIT(4) + +struct apple_rtkit_helper { + struct device *dev; + struct apple_rtkit *rtk; + + void __iomem *asc_base; + + struct resource *sram; + void __iomem *sram_base; +}; + +static int apple_rtkit_helper_shmem_setup(void *cookie, struct apple_rtkit_shmem *bfr) +{ + struct apple_rtkit_helper *helper = cookie; + struct resource res = { + .start = bfr->iova, + .end = bfr->iova + bfr->size - 1, + .name = "rtkit_map", + }; + + if (!bfr->iova) { + bfr->buffer = dma_alloc_coherent(helper->dev, bfr->size, + &bfr->iova, GFP_KERNEL); + if (!bfr->buffer) + return -ENOMEM; + return 0; + } + + if (!helper->sram) { + dev_err(helper->dev, + "RTKit buffer request with no SRAM region: %pR", &res); + return -EFAULT; + } + + res.flags = helper->sram->flags; + + if (res.end < res.start || !resource_contains(helper->sram, &res)) { + dev_err(helper->dev, + "RTKit buffer request outside SRAM region: %pR", &res); + return -EFAULT; + } + + bfr->iomem = helper->sram_base + (res.start - helper->sram->start); + bfr->is_mapped = true; + + return 0; +} + +static void apple_rtkit_helper_shmem_destroy(void *cookie, struct apple_rtkit_shmem *bfr) +{ + // no-op +} + +static const struct apple_rtkit_ops apple_rtkit_helper_ops = { + .shmem_setup = apple_rtkit_helper_shmem_setup, + .shmem_destroy = apple_rtkit_helper_shmem_destroy, +}; + +static int apple_rtkit_helper_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct apple_rtkit_helper *helper; + int ret; + + /* 44 bits for addresses in standard RTKit requests */ + ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(44)); + if (ret) + return ret; + + helper = devm_kzalloc(dev, sizeof(*helper), GFP_KERNEL); + if (!helper) + return -ENOMEM; + + helper->dev = dev; + platform_set_drvdata(pdev, helper); + + helper->asc_base = devm_platform_ioremap_resource_byname(pdev, "asc"); + if (IS_ERR(helper->asc_base)) + return PTR_ERR(helper->asc_base); + + helper->sram = platform_get_resource_byname(pdev, IORESOURCE_MEM, "sram"); + if (helper->sram) { + helper->sram_base = devm_ioremap_resource(dev, helper->sram); + if (IS_ERR(helper->sram_base)) + return dev_err_probe(dev, PTR_ERR(helper->sram_base), + "Failed to map SRAM region"); + } + + helper->rtk = + devm_apple_rtkit_init(dev, helper, NULL, 0, &apple_rtkit_helper_ops); + if (IS_ERR(helper->rtk)) + return dev_err_probe(dev, PTR_ERR(helper->rtk), + "Failed to intialize RTKit"); + + writel_relaxed(APPLE_ASC_CPU_CONTROL_RUN, + helper->asc_base + APPLE_ASC_CPU_CONTROL); + + /* Works for both wake and boot */ + ret = apple_rtkit_wake(helper->rtk); + if (ret != 0) + return dev_err_probe(dev, ret, "Failed to wake up coprocessor"); + + return 0; +} + +static void apple_rtkit_helper_remove(struct platform_device *pdev) +{ + struct apple_rtkit_helper *helper = platform_get_drvdata(pdev); + + if (apple_rtkit_is_running(helper->rtk)) + apple_rtkit_quiesce(helper->rtk); + + writel_relaxed(0, helper->asc_base + APPLE_ASC_CPU_CONTROL); +} + +static const struct of_device_id apple_rtkit_helper_of_match[] = { + { .compatible = "apple,rtk-helper-asc4" }, + {}, +}; +MODULE_DEVICE_TABLE(of, apple_rtkit_helper_of_match); + +static struct platform_driver apple_rtkit_helper_driver = { + .driver = { + .name = "rtkit-helper", + .of_match_table = apple_rtkit_helper_of_match, + }, + .probe = apple_rtkit_helper_probe, + .remove = apple_rtkit_helper_remove, +}; +module_platform_driver(apple_rtkit_helper_driver); + +MODULE_AUTHOR("Hector Martin "); +MODULE_LICENSE("Dual MIT/GPL"); +MODULE_DESCRIPTION("Apple RTKit helper driver"); diff --git a/drivers/soc/apple/rtkit-internal.h b/drivers/soc/apple/rtkit-internal.h index b8d5244678f010..c82065a8bf7b03 100644 --- a/drivers/soc/apple/rtkit-internal.h +++ b/drivers/soc/apple/rtkit-internal.h @@ -15,9 +15,9 @@ #include #include #include +#include #include #include -#include "mailbox.h" #define APPLE_RTKIT_APP_ENDPOINT_START 0x20 #define APPLE_RTKIT_MAX_ENDPOINTS 0x100 diff --git a/drivers/soc/apple/rtkit.c b/drivers/soc/apple/rtkit.c index b8d4da147d23f7..67a21b21a30d98 100644 --- a/drivers/soc/apple/rtkit.c +++ b/drivers/soc/apple/rtkit.c @@ -22,6 +22,7 @@ enum { APPLE_RTKIT_EP_DEBUG = 3, APPLE_RTKIT_EP_IOREPORT = 4, APPLE_RTKIT_EP_OSLOG = 8, + APPLE_RTKIT_EP_TRACEKIT = 0xa, }; #define APPLE_RTKIT_MGMT_TYPE GENMASK_ULL(59, 52) @@ -191,6 +192,7 @@ static void apple_rtkit_management_rx_epmap(struct apple_rtkit *rtk, u64 msg) case APPLE_RTKIT_EP_DEBUG: case APPLE_RTKIT_EP_IOREPORT: case APPLE_RTKIT_EP_OSLOG: + case APPLE_RTKIT_EP_TRACEKIT: dev_dbg(rtk->dev, "RTKit: Starting system endpoint 0x%02x\n", ep); apple_rtkit_start_ep(rtk, ep); @@ -361,7 +363,7 @@ static void apple_rtkit_memcpy(struct apple_rtkit *rtk, void *dst, static void apple_rtkit_crashlog_rx(struct apple_rtkit *rtk, u64 msg) { u8 type = FIELD_GET(APPLE_RTKIT_SYSLOG_TYPE, msg); - u8 *bfr; + u8 *bfr __free(kfree) = NULL; if (type != APPLE_RTKIT_CRASHLOG_CRASH) { dev_warn(rtk->dev, "RTKit: Unknown crashlog message: %llx\n", @@ -394,9 +396,7 @@ static void apple_rtkit_crashlog_rx(struct apple_rtkit *rtk, u64 msg) rtk->crashed = true; if (rtk->ops->crashed) - rtk->ops->crashed(rtk->cookie, bfr, rtk->crashlog_buffer.size); - - kfree(bfr); + rtk->ops->crashed(rtk->cookie, bfr, bfr ? rtk->crashlog_buffer.size : 0); } static void apple_rtkit_ioreport_rx(struct apple_rtkit *rtk, u64 msg) @@ -639,6 +639,12 @@ int apple_rtkit_poll(struct apple_rtkit *rtk) } EXPORT_SYMBOL_GPL(apple_rtkit_poll); +bool apple_rtkit_has_endpoint(struct apple_rtkit *rtk, u8 ep) +{ + return test_bit(ep, rtk->endpoints); +} +EXPORT_SYMBOL_GPL(apple_rtkit_has_endpoint); + int apple_rtkit_start_ep(struct apple_rtkit *rtk, u8 endpoint) { u64 msg; @@ -958,6 +964,12 @@ struct apple_rtkit *devm_apple_rtkit_init(struct device *dev, void *cookie, } EXPORT_SYMBOL_GPL(devm_apple_rtkit_init); +void devm_apple_rtkit_free(struct device *dev, struct apple_rtkit *rtk) +{ + devm_release_action(dev, apple_rtkit_free_wrapper, rtk); +} +EXPORT_SYMBOL_GPL(devm_apple_rtkit_free); + MODULE_LICENSE("Dual MIT/GPL"); MODULE_AUTHOR("Sven Peter "); MODULE_DESCRIPTION("Apple RTKit driver"); diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs new file mode 100644 index 00000000000000..24d16dd80958ad --- /dev/null +++ b/drivers/soc/apple/sep.rs @@ -0,0 +1,353 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#![recursion_limit = "2048"] + +//! Apple SEP driver +//! +//! Copyright (C) The Asahi Linux Contributors + +use core::sync::atomic::{AtomicBool, Ordering}; + +use kernel::{ + bindings, c_str, device, dma, module_platform_driver, new_mutex, of, platform, + prelude::*, + soc::apple::mailbox::{MailCallback, Mailbox, Message}, + sync::{Arc, Mutex}, + types::{ARef, ForeignOwnable}, + workqueue::{self, impl_has_work, new_work, Work, WorkItem}, +}; + +const SHMEM_SIZE: usize = 0x30000; +const MSG_BOOT_TZ0: u64 = 0x5; +const MSG_BOOT_IMG4: u64 = 0x6; +const MSG_SET_SHMEM: u64 = 0x18; +const MSG_BOOT_TZ0_ACK1: u64 = 0x69; +const MSG_BOOT_TZ0_ACK2: u64 = 0xD2; +const MSG_BOOT_IMG4_ACK: u64 = 0x6A; +const MSG_ADVERTISE_EP: u64 = 0; +const EP_DISCOVER: u64 = 0xFD; +const EP_SHMEM: u64 = 0xFE; +const EP_BOOT: u64 = 0xFF; + +const MSG_TYPE_SHIFT: u32 = 16; +const MSG_TYPE_MASK: u64 = 0xFF; +//const MSG_PARAM_SHIFT: u32 = 24; +//const MSG_PARAM_MASK: u64 = 0xFF; + +const MSG_EP_MASK: u64 = 0xFF; +const MSG_DATA_SHIFT: u32 = 32; + +const IOVA_SHIFT: u32 = 0xC; + +type ShMem = dma::CoherentAllocation; + +fn align_up(v: usize, a: usize) -> usize { + (v + a - 1) & !(a - 1) +} + +fn memcpy_to_iomem(iomem: &mut ShMem, off: usize, src: &[u8]) -> Result<()> { + // SAFETY: + // as_slice_mut() checks that off and src.len() are whithin iomem's limits. + // memcpy_to_iomem is only called from within probe() ansuring there are no + // concurrent read and write accesses to the same region while the slice is + // alive per as_slice_mut()'s requiremnts. + unsafe { + let target = iomem.as_slice_mut(off, src.len())?; + target.copy_from_slice(src); + } + Ok(()) +} + +fn build_shmem(dev: &platform::Device) -> Result { + let fwnode = dev.as_ref().fwnode().ok_or(EIO)?; + let mut iomem = + dma::CoherentAllocation::::alloc_coherent(dev.as_ref(), SHMEM_SIZE, GFP_KERNEL)?; + + let panic_offset = 0x4000; + let panic_size = 0x8000; + memcpy_to_iomem(&mut iomem, panic_offset, &1u32.to_le_bytes())?; + + let lpol_offset = panic_offset + panic_size; + let lpol_prop_name = c_str!("local-policy-manifest"); + let lpol_prop_size = fwnode.property_count_elem::(lpol_prop_name)?; + let lpol = fwnode + .property_read_array_vec(lpol_prop_name, lpol_prop_size)? + .required_by(dev.as_ref())?; + memcpy_to_iomem( + &mut iomem, + lpol_offset, + &(lpol_prop_size as u32).to_le_bytes(), + )?; + memcpy_to_iomem(&mut iomem, lpol_offset + 4, &lpol)?; + let lpol_size = align_up(lpol_prop_size + 4, 0x4000); + + let ibot_offset = lpol_offset + lpol_size; + let ibot_prop_name = c_str!("iboot-manifest"); + let ibot_prop_size = fwnode.property_count_elem::(ibot_prop_name)?; + let ibot = fwnode + .property_read_array_vec(ibot_prop_name, ibot_prop_size)? + .required_by(dev.as_ref())?; + memcpy_to_iomem( + &mut iomem, + ibot_offset, + &(ibot_prop_size as u32).to_le_bytes(), + )?; + memcpy_to_iomem(&mut iomem, ibot_offset + 4, &ibot)?; + let ibot_size = align_up(ibot_prop_size + 4, 0x4000); + + memcpy_to_iomem(&mut iomem, 0, b"CNIP")?; + memcpy_to_iomem(&mut iomem, 4, &(panic_size as u32).to_le_bytes())?; + memcpy_to_iomem(&mut iomem, 8, &(panic_offset as u32).to_le_bytes())?; + + memcpy_to_iomem(&mut iomem, 16, b"OPLA")?; + memcpy_to_iomem(&mut iomem, 16 + 4, &(lpol_size as u32).to_le_bytes())?; + memcpy_to_iomem(&mut iomem, 16 + 8, &(lpol_offset as u32).to_le_bytes())?; + + memcpy_to_iomem(&mut iomem, 32, b"IPIS")?; + memcpy_to_iomem(&mut iomem, 32 + 4, &(ibot_size as u32).to_le_bytes())?; + memcpy_to_iomem(&mut iomem, 32 + 8, &(ibot_offset as u32).to_le_bytes())?; + + memcpy_to_iomem(&mut iomem, 48, b"llun")?; + Ok(iomem) +} + +#[pin_data] +struct SepReceiveWork { + data: Arc, + msg: Message, + #[pin] + work: Work, +} + +impl_has_work! { + impl HasWork for SepReceiveWork { self.work } +} + +impl SepReceiveWork { + fn new(data: Arc, msg: Message) -> Result> { + Arc::pin_init( + pin_init!(SepReceiveWork { + data, + msg, + work <- new_work!("SepReceiveWork::work"), + }), + GFP_ATOMIC, + ) + } +} + +impl WorkItem for SepReceiveWork { + type Pointer = Arc; + + fn run(this: Arc) { + this.data.process_message(this.msg); + } +} + +struct FwRegionParams { + addr: u64, + size: usize, +} + +#[pin_data] +struct SepData { + dev: ARef, + #[pin] + mbox: Mutex>>, + shmem: ShMem, + region_params: FwRegionParams, + fw_mapped: AtomicBool, +} + +impl SepData { + fn new( + dev: &platform::Device, + region_params: FwRegionParams, + ) -> Result> { + Arc::pin_init( + try_pin_init!(SepData { + shmem: build_shmem(dev)?, + dev: ARef::::from(dev.as_ref()), + mbox <- new_mutex!(None), + region_params, + fw_mapped: AtomicBool::new(false), + }), + GFP_KERNEL, + ) + } + fn start(&self) -> Result<()> { + self.mbox.lock().as_ref().unwrap().send( + Message { + msg0: EP_BOOT | (MSG_BOOT_TZ0 << MSG_TYPE_SHIFT), + msg1: 0, + }, + false, + ) + } + fn load_fw_and_shmem(&self) -> Result<()> { + let fw_addr = unsafe { + let res = bindings::dma_map_resource( + self.dev.as_raw(), + self.region_params.addr, + self.region_params.size, + bindings::dma_data_direction_DMA_TO_DEVICE, + 0, + ); + if bindings::dma_mapping_error(self.dev.as_raw(), res) != 0 { + dev_err!(self.dev, "Failed to map firmware"); + return Err(ENOMEM); + } + self.fw_mapped.store(true, Ordering::Relaxed); + res >> IOVA_SHIFT + }; + let guard = self.mbox.lock(); + let mbox = guard.as_ref().unwrap(); + mbox.send( + Message { + msg0: EP_BOOT | (MSG_BOOT_IMG4 << MSG_TYPE_SHIFT) | (fw_addr << MSG_DATA_SHIFT), + msg1: 0, + }, + false, + )?; + let shm_addr = self.shmem.dma_handle() >> IOVA_SHIFT; + mbox.send( + Message { + msg0: EP_SHMEM | (MSG_SET_SHMEM << MSG_TYPE_SHIFT) | (shm_addr << MSG_DATA_SHIFT), + msg1: 0, + }, + false, + )?; + Ok(()) + } + fn process_boot_msg(&self, msg: Message) { + let ty = (msg.msg0 >> MSG_TYPE_SHIFT) & MSG_TYPE_MASK; + match ty { + MSG_BOOT_TZ0_ACK1 => {} + MSG_BOOT_TZ0_ACK2 => { + let res = self.load_fw_and_shmem(); + if let Err(e) = res { + dev_err!(self.dev, "Unable to load firmware: {:?}", e); + } + } + MSG_BOOT_IMG4_ACK => {} + _ => { + dev_err!(self.dev, "Unknown boot message type: {}", ty); + } + } + } + fn process_discover_msg(&self, msg: Message) { + let ty = (msg.msg0 >> MSG_TYPE_SHIFT) & MSG_TYPE_MASK; + //let data = (msg.msg0 >> MSG_DATA_SHIFT) as u32; + //let param = (msg.msg0 >> MSG_PARAM_SHIFT) & MSG_PARAM_MASK; + match ty { + MSG_ADVERTISE_EP => { + /*dev_info!( + self.dev, + "Got endpoint {:?} at {}", + core::str::from_utf8(&data.to_be_bytes()), + param + );*/ + } + _ => { + //dev_warn!(self.dev, "Unknown discovery message type: {}", ty); + } + } + } + fn process_message(&self, msg: Message) { + let ep = msg.msg0 & MSG_EP_MASK; + match ep { + EP_BOOT => self.process_boot_msg(msg), + EP_DISCOVER => self.process_discover_msg(msg), + _ => {} // dev_warn!(self.dev, "Message from unknown endpoint: {}", ep), + } + } + fn remove(&self) { + *self.mbox.lock() = None; + if self.fw_mapped.load(Ordering::Relaxed) { + unsafe { + bindings::dma_unmap_resource( + self.dev.as_raw(), + self.region_params.addr, + self.region_params.size, + bindings::dma_data_direction_DMA_TO_DEVICE, + 0, + ); + } + } + } +} + +impl MailCallback for SepData { + type Data = Arc; + fn recv_message(data: ::Borrowed<'_>, msg: Message) { + let work = SepReceiveWork::new(data.into(), msg); + if let Ok(work) = work { + let res = workqueue::system().enqueue(work); + if res.is_err() { + dev_err!( + data.dev, + "Unable to schedule work item for message {}", + msg.msg0 + ); + } + } else { + dev_err!( + data.dev, + "Unable to allocate work item for message {}", + msg.msg0 + ); + } + } +} + +unsafe impl Send for SepData {} +unsafe impl Sync for SepData {} + +struct SepDriver(Arc); + +kernel::of_device_table!( + OF_TABLE, + MODULE_OF_TABLE, + (), + [(of::DeviceId::new(c_str!("apple,sep")), ())] +); + +impl platform::Driver for SepDriver { + type IdInfo = (); + + const OF_ID_TABLE: Option> = Some(&OF_TABLE); + + fn probe( + pdev: &platform::Device, + _info: Option<&()>, + ) -> impl PinInit { + let of = pdev.as_ref().of_node().ok_or(EIO)?; + let res = of.reserved_mem_region_to_resource_byname(c_str!("sepfw"))?; + let data = SepData::new( + pdev, + FwRegionParams { + addr: res.start(), + size: res.size().try_into()?, + }, + )?; + *data.mbox.lock() = Some(Mailbox::new_byname( + pdev.as_ref(), + c_str!("mbox"), + data.clone(), + )?); + data.start()?; + Ok(Self(data)) + } +} + +impl Drop for SepDriver { + fn drop(&mut self) { + self.0.remove(); + } +} + +module_platform_driver! { + type: SepDriver, + name: "apple_sep", + description: "Secure enclave processor stub driver", + license: "Dual MIT/GPL", +} diff --git a/drivers/soc/apple/tunable.c b/drivers/soc/apple/tunable.c new file mode 100644 index 00000000000000..6593238391715b --- /dev/null +++ b/drivers/soc/apple/tunable.c @@ -0,0 +1,80 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Apple Silicon hardware tunable support + * + * Each tunable is a list with each entry containing a offset into the MMIO + * region, a mask of bits to be cleared and a set of bits to be set. These + * tunables are passed along by the previous boot stages and vary from device + * to device such that they cannot be hardcoded in the individual drivers. + * + * Copyright (C) The Asahi Linux Contributors + */ + +#include +#include +#include +#include +#include + +struct apple_tunable *devm_apple_tunable_parse(struct device *dev, + struct device_node *np, + const char *name, + struct resource *res) +{ + struct apple_tunable *tunable; + struct property *prop; + const __be32 *p; + size_t sz; + int i; + + if (resource_size(res) < 4) + return ERR_PTR(-EINVAL); + + prop = of_find_property(np, name, NULL); + if (!prop) + return ERR_PTR(-ENOENT); + + if (prop->length % (3 * sizeof(u32))) + return ERR_PTR(-EINVAL); + sz = prop->length / (3 * sizeof(u32)); + + tunable = devm_kzalloc(dev, struct_size(tunable, values, sz), GFP_KERNEL); + if (!tunable) + return ERR_PTR(-ENOMEM); + tunable->sz = sz; + + for (i = 0, p = NULL; i < tunable->sz; ++i) { + p = of_prop_next_u32(prop, p, &tunable->values[i].offset); + p = of_prop_next_u32(prop, p, &tunable->values[i].mask); + p = of_prop_next_u32(prop, p, &tunable->values[i].value); + + /* Sanity checks to catch bugs in our bootloader */ + if (tunable->values[i].offset % 4) + return ERR_PTR(-EINVAL); + if (tunable->values[i].offset > (resource_size(res) - 4)) + return ERR_PTR(-EINVAL); + } + + return tunable; +} +EXPORT_SYMBOL(devm_apple_tunable_parse); + +void apple_tunable_apply(void __iomem *regs, struct apple_tunable *tunable) +{ + size_t i; + + for (i = 0; i < tunable->sz; ++i) { + u32 val, old_val; + + old_val = readl(regs + tunable->values[i].offset); + val = old_val & ~tunable->values[i].mask; + val |= tunable->values[i].value; + if (val != old_val) + writel(val, regs + tunable->values[i].offset); + } +} +EXPORT_SYMBOL(apple_tunable_apply); + +MODULE_LICENSE("Dual MIT/GPL"); +MODULE_AUTHOR("Sven Peter "); +MODULE_DESCRIPTION("Apple Silicon hardware tunable support"); diff --git a/drivers/soc/aspeed/aspeed-socinfo.c b/drivers/soc/aspeed/aspeed-socinfo.c index 67e9ac3d08ecc8..a90b100f4d101b 100644 --- a/drivers/soc/aspeed/aspeed-socinfo.c +++ b/drivers/soc/aspeed/aspeed-socinfo.c @@ -39,7 +39,7 @@ static const char *siliconid_to_name(u32 siliconid) unsigned int i; for (i = 0 ; i < ARRAY_SIZE(rev_table) ; ++i) { - if (rev_table[i].id == id) + if ((rev_table[i].id & 0xff00ffff) == id) return rev_table[i].name; } diff --git a/drivers/soc/fsl/qbman/qman.c b/drivers/soc/fsl/qbman/qman.c index 6b392b3ad4b155..39a3e7aab6ff23 100644 --- a/drivers/soc/fsl/qbman/qman.c +++ b/drivers/soc/fsl/qbman/qman.c @@ -1827,6 +1827,8 @@ EXPORT_SYMBOL(qman_create_fq); void qman_destroy_fq(struct qman_fq *fq) { + int leaked; + /* * We don't need to lock the FQ as it is a pre-condition that the FQ be * quiesced. Instead, run some checks. @@ -1834,11 +1836,29 @@ void qman_destroy_fq(struct qman_fq *fq) switch (fq->state) { case qman_fq_state_parked: case qman_fq_state_oos: - if (fq_isset(fq, QMAN_FQ_FLAG_DYNAMIC_FQID)) - qman_release_fqid(fq->fqid); + /* + * There's a race condition here on releasing the fqid, + * setting the fq_table to NULL, and freeing the fqid. + * To prevent it, this order should be respected: + */ + if (fq_isset(fq, QMAN_FQ_FLAG_DYNAMIC_FQID)) { + leaked = qman_shutdown_fq(fq->fqid); + if (leaked) + pr_debug("FQID %d leaked\n", fq->fqid); + } DPAA_ASSERT(fq_table[fq->idx]); fq_table[fq->idx] = NULL; + + if (fq_isset(fq, QMAN_FQ_FLAG_DYNAMIC_FQID) && !leaked) { + /* + * fq_table[fq->idx] should be set to null before + * freeing fq->fqid otherwise it could by allocated by + * qman_alloc_fqid() while still being !NULL + */ + smp_wmb(); + gen_pool_free(qm_fqalloc, fq->fqid | DPAA_GENALLOC_OFF, 1); + } return; default: break; diff --git a/drivers/soc/fsl/qe/qmc.c b/drivers/soc/fsl/qe/qmc.c index da5ea6d3561840..6db5ab05c2c1c4 100644 --- a/drivers/soc/fsl/qe/qmc.c +++ b/drivers/soc/fsl/qe/qmc.c @@ -1799,8 +1799,8 @@ static int qmc_qe_init_resources(struct qmc *qmc, struct platform_device *pdev) return -EINVAL; qmc->dpram_offset = res->start - qe_muram_dma(qe_muram_addr(0)); qmc->dpram = devm_ioremap_resource(qmc->dev, res); - if (IS_ERR(qmc->scc_pram)) - return PTR_ERR(qmc->scc_pram); + if (IS_ERR(qmc->dpram)) + return PTR_ERR(qmc->dpram); return 0; } diff --git a/drivers/soc/imx/soc-imx8m.c b/drivers/soc/imx/soc-imx8m.c index 04a1b60f2f2b52..8e2322999f0996 100644 --- a/drivers/soc/imx/soc-imx8m.c +++ b/drivers/soc/imx/soc-imx8m.c @@ -148,7 +148,11 @@ static int imx8m_soc_prepare(struct platform_device *pdev, const char *ocotp_com goto err_clk; } - return clk_prepare_enable(drvdata->clk); + ret = clk_prepare_enable(drvdata->clk); + if (ret) + goto err_clk; + + return 0; err_clk: iounmap(drvdata->ocotp_base); diff --git a/drivers/soc/mediatek/mtk-svs.c b/drivers/soc/mediatek/mtk-svs.c index f45537546553ec..99edecb204f254 100644 --- a/drivers/soc/mediatek/mtk-svs.c +++ b/drivers/soc/mediatek/mtk-svs.c @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -789,7 +790,7 @@ static ssize_t svs_enable_debug_write(struct file *filp, struct svs_bank *svsb = file_inode(filp)->i_private; struct svs_platform *svsp = dev_get_drvdata(svsb->dev); int enabled, ret; - char *buf = NULL; + char *buf __free(kfree) = NULL; if (count >= PAGE_SIZE) return -EINVAL; @@ -807,8 +808,6 @@ static ssize_t svs_enable_debug_write(struct file *filp, svsb->mode_support = SVSB_MODE_ALL_DISABLE; } - kfree(buf); - return count; } diff --git a/drivers/soc/microchip/mpfs-control-scb.c b/drivers/soc/microchip/mpfs-control-scb.c index f0b84b1f49cbca..8dda5704a389fb 100644 --- a/drivers/soc/microchip/mpfs-control-scb.c +++ b/drivers/soc/microchip/mpfs-control-scb.c @@ -14,8 +14,10 @@ static int mpfs_control_scb_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; - return mfd_add_devices(dev, PLATFORM_DEVID_NONE, mpfs_control_scb_devs, - ARRAY_SIZE(mpfs_control_scb_devs), NULL, 0, NULL); + return devm_mfd_add_devices(dev, PLATFORM_DEVID_NONE, + mpfs_control_scb_devs, + ARRAY_SIZE(mpfs_control_scb_devs), NULL, 0, + NULL); } static const struct of_device_id mpfs_control_scb_of_match[] = { diff --git a/drivers/soc/microchip/mpfs-mss-top-sysreg.c b/drivers/soc/microchip/mpfs-mss-top-sysreg.c index b2244e44ff0fa7..b0f42b8dd3ed63 100644 --- a/drivers/soc/microchip/mpfs-mss-top-sysreg.c +++ b/drivers/soc/microchip/mpfs-mss-top-sysreg.c @@ -16,8 +16,10 @@ static int mpfs_mss_top_sysreg_probe(struct platform_device *pdev) struct device *dev = &pdev->dev; int ret; - ret = mfd_add_devices(dev, PLATFORM_DEVID_NONE, mpfs_mss_top_sysreg_devs, - ARRAY_SIZE(mpfs_mss_top_sysreg_devs) , NULL, 0, NULL); + ret = devm_mfd_add_devices(dev, PLATFORM_DEVID_NONE, + mpfs_mss_top_sysreg_devs, + ARRAY_SIZE(mpfs_mss_top_sysreg_devs), NULL, + 0, NULL); if (ret) return ret; diff --git a/drivers/soc/microchip/mpfs-sys-controller.c b/drivers/soc/microchip/mpfs-sys-controller.c index 30bc45d17d3434..81636cfecd37ee 100644 --- a/drivers/soc/microchip/mpfs-sys-controller.c +++ b/drivers/soc/microchip/mpfs-sys-controller.c @@ -142,8 +142,10 @@ static int mpfs_sys_controller_probe(struct platform_device *pdev) sys_controller->flash = of_get_mtd_device_by_node(np); of_node_put(np); - if (IS_ERR(sys_controller->flash)) - return dev_err_probe(dev, PTR_ERR(sys_controller->flash), "Failed to get flash\n"); + if (IS_ERR(sys_controller->flash)) { + ret = dev_err_probe(dev, PTR_ERR(sys_controller->flash), "Failed to get flash\n"); + goto out_free; + } no_flash: sys_controller->client.dev = dev; @@ -155,8 +157,7 @@ static int mpfs_sys_controller_probe(struct platform_device *pdev) if (IS_ERR(sys_controller->chan)) { ret = dev_err_probe(dev, PTR_ERR(sys_controller->chan), "Failed to get mbox channel\n"); - kfree(sys_controller); - return ret; + goto out_free; } init_completion(&sys_controller->c); @@ -174,6 +175,10 @@ static int mpfs_sys_controller_probe(struct platform_device *pdev) dev_info(&pdev->dev, "Registered MPFS system controller\n"); return 0; + +out_free: + kfree(sys_controller); + return ret; } static void mpfs_sys_controller_remove(struct platform_device *pdev) diff --git a/drivers/soc/qcom/cmd-db.c b/drivers/soc/qcom/cmd-db.c index ae66c2623d250d..84a75d8c4b7020 100644 --- a/drivers/soc/qcom/cmd-db.c +++ b/drivers/soc/qcom/cmd-db.c @@ -349,15 +349,16 @@ static int cmd_db_dev_probe(struct platform_device *pdev) return -EINVAL; } - cmd_db_header = memremap(rmem->base, rmem->size, MEMREMAP_WC); - if (!cmd_db_header) { - ret = -ENOMEM; + cmd_db_header = devm_memremap(&pdev->dev, rmem->base, rmem->size, MEMREMAP_WC); + if (IS_ERR(cmd_db_header)) { + ret = PTR_ERR(cmd_db_header); cmd_db_header = NULL; return ret; } if (!cmd_db_magic_matches(cmd_db_header)) { dev_err(&pdev->dev, "Invalid Command DB Magic\n"); + cmd_db_header = NULL; return -EINVAL; } diff --git a/drivers/soc/qcom/pdr_internal.h b/drivers/soc/qcom/pdr_internal.h index 039508c1bbf7d0..047c0160b61788 100644 --- a/drivers/soc/qcom/pdr_internal.h +++ b/drivers/soc/qcom/pdr_internal.h @@ -84,7 +84,7 @@ struct servreg_set_ack_resp { struct servreg_loc_pfr_req { char service[SERVREG_NAME_LENGTH + 1]; - char reason[257]; + char reason[SERVREG_PFR_LENGTH + 1]; }; struct servreg_loc_pfr_resp { diff --git a/drivers/soc/qcom/qcom_pdr_msg.c b/drivers/soc/qcom/qcom_pdr_msg.c index ca98932140d873..02022b11ecf05f 100644 --- a/drivers/soc/qcom/qcom_pdr_msg.c +++ b/drivers/soc/qcom/qcom_pdr_msg.c @@ -325,7 +325,7 @@ const struct qmi_elem_info servreg_loc_pfr_req_ei[] = { }, { .data_type = QMI_STRING, - .elem_len = SERVREG_NAME_LENGTH + 1, + .elem_len = SERVREG_PFR_LENGTH + 1, .elem_size = sizeof(char), .array_type = VAR_LEN_ARRAY, .tlv_type = 0x02, diff --git a/drivers/soc/qcom/smem.c b/drivers/soc/qcom/smem.c index c18a0c946f7627..d5c94b47f431f1 100644 --- a/drivers/soc/qcom/smem.c +++ b/drivers/soc/qcom/smem.c @@ -1219,7 +1219,9 @@ static int qcom_smem_probe(struct platform_device *pdev) smem->item_count = qcom_smem_get_item_count(smem); break; case SMEM_GLOBAL_HEAP_VERSION: - qcom_smem_map_global(smem, size); + ret = qcom_smem_map_global(smem, size); + if (ret < 0) + return ret; smem->item_count = SMEM_ITEM_COUNT; break; default: diff --git a/drivers/soc/rockchip/grf.c b/drivers/soc/rockchip/grf.c index 27bfa09ff2516a..b459607c118aa0 100644 --- a/drivers/soc/rockchip/grf.c +++ b/drivers/soc/rockchip/grf.c @@ -146,7 +146,7 @@ static const struct rockchip_grf_info rk3576_sysgrf __initconst = { .num_values = ARRAY_SIZE(rk3576_defaults_sys_grf), }; -#define RK3576_IOCGRF_MISC_CON 0x04F0 +#define RK3576_IOCGRF_MISC_CON 0x40F0 static const struct rockchip_grf_value rk3576_defaults_ioc_grf[] __initconst = { { "jtag switching", RK3576_IOCGRF_MISC_CON, FIELD_PREP_WM16_CONST(BIT(1), 0) }, @@ -217,34 +217,34 @@ static int __init rockchip_grf_init(void) struct regmap *grf; int ret, i; - np = of_find_matching_node_and_match(NULL, rockchip_grf_dt_match, - &match); - if (!np) - return -ENODEV; - if (!match || !match->data) { - pr_err("%s: missing grf data\n", __func__); - of_node_put(np); - return -EINVAL; - } - - grf_info = match->data; - - grf = syscon_node_to_regmap(np); - of_node_put(np); - if (IS_ERR(grf)) { - pr_err("%s: could not get grf syscon\n", __func__); - return PTR_ERR(grf); - } - - for (i = 0; i < grf_info->num_values; i++) { - const struct rockchip_grf_value *val = &grf_info->values[i]; - - pr_debug("%s: adjusting %s in %#6x to %#10x\n", __func__, - val->desc, val->reg, val->val); - ret = regmap_write(grf, val->reg, val->val); - if (ret < 0) - pr_err("%s: write to %#6x failed with %d\n", - __func__, val->reg, ret); + for_each_matching_node_and_match(np, rockchip_grf_dt_match, &match) { + if (!of_device_is_available(np)) + continue; + if (!match || !match->data) { + pr_err("%s: missing grf data\n", __func__); + of_node_put(np); + return -EINVAL; + } + + grf_info = match->data; + + grf = syscon_node_to_regmap(np); + if (IS_ERR(grf)) { + pr_err("%s: could not get grf syscon\n", __func__); + of_node_put(np); + return PTR_ERR(grf); + } + + for (i = 0; i < grf_info->num_values; i++) { + const struct rockchip_grf_value *val = &grf_info->values[i]; + + pr_debug("%s: adjusting %s in %#6x to %#10x\n", __func__, + val->desc, val->reg, val->val); + ret = regmap_write(grf, val->reg, val->val); + if (ret < 0) + pr_err("%s: write to %#6x failed with %d\n", + __func__, val->reg, ret); + } } return 0; diff --git a/drivers/soc/tegra/pmc.c b/drivers/soc/tegra/pmc.c index f3760a3b3026d6..407fa840814c33 100644 --- a/drivers/soc/tegra/pmc.c +++ b/drivers/soc/tegra/pmc.c @@ -28,6 +28,7 @@ #include #include #include +#include #include #include #include @@ -468,6 +469,10 @@ struct tegra_pmc { unsigned long *wake_sw_status_map; unsigned long *wake_cntrl_level_map; struct syscore syscore; + + /* Pending wake IRQ processing */ + struct irq_work wake_work; + u32 *wake_status; }; static struct tegra_pmc *pmc = &(struct tegra_pmc) { @@ -1905,6 +1910,50 @@ static int tegra_pmc_parse_dt(struct tegra_pmc *pmc, struct device_node *np) return 0; } +/* translate sc7 wake sources back into IRQs to catch edge triggered wakeups */ +static void tegra186_pmc_wake_handler(struct irq_work *work) +{ + struct tegra_pmc *pmc = container_of(work, struct tegra_pmc, wake_work); + unsigned int i, wake; + + for (i = 0; i < pmc->soc->max_wake_vectors; i++) { + unsigned long status = pmc->wake_status[i]; + + for_each_set_bit(wake, &status, 32) { + irq_hw_number_t hwirq = wake + (i * 32); + struct irq_desc *desc; + unsigned int irq; + + irq = irq_find_mapping(pmc->domain, hwirq); + if (!irq) { + dev_warn(pmc->dev, + "No IRQ found for WAKE#%lu!\n", + hwirq); + continue; + } + + dev_dbg(pmc->dev, + "Resume caused by WAKE#%lu mapped to IRQ#%u\n", + hwirq, irq); + + desc = irq_to_desc(irq); + if (!desc) { + dev_warn(pmc->dev, + "No descriptor found for IRQ#%u\n", + irq); + continue; + } + + if (!desc->action || !desc->action->name) + continue; + + generic_handle_irq(irq); + } + + pmc->wake_status[i] = 0; + } +} + static int tegra_pmc_init(struct tegra_pmc *pmc) { if (pmc->soc->max_wake_events > 0) { @@ -1923,6 +1972,18 @@ static int tegra_pmc_init(struct tegra_pmc *pmc) pmc->wake_cntrl_level_map = bitmap_zalloc(pmc->soc->max_wake_events, GFP_KERNEL); if (!pmc->wake_cntrl_level_map) return -ENOMEM; + + pmc->wake_status = kcalloc(pmc->soc->max_wake_vectors, sizeof(u32), GFP_KERNEL); + if (!pmc->wake_status) + return -ENOMEM; + + /* + * Initialize IRQ work for processing wake IRQs. Must use + * HARD_IRQ variant to run in hard IRQ context on PREEMPT_RT + * because we call generic_handle_irq() which requires hard + * IRQ context. + */ + pmc->wake_work = IRQ_WORK_INIT_HARD(tegra186_pmc_wake_handler); } if (pmc->soc->init) @@ -3129,47 +3190,30 @@ static void wke_clear_wake_status(struct tegra_pmc *pmc) } } -/* translate sc7 wake sources back into IRQs to catch edge triggered wakeups */ -static void tegra186_pmc_process_wake_events(struct tegra_pmc *pmc, unsigned int index, - unsigned long status) -{ - unsigned int wake; - - dev_dbg(pmc->dev, "Wake[%d:%d] status=%#lx\n", (index * 32) + 31, index * 32, status); - - for_each_set_bit(wake, &status, 32) { - irq_hw_number_t hwirq = wake + 32 * index; - struct irq_desc *desc; - unsigned int irq; - - irq = irq_find_mapping(pmc->domain, hwirq); - - desc = irq_to_desc(irq); - if (!desc || !desc->action || !desc->action->name) { - dev_dbg(pmc->dev, "Resume caused by WAKE%ld, IRQ %d\n", hwirq, irq); - continue; - } - - dev_dbg(pmc->dev, "Resume caused by WAKE%ld, %s\n", hwirq, desc->action->name); - generic_handle_irq(irq); - } -} - static void tegra186_pmc_wake_syscore_resume(void *data) { - u32 status, mask; unsigned int i; + u32 mask; for (i = 0; i < pmc->soc->max_wake_vectors; i++) { mask = readl(pmc->wake + WAKE_AOWAKE_TIER2_ROUTING(i)); - status = readl(pmc->wake + WAKE_AOWAKE_STATUS_R(i)) & mask; - - tegra186_pmc_process_wake_events(pmc, i, status); + pmc->wake_status[i] = readl(pmc->wake + WAKE_AOWAKE_STATUS_R(i)) & mask; } + + /* Schedule IRQ work to process wake IRQs (if any) */ + irq_work_queue(&pmc->wake_work); } static int tegra186_pmc_wake_syscore_suspend(void *data) { + unsigned int i; + + /* Check if there are unhandled wake IRQs */ + for (i = 0; i < pmc->soc->max_wake_vectors; i++) + if (pmc->wake_status[i]) + dev_warn(pmc->dev, + "Unhandled wake IRQs pending vector[%u]: 0x%x\n", + i, pmc->wake_status[i]); wke_read_sw_wake_status(pmc); /* flip the wakeup trigger for dual-edge triggered pads diff --git a/drivers/soc/ti/k3-socinfo.c b/drivers/soc/ti/k3-socinfo.c index 50c170a995f90b..42275cb5ba1c8d 100644 --- a/drivers/soc/ti/k3-socinfo.c +++ b/drivers/soc/ti/k3-socinfo.c @@ -141,7 +141,7 @@ static int k3_chipinfo_probe(struct platform_device *pdev) if (IS_ERR(base)) return PTR_ERR(base); - regmap = regmap_init_mmio(dev, base, &k3_chipinfo_regmap_cfg); + regmap = devm_regmap_init_mmio(dev, base, &k3_chipinfo_regmap_cfg); if (IS_ERR(regmap)) return PTR_ERR(regmap); diff --git a/drivers/soc/ti/pruss.c b/drivers/soc/ti/pruss.c index 038576805bfa0f..0fd59c73f585d2 100644 --- a/drivers/soc/ti/pruss.c +++ b/drivers/soc/ti/pruss.c @@ -366,12 +366,10 @@ static int pruss_clk_mux_setup(struct pruss *pruss, struct clk *clk_mux, ret = devm_add_action_or_reset(dev, pruss_of_free_clk_provider, clk_mux_np); - if (ret) { + if (ret) dev_err(dev, "failed to add clkmux free action %d", ret); - goto put_clk_mux_np; - } - return 0; + return ret; put_clk_mux_np: of_node_put(clk_mux_np); diff --git a/drivers/soundwire/Kconfig b/drivers/soundwire/Kconfig index ad56393e4c93b8..196a7daaabdb88 100644 --- a/drivers/soundwire/Kconfig +++ b/drivers/soundwire/Kconfig @@ -40,6 +40,7 @@ config SOUNDWIRE_INTEL select AUXILIARY_BUS depends on ACPI && SND_SOC depends on SND_SOC_SOF_HDA_MLINK || !SND_SOC_SOF_HDA_MLINK + depends on SND_HDA_CORE || !SND_HDA_ALIGNED_MMIO help SoundWire Intel Master driver. If you have an Intel platform which has a SoundWire Master then diff --git a/drivers/soundwire/dmi-quirks.c b/drivers/soundwire/dmi-quirks.c index 91ab97a456fa9f..5854218e1a274e 100644 --- a/drivers/soundwire/dmi-quirks.c +++ b/drivers/soundwire/dmi-quirks.c @@ -122,6 +122,17 @@ static const struct dmi_system_id adr_remap_quirk_table[] = { }, .driver_data = (void *)intel_tgl_bios, }, + { + /* + * quirk used for Avell B.ON (OEM rebrand of NUC15 'Bishop County' + * LAPBC510 and LAPBC710) + */ + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "Avell High Performance"), + DMI_MATCH(DMI_PRODUCT_NAME, "B.ON"), + }, + .driver_data = (void *)intel_tgl_bios, + }, { /* quirk used for NUC15 'Rooks County' LAPRC510 and LAPRC710 skews */ .matches = { diff --git a/drivers/soundwire/intel_auxdevice.c b/drivers/soundwire/intel_auxdevice.c index 6df2601fff9099..8752b0e3ce74c3 100644 --- a/drivers/soundwire/intel_auxdevice.c +++ b/drivers/soundwire/intel_auxdevice.c @@ -52,6 +52,7 @@ struct wake_capable_part { static struct wake_capable_part wake_capable_list[] = { {0x01fa, 0x4243}, + {0x01fa, 0x4245}, {0x025d, 0x5682}, {0x025d, 0x700}, {0x025d, 0x711}, diff --git a/drivers/spi/spi-amlogic-spifc-a4.c b/drivers/spi/spi-amlogic-spifc-a4.c index 35a7c4965e1133..3393e1f3057096 100644 --- a/drivers/spi/spi-amlogic-spifc-a4.c +++ b/drivers/spi/spi-amlogic-spifc-a4.c @@ -411,7 +411,7 @@ static int aml_sfc_dma_buffer_setup(struct aml_sfc *sfc, void *databuf, ret = dma_mapping_error(sfc->dev, sfc->daddr); if (ret) { dev_err(sfc->dev, "DMA mapping error\n"); - goto out_map_data; + return ret; } cmd = CMD_DATA_ADDRL(sfc->daddr); @@ -429,7 +429,6 @@ static int aml_sfc_dma_buffer_setup(struct aml_sfc *sfc, void *databuf, ret = dma_mapping_error(sfc->dev, sfc->iaddr); if (ret) { dev_err(sfc->dev, "DMA mapping error\n"); - dma_unmap_single(sfc->dev, sfc->daddr, datalen, dir); goto out_map_data; } @@ -448,7 +447,7 @@ static int aml_sfc_dma_buffer_setup(struct aml_sfc *sfc, void *databuf, return 0; out_map_info: - dma_unmap_single(sfc->dev, sfc->iaddr, datalen, dir); + dma_unmap_single(sfc->dev, sfc->iaddr, infolen, dir); out_map_data: dma_unmap_single(sfc->dev, sfc->daddr, datalen, dir); @@ -1067,6 +1066,13 @@ static const struct nand_ecc_engine_ops aml_sfc_ecc_engine_ops = { .finish_io_req = aml_sfc_ecc_finish_io_req, }; +static void aml_sfc_unregister_ecc_engine(void *data) +{ + struct nand_ecc_engine *eng = data; + + nand_ecc_unregister_on_host_hw_engine(eng); +} + static int aml_sfc_clk_init(struct aml_sfc *sfc) { sfc->gate_clk = devm_clk_get_enabled(sfc->dev, "gate"); @@ -1084,14 +1090,6 @@ static int aml_sfc_clk_init(struct aml_sfc *sfc) return clk_set_rate(sfc->core_clk, SFC_BUS_DEFAULT_CLK); } -static int aml_sfc_disable_clk(struct aml_sfc *sfc) -{ - clk_disable_unprepare(sfc->core_clk); - clk_disable_unprepare(sfc->gate_clk); - - return 0; -} - static int aml_sfc_probe(struct platform_device *pdev) { struct device_node *np = pdev->dev.of_node; @@ -1142,16 +1140,12 @@ static int aml_sfc_probe(struct platform_device *pdev) /* Enable Amlogic flash controller spi mode */ ret = regmap_write(sfc->regmap_base, SFC_SPI_CFG, SPI_MODE_EN); - if (ret) { - dev_err(dev, "failed to enable SPI mode\n"); - goto err_out; - } + if (ret) + return dev_err_probe(dev, ret, "failed to enable SPI mode\n"); ret = dma_set_mask(sfc->dev, DMA_BIT_MASK(32)); - if (ret) { - dev_err(sfc->dev, "failed to set dma mask\n"); - goto err_out; - } + if (ret) + return dev_err_probe(sfc->dev, ret, "failed to set dma mask\n"); sfc->ecc_eng.dev = &pdev->dev; sfc->ecc_eng.integration = NAND_ECC_ENGINE_INTEGRATION_PIPELINED; @@ -1159,10 +1153,13 @@ static int aml_sfc_probe(struct platform_device *pdev) sfc->ecc_eng.priv = sfc; ret = nand_ecc_register_on_host_hw_engine(&sfc->ecc_eng); - if (ret) { - dev_err(&pdev->dev, "failed to register Aml host ecc engine.\n"); - goto err_out; - } + if (ret) + return dev_err_probe(&pdev->dev, ret, "failed to register Aml host ecc engine.\n"); + + ret = devm_add_action_or_reset(dev, aml_sfc_unregister_ecc_engine, + &sfc->ecc_eng); + if (ret) + return dev_err_probe(dev, ret, "failed to add ECC unregister action\n"); ret = of_property_read_u32(np, "amlogic,rx-adj", &val); if (!ret) @@ -1178,24 +1175,7 @@ static int aml_sfc_probe(struct platform_device *pdev) ctrl->min_speed_hz = SFC_MIN_FREQUENCY; ctrl->num_chipselect = SFC_MAX_CS_NUM; - ret = devm_spi_register_controller(dev, ctrl); - if (ret) - goto err_out; - - return 0; - -err_out: - aml_sfc_disable_clk(sfc); - - return ret; -} - -static void aml_sfc_remove(struct platform_device *pdev) -{ - struct spi_controller *ctlr = platform_get_drvdata(pdev); - struct aml_sfc *sfc = spi_controller_get_devdata(ctlr); - - aml_sfc_disable_clk(sfc); + return devm_spi_register_controller(dev, ctrl); } static const struct of_device_id aml_sfc_of_match[] = { @@ -1213,7 +1193,6 @@ static struct platform_driver aml_sfc_driver = { .of_match_table = aml_sfc_of_match, }, .probe = aml_sfc_probe, - .remove = aml_sfc_remove, }; module_platform_driver(aml_sfc_driver); diff --git a/drivers/spi/spi-amlogic-spisg.c b/drivers/spi/spi-amlogic-spisg.c index bcd7ec291ad07c..6045c89c37c830 100644 --- a/drivers/spi/spi-amlogic-spisg.c +++ b/drivers/spi/spi-amlogic-spisg.c @@ -729,9 +729,9 @@ static int aml_spisg_probe(struct platform_device *pdev) }; if (of_property_read_bool(dev->of_node, "spi-slave")) - ctlr = spi_alloc_target(dev, sizeof(*spisg)); + ctlr = devm_spi_alloc_target(dev, sizeof(*spisg)); else - ctlr = spi_alloc_host(dev, sizeof(*spisg)); + ctlr = devm_spi_alloc_host(dev, sizeof(*spisg)); if (!ctlr) return -ENOMEM; @@ -750,10 +750,8 @@ static int aml_spisg_probe(struct platform_device *pdev) return dev_err_probe(dev, PTR_ERR(spisg->map), "regmap init failed\n"); irq = platform_get_irq(pdev, 0); - if (irq < 0) { - ret = irq; - goto out_controller; - } + if (irq < 0) + return irq; ret = device_reset_optional(dev); if (ret) @@ -818,8 +816,6 @@ static int aml_spisg_probe(struct platform_device *pdev) if (spisg->core) clk_disable_unprepare(spisg->core); clk_disable_unprepare(spisg->pclk); -out_controller: - spi_controller_put(ctlr); return ret; } diff --git a/drivers/spi/spi-cadence-quadspi.c b/drivers/spi/spi-cadence-quadspi.c index 965b4cea3388a3..3f8d451297fde3 100644 --- a/drivers/spi/spi-cadence-quadspi.c +++ b/drivers/spi/spi-cadence-quadspi.c @@ -1478,14 +1478,6 @@ static int cqspi_exec_mem_op(struct spi_mem *mem, const struct spi_mem_op *op) if (refcount_read(&cqspi->inflight_ops) == 0) return -ENODEV; - if (!(ddata && (ddata->quirks & CQSPI_DISABLE_RUNTIME_PM))) { - ret = pm_runtime_resume_and_get(dev); - if (ret) { - dev_err(&mem->spi->dev, "resume failed with %d\n", ret); - return ret; - } - } - if (!refcount_read(&cqspi->refcount)) return -EBUSY; @@ -1497,6 +1489,14 @@ static int cqspi_exec_mem_op(struct spi_mem *mem, const struct spi_mem_op *op) return -EBUSY; } + if (!(ddata && (ddata->quirks & CQSPI_DISABLE_RUNTIME_PM))) { + ret = pm_runtime_resume_and_get(dev); + if (ret) { + dev_err(&mem->spi->dev, "resume failed with %d\n", ret); + goto dec_inflight_refcount; + } + } + ret = cqspi_mem_process(mem, op); if (!(ddata && (ddata->quirks & CQSPI_DISABLE_RUNTIME_PM))) @@ -1505,6 +1505,7 @@ static int cqspi_exec_mem_op(struct spi_mem *mem, const struct spi_mem_op *op) if (ret) dev_err(&mem->spi->dev, "operation failed with %d\n", ret); +dec_inflight_refcount: if (refcount_read(&cqspi->inflight_ops) > 1) refcount_dec(&cqspi->inflight_ops); @@ -1844,6 +1845,12 @@ static int cqspi_probe(struct platform_device *pdev) return -ENODEV; } + ret = cqspi_setup_flash(cqspi); + if (ret) { + dev_err(dev, "failed to setup flash parameters %d\n", ret); + return ret; + } + /* Obtain QSPI clock. */ cqspi->clk = devm_clk_get(dev, NULL); if (IS_ERR(cqspi->clk)) { @@ -1885,7 +1892,7 @@ static int cqspi_probe(struct platform_device *pdev) ret = clk_prepare_enable(cqspi->clk); if (ret) { dev_err(dev, "Cannot enable QSPI clock.\n"); - goto probe_clk_failed; + goto disable_rpm; } /* Obtain QSPI reset control */ @@ -1893,14 +1900,14 @@ static int cqspi_probe(struct platform_device *pdev) if (IS_ERR(rstc)) { ret = PTR_ERR(rstc); dev_err(dev, "Cannot get QSPI reset.\n"); - goto probe_reset_failed; + goto disable_clk; } rstc_ocp = devm_reset_control_get_optional_exclusive(dev, "qspi-ocp"); if (IS_ERR(rstc_ocp)) { ret = PTR_ERR(rstc_ocp); dev_err(dev, "Cannot get QSPI OCP reset.\n"); - goto probe_reset_failed; + goto disable_clk; } if (of_device_is_compatible(pdev->dev.of_node, "starfive,jh7110-qspi")) { @@ -1908,7 +1915,7 @@ static int cqspi_probe(struct platform_device *pdev) if (IS_ERR(rstc_ref)) { ret = PTR_ERR(rstc_ref); dev_err(dev, "Cannot get QSPI REF reset.\n"); - goto probe_reset_failed; + goto disable_clk; } reset_control_assert(rstc_ref); reset_control_deassert(rstc_ref); @@ -1950,7 +1957,7 @@ static int cqspi_probe(struct platform_device *pdev) if (ddata->jh7110_clk_init) { ret = cqspi_jh7110_clk_init(pdev, cqspi); if (ret) - goto probe_reset_failed; + goto disable_clk; } if (ddata->quirks & CQSPI_DISABLE_STIG_MODE) cqspi->disable_stig_mode = true; @@ -1958,7 +1965,7 @@ static int cqspi_probe(struct platform_device *pdev) if (ddata->quirks & CQSPI_DMA_SET_MASK) { ret = dma_set_mask(&pdev->dev, DMA_BIT_MASK(64)); if (ret) - goto probe_reset_failed; + goto disable_clks; } } @@ -1969,7 +1976,7 @@ static int cqspi_probe(struct platform_device *pdev) pdev->name, cqspi); if (ret) { dev_err(dev, "Cannot request IRQ.\n"); - goto probe_reset_failed; + goto disable_clks; } cqspi_wait_idle(cqspi); @@ -1987,12 +1994,6 @@ static int cqspi_probe(struct platform_device *pdev) pm_runtime_get_noresume(dev); } - ret = cqspi_setup_flash(cqspi); - if (ret) { - dev_err(dev, "failed to setup flash parameters %d\n", ret); - goto probe_setup_failed; - } - host->num_chipselect = cqspi->num_chipselect; if (ddata && (ddata->quirks & CQSPI_SUPPORT_DEVICE_RESET)) @@ -2002,33 +2003,36 @@ static int cqspi_probe(struct platform_device *pdev) ret = cqspi_request_mmap_dma(cqspi); if (ret == -EPROBE_DEFER) { dev_err_probe(&pdev->dev, ret, "Failed to request mmap DMA\n"); - goto probe_setup_failed; + goto disable_controller; } } ret = spi_register_controller(host); if (ret) { dev_err(&pdev->dev, "failed to register SPI ctlr %d\n", ret); - goto probe_setup_failed; + goto release_dma_chan; } - if (!(ddata && (ddata->quirks & CQSPI_DISABLE_RUNTIME_PM))) { - pm_runtime_mark_last_busy(dev); + if (!(ddata && (ddata->quirks & CQSPI_DISABLE_RUNTIME_PM))) pm_runtime_put_autosuspend(dev); - } return 0; -probe_setup_failed: - if (!(ddata && (ddata->quirks & CQSPI_DISABLE_RUNTIME_PM))) - pm_runtime_disable(dev); + +release_dma_chan: + if (cqspi->rx_chan) + dma_release_channel(cqspi->rx_chan); +disable_controller: cqspi_controller_enable(cqspi, 0); -probe_reset_failed: +disable_clks: if (cqspi->is_jh7110) cqspi_jh7110_disable_clk(pdev, cqspi); - +disable_clk: if (pm_runtime_get_sync(&pdev->dev) >= 0) clk_disable_unprepare(cqspi->clk); -probe_clk_failed: +disable_rpm: + if (!(ddata && (ddata->quirks & CQSPI_DISABLE_RUNTIME_PM))) + pm_runtime_disable(dev); + return ret; } @@ -2037,6 +2041,7 @@ static void cqspi_remove(struct platform_device *pdev) const struct cqspi_driver_platdata *ddata; struct cqspi_st *cqspi = platform_get_drvdata(pdev); struct device *dev = &pdev->dev; + int ret = 0; ddata = of_device_get_match_data(dev); @@ -2046,18 +2051,21 @@ static void cqspi_remove(struct platform_device *pdev) cqspi_wait_idle(cqspi); spi_unregister_controller(cqspi->host); - cqspi_controller_enable(cqspi, 0); if (cqspi->rx_chan) dma_release_channel(cqspi->rx_chan); - if (!(ddata && (ddata->quirks & CQSPI_DISABLE_RUNTIME_PM))) - if (pm_runtime_get_sync(&pdev->dev) >= 0) - clk_disable(cqspi->clk); + cqspi_controller_enable(cqspi, 0); if (cqspi->is_jh7110) cqspi_jh7110_disable_clk(pdev, cqspi); + if (!(ddata && (ddata->quirks & CQSPI_DISABLE_RUNTIME_PM))) + ret = pm_runtime_get_sync(&pdev->dev); + + if (ret >= 0) + clk_disable(cqspi->clk); + if (!(ddata && (ddata->quirks & CQSPI_DISABLE_RUNTIME_PM))) { pm_runtime_put_sync(&pdev->dev); pm_runtime_disable(&pdev->dev); diff --git a/drivers/spi/spi-dw-dma.c b/drivers/spi/spi-dw-dma.c index 65adec7c7524b8..fe726b9b1780d2 100644 --- a/drivers/spi/spi-dw-dma.c +++ b/drivers/spi/spi-dw-dma.c @@ -271,7 +271,7 @@ static int dw_spi_dma_wait(struct dw_spi *dws, unsigned int len, u32 speed) msecs_to_jiffies(ms)); if (ms == 0) { - dev_err(&dws->ctlr->cur_msg->spi->dev, + dev_err(&dws->ctlr->dev, "DMA transaction timed out\n"); return -ETIMEDOUT; } diff --git a/drivers/spi/spi-fsl-lpspi.c b/drivers/spi/spi-fsl-lpspi.c index 065456aba2aea9..47d372557e4f6c 100644 --- a/drivers/spi/spi-fsl-lpspi.c +++ b/drivers/spi/spi-fsl-lpspi.c @@ -972,7 +972,7 @@ static int fsl_lpspi_probe(struct platform_device *pdev) enable_irq(irq); } - ret = devm_spi_register_controller(&pdev->dev, controller); + ret = spi_register_controller(controller); if (ret < 0) { dev_err_probe(&pdev->dev, ret, "spi_register_controller error\n"); goto free_dma; @@ -998,6 +998,7 @@ static void fsl_lpspi_remove(struct platform_device *pdev) struct fsl_lpspi_data *fsl_lpspi = spi_controller_get_devdata(controller); + spi_unregister_controller(controller); fsl_lpspi_dma_exit(controller); pm_runtime_dont_use_autosuspend(fsl_lpspi->dev); diff --git a/drivers/spi/spi-geni-qcom.c b/drivers/spi/spi-geni-qcom.c index a0d8d3425c6c6f..736120107184f9 100644 --- a/drivers/spi/spi-geni-qcom.c +++ b/drivers/spi/spi-geni-qcom.c @@ -160,24 +160,20 @@ static void handle_se_timeout(struct spi_controller *spi, xfer = mas->cur_xfer; mas->cur_xfer = NULL; - if (spi->target) { - /* - * skip CMD Cancel sequnece since spi target - * doesn`t support CMD Cancel sequnece - */ + /* The controller doesn't support the Cancel commnand in target mode */ + if (!spi->target) { + reinit_completion(&mas->cancel_done); + geni_se_cancel_m_cmd(se); + spin_unlock_irq(&mas->lock); - goto reset_if_dma; - } - reinit_completion(&mas->cancel_done); - geni_se_cancel_m_cmd(se); - spin_unlock_irq(&mas->lock); + time_left = wait_for_completion_timeout(&mas->cancel_done, HZ); + if (time_left) + goto reset_if_dma; - time_left = wait_for_completion_timeout(&mas->cancel_done, HZ); - if (time_left) - goto reset_if_dma; + spin_lock_irq(&mas->lock); + } - spin_lock_irq(&mas->lock); reinit_completion(&mas->abort_done); geni_se_abort_m_cmd(se); spin_unlock_irq(&mas->lock); @@ -548,10 +544,10 @@ static u32 get_xfer_len_in_words(struct spi_transfer *xfer, { u32 len; - if (!(mas->cur_bits_per_word % MIN_WORD_LEN)) - len = xfer->len * BITS_PER_BYTE / mas->cur_bits_per_word; + if (!(xfer->bits_per_word % MIN_WORD_LEN)) + len = xfer->len * BITS_PER_BYTE / xfer->bits_per_word; else - len = xfer->len / (mas->cur_bits_per_word / BITS_PER_BYTE + 1); + len = xfer->len / (xfer->bits_per_word / BITS_PER_BYTE + 1); len &= TRANS_LEN_MSK; return len; @@ -571,7 +567,7 @@ static bool geni_can_dma(struct spi_controller *ctlr, return true; len = get_xfer_len_in_words(xfer, mas); - fifo_size = mas->tx_fifo_depth * mas->fifo_width_bits / mas->cur_bits_per_word; + fifo_size = mas->tx_fifo_depth * mas->fifo_width_bits / xfer->bits_per_word; if (len > fifo_size) return true; @@ -724,6 +720,12 @@ static int spi_geni_init(struct spi_geni_master *mas) case 0: mas->cur_xfer_mode = GENI_SE_FIFO; geni_se_select_mode(se, GENI_SE_FIFO); + /* setup_fifo_params assumes that these registers start with a zero value */ + writel(0, se->base + SE_SPI_LOOPBACK); + writel(0, se->base + SE_SPI_DEMUX_SEL); + writel(0, se->base + SE_SPI_CPHA); + writel(0, se->base + SE_SPI_CPOL); + writel(0, se->base + SE_SPI_DEMUX_OUTPUT_INV); ret = 0; break; } @@ -956,10 +958,13 @@ static irqreturn_t geni_spi_isr(int irq, void *data) struct spi_controller *spi = data; struct spi_geni_master *mas = spi_controller_get_devdata(spi); struct geni_se *se = &mas->se; - u32 m_irq; + u32 m_irq, dma_tx_status, dma_rx_status; m_irq = readl(se->base + SE_GENI_M_IRQ_STATUS); - if (!m_irq) + dma_tx_status = readl_relaxed(se->base + SE_DMA_TX_IRQ_STAT); + dma_rx_status = readl_relaxed(se->base + SE_DMA_RX_IRQ_STAT); + + if (!m_irq && !dma_tx_status && !dma_rx_status) return IRQ_NONE; if (m_irq & (M_CMD_OVERRUN_EN | M_ILLEGAL_CMD_EN | M_CMD_FAILURE_EN | @@ -1007,8 +1012,6 @@ static irqreturn_t geni_spi_isr(int irq, void *data) } } else if (mas->cur_xfer_mode == GENI_SE_DMA) { const struct spi_transfer *xfer = mas->cur_xfer; - u32 dma_tx_status = readl_relaxed(se->base + SE_DMA_TX_IRQ_STAT); - u32 dma_rx_status = readl_relaxed(se->base + SE_DMA_RX_IRQ_STAT); if (dma_tx_status) writel(dma_tx_status, se->base + SE_DMA_TX_IRQ_CLR); diff --git a/drivers/spi/spi-intel-pci.c b/drivers/spi/spi-intel-pci.c index bce3d149bea180..d8ef8f89330ac5 100644 --- a/drivers/spi/spi-intel-pci.c +++ b/drivers/spi/spi-intel-pci.c @@ -96,6 +96,7 @@ static const struct pci_device_id intel_spi_pci_ids[] = { { PCI_VDEVICE(INTEL, 0xa324), (unsigned long)&cnl_info }, { PCI_VDEVICE(INTEL, 0xa3a4), (unsigned long)&cnl_info }, { PCI_VDEVICE(INTEL, 0xa823), (unsigned long)&cnl_info }, + { PCI_VDEVICE(INTEL, 0xd323), (unsigned long)&cnl_info }, { PCI_VDEVICE(INTEL, 0xe323), (unsigned long)&cnl_info }, { PCI_VDEVICE(INTEL, 0xe423), (unsigned long)&cnl_info }, { }, diff --git a/drivers/spi/spi-mem.c b/drivers/spi/spi-mem.c index c8b2add2640e55..965673bac98b99 100644 --- a/drivers/spi/spi-mem.c +++ b/drivers/spi/spi-mem.c @@ -178,8 +178,19 @@ bool spi_mem_default_supports_op(struct spi_mem *mem, if (op->data.swap16 && !spi_mem_controller_is_capable(ctlr, swap16)) return false; - if (op->cmd.nbytes != 2) - return false; + /* Extra 8D-8D-8D limitations */ + if (op->cmd.dtr && op->cmd.buswidth == 8) { + if (op->cmd.nbytes != 2) + return false; + + if ((op->addr.nbytes % 2) || + (op->dummy.nbytes % 2) || + (op->data.nbytes % 2)) { + dev_err(&ctlr->dev, + "Even byte numbers not allowed in octal DTR operations\n"); + return false; + } + } } else { if (op->cmd.nbytes != 1) return false; @@ -708,9 +719,18 @@ spi_mem_dirmap_create(struct spi_mem *mem, desc->mem = mem; desc->info = *info; - if (ctlr->mem_ops && ctlr->mem_ops->dirmap_create) + if (ctlr->mem_ops && ctlr->mem_ops->dirmap_create) { + ret = spi_mem_access_start(mem); + if (ret) { + kfree(desc); + return ERR_PTR(ret); + } + ret = ctlr->mem_ops->dirmap_create(desc); + spi_mem_access_end(mem); + } + if (ret) { desc->nodirmap = true; if (!spi_mem_supports_op(desc->mem, &desc->info.op_tmpl)) diff --git a/drivers/spi/spi-meson-spicc.c b/drivers/spi/spi-meson-spicc.c index 6b91373075334e..c99fab392add16 100644 --- a/drivers/spi/spi-meson-spicc.c +++ b/drivers/spi/spi-meson-spicc.c @@ -1102,8 +1102,6 @@ static void meson_spicc_remove(struct platform_device *pdev) /* Disable SPI */ writel(0, spicc->base + SPICC_CONREG); - - spi_controller_put(spicc->host); } static const struct meson_spicc_data meson_spicc_gx_data = { diff --git a/drivers/spi/spi-microchip-core-spi.c b/drivers/spi/spi-microchip-core-spi.c index 89e40fc45d73aa..c8ebb58e0369af 100644 --- a/drivers/spi/spi-microchip-core-spi.c +++ b/drivers/spi/spi-microchip-core-spi.c @@ -161,7 +161,7 @@ static int mchp_corespi_setup(struct spi_device *spi) return -EOPNOTSUPP; } - if (spi->mode & SPI_MODE_X_MASK & ~spi->controller->mode_bits) { + if ((spi->mode ^ spi->controller->mode_bits) & SPI_MODE_X_MASK) { dev_err(&spi->dev, "incompatible CPOL/CPHA, must match controller's Motorola mode\n"); return -EINVAL; } diff --git a/drivers/spi/spi-rockchip-sfc.c b/drivers/spi/spi-rockchip-sfc.c index b3c2b03b11535c..8acf955636977d 100644 --- a/drivers/spi/spi-rockchip-sfc.c +++ b/drivers/spi/spi-rockchip-sfc.c @@ -712,7 +712,7 @@ static int rockchip_sfc_probe(struct platform_device *pdev) } } - ret = devm_spi_register_controller(dev, host); + ret = spi_register_controller(host); if (ret) goto err_register; diff --git a/drivers/spi/spi-sn-f-ospi.c b/drivers/spi/spi-sn-f-ospi.c index c4969f66a0ba95..84a5b327022e89 100644 --- a/drivers/spi/spi-sn-f-ospi.c +++ b/drivers/spi/spi-sn-f-ospi.c @@ -612,7 +612,7 @@ static int f_ospi_probe(struct platform_device *pdev) u32 num_cs = OSPI_NUM_CS; int ret; - ctlr = spi_alloc_host(dev, sizeof(*ospi)); + ctlr = devm_spi_alloc_host(dev, sizeof(*ospi)); if (!ctlr) return -ENOMEM; @@ -636,16 +636,12 @@ static int f_ospi_probe(struct platform_device *pdev) platform_set_drvdata(pdev, ospi); ospi->base = devm_platform_ioremap_resource(pdev, 0); - if (IS_ERR(ospi->base)) { - ret = PTR_ERR(ospi->base); - goto err_put_ctlr; - } + if (IS_ERR(ospi->base)) + return PTR_ERR(ospi->base); ospi->clk = devm_clk_get_enabled(dev, NULL); - if (IS_ERR(ospi->clk)) { - ret = PTR_ERR(ospi->clk); - goto err_put_ctlr; - } + if (IS_ERR(ospi->clk)) + return PTR_ERR(ospi->clk); mutex_init(&ospi->mlock); @@ -662,9 +658,6 @@ static int f_ospi_probe(struct platform_device *pdev) err_destroy_mutex: mutex_destroy(&ospi->mlock); -err_put_ctlr: - spi_controller_put(ctlr); - return ret; } diff --git a/drivers/spi/spi-stm32-ospi.c b/drivers/spi/spi-stm32-ospi.c index f36fd36da26922..2988ff288ff021 100644 --- a/drivers/spi/spi-stm32-ospi.c +++ b/drivers/spi/spi-stm32-ospi.c @@ -939,13 +939,15 @@ static int stm32_ospi_probe(struct platform_device *pdev) if (ret) { /* Disable ospi */ writel_relaxed(0, ospi->regs_base + OSPI_CR); - goto err_pm_resume; + goto err_reset_control; } pm_runtime_put_autosuspend(ospi->dev); return 0; +err_reset_control: + reset_control_release(ospi->rstc); err_pm_resume: pm_runtime_put_sync_suspend(ospi->dev); @@ -963,11 +965,8 @@ static int stm32_ospi_probe(struct platform_device *pdev) static void stm32_ospi_remove(struct platform_device *pdev) { struct stm32_ospi *ospi = platform_get_drvdata(pdev); - int ret; - ret = pm_runtime_resume_and_get(ospi->dev); - if (ret < 0) - return; + pm_runtime_resume_and_get(ospi->dev); spi_unregister_controller(ospi->ctrl); /* Disable ospi */ diff --git a/drivers/spi/spi-stm32.c b/drivers/spi/spi-stm32.c index 2c804c1aef989d..7a6ee93be9bd4e 100644 --- a/drivers/spi/spi-stm32.c +++ b/drivers/spi/spi-stm32.c @@ -1570,6 +1570,9 @@ static int stm32_spi_prepare_rx_dma_mdma_chaining(struct stm32_spi *spi, return -EINVAL; } + *rx_mdma_desc = _mdma_desc; + *rx_dma_desc = _dma_desc; + return 0; } @@ -1906,11 +1909,12 @@ static void stm32h7_spi_data_idleness(struct stm32_spi *spi, struct spi_transfer cfg2_clrb |= STM32H7_SPI_CFG2_MIDI; if ((len > 1) && (spi->cur_midi > 0)) { u32 sck_period_ns = DIV_ROUND_UP(NSEC_PER_SEC, spi->cur_speed); - u32 midi = min_t(u32, - DIV_ROUND_UP(spi->cur_midi, sck_period_ns), - FIELD_GET(STM32H7_SPI_CFG2_MIDI, - STM32H7_SPI_CFG2_MIDI)); + u32 midi = DIV_ROUND_UP(spi->cur_midi, sck_period_ns); + + if ((spi->cur_bpw + midi) < 8) + midi = 8 - spi->cur_bpw; + midi = min_t(u32, midi, FIELD_MAX(STM32H7_SPI_CFG2_MIDI)); dev_dbg(spi->dev, "period=%dns, midi=%d(=%dns)\n", sck_period_ns, midi, midi * sck_period_ns); diff --git a/drivers/spi/spi-wpcm-fiu.c b/drivers/spi/spi-wpcm-fiu.c index a9aee2a6c7dcbc..c47b56f0933f14 100644 --- a/drivers/spi/spi-wpcm-fiu.c +++ b/drivers/spi/spi-wpcm-fiu.c @@ -459,11 +459,11 @@ static int wpcm_fiu_probe(struct platform_device *pdev) res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "memory"); fiu->memory = devm_ioremap_resource(dev, res); - fiu->memory_size = min_t(size_t, resource_size(res), MAX_MEMORY_SIZE_TOTAL); if (IS_ERR(fiu->memory)) return dev_err_probe(dev, PTR_ERR(fiu->memory), "Failed to map flash memory window\n"); + fiu->memory_size = min_t(size_t, resource_size(res), MAX_MEMORY_SIZE_TOTAL); fiu->shm_regmap = syscon_regmap_lookup_by_phandle_optional(dev->of_node, "nuvoton,shm"); wpcm_fiu_hw_init(fiu); diff --git a/drivers/spi/spi.c b/drivers/spi/spi.c index e25df9990f82de..87d829d2a84275 100644 --- a/drivers/spi/spi.c +++ b/drivers/spi/spi.c @@ -50,7 +50,6 @@ static void spidev_release(struct device *dev) struct spi_device *spi = to_spi_device(dev); spi_controller_put(spi->controller); - kfree(spi->driver_override); free_percpu(spi->pcpu_statistics); kfree(spi); } @@ -73,10 +72,9 @@ static ssize_t driver_override_store(struct device *dev, struct device_attribute *a, const char *buf, size_t count) { - struct spi_device *spi = to_spi_device(dev); int ret; - ret = driver_set_override(dev, &spi->driver_override, buf, count); + ret = __device_set_driver_override(dev, buf, count); if (ret) return ret; @@ -86,13 +84,8 @@ static ssize_t driver_override_store(struct device *dev, static ssize_t driver_override_show(struct device *dev, struct device_attribute *a, char *buf) { - const struct spi_device *spi = to_spi_device(dev); - ssize_t len; - - device_lock(dev); - len = sysfs_emit(buf, "%s\n", spi->driver_override ? : ""); - device_unlock(dev); - return len; + guard(spinlock)(&dev->driver_override.lock); + return sysfs_emit(buf, "%s\n", dev->driver_override.name ?: ""); } static DEVICE_ATTR_RW(driver_override); @@ -376,10 +369,12 @@ static int spi_match_device(struct device *dev, const struct device_driver *drv) { const struct spi_device *spi = to_spi_device(dev); const struct spi_driver *sdrv = to_spi_driver(drv); + int ret; /* Check override first, and if set, only use the named driver */ - if (spi->driver_override) - return strcmp(spi->driver_override, drv->name) == 0; + ret = device_match_driver_override(dev, drv); + if (ret >= 0) + return ret; /* Attempt an OF style match */ if (of_driver_match_device(dev, drv)) @@ -2914,6 +2909,8 @@ static void spi_controller_release(struct device *dev) struct spi_controller *ctlr; ctlr = container_of(dev, struct spi_controller, dev); + + free_percpu(ctlr->pcpu_statistics); kfree(ctlr); } @@ -3057,6 +3054,12 @@ struct spi_controller *__spi_alloc_controller(struct device *dev, if (!ctlr) return NULL; + ctlr->pcpu_statistics = spi_alloc_pcpu_stats(NULL); + if (!ctlr->pcpu_statistics) { + kfree(ctlr); + return NULL; + } + device_initialize(&ctlr->dev); INIT_LIST_HEAD(&ctlr->queue); spin_lock_init(&ctlr->queue_lock); @@ -3344,17 +3347,8 @@ int spi_register_controller(struct spi_controller *ctlr) dev_info(dev, "controller is unqueued, this is deprecated\n"); } else if (ctlr->transfer_one || ctlr->transfer_one_message) { status = spi_controller_initialize_queue(ctlr); - if (status) { - device_del(&ctlr->dev); - goto free_bus_id; - } - } - /* Add statistics */ - ctlr->pcpu_statistics = spi_alloc_pcpu_stats(dev); - if (!ctlr->pcpu_statistics) { - dev_err(dev, "Error allocating per-cpu statistics\n"); - status = -ENOMEM; - goto destroy_queue; + if (status) + goto del_ctrl; } mutex_lock(&board_lock); @@ -3368,8 +3362,8 @@ int spi_register_controller(struct spi_controller *ctlr) acpi_register_spi_devices(ctlr); return status; -destroy_queue: - spi_destroy_queue(ctlr); +del_ctrl: + device_del(&ctlr->dev); free_bus_id: mutex_lock(&board_lock); idr_remove(&spi_controller_idr, ctlr->bus_num); diff --git a/drivers/spi/spidev.c b/drivers/spi/spidev.c index 9a0160f6dc3dc6..f28528ed1c24ef 100644 --- a/drivers/spi/spidev.c +++ b/drivers/spi/spidev.c @@ -74,7 +74,6 @@ struct spidev_data { struct list_head device_entry; /* TX/RX buffers are NULL unless this device is open (users > 0) */ - struct mutex buf_lock; unsigned users; u8 *tx_buffer; u8 *rx_buffer; @@ -102,24 +101,6 @@ spidev_sync_unlocked(struct spi_device *spi, struct spi_message *message) return status; } -static ssize_t -spidev_sync(struct spidev_data *spidev, struct spi_message *message) -{ - ssize_t status; - struct spi_device *spi; - - mutex_lock(&spidev->spi_lock); - spi = spidev->spi; - - if (spi == NULL) - status = -ESHUTDOWN; - else - status = spidev_sync_unlocked(spi, message); - - mutex_unlock(&spidev->spi_lock); - return status; -} - static inline ssize_t spidev_sync_write(struct spidev_data *spidev, size_t len) { @@ -132,7 +113,8 @@ spidev_sync_write(struct spidev_data *spidev, size_t len) spi_message_init(&m); spi_message_add_tail(&t, &m); - return spidev_sync(spidev, &m); + + return spidev_sync_unlocked(spidev->spi, &m); } static inline ssize_t @@ -147,7 +129,8 @@ spidev_sync_read(struct spidev_data *spidev, size_t len) spi_message_init(&m); spi_message_add_tail(&t, &m); - return spidev_sync(spidev, &m); + + return spidev_sync_unlocked(spidev->spi, &m); } /*-------------------------------------------------------------------------*/ @@ -157,7 +140,7 @@ static ssize_t spidev_read(struct file *filp, char __user *buf, size_t count, loff_t *f_pos) { struct spidev_data *spidev; - ssize_t status; + ssize_t status = -ESHUTDOWN; /* chipselect only toggles at start or end of operation */ if (count > bufsiz) @@ -165,7 +148,11 @@ spidev_read(struct file *filp, char __user *buf, size_t count, loff_t *f_pos) spidev = filp->private_data; - mutex_lock(&spidev->buf_lock); + mutex_lock(&spidev->spi_lock); + + if (spidev->spi == NULL) + goto err_spi_removed; + status = spidev_sync_read(spidev, count); if (status > 0) { unsigned long missing; @@ -176,7 +163,9 @@ spidev_read(struct file *filp, char __user *buf, size_t count, loff_t *f_pos) else status = status - missing; } - mutex_unlock(&spidev->buf_lock); + +err_spi_removed: + mutex_unlock(&spidev->spi_lock); return status; } @@ -187,7 +176,7 @@ spidev_write(struct file *filp, const char __user *buf, size_t count, loff_t *f_pos) { struct spidev_data *spidev; - ssize_t status; + ssize_t status = -ESHUTDOWN; unsigned long missing; /* chipselect only toggles at start or end of operation */ @@ -196,13 +185,19 @@ spidev_write(struct file *filp, const char __user *buf, spidev = filp->private_data; - mutex_lock(&spidev->buf_lock); + mutex_lock(&spidev->spi_lock); + + if (spidev->spi == NULL) + goto err_spi_removed; + missing = copy_from_user(spidev->tx_buffer, buf, count); if (missing == 0) status = spidev_sync_write(spidev, count); else status = -EFAULT; - mutex_unlock(&spidev->buf_lock); + +err_spi_removed: + mutex_unlock(&spidev->spi_lock); return status; } @@ -379,14 +374,6 @@ spidev_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) ctlr = spi->controller; - /* use the buffer lock here for triple duty: - * - prevent I/O (from us) so calling spi_setup() is safe; - * - prevent concurrent SPI_IOC_WR_* from morphing - * data fields while SPI_IOC_RD_* reads them; - * - SPI_IOC_MESSAGE needs the buffer locked "normally". - */ - mutex_lock(&spidev->buf_lock); - switch (cmd) { /* read requests */ case SPI_IOC_RD_MODE: @@ -510,7 +497,6 @@ spidev_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) break; } - mutex_unlock(&spidev->buf_lock); spi_dev_put(spi); mutex_unlock(&spidev->spi_lock); return retval; @@ -541,9 +527,6 @@ spidev_compat_ioc_message(struct file *filp, unsigned int cmd, return -ESHUTDOWN; } - /* SPI_IOC_MESSAGE needs the buffer locked "normally" */ - mutex_lock(&spidev->buf_lock); - /* Check message and copy into scratch area */ ioc = spidev_get_ioc_message(cmd, u_ioc, &n_ioc); if (IS_ERR(ioc)) { @@ -564,7 +547,6 @@ spidev_compat_ioc_message(struct file *filp, unsigned int cmd, kfree(ioc); done: - mutex_unlock(&spidev->buf_lock); spi_dev_put(spi); mutex_unlock(&spidev->spi_lock); return retval; @@ -802,7 +784,6 @@ static int spidev_probe(struct spi_device *spi) /* Initialize the driver data */ spidev->spi = spi; mutex_init(&spidev->spi_lock); - mutex_init(&spidev->buf_lock); INIT_LIST_HEAD(&spidev->device_entry); diff --git a/drivers/spmi/spmi-apple-controller.c b/drivers/spmi/spmi-apple-controller.c index 697b3e8bb02356..87e3ee9d4f2aa5 100644 --- a/drivers/spmi/spmi-apple-controller.c +++ b/drivers/spmi/spmi-apple-controller.c @@ -149,6 +149,7 @@ static int apple_spmi_probe(struct platform_device *pdev) } static const struct of_device_id apple_spmi_match_table[] = { + { .compatible = "apple,t8103-spmi", }, { .compatible = "apple,spmi", }, {} }; diff --git a/drivers/staging/greybus/light.c b/drivers/staging/greybus/light.c index e509fdc715dbbf..38c233a706c483 100644 --- a/drivers/staging/greybus/light.c +++ b/drivers/staging/greybus/light.c @@ -1008,14 +1008,18 @@ static int gb_lights_light_config(struct gb_lights *glights, u8 id) if (!strlen(conf.name)) return -EINVAL; - light->channels_count = conf.channel_count; light->name = kstrndup(conf.name, NAMES_MAX, GFP_KERNEL); if (!light->name) return -ENOMEM; - light->channels = kcalloc(light->channels_count, + light->channels = kcalloc(conf.channel_count, sizeof(struct gb_channel), GFP_KERNEL); if (!light->channels) return -ENOMEM; + /* + * Publish channels_count only after channels allocation so cleanup + * doesn't walk a NULL channels pointer on allocation failure. + */ + light->channels_count = conf.channel_count; /* First we collect all the configurations for all channels */ for (i = 0; i < light->channels_count; i++) { diff --git a/drivers/staging/media/ipu7/ipu7-buttress.c b/drivers/staging/media/ipu7/ipu7-buttress.c index e5707f5e300bac..40c6c8473357c6 100644 --- a/drivers/staging/media/ipu7/ipu7-buttress.c +++ b/drivers/staging/media/ipu7/ipu7-buttress.c @@ -342,14 +342,23 @@ irqreturn_t ipu_buttress_isr(int irq, void *isp_ptr) u32 disable_irqs = 0; u32 irq_status; unsigned int i; + int active; - pm_runtime_get_noresume(dev); + active = pm_runtime_get_if_active(dev); + if (active <= 0) + return IRQ_NONE; pb_irq = readl(isp->pb_base + INTERRUPT_STATUS); writel(pb_irq, isp->pb_base + INTERRUPT_STATUS); /* check btrs ATS, CFI and IMR errors, BIT(0) is unused for IPU */ pb_local_irq = readl(isp->pb_base + BTRS_LOCAL_INTERRUPT_MASK); + if (pb_local_irq == 0xffffffff) { + dev_warn_once(dev, "invalid PB irq status\n"); + pm_runtime_put_noidle(dev); + return IRQ_NONE; + } + if (pb_local_irq & ~BIT(0)) { dev_warn(dev, "PB interrupt status 0x%x local 0x%x\n", pb_irq, pb_local_irq); @@ -370,6 +379,12 @@ irqreturn_t ipu_buttress_isr(int irq, void *isp_ptr) return IRQ_NONE; } + if (irq_status == 0xffffffff) { + dev_warn_once(dev, "invalid irq status 0x%08x\n", irq_status); + pm_runtime_put_noidle(dev); + return IRQ_NONE; + } + do { writel(irq_status, isp->base + BUTTRESS_REG_IRQ_CLEAR); diff --git a/drivers/staging/media/ipu7/ipu7-isys-csi-phy.c b/drivers/staging/media/ipu7/ipu7-isys-csi-phy.c index 2d57178835188a..3f15af3b4c7990 100644 --- a/drivers/staging/media/ipu7/ipu7-isys-csi-phy.c +++ b/drivers/staging/media/ipu7/ipu7-isys-csi-phy.c @@ -124,6 +124,7 @@ static const struct cdr_fbk_cap_prog_params table7[] = { { 1350, 1589, 4 }, { 1590, 1949, 5 }, { 1950, 2499, 6 }, + { 2500, 3500, 7 }, { } }; @@ -838,9 +839,10 @@ static void ipu7_isys_cphy_config(struct ipu7_isys *isys, u8 id, u8 lanes, dwc_phy_write_mask(isys, id, reg + 0x400 * i, reset_thresh, 9, 11); + /* Tuning ITMINRX to 2 for CPHY */ reg = CORE_DIG_CLANE_0_RW_LP_0; for (i = 0; i < trios; i++) - dwc_phy_write_mask(isys, id, reg + 0x400 * i, 1, 12, 15); + dwc_phy_write_mask(isys, id, reg + 0x400 * i, 2, 12, 15); reg = CORE_DIG_CLANE_0_RW_LP_2; for (i = 0; i < trios; i++) @@ -860,7 +862,11 @@ static void ipu7_isys_cphy_config(struct ipu7_isys *isys, u8 id, u8 lanes, for (i = 0; i < (lanes + 1); i++) { reg = CORE_DIG_IOCTRL_RW_AFE_LANE0_CTRL_2_9 + 0x400 * i; dwc_phy_write_mask(isys, id, reg, 4U, 0, 2); - dwc_phy_write_mask(isys, id, reg, 0U, 3, 4); + /* Set GMODE to 2 when CPHY >= 1.5Gsps */ + if (mbps >= 1500) + dwc_phy_write_mask(isys, id, reg, 2U, 3, 4); + else + dwc_phy_write_mask(isys, id, reg, 0U, 3, 4); reg = CORE_DIG_IOCTRL_RW_AFE_LANE0_CTRL_2_7 + 0x400 * i; dwc_phy_write_mask(isys, id, reg, cap_prog, 10, 12); @@ -930,8 +936,9 @@ static int ipu7_isys_phy_config(struct ipu7_isys *isys, u8 id, u8 lanes, 7, 12, 14); dwc_phy_write_mask(isys, id, CORE_DIG_IOCTRL_RW_AFE_CB_CTRL_2_7, 0, 8, 10); + /* resistance tuning: 1 for 45ohm, 0 for 50ohm */ dwc_phy_write_mask(isys, id, CORE_DIG_IOCTRL_RW_AFE_CB_CTRL_2_5, - 0, 8, 8); + 1, 8, 8); if (aggregation) phy_mode = isys->csi2[0].phy_mode; diff --git a/drivers/staging/media/ipu7/ipu7-mmu.c b/drivers/staging/media/ipu7/ipu7-mmu.c index ded1986eb8ba37..ea35cce4830add 100644 --- a/drivers/staging/media/ipu7/ipu7-mmu.c +++ b/drivers/staging/media/ipu7/ipu7-mmu.c @@ -231,7 +231,7 @@ static u32 *alloc_l2_pt(struct ipu7_mmu_info *mmu_info) dev_dbg(mmu_info->dev, "alloc_l2: get_zeroed_page() = %p\n", pt); - for (i = 0; i < ISP_L1PT_PTES; i++) + for (i = 0; i < ISP_L2PT_PTES; i++) pt[i] = mmu_info->dummy_page_pteval; return pt; diff --git a/drivers/staging/media/ipu7/ipu7.c b/drivers/staging/media/ipu7/ipu7.c index 5cddc09c72bf2c..fa5a1867626f84 100644 --- a/drivers/staging/media/ipu7/ipu7.c +++ b/drivers/staging/media/ipu7/ipu7.c @@ -2620,7 +2620,7 @@ static int ipu7_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) if (!IS_ERR_OR_NULL(isp->isys) && !IS_ERR_OR_NULL(isp->isys->mmu)) ipu7_mmu_cleanup(isp->isys->mmu); if (!IS_ERR_OR_NULL(isp->psys)) - pm_runtime_put(&isp->psys->auxdev.dev); + pm_runtime_put_sync(&isp->psys->auxdev.dev); ipu7_bus_del_devices(pdev); release_firmware(isp->cpd_fw); buttress_exit: @@ -2684,6 +2684,10 @@ static void ipu7_pci_reset_done(struct pci_dev *pdev) */ static int ipu7_suspend(struct device *dev) { + struct pci_dev *pdev = to_pci_dev(dev); + + synchronize_irq(pdev->irq); + return 0; } diff --git a/drivers/staging/media/tegra-video/vi.c b/drivers/staging/media/tegra-video/vi.c index c9276ff76157fa..14b327afe045e6 100644 --- a/drivers/staging/media/tegra-video/vi.c +++ b/drivers/staging/media/tegra-video/vi.c @@ -438,7 +438,7 @@ static int __tegra_channel_try_format(struct tegra_vi_channel *chan, .target = V4L2_SEL_TGT_CROP_BOUNDS, }; struct v4l2_rect *try_crop; - int ret; + int ret = 0; subdev = tegra_channel_get_remote_source_subdev(chan); if (!subdev) @@ -482,8 +482,10 @@ static int __tegra_channel_try_format(struct tegra_vi_channel *chan, } else { ret = v4l2_subdev_call(subdev, pad, get_selection, NULL, &sdsel); - if (ret) - return -EINVAL; + if (ret) { + ret = -EINVAL; + goto out_free; + } try_crop->width = sdsel.r.width; try_crop->height = sdsel.r.height; @@ -495,14 +497,15 @@ static int __tegra_channel_try_format(struct tegra_vi_channel *chan, ret = v4l2_subdev_call(subdev, pad, set_fmt, sd_state, &fmt); if (ret < 0) - return ret; + goto out_free; v4l2_fill_pix_format(pix, &fmt.format); chan->vi->ops->vi_fmt_align(pix, fmtinfo->bpp); +out_free: __v4l2_subdev_state_free(sd_state); - return 0; + return ret; } static int tegra_channel_try_format(struct file *file, void *fh, diff --git a/drivers/staging/rtl8723bs/core/rtw_ieee80211.c b/drivers/staging/rtl8723bs/core/rtw_ieee80211.c index 8fdeeda88a6dc4..3a3161ff1e7f38 100644 --- a/drivers/staging/rtl8723bs/core/rtw_ieee80211.c +++ b/drivers/staging/rtl8723bs/core/rtw_ieee80211.c @@ -185,20 +185,25 @@ u8 *rtw_get_ie_ex(u8 *in_ie, uint in_len, u8 eid, u8 *oui, u8 oui_len, u8 *ie, u cnt = 0; - while (cnt < in_len) { + while (cnt + 2 <= in_len) { + u8 ie_len = in_ie[cnt + 1]; + + if (cnt + 2 + ie_len > in_len) + break; + if (eid == in_ie[cnt] - && (!oui || !memcmp(&in_ie[cnt+2], oui, oui_len))) { + && (!oui || (ie_len >= oui_len && !memcmp(&in_ie[cnt + 2], oui, oui_len)))) { target_ie = &in_ie[cnt]; if (ie) - memcpy(ie, &in_ie[cnt], in_ie[cnt+1]+2); + memcpy(ie, &in_ie[cnt], ie_len + 2); if (ielen) - *ielen = in_ie[cnt+1]+2; + *ielen = ie_len + 2; break; } - cnt += in_ie[cnt+1]+2; /* goto next */ + cnt += ie_len + 2; /* goto next */ } return target_ie; diff --git a/drivers/staging/rtl8723bs/core/rtw_mlme.c b/drivers/staging/rtl8723bs/core/rtw_mlme.c index 98704179ad35ac..2142c2872adf49 100644 --- a/drivers/staging/rtl8723bs/core/rtw_mlme.c +++ b/drivers/staging/rtl8723bs/core/rtw_mlme.c @@ -835,8 +835,10 @@ static void find_network(struct adapter *adapter) struct wlan_network *tgt_network = &pmlmepriv->cur_network; pwlan = rtw_find_network(&pmlmepriv->scanned_queue, tgt_network->network.mac_address); - if (pwlan) - pwlan->fixed = false; + if (!pwlan) + return; + + pwlan->fixed = false; if (check_fwstate(pmlmepriv, WIFI_ADHOC_MASTER_STATE) && (adapter->stapriv.asoc_sta_count == 1)) @@ -2000,7 +2002,10 @@ int rtw_restruct_wmm_ie(struct adapter *adapter, u8 *in_ie, u8 *out_ie, uint in_ while (i < in_len) { ielength = initial_out_len; - if (in_ie[i] == 0xDD && in_ie[i+2] == 0x00 && in_ie[i+3] == 0x50 && in_ie[i+4] == 0xF2 && in_ie[i+5] == 0x02 && i+5 < in_len) { /* WMM element ID and OUI */ + if (i + 5 < in_len && + in_ie[i] == 0xDD && in_ie[i + 2] == 0x00 && + in_ie[i + 3] == 0x50 && in_ie[i + 4] == 0xF2 && + in_ie[i + 5] == 0x02) { for (j = i; j < i + 9; j++) { out_ie[ielength] = in_ie[j]; ielength++; diff --git a/drivers/staging/rtl8723bs/core/rtw_security.c b/drivers/staging/rtl8723bs/core/rtw_security.c index 2f941ffbd46512..5f64d5ae49db2f 100644 --- a/drivers/staging/rtl8723bs/core/rtw_security.c +++ b/drivers/staging/rtl8723bs/core/rtw_security.c @@ -1313,7 +1313,7 @@ u32 rtw_BIP_verify(struct adapter *padapter, u8 *precvframe) u8 mic[16]; struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv; __le16 le_tmp; - __le64 le_tmp64; + __le64 le_tmp64 = 0; ori_len = pattrib->pkt_len - WLAN_HDR_A3_LEN + BIP_AAD_SIZE; BIP_AAD = rtw_zmalloc(ori_len); diff --git a/drivers/staging/rtl8723bs/os_dep/ioctl_cfg80211.c b/drivers/staging/rtl8723bs/os_dep/ioctl_cfg80211.c index 60edeae1cffe71..476ab055e53e5a 100644 --- a/drivers/staging/rtl8723bs/os_dep/ioctl_cfg80211.c +++ b/drivers/staging/rtl8723bs/os_dep/ioctl_cfg80211.c @@ -315,9 +315,10 @@ struct cfg80211_bss *rtw_cfg80211_inform_bss(struct adapter *padapter, struct wl len, notify_signal, GFP_ATOMIC); if (unlikely(!bss)) - goto exit; + goto free_buf; cfg80211_put_bss(wiphy, bss); +free_buf: kfree(buf); exit: diff --git a/drivers/staging/rtl8723bs/os_dep/sdio_intf.c b/drivers/staging/rtl8723bs/os_dep/sdio_intf.c index 1d0239eef114b7..dc787954126fd6 100644 --- a/drivers/staging/rtl8723bs/os_dep/sdio_intf.c +++ b/drivers/staging/rtl8723bs/os_dep/sdio_intf.c @@ -377,7 +377,8 @@ static int rtw_drv_init( if (status != _SUCCESS) goto free_if1; - if (sdio_alloc_irq(dvobj) != _SUCCESS) + status = sdio_alloc_irq(dvobj); + if (status != _SUCCESS) goto free_if1; status = _SUCCESS; diff --git a/drivers/staging/sm750fb/sm750.c b/drivers/staging/sm750fb/sm750.c index fecd7457e615e9..458b528d98b3b6 100644 --- a/drivers/staging/sm750fb/sm750.c +++ b/drivers/staging/sm750fb/sm750.c @@ -481,6 +481,9 @@ static int lynxfb_ops_check_var(struct fb_var_screeninfo *var, struct lynxfb_crtc *crtc; resource_size_t request; + if (!var->pixclock) + return -EINVAL; + ret = 0; par = info->par; crtc = &par->crtc; @@ -1123,6 +1126,7 @@ static void lynxfb_pci_remove(struct pci_dev *pdev) iounmap(sm750_dev->pvReg); iounmap(sm750_dev->pvMem); + pci_release_region(pdev, 1); kfree(g_settings); } diff --git a/drivers/staging/sm750fb/sm750_hw.c b/drivers/staging/sm750fb/sm750_hw.c index ce46f240cbaf1f..b3a16b22359c7f 100644 --- a/drivers/staging/sm750fb/sm750_hw.c +++ b/drivers/staging/sm750fb/sm750_hw.c @@ -36,16 +36,11 @@ int hw_sm750_map(struct sm750_dev *sm750_dev, struct pci_dev *pdev) pr_info("mmio phyAddr = %lx\n", sm750_dev->vidreg_start); - /* - * reserve the vidreg space of smi adaptor - * if you do this, you need to add release region code - * in lynxfb_remove, or memory will not be mapped again - * successfully - */ + /* reserve the vidreg space of smi adaptor */ ret = pci_request_region(pdev, 1, "sm750fb"); if (ret) { pr_err("Can not request PCI regions.\n"); - goto exit; + return ret; } /* now map mmio and vidmem */ @@ -54,7 +49,7 @@ int hw_sm750_map(struct sm750_dev *sm750_dev, struct pci_dev *pdev) if (!sm750_dev->pvReg) { pr_err("mmio failed\n"); ret = -EFAULT; - goto exit; + goto err_release_region; } pr_info("mmio virtual addr = %p\n", sm750_dev->pvReg); @@ -79,13 +74,18 @@ int hw_sm750_map(struct sm750_dev *sm750_dev, struct pci_dev *pdev) sm750_dev->pvMem = ioremap_wc(sm750_dev->vidmem_start, sm750_dev->vidmem_size); if (!sm750_dev->pvMem) { - iounmap(sm750_dev->pvReg); pr_err("Map video memory failed\n"); ret = -EFAULT; - goto exit; + goto err_unmap_reg; } pr_info("video memory vaddr = %p\n", sm750_dev->pvMem); -exit: + + return 0; + +err_unmap_reg: + iounmap(sm750_dev->pvReg); +err_release_region: + pci_release_region(pdev, 1); return ret; } diff --git a/drivers/target/loopback/tcm_loop.c b/drivers/target/loopback/tcm_loop.c index 01a8e349dc4d1a..156f934049f194 100644 --- a/drivers/target/loopback/tcm_loop.c +++ b/drivers/target/loopback/tcm_loop.c @@ -26,6 +26,7 @@ #include #include #include +#include #include #include #include @@ -268,15 +269,27 @@ static int tcm_loop_device_reset(struct scsi_cmnd *sc) return (ret == TMR_FUNCTION_COMPLETE) ? SUCCESS : FAILED; } +static bool tcm_loop_flush_work_iter(struct request *rq, void *data) +{ + struct scsi_cmnd *sc = blk_mq_rq_to_pdu(rq); + struct tcm_loop_cmd *tl_cmd = scsi_cmd_priv(sc); + struct se_cmd *se_cmd = &tl_cmd->tl_se_cmd; + + flush_work(&se_cmd->work); + return true; +} + static int tcm_loop_target_reset(struct scsi_cmnd *sc) { struct tcm_loop_hba *tl_hba; struct tcm_loop_tpg *tl_tpg; + struct Scsi_Host *sh = sc->device->host; + int ret; /* * Locate the tcm_loop_hba_t pointer */ - tl_hba = *(struct tcm_loop_hba **)shost_priv(sc->device->host); + tl_hba = *(struct tcm_loop_hba **)shost_priv(sh); if (!tl_hba) { pr_err("Unable to perform device reset without active I_T Nexus\n"); return FAILED; @@ -285,11 +298,38 @@ static int tcm_loop_target_reset(struct scsi_cmnd *sc) * Locate the tl_tpg pointer from TargetID in sc->device->id */ tl_tpg = &tl_hba->tl_hba_tpgs[sc->device->id]; - if (tl_tpg) { - tl_tpg->tl_transport_status = TCM_TRANSPORT_ONLINE; - return SUCCESS; - } - return FAILED; + if (!tl_tpg) + return FAILED; + + /* + * Issue a LUN_RESET to drain all commands that the target core + * knows about. This handles commands not yet marked CMD_T_COMPLETE. + */ + ret = tcm_loop_issue_tmr(tl_tpg, sc->device->lun, 0, TMR_LUN_RESET); + if (ret != TMR_FUNCTION_COMPLETE) + return FAILED; + + /* + * Flush any deferred target core completion work that may still be + * queued. Commands that already had CMD_T_COMPLETE set before the TMR + * are skipped by the TMR drain, but their async completion work + * (transport_lun_remove_cmd → percpu_ref_put, release_cmd → scsi_done) + * may still be pending in target_completion_wq. + * + * The SCSI EH will reuse in-flight scsi_cmnd structures for recovery + * commands (e.g. TUR) immediately after this handler returns SUCCESS — + * if deferred work is still pending, the memset in queuecommand would + * zero the se_cmd while the work accesses it, leaking the LUN + * percpu_ref and hanging configfs unlink forever. + * + * Use blk_mq_tagset_busy_iter() to find all started requests and + * flush_work() on each — the same pattern used by mpi3mr, scsi_debug, + * and other SCSI drivers to drain outstanding commands during reset. + */ + blk_mq_tagset_busy_iter(&sh->tag_set, tcm_loop_flush_work_iter, NULL); + + tl_tpg->tl_transport_status = TCM_TRANSPORT_ONLINE; + return SUCCESS; } static const struct scsi_host_template tcm_loop_driver_template = { diff --git a/drivers/target/target_core_configfs.c b/drivers/target/target_core_configfs.c index f7868b41c5e61d..749af4a29a7158 100644 --- a/drivers/target/target_core_configfs.c +++ b/drivers/target/target_core_configfs.c @@ -108,8 +108,8 @@ static ssize_t target_core_item_dbroot_store(struct config_item *item, const char *page, size_t count) { ssize_t read_bytes; - struct file *fp; ssize_t r = -EINVAL; + struct path path = {}; mutex_lock(&target_devices_lock); if (target_devices) { @@ -131,17 +131,14 @@ static ssize_t target_core_item_dbroot_store(struct config_item *item, db_root_stage[read_bytes - 1] = '\0'; /* validate new db root before accepting it */ - fp = filp_open(db_root_stage, O_RDONLY, 0); - if (IS_ERR(fp)) { + r = kern_path(db_root_stage, LOOKUP_FOLLOW | LOOKUP_DIRECTORY, &path); + if (r) { pr_err("db_root: cannot open: %s\n", db_root_stage); + if (r == -ENOTDIR) + pr_err("db_root: not a directory: %s\n", db_root_stage); goto unlock; } - if (!S_ISDIR(file_inode(fp)->i_mode)) { - filp_close(fp, NULL); - pr_err("db_root: not a directory: %s\n", db_root_stage); - goto unlock; - } - filp_close(fp, NULL); + path_put(&path); strscpy(db_root, db_root_stage); pr_debug("Target_Core_ConfigFS: db_root set to %s\n", db_root); diff --git a/drivers/target/target_core_file.c b/drivers/target/target_core_file.c index b2610073e8ccab..26d52f1f36df6d 100644 --- a/drivers/target/target_core_file.c +++ b/drivers/target/target_core_file.c @@ -276,7 +276,7 @@ fd_execute_rw_aio(struct se_cmd *cmd, struct scatterlist *sgl, u32 sgl_nents, ssize_t len = 0; int ret = 0, i; - aio_cmd = kmalloc(struct_size(aio_cmd, bvecs, sgl_nents), GFP_KERNEL); + aio_cmd = kzalloc(struct_size(aio_cmd, bvecs, sgl_nents), GFP_KERNEL); if (!aio_cmd) return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE; diff --git a/drivers/tee/tee_shm.c b/drivers/tee/tee_shm.c index 4a47de4bb2e5ca..898707ca21a8e4 100644 --- a/drivers/tee/tee_shm.c +++ b/drivers/tee/tee_shm.c @@ -23,29 +23,11 @@ struct tee_shm_dma_mem { struct page *page; }; -static void shm_put_kernel_pages(struct page **pages, size_t page_count) -{ - size_t n; - - for (n = 0; n < page_count; n++) - put_page(pages[n]); -} - -static void shm_get_kernel_pages(struct page **pages, size_t page_count) -{ - size_t n; - - for (n = 0; n < page_count; n++) - get_page(pages[n]); -} - static void release_registered_pages(struct tee_shm *shm) { if (shm->pages) { if (shm->flags & TEE_SHM_USER_MAPPED) unpin_user_pages(shm->pages, shm->num_pages); - else - shm_put_kernel_pages(shm->pages, shm->num_pages); kfree(shm->pages); } @@ -477,13 +459,6 @@ register_shm_helper(struct tee_context *ctx, struct iov_iter *iter, u32 flags, goto err_put_shm_pages; } - /* - * iov_iter_extract_kvec_pages does not get reference on the pages, - * get a reference on them. - */ - if (iov_iter_is_kvec(iter)) - shm_get_kernel_pages(shm->pages, num_pages); - shm->offset = off; shm->size = len; shm->num_pages = num_pages; @@ -499,8 +474,6 @@ register_shm_helper(struct tee_context *ctx, struct iov_iter *iter, u32 flags, err_put_shm_pages: if (!iov_iter_is_kvec(iter)) unpin_user_pages(shm->pages, shm->num_pages); - else - shm_put_kernel_pages(shm->pages, shm->num_pages); err_free_shm_pages: kfree(shm->pages); err_free_shm: diff --git a/drivers/thermal/intel/int340x_thermal/processor_thermal_rfim.c b/drivers/thermal/intel/int340x_thermal/processor_thermal_rfim.c index 589a3a71f0c4c1..ba88d878c998da 100644 --- a/drivers/thermal/intel/int340x_thermal/processor_thermal_rfim.c +++ b/drivers/thermal/intel/int340x_thermal/processor_thermal_rfim.c @@ -466,8 +466,11 @@ int proc_thermal_rfim_add(struct pci_dev *pdev, struct proc_thermal_device *proc break; } ret = sysfs_create_group(&pdev->dev.kobj, &dlvr_attribute_group); - if (ret) + if (ret) { + if (proc_priv->mmio_feature_mask & PROC_THERMAL_FEATURE_FIVR) + sysfs_remove_group(&pdev->dev.kobj, &fivr_attribute_group); return ret; + } } if (proc_priv->mmio_feature_mask & PROC_THERMAL_FEATURE_DVFS) { diff --git a/drivers/thermal/intel/int340x_thermal/processor_thermal_soc_slider.c b/drivers/thermal/intel/int340x_thermal/processor_thermal_soc_slider.c index 49ff3bae727109..91f291627132aa 100644 --- a/drivers/thermal/intel/int340x_thermal/processor_thermal_soc_slider.c +++ b/drivers/thermal/intel/int340x_thermal/processor_thermal_soc_slider.c @@ -176,15 +176,21 @@ static inline void write_soc_slider(struct proc_thermal_device *proc_priv, u64 v static void set_soc_power_profile(struct proc_thermal_device *proc_priv, int slider) { + u8 offset; u64 val; val = read_soc_slider(proc_priv); val &= ~SLIDER_MASK; val |= FIELD_PREP(SLIDER_MASK, slider) | BIT(SLIDER_ENABLE_BIT); + if (slider == SOC_SLIDER_VALUE_MINIMUM || slider == SOC_SLIDER_VALUE_MAXIMUM) + offset = 0; + else + offset = slider_offset; + /* Set the slider offset from module params */ val &= ~SLIDER_OFFSET_MASK; - val |= FIELD_PREP(SLIDER_OFFSET_MASK, slider_offset); + val |= FIELD_PREP(SLIDER_OFFSET_MASK, offset); write_soc_slider(proc_priv, val); } diff --git a/drivers/thermal/intel/x86_pkg_temp_thermal.c b/drivers/thermal/intel/x86_pkg_temp_thermal.c index 3fc679b6f11b17..aab5f9fca9c33c 100644 --- a/drivers/thermal/intel/x86_pkg_temp_thermal.c +++ b/drivers/thermal/intel/x86_pkg_temp_thermal.c @@ -128,6 +128,9 @@ sys_set_trip_temp(struct thermal_zone_device *tzd, u32 l, h, mask, shift, intr; int tj_max, val, ret; + if (temp == THERMAL_TEMP_INVALID) + temp = 0; + tj_max = intel_tcc_get_tjmax(zonedev->cpu); if (tj_max < 0) return tj_max; diff --git a/drivers/thermal/thermal_core.c b/drivers/thermal/thermal_core.c index 89758c9934ec67..3a19ae8fb5a01d 100644 --- a/drivers/thermal/thermal_core.c +++ b/drivers/thermal/thermal_core.c @@ -41,6 +41,8 @@ static struct thermal_governor *def_governor; static bool thermal_pm_suspended; +static struct workqueue_struct *thermal_wq __ro_after_init; + /* * Governor section: set of functions to handle thermal governors * @@ -313,7 +315,7 @@ static void thermal_zone_device_set_polling(struct thermal_zone_device *tz, if (delay > HZ) delay = round_jiffies_relative(delay); - mod_delayed_work(system_freezable_power_efficient_wq, &tz->poll_queue, delay); + mod_delayed_work(thermal_wq, &tz->poll_queue, delay); } static void thermal_zone_recheck(struct thermal_zone_device *tz, int error) @@ -1636,6 +1638,7 @@ thermal_zone_device_register_with_trips(const char *type, device_del(&tz->device); release_device: put_device(&tz->device); + wait_for_completion(&tz->removal); remove_id: ida_free(&thermal_tz_ida, id); free_tzp: @@ -1781,6 +1784,10 @@ static void thermal_zone_device_resume(struct work_struct *work) guard(thermal_zone)(tz); + /* If the thermal zone is going away, there's nothing to do. */ + if (tz->state & TZ_STATE_FLAG_EXIT) + return; + tz->state &= ~(TZ_STATE_FLAG_SUSPENDED | TZ_STATE_FLAG_RESUMING); thermal_debug_tz_resume(tz); @@ -1807,6 +1814,9 @@ static void thermal_zone_pm_prepare(struct thermal_zone_device *tz) } tz->state |= TZ_STATE_FLAG_SUSPENDED; + + /* Prevent new work from getting to the workqueue subsequently. */ + cancel_delayed_work(&tz->poll_queue); } static void thermal_pm_notify_prepare(void) @@ -1825,8 +1835,6 @@ static void thermal_zone_pm_complete(struct thermal_zone_device *tz) { guard(thermal_zone)(tz); - cancel_delayed_work(&tz->poll_queue); - reinit_completion(&tz->resume); tz->state |= TZ_STATE_FLAG_RESUMING; @@ -1836,7 +1844,7 @@ static void thermal_zone_pm_complete(struct thermal_zone_device *tz) */ INIT_DELAYED_WORK(&tz->poll_queue, thermal_zone_device_resume); /* Queue up the work without a delay. */ - mod_delayed_work(system_freezable_power_efficient_wq, &tz->poll_queue, 0); + mod_delayed_work(thermal_wq, &tz->poll_queue, 0); } static void thermal_pm_notify_complete(void) @@ -1859,6 +1867,11 @@ static int thermal_pm_notify(struct notifier_block *nb, case PM_RESTORE_PREPARE: case PM_SUSPEND_PREPARE: thermal_pm_notify_prepare(); + /* + * Allow any leftover thermal work items already on the + * worqueue to complete so they don't get in the way later. + */ + flush_workqueue(thermal_wq); break; case PM_POST_HIBERNATION: case PM_POST_RESTORE: @@ -1891,9 +1904,16 @@ static int __init thermal_init(void) if (result) goto error; + thermal_wq = alloc_workqueue("thermal_events", + WQ_FREEZABLE | WQ_POWER_EFFICIENT | WQ_PERCPU, 0); + if (!thermal_wq) { + result = -ENOMEM; + goto unregister_netlink; + } + result = thermal_register_governors(); if (result) - goto unregister_netlink; + goto destroy_workqueue; thermal_class = kzalloc(sizeof(*thermal_class), GFP_KERNEL); if (!thermal_class) { @@ -1920,6 +1940,8 @@ static int __init thermal_init(void) unregister_governors: thermal_unregister_governors(); +destroy_workqueue: + destroy_workqueue(thermal_wq); unregister_netlink: thermal_netlink_exit(); error: diff --git a/drivers/thermal/thermal_of.c b/drivers/thermal/thermal_of.c index 1a51a4d240ff60..b6d0c92f5522bd 100644 --- a/drivers/thermal/thermal_of.c +++ b/drivers/thermal/thermal_of.c @@ -280,10 +280,10 @@ static bool thermal_of_cm_lookup(struct device_node *cm_np, struct cooling_spec *c) { for_each_child_of_node_scoped(cm_np, child) { - struct device_node *tr_np; int count, i; - tr_np = of_parse_phandle(child, "trip", 0); + struct device_node *tr_np __free(device_node) = + of_parse_phandle(child, "trip", 0); if (tr_np != trip->priv) continue; diff --git a/drivers/thunderbolt/nhi.c b/drivers/thunderbolt/nhi.c index 6d0c9d37c55d80..b03792390c6ad7 100644 --- a/drivers/thunderbolt/nhi.c +++ b/drivers/thunderbolt/nhi.c @@ -1020,7 +1020,7 @@ static bool nhi_wake_supported(struct pci_dev *pdev) * If power rails are sustainable for wakeup from S4 this * property is set by the BIOS. */ - if (device_property_read_u8(&pdev->dev, "WAKE_SUPPORTED", &val)) + if (!device_property_read_u8(&pdev->dev, "WAKE_SUPPORTED", &val)) return !!val; return true; diff --git a/drivers/tty/serial/8250/8250.h b/drivers/tty/serial/8250/8250.h index 8caecfc85d933c..77fe0588fd6bd5 100644 --- a/drivers/tty/serial/8250/8250.h +++ b/drivers/tty/serial/8250/8250.h @@ -175,7 +175,9 @@ static unsigned int __maybe_unused serial_icr_read(struct uart_8250_port *up, return value; } +void serial8250_clear_fifos(struct uart_8250_port *p); void serial8250_clear_and_reinit_fifos(struct uart_8250_port *p); +void serial8250_fifo_wait_for_lsr_thre(struct uart_8250_port *up, unsigned int count); void serial8250_rpm_get(struct uart_8250_port *p); void serial8250_rpm_put(struct uart_8250_port *p); @@ -400,6 +402,26 @@ static inline bool serial8250_tx_dma_running(struct uart_8250_port *p) return dma && dma->tx_running; } + +static inline void serial8250_tx_dma_pause(struct uart_8250_port *p) +{ + struct uart_8250_dma *dma = p->dma; + + if (!dma->tx_running) + return; + + dmaengine_pause(dma->txchan); +} + +static inline void serial8250_tx_dma_resume(struct uart_8250_port *p) +{ + struct uart_8250_dma *dma = p->dma; + + if (!dma->tx_running) + return; + + dmaengine_resume(dma->txchan); +} #else static inline int serial8250_tx_dma(struct uart_8250_port *p) { @@ -421,6 +443,9 @@ static inline bool serial8250_tx_dma_running(struct uart_8250_port *p) { return false; } + +static inline void serial8250_tx_dma_pause(struct uart_8250_port *p) { } +static inline void serial8250_tx_dma_resume(struct uart_8250_port *p) { } #endif static inline int ns16550a_goto_highspeed(struct uart_8250_port *up) diff --git a/drivers/tty/serial/8250/8250_dma.c b/drivers/tty/serial/8250/8250_dma.c index bdd26c9f34bdf2..3b6452e759d5b5 100644 --- a/drivers/tty/serial/8250/8250_dma.c +++ b/drivers/tty/serial/8250/8250_dma.c @@ -162,7 +162,22 @@ void serial8250_tx_dma_flush(struct uart_8250_port *p) */ dma->tx_size = 0; + /* + * We can't use `dmaengine_terminate_sync` because `uart_flush_buffer` is + * holding the uart port spinlock. + */ dmaengine_terminate_async(dma->txchan); + + /* + * The callback might or might not run. If it doesn't run, we need to ensure + * that `tx_running` is cleared so that we can schedule new transactions. + * If it does run, then the zombie callback will clear `tx_running` again + * and perform a no-op since `tx_size` was cleared above. + * + * In either case, we ASSUME the DMA transaction will terminate before we + * issue a new `serial8250_tx_dma`. + */ + dma->tx_running = 0; } int serial8250_rx_dma(struct uart_8250_port *p) diff --git a/drivers/tty/serial/8250/8250_dw.c b/drivers/tty/serial/8250/8250_dw.c index 27af83f0ff4632..a222ec012861b6 100644 --- a/drivers/tty/serial/8250/8250_dw.c +++ b/drivers/tty/serial/8250/8250_dw.c @@ -9,10 +9,14 @@ * LCR is written whilst busy. If it is, then a busy detect interrupt is * raised, the LCR needs to be rewritten and the uart status register read. */ +#include +#include +#include #include #include #include #include +#include #include #include #include @@ -40,8 +44,12 @@ #define RZN1_UART_RDMACR 0x110 /* DMA Control Register Receive Mode */ /* DesignWare specific register fields */ +#define DW_UART_IIR_IID GENMASK(3, 0) + #define DW_UART_MCR_SIRE BIT(6) +#define DW_UART_USR_BUSY BIT(0) + /* Renesas specific register fields */ #define RZN1_UART_xDMACR_DMA_EN BIT(0) #define RZN1_UART_xDMACR_1_WORD_BURST (0 << 1) @@ -56,6 +64,13 @@ #define DW_UART_QUIRK_IS_DMA_FC BIT(3) #define DW_UART_QUIRK_APMC0D08 BIT(4) #define DW_UART_QUIRK_CPR_VALUE BIT(5) +#define DW_UART_QUIRK_IER_KICK BIT(6) + +/* + * Number of consecutive IIR_NO_INT interrupts required to trigger interrupt + * storm prevention code. + */ +#define DW_UART_QUIRK_IER_KICK_THRES 4 struct dw8250_platform_data { u8 usr_reg; @@ -77,6 +92,9 @@ struct dw8250_data { unsigned int skip_autocfg:1; unsigned int uart_16550_compatible:1; + unsigned int in_idle:1; + + u8 no_int_count; }; static inline struct dw8250_data *to_dw8250_data(struct dw8250_port_data *data) @@ -107,78 +125,167 @@ static inline u32 dw8250_modify_msr(struct uart_port *p, unsigned int offset, u3 return value; } +static void dw8250_idle_exit(struct uart_port *p) +{ + struct dw8250_data *d = to_dw8250_data(p->private_data); + struct uart_8250_port *up = up_to_u8250p(p); + + if (d->uart_16550_compatible) + return; + + if (up->capabilities & UART_CAP_FIFO) + serial_port_out(p, UART_FCR, up->fcr); + serial_port_out(p, UART_MCR, up->mcr); + serial_port_out(p, UART_IER, up->ier); + + /* DMA Rx is restarted by IRQ handler as needed. */ + if (up->dma) + serial8250_tx_dma_resume(up); + + d->in_idle = 0; +} + /* - * This function is being called as part of the uart_port::serial_out() - * routine. Hence, it must not call serial_port_out() or serial_out() - * against the modified registers here, i.e. LCR. + * Ensure BUSY is not asserted. If DW UART is configured with + * !uart_16550_compatible, the writes to LCR, DLL, and DLH fail while + * BUSY is asserted. + * + * Context: port's lock must be held */ -static void dw8250_force_idle(struct uart_port *p) +static int dw8250_idle_enter(struct uart_port *p) { + struct dw8250_data *d = to_dw8250_data(p->private_data); + unsigned int usr_reg = d->pdata ? d->pdata->usr_reg : DW_UART_USR; struct uart_8250_port *up = up_to_u8250p(p); - unsigned int lsr; + int retries; + u32 lsr; - /* - * The following call currently performs serial_out() - * against the FCR register. Because it differs to LCR - * there will be no infinite loop, but if it ever gets - * modified, we might need a new custom version of it - * that avoids infinite recursion. - */ - serial8250_clear_and_reinit_fifos(up); + lockdep_assert_held_once(&p->lock); + + if (d->uart_16550_compatible) + return 0; + + d->in_idle = 1; + + /* Prevent triggering interrupt from RBR filling */ + serial_port_out(p, UART_IER, 0); + + if (up->dma) { + serial8250_rx_dma_flush(up); + if (serial8250_tx_dma_running(up)) + serial8250_tx_dma_pause(up); + } /* - * With PSLVERR_RESP_EN parameter set to 1, the device generates an - * error response when an attempt to read an empty RBR with FIFO - * enabled. + * Wait until Tx becomes empty + one extra frame time to ensure all bits + * have been sent on the wire. + * + * FIXME: frame_time delay is too long with very low baudrates. */ - if (up->fcr & UART_FCR_ENABLE_FIFO) { - lsr = serial_port_in(p, UART_LSR); - if (!(lsr & UART_LSR_DR)) - return; + serial8250_fifo_wait_for_lsr_thre(up, p->fifosize); + ndelay(p->frame_time); + + serial_port_out(p, UART_MCR, up->mcr | UART_MCR_LOOP); + + retries = 4; /* Arbitrary limit, 2 was always enough in tests */ + do { + serial8250_clear_fifos(up); + if (!(serial_port_in(p, usr_reg) & DW_UART_USR_BUSY)) + break; + /* FIXME: frame_time delay is too long with very low baudrates. */ + ndelay(p->frame_time); + } while (--retries); + + lsr = serial_lsr_in(up); + if (lsr & UART_LSR_DR) { + serial_port_in(p, UART_RX); + up->lsr_saved_flags = 0; + } + + /* Now guaranteed to have BUSY deasserted? Just sanity check */ + if (serial_port_in(p, usr_reg) & DW_UART_USR_BUSY) { + dw8250_idle_exit(p); + return -EBUSY; } - serial_port_in(p, UART_RX); + return 0; +} + +static void dw8250_set_divisor(struct uart_port *p, unsigned int baud, + unsigned int quot, unsigned int quot_frac) +{ + struct uart_8250_port *up = up_to_u8250p(p); + int ret; + + ret = dw8250_idle_enter(p); + if (ret < 0) + return; + + serial_port_out(p, UART_LCR, up->lcr | UART_LCR_DLAB); + if (!(serial_port_in(p, UART_LCR) & UART_LCR_DLAB)) + goto idle_failed; + + serial_dl_write(up, quot); + serial_port_out(p, UART_LCR, up->lcr); + +idle_failed: + dw8250_idle_exit(p); } /* * This function is being called as part of the uart_port::serial_out() - * routine. Hence, it must not call serial_port_out() or serial_out() - * against the modified registers here, i.e. LCR. + * routine. Hence, special care must be taken when serial_port_out() or + * serial_out() against the modified registers here, i.e. LCR (d->in_idle is + * used to break recursion loop). */ static void dw8250_check_lcr(struct uart_port *p, unsigned int offset, u32 value) { struct dw8250_data *d = to_dw8250_data(p->private_data); - void __iomem *addr = p->membase + (offset << p->regshift); - int tries = 1000; + u32 lcr; + int ret; if (offset != UART_LCR || d->uart_16550_compatible) return; + lcr = serial_port_in(p, UART_LCR); + /* Make sure LCR write wasn't ignored */ - while (tries--) { - u32 lcr = serial_port_in(p, offset); + if ((value & ~UART_LCR_SPAR) == (lcr & ~UART_LCR_SPAR)) + return; - if ((value & ~UART_LCR_SPAR) == (lcr & ~UART_LCR_SPAR)) - return; + if (d->in_idle) + goto write_err; - dw8250_force_idle(p); + ret = dw8250_idle_enter(p); + if (ret < 0) + goto write_err; -#ifdef CONFIG_64BIT - if (p->type == PORT_OCTEON) - __raw_writeq(value & 0xff, addr); - else -#endif - if (p->iotype == UPIO_MEM32) - writel(value, addr); - else if (p->iotype == UPIO_MEM32BE) - iowrite32be(value, addr); - else - writeb(value, addr); - } + serial_port_out(p, UART_LCR, value); + dw8250_idle_exit(p); + return; + +write_err: /* * FIXME: this deadlocks if port->lock is already held * dev_err(p->dev, "Couldn't set LCR to %d\n", value); */ + return; /* Silences "label at the end of compound statement" */ +} + +/* + * With BUSY, LCR writes can be very expensive (IRQ + complex retry logic). + * If the write does not change the value of the LCR register, skip it entirely. + */ +static bool dw8250_can_skip_reg_write(struct uart_port *p, unsigned int offset, u32 value) +{ + struct dw8250_data *d = to_dw8250_data(p->private_data); + u32 lcr; + + if (offset != UART_LCR || d->uart_16550_compatible) + return false; + + lcr = serial_port_in(p, offset); + return lcr == value; } /* Returns once the transmitter is empty or we run out of retries */ @@ -207,12 +314,18 @@ static void dw8250_tx_wait_empty(struct uart_port *p) static void dw8250_serial_out(struct uart_port *p, unsigned int offset, u32 value) { + if (dw8250_can_skip_reg_write(p, offset, value)) + return; + writeb(value, p->membase + (offset << p->regshift)); dw8250_check_lcr(p, offset, value); } static void dw8250_serial_out38x(struct uart_port *p, unsigned int offset, u32 value) { + if (dw8250_can_skip_reg_write(p, offset, value)) + return; + /* Allow the TX to drain before we reconfigure */ if (offset == UART_LCR) dw8250_tx_wait_empty(p); @@ -237,6 +350,9 @@ static u32 dw8250_serial_inq(struct uart_port *p, unsigned int offset) static void dw8250_serial_outq(struct uart_port *p, unsigned int offset, u32 value) { + if (dw8250_can_skip_reg_write(p, offset, value)) + return; + value &= 0xff; __raw_writeq(value, p->membase + (offset << p->regshift)); /* Read back to ensure register write ordering. */ @@ -248,6 +364,9 @@ static void dw8250_serial_outq(struct uart_port *p, unsigned int offset, u32 val static void dw8250_serial_out32(struct uart_port *p, unsigned int offset, u32 value) { + if (dw8250_can_skip_reg_write(p, offset, value)) + return; + writel(value, p->membase + (offset << p->regshift)); dw8250_check_lcr(p, offset, value); } @@ -261,6 +380,9 @@ static u32 dw8250_serial_in32(struct uart_port *p, unsigned int offset) static void dw8250_serial_out32be(struct uart_port *p, unsigned int offset, u32 value) { + if (dw8250_can_skip_reg_write(p, offset, value)) + return; + iowrite32be(value, p->membase + (offset << p->regshift)); dw8250_check_lcr(p, offset, value); } @@ -272,6 +394,29 @@ static u32 dw8250_serial_in32be(struct uart_port *p, unsigned int offset) return dw8250_modify_msr(p, offset, value); } +/* + * INTC10EE UART can IRQ storm while reporting IIR_NO_INT. Inducing IIR value + * change has been observed to break the storm. + * + * If Tx is empty (THRE asserted), we use here IER_THRI to cause IIR_NO_INT -> + * IIR_THRI transition. + */ +static void dw8250_quirk_ier_kick(struct uart_port *p) +{ + struct uart_8250_port *up = up_to_u8250p(p); + u32 lsr; + + if (up->ier & UART_IER_THRI) + return; + + lsr = serial_lsr_in(up); + if (!(lsr & UART_LSR_THRE)) + return; + + serial_port_out(p, UART_IER, up->ier | UART_IER_THRI); + serial_port_in(p, UART_LCR); /* safe, no side-effects */ + serial_port_out(p, UART_IER, up->ier); +} static int dw8250_handle_irq(struct uart_port *p) { @@ -281,7 +426,31 @@ static int dw8250_handle_irq(struct uart_port *p) bool rx_timeout = (iir & 0x3f) == UART_IIR_RX_TIMEOUT; unsigned int quirks = d->pdata->quirks; unsigned int status; - unsigned long flags; + + guard(uart_port_lock_irqsave)(p); + + switch (FIELD_GET(DW_UART_IIR_IID, iir)) { + case UART_IIR_NO_INT: + if (d->uart_16550_compatible || up->dma) + return 0; + + if (quirks & DW_UART_QUIRK_IER_KICK && + d->no_int_count == (DW_UART_QUIRK_IER_KICK_THRES - 1)) + dw8250_quirk_ier_kick(p); + d->no_int_count = (d->no_int_count + 1) % DW_UART_QUIRK_IER_KICK_THRES; + + return 0; + + case UART_IIR_BUSY: + /* Clear the USR */ + serial_port_in(p, d->pdata->usr_reg); + + d->no_int_count = 0; + + return 1; + } + + d->no_int_count = 0; /* * There are ways to get Designware-based UARTs into a state where @@ -294,20 +463,15 @@ static int dw8250_handle_irq(struct uart_port *p) * so we limit the workaround only to non-DMA mode. */ if (!up->dma && rx_timeout) { - uart_port_lock_irqsave(p, &flags); status = serial_lsr_in(up); if (!(status & (UART_LSR_DR | UART_LSR_BI))) serial_port_in(p, UART_RX); - - uart_port_unlock_irqrestore(p, flags); } /* Manually stop the Rx DMA transfer when acting as flow controller */ if (quirks & DW_UART_QUIRK_IS_DMA_FC && up->dma && up->dma->rx_running && rx_timeout) { - uart_port_lock_irqsave(p, &flags); status = serial_lsr_in(up); - uart_port_unlock_irqrestore(p, flags); if (status & (UART_LSR_DR | UART_LSR_BI)) { dw8250_writel_ext(p, RZN1_UART_RDMACR, 0); @@ -315,17 +479,9 @@ static int dw8250_handle_irq(struct uart_port *p) } } - if (serial8250_handle_irq(p, iir)) - return 1; + serial8250_handle_irq_locked(p, iir); - if ((iir & UART_IIR_BUSY) == UART_IIR_BUSY) { - /* Clear the USR */ - serial_port_in(p, d->pdata->usr_reg); - - return 1; - } - - return 0; + return 1; } static void dw8250_clk_work_cb(struct work_struct *work) @@ -527,6 +683,14 @@ static void dw8250_reset_control_assert(void *data) reset_control_assert(data); } +static void dw8250_shutdown(struct uart_port *port) +{ + struct dw8250_data *d = to_dw8250_data(port->private_data); + + serial8250_do_shutdown(port); + d->no_int_count = 0; +} + static int dw8250_probe(struct platform_device *pdev) { struct uart_8250_port uart = {}, *up = &uart; @@ -545,8 +709,10 @@ static int dw8250_probe(struct platform_device *pdev) p->type = PORT_8250; p->flags = UPF_FIXED_PORT; p->dev = dev; + p->set_ldisc = dw8250_set_ldisc; p->set_termios = dw8250_set_termios; + p->set_divisor = dw8250_set_divisor; data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL); if (!data) @@ -650,10 +816,12 @@ static int dw8250_probe(struct platform_device *pdev) dw8250_quirks(p, data); /* If the Busy Functionality is not implemented, don't handle it */ - if (data->uart_16550_compatible) + if (data->uart_16550_compatible) { p->handle_irq = NULL; - else if (data->pdata) + } else if (data->pdata) { p->handle_irq = dw8250_handle_irq; + p->shutdown = dw8250_shutdown; + } dw8250_setup_dma_filter(p, data); @@ -741,11 +909,18 @@ static int dw8250_runtime_suspend(struct device *dev) static int dw8250_runtime_resume(struct device *dev) { + int ret; struct dw8250_data *data = dev_get_drvdata(dev); - clk_prepare_enable(data->pclk); + ret = clk_prepare_enable(data->pclk); + if (ret) + return ret; - clk_prepare_enable(data->clk); + ret = clk_prepare_enable(data->clk); + if (ret) { + clk_disable_unprepare(data->pclk); + return ret; + } return 0; } @@ -780,6 +955,11 @@ static const struct dw8250_platform_data dw8250_skip_set_rate_data = { .quirks = DW_UART_QUIRK_SKIP_SET_RATE, }; +static const struct dw8250_platform_data dw8250_intc10ee = { + .usr_reg = DW_UART_USR, + .quirks = DW_UART_QUIRK_IER_KICK, +}; + static const struct of_device_id dw8250_of_match[] = { { .compatible = "snps,dw-apb-uart", .data = &dw8250_dw_apb }, { .compatible = "cavium,octeon-3860-uart", .data = &dw8250_octeon_3860_data }, @@ -809,7 +989,7 @@ static const struct acpi_device_id dw8250_acpi_match[] = { { "INT33C5", (kernel_ulong_t)&dw8250_dw_apb }, { "INT3434", (kernel_ulong_t)&dw8250_dw_apb }, { "INT3435", (kernel_ulong_t)&dw8250_dw_apb }, - { "INTC10EE", (kernel_ulong_t)&dw8250_dw_apb }, + { "INTC10EE", (kernel_ulong_t)&dw8250_intc10ee }, { }, }; MODULE_DEVICE_TABLE(acpi, dw8250_acpi_match); @@ -827,6 +1007,7 @@ static struct platform_driver dw8250_platform_driver = { module_platform_driver(dw8250_platform_driver); +MODULE_IMPORT_NS("SERIAL_8250"); MODULE_AUTHOR("Jamie Iles"); MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("Synopsys DesignWare 8250 serial port driver"); diff --git a/drivers/tty/serial/8250/8250_omap.c b/drivers/tty/serial/8250/8250_omap.c index 9e49ef48b851bf..272bc07c9a6b52 100644 --- a/drivers/tty/serial/8250/8250_omap.c +++ b/drivers/tty/serial/8250/8250_omap.c @@ -100,6 +100,9 @@ #define OMAP_UART_REV_52 0x0502 #define OMAP_UART_REV_63 0x0603 +/* Resume register */ +#define UART_OMAP_RESUME 0x0B + /* Interrupt Enable Register 2 */ #define UART_OMAP_IER2 0x1B #define UART_OMAP_IER2_RHR_IT_DIS BIT(2) @@ -119,7 +122,6 @@ /* Timeout low and High */ #define UART_OMAP_TO_L 0x26 #define UART_OMAP_TO_H 0x27 - struct omap8250_priv { void __iomem *membase; int line; @@ -929,7 +931,6 @@ static void __dma_rx_do_complete(struct uart_8250_port *p) goto out; cookie = dma->rx_cookie; - dma->rx_running = 0; /* Re-enable RX FIFO interrupt now that transfer is complete */ if (priv->habit & UART_HAS_RHR_IT_DIS) { @@ -963,6 +964,7 @@ static void __dma_rx_do_complete(struct uart_8250_port *p) goto out; ret = tty_insert_flip_string(tty_port, dma->rx_buf, count); + dma->rx_running = 0; p->port.icount.rx += ret; p->port.icount.buf_overrun += count - ret; out: @@ -1256,6 +1258,20 @@ static u16 omap_8250_handle_rx_dma(struct uart_8250_port *up, u8 iir, u16 status return status; } +static void am654_8250_handle_uart_errors(struct uart_8250_port *up, u8 iir, u16 status) +{ + if (status & UART_LSR_OE) { + serial8250_clear_and_reinit_fifos(up); + serial_in(up, UART_LSR); + serial_in(up, UART_OMAP_RESUME); + } else { + if (status & (UART_LSR_FE | UART_LSR_PE | UART_LSR_BI)) + serial_in(up, UART_RX); + if (iir & UART_IIR_XOFF) + serial_in(up, UART_IIR); + } +} + static void am654_8250_handle_rx_dma(struct uart_8250_port *up, u8 iir, u16 status) { @@ -1266,7 +1282,8 @@ static void am654_8250_handle_rx_dma(struct uart_8250_port *up, u8 iir, * Queue a new transfer if FIFO has data. */ if ((status & (UART_LSR_DR | UART_LSR_BI)) && - (up->ier & UART_IER_RDI)) { + (up->ier & UART_IER_RDI) && !(status & UART_LSR_OE)) { + am654_8250_handle_uart_errors(up, iir, status); omap_8250_rx_dma(up); serial_out(up, UART_OMAP_EFR2, UART_OMAP_EFR2_TIMEOUT_BEHAVE); } else if ((iir & 0x3f) == UART_IIR_RX_TIMEOUT) { @@ -1282,6 +1299,8 @@ static void am654_8250_handle_rx_dma(struct uart_8250_port *up, u8 iir, serial_out(up, UART_OMAP_EFR2, 0x0); up->ier |= UART_IER_RLSI | UART_IER_RDI; serial_out(up, UART_IER, up->ier); + } else { + am654_8250_handle_uart_errors(up, iir, status); } } diff --git a/drivers/tty/serial/8250/8250_pci.c b/drivers/tty/serial/8250/8250_pci.c index 3efe075ef7b210..a85ef082ca59e9 100644 --- a/drivers/tty/serial/8250/8250_pci.c +++ b/drivers/tty/serial/8250/8250_pci.c @@ -137,6 +137,8 @@ struct serial_private { }; #define PCI_DEVICE_ID_HPE_PCI_SERIAL 0x37e +#define PCIE_VENDOR_ID_ASIX 0x125B +#define PCIE_DEVICE_ID_AX99100 0x9100 static const struct pci_device_id pci_use_msi[] = { { PCI_DEVICE_SUB(PCI_VENDOR_ID_NETMOS, PCI_DEVICE_ID_NETMOS_9900, @@ -149,6 +151,8 @@ static const struct pci_device_id pci_use_msi[] = { 0xA000, 0x1000) }, { PCI_DEVICE_SUB(PCI_VENDOR_ID_HP_3PAR, PCI_DEVICE_ID_HPE_PCI_SERIAL, PCI_ANY_ID, PCI_ANY_ID) }, + { PCI_DEVICE_SUB(PCIE_VENDOR_ID_ASIX, PCIE_DEVICE_ID_AX99100, + 0xA000, 0x1000) }, { } }; @@ -920,6 +924,7 @@ static int pci_netmos_init(struct pci_dev *dev) case PCI_DEVICE_ID_NETMOS_9912: case PCI_DEVICE_ID_NETMOS_9922: case PCI_DEVICE_ID_NETMOS_9900: + case PCIE_DEVICE_ID_AX99100: num_serial = pci_netmos_9900_numports(dev); break; @@ -2555,6 +2560,14 @@ static struct pci_serial_quirk pci_serial_quirks[] = { .init = pci_netmos_init, .setup = pci_netmos_9900_setup, }, + { + .vendor = PCIE_VENDOR_ID_ASIX, + .device = PCI_ANY_ID, + .subvendor = PCI_ANY_ID, + .subdevice = PCI_ANY_ID, + .init = pci_netmos_init, + .setup = pci_netmos_9900_setup, + }, /* * EndRun Technologies */ @@ -6076,6 +6089,10 @@ static const struct pci_device_id serial_pci_tbl[] = { 0xA000, 0x3002, 0, 0, pbn_NETMOS9900_2s_115200 }, + { PCIE_VENDOR_ID_ASIX, PCIE_DEVICE_ID_AX99100, + 0xA000, 0x1000, + 0, 0, pbn_b0_1_115200 }, + /* * Best Connectivity and Rosewill PCI Multi I/O cards */ diff --git a/drivers/tty/serial/8250/8250_port.c b/drivers/tty/serial/8250/8250_port.c index 719faf92aa8aed..8785961a2a82e0 100644 --- a/drivers/tty/serial/8250/8250_port.c +++ b/drivers/tty/serial/8250/8250_port.c @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -488,7 +489,7 @@ serial_port_out_sync(struct uart_port *p, int offset, int value) /* * FIFO support. */ -static void serial8250_clear_fifos(struct uart_8250_port *p) +void serial8250_clear_fifos(struct uart_8250_port *p) { if (p->capabilities & UART_CAP_FIFO) { serial_out(p, UART_FCR, UART_FCR_ENABLE_FIFO); @@ -497,6 +498,7 @@ static void serial8250_clear_fifos(struct uart_8250_port *p) serial_out(p, UART_FCR, 0); } } +EXPORT_SYMBOL_NS_GPL(serial8250_clear_fifos, "SERIAL_8250"); static enum hrtimer_restart serial8250_em485_handle_start_tx(struct hrtimer *t); static enum hrtimer_restart serial8250_em485_handle_stop_tx(struct hrtimer *t); @@ -1782,20 +1784,16 @@ static bool handle_rx_dma(struct uart_8250_port *up, unsigned int iir) } /* - * This handles the interrupt from one port. + * Context: port's lock must be held by the caller. */ -int serial8250_handle_irq(struct uart_port *port, unsigned int iir) +void serial8250_handle_irq_locked(struct uart_port *port, unsigned int iir) { struct uart_8250_port *up = up_to_u8250p(port); struct tty_port *tport = &port->state->port; bool skip_rx = false; - unsigned long flags; u16 status; - if (iir & UART_IIR_NO_INT) - return 0; - - uart_port_lock_irqsave(port, &flags); + lockdep_assert_held_once(&port->lock); status = serial_lsr_in(up); @@ -1828,8 +1826,19 @@ int serial8250_handle_irq(struct uart_port *port, unsigned int iir) else if (!up->dma->tx_running) __stop_tx(up); } +} +EXPORT_SYMBOL_NS_GPL(serial8250_handle_irq_locked, "SERIAL_8250"); + +/* + * This handles the interrupt from one port. + */ +int serial8250_handle_irq(struct uart_port *port, unsigned int iir) +{ + if (iir & UART_IIR_NO_INT) + return 0; - uart_unlock_and_check_sysrq_irqrestore(port, flags); + guard(uart_port_lock_irqsave)(port); + serial8250_handle_irq_locked(port, iir); return 1; } @@ -2147,8 +2156,7 @@ static void serial8250_THRE_test(struct uart_port *port) if (up->port.flags & UPF_NO_THRE_TEST) return; - if (port->irqflags & IRQF_SHARED) - disable_irq_nosync(port->irq); + disable_irq(port->irq); /* * Test for UARTs that do not reassert THRE when the transmitter is idle and the interrupt @@ -2170,8 +2178,7 @@ static void serial8250_THRE_test(struct uart_port *port) serial_port_out(port, UART_IER, 0); } - if (port->irqflags & IRQF_SHARED) - enable_irq(port->irq); + enable_irq(port->irq); /* * If the interrupt is not reasserted, or we otherwise don't trust the iir, setup a timer to @@ -2350,6 +2357,7 @@ static int serial8250_startup(struct uart_port *port) void serial8250_do_shutdown(struct uart_port *port) { struct uart_8250_port *up = up_to_u8250p(port); + u32 lcr; serial8250_rpm_get(up); /* @@ -2376,13 +2384,13 @@ void serial8250_do_shutdown(struct uart_port *port) port->mctrl &= ~TIOCM_OUT2; serial8250_set_mctrl(port, port->mctrl); + + /* Disable break condition */ + lcr = serial_port_in(port, UART_LCR); + lcr &= ~UART_LCR_SBC; + serial_port_out(port, UART_LCR, lcr); } - /* - * Disable break condition and FIFOs - */ - serial_port_out(port, UART_LCR, - serial_port_in(port, UART_LCR) & ~UART_LCR_SBC); serial8250_clear_fifos(up); rsa_disable(up); @@ -2392,6 +2400,12 @@ void serial8250_do_shutdown(struct uart_port *port) * the IRQ chain. */ serial_port_in(port, UART_RX); + /* + * LCR writes on DW UART can trigger late (unmaskable) IRQs. + * Handle them before releasing the handler. + */ + synchronize_irq(port->irq); + serial8250_rpm_put(up); up->ops->release_irq(up); @@ -3185,6 +3199,17 @@ void serial8250_set_defaults(struct uart_8250_port *up) } EXPORT_SYMBOL_GPL(serial8250_set_defaults); +void serial8250_fifo_wait_for_lsr_thre(struct uart_8250_port *up, unsigned int count) +{ + unsigned int i; + + for (i = 0; i < count; i++) { + if (wait_for_lsr(up, UART_LSR_THRE)) + return; + } +} +EXPORT_SYMBOL_NS_GPL(serial8250_fifo_wait_for_lsr_thre, "SERIAL_8250"); + #ifdef CONFIG_SERIAL_8250_CONSOLE static void serial8250_console_putchar(struct uart_port *port, unsigned char ch) @@ -3226,16 +3251,6 @@ static void serial8250_console_restore(struct uart_8250_port *up) serial8250_out_MCR(up, up->mcr | UART_MCR_DTR | UART_MCR_RTS); } -static void fifo_wait_for_lsr(struct uart_8250_port *up, unsigned int count) -{ - unsigned int i; - - for (i = 0; i < count; i++) { - if (wait_for_lsr(up, UART_LSR_THRE)) - return; - } -} - /* * Print a string to the serial port using the device FIFO * @@ -3254,7 +3269,7 @@ static void serial8250_console_fifo_write(struct uart_8250_port *up, while (s != end) { /* Allow timeout for each byte of a possibly full FIFO */ - fifo_wait_for_lsr(up, fifosize); + serial8250_fifo_wait_for_lsr_thre(up, fifosize); for (i = 0; i < fifosize && s != end; ++i) { if (*s == '\n' && !cr_sent) { @@ -3272,7 +3287,7 @@ static void serial8250_console_fifo_write(struct uart_8250_port *up, * Allow timeout for each byte written since the caller will only wait * for UART_LSR_BOTH_EMPTY using the timeout of a single character */ - fifo_wait_for_lsr(up, tx_count); + serial8250_fifo_wait_for_lsr_thre(up, tx_count); } /* diff --git a/drivers/tty/serial/Kconfig b/drivers/tty/serial/Kconfig index 59221cce0028f6..3d06c65c2f491d 100644 --- a/drivers/tty/serial/Kconfig +++ b/drivers/tty/serial/Kconfig @@ -486,14 +486,14 @@ config SERIAL_IMX can enable its onboard serial port by enabling this option. config SERIAL_IMX_CONSOLE - tristate "Console on IMX serial port" + bool "Console on IMX serial port" depends on SERIAL_IMX select SERIAL_CORE_CONSOLE help If you have enabled the serial port on the Freescale IMX - CPU you can make it the console by answering Y/M to this option. + CPU you can make it the console by answering Y to this option. - Even if you say Y/M here, the currently visible virtual console + Even if you say Y here, the currently visible virtual console (/dev/tty0) will still be used as the system console by default, but you can alter that using a kernel command line option such as "console=ttymxc0". (Try "man bootparam" or see the documentation of @@ -671,7 +671,7 @@ config SERIAL_SH_SCI_EARLYCON default ARCH_RENESAS config SERIAL_SH_SCI_DMA - bool "DMA support" if EXPERT + bool "Support for DMA on SuperH SCI(F)" if EXPERT depends on SERIAL_SH_SCI && DMA_ENGINE default ARCH_RENESAS diff --git a/drivers/tty/serial/rsci.c b/drivers/tty/serial/rsci.c index b3c48dc1e07dbb..0533a4bb1d03c8 100644 --- a/drivers/tty/serial/rsci.c +++ b/drivers/tty/serial/rsci.c @@ -151,6 +151,22 @@ static void rsci_start_rx(struct uart_port *port) rsci_serial_out(port, CCR0, ctrl); } +static int rsci_scif_set_rtrg(struct uart_port *port, int rx_trig) +{ + u32 fcr = rsci_serial_in(port, FCR); + + if (rx_trig >= port->fifosize) + rx_trig = port->fifosize - 1; + else if (rx_trig < 1) + rx_trig = 0; + + fcr &= ~FCR_RTRG4_0; + fcr |= field_prep(FCR_RTRG4_0, rx_trig); + rsci_serial_out(port, FCR, fcr); + + return rx_trig; +} + static void rsci_set_termios(struct uart_port *port, struct ktermios *termios, const struct ktermios *old) { @@ -454,6 +470,7 @@ static const struct sci_port_ops rsci_port_ops = { .poll_put_char = rsci_poll_put_char, .prepare_console_write = rsci_prepare_console_write, .suspend_regs_size = rsci_suspend_regs_size, + .set_rtrg = rsci_scif_set_rtrg, .shutdown_complete = rsci_shutdown_complete, }; diff --git a/drivers/tty/serial/samsung_tty.c b/drivers/tty/serial/samsung_tty.c index c1fabad6ba1faf..99d83271d8dfbe 100644 --- a/drivers/tty/serial/samsung_tty.c +++ b/drivers/tty/serial/samsung_tty.c @@ -34,6 +34,7 @@ #include #include #include +#include #include #include #include @@ -1298,30 +1299,49 @@ static int apple_s5l_serial_startup(struct uart_port *port) return ret; } +static int __maybe_unused s3c24xx_serial_runtime_suspend(struct device *dev) +{ + struct uart_port *port = dev_get_drvdata(dev); + struct s3c24xx_uart_port *ourport = to_ourport(port); + int timeout = 10000; + + while (--timeout && !s3c24xx_serial_txempty_nofifo(port)) + udelay(100); + + if (!IS_ERR(ourport->baudclk)) + clk_disable_unprepare(ourport->baudclk); + + clk_disable_unprepare(ourport->clk); + return 0; +}; + +static int __maybe_unused s3c24xx_serial_runtime_resume(struct device *dev) +{ + struct uart_port *port = dev_get_drvdata(dev); + struct s3c24xx_uart_port *ourport = to_ourport(port); + + clk_prepare_enable(ourport->clk); + + if (!IS_ERR(ourport->baudclk)) + clk_prepare_enable(ourport->baudclk); + return 0; +}; + static void s3c24xx_serial_pm(struct uart_port *port, unsigned int level, unsigned int old) { struct s3c24xx_uart_port *ourport = to_ourport(port); - int timeout = 10000; ourport->pm_level = level; switch (level) { - case 3: - while (--timeout && !s3c24xx_serial_txempty_nofifo(port)) - udelay(100); - - if (!IS_ERR(ourport->baudclk)) - clk_disable_unprepare(ourport->baudclk); - - clk_disable_unprepare(ourport->clk); + case UART_PM_STATE_OFF: + pm_runtime_mark_last_busy(port->dev); + pm_runtime_put_sync(port->dev); break; - case 0: - clk_prepare_enable(ourport->clk); - - if (!IS_ERR(ourport->baudclk)) - clk_prepare_enable(ourport->baudclk); + case UART_PM_STATE_ON: + pm_runtime_get_sync(port->dev); break; default: dev_err(port->dev, "s3c24xx_serial: unknown pm %d\n", level); @@ -2044,18 +2064,15 @@ static int s3c24xx_serial_probe(struct platform_device *pdev) } } + pm_runtime_get_noresume(&pdev->dev); + pm_runtime_set_active(&pdev->dev); + pm_runtime_enable(&pdev->dev); + dev_dbg(&pdev->dev, "%s: adding port\n", __func__); uart_add_one_port(&s3c24xx_uart_drv, &ourport->port); platform_set_drvdata(pdev, &ourport->port); - /* - * Deactivate the clock enabled in s3c24xx_serial_init_port here, - * so that a potential re-enablement through the pm-callback overlaps - * and keeps the clock enabled in this case. - */ - clk_disable_unprepare(ourport->clk); - if (!IS_ERR(ourport->baudclk)) - clk_disable_unprepare(ourport->baudclk); + pm_runtime_put_sync(&pdev->dev); probe_index++; @@ -2065,26 +2082,40 @@ static int s3c24xx_serial_probe(struct platform_device *pdev) static void s3c24xx_serial_remove(struct platform_device *dev) { struct uart_port *port = s3c24xx_dev_to_port(&dev->dev); + struct s3c24xx_uart_port *ourport = to_ourport(port); - if (port) + if (port) { + pm_runtime_get_sync(&dev->dev); uart_remove_one_port(&s3c24xx_uart_drv, port); + clk_disable_unprepare(ourport->clk); + if (!IS_ERR(ourport->baudclk)) + clk_disable_unprepare(ourport->baudclk); + + pm_runtime_disable(&dev->dev); + pm_runtime_set_suspended(&dev->dev); + pm_runtime_put_noidle(&dev->dev); + } + uart_unregister_driver(&s3c24xx_uart_drv); } /* UART power management code */ -#ifdef CONFIG_PM_SLEEP -static int s3c24xx_serial_suspend(struct device *dev) + +static int __maybe_unused s3c24xx_serial_suspend(struct device *dev) { struct uart_port *port = s3c24xx_dev_to_port(dev); + if (!console_suspend_enabled && uart_console(port)) + device_set_wakeup_path(dev); + if (port) uart_suspend_port(&s3c24xx_uart_drv, port); return 0; } -static int s3c24xx_serial_resume(struct device *dev) +static int __maybe_unused s3c24xx_serial_resume(struct device *dev) { struct uart_port *port = s3c24xx_dev_to_port(dev); struct s3c24xx_uart_port *ourport = to_ourport(port); @@ -2104,7 +2135,7 @@ static int s3c24xx_serial_resume(struct device *dev) return 0; } -static int s3c24xx_serial_resume_noirq(struct device *dev) +static int __maybe_unused s3c24xx_serial_resume_noirq(struct device *dev) { struct uart_port *port = s3c24xx_dev_to_port(dev); struct s3c24xx_uart_port *ourport = to_ourport(port); @@ -2178,13 +2209,9 @@ static int s3c24xx_serial_resume_noirq(struct device *dev) static const struct dev_pm_ops s3c24xx_serial_pm_ops = { SET_SYSTEM_SLEEP_PM_OPS(s3c24xx_serial_suspend, s3c24xx_serial_resume) SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(NULL, s3c24xx_serial_resume_noirq) + SET_RUNTIME_PM_OPS(s3c24xx_serial_runtime_suspend, + s3c24xx_serial_runtime_resume, NULL) }; -#define SERIAL_SAMSUNG_PM_OPS (&s3c24xx_serial_pm_ops) - -#else /* !CONFIG_PM_SLEEP */ - -#define SERIAL_SAMSUNG_PM_OPS NULL -#endif /* CONFIG_PM_SLEEP */ /* Console code */ @@ -2672,7 +2699,7 @@ static struct platform_driver samsung_serial_driver = { .id_table = s3c24xx_serial_driver_ids, .driver = { .name = "samsung-uart", - .pm = SERIAL_SAMSUNG_PM_OPS, + .pm = &s3c24xx_serial_pm_ops, .of_match_table = of_match_ptr(s3c24xx_uart_dt_match), }, }; diff --git a/drivers/tty/serial/serial_core.c b/drivers/tty/serial/serial_core.c index 2805cad105113c..0b2edf185cc7b0 100644 --- a/drivers/tty/serial/serial_core.c +++ b/drivers/tty/serial/serial_core.c @@ -643,7 +643,10 @@ static unsigned int uart_write_room(struct tty_struct *tty) unsigned int ret; port = uart_port_ref_lock(state, &flags); - ret = kfifo_avail(&state->port.xmit_fifo); + if (!state->port.xmit_buf) + ret = 0; + else + ret = kfifo_avail(&state->port.xmit_fifo); uart_port_unlock_deref(port, flags); return ret; } diff --git a/drivers/tty/serial/uartlite.c b/drivers/tty/serial/uartlite.c index 39c1fd1ff9cedd..6240c3d4dfd798 100644 --- a/drivers/tty/serial/uartlite.c +++ b/drivers/tty/serial/uartlite.c @@ -878,6 +878,7 @@ static int ulite_probe(struct platform_device *pdev) pm_runtime_use_autosuspend(&pdev->dev); pm_runtime_set_autosuspend_delay(&pdev->dev, UART_AUTOSUSPEND_TIMEOUT); pm_runtime_set_active(&pdev->dev); + pm_runtime_get_noresume(&pdev->dev); pm_runtime_enable(&pdev->dev); ret = ulite_assign(&pdev->dev, id, res->start, irq, pdata); diff --git a/drivers/tty/vt/keyboard.c b/drivers/tty/vt/keyboard.c index d65fc60dd7beda..3538d54d6a6ac6 100644 --- a/drivers/tty/vt/keyboard.c +++ b/drivers/tty/vt/keyboard.c @@ -1649,134 +1649,143 @@ int __init kbd_init(void) /* Ioctl support code */ -/** - * vt_do_diacrit - diacritical table updates - * @cmd: ioctl request - * @udp: pointer to user data for ioctl - * @perm: permissions check computed by caller - * - * Update the diacritical tables atomically and safely. Lock them - * against simultaneous keypresses - */ -int vt_do_diacrit(unsigned int cmd, void __user *udp, int perm) +static int vt_do_kdgkbdiacr(void __user *udp) { - int asize; - - switch (cmd) { - case KDGKBDIACR: - { - struct kbdiacrs __user *a = udp; - int i; + struct kbdiacrs __user *a = udp; + int i, asize; - struct kbdiacr __free(kfree) *dia = kmalloc_array(MAX_DIACR, sizeof(struct kbdiacr), - GFP_KERNEL); - if (!dia) - return -ENOMEM; + struct kbdiacr __free(kfree) *dia = kmalloc_array(MAX_DIACR, sizeof(struct kbdiacr), + GFP_KERNEL); + if (!dia) + return -ENOMEM; - /* Lock the diacriticals table, make a copy and then - copy it after we unlock */ - scoped_guard(spinlock_irqsave, &kbd_event_lock) { - asize = accent_table_size; - for (i = 0; i < asize; i++) { - dia[i].diacr = conv_uni_to_8bit(accent_table[i].diacr); - dia[i].base = conv_uni_to_8bit(accent_table[i].base); - dia[i].result = conv_uni_to_8bit(accent_table[i].result); - } + /* Lock the diacriticals table, make a copy and then + copy it after we unlock */ + scoped_guard(spinlock_irqsave, &kbd_event_lock) { + asize = accent_table_size; + for (i = 0; i < asize; i++) { + dia[i].diacr = conv_uni_to_8bit(accent_table[i].diacr); + dia[i].base = conv_uni_to_8bit(accent_table[i].base); + dia[i].result = conv_uni_to_8bit(accent_table[i].result); } - - if (put_user(asize, &a->kb_cnt)) - return -EFAULT; - if (copy_to_user(a->kbdiacr, dia, asize * sizeof(struct kbdiacr))) - return -EFAULT; - return 0; } - case KDGKBDIACRUC: - { - struct kbdiacrsuc __user *a = udp; - void __free(kfree) *buf = kmalloc_array(MAX_DIACR, sizeof(struct kbdiacruc), - GFP_KERNEL); - if (buf == NULL) - return -ENOMEM; + if (put_user(asize, &a->kb_cnt)) + return -EFAULT; + if (copy_to_user(a->kbdiacr, dia, asize * sizeof(struct kbdiacr))) + return -EFAULT; + return 0; +} - /* Lock the diacriticals table, make a copy and then - copy it after we unlock */ - scoped_guard(spinlock_irqsave, &kbd_event_lock) { - asize = accent_table_size; - memcpy(buf, accent_table, asize * sizeof(struct kbdiacruc)); - } +static int vt_do_kdgkbdiacruc(void __user *udp) +{ + struct kbdiacrsuc __user *a = udp; + int asize; - if (put_user(asize, &a->kb_cnt)) - return -EFAULT; - if (copy_to_user(a->kbdiacruc, buf, asize * sizeof(struct kbdiacruc))) - return -EFAULT; + void __free(kfree) *buf = kmalloc_array(MAX_DIACR, sizeof(struct kbdiacruc), + GFP_KERNEL); + if (buf == NULL) + return -ENOMEM; - return 0; + /* Lock the diacriticals table, make a copy and then + copy it after we unlock */ + scoped_guard(spinlock_irqsave, &kbd_event_lock) { + asize = accent_table_size; + memcpy(buf, accent_table, asize * sizeof(struct kbdiacruc)); } - case KDSKBDIACR: - { - struct kbdiacrs __user *a = udp; - struct kbdiacr __free(kfree) *dia = NULL; - unsigned int ct; - int i; + if (put_user(asize, &a->kb_cnt)) + return -EFAULT; + if (copy_to_user(a->kbdiacruc, buf, asize * sizeof(struct kbdiacruc))) + return -EFAULT; - if (!perm) - return -EPERM; - if (get_user(ct, &a->kb_cnt)) - return -EFAULT; - if (ct >= MAX_DIACR) - return -EINVAL; + return 0; +} - if (ct) { - dia = memdup_array_user(a->kbdiacr, - ct, sizeof(struct kbdiacr)); - if (IS_ERR(dia)) - return PTR_ERR(dia); - } +static int vt_do_kdskbdiacr(void __user *udp, int perm) +{ + struct kbdiacrs __user *a = udp; + struct kbdiacr __free(kfree) *dia = NULL; + unsigned int ct; + int i; - guard(spinlock_irqsave)(&kbd_event_lock); - accent_table_size = ct; - for (i = 0; i < ct; i++) { - accent_table[i].diacr = - conv_8bit_to_uni(dia[i].diacr); - accent_table[i].base = - conv_8bit_to_uni(dia[i].base); - accent_table[i].result = - conv_8bit_to_uni(dia[i].result); - } + if (!perm) + return -EPERM; + if (get_user(ct, &a->kb_cnt)) + return -EFAULT; + if (ct >= MAX_DIACR) + return -EINVAL; - return 0; + if (ct) { + dia = memdup_array_user(a->kbdiacr, + ct, sizeof(struct kbdiacr)); + if (IS_ERR(dia)) + return PTR_ERR(dia); } - case KDSKBDIACRUC: - { - struct kbdiacrsuc __user *a = udp; - unsigned int ct; - void __free(kfree) *buf = NULL; + guard(spinlock_irqsave)(&kbd_event_lock); + accent_table_size = ct; + for (i = 0; i < ct; i++) { + accent_table[i].diacr = + conv_8bit_to_uni(dia[i].diacr); + accent_table[i].base = + conv_8bit_to_uni(dia[i].base); + accent_table[i].result = + conv_8bit_to_uni(dia[i].result); + } - if (!perm) - return -EPERM; + return 0; +} - if (get_user(ct, &a->kb_cnt)) - return -EFAULT; +static int vt_do_kdskbdiacruc(void __user *udp, int perm) +{ + struct kbdiacrsuc __user *a = udp; + unsigned int ct; + void __free(kfree) *buf = NULL; - if (ct >= MAX_DIACR) - return -EINVAL; + if (!perm) + return -EPERM; - if (ct) { - buf = memdup_array_user(a->kbdiacruc, - ct, sizeof(struct kbdiacruc)); - if (IS_ERR(buf)) - return PTR_ERR(buf); - } - guard(spinlock_irqsave)(&kbd_event_lock); - if (ct) - memcpy(accent_table, buf, - ct * sizeof(struct kbdiacruc)); - accent_table_size = ct; - return 0; + if (get_user(ct, &a->kb_cnt)) + return -EFAULT; + + if (ct >= MAX_DIACR) + return -EINVAL; + + if (ct) { + buf = memdup_array_user(a->kbdiacruc, + ct, sizeof(struct kbdiacruc)); + if (IS_ERR(buf)) + return PTR_ERR(buf); } + guard(spinlock_irqsave)(&kbd_event_lock); + if (ct) + memcpy(accent_table, buf, + ct * sizeof(struct kbdiacruc)); + accent_table_size = ct; + return 0; +} + +/** + * vt_do_diacrit - diacritical table updates + * @cmd: ioctl request + * @udp: pointer to user data for ioctl + * @perm: permissions check computed by caller + * + * Update the diacritical tables atomically and safely. Lock them + * against simultaneous keypresses + */ +int vt_do_diacrit(unsigned int cmd, void __user *udp, int perm) +{ + switch (cmd) { + case KDGKBDIACR: + return vt_do_kdgkbdiacr(udp); + case KDGKBDIACRUC: + return vt_do_kdgkbdiacruc(udp); + case KDSKBDIACR: + return vt_do_kdskbdiacr(udp, perm); + case KDSKBDIACRUC: + return vt_do_kdskbdiacruc(udp, perm); } return 0; } diff --git a/drivers/tty/vt/vt.c b/drivers/tty/vt/vt.c index 59b4b5e126ba1c..738fac2718e632 100644 --- a/drivers/tty/vt/vt.c +++ b/drivers/tty/vt/vt.c @@ -1345,6 +1345,8 @@ struct vc_data *vc_deallocate(unsigned int currcons) kfree(vc->vc_saved_screen); vc->vc_saved_screen = NULL; } + vc_uniscr_free(vc->vc_saved_uni_lines); + vc->vc_saved_uni_lines = NULL; } return vc; } @@ -1890,6 +1892,8 @@ static void enter_alt_screen(struct vc_data *vc) vc->vc_saved_screen = kmemdup((u16 *)vc->vc_origin, size, GFP_KERNEL); if (vc->vc_saved_screen == NULL) return; + vc->vc_saved_uni_lines = vc->vc_uni_lines; + vc->vc_uni_lines = NULL; vc->vc_saved_rows = vc->vc_rows; vc->vc_saved_cols = vc->vc_cols; save_cur(vc); @@ -1911,6 +1915,26 @@ static void leave_alt_screen(struct vc_data *vc) dest = ((u16 *)vc->vc_origin) + r * vc->vc_cols; memcpy(dest, src, 2 * cols); } + /* + * If the console was resized while in the alternate screen, + * resize the saved unicode buffer to the current dimensions. + * On allocation failure new_uniscr is NULL, causing the old + * buffer to be freed and vc_uni_lines to be lazily rebuilt + * via vc_uniscr_check() when next needed. + */ + if (vc->vc_saved_uni_lines && + (vc->vc_saved_rows != vc->vc_rows || + vc->vc_saved_cols != vc->vc_cols)) { + u32 **new_uniscr = vc_uniscr_alloc(vc->vc_cols, vc->vc_rows); + + if (new_uniscr) + vc_uniscr_copy_area(new_uniscr, vc->vc_cols, vc->vc_rows, + vc->vc_saved_uni_lines, cols, 0, rows); + vc_uniscr_free(vc->vc_saved_uni_lines); + vc->vc_saved_uni_lines = new_uniscr; + } + vc_uniscr_set(vc, vc->vc_saved_uni_lines); + vc->vc_saved_uni_lines = NULL; restore_cur(vc); /* Update the entire screen */ if (con_should_update(vc)) @@ -2233,6 +2257,8 @@ static void reset_terminal(struct vc_data *vc, int do_clear) if (vc->vc_saved_screen != NULL) { kfree(vc->vc_saved_screen); vc->vc_saved_screen = NULL; + vc_uniscr_free(vc->vc_saved_uni_lines); + vc->vc_saved_uni_lines = NULL; vc->vc_saved_rows = 0; vc->vc_saved_cols = 0; } diff --git a/drivers/ufs/core/ufshcd.c b/drivers/ufs/core/ufshcd.c index 604043a7533d37..5038b8428fc307 100644 --- a/drivers/ufs/core/ufshcd.c +++ b/drivers/ufs/core/ufshcd.c @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -515,8 +516,8 @@ static void ufshcd_add_command_trace(struct ufs_hba *hba, struct scsi_cmnd *cmd, if (hba->mcq_enabled) { struct ufs_hw_queue *hwq = ufshcd_mcq_req_to_hwq(hba, rq); - - hwq_id = hwq->id; + if (hwq) + hwq_id = hwq->id; } else { doorbell = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL); } @@ -4385,14 +4386,6 @@ static int ufshcd_uic_pwr_ctrl(struct ufs_hba *hba, struct uic_command *cmd) spin_unlock_irqrestore(hba->host->host_lock, flags); mutex_unlock(&hba->uic_cmd_mutex); - /* - * If the h8 exit fails during the runtime resume process, it becomes - * stuck and cannot be recovered through the error handler. To fix - * this, use link recovery instead of the error handler. - */ - if (ret && hba->pm_op_in_progress) - ret = ufshcd_link_recovery(hba); - return ret; } @@ -5245,6 +5238,25 @@ static void ufshcd_lu_init(struct ufs_hba *hba, struct scsi_device *sdev) hba->dev_info.rpmb_region_size[1] = desc_buf[RPMB_UNIT_DESC_PARAM_REGION1_SIZE]; hba->dev_info.rpmb_region_size[2] = desc_buf[RPMB_UNIT_DESC_PARAM_REGION2_SIZE]; hba->dev_info.rpmb_region_size[3] = desc_buf[RPMB_UNIT_DESC_PARAM_REGION3_SIZE]; + + if (hba->dev_info.wspecversion <= 0x0220) { + /* + * These older spec chips have only one RPMB region, + * sized between 128 kB minimum and 16 MB maximum. + * No per region size fields are provided (respective + * REGIONX_SIZE fields always contain zeros), so get + * it from the logical block count and size fields for + * compatibility + * + * (See JESD220C-2_2 Section 14.1.4.6 + * RPMB Unit Descriptor,* offset 13h, 4 bytes) + */ + hba->dev_info.rpmb_region_size[0] = + (get_unaligned_be64(desc_buf + + RPMB_UNIT_DESC_PARAM_LOGICAL_BLK_COUNT) + << desc_buf[RPMB_UNIT_DESC_PARAM_LOGICAL_BLK_SIZE]) + / SZ_128K; + } } @@ -5959,6 +5971,7 @@ static int ufshcd_disable_auto_bkops(struct ufs_hba *hba) hba->auto_bkops_enabled = false; trace_ufshcd_auto_bkops_state(hba, "Disabled"); + hba->urgent_bkops_lvl = BKOPS_STATUS_PERF_IMPACT; hba->is_urgent_bkops_lvl_checked = false; out: return err; @@ -6062,7 +6075,7 @@ static void ufshcd_bkops_exception_event_handler(struct ufs_hba *hba) * impacted or critical. Handle these device by determining their urgent * bkops status at runtime. */ - if (curr_status < BKOPS_STATUS_PERF_IMPACT) { + if ((curr_status > BKOPS_STATUS_NO_OP) && (curr_status < BKOPS_STATUS_PERF_IMPACT)) { dev_err(hba->dev, "%s: device raised urgent BKOPS exception for bkops status %d\n", __func__, curr_status); /* update the current status as the urgent bkops level */ @@ -7093,7 +7106,7 @@ static irqreturn_t ufshcd_handle_mcq_cq_events(struct ufs_hba *hba) ret = ufshcd_vops_get_outstanding_cqs(hba, &outstanding_cqs); if (ret) - outstanding_cqs = (1U << hba->nr_hw_queues) - 1; + outstanding_cqs = (1ULL << hba->nr_hw_queues) - 1; /* Exclude the poll queues */ nr_queues = hba->nr_hw_queues - hba->nr_queues[HCTX_TYPE_POLL]; @@ -9994,6 +10007,8 @@ static int __ufshcd_wl_suspend(struct ufs_hba *hba, enum ufs_pm_op pm_op) if (req_dev_pwr_mode == UFS_ACTIVE_PWR_MODE && req_link_state == UIC_LINK_ACTIVE_STATE) { + ufshcd_disable_auto_bkops(hba); + flush_work(&hba->eeh_work); goto vops_suspend; } @@ -10046,6 +10061,7 @@ static int __ufshcd_wl_suspend(struct ufs_hba *hba, enum ufs_pm_op pm_op) } flush_work(&hba->eeh_work); + cancel_delayed_work_sync(&hba->ufs_rtc_update_work); ret = ufshcd_vops_suspend(hba, pm_op, PRE_CHANGE); if (ret) @@ -10100,7 +10116,6 @@ static int __ufshcd_wl_suspend(struct ufs_hba *hba, enum ufs_pm_op pm_op) if (ret) goto set_link_active; - cancel_delayed_work_sync(&hba->ufs_rtc_update_work); goto out; set_link_active: @@ -10172,7 +10187,15 @@ static int __ufshcd_wl_resume(struct ufs_hba *hba, enum ufs_pm_op pm_op) } else { dev_err(hba->dev, "%s: hibern8 exit failed %d\n", __func__, ret); - goto vendor_suspend; + /* + * If the h8 exit fails during the runtime resume + * process, it becomes stuck and cannot be recovered + * through the error handler. To fix this, use link + * recovery instead of the error handler. + */ + ret = ufshcd_link_recovery(hba); + if (ret) + goto vendor_suspend; } } else if (ufshcd_is_link_off(hba)) { /* diff --git a/drivers/ufs/host/Kconfig b/drivers/ufs/host/Kconfig index 7d5117b2dab4a7..964ae70e7390d0 100644 --- a/drivers/ufs/host/Kconfig +++ b/drivers/ufs/host/Kconfig @@ -72,6 +72,7 @@ config SCSI_UFS_QCOM config SCSI_UFS_MEDIATEK tristate "Mediatek specific hooks to UFS controller platform driver" depends on SCSI_UFSHCD_PLATFORM && ARCH_MEDIATEK + depends on PM depends on RESET_CONTROLLER select PHY_MTK_UFS select RESET_TI_SYSCON diff --git a/drivers/ufs/host/ufs-mediatek-trace.h b/drivers/ufs/host/ufs-mediatek-trace.h index b5f2ec3140748f..0df8ac843379a8 100644 --- a/drivers/ufs/host/ufs-mediatek-trace.h +++ b/drivers/ufs/host/ufs-mediatek-trace.h @@ -33,19 +33,19 @@ TRACE_EVENT(ufs_mtk_clk_scale, TP_ARGS(name, scale_up, clk_rate), TP_STRUCT__entry( - __field(const char*, name) + __string(name, name) __field(bool, scale_up) __field(unsigned long, clk_rate) ), TP_fast_assign( - __entry->name = name; + __assign_str(name); __entry->scale_up = scale_up; __entry->clk_rate = clk_rate; ), TP_printk("ufs: clk (%s) scaled %s @ %lu", - __entry->name, + __get_str(name), __entry->scale_up ? "up" : "down", __entry->clk_rate) ); diff --git a/drivers/ufs/host/ufs-mediatek.c b/drivers/ufs/host/ufs-mediatek.c index 66b11cc0703bd2..b3daaa07e92521 100644 --- a/drivers/ufs/host/ufs-mediatek.c +++ b/drivers/ufs/host/ufs-mediatek.c @@ -2437,7 +2437,6 @@ static void ufs_mtk_remove(struct platform_device *pdev) ufshcd_pltfrm_remove(pdev); } -#ifdef CONFIG_PM_SLEEP static int ufs_mtk_system_suspend(struct device *dev) { struct ufs_hba *hba = dev_get_drvdata(dev); @@ -2484,9 +2483,7 @@ static int ufs_mtk_system_resume(struct device *dev) return ret; } -#endif -#ifdef CONFIG_PM static int ufs_mtk_runtime_suspend(struct device *dev) { struct ufs_hba *hba = dev_get_drvdata(dev); @@ -2525,13 +2522,10 @@ static int ufs_mtk_runtime_resume(struct device *dev) return ufshcd_runtime_resume(dev); } -#endif static const struct dev_pm_ops ufs_mtk_pm_ops = { - SET_SYSTEM_SLEEP_PM_OPS(ufs_mtk_system_suspend, - ufs_mtk_system_resume) - SET_RUNTIME_PM_OPS(ufs_mtk_runtime_suspend, - ufs_mtk_runtime_resume, NULL) + SYSTEM_SLEEP_PM_OPS(ufs_mtk_system_suspend, ufs_mtk_system_resume) + RUNTIME_PM_OPS(ufs_mtk_runtime_suspend, ufs_mtk_runtime_resume, NULL) .prepare = ufshcd_suspend_prepare, .complete = ufshcd_resume_complete, }; @@ -2541,7 +2535,7 @@ static struct platform_driver ufs_mtk_pltform = { .remove = ufs_mtk_remove, .driver = { .name = "ufshcd-mtk", - .pm = &ufs_mtk_pm_ops, + .pm = pm_ptr(&ufs_mtk_pm_ops), .of_match_table = ufs_mtk_of_match, }, }; diff --git a/drivers/usb/cdns3/cdns3-gadget.c b/drivers/usb/cdns3/cdns3-gadget.c index 168707213ed9d3..610ad2adca8889 100644 --- a/drivers/usb/cdns3/cdns3-gadget.c +++ b/drivers/usb/cdns3/cdns3-gadget.c @@ -2589,6 +2589,9 @@ static int __cdns3_gadget_ep_queue(struct usb_ep *ep, struct cdns3_request *priv_req; int ret = 0; + if (!ep->desc) + return -ESHUTDOWN; + request->actual = 0; request->status = -EINPROGRESS; priv_req = to_cdns3_request(request); @@ -3428,6 +3431,7 @@ static int __cdns3_gadget_init(struct cdns *cdns) ret = cdns3_gadget_start(cdns); if (ret) { pm_runtime_put_sync(cdns->dev); + cdns_drd_gadget_off(cdns); return ret; } diff --git a/drivers/usb/cdns3/core.c b/drivers/usb/cdns3/core.c index 1243a5cea91b52..f0e32227c0b791 100644 --- a/drivers/usb/cdns3/core.c +++ b/drivers/usb/cdns3/core.c @@ -551,7 +551,7 @@ int cdns_resume(struct cdns *cdns) } } - if (cdns->roles[cdns->role]->resume) + if (!role_changed && cdns->roles[cdns->role]->resume) cdns->roles[cdns->role]->resume(cdns, power_lost); return 0; diff --git a/drivers/usb/chipidea/udc.c b/drivers/usb/chipidea/udc.c index 64a421ae0f05bf..c8d931d9d43301 100644 --- a/drivers/usb/chipidea/udc.c +++ b/drivers/usb/chipidea/udc.c @@ -931,6 +931,13 @@ __acquires(hwep->lock) list_del_init(&hwreq->queue); hwreq->req.status = -ESHUTDOWN; + /* Unmap DMA and clean up bounce buffers before giving back */ + usb_gadget_unmap_request_by_dev(hwep->ci->dev->parent, + &hwreq->req, hwep->dir); + + if (hwreq->sgt.sgl) + sglist_do_debounce(hwreq, false); + if (hwreq->req.complete != NULL) { spin_unlock(hwep->lock); usb_gadget_giveback_request(&hwep->ep, &hwreq->req); diff --git a/drivers/usb/class/cdc-acm.c b/drivers/usb/class/cdc-acm.c index 54be4aa1dcb2af..782be75fb71b95 100644 --- a/drivers/usb/class/cdc-acm.c +++ b/drivers/usb/class/cdc-acm.c @@ -114,6 +114,8 @@ static int acm_ctrl_msg(struct acm *acm, int request, int value, int retval; retval = usb_autopm_get_interface(acm->control); +#define VENDOR_CLASS_DATA_IFACE BIT(9) /* data interface uses vendor-specific class */ +#define ALWAYS_POLL_CTRL BIT(10) /* keep ctrl URB active even without an open TTY */ if (retval) return retval; @@ -710,12 +712,14 @@ static int acm_port_activate(struct tty_port *port, struct tty_struct *tty) set_bit(TTY_NO_WRITE_SPLIT, &tty->flags); acm->control->needs_remote_wakeup = 1; - acm->ctrlurb->dev = acm->dev; - retval = usb_submit_urb(acm->ctrlurb, GFP_KERNEL); - if (retval) { - dev_err(&acm->control->dev, - "%s - usb_submit_urb(ctrl irq) failed\n", __func__); - goto error_submit_urb; + if (!(acm->quirks & ALWAYS_POLL_CTRL)) { + acm->ctrlurb->dev = acm->dev; + retval = usb_submit_urb(acm->ctrlurb, GFP_KERNEL); + if (retval) { + dev_err(&acm->control->dev, + "%s - usb_submit_urb(ctrl irq) failed\n", __func__); + goto error_submit_urb; + } } acm_tty_set_termios(tty, NULL); @@ -788,6 +792,14 @@ static void acm_port_shutdown(struct tty_port *port) acm_unpoison_urbs(acm); + if (acm->quirks & ALWAYS_POLL_CTRL) { + acm->ctrlurb->dev = acm->dev; + if (usb_submit_urb(acm->ctrlurb, GFP_KERNEL)) + dev_dbg(&acm->control->dev, + "ctrl polling restart failed after port close\n"); + /* port_shutdown() cleared DTR/RTS; restore them */ + acm_set_control(acm, USB_CDC_CTRL_DTR | USB_CDC_CTRL_RTS); + } } static void acm_tty_cleanup(struct tty_struct *tty) @@ -1225,6 +1237,12 @@ static int acm_probe(struct usb_interface *intf, if (!data_interface || !control_interface) return -ENODEV; goto skip_normal_probe; + } else if (quirks == NO_UNION_12) { + data_interface = usb_ifnum_to_if(usb_dev, 2); + control_interface = usb_ifnum_to_if(usb_dev, 1); + if (!data_interface || !control_interface) + return -ENODEV; + goto skip_normal_probe; } /* normal probing*/ @@ -1322,6 +1340,9 @@ static int acm_probe(struct usb_interface *intf, dev_dbg(&intf->dev, "Your device has switched interfaces.\n"); swap(control_interface, data_interface); + } else if (quirks & VENDOR_CLASS_DATA_IFACE) { + dev_dbg(&intf->dev, + "Vendor-specific data interface class, continuing.\n"); } else { return -EINVAL; } @@ -1379,6 +1400,8 @@ static int acm_probe(struct usb_interface *intf, acm->ctrl_caps = h.usb_cdc_acm_descriptor->bmCapabilities; if (quirks & NO_CAP_LINE) acm->ctrl_caps &= ~USB_CDC_CAP_LINE; + if (quirks & MISSING_CAP_BRK) + acm->ctrl_caps |= USB_CDC_CAP_BRK; acm->ctrlsize = ctrlsize; acm->readsize = readsize; acm->rx_buflimit = num_rx_buf; @@ -1514,6 +1537,9 @@ static int acm_probe(struct usb_interface *intf, acm->line.bDataBits = 8; acm_set_line(acm, &acm->line); + if (quirks & ALWAYS_POLL_CTRL) + acm_set_control(acm, USB_CDC_CTRL_DTR | USB_CDC_CTRL_RTS); + if (!acm->combined_interfaces) { rv = usb_driver_claim_interface(&acm_driver, data_interface, acm); if (rv) @@ -1535,6 +1561,13 @@ static int acm_probe(struct usb_interface *intf, dev_info(&intf->dev, "ttyACM%d: USB ACM device\n", minor); + if (acm->quirks & ALWAYS_POLL_CTRL) { + acm->ctrlurb->dev = acm->dev; + if (usb_submit_urb(acm->ctrlurb, GFP_KERNEL)) + dev_warn(&intf->dev, + "failed to start persistent ctrl polling\n"); + } + return 0; err_release_data_interface: @@ -1661,7 +1694,7 @@ static int acm_resume(struct usb_interface *intf) acm_unpoison_urbs(acm); - if (tty_port_initialized(&acm->port)) { + if (tty_port_initialized(&acm->port) || (acm->quirks & ALWAYS_POLL_CTRL)) { rv = usb_submit_urb(acm->ctrlurb, GFP_ATOMIC); for (;;) { @@ -1746,6 +1779,9 @@ static const struct usb_device_id acm_ids[] = { { USB_DEVICE(0x045b, 0x024D), /* Renesas R-Car E3 USB Download mode */ .driver_info = DISABLE_ECHO, /* Don't echo banner */ }, + { USB_DEVICE(0x04b8, 0x0d12), /* EPSON HMD Com&Sens */ + .driver_info = NO_UNION_12, /* union descriptor is garbage */ + }, { USB_DEVICE(0x0e8d, 0x0003), /* FIREFLY, MediaTek Inc; andrey.arapov@gmail.com */ .driver_info = NO_UNION_NORMAL, /* has no union descriptor */ }, @@ -2002,6 +2038,23 @@ static const struct usb_device_id acm_ids[] = { .driver_info = IGNORE_DEVICE, }, + /* CH343 supports CAP_BRK, but doesn't advertise it */ + { USB_DEVICE(0x1a86, 0x55d3), .driver_info = MISSING_CAP_BRK, }, + + /* + * Lenovo Yoga Book 9 14IAH10 (83KJ) — INGENIC 17EF:6161 touchscreen + * composite device. The CDC ACM control interface (0) uses a standard + * Union descriptor, but the data interface (1) is declared as vendor- + * specific class (0xff) with no CDC data descriptors, so cdc-acm would + * normally reject it. The firmware also requires continuous polling of + * the notification endpoint (EP 0x82) to suppress a 20-second watchdog + * reset; ALWAYS_POLL_CTRL keeps the ctrlurb active even when no TTY is + * open. Match only the control interface by class to avoid probing the + * vendor-specific data interface. + */ + { USB_DEVICE_INTERFACE_CLASS(0x17ef, 0x6161, USB_CLASS_COMM), + .driver_info = VENDOR_CLASS_DATA_IFACE | ALWAYS_POLL_CTRL }, + /* control interfaces without any protocol set */ { USB_INTERFACE_INFO(USB_CLASS_COMM, USB_CDC_SUBCLASS_ACM, USB_CDC_PROTO_NONE) }, diff --git a/drivers/usb/class/cdc-acm.h b/drivers/usb/class/cdc-acm.h index 759ac15631d3e5..25fd5329a8781f 100644 --- a/drivers/usb/class/cdc-acm.h +++ b/drivers/usb/class/cdc-acm.h @@ -113,3 +113,5 @@ struct acm { #define CLEAR_HALT_CONDITIONS BIT(5) #define SEND_ZERO_PACKET BIT(6) #define DISABLE_ECHO BIT(7) +#define MISSING_CAP_BRK BIT(8) +#define NO_UNION_12 BIT(9) diff --git a/drivers/usb/class/cdc-wdm.c b/drivers/usb/class/cdc-wdm.c index ecd6d1f39e4984..92567324c5da77 100644 --- a/drivers/usb/class/cdc-wdm.c +++ b/drivers/usb/class/cdc-wdm.c @@ -225,7 +225,8 @@ static void wdm_in_callback(struct urb *urb) /* we may already be in overflow */ if (!test_bit(WDM_OVERFLOW, &desc->flags)) { memmove(desc->ubuf + desc->length, desc->inbuf, length); - desc->length += length; + smp_wmb(); /* against wdm_read() */ + WRITE_ONCE(desc->length, desc->length + length); } } skip_error: @@ -533,6 +534,7 @@ static ssize_t wdm_read return -ERESTARTSYS; cntr = READ_ONCE(desc->length); + smp_rmb(); /* against wdm_in_callback() */ if (cntr == 0) { desc->read = 0; retry: diff --git a/drivers/usb/class/usbtmc.c b/drivers/usb/class/usbtmc.c index 206f1b738ed3a0..2aa964dbb353e9 100644 --- a/drivers/usb/class/usbtmc.c +++ b/drivers/usb/class/usbtmc.c @@ -254,6 +254,9 @@ static int usbtmc_release(struct inode *inode, struct file *file) list_del(&file_data->file_elem); spin_unlock_irq(&file_data->data->dev_lock); + + /* flush anchored URBs */ + usbtmc_draw_down(file_data); mutex_unlock(&file_data->data->io_mutex); kref_put(&file_data->data->kref, usbtmc_delete); @@ -727,7 +730,7 @@ static int usbtmc488_ioctl_trigger(struct usbtmc_file_data *file_data) buffer[1] = data->bTag; buffer[2] = ~data->bTag; - retval = usb_bulk_msg(data->usb_dev, + retval = usb_bulk_msg_killable(data->usb_dev, usb_sndbulkpipe(data->usb_dev, data->bulk_out), buffer, USBTMC_HEADER_SIZE, @@ -1347,7 +1350,7 @@ static int send_request_dev_dep_msg_in(struct usbtmc_file_data *file_data, buffer[11] = 0; /* Reserved */ /* Send bulk URB */ - retval = usb_bulk_msg(data->usb_dev, + retval = usb_bulk_msg_killable(data->usb_dev, usb_sndbulkpipe(data->usb_dev, data->bulk_out), buffer, USBTMC_HEADER_SIZE, @@ -1419,7 +1422,7 @@ static ssize_t usbtmc_read(struct file *filp, char __user *buf, actual = 0; /* Send bulk URB */ - retval = usb_bulk_msg(data->usb_dev, + retval = usb_bulk_msg_killable(data->usb_dev, usb_rcvbulkpipe(data->usb_dev, data->bulk_in), buffer, bufsize, &actual, diff --git a/drivers/usb/common/ulpi.c b/drivers/usb/common/ulpi.c index 4a2ee447b21372..d895cf6532a21f 100644 --- a/drivers/usb/common/ulpi.c +++ b/drivers/usb/common/ulpi.c @@ -331,10 +331,9 @@ struct ulpi *ulpi_register_interface(struct device *dev, ulpi->ops = ops; ret = ulpi_register(dev, ulpi); - if (ret) { - kfree(ulpi); + if (ret) return ERR_PTR(ret); - } + return ulpi; } diff --git a/drivers/usb/core/config.c b/drivers/usb/core/config.c index 2bb1ceb9d621a3..3067e18ec4d8aa 100644 --- a/drivers/usb/core/config.c +++ b/drivers/usb/core/config.c @@ -927,7 +927,11 @@ int usb_get_configuration(struct usb_device *dev) dev->descriptor.bNumConfigurations = ncfg = USB_MAXCONFIG; } - if (ncfg < 1) { + if (ncfg < 1 && dev->quirks & USB_QUIRK_FORCE_ONE_CONFIG) { + dev_info(ddev, "Device claims zero configurations, forcing to 1\n"); + dev->descriptor.bNumConfigurations = 1; + ncfg = 1; + } else if (ncfg < 1) { dev_err(ddev, "no configurations\n"); return -EINVAL; } diff --git a/drivers/usb/core/driver.c b/drivers/usb/core/driver.c index d29edc7c616a28..74b8bdc27dbf54 100644 --- a/drivers/usb/core/driver.c +++ b/drivers/usb/core/driver.c @@ -1415,14 +1415,16 @@ static int usb_suspend_both(struct usb_device *udev, pm_message_t msg) int status = 0; int i = 0, n = 0; struct usb_interface *intf; + bool offload_active = false; if (udev->state == USB_STATE_NOTATTACHED || udev->state == USB_STATE_SUSPENDED) goto done; + usb_offload_set_pm_locked(udev, true); if (msg.event == PM_EVENT_SUSPEND && usb_offload_check(udev)) { dev_dbg(&udev->dev, "device offloaded, skip suspend.\n"); - udev->offload_at_suspend = 1; + offload_active = true; } /* Suspend all the interfaces and then udev itself */ @@ -1436,8 +1438,7 @@ static int usb_suspend_both(struct usb_device *udev, pm_message_t msg) * interrupt urbs, allowing interrupt events to be * handled during system suspend. */ - if (udev->offload_at_suspend && - intf->needs_remote_wakeup) { + if (offload_active && intf->needs_remote_wakeup) { dev_dbg(&intf->dev, "device offloaded, skip suspend.\n"); continue; @@ -1452,7 +1453,7 @@ static int usb_suspend_both(struct usb_device *udev, pm_message_t msg) } } if (status == 0) { - if (!udev->offload_at_suspend) + if (!offload_active) status = usb_suspend_device(udev, msg); /* @@ -1498,7 +1499,7 @@ static int usb_suspend_both(struct usb_device *udev, pm_message_t msg) */ } else { udev->can_submit = 0; - if (!udev->offload_at_suspend) { + if (!offload_active) { for (i = 0; i < 16; ++i) { usb_hcd_flush_endpoint(udev, udev->ep_out[i]); usb_hcd_flush_endpoint(udev, udev->ep_in[i]); @@ -1507,6 +1508,8 @@ static int usb_suspend_both(struct usb_device *udev, pm_message_t msg) } done: + if (status != 0) + usb_offload_set_pm_locked(udev, false); dev_vdbg(&udev->dev, "%s: status %d\n", __func__, status); return status; } @@ -1536,16 +1539,19 @@ static int usb_resume_both(struct usb_device *udev, pm_message_t msg) int status = 0; int i; struct usb_interface *intf; + bool offload_active = false; if (udev->state == USB_STATE_NOTATTACHED) { status = -ENODEV; goto done; } udev->can_submit = 1; + if (msg.event == PM_EVENT_RESUME) + offload_active = usb_offload_check(udev); /* Resume the device */ if (udev->state == USB_STATE_SUSPENDED || udev->reset_resume) { - if (!udev->offload_at_suspend) + if (!offload_active) status = usb_resume_device(udev, msg); else dev_dbg(&udev->dev, @@ -1562,8 +1568,7 @@ static int usb_resume_both(struct usb_device *udev, pm_message_t msg) * pending interrupt urbs, allowing interrupt events * to be handled during system suspend. */ - if (udev->offload_at_suspend && - intf->needs_remote_wakeup) { + if (offload_active && intf->needs_remote_wakeup) { dev_dbg(&intf->dev, "device offloaded, skip resume.\n"); continue; @@ -1572,11 +1577,11 @@ static int usb_resume_both(struct usb_device *udev, pm_message_t msg) udev->reset_resume); } } - udev->offload_at_suspend = 0; usb_mark_last_busy(udev); done: dev_vdbg(&udev->dev, "%s: status %d\n", __func__, status); + usb_offload_set_pm_locked(udev, false); if (!status) udev->reset_resume = 0; return status; diff --git a/drivers/usb/core/message.c b/drivers/usb/core/message.c index 6138468c67c472..43522f1d6b2bae 100644 --- a/drivers/usb/core/message.c +++ b/drivers/usb/core/message.c @@ -42,16 +42,19 @@ static void usb_api_blocking_completion(struct urb *urb) /* - * Starts urb and waits for completion or timeout. Note that this call - * is NOT interruptible. Many device driver i/o requests should be - * interruptible and therefore these drivers should implement their - * own interruptible routines. + * Starts urb and waits for completion or timeout. + * Whether or not the wait is killable depends on the flag passed in. + * For example, compare usb_bulk_msg() and usb_bulk_msg_killable(). + * + * For non-killable waits, we enforce a maximum limit on the timeout value. */ -static int usb_start_wait_urb(struct urb *urb, int timeout, int *actual_length) +static int usb_start_wait_urb(struct urb *urb, int timeout, int *actual_length, + bool killable) { struct api_context ctx; unsigned long expire; int retval; + long rc; init_completion(&ctx.done); urb->context = &ctx; @@ -60,13 +63,24 @@ static int usb_start_wait_urb(struct urb *urb, int timeout, int *actual_length) if (unlikely(retval)) goto out; - expire = timeout ? msecs_to_jiffies(timeout) : MAX_SCHEDULE_TIMEOUT; - if (!wait_for_completion_timeout(&ctx.done, expire)) { + if (!killable && (timeout <= 0 || timeout > USB_MAX_SYNCHRONOUS_TIMEOUT)) + timeout = USB_MAX_SYNCHRONOUS_TIMEOUT; + expire = (timeout > 0) ? msecs_to_jiffies(timeout) : MAX_SCHEDULE_TIMEOUT; + if (killable) + rc = wait_for_completion_killable_timeout(&ctx.done, expire); + else + rc = wait_for_completion_timeout(&ctx.done, expire); + if (rc <= 0) { usb_kill_urb(urb); - retval = (ctx.status == -ENOENT ? -ETIMEDOUT : ctx.status); + if (ctx.status != -ENOENT) + retval = ctx.status; + else if (rc == 0) + retval = -ETIMEDOUT; + else + retval = rc; dev_dbg(&urb->dev->dev, - "%s timed out on ep%d%s len=%u/%u\n", + "%s timed out or killed on ep%d%s len=%u/%u\n", current->comm, usb_endpoint_num(&urb->ep->desc), usb_urb_dir_in(urb) ? "in" : "out", @@ -100,7 +114,7 @@ static int usb_internal_control_msg(struct usb_device *usb_dev, usb_fill_control_urb(urb, usb_dev, pipe, (unsigned char *)cmd, data, len, usb_api_blocking_completion, NULL); - retv = usb_start_wait_urb(urb, timeout, &length); + retv = usb_start_wait_urb(urb, timeout, &length, false); if (retv < 0) return retv; else @@ -117,8 +131,7 @@ static int usb_internal_control_msg(struct usb_device *usb_dev, * @index: USB message index value * @data: pointer to the data to send * @size: length in bytes of the data to send - * @timeout: time in msecs to wait for the message to complete before timing - * out (if 0 the wait is forever) + * @timeout: time in msecs to wait for the message to complete before timing out * * Context: task context, might sleep. * @@ -173,8 +186,7 @@ EXPORT_SYMBOL_GPL(usb_control_msg); * @index: USB message index value * @driver_data: pointer to the data to send * @size: length in bytes of the data to send - * @timeout: time in msecs to wait for the message to complete before timing - * out (if 0 the wait is forever) + * @timeout: time in msecs to wait for the message to complete before timing out * @memflags: the flags for memory allocation for buffers * * Context: !in_interrupt () @@ -232,8 +244,7 @@ EXPORT_SYMBOL_GPL(usb_control_msg_send); * @index: USB message index value * @driver_data: pointer to the data to be filled in by the message * @size: length in bytes of the data to be received - * @timeout: time in msecs to wait for the message to complete before timing - * out (if 0 the wait is forever) + * @timeout: time in msecs to wait for the message to complete before timing out * @memflags: the flags for memory allocation for buffers * * Context: !in_interrupt () @@ -304,8 +315,7 @@ EXPORT_SYMBOL_GPL(usb_control_msg_recv); * @len: length in bytes of the data to send * @actual_length: pointer to a location to put the actual length transferred * in bytes - * @timeout: time in msecs to wait for the message to complete before - * timing out (if 0 the wait is forever) + * @timeout: time in msecs to wait for the message to complete before timing out * * Context: task context, might sleep. * @@ -337,8 +347,7 @@ EXPORT_SYMBOL_GPL(usb_interrupt_msg); * @len: length in bytes of the data to send * @actual_length: pointer to a location to put the actual length transferred * in bytes - * @timeout: time in msecs to wait for the message to complete before - * timing out (if 0 the wait is forever) + * @timeout: time in msecs to wait for the message to complete before timing out * * Context: task context, might sleep. * @@ -385,10 +394,59 @@ int usb_bulk_msg(struct usb_device *usb_dev, unsigned int pipe, usb_fill_bulk_urb(urb, usb_dev, pipe, data, len, usb_api_blocking_completion, NULL); - return usb_start_wait_urb(urb, timeout, actual_length); + return usb_start_wait_urb(urb, timeout, actual_length, false); } EXPORT_SYMBOL_GPL(usb_bulk_msg); +/** + * usb_bulk_msg_killable - Builds a bulk urb, sends it off and waits for completion in a killable state + * @usb_dev: pointer to the usb device to send the message to + * @pipe: endpoint "pipe" to send the message to + * @data: pointer to the data to send + * @len: length in bytes of the data to send + * @actual_length: pointer to a location to put the actual length transferred + * in bytes + * @timeout: time in msecs to wait for the message to complete before + * timing out (if <= 0, the wait is as long as possible) + * + * Context: task context, might sleep. + * + * This function is just like usb_blk_msg(), except that it waits in a + * killable state and there is no limit on the timeout length. + * + * Return: + * If successful, 0. Otherwise a negative error number. The number of actual + * bytes transferred will be stored in the @actual_length parameter. + * + */ +int usb_bulk_msg_killable(struct usb_device *usb_dev, unsigned int pipe, + void *data, int len, int *actual_length, int timeout) +{ + struct urb *urb; + struct usb_host_endpoint *ep; + + ep = usb_pipe_endpoint(usb_dev, pipe); + if (!ep || len < 0) + return -EINVAL; + + urb = usb_alloc_urb(0, GFP_KERNEL); + if (!urb) + return -ENOMEM; + + if ((ep->desc.bmAttributes & USB_ENDPOINT_XFERTYPE_MASK) == + USB_ENDPOINT_XFER_INT) { + pipe = (pipe & ~(3 << 30)) | (PIPE_INTERRUPT << 30); + usb_fill_int_urb(urb, usb_dev, pipe, data, len, + usb_api_blocking_completion, NULL, + ep->desc.bInterval); + } else + usb_fill_bulk_urb(urb, usb_dev, pipe, data, len, + usb_api_blocking_completion, NULL); + + return usb_start_wait_urb(urb, timeout, actual_length, true); +} +EXPORT_SYMBOL_GPL(usb_bulk_msg_killable); + /*-------------------------------------------------------------------*/ static void sg_clean(struct usb_sg_request *io) diff --git a/drivers/usb/core/offload.c b/drivers/usb/core/offload.c index 7c699f1b8d2b78..9db3cfedd29c7b 100644 --- a/drivers/usb/core/offload.c +++ b/drivers/usb/core/offload.c @@ -25,33 +25,30 @@ */ int usb_offload_get(struct usb_device *udev) { - int ret; + int ret = 0; - usb_lock_device(udev); - if (udev->state == USB_STATE_NOTATTACHED) { - usb_unlock_device(udev); + if (!usb_get_dev(udev)) return -ENODEV; - } - if (udev->state == USB_STATE_SUSPENDED || - udev->offload_at_suspend) { - usb_unlock_device(udev); - return -EBUSY; + if (pm_runtime_get_if_active(&udev->dev) != 1) { + ret = -EBUSY; + goto err_rpm; } - /* - * offload_usage could only be modified when the device is active, since - * it will alter the suspend flow of the device. - */ - ret = usb_autoresume_device(udev); - if (ret < 0) { - usb_unlock_device(udev); - return ret; + spin_lock(&udev->offload_lock); + + if (udev->offload_pm_locked) { + ret = -EAGAIN; + goto err; } udev->offload_usage++; - usb_autosuspend_device(udev); - usb_unlock_device(udev); + +err: + spin_unlock(&udev->offload_lock); + pm_runtime_put_autosuspend(&udev->dev); +err_rpm: + usb_put_dev(udev); return ret; } @@ -69,35 +66,32 @@ EXPORT_SYMBOL_GPL(usb_offload_get); */ int usb_offload_put(struct usb_device *udev) { - int ret; + int ret = 0; - usb_lock_device(udev); - if (udev->state == USB_STATE_NOTATTACHED) { - usb_unlock_device(udev); + if (!usb_get_dev(udev)) return -ENODEV; - } - if (udev->state == USB_STATE_SUSPENDED || - udev->offload_at_suspend) { - usb_unlock_device(udev); - return -EBUSY; + if (pm_runtime_get_if_active(&udev->dev) != 1) { + ret = -EBUSY; + goto err_rpm; } - /* - * offload_usage could only be modified when the device is active, since - * it will alter the suspend flow of the device. - */ - ret = usb_autoresume_device(udev); - if (ret < 0) { - usb_unlock_device(udev); - return ret; + spin_lock(&udev->offload_lock); + + if (udev->offload_pm_locked) { + ret = -EAGAIN; + goto err; } /* Drop the count when it wasn't 0, ignore the operation otherwise. */ if (udev->offload_usage) udev->offload_usage--; - usb_autosuspend_device(udev); - usb_unlock_device(udev); + +err: + spin_unlock(&udev->offload_lock); + pm_runtime_put_autosuspend(&udev->dev); +err_rpm: + usb_put_dev(udev); return ret; } @@ -112,25 +106,47 @@ EXPORT_SYMBOL_GPL(usb_offload_put); * management. * * The caller must hold @udev's device lock. In addition, the caller should - * ensure downstream usb devices are all either suspended or marked as - * "offload_at_suspend" to ensure the correctness of the return value. + * ensure the device itself and the downstream usb devices are all marked as + * "offload_pm_locked" to ensure the correctness of the return value. * * Returns true on any offload activity, false otherwise. */ bool usb_offload_check(struct usb_device *udev) __must_hold(&udev->dev->mutex) { struct usb_device *child; - bool active; + bool active = false; int port1; + if (udev->offload_usage) + return true; + usb_hub_for_each_child(udev, port1, child) { usb_lock_device(child); active = usb_offload_check(child); usb_unlock_device(child); + if (active) - return true; + break; } - return !!udev->offload_usage; + return active; } EXPORT_SYMBOL_GPL(usb_offload_check); + +/** + * usb_offload_set_pm_locked - set the PM lock state of a USB device + * @udev: the USB device to modify + * @locked: the new lock state + * + * Setting @locked to true prevents offload_usage from being modified. This + * ensures that offload activities cannot be started or stopped during critical + * power management transitions, maintaining a stable state for the duration + * of the transition. + */ +void usb_offload_set_pm_locked(struct usb_device *udev, bool locked) +{ + spin_lock(&udev->offload_lock); + udev->offload_pm_locked = locked; + spin_unlock(&udev->offload_lock); +} +EXPORT_SYMBOL_GPL(usb_offload_set_pm_locked); diff --git a/drivers/usb/core/phy.c b/drivers/usb/core/phy.c index faa20054ad5a1c..4d966cc9cdc951 100644 --- a/drivers/usb/core/phy.c +++ b/drivers/usb/core/phy.c @@ -114,7 +114,7 @@ EXPORT_SYMBOL_GPL(usb_phy_roothub_alloc); struct usb_phy_roothub *usb_phy_roothub_alloc_usb3_phy(struct device *dev) { struct usb_phy_roothub *phy_roothub; - int num_phys; + int num_phys, usb2_phy_index; if (!IS_ENABLED(CONFIG_GENERIC_PHY)) return NULL; @@ -124,6 +124,16 @@ struct usb_phy_roothub *usb_phy_roothub_alloc_usb3_phy(struct device *dev) if (num_phys <= 0) return NULL; + /* + * If 'usb2-phy' is not present, usb_phy_roothub_alloc() added + * all PHYs to the primary HCD's phy_roothub already, so skip + * adding 'usb3-phy' here to avoid double use of that. + */ + usb2_phy_index = of_property_match_string(dev->of_node, "phy-names", + "usb2-phy"); + if (usb2_phy_index < 0) + return NULL; + phy_roothub = devm_kzalloc(dev, sizeof(*phy_roothub), GFP_KERNEL); if (!phy_roothub) return ERR_PTR(-ENOMEM); @@ -200,16 +210,10 @@ int usb_phy_roothub_set_mode(struct usb_phy_roothub *phy_roothub, list_for_each_entry(roothub_entry, head, list) { err = phy_set_mode(roothub_entry->phy, mode); if (err) - goto err_out; + return err; } return 0; - -err_out: - list_for_each_entry_continue_reverse(roothub_entry, head, list) - phy_power_off(roothub_entry->phy); - - return err; } EXPORT_SYMBOL_GPL(usb_phy_roothub_set_mode); diff --git a/drivers/usb/core/port.c b/drivers/usb/core/port.c index f54198171b6a3f..a47df5d32f7c4e 100644 --- a/drivers/usb/core/port.c +++ b/drivers/usb/core/port.c @@ -141,6 +141,7 @@ static ssize_t disable_store(struct device *dev, struct device_attribute *attr, usb_disconnect(&port_dev->child); rc = usb_hub_set_port_power(hdev, hub, port1, !disabled); + msleep(2 * hub_power_on_good_delay(hub)); if (disabled) { usb_clear_port_feature(hdev, port1, USB_PORT_FEAT_C_CONNECTION); diff --git a/drivers/usb/core/quirks.c b/drivers/usb/core/quirks.c index c4d85089d19b13..34b1f7df3529a1 100644 --- a/drivers/usb/core/quirks.c +++ b/drivers/usb/core/quirks.c @@ -141,6 +141,8 @@ static int quirks_param_set(const char *value, const struct kernel_param *kp) case 'p': flags |= USB_QUIRK_SHORT_SET_ADDRESS_REQ_TIMEOUT; break; + case 'q': + flags |= USB_QUIRK_FORCE_ONE_CONFIG; /* Ignore unrecognized flag characters */ } } @@ -208,6 +210,10 @@ static const struct usb_device_id usb_quirk_list[] = { /* HP v222w 16GB Mini USB Drive */ { USB_DEVICE(0x03f0, 0x3f40), .driver_info = USB_QUIRK_DELAY_INIT }, + /* Huawei 4G LTE module ME906S */ + { USB_DEVICE(0x03f0, 0xa31d), .driver_info = + USB_QUIRK_DISCONNECT_SUSPEND }, + /* Creative SB Audigy 2 NX */ { USB_DEVICE(0x041e, 0x3020), .driver_info = USB_QUIRK_RESET_RESUME }, @@ -377,6 +383,9 @@ static const struct usb_device_id usb_quirk_list[] = { /* SanDisk Extreme 55AE */ { USB_DEVICE(0x0781, 0x55ae), .driver_info = USB_QUIRK_NO_LPM }, + /* Avermedia Live Gamer Ultra 2.1 (GC553G2) - BOS descriptor fetch hangs at SuperSpeed Plus */ + { USB_DEVICE(0x07ca, 0x2553), .driver_info = USB_QUIRK_NO_BOS }, + /* Realforce 87U Keyboard */ { USB_DEVICE(0x0853, 0x011b), .driver_info = USB_QUIRK_NO_LPM }, @@ -393,6 +402,7 @@ static const struct usb_device_id usb_quirk_list[] = { /* Silicon Motion Flash Drive */ { USB_DEVICE(0x090c, 0x1000), .driver_info = USB_QUIRK_DELAY_INIT }, + { USB_DEVICE(0x090c, 0x2000), .driver_info = USB_QUIRK_DELAY_INIT }, /* Sound Devices USBPre2 */ { USB_DEVICE(0x0926, 0x0202), .driver_info = @@ -437,6 +447,9 @@ static const struct usb_device_id usb_quirk_list[] = { { USB_DEVICE(0x0b05, 0x17e0), .driver_info = USB_QUIRK_IGNORE_REMOTE_WAKEUP }, + /* ASUS TUF 4K PRO - BOS descriptor fetch hangs at SuperSpeed Plus */ + { USB_DEVICE(0x0b05, 0x1ab9), .driver_info = USB_QUIRK_NO_BOS }, + /* Realtek Semiconductor Corp. Mass Storage Device (Multicard Reader)*/ { USB_DEVICE(0x0bda, 0x0151), .driver_info = USB_QUIRK_CONFIG_INTF_STRINGS }, @@ -481,6 +494,8 @@ static const struct usb_device_id usb_quirk_list[] = { /* Razer - Razer Blade Keyboard */ { USB_DEVICE(0x1532, 0x0116), .driver_info = USB_QUIRK_LINEAR_UFRAME_INTR_BINTERVAL }, + /* Razer - Razer Kiyo Pro Webcam */ + { USB_DEVICE(0x1532, 0x0e05), .driver_info = USB_QUIRK_NO_LPM }, /* Lenovo ThinkPad OneLink+ Dock twin hub controllers (VIA Labs VL812) */ { USB_DEVICE(0x17ef, 0x1018), .driver_info = USB_QUIRK_RESET_RESUME }, @@ -565,6 +580,9 @@ static const struct usb_device_id usb_quirk_list[] = { { USB_DEVICE(0x2386, 0x350e), .driver_info = USB_QUIRK_NO_LPM }, + /* UGREEN 35871 - BOS descriptor fetch hangs at SuperSpeed Plus */ + { USB_DEVICE(0x2b89, 0x5871), .driver_info = USB_QUIRK_NO_BOS }, + /* APTIV AUTOMOTIVE HUB */ { USB_DEVICE(0x2c48, 0x0132), .driver_info = USB_QUIRK_SHORT_SET_ADDRESS_REQ_TIMEOUT }, @@ -575,12 +593,18 @@ static const struct usb_device_id usb_quirk_list[] = { /* Alcor Link AK9563 SC Reader used in 2022 Lenovo ThinkPads */ { USB_DEVICE(0x2ce3, 0x9563), .driver_info = USB_QUIRK_NO_LPM }, + /* ezcap401 - BOS descriptor fetch hangs at SuperSpeed Plus */ + { USB_DEVICE(0x32ed, 0x0401), .driver_info = USB_QUIRK_NO_BOS }, + /* DELL USB GEN2 */ { USB_DEVICE(0x413c, 0xb062), .driver_info = USB_QUIRK_NO_LPM | USB_QUIRK_RESET_RESUME }, /* VCOM device */ { USB_DEVICE(0x4296, 0x7570), .driver_info = USB_QUIRK_CONFIG_INTF_STRINGS }, + /* Noji-MCS SmartCard Reader */ + { USB_DEVICE(0x5131, 0x2007), .driver_info = USB_QUIRK_FORCE_ONE_CONFIG }, + /* INTEL VALUE SSD */ { USB_DEVICE(0x8086, 0xf1a5), .driver_info = USB_QUIRK_RESET_RESUME }, diff --git a/drivers/usb/core/usb.c b/drivers/usb/core/usb.c index e740f7852bcdeb..8f7ca084010f2f 100644 --- a/drivers/usb/core/usb.c +++ b/drivers/usb/core/usb.c @@ -671,6 +671,7 @@ struct usb_device *usb_alloc_dev(struct usb_device *parent, set_dev_node(&dev->dev, dev_to_node(bus->sysdev)); dev->state = USB_STATE_ATTACHED; dev->lpm_disable_count = 1; + spin_lock_init(&dev->offload_lock); dev->offload_usage = 0; atomic_set(&dev->urbnum, 0); diff --git a/drivers/usb/dwc2/core.c b/drivers/usb/dwc2/core.c index c3d24312db0fec..f375c5185bfe22 100644 --- a/drivers/usb/dwc2/core.c +++ b/drivers/usb/dwc2/core.c @@ -578,6 +578,7 @@ void dwc2_force_dr_mode(struct dwc2_hsotg *hsotg) { switch (hsotg->dr_mode) { case USB_DR_MODE_HOST: + dwc2_force_mode(hsotg, true); /* * NOTE: This is required for some rockchip soc based * platforms on their host-only dwc2. diff --git a/drivers/usb/dwc2/gadget.c b/drivers/usb/dwc2/gadget.c index 0637bfbc054e23..a6c2b8de390834 100644 --- a/drivers/usb/dwc2/gadget.c +++ b/drivers/usb/dwc2/gadget.c @@ -4607,7 +4607,9 @@ static int dwc2_hsotg_udc_stop(struct usb_gadget *gadget) /* Exit clock gating when driver is stopped. */ if (hsotg->params.power_down == DWC2_POWER_DOWN_PARAM_NONE && hsotg->bus_suspended && !hsotg->params.no_clock_gating) { + spin_lock_irqsave(&hsotg->lock, flags); dwc2_gadget_exit_clock_gating(hsotg, 0); + spin_unlock_irqrestore(&hsotg->lock, flags); } /* all endpoints should be shutdown */ diff --git a/drivers/usb/dwc3/core.c b/drivers/usb/dwc3/core.c index 93fd5fdf95cb1d..59801611dd756e 100644 --- a/drivers/usb/dwc3/core.c +++ b/drivers/usb/dwc3/core.c @@ -2155,6 +2155,20 @@ static int dwc3_get_num_ports(struct dwc3 *dwc) return 0; } +static void dwc3_vbus_draw_work(struct work_struct *work) +{ + struct dwc3 *dwc = container_of(work, struct dwc3, vbus_draw_work); + union power_supply_propval val = {0}; + int ret; + + val.intval = 1000 * (dwc->current_limit); + ret = power_supply_set_property(dwc->usb_psy, POWER_SUPPLY_PROP_INPUT_CURRENT_LIMIT, &val); + + if (ret < 0) + dev_dbg(dwc->dev, "Error (%d) setting vbus draw (%d mA)\n", + ret, dwc->current_limit); +} + static struct power_supply *dwc3_get_usb_power_supply(struct dwc3 *dwc) { struct power_supply *usb_psy; @@ -2169,6 +2183,7 @@ static struct power_supply *dwc3_get_usb_power_supply(struct dwc3 *dwc) if (!usb_psy) return ERR_PTR(-EPROBE_DEFER); + INIT_WORK(&dwc->vbus_draw_work, dwc3_vbus_draw_work); return usb_psy; } @@ -2395,8 +2410,10 @@ void dwc3_core_remove(struct dwc3 *dwc) dwc3_free_event_buffers(dwc); - if (dwc->usb_psy) + if (dwc->usb_psy) { + cancel_work_sync(&dwc->vbus_draw_work); power_supply_put(dwc->usb_psy); + } } EXPORT_SYMBOL_GPL(dwc3_core_remove); diff --git a/drivers/usb/dwc3/core.h b/drivers/usb/dwc3/core.h index 45757169b672fd..9cfc36d4bc259d 100644 --- a/drivers/usb/dwc3/core.h +++ b/drivers/usb/dwc3/core.h @@ -1060,6 +1060,8 @@ struct dwc3_glue_ops { * @role_switch_default_mode: default operation mode of controller while * usb role is USB_ROLE_NONE. * @usb_psy: pointer to power supply interface. + * @vbus_draw_work: Work to set the vbus drawing limit + * @current_limit: How much current to draw from vbus, in milliAmperes. * @usb2_phy: pointer to USB2 PHY * @usb3_phy: pointer to USB3 PHY * @usb2_generic_phy: pointer to array of USB2 PHYs @@ -1246,6 +1248,8 @@ struct dwc3 { enum usb_dr_mode role_switch_default_mode; struct power_supply *usb_psy; + struct work_struct vbus_draw_work; + unsigned int current_limit; u32 fladj; u32 ref_clk_per; diff --git a/drivers/usb/dwc3/dwc3-pci.c b/drivers/usb/dwc3/dwc3-pci.c index 6ecadc81bd6ba5..6c1cbb722ca858 100644 --- a/drivers/usb/dwc3/dwc3-pci.c +++ b/drivers/usb/dwc3/dwc3-pci.c @@ -56,6 +56,7 @@ #define PCI_DEVICE_ID_INTEL_CNPH 0xa36e #define PCI_DEVICE_ID_INTEL_CNPV 0xa3b0 #define PCI_DEVICE_ID_INTEL_RPL 0xa70e +#define PCI_DEVICE_ID_INTEL_NVLH 0xd37f #define PCI_DEVICE_ID_INTEL_PTLH 0xe332 #define PCI_DEVICE_ID_INTEL_PTLH_PCH 0xe37e #define PCI_DEVICE_ID_INTEL_PTLU 0xe432 @@ -447,6 +448,7 @@ static const struct pci_device_id dwc3_pci_id_table[] = { { PCI_DEVICE_DATA(INTEL, CNPH, &dwc3_pci_intel_swnode) }, { PCI_DEVICE_DATA(INTEL, CNPV, &dwc3_pci_intel_swnode) }, { PCI_DEVICE_DATA(INTEL, RPL, &dwc3_pci_intel_swnode) }, + { PCI_DEVICE_DATA(INTEL, NVLH, &dwc3_pci_intel_swnode) }, { PCI_DEVICE_DATA(INTEL, PTLH, &dwc3_pci_intel_swnode) }, { PCI_DEVICE_DATA(INTEL, PTLH_PCH, &dwc3_pci_intel_swnode) }, { PCI_DEVICE_DATA(INTEL, PTLU, &dwc3_pci_intel_swnode) }, diff --git a/drivers/usb/dwc3/gadget.c b/drivers/usb/dwc3/gadget.c index 8a35a6901db7d7..5732d414e6a643 100644 --- a/drivers/usb/dwc3/gadget.c +++ b/drivers/usb/dwc3/gadget.c @@ -3123,8 +3123,6 @@ static void dwc3_gadget_set_ssp_rate(struct usb_gadget *g, static int dwc3_gadget_vbus_draw(struct usb_gadget *g, unsigned int mA) { struct dwc3 *dwc = gadget_to_dwc(g); - union power_supply_propval val = {0}; - int ret; if (dwc->usb2_phy) return usb_phy_set_power(dwc->usb2_phy, mA); @@ -3132,10 +3130,10 @@ static int dwc3_gadget_vbus_draw(struct usb_gadget *g, unsigned int mA) if (!dwc->usb_psy) return -EOPNOTSUPP; - val.intval = 1000 * mA; - ret = power_supply_set_property(dwc->usb_psy, POWER_SUPPLY_PROP_INPUT_CURRENT_LIMIT, &val); + dwc->current_limit = mA; + schedule_work(&dwc->vbus_draw_work); - return ret; + return 0; } /** diff --git a/drivers/usb/gadget/function/f_ecm.c b/drivers/usb/gadget/function/f_ecm.c index 675d2bc538a457..e315f18b7f9f04 100644 --- a/drivers/usb/gadget/function/f_ecm.c +++ b/drivers/usb/gadget/function/f_ecm.c @@ -681,6 +681,7 @@ ecm_bind(struct usb_configuration *c, struct usb_function *f) struct usb_ep *ep; struct f_ecm_opts *ecm_opts; + struct net_device *net __free(detach_gadget) = NULL; struct usb_request *request __free(free_usb_request) = NULL; if (!can_support_ecm(cdev->gadget)) @@ -688,18 +689,18 @@ ecm_bind(struct usb_configuration *c, struct usb_function *f) ecm_opts = container_of(f->fi, struct f_ecm_opts, func_inst); - mutex_lock(&ecm_opts->lock); - - gether_set_gadget(ecm_opts->net, cdev->gadget); - - if (!ecm_opts->bound) { - status = gether_register_netdev(ecm_opts->net); - ecm_opts->bound = true; - } - - mutex_unlock(&ecm_opts->lock); - if (status) - return status; + scoped_guard(mutex, &ecm_opts->lock) + if (ecm_opts->bind_count == 0 && !ecm_opts->bound) { + if (!device_is_registered(&ecm_opts->net->dev)) { + gether_set_gadget(ecm_opts->net, cdev->gadget); + status = gether_register_netdev(ecm_opts->net); + } else + status = gether_attach_gadget(ecm_opts->net, cdev->gadget); + + if (status) + return status; + net = ecm_opts->net; + } ecm_string_defs[1].s = ecm->ethaddr; @@ -790,6 +791,9 @@ ecm_bind(struct usb_configuration *c, struct usb_function *f) ecm->notify_req = no_free_ptr(request); + ecm_opts->bind_count++; + retain_and_null_ptr(net); + DBG(cdev, "CDC Ethernet: IN/%s OUT/%s NOTIFY/%s\n", ecm->port.in_ep->name, ecm->port.out_ep->name, ecm->notify->name); @@ -836,7 +840,7 @@ static void ecm_free_inst(struct usb_function_instance *f) struct f_ecm_opts *opts; opts = container_of(f, struct f_ecm_opts, func_inst); - if (opts->bound) + if (device_is_registered(&opts->net->dev)) gether_cleanup(netdev_priv(opts->net)); else free_netdev(opts->net); @@ -906,9 +910,12 @@ static void ecm_free(struct usb_function *f) static void ecm_unbind(struct usb_configuration *c, struct usb_function *f) { struct f_ecm *ecm = func_to_ecm(f); + struct f_ecm_opts *ecm_opts; DBG(c->cdev, "ecm unbind\n"); + ecm_opts = container_of(f->fi, struct f_ecm_opts, func_inst); + usb_free_all_descriptors(f); if (atomic_read(&ecm->notify_count)) { @@ -918,6 +925,10 @@ static void ecm_unbind(struct usb_configuration *c, struct usb_function *f) kfree(ecm->notify_req->buf); usb_ep_free_request(ecm->notify, ecm->notify_req); + + ecm_opts->bind_count--; + if (ecm_opts->bind_count == 0 && !ecm_opts->bound) + gether_detach_gadget(ecm_opts->net); } static struct usb_function *ecm_alloc(struct usb_function_instance *fi) diff --git a/drivers/usb/gadget/function/f_eem.c b/drivers/usb/gadget/function/f_eem.c index edbbadad613815..36f4295e6d63d7 100644 --- a/drivers/usb/gadget/function/f_eem.c +++ b/drivers/usb/gadget/function/f_eem.c @@ -7,6 +7,7 @@ * Copyright (C) 2009 EF Johnson Technologies */ +#include #include #include #include @@ -251,24 +252,22 @@ static int eem_bind(struct usb_configuration *c, struct usb_function *f) struct usb_ep *ep; struct f_eem_opts *eem_opts; + struct net_device *net __free(detach_gadget) = NULL; eem_opts = container_of(f->fi, struct f_eem_opts, func_inst); - /* - * in drivers/usb/gadget/configfs.c:configfs_composite_bind() - * configurations are bound in sequence with list_for_each_entry, - * in each configuration its functions are bound in sequence - * with list_for_each_entry, so we assume no race condition - * with regard to eem_opts->bound access - */ - if (!eem_opts->bound) { - mutex_lock(&eem_opts->lock); - gether_set_gadget(eem_opts->net, cdev->gadget); - status = gether_register_netdev(eem_opts->net); - mutex_unlock(&eem_opts->lock); - if (status) - return status; - eem_opts->bound = true; - } + + scoped_guard(mutex, &eem_opts->lock) + if (eem_opts->bind_count == 0 && !eem_opts->bound) { + if (!device_is_registered(&eem_opts->net->dev)) { + gether_set_gadget(eem_opts->net, cdev->gadget); + status = gether_register_netdev(eem_opts->net); + } else + status = gether_attach_gadget(eem_opts->net, cdev->gadget); + + if (status) + return status; + net = eem_opts->net; + } us = usb_gstrings_attach(cdev, eem_strings, ARRAY_SIZE(eem_string_defs)); @@ -279,21 +278,19 @@ static int eem_bind(struct usb_configuration *c, struct usb_function *f) /* allocate instance-specific interface IDs */ status = usb_interface_id(c, f); if (status < 0) - goto fail; + return status; eem->ctrl_id = status; eem_intf.bInterfaceNumber = status; - status = -ENODEV; - /* allocate instance-specific endpoints */ ep = usb_ep_autoconfig(cdev->gadget, &eem_fs_in_desc); if (!ep) - goto fail; + return -ENODEV; eem->port.in_ep = ep; ep = usb_ep_autoconfig(cdev->gadget, &eem_fs_out_desc); if (!ep) - goto fail; + return -ENODEV; eem->port.out_ep = ep; /* support all relevant hardware speeds... we expect that when @@ -309,16 +306,14 @@ static int eem_bind(struct usb_configuration *c, struct usb_function *f) status = usb_assign_descriptors(f, eem_fs_function, eem_hs_function, eem_ss_function, eem_ss_function); if (status) - goto fail; + return status; + + eem_opts->bind_count++; + retain_and_null_ptr(net); DBG(cdev, "CDC Ethernet (EEM): IN/%s OUT/%s\n", eem->port.in_ep->name, eem->port.out_ep->name); return 0; - -fail: - ERROR(cdev, "%s: can't bind, err %d\n", f->name, status); - - return status; } static void eem_cmd_complete(struct usb_ep *ep, struct usb_request *req) @@ -597,7 +592,7 @@ static void eem_free_inst(struct usb_function_instance *f) struct f_eem_opts *opts; opts = container_of(f, struct f_eem_opts, func_inst); - if (opts->bound) + if (device_is_registered(&opts->net->dev)) gether_cleanup(netdev_priv(opts->net)); else free_netdev(opts->net); @@ -640,9 +635,17 @@ static void eem_free(struct usb_function *f) static void eem_unbind(struct usb_configuration *c, struct usb_function *f) { + struct f_eem_opts *opts; + DBG(c->cdev, "eem unbind\n"); + opts = container_of(f->fi, struct f_eem_opts, func_inst); + usb_free_all_descriptors(f); + + opts->bind_count--; + if (opts->bind_count == 0 && !opts->bound) + gether_detach_gadget(opts->net); } static struct usb_function *eem_alloc(struct usb_function_instance *fi) diff --git a/drivers/usb/gadget/function/f_fs.c b/drivers/usb/gadget/function/f_fs.c index fa467a40949d2e..e75d5d8b5ac917 100644 --- a/drivers/usb/gadget/function/f_fs.c +++ b/drivers/usb/gadget/function/f_fs.c @@ -1509,7 +1509,7 @@ static int ffs_dmabuf_attach(struct file *file, int fd) goto err_dmabuf_detach; } - dir = epfile->in ? DMA_FROM_DEVICE : DMA_TO_DEVICE; + dir = epfile->in ? DMA_TO_DEVICE : DMA_FROM_DEVICE; err = ffs_dma_resv_lock(dmabuf, nonblock); if (err) @@ -1639,7 +1639,7 @@ static int ffs_dmabuf_transfer(struct file *file, /* Make sure we don't have writers */ timeout = nonblock ? 0 : msecs_to_jiffies(DMABUF_ENQUEUE_TIMEOUT_MS); retl = dma_resv_wait_timeout(dmabuf->resv, - dma_resv_usage_rw(epfile->in), + dma_resv_usage_rw(!epfile->in), true, timeout); if (retl == 0) retl = -EBUSY; @@ -1684,7 +1684,7 @@ static int ffs_dmabuf_transfer(struct file *file, dma_fence_init(&fence->base, &ffs_dmabuf_fence_ops, &priv->lock, priv->context, seqno); - resv_dir = epfile->in ? DMA_RESV_USAGE_WRITE : DMA_RESV_USAGE_READ; + resv_dir = epfile->in ? DMA_RESV_USAGE_READ : DMA_RESV_USAGE_WRITE; dma_resv_add_fence(dmabuf->resv, &fence->base, resv_dir); dma_resv_unlock(dmabuf->resv); @@ -1744,10 +1744,8 @@ static long ffs_epfile_ioctl(struct file *file, unsigned code, { int fd; - if (copy_from_user(&fd, (void __user *)value, sizeof(fd))) { - ret = -EFAULT; - break; - } + if (copy_from_user(&fd, (void __user *)value, sizeof(fd))) + return -EFAULT; return ffs_dmabuf_attach(file, fd); } @@ -1755,10 +1753,8 @@ static long ffs_epfile_ioctl(struct file *file, unsigned code, { int fd; - if (copy_from_user(&fd, (void __user *)value, sizeof(fd))) { - ret = -EFAULT; - break; - } + if (copy_from_user(&fd, (void __user *)value, sizeof(fd))) + return -EFAULT; return ffs_dmabuf_detach(file, fd); } @@ -1766,10 +1762,8 @@ static long ffs_epfile_ioctl(struct file *file, unsigned code, { struct usb_ffs_dmabuf_transfer_req req; - if (copy_from_user(&req, (void __user *)value, sizeof(req))) { - ret = -EFAULT; - break; - } + if (copy_from_user(&req, (void __user *)value, sizeof(req))) + return -EFAULT; return ffs_dmabuf_transfer(file, &req); } diff --git a/drivers/usb/gadget/function/f_hid.c b/drivers/usb/gadget/function/f_hid.c index 3ddfd4f66f0b09..4fc82e51d6eae0 100644 --- a/drivers/usb/gadget/function/f_hid.c +++ b/drivers/usb/gadget/function/f_hid.c @@ -106,7 +106,7 @@ struct f_hidg { struct list_head report_list; struct device dev; - struct cdev cdev; + struct cdev *cdev; struct usb_function func; struct usb_ep *in_ep; @@ -749,8 +749,9 @@ static int f_hidg_release(struct inode *inode, struct file *fd) static int f_hidg_open(struct inode *inode, struct file *fd) { + struct kobject *parent = inode->i_cdev->kobj.parent; struct f_hidg *hidg = - container_of(inode->i_cdev, struct f_hidg, cdev); + container_of(parent, struct f_hidg, dev.kobj); fd->private_data = hidg; @@ -1207,9 +1208,11 @@ static int hidg_bind(struct usb_configuration *c, struct usb_function *f) if (!hidg->interval_user_set) { hidg_fs_in_ep_desc.bInterval = 10; hidg_hs_in_ep_desc.bInterval = 4; + hidg_ss_in_ep_desc.bInterval = 4; } else { hidg_fs_in_ep_desc.bInterval = hidg->interval; hidg_hs_in_ep_desc.bInterval = hidg->interval; + hidg_ss_in_ep_desc.bInterval = hidg->interval; } hidg_ss_out_comp_desc.wBytesPerInterval = @@ -1239,9 +1242,11 @@ static int hidg_bind(struct usb_configuration *c, struct usb_function *f) if (!hidg->interval_user_set) { hidg_fs_out_ep_desc.bInterval = 10; hidg_hs_out_ep_desc.bInterval = 4; + hidg_ss_out_ep_desc.bInterval = 4; } else { hidg_fs_out_ep_desc.bInterval = hidg->interval; hidg_hs_out_ep_desc.bInterval = hidg->interval; + hidg_ss_out_ep_desc.bInterval = hidg->interval; } status = usb_assign_descriptors(f, hidg_fs_descriptors_intout, @@ -1258,17 +1263,8 @@ static int hidg_bind(struct usb_configuration *c, struct usb_function *f) if (status) goto fail; - spin_lock_init(&hidg->write_spinlock); hidg->write_pending = 1; hidg->req = NULL; - spin_lock_init(&hidg->read_spinlock); - spin_lock_init(&hidg->get_report_spinlock); - init_waitqueue_head(&hidg->write_queue); - init_waitqueue_head(&hidg->read_queue); - init_waitqueue_head(&hidg->get_queue); - init_waitqueue_head(&hidg->get_id_queue); - INIT_LIST_HEAD(&hidg->completed_out_req); - INIT_LIST_HEAD(&hidg->report_list); INIT_WORK(&hidg->work, get_report_workqueue_handler); hidg->workqueue = alloc_workqueue("report_work", @@ -1281,8 +1277,12 @@ static int hidg_bind(struct usb_configuration *c, struct usb_function *f) } /* create char device */ - cdev_init(&hidg->cdev, &f_hidg_fops); - status = cdev_device_add(&hidg->cdev, &hidg->dev); + hidg->cdev = cdev_alloc(); + if (!hidg->cdev) + goto fail_free_all; + hidg->cdev->ops = &f_hidg_fops; + + status = cdev_device_add(hidg->cdev, &hidg->dev); if (status) goto fail_free_all; @@ -1584,7 +1584,7 @@ static void hidg_unbind(struct usb_configuration *c, struct usb_function *f) { struct f_hidg *hidg = func_to_hidg(f); - cdev_device_del(&hidg->cdev, &hidg->dev); + cdev_device_del(hidg->cdev, &hidg->dev); destroy_workqueue(hidg->workqueue); usb_free_all_descriptors(f); } @@ -1604,6 +1604,16 @@ static struct usb_function *hidg_alloc(struct usb_function_instance *fi) mutex_lock(&opts->lock); + spin_lock_init(&hidg->write_spinlock); + spin_lock_init(&hidg->read_spinlock); + spin_lock_init(&hidg->get_report_spinlock); + init_waitqueue_head(&hidg->write_queue); + init_waitqueue_head(&hidg->read_queue); + init_waitqueue_head(&hidg->get_queue); + init_waitqueue_head(&hidg->get_id_queue); + INIT_LIST_HEAD(&hidg->completed_out_req); + INIT_LIST_HEAD(&hidg->report_list); + device_initialize(&hidg->dev); hidg->dev.release = hidg_release; hidg->dev.class = &hidg_class; diff --git a/drivers/usb/gadget/function/f_mass_storage.c b/drivers/usb/gadget/function/f_mass_storage.c index 94d478b6bcd3d4..6f275c3d11ac5a 100644 --- a/drivers/usb/gadget/function/f_mass_storage.c +++ b/drivers/usb/gadget/function/f_mass_storage.c @@ -180,6 +180,7 @@ #include #include #include +#include #include #include #include @@ -1853,8 +1854,15 @@ static int check_command_size_in_blocks(struct fsg_common *common, int cmnd_size, enum data_direction data_dir, unsigned int mask, int needs_medium, const char *name) { - if (common->curlun) - common->data_size_from_cmnd <<= common->curlun->blkbits; + if (common->curlun) { + if (check_shl_overflow(common->data_size_from_cmnd, + common->curlun->blkbits, + &common->data_size_from_cmnd)) { + common->phase_error = 1; + return -EINVAL; + } + } + return check_command(common, cmnd_size, data_dir, mask, needs_medium, name); } diff --git a/drivers/usb/gadget/function/f_ncm.c b/drivers/usb/gadget/function/f_ncm.c index 0e38330271d5ac..04be7047aa3d2c 100644 --- a/drivers/usb/gadget/function/f_ncm.c +++ b/drivers/usb/gadget/function/f_ncm.c @@ -1210,8 +1210,8 @@ static int ncm_unwrap_ntb(struct gether *port, block_len = get_ncm(&tmp, opts->block_length); /* (d)wBlockLength */ - if (block_len > ntb_max) { - INFO(port->func.config->cdev, "OUT size exceeded\n"); + if ((block_len < opts->nth_size + opts->ndp_size) || (block_len > ntb_max)) { + INFO(port->func.config->cdev, "Bad block length: %#X\n", block_len); goto err; } @@ -1439,6 +1439,7 @@ static int ncm_bind(struct usb_configuration *c, struct usb_function *f) struct f_ncm_opts *ncm_opts; struct usb_os_desc_table *os_desc_table __free(kfree) = NULL; + struct net_device *net __free(detach_gadget) = NULL; struct usb_request *request __free(free_usb_request) = NULL; if (!can_support_ecm(cdev->gadget)) @@ -1452,18 +1453,19 @@ static int ncm_bind(struct usb_configuration *c, struct usb_function *f) return -ENOMEM; } - mutex_lock(&ncm_opts->lock); - gether_set_gadget(ncm_opts->net, cdev->gadget); - if (!ncm_opts->bound) { - ncm_opts->net->mtu = (ncm_opts->max_segment_size - ETH_HLEN); - status = gether_register_netdev(ncm_opts->net); - } - mutex_unlock(&ncm_opts->lock); - - if (status) - return status; - - ncm_opts->bound = true; + scoped_guard(mutex, &ncm_opts->lock) + if (ncm_opts->bind_count == 0) { + if (!device_is_registered(&ncm_opts->net->dev)) { + ncm_opts->net->mtu = (ncm_opts->max_segment_size - ETH_HLEN); + gether_set_gadget(ncm_opts->net, cdev->gadget); + status = gether_register_netdev(ncm_opts->net); + } else + status = gether_attach_gadget(ncm_opts->net, cdev->gadget); + + if (status) + return status; + net = ncm_opts->net; + } ncm_string_defs[1].s = ncm->ethaddr; @@ -1564,6 +1566,9 @@ static int ncm_bind(struct usb_configuration *c, struct usb_function *f) } ncm->notify_req = no_free_ptr(request); + ncm_opts->bind_count++; + retain_and_null_ptr(net); + DBG(cdev, "CDC Network: IN/%s OUT/%s NOTIFY/%s\n", ncm->port.in_ep->name, ncm->port.out_ep->name, ncm->notify->name); @@ -1655,7 +1660,7 @@ static void ncm_free_inst(struct usb_function_instance *f) struct f_ncm_opts *opts; opts = container_of(f, struct f_ncm_opts, func_inst); - if (opts->bound) + if (device_is_registered(&opts->net->dev)) gether_cleanup(netdev_priv(opts->net)); else free_netdev(opts->net); @@ -1718,9 +1723,12 @@ static void ncm_free(struct usb_function *f) static void ncm_unbind(struct usb_configuration *c, struct usb_function *f) { struct f_ncm *ncm = func_to_ncm(f); + struct f_ncm_opts *ncm_opts; DBG(c->cdev, "ncm unbind\n"); + ncm_opts = container_of(f->fi, struct f_ncm_opts, func_inst); + hrtimer_cancel(&ncm->task_timer); kfree(f->os_desc_table); @@ -1736,6 +1744,10 @@ static void ncm_unbind(struct usb_configuration *c, struct usb_function *f) kfree(ncm->notify_req->buf); usb_ep_free_request(ncm->notify, ncm->notify_req); + + ncm_opts->bind_count--; + if (ncm_opts->bind_count == 0) + gether_detach_gadget(ncm_opts->net); } static struct usb_function *ncm_alloc(struct usb_function_instance *fi) diff --git a/drivers/usb/gadget/function/f_phonet.c b/drivers/usb/gadget/function/f_phonet.c index 0aa9e8224cae57..a3e11c2011a879 100644 --- a/drivers/usb/gadget/function/f_phonet.c +++ b/drivers/usb/gadget/function/f_phonet.c @@ -333,6 +333,15 @@ static void pn_rx_complete(struct usb_ep *ep, struct usb_request *req) if (unlikely(!skb)) break; + if (unlikely(skb_shinfo(skb)->nr_frags >= MAX_SKB_FRAGS)) { + /* Frame count from host exceeds frags[] capacity */ + dev_kfree_skb_any(skb); + if (fp->rx.skb == skb) + fp->rx.skb = NULL; + dev->stats.rx_length_errors++; + break; + } + if (skb->len == 0) { /* First fragment */ skb->protocol = htons(ETH_P_PHONET); skb_reset_mac_header(skb); diff --git a/drivers/usb/gadget/function/f_rndis.c b/drivers/usb/gadget/function/f_rndis.c index 7451e7cb7a8523..ec397291e40b18 100644 --- a/drivers/usb/gadget/function/f_rndis.c +++ b/drivers/usb/gadget/function/f_rndis.c @@ -11,6 +11,7 @@ /* #define VERBOSE_DEBUG */ +#include #include #include #include @@ -665,6 +666,7 @@ rndis_bind(struct usb_configuration *c, struct usb_function *f) struct f_rndis_opts *rndis_opts; struct usb_os_desc_table *os_desc_table __free(kfree) = NULL; + struct net_device *net __free(detach_gadget) = NULL; struct usb_request *request __free(free_usb_request) = NULL; if (!can_support_rndis(c)) @@ -678,23 +680,22 @@ rndis_bind(struct usb_configuration *c, struct usb_function *f) return -ENOMEM; } - rndis_iad_descriptor.bFunctionClass = rndis_opts->class; - rndis_iad_descriptor.bFunctionSubClass = rndis_opts->subclass; - rndis_iad_descriptor.bFunctionProtocol = rndis_opts->protocol; - - /* - * in drivers/usb/gadget/configfs.c:configfs_composite_bind() - * configurations are bound in sequence with list_for_each_entry, - * in each configuration its functions are bound in sequence - * with list_for_each_entry, so we assume no race condition - * with regard to rndis_opts->bound access - */ - if (!rndis_opts->bound) { - gether_set_gadget(rndis_opts->net, cdev->gadget); - status = gether_register_netdev(rndis_opts->net); - if (status) - return status; - rndis_opts->bound = true; + scoped_guard(mutex, &rndis_opts->lock) { + rndis_iad_descriptor.bFunctionClass = rndis_opts->class; + rndis_iad_descriptor.bFunctionSubClass = rndis_opts->subclass; + rndis_iad_descriptor.bFunctionProtocol = rndis_opts->protocol; + + if (rndis_opts->bind_count == 0 && !rndis_opts->borrowed_net) { + if (!device_is_registered(&rndis_opts->net->dev)) { + gether_set_gadget(rndis_opts->net, cdev->gadget); + status = gether_register_netdev(rndis_opts->net); + } else + status = gether_attach_gadget(rndis_opts->net, cdev->gadget); + + if (status) + return status; + net = rndis_opts->net; + } } us = usb_gstrings_attach(cdev, rndis_strings, @@ -793,6 +794,9 @@ rndis_bind(struct usb_configuration *c, struct usb_function *f) } rndis->notify_req = no_free_ptr(request); + rndis_opts->bind_count++; + retain_and_null_ptr(net); + /* NOTE: all that is done without knowing or caring about * the network link ... which is unavailable to this code * until we're activated via set_alt(). @@ -809,11 +813,11 @@ void rndis_borrow_net(struct usb_function_instance *f, struct net_device *net) struct f_rndis_opts *opts; opts = container_of(f, struct f_rndis_opts, func_inst); - if (opts->bound) + if (device_is_registered(&opts->net->dev)) gether_cleanup(netdev_priv(opts->net)); else free_netdev(opts->net); - opts->borrowed_net = opts->bound = true; + opts->borrowed_net = true; opts->net = net; } EXPORT_SYMBOL_GPL(rndis_borrow_net); @@ -871,7 +875,7 @@ static void rndis_free_inst(struct usb_function_instance *f) opts = container_of(f, struct f_rndis_opts, func_inst); if (!opts->borrowed_net) { - if (opts->bound) + if (device_is_registered(&opts->net->dev)) gether_cleanup(netdev_priv(opts->net)); else free_netdev(opts->net); @@ -940,6 +944,9 @@ static void rndis_free(struct usb_function *f) static void rndis_unbind(struct usb_configuration *c, struct usb_function *f) { struct f_rndis *rndis = func_to_rndis(f); + struct f_rndis_opts *rndis_opts; + + rndis_opts = container_of(f->fi, struct f_rndis_opts, func_inst); kfree(f->os_desc_table); f->os_desc_n = 0; @@ -947,6 +954,10 @@ static void rndis_unbind(struct usb_configuration *c, struct usb_function *f) kfree(rndis->notify_req->buf); usb_ep_free_request(rndis->notify, rndis->notify_req); + + rndis_opts->bind_count--; + if (rndis_opts->bind_count == 0 && !rndis_opts->borrowed_net) + gether_detach_gadget(rndis_opts->net); } static struct usb_function *rndis_alloc(struct usb_function_instance *fi) diff --git a/drivers/usb/gadget/function/f_subset.c b/drivers/usb/gadget/function/f_subset.c index ea3fdd84246290..638e3138f8a0b7 100644 --- a/drivers/usb/gadget/function/f_subset.c +++ b/drivers/usb/gadget/function/f_subset.c @@ -6,6 +6,7 @@ * Copyright (C) 2008 Nokia Corporation */ +#include #include #include #include @@ -298,25 +299,22 @@ geth_bind(struct usb_configuration *c, struct usb_function *f) struct usb_ep *ep; struct f_gether_opts *gether_opts; + struct net_device *net __free(detach_gadget) = NULL; gether_opts = container_of(f->fi, struct f_gether_opts, func_inst); - /* - * in drivers/usb/gadget/configfs.c:configfs_composite_bind() - * configurations are bound in sequence with list_for_each_entry, - * in each configuration its functions are bound in sequence - * with list_for_each_entry, so we assume no race condition - * with regard to gether_opts->bound access - */ - if (!gether_opts->bound) { - mutex_lock(&gether_opts->lock); - gether_set_gadget(gether_opts->net, cdev->gadget); - status = gether_register_netdev(gether_opts->net); - mutex_unlock(&gether_opts->lock); - if (status) - return status; - gether_opts->bound = true; - } + scoped_guard(mutex, &gether_opts->lock) + if (gether_opts->bind_count == 0 && !gether_opts->bound) { + if (!device_is_registered(&gether_opts->net->dev)) { + gether_set_gadget(gether_opts->net, cdev->gadget); + status = gether_register_netdev(gether_opts->net); + } else + status = gether_attach_gadget(gether_opts->net, cdev->gadget); + + if (status) + return status; + net = gether_opts->net; + } us = usb_gstrings_attach(cdev, geth_strings, ARRAY_SIZE(geth_string_defs)); @@ -329,20 +327,18 @@ geth_bind(struct usb_configuration *c, struct usb_function *f) /* allocate instance-specific interface IDs */ status = usb_interface_id(c, f); if (status < 0) - goto fail; + return status; subset_data_intf.bInterfaceNumber = status; - status = -ENODEV; - /* allocate instance-specific endpoints */ ep = usb_ep_autoconfig(cdev->gadget, &fs_subset_in_desc); if (!ep) - goto fail; + return -ENODEV; geth->port.in_ep = ep; ep = usb_ep_autoconfig(cdev->gadget, &fs_subset_out_desc); if (!ep) - goto fail; + return -ENODEV; geth->port.out_ep = ep; /* support all relevant hardware speeds... we expect that when @@ -360,21 +356,19 @@ geth_bind(struct usb_configuration *c, struct usb_function *f) status = usb_assign_descriptors(f, fs_eth_function, hs_eth_function, ss_eth_function, ss_eth_function); if (status) - goto fail; + return status; /* NOTE: all that is done without knowing or caring about * the network link ... which is unavailable to this code * until we're activated via set_alt(). */ + gether_opts->bind_count++; + retain_and_null_ptr(net); + DBG(cdev, "CDC Subset: IN/%s OUT/%s\n", geth->port.in_ep->name, geth->port.out_ep->name); return 0; - -fail: - ERROR(cdev, "%s: can't bind, err %d\n", f->name, status); - - return status; } static inline struct f_gether_opts *to_f_gether_opts(struct config_item *item) @@ -417,7 +411,7 @@ static void geth_free_inst(struct usb_function_instance *f) struct f_gether_opts *opts; opts = container_of(f, struct f_gether_opts, func_inst); - if (opts->bound) + if (device_is_registered(&opts->net->dev)) gether_cleanup(netdev_priv(opts->net)); else free_netdev(opts->net); @@ -449,15 +443,28 @@ static struct usb_function_instance *geth_alloc_inst(void) static void geth_free(struct usb_function *f) { struct f_gether *eth; + struct f_gether_opts *opts; + + opts = container_of(f->fi, struct f_gether_opts, func_inst); eth = func_to_geth(f); + scoped_guard(mutex, &opts->lock) + opts->refcnt--; kfree(eth); } static void geth_unbind(struct usb_configuration *c, struct usb_function *f) { + struct f_gether_opts *opts; + + opts = container_of(f->fi, struct f_gether_opts, func_inst); + geth_string_defs[0].id = 0; usb_free_all_descriptors(f); + + opts->bind_count--; + if (opts->bind_count == 0 && !opts->bound) + gether_detach_gadget(opts->net); } static struct usb_function *geth_alloc(struct usb_function_instance *fi) diff --git a/drivers/usb/gadget/function/f_tcm.c b/drivers/usb/gadget/function/f_tcm.c index 6e8804f04baa77..7b27f8082ace66 100644 --- a/drivers/usb/gadget/function/f_tcm.c +++ b/drivers/usb/gadget/function/f_tcm.c @@ -1222,6 +1222,13 @@ static void usbg_submit_cmd(struct usbg_cmd *cmd) se_cmd = &cmd->se_cmd; tpg = cmd->fu->tpg; tv_nexus = tpg->tpg_nexus; + if (!tv_nexus) { + struct usb_gadget *gadget = fuas_to_gadget(cmd->fu); + + dev_err(&gadget->dev, "Missing nexus, ignoring command\n"); + return; + } + dir = get_cmd_dir(cmd->cmd_buf); if (dir < 0) goto out; @@ -1482,6 +1489,13 @@ static void bot_cmd_work(struct work_struct *work) se_cmd = &cmd->se_cmd; tpg = cmd->fu->tpg; tv_nexus = tpg->tpg_nexus; + if (!tv_nexus) { + struct usb_gadget *gadget = fuas_to_gadget(cmd->fu); + + dev_err(&gadget->dev, "Missing nexus, ignoring command\n"); + return; + } + dir = get_cmd_dir(cmd->cmd_buf); if (dir < 0) goto out; diff --git a/drivers/usb/gadget/function/f_uac1_legacy.c b/drivers/usb/gadget/function/f_uac1_legacy.c index 49cf5aae90ca3e..4981af8337ab82 100644 --- a/drivers/usb/gadget/function/f_uac1_legacy.c +++ b/drivers/usb/gadget/function/f_uac1_legacy.c @@ -360,19 +360,46 @@ static int f_audio_out_ep_complete(struct usb_ep *ep, struct usb_request *req) static void f_audio_complete(struct usb_ep *ep, struct usb_request *req) { struct f_audio *audio = req->context; - int status = req->status; - u32 data = 0; struct usb_ep *out_ep = audio->out_ep; - switch (status) { - - case 0: /* normal completion? */ - if (ep == out_ep) + switch (req->status) { + case 0: + if (ep == out_ep) { f_audio_out_ep_complete(ep, req); - else if (audio->set_con) { - memcpy(&data, req->buf, req->length); - audio->set_con->set(audio->set_con, audio->set_cmd, - le16_to_cpu(data)); + } else if (audio->set_con) { + struct usb_audio_control *con = audio->set_con; + u8 type = con->type; + u32 data; + bool valid_request = false; + + switch (type) { + case UAC_FU_MUTE: { + u8 value; + + if (req->actual == sizeof(value)) { + memcpy(&value, req->buf, sizeof(value)); + data = value; + valid_request = true; + } + break; + } + case UAC_FU_VOLUME: { + __le16 value; + + if (req->actual == sizeof(value)) { + memcpy(&value, req->buf, sizeof(value)); + data = le16_to_cpu(value); + valid_request = true; + } + break; + } + } + + if (valid_request) + con->set(con, audio->set_cmd, data); + else + usb_ep_set_halt(ep); + audio->set_con = NULL; } break; diff --git a/drivers/usb/gadget/function/f_uvc.c b/drivers/usb/gadget/function/f_uvc.c index a96476507d2fdf..842187a09cc032 100644 --- a/drivers/usb/gadget/function/f_uvc.c +++ b/drivers/usb/gadget/function/f_uvc.c @@ -413,6 +413,12 @@ uvc_function_disconnect(struct uvc_device *uvc) { int ret; + guard(mutex)(&uvc->lock); + if (uvc->func_unbound) { + dev_dbg(&uvc->vdev.dev, "skipping function deactivate (unbound)\n"); + return; + } + if ((ret = usb_function_deactivate(&uvc->func)) < 0) uvcg_info(&uvc->func, "UVC disconnect failed with %d\n", ret); } @@ -431,6 +437,15 @@ static ssize_t function_name_show(struct device *dev, static DEVICE_ATTR_RO(function_name); +static void uvc_vdev_release(struct video_device *vdev) +{ + struct uvc_device *uvc = video_get_drvdata(vdev); + + /* Signal uvc_function_unbind() that the video device has been released */ + if (uvc->vdev_release_done) + complete(uvc->vdev_release_done); +} + static int uvc_register_video(struct uvc_device *uvc) { @@ -443,7 +458,7 @@ uvc_register_video(struct uvc_device *uvc) uvc->vdev.v4l2_dev->dev = &cdev->gadget->dev; uvc->vdev.fops = &uvc_v4l2_fops; uvc->vdev.ioctl_ops = &uvc_v4l2_ioctl_ops; - uvc->vdev.release = video_device_release_empty; + uvc->vdev.release = uvc_vdev_release; uvc->vdev.vfl_dir = VFL_DIR_TX; uvc->vdev.lock = &uvc->video.mutex; uvc->vdev.device_caps = V4L2_CAP_VIDEO_OUTPUT | V4L2_CAP_STREAMING; @@ -659,6 +674,8 @@ uvc_function_bind(struct usb_configuration *c, struct usb_function *f) int ret = -EINVAL; uvcg_info(f, "%s()\n", __func__); + scoped_guard(mutex, &uvc->lock) + uvc->func_unbound = false; opts = fi_to_f_uvc_opts(f->fi); /* Sanity check the streaming endpoint module parameters. */ @@ -988,12 +1005,19 @@ static void uvc_free(struct usb_function *f) static void uvc_function_unbind(struct usb_configuration *c, struct usb_function *f) { + DECLARE_COMPLETION_ONSTACK(vdev_release_done); struct usb_composite_dev *cdev = c->cdev; struct uvc_device *uvc = to_uvc(f); struct uvc_video *video = &uvc->video; long wait_ret = 1; + bool connected; uvcg_info(f, "%s()\n", __func__); + scoped_guard(mutex, &uvc->lock) { + uvc->func_unbound = true; + uvc->vdev_release_done = &vdev_release_done; + connected = uvc->func_connected; + } kthread_cancel_work_sync(&video->hw_submit); @@ -1006,7 +1030,7 @@ static void uvc_function_unbind(struct usb_configuration *c, * though the video device removal uevent. Allow some time for the * application to close out before things get deleted. */ - if (uvc->func_connected) { + if (connected) { uvcg_dbg(f, "waiting for clean disconnect\n"); wait_ret = wait_event_interruptible_timeout(uvc->func_connected_queue, uvc->func_connected == false, msecs_to_jiffies(500)); @@ -1017,7 +1041,10 @@ static void uvc_function_unbind(struct usb_configuration *c, video_unregister_device(&uvc->vdev); v4l2_device_unregister(&uvc->v4l2_dev); - if (uvc->func_connected) { + scoped_guard(mutex, &uvc->lock) + connected = uvc->func_connected; + + if (connected) { /* * Wait for the release to occur to ensure there are no longer any * pending operations that may cause panics when resources are cleaned @@ -1029,6 +1056,10 @@ static void uvc_function_unbind(struct usb_configuration *c, uvcg_dbg(f, "done waiting for release with ret: %ld\n", wait_ret); } + /* Wait for the video device to be released */ + wait_for_completion(&vdev_release_done); + uvc->vdev_release_done = NULL; + usb_ep_free_request(cdev->gadget->ep0, uvc->control_req); kfree(uvc->control_buf); @@ -1047,6 +1078,8 @@ static struct usb_function *uvc_alloc(struct usb_function_instance *fi) return ERR_PTR(-ENOMEM); mutex_init(&uvc->video.mutex); + mutex_init(&uvc->lock); + uvc->func_unbound = true; uvc->state = UVC_STATE_DISCONNECTED; init_waitqueue_head(&uvc->func_connected_queue); opts = fi_to_f_uvc_opts(fi); diff --git a/drivers/usb/gadget/function/u_ecm.h b/drivers/usb/gadget/function/u_ecm.h index 77cfb89932bea7..7f666b9dea0265 100644 --- a/drivers/usb/gadget/function/u_ecm.h +++ b/drivers/usb/gadget/function/u_ecm.h @@ -15,17 +15,26 @@ #include +/** + * struct f_ecm_opts - ECM function options + * @func_inst: USB function instance. + * @net: The net_device associated with the ECM function. + * @bound: True if the net_device is shared and pre-registered during the + * legacy composite driver's bind phase (e.g., multi.c). If false, + * the ECM function will register the net_device during its own + * bind phase. + * @bind_count: Tracks the number of configurations the ECM function is + * bound to, preventing double-registration of the @net device. + * @lock: Protects the data from concurrent access by configfs read/write + * and create symlink/remove symlink operations. + * @refcnt: Reference counter for the function instance. + */ struct f_ecm_opts { struct usb_function_instance func_inst; struct net_device *net; bool bound; + int bind_count; - /* - * Read/write access to configfs attributes is handled by configfs. - * - * This is to protect the data from concurrent access by read/write - * and create symlink/remove symlink. - */ struct mutex lock; int refcnt; }; diff --git a/drivers/usb/gadget/function/u_eem.h b/drivers/usb/gadget/function/u_eem.h index 3bd85dfcd71c81..78ef5581521972 100644 --- a/drivers/usb/gadget/function/u_eem.h +++ b/drivers/usb/gadget/function/u_eem.h @@ -15,17 +15,26 @@ #include +/** + * struct f_eem_opts - EEM function options + * @func_inst: USB function instance. + * @net: The net_device associated with the EEM function. + * @bound: True if the net_device is shared and pre-registered during the + * legacy composite driver's bind phase (e.g., multi.c). If false, + * the EEM function will register the net_device during its own + * bind phase. + * @bind_count: Tracks the number of configurations the EEM function is + * bound to, preventing double-registration of the @net device. + * @lock: Protects the data from concurrent access by configfs read/write + * and create symlink/remove symlink operations. + * @refcnt: Reference counter for the function instance. + */ struct f_eem_opts { struct usb_function_instance func_inst; struct net_device *net; bool bound; + int bind_count; - /* - * Read/write access to configfs attributes is handled by configfs. - * - * This is to protect the data from concurrent access by read/write - * and create symlink/remove symlink. - */ struct mutex lock; int refcnt; }; diff --git a/drivers/usb/gadget/function/u_ether.c b/drivers/usb/gadget/function/u_ether.c index f58590bf5e02f5..2b824db4d31b78 100644 --- a/drivers/usb/gadget/function/u_ether.c +++ b/drivers/usb/gadget/function/u_ether.c @@ -112,8 +112,10 @@ static void eth_get_drvinfo(struct net_device *net, struct ethtool_drvinfo *p) strscpy(p->driver, "g_ether", sizeof(p->driver)); strscpy(p->version, UETH__VERSION, sizeof(p->version)); - strscpy(p->fw_version, dev->gadget->name, sizeof(p->fw_version)); - strscpy(p->bus_info, dev_name(&dev->gadget->dev), sizeof(p->bus_info)); + if (dev->gadget) { + strscpy(p->fw_version, dev->gadget->name, sizeof(p->fw_version)); + strscpy(p->bus_info, dev_name(&dev->gadget->dev), sizeof(p->bus_info)); + } } /* REVISIT can also support: @@ -896,6 +898,28 @@ void gether_set_gadget(struct net_device *net, struct usb_gadget *g) } EXPORT_SYMBOL_GPL(gether_set_gadget); +int gether_attach_gadget(struct net_device *net, struct usb_gadget *g) +{ + int ret; + + ret = device_move(&net->dev, &g->dev, DPM_ORDER_DEV_AFTER_PARENT); + if (ret) + return ret; + + gether_set_gadget(net, g); + return 0; +} +EXPORT_SYMBOL_GPL(gether_attach_gadget); + +void gether_detach_gadget(struct net_device *net) +{ + struct eth_dev *dev = netdev_priv(net); + + device_move(&net->dev, NULL, DPM_ORDER_NONE); + dev->gadget = NULL; +} +EXPORT_SYMBOL_GPL(gether_detach_gadget); + int gether_set_dev_addr(struct net_device *net, const char *dev_addr) { struct eth_dev *dev; @@ -1200,6 +1224,11 @@ void gether_disconnect(struct gether *link) DBG(dev, "%s\n", __func__); + spin_lock(&dev->lock); + dev->port_usb = NULL; + link->is_suspend = false; + spin_unlock(&dev->lock); + netif_stop_queue(dev->net); netif_carrier_off(dev->net); @@ -1237,11 +1266,6 @@ void gether_disconnect(struct gether *link) dev->header_len = 0; dev->unwrap = NULL; dev->wrap = NULL; - - spin_lock(&dev->lock); - dev->port_usb = NULL; - link->is_suspend = false; - spin_unlock(&dev->lock); } EXPORT_SYMBOL_GPL(gether_disconnect); diff --git a/drivers/usb/gadget/function/u_ether.h b/drivers/usb/gadget/function/u_ether.h index 34be220cef77c4..c85a1cf3c115d2 100644 --- a/drivers/usb/gadget/function/u_ether.h +++ b/drivers/usb/gadget/function/u_ether.h @@ -150,6 +150,32 @@ static inline struct net_device *gether_setup_default(void) */ void gether_set_gadget(struct net_device *net, struct usb_gadget *g); +/** + * gether_attach_gadget - Reparent net_device to the gadget device. + * @net: The network device to reparent. + * @g: The target USB gadget device to parent to. + * + * This function moves the network device to be a child of the USB gadget + * device in the device hierarchy. This is typically done when the function + * is bound to a configuration. + * + * Returns 0 on success, or a negative error code on failure. + */ +int gether_attach_gadget(struct net_device *net, struct usb_gadget *g); + +/** + * gether_detach_gadget - Detach net_device from its gadget parent. + * @net: The network device to detach. + * + * This function moves the network device to be a child of the virtual + * devices parent, effectively detaching it from the USB gadget device + * hierarchy. This is typically done when the function is unbound + * from a configuration but the instance is not yet freed. + */ +void gether_detach_gadget(struct net_device *net); + +DEFINE_FREE(detach_gadget, struct net_device *, if (_T) gether_detach_gadget(_T)) + /** * gether_set_dev_addr - initialize an ethernet-over-usb link with eth address * @net: device representing this link diff --git a/drivers/usb/gadget/function/u_gether.h b/drivers/usb/gadget/function/u_gether.h index 2f7a373ed4496a..e7b6b51f69c12a 100644 --- a/drivers/usb/gadget/function/u_gether.h +++ b/drivers/usb/gadget/function/u_gether.h @@ -15,17 +15,25 @@ #include +/** + * struct f_gether_opts - subset function options + * @func_inst: USB function instance. + * @net: The net_device associated with the subset function. + * @bound: True if the net_device is shared and pre-registered during the + * legacy composite driver's bind phase (e.g., multi.c). If false, + * the subset function will register the net_device during its own + * bind phase. + * @bind_count: Tracks the number of configurations the subset function is + * bound to, preventing double-registration of the @net device. + * @lock: Protects the data from concurrent access by configfs read/write + * and create symlink/remove symlink operations. + * @refcnt: Reference counter for the function instance. + */ struct f_gether_opts { struct usb_function_instance func_inst; struct net_device *net; bool bound; - - /* - * Read/write access to configfs attributes is handled by configfs. - * - * This is to protect the data from concurrent access by read/write - * and create symlink/remove symlink. - */ + int bind_count; struct mutex lock; int refcnt; }; diff --git a/drivers/usb/gadget/function/u_ncm.h b/drivers/usb/gadget/function/u_ncm.h index 49ec095cdb4b6d..b1f3db8b68c15e 100644 --- a/drivers/usb/gadget/function/u_ncm.h +++ b/drivers/usb/gadget/function/u_ncm.h @@ -18,7 +18,7 @@ struct f_ncm_opts { struct usb_function_instance func_inst; struct net_device *net; - bool bound; + int bind_count; struct config_group *ncm_interf_group; struct usb_os_desc ncm_os_desc; diff --git a/drivers/usb/gadget/function/u_rndis.h b/drivers/usb/gadget/function/u_rndis.h index a8c409b2f52f9e..4e64619714dc2d 100644 --- a/drivers/usb/gadget/function/u_rndis.h +++ b/drivers/usb/gadget/function/u_rndis.h @@ -15,12 +15,34 @@ #include +/** + * struct f_rndis_opts - RNDIS function options + * @func_inst: USB function instance. + * @vendor_id: Vendor ID. + * @manufacturer: Manufacturer string. + * @net: The net_device associated with the RNDIS function. + * @bind_count: Tracks the number of configurations the RNDIS function is + * bound to, preventing double-registration of the @net device. + * @borrowed_net: True if the net_device is shared and pre-registered during + * the legacy composite driver's bind phase (e.g., multi.c). + * If false, the RNDIS function will register the net_device + * during its own bind phase. + * @rndis_interf_group: ConfigFS group for RNDIS interface. + * @rndis_os_desc: USB OS descriptor for RNDIS. + * @rndis_ext_compat_id: Extended compatibility ID. + * @class: USB class. + * @subclass: USB subclass. + * @protocol: USB protocol. + * @lock: Protects the data from concurrent access by configfs read/write + * and create symlink/remove symlink operations. + * @refcnt: Reference counter for the function instance. + */ struct f_rndis_opts { struct usb_function_instance func_inst; u32 vendor_id; const char *manufacturer; struct net_device *net; - bool bound; + int bind_count; bool borrowed_net; struct config_group *rndis_interf_group; @@ -30,13 +52,6 @@ struct f_rndis_opts { u8 class; u8 subclass; u8 protocol; - - /* - * Read/write access to configfs attributes is handled by configfs. - * - * This is to protect the data from concurrent access by read/write - * and create symlink/remove symlink. - */ struct mutex lock; int refcnt; }; diff --git a/drivers/usb/gadget/function/uvc.h b/drivers/usb/gadget/function/uvc.h index 676419a049762f..7abfdd5e1eefb6 100644 --- a/drivers/usb/gadget/function/uvc.h +++ b/drivers/usb/gadget/function/uvc.h @@ -155,6 +155,9 @@ struct uvc_device { enum uvc_state state; struct usb_function func; struct uvc_video video; + struct completion *vdev_release_done; + struct mutex lock; /* protects func_unbound and func_connected */ + bool func_unbound; bool func_connected; wait_queue_head_t func_connected_queue; diff --git a/drivers/usb/gadget/function/uvc_v4l2.c b/drivers/usb/gadget/function/uvc_v4l2.c index fd4b998ccd1605..23bafb07133f25 100644 --- a/drivers/usb/gadget/function/uvc_v4l2.c +++ b/drivers/usb/gadget/function/uvc_v4l2.c @@ -574,6 +574,8 @@ uvc_v4l2_subscribe_event(struct v4l2_fh *fh, if (sub->type < UVC_EVENT_FIRST || sub->type > UVC_EVENT_LAST) return -EINVAL; + guard(mutex)(&uvc->lock); + if (sub->type == UVC_EVENT_SETUP && uvc->func_connected) return -EBUSY; @@ -595,7 +597,8 @@ static void uvc_v4l2_disable(struct uvc_device *uvc) uvc_function_disconnect(uvc); uvcg_video_disable(&uvc->video); uvcg_free_buffers(&uvc->video.queue); - uvc->func_connected = false; + scoped_guard(mutex, &uvc->lock) + uvc->func_connected = false; wake_up_interruptible(&uvc->func_connected_queue); } diff --git a/drivers/usb/gadget/function/uvc_video.c b/drivers/usb/gadget/function/uvc_video.c index f568dee08b3b72..bbde1db6833b5a 100644 --- a/drivers/usb/gadget/function/uvc_video.c +++ b/drivers/usb/gadget/function/uvc_video.c @@ -513,7 +513,7 @@ uvc_video_prep_requests(struct uvc_video *video) return; } - interval_duration = 2 << (video->ep->desc->bInterval - 1); + interval_duration = 1 << (video->ep->desc->bInterval - 1); if (cdev->gadget->speed < USB_SPEED_HIGH) interval_duration *= 10000; else diff --git a/drivers/usb/gadget/udc/bdc/bdc_core.c b/drivers/usb/gadget/udc/bdc/bdc_core.c index 5c3d8b64c0e76e..f47aac078f6be6 100644 --- a/drivers/usb/gadget/udc/bdc/bdc_core.c +++ b/drivers/usb/gadget/udc/bdc/bdc_core.c @@ -35,8 +35,8 @@ static int poll_oip(struct bdc *bdc, u32 usec) u32 status; int ret; - ret = readl_poll_timeout(bdc->regs + BDC_BDCSC, status, - (BDC_CSTS(status) != BDC_OIP), 10, usec); + ret = readl_poll_timeout_atomic(bdc->regs + BDC_BDCSC, status, + (BDC_CSTS(status) != BDC_OIP), 10, usec); if (ret) dev_err(bdc->dev, "operation timedout BDCSC: 0x%08x\n", status); else diff --git a/drivers/usb/gadget/udc/dummy_hcd.c b/drivers/usb/gadget/udc/dummy_hcd.c index 1cefca660773c4..da271308d75345 100644 --- a/drivers/usb/gadget/udc/dummy_hcd.c +++ b/drivers/usb/gadget/udc/dummy_hcd.c @@ -462,8 +462,13 @@ static void set_link_state(struct dummy_hcd *dum_hcd) /* Report reset and disconnect events to the driver */ if (dum->ints_enabled && (disconnect || reset)) { - stop_activity(dum); ++dum->callback_usage; + /* + * stop_activity() can drop dum->lock, so it must + * not come between the dum->ints_enabled test + * and the ++dum->callback_usage. + */ + stop_activity(dum); spin_unlock(&dum->lock); if (reset) usb_gadget_udc_reset(&dum->gadget, dum->driver); @@ -908,21 +913,6 @@ static int dummy_pullup(struct usb_gadget *_gadget, int value) spin_lock_irqsave(&dum->lock, flags); dum->pullup = (value != 0); set_link_state(dum_hcd); - if (value == 0) { - /* - * Emulate synchronize_irq(): wait for callbacks to finish. - * This seems to be the best place to emulate the call to - * synchronize_irq() that's in usb_gadget_remove_driver(). - * Doing it in dummy_udc_stop() would be too late since it - * is called after the unbind callback and unbind shouldn't - * be invoked until all the other callbacks are finished. - */ - while (dum->callback_usage > 0) { - spin_unlock_irqrestore(&dum->lock, flags); - usleep_range(1000, 2000); - spin_lock_irqsave(&dum->lock, flags); - } - } spin_unlock_irqrestore(&dum->lock, flags); usb_hcd_poll_rh_status(dummy_hcd_to_hcd(dum_hcd)); @@ -945,6 +935,20 @@ static void dummy_udc_async_callbacks(struct usb_gadget *_gadget, bool enable) spin_lock_irq(&dum->lock); dum->ints_enabled = enable; + if (!enable) { + /* + * Emulate synchronize_irq(): wait for callbacks to finish. + * This has to happen after emulated interrupts are disabled + * (dum->ints_enabled is clear) and before the unbind callback, + * just like the call to synchronize_irq() in + * gadget/udc/core:gadget_unbind_driver(). + */ + while (dum->callback_usage > 0) { + spin_unlock_irq(&dum->lock); + usleep_range(1000, 2000); + spin_lock_irq(&dum->lock); + } + } spin_unlock_irq(&dum->lock); } @@ -1534,6 +1538,12 @@ static int transfer(struct dummy_hcd *dum_hcd, struct urb *urb, /* rescan to continue with any other queued i/o */ if (rescan) goto top; + + /* request not fully transferred; stop iterating to + * preserve data ordering across queued requests. + */ + if (req->req.actual < req->req.length) + break; } return sent; } diff --git a/drivers/usb/gadget/udc/renesas_usb3.c b/drivers/usb/gadget/udc/renesas_usb3.c index 7cdcc9d16b8b13..ea89bcb9ad35a1 100644 --- a/drivers/usb/gadget/udc/renesas_usb3.c +++ b/drivers/usb/gadget/udc/renesas_usb3.c @@ -1669,6 +1669,10 @@ static bool usb3_std_req_get_status(struct renesas_usb3 *usb3, break; case USB_RECIP_ENDPOINT: num = le16_to_cpu(ctrl->wIndex) & USB_ENDPOINT_NUMBER_MASK; + if (num >= usb3->num_usb3_eps) { + stall = true; + break; + } usb3_ep = usb3_get_ep(usb3, num); if (usb3_ep->halt) status |= 1 << USB_ENDPOINT_HALT; @@ -1781,7 +1785,8 @@ static bool usb3_std_req_feature_endpoint(struct renesas_usb3 *usb3, struct renesas_usb3_ep *usb3_ep; struct renesas_usb3_request *usb3_req; - if (le16_to_cpu(ctrl->wValue) != USB_ENDPOINT_HALT) + if ((le16_to_cpu(ctrl->wValue) != USB_ENDPOINT_HALT) || + (num >= usb3->num_usb3_eps)) return true; /* stall */ usb3_ep = usb3_get_ep(usb3, num); diff --git a/drivers/usb/gadget/udc/tegra-xudc.c b/drivers/usb/gadget/udc/tegra-xudc.c index 9d2007f448c049..7f7251c10e9526 100644 --- a/drivers/usb/gadget/udc/tegra-xudc.c +++ b/drivers/usb/gadget/udc/tegra-xudc.c @@ -3392,17 +3392,18 @@ static void tegra_xudc_device_params_init(struct tegra_xudc *xudc) { u32 val, imod; + val = xudc_readl(xudc, BLCG); if (xudc->soc->has_ipfs) { - val = xudc_readl(xudc, BLCG); val |= BLCG_ALL; val &= ~(BLCG_DFPCI | BLCG_UFPCI | BLCG_FE | BLCG_COREPLL_PWRDN); val |= BLCG_IOPLL_0_PWRDN; val |= BLCG_IOPLL_1_PWRDN; val |= BLCG_IOPLL_2_PWRDN; - - xudc_writel(xudc, val, BLCG); + } else { + val &= ~BLCG_COREPLL_PWRDN; } + xudc_writel(xudc, val, BLCG); if (xudc->soc->port_speed_quirk) tegra_xudc_limit_port_speed(xudc); @@ -3953,6 +3954,7 @@ static void tegra_xudc_remove(struct platform_device *pdev) static int __maybe_unused tegra_xudc_powergate(struct tegra_xudc *xudc) { unsigned long flags; + u32 val; dev_dbg(xudc->dev, "entering ELPG\n"); @@ -3965,6 +3967,10 @@ static int __maybe_unused tegra_xudc_powergate(struct tegra_xudc *xudc) spin_unlock_irqrestore(&xudc->lock, flags); + val = xudc_readl(xudc, BLCG); + val |= BLCG_COREPLL_PWRDN; + xudc_writel(xudc, val, BLCG); + clk_bulk_disable_unprepare(xudc->soc->num_clks, xudc->clks); regulator_bulk_disable(xudc->soc->num_supplies, xudc->supplies); diff --git a/drivers/usb/host/Kconfig b/drivers/usb/host/Kconfig index c4f17ce5c77b15..5f3ecce8398b39 100644 --- a/drivers/usb/host/Kconfig +++ b/drivers/usb/host/Kconfig @@ -51,6 +51,15 @@ config USB_XHCI_PCI_RENESAS installed on your system for this device to work. If unsure, say 'N'. +config USB_XHCI_PCI_ASMEDIA + bool "Support firmware loading for ASMedia xHCI controllers" + default USB_XHCI_PCI if ARCH_APPLE + depends on USB_XHCI_PCI + help + Say 'Y' to enable support for ASMedia xHCI controllers with + host-supplied firmware. These are usually present on Apple devices. + If unsure, say 'N'. + config USB_XHCI_PLATFORM tristate "Generic xHCI driver for a platform device" help diff --git a/drivers/usb/host/Makefile b/drivers/usb/host/Makefile index 4df946c05ba0ee..a197e0979f89bd 100644 --- a/drivers/usb/host/Makefile +++ b/drivers/usb/host/Makefile @@ -72,6 +72,8 @@ obj-$(CONFIG_USB_UHCI_HCD) += uhci-hcd.o obj-$(CONFIG_USB_FHCI_HCD) += fhci.o obj-$(CONFIG_USB_XHCI_HCD) += xhci-hcd.o obj-$(CONFIG_USB_XHCI_PCI) += xhci-pci.o +xhci-pci-y += xhci-pci-core.o +xhci-pci-$(CONFIG_USB_XHCI_PCI_ASMEDIA) += xhci-pci-asmedia.o obj-$(CONFIG_USB_XHCI_PCI_RENESAS) += xhci-pci-renesas.o obj-$(CONFIG_USB_XHCI_PLATFORM) += xhci-plat-hcd.o obj-$(CONFIG_USB_XHCI_HISTB) += xhci-histb.o diff --git a/drivers/usb/host/ehci-brcm.c b/drivers/usb/host/ehci-brcm.c index 888e8f6670d2a1..5e3156f94cc690 100644 --- a/drivers/usb/host/ehci-brcm.c +++ b/drivers/usb/host/ehci-brcm.c @@ -31,8 +31,8 @@ static inline void ehci_brcm_wait_for_sof(struct ehci_hcd *ehci, u32 delay) int res; /* Wait for next microframe (every 125 usecs) */ - res = readl_relaxed_poll_timeout(&ehci->regs->frame_index, val, - val != frame_idx, 1, 130); + res = readl_relaxed_poll_timeout_atomic(&ehci->regs->frame_index, + val, val != frame_idx, 1, 130); if (res) ehci_err(ehci, "Error waiting for SOF\n"); udelay(delay); diff --git a/drivers/usb/host/xhci-debugfs.c b/drivers/usb/host/xhci-debugfs.c index c1eb1036ede954..5ff5b761bccf8e 100644 --- a/drivers/usb/host/xhci-debugfs.c +++ b/drivers/usb/host/xhci-debugfs.c @@ -386,11 +386,19 @@ static const struct file_operations port_fops = { static int xhci_portli_show(struct seq_file *s, void *unused) { struct xhci_port *port = s->private; - struct xhci_hcd *xhci = hcd_to_xhci(port->rhub->hcd); + struct xhci_hcd *xhci; u32 portli; portli = readl(&port->port_reg->portli); + /* port without protocol capability isn't added to a roothub */ + if (!port->rhub) { + seq_printf(s, "0x%08x\n", portli); + return 0; + } + + xhci = hcd_to_xhci(port->rhub->hcd); + /* PORTLI fields are valid if port is a USB3 or eUSB2V2 port */ if (port->rhub == &xhci->usb3_rhub) seq_printf(s, "0x%08x LEC=%u RLC=%u TLC=%u\n", portli, diff --git a/drivers/usb/host/xhci-pci-asmedia.c b/drivers/usb/host/xhci-pci-asmedia.c new file mode 100644 index 00000000000000..d6b12f5c540296 --- /dev/null +++ b/drivers/usb/host/xhci-pci-asmedia.c @@ -0,0 +1,414 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * ASMedia xHCI firmware loader + * Copyright (C) The Asahi Linux Contributors + */ + +#include +#include +#include +#include +#include +#include + +#include "xhci.h" +#include "xhci-trace.h" +#include "xhci-pci.h" + +/* Configuration space registers */ +#define ASMT_CFG_CONTROL 0xe0 +#define ASMT_CFG_CONTROL_WRITE BIT(1) +#define ASMT_CFG_CONTROL_READ BIT(0) + +#define ASMT_CFG_SRAM_ADDR 0xe2 + +#define ASMT_CFG_SRAM_ACCESS 0xef +#define ASMT_CFG_SRAM_ACCESS_READ BIT(6) +#define ASMT_CFG_SRAM_ACCESS_ENABLE BIT(7) + +#define ASMT_CFG_DATA_READ0 0xf0 +#define ASMT_CFG_DATA_READ1 0xf4 + +#define ASMT_CFG_DATA_WRITE0 0xf8 +#define ASMT_CFG_DATA_WRITE1 0xfc + +#define ASMT_CMD_GET_FWVER 0x8000060840 +#define ASMT_FWVER_ROM 0x010250090816 + +/* BAR0 registers */ +#define ASMT_REG_ADDR 0x3000 + +#define ASMT_REG_WDATA 0x3004 +#define ASMT_REG_RDATA 0x3008 + +#define ASMT_REG_STATUS 0x3009 +#define ASMT_REG_STATUS_BUSY BIT(7) + +#define ASMT_REG_CODE_WDATA 0x3010 +#define ASMT_REG_CODE_RDATA 0x3018 + +#define ASMT_MMIO_CPU_MISC 0x500e +#define ASMT_MMIO_CPU_MISC_CODE_RAM_WR BIT(0) + +#define ASMT_MMIO_CPU_MODE_NEXT 0x5040 +#define ASMT_MMIO_CPU_MODE_CUR 0x5041 + +#define ASMT_MMIO_CPU_MODE_RAM BIT(0) +#define ASMT_MMIO_CPU_MODE_HALFSPEED BIT(1) + +#define ASMT_MMIO_CPU_EXEC_CTRL 0x5042 +#define ASMT_MMIO_CPU_EXEC_CTRL_RESET BIT(0) +#define ASMT_MMIO_CPU_EXEC_CTRL_HALT BIT(1) + +#define TIMEOUT_USEC 10000 +#define RESET_TIMEOUT_USEC 500000 + +static int asmedia_mbox_tx(struct pci_dev *pdev, u64 data) +{ + u8 op; + int ret, err; + + ret = read_poll_timeout(pci_read_config_byte, err, + err || !(op & ASMT_CFG_CONTROL_WRITE), + 1, TIMEOUT_USEC, false, pdev, ASMT_CFG_CONTROL, + &op); + if (ret) { + dev_err(&pdev->dev, + "Timed out on mailbox tx: 0x%llx\n", + data); + return ret; + } + if (err) + return err; + + pci_write_config_dword(pdev, ASMT_CFG_DATA_WRITE0, data); + pci_write_config_dword(pdev, ASMT_CFG_DATA_WRITE1, data >> 32); + pci_write_config_byte(pdev, ASMT_CFG_CONTROL, ASMT_CFG_CONTROL_WRITE); + + return 0; +} + +static int asmedia_mbox_rx(struct pci_dev *pdev, u64 *data) +{ + u8 op; + u32 low, high; + int ret, err; + + ret = read_poll_timeout(pci_read_config_byte, err, + err || (op & ASMT_CFG_CONTROL_READ), + 1, TIMEOUT_USEC, false, pdev, ASMT_CFG_CONTROL, + &op); + if (ret) { + dev_err(&pdev->dev, "Timed out on mailbox rx\n"); + return ret; + } + if (err) + return err; + + pci_read_config_dword(pdev, ASMT_CFG_DATA_READ0, &low); + pci_read_config_dword(pdev, ASMT_CFG_DATA_READ1, &high); + pci_write_config_byte(pdev, ASMT_CFG_CONTROL, ASMT_CFG_CONTROL_READ); + + *data = ((u64)high << 32) | low; + return 0; +} + +static int asmedia_get_fw_version(struct pci_dev *pdev, u64 *version) +{ + int err = 0; + u64 cmd; + + err = asmedia_mbox_tx(pdev, ASMT_CMD_GET_FWVER); + if (err) + return err; + err = asmedia_mbox_tx(pdev, 0); + if (err) + return err; + + err = asmedia_mbox_rx(pdev, &cmd); + if (err) + return err; + err = asmedia_mbox_rx(pdev, version); + if (err) + return err; + + if (cmd != ASMT_CMD_GET_FWVER) { + dev_err(&pdev->dev, "Unexpected reply command 0x%llx\n", cmd); + return -EIO; + } + + return 0; +} + +static bool asmedia_check_firmware(struct pci_dev *pdev) +{ + u64 fwver; + int ret; + + ret = asmedia_get_fw_version(pdev, &fwver); + if (ret) + return ret; + + dev_info(&pdev->dev, "Firmware version: 0x%llx\n", fwver); + + return fwver != ASMT_FWVER_ROM; +} + +static int asmedia_wait_reset(struct pci_dev *pdev) +{ + struct usb_hcd *hcd = dev_get_drvdata(&pdev->dev); + struct xhci_cap_regs __iomem *cap = hcd->regs; + struct xhci_op_regs __iomem *op; + u32 val; + int ret; + + op = hcd->regs + HC_LENGTH(readl(&cap->hc_capbase)); + + ret = readl_poll_timeout(&op->command, + val, !(val & CMD_RESET), + 1000, RESET_TIMEOUT_USEC); + + if (!ret) + return 0; + + dev_err(hcd->self.controller, "Reset timed out, trying to kick it\n"); + + pci_write_config_byte(pdev, ASMT_CFG_SRAM_ACCESS, + ASMT_CFG_SRAM_ACCESS_ENABLE); + + pci_write_config_byte(pdev, ASMT_CFG_SRAM_ACCESS, 0); + + ret = readl_poll_timeout(&op->command, + val, !(val & CMD_RESET), + 1000, RESET_TIMEOUT_USEC); + + if (ret) + dev_err(hcd->self.controller, "Reset timed out, giving up\n"); + + return ret; +} + +static int asmedia_read_reg(struct usb_hcd *hcd, u16 addr, u8 *val) { + void __iomem *regs = hcd->regs; + u8 status; + int ret; + + ret = readb_poll_timeout(regs + ASMT_REG_STATUS, status, + !(status & ASMT_REG_STATUS_BUSY), + 1000, TIMEOUT_USEC); + + if (ret) { + dev_err(hcd->self.controller, + "Read reg wait timed out ([%04x])\n", addr); + return ret; + } + + writew_relaxed(addr, regs + ASMT_REG_ADDR); + + ret = readb_poll_timeout(regs + ASMT_REG_STATUS, status, + !(status & ASMT_REG_STATUS_BUSY), + 1000, TIMEOUT_USEC); + + if (ret) { + dev_err(hcd->self.controller, + "Read reg addr timed out ([%04x])\n", addr); + return ret; + } + + *val = readb_relaxed(regs + ASMT_REG_RDATA); + return 0; +} + +static int asmedia_write_reg(struct usb_hcd *hcd, u16 addr, u8 data, bool wait) { + void __iomem *regs = hcd->regs; + u8 status, val; + int ret, err; + + writew_relaxed(addr, regs + ASMT_REG_ADDR); + + ret = readb_poll_timeout(regs + ASMT_REG_STATUS, status, + !(status & ASMT_REG_STATUS_BUSY), + 1000, TIMEOUT_USEC); + + if (ret) { + dev_err(hcd->self.controller, + "Write reg addr timed out ([%04x] = %02x)\n", + addr, data); + return ret; + } + + writeb_relaxed(data, regs + ASMT_REG_WDATA); + + ret = readb_poll_timeout(regs + ASMT_REG_STATUS, status, + !(status & ASMT_REG_STATUS_BUSY), + 1000, TIMEOUT_USEC); + + if (ret) { + dev_err(hcd->self.controller, + "Write reg data timed out ([%04x] = %02x)\n", + addr, data); + return ret; + } + + if (!wait) + return 0; + + ret = read_poll_timeout(asmedia_read_reg, err, err || val == data, + 0, TIMEOUT_USEC, false, hcd, addr, &val); + if (ret) { + dev_err(hcd->self.controller, + "Verify register timed out ([%04x] = %02x)\n", + addr, data); + return ret; + } + if (err) { + dev_err(hcd->self.controller, + "Verify register read error ([%04x] = %02x)\n", + addr, data); + return err; + } + + return 0; +} + +static int asmedia_load_fw(struct pci_dev *pdev, const struct firmware *fw) +{ + struct usb_hcd *hcd; + void __iomem *regs; + const u16 *fw_data = (const u16 *)fw->data; + u16 raddr; + u32 data; + size_t index = 0, addr = 0; + size_t words = fw->size >> 1; + int ret, err; + + hcd = dev_get_drvdata(&pdev->dev); + regs = hcd->regs; + + ret = asmedia_write_reg(hcd, ASMT_MMIO_CPU_MODE_NEXT, + ASMT_MMIO_CPU_MODE_HALFSPEED, false); + if (ret) + return ret; + + ret = asmedia_write_reg(hcd, ASMT_MMIO_CPU_EXEC_CTRL, + ASMT_MMIO_CPU_EXEC_CTRL_RESET, false); + if (ret) + return ret; + + ret = asmedia_wait_reset(pdev); + if (ret) { + dev_err(hcd->self.controller, "Failed pre-upload reset\n"); + return ret; + } + + ret = asmedia_write_reg(hcd, ASMT_MMIO_CPU_EXEC_CTRL, + ASMT_MMIO_CPU_EXEC_CTRL_HALT, false); + if (ret) + return ret; + + ret = asmedia_write_reg(hcd, ASMT_MMIO_CPU_MISC, + ASMT_MMIO_CPU_MISC_CODE_RAM_WR, true); + if (ret) + return ret; + + pci_write_config_byte(pdev, ASMT_CFG_SRAM_ACCESS, + ASMT_CFG_SRAM_ACCESS_ENABLE); + + /* The firmware upload is interleaved in 0x4000 word blocks */ + addr = index = 0; + while (index < words) { + data = fw_data[index]; + if ((index | 0x4000) < words) + data |= fw_data[index | 0x4000] << 16; + + pci_write_config_word(pdev, ASMT_CFG_SRAM_ADDR, + addr); + + writel_relaxed(data, regs + ASMT_REG_CODE_WDATA); + + ret = read_poll_timeout(pci_read_config_word, err, + err || (raddr != addr), + 1, TIMEOUT_USEC, false, pdev, + ASMT_CFG_SRAM_ADDR, &raddr); + if (ret) { + dev_err(hcd->self.controller, "Word write timed out\n"); + return ret; + } + if (err) + return err; + + if (++index & 0x4000) + index += 0x4000; + addr += 2; + } + + pci_write_config_byte(pdev, ASMT_CFG_SRAM_ACCESS, 0); + + ret = asmedia_write_reg(hcd, ASMT_MMIO_CPU_MISC, 0, true); + if (ret) + return ret; + + ret = asmedia_write_reg(hcd, ASMT_MMIO_CPU_MODE_NEXT, + ASMT_MMIO_CPU_MODE_RAM | + ASMT_MMIO_CPU_MODE_HALFSPEED, false); + if (ret) + return ret; + + ret = asmedia_write_reg(hcd, ASMT_MMIO_CPU_EXEC_CTRL, 0, false); + if (ret) + return ret; + + ret = asmedia_wait_reset(pdev); + if (ret) { + dev_err(hcd->self.controller, "Failed post-upload reset\n"); + return ret; + } + + return 0; +} + +int asmedia_xhci_check_request_fw(struct pci_dev *pdev, + const struct pci_device_id *id) +{ + struct xhci_driver_data *driver_data = + (struct xhci_driver_data *)id->driver_data; + const char *fw_name = driver_data->firmware; + const struct firmware *fw; + int ret; + + /* Check if device has firmware, if so skip everything */ + ret = asmedia_check_firmware(pdev); + if (ret < 0) + return ret; + else if (ret == 1) + return 0; + + pci_dev_get(pdev); + ret = request_firmware(&fw, fw_name, &pdev->dev); + pci_dev_put(pdev); + if (ret) { + dev_err(&pdev->dev, "Could not load firmware %s: %d\n", + fw_name, ret); + return ret; + } + + ret = asmedia_load_fw(pdev, fw); + if (ret) { + dev_err(&pdev->dev, "Firmware upload failed: %d\n", ret); + goto err; + } + + ret = asmedia_check_firmware(pdev); + if (ret < 0) { + goto err; + } else if (ret != 1) { + dev_err(&pdev->dev, "Firmware version is too old after upload\n"); + ret = -EIO; + } else { + ret = 0; + } + +err: + release_firmware(fw); + return ret; +} diff --git a/drivers/usb/host/xhci-pci.c b/drivers/usb/host/xhci-pci-core.c similarity index 97% rename from drivers/usb/host/xhci-pci.c rename to drivers/usb/host/xhci-pci-core.c index 585b2f3117b08a..c58cdcdb3cc862 100644 --- a/drivers/usb/host/xhci-pci.c +++ b/drivers/usb/host/xhci-pci-core.c @@ -569,6 +569,18 @@ static int xhci_pci_setup(struct usb_hcd *hcd) struct pci_dev *pdev = to_pci_dev(hcd->self.controller); int retval; u8 sbrn; + struct xhci_driver_data *driver_data; + const struct pci_device_id *id; + + id = pci_match_id(to_pci_driver(pdev->dev.driver)->id_table, pdev); + if (id && id->driver_data && usb_hcd_is_primary_hcd(hcd)) { + driver_data = (struct xhci_driver_data *)id->driver_data; + if (driver_data->quirks & XHCI_ASMEDIA_FW_QUIRK) { + retval = asmedia_xhci_check_request_fw(pdev, id); + if (retval < 0) + return retval; + } + } xhci = hcd_to_xhci(hcd); @@ -938,10 +950,19 @@ static void xhci_pci_shutdown(struct usb_hcd *hcd) pci_set_power_state(pdev, PCI_D3hot); } +#define ASMEDIA_APPLE_FW_NAME "asmedia/asm2214a-apple.bin" + /*-------------------------------------------------------------------------*/ +static const struct xhci_driver_data asmedia_data = { + .quirks = XHCI_ASMEDIA_FW_QUIRK, + .firmware = ASMEDIA_APPLE_FW_NAME, +}; /* PCI driver selection metadata; PCI hotplugging uses this */ static const struct pci_device_id pci_ids[] = { + { PCI_DEVICE(PCI_VENDOR_ID_ASMEDIA, 0x2142), + .driver_data = (unsigned long)&asmedia_data, + }, /* handle any USB 3.0 xHCI controller */ { PCI_DEVICE_CLASS(PCI_CLASS_SERIAL_USB_XHCI, ~0), }, @@ -949,6 +970,10 @@ static const struct pci_device_id pci_ids[] = { }; MODULE_DEVICE_TABLE(pci, pci_ids); +#if IS_ENABLED(CONFIG_USB_XHCI_PCI_ASMEDIA) +MODULE_FIRMWARE(ASMEDIA_APPLE_FW_NAME); +#endif + /* pci driver glue; this is a "new style" PCI driver module */ static struct pci_driver xhci_pci_driver = { .name = hcd_name, diff --git a/drivers/usb/host/xhci-pci.h b/drivers/usb/host/xhci-pci.h index e87c7d9d76b8e2..452908d1c069ba 100644 --- a/drivers/usb/host/xhci-pci.h +++ b/drivers/usb/host/xhci-pci.h @@ -7,4 +7,22 @@ int xhci_pci_common_probe(struct pci_dev *dev, const struct pci_device_id *id); void xhci_pci_remove(struct pci_dev *dev); +struct xhci_driver_data { + u64 quirks; + const char *firmware; +}; + +#if IS_ENABLED(CONFIG_USB_XHCI_PCI_ASMEDIA) +int asmedia_xhci_check_request_fw(struct pci_dev *dev, + const struct pci_device_id *id); + +#else +static inline int asmedia_xhci_check_request_fw(struct pci_dev *dev, + const struct pci_device_id *id) +{ + return 0; +} + +#endif + #endif diff --git a/drivers/usb/host/xhci-ring.c b/drivers/usb/host/xhci-ring.c index 9315ba18310d68..1cbefee3c4cac3 100644 --- a/drivers/usb/host/xhci-ring.c +++ b/drivers/usb/host/xhci-ring.c @@ -3195,6 +3195,7 @@ irqreturn_t xhci_irq(struct usb_hcd *hcd) if (status & STS_HCE) { xhci_warn(xhci, "WARNING: Host Controller Error\n"); + xhci_halt(xhci); goto out; } diff --git a/drivers/usb/host/xhci-sideband.c b/drivers/usb/host/xhci-sideband.c index 2bd77255032b97..65197360613769 100644 --- a/drivers/usb/host/xhci-sideband.c +++ b/drivers/usb/host/xhci-sideband.c @@ -93,8 +93,6 @@ __xhci_sideband_remove_endpoint(struct xhci_sideband *sb, struct xhci_virt_ep *e static void __xhci_sideband_remove_interrupter(struct xhci_sideband *sb) { - struct usb_device *udev; - lockdep_assert_held(&sb->mutex); if (!sb->ir) @@ -102,10 +100,6 @@ __xhci_sideband_remove_interrupter(struct xhci_sideband *sb) xhci_remove_secondary_interrupter(xhci_to_hcd(sb->xhci), sb->ir); sb->ir = NULL; - udev = sb->vdev->udev; - - if (udev->state != USB_STATE_NOTATTACHED) - usb_offload_put(udev); } /* sideband api functions */ @@ -291,8 +285,8 @@ EXPORT_SYMBOL_GPL(xhci_sideband_get_event_buffer); * Allow other drivers, such as usb controller driver, to check if there are * any sideband activity on the host controller. This information could be used * for power management or other forms of resource management. The caller should - * ensure downstream usb devices are all either suspended or marked as - * "offload_at_suspend" to ensure the correctness of the return value. + * ensure downstream usb devices are all marked as "offload_pm_locked" to ensure + * the correctness of the return value. * * Returns true on any active sideband existence, false otherwise. */ @@ -328,9 +322,6 @@ int xhci_sideband_create_interrupter(struct xhci_sideband *sb, int num_seg, bool ip_autoclear, u32 imod_interval, int intr_num) { - int ret = 0; - struct usb_device *udev; - if (!sb || !sb->xhci) return -ENODEV; @@ -348,12 +339,9 @@ xhci_sideband_create_interrupter(struct xhci_sideband *sb, int num_seg, if (!sb->ir) return -ENOMEM; - udev = sb->vdev->udev; - ret = usb_offload_get(udev); - sb->ir->ip_autoclear = ip_autoclear; - return ret; + return 0; } EXPORT_SYMBOL_GPL(xhci_sideband_create_interrupter); diff --git a/drivers/usb/host/xhci-tegra.c b/drivers/usb/host/xhci-tegra.c index 8b492871d21d66..3f6aa2440b05b9 100644 --- a/drivers/usb/host/xhci-tegra.c +++ b/drivers/usb/host/xhci-tegra.c @@ -1570,7 +1570,6 @@ static int tegra_xusb_setup_wakeup(struct platform_device *pdev, struct tegra_xu data = irq_get_irq_data(tegra->wake_irqs[i]); if (!data) { dev_warn(tegra->dev, "get wake event %d irq data fail\n", i); - irq_dispose_mapping(tegra->wake_irqs[i]); break; } @@ -1583,16 +1582,6 @@ static int tegra_xusb_setup_wakeup(struct platform_device *pdev, struct tegra_xu return 0; } -static void tegra_xusb_dispose_wake(struct tegra_xusb *tegra) -{ - unsigned int i; - - for (i = 0; i < tegra->num_wakes; i++) - irq_dispose_mapping(tegra->wake_irqs[i]); - - tegra->num_wakes = 0; -} - static int tegra_xusb_probe(struct platform_device *pdev) { struct tegra_xusb *tegra; @@ -1648,10 +1637,8 @@ static int tegra_xusb_probe(struct platform_device *pdev) return err; tegra->padctl = tegra_xusb_padctl_get(&pdev->dev); - if (IS_ERR(tegra->padctl)) { - err = PTR_ERR(tegra->padctl); - goto dispose_wake; - } + if (IS_ERR(tegra->padctl)) + return PTR_ERR(tegra->padctl); np = of_parse_phandle(pdev->dev.of_node, "nvidia,xusb-padctl", 0); if (!np) { @@ -1975,8 +1962,6 @@ static int tegra_xusb_probe(struct platform_device *pdev) put_padctl: of_node_put(np); tegra_xusb_padctl_put(tegra->padctl); -dispose_wake: - tegra_xusb_dispose_wake(tegra); return err; } @@ -2009,8 +1994,6 @@ static void tegra_xusb_remove(struct platform_device *pdev) if (tegra->padctl_irq) pm_runtime_disable(&pdev->dev); - tegra_xusb_dispose_wake(tegra); - pm_runtime_put(&pdev->dev); tegra_xusb_disable(tegra); diff --git a/drivers/usb/host/xhci.c b/drivers/usb/host/xhci.c index b3ba16b9718cf6..e94cf92e012baf 100644 --- a/drivers/usb/host/xhci.c +++ b/drivers/usb/host/xhci.c @@ -4146,7 +4146,7 @@ int xhci_disable_slot(struct xhci_hcd *xhci, u32 slot_id) if (state == 0xffffffff || (xhci->xhc_state & XHCI_STATE_DYING) || (xhci->xhc_state & XHCI_STATE_HALTED)) { spin_unlock_irqrestore(&xhci->lock, flags); - kfree(command); + xhci_free_command(xhci, command); return -ENODEV; } @@ -4154,7 +4154,7 @@ int xhci_disable_slot(struct xhci_hcd *xhci, u32 slot_id) slot_id); if (ret) { spin_unlock_irqrestore(&xhci->lock, flags); - kfree(command); + xhci_free_command(xhci, command); return ret; } xhci_ring_cmd_db(xhci); diff --git a/drivers/usb/host/xhci.h b/drivers/usb/host/xhci.h index 2b0796f6d00eae..d262671a3803ee 100644 --- a/drivers/usb/host/xhci.h +++ b/drivers/usb/host/xhci.h @@ -1644,6 +1644,7 @@ struct xhci_hcd { #define XHCI_CDNS_SCTX_QUIRK BIT_ULL(48) #define XHCI_ETRON_HOST BIT_ULL(49) #define XHCI_LIMIT_ENDPOINT_INTERVAL_9 BIT_ULL(50) +#define XHCI_ASMEDIA_FW_QUIRK BIT_ULL(51) unsigned int num_active_eps; unsigned int limit_active_eps; diff --git a/drivers/usb/image/mdc800.c b/drivers/usb/image/mdc800.c index 7b7e1554ea20e2..10d72562e4d295 100644 --- a/drivers/usb/image/mdc800.c +++ b/drivers/usb/image/mdc800.c @@ -707,7 +707,7 @@ static ssize_t mdc800_device_read (struct file *file, char __user *buf, size_t l if (signal_pending (current)) { mutex_unlock(&mdc800->io_lock); - return -EINTR; + return len == left ? -EINTR : len-left; } sts=left > (mdc800->out_count-mdc800->out_ptr)?mdc800->out_count-mdc800->out_ptr:left; @@ -730,9 +730,11 @@ static ssize_t mdc800_device_read (struct file *file, char __user *buf, size_t l mutex_unlock(&mdc800->io_lock); return len-left; } - wait_event_timeout(mdc800->download_wait, + retval = wait_event_timeout(mdc800->download_wait, mdc800->downloaded, msecs_to_jiffies(TO_DOWNLOAD_GET_READY)); + if (!retval) + usb_kill_urb(mdc800->download_urb); mdc800->downloaded = 0; if (mdc800->download_urb->status != 0) { diff --git a/drivers/usb/misc/usbio.c b/drivers/usb/misc/usbio.c index 37644dddf157ee..64815f8410acd3 100644 --- a/drivers/usb/misc/usbio.c +++ b/drivers/usb/misc/usbio.c @@ -614,8 +614,10 @@ static int usbio_probe(struct usb_interface *intf, const struct usb_device_id *i usb_fill_bulk_urb(usbio->urb, udev, usbio->rx_pipe, usbio->rxbuf, usbio->rxbuf_len, usbio_bulk_recv, usbio); ret = usb_submit_urb(usbio->urb, GFP_KERNEL); - if (ret) - return dev_err_probe(dev, ret, "Submitting usb urb\n"); + if (ret) { + dev_err_probe(dev, ret, "Submitting usb urb\n"); + goto err_free_urb; + } mutex_lock(&usbio->ctrl_mutex); @@ -663,6 +665,7 @@ static int usbio_probe(struct usb_interface *intf, const struct usb_device_id *i err_unlock: mutex_unlock(&usbio->ctrl_mutex); usb_kill_urb(usbio->urb); +err_free_urb: usb_free_urb(usbio->urb); return ret; diff --git a/drivers/usb/misc/uss720.c b/drivers/usb/misc/uss720.c index b26c1d382d5990..3138f5dca6da48 100644 --- a/drivers/usb/misc/uss720.c +++ b/drivers/usb/misc/uss720.c @@ -736,7 +736,7 @@ static int uss720_probe(struct usb_interface *intf, ret = get_1284_register(pp, 0, ®, GFP_KERNEL); dev_dbg(&intf->dev, "reg: %7ph\n", priv->reg); if (ret < 0) - return ret; + goto probe_abort; ret = usb_find_last_int_in_endpoint(interface, &epd); if (!ret) { diff --git a/drivers/usb/misc/yurex.c b/drivers/usb/misc/yurex.c index 70dff0db5354ff..6d03e689850a60 100644 --- a/drivers/usb/misc/yurex.c +++ b/drivers/usb/misc/yurex.c @@ -272,6 +272,7 @@ static int yurex_probe(struct usb_interface *interface, const struct usb_device_ dev->int_buffer, YUREX_BUF_SIZE, yurex_interrupt, dev, 1); dev->urb->transfer_flags |= URB_NO_TRANSFER_DMA_MAP; + dev->bbu = -1; if (usb_submit_urb(dev->urb, GFP_KERNEL)) { retval = -EIO; dev_err(&interface->dev, "Could not submitting URB\n"); @@ -280,7 +281,6 @@ static int yurex_probe(struct usb_interface *interface, const struct usb_device_ /* save our data pointer in this interface device */ usb_set_intfdata(interface, dev); - dev->bbu = -1; /* we can register the device now, as it is ready */ retval = usb_register_dev(interface, &yurex_class); diff --git a/drivers/usb/renesas_usbhs/common.c b/drivers/usb/renesas_usbhs/common.c index cf4a0367d6d60e..8c93bde4b81673 100644 --- a/drivers/usb/renesas_usbhs/common.c +++ b/drivers/usb/renesas_usbhs/common.c @@ -815,6 +815,15 @@ static void usbhs_remove(struct platform_device *pdev) usbhs_platform_call(priv, hardware_exit, pdev); reset_control_assert(priv->rsts); + + /* + * Explicitly free the IRQ to ensure the interrupt handler is + * disabled and synchronized before freeing resources. + * devm_free_irq() calls free_irq() which waits for any running + * ISR to complete, preventing UAF. + */ + devm_free_irq(&pdev->dev, priv->irq, priv); + usbhs_mod_remove(priv); usbhs_fifo_remove(priv); usbhs_pipe_remove(priv); diff --git a/drivers/usb/roles/class.c b/drivers/usb/roles/class.c index 30482d4cf82678..b7cdc62d420fbe 100644 --- a/drivers/usb/roles/class.c +++ b/drivers/usb/roles/class.c @@ -139,9 +139,14 @@ static void *usb_role_switch_match(const struct fwnode_handle *fwnode, const cha static struct usb_role_switch * usb_role_switch_is_parent(struct fwnode_handle *fwnode) { - struct fwnode_handle *parent = fwnode_get_parent(fwnode); + struct fwnode_handle *parent; struct device *dev; + if (!fwnode_device_is_compatible(fwnode, "usb-b-connector")) + return NULL; + + parent = fwnode_get_parent(fwnode); + if (!fwnode_property_present(parent, "usb-role-switch")) { fwnode_handle_put(parent); return NULL; diff --git a/drivers/usb/serial/io_edgeport.c b/drivers/usb/serial/io_edgeport.c index 1fffda7647f989..ad73040b30c8d7 100644 --- a/drivers/usb/serial/io_edgeport.c +++ b/drivers/usb/serial/io_edgeport.c @@ -73,6 +73,7 @@ static const struct usb_device_id edgeport_4port_id_table[] = { { USB_DEVICE(USB_VENDOR_ID_ION, ION_DEVICE_ID_EDGEPORT_22I) }, { USB_DEVICE(USB_VENDOR_ID_ION, ION_DEVICE_ID_EDGEPORT_412_4) }, { USB_DEVICE(USB_VENDOR_ID_ION, ION_DEVICE_ID_EDGEPORT_COMPATIBLE) }, + { USB_DEVICE(USB_VENDOR_ID_ION, ION_DEVICE_ID_BLACKBOX_IC135A) }, { } }; @@ -121,6 +122,7 @@ static const struct usb_device_id id_table_combined[] = { { USB_DEVICE(USB_VENDOR_ID_ION, ION_DEVICE_ID_EDGEPORT_8R) }, { USB_DEVICE(USB_VENDOR_ID_ION, ION_DEVICE_ID_EDGEPORT_8RR) }, { USB_DEVICE(USB_VENDOR_ID_ION, ION_DEVICE_ID_EDGEPORT_412_8) }, + { USB_DEVICE(USB_VENDOR_ID_ION, ION_DEVICE_ID_BLACKBOX_IC135A) }, { USB_DEVICE(USB_VENDOR_ID_NCR, NCR_DEVICE_ID_EPIC_0202) }, { USB_DEVICE(USB_VENDOR_ID_NCR, NCR_DEVICE_ID_EPIC_0203) }, { USB_DEVICE(USB_VENDOR_ID_NCR, NCR_DEVICE_ID_EPIC_0310) }, @@ -470,6 +472,7 @@ static void get_product_info(struct edgeport_serial *edge_serial) case ION_DEVICE_ID_EDGEPORT_2_DIN: case ION_DEVICE_ID_EDGEPORT_4_DIN: case ION_DEVICE_ID_EDGEPORT_16_DUAL_CPU: + case ION_DEVICE_ID_BLACKBOX_IC135A: product_info->IsRS232 = 1; break; diff --git a/drivers/usb/serial/io_usbvend.h b/drivers/usb/serial/io_usbvend.h index 9a6f742ad3abd0..c82a275e8e76ec 100644 --- a/drivers/usb/serial/io_usbvend.h +++ b/drivers/usb/serial/io_usbvend.h @@ -211,6 +211,7 @@ // // Definitions for other product IDs +#define ION_DEVICE_ID_BLACKBOX_IC135A 0x0801 // OEM device (rebranded Edgeport/4) #define ION_DEVICE_ID_MT4X56USB 0x1403 // OEM device #define ION_DEVICE_ID_E5805A 0x1A01 // OEM device (rebranded Edgeport/4) diff --git a/drivers/usb/serial/option.c b/drivers/usb/serial/option.c index 9f2cc5fb9f4562..5f16ea44084fec 100644 --- a/drivers/usb/serial/option.c +++ b/drivers/usb/serial/option.c @@ -1383,6 +1383,8 @@ static const struct usb_device_id option_ids[] = { .driver_info = NCTRL(2) | RSVD(3) }, { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x1073, 0xff), /* Telit FN990A (ECM) */ .driver_info = NCTRL(0) | RSVD(1) }, + { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x1074, 0xff), /* Telit FN990A (MBIM) */ + .driver_info = NCTRL(5) | RSVD(6) | RSVD(7) }, { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x1075, 0xff), /* Telit FN990A (PCIe) */ .driver_info = RSVD(0) }, { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x1077, 0xff), /* Telit FN990A (rmnet + audio) */ @@ -1401,12 +1403,16 @@ static const struct usb_device_id option_ids[] = { .driver_info = NCTRL(0) | RSVD(1) }, { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x10a0, 0xff), /* Telit FN20C04 (rmnet) */ .driver_info = RSVD(0) | NCTRL(3) }, + { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x10a1, 0xff), /* Telit FN20C04 (RNDIS) */ + .driver_info = NCTRL(4) }, { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x10a2, 0xff), /* Telit FN920C04 (MBIM) */ .driver_info = NCTRL(4) }, { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x10a3, 0xff), /* Telit FN920C04 (ECM) */ .driver_info = NCTRL(4) }, { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x10a4, 0xff), /* Telit FN20C04 (rmnet) */ .driver_info = RSVD(0) | NCTRL(3) }, + { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x10a6, 0xff), /* Telit FN920C04 (RNDIS) */ + .driver_info = NCTRL(4) }, { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x10a7, 0xff), /* Telit FN920C04 (MBIM) */ .driver_info = NCTRL(4) }, { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x10a8, 0xff), /* Telit FN920C04 (ECM) */ @@ -1415,6 +1421,8 @@ static const struct usb_device_id option_ids[] = { .driver_info = RSVD(0) | NCTRL(2) | RSVD(3) | RSVD(4) }, { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x10aa, 0xff), /* Telit FN920C04 (MBIM) */ .driver_info = NCTRL(3) | RSVD(4) | RSVD(5) }, + { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x10ab, 0xff), /* Telit FN920C04 (RNDIS) */ + .driver_info = NCTRL(3) | RSVD(4) | RSVD(5) }, { USB_DEVICE_AND_INTERFACE_INFO(TELIT_VENDOR_ID, 0x10b0, 0xff, 0xff, 0x30), /* Telit FE990B (rmnet) */ .driver_info = NCTRL(5) }, { USB_DEVICE_AND_INTERFACE_INFO(TELIT_VENDOR_ID, 0x10b0, 0xff, 0xff, 0x40) }, @@ -2435,6 +2443,9 @@ static const struct usb_device_id option_ids[] = { { USB_DEVICE_AND_INTERFACE_INFO(0x2dee, 0x4d22, 0xff, 0xff, 0x30) }, /* MeiG Smart SRM815 and SRM825L */ { USB_DEVICE_AND_INTERFACE_INFO(0x2dee, 0x4d22, 0xff, 0xff, 0x40) }, /* MeiG Smart SRM825L */ { USB_DEVICE_AND_INTERFACE_INFO(0x2dee, 0x4d22, 0xff, 0xff, 0x60) }, /* MeiG Smart SRM825L */ + { USB_DEVICE_AND_INTERFACE_INFO(0x2dee, 0x4d38, 0xff, 0xff, 0x30) }, /* MeiG Smart SRM825WN (Diag) */ + { USB_DEVICE_AND_INTERFACE_INFO(0x2dee, 0x4d38, 0xff, 0xff, 0x40) }, /* MeiG Smart SRM825WN (AT) */ + { USB_DEVICE_AND_INTERFACE_INFO(0x2dee, 0x4d38, 0xff, 0xff, 0x60) }, /* MeiG Smart SRM825WN (NMEA) */ { USB_DEVICE_INTERFACE_CLASS(0x2df3, 0x9d03, 0xff) }, /* LongSung M5710 */ { USB_DEVICE_INTERFACE_CLASS(0x305a, 0x1404, 0xff) }, /* GosunCn GM500 RNDIS */ { USB_DEVICE_INTERFACE_CLASS(0x305a, 0x1405, 0xff) }, /* GosunCn GM500 MBIM */ @@ -2455,6 +2466,7 @@ static const struct usb_device_id option_ids[] = { { USB_DEVICE_INTERFACE_CLASS(0x33f8, 0x0302, 0xff) }, /* Rolling RW101R-GL (laptop MBIM) */ { USB_DEVICE_INTERFACE_CLASS(0x33f8, 0x0802, 0xff), /* Rolling RW350-GL (laptop MBIM) */ .driver_info = RSVD(5) }, + { USB_DEVICE_INTERFACE_CLASS(0x33f8, 0x1003, 0xff) }, /* Rolling RW135R-GL (laptop MBIM) */ { USB_DEVICE_AND_INTERFACE_INFO(0x3731, 0x0100, 0xff, 0xff, 0x30) }, /* NetPrisma LCUK54-WWD for Global */ { USB_DEVICE_AND_INTERFACE_INFO(0x3731, 0x0100, 0xff, 0x00, 0x40) }, { USB_DEVICE_AND_INTERFACE_INFO(0x3731, 0x0100, 0xff, 0xff, 0x40) }, diff --git a/drivers/usb/storage/unusual_devs.h b/drivers/usb/storage/unusual_devs.h index 47f50d7a385c8a..255968f9ca42ae 100644 --- a/drivers/usb/storage/unusual_devs.h +++ b/drivers/usb/storage/unusual_devs.h @@ -2350,10 +2350,11 @@ UNUSUAL_DEV( 0x2027, 0xa001, 0x0000, 0x9999, US_FL_SCM_MULT_TARG ), /* - * Reported by DocMAX - * and Thomas Weißschuh + * Reported by DocMAX , + * Thomas Weißschuh + * and Daniel Brát */ -UNUSUAL_DEV( 0x2109, 0x0715, 0x9999, 0x9999, +UNUSUAL_DEV( 0x2109, 0x0715, 0x0000, 0x9999, "VIA Labs, Inc.", "VL817 SATA Bridge", USB_SC_DEVICE, USB_PR_DEVICE, NULL, diff --git a/drivers/usb/typec/altmodes/displayport.c b/drivers/usb/typec/altmodes/displayport.c index d96ab106a980bd..8b51b32ec5d458 100644 --- a/drivers/usb/typec/altmodes/displayport.c +++ b/drivers/usb/typec/altmodes/displayport.c @@ -100,9 +100,14 @@ static int dp_altmode_configure(struct dp_altmode *dp, u8 con) { u8 pin_assign = 0; u32 conf; + u32 signal; /* DP Signalling */ - conf = (dp->data.conf & DP_CONF_SIGNALLING_MASK) >> DP_CONF_SIGNALLING_SHIFT; + signal = DP_CAP_DP_SIGNALLING(dp->port->vdo) & DP_CAP_DP_SIGNALLING(dp->alt->vdo); + if (dp->plug_prime) + signal &= DP_CAP_DP_SIGNALLING(dp->plug_prime->vdo); + + conf = signal << DP_CONF_SIGNALLING_SHIFT; switch (con) { case DP_STATUS_CON_DISABLED: diff --git a/drivers/usb/typec/altmodes/thunderbolt.c b/drivers/usb/typec/altmodes/thunderbolt.c index 6eadf7835f8f6c..d09dd09cf1c314 100644 --- a/drivers/usb/typec/altmodes/thunderbolt.c +++ b/drivers/usb/typec/altmodes/thunderbolt.c @@ -39,28 +39,7 @@ static bool tbt_ready(struct typec_altmode *alt); static int tbt_enter_mode(struct tbt_altmode *tbt) { - struct typec_altmode *plug = tbt->plug[TYPEC_PLUG_SOP_P]; - u32 vdo; - - vdo = tbt->alt->vdo & (TBT_VENDOR_SPECIFIC_B0 | TBT_VENDOR_SPECIFIC_B1); - vdo |= tbt->alt->vdo & TBT_INTEL_SPECIFIC_B0; - vdo |= TBT_MODE; - - if (plug) { - if (typec_cable_is_active(tbt->cable)) - vdo |= TBT_ENTER_MODE_ACTIVE_CABLE; - - vdo |= TBT_ENTER_MODE_CABLE_SPEED(TBT_CABLE_SPEED(plug->vdo)); - vdo |= plug->vdo & TBT_CABLE_ROUNDED; - vdo |= plug->vdo & TBT_CABLE_OPTICAL; - vdo |= plug->vdo & TBT_CABLE_RETIMER; - vdo |= plug->vdo & TBT_CABLE_LINK_TRAINING; - } else { - vdo |= TBT_ENTER_MODE_CABLE_SPEED(TBT_CABLE_USB3_PASSIVE); - } - - tbt->enter_vdo = vdo; - return typec_altmode_enter(tbt->alt, &vdo); + return typec_altmode_enter(tbt->alt, &tbt->enter_vdo); } static void tbt_altmode_work(struct work_struct *work) @@ -337,6 +316,7 @@ static bool tbt_ready(struct typec_altmode *alt) { struct tbt_altmode *tbt = typec_altmode_get_drvdata(alt); struct typec_altmode *plug; + u32 vdo; if (tbt->cable) return true; @@ -364,6 +344,26 @@ static bool tbt_ready(struct typec_altmode *alt) tbt->plug[i] = plug; } + vdo = tbt->alt->vdo & (TBT_VENDOR_SPECIFIC_B0 | TBT_VENDOR_SPECIFIC_B1); + vdo |= tbt->alt->vdo & TBT_INTEL_SPECIFIC_B0; + vdo |= TBT_MODE; + plug = tbt->plug[TYPEC_PLUG_SOP_P]; + + if (plug) { + if (typec_cable_is_active(tbt->cable)) + vdo |= TBT_ENTER_MODE_ACTIVE_CABLE; + + vdo |= TBT_ENTER_MODE_CABLE_SPEED(TBT_CABLE_SPEED(plug->vdo)); + vdo |= plug->vdo & TBT_CABLE_ROUNDED; + vdo |= plug->vdo & TBT_CABLE_OPTICAL; + vdo |= plug->vdo & TBT_CABLE_RETIMER; + vdo |= plug->vdo & TBT_CABLE_LINK_TRAINING; + } else { + vdo |= TBT_ENTER_MODE_CABLE_SPEED(TBT_CABLE_USB3_PASSIVE); + } + + tbt->enter_vdo = vdo; + return true; } diff --git a/drivers/usb/typec/tcpm/fusb302.c b/drivers/usb/typec/tcpm/fusb302.c index 870a71f953f6cd..5b1f2750cfc3bb 100644 --- a/drivers/usb/typec/tcpm/fusb302.c +++ b/drivers/usb/typec/tcpm/fusb302.c @@ -1755,9 +1755,9 @@ static int fusb302_probe(struct i2c_client *client) goto destroy_workqueue; } - ret = request_irq(chip->gpio_int_n_irq, fusb302_irq_intn, - IRQF_ONESHOT | IRQF_TRIGGER_LOW, - "fsc_interrupt_int_n", chip); + ret = request_threaded_irq(chip->gpio_int_n_irq, NULL, fusb302_irq_intn, + IRQF_ONESHOT | IRQF_TRIGGER_LOW, + "fsc_interrupt_int_n", chip); if (ret < 0) { dev_err(dev, "cannot request IRQ for GPIO Int_N, ret=%d", ret); goto tcpm_unregister_port; diff --git a/drivers/usb/typec/tcpm/tcpm.c b/drivers/usb/typec/tcpm/tcpm.c index be49a976428fc4..4ca2746ce16bc0 100644 --- a/drivers/usb/typec/tcpm/tcpm.c +++ b/drivers/usb/typec/tcpm/tcpm.c @@ -7890,7 +7890,7 @@ struct tcpm_port *tcpm_register_port(struct device *dev, struct tcpc_dev *tcpc) port->partner_desc.identity = &port->partner_ident; port->role_sw = fwnode_usb_role_switch_get(tcpc->fwnode); - if (IS_ERR_OR_NULL(port->role_sw)) + if (!port->role_sw) port->role_sw = usb_role_switch_get(port->dev); if (IS_ERR(port->role_sw)) { err = PTR_ERR(port->role_sw); diff --git a/drivers/usb/typec/ucsi/Kconfig b/drivers/usb/typec/ucsi/Kconfig index b812be4d0e674e..87dd992a4b9e9a 100644 --- a/drivers/usb/typec/ucsi/Kconfig +++ b/drivers/usb/typec/ucsi/Kconfig @@ -73,7 +73,6 @@ config CROS_EC_UCSI tristate "UCSI Driver for ChromeOS EC" depends on MFD_CROS_EC_DEV depends on CROS_USBPD_NOTIFY - depends on !EXTCON_TCSS_CROS_EC default MFD_CROS_EC_DEV help This driver enables UCSI support for a ChromeOS EC. The EC is diff --git a/drivers/usb/typec/ucsi/psy.c b/drivers/usb/typec/ucsi/psy.c index 3abe9370ffaaf0..62160c41917180 100644 --- a/drivers/usb/typec/ucsi/psy.c +++ b/drivers/usb/typec/ucsi/psy.c @@ -112,15 +112,20 @@ static int ucsi_psy_get_voltage_max(struct ucsi_connector *con, union power_supply_propval *val) { u32 pdo; + int max_voltage = 0; switch (UCSI_CONSTAT(con, PWR_OPMODE)) { case UCSI_CONSTAT_PWR_OPMODE_PD: - if (con->num_pdos > 0) { - pdo = con->src_pdos[con->num_pdos - 1]; - val->intval = pdo_fixed_voltage(pdo) * 1000; - } else { - val->intval = 0; + for (int i = 0; i < con->num_pdos; i++) { + int pdo_voltage = 0; + + pdo = con->src_pdos[i]; + if (pdo_type(pdo) == PDO_TYPE_FIXED) + pdo_voltage = pdo_fixed_voltage(pdo) * 1000; + max_voltage = (pdo_voltage > max_voltage) ? pdo_voltage + : max_voltage; } + val->intval = max_voltage; break; case UCSI_CONSTAT_PWR_OPMODE_TYPEC3_0: case UCSI_CONSTAT_PWR_OPMODE_TYPEC1_5: @@ -168,6 +173,7 @@ static int ucsi_psy_get_current_max(struct ucsi_connector *con, union power_supply_propval *val) { u32 pdo; + int max_current = 0; if (!UCSI_CONSTAT(con, CONNECTED)) { val->intval = 0; @@ -176,12 +182,16 @@ static int ucsi_psy_get_current_max(struct ucsi_connector *con, switch (UCSI_CONSTAT(con, PWR_OPMODE)) { case UCSI_CONSTAT_PWR_OPMODE_PD: - if (con->num_pdos > 0) { - pdo = con->src_pdos[con->num_pdos - 1]; - val->intval = pdo_max_current(pdo) * 1000; - } else { - val->intval = 0; + for (int i = 0; i < con->num_pdos; i++) { + int pdo_current = 0; + + pdo = con->src_pdos[i]; + if (pdo_type(pdo) == PDO_TYPE_FIXED) + pdo_current = pdo_max_current(pdo) * 1000; + max_current = (pdo_current > max_current) ? pdo_current + : max_current; } + val->intval = max_current; break; case UCSI_CONSTAT_PWR_OPMODE_TYPEC1_5: val->intval = UCSI_TYPEC_1_5_CURRENT * 1000; diff --git a/drivers/usb/typec/ucsi/ucsi.c b/drivers/usb/typec/ucsi/ucsi.c index a7b388dc7fa0fa..7df3a7b94a40f5 100644 --- a/drivers/usb/typec/ucsi/ucsi.c +++ b/drivers/usb/typec/ucsi/ucsi.c @@ -42,8 +42,14 @@ void ucsi_notify_common(struct ucsi *ucsi, u32 cci) if (cci & UCSI_CCI_BUSY) return; - if (UCSI_CCI_CONNECTOR(cci)) - ucsi_connector_change(ucsi, UCSI_CCI_CONNECTOR(cci)); + if (UCSI_CCI_CONNECTOR(cci)) { + if (!ucsi->cap.num_connectors || + UCSI_CCI_CONNECTOR(cci) <= ucsi->cap.num_connectors) + ucsi_connector_change(ucsi, UCSI_CCI_CONNECTOR(cci)); + else + dev_err(ucsi->dev, "bogus connector number in CCI: %lu\n", + UCSI_CCI_CONNECTOR(cci)); + } if (cci & UCSI_CCI_ACK_COMPLETE && test_and_clear_bit(ACK_PENDING, &ucsi->flags)) diff --git a/drivers/usb/usbip/usbip_common.c b/drivers/usb/usbip/usbip_common.c index a2b2da1255dda0..ba9e7c616e1294 100644 --- a/drivers/usb/usbip/usbip_common.c +++ b/drivers/usb/usbip/usbip_common.c @@ -470,6 +470,18 @@ static void usbip_pack_ret_submit(struct usbip_header *pdu, struct urb *urb, urb->status = rpdu->status; urb->actual_length = rpdu->actual_length; urb->start_frame = rpdu->start_frame; + /* + * The number_of_packets field determines the length of + * iso_frame_desc[], which is a flexible array allocated + * at URB creation time. A response must never claim more + * packets than originally submitted; doing so would cause + * an out-of-bounds write in usbip_recv_iso() and + * usbip_pad_iso(). Clamp to zero on violation so both + * functions safely return early. + */ + if (rpdu->number_of_packets < 0 || + rpdu->number_of_packets > urb->number_of_packets) + rpdu->number_of_packets = 0; urb->number_of_packets = rpdu->number_of_packets; urb->error_count = rpdu->error_count; } diff --git a/drivers/vdpa/mlx5/net/mlx5_vnet.c b/drivers/vdpa/mlx5/net/mlx5_vnet.c index ddaa1366704bb0..44062e9d68f006 100644 --- a/drivers/vdpa/mlx5/net/mlx5_vnet.c +++ b/drivers/vdpa/mlx5/net/mlx5_vnet.c @@ -3640,9 +3640,6 @@ static int mlx5_set_group_asid(struct vdpa_device *vdev, u32 group, struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); int err = 0; - if (group >= MLX5_VDPA_NUMVQ_GROUPS) - return -EINVAL; - mvdev->mres.group2asid[group] = asid; mutex_lock(&mvdev->mres.lock); diff --git a/drivers/vdpa/vdpa_sim/vdpa_sim.c b/drivers/vdpa/vdpa_sim/vdpa_sim.c index c1c6431950e1b1..df9c7ddc5d7827 100644 --- a/drivers/vdpa/vdpa_sim/vdpa_sim.c +++ b/drivers/vdpa/vdpa_sim/vdpa_sim.c @@ -606,12 +606,6 @@ static int vdpasim_set_group_asid(struct vdpa_device *vdpa, unsigned int group, struct vhost_iotlb *iommu; int i; - if (group > vdpasim->dev_attr.ngroups) - return -EINVAL; - - if (asid >= vdpasim->dev_attr.nas) - return -EINVAL; - iommu = &vdpasim->iommu[asid]; mutex_lock(&vdpasim->mutex); diff --git a/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c b/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c index cf45f6370c3698..e61df3fe0db99a 100644 --- a/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c +++ b/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c @@ -426,7 +426,7 @@ static int vf_qm_check_match(struct hisi_acc_vf_core_device *hisi_acc_vdev, ret = qm_get_vft(vf_qm, &vf_qm->qp_base); if (ret <= 0) { dev_err(dev, "failed to get vft qp nums\n"); - return ret; + return ret < 0 ? ret : -EINVAL; } if (ret != vf_data->qp_num) { @@ -1188,12 +1188,34 @@ hisi_acc_vfio_pci_get_device_state(struct vfio_device *vdev, return 0; } +static void hisi_acc_vf_pci_reset_prepare(struct pci_dev *pdev) +{ + struct hisi_acc_vf_core_device *hisi_acc_vdev = hisi_acc_drvdata(pdev); + struct hisi_qm *qm = hisi_acc_vdev->pf_qm; + struct device *dev = &qm->pdev->dev; + u32 delay = 0; + + /* All reset requests need to be queued for processing */ + while (test_and_set_bit(QM_RESETTING, &qm->misc_ctl)) { + msleep(1); + if (++delay > QM_RESET_WAIT_TIMEOUT) { + dev_err(dev, "reset prepare failed\n"); + return; + } + } + + hisi_acc_vdev->set_reset_flag = true; +} + static void hisi_acc_vf_pci_aer_reset_done(struct pci_dev *pdev) { struct hisi_acc_vf_core_device *hisi_acc_vdev = hisi_acc_drvdata(pdev); + struct hisi_qm *qm = hisi_acc_vdev->pf_qm; + + if (hisi_acc_vdev->set_reset_flag) + clear_bit(QM_RESETTING, &qm->misc_ctl); - if (hisi_acc_vdev->core_device.vdev.migration_flags != - VFIO_MIGRATION_STOP_COPY) + if (!hisi_acc_vdev->core_device.vdev.mig_ops) return; mutex_lock(&hisi_acc_vdev->state_mutex); @@ -1547,6 +1569,7 @@ static int hisi_acc_vfio_pci_open_device(struct vfio_device *core_vdev) } hisi_acc_vdev->mig_state = VFIO_DEVICE_STATE_RUNNING; hisi_acc_vdev->dev_opened = true; + hisi_acc_vdev->match_done = 0; mutex_unlock(&hisi_acc_vdev->open_mutex); } @@ -1734,6 +1757,7 @@ static const struct pci_device_id hisi_acc_vfio_pci_table[] = { MODULE_DEVICE_TABLE(pci, hisi_acc_vfio_pci_table); static const struct pci_error_handlers hisi_acc_vf_err_handlers = { + .reset_prepare = hisi_acc_vf_pci_reset_prepare, .reset_done = hisi_acc_vf_pci_aer_reset_done, .error_detected = vfio_pci_core_aer_err_detected, }; diff --git a/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.h b/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.h index cd55eba64dfb23..a3d91a31e3d886 100644 --- a/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.h +++ b/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.h @@ -27,6 +27,7 @@ #define ERROR_CHECK_TIMEOUT 100 #define CHECK_DELAY_TIME 100 +#define QM_RESET_WAIT_TIMEOUT 60000 #define QM_SQC_VFT_BASE_SHIFT_V2 28 #define QM_SQC_VFT_BASE_MASK_V2 GENMASK(15, 0) @@ -128,6 +129,7 @@ struct hisi_acc_vf_migration_file { struct hisi_acc_vf_core_device { struct vfio_pci_core_device core_device; u8 match_done; + bool set_reset_flag; /* * io_base is only valid when dev_opened is true, * which is protected by open_mutex. diff --git a/drivers/vfio/pci/vfio_pci_core.c b/drivers/vfio/pci/vfio_pci_core.c index 3a11e6f450f701..72c33b399800eb 100644 --- a/drivers/vfio/pci/vfio_pci_core.c +++ b/drivers/vfio/pci/vfio_pci_core.c @@ -588,6 +588,7 @@ EXPORT_SYMBOL_GPL(vfio_pci_core_enable); void vfio_pci_core_disable(struct vfio_pci_core_device *vdev) { + struct pci_dev *bridge; struct pci_dev *pdev = vdev->pdev; struct vfio_pci_dummy_resource *dummy_res, *tmp; struct vfio_pci_ioeventfd *ioeventfd, *ioeventfd_tmp; @@ -694,12 +695,20 @@ void vfio_pci_core_disable(struct vfio_pci_core_device *vdev) * We can not use the "try" reset interface here, which will * overwrite the previously restored configuration information. */ - if (vdev->reset_works && pci_dev_trylock(pdev)) { - if (!__pci_reset_function_locked(pdev)) - vdev->needs_reset = false; - pci_dev_unlock(pdev); + if (vdev->reset_works) { + bridge = pci_upstream_bridge(pdev); + if (bridge && !pci_dev_trylock(bridge)) + goto out_restore_state; + if (pci_dev_trylock(pdev)) { + if (!__pci_reset_function_locked(pdev)) + vdev->needs_reset = false; + pci_dev_unlock(pdev); + } + if (bridge) + pci_dev_unlock(bridge); } +out_restore_state: pci_restore_state(pdev); out: pci_disable_device(pdev); diff --git a/drivers/vfio/pci/vfio_pci_dmabuf.c b/drivers/vfio/pci/vfio_pci_dmabuf.c index 4be4a85005cbcb..2229f6939cfeb9 100644 --- a/drivers/vfio/pci/vfio_pci_dmabuf.c +++ b/drivers/vfio/pci/vfio_pci_dmabuf.c @@ -302,11 +302,10 @@ int vfio_pci_core_feature_dma_buf(struct vfio_pci_core_device *vdev, u32 flags, */ ret = dma_buf_fd(priv->dmabuf, get_dma_buf.open_flags); if (ret < 0) - goto err_dma_buf; + dma_buf_put(priv->dmabuf); + return ret; -err_dma_buf: - dma_buf_put(priv->dmabuf); err_dev_put: vfio_device_put_registration(&vdev->vdev); err_free_phys: diff --git a/drivers/vfio/pci/xe/main.c b/drivers/vfio/pci/xe/main.c index 2a5eb9260ec7b5..ef698ca1565e79 100644 --- a/drivers/vfio/pci/xe/main.c +++ b/drivers/vfio/pci/xe/main.c @@ -454,39 +454,46 @@ static const struct vfio_migration_ops xe_vfio_pci_migration_ops = { static void xe_vfio_pci_migration_init(struct xe_vfio_pci_core_device *xe_vdev) { struct vfio_device *core_vdev = &xe_vdev->core_device.vdev; - struct pci_dev *pdev = to_pci_dev(core_vdev->dev); - struct xe_device *xe = xe_sriov_vfio_get_pf(pdev); - if (!xe) + if (!xe_sriov_vfio_migration_supported(xe_vdev->xe)) return; - if (!xe_sriov_vfio_migration_supported(xe)) - return; - - mutex_init(&xe_vdev->state_mutex); - spin_lock_init(&xe_vdev->reset_lock); - - /* PF internal control uses vfid index starting from 1 */ - xe_vdev->vfid = pci_iov_vf_id(pdev) + 1; - xe_vdev->xe = xe; core_vdev->migration_flags = VFIO_MIGRATION_STOP_COPY | VFIO_MIGRATION_P2P; core_vdev->mig_ops = &xe_vfio_pci_migration_ops; } -static void xe_vfio_pci_migration_fini(struct xe_vfio_pci_core_device *xe_vdev) +static int xe_vfio_pci_vf_init(struct xe_vfio_pci_core_device *xe_vdev) { - if (!xe_vdev->vfid) - return; + struct vfio_device *core_vdev = &xe_vdev->core_device.vdev; + struct pci_dev *pdev = to_pci_dev(core_vdev->dev); + struct xe_device *xe = xe_sriov_vfio_get_pf(pdev); - mutex_destroy(&xe_vdev->state_mutex); + if (!pdev->is_virtfn) + return 0; + if (!xe) + return -ENODEV; + xe_vdev->xe = xe; + + /* PF internal control uses vfid index starting from 1 */ + xe_vdev->vfid = pci_iov_vf_id(pdev) + 1; + + xe_vfio_pci_migration_init(xe_vdev); + + return 0; } static int xe_vfio_pci_init_dev(struct vfio_device *core_vdev) { struct xe_vfio_pci_core_device *xe_vdev = container_of(core_vdev, struct xe_vfio_pci_core_device, core_device.vdev); + int ret; - xe_vfio_pci_migration_init(xe_vdev); + mutex_init(&xe_vdev->state_mutex); + spin_lock_init(&xe_vdev->reset_lock); + + ret = xe_vfio_pci_vf_init(xe_vdev); + if (ret) + return ret; return vfio_pci_core_init_dev(core_vdev); } @@ -496,7 +503,7 @@ static void xe_vfio_pci_release_dev(struct vfio_device *core_vdev) struct xe_vfio_pci_core_device *xe_vdev = container_of(core_vdev, struct xe_vfio_pci_core_device, core_device.vdev); - xe_vfio_pci_migration_fini(xe_vdev); + mutex_destroy(&xe_vdev->state_mutex); } static const struct vfio_device_ops xe_vfio_pci_ops = { diff --git a/drivers/vhost/vdpa.c b/drivers/vhost/vdpa.c index 05a481e4c385a0..7e51eec842b8cc 100644 --- a/drivers/vhost/vdpa.c +++ b/drivers/vhost/vdpa.c @@ -680,7 +680,7 @@ static long vhost_vdpa_vring_ioctl(struct vhost_vdpa *v, unsigned int cmd, case VHOST_VDPA_SET_GROUP_ASID: if (copy_from_user(&s, argp, sizeof(s))) return -EFAULT; - if (s.num >= vdpa->nas) + if (idx >= vdpa->ngroups || s.num >= vdpa->nas) return -EINVAL; if (!ops->set_group_asid) return -EOPNOTSUPP; @@ -1527,6 +1527,7 @@ static int vhost_vdpa_mmap(struct file *file, struct vm_area_struct *vma) if (vma->vm_end - vma->vm_start != notify.size) return -ENOTSUPP; + vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); vm_flags_set(vma, VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP); vma->vm_ops = &vhost_vdpa_vm_ops; return 0; diff --git a/drivers/video/backlight/aw99706.c b/drivers/video/backlight/aw99706.c index df5b23b2f75340..938f352aaab7f8 100644 --- a/drivers/video/backlight/aw99706.c +++ b/drivers/video/backlight/aw99706.c @@ -12,7 +12,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/drivers/video/backlight/qcom-wled.c b/drivers/video/backlight/qcom-wled.c index a63bb42c8f8b03..8054e4787725ef 100644 --- a/drivers/video/backlight/qcom-wled.c +++ b/drivers/video/backlight/qcom-wled.c @@ -1244,6 +1244,15 @@ static const struct wled_var_cfg wled4_ovp_cfg = { .size = ARRAY_SIZE(wled4_ovp_values), }; +static const u32 pmi8994_wled_ovp_values[] = { + 31000, 29500, 19400, 17800, +}; + +static const struct wled_var_cfg pmi8994_wled_ovp_cfg = { + .values = pmi8994_wled_ovp_values, + .size = ARRAY_SIZE(pmi8994_wled_ovp_values), +}; + static inline u32 wled5_ovp_values_fn(u32 idx) { /* @@ -1357,6 +1366,29 @@ static int wled_configure(struct wled *wled) }, }; + const struct wled_u32_opts pmi8994_wled_opts[] = { + { + .name = "qcom,current-boost-limit", + .val_ptr = &cfg->boost_i_limit, + .cfg = &wled4_boost_i_limit_cfg, + }, + { + .name = "qcom,current-limit-microamp", + .val_ptr = &cfg->string_i_limit, + .cfg = &wled4_string_i_limit_cfg, + }, + { + .name = "qcom,ovp-millivolt", + .val_ptr = &cfg->ovp, + .cfg = &pmi8994_wled_ovp_cfg, + }, + { + .name = "qcom,switching-freq", + .val_ptr = &cfg->switch_freq, + .cfg = &wled3_switch_freq_cfg, + }, + }; + const struct wled_u32_opts wled5_opts[] = { { .name = "qcom,current-boost-limit", @@ -1423,8 +1455,14 @@ static int wled_configure(struct wled *wled) break; case 4: - u32_opts = wled4_opts; - size = ARRAY_SIZE(wled4_opts); + if (of_device_is_compatible(dev->of_node, "qcom,pmi8950-wled") || + of_device_is_compatible(dev->of_node, "qcom,pmi8994-wled")) { + u32_opts = pmi8994_wled_opts; + size = ARRAY_SIZE(pmi8994_wled_opts); + } else { + u32_opts = wled4_opts; + size = ARRAY_SIZE(wled4_opts); + } *cfg = wled4_config_defaults; wled->wled_set_brightness = wled4_set_brightness; wled->wled_sync_toggle = wled3_sync_toggle; diff --git a/drivers/video/fbdev/au1200fb.c b/drivers/video/fbdev/au1200fb.c index ed770222660b51..685e629e7e1642 100644 --- a/drivers/video/fbdev/au1200fb.c +++ b/drivers/video/fbdev/au1200fb.c @@ -1724,8 +1724,10 @@ static int au1200fb_drv_probe(struct platform_device *dev) /* Now hook interrupt too */ irq = platform_get_irq(dev, 0); - if (irq < 0) - return irq; + if (irq < 0) { + ret = irq; + goto failed; + } ret = request_irq(irq, au1200fb_handle_irq, IRQF_SHARED, "lcd", (void *)dev); diff --git a/drivers/video/fbdev/core/fbcon.c b/drivers/video/fbdev/core/fbcon.c index 7be9e865325d95..a07737dcb45adc 100644 --- a/drivers/video/fbdev/core/fbcon.c +++ b/drivers/video/fbdev/core/fbcon.c @@ -1068,7 +1068,8 @@ static void fbcon_init(struct vc_data *vc, bool init) return; if (!info->fbcon_par) - con2fb_acquire_newinfo(vc, info, vc->vc_num); + if (con2fb_acquire_newinfo(vc, info, vc->vc_num)) + return; /* If we are not the first console on this fb, copy the font from that console */ diff --git a/drivers/video/fbdev/core/fbcon.h b/drivers/video/fbdev/core/fbcon.h index 44ea4ae4bba0d1..9cabafd2d91ca6 100644 --- a/drivers/video/fbdev/core/fbcon.h +++ b/drivers/video/fbdev/core/fbcon.h @@ -30,7 +30,6 @@ struct fbcon_display { #ifdef CONFIG_FRAMEBUFFER_CONSOLE_LEGACY_ACCELERATION u_short scrollmode; /* Scroll Method, use fb_scrollmode() */ #endif - u_short inverse; /* != 0 text black on white as default */ short yscroll; /* Hardware scrolling */ int vrows; /* number of virtual rows */ int cursor_shape; diff --git a/drivers/video/fbdev/core/fbsysfs.c b/drivers/video/fbdev/core/fbsysfs.c index b8344c40073b41..baa2bae0fb5b30 100644 --- a/drivers/video/fbdev/core/fbsysfs.c +++ b/drivers/video/fbdev/core/fbsysfs.c @@ -12,8 +12,6 @@ #include "fb_internal.h" -#define FB_SYSFS_FLAG_ATTR 1 - static int activate(struct fb_info *fb_info, struct fb_var_screeninfo *var) { int err; @@ -451,33 +449,7 @@ static struct attribute *fb_device_attrs[] = { NULL, }; -static const struct attribute_group fb_device_attr_group = { - .attrs = fb_device_attrs, -}; - -static int fb_init_device(struct fb_info *fb_info) -{ - int ret; - - dev_set_drvdata(fb_info->dev, fb_info); - - fb_info->class_flag |= FB_SYSFS_FLAG_ATTR; - - ret = device_add_group(fb_info->dev, &fb_device_attr_group); - if (ret) - fb_info->class_flag &= ~FB_SYSFS_FLAG_ATTR; - - return 0; -} - -static void fb_cleanup_device(struct fb_info *fb_info) -{ - if (fb_info->class_flag & FB_SYSFS_FLAG_ATTR) { - device_remove_group(fb_info->dev, &fb_device_attr_group); - - fb_info->class_flag &= ~FB_SYSFS_FLAG_ATTR; - } -} +ATTRIBUTE_GROUPS(fb_device); int fb_device_create(struct fb_info *fb_info) { @@ -485,14 +457,13 @@ int fb_device_create(struct fb_info *fb_info) dev_t devt = MKDEV(FB_MAJOR, node); int ret; - fb_info->dev = device_create(fb_class, fb_info->device, devt, NULL, "fb%d", node); + fb_info->dev = device_create_with_groups(fb_class, fb_info->device, devt, fb_info, + fb_device_groups, "fb%d", node); if (IS_ERR(fb_info->dev)) { /* Not fatal */ ret = PTR_ERR(fb_info->dev); pr_warn("Unable to create device for framebuffer %d; error %d\n", node, ret); fb_info->dev = NULL; - } else { - fb_init_device(fb_info); } return 0; @@ -505,7 +476,6 @@ void fb_device_destroy(struct fb_info *fb_info) if (!fb_info->dev) return; - fb_cleanup_device(fb_info); device_destroy(fb_class, devt); fb_info->dev = NULL; } diff --git a/drivers/video/fbdev/ffb.c b/drivers/video/fbdev/ffb.c index 34b6abff9493e3..da531b4cb4513f 100644 --- a/drivers/video/fbdev/ffb.c +++ b/drivers/video/fbdev/ffb.c @@ -335,6 +335,9 @@ struct ffb_dac { }; #define FFB_DAC_UCTRL 0x1001 /* User Control */ +#define FFB_DAC_UCTRL_OVENAB 0x00000008 /* Overlay Enable */ +#define FFB_DAC_UCTRL_WMODE 0x00000030 /* Window Mode */ +#define FFB_DAC_UCTRL_WM_COMB 0x00000000 /* Window Mode = Combined */ #define FFB_DAC_UCTRL_MANREV 0x00000f00 /* 4-bit Manufacturing Revision */ #define FFB_DAC_UCTRL_MANREV_SHIFT 8 #define FFB_DAC_TGEN 0x6000 /* Timing Generator */ @@ -425,7 +428,7 @@ static void ffb_switch_from_graph(struct ffb_par *par) { struct ffb_fbc __iomem *fbc = par->fbc; struct ffb_dac __iomem *dac = par->dac; - unsigned long flags; + unsigned long flags, uctrl; spin_lock_irqsave(&par->lock, flags); FFBWait(par); @@ -450,6 +453,15 @@ static void ffb_switch_from_graph(struct ffb_par *par) upa_writel((FFB_DAC_CUR_CTRL_P0 | FFB_DAC_CUR_CTRL_P1), &dac->value2); + /* Disable overlay and window modes. */ + upa_writel(FFB_DAC_UCTRL, &dac->type); + uctrl = upa_readl(&dac->value); + uctrl &= ~FFB_DAC_UCTRL_WMODE; + uctrl |= FFB_DAC_UCTRL_WM_COMB; + uctrl &= ~FFB_DAC_UCTRL_OVENAB; + upa_writel(FFB_DAC_UCTRL, &dac->type); + upa_writel(uctrl, &dac->value); + spin_unlock_irqrestore(&par->lock, flags); } diff --git a/drivers/video/fbdev/riva/riva_hw.c b/drivers/video/fbdev/riva/riva_hw.c index 8b829b7200642f..f292079566cfca 100644 --- a/drivers/video/fbdev/riva/riva_hw.c +++ b/drivers/video/fbdev/riva/riva_hw.c @@ -436,6 +436,9 @@ static char nv3_arb(nv3_fifo_info * res_info, nv3_sim_state * state, nv3_arb_in vmisses = 2; eburst_size = state->memory_width * 1; mburst_size = 32; + if (!state->mclk_khz) + return (0); + gns = 1000000 * (gmisses*state->mem_page_miss + state->mem_latency)/state->mclk_khz; ainfo->by_gfacc = gns*ainfo->gdrain_rate/1000000; ainfo->wcmocc = 0; diff --git a/drivers/video/fbdev/smscufx.c b/drivers/video/fbdev/smscufx.c index 5f0dd01fd83495..891ce7b76d637e 100644 --- a/drivers/video/fbdev/smscufx.c +++ b/drivers/video/fbdev/smscufx.c @@ -932,7 +932,6 @@ static int ufx_ops_ioctl(struct fb_info *info, unsigned int cmd, unsigned long arg) { struct ufx_data *dev = info->par; - struct dloarea *area = NULL; if (!atomic_read(&dev->usb_active)) return 0; @@ -947,6 +946,10 @@ static int ufx_ops_ioctl(struct fb_info *info, unsigned int cmd, /* TODO: Help propose a standard fb.h ioctl to report mmap damage */ if (cmd == UFX_IOCTL_REPORT_DAMAGE) { + struct dloarea *area __free(kfree) = kmalloc(sizeof(*area), GFP_KERNEL); + if (!area) + return -ENOMEM; + /* If we have a damage-aware client, turn fb_defio "off" * To avoid perf imact of unnecessary page fault handling. * Done by resetting the delay for this fb_info to a very @@ -956,7 +959,8 @@ static int ufx_ops_ioctl(struct fb_info *info, unsigned int cmd, if (info->fbdefio) info->fbdefio->delay = UFX_DEFIO_WRITE_DISABLE; - area = (struct dloarea *)arg; + if (copy_from_user(area, (u8 __user *)arg, sizeof(*area))) + return -EFAULT; if (area->x < 0) area->x = 0; diff --git a/drivers/video/fbdev/tdfxfb.c b/drivers/video/fbdev/tdfxfb.c index 51ebe78359ec34..531fb8478e2071 100644 --- a/drivers/video/fbdev/tdfxfb.c +++ b/drivers/video/fbdev/tdfxfb.c @@ -496,6 +496,9 @@ static int tdfxfb_check_var(struct fb_var_screeninfo *var, struct fb_info *info) } } + if (!var->pixclock) + return -EINVAL; + if (PICOS2KHZ(var->pixclock) > par->max_pixclock) { DPRINTK("pixclock too high (%ldKHz)\n", PICOS2KHZ(var->pixclock)); diff --git a/drivers/video/fbdev/udlfb.c b/drivers/video/fbdev/udlfb.c index ccede85df1e1af..28e6d75e13ed35 100644 --- a/drivers/video/fbdev/udlfb.c +++ b/drivers/video/fbdev/udlfb.c @@ -1018,6 +1018,9 @@ static int dlfb_ops_check_var(struct fb_var_screeninfo *var, struct fb_videomode mode; struct dlfb_data *dlfb = info->par; + if (!var->pixclock) + return -EINVAL; + /* set device-specific elements of var unrelated to mode */ dlfb_var_color_format(var); diff --git a/drivers/video/fbdev/vt8500lcdfb.c b/drivers/video/fbdev/vt8500lcdfb.c index b08a6fdc53fd2f..85c7a99a7d6482 100644 --- a/drivers/video/fbdev/vt8500lcdfb.c +++ b/drivers/video/fbdev/vt8500lcdfb.c @@ -369,7 +369,7 @@ static int vt8500lcd_probe(struct platform_device *pdev) if (fbi->palette_cpu == NULL) { dev_err(&pdev->dev, "Failed to allocate palette buffer\n"); ret = -ENOMEM; - goto failed_free_io; + goto failed_free_mem_virt; } irq = platform_get_irq(pdev, 0); @@ -432,6 +432,9 @@ static int vt8500lcd_probe(struct platform_device *pdev) failed_free_palette: dma_free_coherent(&pdev->dev, fbi->palette_size, fbi->palette_cpu, fbi->palette_phys); +failed_free_mem_virt: + dma_free_coherent(&pdev->dev, fbi->fb.fix.smem_len, + fbi->fb.screen_buffer, fbi->fb.fix.smem_start); failed_free_io: iounmap(fbi->regbase); failed_free_res: diff --git a/drivers/video/of_display_timing.c b/drivers/video/of_display_timing.c index bebd371c6b93ea..a6ec392253c3ee 100644 --- a/drivers/video/of_display_timing.c +++ b/drivers/video/of_display_timing.c @@ -181,7 +181,7 @@ struct display_timings *of_get_display_timings(const struct device_node *np) if (disp->num_timings == 0) { /* should never happen, as entry was already found above */ pr_err("%pOF: no timings specified\n", np); - goto entryfail; + goto timingfail; } disp->timings = kcalloc(disp->num_timings, @@ -189,13 +189,13 @@ struct display_timings *of_get_display_timings(const struct device_node *np) GFP_KERNEL); if (!disp->timings) { pr_err("%pOF: could not allocate timings array\n", np); - goto entryfail; + goto timingfail; } disp->num_timings = 0; disp->native_mode = 0; - for_each_child_of_node(timings_np, entry) { + for_each_child_of_node_scoped(timings_np, child) { struct display_timing *dt; int r; @@ -206,7 +206,7 @@ struct display_timings *of_get_display_timings(const struct device_node *np) goto timingfail; } - r = of_parse_display_timing(entry, dt); + r = of_parse_display_timing(child, dt); if (r) { /* * to not encourage wrong devicetrees, fail in case of @@ -218,7 +218,7 @@ struct display_timings *of_get_display_timings(const struct device_node *np) goto timingfail; } - if (native_mode == entry) + if (native_mode == child) disp->native_mode = disp->num_timings; disp->timings[disp->num_timings] = dt; diff --git a/drivers/virt/coco/tdx-guest/tdx-guest.c b/drivers/virt/coco/tdx-guest/tdx-guest.c index 4e239ec960c9b5..40b53b4da72813 100644 --- a/drivers/virt/coco/tdx-guest/tdx-guest.c +++ b/drivers/virt/coco/tdx-guest/tdx-guest.c @@ -169,6 +169,8 @@ static void tdx_mr_deinit(const struct attribute_group *mr_grp) #define GET_QUOTE_SUCCESS 0 #define GET_QUOTE_IN_FLIGHT 0xffffffffffffffff +#define TDX_QUOTE_MAX_LEN (GET_QUOTE_BUF_SIZE - sizeof(struct tdx_quote_buf)) + /* struct tdx_quote_buf: Format of Quote request buffer. * @version: Quote format version, filled by TD. * @status: Status code of Quote request, filled by VMM. @@ -267,6 +269,7 @@ static int tdx_report_new_locked(struct tsm_report *report, void *data) u8 *buf; struct tdx_quote_buf *quote_buf = quote_data; struct tsm_report_desc *desc = &report->desc; + u32 out_len; int ret; u64 err; @@ -304,12 +307,17 @@ static int tdx_report_new_locked(struct tsm_report *report, void *data) return ret; } - buf = kvmemdup(quote_buf->data, quote_buf->out_len, GFP_KERNEL); + out_len = READ_ONCE(quote_buf->out_len); + + if (out_len > TDX_QUOTE_MAX_LEN) + return -EFBIG; + + buf = kvmemdup(quote_buf->data, out_len, GFP_KERNEL); if (!buf) return -ENOMEM; report->outblob = buf; - report->outblob_len = quote_buf->out_len; + report->outblob_len = out_len; /* * TODO: parse the PEM-formatted cert chain out of the quote buffer when diff --git a/drivers/watchdog/apple_wdt.c b/drivers/watchdog/apple_wdt.c index 66a158f67a712b..6b9b0f9b05cedf 100644 --- a/drivers/watchdog/apple_wdt.c +++ b/drivers/watchdog/apple_wdt.c @@ -218,6 +218,7 @@ static int apple_wdt_suspend(struct device *dev) static DEFINE_SIMPLE_DEV_PM_OPS(apple_wdt_pm_ops, apple_wdt_suspend, apple_wdt_resume); static const struct of_device_id apple_wdt_of_match[] = { + { .compatible = "apple,t8103-wdt" }, { .compatible = "apple,wdt" }, {}, }; diff --git a/drivers/watchdog/imx7ulp_wdt.c b/drivers/watchdog/imx7ulp_wdt.c index 0f13a30533574a..03479110453ce7 100644 --- a/drivers/watchdog/imx7ulp_wdt.c +++ b/drivers/watchdog/imx7ulp_wdt.c @@ -346,6 +346,7 @@ static int imx7ulp_wdt_probe(struct platform_device *pdev) watchdog_stop_on_reboot(wdog); watchdog_stop_on_unregister(wdog); watchdog_set_drvdata(wdog, imx7ulp_wdt); + watchdog_set_nowayout(wdog, nowayout); imx7ulp_wdt->hw = of_device_get_match_data(dev); ret = imx7ulp_wdt_init(imx7ulp_wdt, wdog->timeout * imx7ulp_wdt->hw->wdog_clock_rate); diff --git a/drivers/watchdog/it87_wdt.c b/drivers/watchdog/it87_wdt.c index 3b8488c86a2f3a..1d9f8591f38d87 100644 --- a/drivers/watchdog/it87_wdt.c +++ b/drivers/watchdog/it87_wdt.c @@ -188,6 +188,12 @@ static void _wdt_update_timeout(unsigned int t) superio_outb(t >> 8, WDTVALMSB); } +/* Internal function, should be called after superio_select(GPIO) */ +static bool _wdt_running(void) +{ + return superio_inb(WDTVALLSB) || (max_units > 255 && superio_inb(WDTVALMSB)); +} + static int wdt_update_timeout(unsigned int t) { int ret; @@ -374,6 +380,12 @@ static int __init it87_wdt_init(void) } } + /* wdt already left running by firmware? */ + if (_wdt_running()) { + pr_info("Left running by firmware.\n"); + set_bit(WDOG_HW_RUNNING, &wdt_dev.status); + } + superio_exit(); if (timeout < 1 || timeout > max_units * 60) { diff --git a/drivers/watchdog/rzv2h_wdt.c b/drivers/watchdog/rzv2h_wdt.c index a694786837e114..f9bb4ef3d327bf 100644 --- a/drivers/watchdog/rzv2h_wdt.c +++ b/drivers/watchdog/rzv2h_wdt.c @@ -270,9 +270,7 @@ static int rzt2h_wdt_wdtdcr_init(struct platform_device *pdev, rzt2h_wdt_wdtdcr_count_stop(priv); - ret = pm_runtime_put(&pdev->dev); - if (ret < 0) - return ret; + pm_runtime_put(&pdev->dev); return 0; } diff --git a/drivers/watchdog/starfive-wdt.c b/drivers/watchdog/starfive-wdt.c index ed71d3960a0f26..af55adc4a3c69b 100644 --- a/drivers/watchdog/starfive-wdt.c +++ b/drivers/watchdog/starfive-wdt.c @@ -446,7 +446,7 @@ static int starfive_wdt_probe(struct platform_device *pdev) platform_set_drvdata(pdev, wdt); pm_runtime_enable(&pdev->dev); if (pm_runtime_enabled(&pdev->dev)) { - ret = pm_runtime_get_sync(&pdev->dev); + ret = pm_runtime_resume_and_get(&pdev->dev); if (ret < 0) return ret; } else { diff --git a/drivers/xen/balloon.c b/drivers/xen/balloon.c index 49c3f992639435..8c44a25a7d2b95 100644 --- a/drivers/xen/balloon.c +++ b/drivers/xen/balloon.c @@ -724,6 +724,7 @@ static int __init balloon_add_regions(void) static int __init balloon_init(void) { struct task_struct *task; + unsigned long current_pages; int rc; if (!xen_domain()) @@ -731,12 +732,18 @@ static int __init balloon_init(void) pr_info("Initialising balloon driver\n"); - if (xen_released_pages >= get_num_physpages()) { - WARN(1, "Released pages underflow current target"); - return -ERANGE; + if (xen_pv_domain()) { + if (xen_released_pages >= xen_start_info->nr_pages) + goto underflow; + current_pages = min(xen_start_info->nr_pages - + xen_released_pages, max_pfn); + } else { + if (xen_unpopulated_pages >= get_num_physpages()) + goto underflow; + current_pages = get_num_physpages() - xen_unpopulated_pages; } - balloon_stats.current_pages = get_num_physpages() - xen_released_pages; + balloon_stats.current_pages = current_pages; balloon_stats.target_pages = balloon_stats.current_pages; balloon_stats.balloon_low = 0; balloon_stats.balloon_high = 0; @@ -767,6 +774,10 @@ static int __init balloon_init(void) xen_balloon_init(); return 0; + + underflow: + WARN(1, "Released pages underflow current target"); + return -ERANGE; } subsys_initcall(balloon_init); diff --git a/drivers/xen/grant-dma-ops.c b/drivers/xen/grant-dma-ops.c index 14077d23f2a19e..c2603e70017863 100644 --- a/drivers/xen/grant-dma-ops.c +++ b/drivers/xen/grant-dma-ops.c @@ -366,7 +366,8 @@ static int xen_grant_init_backend_domid(struct device *dev, if (np) { ret = xen_dt_grant_init_backend_domid(dev, np, backend_domid); of_node_put(np); - } else if (IS_ENABLED(CONFIG_XEN_VIRTIO_FORCE_GRANT) || xen_pv_domain()) { + } else if (!xen_initial_domain() && + (IS_ENABLED(CONFIG_XEN_VIRTIO_FORCE_GRANT) || xen_pv_domain())) { dev_info(dev, "Using dom0 as backend\n"); *backend_domid = 0; ret = 0; diff --git a/drivers/xen/privcmd.c b/drivers/xen/privcmd.c index f52a457b302d9c..cbc62f0df11b75 100644 --- a/drivers/xen/privcmd.c +++ b/drivers/xen/privcmd.c @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -30,7 +31,10 @@ #include #include #include +#include +#include #include +#include #include #include @@ -46,6 +50,7 @@ #include #include #include +#include #ifdef CONFIG_XEN_ACPI #include #endif @@ -68,10 +73,20 @@ module_param_named(dm_op_buf_max_size, privcmd_dm_op_buf_max_size, uint, MODULE_PARM_DESC(dm_op_buf_max_size, "Maximum size of a dm_op hypercall buffer"); +static bool unrestricted; +module_param(unrestricted, bool, 0); +MODULE_PARM_DESC(unrestricted, + "Don't restrict hypercalls to target domain if running in a domU"); + struct privcmd_data { domid_t domid; }; +/* DOMID_INVALID implies no restriction */ +static domid_t target_domain = DOMID_INVALID; +static bool restrict_wait; +static DECLARE_WAIT_QUEUE_HEAD(restrict_wait_wq); + static int privcmd_vma_range_is_mapped( struct vm_area_struct *vma, unsigned long addr, @@ -1562,13 +1577,16 @@ static long privcmd_ioctl(struct file *file, static int privcmd_open(struct inode *ino, struct file *file) { - struct privcmd_data *data = kzalloc(sizeof(*data), GFP_KERNEL); + struct privcmd_data *data; + if (wait_event_interruptible(restrict_wait_wq, !restrict_wait) < 0) + return -EINTR; + + data = kzalloc(sizeof(*data), GFP_KERNEL); if (!data) return -ENOMEM; - /* DOMID_INVALID implies no restriction */ - data->domid = DOMID_INVALID; + data->domid = target_domain; file->private_data = data; return 0; @@ -1661,6 +1679,52 @@ static struct miscdevice privcmd_dev = { .fops = &xen_privcmd_fops, }; +static int init_restrict(struct notifier_block *notifier, + unsigned long event, + void *data) +{ + char *target; + unsigned int domid; + + /* Default to an guaranteed unused domain-id. */ + target_domain = DOMID_IDLE; + + target = xenbus_read(XBT_NIL, "target", "", NULL); + if (IS_ERR(target) || kstrtouint(target, 10, &domid)) { + pr_err("No target domain found, blocking all hypercalls\n"); + goto out; + } + + target_domain = domid; + + out: + if (!IS_ERR(target)) + kfree(target); + + restrict_wait = false; + wake_up_all(&restrict_wait_wq); + + return NOTIFY_DONE; +} + +static struct notifier_block xenstore_notifier = { + .notifier_call = init_restrict, +}; + +static void __init restrict_driver(void) +{ + if (unrestricted) { + if (security_locked_down(LOCKDOWN_XEN_USER_ACTIONS)) + pr_warn("Kernel is locked down, parameter \"unrestricted\" ignored\n"); + else + return; + } + + restrict_wait = true; + + register_xenstore_notifier(&xenstore_notifier); +} + static int __init privcmd_init(void) { int err; @@ -1668,6 +1732,9 @@ static int __init privcmd_init(void) if (!xen_domain()) return -ENODEV; + if (!xen_initial_domain()) + restrict_driver(); + err = misc_register(&privcmd_dev); if (err != 0) { pr_err("Could not register Xen privcmd device\n"); @@ -1697,6 +1764,9 @@ static int __init privcmd_init(void) static void __exit privcmd_exit(void) { + if (!xen_initial_domain()) + unregister_xenstore_notifier(&xenstore_notifier); + privcmd_ioeventfd_exit(); privcmd_irqfd_exit(); misc_deregister(&privcmd_dev); diff --git a/drivers/xen/unpopulated-alloc.c b/drivers/xen/unpopulated-alloc.c index d6fc2aefe2646b..1dc0b495c8e597 100644 --- a/drivers/xen/unpopulated-alloc.c +++ b/drivers/xen/unpopulated-alloc.c @@ -18,6 +18,9 @@ static unsigned int list_count; static struct resource *target_resource; +/* Pages to subtract from the memory count when setting balloon target. */ +unsigned long xen_unpopulated_pages __initdata; + /* * If arch is not happy with system "iomem_resource" being used for * the region allocation it can provide it's own view by creating specific diff --git a/drivers/xen/xen-acpi-processor.c b/drivers/xen/xen-acpi-processor.c index f2e8eaf684ba6e..8d1860bd5d578b 100644 --- a/drivers/xen/xen-acpi-processor.c +++ b/drivers/xen/xen-acpi-processor.c @@ -379,11 +379,8 @@ read_acpi_id(acpi_handle handle, u32 lvl, void *context, void **rv) acpi_psd[acpi_id].domain); } - status = acpi_evaluate_object(handle, "_CST", NULL, &buffer); - if (ACPI_FAILURE(status)) { - if (!pblk) - return AE_OK; - } + if (!pblk && !acpi_has_method(handle, "_CST")) + return AE_OK; /* .. and it has a C-state */ __set_bit(acpi_id, acpi_id_cst_present); diff --git a/drivers/xen/xenbus/xenbus_probe_frontend.c b/drivers/xen/xenbus/xenbus_probe_frontend.c index 6d1819269cbe53..199917b6f77ca9 100644 --- a/drivers/xen/xenbus/xenbus_probe_frontend.c +++ b/drivers/xen/xenbus/xenbus_probe_frontend.c @@ -148,11 +148,9 @@ static void xenbus_frontend_dev_shutdown(struct device *_dev) } static const struct dev_pm_ops xenbus_pm_ops = { - .suspend = xenbus_dev_suspend, - .resume = xenbus_frontend_dev_resume, .freeze = xenbus_dev_suspend, .thaw = xenbus_dev_cancel, - .restore = xenbus_dev_resume, + .restore = xenbus_frontend_dev_resume, }; static struct xen_bus_type xenbus_frontend = { diff --git a/fs/afs/addr_list.c b/fs/afs/addr_list.c index e941da5b6dd92d..b1704de3d95f55 100644 --- a/fs/afs/addr_list.c +++ b/fs/afs/addr_list.c @@ -298,8 +298,8 @@ int afs_merge_fs_addr4(struct afs_net *net, struct afs_addr_list *alist, srx.transport.sin.sin_addr.s_addr = xdr; peer = rxrpc_kernel_lookup_peer(net->socket, &srx, GFP_KERNEL); - if (!peer) - return -ENOMEM; + if (IS_ERR(peer)) + return PTR_ERR(peer); for (i = 0; i < alist->nr_ipv4; i++) { if (peer == alist->addrs[i].peer) { @@ -342,8 +342,8 @@ int afs_merge_fs_addr6(struct afs_net *net, struct afs_addr_list *alist, memcpy(&srx.transport.sin6.sin6_addr, xdr, 16); peer = rxrpc_kernel_lookup_peer(net->socket, &srx, GFP_KERNEL); - if (!peer) - return -ENOMEM; + if (IS_ERR(peer)) + return PTR_ERR(peer); for (i = alist->nr_ipv4; i < alist->nr_addrs; i++) { if (peer == alist->addrs[i].peer) { diff --git a/fs/binfmt_elf_fdpic.c b/fs/binfmt_elf_fdpic.c index 48fd2de3bca052..a3d4e6973b299b 100644 --- a/fs/binfmt_elf_fdpic.c +++ b/fs/binfmt_elf_fdpic.c @@ -595,6 +595,12 @@ static int create_elf_fdpic_tables(struct linux_binprm *bprm, #ifdef ELF_HWCAP2 nitems++; #endif +#ifdef ELF_HWCAP3 + nitems++; +#endif +#ifdef ELF_HWCAP4 + nitems++; +#endif csp = sp; sp -= nitems * 2 * sizeof(unsigned long); diff --git a/fs/btrfs/bio.c b/fs/btrfs/bio.c index fa1d321a2fb838..1de1b408c6a6d7 100644 --- a/fs/btrfs/bio.c +++ b/fs/btrfs/bio.c @@ -480,6 +480,8 @@ static void btrfs_clone_write_end_io(struct bio *bio) static void btrfs_submit_dev_bio(struct btrfs_device *dev, struct bio *bio) { + u64 physical = bio->bi_iter.bi_sector << SECTOR_SHIFT; + if (!dev || !dev->bdev || test_bit(BTRFS_DEV_STATE_MISSING, &dev->dev_state) || (btrfs_op(bio) == BTRFS_MAP_WRITE && @@ -494,12 +496,13 @@ static void btrfs_submit_dev_bio(struct btrfs_device *dev, struct bio *bio) * For zone append writing, bi_sector must point the beginning of the * zone */ - if (bio_op(bio) == REQ_OP_ZONE_APPEND) { - u64 physical = bio->bi_iter.bi_sector << SECTOR_SHIFT; + if (btrfs_bio(bio)->can_use_append && btrfs_dev_is_sequential(dev, physical)) { u64 zone_start = round_down(physical, dev->fs_info->zone_size); ASSERT(btrfs_dev_is_sequential(dev, physical)); bio->bi_iter.bi_sector = zone_start >> SECTOR_SHIFT; + bio->bi_opf &= ~REQ_OP_WRITE; + bio->bi_opf |= REQ_OP_ZONE_APPEND; } btrfs_debug(dev->fs_info, "%s: rw %d 0x%x, sector=%llu, dev=%lu (%s id %llu), size=%u", @@ -747,7 +750,6 @@ static bool btrfs_submit_chunk(struct btrfs_bio *bbio, int mirror_num) u64 logical = bio->bi_iter.bi_sector << SECTOR_SHIFT; u64 length = bio->bi_iter.bi_size; u64 map_length = length; - bool use_append = btrfs_use_zone_append(bbio); struct btrfs_io_context *bioc = NULL; struct btrfs_io_stripe smap; blk_status_t status; @@ -775,8 +777,10 @@ static bool btrfs_submit_chunk(struct btrfs_bio *bbio, int mirror_num) if (bio_op(bio) == REQ_OP_WRITE && is_data_bbio(bbio)) bbio->orig_logical = logical; + bbio->can_use_append = btrfs_use_zone_append(bbio); + map_length = min(map_length, length); - if (use_append) + if (bbio->can_use_append) map_length = btrfs_append_map_length(bbio, map_length); if (map_length < length) { @@ -805,11 +809,6 @@ static bool btrfs_submit_chunk(struct btrfs_bio *bbio, int mirror_num) } if (btrfs_op(bio) == BTRFS_MAP_WRITE) { - if (use_append) { - bio->bi_opf &= ~REQ_OP_WRITE; - bio->bi_opf |= REQ_OP_ZONE_APPEND; - } - if (is_data_bbio(bbio) && bioc && bioc->use_rst) { /* * No locking for the list update, as we only add to @@ -836,7 +835,7 @@ static bool btrfs_submit_chunk(struct btrfs_bio *bbio, int mirror_num) status = errno_to_blk_status(ret); if (status) goto fail; - } else if (use_append || + } else if (bbio->can_use_append || (btrfs_is_zoned(fs_info) && inode && inode->flags & BTRFS_INODE_NODATASUM)) { ret = btrfs_alloc_dummy_sum(bbio); @@ -935,7 +934,6 @@ int btrfs_repair_io_failure(struct btrfs_fs_info *fs_info, u64 ino, u64 fileoff, struct bio *bio = NULL; int ret = 0; - ASSERT(!(fs_info->sb->s_flags & SB_RDONLY)); BUG_ON(!mirror_num); /* Basic alignment checks. */ @@ -947,6 +945,13 @@ int btrfs_repair_io_failure(struct btrfs_fs_info *fs_info, u64 ino, u64 fileoff, ASSERT(step <= length); ASSERT(is_power_of_2(step)); + /* + * The fs either mounted RO or hit critical errors, no need + * to continue repairing. + */ + if (unlikely(sb_rdonly(fs_info->sb))) + return 0; + if (btrfs_repair_one_zone(fs_info, logical)) return 0; diff --git a/fs/btrfs/bio.h b/fs/btrfs/bio.h index 1be74209f0b8db..246c7519dff39c 100644 --- a/fs/btrfs/bio.h +++ b/fs/btrfs/bio.h @@ -92,6 +92,9 @@ struct btrfs_bio { /* Whether the csum generation for data write is async. */ bool async_csum; + /* Whether the bio is written using zone append. */ + bool can_use_append; + /* * This member must come last, bio_alloc_bioset will allocate enough * bytes for entire btrfs_bio but relies on bio being last. diff --git a/fs/btrfs/block-group.c b/fs/btrfs/block-group.c index 08b14449fabeba..4b73ccefcbcbaa 100644 --- a/fs/btrfs/block-group.c +++ b/fs/btrfs/block-group.c @@ -1872,6 +1872,7 @@ void btrfs_reclaim_bgs_work(struct work_struct *work) while (!list_empty(&fs_info->reclaim_bgs)) { u64 used; u64 reserved; + u64 old_total; int ret = 0; bg = list_first_entry(&fs_info->reclaim_bgs, @@ -1937,6 +1938,7 @@ void btrfs_reclaim_bgs_work(struct work_struct *work) } spin_unlock(&bg->lock); + old_total = space_info->total_bytes; spin_unlock(&space_info->lock); /* @@ -1989,14 +1991,14 @@ void btrfs_reclaim_bgs_work(struct work_struct *work) reserved = 0; spin_lock(&space_info->lock); space_info->reclaim_errors++; - if (READ_ONCE(space_info->periodic_reclaim)) - space_info->periodic_reclaim_ready = false; spin_unlock(&space_info->lock); } spin_lock(&space_info->lock); space_info->reclaim_count++; space_info->reclaim_bytes += used; space_info->reclaim_bytes += reserved; + if (space_info->total_bytes < old_total) + btrfs_set_periodic_reclaim_ready(space_info, true); spin_unlock(&space_info->lock); next: @@ -3675,6 +3677,14 @@ int btrfs_write_dirty_block_groups(struct btrfs_trans_handle *trans) return ret; } +static void btrfs_maybe_reset_size_class(struct btrfs_block_group *bg) +{ + lockdep_assert_held(&bg->lock); + if (btrfs_block_group_should_use_size_class(bg) && + bg->used == 0 && bg->reserved == 0) + bg->size_class = BTRFS_BG_SZ_NONE; +} + int btrfs_update_block_group(struct btrfs_trans_handle *trans, u64 bytenr, u64 num_bytes, bool alloc) { @@ -3739,6 +3749,7 @@ int btrfs_update_block_group(struct btrfs_trans_handle *trans, old_val -= num_bytes; cache->used = old_val; cache->pinned += num_bytes; + btrfs_maybe_reset_size_class(cache); btrfs_space_info_update_bytes_pinned(space_info, num_bytes); space_info->bytes_used -= num_bytes; space_info->disk_used -= num_bytes * factor; @@ -3867,6 +3878,7 @@ void btrfs_free_reserved_bytes(struct btrfs_block_group *cache, u64 num_bytes, spin_lock(&cache->lock); bg_ro = cache->ro; cache->reserved -= num_bytes; + btrfs_maybe_reset_size_class(cache); if (is_delalloc) cache->delalloc_bytes -= num_bytes; spin_unlock(&cache->lock); @@ -4454,7 +4466,7 @@ static void check_removing_space_info(struct btrfs_space_info *space_info) for (int i = 0; i < BTRFS_SPACE_INFO_SUB_GROUP_MAX; i++) { if (space_info->sub_group[i]) { check_removing_space_info(space_info->sub_group[i]); - kfree(space_info->sub_group[i]); + btrfs_sysfs_remove_space_info(space_info->sub_group[i]); space_info->sub_group[i] = NULL; } } diff --git a/fs/btrfs/block-rsv.c b/fs/btrfs/block-rsv.c index 96cf7a1629870f..52fb7d9425917e 100644 --- a/fs/btrfs/block-rsv.c +++ b/fs/btrfs/block-rsv.c @@ -276,10 +276,11 @@ u64 btrfs_block_rsv_release(struct btrfs_fs_info *fs_info, struct btrfs_block_rsv *target = NULL; /* - * If we are a delayed block reserve then push to the global rsv, - * otherwise dump into the global delayed reserve if it is not full. + * If we are a delayed refs block reserve then push to the global + * reserve, otherwise dump into the global delayed refs reserve if it is + * not full. */ - if (block_rsv->type == BTRFS_BLOCK_RSV_DELOPS) + if (block_rsv->type == BTRFS_BLOCK_RSV_DELREFS) target = global_rsv; else if (block_rsv != global_rsv && !btrfs_block_rsv_full(delayed_rsv)) target = delayed_rsv; diff --git a/fs/btrfs/delayed-inode.c b/fs/btrfs/delayed-inode.c index 4b7d9015e0dade..7e3d294a6dced9 100644 --- a/fs/btrfs/delayed-inode.c +++ b/fs/btrfs/delayed-inode.c @@ -1673,7 +1673,7 @@ int btrfs_delete_delayed_dir_index(struct btrfs_trans_handle *trans, if (unlikely(ret)) { btrfs_err(trans->fs_info, "failed to add delayed dir index item, root: %llu, inode: %llu, index: %llu, error: %d", - index, btrfs_root_id(node->root), node->inode_id, ret); + btrfs_root_id(node->root), node->inode_id, index, ret); btrfs_delayed_item_release_metadata(dir->root, item); btrfs_release_delayed_item(item); } diff --git a/fs/btrfs/direct-io.c b/fs/btrfs/direct-io.c index 07e19e88ba4b3f..5443d69efe956a 100644 --- a/fs/btrfs/direct-io.c +++ b/fs/btrfs/direct-io.c @@ -814,6 +814,8 @@ ssize_t btrfs_direct_write(struct kiocb *iocb, struct iov_iter *from) ssize_t ret; unsigned int ilock_flags = 0; struct iomap_dio *dio; + const u64 data_profile = btrfs_data_alloc_profile(fs_info) & + BTRFS_BLOCK_GROUP_PROFILE_MASK; if (iocb->ki_flags & IOCB_NOWAIT) ilock_flags |= BTRFS_ILOCK_TRY; @@ -827,6 +829,16 @@ ssize_t btrfs_direct_write(struct kiocb *iocb, struct iov_iter *from) if (iocb->ki_pos + iov_iter_count(from) <= i_size_read(inode) && IS_NOSEC(inode)) ilock_flags |= BTRFS_ILOCK_SHARED; + /* + * If our data profile has duplication (either extra mirrors or RAID56), + * we can not trust the direct IO buffer, the content may change during + * writeback and cause different contents written to different mirrors. + * + * Thus only RAID0 and SINGLE can go true zero-copy direct IO. + */ + if (data_profile != BTRFS_BLOCK_GROUP_RAID0 && data_profile != 0) + goto buffered; + relock: ret = btrfs_inode_lock(BTRFS_I(inode), ilock_flags); if (ret < 0) diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c index 2833b44f4b4f22..8df7eb7f01e90c 100644 --- a/fs/btrfs/disk-io.c +++ b/fs/btrfs/disk-io.c @@ -2503,8 +2503,8 @@ int btrfs_validate_super(const struct btrfs_fs_info *fs_info, if (mirror_num >= 0 && btrfs_super_bytenr(sb) != btrfs_sb_offset(mirror_num)) { - btrfs_err(fs_info, "super offset mismatch %llu != %u", - btrfs_super_bytenr(sb), BTRFS_SUPER_INFO_OFFSET); + btrfs_err(fs_info, "super offset mismatch %llu != %llu", + btrfs_super_bytenr(sb), btrfs_sb_offset(mirror_num)); ret = -EINVAL; } @@ -3150,7 +3150,7 @@ int btrfs_check_features(struct btrfs_fs_info *fs_info, bool is_rw_mount) if (incompat & ~BTRFS_FEATURE_INCOMPAT_SUPP) { btrfs_err(fs_info, "cannot mount because of unknown incompat features (0x%llx)", - incompat); + incompat & ~BTRFS_FEATURE_INCOMPAT_SUPP); return -EINVAL; } @@ -3182,7 +3182,7 @@ int btrfs_check_features(struct btrfs_fs_info *fs_info, bool is_rw_mount) if (compat_ro_unsupp && is_rw_mount) { btrfs_err(fs_info, "cannot mount read-write because of unknown compat_ro features (0x%llx)", - compat_ro); + compat_ro_unsupp); return -EINVAL; } @@ -3195,7 +3195,7 @@ int btrfs_check_features(struct btrfs_fs_info *fs_info, bool is_rw_mount) !btrfs_test_opt(fs_info, NOLOGREPLAY)) { btrfs_err(fs_info, "cannot replay dirty log with unsupported compat_ro features (0x%llx), try rescue=nologreplay", - compat_ro); + compat_ro_unsupp); return -EINVAL; } diff --git a/fs/btrfs/extent-tree.c b/fs/btrfs/extent-tree.c index e4cae34620d199..f5ca544e354316 100644 --- a/fs/btrfs/extent-tree.c +++ b/fs/btrfs/extent-tree.c @@ -476,7 +476,7 @@ static noinline int lookup_extent_data_ref(struct btrfs_trans_handle *trans, btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); if (key.objectid != bytenr || key.type != BTRFS_EXTENT_DATA_REF_KEY) - goto fail; + return -ENOENT; ref = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_data_ref); @@ -487,12 +487,11 @@ static noinline int lookup_extent_data_ref(struct btrfs_trans_handle *trans, btrfs_release_path(path); goto again; } - ret = 0; - break; + return 0; } path->slots[0]++; } -fail: + return ret; } @@ -1761,32 +1760,36 @@ static int run_one_delayed_ref(struct btrfs_trans_handle *trans, struct btrfs_delayed_extent_op *extent_op, bool insert_reserved) { + struct btrfs_fs_info *fs_info = trans->fs_info; int ret = 0; if (TRANS_ABORTED(trans)) { if (insert_reserved) { btrfs_pin_extent(trans, node->bytenr, node->num_bytes); - free_head_ref_squota_rsv(trans->fs_info, href); + free_head_ref_squota_rsv(fs_info, href); } return 0; } if (node->type == BTRFS_TREE_BLOCK_REF_KEY || - node->type == BTRFS_SHARED_BLOCK_REF_KEY) + node->type == BTRFS_SHARED_BLOCK_REF_KEY) { ret = run_delayed_tree_ref(trans, href, node, extent_op, insert_reserved); - else if (node->type == BTRFS_EXTENT_DATA_REF_KEY || - node->type == BTRFS_SHARED_DATA_REF_KEY) + } else if (node->type == BTRFS_EXTENT_DATA_REF_KEY || + node->type == BTRFS_SHARED_DATA_REF_KEY) { ret = run_delayed_data_ref(trans, href, node, extent_op, insert_reserved); - else if (node->type == BTRFS_EXTENT_OWNER_REF_KEY) + } else if (node->type == BTRFS_EXTENT_OWNER_REF_KEY) { ret = 0; - else - BUG(); + } else { + ret = -EUCLEAN; + btrfs_err(fs_info, "unexpected delayed ref node type: %u", node->type); + } + if (ret && insert_reserved) btrfs_pin_extent(trans, node->bytenr, node->num_bytes); if (ret < 0) - btrfs_err(trans->fs_info, + btrfs_err(fs_info, "failed to run delayed ref for logical %llu num_bytes %llu type %u action %u ref_mod %d: %d", node->bytenr, node->num_bytes, node->type, node->action, node->ref_mod, ret); @@ -2470,7 +2473,7 @@ static int __btrfs_mod_ref(struct btrfs_trans_handle *trans, int i; int action; int level; - int ret = 0; + int ret; if (btrfs_is_testing(fs_info)) return 0; @@ -2522,7 +2525,7 @@ static int __btrfs_mod_ref(struct btrfs_trans_handle *trans, else ret = btrfs_free_extent(trans, &ref); if (ret) - goto fail; + return ret; } else { /* We don't know the owning_root, leave as 0. */ ref.bytenr = btrfs_node_blockptr(buf, i); @@ -2535,12 +2538,10 @@ static int __btrfs_mod_ref(struct btrfs_trans_handle *trans, else ret = btrfs_free_extent(trans, &ref); if (ret) - goto fail; + return ret; } } return 0; -fail: - return ret; } int btrfs_inc_ref(struct btrfs_trans_handle *trans, struct btrfs_root *root, @@ -3462,12 +3463,12 @@ int btrfs_free_tree_block(struct btrfs_trans_handle *trans, return 0; if (btrfs_header_generation(buf) != trans->transid) - goto out; + return 0; if (root_id != BTRFS_TREE_LOG_OBJECTID) { ret = check_ref_cleanup(trans, buf->start); if (!ret) - goto out; + return 0; } bg = btrfs_lookup_block_group(fs_info, buf->start); @@ -3475,7 +3476,7 @@ int btrfs_free_tree_block(struct btrfs_trans_handle *trans, if (btrfs_header_flag(buf, BTRFS_HEADER_FLAG_WRITTEN)) { pin_down_extent(trans, bg, buf->start, buf->len, true); btrfs_put_block_group(bg); - goto out; + return 0; } /* @@ -3499,7 +3500,7 @@ int btrfs_free_tree_block(struct btrfs_trans_handle *trans, || btrfs_is_zoned(fs_info)) { pin_down_extent(trans, bg, buf->start, buf->len, true); btrfs_put_block_group(bg); - goto out; + return 0; } WARN_ON(test_bit(EXTENT_BUFFER_DIRTY, &buf->bflags)); @@ -3509,7 +3510,6 @@ int btrfs_free_tree_block(struct btrfs_trans_handle *trans, btrfs_put_block_group(bg); trace_btrfs_reserved_extent_free(fs_info, buf->start, buf->len); -out: return 0; } @@ -6551,6 +6551,10 @@ int btrfs_trim_fs(struct btrfs_fs_info *fs_info, struct fstrim_range *range) range->minlen); trimmed += group_trimmed; + if (ret == -ERESTARTSYS || ret == -EINTR) { + btrfs_put_block_group(cache); + break; + } if (ret) { bg_failed++; bg_ret = ret; @@ -6564,6 +6568,9 @@ int btrfs_trim_fs(struct btrfs_fs_info *fs_info, struct fstrim_range *range) "failed to trim %llu block group(s), last error %d", bg_failed, bg_ret); + if (ret == -ERESTARTSYS || ret == -EINTR) + return ret; + mutex_lock(&fs_devices->device_list_mutex); list_for_each_entry(device, &fs_devices->devices, dev_list) { if (test_bit(BTRFS_DEV_STATE_MISSING, &device->dev_state)) @@ -6572,10 +6579,12 @@ int btrfs_trim_fs(struct btrfs_fs_info *fs_info, struct fstrim_range *range) ret = btrfs_trim_free_extents(device, &group_trimmed); trimmed += group_trimmed; + if (ret == -ERESTARTSYS || ret == -EINTR) + break; if (ret) { dev_failed++; dev_ret = ret; - break; + continue; } } mutex_unlock(&fs_devices->device_list_mutex); @@ -6585,6 +6594,8 @@ int btrfs_trim_fs(struct btrfs_fs_info *fs_info, struct fstrim_range *range) "failed to trim %llu device(s), last error %d", dev_failed, dev_ret); range->len = trimmed; + if (ret == -ERESTARTSYS || ret == -EINTR) + return ret; if (bg_ret) return bg_ret; return dev_ret; diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c index f6cca3c97166f7..3e6be1911237f4 100644 --- a/fs/btrfs/extent_io.c +++ b/fs/btrfs/extent_io.c @@ -4475,6 +4475,7 @@ static int try_release_subpage_extent_buffer(struct folio *folio) */ if (!test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags)) { spin_unlock(&eb->refs_lock); + rcu_read_lock(); break; } diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index a2b5b440637e67..13f1f3b52b0499 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -4720,7 +4720,7 @@ int btrfs_delete_subvolume(struct btrfs_inode *dir, struct dentry *dentry) spin_unlock(&dest->root_item_lock); btrfs_warn(fs_info, "attempt to delete subvolume %llu with active swapfile", - btrfs_root_id(root)); + btrfs_root_id(dest)); ret = -EPERM; goto out_up_write; } @@ -6542,6 +6542,25 @@ int btrfs_create_new_inode(struct btrfs_trans_handle *trans, int ret; bool xa_reserved = false; + if (!args->orphan && !args->subvol) { + /* + * Before anything else, check if we can add the name to the + * parent directory. We want to avoid a dir item overflow in + * case we have an existing dir item due to existing name + * hash collisions. We do this check here before we call + * btrfs_add_link() down below so that we can avoid a + * transaction abort (which could be exploited by malicious + * users). + * + * For subvolumes we already do this in btrfs_mksubvol(). + */ + ret = btrfs_check_dir_item_collision(BTRFS_I(dir)->root, + btrfs_ino(BTRFS_I(dir)), + name); + if (ret < 0) + return ret; + } + path = btrfs_alloc_path(); if (!path) return -ENOMEM; diff --git a/fs/btrfs/ioctl.c b/fs/btrfs/ioctl.c index acb484546b1da1..16c9b242e917ff 100644 --- a/fs/btrfs/ioctl.c +++ b/fs/btrfs/ioctl.c @@ -672,6 +672,13 @@ static noinline int create_subvol(struct mnt_idmap *idmap, goto out; } + /* + * Subvolumes have orphans cleaned on first dentry lookup. A new + * subvolume cannot have any orphans, so we should set the bit before we + * add the subvolume dentry to the dentry cache, so that it is in the + * same state as a subvolume after first lookup. + */ + set_bit(BTRFS_ROOT_ORPHAN_CLEANUP, &new_root->state); d_instantiate_new(dentry, new_inode_args.inode); new_inode_args.inode = NULL; @@ -3690,7 +3697,8 @@ static long btrfs_ioctl_qgroup_assign(struct file *file, void __user *arg) } } - trans = btrfs_join_transaction(root); + /* 2 BTRFS_QGROUP_RELATION_KEY items. */ + trans = btrfs_start_transaction(root, 2); if (IS_ERR(trans)) { ret = PTR_ERR(trans); goto out; @@ -3762,7 +3770,11 @@ static long btrfs_ioctl_qgroup_create(struct file *file, void __user *arg) goto out; } - trans = btrfs_join_transaction(root); + /* + * 1 BTRFS_QGROUP_INFO_KEY item. + * 1 BTRFS_QGROUP_LIMIT_KEY item. + */ + trans = btrfs_start_transaction(root, 2); if (IS_ERR(trans)) { ret = PTR_ERR(trans); goto out; @@ -3811,7 +3823,8 @@ static long btrfs_ioctl_qgroup_limit(struct file *file, void __user *arg) goto drop_write; } - trans = btrfs_join_transaction(root); + /* 1 BTRFS_QGROUP_LIMIT_KEY item. */ + trans = btrfs_start_transaction(root, 1); if (IS_ERR(trans)) { ret = PTR_ERR(trans); goto out; @@ -3932,6 +3945,25 @@ static long _btrfs_ioctl_set_received_subvol(struct file *file, goto out; } + received_uuid_changed = memcmp(root_item->received_uuid, sa->uuid, + BTRFS_UUID_SIZE); + + /* + * Before we attempt to add the new received uuid, check if we have room + * for it in case there's already an item. If the size of the existing + * item plus this root's ID (u64) exceeds the maximum item size, we can + * return here without the need to abort a transaction. If we don't do + * this check, the btrfs_uuid_tree_add() call below would fail with + * -EOVERFLOW and result in a transaction abort. Malicious users could + * exploit this to turn the fs into RO mode. + */ + if (received_uuid_changed && !btrfs_is_empty_uuid(sa->uuid)) { + ret = btrfs_uuid_tree_check_overflow(fs_info, sa->uuid, + BTRFS_UUID_KEY_RECEIVED_SUBVOL); + if (ret < 0) + goto out; + } + /* * 1 - root item * 2 - uuid items (received uuid + subvol uuid) @@ -3947,8 +3979,6 @@ static long _btrfs_ioctl_set_received_subvol(struct file *file, sa->rtime.sec = ct.tv_sec; sa->rtime.nsec = ct.tv_nsec; - received_uuid_changed = memcmp(root_item->received_uuid, sa->uuid, - BTRFS_UUID_SIZE); if (received_uuid_changed && !btrfs_is_empty_uuid(root_item->received_uuid)) { ret = btrfs_uuid_tree_remove(trans, root_item->received_uuid, @@ -3970,7 +4000,8 @@ static long _btrfs_ioctl_set_received_subvol(struct file *file, ret = btrfs_update_root(trans, fs_info->tree_root, &root->root_key, &root->root_item); - if (ret < 0) { + if (unlikely(ret < 0)) { + btrfs_abort_transaction(trans, ret); btrfs_end_transaction(trans); goto out; } @@ -4661,7 +4692,7 @@ static int btrfs_uring_read_extent(struct kiocb *iocb, struct iov_iter *iter, { struct btrfs_inode *inode = BTRFS_I(file_inode(iocb->ki_filp)); struct extent_io_tree *io_tree = &inode->io_tree; - struct page **pages; + struct page **pages = NULL; struct btrfs_uring_priv *priv = NULL; unsigned long nr_pages; int ret; @@ -4719,6 +4750,11 @@ static int btrfs_uring_read_extent(struct kiocb *iocb, struct iov_iter *iter, btrfs_unlock_extent(io_tree, start, lockend, &cached_state); btrfs_inode_unlock(inode, BTRFS_ILOCK_SHARED); kfree(priv); + for (int i = 0; i < nr_pages; i++) { + if (pages[i]) + __free_page(pages[i]); + } + kfree(pages); return ret; } diff --git a/fs/btrfs/qgroup.c b/fs/btrfs/qgroup.c index 206587820fec09..bed9d1c11c67a0 100644 --- a/fs/btrfs/qgroup.c +++ b/fs/btrfs/qgroup.c @@ -1137,11 +1137,14 @@ int btrfs_quota_enable(struct btrfs_fs_info *fs_info, } if (ret > 0) { /* - * Shouldn't happen, but in case it does we - * don't need to do the btrfs_next_item, just - * continue. + * Shouldn't happen because the key should still + * be there (return 0), but in case it does it + * means we have reached the end of the tree - + * there are no more leaves with items that have + * a key greater than or equals to @found_key, + * so just stop the search loop. */ - continue; + break; } } ret = btrfs_next_item(tree_root, path); @@ -1607,8 +1610,10 @@ static int __del_qgroup_relation(struct btrfs_trans_handle *trans, u64 src, if (ret < 0 && ret != -ENOENT) goto out; ret2 = del_qgroup_relation_item(trans, dst, src); - if (ret2 < 0 && ret2 != -ENOENT) + if (ret2 < 0 && ret2 != -ENOENT) { + ret = ret2; goto out; + } /* At least one deletion succeeded, return 0 */ if (!ret || !ret2) diff --git a/fs/btrfs/scrub.c b/fs/btrfs/scrub.c index a40ee41f42c681..4fc69b2d213a67 100644 --- a/fs/btrfs/scrub.c +++ b/fs/btrfs/scrub.c @@ -745,7 +745,7 @@ static void scrub_verify_one_metadata(struct scrub_stripe *stripe, int sector_nr btrfs_warn_rl(fs_info, "scrub: tree block %llu mirror %u has bad fsid, has %pU want %pU", logical, stripe->mirror_num, - header->fsid, fs_info->fs_devices->fsid); + header->fsid, fs_info->fs_devices->metadata_uuid); return; } if (memcmp(header->chunk_tree_uuid, fs_info->chunk_tree_uuid, diff --git a/fs/btrfs/space-info.c b/fs/btrfs/space-info.c index 3f08e450f7961e..13b2bbe6743081 100644 --- a/fs/btrfs/space-info.c +++ b/fs/btrfs/space-info.c @@ -2099,11 +2099,11 @@ static bool is_reclaim_urgent(struct btrfs_space_info *space_info) return unalloc < data_chunk_size; } -static void do_reclaim_sweep(struct btrfs_space_info *space_info, int raid) +static bool do_reclaim_sweep(struct btrfs_space_info *space_info, int raid) { struct btrfs_block_group *bg; int thresh_pct; - bool try_again = true; + bool will_reclaim = false; bool urgent; spin_lock(&space_info->lock); @@ -2121,7 +2121,7 @@ static void do_reclaim_sweep(struct btrfs_space_info *space_info, int raid) spin_lock(&bg->lock); thresh = mult_perc(bg->length, thresh_pct); if (bg->used < thresh && bg->reclaim_mark) { - try_again = false; + will_reclaim = true; reclaim = true; } bg->reclaim_mark++; @@ -2138,12 +2138,13 @@ static void do_reclaim_sweep(struct btrfs_space_info *space_info, int raid) * If we have any staler groups, we don't touch the fresher ones, but if we * really need a block group, do take a fresh one. */ - if (try_again && urgent) { - try_again = false; + if (!will_reclaim && urgent) { + urgent = false; goto again; } up_read(&space_info->groups_sem); + return will_reclaim; } void btrfs_space_info_update_reclaimable(struct btrfs_space_info *space_info, s64 bytes) @@ -2153,7 +2154,8 @@ void btrfs_space_info_update_reclaimable(struct btrfs_space_info *space_info, s6 lockdep_assert_held(&space_info->lock); space_info->reclaimable_bytes += bytes; - if (space_info->reclaimable_bytes >= chunk_sz) + if (space_info->reclaimable_bytes > 0 && + space_info->reclaimable_bytes >= chunk_sz) btrfs_set_periodic_reclaim_ready(space_info, true); } @@ -2180,7 +2182,6 @@ static bool btrfs_should_periodic_reclaim(struct btrfs_space_info *space_info) spin_lock(&space_info->lock); ret = space_info->periodic_reclaim_ready; - btrfs_set_periodic_reclaim_ready(space_info, false); spin_unlock(&space_info->lock); return ret; @@ -2194,8 +2195,13 @@ void btrfs_reclaim_sweep(const struct btrfs_fs_info *fs_info) list_for_each_entry(space_info, &fs_info->space_info, list) { if (!btrfs_should_periodic_reclaim(space_info)) continue; - for (raid = 0; raid < BTRFS_NR_RAID_TYPES; raid++) - do_reclaim_sweep(space_info, raid); + for (raid = 0; raid < BTRFS_NR_RAID_TYPES; raid++) { + if (do_reclaim_sweep(space_info, raid)) { + spin_lock(&space_info->lock); + btrfs_set_periodic_reclaim_ready(space_info, false); + spin_unlock(&space_info->lock); + } + } } } diff --git a/fs/btrfs/transaction.c b/fs/btrfs/transaction.c index bd03f465e2d3ee..5fc3735e790e24 100644 --- a/fs/btrfs/transaction.c +++ b/fs/btrfs/transaction.c @@ -726,7 +726,7 @@ start_transaction(struct btrfs_root *root, unsigned int num_items, h->type = type; INIT_LIST_HEAD(&h->new_bgs); - btrfs_init_metadata_block_rsv(fs_info, &h->delayed_rsv, BTRFS_BLOCK_RSV_DELOPS); + btrfs_init_metadata_block_rsv(fs_info, &h->delayed_rsv, BTRFS_BLOCK_RSV_DELREFS); smp_mb(); if (cur_trans->state >= TRANS_STATE_COMMIT_START && @@ -1904,6 +1904,22 @@ static noinline int create_pending_snapshot(struct btrfs_trans_handle *trans, ret = btrfs_uuid_tree_add(trans, new_root_item->received_uuid, BTRFS_UUID_KEY_RECEIVED_SUBVOL, objectid); + /* + * We are creating of lot of snapshots of the same root that was + * received (has a received UUID) and reached a leaf's limit for + * an item. We can safely ignore this and avoid a transaction + * abort. A deletion of this snapshot will still work since we + * ignore if an item with a BTRFS_UUID_KEY_RECEIVED_SUBVOL key + * is missing (see btrfs_delete_subvolume()). Send/receive will + * work too since it peeks the first root id from the existing + * item (it could peek any), and in case it's missing it + * falls back to search by BTRFS_UUID_KEY_SUBVOL keys. + * Creation of a snapshot does not require CAP_SYS_ADMIN, so + * we don't want users triggering transaction aborts, either + * intentionally or not. + */ + if (ret == -EOVERFLOW) + ret = 0; if (unlikely(ret && ret != -EEXIST)) { btrfs_abort_transaction(trans, ret); goto fail; @@ -2500,13 +2516,6 @@ int btrfs_commit_transaction(struct btrfs_trans_handle *trans) list_add_tail(&fs_info->chunk_root->dirty_list, &cur_trans->switch_commits); - if (btrfs_fs_incompat(fs_info, EXTENT_TREE_V2)) { - btrfs_set_root_node(&fs_info->block_group_root->root_item, - fs_info->block_group_root->node); - list_add_tail(&fs_info->block_group_root->dirty_list, - &cur_trans->switch_commits); - } - switch_commit_roots(trans); ASSERT(list_empty(&cur_trans->dirty_bgs)); diff --git a/fs/btrfs/tree-checker.c b/fs/btrfs/tree-checker.c index c21c21adf61ed1..1c8f61cbbbe5b1 100644 --- a/fs/btrfs/tree-checker.c +++ b/fs/btrfs/tree-checker.c @@ -1256,10 +1256,27 @@ static int check_root_item(struct extent_buffer *leaf, struct btrfs_key *key, } if (unlikely(btrfs_root_drop_level(&ri) >= BTRFS_MAX_LEVEL)) { generic_err(leaf, slot, - "invalid root level, have %u expect [0, %u]", + "invalid root drop_level, have %u expect [0, %u]", btrfs_root_drop_level(&ri), BTRFS_MAX_LEVEL - 1); return -EUCLEAN; } + /* + * If drop_progress.objectid is non-zero, a btrfs_drop_snapshot() was + * interrupted and the resume point was recorded in drop_progress and + * drop_level. In that case drop_level must be >= 1: level 0 is the + * leaf level and drop_snapshot never saves a checkpoint there (it + * only records checkpoints at internal node levels in DROP_REFERENCE + * stage). A zero drop_level combined with a non-zero drop_progress + * objectid indicates on-disk corruption and would cause a BUG_ON in + * merge_reloc_root() and btrfs_drop_snapshot() at mount time. + */ + if (unlikely(btrfs_disk_key_objectid(&ri.drop_progress) != 0 && + btrfs_root_drop_level(&ri) == 0)) { + generic_err(leaf, slot, + "invalid root drop_level 0 with non-zero drop_progress objectid %llu", + btrfs_disk_key_objectid(&ri.drop_progress)); + return -EUCLEAN; + } /* Flags check */ if (unlikely(btrfs_root_flags(&ri) & ~valid_root_flags)) { @@ -1712,7 +1729,7 @@ static int check_extent_data_ref(struct extent_buffer *leaf, objectid > BTRFS_LAST_FREE_OBJECTID)) { extent_err(leaf, slot, "invalid extent data backref objectid value %llu", - root); + objectid); return -EUCLEAN; } if (unlikely(!IS_ALIGNED(offset, leaf->fs_info->sectorsize))) { @@ -1893,7 +1910,7 @@ static int check_dev_extent_item(const struct extent_buffer *leaf, if (unlikely(prev_key->offset + prev_len > key->offset)) { generic_err(leaf, slot, "dev extent overlap, prev offset %llu len %llu current offset %llu", - prev_key->objectid, prev_len, key->offset); + prev_key->offset, prev_len, key->offset); return -EUCLEAN; } } diff --git a/fs/btrfs/tree-log.c b/fs/btrfs/tree-log.c index 6cffcf0c3e7af9..4cea0489f121cf 100644 --- a/fs/btrfs/tree-log.c +++ b/fs/btrfs/tree-log.c @@ -4609,21 +4609,32 @@ static void fill_inode_item(struct btrfs_trans_handle *trans, struct inode *inode, bool log_inode_only, u64 logged_isize) { + u64 gen = BTRFS_I(inode)->generation; u64 flags; if (log_inode_only) { - /* set the generation to zero so the recover code - * can tell the difference between an logging - * just to say 'this inode exists' and a logging - * to say 'update this inode with these values' + /* + * Set the generation to zero so the recover code can tell the + * difference between a logging just to say 'this inode exists' + * and a logging to say 'update this inode with these values'. + * But only if the inode was not already logged before. + * We access ->logged_trans directly since it was already set + * up in the call chain by btrfs_log_inode(), and data_race() + * to avoid false alerts from KCSAN and since it was set already + * and one can set it to 0 since that only happens on eviction + * and we are holding a ref on the inode. */ - btrfs_set_inode_generation(leaf, item, 0); + ASSERT(data_race(BTRFS_I(inode)->logged_trans) > 0); + if (data_race(BTRFS_I(inode)->logged_trans) < trans->transid) + gen = 0; + btrfs_set_inode_size(leaf, item, logged_isize); } else { - btrfs_set_inode_generation(leaf, item, BTRFS_I(inode)->generation); btrfs_set_inode_size(leaf, item, inode->i_size); } + btrfs_set_inode_generation(leaf, item, gen); + btrfs_set_inode_uid(leaf, item, i_uid_read(inode)); btrfs_set_inode_gid(leaf, item, i_gid_read(inode)); btrfs_set_inode_mode(leaf, item, inode->i_mode); @@ -5427,42 +5438,63 @@ static int btrfs_log_changed_extents(struct btrfs_trans_handle *trans, return 0; } -static int logged_inode_size(struct btrfs_root *log, struct btrfs_inode *inode, - struct btrfs_path *path, u64 *size_ret) +static int get_inode_size_to_log(struct btrfs_trans_handle *trans, + struct btrfs_inode *inode, + struct btrfs_path *path, u64 *size_ret) { struct btrfs_key key; + struct btrfs_inode_item *item; int ret; key.objectid = btrfs_ino(inode); key.type = BTRFS_INODE_ITEM_KEY; key.offset = 0; - ret = btrfs_search_slot(NULL, log, &key, path, 0, 0); - if (ret < 0) { - return ret; - } else if (ret > 0) { - *size_ret = 0; - } else { - struct btrfs_inode_item *item; + /* + * Our caller called inode_logged(), so logged_trans is up to date. + * Use data_race() to silence any warning from KCSAN. Once logged_trans + * is set, it can only be reset to 0 after inode eviction. + */ + if (data_race(inode->logged_trans) == trans->transid) { + ret = btrfs_search_slot(NULL, inode->root->log_root, &key, path, 0, 0); + } else if (inode->generation < trans->transid) { + path->search_commit_root = true; + path->skip_locking = true; + ret = btrfs_search_slot(NULL, inode->root, &key, path, 0, 0); + path->search_commit_root = false; + path->skip_locking = false; - item = btrfs_item_ptr(path->nodes[0], path->slots[0], - struct btrfs_inode_item); - *size_ret = btrfs_inode_size(path->nodes[0], item); - /* - * If the in-memory inode's i_size is smaller then the inode - * size stored in the btree, return the inode's i_size, so - * that we get a correct inode size after replaying the log - * when before a power failure we had a shrinking truncate - * followed by addition of a new name (rename / new hard link). - * Otherwise return the inode size from the btree, to avoid - * data loss when replaying a log due to previously doing a - * write that expands the inode's size and logging a new name - * immediately after. - */ - if (*size_ret > inode->vfs_inode.i_size) - *size_ret = inode->vfs_inode.i_size; + } else { + *size_ret = 0; + return 0; } + /* + * If the inode was logged before or is from a past transaction, then + * its inode item must exist in the log root or in the commit root. + */ + ASSERT(ret <= 0); + if (WARN_ON_ONCE(ret > 0)) + ret = -ENOENT; + + if (ret < 0) + return ret; + + item = btrfs_item_ptr(path->nodes[0], path->slots[0], + struct btrfs_inode_item); + *size_ret = btrfs_inode_size(path->nodes[0], item); + /* + * If the in-memory inode's i_size is smaller then the inode size stored + * in the btree, return the inode's i_size, so that we get a correct + * inode size after replaying the log when before a power failure we had + * a shrinking truncate followed by addition of a new name (rename / new + * hard link). Otherwise return the inode size from the btree, to avoid + * data loss when replaying a log due to previously doing a write that + * expands the inode's size and logging a new name immediately after. + */ + if (*size_ret > inode->vfs_inode.i_size) + *size_ret = inode->vfs_inode.i_size; + btrfs_release_path(path); return 0; } @@ -6195,6 +6227,7 @@ static int log_conflicting_inodes(struct btrfs_trans_handle *trans, struct btrfs_root *root, struct btrfs_log_ctx *ctx) { + const bool orig_log_new_dentries = ctx->log_new_dentries; int ret = 0; /* @@ -6256,7 +6289,11 @@ static int log_conflicting_inodes(struct btrfs_trans_handle *trans, * dir index key range logged for the directory. So we * must make sure the deletion is recorded. */ + ctx->log_new_dentries = false; ret = btrfs_log_inode(trans, inode, LOG_INODE_ALL, ctx); + if (!ret && ctx->log_new_dentries) + ret = log_new_dir_dentries(trans, inode, ctx); + btrfs_add_delayed_iput(inode); if (ret) break; @@ -6291,6 +6328,7 @@ static int log_conflicting_inodes(struct btrfs_trans_handle *trans, break; } + ctx->log_new_dentries = orig_log_new_dentries; ctx->logging_conflict_inodes = false; if (ret) free_conflicting_inodes(ctx); @@ -6969,7 +7007,7 @@ static int btrfs_log_inode(struct btrfs_trans_handle *trans, ret = drop_inode_items(trans, log, path, inode, BTRFS_XATTR_ITEM_KEY); } else { - if (inode_only == LOG_INODE_EXISTS && ctx->logged_before) { + if (inode_only == LOG_INODE_EXISTS) { /* * Make sure the new inode item we write to the log has * the same isize as the current one (if it exists). @@ -6983,7 +7021,7 @@ static int btrfs_log_inode(struct btrfs_trans_handle *trans, * (zeroes), as if an expanding truncate happened, * instead of getting a file of 4Kb only. */ - ret = logged_inode_size(log, inode, path, &logged_isize); + ret = get_inode_size_to_log(trans, inode, path, &logged_isize); if (ret) goto out_unlock; } diff --git a/fs/btrfs/uuid-tree.c b/fs/btrfs/uuid-tree.c index e3a1310fa7d5c5..a3c244ff3a0c57 100644 --- a/fs/btrfs/uuid-tree.c +++ b/fs/btrfs/uuid-tree.c @@ -199,6 +199,44 @@ int btrfs_uuid_tree_remove(struct btrfs_trans_handle *trans, const u8 *uuid, u8 return 0; } +/* + * Check if we can add one root ID to a UUID key. + * If the key does not yet exists, we can, otherwise only if extended item does + * not exceeds the maximum item size permitted by the leaf size. + * + * Returns 0 on success, negative value on error. + */ +int btrfs_uuid_tree_check_overflow(struct btrfs_fs_info *fs_info, + const u8 *uuid, u8 type) +{ + BTRFS_PATH_AUTO_FREE(path); + int ret; + u32 item_size; + struct btrfs_key key; + + if (WARN_ON_ONCE(!fs_info->uuid_root)) + return -EINVAL; + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + btrfs_uuid_to_key(uuid, type, &key); + ret = btrfs_search_slot(NULL, fs_info->uuid_root, &key, path, 0, 0); + if (ret < 0) + return ret; + if (ret > 0) + return 0; + + item_size = btrfs_item_size(path->nodes[0], path->slots[0]); + + if (sizeof(struct btrfs_item) + item_size + sizeof(u64) > + BTRFS_LEAF_DATA_SIZE(fs_info)) + return -EOVERFLOW; + + return 0; +} + static int btrfs_uuid_iter_rem(struct btrfs_root *uuid_root, u8 *uuid, u8 type, u64 subid) { diff --git a/fs/btrfs/uuid-tree.h b/fs/btrfs/uuid-tree.h index c60ad20325cce0..02b235a3653f06 100644 --- a/fs/btrfs/uuid-tree.h +++ b/fs/btrfs/uuid-tree.h @@ -12,6 +12,8 @@ int btrfs_uuid_tree_add(struct btrfs_trans_handle *trans, const u8 *uuid, u8 typ u64 subid); int btrfs_uuid_tree_remove(struct btrfs_trans_handle *trans, const u8 *uuid, u8 type, u64 subid); +int btrfs_uuid_tree_check_overflow(struct btrfs_fs_info *fs_info, + const u8 *uuid, u8 type); int btrfs_uuid_tree_iterate(struct btrfs_fs_info *fs_info); int btrfs_create_uuid_tree(struct btrfs_fs_info *fs_info); int btrfs_uuid_scan_kthread(void *data); diff --git a/fs/btrfs/volumes.c b/fs/btrfs/volumes.c index 8a08412f3529a1..052b830a0b66e6 100644 --- a/fs/btrfs/volumes.c +++ b/fs/btrfs/volumes.c @@ -1505,30 +1505,158 @@ struct btrfs_device *btrfs_scan_one_device(const char *path, } /* - * Try to find a chunk that intersects [start, start + len] range and when one - * such is found, record the end of it in *start + * Find the first pending extent intersecting a range. + * + * @device: the device to search + * @start: start of the range to check + * @len: length of the range to check + * @pending_start: output pointer for the start of the found pending extent + * @pending_end: output pointer for the end of the found pending extent (inclusive) + * + * Search for a pending chunk allocation that intersects the half-open range + * [start, start + len). + * + * Return: true if a pending extent was found, false otherwise. + * If the return value is true, store the first pending extent in + * [*pending_start, *pending_end]. Otherwise, the two output variables + * may still be modified, to something outside the range and should not + * be used. */ -static bool contains_pending_extent(struct btrfs_device *device, u64 *start, - u64 len) +static bool first_pending_extent(struct btrfs_device *device, u64 start, u64 len, + u64 *pending_start, u64 *pending_end) { - u64 physical_start, physical_end; - lockdep_assert_held(&device->fs_info->chunk_mutex); - if (btrfs_find_first_extent_bit(&device->alloc_state, *start, - &physical_start, &physical_end, + if (btrfs_find_first_extent_bit(&device->alloc_state, start, + pending_start, pending_end, CHUNK_ALLOCATED, NULL)) { - if (in_range(physical_start, *start, len) || - in_range(*start, physical_start, - physical_end + 1 - physical_start)) { - *start = physical_end + 1; + if (in_range(*pending_start, start, len) || + in_range(start, *pending_start, *pending_end + 1 - *pending_start)) { return true; } } return false; } +/* + * Find the first real hole accounting for pending extents. + * + * @device: the device containing the candidate hole + * @start: input/output pointer for the hole start position + * @len: input/output pointer for the hole length + * @min_hole_size: the size of hole we are looking for + * + * Given a potential hole specified by [*start, *start + *len), check for pending + * chunk allocations within that range. If pending extents are found, the hole is + * adjusted to represent the first true free space that is large enough when + * accounting for pending chunks. + * + * Note that this function must handle various cases involving non consecutive + * pending extents. + * + * Returns: true if a suitable hole was found and false otherwise. + * If the return value is true, then *start and *len are set to represent the hole. + * If the return value is false, then *start is set to the largest hole we + * found and *len is set to its length. + * If there are no holes at all, then *start is set to the end of the range and + * *len is set to 0. + */ +static bool find_hole_in_pending_extents(struct btrfs_device *device, u64 *start, + u64 *len, u64 min_hole_size) +{ + u64 pending_start, pending_end; + u64 end; + u64 max_hole_start = 0; + u64 max_hole_len = 0; + + lockdep_assert_held(&device->fs_info->chunk_mutex); + + if (*len == 0) + return false; + + end = *start + *len - 1; + + /* + * Loop until we either see a large enough hole or check every pending + * extent overlapping the candidate hole. + * At every hole that we observe, record it if it is the new max. + * At the end of the iteration, set the output variables to the max hole. + */ + while (true) { + if (first_pending_extent(device, *start, *len, &pending_start, &pending_end)) { + /* + * Case 1: the pending extent overlaps the start of + * candidate hole. That means the true hole is after the + * pending extent, but we need to find the next pending + * extent to properly size the hole. In the next loop, + * we will reduce to case 2 or 3. + * e.g., + * + * |----pending A----| real hole |----pending B----| + * | candidate hole | + * *start end + */ + if (pending_start <= *start) { + *start = pending_end + 1; + goto next; + } + /* + * Case 2: The pending extent starts after *start (and overlaps + * [*start, end), so the first hole just goes up to the start + * of the pending extent. + * e.g., + * + * | real hole |----pending A----| + * | candidate hole | + * *start end + */ + *len = pending_start - *start; + if (*len > max_hole_len) { + max_hole_start = *start; + max_hole_len = *len; + } + if (*len >= min_hole_size) + break; + /* + * If the hole wasn't big enough, then we advance past + * the pending extent and keep looking. + */ + *start = pending_end + 1; + goto next; + } else { + /* + * Case 3: There is no pending extent overlapping the + * range [*start, *start + *len - 1], so the only remaining + * hole is the remaining range. + * e.g., + * + * | candidate hole | + * | real hole | + * *start end + */ + + if (*len > max_hole_len) { + max_hole_start = *start; + max_hole_len = *len; + } + break; + } +next: + if (*start > end) + break; + *len = end - *start + 1; + } + if (max_hole_len) { + *start = max_hole_start; + *len = max_hole_len; + } else { + *start = end + 1; + *len = 0; + } + return max_hole_len >= min_hole_size; +} + static u64 dev_extent_search_start(struct btrfs_device *device) { switch (device->fs_devices->chunk_alloc_policy) { @@ -1593,59 +1721,57 @@ static bool dev_extent_hole_check_zoned(struct btrfs_device *device, } /* - * Check if specified hole is suitable for allocation. + * Validate and adjust a hole for chunk allocation + * + * @device: the device containing the candidate hole + * @hole_start: input/output pointer for the hole start position + * @hole_size: input/output pointer for the hole size + * @num_bytes: minimum allocation size required * - * @device: the device which we have the hole - * @hole_start: starting position of the hole - * @hole_size: the size of the hole - * @num_bytes: the size of the free space that we need + * Check if the specified hole is suitable for allocation and adjust it if + * necessary. The hole may be modified to skip over pending chunk allocations + * and to satisfy stricter zoned requirements on zoned filesystems. * - * This function may modify @hole_start and @hole_size to reflect the suitable - * position for allocation. Returns 1 if hole position is updated, 0 otherwise. + * For regular (non-zoned) allocation, if the hole after adjustment is smaller + * than @num_bytes, the search continues past additional pending extents until + * either a sufficiently large hole is found or no more pending extents exist. + * + * Return: true if a suitable hole was found and false otherwise. + * If the return value is true, then *hole_start and *hole_size are set to + * represent the hole we found. + * If the return value is false, then *hole_start is set to the largest + * hole we found and *hole_size is set to its length. + * If there are no holes at all, then *hole_start is set to the end of the range + * and *hole_size is set to 0. */ static bool dev_extent_hole_check(struct btrfs_device *device, u64 *hole_start, u64 *hole_size, u64 num_bytes) { - bool changed = false; - u64 hole_end = *hole_start + *hole_size; + bool found = false; + const u64 hole_end = *hole_start + *hole_size - 1; - for (;;) { - /* - * Check before we set max_hole_start, otherwise we could end up - * sending back this offset anyway. - */ - if (contains_pending_extent(device, hole_start, *hole_size)) { - if (hole_end >= *hole_start) - *hole_size = hole_end - *hole_start; - else - *hole_size = 0; - changed = true; - } + ASSERT(*hole_size > 0); - switch (device->fs_devices->chunk_alloc_policy) { - default: - btrfs_warn_unknown_chunk_allocation(device->fs_devices->chunk_alloc_policy); - fallthrough; - case BTRFS_CHUNK_ALLOC_REGULAR: - /* No extra check */ - break; - case BTRFS_CHUNK_ALLOC_ZONED: - if (dev_extent_hole_check_zoned(device, hole_start, - hole_size, num_bytes)) { - changed = true; - /* - * The changed hole can contain pending extent. - * Loop again to check that. - */ - continue; - } - break; - } +again: + *hole_size = hole_end - *hole_start + 1; + found = find_hole_in_pending_extents(device, hole_start, hole_size, num_bytes); + if (!found) + return found; + ASSERT(*hole_size >= num_bytes); + switch (device->fs_devices->chunk_alloc_policy) { + default: + btrfs_warn_unknown_chunk_allocation(device->fs_devices->chunk_alloc_policy); + fallthrough; + case BTRFS_CHUNK_ALLOC_REGULAR: + return found; + case BTRFS_CHUNK_ALLOC_ZONED: + if (dev_extent_hole_check_zoned(device, hole_start, hole_size, num_bytes)) + goto again; break; } - return changed; + return found; } /* @@ -1704,7 +1830,7 @@ static int find_free_dev_extent(struct btrfs_device *device, u64 num_bytes, ret = -ENOMEM; goto out; } -again: + if (search_start >= search_end || test_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state)) { ret = -ENOSPC; @@ -1791,11 +1917,7 @@ static int find_free_dev_extent(struct btrfs_device *device, u64 num_bytes, */ if (search_end > search_start) { hole_size = search_end - search_start; - if (dev_extent_hole_check(device, &search_start, &hole_size, - num_bytes)) { - btrfs_release_path(path); - goto again; - } + dev_extent_hole_check(device, &search_start, &hole_size, num_bytes); if (hole_size > max_hole_size) { max_hole_start = search_start; @@ -4112,8 +4234,14 @@ static int __btrfs_balance(struct btrfs_fs_info *fs_info) * this shouldn't happen, it means the last relocate * failed */ - if (ret == 0) - BUG(); /* FIXME break ? */ + if (unlikely(ret == 0)) { + btrfs_err(fs_info, + "unexpected exact match of CHUNK_ITEM in chunk tree, offset 0x%llx", + key.offset); + mutex_unlock(&fs_info->reclaim_bgs_lock); + ret = -EUCLEAN; + goto error; + } ret = btrfs_previous_item(chunk_root, path, 0, BTRFS_CHUNK_ITEM_KEY); @@ -4844,6 +4972,7 @@ int btrfs_shrink_device(struct btrfs_device *device, u64 new_size) u64 diff; u64 start; u64 free_diff = 0; + u64 pending_start, pending_end; new_size = round_down(new_size, fs_info->sectorsize); start = new_size; @@ -4889,7 +5018,7 @@ int btrfs_shrink_device(struct btrfs_device *device, u64 new_size) * in-memory chunks are synced to disk so that the loop below sees them * and relocates them accordingly. */ - if (contains_pending_extent(device, &start, diff)) { + if (first_pending_extent(device, start, diff, &pending_start, &pending_end)) { mutex_unlock(&fs_info->chunk_mutex); ret = btrfs_commit_transaction(trans); if (ret) @@ -6578,8 +6707,10 @@ int btrfs_map_block(struct btrfs_fs_info *fs_info, enum btrfs_map_op op, return PTR_ERR(map); num_copies = btrfs_chunk_map_num_copies(map); - if (io_geom.mirror_num > num_copies) - return -EINVAL; + if (io_geom.mirror_num > num_copies) { + ret = -EINVAL; + goto out; + } map_offset = logical - map->start; io_geom.raid56_full_stripe_start = (u64)-1; @@ -7743,8 +7874,9 @@ int btrfs_run_dev_stats(struct btrfs_trans_handle *trans) smp_rmb(); ret = update_dev_stat_item(trans, device); - if (!ret) - atomic_sub(stats_cnt, &device->dev_stats_ccnt); + if (ret) + break; + atomic_sub(stats_cnt, &device->dev_stats_ccnt); } mutex_unlock(&fs_devices->device_list_mutex); diff --git a/fs/btrfs/zoned.c b/fs/btrfs/zoned.c index 359a98e6de8513..ab7cc302127024 100644 --- a/fs/btrfs/zoned.c +++ b/fs/btrfs/zoned.c @@ -337,7 +337,10 @@ int btrfs_get_dev_zone_info_all_devices(struct btrfs_fs_info *fs_info) if (!btrfs_fs_incompat(fs_info, ZONED)) return 0; - mutex_lock(&fs_devices->device_list_mutex); + /* + * No need to take the device_list mutex here, we're still in the mount + * path and devices cannot be added to or removed from the list yet. + */ list_for_each_entry(device, &fs_devices->devices, dev_list) { /* We can skip reading of zone info for missing devices */ if (!device->bdev) @@ -347,7 +350,6 @@ int btrfs_get_dev_zone_info_all_devices(struct btrfs_fs_info *fs_info) if (ret) break; } - mutex_unlock(&fs_devices->device_list_mutex); return ret; } @@ -1449,6 +1451,20 @@ static int btrfs_load_block_group_dup(struct btrfs_block_group *bg, return -EIO; } + /* + * When the last extent is removed, last_alloc can be smaller than the other write + * pointer. In that case, last_alloc should be moved to the corresponding write + * pointer position. + */ + for (int i = 0; i < map->num_stripes; i++) { + if (zone_info[i].alloc_offset == WP_CONVENTIONAL) + continue; + if (last_alloc <= zone_info[i].alloc_offset) { + last_alloc = zone_info[i].alloc_offset; + break; + } + } + if (zone_info[0].alloc_offset == WP_CONVENTIONAL) zone_info[0].alloc_offset = last_alloc; @@ -1490,6 +1506,21 @@ static int btrfs_load_block_group_raid1(struct btrfs_block_group *bg, /* In case a device is missing we have a cap of 0, so don't use it. */ bg->zone_capacity = min_not_zero(zone_info[0].capacity, zone_info[1].capacity); + /* + * When the last extent is removed, last_alloc can be smaller than the other write + * pointer. In that case, last_alloc should be moved to the corresponding write + * pointer position. + */ + for (i = 0; i < map->num_stripes; i++) { + if (zone_info[i].alloc_offset == WP_MISSING_DEV || + zone_info[i].alloc_offset == WP_CONVENTIONAL) + continue; + if (last_alloc <= zone_info[i].alloc_offset) { + last_alloc = zone_info[i].alloc_offset; + break; + } + } + for (i = 0; i < map->num_stripes; i++) { if (zone_info[i].alloc_offset == WP_MISSING_DEV) continue; @@ -1531,7 +1562,9 @@ static int btrfs_load_block_group_raid0(struct btrfs_block_group *bg, { struct btrfs_fs_info *fs_info = bg->fs_info; u64 stripe_nr = 0, stripe_offset = 0; + u64 prev_offset = 0; u32 stripe_index = 0; + bool has_partial = false, has_conventional = false; if ((map->type & BTRFS_BLOCK_GROUP_DATA) && !fs_info->stripe_root) { btrfs_err(fs_info, "zoned: data %s needs raid-stripe-tree", @@ -1539,6 +1572,35 @@ static int btrfs_load_block_group_raid0(struct btrfs_block_group *bg, return -EINVAL; } + /* + * When the last extent is removed, last_alloc can be smaller than the other write + * pointer. In that case, last_alloc should be moved to the corresponding write + * pointer position. + */ + for (int i = 0; i < map->num_stripes; i++) { + u64 alloc; + + if (zone_info[i].alloc_offset == WP_MISSING_DEV || + zone_info[i].alloc_offset == WP_CONVENTIONAL) + continue; + + stripe_nr = zone_info[i].alloc_offset >> BTRFS_STRIPE_LEN_SHIFT; + stripe_offset = zone_info[i].alloc_offset & BTRFS_STRIPE_LEN_MASK; + if (stripe_offset == 0 && stripe_nr > 0) { + stripe_nr--; + stripe_offset = BTRFS_STRIPE_LEN; + } + alloc = ((stripe_nr * map->num_stripes + i) << BTRFS_STRIPE_LEN_SHIFT) + + stripe_offset; + last_alloc = max(last_alloc, alloc); + + /* Partially written stripe found. It should be last. */ + if (zone_info[i].alloc_offset & BTRFS_STRIPE_LEN_MASK) + break; + } + stripe_nr = 0; + stripe_offset = 0; + if (last_alloc) { u32 factor = map->num_stripes; @@ -1552,7 +1614,7 @@ static int btrfs_load_block_group_raid0(struct btrfs_block_group *bg, continue; if (zone_info[i].alloc_offset == WP_CONVENTIONAL) { - + has_conventional = true; zone_info[i].alloc_offset = btrfs_stripe_nr_to_offset(stripe_nr); if (stripe_index > i) @@ -1561,6 +1623,28 @@ static int btrfs_load_block_group_raid0(struct btrfs_block_group *bg, zone_info[i].alloc_offset += stripe_offset; } + /* Verification */ + if (i != 0) { + if (unlikely(prev_offset < zone_info[i].alloc_offset)) { + btrfs_err(fs_info, + "zoned: stripe position disorder found in block group %llu", + bg->start); + return -EIO; + } + + if (unlikely(has_partial && + (zone_info[i].alloc_offset & BTRFS_STRIPE_LEN_MASK))) { + btrfs_err(fs_info, + "zoned: multiple partial written stripe found in block group %llu", + bg->start); + return -EIO; + } + } + prev_offset = zone_info[i].alloc_offset; + + if ((zone_info[i].alloc_offset & BTRFS_STRIPE_LEN_MASK) != 0) + has_partial = true; + if (test_bit(0, active) != test_bit(i, active)) { if (unlikely(!btrfs_zone_activate(bg))) return -EIO; @@ -1572,6 +1656,19 @@ static int btrfs_load_block_group_raid0(struct btrfs_block_group *bg, bg->alloc_offset += zone_info[i].alloc_offset; } + /* Check if all devices stay in the same stripe row. */ + if (unlikely(zone_info[0].alloc_offset - + zone_info[map->num_stripes - 1].alloc_offset > BTRFS_STRIPE_LEN)) { + btrfs_err(fs_info, "zoned: stripe gap too large in block group %llu", bg->start); + return -EIO; + } + + if (unlikely(has_conventional && bg->alloc_offset < last_alloc)) { + btrfs_err(fs_info, "zoned: allocated extent stays beyond write pointers %llu %llu", + bg->alloc_offset, last_alloc); + return -EIO; + } + return 0; } @@ -1582,8 +1679,11 @@ static int btrfs_load_block_group_raid10(struct btrfs_block_group *bg, u64 last_alloc) { struct btrfs_fs_info *fs_info = bg->fs_info; + u64 AUTO_KFREE(raid0_allocs); u64 stripe_nr = 0, stripe_offset = 0; u32 stripe_index = 0; + bool has_partial = false, has_conventional = false; + u64 prev_offset = 0; if ((map->type & BTRFS_BLOCK_GROUP_DATA) && !fs_info->stripe_root) { btrfs_err(fs_info, "zoned: data %s needs raid-stripe-tree", @@ -1591,6 +1691,60 @@ static int btrfs_load_block_group_raid10(struct btrfs_block_group *bg, return -EINVAL; } + raid0_allocs = kcalloc(map->num_stripes / map->sub_stripes, sizeof(*raid0_allocs), + GFP_NOFS); + if (!raid0_allocs) + return -ENOMEM; + + /* + * When the last extent is removed, last_alloc can be smaller than the other write + * pointer. In that case, last_alloc should be moved to the corresponding write + * pointer position. + */ + for (int i = 0; i < map->num_stripes; i += map->sub_stripes) { + u64 alloc = zone_info[i].alloc_offset; + + for (int j = 1; j < map->sub_stripes; j++) { + int idx = i + j; + + if (zone_info[idx].alloc_offset == WP_MISSING_DEV || + zone_info[idx].alloc_offset == WP_CONVENTIONAL) + continue; + if (alloc == WP_MISSING_DEV || alloc == WP_CONVENTIONAL) { + alloc = zone_info[idx].alloc_offset; + } else if (unlikely(zone_info[idx].alloc_offset != alloc)) { + btrfs_err(fs_info, + "zoned: write pointer mismatch found in block group %llu", + bg->start); + return -EIO; + } + } + + raid0_allocs[i / map->sub_stripes] = alloc; + if (alloc == WP_CONVENTIONAL) + continue; + if (unlikely(alloc == WP_MISSING_DEV)) { + btrfs_err(fs_info, + "zoned: cannot recover write pointer of block group %llu due to missing device", + bg->start); + return -EIO; + } + + stripe_nr = alloc >> BTRFS_STRIPE_LEN_SHIFT; + stripe_offset = alloc & BTRFS_STRIPE_LEN_MASK; + if (stripe_offset == 0 && stripe_nr > 0) { + stripe_nr--; + stripe_offset = BTRFS_STRIPE_LEN; + } + + alloc = ((stripe_nr * (map->num_stripes / map->sub_stripes) + + (i / map->sub_stripes)) << + BTRFS_STRIPE_LEN_SHIFT) + stripe_offset; + last_alloc = max(last_alloc, alloc); + } + stripe_nr = 0; + stripe_offset = 0; + if (last_alloc) { u32 factor = map->num_stripes / map->sub_stripes; @@ -1600,24 +1754,51 @@ static int btrfs_load_block_group_raid10(struct btrfs_block_group *bg, } for (int i = 0; i < map->num_stripes; i++) { - if (zone_info[i].alloc_offset == WP_MISSING_DEV) - continue; + int idx = i / map->sub_stripes; - if (test_bit(0, active) != test_bit(i, active)) { - if (unlikely(!btrfs_zone_activate(bg))) - return -EIO; - } else { - if (test_bit(0, active)) - set_bit(BLOCK_GROUP_FLAG_ZONE_IS_ACTIVE, &bg->runtime_flags); + if (raid0_allocs[idx] == WP_CONVENTIONAL) { + has_conventional = true; + raid0_allocs[idx] = btrfs_stripe_nr_to_offset(stripe_nr); + + if (stripe_index > idx) + raid0_allocs[idx] += BTRFS_STRIPE_LEN; + else if (stripe_index == idx) + raid0_allocs[idx] += stripe_offset; } - if (zone_info[i].alloc_offset == WP_CONVENTIONAL) { - zone_info[i].alloc_offset = btrfs_stripe_nr_to_offset(stripe_nr); + if ((i % map->sub_stripes) == 0) { + /* Verification */ + if (i != 0) { + if (unlikely(prev_offset < raid0_allocs[idx])) { + btrfs_err(fs_info, + "zoned: stripe position disorder found in block group %llu", + bg->start); + return -EIO; + } - if (stripe_index > (i / map->sub_stripes)) - zone_info[i].alloc_offset += BTRFS_STRIPE_LEN; - else if (stripe_index == (i / map->sub_stripes)) - zone_info[i].alloc_offset += stripe_offset; + if (unlikely(has_partial && + (raid0_allocs[idx] & BTRFS_STRIPE_LEN_MASK))) { + btrfs_err(fs_info, + "zoned: multiple partial written stripe found in block group %llu", + bg->start); + return -EIO; + } + } + prev_offset = raid0_allocs[idx]; + + if ((raid0_allocs[idx] & BTRFS_STRIPE_LEN_MASK) != 0) + has_partial = true; + } + + if (zone_info[i].alloc_offset == WP_MISSING_DEV || + zone_info[i].alloc_offset == WP_CONVENTIONAL) + zone_info[i].alloc_offset = raid0_allocs[idx]; + + if (test_bit(0, active) != test_bit(i, active)) { + if (unlikely(!btrfs_zone_activate(bg))) + return -EIO; + } else if (test_bit(0, active)) { + set_bit(BLOCK_GROUP_FLAG_ZONE_IS_ACTIVE, &bg->runtime_flags); } if ((i % map->sub_stripes) == 0) { @@ -1626,6 +1807,20 @@ static int btrfs_load_block_group_raid10(struct btrfs_block_group *bg, } } + /* Check if all devices stay in the same stripe row. */ + if (unlikely(zone_info[0].alloc_offset - + zone_info[map->num_stripes - 1].alloc_offset > BTRFS_STRIPE_LEN)) { + btrfs_err(fs_info, "zoned: stripe gap too large in block group %llu", + bg->start); + return -EIO; + } + + if (unlikely(has_conventional && bg->alloc_offset < last_alloc)) { + btrfs_err(fs_info, "zoned: allocated extent stays beyond write pointers %llu %llu", + bg->alloc_offset, last_alloc); + return -EIO; + } + return 0; } diff --git a/fs/buffer.c b/fs/buffer.c index 838c0c5710229e..28e4d53f17173a 100644 --- a/fs/buffer.c +++ b/fs/buffer.c @@ -2948,6 +2948,10 @@ bool try_to_free_buffers(struct folio *folio) if (folio_test_writeback(folio)) return false; + /* Misconfigured folio check */ + if (WARN_ON_ONCE(!folio_buffers(folio))) + return true; + if (mapping == NULL) { /* can this still happen? */ ret = drop_buffers(folio, &buffers_to_free); goto out; diff --git a/fs/cachefiles/namei.c b/fs/cachefiles/namei.c index e5ec90dccc27f7..eb9eb7683e3ccf 100644 --- a/fs/cachefiles/namei.c +++ b/fs/cachefiles/namei.c @@ -810,6 +810,11 @@ int cachefiles_cull(struct cachefiles_cache *cache, struct dentry *dir, if (ret < 0) goto error_unlock; + /* + * cachefiles_bury_object() expects 2 references to 'victim', + * and drops one. + */ + dget(victim); ret = cachefiles_bury_object(cache, NULL, dir, victim, FSCACHE_OBJECT_WAS_CULLED); dput(victim); diff --git a/fs/ceph/addr.c b/fs/ceph/addr.c index 63b75d2142102e..20ac6371ba2cba 100644 --- a/fs/ceph/addr.c +++ b/fs/ceph/addr.c @@ -1000,7 +1000,8 @@ unsigned int ceph_define_write_size(struct address_space *mapping) { struct inode *inode = mapping->host; struct ceph_fs_client *fsc = ceph_inode_to_fs_client(inode); - unsigned int wsize = i_blocksize(inode); + struct ceph_inode_info *ci = ceph_inode(inode); + unsigned int wsize = ci->i_layout.stripe_unit; if (fsc->mount_options->wsize < wsize) wsize = fsc->mount_options->wsize; @@ -1329,7 +1330,6 @@ int ceph_process_folio_batch(struct address_space *mapping, } else if (rc == -E2BIG) { rc = 0; folio_unlock(folio); - ceph_wbc->fbatch.folios[i] = NULL; break; } @@ -1369,6 +1369,7 @@ int ceph_process_folio_batch(struct address_space *mapping, rc = move_dirty_folio_in_page_array(mapping, wbc, ceph_wbc, folio); if (rc) { + rc = 0; folio_redirty_for_writepage(wbc, folio); folio_unlock(folio); break; @@ -2199,6 +2200,7 @@ int ceph_uninline_data(struct file *file) struct ceph_osd_request *req = NULL; struct ceph_cap_flush *prealloc_cf = NULL; struct folio *folio = NULL; + struct ceph_snap_context *snapc = NULL; u64 inline_version = CEPH_INLINE_NONE; struct page *pages[1]; int err = 0; @@ -2226,6 +2228,24 @@ int ceph_uninline_data(struct file *file) if (inline_version == 1) /* initial version, no data */ goto out_uninline; + down_read(&fsc->mdsc->snap_rwsem); + spin_lock(&ci->i_ceph_lock); + if (__ceph_have_pending_cap_snap(ci)) { + struct ceph_cap_snap *capsnap = + list_last_entry(&ci->i_cap_snaps, + struct ceph_cap_snap, + ci_item); + snapc = ceph_get_snap_context(capsnap->context); + } else { + if (!ci->i_head_snapc) { + ci->i_head_snapc = ceph_get_snap_context( + ci->i_snap_realm->cached_context); + } + snapc = ceph_get_snap_context(ci->i_head_snapc); + } + spin_unlock(&ci->i_ceph_lock); + up_read(&fsc->mdsc->snap_rwsem); + folio = read_mapping_folio(inode->i_mapping, 0, file); if (IS_ERR(folio)) { err = PTR_ERR(folio); @@ -2241,7 +2261,7 @@ int ceph_uninline_data(struct file *file) req = ceph_osdc_new_request(&fsc->client->osdc, &ci->i_layout, ceph_vino(inode), 0, &len, 0, 1, CEPH_OSD_OP_CREATE, CEPH_OSD_FLAG_WRITE, - NULL, 0, 0, false); + snapc, 0, 0, false); if (IS_ERR(req)) { err = PTR_ERR(req); goto out_unlock; @@ -2257,7 +2277,7 @@ int ceph_uninline_data(struct file *file) req = ceph_osdc_new_request(&fsc->client->osdc, &ci->i_layout, ceph_vino(inode), 0, &len, 1, 3, CEPH_OSD_OP_WRITE, CEPH_OSD_FLAG_WRITE, - NULL, ci->i_truncate_seq, + snapc, ci->i_truncate_seq, ci->i_truncate_size, false); if (IS_ERR(req)) { err = PTR_ERR(req); @@ -2320,6 +2340,7 @@ int ceph_uninline_data(struct file *file) folio_put(folio); } out: + ceph_put_snap_context(snapc); ceph_free_cap_flush(prealloc_cf); doutc(cl, "%llx.%llx inline_version %llu = %d\n", ceph_vinop(inode), inline_version, err); diff --git a/fs/ceph/debugfs.c b/fs/ceph/debugfs.c index f3fe786b4143d4..7dc3077902401e 100644 --- a/fs/ceph/debugfs.c +++ b/fs/ceph/debugfs.c @@ -79,7 +79,7 @@ static int mdsc_show(struct seq_file *s, void *p) if (req->r_inode) { seq_printf(s, " #%llx", ceph_ino(req->r_inode)); } else if (req->r_dentry) { - struct ceph_path_info path_info; + struct ceph_path_info path_info = {0}; path = ceph_mdsc_build_path(mdsc, req->r_dentry, &path_info, 0); if (IS_ERR(path)) path = NULL; @@ -98,7 +98,7 @@ static int mdsc_show(struct seq_file *s, void *p) } if (req->r_old_dentry) { - struct ceph_path_info path_info; + struct ceph_path_info path_info = {0}; path = ceph_mdsc_build_path(mdsc, req->r_old_dentry, &path_info, 0); if (IS_ERR(path)) path = NULL; diff --git a/fs/ceph/dir.c b/fs/ceph/dir.c index 804588524cd570..cb5461b9300ee9 100644 --- a/fs/ceph/dir.c +++ b/fs/ceph/dir.c @@ -1339,6 +1339,7 @@ static int ceph_unlink(struct inode *dir, struct dentry *dentry) struct ceph_client *cl = fsc->client; struct ceph_mds_client *mdsc = fsc->mdsc; struct inode *inode = d_inode(dentry); + struct ceph_inode_info *ci = ceph_inode(inode); struct ceph_mds_request *req; bool try_async = ceph_test_mount_opt(fsc, ASYNC_DIROPS); struct dentry *dn; @@ -1363,7 +1364,7 @@ static int ceph_unlink(struct inode *dir, struct dentry *dentry) if (!dn) { try_async = false; } else { - struct ceph_path_info path_info; + struct ceph_path_info path_info = {0}; path = ceph_mdsc_build_path(mdsc, dn, &path_info, 0); if (IS_ERR(path)) { try_async = false; @@ -1424,7 +1425,19 @@ static int ceph_unlink(struct inode *dir, struct dentry *dentry) * We have enough caps, so we assume that the unlink * will succeed. Fix up the target inode and dcache. */ - drop_nlink(inode); + + /* + * Protect the i_nlink update with i_ceph_lock + * to precent racing against ceph_fill_inode() + * handling our completion on a worker thread + * and don't decrement if i_nlink has already + * been updated to zero by this completion. + */ + spin_lock(&ci->i_ceph_lock); + if (inode->i_nlink > 0) + drop_nlink(inode); + spin_unlock(&ci->i_ceph_lock); + d_delete(dentry); } else { spin_lock(&fsc->async_unlink_conflict_lock); diff --git a/fs/ceph/file.c b/fs/ceph/file.c index 983390069f7372..10c4df48dbdca0 100644 --- a/fs/ceph/file.c +++ b/fs/ceph/file.c @@ -397,7 +397,7 @@ int ceph_open(struct inode *inode, struct file *file) if (!dentry) { do_sync = true; } else { - struct ceph_path_info path_info; + struct ceph_path_info path_info = {0}; path = ceph_mdsc_build_path(mdsc, dentry, &path_info, 0); if (IS_ERR(path)) { do_sync = true; @@ -807,7 +807,7 @@ int ceph_atomic_open(struct inode *dir, struct dentry *dentry, if (!dn) { try_async = false; } else { - struct ceph_path_info path_info; + struct ceph_path_info path_info = {0}; path = ceph_mdsc_build_path(mdsc, dn, &path_info, 0); if (IS_ERR(path)) { try_async = false; @@ -2568,6 +2568,7 @@ static int ceph_zero_partial_object(struct inode *inode, struct ceph_inode_info *ci = ceph_inode(inode); struct ceph_fs_client *fsc = ceph_inode_to_fs_client(inode); struct ceph_osd_request *req; + struct ceph_snap_context *snapc; int ret = 0; loff_t zero = 0; int op; @@ -2582,12 +2583,25 @@ static int ceph_zero_partial_object(struct inode *inode, op = CEPH_OSD_OP_ZERO; } + spin_lock(&ci->i_ceph_lock); + if (__ceph_have_pending_cap_snap(ci)) { + struct ceph_cap_snap *capsnap = + list_last_entry(&ci->i_cap_snaps, + struct ceph_cap_snap, + ci_item); + snapc = ceph_get_snap_context(capsnap->context); + } else { + BUG_ON(!ci->i_head_snapc); + snapc = ceph_get_snap_context(ci->i_head_snapc); + } + spin_unlock(&ci->i_ceph_lock); + req = ceph_osdc_new_request(&fsc->client->osdc, &ci->i_layout, ceph_vino(inode), offset, length, 0, 1, op, CEPH_OSD_FLAG_WRITE, - NULL, 0, 0, false); + snapc, 0, 0, false); if (IS_ERR(req)) { ret = PTR_ERR(req); goto out; @@ -2601,6 +2615,7 @@ static int ceph_zero_partial_object(struct inode *inode, ceph_osdc_put_request(req); out: + ceph_put_snap_context(snapc); return ret; } diff --git a/fs/ceph/inode.c b/fs/ceph/inode.c index 2966f88310e34a..5781412bfbc126 100644 --- a/fs/ceph/inode.c +++ b/fs/ceph/inode.c @@ -2551,7 +2551,7 @@ int __ceph_setattr(struct mnt_idmap *idmap, struct inode *inode, if (!dentry) { do_sync = true; } else { - struct ceph_path_info path_info; + struct ceph_path_info path_info = {0}; path = ceph_mdsc_build_path(mdsc, dentry, &path_info, 0); if (IS_ERR(path)) { do_sync = true; diff --git a/fs/ceph/mds_client.c b/fs/ceph/mds_client.c index c45bd19d4b1ca7..109658e4d0231a 100644 --- a/fs/ceph/mds_client.c +++ b/fs/ceph/mds_client.c @@ -2768,6 +2768,7 @@ char *ceph_mdsc_build_path(struct ceph_mds_client *mdsc, struct dentry *dentry, if (ret < 0) { dput(parent); dput(cur); + __putname(path); return ERR_PTR(ret); } @@ -2777,6 +2778,7 @@ char *ceph_mdsc_build_path(struct ceph_mds_client *mdsc, struct dentry *dentry, if (len < 0) { dput(parent); dput(cur); + __putname(path); return ERR_PTR(len); } } @@ -2813,6 +2815,7 @@ char *ceph_mdsc_build_path(struct ceph_mds_client *mdsc, struct dentry *dentry, * cannot ever succeed. Creating paths that long is * possible with Ceph, but Linux cannot use them. */ + __putname(path); return ERR_PTR(-ENAMETOOLONG); } diff --git a/fs/dcache.c b/fs/dcache.c index 66dd1bb830d1a8..957a44d2c44af1 100644 --- a/fs/dcache.c +++ b/fs/dcache.c @@ -3260,7 +3260,7 @@ static void __init dcache_init_early(void) HASH_EARLY | HASH_ZERO, &d_hash_shift, NULL, - 0, + 2, 0); d_hash_shift = 32 - d_hash_shift; @@ -3292,7 +3292,7 @@ static void __init dcache_init(void) HASH_ZERO, &d_hash_shift, NULL, - 0, + 2, 0); d_hash_shift = 32 - d_hash_shift; diff --git a/fs/dlm/lock.c b/fs/dlm/lock.c index be938fdf17d967..a393ecaf3442ac 100644 --- a/fs/dlm/lock.c +++ b/fs/dlm/lock.c @@ -626,7 +626,8 @@ int dlm_search_rsb_tree(struct rhashtable *rhash, const void *name, int len, struct dlm_rsb **r_ret) { char key[DLM_RESNAME_MAXLEN] = {}; - + if (len > DLM_RESNAME_MAXLEN) + return -EINVAL; memcpy(key, name, len); *r_ret = rhashtable_lookup_fast(rhash, &key, dlm_rhash_rsb_params); if (*r_ret) @@ -5014,25 +5015,8 @@ void dlm_receive_buffer(const union dlm_packet *p, int nodeid) static void recover_convert_waiter(struct dlm_ls *ls, struct dlm_lkb *lkb, struct dlm_message *ms_local) { - if (middle_conversion(lkb)) { - log_rinfo(ls, "%s %x middle convert in progress", __func__, - lkb->lkb_id); - - /* We sent this lock to the new master. The new master will - * tell us when it's granted. We no longer need a reply, so - * use a fake reply to put the lkb into the right state. - */ - hold_lkb(lkb); - memset(ms_local, 0, sizeof(struct dlm_message)); - ms_local->m_type = cpu_to_le32(DLM_MSG_CONVERT_REPLY); - ms_local->m_result = cpu_to_le32(to_dlm_errno(-EINPROGRESS)); - ms_local->m_header.h_nodeid = cpu_to_le32(lkb->lkb_nodeid); - _receive_convert_reply(lkb, ms_local, true); - unhold_lkb(lkb); - - } else if (lkb->lkb_rqmode >= lkb->lkb_grmode) { + if (middle_conversion(lkb) || lkb->lkb_rqmode >= lkb->lkb_grmode) set_bit(DLM_IFL_RESEND_BIT, &lkb->lkb_iflags); - } /* lkb->lkb_rqmode < lkb->lkb_grmode shouldn't happen since down conversions are async; there's no reply from the remote master */ diff --git a/fs/erofs/fileio.c b/fs/erofs/fileio.c index 932e8b353ba11e..aa580c59fe6458 100644 --- a/fs/erofs/fileio.c +++ b/fs/erofs/fileio.c @@ -10,6 +10,7 @@ struct erofs_fileio_rq { struct bio bio; struct kiocb iocb; struct super_block *sb; + refcount_t ref; }; struct erofs_fileio { @@ -24,31 +25,26 @@ static void erofs_fileio_ki_complete(struct kiocb *iocb, long ret) container_of(iocb, struct erofs_fileio_rq, iocb); struct folio_iter fi; - if (ret > 0) { - if (ret != rq->bio.bi_iter.bi_size) { - bio_advance(&rq->bio, ret); - zero_fill_bio(&rq->bio); - } - ret = 0; - } - if (rq->bio.bi_end_io) { - if (ret < 0 && !rq->bio.bi_status) - rq->bio.bi_status = errno_to_blk_status(ret); - } else { + if (ret >= 0 && ret != rq->bio.bi_iter.bi_size) + ret = -EIO; + if (!rq->bio.bi_end_io) { bio_for_each_folio_all(fi, &rq->bio) { DBG_BUGON(folio_test_uptodate(fi.folio)); - erofs_onlinefolio_end(fi.folio, ret, false); + erofs_onlinefolio_end(fi.folio, ret < 0, false); } + } else if (ret < 0 && !rq->bio.bi_status) { + rq->bio.bi_status = errno_to_blk_status(ret); } bio_endio(&rq->bio); bio_uninit(&rq->bio); - kfree(rq); + if (refcount_dec_and_test(&rq->ref)) + kfree(rq); } static void erofs_fileio_rq_submit(struct erofs_fileio_rq *rq) { struct iov_iter iter; - int ret; + ssize_t ret; if (!rq) return; @@ -64,6 +60,8 @@ static void erofs_fileio_rq_submit(struct erofs_fileio_rq *rq) ret = vfs_iocb_iter_read(rq->iocb.ki_filp, &rq->iocb, &iter); if (ret != -EIOCBQUEUED) erofs_fileio_ki_complete(&rq->iocb, ret); + if (refcount_dec_and_test(&rq->ref)) + kfree(rq); } static struct erofs_fileio_rq *erofs_fileio_rq_alloc(struct erofs_map_dev *mdev) @@ -74,6 +72,7 @@ static struct erofs_fileio_rq *erofs_fileio_rq_alloc(struct erofs_map_dev *mdev) bio_init(&rq->bio, NULL, rq->bvecs, ARRAY_SIZE(rq->bvecs), REQ_OP_READ); rq->iocb.ki_filp = mdev->m_dif->file; rq->sb = mdev->m_sb; + refcount_set(&rq->ref, 2); return rq; } diff --git a/fs/erofs/super.c b/fs/erofs/super.c index 5136cda5972a98..ee37628ec99fb3 100644 --- a/fs/erofs/super.c +++ b/fs/erofs/super.c @@ -330,12 +330,13 @@ static int erofs_read_superblock(struct super_block *sb) } sbi->packed_nid = le64_to_cpu(dsb->packed_nid); if (erofs_sb_has_metabox(sbi)) { + ret = -EFSCORRUPTED; if (sbi->sb_size <= offsetof(struct erofs_super_block, metabox_nid)) - return -EFSCORRUPTED; + goto out; sbi->metabox_nid = le64_to_cpu(dsb->metabox_nid); if (sbi->metabox_nid & BIT_ULL(EROFS_DIRENT_NID_METABOX_BIT)) - return -EFSCORRUPTED; /* self-loop detection */ + goto out; /* self-loop detection */ } sbi->inos = le64_to_cpu(dsb->inos); @@ -346,8 +347,10 @@ static int erofs_read_superblock(struct super_block *sb) if (dsb->volume_name[0]) { sbi->volume_name = kstrndup(dsb->volume_name, sizeof(dsb->volume_name), GFP_KERNEL); - if (!sbi->volume_name) - return -ENOMEM; + if (!sbi->volume_name) { + ret = -ENOMEM; + goto out; + } } /* parse on-disk compression configurations */ diff --git a/fs/erofs/zdata.c b/fs/erofs/zdata.c index 3d31f7840ca04d..8ba409df1ca70a 100644 --- a/fs/erofs/zdata.c +++ b/fs/erofs/zdata.c @@ -805,14 +805,26 @@ static int z_erofs_pcluster_begin(struct z_erofs_frontend *fe) struct erofs_map_blocks *map = &fe->map; struct super_block *sb = fe->inode->i_sb; struct z_erofs_pcluster *pcl = NULL; - void *ptr; + void *ptr = NULL; int ret; DBG_BUGON(fe->pcl); /* must be Z_EROFS_PCLUSTER_TAIL or pointed to previous pcluster */ DBG_BUGON(!fe->head); - if (!(map->m_flags & EROFS_MAP_META)) { + if (map->m_flags & EROFS_MAP_META) { + ret = erofs_init_metabuf(&map->buf, sb, + erofs_inode_in_metabox(fe->inode)); + if (ret) + return ret; + ptr = erofs_bread(&map->buf, map->m_pa, false); + if (IS_ERR(ptr)) { + erofs_err(sb, "failed to read inline data %pe @ pa %llu of nid %llu", + ptr, map->m_pa, EROFS_I(fe->inode)->nid); + return PTR_ERR(ptr); + } + ptr = map->buf.page; + } else { while (1) { rcu_read_lock(); pcl = xa_load(&EROFS_SB(sb)->managed_pslots, map->m_pa); @@ -852,18 +864,8 @@ static int z_erofs_pcluster_begin(struct z_erofs_frontend *fe) /* bind cache first when cached decompression is preferred */ z_erofs_bind_cache(fe); } else { - ret = erofs_init_metabuf(&map->buf, sb, - erofs_inode_in_metabox(fe->inode)); - if (ret) - return ret; - ptr = erofs_bread(&map->buf, map->m_pa, false); - if (IS_ERR(ptr)) { - ret = PTR_ERR(ptr); - erofs_err(sb, "failed to get inline folio %d", ret); - return ret; - } - folio_get(page_folio(map->buf.page)); - WRITE_ONCE(fe->pcl->compressed_bvecs[0].page, map->buf.page); + folio_get(page_folio((struct page *)ptr)); + WRITE_ONCE(fe->pcl->compressed_bvecs[0].page, ptr); fe->pcl->pageofs_in = map->m_pa & ~PAGE_MASK; fe->mode = Z_EROFS_PCLUSTER_FOLLOWED_NOINPLACE; } @@ -1324,9 +1326,10 @@ static int z_erofs_decompress_pcluster(struct z_erofs_backend *be, bool eio) GFP_NOWAIT | __GFP_NORETRY }, be->pagepool); if (IS_ERR(reason)) { - erofs_err(be->sb, "failed to decompress (%s) %ld @ pa %llu size %u => %u", - alg->name, PTR_ERR(reason), pcl->pos, - pcl->pclustersize, pcl->length); + if (pcl->besteffort || reason != ERR_PTR(-ENOMEM)) + erofs_err(be->sb, "failed to decompress (%s) %pe @ pa %llu size %u => %u", + alg->name, reason, pcl->pos, + pcl->pclustersize, pcl->length); err = PTR_ERR(reason); } else if (unlikely(reason)) { erofs_err(be->sb, "failed to decompress (%s) %s @ pa %llu size %u => %u", @@ -1456,6 +1459,7 @@ static void z_erofs_decompress_kickoff(struct z_erofs_decompressqueue *io, int bios) { struct erofs_sb_info *const sbi = EROFS_SB(io->sb); + int gfp_flag; /* wake up the caller thread for sync decompression */ if (io->sync) { @@ -1488,7 +1492,9 @@ static void z_erofs_decompress_kickoff(struct z_erofs_decompressqueue *io, sbi->opt.sync_decompress = EROFS_SYNC_DECOMPRESS_FORCE_ON; return; } + gfp_flag = memalloc_noio_save(); z_erofs_decompressqueue_work(&io->u.work); + memalloc_noio_restore(gfp_flag); } static void z_erofs_fill_bio_vec(struct bio_vec *bvec, diff --git a/fs/erofs/zmap.c b/fs/erofs/zmap.c index c8d8e129eb4bad..30775502b56da2 100644 --- a/fs/erofs/zmap.c +++ b/fs/erofs/zmap.c @@ -513,6 +513,7 @@ static int z_erofs_map_blocks_ext(struct inode *inode, unsigned int recsz = z_erofs_extent_recsize(vi->z_advise); erofs_off_t pos = round_up(Z_EROFS_MAP_HEADER_END(erofs_iloc(inode) + vi->inode_isize + vi->xattr_isize), recsz); + unsigned int bmask = sb->s_blocksize - 1; bool in_mbox = erofs_inode_in_metabox(inode); erofs_off_t lend = inode->i_size; erofs_off_t l, r, mid, pa, la, lstart; @@ -596,17 +597,17 @@ static int z_erofs_map_blocks_ext(struct inode *inode, map->m_flags |= EROFS_MAP_MAPPED | EROFS_MAP_FULL_MAPPED | EROFS_MAP_ENCODED; fmt = map->m_plen >> Z_EROFS_EXTENT_PLEN_FMT_BIT; + if (map->m_plen & Z_EROFS_EXTENT_PLEN_PARTIAL) + map->m_flags |= EROFS_MAP_PARTIAL_REF; + map->m_plen &= Z_EROFS_EXTENT_PLEN_MASK; if (fmt) map->m_algorithmformat = fmt - 1; - else if (interlaced && !erofs_blkoff(sb, map->m_pa)) + else if (interlaced && !((map->m_pa | map->m_plen) & bmask)) map->m_algorithmformat = Z_EROFS_COMPRESSION_INTERLACED; else map->m_algorithmformat = Z_EROFS_COMPRESSION_SHIFTED; - if (map->m_plen & Z_EROFS_EXTENT_PLEN_PARTIAL) - map->m_flags |= EROFS_MAP_PARTIAL_REF; - map->m_plen &= Z_EROFS_EXTENT_PLEN_MASK; } } map->m_llen = lend - map->m_la; diff --git a/fs/eventpoll.c b/fs/eventpoll.c index 6c36d9dc6926f8..3bdbaf202d4dba 100644 --- a/fs/eventpoll.c +++ b/fs/eventpoll.c @@ -226,6 +226,9 @@ struct eventpoll { */ refcount_t refcount; + /* used to defer freeing past ep_get_upwards_depth_proc() RCU walk */ + struct rcu_head rcu; + #ifdef CONFIG_NET_RX_BUSY_POLL /* used to track busy poll napi_id */ unsigned int napi_id; @@ -819,7 +822,8 @@ static void ep_free(struct eventpoll *ep) mutex_destroy(&ep->mtx); free_uid(ep->user); wakeup_source_unregister(ep->ws); - kfree(ep); + /* ep_get_upwards_depth_proc() may still hold epi->ep under RCU */ + kfree_rcu(ep, rcu); } /* @@ -2061,7 +2065,8 @@ static int ep_poll(struct eventpoll *ep, struct epoll_event __user *events, * @ep: the &struct eventpoll to be currently checked. * @depth: Current depth of the path being checked. * - * Return: depth of the subtree, or INT_MAX if we found a loop or went too deep. + * Return: depth of the subtree, or a value bigger than EP_MAX_NESTS if we found + * a loop or went too deep. */ static int ep_loop_check_proc(struct eventpoll *ep, int depth) { @@ -2080,7 +2085,7 @@ static int ep_loop_check_proc(struct eventpoll *ep, int depth) struct eventpoll *ep_tovisit; ep_tovisit = epi->ffd.file->private_data; if (ep_tovisit == inserting_into || depth > EP_MAX_NESTS) - result = INT_MAX; + result = EP_MAX_NESTS+1; else result = max(result, ep_loop_check_proc(ep_tovisit, depth + 1) + 1); if (result > EP_MAX_NESTS) diff --git a/fs/ext4/Makefile b/fs/ext4/Makefile index 72206a2926765f..d836c3fe311b58 100644 --- a/fs/ext4/Makefile +++ b/fs/ext4/Makefile @@ -14,7 +14,7 @@ ext4-y := balloc.o bitmap.o block_validity.o dir.o ext4_jbd2.o extents.o \ ext4-$(CONFIG_EXT4_FS_POSIX_ACL) += acl.o ext4-$(CONFIG_EXT4_FS_SECURITY) += xattr_security.o -ext4-inode-test-objs += inode-test.o -obj-$(CONFIG_EXT4_KUNIT_TESTS) += ext4-inode-test.o +ext4-test-objs += inode-test.o mballoc-test.o +obj-$(CONFIG_EXT4_KUNIT_TESTS) += ext4-test.o ext4-$(CONFIG_FS_VERITY) += verity.o ext4-$(CONFIG_FS_ENCRYPTION) += crypto.o diff --git a/fs/ext4/crypto.c b/fs/ext4/crypto.c index cf0a0970c09562..f41f320f4437b9 100644 --- a/fs/ext4/crypto.c +++ b/fs/ext4/crypto.c @@ -163,10 +163,17 @@ static int ext4_set_context(struct inode *inode, const void *ctx, size_t len, */ if (handle) { + /* + * Since the inode is new it is ok to pass the + * XATTR_CREATE flag. This is necessary to match the + * remaining journal credits check in the set_handle + * function with the credits allocated for the new + * inode. + */ res = ext4_xattr_set_handle(handle, inode, EXT4_XATTR_INDEX_ENCRYPTION, EXT4_XATTR_NAME_ENCRYPTION_CONTEXT, - ctx, len, 0); + ctx, len, XATTR_CREATE); if (!res) { ext4_set_inode_flag(inode, EXT4_INODE_ENCRYPT); ext4_clear_inode_state(inode, diff --git a/fs/ext4/ext4.h b/fs/ext4/ext4.h index 56112f201cace7..f1c476303f3a97 100644 --- a/fs/ext4/ext4.h +++ b/fs/ext4/ext4.h @@ -1583,6 +1583,7 @@ struct ext4_sb_info { struct proc_dir_entry *s_proc; struct kobject s_kobj; struct completion s_kobj_unregister; + struct mutex s_error_notify_mutex; /* protects sysfs_notify vs kobject_del */ struct super_block *s_sb; struct buffer_head *s_mmp_bh; @@ -1795,6 +1796,10 @@ struct ext4_sb_info { * Main fast commit lock. This lock protects accesses to the * following fields: * ei->i_fc_list, s_fc_dentry_q, s_fc_q, s_fc_bytes, s_fc_bh. + * + * s_fc_lock can be taken from reclaim context (inode eviction) and is + * thus reclaim unsafe. Use ext4_fc_lock()/ext4_fc_unlock() helpers + * when acquiring / releasing the lock. */ struct mutex s_fc_lock; struct buffer_head *s_fc_bh; @@ -1839,6 +1844,18 @@ static inline void ext4_writepages_up_write(struct super_block *sb, int ctx) percpu_up_write(&EXT4_SB(sb)->s_writepages_rwsem); } +static inline int ext4_fc_lock(struct super_block *sb) +{ + mutex_lock(&EXT4_SB(sb)->s_fc_lock); + return memalloc_nofs_save(); +} + +static inline void ext4_fc_unlock(struct super_block *sb, int ctx) +{ + memalloc_nofs_restore(ctx); + mutex_unlock(&EXT4_SB(sb)->s_fc_lock); +} + static inline int ext4_valid_inum(struct super_block *sb, unsigned long ino) { return ino == EXT4_ROOT_INO || @@ -3936,6 +3953,11 @@ static inline bool ext4_inode_can_atomic_write(struct inode *inode) extern int ext4_block_write_begin(handle_t *handle, struct folio *folio, loff_t pos, unsigned len, get_block_t *get_block); + +#if IS_ENABLED(CONFIG_EXT4_KUNIT_TESTS) +#define EXPORT_SYMBOL_FOR_EXT4_TEST(sym) \ + EXPORT_SYMBOL_FOR_MODULES(sym, "ext4-test") +#endif #endif /* __KERNEL__ */ #define EFSBADCRC EBADMSG /* Bad CRC detected */ diff --git a/fs/ext4/extents.c b/fs/ext4/extents.c index 2cf5759ba68940..6857602081c966 100644 --- a/fs/ext4/extents.c +++ b/fs/ext4/extents.c @@ -43,8 +43,13 @@ #define EXT4_EXT_MARK_UNWRIT1 0x2 /* mark first half unwritten */ #define EXT4_EXT_MARK_UNWRIT2 0x4 /* mark second half unwritten */ -#define EXT4_EXT_DATA_VALID1 0x8 /* first half contains valid data */ -#define EXT4_EXT_DATA_VALID2 0x10 /* second half contains valid data */ +/* first half contains valid data */ +#define EXT4_EXT_DATA_ENTIRE_VALID1 0x8 /* has entirely valid data */ +#define EXT4_EXT_DATA_PARTIAL_VALID1 0x10 /* has partially valid data */ +#define EXT4_EXT_DATA_VALID1 (EXT4_EXT_DATA_ENTIRE_VALID1 | \ + EXT4_EXT_DATA_PARTIAL_VALID1) + +#define EXT4_EXT_DATA_VALID2 0x20 /* second half contains valid data */ static __le32 ext4_extent_block_csum(struct inode *inode, struct ext4_extent_header *eh) @@ -1736,6 +1741,13 @@ static int ext4_ext_correct_indexes(handle_t *handle, struct inode *inode, err = ext4_ext_get_access(handle, inode, path + k); if (err) return err; + if (unlikely(path[k].p_idx > EXT_LAST_INDEX(path[k].p_hdr))) { + EXT4_ERROR_INODE(inode, + "path[%d].p_idx %p > EXT_LAST_INDEX %p", + k, path[k].p_idx, + EXT_LAST_INDEX(path[k].p_hdr)); + return -EFSCORRUPTED; + } path[k].p_idx->ei_block = border; err = ext4_ext_dirty(handle, inode, path + k); if (err) @@ -1748,6 +1760,14 @@ static int ext4_ext_correct_indexes(handle_t *handle, struct inode *inode, err = ext4_ext_get_access(handle, inode, path + k); if (err) goto clean; + if (unlikely(path[k].p_idx > EXT_LAST_INDEX(path[k].p_hdr))) { + EXT4_ERROR_INODE(inode, + "path[%d].p_idx %p > EXT_LAST_INDEX %p", + k, path[k].p_idx, + EXT_LAST_INDEX(path[k].p_hdr)); + err = -EFSCORRUPTED; + goto clean; + } path[k].p_idx->ei_block = border; err = ext4_ext_dirty(handle, inode, path + k); if (err) @@ -3190,8 +3210,12 @@ static struct ext4_ext_path *ext4_split_extent_at(handle_t *handle, unsigned int ee_len, depth; int err = 0; - BUG_ON((split_flag & (EXT4_EXT_DATA_VALID1 | EXT4_EXT_DATA_VALID2)) == - (EXT4_EXT_DATA_VALID1 | EXT4_EXT_DATA_VALID2)); + BUG_ON((split_flag & EXT4_EXT_DATA_VALID1) == EXT4_EXT_DATA_VALID1); + BUG_ON((split_flag & EXT4_EXT_DATA_VALID1) && + (split_flag & EXT4_EXT_DATA_VALID2)); + + /* Do not cache extents that are in the process of being modified. */ + flags |= EXT4_EX_NOCACHE; ext_debug(inode, "logical block %llu\n", (unsigned long long)split); @@ -3258,7 +3282,7 @@ static struct ext4_ext_path *ext4_split_extent_at(handle_t *handle, err = PTR_ERR(path); if (err != -ENOSPC && err != -EDQUOT && err != -ENOMEM) - return path; + goto out_path; /* * Get a new path to try to zeroout or fix the extent length. @@ -3272,7 +3296,7 @@ static struct ext4_ext_path *ext4_split_extent_at(handle_t *handle, if (IS_ERR(path)) { EXT4_ERROR_INODE(inode, "Failed split extent on %u, err %ld", split, PTR_ERR(path)); - return path; + goto out_path; } depth = ext_depth(inode); ex = path[depth].p_ext; @@ -3304,6 +3328,23 @@ static struct ext4_ext_path *ext4_split_extent_at(handle_t *handle, } if (!err) { + /* + * The first half contains partially valid data, the + * splitting of this extent has not been completed, fix + * extent length and ext4_split_extent() split will the + * first half again. + */ + if (split_flag & EXT4_EXT_DATA_PARTIAL_VALID1) { + /* + * Drop extent cache to prevent stale unwritten + * extents remaining after zeroing out. + */ + ext4_es_remove_extent(inode, + le32_to_cpu(zero_ex.ee_block), + ext4_ext_get_actual_len(&zero_ex)); + goto fix_extent_len; + } + /* update the extent length and mark as initialized */ ex->ee_len = cpu_to_le16(ee_len); ext4_ext_try_to_merge(handle, inode, path, ex); @@ -3332,6 +3373,10 @@ static struct ext4_ext_path *ext4_split_extent_at(handle_t *handle, ext4_free_ext_path(path); path = ERR_PTR(err); } +out_path: + if (IS_ERR(path)) + /* Remove all remaining potentially stale extents. */ + ext4_es_remove_extent(inode, ee_block, ee_len); ext4_ext_show_leaf(inode, path); return path; } @@ -3366,6 +3411,9 @@ static struct ext4_ext_path *ext4_split_extent(handle_t *handle, ee_len = ext4_ext_get_actual_len(ex); unwritten = ext4_ext_is_unwritten(ex); + /* Do not cache extents that are in the process of being modified. */ + flags |= EXT4_EX_NOCACHE; + if (map->m_lblk + map->m_len < ee_block + ee_len) { split_flag1 = split_flag & EXT4_EXT_MAY_ZEROOUT; flags1 = flags | EXT4_GET_BLOCKS_SPLIT_NOMERGE; @@ -3373,7 +3421,9 @@ static struct ext4_ext_path *ext4_split_extent(handle_t *handle, split_flag1 |= EXT4_EXT_MARK_UNWRIT1 | EXT4_EXT_MARK_UNWRIT2; if (split_flag & EXT4_EXT_DATA_VALID2) - split_flag1 |= EXT4_EXT_DATA_VALID1; + split_flag1 |= map->m_lblk > ee_block ? + EXT4_EXT_DATA_PARTIAL_VALID1 : + EXT4_EXT_DATA_ENTIRE_VALID1; path = ext4_split_extent_at(handle, inode, path, map->m_lblk + map->m_len, split_flag1, flags1); if (IS_ERR(path)) @@ -3728,16 +3778,20 @@ static struct ext4_ext_path *ext4_split_convert_extents(handle_t *handle, /* Convert to unwritten */ if (flags & EXT4_GET_BLOCKS_CONVERT_UNWRITTEN) { - split_flag |= EXT4_EXT_DATA_VALID1; - /* Convert to initialized */ - } else if (flags & EXT4_GET_BLOCKS_CONVERT) { + split_flag |= EXT4_EXT_DATA_ENTIRE_VALID1; + /* Split the existing unwritten extent */ + } else if (flags & (EXT4_GET_BLOCKS_UNWRIT_EXT | + EXT4_GET_BLOCKS_CONVERT)) { /* * It is safe to convert extent to initialized via explicit * zeroout only if extent is fully inside i_size or new_size. */ split_flag |= ee_block + ee_len <= eof_block ? EXT4_EXT_MAY_ZEROOUT : 0; - split_flag |= (EXT4_EXT_MARK_UNWRIT2 | EXT4_EXT_DATA_VALID2); + split_flag |= EXT4_EXT_MARK_UNWRIT2; + /* Convert to initialized */ + if (flags & EXT4_GET_BLOCKS_CONVERT) + split_flag |= EXT4_EXT_DATA_VALID2; } flags |= EXT4_GET_BLOCKS_SPLIT_NOMERGE; return ext4_split_extent(handle, inode, path, map, split_flag, flags, @@ -3770,6 +3824,8 @@ ext4_convert_unwritten_extents_endio(handle_t *handle, struct inode *inode, * illegal. */ if (ee_block != map->m_lblk || ee_len > map->m_len) { + int flags = EXT4_GET_BLOCKS_CONVERT | + EXT4_GET_BLOCKS_METADATA_NOFAIL; #ifdef CONFIG_EXT4_DEBUG ext4_warning(inode->i_sb, "Inode (%ld) finished: extent logical block %llu," " len %u; IO logical block %llu, len %u", @@ -3777,7 +3833,7 @@ ext4_convert_unwritten_extents_endio(handle_t *handle, struct inode *inode, (unsigned long long)map->m_lblk, map->m_len); #endif path = ext4_split_convert_extents(handle, inode, map, path, - EXT4_GET_BLOCKS_CONVERT, NULL); + flags, NULL); if (IS_ERR(path)) return path; @@ -3816,6 +3872,7 @@ static struct ext4_ext_path * convert_initialized_extent(handle_t *handle, struct inode *inode, struct ext4_map_blocks *map, struct ext4_ext_path *path, + int flags, unsigned int *allocated) { struct ext4_extent *ex; @@ -3841,11 +3898,11 @@ convert_initialized_extent(handle_t *handle, struct inode *inode, if (ee_block != map->m_lblk || ee_len > map->m_len) { path = ext4_split_convert_extents(handle, inode, map, path, - EXT4_GET_BLOCKS_CONVERT_UNWRITTEN, NULL); + flags, NULL); if (IS_ERR(path)) return path; - path = ext4_find_extent(inode, map->m_lblk, path, 0); + path = ext4_find_extent(inode, map->m_lblk, path, flags); if (IS_ERR(path)) return path; depth = ext_depth(inode); @@ -3913,7 +3970,7 @@ ext4_ext_handle_unwritten_extents(handle_t *handle, struct inode *inode, /* get_block() before submitting IO, split the extent */ if (flags & EXT4_GET_BLOCKS_SPLIT_NOMERGE) { path = ext4_split_convert_extents(handle, inode, map, path, - flags | EXT4_GET_BLOCKS_CONVERT, allocated); + flags, allocated); if (IS_ERR(path)) return path; /* @@ -4257,7 +4314,7 @@ int ext4_ext_map_blocks(handle_t *handle, struct inode *inode, if ((!ext4_ext_is_unwritten(ex)) && (flags & EXT4_GET_BLOCKS_CONVERT_UNWRITTEN)) { path = convert_initialized_extent(handle, - inode, map, path, &allocated); + inode, map, path, flags, &allocated); if (IS_ERR(path)) err = PTR_ERR(path); goto out; @@ -4404,9 +4461,13 @@ int ext4_ext_map_blocks(handle_t *handle, struct inode *inode, path = ext4_ext_insert_extent(handle, inode, path, &newex, flags); if (IS_ERR(path)) { err = PTR_ERR(path); - if (allocated_clusters) { + /* + * Gracefully handle out of space conditions. If the filesystem + * is inconsistent, we'll just leak allocated blocks to avoid + * causing even more damage. + */ + if (allocated_clusters && (err == -EDQUOT || err == -ENOSPC)) { int fb_flags = 0; - /* * free data blocks we just allocated. * not a good idea to call discard here directly, @@ -5375,7 +5436,8 @@ ext4_ext_shift_extents(struct inode *inode, handle_t *handle, if (!extent) { EXT4_ERROR_INODE(inode, "unexpected hole at %lu", (unsigned long) *iterator); - return -EFSCORRUPTED; + ret = -EFSCORRUPTED; + goto out; } if (SHIFT == SHIFT_LEFT && *iterator > le32_to_cpu(extent->ee_block)) { diff --git a/fs/ext4/fast_commit.c b/fs/ext4/fast_commit.c index fa66b08de9994e..bab522ca1573df 100644 --- a/fs/ext4/fast_commit.c +++ b/fs/ext4/fast_commit.c @@ -231,16 +231,16 @@ static bool ext4_fc_disabled(struct super_block *sb) void ext4_fc_del(struct inode *inode) { struct ext4_inode_info *ei = EXT4_I(inode); - struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); struct ext4_fc_dentry_update *fc_dentry; wait_queue_head_t *wq; + int alloc_ctx; if (ext4_fc_disabled(inode->i_sb)) return; - mutex_lock(&sbi->s_fc_lock); + alloc_ctx = ext4_fc_lock(inode->i_sb); if (list_empty(&ei->i_fc_list) && list_empty(&ei->i_fc_dilist)) { - mutex_unlock(&sbi->s_fc_lock); + ext4_fc_unlock(inode->i_sb, alloc_ctx); return; } @@ -275,9 +275,9 @@ void ext4_fc_del(struct inode *inode) #endif prepare_to_wait(wq, &wait.wq_entry, TASK_UNINTERRUPTIBLE); if (ext4_test_inode_state(inode, EXT4_STATE_FC_FLUSHING_DATA)) { - mutex_unlock(&sbi->s_fc_lock); + ext4_fc_unlock(inode->i_sb, alloc_ctx); schedule(); - mutex_lock(&sbi->s_fc_lock); + alloc_ctx = ext4_fc_lock(inode->i_sb); } finish_wait(wq, &wait.wq_entry); } @@ -288,7 +288,7 @@ void ext4_fc_del(struct inode *inode) * dentry create references, since it is not needed to log it anyways. */ if (list_empty(&ei->i_fc_dilist)) { - mutex_unlock(&sbi->s_fc_lock); + ext4_fc_unlock(inode->i_sb, alloc_ctx); return; } @@ -298,7 +298,7 @@ void ext4_fc_del(struct inode *inode) list_del_init(&fc_dentry->fcd_dilist); WARN_ON(!list_empty(&ei->i_fc_dilist)); - mutex_unlock(&sbi->s_fc_lock); + ext4_fc_unlock(inode->i_sb, alloc_ctx); release_dentry_name_snapshot(&fc_dentry->fcd_name); kmem_cache_free(ext4_fc_dentry_cachep, fc_dentry); @@ -315,6 +315,7 @@ void ext4_fc_mark_ineligible(struct super_block *sb, int reason, handle_t *handl tid_t tid; bool has_transaction = true; bool is_ineligible; + int alloc_ctx; if (ext4_fc_disabled(sb)) return; @@ -329,12 +330,12 @@ void ext4_fc_mark_ineligible(struct super_block *sb, int reason, handle_t *handl has_transaction = false; read_unlock(&sbi->s_journal->j_state_lock); } - mutex_lock(&sbi->s_fc_lock); + alloc_ctx = ext4_fc_lock(sb); is_ineligible = ext4_test_mount_flag(sb, EXT4_MF_FC_INELIGIBLE); if (has_transaction && (!is_ineligible || tid_gt(tid, sbi->s_fc_ineligible_tid))) sbi->s_fc_ineligible_tid = tid; ext4_set_mount_flag(sb, EXT4_MF_FC_INELIGIBLE); - mutex_unlock(&sbi->s_fc_lock); + ext4_fc_unlock(sb, alloc_ctx); WARN_ON(reason >= EXT4_FC_REASON_MAX); sbi->s_fc_stats.fc_ineligible_reason_count[reason]++; } @@ -358,6 +359,7 @@ static int ext4_fc_track_template( struct ext4_inode_info *ei = EXT4_I(inode); struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); tid_t tid = 0; + int alloc_ctx; int ret; tid = handle->h_transaction->t_tid; @@ -373,14 +375,14 @@ static int ext4_fc_track_template( if (!enqueue) return ret; - mutex_lock(&sbi->s_fc_lock); + alloc_ctx = ext4_fc_lock(inode->i_sb); if (list_empty(&EXT4_I(inode)->i_fc_list)) list_add_tail(&EXT4_I(inode)->i_fc_list, (sbi->s_journal->j_flags & JBD2_FULL_COMMIT_ONGOING || sbi->s_journal->j_flags & JBD2_FAST_COMMIT_ONGOING) ? &sbi->s_fc_q[FC_Q_STAGING] : &sbi->s_fc_q[FC_Q_MAIN]); - mutex_unlock(&sbi->s_fc_lock); + ext4_fc_unlock(inode->i_sb, alloc_ctx); return ret; } @@ -402,6 +404,7 @@ static int __track_dentry_update(handle_t *handle, struct inode *inode, struct inode *dir = dentry->d_parent->d_inode; struct super_block *sb = inode->i_sb; struct ext4_sb_info *sbi = EXT4_SB(sb); + int alloc_ctx; spin_unlock(&ei->i_fc_lock); @@ -425,7 +428,7 @@ static int __track_dentry_update(handle_t *handle, struct inode *inode, take_dentry_name_snapshot(&node->fcd_name, dentry); INIT_LIST_HEAD(&node->fcd_dilist); INIT_LIST_HEAD(&node->fcd_list); - mutex_lock(&sbi->s_fc_lock); + alloc_ctx = ext4_fc_lock(sb); if (sbi->s_journal->j_flags & JBD2_FULL_COMMIT_ONGOING || sbi->s_journal->j_flags & JBD2_FAST_COMMIT_ONGOING) list_add_tail(&node->fcd_list, @@ -446,7 +449,7 @@ static int __track_dentry_update(handle_t *handle, struct inode *inode, WARN_ON(!list_empty(&ei->i_fc_dilist)); list_add_tail(&node->fcd_dilist, &ei->i_fc_dilist); } - mutex_unlock(&sbi->s_fc_lock); + ext4_fc_unlock(sb, alloc_ctx); spin_lock(&ei->i_fc_lock); return 0; @@ -972,13 +975,13 @@ static int ext4_fc_flush_data(journal_t *journal) int ret = 0; list_for_each_entry(ei, &sbi->s_fc_q[FC_Q_MAIN], i_fc_list) { - ret = jbd2_submit_inode_data(journal, ei->jinode); + ret = jbd2_submit_inode_data(journal, READ_ONCE(ei->jinode)); if (ret) return ret; } list_for_each_entry(ei, &sbi->s_fc_q[FC_Q_MAIN], i_fc_list) { - ret = jbd2_wait_inode_data(journal, ei->jinode); + ret = jbd2_wait_inode_data(journal, READ_ONCE(ei->jinode)); if (ret) return ret; } @@ -1046,18 +1049,19 @@ static int ext4_fc_perform_commit(journal_t *journal) struct blk_plug plug; int ret = 0; u32 crc = 0; + int alloc_ctx; /* * Step 1: Mark all inodes on s_fc_q[MAIN] with * EXT4_STATE_FC_FLUSHING_DATA. This prevents these inodes from being * freed until the data flush is over. */ - mutex_lock(&sbi->s_fc_lock); + alloc_ctx = ext4_fc_lock(sb); list_for_each_entry(iter, &sbi->s_fc_q[FC_Q_MAIN], i_fc_list) { ext4_set_inode_state(&iter->vfs_inode, EXT4_STATE_FC_FLUSHING_DATA); } - mutex_unlock(&sbi->s_fc_lock); + ext4_fc_unlock(sb, alloc_ctx); /* Step 2: Flush data for all the eligible inodes. */ ret = ext4_fc_flush_data(journal); @@ -1067,7 +1071,7 @@ static int ext4_fc_perform_commit(journal_t *journal) * any error from step 2. This ensures that waiters waiting on * EXT4_STATE_FC_FLUSHING_DATA can resume. */ - mutex_lock(&sbi->s_fc_lock); + alloc_ctx = ext4_fc_lock(sb); list_for_each_entry(iter, &sbi->s_fc_q[FC_Q_MAIN], i_fc_list) { ext4_clear_inode_state(&iter->vfs_inode, EXT4_STATE_FC_FLUSHING_DATA); @@ -1084,7 +1088,7 @@ static int ext4_fc_perform_commit(journal_t *journal) * prepare_to_wait() in ext4_fc_del(). */ smp_mb(); - mutex_unlock(&sbi->s_fc_lock); + ext4_fc_unlock(sb, alloc_ctx); /* * If we encountered error in Step 2, return it now after clearing @@ -1101,12 +1105,12 @@ static int ext4_fc_perform_commit(journal_t *journal) * previous handles are now drained. We now mark the inodes on the * commit queue as being committed. */ - mutex_lock(&sbi->s_fc_lock); + alloc_ctx = ext4_fc_lock(sb); list_for_each_entry(iter, &sbi->s_fc_q[FC_Q_MAIN], i_fc_list) { ext4_set_inode_state(&iter->vfs_inode, EXT4_STATE_FC_COMMITTING); } - mutex_unlock(&sbi->s_fc_lock); + ext4_fc_unlock(sb, alloc_ctx); jbd2_journal_unlock_updates(journal); /* @@ -1117,6 +1121,7 @@ static int ext4_fc_perform_commit(journal_t *journal) blkdev_issue_flush(journal->j_fs_dev); blk_start_plug(&plug); + alloc_ctx = ext4_fc_lock(sb); /* Step 6: Write fast commit blocks to disk. */ if (sbi->s_fc_bytes == 0) { /* @@ -1134,7 +1139,6 @@ static int ext4_fc_perform_commit(journal_t *journal) } /* Step 6.2: Now write all the dentry updates. */ - mutex_lock(&sbi->s_fc_lock); ret = ext4_fc_commit_dentry_updates(journal, &crc); if (ret) goto out; @@ -1156,7 +1160,7 @@ static int ext4_fc_perform_commit(journal_t *journal) ret = ext4_fc_write_tail(sb, crc); out: - mutex_unlock(&sbi->s_fc_lock); + ext4_fc_unlock(sb, alloc_ctx); blk_finish_plug(&plug); return ret; } @@ -1290,6 +1294,7 @@ static void ext4_fc_cleanup(journal_t *journal, int full, tid_t tid) struct ext4_sb_info *sbi = EXT4_SB(sb); struct ext4_inode_info *ei; struct ext4_fc_dentry_update *fc_dentry; + int alloc_ctx; if (full && sbi->s_fc_bh) sbi->s_fc_bh = NULL; @@ -1297,7 +1302,7 @@ static void ext4_fc_cleanup(journal_t *journal, int full, tid_t tid) trace_ext4_fc_cleanup(journal, full, tid); jbd2_fc_release_bufs(journal); - mutex_lock(&sbi->s_fc_lock); + alloc_ctx = ext4_fc_lock(sb); while (!list_empty(&sbi->s_fc_q[FC_Q_MAIN])) { ei = list_first_entry(&sbi->s_fc_q[FC_Q_MAIN], struct ext4_inode_info, @@ -1356,7 +1361,7 @@ static void ext4_fc_cleanup(journal_t *journal, int full, tid_t tid) if (full) sbi->s_fc_bytes = 0; - mutex_unlock(&sbi->s_fc_lock); + ext4_fc_unlock(sb, alloc_ctx); trace_ext4_fc_stats(sb); } @@ -1608,19 +1613,21 @@ static int ext4_fc_replay_inode(struct super_block *sb, /* Immediately update the inode on disk. */ ret = ext4_handle_dirty_metadata(NULL, NULL, iloc.bh); if (ret) - goto out; + goto out_brelse; ret = sync_dirty_buffer(iloc.bh); if (ret) - goto out; + goto out_brelse; ret = ext4_mark_inode_used(sb, ino); if (ret) - goto out; + goto out_brelse; /* Given that we just wrote the inode on disk, this SHOULD succeed. */ inode = ext4_iget(sb, ino, EXT4_IGET_NORMAL); if (IS_ERR(inode)) { ext4_debug("Inode not found."); - return -EFSCORRUPTED; + inode = NULL; + ret = -EFSCORRUPTED; + goto out_brelse; } /* @@ -1637,13 +1644,14 @@ static int ext4_fc_replay_inode(struct super_block *sb, ext4_inode_csum_set(inode, ext4_raw_inode(&iloc), EXT4_I(inode)); ret = ext4_handle_dirty_metadata(NULL, NULL, iloc.bh); sync_dirty_buffer(iloc.bh); +out_brelse: brelse(iloc.bh); out: iput(inode); if (!ret) blkdev_issue_flush(sb->s_bdev); - return 0; + return ret; } /* diff --git a/fs/ext4/fsync.c b/fs/ext4/fsync.c index e476c6de307407..bd8f230fa507e7 100644 --- a/fs/ext4/fsync.c +++ b/fs/ext4/fsync.c @@ -83,11 +83,23 @@ static int ext4_fsync_nojournal(struct file *file, loff_t start, loff_t end, int datasync, bool *needs_barrier) { struct inode *inode = file->f_inode; + struct writeback_control wbc = { + .sync_mode = WB_SYNC_ALL, + .nr_to_write = 0, + }; int ret; ret = generic_buffers_fsync_noflush(file, start, end, datasync); - if (!ret) - ret = ext4_sync_parent(inode); + if (ret) + return ret; + + /* Force writeout of inode table buffer to disk */ + ret = ext4_write_inode(inode, &wbc); + if (ret) + return ret; + + ret = ext4_sync_parent(inode); + if (test_opt(inode->i_sb, BARRIER)) *needs_barrier = true; diff --git a/fs/ext4/ialloc.c b/fs/ext4/ialloc.c index b20a1bf866abed..b1bc1950c9f03b 100644 --- a/fs/ext4/ialloc.c +++ b/fs/ext4/ialloc.c @@ -686,6 +686,12 @@ static int recently_deleted(struct super_block *sb, ext4_group_t group, int ino) if (unlikely(!gdp)) return 0; + /* Inode was never used in this filesystem? */ + if (ext4_has_group_desc_csum(sb) && + (gdp->bg_flags & cpu_to_le16(EXT4_BG_INODE_UNINIT) || + ino >= EXT4_INODES_PER_GROUP(sb) - ext4_itable_unused_count(sb, gdp))) + return 0; + bh = sb_find_get_block(sb, ext4_inode_table(sb, gdp) + (ino / inodes_per_block)); if (!bh || !buffer_uptodate(bh)) diff --git a/fs/ext4/inline.c b/fs/ext4/inline.c index 1f6bc05593df16..408677fa819672 100644 --- a/fs/ext4/inline.c +++ b/fs/ext4/inline.c @@ -522,7 +522,15 @@ static int ext4_read_inline_folio(struct inode *inode, struct folio *folio) goto out; len = min_t(size_t, ext4_get_inline_size(inode), i_size_read(inode)); - BUG_ON(len > PAGE_SIZE); + + if (len > PAGE_SIZE) { + ext4_error_inode(inode, __func__, __LINE__, 0, + "inline size %zu exceeds PAGE_SIZE", len); + ret = -EFSCORRUPTED; + brelse(iloc.bh); + goto out; + } + kaddr = kmap_local_folio(folio, 0); ret = ext4_read_inline_data(inode, kaddr, len, &iloc); kaddr = folio_zero_tail(folio, len, kaddr + len); diff --git a/fs/ext4/inode.c b/fs/ext4/inode.c index 0c466ccbed6967..625cfbf61582c2 100644 --- a/fs/ext4/inode.c +++ b/fs/ext4/inode.c @@ -126,6 +126,8 @@ void ext4_inode_csum_set(struct inode *inode, struct ext4_inode *raw, static inline int ext4_begin_ordered_truncate(struct inode *inode, loff_t new_size) { + struct jbd2_inode *jinode = READ_ONCE(EXT4_I(inode)->jinode); + trace_ext4_begin_ordered_truncate(inode, new_size); /* * If jinode is zero, then we never opened the file for @@ -133,10 +135,10 @@ static inline int ext4_begin_ordered_truncate(struct inode *inode, * jbd2_journal_begin_ordered_truncate() since there's no * outstanding writes we need to flush. */ - if (!EXT4_I(inode)->jinode) + if (!jinode) return 0; return jbd2_journal_begin_ordered_truncate(EXT4_JOURNAL(inode), - EXT4_I(inode)->jinode, + jinode, new_size); } @@ -182,6 +184,14 @@ void ext4_evict_inode(struct inode *inode) if (EXT4_I(inode)->i_flags & EXT4_EA_INODE_FL) ext4_evict_ea_inode(inode); if (inode->i_nlink) { + /* + * If there's dirty page will lead to data loss, user + * could see stale data. + */ + if (unlikely(!ext4_emergency_state(inode->i_sb) && + mapping_tagged(&inode->i_data, PAGECACHE_TAG_DIRTY))) + ext4_warning_inode(inode, "data will be lost"); + truncate_inode_pages_final(&inode->i_data); goto no_delete; @@ -4499,8 +4509,13 @@ int ext4_inode_attach_jinode(struct inode *inode) spin_unlock(&inode->i_lock); return -ENOMEM; } - ei->jinode = jinode; - jbd2_journal_init_jbd_inode(ei->jinode, inode); + jbd2_journal_init_jbd_inode(jinode, inode); + /* + * Publish ->jinode only after it is fully initialized so that + * readers never observe a partially initialized jbd2_inode. + */ + smp_wmb(); + WRITE_ONCE(ei->jinode, jinode); jinode = NULL; } spin_unlock(&inode->i_lock); @@ -5449,18 +5464,36 @@ struct inode *__ext4_iget(struct super_block *sb, unsigned long ino, inode->i_op = &ext4_encrypted_symlink_inode_operations; } else if (ext4_inode_is_fast_symlink(inode)) { inode->i_op = &ext4_fast_symlink_inode_operations; - if (inode->i_size == 0 || - inode->i_size >= sizeof(ei->i_data) || - strnlen((char *)ei->i_data, inode->i_size + 1) != - inode->i_size) { - ext4_error_inode(inode, function, line, 0, - "invalid fast symlink length %llu", - (unsigned long long)inode->i_size); - ret = -EFSCORRUPTED; - goto bad_inode; + + /* + * Orphan cleanup can see inodes with i_size == 0 + * and i_data uninitialized. Skip size checks in + * that case. This is safe because the first thing + * ext4_evict_inode() does for fast symlinks is + * clearing of i_data and i_size. + */ + if ((EXT4_SB(sb)->s_mount_state & EXT4_ORPHAN_FS)) { + if (inode->i_nlink != 0) { + ext4_error_inode(inode, function, line, 0, + "invalid orphan symlink nlink %d", + inode->i_nlink); + ret = -EFSCORRUPTED; + goto bad_inode; + } + } else { + if (inode->i_size == 0 || + inode->i_size >= sizeof(ei->i_data) || + strnlen((char *)ei->i_data, inode->i_size + 1) != + inode->i_size) { + ext4_error_inode(inode, function, line, 0, + "invalid fast symlink length %llu", + (unsigned long long)inode->i_size); + ret = -EFSCORRUPTED; + goto bad_inode; + } + inode_set_cached_link(inode, (char *)ei->i_data, + inode->i_size); } - inode_set_cached_link(inode, (char *)ei->i_data, - inode->i_size); } else { inode->i_op = &ext4_symlink_inode_operations; } @@ -5901,6 +5934,18 @@ int ext4_setattr(struct mnt_idmap *idmap, struct dentry *dentry, if (attr->ia_size == inode->i_size) inc_ivers = false; + /* + * If file has inline data but new size exceeds inline capacity, + * convert to extent-based storage first to prevent inconsistent + * state (inline flag set but size exceeds inline capacity). + */ + if (ext4_has_inline_data(inode) && + attr->ia_size > EXT4_I(inode)->i_inline_size) { + error = ext4_convert_inline_data(inode); + if (error) + goto err_out; + } + if (shrink) { if (ext4_should_order_data(inode)) { error = ext4_begin_ordered_truncate(inode, diff --git a/fs/ext4/ioctl.c b/fs/ext4/ioctl.c index 7ce0fc40aec2fb..e5e197ac7d88b2 100644 --- a/fs/ext4/ioctl.c +++ b/fs/ext4/ioctl.c @@ -966,6 +966,7 @@ static long ext4_ioctl_group_add(struct file *file, err = ext4_group_add(sb, input); if (EXT4_SB(sb)->s_journal) { + ext4_fc_mark_ineligible(sb, EXT4_FC_REASON_RESIZE, NULL); jbd2_journal_lock_updates(EXT4_SB(sb)->s_journal); err2 = jbd2_journal_flush(EXT4_SB(sb)->s_journal, 0); jbd2_journal_unlock_updates(EXT4_SB(sb)->s_journal); @@ -1611,6 +1612,8 @@ static long __ext4_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) err = ext4_group_extend(sb, EXT4_SB(sb)->s_es, n_blocks_count); if (EXT4_SB(sb)->s_journal) { + ext4_fc_mark_ineligible(sb, EXT4_FC_REASON_RESIZE, + NULL); jbd2_journal_lock_updates(EXT4_SB(sb)->s_journal); err2 = jbd2_journal_flush(EXT4_SB(sb)->s_journal, 0); jbd2_journal_unlock_updates(EXT4_SB(sb)->s_journal); diff --git a/fs/ext4/mballoc-test.c b/fs/ext4/mballoc-test.c index a9416b20ff64c9..749ed2fc224159 100644 --- a/fs/ext4/mballoc-test.c +++ b/fs/ext4/mballoc-test.c @@ -8,6 +8,7 @@ #include #include "ext4.h" +#include "mballoc.h" struct mbt_grp_ctx { struct buffer_head bitmap_bh; @@ -337,7 +338,7 @@ ext4_mb_mark_context_stub(handle_t *handle, struct super_block *sb, bool state, if (state) mb_set_bits(bitmap_bh->b_data, blkoff, len); else - mb_clear_bits(bitmap_bh->b_data, blkoff, len); + mb_clear_bits_test(bitmap_bh->b_data, blkoff, len); return 0; } @@ -414,14 +415,14 @@ static void test_new_blocks_simple(struct kunit *test) /* get block at goal */ ar.goal = ext4_group_first_block_no(sb, goal_group); - found = ext4_mb_new_blocks_simple(&ar, &err); + found = ext4_mb_new_blocks_simple_test(&ar, &err); KUNIT_ASSERT_EQ_MSG(test, ar.goal, found, "failed to alloc block at goal, expected %llu found %llu", ar.goal, found); /* get block after goal in goal group */ ar.goal = ext4_group_first_block_no(sb, goal_group); - found = ext4_mb_new_blocks_simple(&ar, &err); + found = ext4_mb_new_blocks_simple_test(&ar, &err); KUNIT_ASSERT_EQ_MSG(test, ar.goal + EXT4_C2B(sbi, 1), found, "failed to alloc block after goal in goal group, expected %llu found %llu", ar.goal + 1, found); @@ -429,7 +430,7 @@ static void test_new_blocks_simple(struct kunit *test) /* get block after goal group */ mbt_ctx_mark_used(sb, goal_group, 0, EXT4_CLUSTERS_PER_GROUP(sb)); ar.goal = ext4_group_first_block_no(sb, goal_group); - found = ext4_mb_new_blocks_simple(&ar, &err); + found = ext4_mb_new_blocks_simple_test(&ar, &err); KUNIT_ASSERT_EQ_MSG(test, ext4_group_first_block_no(sb, goal_group + 1), found, "failed to alloc block after goal group, expected %llu found %llu", @@ -439,7 +440,7 @@ static void test_new_blocks_simple(struct kunit *test) for (i = goal_group; i < ext4_get_groups_count(sb); i++) mbt_ctx_mark_used(sb, i, 0, EXT4_CLUSTERS_PER_GROUP(sb)); ar.goal = ext4_group_first_block_no(sb, goal_group); - found = ext4_mb_new_blocks_simple(&ar, &err); + found = ext4_mb_new_blocks_simple_test(&ar, &err); KUNIT_ASSERT_EQ_MSG(test, ext4_group_first_block_no(sb, 0) + EXT4_C2B(sbi, 1), found, "failed to alloc block before goal group, expected %llu found %llu", @@ -449,7 +450,7 @@ static void test_new_blocks_simple(struct kunit *test) for (i = 0; i < ext4_get_groups_count(sb); i++) mbt_ctx_mark_used(sb, i, 0, EXT4_CLUSTERS_PER_GROUP(sb)); ar.goal = ext4_group_first_block_no(sb, goal_group); - found = ext4_mb_new_blocks_simple(&ar, &err); + found = ext4_mb_new_blocks_simple_test(&ar, &err); KUNIT_ASSERT_NE_MSG(test, err, 0, "unexpectedly get block when no block is available"); } @@ -493,16 +494,16 @@ validate_free_blocks_simple(struct kunit *test, struct super_block *sb, continue; bitmap = mbt_ctx_bitmap(sb, i); - bit = mb_find_next_zero_bit(bitmap, max, 0); + bit = mb_find_next_zero_bit_test(bitmap, max, 0); KUNIT_ASSERT_EQ_MSG(test, bit, max, "free block on unexpected group %d", i); } bitmap = mbt_ctx_bitmap(sb, goal_group); - bit = mb_find_next_zero_bit(bitmap, max, 0); + bit = mb_find_next_zero_bit_test(bitmap, max, 0); KUNIT_ASSERT_EQ(test, bit, start); - bit = mb_find_next_bit(bitmap, max, bit + 1); + bit = mb_find_next_bit_test(bitmap, max, bit + 1); KUNIT_ASSERT_EQ(test, bit, start + len); } @@ -525,7 +526,7 @@ test_free_blocks_simple_range(struct kunit *test, ext4_group_t goal_group, block = ext4_group_first_block_no(sb, goal_group) + EXT4_C2B(sbi, start); - ext4_free_blocks_simple(inode, block, len); + ext4_free_blocks_simple_test(inode, block, len); validate_free_blocks_simple(test, sb, goal_group, start, len); mbt_ctx_mark_used(sb, goal_group, 0, EXT4_CLUSTERS_PER_GROUP(sb)); } @@ -567,15 +568,15 @@ test_mark_diskspace_used_range(struct kunit *test, bitmap = mbt_ctx_bitmap(sb, TEST_GOAL_GROUP); memset(bitmap, 0, sb->s_blocksize); - ret = ext4_mb_mark_diskspace_used(ac, NULL, 0); + ret = ext4_mb_mark_diskspace_used_test(ac, NULL); KUNIT_ASSERT_EQ(test, ret, 0); max = EXT4_CLUSTERS_PER_GROUP(sb); - i = mb_find_next_bit(bitmap, max, 0); + i = mb_find_next_bit_test(bitmap, max, 0); KUNIT_ASSERT_EQ(test, i, start); - i = mb_find_next_zero_bit(bitmap, max, i + 1); + i = mb_find_next_zero_bit_test(bitmap, max, i + 1); KUNIT_ASSERT_EQ(test, i, start + len); - i = mb_find_next_bit(bitmap, max, i + 1); + i = mb_find_next_bit_test(bitmap, max, i + 1); KUNIT_ASSERT_EQ(test, max, i); } @@ -618,54 +619,54 @@ static void mbt_generate_buddy(struct super_block *sb, void *buddy, max = EXT4_CLUSTERS_PER_GROUP(sb); bb_h = buddy + sbi->s_mb_offsets[1]; - off = mb_find_next_zero_bit(bb, max, 0); + off = mb_find_next_zero_bit_test(bb, max, 0); grp->bb_first_free = off; while (off < max) { grp->bb_counters[0]++; grp->bb_free++; - if (!(off & 1) && !mb_test_bit(off + 1, bb)) { + if (!(off & 1) && !mb_test_bit_test(off + 1, bb)) { grp->bb_free++; grp->bb_counters[0]--; - mb_clear_bit(off >> 1, bb_h); + mb_clear_bit_test(off >> 1, bb_h); grp->bb_counters[1]++; grp->bb_largest_free_order = 1; off++; } - off = mb_find_next_zero_bit(bb, max, off + 1); + off = mb_find_next_zero_bit_test(bb, max, off + 1); } for (order = 1; order < MB_NUM_ORDERS(sb) - 1; order++) { bb = buddy + sbi->s_mb_offsets[order]; bb_h = buddy + sbi->s_mb_offsets[order + 1]; max = max >> 1; - off = mb_find_next_zero_bit(bb, max, 0); + off = mb_find_next_zero_bit_test(bb, max, 0); while (off < max) { - if (!(off & 1) && !mb_test_bit(off + 1, bb)) { + if (!(off & 1) && !mb_test_bit_test(off + 1, bb)) { mb_set_bits(bb, off, 2); grp->bb_counters[order] -= 2; - mb_clear_bit(off >> 1, bb_h); + mb_clear_bit_test(off >> 1, bb_h); grp->bb_counters[order + 1]++; grp->bb_largest_free_order = order + 1; off++; } - off = mb_find_next_zero_bit(bb, max, off + 1); + off = mb_find_next_zero_bit_test(bb, max, off + 1); } } max = EXT4_CLUSTERS_PER_GROUP(sb); - off = mb_find_next_zero_bit(bitmap, max, 0); + off = mb_find_next_zero_bit_test(bitmap, max, 0); while (off < max) { grp->bb_fragments++; - off = mb_find_next_bit(bitmap, max, off + 1); + off = mb_find_next_bit_test(bitmap, max, off + 1); if (off + 1 >= max) break; - off = mb_find_next_zero_bit(bitmap, max, off + 1); + off = mb_find_next_zero_bit_test(bitmap, max, off + 1); } } @@ -707,7 +708,7 @@ do_test_generate_buddy(struct kunit *test, struct super_block *sb, void *bitmap, /* needed by validation in ext4_mb_generate_buddy */ ext4_grp->bb_free = mbt_grp->bb_free; memset(ext4_buddy, 0xff, sb->s_blocksize); - ext4_mb_generate_buddy(sb, ext4_buddy, bitmap, TEST_GOAL_GROUP, + ext4_mb_generate_buddy_test(sb, ext4_buddy, bitmap, TEST_GOAL_GROUP, ext4_grp); KUNIT_ASSERT_EQ(test, memcmp(mbt_buddy, ext4_buddy, sb->s_blocksize), @@ -761,7 +762,7 @@ test_mb_mark_used_range(struct kunit *test, struct ext4_buddy *e4b, ex.fe_group = TEST_GOAL_GROUP; ext4_lock_group(sb, TEST_GOAL_GROUP); - mb_mark_used(e4b, &ex); + mb_mark_used_test(e4b, &ex); ext4_unlock_group(sb, TEST_GOAL_GROUP); mb_set_bits(bitmap, start, len); @@ -770,7 +771,7 @@ test_mb_mark_used_range(struct kunit *test, struct ext4_buddy *e4b, memset(buddy, 0xff, sb->s_blocksize); for (i = 0; i < MB_NUM_ORDERS(sb); i++) grp->bb_counters[i] = 0; - ext4_mb_generate_buddy(sb, buddy, bitmap, 0, grp); + ext4_mb_generate_buddy_test(sb, buddy, bitmap, 0, grp); KUNIT_ASSERT_EQ(test, memcmp(buddy, e4b->bd_buddy, sb->s_blocksize), 0); @@ -799,7 +800,7 @@ static void test_mb_mark_used(struct kunit *test) bb_counters[MB_NUM_ORDERS(sb)]), GFP_KERNEL); KUNIT_ASSERT_NOT_ERR_OR_NULL(test, grp); - ret = ext4_mb_load_buddy(sb, TEST_GOAL_GROUP, &e4b); + ret = ext4_mb_load_buddy_test(sb, TEST_GOAL_GROUP, &e4b); KUNIT_ASSERT_EQ(test, ret, 0); grp->bb_free = EXT4_CLUSTERS_PER_GROUP(sb); @@ -810,7 +811,7 @@ static void test_mb_mark_used(struct kunit *test) test_mb_mark_used_range(test, &e4b, ranges[i].start, ranges[i].len, bitmap, buddy, grp); - ext4_mb_unload_buddy(&e4b); + ext4_mb_unload_buddy_test(&e4b); } static void @@ -826,16 +827,16 @@ test_mb_free_blocks_range(struct kunit *test, struct ext4_buddy *e4b, return; ext4_lock_group(sb, e4b->bd_group); - mb_free_blocks(NULL, e4b, start, len); + mb_free_blocks_test(NULL, e4b, start, len); ext4_unlock_group(sb, e4b->bd_group); - mb_clear_bits(bitmap, start, len); + mb_clear_bits_test(bitmap, start, len); /* bypass bb_free validatoin in ext4_mb_generate_buddy */ grp->bb_free += len; memset(buddy, 0xff, sb->s_blocksize); for (i = 0; i < MB_NUM_ORDERS(sb); i++) grp->bb_counters[i] = 0; - ext4_mb_generate_buddy(sb, buddy, bitmap, 0, grp); + ext4_mb_generate_buddy_test(sb, buddy, bitmap, 0, grp); KUNIT_ASSERT_EQ(test, memcmp(buddy, e4b->bd_buddy, sb->s_blocksize), 0); @@ -866,7 +867,7 @@ static void test_mb_free_blocks(struct kunit *test) bb_counters[MB_NUM_ORDERS(sb)]), GFP_KERNEL); KUNIT_ASSERT_NOT_ERR_OR_NULL(test, grp); - ret = ext4_mb_load_buddy(sb, TEST_GOAL_GROUP, &e4b); + ret = ext4_mb_load_buddy_test(sb, TEST_GOAL_GROUP, &e4b); KUNIT_ASSERT_EQ(test, ret, 0); ex.fe_start = 0; @@ -874,7 +875,7 @@ static void test_mb_free_blocks(struct kunit *test) ex.fe_group = TEST_GOAL_GROUP; ext4_lock_group(sb, TEST_GOAL_GROUP); - mb_mark_used(&e4b, &ex); + mb_mark_used_test(&e4b, &ex); ext4_unlock_group(sb, TEST_GOAL_GROUP); grp->bb_free = 0; @@ -887,7 +888,7 @@ static void test_mb_free_blocks(struct kunit *test) test_mb_free_blocks_range(test, &e4b, ranges[i].start, ranges[i].len, bitmap, buddy, grp); - ext4_mb_unload_buddy(&e4b); + ext4_mb_unload_buddy_test(&e4b); } #define COUNT_FOR_ESTIMATE 100000 @@ -905,7 +906,7 @@ static void test_mb_mark_used_cost(struct kunit *test) if (sb->s_blocksize > PAGE_SIZE) kunit_skip(test, "blocksize exceeds pagesize"); - ret = ext4_mb_load_buddy(sb, TEST_GOAL_GROUP, &e4b); + ret = ext4_mb_load_buddy_test(sb, TEST_GOAL_GROUP, &e4b); KUNIT_ASSERT_EQ(test, ret, 0); ex.fe_group = TEST_GOAL_GROUP; @@ -919,7 +920,7 @@ static void test_mb_mark_used_cost(struct kunit *test) ex.fe_start = ranges[i].start; ex.fe_len = ranges[i].len; ext4_lock_group(sb, TEST_GOAL_GROUP); - mb_mark_used(&e4b, &ex); + mb_mark_used_test(&e4b, &ex); ext4_unlock_group(sb, TEST_GOAL_GROUP); } end = jiffies; @@ -930,14 +931,14 @@ static void test_mb_mark_used_cost(struct kunit *test) continue; ext4_lock_group(sb, TEST_GOAL_GROUP); - mb_free_blocks(NULL, &e4b, ranges[i].start, + mb_free_blocks_test(NULL, &e4b, ranges[i].start, ranges[i].len); ext4_unlock_group(sb, TEST_GOAL_GROUP); } } kunit_info(test, "costed jiffies %lu\n", all); - ext4_mb_unload_buddy(&e4b); + ext4_mb_unload_buddy_test(&e4b); } static const struct mbt_ext4_block_layout mbt_test_layouts[] = { diff --git a/fs/ext4/mballoc.c b/fs/ext4/mballoc.c index 56d50fd3310b47..88dcf218f456ac 100644 --- a/fs/ext4/mballoc.c +++ b/fs/ext4/mballoc.c @@ -892,6 +892,21 @@ mb_update_avg_fragment_size(struct super_block *sb, struct ext4_group_info *grp) } } +static ext4_group_t ext4_get_allocation_groups_count( + struct ext4_allocation_context *ac) +{ + ext4_group_t ngroups = ext4_get_groups_count(ac->ac_sb); + + /* non-extent files are limited to low blocks/groups */ + if (!(ext4_test_inode_flag(ac->ac_inode, EXT4_INODE_EXTENTS))) + ngroups = EXT4_SB(ac->ac_sb)->s_blockfile_groups; + + /* Pairs with smp_wmb() in ext4_update_super() */ + smp_rmb(); + + return ngroups; +} + static int ext4_mb_scan_groups_xa_range(struct ext4_allocation_context *ac, struct xarray *xa, ext4_group_t start, ext4_group_t end) @@ -899,7 +914,7 @@ static int ext4_mb_scan_groups_xa_range(struct ext4_allocation_context *ac, struct super_block *sb = ac->ac_sb; struct ext4_sb_info *sbi = EXT4_SB(sb); enum criteria cr = ac->ac_criteria; - ext4_group_t ngroups = ext4_get_groups_count(sb); + ext4_group_t ngroups = ext4_get_allocation_groups_count(ac); unsigned long group = start; struct ext4_group_info *grp; @@ -951,7 +966,7 @@ static int ext4_mb_scan_groups_p2_aligned(struct ext4_allocation_context *ac, ext4_group_t start, end; start = group; - end = ext4_get_groups_count(ac->ac_sb); + end = ext4_get_allocation_groups_count(ac); wrap_around: for (i = ac->ac_2order; i < MB_NUM_ORDERS(ac->ac_sb); i++) { ret = ext4_mb_scan_groups_largest_free_order_range(ac, i, @@ -1001,7 +1016,7 @@ static int ext4_mb_scan_groups_goal_fast(struct ext4_allocation_context *ac, ext4_group_t start, end; start = group; - end = ext4_get_groups_count(ac->ac_sb); + end = ext4_get_allocation_groups_count(ac); wrap_around: i = mb_avg_fragment_size_order(ac->ac_sb, ac->ac_g_ex.fe_len); for (; i < MB_NUM_ORDERS(ac->ac_sb); i++) { @@ -1083,7 +1098,7 @@ static int ext4_mb_scan_groups_best_avail(struct ext4_allocation_context *ac, min_order = fls(ac->ac_o_ex.fe_len); start = group; - end = ext4_get_groups_count(ac->ac_sb); + end = ext4_get_allocation_groups_count(ac); wrap_around: for (i = order; i >= min_order; i--) { int frag_order; @@ -1133,8 +1148,6 @@ static inline int should_optimize_scan(struct ext4_allocation_context *ac) return 0; if (ac->ac_criteria >= CR_GOAL_LEN_SLOW) return 0; - if (!ext4_test_inode_flag(ac->ac_inode, EXT4_INODE_EXTENTS)) - return 0; return 1; } @@ -1182,14 +1195,12 @@ static int ext4_mb_scan_groups(struct ext4_allocation_context *ac) int ret = 0; ext4_group_t start; struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); - ext4_group_t ngroups = ext4_get_groups_count(ac->ac_sb); - - /* non-extent files are limited to low blocks/groups */ - if (!(ext4_test_inode_flag(ac->ac_inode, EXT4_INODE_EXTENTS))) - ngroups = sbi->s_blockfile_groups; + ext4_group_t ngroups = ext4_get_allocation_groups_count(ac); /* searching for the right group start from the goal value specified */ start = ac->ac_g_ex.fe_group; + if (start >= ngroups) + start = 0; ac->ac_prefetch_grp = start; ac->ac_prefetch_nr = 0; @@ -1706,16 +1717,17 @@ ext4_mb_load_buddy_gfp(struct super_block *sb, ext4_group_t group, /* Avoid locking the folio in the fast path ... */ folio = __filemap_get_folio(inode->i_mapping, pnum, FGP_ACCESSED, 0); - if (IS_ERR(folio) || !folio_test_uptodate(folio)) { + if (IS_ERR(folio) || !folio_test_uptodate(folio) || folio_test_locked(folio)) { + /* + * folio_test_locked is employed to detect ongoing folio + * migrations, since concurrent migrations can lead to + * bitmap inconsistency. And if we are not uptodate that + * implies somebody just created the folio but is yet to + * initialize it. We can drop the folio reference and + * try to get the folio with lock in both cases to avoid + * concurrency. + */ if (!IS_ERR(folio)) - /* - * drop the folio reference and try - * to get the folio with lock. If we - * are not uptodate that implies - * somebody just created the folio but - * is yet to initialize it. So - * wait for it to initialize. - */ folio_put(folio); folio = __filemap_get_folio(inode->i_mapping, pnum, FGP_LOCK | FGP_ACCESSED | FGP_CREAT, gfp); @@ -1764,7 +1776,7 @@ ext4_mb_load_buddy_gfp(struct super_block *sb, ext4_group_t group, /* we need another folio for the buddy */ folio = __filemap_get_folio(inode->i_mapping, pnum, FGP_ACCESSED, 0); - if (IS_ERR(folio) || !folio_test_uptodate(folio)) { + if (IS_ERR(folio) || !folio_test_uptodate(folio) || folio_test_locked(folio)) { if (!IS_ERR(folio)) folio_put(folio); folio = __filemap_get_folio(inode->i_mapping, pnum, @@ -2433,8 +2445,12 @@ int ext4_mb_find_by_goal(struct ext4_allocation_context *ac, return 0; err = ext4_mb_load_buddy(ac->ac_sb, group, e4b); - if (err) + if (err) { + if (EXT4_MB_GRP_BBITMAP_CORRUPT(e4b->bd_info) && + !(ac->ac_flags & EXT4_MB_HINT_GOAL_ONLY)) + return 0; return err; + } ext4_lock_group(ac->ac_sb, group); if (unlikely(EXT4_MB_GRP_BBITMAP_CORRUPT(e4b->bd_info))) @@ -3570,9 +3586,7 @@ static int ext4_mb_init_backend(struct super_block *sb) rcu_read_unlock(); iput(sbi->s_buddy_cache); err_freesgi: - rcu_read_lock(); - kvfree(rcu_dereference(sbi->s_group_info)); - rcu_read_unlock(); + kvfree(rcu_access_pointer(sbi->s_group_info)); return -ENOMEM; } @@ -3881,15 +3895,14 @@ void ext4_mb_release(struct super_block *sb) struct kmem_cache *cachep = get_groupinfo_cache(sb->s_blocksize_bits); int count; - if (test_opt(sb, DISCARD)) { - /* - * wait the discard work to drain all of ext4_free_data - */ - flush_work(&sbi->s_discard_work); - WARN_ON_ONCE(!list_empty(&sbi->s_discard_list)); - } + /* + * wait the discard work to drain all of ext4_free_data + */ + flush_work(&sbi->s_discard_work); + WARN_ON_ONCE(!list_empty(&sbi->s_discard_list)); - if (sbi->s_group_info) { + group_info = rcu_access_pointer(sbi->s_group_info); + if (group_info) { for (i = 0; i < ngroups; i++) { cond_resched(); grinfo = ext4_get_group_info(sb, i); @@ -3907,12 +3920,9 @@ void ext4_mb_release(struct super_block *sb) num_meta_group_infos = (ngroups + EXT4_DESC_PER_BLOCK(sb) - 1) >> EXT4_DESC_PER_BLOCK_BITS(sb); - rcu_read_lock(); - group_info = rcu_dereference(sbi->s_group_info); for (i = 0; i < num_meta_group_infos; i++) kfree(group_info[i]); kvfree(group_info); - rcu_read_unlock(); } ext4_mb_avg_fragment_size_destroy(sbi); ext4_mb_largest_free_orders_destroy(sbi); @@ -4076,7 +4086,7 @@ void ext4_exit_mballoc(void) #define EXT4_MB_BITMAP_MARKED_CHECK 0x0001 #define EXT4_MB_SYNC_UPDATE 0x0002 -static int +int ext4_mb_mark_context(handle_t *handle, struct super_block *sb, bool state, ext4_group_t group, ext4_grpblk_t blkoff, ext4_grpblk_t len, int flags, ext4_grpblk_t *ret_changed) @@ -4185,8 +4195,7 @@ ext4_mb_mark_context(handle_t *handle, struct super_block *sb, bool state, * Returns 0 if success or error code */ static noinline_for_stack int -ext4_mb_mark_diskspace_used(struct ext4_allocation_context *ac, - handle_t *handle, unsigned int reserv_clstrs) +ext4_mb_mark_diskspace_used(struct ext4_allocation_context *ac, handle_t *handle) { struct ext4_group_desc *gdp; struct ext4_sb_info *sbi; @@ -4241,13 +4250,6 @@ ext4_mb_mark_diskspace_used(struct ext4_allocation_context *ac, BUG_ON(changed != ac->ac_b_ex.fe_len); #endif percpu_counter_sub(&sbi->s_freeclusters_counter, ac->ac_b_ex.fe_len); - /* - * Now reduce the dirty block count also. Should not go negative - */ - if (!(ac->ac_flags & EXT4_MB_DELALLOC_RESERVED)) - /* release all the reserved blocks if non delalloc */ - percpu_counter_sub(&sbi->s_dirtyclusters_counter, - reserv_clstrs); return err; } @@ -6332,7 +6334,7 @@ ext4_fsblk_t ext4_mb_new_blocks(handle_t *handle, ext4_mb_pa_put_free(ac); } if (likely(ac->ac_status == AC_STATUS_FOUND)) { - *errp = ext4_mb_mark_diskspace_used(ac, handle, reserv_clstrs); + *errp = ext4_mb_mark_diskspace_used(ac, handle); if (*errp) { ext4_discard_allocated_blocks(ac); goto errout; @@ -6363,12 +6365,9 @@ ext4_fsblk_t ext4_mb_new_blocks(handle_t *handle, out: if (inquota && ar->len < inquota) dquot_free_block(ar->inode, EXT4_C2B(sbi, inquota - ar->len)); - if (!ar->len) { - if ((ar->flags & EXT4_MB_DELALLOC_RESERVED) == 0) - /* release all the reserved blocks if non delalloc */ - percpu_counter_sub(&sbi->s_dirtyclusters_counter, - reserv_clstrs); - } + /* release any reserved blocks */ + if (reserv_clstrs) + percpu_counter_sub(&sbi->s_dirtyclusters_counter, reserv_clstrs); trace_ext4_allocate_blocks(ar, (unsigned long long)block); @@ -7192,6 +7191,102 @@ ext4_mballoc_query_range( return error; } -#ifdef CONFIG_EXT4_KUNIT_TESTS -#include "mballoc-test.c" +#if IS_ENABLED(CONFIG_EXT4_KUNIT_TESTS) +void mb_clear_bits_test(void *bm, int cur, int len) +{ + mb_clear_bits(bm, cur, len); +} +EXPORT_SYMBOL_FOR_EXT4_TEST(mb_clear_bits_test); + +ext4_fsblk_t +ext4_mb_new_blocks_simple_test(struct ext4_allocation_request *ar, + int *errp) +{ + return ext4_mb_new_blocks_simple(ar, errp); +} +EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_mb_new_blocks_simple_test); + +int mb_find_next_zero_bit_test(void *addr, int max, int start) +{ + return mb_find_next_zero_bit(addr, max, start); +} +EXPORT_SYMBOL_FOR_EXT4_TEST(mb_find_next_zero_bit_test); + +int mb_find_next_bit_test(void *addr, int max, int start) +{ + return mb_find_next_bit(addr, max, start); +} +EXPORT_SYMBOL_FOR_EXT4_TEST(mb_find_next_bit_test); + +void mb_clear_bit_test(int bit, void *addr) +{ + mb_clear_bit(bit, addr); +} +EXPORT_SYMBOL_FOR_EXT4_TEST(mb_clear_bit_test); + +int mb_test_bit_test(int bit, void *addr) +{ + return mb_test_bit(bit, addr); +} +EXPORT_SYMBOL_FOR_EXT4_TEST(mb_test_bit_test); + +int ext4_mb_mark_diskspace_used_test(struct ext4_allocation_context *ac, + handle_t *handle) +{ + return ext4_mb_mark_diskspace_used(ac, handle); +} +EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_mb_mark_diskspace_used_test); + +int mb_mark_used_test(struct ext4_buddy *e4b, struct ext4_free_extent *ex) +{ + return mb_mark_used(e4b, ex); +} +EXPORT_SYMBOL_FOR_EXT4_TEST(mb_mark_used_test); + +void ext4_mb_generate_buddy_test(struct super_block *sb, void *buddy, + void *bitmap, ext4_group_t group, + struct ext4_group_info *grp) +{ + ext4_mb_generate_buddy(sb, buddy, bitmap, group, grp); +} +EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_mb_generate_buddy_test); + +int ext4_mb_load_buddy_test(struct super_block *sb, ext4_group_t group, + struct ext4_buddy *e4b) +{ + return ext4_mb_load_buddy(sb, group, e4b); +} +EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_mb_load_buddy_test); + +void ext4_mb_unload_buddy_test(struct ext4_buddy *e4b) +{ + ext4_mb_unload_buddy(e4b); +} +EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_mb_unload_buddy_test); + +void mb_free_blocks_test(struct inode *inode, struct ext4_buddy *e4b, + int first, int count) +{ + mb_free_blocks(inode, e4b, first, count); +} +EXPORT_SYMBOL_FOR_EXT4_TEST(mb_free_blocks_test); + +void ext4_free_blocks_simple_test(struct inode *inode, ext4_fsblk_t block, + unsigned long count) +{ + return ext4_free_blocks_simple(inode, block, count); +} +EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_free_blocks_simple_test); + +EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_wait_block_bitmap); +EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_mb_init); +EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_get_group_desc); +EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_count_free_clusters); +EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_get_group_info); +EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_free_group_clusters_set); +EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_mb_release); +EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_read_block_bitmap_nowait); +EXPORT_SYMBOL_FOR_EXT4_TEST(mb_set_bits); +EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_fc_init_inode); +EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_mb_mark_context); #endif diff --git a/fs/ext4/mballoc.h b/fs/ext4/mballoc.h index 15a049f05d04ad..39333ce72cbd50 100644 --- a/fs/ext4/mballoc.h +++ b/fs/ext4/mballoc.h @@ -270,4 +270,34 @@ ext4_mballoc_query_range( ext4_mballoc_query_range_fn formatter, void *priv); +extern int ext4_mb_mark_context(handle_t *handle, + struct super_block *sb, bool state, + ext4_group_t group, ext4_grpblk_t blkoff, + ext4_grpblk_t len, int flags, + ext4_grpblk_t *ret_changed); +#if IS_ENABLED(CONFIG_EXT4_KUNIT_TESTS) +extern void mb_clear_bits_test(void *bm, int cur, int len); +extern ext4_fsblk_t +ext4_mb_new_blocks_simple_test(struct ext4_allocation_request *ar, + int *errp); +extern int mb_find_next_zero_bit_test(void *addr, int max, int start); +extern int mb_find_next_bit_test(void *addr, int max, int start); +extern void mb_clear_bit_test(int bit, void *addr); +extern int mb_test_bit_test(int bit, void *addr); +extern int +ext4_mb_mark_diskspace_used_test(struct ext4_allocation_context *ac, + handle_t *handle); +extern int mb_mark_used_test(struct ext4_buddy *e4b, + struct ext4_free_extent *ex); +extern void ext4_mb_generate_buddy_test(struct super_block *sb, + void *buddy, void *bitmap, ext4_group_t group, + struct ext4_group_info *grp); +extern int ext4_mb_load_buddy_test(struct super_block *sb, + ext4_group_t group, struct ext4_buddy *e4b); +extern void ext4_mb_unload_buddy_test(struct ext4_buddy *e4b); +extern void mb_free_blocks_test(struct inode *inode, + struct ext4_buddy *e4b, int first, int count); +extern void ext4_free_blocks_simple_test(struct inode *inode, + ext4_fsblk_t block, unsigned long count); +#endif #endif diff --git a/fs/ext4/page-io.c b/fs/ext4/page-io.c index 39abfeec5f36c6..0a3ef9bd680304 100644 --- a/fs/ext4/page-io.c +++ b/fs/ext4/page-io.c @@ -523,9 +523,15 @@ int ext4_bio_write_folio(struct ext4_io_submit *io, struct folio *folio, nr_to_submit++; } while ((bh = bh->b_this_page) != head); - /* Nothing to submit? Just unlock the folio... */ - if (!nr_to_submit) + if (!nr_to_submit) { + /* + * We have nothing to submit. Just cycle the folio through + * writeback state to properly update xarray tags. + */ + __folio_start_writeback(folio, keep_towrite); + folio_end_writeback(folio); return 0; + } bh = head = folio_buffers(folio); diff --git a/fs/ext4/super.c b/fs/ext4/super.c index 87205660c5d026..a839dbc9c34bfe 100644 --- a/fs/ext4/super.c +++ b/fs/ext4/super.c @@ -1249,12 +1249,10 @@ static void ext4_group_desc_free(struct ext4_sb_info *sbi) struct buffer_head **group_desc; int i; - rcu_read_lock(); - group_desc = rcu_dereference(sbi->s_group_desc); + group_desc = rcu_access_pointer(sbi->s_group_desc); for (i = 0; i < sbi->s_gdb_count; i++) brelse(group_desc[i]); kvfree(group_desc); - rcu_read_unlock(); } static void ext4_flex_groups_free(struct ext4_sb_info *sbi) @@ -1262,14 +1260,12 @@ static void ext4_flex_groups_free(struct ext4_sb_info *sbi) struct flex_groups **flex_groups; int i; - rcu_read_lock(); - flex_groups = rcu_dereference(sbi->s_flex_groups); + flex_groups = rcu_access_pointer(sbi->s_flex_groups); if (flex_groups) { for (i = 0; i < sbi->s_flex_groups_allocated; i++) kvfree(flex_groups[i]); kvfree(flex_groups); } - rcu_read_unlock(); } static void ext4_put_super(struct super_block *sb) @@ -3625,6 +3621,13 @@ int ext4_feature_set_ok(struct super_block *sb, int readonly) "extents feature\n"); return 0; } + if (ext4_has_feature_bigalloc(sb) && + le32_to_cpu(EXT4_SB(sb)->s_es->s_first_data_block)) { + ext4_msg(sb, KERN_WARNING, + "bad geometry: bigalloc file system with non-zero " + "first_data_block\n"); + return 0; + } #if !IS_ENABLED(CONFIG_QUOTA) || !IS_ENABLED(CONFIG_QFMT_V2) if (!readonly && (ext4_has_feature_quota(sb) || @@ -5393,6 +5396,7 @@ static int __ext4_fill_super(struct fs_context *fc, struct super_block *sb) timer_setup(&sbi->s_err_report, print_daily_error_info, 0); spin_lock_init(&sbi->s_error_lock); + mutex_init(&sbi->s_error_notify_mutex); INIT_WORK(&sbi->s_sb_upd_work, update_super_work); err = ext4_group_desc_init(sb, es, logical_sb_block, &first_not_zeroed); @@ -5604,6 +5608,10 @@ static int __ext4_fill_super(struct fs_context *fc, struct super_block *sb) clear_opt2(sb, MB_OPTIMIZE_SCAN); } + err = ext4_percpu_param_init(sbi); + if (err) + goto failed_mount5; + err = ext4_mb_init(sb); if (err) { ext4_msg(sb, KERN_ERR, "failed to initialize mballoc (%d)", @@ -5619,10 +5627,6 @@ static int __ext4_fill_super(struct fs_context *fc, struct super_block *sb) sbi->s_journal->j_commit_callback = ext4_journal_commit_callback; - err = ext4_percpu_param_init(sbi); - if (err) - goto failed_mount6; - if (ext4_has_feature_flex_bg(sb)) if (!ext4_fill_flex_info(sb)) { ext4_msg(sb, KERN_ERR, @@ -5704,8 +5708,8 @@ failed_mount8: __maybe_unused failed_mount6: ext4_mb_release(sb); ext4_flex_groups_free(sbi); - ext4_percpu_param_destroy(sbi); failed_mount5: + ext4_percpu_param_destroy(sbi); ext4_ext_release(sb); ext4_release_system_zone(sb); failed_mount4a: diff --git a/fs/ext4/sysfs.c b/fs/ext4/sysfs.c index 0018e09b867ec3..a14882a9617368 100644 --- a/fs/ext4/sysfs.c +++ b/fs/ext4/sysfs.c @@ -561,7 +561,10 @@ static const struct kobj_type ext4_feat_ktype = { void ext4_notify_error_sysfs(struct ext4_sb_info *sbi) { - sysfs_notify(&sbi->s_kobj, NULL, "errors_count"); + mutex_lock(&sbi->s_error_notify_mutex); + if (sbi->s_kobj.state_in_sysfs) + sysfs_notify(&sbi->s_kobj, NULL, "errors_count"); + mutex_unlock(&sbi->s_error_notify_mutex); } static struct kobject *ext4_root; @@ -574,8 +577,10 @@ int ext4_register_sysfs(struct super_block *sb) int err; init_completion(&sbi->s_kobj_unregister); + mutex_lock(&sbi->s_error_notify_mutex); err = kobject_init_and_add(&sbi->s_kobj, &ext4_sb_ktype, ext4_root, "%s", sb->s_id); + mutex_unlock(&sbi->s_error_notify_mutex); if (err) { kobject_put(&sbi->s_kobj); wait_for_completion(&sbi->s_kobj_unregister); @@ -608,7 +613,10 @@ void ext4_unregister_sysfs(struct super_block *sb) if (sbi->s_proc) remove_proc_subtree(sb->s_id, ext4_proc_root); + + mutex_lock(&sbi->s_error_notify_mutex); kobject_del(&sbi->s_kobj); + mutex_unlock(&sbi->s_error_notify_mutex); } int __init ext4_init_sysfs(void) diff --git a/fs/f2fs/data.c b/fs/f2fs/data.c index c30e69392a6236..86e00b9e0d1cfc 100644 --- a/fs/f2fs/data.c +++ b/fs/f2fs/data.c @@ -151,6 +151,12 @@ static void f2fs_finish_read_bio(struct bio *bio, bool in_task) } dec_page_count(F2FS_F_SB(folio), __read_io_type(folio)); + + if (F2FS_F_SB(folio)->node_inode && is_node_folio(folio) && + f2fs_sanity_check_node_footer(F2FS_F_SB(folio), + folio, folio->index, NODE_TYPE_REGULAR, true)) + bio->bi_status = BLK_STS_IOERR; + folio_end_read(folio, bio->bi_status == BLK_STS_OK); } @@ -352,18 +358,27 @@ static void f2fs_write_end_io(struct bio *bio) STOP_CP_REASON_WRITE_FAIL); } - f2fs_bug_on(sbi, is_node_folio(folio) && - folio->index != nid_of_node(folio)); + if (is_node_folio(folio)) { + f2fs_sanity_check_node_footer(sbi, folio, + folio->index, NODE_TYPE_REGULAR, true); + f2fs_bug_on(sbi, folio->index != nid_of_node(folio)); + } dec_page_count(sbi, type); + + /* + * we should access sbi before folio_end_writeback() to + * avoid racing w/ kill_f2fs_super() + */ + if (type == F2FS_WB_CP_DATA && !get_pages(sbi, type) && + wq_has_sleeper(&sbi->cp_wait)) + wake_up(&sbi->cp_wait); + if (f2fs_in_warm_node_list(sbi, folio)) f2fs_del_fsync_node_entry(sbi, folio); folio_clear_f2fs_gcing(folio); folio_end_writeback(folio); } - if (!get_pages(sbi, F2FS_WB_CP_DATA) && - wq_has_sleeper(&sbi->cp_wait)) - wake_up(&sbi->cp_wait); bio_put(bio); } @@ -1418,7 +1433,6 @@ static int __allocate_data_block(struct dnode_of_data *dn, int seg_type) static void f2fs_map_lock(struct f2fs_sb_info *sbi, int flag) { - f2fs_down_read(&sbi->cp_enable_rwsem); if (flag == F2FS_GET_BLOCK_PRE_AIO) f2fs_down_read(&sbi->node_change); else @@ -1431,7 +1445,6 @@ static void f2fs_map_unlock(struct f2fs_sb_info *sbi, int flag) f2fs_up_read(&sbi->node_change); else f2fs_unlock_op(sbi); - f2fs_up_read(&sbi->cp_enable_rwsem); } int f2fs_get_block_locked(struct dnode_of_data *dn, pgoff_t index) @@ -1793,7 +1806,8 @@ int f2fs_map_blocks(struct inode *inode, struct f2fs_map_blocks *map, int flag) return err; } -bool f2fs_overwrite_io(struct inode *inode, loff_t pos, size_t len) +static bool __f2fs_overwrite_io(struct inode *inode, loff_t pos, size_t len, + bool check_first) { struct f2fs_map_blocks map; block_t last_lblk; @@ -1815,10 +1829,17 @@ bool f2fs_overwrite_io(struct inode *inode, loff_t pos, size_t len) if (err || map.m_len == 0) return false; map.m_lblk += map.m_len; + if (check_first) + break; } return true; } +bool f2fs_overwrite_io(struct inode *inode, loff_t pos, size_t len) +{ + return __f2fs_overwrite_io(inode, pos, len, false); +} + static int f2fs_xattr_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo) { @@ -3929,6 +3950,7 @@ static int check_swap_activate(struct swap_info_struct *sis, while (cur_lblock < last_lblock && cur_lblock < sis->max) { struct f2fs_map_blocks map; + bool last_extent = false; retry: cond_resched(); @@ -3954,11 +3976,10 @@ static int check_swap_activate(struct swap_info_struct *sis, pblock = map.m_pblk; nr_pblocks = map.m_len; - if ((pblock - SM_I(sbi)->main_blkaddr) % blks_per_sec || - nr_pblocks % blks_per_sec || - f2fs_is_sequential_zone_area(sbi, pblock)) { - bool last_extent = false; - + if (!last_extent && + ((pblock - SM_I(sbi)->main_blkaddr) % blks_per_sec || + nr_pblocks % blks_per_sec || + f2fs_is_sequential_zone_area(sbi, pblock))) { not_aligned++; nr_pblocks = roundup(nr_pblocks, blks_per_sec); @@ -3979,8 +4000,8 @@ static int check_swap_activate(struct swap_info_struct *sis, goto out; } - if (!last_extent) - goto retry; + /* lookup block mapping info after block migration */ + goto retry; } if (cur_lblock + nr_pblocks >= sis->max) @@ -4181,7 +4202,7 @@ static int f2fs_iomap_begin(struct inode *inode, loff_t offset, loff_t length, * f2fs_map_lock and f2fs_balance_fs are not necessary. */ if ((flags & IOMAP_WRITE) && - !f2fs_overwrite_io(inode, offset, length)) + !__f2fs_overwrite_io(inode, offset, length, true)) map.m_may_create = true; err = f2fs_map_blocks(inode, &map, F2FS_GET_BLOCK_DIO); diff --git a/fs/f2fs/f2fs.h b/fs/f2fs/f2fs.h index 20edbb99b814a7..d9a8465cb2f4de 100644 --- a/fs/f2fs/f2fs.h +++ b/fs/f2fs/f2fs.h @@ -287,7 +287,7 @@ enum { #define DEF_CP_INTERVAL 60 /* 60 secs */ #define DEF_IDLE_INTERVAL 5 /* 5 secs */ #define DEF_DISABLE_INTERVAL 5 /* 5 secs */ -#define DEF_ENABLE_INTERVAL 5 /* 5 secs */ +#define DEF_ENABLE_INTERVAL 16 /* 16 secs */ #define DEF_DISABLE_QUICK_INTERVAL 1 /* 1 secs */ #define DEF_UMOUNT_DISCARD_TIMEOUT 5 /* 5 secs */ @@ -521,13 +521,25 @@ struct fsync_inode_entry { #define nats_in_cursum(jnl) (le16_to_cpu((jnl)->n_nats)) #define sits_in_cursum(jnl) (le16_to_cpu((jnl)->n_sits)) -#define nat_in_journal(jnl, i) ((jnl)->nat_j.entries[i].ne) -#define nid_in_journal(jnl, i) ((jnl)->nat_j.entries[i].nid) -#define sit_in_journal(jnl, i) ((jnl)->sit_j.entries[i].se) -#define segno_in_journal(jnl, i) ((jnl)->sit_j.entries[i].segno) - -#define MAX_NAT_JENTRIES(jnl) (NAT_JOURNAL_ENTRIES - nats_in_cursum(jnl)) -#define MAX_SIT_JENTRIES(jnl) (SIT_JOURNAL_ENTRIES - sits_in_cursum(jnl)) +#define nat_in_journal(jnl, i) \ + (((struct nat_journal_entry *)(jnl)->nat_j.entries)[i].ne) +#define nid_in_journal(jnl, i) \ + (((struct nat_journal_entry *)(jnl)->nat_j.entries)[i].nid) +#define sit_in_journal(jnl, i) \ + (((struct sit_journal_entry *)(jnl)->sit_j.entries)[i].se) +#define segno_in_journal(jnl, i) \ + (((struct sit_journal_entry *)(jnl)->sit_j.entries)[i].segno) + +#define sum_entries(sum) ((struct f2fs_summary *)(sum)) +#define sum_journal(sbi, sum) \ + ((struct f2fs_journal *)((char *)(sum) + \ + ((sbi)->entries_in_sum * sizeof(struct f2fs_summary)))) +#define sum_footer(sbi, sum) \ + ((struct summary_footer *)((char *)(sum) + (sbi)->sum_blocksize - \ + sizeof(struct summary_footer))) + +#define MAX_NAT_JENTRIES(sbi, jnl) ((sbi)->nat_journal_entries - nats_in_cursum(jnl)) +#define MAX_SIT_JENTRIES(sbi, jnl) ((sbi)->sit_journal_entries - sits_in_cursum(jnl)) static inline int update_nats_in_cursum(struct f2fs_journal *journal, int i) { @@ -545,14 +557,6 @@ static inline int update_sits_in_cursum(struct f2fs_journal *journal, int i) return before; } -static inline bool __has_cursum_space(struct f2fs_journal *journal, - int size, int type) -{ - if (type == NAT_JOURNAL) - return size <= MAX_NAT_JENTRIES(journal); - return size <= MAX_SIT_JENTRIES(journal); -} - /* for inline stuff */ #define DEF_INLINE_RESERVED_SIZE 1 static inline int get_extra_isize(struct inode *inode); @@ -1525,6 +1529,15 @@ enum f2fs_lookup_mode { LOOKUP_AUTO, }; +/* For node type in __get_node_folio() */ +enum node_type { + NODE_TYPE_REGULAR, + NODE_TYPE_INODE, + NODE_TYPE_XATTR, + NODE_TYPE_NON_INODE, +}; + + static inline int f2fs_test_bit(unsigned int nr, char *addr); static inline void f2fs_set_bit(unsigned int nr, char *addr); static inline void f2fs_clear_bit(unsigned int nr, char *addr); @@ -1716,7 +1729,6 @@ struct f2fs_sb_info { long interval_time[MAX_TIME]; /* to store thresholds */ struct ckpt_req_control cprc_info; /* for checkpoint request control */ struct cp_stats cp_stats; /* for time stat of checkpoint */ - struct f2fs_rwsem cp_enable_rwsem; /* block cache/dio write */ struct inode_management im[MAX_INO_ENTRY]; /* manage inode cache */ @@ -1764,6 +1776,15 @@ struct f2fs_sb_info { bool readdir_ra; /* readahead inode in readdir */ u64 max_io_bytes; /* max io bytes to merge IOs */ + /* variable summary block units */ + unsigned int sum_blocksize; /* sum block size */ + unsigned int sums_per_block; /* sum block count per block */ + unsigned int entries_in_sum; /* entry count in sum block */ + unsigned int sum_entry_size; /* total entry size in sum block */ + unsigned int sum_journal_size; /* journal size in sum block */ + unsigned int nat_journal_entries; /* nat journal entry count in the journal */ + unsigned int sit_journal_entries; /* sit journal entry count in the journal */ + block_t user_block_count; /* # of user blocks */ block_t total_valid_block_count; /* # of valid blocks */ block_t discard_blks; /* discard command candidats */ @@ -2813,6 +2834,14 @@ static inline block_t __start_sum_addr(struct f2fs_sb_info *sbi) return le32_to_cpu(F2FS_CKPT(sbi)->cp_pack_start_sum); } +static inline bool __has_cursum_space(struct f2fs_sb_info *sbi, + struct f2fs_journal *journal, int size, int type) +{ + if (type == NAT_JOURNAL) + return size <= MAX_NAT_JENTRIES(sbi, journal); + return size <= MAX_SIT_JENTRIES(sbi, journal); +} + extern void f2fs_mark_inode_dirty_sync(struct inode *inode, bool sync); static inline int inc_valid_node_count(struct f2fs_sb_info *sbi, struct inode *inode, bool is_inode) @@ -3857,6 +3886,9 @@ struct folio *f2fs_new_node_folio(struct dnode_of_data *dn, unsigned int ofs); void f2fs_ra_node_page(struct f2fs_sb_info *sbi, nid_t nid); struct folio *f2fs_get_node_folio(struct f2fs_sb_info *sbi, pgoff_t nid, enum node_type node_type); +int f2fs_sanity_check_node_footer(struct f2fs_sb_info *sbi, + struct folio *folio, pgoff_t nid, + enum node_type ntype, bool in_irq); struct folio *f2fs_get_inode_folio(struct f2fs_sb_info *sbi, pgoff_t ino); struct folio *f2fs_get_xnode_folio(struct f2fs_sb_info *sbi, pgoff_t xnid); int f2fs_move_node_folio(struct folio *node_folio, int gc_type); @@ -3956,7 +3988,8 @@ void f2fs_wait_on_block_writeback_range(struct inode *inode, block_t blkaddr, block_t len); void f2fs_write_data_summaries(struct f2fs_sb_info *sbi, block_t start_blk); void f2fs_write_node_summaries(struct f2fs_sb_info *sbi, block_t start_blk); -int f2fs_lookup_journal_in_cursum(struct f2fs_journal *journal, int type, +int f2fs_lookup_journal_in_cursum(struct f2fs_sb_info *sbi, + struct f2fs_journal *journal, int type, unsigned int val, int alloc); void f2fs_flush_sit_entries(struct f2fs_sb_info *sbi, struct cp_control *cpc); int f2fs_check_and_fix_write_pointer(struct f2fs_sb_info *sbi); diff --git a/fs/f2fs/gc.c b/fs/f2fs/gc.c index 384fa7e2085bf9..fa20bcd700588f 100644 --- a/fs/f2fs/gc.c +++ b/fs/f2fs/gc.c @@ -1769,8 +1769,8 @@ static int do_garbage_collect(struct f2fs_sb_info *sbi, sanity_check_seg_type(sbi, get_seg_entry(sbi, segno)->type); - segno = rounddown(segno, SUMS_PER_BLOCK); - sum_blk_cnt = DIV_ROUND_UP(end_segno - segno, SUMS_PER_BLOCK); + segno = rounddown(segno, sbi->sums_per_block); + sum_blk_cnt = DIV_ROUND_UP(end_segno - segno, sbi->sums_per_block); /* readahead multi ssa blocks those have contiguous address */ if (__is_large_section(sbi)) f2fs_ra_meta_pages(sbi, GET_SUM_BLOCK(sbi, segno), @@ -1780,17 +1780,17 @@ static int do_garbage_collect(struct f2fs_sb_info *sbi, while (segno < end_segno) { struct folio *sum_folio = f2fs_get_sum_folio(sbi, segno); - segno += SUMS_PER_BLOCK; + segno += sbi->sums_per_block; if (IS_ERR(sum_folio)) { int err = PTR_ERR(sum_folio); - end_segno = segno - SUMS_PER_BLOCK; - segno = rounddown(start_segno, SUMS_PER_BLOCK); + end_segno = segno - sbi->sums_per_block; + segno = rounddown(start_segno, sbi->sums_per_block); while (segno < end_segno) { sum_folio = filemap_get_folio(META_MAPPING(sbi), GET_SUM_BLOCK(sbi, segno)); folio_put_refs(sum_folio, 2); - segno += SUMS_PER_BLOCK; + segno += sbi->sums_per_block; } return err; } @@ -1806,8 +1806,8 @@ static int do_garbage_collect(struct f2fs_sb_info *sbi, /* find segment summary of victim */ struct folio *sum_folio = filemap_get_folio(META_MAPPING(sbi), GET_SUM_BLOCK(sbi, segno)); - unsigned int block_end_segno = rounddown(segno, SUMS_PER_BLOCK) - + SUMS_PER_BLOCK; + unsigned int block_end_segno = rounddown(segno, sbi->sums_per_block) + + sbi->sums_per_block; if (block_end_segno > end_segno) block_end_segno = end_segno; @@ -1833,12 +1833,13 @@ static int do_garbage_collect(struct f2fs_sb_info *sbi, migrated >= sbi->migration_granularity) continue; - sum = SUM_BLK_PAGE_ADDR(sum_folio, cur_segno); - if (type != GET_SUM_TYPE((&sum->footer))) { + sum = SUM_BLK_PAGE_ADDR(sbi, sum_folio, cur_segno); + if (type != GET_SUM_TYPE(sum_footer(sbi, sum))) { f2fs_err(sbi, "Inconsistent segment (%u) type " "[%d, %d] in SSA and SIT", cur_segno, type, - GET_SUM_TYPE((&sum->footer))); + GET_SUM_TYPE( + sum_footer(sbi, sum))); f2fs_stop_checkpoint(sbi, false, STOP_CP_REASON_CORRUPTED_SUMMARY); continue; @@ -2096,6 +2097,7 @@ int f2fs_gc_range(struct f2fs_sb_info *sbi, if (unlikely(f2fs_cp_error(sbi))) return -EIO; + stat_inc_gc_call_count(sbi, FOREGROUND); for (segno = start_seg; segno <= end_seg; segno += SEGS_PER_SEC(sbi)) { struct gc_inode_list gc_list = { .ilist = LIST_HEAD_INIT(gc_list.ilist), diff --git a/fs/f2fs/node.c b/fs/f2fs/node.c index 482a362f262543..591fcdf3ba77b9 100644 --- a/fs/f2fs/node.c +++ b/fs/f2fs/node.c @@ -606,7 +606,7 @@ int f2fs_get_node_info(struct f2fs_sb_info *sbi, nid_t nid, goto retry; } - i = f2fs_lookup_journal_in_cursum(journal, NAT_JOURNAL, nid, 0); + i = f2fs_lookup_journal_in_cursum(sbi, journal, NAT_JOURNAL, nid, 0); if (i >= 0) { ne = nat_in_journal(journal, i); node_info_from_raw_nat(ni, &ne); @@ -1500,9 +1500,9 @@ void f2fs_ra_node_page(struct f2fs_sb_info *sbi, nid_t nid) f2fs_folio_put(afolio, err ? true : false); } -static int sanity_check_node_footer(struct f2fs_sb_info *sbi, +int f2fs_sanity_check_node_footer(struct f2fs_sb_info *sbi, struct folio *folio, pgoff_t nid, - enum node_type ntype) + enum node_type ntype, bool in_irq) { if (unlikely(nid != nid_of_node(folio))) goto out_err; @@ -1527,12 +1527,13 @@ static int sanity_check_node_footer(struct f2fs_sb_info *sbi, goto out_err; return 0; out_err: - f2fs_warn(sbi, "inconsistent node block, node_type:%d, nid:%lu, " - "node_footer[nid:%u,ino:%u,ofs:%u,cpver:%llu,blkaddr:%u]", - ntype, nid, nid_of_node(folio), ino_of_node(folio), - ofs_of_node(folio), cpver_of_node(folio), - next_blkaddr_of_node(folio)); set_sbi_flag(sbi, SBI_NEED_FSCK); + f2fs_warn_ratelimited(sbi, "inconsistent node block, node_type:%d, nid:%lu, " + "node_footer[nid:%u,ino:%u,ofs:%u,cpver:%llu,blkaddr:%u]", + ntype, nid, nid_of_node(folio), ino_of_node(folio), + ofs_of_node(folio), cpver_of_node(folio), + next_blkaddr_of_node(folio)); + f2fs_handle_error(sbi, ERROR_INCONSISTENT_FOOTER); return -EFSCORRUPTED; } @@ -1578,7 +1579,7 @@ static struct folio *__get_node_folio(struct f2fs_sb_info *sbi, pgoff_t nid, goto out_err; } page_hit: - err = sanity_check_node_footer(sbi, folio, nid, ntype); + err = f2fs_sanity_check_node_footer(sbi, folio, nid, ntype, false); if (!err) return folio; out_err: @@ -1751,7 +1752,12 @@ static bool __write_node_folio(struct folio *folio, bool atomic, bool *submitted /* get old block addr of this node page */ nid = nid_of_node(folio); - f2fs_bug_on(sbi, folio->index != nid); + + if (f2fs_sanity_check_node_footer(sbi, folio, nid, + NODE_TYPE_REGULAR, false)) { + f2fs_handle_critical_error(sbi, STOP_CP_REASON_CORRUPTED_NID); + goto redirty_out; + } if (f2fs_get_node_info(sbi, nid, &ni, !do_balance)) goto redirty_out; @@ -1774,8 +1780,13 @@ static bool __write_node_folio(struct folio *folio, bool atomic, bool *submitted goto redirty_out; } - if (atomic && !test_opt(sbi, NOBARRIER)) - fio.op_flags |= REQ_PREFLUSH | REQ_FUA; + if (atomic) { + if (!test_opt(sbi, NOBARRIER)) + fio.op_flags |= REQ_PREFLUSH | REQ_FUA; + if (IS_INODE(folio)) + set_dentry_mark(folio, + f2fs_need_dentry_mark(sbi, ino_of_node(folio))); + } /* should add to global list before clearing PAGECACHE status */ if (f2fs_in_warm_node_list(sbi, folio)) { @@ -1916,8 +1927,9 @@ int f2fs_fsync_node_pages(struct f2fs_sb_info *sbi, struct inode *inode, if (is_inode_flag_set(inode, FI_DIRTY_INODE)) f2fs_update_inode(inode, folio); - set_dentry_mark(folio, - f2fs_need_dentry_mark(sbi, ino)); + if (!atomic) + set_dentry_mark(folio, + f2fs_need_dentry_mark(sbi, ino)); } /* may be written by other thread */ if (!folio_test_dirty(folio)) @@ -2937,7 +2949,7 @@ int f2fs_restore_node_summary(struct f2fs_sb_info *sbi, /* scan the node segment */ last_offset = BLKS_PER_SEG(sbi); addr = START_BLOCK(sbi, segno); - sum_entry = &sum->entries[0]; + sum_entry = sum_entries(sum); for (i = 0; i < last_offset; i += nrpages, addr += nrpages) { nrpages = bio_max_segs(last_offset - i); @@ -3078,7 +3090,7 @@ static int __flush_nat_entry_set(struct f2fs_sb_info *sbi, * #2, flush nat entries to nat page. */ if (enabled_nat_bits(sbi, cpc) || - !__has_cursum_space(journal, set->entry_cnt, NAT_JOURNAL)) + !__has_cursum_space(sbi, journal, set->entry_cnt, NAT_JOURNAL)) to_journal = false; if (to_journal) { @@ -3101,7 +3113,7 @@ static int __flush_nat_entry_set(struct f2fs_sb_info *sbi, f2fs_bug_on(sbi, nat_get_blkaddr(ne) == NEW_ADDR); if (to_journal) { - offset = f2fs_lookup_journal_in_cursum(journal, + offset = f2fs_lookup_journal_in_cursum(sbi, journal, NAT_JOURNAL, nid, 1); f2fs_bug_on(sbi, offset < 0); raw_ne = &nat_in_journal(journal, offset); @@ -3172,7 +3184,7 @@ int f2fs_flush_nat_entries(struct f2fs_sb_info *sbi, struct cp_control *cpc) * into nat entry set. */ if (enabled_nat_bits(sbi, cpc) || - !__has_cursum_space(journal, + !__has_cursum_space(sbi, journal, nm_i->nat_cnt[DIRTY_NAT], NAT_JOURNAL)) remove_nats_in_journal(sbi); @@ -3183,7 +3195,7 @@ int f2fs_flush_nat_entries(struct f2fs_sb_info *sbi, struct cp_control *cpc) set_idx = setvec[found - 1]->set + 1; for (idx = 0; idx < found; idx++) __adjust_nat_entry_set(setvec[idx], &sets, - MAX_NAT_JENTRIES(journal)); + MAX_NAT_JENTRIES(sbi, journal)); } /* flush dirty nats in nat entry set */ diff --git a/fs/f2fs/node.h b/fs/f2fs/node.h index 9cb8dcf8d41760..824ac9f0e6e42a 100644 --- a/fs/f2fs/node.h +++ b/fs/f2fs/node.h @@ -52,14 +52,6 @@ enum { IS_PREALLOC, /* nat entry is preallocated */ }; -/* For node type in __get_node_folio() */ -enum node_type { - NODE_TYPE_REGULAR, - NODE_TYPE_INODE, - NODE_TYPE_XATTR, - NODE_TYPE_NON_INODE, -}; - /* * For node information */ diff --git a/fs/f2fs/recovery.c b/fs/f2fs/recovery.c index c3415ebb9f5053..a6bfc8e759cf40 100644 --- a/fs/f2fs/recovery.c +++ b/fs/f2fs/recovery.c @@ -514,7 +514,7 @@ static int check_index_in_prev_nodes(struct f2fs_sb_info *sbi, struct curseg_info *curseg = CURSEG_I(sbi, i); if (curseg->segno == segno) { - sum = curseg->sum_blk->entries[blkoff]; + sum = sum_entries(curseg->sum_blk)[blkoff]; goto got_it; } } @@ -522,8 +522,8 @@ static int check_index_in_prev_nodes(struct f2fs_sb_info *sbi, sum_folio = f2fs_get_sum_folio(sbi, segno); if (IS_ERR(sum_folio)) return PTR_ERR(sum_folio); - sum_node = SUM_BLK_PAGE_ADDR(sum_folio, segno); - sum = sum_node->entries[blkoff]; + sum_node = SUM_BLK_PAGE_ADDR(sbi, sum_folio, segno); + sum = sum_entries(sum_node)[blkoff]; f2fs_folio_put(sum_folio, true); got_it: /* Use the locked dnode page and inode */ diff --git a/fs/f2fs/segment.c b/fs/f2fs/segment.c index c26424f4768635..90d0bac9d29400 100644 --- a/fs/f2fs/segment.c +++ b/fs/f2fs/segment.c @@ -2685,12 +2685,12 @@ int f2fs_npages_for_summary_flush(struct f2fs_sb_info *sbi, bool for_ra) valid_sum_count += f2fs_curseg_valid_blocks(sbi, i); } - sum_in_page = (PAGE_SIZE - 2 * SUM_JOURNAL_SIZE - + sum_in_page = (sbi->blocksize - 2 * sbi->sum_journal_size - SUM_FOOTER_SIZE) / SUMMARY_SIZE; if (valid_sum_count <= sum_in_page) return 1; else if ((valid_sum_count - sum_in_page) <= - (PAGE_SIZE - SUM_FOOTER_SIZE) / SUMMARY_SIZE) + (sbi->blocksize - SUM_FOOTER_SIZE) / SUMMARY_SIZE) return 2; return 3; } @@ -2710,7 +2710,7 @@ void f2fs_update_meta_page(struct f2fs_sb_info *sbi, { struct folio *folio; - if (SUMS_PER_BLOCK == 1) + if (!f2fs_sb_has_packed_ssa(sbi)) folio = f2fs_grab_meta_folio(sbi, blk_addr); else folio = f2fs_get_meta_folio_retry(sbi, blk_addr); @@ -2728,7 +2728,7 @@ static void write_sum_page(struct f2fs_sb_info *sbi, { struct folio *folio; - if (SUMS_PER_BLOCK == 1) + if (!f2fs_sb_has_packed_ssa(sbi)) return f2fs_update_meta_page(sbi, (void *)sum_blk, GET_SUM_BLOCK(sbi, segno)); @@ -2736,7 +2736,8 @@ static void write_sum_page(struct f2fs_sb_info *sbi, if (IS_ERR(folio)) return; - memcpy(SUM_BLK_PAGE_ADDR(folio, segno), sum_blk, sizeof(*sum_blk)); + memcpy(SUM_BLK_PAGE_ADDR(sbi, folio, segno), sum_blk, + sbi->sum_blocksize); folio_mark_dirty(folio); f2fs_folio_put(folio, true); } @@ -2755,11 +2756,11 @@ static void write_current_sum_page(struct f2fs_sb_info *sbi, mutex_lock(&curseg->curseg_mutex); down_read(&curseg->journal_rwsem); - memcpy(&dst->journal, curseg->journal, SUM_JOURNAL_SIZE); + memcpy(sum_journal(sbi, dst), curseg->journal, sbi->sum_journal_size); up_read(&curseg->journal_rwsem); - memcpy(dst->entries, src->entries, SUM_ENTRY_SIZE); - memcpy(&dst->footer, &src->footer, SUM_FOOTER_SIZE); + memcpy(sum_entries(dst), sum_entries(src), sbi->sum_entry_size); + memcpy(sum_footer(sbi, dst), sum_footer(sbi, src), SUM_FOOTER_SIZE); mutex_unlock(&curseg->curseg_mutex); @@ -2932,7 +2933,7 @@ static void reset_curseg(struct f2fs_sb_info *sbi, int type, int modified) curseg->next_blkoff = 0; curseg->next_segno = NULL_SEGNO; - sum_footer = &(curseg->sum_blk->footer); + sum_footer = sum_footer(sbi, curseg->sum_blk); memset(sum_footer, 0, sizeof(struct summary_footer)); sanity_check_seg_type(sbi, seg_type); @@ -3078,11 +3079,11 @@ static int change_curseg(struct f2fs_sb_info *sbi, int type) sum_folio = f2fs_get_sum_folio(sbi, new_segno); if (IS_ERR(sum_folio)) { /* GC won't be able to use stale summary pages by cp_error */ - memset(curseg->sum_blk, 0, SUM_ENTRY_SIZE); + memset(curseg->sum_blk, 0, sbi->sum_entry_size); return PTR_ERR(sum_folio); } - sum_node = SUM_BLK_PAGE_ADDR(sum_folio, new_segno); - memcpy(curseg->sum_blk, sum_node, SUM_ENTRY_SIZE); + sum_node = SUM_BLK_PAGE_ADDR(sbi, sum_folio, new_segno); + memcpy(curseg->sum_blk, sum_node, sbi->sum_entry_size); f2fs_folio_put(sum_folio, true); return 0; } @@ -3814,7 +3815,7 @@ int f2fs_allocate_data_block(struct f2fs_sb_info *sbi, struct folio *folio, f2fs_wait_discard_bio(sbi, *new_blkaddr); - curseg->sum_blk->entries[curseg->next_blkoff] = *sum; + sum_entries(curseg->sum_blk)[curseg->next_blkoff] = *sum; if (curseg->alloc_type == SSR) { curseg->next_blkoff = f2fs_find_next_ssr_block(sbi, curseg); } else { @@ -4183,7 +4184,7 @@ void f2fs_do_replace_block(struct f2fs_sb_info *sbi, struct f2fs_summary *sum, } curseg->next_blkoff = GET_BLKOFF_FROM_SEG0(sbi, new_blkaddr); - curseg->sum_blk->entries[curseg->next_blkoff] = *sum; + sum_entries(curseg->sum_blk)[curseg->next_blkoff] = *sum; if (!recover_curseg || recover_newaddr) { if (!from_gc) @@ -4303,12 +4304,12 @@ static int read_compacted_summaries(struct f2fs_sb_info *sbi) /* Step 1: restore nat cache */ seg_i = CURSEG_I(sbi, CURSEG_HOT_DATA); - memcpy(seg_i->journal, kaddr, SUM_JOURNAL_SIZE); + memcpy(seg_i->journal, kaddr, sbi->sum_journal_size); /* Step 2: restore sit cache */ seg_i = CURSEG_I(sbi, CURSEG_COLD_DATA); - memcpy(seg_i->journal, kaddr + SUM_JOURNAL_SIZE, SUM_JOURNAL_SIZE); - offset = 2 * SUM_JOURNAL_SIZE; + memcpy(seg_i->journal, kaddr + sbi->sum_journal_size, sbi->sum_journal_size); + offset = 2 * sbi->sum_journal_size; /* Step 3: restore summary entries */ for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) { @@ -4330,9 +4331,9 @@ static int read_compacted_summaries(struct f2fs_sb_info *sbi) struct f2fs_summary *s; s = (struct f2fs_summary *)(kaddr + offset); - seg_i->sum_blk->entries[j] = *s; + sum_entries(seg_i->sum_blk)[j] = *s; offset += SUMMARY_SIZE; - if (offset + SUMMARY_SIZE <= PAGE_SIZE - + if (offset + SUMMARY_SIZE <= sbi->blocksize - SUM_FOOTER_SIZE) continue; @@ -4388,7 +4389,7 @@ static int read_normal_summaries(struct f2fs_sb_info *sbi, int type) if (IS_NODESEG(type)) { if (__exist_node_summaries(sbi)) { - struct f2fs_summary *ns = &sum->entries[0]; + struct f2fs_summary *ns = sum_entries(sum); int i; for (i = 0; i < BLKS_PER_SEG(sbi); i++, ns++) { @@ -4408,11 +4409,13 @@ static int read_normal_summaries(struct f2fs_sb_info *sbi, int type) /* update journal info */ down_write(&curseg->journal_rwsem); - memcpy(curseg->journal, &sum->journal, SUM_JOURNAL_SIZE); + memcpy(curseg->journal, sum_journal(sbi, sum), sbi->sum_journal_size); up_write(&curseg->journal_rwsem); - memcpy(curseg->sum_blk->entries, sum->entries, SUM_ENTRY_SIZE); - memcpy(&curseg->sum_blk->footer, &sum->footer, SUM_FOOTER_SIZE); + memcpy(sum_entries(curseg->sum_blk), sum_entries(sum), + sbi->sum_entry_size); + memcpy(sum_footer(sbi, curseg->sum_blk), sum_footer(sbi, sum), + SUM_FOOTER_SIZE); curseg->next_segno = segno; reset_curseg(sbi, type, 0); curseg->alloc_type = ckpt->alloc_type[type]; @@ -4456,8 +4459,8 @@ static int restore_curseg_summaries(struct f2fs_sb_info *sbi) } /* sanity check for summary blocks */ - if (nats_in_cursum(nat_j) > NAT_JOURNAL_ENTRIES || - sits_in_cursum(sit_j) > SIT_JOURNAL_ENTRIES) { + if (nats_in_cursum(nat_j) > sbi->nat_journal_entries || + sits_in_cursum(sit_j) > sbi->sit_journal_entries) { f2fs_err(sbi, "invalid journal entries nats %u sits %u", nats_in_cursum(nat_j), sits_in_cursum(sit_j)); return -EINVAL; @@ -4481,13 +4484,13 @@ static void write_compacted_summaries(struct f2fs_sb_info *sbi, block_t blkaddr) /* Step 1: write nat cache */ seg_i = CURSEG_I(sbi, CURSEG_HOT_DATA); - memcpy(kaddr, seg_i->journal, SUM_JOURNAL_SIZE); - written_size += SUM_JOURNAL_SIZE; + memcpy(kaddr, seg_i->journal, sbi->sum_journal_size); + written_size += sbi->sum_journal_size; /* Step 2: write sit cache */ seg_i = CURSEG_I(sbi, CURSEG_COLD_DATA); - memcpy(kaddr + written_size, seg_i->journal, SUM_JOURNAL_SIZE); - written_size += SUM_JOURNAL_SIZE; + memcpy(kaddr + written_size, seg_i->journal, sbi->sum_journal_size); + written_size += sbi->sum_journal_size; /* Step 3: write summary entries */ for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) { @@ -4500,10 +4503,10 @@ static void write_compacted_summaries(struct f2fs_sb_info *sbi, block_t blkaddr) written_size = 0; } summary = (struct f2fs_summary *)(kaddr + written_size); - *summary = seg_i->sum_blk->entries[j]; + *summary = sum_entries(seg_i->sum_blk)[j]; written_size += SUMMARY_SIZE; - if (written_size + SUMMARY_SIZE <= PAGE_SIZE - + if (written_size + SUMMARY_SIZE <= sbi->blocksize - SUM_FOOTER_SIZE) continue; @@ -4545,8 +4548,9 @@ void f2fs_write_node_summaries(struct f2fs_sb_info *sbi, block_t start_blk) write_normal_summaries(sbi, start_blk, CURSEG_HOT_NODE); } -int f2fs_lookup_journal_in_cursum(struct f2fs_journal *journal, int type, - unsigned int val, int alloc) +int f2fs_lookup_journal_in_cursum(struct f2fs_sb_info *sbi, + struct f2fs_journal *journal, int type, + unsigned int val, int alloc) { int i; @@ -4555,13 +4559,13 @@ int f2fs_lookup_journal_in_cursum(struct f2fs_journal *journal, int type, if (le32_to_cpu(nid_in_journal(journal, i)) == val) return i; } - if (alloc && __has_cursum_space(journal, 1, NAT_JOURNAL)) + if (alloc && __has_cursum_space(sbi, journal, 1, NAT_JOURNAL)) return update_nats_in_cursum(journal, 1); } else if (type == SIT_JOURNAL) { for (i = 0; i < sits_in_cursum(journal); i++) if (le32_to_cpu(segno_in_journal(journal, i)) == val) return i; - if (alloc && __has_cursum_space(journal, 1, SIT_JOURNAL)) + if (alloc && __has_cursum_space(sbi, journal, 1, SIT_JOURNAL)) return update_sits_in_cursum(journal, 1); } return -1; @@ -4709,8 +4713,8 @@ void f2fs_flush_sit_entries(struct f2fs_sb_info *sbi, struct cp_control *cpc) * entries, remove all entries from journal and add and account * them in sit entry set. */ - if (!__has_cursum_space(journal, sit_i->dirty_sentries, SIT_JOURNAL) || - !to_journal) + if (!__has_cursum_space(sbi, journal, + sit_i->dirty_sentries, SIT_JOURNAL) || !to_journal) remove_sits_in_journal(sbi); /* @@ -4727,7 +4731,8 @@ void f2fs_flush_sit_entries(struct f2fs_sb_info *sbi, struct cp_control *cpc) unsigned int segno = start_segno; if (to_journal && - !__has_cursum_space(journal, ses->entry_cnt, SIT_JOURNAL)) + !__has_cursum_space(sbi, journal, ses->entry_cnt, + SIT_JOURNAL)) to_journal = false; if (to_journal) { @@ -4755,7 +4760,7 @@ void f2fs_flush_sit_entries(struct f2fs_sb_info *sbi, struct cp_control *cpc) } if (to_journal) { - offset = f2fs_lookup_journal_in_cursum(journal, + offset = f2fs_lookup_journal_in_cursum(sbi, journal, SIT_JOURNAL, segno, 1); f2fs_bug_on(sbi, offset < 0); segno_in_journal(journal, offset) = @@ -4962,12 +4967,13 @@ static int build_curseg(struct f2fs_sb_info *sbi) for (i = 0; i < NO_CHECK_TYPE; i++) { mutex_init(&array[i].curseg_mutex); - array[i].sum_blk = f2fs_kzalloc(sbi, PAGE_SIZE, GFP_KERNEL); + array[i].sum_blk = f2fs_kzalloc(sbi, sbi->sum_blocksize, + GFP_KERNEL); if (!array[i].sum_blk) return -ENOMEM; init_rwsem(&array[i].journal_rwsem); array[i].journal = f2fs_kzalloc(sbi, - sizeof(struct f2fs_journal), GFP_KERNEL); + sbi->sum_journal_size, GFP_KERNEL); if (!array[i].journal) return -ENOMEM; array[i].seg_type = log_type_to_seg_type(i); diff --git a/fs/f2fs/segment.h b/fs/f2fs/segment.h index 07dcbcbeb7c6d3..3094f2de37b636 100644 --- a/fs/f2fs/segment.h +++ b/fs/f2fs/segment.h @@ -90,12 +90,11 @@ static inline void sanity_check_seg_type(struct f2fs_sb_info *sbi, #define GET_ZONE_FROM_SEG(sbi, segno) \ GET_ZONE_FROM_SEC(sbi, GET_SEC_FROM_SEG(sbi, segno)) -#define SUMS_PER_BLOCK (F2FS_BLKSIZE / F2FS_SUM_BLKSIZE) #define GET_SUM_BLOCK(sbi, segno) \ - (SM_I(sbi)->ssa_blkaddr + (segno / SUMS_PER_BLOCK)) -#define GET_SUM_BLKOFF(segno) (segno % SUMS_PER_BLOCK) -#define SUM_BLK_PAGE_ADDR(folio, segno) \ - (folio_address(folio) + GET_SUM_BLKOFF(segno) * F2FS_SUM_BLKSIZE) + (SM_I(sbi)->ssa_blkaddr + (segno / (sbi)->sums_per_block)) +#define GET_SUM_BLKOFF(sbi, segno) (segno % (sbi)->sums_per_block) +#define SUM_BLK_PAGE_ADDR(sbi, folio, segno) \ + (folio_address(folio) + GET_SUM_BLKOFF(sbi, segno) * (sbi)->sum_blocksize) #define GET_SUM_TYPE(footer) ((footer)->entry_type) #define SET_SUM_TYPE(footer, type) ((footer)->entry_type = (type)) diff --git a/fs/f2fs/super.c b/fs/f2fs/super.c index c4c225e09dc470..6be6d7372badf5 100644 --- a/fs/f2fs/super.c +++ b/fs/f2fs/super.c @@ -2636,11 +2636,10 @@ static int f2fs_disable_checkpoint(struct f2fs_sb_info *sbi) static int f2fs_enable_checkpoint(struct f2fs_sb_info *sbi) { unsigned int nr_pages = get_pages(sbi, F2FS_DIRTY_DATA) / 16; - long long start, writeback, lock, sync_inode, end; + long long start, writeback, end; int ret; - f2fs_info(sbi, "%s start, meta: %lld, node: %lld, data: %lld", - __func__, + f2fs_info(sbi, "f2fs_enable_checkpoint() starts, meta: %lld, node: %lld, data: %lld", get_pages(sbi, F2FS_DIRTY_META), get_pages(sbi, F2FS_DIRTY_NODES), get_pages(sbi, F2FS_DIRTY_DATA)); @@ -2659,18 +2658,11 @@ static int f2fs_enable_checkpoint(struct f2fs_sb_info *sbi) } writeback = ktime_get(); - f2fs_down_write(&sbi->cp_enable_rwsem); - - lock = ktime_get(); - - if (get_pages(sbi, F2FS_DIRTY_DATA)) - sync_inodes_sb(sbi->sb); + sync_inodes_sb(sbi->sb); if (unlikely(get_pages(sbi, F2FS_DIRTY_DATA))) - f2fs_warn(sbi, "%s: has some unwritten data: %lld", - __func__, get_pages(sbi, F2FS_DIRTY_DATA)); - - sync_inode = ktime_get(); + f2fs_warn(sbi, "checkpoint=enable has some unwritten data: %lld", + get_pages(sbi, F2FS_DIRTY_DATA)); f2fs_down_write(&sbi->gc_lock); f2fs_dirty_to_prefree(sbi); @@ -2679,13 +2671,6 @@ static int f2fs_enable_checkpoint(struct f2fs_sb_info *sbi) set_sbi_flag(sbi, SBI_IS_DIRTY); f2fs_up_write(&sbi->gc_lock); - f2fs_info(sbi, "%s sync_fs, meta: %lld, imeta: %lld, node: %lld, dents: %lld, qdata: %lld", - __func__, - get_pages(sbi, F2FS_DIRTY_META), - get_pages(sbi, F2FS_DIRTY_IMETA), - get_pages(sbi, F2FS_DIRTY_NODES), - get_pages(sbi, F2FS_DIRTY_DENTS), - get_pages(sbi, F2FS_DIRTY_QDATA)); ret = f2fs_sync_fs(sbi->sb, 1); if (ret) f2fs_err(sbi, "%s sync_fs failed, ret: %d", __func__, ret); @@ -2693,17 +2678,11 @@ static int f2fs_enable_checkpoint(struct f2fs_sb_info *sbi) /* Let's ensure there's no pending checkpoint anymore */ f2fs_flush_ckpt_thread(sbi); - f2fs_up_write(&sbi->cp_enable_rwsem); - end = ktime_get(); - f2fs_info(sbi, "%s end, writeback:%llu, " - "lock:%llu, sync_inode:%llu, sync_fs:%llu", - __func__, - ktime_ms_delta(writeback, start), - ktime_ms_delta(lock, writeback), - ktime_ms_delta(sync_inode, lock), - ktime_ms_delta(end, sync_inode)); + f2fs_info(sbi, "f2fs_enable_checkpoint() finishes, writeback:%llu, sync:%llu", + ktime_ms_delta(writeback, start), + ktime_ms_delta(end, writeback)); return ret; } @@ -4080,20 +4059,6 @@ static int sanity_check_raw_super(struct f2fs_sb_info *sbi, if (sanity_check_area_boundary(sbi, folio, index)) return -EFSCORRUPTED; - /* - * Check for legacy summary layout on 16KB+ block devices. - * Modern f2fs-tools packs multiple 4KB summary areas into one block, - * whereas legacy versions used one block per summary, leading - * to a much larger SSA. - */ - if (SUMS_PER_BLOCK > 1 && - !(__F2FS_HAS_FEATURE(raw_super, F2FS_FEATURE_PACKED_SSA))) { - f2fs_info(sbi, "Error: Device formatted with a legacy version. " - "Please reformat with a tool supporting the packed ssa " - "feature for block sizes larger than 4kb."); - return -EOPNOTSUPP; - } - return 0; } @@ -4304,6 +4269,18 @@ static void init_sb_info(struct f2fs_sb_info *sbi) spin_lock_init(&sbi->gc_remaining_trials_lock); atomic64_set(&sbi->current_atomic_write, 0); + sbi->sum_blocksize = f2fs_sb_has_packed_ssa(sbi) ? + 4096 : sbi->blocksize; + sbi->sums_per_block = sbi->blocksize / sbi->sum_blocksize; + sbi->entries_in_sum = sbi->sum_blocksize / 8; + sbi->sum_entry_size = SUMMARY_SIZE * sbi->entries_in_sum; + sbi->sum_journal_size = sbi->sum_blocksize - SUM_FOOTER_SIZE - + sbi->sum_entry_size; + sbi->nat_journal_entries = (sbi->sum_journal_size - 2) / + sizeof(struct nat_journal_entry); + sbi->sit_journal_entries = (sbi->sum_journal_size - 2) / + sizeof(struct sit_journal_entry); + sbi->dir_level = DEF_DIR_LEVEL; sbi->interval_time[CP_TIME] = DEF_CP_INTERVAL; sbi->interval_time[REQ_TIME] = DEF_IDLE_INTERVAL; @@ -4906,7 +4883,6 @@ static int f2fs_fill_super(struct super_block *sb, struct fs_context *fc) init_f2fs_rwsem(&sbi->node_change); spin_lock_init(&sbi->stat_lock); init_f2fs_rwsem(&sbi->cp_rwsem); - init_f2fs_rwsem(&sbi->cp_enable_rwsem); init_f2fs_rwsem(&sbi->quota_sem); init_waitqueue_head(&sbi->cp_wait); spin_lock_init(&sbi->error_lock); diff --git a/fs/f2fs/sysfs.c b/fs/f2fs/sysfs.c index c42f4f979d13f3..353bf47959f36a 100644 --- a/fs/f2fs/sysfs.c +++ b/fs/f2fs/sysfs.c @@ -58,6 +58,7 @@ struct f2fs_attr { const char *buf, size_t len); int struct_type; int offset; + int size; int id; }; @@ -344,11 +345,30 @@ static ssize_t main_blkaddr_show(struct f2fs_attr *a, (unsigned long long)MAIN_BLKADDR(sbi)); } +static ssize_t __sbi_show_value(struct f2fs_attr *a, + struct f2fs_sb_info *sbi, char *buf, + unsigned char *value) +{ + switch (a->size) { + case 1: + return sysfs_emit(buf, "%u\n", *(u8 *)value); + case 2: + return sysfs_emit(buf, "%u\n", *(u16 *)value); + case 4: + return sysfs_emit(buf, "%u\n", *(u32 *)value); + case 8: + return sysfs_emit(buf, "%llu\n", *(u64 *)value); + default: + f2fs_bug_on(sbi, 1); + return sysfs_emit(buf, + "show sysfs node value with wrong type\n"); + } +} + static ssize_t f2fs_sbi_show(struct f2fs_attr *a, struct f2fs_sb_info *sbi, char *buf) { unsigned char *ptr = NULL; - unsigned int *ui; ptr = __struct_ptr(sbi, a->struct_type); if (!ptr) @@ -428,9 +448,30 @@ static ssize_t f2fs_sbi_show(struct f2fs_attr *a, atomic_read(&sbi->cp_call_count[BACKGROUND])); #endif - ui = (unsigned int *)(ptr + a->offset); + return __sbi_show_value(a, sbi, buf, ptr + a->offset); +} - return sysfs_emit(buf, "%u\n", *ui); +static void __sbi_store_value(struct f2fs_attr *a, + struct f2fs_sb_info *sbi, + unsigned char *ui, unsigned long value) +{ + switch (a->size) { + case 1: + *(u8 *)ui = value; + break; + case 2: + *(u16 *)ui = value; + break; + case 4: + *(u32 *)ui = value; + break; + case 8: + *(u64 *)ui = value; + break; + default: + f2fs_bug_on(sbi, 1); + f2fs_err(sbi, "store sysfs node value with wrong type"); + } } static ssize_t __sbi_store(struct f2fs_attr *a, @@ -749,7 +790,7 @@ static ssize_t __sbi_store(struct f2fs_attr *a, return count; } - if (!strcmp(a->attr.name, "gc_pin_file_threshold")) { + if (!strcmp(a->attr.name, "gc_pin_file_thresh")) { if (t > MAX_GC_FAILED_PINNED_FILES) return -EINVAL; sbi->gc_pin_file_threshold = t; @@ -906,7 +947,7 @@ static ssize_t __sbi_store(struct f2fs_attr *a, return count; } - *ui = (unsigned int)t; + __sbi_store_value(a, sbi, ptr + a->offset, t); return count; } @@ -1053,24 +1094,27 @@ static struct f2fs_attr f2fs_attr_sb_##_name = { \ .id = F2FS_FEATURE_##_feat, \ } -#define F2FS_ATTR_OFFSET(_struct_type, _name, _mode, _show, _store, _offset) \ +#define F2FS_ATTR_OFFSET(_struct_type, _name, _mode, _show, _store, _offset, _size) \ static struct f2fs_attr f2fs_attr_##_name = { \ .attr = {.name = __stringify(_name), .mode = _mode }, \ .show = _show, \ .store = _store, \ .struct_type = _struct_type, \ - .offset = _offset \ + .offset = _offset, \ + .size = _size \ } #define F2FS_RO_ATTR(struct_type, struct_name, name, elname) \ F2FS_ATTR_OFFSET(struct_type, name, 0444, \ f2fs_sbi_show, NULL, \ - offsetof(struct struct_name, elname)) + offsetof(struct struct_name, elname), \ + sizeof_field(struct struct_name, elname)) #define F2FS_RW_ATTR(struct_type, struct_name, name, elname) \ F2FS_ATTR_OFFSET(struct_type, name, 0644, \ f2fs_sbi_show, f2fs_sbi_store, \ - offsetof(struct struct_name, elname)) + offsetof(struct struct_name, elname), \ + sizeof_field(struct struct_name, elname)) #define F2FS_GENERAL_RO_ATTR(name) \ static struct f2fs_attr f2fs_attr_##name = __ATTR(name, 0444, name##_show, NULL) diff --git a/fs/fat/namei_msdos.c b/fs/fat/namei_msdos.c index 0b920ee40a7f9f..262ec1b790b560 100644 --- a/fs/fat/namei_msdos.c +++ b/fs/fat/namei_msdos.c @@ -325,7 +325,12 @@ static int msdos_rmdir(struct inode *dir, struct dentry *dentry) err = fat_remove_entries(dir, &sinfo); /* and releases bh */ if (err) goto out; - drop_nlink(dir); + if (dir->i_nlink >= 3) + drop_nlink(dir); + else { + fat_fs_error(sb, "parent dir link count too low (%u)", + dir->i_nlink); + } clear_nlink(inode); fat_truncate_time(inode, NULL, S_CTIME); diff --git a/fs/fat/namei_vfat.c b/fs/fat/namei_vfat.c index 5dbc4cbb8fce3d..47ff083cfc7e66 100644 --- a/fs/fat/namei_vfat.c +++ b/fs/fat/namei_vfat.c @@ -803,7 +803,12 @@ static int vfat_rmdir(struct inode *dir, struct dentry *dentry) err = fat_remove_entries(dir, &sinfo); /* and releases bh */ if (err) goto out; - drop_nlink(dir); + if (dir->i_nlink >= 3) + drop_nlink(dir); + else { + fat_fs_error(sb, "parent dir link count too low (%u)", + dir->i_nlink); + } clear_nlink(inode); fat_truncate_time(inode, NULL, S_ATIME|S_MTIME); diff --git a/fs/file_attr.c b/fs/file_attr.c index 13cdb31a3e9472..4889cf59b25626 100644 --- a/fs/file_attr.c +++ b/fs/file_attr.c @@ -377,7 +377,7 @@ SYSCALL_DEFINE5(file_getattr, int, dfd, const char __user *, filename, struct filename *name __free(putname) = NULL; unsigned int lookup_flags = 0; struct file_attr fattr; - struct file_kattr fa; + struct file_kattr fa = { .flags_valid = true }; /* hint only */ int error; BUILD_BUG_ON(sizeof(struct file_attr) < FILE_ATTR_SIZE_VER0); diff --git a/fs/file_table.c b/fs/file_table.c index cd4a3db4659ac4..34244fccf2edf0 100644 --- a/fs/file_table.c +++ b/fs/file_table.c @@ -176,6 +176,11 @@ static int init_file(struct file *f, int flags, const struct cred *cred) f->f_flags = flags; f->f_mode = OPEN_FMODE(flags); + /* + * Disable permission and pre-content events for all files by default. + * They may be enabled later by fsnotify_open_perm_and_set_mode(). + */ + file_set_fsnotify_mode(f, FMODE_NONOTIFY_PERM); f->f_op = NULL; f->f_mapping = NULL; @@ -197,11 +202,6 @@ static int init_file(struct file *f, int flags, const struct cred *cred) * refcount bumps we should reinitialize the reused file first. */ file_ref_init(&f->f_ref, 1); - /* - * Disable permission and pre-content events for all files by default. - * They may be enabled later by fsnotify_open_perm_and_set_mode(). - */ - file_set_fsnotify_mode(f, FMODE_NONOTIFY_PERM); return 0; } diff --git a/fs/fs-writeback.c b/fs/fs-writeback.c index 5444fc706ac7d6..993f865c9156f6 100644 --- a/fs/fs-writeback.c +++ b/fs/fs-writeback.c @@ -198,10 +198,11 @@ static void wb_queue_work(struct bdi_writeback *wb, static bool wb_wait_for_completion_cb(struct wb_completion *done) { + unsigned long timeout = sysctl_hung_task_timeout_secs; unsigned long waited_secs = (jiffies - done->wait_start) / HZ; done->progress_stamp = jiffies; - if (waited_secs > sysctl_hung_task_timeout_secs) + if (timeout && (waited_secs > timeout)) pr_info("INFO: The task %s:%d has been waiting for writeback " "completion for more than %lu seconds.", current->comm, current->pid, waited_secs); @@ -1944,6 +1945,7 @@ static long writeback_sb_inodes(struct super_block *sb, .range_end = LLONG_MAX, }; unsigned long start_time = jiffies; + unsigned long timeout = sysctl_hung_task_timeout_secs; long write_chunk; long total_wrote = 0; /* count both pages and inodes */ unsigned long dirtied_before = jiffies; @@ -2030,9 +2032,8 @@ static long writeback_sb_inodes(struct super_block *sb, __writeback_single_inode(inode, &wbc); /* Report progress to inform the hung task detector of the progress. */ - if (work->done && work->done->progress_stamp && - (jiffies - work->done->progress_stamp) > HZ * - sysctl_hung_task_timeout_secs / 2) + if (work->done && work->done->progress_stamp && timeout && + (jiffies - work->done->progress_stamp) > HZ * timeout / 2) wake_up_all(work->done->waitq); wbc_detach_inode(&wbc); @@ -2758,13 +2759,8 @@ static void wait_sb_inodes(struct super_block *sb) * The mapping can appear untagged while still on-list since we * do not have the mapping lock. Skip it here, wb completion * will remove it. - * - * If the mapping does not have data integrity semantics, - * there's no need to wait for the writeout to complete, as the - * mapping cannot guarantee that data is persistently stored. */ - if (!mapping_tagged(mapping, PAGECACHE_TAG_WRITEBACK) || - mapping_no_data_integrity(mapping)) + if (!mapping_tagged(mapping, PAGECACHE_TAG_WRITEBACK)) continue; spin_unlock_irq(&sb->s_inode_wblist_lock); @@ -2899,6 +2895,17 @@ void sync_inodes_sb(struct super_block *sb) */ if (bdi == &noop_backing_dev_info) return; + + /* + * If the superblock has SB_I_NO_DATA_INTEGRITY set, there's no need to + * wait for the writeout to complete, as the filesystem cannot guarantee + * data persistence on sync. Just kick off writeback and return. + */ + if (sb->s_iflags & SB_I_NO_DATA_INTEGRITY) { + wakeup_flusher_threads_bdi(bdi, WB_REASON_SYNC); + return; + } + WARN_ON(!rwsem_is_locked(&sb->s_umount)); /* protect against inode wb switch, see inode_switch_wbs_work_fn() */ diff --git a/fs/fs_struct.c b/fs/fs_struct.c index b8c46c5a38a057..394875d06fd606 100644 --- a/fs/fs_struct.c +++ b/fs/fs_struct.c @@ -6,6 +6,7 @@ #include #include #include +#include #include "internal.h" /* diff --git a/fs/fuse/file.c b/fs/fuse/file.c index 3b2a171e652f0c..01bc894e9c2bae 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -3200,10 +3200,8 @@ void fuse_init_file_inode(struct inode *inode, unsigned int flags) inode->i_fop = &fuse_file_operations; inode->i_data.a_ops = &fuse_file_aops; - if (fc->writeback_cache) { + if (fc->writeback_cache) mapping_set_writeback_may_deadlock_on_reclaim(&inode->i_data); - mapping_set_no_data_integrity(&inode->i_data); - } INIT_LIST_HEAD(&fi->write_files); INIT_LIST_HEAD(&fi->queued_writes); diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c index 819e50d666224a..d7a226e6af1f97 100644 --- a/fs/fuse/inode.c +++ b/fs/fuse/inode.c @@ -1709,6 +1709,7 @@ static void fuse_sb_defaults(struct super_block *sb) sb->s_export_op = &fuse_export_operations; sb->s_iflags |= SB_I_IMA_UNVERIFIABLE_SIGNATURE; sb->s_iflags |= SB_I_NOIDMAP; + sb->s_iflags |= SB_I_NO_DATA_INTEGRITY; if (sb->s_user_ns != &init_user_ns) sb->s_iflags |= SB_I_UNTRUSTED_MOUNTER; sb->s_flags &= ~(SB_NOSEC | SB_I_VERSION); diff --git a/fs/gfs2/bmap.c b/fs/gfs2/bmap.c index 131091520de6be..fdcac8e3f2ba26 100644 --- a/fs/gfs2/bmap.c +++ b/fs/gfs2/bmap.c @@ -1127,10 +1127,18 @@ static int gfs2_iomap_begin(struct inode *inode, loff_t pos, loff_t length, goto out_unlock; break; default: - goto out_unlock; + goto out; } ret = gfs2_iomap_begin_write(inode, pos, length, flags, iomap, &mp); + if (ret) + goto out_unlock; + +out: + if (iomap->type == IOMAP_INLINE) { + iomap->private = metapath_dibh(&mp); + get_bh(iomap->private); + } out_unlock: release_metapath(&mp); @@ -1144,6 +1152,9 @@ static int gfs2_iomap_end(struct inode *inode, loff_t pos, loff_t length, struct gfs2_inode *ip = GFS2_I(inode); struct gfs2_sbd *sdp = GFS2_SB(inode); + if (iomap->private) + brelse(iomap->private); + switch (flags & (IOMAP_WRITE | IOMAP_ZERO)) { case IOMAP_WRITE: if (flags & IOMAP_DIRECT) diff --git a/fs/gfs2/glock.c b/fs/gfs2/glock.c index 92e029104d8a42..289851d70130bd 100644 --- a/fs/gfs2/glock.c +++ b/fs/gfs2/glock.c @@ -1284,31 +1284,45 @@ static int glocks_pending(unsigned int num_gh, struct gfs2_holder *ghs) * gfs2_glock_async_wait - wait on multiple asynchronous glock acquisitions * @num_gh: the number of holders in the array * @ghs: the glock holder array + * @retries: number of retries attempted so far * * Returns: 0 on success, meaning all glocks have been granted and are held. * -ESTALE if the request timed out, meaning all glocks were released, * and the caller should retry the operation. */ -int gfs2_glock_async_wait(unsigned int num_gh, struct gfs2_holder *ghs) +int gfs2_glock_async_wait(unsigned int num_gh, struct gfs2_holder *ghs, + unsigned int retries) { struct gfs2_sbd *sdp = ghs[0].gh_gl->gl_name.ln_sbd; - int i, ret = 0, timeout = 0; unsigned long start_time = jiffies; + int i, ret = 0; + long timeout; might_sleep(); - /* - * Total up the (minimum hold time * 2) of all glocks and use that to - * determine the max amount of time we should wait. - */ - for (i = 0; i < num_gh; i++) - timeout += ghs[i].gh_gl->gl_hold_time << 1; - if (!wait_event_timeout(sdp->sd_async_glock_wait, + timeout = GL_GLOCK_MIN_HOLD; + if (retries) { + unsigned int max_shift; + long incr; + + /* Add a random delay and increase the timeout exponentially. */ + max_shift = BITS_PER_LONG - 2 - __fls(GL_GLOCK_HOLD_INCR); + incr = min(GL_GLOCK_HOLD_INCR << min(retries - 1, max_shift), + 10 * HZ - GL_GLOCK_MIN_HOLD); + schedule_timeout_interruptible(get_random_long() % (incr / 3)); + if (signal_pending(current)) + goto interrupted; + timeout += (incr / 3) + get_random_long() % (incr / 3); + } + + if (!wait_event_interruptible_timeout(sdp->sd_async_glock_wait, !glocks_pending(num_gh, ghs), timeout)) { ret = -ESTALE; /* request timed out. */ goto out; } + if (signal_pending(current)) + goto interrupted; for (i = 0; i < num_gh; i++) { struct gfs2_holder *gh = &ghs[i]; @@ -1332,6 +1346,10 @@ int gfs2_glock_async_wait(unsigned int num_gh, struct gfs2_holder *ghs) } } return ret; + +interrupted: + ret = -EINTR; + goto out; } /** diff --git a/fs/gfs2/glock.h b/fs/gfs2/glock.h index 55d5985f32a080..dccbf36b8cb108 100644 --- a/fs/gfs2/glock.h +++ b/fs/gfs2/glock.h @@ -204,7 +204,8 @@ int gfs2_glock_poll(struct gfs2_holder *gh); int gfs2_instantiate(struct gfs2_holder *gh); int gfs2_glock_holder_ready(struct gfs2_holder *gh); int gfs2_glock_wait(struct gfs2_holder *gh); -int gfs2_glock_async_wait(unsigned int num_gh, struct gfs2_holder *ghs); +int gfs2_glock_async_wait(unsigned int num_gh, struct gfs2_holder *ghs, + unsigned int retries); void gfs2_glock_dq(struct gfs2_holder *gh); void gfs2_glock_dq_wait(struct gfs2_holder *gh); void gfs2_glock_dq_uninit(struct gfs2_holder *gh); diff --git a/fs/gfs2/inode.c b/fs/gfs2/inode.c index 36618e35319965..4d65e4a752626c 100644 --- a/fs/gfs2/inode.c +++ b/fs/gfs2/inode.c @@ -1495,7 +1495,7 @@ static int gfs2_rename(struct inode *odir, struct dentry *odentry, unsigned int num_gh; int dir_rename = 0; struct gfs2_diradd da = { .nr_blocks = 0, .save_loc = 0, }; - unsigned int x; + unsigned int retries = 0, x; int error; gfs2_holder_mark_uninitialized(&r_gh); @@ -1545,12 +1545,17 @@ static int gfs2_rename(struct inode *odir, struct dentry *odentry, num_gh++; } +again: for (x = 0; x < num_gh; x++) { error = gfs2_glock_nq(ghs + x); if (error) goto out_gunlock; } - error = gfs2_glock_async_wait(num_gh, ghs); + error = gfs2_glock_async_wait(num_gh, ghs, retries); + if (error == -ESTALE) { + retries++; + goto again; + } if (error) goto out_gunlock; @@ -1739,7 +1744,7 @@ static int gfs2_exchange(struct inode *odir, struct dentry *odentry, struct gfs2_sbd *sdp = GFS2_SB(odir); struct gfs2_holder ghs[4], r_gh; unsigned int num_gh; - unsigned int x; + unsigned int retries = 0, x; umode_t old_mode = oip->i_inode.i_mode; umode_t new_mode = nip->i_inode.i_mode; int error; @@ -1783,13 +1788,18 @@ static int gfs2_exchange(struct inode *odir, struct dentry *odentry, gfs2_holder_init(nip->i_gl, LM_ST_EXCLUSIVE, GL_ASYNC, ghs + num_gh); num_gh++; +again: for (x = 0; x < num_gh; x++) { error = gfs2_glock_nq(ghs + x); if (error) goto out_gunlock; } - error = gfs2_glock_async_wait(num_gh, ghs); + error = gfs2_glock_async_wait(num_gh, ghs, retries); + if (error == -ESTALE) { + retries++; + goto again; + } if (error) goto out_gunlock; @@ -2182,6 +2192,14 @@ static int gfs2_getattr(struct mnt_idmap *idmap, return 0; } +static bool fault_in_fiemap(struct fiemap_extent_info *fi) +{ + struct fiemap_extent __user *dest = fi->fi_extents_start; + size_t size = sizeof(*dest) * fi->fi_extents_max; + + return fault_in_safe_writeable((char __user *)dest, size) == 0; +} + static int gfs2_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, u64 start, u64 len) { @@ -2191,14 +2209,22 @@ static int gfs2_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, inode_lock_shared(inode); +retry: ret = gfs2_glock_nq_init(ip->i_gl, LM_ST_SHARED, 0, &gh); if (ret) goto out; + pagefault_disable(); ret = iomap_fiemap(inode, fieinfo, start, len, &gfs2_iomap_ops); + pagefault_enable(); gfs2_glock_dq_uninit(&gh); + if (ret == -EFAULT && fault_in_fiemap(fieinfo)) { + fieinfo->fi_extents_mapped = 0; + goto retry; + } + out: inode_unlock_shared(inode); return ret; diff --git a/fs/gfs2/log.c b/fs/gfs2/log.c index 8312cd2cdae471..347df29d610e67 100644 --- a/fs/gfs2/log.c +++ b/fs/gfs2/log.c @@ -888,8 +888,9 @@ void gfs2_write_log_header(struct gfs2_sbd *sdp, struct gfs2_jdesc *jd, sb->s_blocksize - LH_V1_SIZE - 4); lh->lh_crc = cpu_to_be32(crc); - gfs2_log_write(sdp, jd, page, sb->s_blocksize, 0, dblock); - gfs2_log_submit_bio(&jd->jd_log_bio, REQ_OP_WRITE | op_flags); + gfs2_log_write(sdp, jd, page, sb->s_blocksize, 0, dblock, + REQ_OP_WRITE | op_flags); + gfs2_log_submit_write(&jd->jd_log_bio); } /** @@ -1096,7 +1097,7 @@ void gfs2_log_flush(struct gfs2_sbd *sdp, struct gfs2_glock *gl, u32 flags) if (gfs2_withdrawn(sdp)) goto out_withdraw; if (sdp->sd_jdesc) - gfs2_log_submit_bio(&sdp->sd_jdesc->jd_log_bio, REQ_OP_WRITE); + gfs2_log_submit_write(&sdp->sd_jdesc->jd_log_bio); if (gfs2_withdrawn(sdp)) goto out_withdraw; diff --git a/fs/gfs2/lops.c b/fs/gfs2/lops.c index d27a0b1080a97f..c3317432a25b38 100644 --- a/fs/gfs2/lops.c +++ b/fs/gfs2/lops.c @@ -229,21 +229,19 @@ static void gfs2_end_log_write(struct bio *bio) } /** - * gfs2_log_submit_bio - Submit any pending log bio + * gfs2_log_submit_write - Submit a pending log write bio * @biop: Address of the bio pointer - * @opf: REQ_OP | op_flags * * Submit any pending part-built or full bio to the block device. If * there is no pending bio, then this is a no-op. */ -void gfs2_log_submit_bio(struct bio **biop, blk_opf_t opf) +void gfs2_log_submit_write(struct bio **biop) { struct bio *bio = *biop; if (bio) { struct gfs2_sbd *sdp = bio->bi_private; atomic_inc(&sdp->sd_log_in_flight); - bio->bi_opf = opf; submit_bio(bio); *biop = NULL; } @@ -254,6 +252,7 @@ void gfs2_log_submit_bio(struct bio **biop, blk_opf_t opf) * @sdp: The super block * @blkno: The device block number we want to write to * @end_io: The bi_end_io callback + * @opf: REQ_OP | op_flags * * Allocate a new bio, initialize it with the given parameters and return it. * @@ -261,10 +260,10 @@ void gfs2_log_submit_bio(struct bio **biop, blk_opf_t opf) */ static struct bio *gfs2_log_alloc_bio(struct gfs2_sbd *sdp, u64 blkno, - bio_end_io_t *end_io) + bio_end_io_t *end_io, blk_opf_t opf) { struct super_block *sb = sdp->sd_vfs; - struct bio *bio = bio_alloc(sb->s_bdev, BIO_MAX_VECS, 0, GFP_NOIO); + struct bio *bio = bio_alloc(sb->s_bdev, BIO_MAX_VECS, opf, GFP_NOIO); bio->bi_iter.bi_sector = blkno << sdp->sd_fsb2bb_shift; bio->bi_end_io = end_io; @@ -303,10 +302,10 @@ static struct bio *gfs2_log_get_bio(struct gfs2_sbd *sdp, u64 blkno, nblk >>= sdp->sd_fsb2bb_shift; if (blkno == nblk && !flush) return bio; - gfs2_log_submit_bio(biop, op); + gfs2_log_submit_write(biop); } - *biop = gfs2_log_alloc_bio(sdp, blkno, end_io); + *biop = gfs2_log_alloc_bio(sdp, blkno, end_io, op); return *biop; } @@ -318,6 +317,7 @@ static struct bio *gfs2_log_get_bio(struct gfs2_sbd *sdp, u64 blkno, * @size: the size of the data to write * @offset: the offset within the page * @blkno: block number of the log entry + * @opf: REQ_OP | op_flags * * Try and add the page segment to the current bio. If that fails, * submit the current bio to the device and create a new one, and @@ -326,17 +326,17 @@ static struct bio *gfs2_log_get_bio(struct gfs2_sbd *sdp, u64 blkno, void gfs2_log_write(struct gfs2_sbd *sdp, struct gfs2_jdesc *jd, struct page *page, unsigned size, unsigned offset, - u64 blkno) + u64 blkno, blk_opf_t opf) { struct bio *bio; int ret; - bio = gfs2_log_get_bio(sdp, blkno, &jd->jd_log_bio, REQ_OP_WRITE, + bio = gfs2_log_get_bio(sdp, blkno, &jd->jd_log_bio, opf, gfs2_end_log_write, false); ret = bio_add_page(bio, page, size, offset); if (ret == 0) { bio = gfs2_log_get_bio(sdp, blkno, &jd->jd_log_bio, - REQ_OP_WRITE, gfs2_end_log_write, true); + opf, gfs2_end_log_write, true); ret = bio_add_page(bio, page, size, offset); WARN_ON(ret == 0); } @@ -359,7 +359,7 @@ static void gfs2_log_write_bh(struct gfs2_sbd *sdp, struct buffer_head *bh) dblock = gfs2_log_bmap(sdp->sd_jdesc, sdp->sd_log_flush_head); gfs2_log_incr_head(sdp); gfs2_log_write(sdp, sdp->sd_jdesc, folio_page(bh->b_folio, 0), - bh->b_size, bh_offset(bh), dblock); + bh->b_size, bh_offset(bh), dblock, REQ_OP_WRITE); } /** @@ -380,7 +380,8 @@ static void gfs2_log_write_page(struct gfs2_sbd *sdp, struct page *page) dblock = gfs2_log_bmap(sdp->sd_jdesc, sdp->sd_log_flush_head); gfs2_log_incr_head(sdp); - gfs2_log_write(sdp, sdp->sd_jdesc, page, sb->s_blocksize, 0, dblock); + gfs2_log_write(sdp, sdp->sd_jdesc, page, sb->s_blocksize, 0, dblock, + REQ_OP_WRITE); } /** @@ -477,11 +478,12 @@ static void gfs2_jhead_process_page(struct gfs2_jdesc *jd, unsigned long index, folio_put_refs(folio, 2); } -static struct bio *gfs2_chain_bio(struct bio *prev, unsigned int nr_iovecs) +static struct bio *gfs2_chain_bio(struct bio *prev, unsigned int nr_iovecs, + blk_opf_t opf) { struct bio *new; - new = bio_alloc(prev->bi_bdev, nr_iovecs, prev->bi_opf, GFP_NOIO); + new = bio_alloc(prev->bi_bdev, nr_iovecs, opf, GFP_NOIO); bio_clone_blkg_association(new, prev); new->bi_iter.bi_sector = bio_end_sector(prev); bio_chain(new, prev); @@ -546,7 +548,8 @@ int gfs2_find_jhead(struct gfs2_jdesc *jd, struct gfs2_log_header_host *head) unsigned int blocks = (PAGE_SIZE - off) >> bsize_shift; - bio = gfs2_chain_bio(bio, blocks); + bio = gfs2_chain_bio(bio, blocks, + REQ_OP_READ); goto add_block_to_new_bio; } } @@ -556,8 +559,8 @@ int gfs2_find_jhead(struct gfs2_jdesc *jd, struct gfs2_log_header_host *head) submit_bio(bio); } - bio = gfs2_log_alloc_bio(sdp, dblock, gfs2_end_log_read); - bio->bi_opf = REQ_OP_READ; + bio = gfs2_log_alloc_bio(sdp, dblock, gfs2_end_log_read, + REQ_OP_READ); add_block_to_new_bio: bio_add_folio_nofail(bio, folio, bsize, off); block_added: diff --git a/fs/gfs2/lops.h b/fs/gfs2/lops.h index be740bf3366646..772557b63b48b7 100644 --- a/fs/gfs2/lops.h +++ b/fs/gfs2/lops.h @@ -16,8 +16,8 @@ void gfs2_log_incr_head(struct gfs2_sbd *sdp); u64 gfs2_log_bmap(struct gfs2_jdesc *jd, unsigned int lbn); void gfs2_log_write(struct gfs2_sbd *sdp, struct gfs2_jdesc *jd, struct page *page, unsigned size, unsigned offset, - u64 blkno); -void gfs2_log_submit_bio(struct bio **biop, blk_opf_t opf); + u64 blkno, blk_opf_t opf); +void gfs2_log_submit_write(struct bio **biop); void gfs2_pin(struct gfs2_sbd *sdp, struct buffer_head *bh); int gfs2_find_jhead(struct gfs2_jdesc *jd, struct gfs2_log_header_host *head); diff --git a/fs/gfs2/ops_fstype.c b/fs/gfs2/ops_fstype.c index e7a88b717991ae..c7d57de7c8f06b 100644 --- a/fs/gfs2/ops_fstype.c +++ b/fs/gfs2/ops_fstype.c @@ -1276,7 +1276,6 @@ static int gfs2_fill_super(struct super_block *sb, struct fs_context *fc) if (error) { gfs2_freeze_unlock(sdp); - gfs2_destroy_threads(sdp); fs_err(sdp, "can't make FS RW: %d\n", error); goto fail_per_node; } @@ -1286,6 +1285,7 @@ static int gfs2_fill_super(struct super_block *sb, struct fs_context *fc) fail_per_node: init_per_node(sdp, UNDO); + gfs2_destroy_threads(sdp); fail_inodes: init_inodes(sdp, UNDO); fail_sb: diff --git a/fs/gfs2/quota.c b/fs/gfs2/quota.c index b1692f12a602a6..2b499b554e876a 100644 --- a/fs/gfs2/quota.c +++ b/fs/gfs2/quota.c @@ -334,6 +334,7 @@ static void qd_put(struct gfs2_quota_data *qd) lockref_mark_dead(&qd->qd_lockref); spin_unlock(&qd->qd_lockref.lock); + list_lru_del_obj(&gfs2_qd_lru, &qd->qd_lru); gfs2_qd_dispose(qd); return; } diff --git a/fs/gfs2/super.c b/fs/gfs2/super.c index f6cd907b3ec6c4..d96160636161c8 100644 --- a/fs/gfs2/super.c +++ b/fs/gfs2/super.c @@ -147,8 +147,10 @@ int gfs2_make_fs_rw(struct gfs2_sbd *sdp) } error = gfs2_quota_init(sdp); - if (!error && gfs2_withdrawn(sdp)) + if (!error && gfs2_withdrawn(sdp)) { + gfs2_quota_cleanup(sdp); error = -EIO; + } if (!error) set_bit(SDF_JOURNAL_LIVE, &sdp->sd_flags); return error; diff --git a/fs/hfs/dir.c b/fs/hfs/dir.c index 86a6b317b474a9..0c615c078650cc 100644 --- a/fs/hfs/dir.c +++ b/fs/hfs/dir.c @@ -196,8 +196,8 @@ static int hfs_create(struct mnt_idmap *idmap, struct inode *dir, int res; inode = hfs_new_inode(dir, &dentry->d_name, mode); - if (!inode) - return -ENOMEM; + if (IS_ERR(inode)) + return PTR_ERR(inode); res = hfs_cat_create(inode->i_ino, dir, &dentry->d_name, inode); if (res) { @@ -226,8 +226,8 @@ static struct dentry *hfs_mkdir(struct mnt_idmap *idmap, struct inode *dir, int res; inode = hfs_new_inode(dir, &dentry->d_name, S_IFDIR | mode); - if (!inode) - return ERR_PTR(-ENOMEM); + if (IS_ERR(inode)) + return ERR_CAST(inode); res = hfs_cat_create(inode->i_ino, dir, &dentry->d_name, inode); if (res) { @@ -254,11 +254,18 @@ static struct dentry *hfs_mkdir(struct mnt_idmap *idmap, struct inode *dir, */ static int hfs_remove(struct inode *dir, struct dentry *dentry) { + struct super_block *sb = dir->i_sb; struct inode *inode = d_inode(dentry); int res; if (S_ISDIR(inode->i_mode) && inode->i_size != 2) return -ENOTEMPTY; + + if (unlikely(!is_hfs_cnid_counts_valid(sb))) { + pr_err("cannot remove file/folder\n"); + return -ERANGE; + } + res = hfs_cat_delete(inode->i_ino, dir, &dentry->d_name); if (res) return res; diff --git a/fs/hfs/hfs_fs.h b/fs/hfs/hfs_fs.h index e94dbc04a1e434..ac0e83f77a0f19 100644 --- a/fs/hfs/hfs_fs.h +++ b/fs/hfs/hfs_fs.h @@ -199,6 +199,7 @@ extern void hfs_delete_inode(struct inode *inode); extern const struct xattr_handler * const hfs_xattr_handlers[]; /* mdb.c */ +extern bool is_hfs_cnid_counts_valid(struct super_block *sb); extern int hfs_mdb_get(struct super_block *sb); extern void hfs_mdb_commit(struct super_block *sb); extern void hfs_mdb_close(struct super_block *sb); diff --git a/fs/hfs/inode.c b/fs/hfs/inode.c index 524db1389737d8..878535db64d679 100644 --- a/fs/hfs/inode.c +++ b/fs/hfs/inode.c @@ -187,16 +187,23 @@ struct inode *hfs_new_inode(struct inode *dir, const struct qstr *name, umode_t s64 next_id; s64 file_count; s64 folder_count; + int err = -ENOMEM; if (!inode) - return NULL; + goto out_err; + + err = -ERANGE; mutex_init(&HFS_I(inode)->extents_lock); INIT_LIST_HEAD(&HFS_I(inode)->open_dir_list); spin_lock_init(&HFS_I(inode)->open_dir_lock); hfs_cat_build_key(sb, (btree_key *)&HFS_I(inode)->cat_key, dir->i_ino, name); next_id = atomic64_inc_return(&HFS_SB(sb)->next_id); - BUG_ON(next_id > U32_MAX); + if (next_id > U32_MAX) { + atomic64_dec(&HFS_SB(sb)->next_id); + pr_err("cannot create new inode: next CNID exceeds limit\n"); + goto out_discard; + } inode->i_ino = (u32)next_id; inode->i_mode = mode; inode->i_uid = current_fsuid(); @@ -210,7 +217,11 @@ struct inode *hfs_new_inode(struct inode *dir, const struct qstr *name, umode_t if (S_ISDIR(mode)) { inode->i_size = 2; folder_count = atomic64_inc_return(&HFS_SB(sb)->folder_count); - BUG_ON(folder_count > U32_MAX); + if (folder_count> U32_MAX) { + atomic64_dec(&HFS_SB(sb)->folder_count); + pr_err("cannot create new inode: folder count exceeds limit\n"); + goto out_discard; + } if (dir->i_ino == HFS_ROOT_CNID) HFS_SB(sb)->root_dirs++; inode->i_op = &hfs_dir_inode_operations; @@ -220,7 +231,11 @@ struct inode *hfs_new_inode(struct inode *dir, const struct qstr *name, umode_t } else if (S_ISREG(mode)) { HFS_I(inode)->clump_blocks = HFS_SB(sb)->clumpablks; file_count = atomic64_inc_return(&HFS_SB(sb)->file_count); - BUG_ON(file_count > U32_MAX); + if (file_count > U32_MAX) { + atomic64_dec(&HFS_SB(sb)->file_count); + pr_err("cannot create new inode: file count exceeds limit\n"); + goto out_discard; + } if (dir->i_ino == HFS_ROOT_CNID) HFS_SB(sb)->root_files++; inode->i_op = &hfs_file_inode_operations; @@ -244,6 +259,11 @@ struct inode *hfs_new_inode(struct inode *dir, const struct qstr *name, umode_t hfs_mark_mdb_dirty(sb); return inode; + + out_discard: + iput(inode); + out_err: + return ERR_PTR(err); } void hfs_delete_inode(struct inode *inode) @@ -252,7 +272,6 @@ void hfs_delete_inode(struct inode *inode) hfs_dbg("ino %lu\n", inode->i_ino); if (S_ISDIR(inode->i_mode)) { - BUG_ON(atomic64_read(&HFS_SB(sb)->folder_count) > U32_MAX); atomic64_dec(&HFS_SB(sb)->folder_count); if (HFS_I(inode)->cat_key.ParID == cpu_to_be32(HFS_ROOT_CNID)) HFS_SB(sb)->root_dirs--; @@ -261,7 +280,6 @@ void hfs_delete_inode(struct inode *inode) return; } - BUG_ON(atomic64_read(&HFS_SB(sb)->file_count) > U32_MAX); atomic64_dec(&HFS_SB(sb)->file_count); if (HFS_I(inode)->cat_key.ParID == cpu_to_be32(HFS_ROOT_CNID)) HFS_SB(sb)->root_files--; diff --git a/fs/hfs/mdb.c b/fs/hfs/mdb.c index 53f3fae6021797..a97cea35ca2e12 100644 --- a/fs/hfs/mdb.c +++ b/fs/hfs/mdb.c @@ -64,6 +64,27 @@ static int hfs_get_last_session(struct super_block *sb, return 0; } +bool is_hfs_cnid_counts_valid(struct super_block *sb) +{ + struct hfs_sb_info *sbi = HFS_SB(sb); + bool corrupted = false; + + if (unlikely(atomic64_read(&sbi->next_id) > U32_MAX)) { + pr_warn("next CNID exceeds limit\n"); + corrupted = true; + } + if (unlikely(atomic64_read(&sbi->file_count) > U32_MAX)) { + pr_warn("file count exceeds limit\n"); + corrupted = true; + } + if (unlikely(atomic64_read(&sbi->folder_count) > U32_MAX)) { + pr_warn("folder count exceeds limit\n"); + corrupted = true; + } + + return !corrupted; +} + /* * hfs_mdb_get() * @@ -92,7 +113,7 @@ int hfs_mdb_get(struct super_block *sb) /* See if this is an HFS filesystem */ bh = sb_bread512(sb, part_start + HFS_MDB_BLK, mdb); if (!bh) - goto out; + return -EIO; if (mdb->drSigWord == cpu_to_be16(HFS_SUPER_MAGIC)) break; @@ -102,13 +123,14 @@ int hfs_mdb_get(struct super_block *sb) * (should do this only for cdrom/loop though) */ if (hfs_part_find(sb, &part_start, &part_size)) - goto out; + return -EIO; } HFS_SB(sb)->alloc_blksz = size = be32_to_cpu(mdb->drAlBlkSiz); if (!size || (size & (HFS_SECTOR_SIZE - 1))) { pr_err("bad allocation block size %d\n", size); - goto out_bh; + brelse(bh); + return -EIO; } size = min(HFS_SB(sb)->alloc_blksz, (u32)PAGE_SIZE); @@ -125,14 +147,16 @@ int hfs_mdb_get(struct super_block *sb) brelse(bh); if (!sb_set_blocksize(sb, size)) { pr_err("unable to set blocksize to %u\n", size); - goto out; + return -EIO; } bh = sb_bread512(sb, part_start + HFS_MDB_BLK, mdb); if (!bh) - goto out; - if (mdb->drSigWord != cpu_to_be16(HFS_SUPER_MAGIC)) - goto out_bh; + return -EIO; + if (mdb->drSigWord != cpu_to_be16(HFS_SUPER_MAGIC)) { + brelse(bh); + return -EIO; + } HFS_SB(sb)->mdb_bh = bh; HFS_SB(sb)->mdb = mdb; @@ -156,6 +180,11 @@ int hfs_mdb_get(struct super_block *sb) atomic64_set(&HFS_SB(sb)->file_count, be32_to_cpu(mdb->drFilCnt)); atomic64_set(&HFS_SB(sb)->folder_count, be32_to_cpu(mdb->drDirCnt)); + if (!is_hfs_cnid_counts_valid(sb)) { + pr_warn("filesystem possibly corrupted, running fsck.hfs is recommended. Mounting read-only.\n"); + sb->s_flags |= SB_RDONLY; + } + /* TRY to get the alternate (backup) MDB. */ sect = part_start + part_size - 2; bh = sb_bread512(sb, sect, mdb2); @@ -174,7 +203,7 @@ int hfs_mdb_get(struct super_block *sb) HFS_SB(sb)->bitmap = kzalloc(8192, GFP_KERNEL); if (!HFS_SB(sb)->bitmap) - goto out; + return -EIO; /* read in the bitmap */ block = be16_to_cpu(mdb->drVBMSt) + part_start; @@ -185,7 +214,7 @@ int hfs_mdb_get(struct super_block *sb) bh = sb_bread(sb, off >> sb->s_blocksize_bits); if (!bh) { pr_err("unable to read volume bitmap\n"); - goto out; + return -EIO; } off2 = off & (sb->s_blocksize - 1); len = min((int)sb->s_blocksize - off2, size); @@ -199,17 +228,17 @@ int hfs_mdb_get(struct super_block *sb) HFS_SB(sb)->ext_tree = hfs_btree_open(sb, HFS_EXT_CNID, hfs_ext_keycmp); if (!HFS_SB(sb)->ext_tree) { pr_err("unable to open extent tree\n"); - goto out; + return -EIO; } HFS_SB(sb)->cat_tree = hfs_btree_open(sb, HFS_CAT_CNID, hfs_cat_keycmp); if (!HFS_SB(sb)->cat_tree) { pr_err("unable to open catalog tree\n"); - goto out; + return -EIO; } attrib = mdb->drAtrb; if (!(attrib & cpu_to_be16(HFS_SB_ATTRIB_UNMNT))) { - pr_warn("filesystem was not cleanly unmounted, running fsck.hfs is recommended. mounting read-only.\n"); + pr_warn("filesystem was not cleanly unmounted, running fsck.hfs is recommended. Mounting read-only.\n"); sb->s_flags |= SB_RDONLY; } if ((attrib & cpu_to_be16(HFS_SB_ATTRIB_SLOCK))) { @@ -229,12 +258,6 @@ int hfs_mdb_get(struct super_block *sb) } return 0; - -out_bh: - brelse(bh); -out: - hfs_mdb_put(sb); - return -EIO; } /* @@ -273,15 +296,12 @@ void hfs_mdb_commit(struct super_block *sb) /* These parameters may have been modified, so write them back */ mdb->drLsMod = hfs_mtime(); mdb->drFreeBks = cpu_to_be16(HFS_SB(sb)->free_ablocks); - BUG_ON(atomic64_read(&HFS_SB(sb)->next_id) > U32_MAX); mdb->drNxtCNID = cpu_to_be32((u32)atomic64_read(&HFS_SB(sb)->next_id)); mdb->drNmFls = cpu_to_be16(HFS_SB(sb)->root_files); mdb->drNmRtDirs = cpu_to_be16(HFS_SB(sb)->root_dirs); - BUG_ON(atomic64_read(&HFS_SB(sb)->file_count) > U32_MAX); mdb->drFilCnt = cpu_to_be32((u32)atomic64_read(&HFS_SB(sb)->file_count)); - BUG_ON(atomic64_read(&HFS_SB(sb)->folder_count) > U32_MAX); mdb->drDirCnt = cpu_to_be32((u32)atomic64_read(&HFS_SB(sb)->folder_count)); @@ -359,8 +379,6 @@ void hfs_mdb_close(struct super_block *sb) * Release the resources associated with the in-core MDB. */ void hfs_mdb_put(struct super_block *sb) { - if (!HFS_SB(sb)) - return; /* free the B-trees */ hfs_btree_close(HFS_SB(sb)->ext_tree); hfs_btree_close(HFS_SB(sb)->cat_tree); @@ -373,6 +391,4 @@ void hfs_mdb_put(struct super_block *sb) unload_nls(HFS_SB(sb)->nls_disk); kfree(HFS_SB(sb)->bitmap); - kfree(HFS_SB(sb)); - sb->s_fs_info = NULL; } diff --git a/fs/hfs/super.c b/fs/hfs/super.c index 47f50fa555a457..97546d6b41f477 100644 --- a/fs/hfs/super.c +++ b/fs/hfs/super.c @@ -34,6 +34,7 @@ MODULE_LICENSE("GPL"); static int hfs_sync_fs(struct super_block *sb, int wait) { + is_hfs_cnid_counts_valid(sb); hfs_mdb_commit(sb); return 0; } @@ -65,6 +66,8 @@ static void flush_mdb(struct work_struct *work) sbi->work_queued = 0; spin_unlock(&sbi->work_lock); + is_hfs_cnid_counts_valid(sb); + hfs_mdb_commit(sb); } @@ -431,10 +434,18 @@ static int hfs_init_fs_context(struct fs_context *fc) return 0; } +static void hfs_kill_super(struct super_block *sb) +{ + struct hfs_sb_info *hsb = HFS_SB(sb); + + kill_block_super(sb); + kfree(hsb); +} + static struct file_system_type hfs_fs_type = { .owner = THIS_MODULE, .name = "hfs", - .kill_sb = kill_block_super, + .kill_sb = hfs_kill_super, .fs_flags = FS_REQUIRES_DEV, .init_fs_context = hfs_init_fs_context, }; diff --git a/fs/hfsplus/bnode.c b/fs/hfsplus/bnode.c index 191661af967789..250a226336ea7a 100644 --- a/fs/hfsplus/bnode.c +++ b/fs/hfsplus/bnode.c @@ -629,7 +629,7 @@ struct hfs_bnode *hfs_bnode_create(struct hfs_btree *tree, u32 num) if (node) { pr_crit("new node %u already hashed?\n", num); WARN_ON(1); - return node; + return ERR_PTR(-EEXIST); } node = __hfs_bnode_create(tree, num); if (!node) diff --git a/fs/hfsplus/dir.c b/fs/hfsplus/dir.c index cadf0b5f93422e..ca5f74a140ec19 100644 --- a/fs/hfsplus/dir.c +++ b/fs/hfsplus/dir.c @@ -313,6 +313,9 @@ static int hfsplus_link(struct dentry *src_dentry, struct inode *dst_dir, if (!S_ISREG(inode->i_mode)) return -EPERM; + hfs_dbg("src_dir->i_ino %lu, dst_dir->i_ino %lu, inode->i_ino %lu\n", + src_dir->i_ino, dst_dir->i_ino, inode->i_ino); + mutex_lock(&sbi->vh_mutex); if (inode->i_ino == (u32)(unsigned long)src_dentry->d_fsdata) { for (;;) { @@ -332,7 +335,7 @@ static int hfsplus_link(struct dentry *src_dentry, struct inode *dst_dir, cnid = sbi->next_cnid++; src_dentry->d_fsdata = (void *)(unsigned long)cnid; res = hfsplus_create_cat(cnid, src_dir, - &src_dentry->d_name, inode); + &src_dentry->d_name, inode); if (res) /* panic? */ goto out; @@ -350,6 +353,21 @@ static int hfsplus_link(struct dentry *src_dentry, struct inode *dst_dir, mark_inode_dirty(inode); sbi->file_count++; hfsplus_mark_mdb_dirty(dst_dir->i_sb); + + res = hfsplus_cat_write_inode(src_dir); + if (res) + goto out; + + res = hfsplus_cat_write_inode(dst_dir); + if (res) + goto out; + + res = hfsplus_cat_write_inode(sbi->hidden_dir); + if (res) + goto out; + + res = hfsplus_cat_write_inode(inode); + out: mutex_unlock(&sbi->vh_mutex); return res; @@ -367,6 +385,9 @@ static int hfsplus_unlink(struct inode *dir, struct dentry *dentry) if (HFSPLUS_IS_RSRC(inode)) return -EPERM; + hfs_dbg("dir->i_ino %lu, inode->i_ino %lu\n", + dir->i_ino, inode->i_ino); + mutex_lock(&sbi->vh_mutex); cnid = (u32)(unsigned long)dentry->d_fsdata; if (inode->i_ino == cnid && @@ -408,6 +429,15 @@ static int hfsplus_unlink(struct inode *dir, struct dentry *dentry) inode_set_ctime_current(inode); mark_inode_dirty(inode); out: + if (!res) { + res = hfsplus_cat_write_inode(dir); + if (!res) { + res = hfsplus_cat_write_inode(sbi->hidden_dir); + if (!res) + res = hfsplus_cat_write_inode(inode); + } + } + mutex_unlock(&sbi->vh_mutex); return res; } @@ -429,6 +459,8 @@ static int hfsplus_rmdir(struct inode *dir, struct dentry *dentry) inode_set_ctime_current(inode); hfsplus_delete_inode(inode); mark_inode_dirty(inode); + + res = hfsplus_cat_write_inode(dir); out: mutex_unlock(&sbi->vh_mutex); return res; @@ -465,6 +497,12 @@ static int hfsplus_symlink(struct mnt_idmap *idmap, struct inode *dir, hfsplus_instantiate(dentry, inode, inode->i_ino); mark_inode_dirty(inode); + + res = hfsplus_cat_write_inode(dir); + if (res) + goto out; + + res = hfsplus_cat_write_inode(inode); goto out; out_err: @@ -506,6 +544,12 @@ static int hfsplus_mknod(struct mnt_idmap *idmap, struct inode *dir, hfsplus_instantiate(dentry, inode, inode->i_ino); mark_inode_dirty(inode); + + res = hfsplus_cat_write_inode(dir); + if (res) + goto out; + + res = hfsplus_cat_write_inode(inode); goto out; failed_mknod: diff --git a/fs/hfsplus/inode.c b/fs/hfsplus/inode.c index 7ae6745ca7ae12..6153e5cc6eb65e 100644 --- a/fs/hfsplus/inode.c +++ b/fs/hfsplus/inode.c @@ -328,6 +328,9 @@ int hfsplus_file_fsync(struct file *file, loff_t start, loff_t end, struct hfsplus_vh *vhdr = sbi->s_vhdr; int error = 0, error2; + hfs_dbg("inode->i_ino %lu, start %llu, end %llu\n", + inode->i_ino, start, end); + error = file_write_and_wait_range(file, start, end); if (error) return error; @@ -612,17 +615,20 @@ int hfsplus_cat_read_inode(struct inode *inode, struct hfs_find_data *fd) int hfsplus_cat_write_inode(struct inode *inode) { struct inode *main_inode = inode; + struct hfs_btree *tree = HFSPLUS_SB(inode->i_sb)->cat_tree; struct hfs_find_data fd; hfsplus_cat_entry entry; int res = 0; + hfs_dbg("inode->i_ino %lu\n", inode->i_ino); + if (HFSPLUS_IS_RSRC(inode)) main_inode = HFSPLUS_I(inode)->rsrc_inode; if (!main_inode->i_nlink) return 0; - if (hfs_find_init(HFSPLUS_SB(main_inode->i_sb)->cat_tree, &fd)) + if (hfs_find_init(tree, &fd)) /* panic? */ return -EIO; @@ -687,6 +693,15 @@ int hfsplus_cat_write_inode(struct inode *inode) set_bit(HFSPLUS_I_CAT_DIRTY, &HFSPLUS_I(inode)->flags); out: hfs_find_exit(&fd); + + if (!res) { + res = hfs_btree_write(tree); + if (res) { + pr_err("b-tree write err: %d, ino %lu\n", + res, inode->i_ino); + } + } + return res; } diff --git a/fs/hfsplus/super.c b/fs/hfsplus/super.c index aaffa9e060a0ae..942b8ff01ad070 100644 --- a/fs/hfsplus/super.c +++ b/fs/hfsplus/super.c @@ -53,6 +53,12 @@ static int hfsplus_system_read_inode(struct inode *inode) return -EIO; } + /* + * Assign a dummy file type, for may_open() requires that + * an inode has a valid file type. + */ + inode->i_mode = S_IFREG; + return 0; } @@ -344,8 +350,6 @@ static void hfsplus_put_super(struct super_block *sb) hfs_btree_close(sbi->ext_tree); kfree(sbi->s_vhdr_buf); kfree(sbi->s_backup_vhdr_buf); - call_rcu(&sbi->rcu, delayed_free); - hfs_dbg("finished\n"); } @@ -650,7 +654,6 @@ static int hfsplus_fill_super(struct super_block *sb, struct fs_context *fc) out_unload_nls: unload_nls(sbi->nls); unload_nls(nls); - kfree(sbi); return err; } @@ -709,10 +712,18 @@ static int hfsplus_init_fs_context(struct fs_context *fc) return 0; } +static void hfsplus_kill_super(struct super_block *sb) +{ + struct hfsplus_sb_info *sbi = HFSPLUS_SB(sb); + + kill_block_super(sb); + call_rcu(&sbi->rcu, delayed_free); +} + static struct file_system_type hfsplus_fs_type = { .owner = THIS_MODULE, .name = "hfsplus", - .kill_sb = kill_block_super, + .kill_sb = hfsplus_kill_super, .fs_flags = FS_REQUIRES_DEV, .init_fs_context = hfsplus_init_fs_context, }; diff --git a/fs/iomap/buffered-io.c b/fs/iomap/buffered-io.c index 6beb876658c09b..67eda23faac8de 100644 --- a/fs/iomap/buffered-io.c +++ b/fs/iomap/buffered-io.c @@ -79,18 +79,27 @@ static void iomap_set_range_uptodate(struct folio *folio, size_t off, { struct iomap_folio_state *ifs = folio->private; unsigned long flags; - bool uptodate = true; + bool mark_uptodate = true; if (folio_test_uptodate(folio)) return; if (ifs) { spin_lock_irqsave(&ifs->state_lock, flags); - uptodate = ifs_set_range_uptodate(folio, ifs, off, len); + /* + * If a read with bytes pending is in progress, we must not call + * folio_mark_uptodate(). The read completion path + * (iomap_read_end()) will call folio_end_read(), which uses XOR + * semantics to set the uptodate bit. If we set it here, the XOR + * in folio_end_read() will clear it, leaving the folio not + * uptodate. + */ + mark_uptodate = ifs_set_range_uptodate(folio, ifs, off, len) && + !ifs->read_bytes_pending; spin_unlock_irqrestore(&ifs->state_lock, flags); } - if (uptodate) + if (mark_uptodate) folio_mark_uptodate(folio); } @@ -409,8 +418,6 @@ static void iomap_read_init(struct folio *folio) struct iomap_folio_state *ifs = folio->private; if (ifs) { - size_t len = folio_size(folio); - /* * ifs->read_bytes_pending is used to track how many bytes are * read in asynchronously by the IO helper. We need to track @@ -418,23 +425,19 @@ static void iomap_read_init(struct folio *folio) * reading in all the necessary ranges of the folio and can end * the read. * - * Increase ->read_bytes_pending by the folio size to start, and - * add a +1 bias. We'll subtract the bias and any uptodate / - * zeroed ranges that did not require IO in iomap_read_end() - * after we're done processing the folio. + * Increase ->read_bytes_pending by the folio size to start. + * We'll subtract any uptodate / zeroed ranges that did not + * require IO in iomap_read_end() after we're done processing + * the folio. * * We do this because otherwise, we would have to increment * ifs->read_bytes_pending every time a range in the folio needs * to be read in, which can get expensive since the spinlock * needs to be held whenever modifying ifs->read_bytes_pending. - * - * We add the bias to ensure the read has not been ended on the - * folio when iomap_read_end() is called, even if the IO helper - * has already finished reading in the entire folio. */ spin_lock_irq(&ifs->state_lock); WARN_ON_ONCE(ifs->read_bytes_pending != 0); - ifs->read_bytes_pending = len + 1; + ifs->read_bytes_pending = folio_size(folio); spin_unlock_irq(&ifs->state_lock); } } @@ -465,11 +468,9 @@ static void iomap_read_end(struct folio *folio, size_t bytes_submitted) /* * Subtract any bytes that were initially accounted to - * read_bytes_pending but skipped for IO. The +1 accounts for - * the bias we added in iomap_read_init(). + * read_bytes_pending but skipped for IO. */ - ifs->read_bytes_pending -= - (folio_size(folio) + 1 - bytes_submitted); + ifs->read_bytes_pending -= folio_size(folio) - bytes_submitted; /* * If !ifs->read_bytes_pending, this means all pending reads by @@ -483,14 +484,16 @@ static void iomap_read_end(struct folio *folio, size_t bytes_submitted) spin_unlock_irq(&ifs->state_lock); if (end_read) folio_end_read(folio, uptodate); - } else if (!bytes_submitted) { + } else { /* - * If there were no bytes submitted, this means we are - * responsible for unlocking the folio here, since no IO helper - * has taken ownership of it. If there were bytes submitted, - * then the IO helper will end the read via - * iomap_finish_folio_read(). + * If a folio without an ifs is submitted to the IO helper, the + * read must be on the entire folio and the IO helper takes + * ownership of the folio. This means we should only enter + * iomap_read_end() for the !ifs case if no bytes were submitted + * to the IO helper, in which case we are responsible for + * unlocking the folio here. */ + WARN_ON_ONCE(bytes_submitted); folio_unlock(folio); } } @@ -502,6 +505,8 @@ static int iomap_read_folio_iter(struct iomap_iter *iter, loff_t pos = iter->pos; loff_t length = iomap_length(iter); struct folio *folio = ctx->cur_folio; + size_t folio_len = folio_size(folio); + struct iomap_folio_state *ifs; size_t poff, plen; loff_t pos_diff; int ret; @@ -513,10 +518,9 @@ static int iomap_read_folio_iter(struct iomap_iter *iter, return iomap_iter_advance(iter, length); } - ifs_alloc(iter->inode, folio, iter->flags); + ifs = ifs_alloc(iter->inode, folio, iter->flags); - length = min_t(loff_t, length, - folio_size(folio) - offset_in_folio(folio, pos)); + length = min_t(loff_t, length, folio_len - offset_in_folio(folio, pos)); while (length) { iomap_adjust_read_range(iter->inode, folio, &pos, length, &poff, &plen); @@ -542,7 +546,19 @@ static int iomap_read_folio_iter(struct iomap_iter *iter, ret = ctx->ops->read_folio_range(iter, ctx, plen); if (ret) return ret; + *bytes_submitted += plen; + /* + * Hand off folio ownership to the IO helper when: + * 1) The entire folio has been submitted for IO, or + * 2) There is no ifs attached to the folio + * + * Case (2) occurs when 1 << i_blkbits matches the folio + * size but the underlying filesystem or block device + * uses a smaller granularity for IO. + */ + if (*bytes_submitted == folio_len || !ifs) + ctx->cur_folio = NULL; } ret = iomap_iter_advance(iter, plen); @@ -575,7 +591,8 @@ void iomap_read_folio(const struct iomap_ops *ops, if (ctx->ops->submit_read) ctx->ops->submit_read(ctx); - iomap_read_end(folio, bytes_submitted); + if (ctx->cur_folio) + iomap_read_end(ctx->cur_folio, bytes_submitted); } EXPORT_SYMBOL_GPL(iomap_read_folio); diff --git a/fs/iomap/direct-io.c b/fs/iomap/direct-io.c index 8e273408453a9c..6ec4940e019c64 100644 --- a/fs/iomap/direct-io.c +++ b/fs/iomap/direct-io.c @@ -442,9 +442,13 @@ static int iomap_dio_bio_iter(struct iomap_iter *iter, struct iomap_dio *dio) nr_pages = bio_iov_vecs_to_alloc(dio->submit.iter, BIO_MAX_VECS); do { size_t n; - if (dio->error) { - iov_iter_revert(dio->submit.iter, copied); - copied = ret = 0; + + /* + * If completions already occurred and reported errors, give up now and + * don't bother submitting more bios. + */ + if (unlikely(data_race(dio->error))) { + ret = 0; goto out; } diff --git a/fs/iomap/ioend.c b/fs/iomap/ioend.c index 86f44922ed3b6a..f418cfde32764d 100644 --- a/fs/iomap/ioend.c +++ b/fs/iomap/ioend.c @@ -163,17 +163,18 @@ ssize_t iomap_add_to_ioend(struct iomap_writepage_ctx *wpc, struct folio *folio, WARN_ON_ONCE(!folio->private && map_len < dirty_len); switch (wpc->iomap.type) { - case IOMAP_INLINE: - WARN_ON_ONCE(1); - return -EIO; + case IOMAP_UNWRITTEN: + ioend_flags |= IOMAP_IOEND_UNWRITTEN; + break; + case IOMAP_MAPPED: + break; case IOMAP_HOLE: return map_len; default: - break; + WARN_ON_ONCE(1); + return -EIO; } - if (wpc->iomap.type == IOMAP_UNWRITTEN) - ioend_flags |= IOMAP_IOEND_UNWRITTEN; if (wpc->iomap.flags & IOMAP_F_SHARED) ioend_flags |= IOMAP_IOEND_SHARED; if (folio_test_dropbehind(folio)) diff --git a/fs/jbd2/checkpoint.c b/fs/jbd2/checkpoint.c index de89c5bef60749..1508e2f5446211 100644 --- a/fs/jbd2/checkpoint.c +++ b/fs/jbd2/checkpoint.c @@ -267,7 +267,15 @@ int jbd2_log_do_checkpoint(journal_t *journal) */ BUFFER_TRACE(bh, "queue"); get_bh(bh); - J_ASSERT_BH(bh, !buffer_jwrite(bh)); + if (WARN_ON_ONCE(buffer_jwrite(bh))) { + put_bh(bh); /* drop the ref we just took */ + spin_unlock(&journal->j_list_lock); + /* Clean up any previously batched buffers */ + if (batch_count) + __flush_batch(journal, &batch_count); + jbd2_journal_abort(journal, -EFSCORRUPTED); + return -EFSCORRUPTED; + } journal->j_chkpt_bhs[batch_count++] = bh; transaction->t_chp_stats.cs_written++; transaction->t_checkpoint_list = jh->b_cpnext; @@ -325,7 +333,10 @@ int jbd2_cleanup_journal_tail(journal_t *journal) if (!jbd2_journal_get_log_tail(journal, &first_tid, &blocknr)) return 1; - J_ASSERT(blocknr != 0); + if (WARN_ON_ONCE(blocknr == 0)) { + jbd2_journal_abort(journal, -EFSCORRUPTED); + return -EFSCORRUPTED; + } /* * We need to make sure that any blocks that were recently written out diff --git a/fs/jfs/jfs_dtree.c b/fs/jfs/jfs_dtree.c index 0ab83bb7bbdf9a..9ab3f2fc61d176 100644 --- a/fs/jfs/jfs_dtree.c +++ b/fs/jfs/jfs_dtree.c @@ -2903,7 +2903,7 @@ int jfs_readdir(struct file *file, struct dir_context *ctx) stbl = DT_GETSTBL(p); for (i = index; i < p->header.nextindex; i++) { - if (stbl[i] < 0 || stbl[i] >= DTPAGEMAXSLOT) { + if (stbl[i] < 0) { jfs_err("JFS: Invalid stbl[%d] = %d for inode %ld, block = %lld", i, stbl[i], (long)ip->i_ino, (long long)bn); free_page(dirent_buf); @@ -3108,7 +3108,7 @@ static int dtReadFirst(struct inode *ip, struct btstack * btstack) /* get the leftmost entry */ stbl = DT_GETSTBL(p); - if (stbl[0] < 0 || stbl[0] >= DTPAGEMAXSLOT) { + if (stbl[0] < 0) { DT_PUTPAGE(mp); jfs_error(ip->i_sb, "stbl[0] out of bound\n"); return -EIO; diff --git a/fs/jfs/jfs_logmgr.c b/fs/jfs/jfs_logmgr.c index b343c5ea11592d..5b1c5da041630a 100644 --- a/fs/jfs/jfs_logmgr.c +++ b/fs/jfs/jfs_logmgr.c @@ -2311,6 +2311,7 @@ int jfsIOWait(void *arg) { struct lbuf *bp; + set_freezable(); do { spin_lock_irq(&log_redrive_lock); while ((bp = log_redrive_list)) { diff --git a/fs/jfs/namei.c b/fs/jfs/namei.c index 65a218eba8faf9..7879c049632b3d 100644 --- a/fs/jfs/namei.c +++ b/fs/jfs/namei.c @@ -1228,7 +1228,7 @@ static int jfs_rename(struct mnt_idmap *idmap, struct inode *old_dir, jfs_err("jfs_rename: dtInsert returned -EIO"); goto out_tx; } - if (S_ISDIR(old_ip->i_mode)) + if (S_ISDIR(old_ip->i_mode) && old_dir != new_dir) inc_nlink(new_dir); } /* @@ -1244,7 +1244,9 @@ static int jfs_rename(struct mnt_idmap *idmap, struct inode *old_dir, goto out_tx; } if (S_ISDIR(old_ip->i_mode)) { - drop_nlink(old_dir); + if (new_ip || old_dir != new_dir) + drop_nlink(old_dir); + if (old_dir != new_dir) { /* * Change inode number of parent for moved directory diff --git a/fs/minix/inode.c b/fs/minix/inode.c index 51ea9bdc813f77..c8c6b2135abe79 100644 --- a/fs/minix/inode.c +++ b/fs/minix/inode.c @@ -170,10 +170,38 @@ static int minix_reconfigure(struct fs_context *fc) static bool minix_check_superblock(struct super_block *sb) { struct minix_sb_info *sbi = minix_sb(sb); + unsigned long block; - if (sbi->s_imap_blocks == 0 || sbi->s_zmap_blocks == 0) + if (sbi->s_log_zone_size != 0) { + printk("minix-fs error: zone size must equal block size. " + "s_log_zone_size > 0 is not supported.\n"); + return false; + } + + if (sbi->s_ninodes < 1 || sbi->s_firstdatazone <= 4 || + sbi->s_firstdatazone >= sbi->s_nzones) return false; + /* Apparently minix can create filesystems that allocate more blocks for + * the bitmaps than needed. We simply ignore that, but verify it didn't + * create one with not enough blocks and bail out if so. + */ + block = minix_blocks_needed(sbi->s_ninodes, sb->s_blocksize); + if (sbi->s_imap_blocks < block) { + printk("MINIX-fs: file system does not have enough " + "imap blocks allocated. Refusing to mount.\n"); + return false; + } + + block = minix_blocks_needed( + (sbi->s_nzones - sbi->s_firstdatazone + 1), + sb->s_blocksize); + if (sbi->s_zmap_blocks < block) { + printk("MINIX-fs: file system does not have enough " + "zmap blocks allocated. Refusing to mount.\n"); + return false; + } + /* * s_max_size must not exceed the block mapping limitation. This check * is only needed for V1 filesystems, since V2/V3 support an extra level @@ -293,26 +321,6 @@ static int minix_fill_super(struct super_block *s, struct fs_context *fc) minix_set_bit(0,sbi->s_imap[0]->b_data); minix_set_bit(0,sbi->s_zmap[0]->b_data); - /* Apparently minix can create filesystems that allocate more blocks for - * the bitmaps than needed. We simply ignore that, but verify it didn't - * create one with not enough blocks and bail out if so. - */ - block = minix_blocks_needed(sbi->s_ninodes, s->s_blocksize); - if (sbi->s_imap_blocks < block) { - printk("MINIX-fs: file system does not have enough " - "imap blocks allocated. Refusing to mount.\n"); - goto out_no_bitmap; - } - - block = minix_blocks_needed( - (sbi->s_nzones - sbi->s_firstdatazone + 1), - s->s_blocksize); - if (sbi->s_zmap_blocks < block) { - printk("MINIX-fs: file system does not have enough " - "zmap blocks allocated. Refusing to mount.\n"); - goto out_no_bitmap; - } - /* set up enough so that it can read an inode */ s->s_op = &minix_sops; s->s_time_min = 0; diff --git a/fs/namespace.c b/fs/namespace.c index c58674a20cad54..9e5e3f1db02f97 100644 --- a/fs/namespace.c +++ b/fs/namespace.c @@ -221,7 +221,7 @@ static int mnt_alloc_id(struct mount *mnt) int res; xa_lock(&mnt_id_xa); - res = __xa_alloc(&mnt_id_xa, &mnt->mnt_id, mnt, XA_LIMIT(1, INT_MAX), GFP_KERNEL); + res = __xa_alloc(&mnt_id_xa, &mnt->mnt_id, mnt, xa_limit_31b, GFP_KERNEL); if (!res) mnt->mnt_id_unique = ++mnt_id_ctr; xa_unlock(&mnt_id_xa); @@ -1537,23 +1537,33 @@ static struct mount *mnt_find_id_at_reverse(struct mnt_namespace *ns, u64 mnt_id static void *m_start(struct seq_file *m, loff_t *pos) { struct proc_mounts *p = m->private; + struct mount *mnt; down_read(&namespace_sem); - return mnt_find_id_at(p->ns, *pos); + mnt = mnt_find_id_at(p->ns, *pos); + if (mnt) + *pos = mnt->mnt_id_unique; + return mnt; } static void *m_next(struct seq_file *m, void *v, loff_t *pos) { - struct mount *next = NULL, *mnt = v; + struct mount *mnt = v; struct rb_node *node = rb_next(&mnt->mnt_node); - ++*pos; if (node) { - next = node_to_mount(node); + struct mount *next = node_to_mount(node); *pos = next->mnt_id_unique; + return next; } - return next; + + /* + * No more mounts. Set pos past current mount's ID so that if + * iteration restarts, mnt_find_id_at() returns NULL. + */ + *pos = mnt->mnt_id_unique + 1; + return NULL; } static void m_stop(struct seq_file *m, void *v) @@ -5780,7 +5790,7 @@ SYSCALL_DEFINE4(statmount, const struct mnt_id_req __user *, req, if (kreq.mnt_ns_id && (ns != current->nsproxy->mnt_ns) && !ns_capable_noaudit(ns->user_ns, CAP_SYS_ADMIN)) - return -ENOENT; + return -EPERM; ks = kmalloc(sizeof(*ks), GFP_KERNEL_ACCOUNT); if (!ks) diff --git a/fs/netfs/buffered_read.c b/fs/netfs/buffered_read.c index 37ab6f28b5ad0e..88361e8c709614 100644 --- a/fs/netfs/buffered_read.c +++ b/fs/netfs/buffered_read.c @@ -171,9 +171,8 @@ static void netfs_queue_read(struct netfs_io_request *rreq, spin_lock(&rreq->lock); list_add_tail(&subreq->rreq_link, &stream->subrequests); if (list_is_first(&subreq->rreq_link, &stream->subrequests)) { - stream->front = subreq; if (!stream->active) { - stream->collected_to = stream->front->start; + stream->collected_to = subreq->start; /* Store list pointers before active flag */ smp_store_release(&stream->active, true); } diff --git a/fs/netfs/direct_read.c b/fs/netfs/direct_read.c index a498ee8d66745f..f72e6da88cca7c 100644 --- a/fs/netfs/direct_read.c +++ b/fs/netfs/direct_read.c @@ -71,9 +71,8 @@ static int netfs_dispatch_unbuffered_reads(struct netfs_io_request *rreq) spin_lock(&rreq->lock); list_add_tail(&subreq->rreq_link, &stream->subrequests); if (list_is_first(&subreq->rreq_link, &stream->subrequests)) { - stream->front = subreq; if (!stream->active) { - stream->collected_to = stream->front->start; + stream->collected_to = subreq->start; /* Store list pointers before active flag */ smp_store_release(&stream->active, true); } diff --git a/fs/netfs/direct_write.c b/fs/netfs/direct_write.c index a9d1c3b2c08426..f9ab69de3e298a 100644 --- a/fs/netfs/direct_write.c +++ b/fs/netfs/direct_write.c @@ -9,6 +9,209 @@ #include #include "internal.h" +/* + * Perform the cleanup rituals after an unbuffered write is complete. + */ +static void netfs_unbuffered_write_done(struct netfs_io_request *wreq) +{ + struct netfs_inode *ictx = netfs_inode(wreq->inode); + + _enter("R=%x", wreq->debug_id); + + /* Okay, declare that all I/O is complete. */ + trace_netfs_rreq(wreq, netfs_rreq_trace_write_done); + + if (!wreq->error) + netfs_update_i_size(ictx, &ictx->inode, wreq->start, wreq->transferred); + + if (wreq->origin == NETFS_DIO_WRITE && + wreq->mapping->nrpages) { + /* mmap may have got underfoot and we may now have folios + * locally covering the region we just wrote. Attempt to + * discard the folios, but leave in place any modified locally. + * ->write_iter() is prevented from interfering by the DIO + * counter. + */ + pgoff_t first = wreq->start >> PAGE_SHIFT; + pgoff_t last = (wreq->start + wreq->transferred - 1) >> PAGE_SHIFT; + + invalidate_inode_pages2_range(wreq->mapping, first, last); + } + + if (wreq->origin == NETFS_DIO_WRITE) + inode_dio_end(wreq->inode); + + _debug("finished"); + netfs_wake_rreq_flag(wreq, NETFS_RREQ_IN_PROGRESS, netfs_rreq_trace_wake_ip); + /* As we cleared NETFS_RREQ_IN_PROGRESS, we acquired its ref. */ + + if (wreq->iocb) { + size_t written = umin(wreq->transferred, wreq->len); + + wreq->iocb->ki_pos += written; + if (wreq->iocb->ki_complete) { + trace_netfs_rreq(wreq, netfs_rreq_trace_ki_complete); + wreq->iocb->ki_complete(wreq->iocb, wreq->error ?: written); + } + wreq->iocb = VFS_PTR_POISON; + } + + netfs_clear_subrequests(wreq); +} + +/* + * Collect the subrequest results of unbuffered write subrequests. + */ +static void netfs_unbuffered_write_collect(struct netfs_io_request *wreq, + struct netfs_io_stream *stream, + struct netfs_io_subrequest *subreq) +{ + trace_netfs_collect_sreq(wreq, subreq); + + spin_lock(&wreq->lock); + list_del_init(&subreq->rreq_link); + spin_unlock(&wreq->lock); + + wreq->transferred += subreq->transferred; + iov_iter_advance(&wreq->buffer.iter, subreq->transferred); + + stream->collected_to = subreq->start + subreq->transferred; + wreq->collected_to = stream->collected_to; + netfs_put_subrequest(subreq, netfs_sreq_trace_put_done); + + trace_netfs_collect_stream(wreq, stream); + trace_netfs_collect_state(wreq, wreq->collected_to, 0); +} + +/* + * Write data to the server without going through the pagecache and without + * writing it to the local cache. We dispatch the subrequests serially and + * wait for each to complete before dispatching the next, lest we leave a gap + * in the data written due to a failure such as ENOSPC. We could, however + * attempt to do preparation such as content encryption for the next subreq + * whilst the current is in progress. + */ +static int netfs_unbuffered_write(struct netfs_io_request *wreq) +{ + struct netfs_io_subrequest *subreq = NULL; + struct netfs_io_stream *stream = &wreq->io_streams[0]; + int ret; + + _enter("%llx", wreq->len); + + if (wreq->origin == NETFS_DIO_WRITE) + inode_dio_begin(wreq->inode); + + stream->collected_to = wreq->start; + + for (;;) { + bool retry = false; + + if (!subreq) { + netfs_prepare_write(wreq, stream, wreq->start + wreq->transferred); + subreq = stream->construct; + stream->construct = NULL; + } + + /* Check if (re-)preparation failed. */ + if (unlikely(test_bit(NETFS_SREQ_FAILED, &subreq->flags))) { + netfs_write_subrequest_terminated(subreq, subreq->error); + wreq->error = subreq->error; + break; + } + + iov_iter_truncate(&subreq->io_iter, wreq->len - wreq->transferred); + if (!iov_iter_count(&subreq->io_iter)) + break; + + subreq->len = netfs_limit_iter(&subreq->io_iter, 0, + stream->sreq_max_len, + stream->sreq_max_segs); + iov_iter_truncate(&subreq->io_iter, subreq->len); + stream->submit_extendable_to = subreq->len; + + trace_netfs_sreq(subreq, netfs_sreq_trace_submit); + stream->issue_write(subreq); + + /* Async, need to wait. */ + netfs_wait_for_in_progress_stream(wreq, stream); + + if (test_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags)) { + retry = true; + } else if (test_bit(NETFS_SREQ_FAILED, &subreq->flags)) { + ret = subreq->error; + wreq->error = ret; + netfs_see_subrequest(subreq, netfs_sreq_trace_see_failed); + subreq = NULL; + break; + } + ret = 0; + + if (!retry) { + netfs_unbuffered_write_collect(wreq, stream, subreq); + subreq = NULL; + if (wreq->transferred >= wreq->len) + break; + if (!wreq->iocb && signal_pending(current)) { + ret = wreq->transferred ? -EINTR : -ERESTARTSYS; + trace_netfs_rreq(wreq, netfs_rreq_trace_intr); + break; + } + continue; + } + + /* We need to retry the last subrequest, so first reset the + * iterator, taking into account what, if anything, we managed + * to transfer. + */ + subreq->error = -EAGAIN; + trace_netfs_sreq(subreq, netfs_sreq_trace_retry); + if (subreq->transferred > 0) + iov_iter_advance(&wreq->buffer.iter, subreq->transferred); + + if (stream->source == NETFS_UPLOAD_TO_SERVER && + wreq->netfs_ops->retry_request) + wreq->netfs_ops->retry_request(wreq, stream); + + __clear_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags); + __clear_bit(NETFS_SREQ_BOUNDARY, &subreq->flags); + __clear_bit(NETFS_SREQ_FAILED, &subreq->flags); + subreq->io_iter = wreq->buffer.iter; + subreq->start = wreq->start + wreq->transferred; + subreq->len = wreq->len - wreq->transferred; + subreq->transferred = 0; + subreq->retry_count += 1; + stream->sreq_max_len = UINT_MAX; + stream->sreq_max_segs = INT_MAX; + + netfs_get_subrequest(subreq, netfs_sreq_trace_get_resubmit); + + if (stream->prepare_write) { + stream->prepare_write(subreq); + __set_bit(NETFS_SREQ_IN_PROGRESS, &subreq->flags); + netfs_stat(&netfs_n_wh_retry_write_subreq); + } else { + struct iov_iter source; + + netfs_reset_iter(subreq); + source = subreq->io_iter; + netfs_reissue_write(stream, subreq, &source); + } + } + + netfs_unbuffered_write_done(wreq); + _leave(" = %d", ret); + return ret; +} + +static void netfs_unbuffered_write_async(struct work_struct *work) +{ + struct netfs_io_request *wreq = container_of(work, struct netfs_io_request, work); + + netfs_unbuffered_write(wreq); + netfs_put_request(wreq, netfs_rreq_trace_put_complete); +} + /* * Perform an unbuffered write where we may have to do an RMW operation on an * encrypted file. This can also be used for direct I/O writes. @@ -70,35 +273,35 @@ ssize_t netfs_unbuffered_write_iter_locked(struct kiocb *iocb, struct iov_iter * */ wreq->buffer.iter = *iter; } + + wreq->len = iov_iter_count(&wreq->buffer.iter); } __set_bit(NETFS_RREQ_USE_IO_ITER, &wreq->flags); - if (async) - __set_bit(NETFS_RREQ_OFFLOAD_COLLECTION, &wreq->flags); /* Copy the data into the bounce buffer and encrypt it. */ // TODO /* Dispatch the write. */ __set_bit(NETFS_RREQ_UPLOAD_TO_SERVER, &wreq->flags); - if (async) - wreq->iocb = iocb; - wreq->len = iov_iter_count(&wreq->buffer.iter); - ret = netfs_unbuffered_write(wreq, is_sync_kiocb(iocb), wreq->len); - if (ret < 0) { - _debug("begin = %zd", ret); - goto out; - } - if (!async) { - ret = netfs_wait_for_write(wreq); - if (ret > 0) - iocb->ki_pos += ret; - } else { + if (async) { + INIT_WORK(&wreq->work, netfs_unbuffered_write_async); + wreq->iocb = iocb; + queue_work(system_dfl_wq, &wreq->work); ret = -EIOCBQUEUED; + } else { + ret = netfs_unbuffered_write(wreq); + if (ret < 0) { + _debug("begin = %zd", ret); + } else { + iocb->ki_pos += wreq->transferred; + ret = wreq->transferred ?: wreq->error; + } + + netfs_put_request(wreq, netfs_rreq_trace_put_complete); } -out: netfs_put_request(wreq, netfs_rreq_trace_put_return); return ret; diff --git a/fs/netfs/internal.h b/fs/netfs/internal.h index 4319611f535449..d436e20d341852 100644 --- a/fs/netfs/internal.h +++ b/fs/netfs/internal.h @@ -198,6 +198,9 @@ struct netfs_io_request *netfs_create_write_req(struct address_space *mapping, struct file *file, loff_t start, enum netfs_io_origin origin); +void netfs_prepare_write(struct netfs_io_request *wreq, + struct netfs_io_stream *stream, + loff_t start); void netfs_reissue_write(struct netfs_io_stream *stream, struct netfs_io_subrequest *subreq, struct iov_iter *source); @@ -212,7 +215,6 @@ int netfs_advance_writethrough(struct netfs_io_request *wreq, struct writeback_c struct folio **writethrough_cache); ssize_t netfs_end_writethrough(struct netfs_io_request *wreq, struct writeback_control *wbc, struct folio *writethrough_cache); -int netfs_unbuffered_write(struct netfs_io_request *wreq, bool may_wait, size_t len); /* * write_retry.c diff --git a/fs/netfs/iterator.c b/fs/netfs/iterator.c index 72a435e5fc6da4..154a14bb2d7f78 100644 --- a/fs/netfs/iterator.c +++ b/fs/netfs/iterator.c @@ -142,6 +142,47 @@ static size_t netfs_limit_bvec(const struct iov_iter *iter, size_t start_offset, return min(span, max_size); } +/* + * Select the span of a kvec iterator we're going to use. Limit it by both + * maximum size and maximum number of segments. Returns the size of the span + * in bytes. + */ +static size_t netfs_limit_kvec(const struct iov_iter *iter, size_t start_offset, + size_t max_size, size_t max_segs) +{ + const struct kvec *kvecs = iter->kvec; + unsigned int nkv = iter->nr_segs, ix = 0, nsegs = 0; + size_t len, span = 0, n = iter->count; + size_t skip = iter->iov_offset + start_offset; + + if (WARN_ON(!iov_iter_is_kvec(iter)) || + WARN_ON(start_offset > n) || + n == 0) + return 0; + + while (n && ix < nkv && skip) { + len = kvecs[ix].iov_len; + if (skip < len) + break; + skip -= len; + n -= len; + ix++; + } + + while (n && ix < nkv) { + len = min3(n, kvecs[ix].iov_len - skip, max_size); + span += len; + nsegs++; + ix++; + if (span >= max_size || nsegs >= max_segs) + break; + skip = 0; + n -= len; + } + + return min(span, max_size); +} + /* * Select the span of an xarray iterator we're going to use. Limit it by both * maximum size and maximum number of segments. It is assumed that segments @@ -245,6 +286,8 @@ size_t netfs_limit_iter(const struct iov_iter *iter, size_t start_offset, return netfs_limit_bvec(iter, start_offset, max_size, max_segs); if (iov_iter_is_xarray(iter)) return netfs_limit_xarray(iter, start_offset, max_size, max_segs); + if (iov_iter_is_kvec(iter)) + return netfs_limit_kvec(iter, start_offset, max_size, max_segs); BUG(); } EXPORT_SYMBOL(netfs_limit_iter); diff --git a/fs/netfs/read_collect.c b/fs/netfs/read_collect.c index 7a0ffa675fb17a..e5f6665b3341ee 100644 --- a/fs/netfs/read_collect.c +++ b/fs/netfs/read_collect.c @@ -205,7 +205,8 @@ static void netfs_collect_read_results(struct netfs_io_request *rreq) * in progress. The issuer thread may be adding stuff to the tail * whilst we're doing this. */ - front = READ_ONCE(stream->front); + front = list_first_entry_or_null(&stream->subrequests, + struct netfs_io_subrequest, rreq_link); while (front) { size_t transferred; @@ -301,7 +302,6 @@ static void netfs_collect_read_results(struct netfs_io_request *rreq) list_del_init(&front->rreq_link); front = list_first_entry_or_null(&stream->subrequests, struct netfs_io_subrequest, rreq_link); - stream->front = front; spin_unlock(&rreq->lock); netfs_put_subrequest(remove, notes & ABANDON_SREQ ? @@ -546,6 +546,15 @@ void netfs_read_subreq_terminated(struct netfs_io_subrequest *subreq) } } + /* If need retry is set, error should not matter unless we hit too many + * retries. Pause the generation of new subreqs + */ + if (test_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags)) { + trace_netfs_rreq(rreq, netfs_rreq_trace_set_pause); + set_bit(NETFS_RREQ_PAUSE, &rreq->flags); + goto skip_error_checks; + } + if (unlikely(subreq->error < 0)) { trace_netfs_failure(rreq, subreq, subreq->error, netfs_fail_read); if (subreq->source == NETFS_READ_FROM_CACHE) { @@ -559,6 +568,7 @@ void netfs_read_subreq_terminated(struct netfs_io_subrequest *subreq) set_bit(NETFS_RREQ_PAUSE, &rreq->flags); } +skip_error_checks: trace_netfs_sreq(subreq, netfs_sreq_trace_terminated); netfs_subreq_clear_in_progress(subreq); netfs_put_subrequest(subreq, netfs_sreq_trace_put_terminated); diff --git a/fs/netfs/read_retry.c b/fs/netfs/read_retry.c index b99e84a8170af2..cca9ac43c07738 100644 --- a/fs/netfs/read_retry.c +++ b/fs/netfs/read_retry.c @@ -12,6 +12,7 @@ static void netfs_reissue_read(struct netfs_io_request *rreq, struct netfs_io_subrequest *subreq) { + subreq->error = 0; __clear_bit(NETFS_SREQ_MADE_PROGRESS, &subreq->flags); __set_bit(NETFS_SREQ_IN_PROGRESS, &subreq->flags); netfs_stat(&netfs_n_rh_retry_read_subreq); @@ -92,8 +93,10 @@ static void netfs_retry_read_subrequests(struct netfs_io_request *rreq) from->start, from->transferred, from->len); if (test_bit(NETFS_SREQ_FAILED, &from->flags) || - !test_bit(NETFS_SREQ_NEED_RETRY, &from->flags)) + !test_bit(NETFS_SREQ_NEED_RETRY, &from->flags)) { + subreq = from; goto abandon; + } list_for_each_continue(next, &stream->subrequests) { subreq = list_entry(next, struct netfs_io_subrequest, rreq_link); @@ -177,6 +180,7 @@ static void netfs_retry_read_subrequests(struct netfs_io_request *rreq) if (subreq == to) break; } + subreq = NULL; continue; } @@ -242,8 +246,7 @@ static void netfs_retry_read_subrequests(struct netfs_io_request *rreq) subreq = list_next_entry(subreq, rreq_link); abandon: list_for_each_entry_from(subreq, &stream->subrequests, rreq_link) { - if (!subreq->error && - !test_bit(NETFS_SREQ_FAILED, &subreq->flags) && + if (!test_bit(NETFS_SREQ_FAILED, &subreq->flags) && !test_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags)) continue; subreq->error = -ENOMEM; diff --git a/fs/netfs/read_single.c b/fs/netfs/read_single.c index 8e6264f62a8f38..d0e23bc42445ff 100644 --- a/fs/netfs/read_single.c +++ b/fs/netfs/read_single.c @@ -107,7 +107,6 @@ static int netfs_single_dispatch_read(struct netfs_io_request *rreq) spin_lock(&rreq->lock); list_add_tail(&subreq->rreq_link, &stream->subrequests); trace_netfs_sreq(subreq, netfs_sreq_trace_added); - stream->front = subreq; /* Store list pointers before active flag */ smp_store_release(&stream->active, true); spin_unlock(&rreq->lock); diff --git a/fs/netfs/write_collect.c b/fs/netfs/write_collect.c index cbf3d9194c7bf6..b194447f4b1117 100644 --- a/fs/netfs/write_collect.c +++ b/fs/netfs/write_collect.c @@ -228,7 +228,8 @@ static void netfs_collect_write_results(struct netfs_io_request *wreq) if (!smp_load_acquire(&stream->active)) continue; - front = stream->front; + front = list_first_entry_or_null(&stream->subrequests, + struct netfs_io_subrequest, rreq_link); while (front) { trace_netfs_collect_sreq(wreq, front); //_debug("sreq [%x] %llx %zx/%zx", @@ -279,7 +280,6 @@ static void netfs_collect_write_results(struct netfs_io_request *wreq) list_del_init(&front->rreq_link); front = list_first_entry_or_null(&stream->subrequests, struct netfs_io_subrequest, rreq_link); - stream->front = front; spin_unlock(&wreq->lock); netfs_put_subrequest(remove, notes & SAW_FAILURE ? @@ -399,27 +399,6 @@ bool netfs_write_collection(struct netfs_io_request *wreq) ictx->ops->invalidate_cache(wreq); } - if ((wreq->origin == NETFS_UNBUFFERED_WRITE || - wreq->origin == NETFS_DIO_WRITE) && - !wreq->error) - netfs_update_i_size(ictx, &ictx->inode, wreq->start, wreq->transferred); - - if (wreq->origin == NETFS_DIO_WRITE && - wreq->mapping->nrpages) { - /* mmap may have got underfoot and we may now have folios - * locally covering the region we just wrote. Attempt to - * discard the folios, but leave in place any modified locally. - * ->write_iter() is prevented from interfering by the DIO - * counter. - */ - pgoff_t first = wreq->start >> PAGE_SHIFT; - pgoff_t last = (wreq->start + wreq->transferred - 1) >> PAGE_SHIFT; - invalidate_inode_pages2_range(wreq->mapping, first, last); - } - - if (wreq->origin == NETFS_DIO_WRITE) - inode_dio_end(wreq->inode); - _debug("finished"); netfs_wake_rreq_flag(wreq, NETFS_RREQ_IN_PROGRESS, netfs_rreq_trace_wake_ip); /* As we cleared NETFS_RREQ_IN_PROGRESS, we acquired its ref. */ @@ -492,11 +471,11 @@ void netfs_write_subrequest_terminated(void *_op, ssize_t transferred_or_error) if (IS_ERR_VALUE(transferred_or_error)) { subreq->error = transferred_or_error; - if (subreq->error == -EAGAIN) - set_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags); - else + /* if need retry is set, error should not matter */ + if (!test_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags)) { set_bit(NETFS_SREQ_FAILED, &subreq->flags); - trace_netfs_failure(wreq, subreq, transferred_or_error, netfs_fail_write); + trace_netfs_failure(wreq, subreq, transferred_or_error, netfs_fail_write); + } switch (subreq->source) { case NETFS_WRITE_TO_CACHE: diff --git a/fs/netfs/write_issue.c b/fs/netfs/write_issue.c index dd8743bc8d7fe3..2db688f9412519 100644 --- a/fs/netfs/write_issue.c +++ b/fs/netfs/write_issue.c @@ -154,9 +154,9 @@ EXPORT_SYMBOL(netfs_prepare_write_failed); * Prepare a write subrequest. We need to allocate a new subrequest * if we don't have one. */ -static void netfs_prepare_write(struct netfs_io_request *wreq, - struct netfs_io_stream *stream, - loff_t start) +void netfs_prepare_write(struct netfs_io_request *wreq, + struct netfs_io_stream *stream, + loff_t start) { struct netfs_io_subrequest *subreq; struct iov_iter *wreq_iter = &wreq->buffer.iter; @@ -206,9 +206,8 @@ static void netfs_prepare_write(struct netfs_io_request *wreq, spin_lock(&wreq->lock); list_add_tail(&subreq->rreq_link, &stream->subrequests); if (list_is_first(&subreq->rreq_link, &stream->subrequests)) { - stream->front = subreq; if (!stream->active) { - stream->collected_to = stream->front->start; + stream->collected_to = subreq->start; /* Write list pointers before active flag */ smp_store_release(&stream->active, true); } @@ -250,6 +249,7 @@ void netfs_reissue_write(struct netfs_io_stream *stream, iov_iter_truncate(&subreq->io_iter, size); subreq->retry_count++; + subreq->error = 0; __clear_bit(NETFS_SREQ_MADE_PROGRESS, &subreq->flags); __set_bit(NETFS_SREQ_IN_PROGRESS, &subreq->flags); netfs_stat(&netfs_n_wh_retry_write_subreq); @@ -697,41 +697,6 @@ ssize_t netfs_end_writethrough(struct netfs_io_request *wreq, struct writeback_c return ret; } -/* - * Write data to the server without going through the pagecache and without - * writing it to the local cache. - */ -int netfs_unbuffered_write(struct netfs_io_request *wreq, bool may_wait, size_t len) -{ - struct netfs_io_stream *upload = &wreq->io_streams[0]; - ssize_t part; - loff_t start = wreq->start; - int error = 0; - - _enter("%zx", len); - - if (wreq->origin == NETFS_DIO_WRITE) - inode_dio_begin(wreq->inode); - - while (len) { - // TODO: Prepare content encryption - - _debug("unbuffered %zx", len); - part = netfs_advance_write(wreq, upload, start, len, false); - start += part; - len -= part; - rolling_buffer_advance(&wreq->buffer, part); - if (test_bit(NETFS_RREQ_PAUSE, &wreq->flags)) - netfs_wait_for_paused_write(wreq); - if (test_bit(NETFS_RREQ_FAILED, &wreq->flags)) - break; - } - - netfs_end_issue_write(wreq); - _leave(" = %d", error); - return error; -} - /* * Write some of a pending folio data back to the server and/or the cache. */ diff --git a/fs/netfs/write_retry.c b/fs/netfs/write_retry.c index fc9c3e0d34d813..29489a23a22093 100644 --- a/fs/netfs/write_retry.c +++ b/fs/netfs/write_retry.c @@ -98,7 +98,6 @@ static void netfs_retry_write_stream(struct netfs_io_request *wreq, subreq->start = start; subreq->len = len; __clear_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags); - subreq->retry_count++; trace_netfs_sreq(subreq, netfs_sreq_trace_retry); /* Renegotiate max_len (wsize) */ diff --git a/fs/nfs/dir.c b/fs/nfs/dir.c index e3654f4a1b9aab..571bb40880b2d5 100644 --- a/fs/nfs/dir.c +++ b/fs/nfs/dir.c @@ -73,7 +73,7 @@ const struct address_space_operations nfs_dir_aops = { .free_folio = nfs_readdir_clear_array, }; -#define NFS_INIT_DTSIZE PAGE_SIZE +#define NFS_INIT_DTSIZE SZ_64K static struct nfs_open_dir_context * alloc_nfs_open_dir_context(struct inode *dir) @@ -84,7 +84,7 @@ alloc_nfs_open_dir_context(struct inode *dir) ctx = kzalloc(sizeof(*ctx), GFP_KERNEL_ACCOUNT); if (ctx != NULL) { ctx->attr_gencount = nfsi->attr_gencount; - ctx->dtsize = NFS_INIT_DTSIZE; + ctx->dtsize = min(NFS_SERVER(dir)->dtsize, NFS_INIT_DTSIZE); spin_lock(&dir->i_lock); if (list_empty(&nfsi->open_files) && (nfsi->cache_validity & NFS_INO_DATA_INVAL_DEFER)) diff --git a/fs/nfs/localio.c b/fs/nfs/localio.c index 41fbcb3f9167e9..3bd4f28c8afc26 100644 --- a/fs/nfs/localio.c +++ b/fs/nfs/localio.c @@ -58,6 +58,11 @@ struct nfs_local_fsync_ctx { static bool localio_enabled __read_mostly = true; module_param(localio_enabled, bool, 0644); +static int nfs_local_do_read(struct nfs_local_kiocb *iocb, + const struct rpc_call_ops *call_ops); +static int nfs_local_do_write(struct nfs_local_kiocb *iocb, + const struct rpc_call_ops *call_ops); + static inline bool nfs_client_is_local(const struct nfs_client *clp) { return !!rcu_access_pointer(clp->cl_uuid.net); @@ -286,6 +291,18 @@ nfs_local_open_fh(struct nfs_client *clp, const struct cred *cred, } EXPORT_SYMBOL_GPL(nfs_local_open_fh); +/* + * Ensure all page cache allocations are done from GFP_NOFS context to + * prevent direct reclaim recursion back into NFS via nfs_writepages. + */ +static void +nfs_local_mapping_set_gfp_nofs_context(struct address_space *m) +{ + gfp_t gfp_mask = mapping_gfp_mask(m); + + mapping_set_gfp_mask(m, (gfp_mask & ~(__GFP_FS))); +} + static void nfs_local_iocb_free(struct nfs_local_kiocb *iocb) { @@ -310,6 +327,7 @@ nfs_local_iocb_alloc(struct nfs_pgio_header *hdr, return NULL; } + nfs_local_mapping_set_gfp_nofs_context(file->f_mapping); init_sync_kiocb(&iocb->kiocb, file); iocb->hdr = hdr; @@ -542,13 +560,50 @@ nfs_local_iocb_release(struct nfs_local_kiocb *iocb) nfs_local_iocb_free(iocb); } -static void -nfs_local_pgio_release(struct nfs_local_kiocb *iocb) +static void nfs_local_pgio_restart(struct nfs_local_kiocb *iocb, + struct nfs_pgio_header *hdr) +{ + int status = 0; + + iocb->kiocb.ki_pos = hdr->args.offset; + iocb->kiocb.ki_flags &= ~(IOCB_DSYNC | IOCB_SYNC | IOCB_DIRECT); + iocb->kiocb.ki_complete = NULL; + iocb->aio_complete_work = NULL; + iocb->end_iter_index = -1; + + switch (hdr->rw_mode) { + case FMODE_READ: + nfs_local_iters_init(iocb, ITER_DEST); + status = nfs_local_do_read(iocb, hdr->task.tk_ops); + break; + case FMODE_WRITE: + nfs_local_iters_init(iocb, ITER_SOURCE); + status = nfs_local_do_write(iocb, hdr->task.tk_ops); + break; + default: + status = -EOPNOTSUPP; + } + + if (status != 0) { + nfs_local_iocb_release(iocb); + hdr->task.tk_status = status; + nfs_local_hdr_release(hdr, hdr->task.tk_ops); + } +} + +static void nfs_local_pgio_release(struct nfs_local_kiocb *iocb) { struct nfs_pgio_header *hdr = iocb->hdr; + struct rpc_task *task = &hdr->task; + + task->tk_action = NULL; + task->tk_ops->rpc_call_done(task, hdr); - nfs_local_iocb_release(iocb); - nfs_local_hdr_release(hdr, hdr->task.tk_ops); + if (task->tk_action == NULL) { + nfs_local_iocb_release(iocb); + task->tk_ops->rpc_release(hdr); + } else + nfs_local_pgio_restart(iocb, hdr); } /* @@ -773,19 +828,7 @@ static void nfs_local_write_done(struct nfs_local_kiocb *iocb) pr_info_ratelimited("nfs: Unexpected direct I/O write alignment failure\n"); } - /* Handle short writes as if they are ENOSPC */ - status = hdr->res.count; - if (status > 0 && status < hdr->args.count) { - hdr->mds_offset += status; - hdr->args.offset += status; - hdr->args.pgbase += status; - hdr->args.count -= status; - nfs_set_pgio_error(hdr, -ENOSPC, hdr->args.offset); - status = -ENOSPC; - /* record -ENOSPC in terms of nfs_local_pgio_done */ - (void) nfs_local_pgio_done(iocb, status, true); - } - if (hdr->task.tk_status < 0) + if (status < 0) nfs_reset_boot_verifier(hdr->inode); } @@ -946,8 +989,6 @@ int nfs_local_doio(struct nfs_client *clp, struct nfsd_file *localio, } if (status != 0) { - if (status == -EAGAIN) - nfs_localio_disable_client(clp); nfs_local_iocb_release(iocb); hdr->task.tk_status = status; nfs_local_hdr_release(hdr, call_ops); @@ -974,6 +1015,8 @@ nfs_local_run_commit(struct file *filp, struct nfs_commit_data *data) end = LLONG_MAX; } + nfs_local_mapping_set_gfp_nofs_context(filp->f_mapping); + dprintk("%s: commit %llu - %llu\n", __func__, start, end); return vfs_fsync_range(filp, start, end, 0); } @@ -1015,17 +1058,22 @@ nfs_local_fsync_ctx_free(struct nfs_local_fsync_ctx *ctx) static void nfs_local_fsync_work(struct work_struct *work) { + unsigned long old_flags = current->flags; struct nfs_local_fsync_ctx *ctx; int status; ctx = container_of(work, struct nfs_local_fsync_ctx, work); + current->flags |= PF_LOCAL_THROTTLE | PF_MEMALLOC_NOIO; + status = nfs_local_run_commit(nfs_to->nfsd_file_file(ctx->localio), ctx->data); nfs_local_commit_done(ctx->data, status); if (ctx->done != NULL) complete(ctx->done); nfs_local_fsync_ctx_free(ctx); + + current->flags = old_flags; } static struct nfs_local_fsync_ctx * @@ -1049,7 +1097,7 @@ int nfs_local_commit(struct nfsd_file *localio, { struct nfs_local_fsync_ctx *ctx; - ctx = nfs_local_fsync_ctx_alloc(data, localio, GFP_KERNEL); + ctx = nfs_local_fsync_ctx_alloc(data, localio, GFP_NOIO); if (!ctx) { nfs_local_commit_done(data, -ENOMEM); nfs_local_release_commit_data(localio, data, call_ops); @@ -1061,10 +1109,10 @@ int nfs_local_commit(struct nfsd_file *localio, if (how & FLUSH_SYNC) { DECLARE_COMPLETION_ONSTACK(done); ctx->done = &done; - queue_work(nfsiod_workqueue, &ctx->work); + queue_work(nfslocaliod_workqueue, &ctx->work); wait_for_completion(&done); } else - queue_work(nfsiod_workqueue, &ctx->work); + queue_work(nfslocaliod_workqueue, &ctx->work); return 0; } diff --git a/fs/nfs/nfs3proc.c b/fs/nfs/nfs3proc.c index 1181f9cc6dbdb0..f8bc9bffdad90b 100644 --- a/fs/nfs/nfs3proc.c +++ b/fs/nfs/nfs3proc.c @@ -392,8 +392,13 @@ nfs3_proc_create(struct inode *dir, struct dentry *dentry, struct iattr *sattr, if (status != 0) goto out_release_acls; - if (d_alias) + if (d_alias) { + if (d_is_dir(d_alias)) { + status = -EISDIR; + goto out_dput; + } dentry = d_alias; + } /* When we created the file with exclusive semantics, make * sure we set the attributes afterwards. */ diff --git a/fs/nfs/pnfs.c b/fs/nfs/pnfs.c index cff225721d1ce5..ff8483d3373a8b 100644 --- a/fs/nfs/pnfs.c +++ b/fs/nfs/pnfs.c @@ -463,7 +463,8 @@ pnfs_mark_layout_stateid_invalid(struct pnfs_layout_hdr *lo, }; struct pnfs_layout_segment *lseg, *next; - set_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags); + if (test_and_set_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags)) + return !list_empty(&lo->plh_segs); clear_bit(NFS_INO_LAYOUTCOMMIT, &NFS_I(lo->plh_inode)->flags); list_for_each_entry_safe(lseg, next, &lo->plh_segs, pls_list) pnfs_clear_lseg_state(lseg, lseg_list); diff --git a/fs/nfsd/export.c b/fs/nfsd/export.c index 2a1499f2ad196a..0d2feaab545b30 100644 --- a/fs/nfsd/export.c +++ b/fs/nfsd/export.c @@ -36,19 +36,30 @@ * second map contains a reference to the entry in the first map. */ +static struct workqueue_struct *nfsd_export_wq; + #define EXPKEY_HASHBITS 8 #define EXPKEY_HASHMAX (1 << EXPKEY_HASHBITS) #define EXPKEY_HASHMASK (EXPKEY_HASHMAX -1) -static void expkey_put(struct kref *ref) +static void expkey_release(struct work_struct *work) { - struct svc_expkey *key = container_of(ref, struct svc_expkey, h.ref); + struct svc_expkey *key = container_of(to_rcu_work(work), + struct svc_expkey, ek_rwork); if (test_bit(CACHE_VALID, &key->h.flags) && !test_bit(CACHE_NEGATIVE, &key->h.flags)) path_put(&key->ek_path); auth_domain_put(key->ek_client); - kfree_rcu(key, ek_rcu); + kfree(key); +} + +static void expkey_put(struct kref *ref) +{ + struct svc_expkey *key = container_of(ref, struct svc_expkey, h.ref); + + INIT_RCU_WORK(&key->ek_rwork, expkey_release); + queue_rcu_work(nfsd_export_wq, &key->ek_rwork); } static int expkey_upcall(struct cache_detail *cd, struct cache_head *h) @@ -353,11 +364,13 @@ static void export_stats_destroy(struct export_stats *stats) EXP_STATS_COUNTERS_NUM); } -static void svc_export_release(struct rcu_head *rcu_head) +static void svc_export_release(struct work_struct *work) { - struct svc_export *exp = container_of(rcu_head, struct svc_export, - ex_rcu); + struct svc_export *exp = container_of(to_rcu_work(work), + struct svc_export, ex_rwork); + path_put(&exp->ex_path); + auth_domain_put(exp->ex_client); nfsd4_fslocs_free(&exp->ex_fslocs); export_stats_destroy(exp->ex_stats); kfree(exp->ex_stats); @@ -369,9 +382,8 @@ static void svc_export_put(struct kref *ref) { struct svc_export *exp = container_of(ref, struct svc_export, h.ref); - path_put(&exp->ex_path); - auth_domain_put(exp->ex_client); - call_rcu(&exp->ex_rcu, svc_export_release); + INIT_RCU_WORK(&exp->ex_rwork, svc_export_release); + queue_rcu_work(nfsd_export_wq, &exp->ex_rwork); } static int svc_export_upcall(struct cache_detail *cd, struct cache_head *h) @@ -427,7 +439,8 @@ static int check_export(const struct path *path, int *flags, unsigned char *uuid * either a device number (so FS_REQUIRES_DEV needed) * or an FSID number (so NFSEXP_FSID or ->uuid is needed). * 2: We must be able to find an inode from a filehandle. - * This means that s_export_op must be set. + * This means that s_export_op must be set and comply with + * the requirements for remote filesystem export. * 3: We must not currently be on an idmapped mount. */ if (!(inode->i_sb->s_type->fs_flags & FS_REQUIRES_DEV) && @@ -437,8 +450,9 @@ static int check_export(const struct path *path, int *flags, unsigned char *uuid return -EINVAL; } - if (!exportfs_can_decode_fh(inode->i_sb->s_export_op)) { - dprintk("exp_export: export of invalid fs type.\n"); + if (!exportfs_may_export(inode->i_sb->s_export_op)) { + dprintk("exp_export: export of invalid fs type (%s).\n", + inode->i_sb->s_type->name); return -EINVAL; } @@ -1478,6 +1492,36 @@ const struct seq_operations nfs_exports_op = { .show = e_show, }; +/** + * nfsd_export_wq_init - allocate the export release workqueue + * + * Called once at module load. The workqueue runs deferred svc_export and + * svc_expkey release work scheduled by queue_rcu_work() in the cache put + * callbacks. + * + * Return values: + * %0: workqueue allocated + * %-ENOMEM: allocation failed + */ +int nfsd_export_wq_init(void) +{ + nfsd_export_wq = alloc_workqueue("nfsd_export", WQ_UNBOUND, 0); + if (!nfsd_export_wq) + return -ENOMEM; + return 0; +} + +/** + * nfsd_export_wq_shutdown - drain and free the export release workqueue + * + * Called once at module unload. Per-namespace teardown in + * nfsd_export_shutdown() has already drained all deferred work. + */ +void nfsd_export_wq_shutdown(void) +{ + destroy_workqueue(nfsd_export_wq); +} + /* * Initialize the exports module. */ @@ -1539,6 +1583,9 @@ nfsd_export_shutdown(struct net *net) cache_unregister_net(nn->svc_expkey_cache, net); cache_unregister_net(nn->svc_export_cache, net); + /* Drain deferred export and expkey release work. */ + rcu_barrier(); + flush_workqueue(nfsd_export_wq); cache_destroy_net(nn->svc_expkey_cache, net); cache_destroy_net(nn->svc_export_cache, net); svcauth_unix_purge(net); diff --git a/fs/nfsd/export.h b/fs/nfsd/export.h index d2b09cd761453d..b053993745749e 100644 --- a/fs/nfsd/export.h +++ b/fs/nfsd/export.h @@ -7,6 +7,7 @@ #include #include +#include #include #include @@ -75,7 +76,7 @@ struct svc_export { u32 ex_layout_types; struct nfsd4_deviceid_map *ex_devid_map; struct cache_detail *cd; - struct rcu_head ex_rcu; + struct rcu_work ex_rwork; unsigned long ex_xprtsec_modes; struct export_stats *ex_stats; }; @@ -92,7 +93,7 @@ struct svc_expkey { u32 ek_fsid[6]; struct path ek_path; - struct rcu_head ek_rcu; + struct rcu_work ek_rwork; }; #define EX_ISSYNC(exp) (!((exp)->ex_flags & NFSEXP_ASYNC)) @@ -110,6 +111,8 @@ __be32 check_nfsd_access(struct svc_export *exp, struct svc_rqst *rqstp, /* * Function declarations */ +int nfsd_export_wq_init(void); +void nfsd_export_wq_shutdown(void); int nfsd_export_init(struct net *); void nfsd_export_shutdown(struct net *); void nfsd_export_flush(struct net *); diff --git a/fs/nfsd/nfs2acl.c b/fs/nfsd/nfs2acl.c index 5fb202acb0fd00..0ac538c7618009 100644 --- a/fs/nfsd/nfs2acl.c +++ b/fs/nfsd/nfs2acl.c @@ -45,7 +45,7 @@ static __be32 nfsacld_proc_getacl(struct svc_rqst *rqstp) inode = d_inode(fh->fh_dentry); if (argp->mask & ~NFS_ACL_MASK) { - resp->status = nfserr_inval; + resp->status = nfserr_io; goto out; } resp->mask = argp->mask; diff --git a/fs/nfsd/nfs4idmap.c b/fs/nfsd/nfs4idmap.c index 8cca1329f3485c..c319c31b0f6476 100644 --- a/fs/nfsd/nfs4idmap.c +++ b/fs/nfsd/nfs4idmap.c @@ -643,34 +643,74 @@ static __be32 encode_name_from_id(struct xdr_stream *xdr, return idmap_id_to_name(xdr, rqstp, type, id); } -__be32 -nfsd_map_name_to_uid(struct svc_rqst *rqstp, const char *name, size_t namelen, - kuid_t *uid) +/** + * nfsd_map_name_to_uid - Map user@domain to local UID + * @rqstp: RPC execution context + * @name: user@domain name to be mapped + * @namelen: length of name, in bytes + * @uid: OUT: mapped local UID value + * + * Returns nfs_ok on success or an NFSv4 status code on failure. + */ +__be32 nfsd_map_name_to_uid(struct svc_rqst *rqstp, const char *name, + size_t namelen, kuid_t *uid) { __be32 status; u32 id = -1; + /* + * The idmap lookup below triggers an upcall that invokes + * cache_check(). RQ_USEDEFERRAL must be clear to prevent + * cache_check() from setting RQ_DROPME via svc_defer(). + * NFSv4 servers are not permitted to drop requests. Also + * RQ_DROPME will force NFSv4.1 session slot processing to + * be skipped. + */ + WARN_ON_ONCE(test_bit(RQ_USEDEFERRAL, &rqstp->rq_flags)); + if (name == NULL || namelen == 0) return nfserr_inval; status = do_name_to_id(rqstp, IDMAP_TYPE_USER, name, namelen, &id); + if (status) + return status; *uid = make_kuid(nfsd_user_namespace(rqstp), id); if (!uid_valid(*uid)) status = nfserr_badowner; return status; } -__be32 -nfsd_map_name_to_gid(struct svc_rqst *rqstp, const char *name, size_t namelen, - kgid_t *gid) +/** + * nfsd_map_name_to_gid - Map user@domain to local GID + * @rqstp: RPC execution context + * @name: user@domain name to be mapped + * @namelen: length of name, in bytes + * @gid: OUT: mapped local GID value + * + * Returns nfs_ok on success or an NFSv4 status code on failure. + */ +__be32 nfsd_map_name_to_gid(struct svc_rqst *rqstp, const char *name, + size_t namelen, kgid_t *gid) { __be32 status; u32 id = -1; + /* + * The idmap lookup below triggers an upcall that invokes + * cache_check(). RQ_USEDEFERRAL must be clear to prevent + * cache_check() from setting RQ_DROPME via svc_defer(). + * NFSv4 servers are not permitted to drop requests. Also + * RQ_DROPME will force NFSv4.1 session slot processing to + * be skipped. + */ + WARN_ON_ONCE(test_bit(RQ_USEDEFERRAL, &rqstp->rq_flags)); + if (name == NULL || namelen == 0) return nfserr_inval; status = do_name_to_id(rqstp, IDMAP_TYPE_GROUP, name, namelen, &id); + if (status) + return status; *gid = make_kgid(nfsd_user_namespace(rqstp), id); if (!gid_valid(*gid)) status = nfserr_badowner; diff --git a/fs/nfsd/nfs4proc.c b/fs/nfsd/nfs4proc.c index 9ec08dd4fe823b..f780024f9a0887 100644 --- a/fs/nfsd/nfs4proc.c +++ b/fs/nfsd/nfs4proc.c @@ -3011,8 +3011,6 @@ nfsd4_proc_compound(struct svc_rqst *rqstp) BUG_ON(cstate->replay_owner); out: cstate->status = status; - /* Reset deferral mechanism for RPC deferrals */ - set_bit(RQ_USEDEFERRAL, &rqstp->rq_flags); return rpc_success; } diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c index d5e0f3a52d4f02..c298ec2621ec9c 100644 --- a/fs/nfsd/nfs4state.c +++ b/fs/nfsd/nfs4state.c @@ -6353,7 +6353,8 @@ nfs4_open_delegation(struct svc_rqst *rqstp, struct nfsd4_open *open, dp->dl_ctime = stat.ctime; dp->dl_mtime = stat.mtime; spin_lock(&f->f_lock); - f->f_mode |= FMODE_NOCMTIME; + if (deleg_ts) + f->f_mode |= FMODE_NOCMTIME; spin_unlock(&f->f_lock); trace_nfsd_deleg_write(&dp->dl_stid.sc_stateid); } else { @@ -9520,8 +9521,10 @@ nfsd_get_dir_deleg(struct nfsd4_compound_state *cstate, spin_unlock(&clp->cl_lock); spin_unlock(&state_lock); - if (!status) + if (!status) { + put_nfs4_file(fp); return dp; + } /* Something failed. Drop the lease and clean up the stid */ kernel_setlease(fp->fi_deleg_file->nf_file, F_UNLCK, NULL, (void **)&dp); @@ -9529,5 +9532,6 @@ nfsd_get_dir_deleg(struct nfsd4_compound_state *cstate, nfs4_put_stid(&dp->dl_stid); out_delegees: put_deleg_file(fp); + put_nfs4_file(fp); return ERR_PTR(status); } diff --git a/fs/nfsd/nfs4xdr.c b/fs/nfsd/nfs4xdr.c index 51ef97c2545688..ddc6e4fb481e41 100644 --- a/fs/nfsd/nfs4xdr.c +++ b/fs/nfsd/nfs4xdr.c @@ -5946,9 +5946,14 @@ nfsd4_encode_operation(struct nfsd4_compoundres *resp, struct nfsd4_op *op) int len = xdr->buf->len - (op_status_offset + XDR_UNIT); so->so_replay.rp_status = op->status; - so->so_replay.rp_buflen = len; - read_bytes_from_xdr_buf(xdr->buf, op_status_offset + XDR_UNIT, + if (len <= NFSD4_REPLAY_ISIZE) { + so->so_replay.rp_buflen = len; + read_bytes_from_xdr_buf(xdr->buf, + op_status_offset + XDR_UNIT, so->so_replay.rp_buf, len); + } else { + so->so_replay.rp_buflen = 0; + } } status: op->status = nfsd4_map_status(op->status, @@ -6013,6 +6018,22 @@ nfs4svc_decode_compoundargs(struct svc_rqst *rqstp, struct xdr_stream *xdr) args->ops = args->iops; args->rqstp = rqstp; + /* + * NFSv4 operation decoders can invoke svc cache lookups + * that trigger svc_defer() when RQ_USEDEFERRAL is set, + * setting RQ_DROPME. This creates two problems: + * + * 1. Non-idempotency: Compounds make it too hard to avoid + * problems if a request is deferred and replayed. + * + * 2. Session slot leakage (NFSv4.1+): If RQ_DROPME is set + * during decode but SEQUENCE executes successfully, the + * session slot will be marked INUSE. The request is then + * dropped before encoding, so the slot is never released, + * rendering it permanently unusable by the client. + */ + clear_bit(RQ_USEDEFERRAL, &rqstp->rq_flags); + return nfsd4_decode_compound(args); } diff --git a/fs/nfsd/nfsctl.c b/fs/nfsd/nfsctl.c index 084fc517e9e160..6b198e75282126 100644 --- a/fs/nfsd/nfsctl.c +++ b/fs/nfsd/nfsctl.c @@ -149,9 +149,19 @@ static int exports_net_open(struct net *net, struct file *file) seq = file->private_data; seq->private = nn->svc_export_cache; + get_net(net); return 0; } +static int exports_release(struct inode *inode, struct file *file) +{ + struct seq_file *seq = file->private_data; + struct cache_detail *cd = seq->private; + + put_net(cd->net); + return seq_release(inode, file); +} + static int exports_nfsd_open(struct inode *inode, struct file *file) { return exports_net_open(inode->i_sb->s_fs_info, file); @@ -161,7 +171,7 @@ static const struct file_operations exports_nfsd_operations = { .open = exports_nfsd_open, .read = seq_read, .llseek = seq_lseek, - .release = seq_release, + .release = exports_release, }; static int export_features_show(struct seq_file *m, void *v) @@ -1375,7 +1385,7 @@ static const struct proc_ops exports_proc_ops = { .proc_open = exports_proc_open, .proc_read = seq_read, .proc_lseek = seq_lseek, - .proc_release = seq_release, + .proc_release = exports_release, }; static int create_proc_exports_entry(void) @@ -1642,7 +1652,7 @@ int nfsd_nl_threads_set_doit(struct sk_buff *skb, struct genl_info *info) scope = nla_data(attr); } - ret = nfsd_svc(nrpools, nthreads, net, get_current_cred(), scope); + ret = nfsd_svc(nrpools, nthreads, net, current_cred(), scope); if (ret > 0) ret = 0; out_unlock: @@ -1993,7 +2003,7 @@ int nfsd_nl_listener_set_doit(struct sk_buff *skb, struct genl_info *info) } ret = svc_xprt_create_from_sa(serv, xcl_name, net, sa, 0, - get_current_cred()); + current_cred()); /* always save the latest error */ if (ret < 0) err = ret; @@ -2252,9 +2262,12 @@ static int __init init_nfsd(void) if (retval) goto out_free_pnfs; nfsd_lockd_init(); /* lockd->nfsd callbacks */ + retval = nfsd_export_wq_init(); + if (retval) + goto out_free_lockd; retval = register_pernet_subsys(&nfsd_net_ops); if (retval < 0) - goto out_free_lockd; + goto out_free_export_wq; retval = register_cld_notifier(); if (retval) goto out_free_subsys; @@ -2283,6 +2296,8 @@ static int __init init_nfsd(void) unregister_cld_notifier(); out_free_subsys: unregister_pernet_subsys(&nfsd_net_ops); +out_free_export_wq: + nfsd_export_wq_shutdown(); out_free_lockd: nfsd_lockd_shutdown(); nfsd_drc_slab_free(); @@ -2303,6 +2318,7 @@ static void __exit exit_nfsd(void) nfsd4_destroy_laundry_wq(); unregister_cld_notifier(); unregister_pernet_subsys(&nfsd_net_ops); + nfsd_export_wq_shutdown(); nfsd_drc_slab_free(); nfsd_lockd_shutdown(); nfsd4_free_slabs(); diff --git a/fs/nfsd/nfsproc.c b/fs/nfsd/nfsproc.c index 481e789a769749..8873033d1e82f4 100644 --- a/fs/nfsd/nfsproc.c +++ b/fs/nfsd/nfsproc.c @@ -33,7 +33,7 @@ static __be32 nfsd_map_status(__be32 status) break; case nfserr_symlink: case nfserr_wrong_type: - status = nfserr_inval; + status = nfserr_io; break; } return status; diff --git a/fs/nfsd/state.h b/fs/nfsd/state.h index 508b7e36d846d8..97c5675cd1cae1 100644 --- a/fs/nfsd/state.h +++ b/fs/nfsd/state.h @@ -541,11 +541,18 @@ struct nfs4_client_reclaim { struct xdr_netobj cr_princhash; }; -/* A reasonable value for REPLAY_ISIZE was estimated as follows: - * The OPEN response, typically the largest, requires - * 4(status) + 8(stateid) + 20(changeinfo) + 4(rflags) + 8(verifier) + - * 4(deleg. type) + 8(deleg. stateid) + 4(deleg. recall flag) + - * 20(deleg. space limit) + ~32(deleg. ace) = 112 bytes +/* + * REPLAY_ISIZE is sized for an OPEN response with delegation: + * 4(status) + 8(stateid) + 20(changeinfo) + 4(rflags) + + * 8(verifier) + 4(deleg. type) + 8(deleg. stateid) + + * 4(deleg. recall flag) + 20(deleg. space limit) + + * ~32(deleg. ace) = 112 bytes + * + * Some responses can exceed this. A LOCK denial includes the conflicting + * lock owner, which can be up to 1024 bytes (NFS4_OPAQUE_LIMIT). Responses + * larger than REPLAY_ISIZE are not cached in rp_ibuf; only rp_status is + * saved. Enlarging this constant increases the size of every + * nfs4_stateowner. */ #define NFSD4_REPLAY_ISIZE 112 diff --git a/fs/nilfs2/dat.c b/fs/nilfs2/dat.c index 674380837ab98b..888dc1831c86ee 100644 --- a/fs/nilfs2/dat.c +++ b/fs/nilfs2/dat.c @@ -524,6 +524,9 @@ int nilfs_dat_read(struct super_block *sb, size_t entry_size, if (err) goto failed; + err = nilfs_attach_btree_node_cache(dat); + if (err) + goto failed; err = nilfs_read_inode_common(dat, raw_inode); if (err) goto failed; diff --git a/fs/nilfs2/sufile.c b/fs/nilfs2/sufile.c index 83f93337c01b24..eceedca0269725 100644 --- a/fs/nilfs2/sufile.c +++ b/fs/nilfs2/sufile.c @@ -1093,6 +1093,9 @@ int nilfs_sufile_trim_fs(struct inode *sufile, struct fstrim_range *range) else end_block = start_block + len - 1; + if (end_block < nilfs->ns_first_data_block) + goto out; + segnum = nilfs_get_segnum_of_block(nilfs, start_block); segnum_end = nilfs_get_segnum_of_block(nilfs, end_block); @@ -1191,6 +1194,7 @@ int nilfs_sufile_trim_fs(struct inode *sufile, struct fstrim_range *range) out_sem: up_read(&NILFS_MDT(sufile)->mi_sem); +out: range->len = ndiscarded << nilfs->ns_blocksize_bits; return ret; } diff --git a/fs/nsfs.c b/fs/nsfs.c index bf27d5da91f1e7..392d2d7f85f747 100644 --- a/fs/nsfs.c +++ b/fs/nsfs.c @@ -186,6 +186,17 @@ static bool nsfs_ioctl_valid(unsigned int cmd) return false; } +static bool may_use_nsfs_ioctl(unsigned int cmd) +{ + switch (_IOC_NR(cmd)) { + case _IOC_NR(NS_MNT_GET_NEXT): + fallthrough; + case _IOC_NR(NS_MNT_GET_PREV): + return may_see_all_namespaces(); + } + return true; +} + static long ns_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) { @@ -201,6 +212,8 @@ static long ns_ioctl(struct file *filp, unsigned int ioctl, if (!nsfs_ioctl_valid(ioctl)) return -ENOIOCTLCMD; + if (!may_use_nsfs_ioctl(ioctl)) + return -EPERM; ns = get_proc_ns(file_inode(filp)); switch (ioctl) { @@ -601,7 +614,7 @@ static struct dentry *nsfs_fh_to_dentry(struct super_block *sb, struct fid *fh, return ERR_PTR(-EOPNOTSUPP); } - if (owning_ns && !ns_capable(owning_ns, CAP_SYS_ADMIN)) { + if (owning_ns && !may_see_all_namespaces()) { ns->ops->put(ns); return ERR_PTR(-EPERM); } diff --git a/fs/ntfs3/attrib.c b/fs/ntfs3/attrib.c index 980ae9157248da..0cd15a0983fee3 100644 --- a/fs/ntfs3/attrib.c +++ b/fs/ntfs3/attrib.c @@ -448,8 +448,10 @@ int attr_set_size(struct ntfs_inode *ni, enum ATTR_TYPE type, is_ext = is_attr_ext(attr_b); align = sbi->cluster_size; - if (is_ext) + if (is_ext) { align <<= attr_b->nres.c_unit; + keep_prealloc = false; + } old_valid = le64_to_cpu(attr_b->nres.valid_size); old_size = le64_to_cpu(attr_b->nres.data_size); @@ -1354,19 +1356,28 @@ int attr_load_runs_range(struct ntfs_inode *ni, enum ATTR_TYPE type, CLST vcn; CLST vcn_last = (to - 1) >> cluster_bits; CLST lcn, clen; - int err; + int err = 0; + int retry = 0; for (vcn = from >> cluster_bits; vcn <= vcn_last; vcn += clen) { if (!run_lookup_entry(run, vcn, &lcn, &clen, NULL)) { + if (retry != 0) { /* Next run_lookup_entry(vcn) also failed. */ + err = -EINVAL; + break; + } err = attr_load_runs_vcn(ni, type, name, name_len, run, vcn); if (err) - return err; + break; + clen = 0; /* Next run_lookup_entry(vcn) must be success. */ + retry++; } + else + retry = 0; } - return 0; + return err; } #ifdef CONFIG_NTFS3_LZX_XPRESS diff --git a/fs/ntfs3/attrlist.c b/fs/ntfs3/attrlist.c index a4d74bed74fab0..098bd7e8c3d646 100644 --- a/fs/ntfs3/attrlist.c +++ b/fs/ntfs3/attrlist.c @@ -52,6 +52,11 @@ int ntfs_load_attr_list(struct ntfs_inode *ni, struct ATTRIB *attr) if (!attr->non_res) { lsize = le32_to_cpu(attr->res.data_size); + if (!lsize) { + err = -EINVAL; + goto out; + } + /* attr is resident: lsize < record_size (1K or 4K) */ le = kvmalloc(al_aligned(lsize), GFP_KERNEL); if (!le) { @@ -66,6 +71,10 @@ int ntfs_load_attr_list(struct ntfs_inode *ni, struct ATTRIB *attr) u16 run_off = le16_to_cpu(attr->nres.run_off); lsize = le64_to_cpu(attr->nres.data_size); + if (!lsize) { + err = -EINVAL; + goto out; + } run_init(&ni->attr_list.run); diff --git a/fs/ntfs3/dir.c b/fs/ntfs3/dir.c index b98e95d6b4d993..cf038d713f507f 100644 --- a/fs/ntfs3/dir.c +++ b/fs/ntfs3/dir.c @@ -423,8 +423,7 @@ static int ntfs_readdir(struct file *file, struct dir_context *ctx) if (!dir_emit_dots(file, ctx)) return 0; - /* Allocate PATH_MAX bytes. */ - name = __getname(); + name = kmalloc(PATH_MAX, GFP_KERNEL); if (!name) return -ENOMEM; @@ -502,7 +501,7 @@ static int ntfs_readdir(struct file *file, struct dir_context *ctx) out: - __putname(name); + kfree(name); put_indx_node(node); if (err == 1) { diff --git a/fs/ntfs3/file.c b/fs/ntfs3/file.c index 2e7b2e566ebe18..13d014b878f6c8 100644 --- a/fs/ntfs3/file.c +++ b/fs/ntfs3/file.c @@ -505,8 +505,8 @@ static int ntfs_truncate(struct inode *inode, loff_t new_size) { struct super_block *sb = inode->i_sb; struct ntfs_inode *ni = ntfs_i(inode); - int err, dirty = 0; u64 new_valid; + int err; if (!S_ISREG(inode->i_mode)) return 0; @@ -522,7 +522,6 @@ static int ntfs_truncate(struct inode *inode, loff_t new_size) } new_valid = ntfs_up_block(sb, min_t(u64, ni->i_valid, new_size)); - truncate_setsize(inode, new_size); ni_lock(ni); @@ -536,20 +535,19 @@ static int ntfs_truncate(struct inode *inode, loff_t new_size) ni->i_valid = new_valid; ni_unlock(ni); + if (unlikely(err)) + return err; ni->std_fa |= FILE_ATTRIBUTE_ARCHIVE; inode_set_mtime_to_ts(inode, inode_set_ctime_current(inode)); if (!IS_DIRSYNC(inode)) { - dirty = 1; + mark_inode_dirty(inode); } else { err = ntfs_sync_inode(inode); if (err) return err; } - if (dirty) - mark_inode_dirty(inode); - return 0; } @@ -995,7 +993,7 @@ static int ntfs_get_frame_pages(struct address_space *mapping, pgoff_t index, folio = __filemap_get_folio(mapping, index, FGP_LOCK | FGP_ACCESSED | FGP_CREAT, - gfp_mask); + gfp_mask | __GFP_ZERO); if (IS_ERR(folio)) { while (npages--) { folio = page_folio(pages[npages]); @@ -1077,8 +1075,12 @@ static ssize_t ntfs_compress_write(struct kiocb *iocb, struct iov_iter *from) goto out; if (lcn == SPARSE_LCN) { - ni->i_valid = valid = - frame_vbo + ((u64)clen << sbi->cluster_bits); + valid = frame_vbo + ((u64)clen << sbi->cluster_bits); + if (ni->i_valid == valid) { + err = -EINVAL; + goto out; + } + ni->i_valid = valid; continue; } diff --git a/fs/ntfs3/frecord.c b/fs/ntfs3/frecord.c index 641ddaf8d4a07e..d5bbd47e1ee9db 100644 --- a/fs/ntfs3/frecord.c +++ b/fs/ntfs3/frecord.c @@ -2046,18 +2046,18 @@ static struct page *ntfs_lock_new_page(struct address_space *mapping, } /* - * ni_readpage_cmpr + * ni_read_folio_cmpr * * When decompressing, we typically obtain more than one page per reference. * We inject the additional pages into the page cache. */ -int ni_readpage_cmpr(struct ntfs_inode *ni, struct folio *folio) +int ni_read_folio_cmpr(struct ntfs_inode *ni, struct folio *folio) { int err; struct ntfs_sb_info *sbi = ni->mi.sbi; struct address_space *mapping = folio->mapping; - pgoff_t index = folio->index; - u64 frame_vbo, vbo = (u64)index << PAGE_SHIFT; + pgoff_t index; + u64 frame_vbo, vbo = folio_pos(folio); struct page **pages = NULL; /* Array of at most 16 pages. stack? */ u8 frame_bits; CLST frame; @@ -2107,7 +2107,9 @@ int ni_readpage_cmpr(struct ntfs_inode *ni, struct folio *folio) pages[i] = pg; } + ni_lock(ni); err = ni_read_frame(ni, frame_vbo, pages, pages_per_frame, 0); + ni_unlock(ni); out1: for (i = 0; i < pages_per_frame; i++) { diff --git a/fs/ntfs3/fslog.c b/fs/ntfs3/fslog.c index 38934e6978ecec..28bd611f580d9c 100644 --- a/fs/ntfs3/fslog.c +++ b/fs/ntfs3/fslog.c @@ -3429,6 +3429,9 @@ static int do_action(struct ntfs_log *log, struct OPEN_ATTR_ENRTY *oe, e1 = Add2Ptr(attr, le16_to_cpu(lrh->attr_off)); esize = le16_to_cpu(e1->size); + if (PtrOffset(e1, Add2Ptr(hdr, used)) < esize) + goto dirty_vol; + e2 = Add2Ptr(e1, esize); memmove(e1, e2, PtrOffset(e2, Add2Ptr(hdr, used))); diff --git a/fs/ntfs3/fsntfs.c b/fs/ntfs3/fsntfs.c index 5f138f71583577..ea5b673462c358 100644 --- a/fs/ntfs3/fsntfs.c +++ b/fs/ntfs3/fsntfs.c @@ -1252,6 +1252,12 @@ int ntfs_read_run_nb(struct ntfs_sb_info *sbi, const struct runs_tree *run, } while (len32); + if (!run) { + err = -EINVAL; + goto out; + } + + /* Get next fragment to read. */ vcn_next = vcn + clen; if (!run_get_entry(run, ++idx, &vcn, &lcn, &clen) || vcn != vcn_next) { @@ -2627,7 +2633,7 @@ int ntfs_set_label(struct ntfs_sb_info *sbi, u8 *label, int len) u32 uni_bytes; struct ntfs_inode *ni = sbi->volume.ni; /* Allocate PATH_MAX bytes. */ - struct cpu_str *uni = __getname(); + struct cpu_str *uni = kmalloc(PATH_MAX, GFP_KERNEL); if (!uni) return -ENOMEM; @@ -2671,6 +2677,6 @@ int ntfs_set_label(struct ntfs_sb_info *sbi, u8 *label, int len) err = _ni_write_inode(&ni->vfs_inode, 0); out: - __putname(uni); + kfree(uni); return err; } diff --git a/fs/ntfs3/index.c b/fs/ntfs3/index.c index 7157cfd70fdcb4..75b94beac1613a 100644 --- a/fs/ntfs3/index.c +++ b/fs/ntfs3/index.c @@ -1190,7 +1190,12 @@ int indx_find(struct ntfs_index *indx, struct ntfs_inode *ni, return -EINVAL; } - fnd_push(fnd, node, e); + err = fnd_push(fnd, node, e); + + if (err) { + put_indx_node(node); + return err; + } } *entry = e; diff --git a/fs/ntfs3/inode.c b/fs/ntfs3/inode.c index 0a9ac5efeb67c2..bf6ef525d9562d 100644 --- a/fs/ntfs3/inode.c +++ b/fs/ntfs3/inode.c @@ -735,9 +735,8 @@ static int ntfs_read_folio(struct file *file, struct folio *folio) } if (is_compressed(ni)) { - ni_lock(ni); - err = ni_readpage_cmpr(ni, folio); - ni_unlock(ni); + /* ni_lock is taken inside ni_read_folio_cmpr after page locks */ + err = ni_read_folio_cmpr(ni, folio); return err; } @@ -1281,7 +1280,7 @@ int ntfs_create_inode(struct mnt_idmap *idmap, struct inode *dir, fa |= FILE_ATTRIBUTE_READONLY; /* Allocate PATH_MAX bytes. */ - new_de = kmem_cache_zalloc(names_cachep, GFP_KERNEL); + new_de = kzalloc(PATH_MAX, GFP_KERNEL); if (!new_de) { err = -ENOMEM; goto out1; @@ -1702,7 +1701,7 @@ int ntfs_create_inode(struct mnt_idmap *idmap, struct inode *dir, ntfs_mark_rec_free(sbi, ino, false); out2: - __putname(new_de); + kfree(new_de); kfree(rp); out1: @@ -1723,7 +1722,7 @@ int ntfs_link_inode(struct inode *inode, struct dentry *dentry) struct NTFS_DE *de; /* Allocate PATH_MAX bytes. */ - de = kmem_cache_zalloc(names_cachep, GFP_KERNEL); + de = kzalloc(PATH_MAX, GFP_KERNEL); if (!de) return -ENOMEM; @@ -1737,7 +1736,7 @@ int ntfs_link_inode(struct inode *inode, struct dentry *dentry) err = ni_add_name(ntfs_i(d_inode(dentry->d_parent)), ni, de); out: - __putname(de); + kfree(de); return err; } @@ -1760,8 +1759,7 @@ int ntfs_unlink_inode(struct inode *dir, const struct dentry *dentry) if (ntfs_is_meta_file(sbi, ni->mi.rno)) return -EINVAL; - /* Allocate PATH_MAX bytes. */ - de = kmem_cache_zalloc(names_cachep, GFP_KERNEL); + de = kzalloc(PATH_MAX, GFP_KERNEL); if (!de) return -ENOMEM; @@ -1797,7 +1795,7 @@ int ntfs_unlink_inode(struct inode *dir, const struct dentry *dentry) out: ni_unlock(ni); - __putname(de); + kfree(de); return err; } diff --git a/fs/ntfs3/namei.c b/fs/ntfs3/namei.c index 3b24ca02de6143..b2af8f695e60fc 100644 --- a/fs/ntfs3/namei.c +++ b/fs/ntfs3/namei.c @@ -68,7 +68,7 @@ static struct dentry *ntfs_lookup(struct inode *dir, struct dentry *dentry, u32 flags) { struct ntfs_inode *ni = ntfs_i(dir); - struct cpu_str *uni = __getname(); + struct cpu_str *uni = kmalloc(PATH_MAX, GFP_KERNEL); struct inode *inode; int err; @@ -85,7 +85,7 @@ static struct dentry *ntfs_lookup(struct inode *dir, struct dentry *dentry, inode = dir_search_u(dir, uni, NULL); ni_unlock(ni); } - __putname(uni); + kfree(uni); } /* @@ -303,8 +303,7 @@ static int ntfs_rename(struct mnt_idmap *idmap, struct inode *dir, return err; } - /* Allocate PATH_MAX bytes. */ - de = __getname(); + de = kmalloc(PATH_MAX, GFP_KERNEL); if (!de) return -ENOMEM; @@ -349,7 +348,7 @@ static int ntfs_rename(struct mnt_idmap *idmap, struct inode *dir, ni_unlock(ni); ni_unlock(dir_ni); out: - __putname(de); + kfree(de); return err; } @@ -407,7 +406,7 @@ static int ntfs_d_hash(const struct dentry *dentry, struct qstr *name) /* * Try slow way with current upcase table */ - uni = kmem_cache_alloc(names_cachep, GFP_NOWAIT); + uni = kmalloc(PATH_MAX, GFP_NOWAIT); if (!uni) return -ENOMEM; @@ -429,7 +428,7 @@ static int ntfs_d_hash(const struct dentry *dentry, struct qstr *name) err = 0; out: - kmem_cache_free(names_cachep, uni); + kfree(uni); return err; } @@ -468,7 +467,7 @@ static int ntfs_d_compare(const struct dentry *dentry, unsigned int len1, * Try slow way with current upcase table */ sbi = dentry->d_sb->s_fs_info; - uni1 = __getname(); + uni1 = kmalloc(PATH_MAX, GFP_NOWAIT); if (!uni1) return -ENOMEM; @@ -498,7 +497,7 @@ static int ntfs_d_compare(const struct dentry *dentry, unsigned int len1, ret = !ntfs_cmp_names_cpu(uni1, uni2, sbi->upcase, false) ? 0 : 1; out: - __putname(uni1); + kfree(uni1); return ret; } diff --git a/fs/ntfs3/ntfs_fs.h b/fs/ntfs3/ntfs_fs.h index a4559c9f64e686..7b619bb151ce9a 100644 --- a/fs/ntfs3/ntfs_fs.h +++ b/fs/ntfs3/ntfs_fs.h @@ -568,7 +568,7 @@ int ni_write_inode(struct inode *inode, int sync, const char *hint); #define _ni_write_inode(i, w) ni_write_inode(i, w, __func__) int ni_fiemap(struct ntfs_inode *ni, struct fiemap_extent_info *fieinfo, __u64 vbo, __u64 len); -int ni_readpage_cmpr(struct ntfs_inode *ni, struct folio *folio); +int ni_read_folio_cmpr(struct ntfs_inode *ni, struct folio *folio); int ni_decompress_file(struct ntfs_inode *ni); int ni_read_frame(struct ntfs_inode *ni, u64 frame_vbo, struct page **pages, u32 pages_per_frame, int copy); diff --git a/fs/ntfs3/run.c b/fs/ntfs3/run.c index 395b2049252583..dc59cad4fa3764 100644 --- a/fs/ntfs3/run.c +++ b/fs/ntfs3/run.c @@ -1131,11 +1131,14 @@ int run_unpack_ex(struct runs_tree *run, struct ntfs_sb_info *sbi, CLST ino, struct rw_semaphore *lock = is_mounted(sbi) ? &sbi->mft.ni->file.run_lock : NULL; - if (lock) - down_read(lock); - ntfs_refresh_zone(sbi); - if (lock) - up_read(lock); + if (lock) { + if (down_read_trylock(lock)) { + ntfs_refresh_zone(sbi); + up_read(lock); + } + } else { + ntfs_refresh_zone(sbi); + } } up_write(&wnd->rw_lock); if (err) diff --git a/fs/ntfs3/super.c b/fs/ntfs3/super.c index 8b0cf0ed4f72cc..0567a3b224ed3d 100644 --- a/fs/ntfs3/super.c +++ b/fs/ntfs3/super.c @@ -705,9 +705,7 @@ static void ntfs_put_super(struct super_block *sb) ntfs_set_state(sbi, NTFS_DIRTY_CLEAR); if (sbi->options) { - unload_nls(sbi->options->nls); - kfree(sbi->options->nls_name); - kfree(sbi->options); + put_mount_options(sbi->options); sbi->options = NULL; } @@ -1253,7 +1251,6 @@ static int ntfs_fill_super(struct super_block *sb, struct fs_context *fc) } } sbi->options = options; - fc->fs_private = NULL; sb->s_flags |= SB_NODIRATIME; sb->s_magic = 0x7366746e; // "ntfs" sb->s_op = &ntfs_sops; @@ -1679,9 +1676,7 @@ static int ntfs_fill_super(struct super_block *sb, struct fs_context *fc) out: /* sbi->options == options */ if (options) { - unload_nls(options->nls); - kfree(options->nls_name); - kfree(options); + put_mount_options(sbi->options); sbi->options = NULL; } diff --git a/fs/ntfs3/xattr.c b/fs/ntfs3/xattr.c index c93df55e98d079..f3bb2c41c000f1 100644 --- a/fs/ntfs3/xattr.c +++ b/fs/ntfs3/xattr.c @@ -556,8 +556,7 @@ struct posix_acl *ntfs_get_acl(struct mnt_idmap *idmap, struct dentry *dentry, if (unlikely(is_bad_ni(ni))) return ERR_PTR(-EINVAL); - /* Allocate PATH_MAX bytes. */ - buf = __getname(); + buf = kmalloc(PATH_MAX, GFP_KERNEL); if (!buf) return ERR_PTR(-ENOMEM); @@ -588,7 +587,7 @@ struct posix_acl *ntfs_get_acl(struct mnt_idmap *idmap, struct dentry *dentry, if (!IS_ERR(acl)) set_cached_acl(inode, type, acl); - __putname(buf); + kfree(buf); return acl; } diff --git a/fs/ocfs2/aops.c b/fs/ocfs2/aops.c index 76c86f1c2b1ca3..7a65d5a36a3e12 100644 --- a/fs/ocfs2/aops.c +++ b/fs/ocfs2/aops.c @@ -2295,8 +2295,6 @@ static int ocfs2_dio_end_io_write(struct inode *inode, goto out; } - down_write(&oi->ip_alloc_sem); - /* Delete orphan before acquire i_rwsem. */ if (dwc->dw_orphaned) { BUG_ON(dwc->dw_writer_pid != task_pid_nr(current)); @@ -2309,6 +2307,7 @@ static int ocfs2_dio_end_io_write(struct inode *inode, mlog_errno(ret); } + down_write(&oi->ip_alloc_sem); di = (struct ocfs2_dinode *)di_bh->b_data; ocfs2_init_dinode_extent_tree(&et, INODE_CACHE(inode), di_bh); diff --git a/fs/ocfs2/inode.c b/fs/ocfs2/inode.c index b5fcc2725a2962..121560052b71c2 100644 --- a/fs/ocfs2/inode.c +++ b/fs/ocfs2/inode.c @@ -1494,12 +1494,35 @@ int ocfs2_validate_inode_block(struct super_block *sb, goto bail; } - if ((le16_to_cpu(di->i_dyn_features) & OCFS2_INLINE_DATA_FL) && - le32_to_cpu(di->i_clusters)) { - rc = ocfs2_error(sb, "Invalid dinode %llu: %u clusters\n", - (unsigned long long)bh->b_blocknr, - le32_to_cpu(di->i_clusters)); - goto bail; + if (le16_to_cpu(di->i_dyn_features) & OCFS2_INLINE_DATA_FL) { + struct ocfs2_inline_data *data = &di->id2.i_data; + + if (le32_to_cpu(di->i_clusters)) { + rc = ocfs2_error(sb, + "Invalid dinode %llu: %u clusters\n", + (unsigned long long)bh->b_blocknr, + le32_to_cpu(di->i_clusters)); + goto bail; + } + + if (le16_to_cpu(data->id_count) > + ocfs2_max_inline_data_with_xattr(sb, di)) { + rc = ocfs2_error(sb, + "Invalid dinode #%llu: inline data id_count %u exceeds max %d\n", + (unsigned long long)bh->b_blocknr, + le16_to_cpu(data->id_count), + ocfs2_max_inline_data_with_xattr(sb, di)); + goto bail; + } + + if (le64_to_cpu(di->i_size) > le16_to_cpu(data->id_count)) { + rc = ocfs2_error(sb, + "Invalid dinode #%llu: inline data i_size %llu exceeds id_count %u\n", + (unsigned long long)bh->b_blocknr, + (unsigned long long)le64_to_cpu(di->i_size), + le16_to_cpu(data->id_count)); + goto bail; + } } if (le32_to_cpu(di->i_flags) & OCFS2_CHAIN_FL) { diff --git a/fs/ocfs2/mmap.c b/fs/ocfs2/mmap.c index 50e2faf64c1976..6c570157caf16e 100644 --- a/fs/ocfs2/mmap.c +++ b/fs/ocfs2/mmap.c @@ -30,7 +30,8 @@ static vm_fault_t ocfs2_fault(struct vm_fault *vmf) { - struct vm_area_struct *vma = vmf->vma; + unsigned long long ip_blkno = + OCFS2_I(file_inode(vmf->vma->vm_file))->ip_blkno; sigset_t oldset; vm_fault_t ret; @@ -38,11 +39,9 @@ static vm_fault_t ocfs2_fault(struct vm_fault *vmf) ret = filemap_fault(vmf); ocfs2_unblock_signals(&oldset); - trace_ocfs2_fault(OCFS2_I(vma->vm_file->f_mapping->host)->ip_blkno, - vma, vmf->page, vmf->pgoff); + trace_ocfs2_fault(ip_blkno, vmf->page, vmf->pgoff); return ret; } - static vm_fault_t __ocfs2_page_mkwrite(struct file *file, struct buffer_head *di_bh, struct folio *folio) { diff --git a/fs/ocfs2/ocfs2_trace.h b/fs/ocfs2/ocfs2_trace.h index 4b32fb5658ad7a..6c2c97a9804fb0 100644 --- a/fs/ocfs2/ocfs2_trace.h +++ b/fs/ocfs2/ocfs2_trace.h @@ -1246,22 +1246,20 @@ TRACE_EVENT(ocfs2_write_end_inline, TRACE_EVENT(ocfs2_fault, TP_PROTO(unsigned long long ino, - void *area, void *page, unsigned long pgoff), - TP_ARGS(ino, area, page, pgoff), + void *page, unsigned long pgoff), + TP_ARGS(ino, page, pgoff), TP_STRUCT__entry( __field(unsigned long long, ino) - __field(void *, area) __field(void *, page) __field(unsigned long, pgoff) ), TP_fast_assign( __entry->ino = ino; - __entry->area = area; __entry->page = page; __entry->pgoff = pgoff; ), - TP_printk("%llu %p %p %lu", - __entry->ino, __entry->area, __entry->page, __entry->pgoff) + TP_printk("%llu %p %lu", + __entry->ino, __entry->page, __entry->pgoff) ); /* End of trace events for fs/ocfs2/mmap.c. */ diff --git a/fs/ocfs2/resize.c b/fs/ocfs2/resize.c index ac3ec2c2111963..09724e7dc01ba4 100644 --- a/fs/ocfs2/resize.c +++ b/fs/ocfs2/resize.c @@ -303,9 +303,13 @@ int ocfs2_group_extend(struct inode * inode, int new_clusters) fe = (struct ocfs2_dinode *)main_bm_bh->b_data; - /* main_bm_bh is validated by inode read inside ocfs2_inode_lock(), - * so any corruption is a code bug. */ - BUG_ON(!OCFS2_IS_VALID_DINODE(fe)); + /* JBD-managed buffers can bypass validation, so treat this as corruption. */ + if (!OCFS2_IS_VALID_DINODE(fe)) { + ret = ocfs2_error(main_bm_inode->i_sb, + "Invalid dinode #%llu\n", + (unsigned long long)OCFS2_I(main_bm_inode)->ip_blkno); + goto out_unlock; + } if (le16_to_cpu(fe->id2.i_chain.cl_cpg) != ocfs2_group_bitmap_size(osb->sb, 0, diff --git a/fs/ocfs2/xattr.c b/fs/ocfs2/xattr.c index 1b21fbc16d73a2..1bff4f2d1345ed 100644 --- a/fs/ocfs2/xattr.c +++ b/fs/ocfs2/xattr.c @@ -6394,6 +6394,10 @@ static int ocfs2_reflink_xattr_header(handle_t *handle, (void *)last - (void *)xe); memset(last, 0, sizeof(struct ocfs2_xattr_entry)); + last = &new_xh->xh_entries[le16_to_cpu(new_xh->xh_count)] - 1; + } else { + memset(xe, 0, sizeof(struct ocfs2_xattr_entry)); + last = NULL; } /* diff --git a/fs/overlayfs/copy_up.c b/fs/overlayfs/copy_up.c index 758611ee4475f0..13cb60b52bd6e3 100644 --- a/fs/overlayfs/copy_up.c +++ b/fs/overlayfs/copy_up.c @@ -1146,15 +1146,15 @@ static int ovl_copy_up_one(struct dentry *parent, struct dentry *dentry, return -EOVERFLOW; /* - * With metacopy disabled, we fsync after final metadata copyup, for + * With "fsync=strict", we fsync after final metadata copyup, for * both regular files and directories to get atomic copyup semantics * on filesystems that do not use strict metadata ordering (e.g. ubifs). * - * With metacopy enabled we want to avoid fsync on all meta copyup + * By default, we want to avoid fsync on all meta copyup, because * that will hurt performance of workloads such as chown -R, so we * only fsync on data copyup as legacy behavior. */ - ctx.metadata_fsync = !OVL_FS(dentry->d_sb)->config.metacopy && + ctx.metadata_fsync = ovl_should_sync_metadata(OVL_FS(dentry->d_sb)) && (S_ISREG(ctx.stat.mode) || S_ISDIR(ctx.stat.mode)); ctx.metacopy = ovl_need_meta_copy_up(dentry, ctx.stat.mode, flags); diff --git a/fs/overlayfs/overlayfs.h b/fs/overlayfs/overlayfs.h index f9ac9bdde83059..d0708a6630c11b 100644 --- a/fs/overlayfs/overlayfs.h +++ b/fs/overlayfs/overlayfs.h @@ -99,6 +99,12 @@ enum { OVL_VERITY_REQUIRE, }; +enum { + OVL_FSYNC_VOLATILE, + OVL_FSYNC_AUTO, + OVL_FSYNC_STRICT, +}; + /* * The tuple (fh,uuid) is a universal unique identifier for a copy up origin, * where: @@ -656,6 +662,21 @@ static inline bool ovl_xino_warn(struct ovl_fs *ofs) return ofs->config.xino == OVL_XINO_ON; } +static inline bool ovl_should_sync(struct ovl_fs *ofs) +{ + return ofs->config.fsync_mode != OVL_FSYNC_VOLATILE; +} + +static inline bool ovl_should_sync_metadata(struct ovl_fs *ofs) +{ + return ofs->config.fsync_mode == OVL_FSYNC_STRICT; +} + +static inline bool ovl_is_volatile(struct ovl_config *config) +{ + return config->fsync_mode == OVL_FSYNC_VOLATILE; +} + /* * To avoid regressions in existing setups with overlay lower offline changes, * we allow lower changes only if none of the new features are used. diff --git a/fs/overlayfs/ovl_entry.h b/fs/overlayfs/ovl_entry.h index 1d4828dbcf7ac4..80cad4ea96a3ed 100644 --- a/fs/overlayfs/ovl_entry.h +++ b/fs/overlayfs/ovl_entry.h @@ -18,7 +18,7 @@ struct ovl_config { int xino; bool metacopy; bool userxattr; - bool ovl_volatile; + int fsync_mode; }; struct ovl_sb { @@ -120,11 +120,6 @@ static inline struct ovl_fs *OVL_FS(struct super_block *sb) return (struct ovl_fs *)sb->s_fs_info; } -static inline bool ovl_should_sync(struct ovl_fs *ofs) -{ - return !ofs->config.ovl_volatile; -} - static inline unsigned int ovl_numlower(struct ovl_entry *oe) { return oe ? oe->__numlower : 0; diff --git a/fs/overlayfs/params.c b/fs/overlayfs/params.c index 63b7346c5ee1c1..7d8bc6d3863c1a 100644 --- a/fs/overlayfs/params.c +++ b/fs/overlayfs/params.c @@ -58,6 +58,7 @@ enum ovl_opt { Opt_xino, Opt_metacopy, Opt_verity, + Opt_fsync, Opt_volatile, Opt_override_creds, }; @@ -140,6 +141,23 @@ static int ovl_verity_mode_def(void) return OVL_VERITY_OFF; } +static const struct constant_table ovl_parameter_fsync[] = { + { "volatile", OVL_FSYNC_VOLATILE }, + { "auto", OVL_FSYNC_AUTO }, + { "strict", OVL_FSYNC_STRICT }, + {} +}; + +static const char *ovl_fsync_mode(struct ovl_config *config) +{ + return ovl_parameter_fsync[config->fsync_mode].name; +} + +static int ovl_fsync_mode_def(void) +{ + return OVL_FSYNC_AUTO; +} + const struct fs_parameter_spec ovl_parameter_spec[] = { fsparam_string_empty("lowerdir", Opt_lowerdir), fsparam_file_or_string("lowerdir+", Opt_lowerdir_add), @@ -155,6 +173,7 @@ const struct fs_parameter_spec ovl_parameter_spec[] = { fsparam_enum("xino", Opt_xino, ovl_parameter_xino), fsparam_enum("metacopy", Opt_metacopy, ovl_parameter_bool), fsparam_enum("verity", Opt_verity, ovl_parameter_verity), + fsparam_enum("fsync", Opt_fsync, ovl_parameter_fsync), fsparam_flag("volatile", Opt_volatile), fsparam_flag_no("override_creds", Opt_override_creds), {} @@ -665,8 +684,11 @@ static int ovl_parse_param(struct fs_context *fc, struct fs_parameter *param) case Opt_verity: config->verity_mode = result.uint_32; break; + case Opt_fsync: + config->fsync_mode = result.uint_32; + break; case Opt_volatile: - config->ovl_volatile = true; + config->fsync_mode = OVL_FSYNC_VOLATILE; break; case Opt_userxattr: config->userxattr = true; @@ -800,6 +822,7 @@ int ovl_init_fs_context(struct fs_context *fc) ofs->config.nfs_export = ovl_nfs_export_def; ofs->config.xino = ovl_xino_def(); ofs->config.metacopy = ovl_metacopy_def; + ofs->config.fsync_mode = ovl_fsync_mode_def(); fc->s_fs_info = ofs; fc->fs_private = ctx; @@ -870,9 +893,9 @@ int ovl_fs_params_verify(const struct ovl_fs_context *ctx, config->index = false; } - if (!config->upperdir && config->ovl_volatile) { + if (!config->upperdir && ovl_is_volatile(config)) { pr_info("option \"volatile\" is meaningless in a non-upper mount, ignoring it.\n"); - config->ovl_volatile = false; + config->fsync_mode = ovl_fsync_mode_def(); } if (!config->upperdir && config->uuid == OVL_UUID_ON) { @@ -1070,8 +1093,8 @@ int ovl_show_options(struct seq_file *m, struct dentry *dentry) seq_printf(m, ",xino=%s", ovl_xino_mode(&ofs->config)); if (ofs->config.metacopy != ovl_metacopy_def) seq_printf(m, ",metacopy=%s", str_on_off(ofs->config.metacopy)); - if (ofs->config.ovl_volatile) - seq_puts(m, ",volatile"); + if (ofs->config.fsync_mode != ovl_fsync_mode_def()) + seq_printf(m, ",fsync=%s", ovl_fsync_mode(&ofs->config)); if (ofs->config.userxattr) seq_puts(m, ",userxattr"); if (ofs->config.verity_mode != ovl_verity_mode_def()) diff --git a/fs/overlayfs/readdir.c b/fs/overlayfs/readdir.c index 160960bb0ad0b0..724ec9d93fc829 100644 --- a/fs/overlayfs/readdir.c +++ b/fs/overlayfs/readdir.c @@ -755,7 +755,7 @@ static bool ovl_fill_real(struct dir_context *ctx, const char *name, struct dir_context *orig_ctx = rdt->orig_ctx; bool res; - if (rdt->parent_ino && strcmp(name, "..") == 0) { + if (rdt->parent_ino && namelen == 2 && !strncmp(name, "..", 2)) { ino = rdt->parent_ino; } else if (rdt->cache) { struct ovl_cache_entry *p; diff --git a/fs/overlayfs/super.c b/fs/overlayfs/super.c index ba9146f22a2cc3..18804cda21c482 100644 --- a/fs/overlayfs/super.c +++ b/fs/overlayfs/super.c @@ -776,7 +776,7 @@ static int ovl_make_workdir(struct super_block *sb, struct ovl_fs *ofs, * For volatile mount, create a incompat/volatile/dirty file to keep * track of it. */ - if (ofs->config.ovl_volatile) { + if (ovl_is_volatile(&ofs->config)) { err = ovl_create_volatile_dirty(ofs); if (err < 0) { pr_err("Failed to create volatile/dirty file.\n"); diff --git a/fs/overlayfs/util.c b/fs/overlayfs/util.c index 94986d11a166c6..bfd8d7a6f1e781 100644 --- a/fs/overlayfs/util.c +++ b/fs/overlayfs/util.c @@ -85,7 +85,10 @@ int ovl_can_decode_fh(struct super_block *sb) if (!exportfs_can_decode_fh(sb->s_export_op)) return 0; - return sb->s_export_op->encode_fh ? -1 : FILEID_INO32_GEN; + if (sb->s_export_op->encode_fh == generic_encode_ino32_fh) + return FILEID_INO32_GEN; + + return -1; } struct dentry *ovl_indexdir(struct super_block *sb) diff --git a/fs/pidfs.c b/fs/pidfs.c index 1e20e36e0ed559..d18c51513f6c51 100644 --- a/fs/pidfs.c +++ b/fs/pidfs.c @@ -329,7 +329,7 @@ static long pidfd_info(struct file *file, unsigned int cmd, unsigned long arg) * namespace hierarchy. */ if (!pid_in_current_pidns(pid)) - return -ESRCH; + return -EREMOTE; attr = READ_ONCE(pid->attr); if (mask & PIDFD_INFO_EXIT) { diff --git a/fs/proc/array.c b/fs/proc/array.c index 42932f88141a97..5571177e0435d0 100644 --- a/fs/proc/array.c +++ b/fs/proc/array.c @@ -528,7 +528,7 @@ static int do_task_stat(struct seq_file *m, struct pid_namespace *ns, } sid = task_session_nr_ns(task, ns); - ppid = task_tgid_nr_ns(task->real_parent, ns); + ppid = task_ppid_nr_ns(task, ns); pgid = task_pgrp_nr_ns(task, ns); unlock_task_sighand(task, &flags); diff --git a/fs/proc/base.c b/fs/proc/base.c index 4eec684baca9f7..4c863d17dfb4c7 100644 --- a/fs/proc/base.c +++ b/fs/proc/base.c @@ -2128,6 +2128,9 @@ bool proc_fill_cache(struct file *file, struct dir_context *ctx, ino_t ino = 1; child = try_lookup_noperm(&qname, dir); + if (IS_ERR(child)) + goto end_instantiate; + if (!child) { DECLARE_WAIT_QUEUE_HEAD_ONSTACK(wq); child = d_alloc_parallel(dir, &qname, &wq); diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c index 26188a4ad1abd9..2f55efc368162d 100644 --- a/fs/proc/task_mmu.c +++ b/fs/proc/task_mmu.c @@ -780,7 +780,7 @@ static int do_procmap_query(struct mm_struct *mm, void __user *uarg) } else { if (karg.build_id_size < build_id_sz) { err = -ENAMETOOLONG; - goto out; + goto out_file; } karg.build_id_size = build_id_sz; } @@ -808,6 +808,7 @@ static int do_procmap_query(struct mm_struct *mm, void __user *uarg) out: query_vma_teardown(&lock_ctx); mmput(mm); +out_file: if (vm_file) fput(vm_file); kfree(name_buf); diff --git a/fs/pstore/ram_core.c b/fs/pstore/ram_core.c index f1848cdd6d3485..7b6d6378a3b87d 100644 --- a/fs/pstore/ram_core.c +++ b/fs/pstore/ram_core.c @@ -298,6 +298,17 @@ void persistent_ram_save_old(struct persistent_ram_zone *prz) if (!size) return; + /* + * If the existing buffer is differently sized, free it so a new + * one is allocated. This can happen when persistent_ram_save_old() + * is called early in boot and later for a timer-triggered + * survivable crash when the crash dumps don't match in size + * (which would be extremely unlikely given kmsg buffers usually + * exceed prz buffer sizes). + */ + if (prz->old_log && prz->old_log_size != size) + persistent_ram_free_old(prz); + if (!prz->old_log) { persistent_ram_ecc_old(prz); prz->old_log = kvzalloc(size, GFP_KERNEL); @@ -446,6 +457,13 @@ static void *persistent_ram_vmap(phys_addr_t start, size_t size, vaddr = vmap(pages, page_count, VM_MAP | VM_IOREMAP, prot); kfree(pages); + /* + * vmap() may fail and return NULL. Do not add the offset in this + * case, otherwise a NULL mapping would appear successful. + */ + if (!vaddr) + return NULL; + /* * Since vmap() uses page granularity, we must add the offset * into the page here, to get the byte granularity address diff --git a/fs/quota/quota.c b/fs/quota/quota.c index 7c2b75a4448528..de4379a9c79208 100644 --- a/fs/quota/quota.c +++ b/fs/quota/quota.c @@ -899,6 +899,7 @@ static struct super_block *quotactl_block(const char __user *special, int cmd) sb_start_write(sb); sb_end_write(sb); put_super(sb); + cond_resched(); goto retry; } return sb; diff --git a/fs/smb/client/cached_dir.c b/fs/smb/client/cached_dir.c index 1db7ab6c2529ce..569030b3e68d4a 100644 --- a/fs/smb/client/cached_dir.c +++ b/fs/smb/client/cached_dir.c @@ -788,11 +788,11 @@ static void cfids_laundromat_worker(struct work_struct *work) cfid->dentry = NULL; if (cfid->is_open) { - spin_lock(&cifs_tcp_ses_lock); + spin_lock(&cfid->tcon->tc_lock); ++cfid->tcon->tc_count; trace_smb3_tcon_ref(cfid->tcon->debug_id, cfid->tcon->tc_count, netfs_trace_tcon_ref_get_cached_laundromat); - spin_unlock(&cifs_tcp_ses_lock); + spin_unlock(&cfid->tcon->tc_lock); queue_work(serverclose_wq, &cfid->close_work); } else /* diff --git a/fs/smb/client/cached_dir.h b/fs/smb/client/cached_dir.h index 1e383db7c33743..5091bf45345e81 100644 --- a/fs/smb/client/cached_dir.h +++ b/fs/smb/client/cached_dir.h @@ -36,10 +36,10 @@ struct cached_fid { struct list_head entry; struct cached_fids *cfids; const char *path; - bool has_lease:1; - bool is_open:1; - bool on_list:1; - bool file_all_info_is_valid:1; + bool has_lease; + bool is_open; + bool on_list; + bool file_all_info_is_valid; unsigned long time; /* jiffies of when lease was taken */ unsigned long last_access_time; /* jiffies of when last accessed */ struct kref refcount; diff --git a/fs/smb/client/cifsfs.c b/fs/smb/client/cifsfs.c index a3dc7cb1ab541d..ce077a62da7973 100644 --- a/fs/smb/client/cifsfs.c +++ b/fs/smb/client/cifsfs.c @@ -331,10 +331,14 @@ static void cifs_kill_sb(struct super_block *sb) /* * We need to release all dentries for the cached directories - * before we kill the sb. + * and close all deferred file handles before we kill the sb. */ if (cifs_sb->root) { close_all_cached_dirs(cifs_sb); + cifs_close_all_deferred_files_sb(cifs_sb); + + /* Wait for all pending oplock breaks to complete */ + flush_workqueue(cifsoplockd_wq); /* finally release root dentry */ dput(cifs_sb->root); @@ -865,7 +869,6 @@ static void cifs_umount_begin(struct super_block *sb) spin_unlock(&tcon->tc_lock); spin_unlock(&cifs_tcp_ses_lock); - cifs_close_all_deferred_files(tcon); /* cancel_brl_requests(tcon); */ /* BB mark all brl mids as exiting */ /* cancel_notify_requests(tcon); */ if (tcon->ses && tcon->ses->server) { diff --git a/fs/smb/client/cifsglob.h b/fs/smb/client/cifsglob.h index 3eca5bfb703030..474d7b2aa2ef50 100644 --- a/fs/smb/client/cifsglob.h +++ b/fs/smb/client/cifsglob.h @@ -20,6 +20,7 @@ #include #include #include +#include #include "cifs_fs_sb.h" #include "cifsacl.h" #include @@ -2313,4 +2314,20 @@ static inline void cifs_requeue_server_reconn(struct TCP_Server_Info *server) queue_delayed_work(cifsiod_wq, &server->reconnect, delay * HZ); } +static inline int cifs_open_create_options(unsigned int oflags, int opts) +{ + /* O_SYNC also has bit for O_DSYNC so following check picks up either */ + if (oflags & O_SYNC) + opts |= CREATE_WRITE_THROUGH; + if (oflags & O_DIRECT) + opts |= CREATE_NO_BUFFER; + return opts; +} + +/* + * The number of blocks is not related to (i_size / i_blksize), but instead + * 512 byte (2**9) size is required for calculating num blocks. + */ +#define CIFS_INO_BLOCKS(size) DIV_ROUND_UP_ULL((u64)(size), 512) + #endif /* _CIFS_GLOB_H */ diff --git a/fs/smb/client/cifsproto.h b/fs/smb/client/cifsproto.h index f8c0615d4ee422..043cea4ff77585 100644 --- a/fs/smb/client/cifsproto.h +++ b/fs/smb/client/cifsproto.h @@ -302,6 +302,7 @@ extern void cifs_close_deferred_file(struct cifsInodeInfo *cifs_inode); extern void cifs_close_all_deferred_files(struct cifs_tcon *cifs_tcon); +void cifs_close_all_deferred_files_sb(struct cifs_sb_info *cifs_sb); void cifs_close_deferred_file_under_dentry(struct cifs_tcon *cifs_tcon, struct dentry *dentry); diff --git a/fs/smb/client/connect.c b/fs/smb/client/connect.c index ce620503e9f702..97f7ef655120d9 100644 --- a/fs/smb/client/connect.c +++ b/fs/smb/client/connect.c @@ -1952,6 +1952,10 @@ static int match_session(struct cifs_ses *ses, case Kerberos: if (!uid_eq(ctx->cred_uid, ses->cred_uid)) return 0; + if (strncmp(ses->user_name ?: "", + ctx->username ?: "", + CIFS_MAX_USERNAME_LEN)) + return 0; break; case NTLMv2: case RawNTLMSSP: @@ -2233,7 +2237,6 @@ cifs_set_cifscreds(struct smb3_fs_context *ctx, struct cifs_ses *ses) /* find first : in payload */ payload = upayload->data; delim = strnchr(payload, upayload->datalen, ':'); - cifs_dbg(FYI, "payload=%s\n", payload); if (!delim) { cifs_dbg(FYI, "Unable to find ':' in payload (datalen=%d)\n", upayload->datalen); @@ -4270,7 +4273,9 @@ cifs_setup_session(const unsigned int xid, struct cifs_ses *ses, ses->ses_status = SES_IN_SETUP; /* force iface_list refresh */ + spin_lock(&ses->iface_lock); ses->iface_last_update = 0; + spin_unlock(&ses->iface_lock); } spin_unlock(&ses->ses_lock); diff --git a/fs/smb/client/dir.c b/fs/smb/client/dir.c index 747256025e49bd..462d8e751ecfae 100644 --- a/fs/smb/client/dir.c +++ b/fs/smb/client/dir.c @@ -307,6 +307,7 @@ static int cifs_do_create(struct inode *inode, struct dentry *direntry, unsigned goto out; } + create_options |= cifs_open_create_options(oflags, create_options); /* * if we're not using unix extensions, see if we need to set * ATTR_READONLY on the create call diff --git a/fs/smb/client/file.c b/fs/smb/client/file.c index 7ff5cc9c5c5b74..9d703a24745096 100644 --- a/fs/smb/client/file.c +++ b/fs/smb/client/file.c @@ -585,15 +585,8 @@ static int cifs_nt_open(const char *full_path, struct inode *inode, struct cifs_ *********************************************************************/ disposition = cifs_get_disposition(f_flags); - /* BB pass O_SYNC flag through on file attributes .. BB */ - - /* O_SYNC also has bit for O_DSYNC so following check picks up either */ - if (f_flags & O_SYNC) - create_options |= CREATE_WRITE_THROUGH; - - if (f_flags & O_DIRECT) - create_options |= CREATE_NO_BUFFER; + create_options |= cifs_open_create_options(f_flags, create_options); retry_open: oparms = (struct cifs_open_parms) { @@ -712,8 +705,6 @@ struct cifsFileInfo *cifs_new_fileinfo(struct cifs_fid *fid, struct file *file, mutex_init(&cfile->fh_mutex); spin_lock_init(&cfile->file_info_lock); - cifs_sb_active(inode->i_sb); - /* * If the server returned a read oplock and we have mandatory brlocks, * set oplock level to None. @@ -768,7 +759,6 @@ static void cifsFileInfo_put_final(struct cifsFileInfo *cifs_file) struct inode *inode = d_inode(cifs_file->dentry); struct cifsInodeInfo *cifsi = CIFS_I(inode); struct cifsLockInfo *li, *tmp; - struct super_block *sb = inode->i_sb; /* * Delete any outstanding lock records. We'll lose them when the file @@ -786,7 +776,6 @@ static void cifsFileInfo_put_final(struct cifsFileInfo *cifs_file) cifs_put_tlink(cifs_file->tlink); dput(cifs_file->dentry); - cifs_sb_deactive(sb); kfree(cifs_file->symlink_target); kfree(cifs_file); } @@ -1005,7 +994,6 @@ static int cifs_do_truncate(const unsigned int xid, struct dentry *dentry) if (!rc) { netfs_resize_file(&cinode->netfs, 0, true); cifs_setsize(inode, 0); - inode->i_blocks = 0; } } if (cfile) @@ -1319,13 +1307,8 @@ cifs_reopen_file(struct cifsFileInfo *cfile, bool can_flush) rdwr_for_fscache = 1; desired_access = cifs_convert_flags(cfile->f_flags, rdwr_for_fscache); - - /* O_SYNC also has bit for O_DSYNC so following check picks up either */ - if (cfile->f_flags & O_SYNC) - create_options |= CREATE_WRITE_THROUGH; - - if (cfile->f_flags & O_DIRECT) - create_options |= CREATE_NO_BUFFER; + create_options |= cifs_open_create_options(cfile->f_flags, + create_options); if (server->ops->get_lease_key) server->ops->get_lease_key(inode, &cfile->fid); @@ -3163,12 +3146,6 @@ void cifs_oplock_break(struct work_struct *work) __u64 persistent_fid, volatile_fid; __u16 net_fid; - /* - * Hold a reference to the superblock to prevent it and its inodes from - * being freed while we are accessing cinode. Otherwise, _cifsFileInfo_put() - * may release the last reference to the sb and trigger inode eviction. - */ - cifs_sb_active(sb); wait_on_bit(&cinode->flags, CIFS_INODE_PENDING_WRITERS, TASK_UNINTERRUPTIBLE); @@ -3241,7 +3218,6 @@ void cifs_oplock_break(struct work_struct *work) cifs_put_tlink(tlink); out: cifs_done_oplock_break(cinode); - cifs_sb_deactive(sb); } static int cifs_swap_activate(struct swap_info_struct *sis, diff --git a/fs/smb/client/fs_context.c b/fs/smb/client/fs_context.c index d4291d3a9a4858..e61bb6ac1d1115 100644 --- a/fs/smb/client/fs_context.c +++ b/fs/smb/client/fs_context.c @@ -589,6 +589,10 @@ char *cifs_sanitize_prepath(char *prepath, gfp_t gfp) while (IS_DELIM(*cursor1)) cursor1++; + /* exit in case of only delimiters */ + if (!*cursor1) + return NULL; + /* copy the first letter */ *cursor2 = *cursor1; @@ -826,9 +830,7 @@ static int smb3_fs_context_parse_monolithic(struct fs_context *fc, if (ret < 0) break; } - ret = smb3_handle_conflicting_options(fc); - - return ret; + return ret ?: smb3_handle_conflicting_options(fc); } /* @@ -2000,7 +2002,7 @@ int smb3_init_fs_context(struct fs_context *fc) ctx->backupuid_specified = false; /* no backup intent for a user */ ctx->backupgid_specified = false; /* no backup intent for a group */ - ctx->retrans = 1; + ctx->retrans = 0; ctx->reparse_type = CIFS_REPARSE_TYPE_DEFAULT; ctx->symlink_type = CIFS_SYMLINK_TYPE_DEFAULT; ctx->nonativesocket = 0; diff --git a/fs/smb/client/inode.c b/fs/smb/client/inode.c index f9ee95953fa4a5..c5d89ddc87c00d 100644 --- a/fs/smb/client/inode.c +++ b/fs/smb/client/inode.c @@ -219,13 +219,7 @@ cifs_fattr_to_inode(struct inode *inode, struct cifs_fattr *fattr, */ if (is_size_safe_to_change(cifs_i, fattr->cf_eof, from_readdir)) { i_size_write(inode, fattr->cf_eof); - - /* - * i_blocks is not related to (i_size / i_blksize), - * but instead 512 byte (2**9) size is required for - * calculating num blocks. - */ - inode->i_blocks = (512 - 1 + fattr->cf_bytes) >> 9; + inode->i_blocks = CIFS_INO_BLOCKS(fattr->cf_bytes); } if (S_ISLNK(fattr->cf_mode) && fattr->cf_symlink_target) { @@ -3009,6 +3003,11 @@ void cifs_setsize(struct inode *inode, loff_t offset) { spin_lock(&inode->i_lock); i_size_write(inode, offset); + /* + * Until we can query the server for actual allocation size, + * this is best estimate we have for blocks allocated for a file. + */ + inode->i_blocks = CIFS_INO_BLOCKS(offset); spin_unlock(&inode->i_lock); inode_set_mtime_to_ts(inode, inode_set_ctime_current(inode)); truncate_pagecache(inode, offset); @@ -3081,14 +3080,6 @@ int cifs_file_set_size(const unsigned int xid, struct dentry *dentry, if (rc == 0) { netfs_resize_file(&cifsInode->netfs, size, true); cifs_setsize(inode, size); - /* - * i_blocks is not related to (i_size / i_blksize), but instead - * 512 byte (2**9) size is required for calculating num blocks. - * Until we can query the server for actual allocation size, - * this is best estimate we have for blocks allocated for a file - * Number of blocks must be rounded up so size 1 is not 0 blocks - */ - inode->i_blocks = (512 - 1 + size) >> 9; } return rc; diff --git a/fs/smb/client/misc.c b/fs/smb/client/misc.c index 9529fa385938e0..c841cfab104606 100644 --- a/fs/smb/client/misc.c +++ b/fs/smb/client/misc.c @@ -28,6 +28,11 @@ #include "fs_context.h" #include "cached_dir.h" +struct tcon_list { + struct list_head entry; + struct cifs_tcon *tcon; +}; + /* The xid serves as a useful identifier for each incoming vfs request, in a similar way to the mid which is useful to track each sent smb, and CurrentXid can also provide a running counter (although it @@ -840,6 +845,43 @@ cifs_close_all_deferred_files(struct cifs_tcon *tcon) } } +void cifs_close_all_deferred_files_sb(struct cifs_sb_info *cifs_sb) +{ + struct rb_root *root = &cifs_sb->tlink_tree; + struct rb_node *node; + struct cifs_tcon *tcon; + struct tcon_link *tlink; + struct tcon_list *tmp_list, *q; + LIST_HEAD(tcon_head); + + spin_lock(&cifs_sb->tlink_tree_lock); + for (node = rb_first(root); node; node = rb_next(node)) { + tlink = rb_entry(node, struct tcon_link, tl_rbnode); + tcon = tlink_tcon(tlink); + if (IS_ERR(tcon)) + continue; + tmp_list = kmalloc(sizeof(struct tcon_list), GFP_ATOMIC); + if (tmp_list == NULL) + break; + tmp_list->tcon = tcon; + /* Take a reference on tcon to prevent it from being freed */ + spin_lock(&tcon->tc_lock); + ++tcon->tc_count; + trace_smb3_tcon_ref(tcon->debug_id, tcon->tc_count, + netfs_trace_tcon_ref_get_close_defer_files); + spin_unlock(&tcon->tc_lock); + list_add_tail(&tmp_list->entry, &tcon_head); + } + spin_unlock(&cifs_sb->tlink_tree_lock); + + list_for_each_entry_safe(tmp_list, q, &tcon_head, entry) { + cifs_close_all_deferred_files(tmp_list->tcon); + list_del(&tmp_list->entry); + cifs_put_tcon(tmp_list->tcon, netfs_trace_tcon_ref_put_close_defer_files); + kfree(tmp_list); + } +} + void cifs_close_deferred_file_under_dentry(struct cifs_tcon *tcon, struct dentry *dentry) { diff --git a/fs/smb/client/smb2file.c b/fs/smb/client/smb2file.c index 2dd08388ea8733..b2ddcecd00b96e 100644 --- a/fs/smb/client/smb2file.c +++ b/fs/smb/client/smb2file.c @@ -27,10 +27,11 @@ static struct smb2_symlink_err_rsp *symlink_data(const struct kvec *iov) { struct smb2_err_rsp *err = iov->iov_base; struct smb2_symlink_err_rsp *sym = ERR_PTR(-EINVAL); + u8 *end = (u8 *)err + iov->iov_len; u32 len; if (err->ErrorContextCount) { - struct smb2_error_context_rsp *p, *end; + struct smb2_error_context_rsp *p; len = (u32)err->ErrorContextCount * (offsetof(struct smb2_error_context_rsp, ErrorContextData) + @@ -39,8 +40,7 @@ static struct smb2_symlink_err_rsp *symlink_data(const struct kvec *iov) return ERR_PTR(-EINVAL); p = (struct smb2_error_context_rsp *)err->ErrorData; - end = (struct smb2_error_context_rsp *)((u8 *)err + iov->iov_len); - do { + while ((u8 *)p + sizeof(*p) <= end) { if (le32_to_cpu(p->ErrorId) == SMB2_ERROR_ID_DEFAULT) { sym = (struct smb2_symlink_err_rsp *)p->ErrorContextData; break; @@ -50,14 +50,16 @@ static struct smb2_symlink_err_rsp *symlink_data(const struct kvec *iov) len = ALIGN(le32_to_cpu(p->ErrorDataLength), 8); p = (struct smb2_error_context_rsp *)(p->ErrorContextData + len); - } while (p < end); + } } else if (le32_to_cpu(err->ByteCount) >= sizeof(*sym) && iov->iov_len >= SMB2_SYMLINK_STRUCT_SIZE) { sym = (struct smb2_symlink_err_rsp *)err->ErrorData; } - if (!IS_ERR(sym) && (le32_to_cpu(sym->SymLinkErrorTag) != SYMLINK_ERROR_TAG || - le32_to_cpu(sym->ReparseTag) != IO_REPARSE_TAG_SYMLINK)) + if (!IS_ERR(sym) && + ((u8 *)sym + sizeof(*sym) > end || + le32_to_cpu(sym->SymLinkErrorTag) != SYMLINK_ERROR_TAG || + le32_to_cpu(sym->ReparseTag) != IO_REPARSE_TAG_SYMLINK)) sym = ERR_PTR(-EINVAL); return sym; @@ -128,8 +130,10 @@ int smb2_parse_symlink_response(struct cifs_sb_info *cifs_sb, const struct kvec print_len = le16_to_cpu(sym->PrintNameLength); print_offs = le16_to_cpu(sym->PrintNameOffset); - if (iov->iov_len < SMB2_SYMLINK_STRUCT_SIZE + sub_offs + sub_len || - iov->iov_len < SMB2_SYMLINK_STRUCT_SIZE + print_offs + print_len) + if ((char *)sym->PathBuffer + sub_offs + sub_len > + (char *)iov->iov_base + iov->iov_len || + (char *)sym->PathBuffer + print_offs + print_len > + (char *)iov->iov_base + iov->iov_len) return -EINVAL; return smb2_parse_native_symlink(path, @@ -179,6 +183,8 @@ int smb2_open_file(const unsigned int xid, struct cifs_open_parms *oparms, &err_buftype); if (rc == -EACCES && retry_without_read_attributes) { free_rsp_buf(err_buftype, err_iov.iov_base); + memset(&err_iov, 0, sizeof(err_iov)); + err_buftype = CIFS_NO_BUFFER; oparms->desired_access &= ~FILE_READ_ATTRIBUTES; rc = SMB2_open(xid, oparms, smb2_path, &smb2_oplock, smb2_data, NULL, &err_iov, &err_buftype); diff --git a/fs/smb/client/smb2inode.c b/fs/smb/client/smb2inode.c index 2ded3246600c01..ed378fbe537566 100644 --- a/fs/smb/client/smb2inode.c +++ b/fs/smb/client/smb2inode.c @@ -128,7 +128,7 @@ static int check_wsl_eas(struct kvec *rsp_iov) nlen = ea->ea_name_length; vlen = le16_to_cpu(ea->ea_value_length); if (nlen != SMB2_WSL_XATTR_NAME_LEN || - (u8 *)ea + nlen + 1 + vlen > end) + (u8 *)ea->ea_data + nlen + 1 + vlen > end) return -EINVAL; switch (vlen) { @@ -325,7 +325,7 @@ static int smb2_compound_op(const unsigned int xid, struct cifs_tcon *tcon, cfile->fid.volatile_fid, SMB_FIND_FILE_POSIX_INFO, SMB2_O_INFO_FILE, 0, - sizeof(struct smb311_posix_qinfo *) + + sizeof(struct smb311_posix_qinfo) + (PATH_MAX * 2) + (sizeof(struct smb_sid) * 2), 0, NULL); } else { @@ -335,7 +335,7 @@ static int smb2_compound_op(const unsigned int xid, struct cifs_tcon *tcon, COMPOUND_FID, SMB_FIND_FILE_POSIX_INFO, SMB2_O_INFO_FILE, 0, - sizeof(struct smb311_posix_qinfo *) + + sizeof(struct smb311_posix_qinfo) + (PATH_MAX * 2) + (sizeof(struct smb_sid) * 2), 0, NULL); } @@ -1208,6 +1208,7 @@ smb2_unlink(const unsigned int xid, struct cifs_tcon *tcon, const char *name, memset(resp_buftype, 0, sizeof(resp_buftype)); memset(rsp_iov, 0, sizeof(rsp_iov)); + memset(open_iov, 0, sizeof(open_iov)); rqst[0].rq_iov = open_iov; rqst[0].rq_nvec = ARRAY_SIZE(open_iov); @@ -1232,14 +1233,15 @@ smb2_unlink(const unsigned int xid, struct cifs_tcon *tcon, const char *name, creq = rqst[0].rq_iov[0].iov_base; creq->ShareAccess = FILE_SHARE_DELETE_LE; + memset(&close_iov, 0, sizeof(close_iov)); rqst[1].rq_iov = &close_iov; rqst[1].rq_nvec = 1; rc = SMB2_close_init(tcon, server, &rqst[1], COMPOUND_FID, COMPOUND_FID, false); - smb2_set_related(&rqst[1]); if (rc) goto err_free; + smb2_set_related(&rqst[1]); if (retries) { for (int i = 0; i < ARRAY_SIZE(rqst); i++) diff --git a/fs/smb/client/smb2misc.c b/fs/smb/client/smb2misc.c index f3cb62d914502f..0871b9f1f86a6f 100644 --- a/fs/smb/client/smb2misc.c +++ b/fs/smb/client/smb2misc.c @@ -820,14 +820,14 @@ smb2_handle_cancelled_close(struct cifs_tcon *tcon, __u64 persistent_fid, int rc; cifs_dbg(FYI, "%s: tc_count=%d\n", __func__, tcon->tc_count); - spin_lock(&cifs_tcp_ses_lock); + spin_lock(&tcon->tc_lock); if (tcon->tc_count <= 0) { struct TCP_Server_Info *server = NULL; trace_smb3_tcon_ref(tcon->debug_id, tcon->tc_count, netfs_trace_tcon_ref_see_cancelled_close); WARN_ONCE(tcon->tc_count < 0, "tcon refcount is negative"); - spin_unlock(&cifs_tcp_ses_lock); + spin_unlock(&tcon->tc_lock); if (tcon->ses) { server = tcon->ses->server; @@ -841,7 +841,7 @@ smb2_handle_cancelled_close(struct cifs_tcon *tcon, __u64 persistent_fid, tcon->tc_count++; trace_smb3_tcon_ref(tcon->debug_id, tcon->tc_count, netfs_trace_tcon_ref_get_cancelled_close); - spin_unlock(&cifs_tcp_ses_lock); + spin_unlock(&tcon->tc_lock); rc = __smb2_handle_cancelled_cmd(tcon, SMB2_CLOSE_HE, 0, persistent_fid, volatile_fid); diff --git a/fs/smb/client/smb2ops.c b/fs/smb/client/smb2ops.c index c1aaf77e187b6d..067e3132832912 100644 --- a/fs/smb/client/smb2ops.c +++ b/fs/smb/client/smb2ops.c @@ -628,6 +628,7 @@ parse_server_interfaces(struct network_interface_info_ioctl_rsp *buf, struct smb_sockaddr_in6 *p6; struct cifs_server_iface *info = NULL, *iface = NULL, *niface = NULL; struct cifs_server_iface tmp_iface; + __be16 port; ssize_t bytes_left; size_t next = 0; int nb_iface = 0; @@ -637,13 +638,6 @@ parse_server_interfaces(struct network_interface_info_ioctl_rsp *buf, p = buf; spin_lock(&ses->iface_lock); - /* do not query too frequently, this time with lock held */ - if (ses->iface_last_update && - time_before(jiffies, ses->iface_last_update + - (SMB_INTERFACE_POLL_INTERVAL * HZ))) { - spin_unlock(&ses->iface_lock); - return 0; - } /* * Go through iface_list and mark them as inactive @@ -666,10 +660,18 @@ parse_server_interfaces(struct network_interface_info_ioctl_rsp *buf, "Empty network interface list returned by server %s\n", ses->server->hostname); rc = -EOPNOTSUPP; - ses->iface_last_update = jiffies; goto out; } + spin_lock(&ses->server->srv_lock); + if (ses->server->dstaddr.ss_family == AF_INET) + port = ((struct sockaddr_in *)&ses->server->dstaddr)->sin_port; + else if (ses->server->dstaddr.ss_family == AF_INET6) + port = ((struct sockaddr_in6 *)&ses->server->dstaddr)->sin6_port; + else + port = cpu_to_be16(CIFS_PORT); + spin_unlock(&ses->server->srv_lock); + while (bytes_left >= (ssize_t)sizeof(*p)) { memset(&tmp_iface, 0, sizeof(tmp_iface)); /* default to 1Gbps when link speed is unset */ @@ -690,7 +692,7 @@ parse_server_interfaces(struct network_interface_info_ioctl_rsp *buf, memcpy(&addr4->sin_addr, &p4->IPv4Address, 4); /* [MS-SMB2] 2.2.32.5.1.1 Clients MUST ignore these */ - addr4->sin_port = cpu_to_be16(CIFS_PORT); + addr4->sin_port = port; cifs_dbg(FYI, "%s: ipv4 %pI4\n", __func__, &addr4->sin_addr); @@ -704,7 +706,7 @@ parse_server_interfaces(struct network_interface_info_ioctl_rsp *buf, /* [MS-SMB2] 2.2.32.5.1.2 Clients MUST ignore these */ addr6->sin6_flowinfo = 0; addr6->sin6_scope_id = 0; - addr6->sin6_port = cpu_to_be16(CIFS_PORT); + addr6->sin6_port = port; cifs_dbg(FYI, "%s: ipv6 %pI6\n", __func__, &addr6->sin6_addr); @@ -795,8 +797,6 @@ parse_server_interfaces(struct network_interface_info_ioctl_rsp *buf, + sizeof(p->Next) && p->Next)) cifs_dbg(VFS, "%s: incomplete interface info\n", __func__); - ses->iface_last_update = jiffies; - out: /* * Go through the list again and put the inactive entries @@ -825,10 +825,17 @@ SMB3_request_interfaces(const unsigned int xid, struct cifs_tcon *tcon, bool in_ struct TCP_Server_Info *pserver; /* do not query too frequently */ + spin_lock(&ses->iface_lock); if (ses->iface_last_update && time_before(jiffies, ses->iface_last_update + - (SMB_INTERFACE_POLL_INTERVAL * HZ))) + (SMB_INTERFACE_POLL_INTERVAL * HZ))) { + spin_unlock(&ses->iface_lock); return 0; + } + + ses->iface_last_update = jiffies; + + spin_unlock(&ses->iface_lock); rc = SMB2_ioctl(xid, tcon, NO_FILE_ID, NO_FILE_ID, FSCTL_QUERY_NETWORK_INTERFACE_INFO, @@ -1188,6 +1195,7 @@ smb2_set_ea(const unsigned int xid, struct cifs_tcon *tcon, replay_again: /* reinitialize for possible replay */ + used_len = 0; flags = CIFS_CP_CREATE_CLOSE_OP; oplock = SMB2_OPLOCK_LEVEL_NONE; server = cifs_pick_channel(ses); @@ -1485,6 +1493,7 @@ smb2_close_getattr(const unsigned int xid, struct cifs_tcon *tcon, { struct smb2_file_network_open_info file_inf; struct inode *inode; + u64 asize; int rc; rc = __SMB2_close(xid, tcon, cfile->fid.persistent_fid, @@ -1508,14 +1517,9 @@ smb2_close_getattr(const unsigned int xid, struct cifs_tcon *tcon, inode_set_atime_to_ts(inode, cifs_NTtimeToUnix(file_inf.LastAccessTime)); - /* - * i_blocks is not related to (i_size / i_blksize), - * but instead 512 byte (2**9) size is required for - * calculating num blocks. - */ - if (le64_to_cpu(file_inf.AllocationSize) > 4096) - inode->i_blocks = - (512 - 1 + le64_to_cpu(file_inf.AllocationSize)) >> 9; + asize = le64_to_cpu(file_inf.AllocationSize); + if (asize > 4096) + inode->i_blocks = CIFS_INO_BLOCKS(asize); /* End of file and Attributes should not have to be updated on close */ spin_unlock(&inode->i_lock); @@ -1586,6 +1590,7 @@ smb2_ioctl_query_info(const unsigned int xid, replay_again: /* reinitialize for possible replay */ + buffer = NULL; flags = CIFS_CP_CREATE_CLOSE_OP; oplock = SMB2_OPLOCK_LEVEL_NONE; server = cifs_pick_channel(ses); @@ -2188,14 +2193,6 @@ smb2_duplicate_extents(const unsigned int xid, rc = smb2_set_file_size(xid, tcon, trgtfile, dest_off + len, false); if (rc) goto duplicate_extents_out; - - /* - * Although also could set plausible allocation size (i_blocks) - * here in addition to setting the file size, in reflink - * it is likely that the target file is sparse. Its allocation - * size will be queried on next revalidate, but it is important - * to make sure that file's cached size is updated immediately - */ netfs_resize_file(netfs_inode(inode), dest_off + len, true); cifs_setsize(inode, dest_off + len); } @@ -3091,7 +3088,9 @@ smb2_get_dfs_refer(const unsigned int xid, struct cifs_ses *ses, struct cifs_tcon, tcon_list); if (tcon) { + spin_lock(&tcon->tc_lock); tcon->tc_count++; + spin_unlock(&tcon->tc_lock); trace_smb3_tcon_ref(tcon->debug_id, tcon->tc_count, netfs_trace_tcon_ref_get_dfs_refer); } @@ -3160,13 +3159,9 @@ smb2_get_dfs_refer(const unsigned int xid, struct cifs_ses *ses, out: if (tcon && !tcon->ipc) { /* ipc tcons are not refcounted */ - spin_lock(&cifs_tcp_ses_lock); - tcon->tc_count--; + cifs_put_tcon(tcon, netfs_trace_tcon_ref_put_dfs_refer); trace_smb3_tcon_ref(tcon->debug_id, tcon->tc_count, netfs_trace_tcon_ref_dec_dfs_refer); - /* tc_count can never go negative */ - WARN_ON(tcon->tc_count < 0); - spin_unlock(&cifs_tcp_ses_lock); } kfree(utf16_path); kfree(dfs_req); diff --git a/fs/smb/client/smb2pdu.c b/fs/smb/client/smb2pdu.c index 5d57c895ca37a7..a00bcfd08152ea 100644 --- a/fs/smb/client/smb2pdu.c +++ b/fs/smb/client/smb2pdu.c @@ -1715,19 +1715,17 @@ SMB2_auth_kerberos(struct SMB2_sess_data *sess_data) is_binding = (ses->ses_status == SES_GOOD); spin_unlock(&ses->ses_lock); - /* keep session key if binding */ - if (!is_binding) { - kfree_sensitive(ses->auth_key.response); - ses->auth_key.response = kmemdup(msg->data, msg->sesskey_len, - GFP_KERNEL); - if (!ses->auth_key.response) { - cifs_dbg(VFS, "Kerberos can't allocate (%u bytes) memory\n", - msg->sesskey_len); - rc = -ENOMEM; - goto out_put_spnego_key; - } - ses->auth_key.len = msg->sesskey_len; + kfree_sensitive(ses->auth_key.response); + ses->auth_key.response = kmemdup(msg->data, + msg->sesskey_len, + GFP_KERNEL); + if (!ses->auth_key.response) { + cifs_dbg(VFS, "%s: can't allocate (%u bytes) memory\n", + __func__, msg->sesskey_len); + rc = -ENOMEM; + goto out_put_spnego_key; } + ses->auth_key.len = msg->sesskey_len; sess_data->iov[1].iov_base = msg->data + msg->sesskey_len; sess_data->iov[1].iov_len = msg->secblob_len; @@ -2908,6 +2906,7 @@ int smb311_posix_mkdir(const unsigned int xid, struct inode *inode, replay_again: /* reinitialize for possible replay */ + pc_buf = NULL; flags = 0; n_iov = 2; server = cifs_pick_channel(ses); @@ -3982,7 +3981,7 @@ SMB311_posix_query_info(const unsigned int xid, struct cifs_tcon *tcon, u64 persistent_fid, u64 volatile_fid, struct smb311_posix_qinfo *data, u32 *plen) { - size_t output_len = sizeof(struct smb311_posix_qinfo *) + + size_t output_len = sizeof(struct smb311_posix_qinfo) + (sizeof(struct smb_sid) * 2) + (PATH_MAX * 2); *plen = 0; @@ -4239,7 +4238,9 @@ void smb2_reconnect_server(struct work_struct *work) list_for_each_entry(tcon, &ses->tcon_list, tcon_list) { if (tcon->need_reconnect || tcon->need_reopen_files) { + spin_lock(&tcon->tc_lock); tcon->tc_count++; + spin_unlock(&tcon->tc_lock); trace_smb3_tcon_ref(tcon->debug_id, tcon->tc_count, netfs_trace_tcon_ref_get_reconnect_server); list_add_tail(&tcon->rlist, &tmp_list); @@ -5236,7 +5237,10 @@ SMB2_write(const unsigned int xid, struct cifs_io_parms *io_parms, memset(&rqst, 0, sizeof(struct smb_rqst)); rqst.rq_iov = iov; - rqst.rq_nvec = n_vec + 1; + /* iov[0] is the SMB header; move payload to rq_iter for encryption safety */ + rqst.rq_nvec = 1; + iov_iter_kvec(&rqst.rq_iter, ITER_SOURCE, &iov[1], n_vec, + io_parms->length); if (retries) smb2_set_replay(server, &rqst); diff --git a/fs/smb/client/smbdirect.c b/fs/smb/client/smbdirect.c index 788a0670c4a8da..ff44a2dc49938f 100644 --- a/fs/smb/client/smbdirect.c +++ b/fs/smb/client/smbdirect.c @@ -35,6 +35,10 @@ static void enqueue_reassembly( static struct smbdirect_recv_io *_get_first_reassembly( struct smbdirect_socket *sc); +static int smbd_post_send(struct smbdirect_socket *sc, + struct smbdirect_send_batch *batch, + struct smbdirect_send_io *request); + static int smbd_post_recv( struct smbdirect_socket *sc, struct smbdirect_recv_io *response); @@ -97,8 +101,23 @@ int smbd_send_credit_target = 255; /* The maximum single message size can be sent to remote peer */ int smbd_max_send_size = 1364; -/* The maximum fragmented upper-layer payload receive size supported */ -int smbd_max_fragmented_recv_size = 1024 * 1024; +/* + * The maximum fragmented upper-layer payload receive size supported + * + * Assume max_payload_per_credit is + * smbd_max_receive_size - 24 = 1340 + * + * The maximum number would be + * smbd_receive_credit_max * max_payload_per_credit + * + * 1340 * 255 = 341700 (0x536C4) + * + * The minimum value from the spec is 131072 (0x20000) + * + * For now we use the logic we used in ksmbd before: + * (1364 * 255) / 2 = 173910 (0x2A756) + */ +int smbd_max_fragmented_recv_size = (1364 * 255) / 2; /* The maximum single-message size which can be received */ int smbd_max_receive_size = 1364; @@ -493,27 +512,103 @@ static inline void *smbdirect_recv_io_payload(struct smbdirect_recv_io *response return (void *)response->packet; } +static struct smbdirect_send_io *smbd_alloc_send_io(struct smbdirect_socket *sc) +{ + struct smbdirect_send_io *msg; + + msg = mempool_alloc(sc->send_io.mem.pool, GFP_KERNEL); + if (!msg) + return ERR_PTR(-ENOMEM); + msg->socket = sc; + INIT_LIST_HEAD(&msg->sibling_list); + msg->num_sge = 0; + + return msg; +} + +static void smbd_free_send_io(struct smbdirect_send_io *msg) +{ + struct smbdirect_socket *sc = msg->socket; + size_t i; + + /* + * The list needs to be empty! + * The caller should take care of it. + */ + WARN_ON_ONCE(!list_empty(&msg->sibling_list)); + + /* + * Note we call ib_dma_unmap_page(), even if some sges are mapped using + * ib_dma_map_single(). + * + * The difference between _single() and _page() only matters for the + * ib_dma_map_*() case. + * + * For the ib_dma_unmap_*() case it does not matter as both take the + * dma_addr_t and dma_unmap_single_attrs() is just an alias to + * dma_unmap_page_attrs(). + */ + for (i = 0; i < msg->num_sge; i++) + ib_dma_unmap_page(sc->ib.dev, + msg->sge[i].addr, + msg->sge[i].length, + DMA_TO_DEVICE); + + mempool_free(msg, sc->send_io.mem.pool); +} + /* Called when a RDMA send is done */ static void send_done(struct ib_cq *cq, struct ib_wc *wc) { - int i; struct smbdirect_send_io *request = container_of(wc->wr_cqe, struct smbdirect_send_io, cqe); struct smbdirect_socket *sc = request->socket; + struct smbdirect_send_io *sibling, *next; int lcredits = 0; log_rdma_send(INFO, "smbdirect_send_io 0x%p completed wc->status=%s\n", request, ib_wc_status_msg(wc->status)); - for (i = 0; i < request->num_sge; i++) - ib_dma_unmap_single(sc->ib.dev, - request->sge[i].addr, - request->sge[i].length, - DMA_TO_DEVICE); - mempool_free(request, sc->send_io.mem.pool); + if (unlikely(!(request->wr.send_flags & IB_SEND_SIGNALED))) { + /* + * This happens when smbdirect_send_io is a sibling + * before the final message, it is signaled on + * error anyway, so we need to skip + * smbdirect_connection_free_send_io here, + * otherwise is will destroy the memory + * of the siblings too, which will cause + * use after free problems for the others + * triggered from ib_drain_qp(). + */ + if (wc->status != IB_WC_SUCCESS) + goto skip_free; + + /* + * This should not happen! + * But we better just close the + * connection... + */ + log_rdma_send(ERR, + "unexpected send completion wc->status=%s (%d) wc->opcode=%d\n", + ib_wc_status_msg(wc->status), wc->status, wc->opcode); + smbd_disconnect_rdma_connection(sc); + return; + } + + /* + * Free possible siblings and then the main send_io + */ + list_for_each_entry_safe(sibling, next, &request->sibling_list, sibling_list) { + list_del_init(&sibling->sibling_list); + smbd_free_send_io(sibling); + lcredits += 1; + } + /* Note this frees wc->wr_cqe, but not wc */ + smbd_free_send_io(request); lcredits += 1; if (wc->status != IB_WC_SUCCESS || wc->opcode != IB_WC_SEND) { +skip_free: if (wc->status != IB_WC_WR_FLUSH_ERR) log_rdma_send(ERR, "wc->status=%s wc->opcode=%d\n", ib_wc_status_msg(wc->status), wc->opcode); @@ -608,6 +703,7 @@ static bool process_negotiation_response( sp->max_frmr_depth * PAGE_SIZE); sp->max_frmr_depth = sp->max_read_write_size / PAGE_SIZE; + atomic_set(&sc->send_io.bcredits.count, 1); sc->recv_io.expected = SMBDIRECT_EXPECT_DATA_TRANSFER; return true; } @@ -618,6 +714,7 @@ static void smbd_post_send_credits(struct work_struct *work) struct smbdirect_recv_io *response; struct smbdirect_socket *sc = container_of(work, struct smbdirect_socket, recv_io.posted.refill_work); + int posted = 0; if (sc->status != SMBDIRECT_SOCKET_CONNECTED) { return; @@ -640,9 +737,21 @@ static void smbd_post_send_credits(struct work_struct *work) } atomic_inc(&sc->recv_io.posted.count); + posted += 1; } } + atomic_add(posted, &sc->recv_io.credits.available); + + /* + * If the last send credit is waiting for credits + * it can grant we need to wake it up + */ + if (posted && + atomic_read(&sc->send_io.bcredits.count) == 0 && + atomic_read(&sc->send_io.credits.count) == 0) + wake_up(&sc->send_io.credits.wait_queue); + /* Promptly send an immediate packet as defined in [MS-SMBD] 3.1.1.1 */ if (atomic_read(&sc->recv_io.credits.count) < sc->recv_io.credits.target - 1) { @@ -659,6 +768,7 @@ static void recv_done(struct ib_cq *cq, struct ib_wc *wc) container_of(wc->wr_cqe, struct smbdirect_recv_io, cqe); struct smbdirect_socket *sc = response->socket; struct smbdirect_socket_parameters *sp = &sc->parameters; + int current_recv_credits; u16 old_recv_credit_target; u32 data_offset = 0; u32 data_length = 0; @@ -743,7 +853,8 @@ static void recv_done(struct ib_cq *cq, struct ib_wc *wc) } atomic_dec(&sc->recv_io.posted.count); - atomic_dec(&sc->recv_io.credits.count); + current_recv_credits = atomic_dec_return(&sc->recv_io.credits.count); + old_recv_credit_target = sc->recv_io.credits.target; sc->recv_io.credits.target = le16_to_cpu(data_transfer->credits_requested); @@ -779,7 +890,8 @@ static void recv_done(struct ib_cq *cq, struct ib_wc *wc) * reassembly queue and wake up the reading thread */ if (data_length) { - if (sc->recv_io.credits.target > old_recv_credit_target) + if (current_recv_credits <= (sc->recv_io.credits.target / 4) || + sc->recv_io.credits.target > old_recv_credit_target) queue_work(sc->workqueue, &sc->recv_io.posted.refill_work); enqueue_reassembly(sc, response, data_length); @@ -955,16 +1067,13 @@ static int smbd_ia_open( static int smbd_post_send_negotiate_req(struct smbdirect_socket *sc) { struct smbdirect_socket_parameters *sp = &sc->parameters; - struct ib_send_wr send_wr; - int rc = -ENOMEM; + int rc; struct smbdirect_send_io *request; struct smbdirect_negotiate_req *packet; - request = mempool_alloc(sc->send_io.mem.pool, GFP_KERNEL); - if (!request) - return rc; - - request->socket = sc; + request = smbd_alloc_send_io(sc); + if (IS_ERR(request)) + return PTR_ERR(request); packet = smbdirect_send_io_payload(request); packet->min_version = cpu_to_le16(SMBDIRECT_V1); @@ -976,7 +1085,6 @@ static int smbd_post_send_negotiate_req(struct smbdirect_socket *sc) packet->max_fragmented_size = cpu_to_le32(sp->max_fragmented_recv_size); - request->num_sge = 1; request->sge[0].addr = ib_dma_map_single( sc->ib.dev, (void *)packet, sizeof(*packet), DMA_TO_DEVICE); @@ -984,42 +1092,20 @@ static int smbd_post_send_negotiate_req(struct smbdirect_socket *sc) rc = -EIO; goto dma_mapping_failed; } + request->num_sge = 1; request->sge[0].length = sizeof(*packet); request->sge[0].lkey = sc->ib.pd->local_dma_lkey; - ib_dma_sync_single_for_device( - sc->ib.dev, request->sge[0].addr, - request->sge[0].length, DMA_TO_DEVICE); - - request->cqe.done = send_done; - - send_wr.next = NULL; - send_wr.wr_cqe = &request->cqe; - send_wr.sg_list = request->sge; - send_wr.num_sge = request->num_sge; - send_wr.opcode = IB_WR_SEND; - send_wr.send_flags = IB_SEND_SIGNALED; - - log_rdma_send(INFO, "sge addr=0x%llx length=%u lkey=0x%x\n", - request->sge[0].addr, - request->sge[0].length, request->sge[0].lkey); - - atomic_inc(&sc->send_io.pending.count); - rc = ib_post_send(sc->ib.qp, &send_wr, NULL); + rc = smbd_post_send(sc, NULL, request); if (!rc) return 0; - /* if we reach here, post send failed */ - log_rdma_send(ERR, "ib_post_send failed rc=%d\n", rc); - atomic_dec(&sc->send_io.pending.count); - ib_dma_unmap_single(sc->ib.dev, request->sge[0].addr, - request->sge[0].length, DMA_TO_DEVICE); - - smbd_disconnect_rdma_connection(sc); + if (rc == -EAGAIN) + rc = -EIO; dma_mapping_failed: - mempool_free(request, sc->send_io.mem.pool); + smbd_free_send_io(request); return rc; } @@ -1033,19 +1119,38 @@ static int smbd_post_send_negotiate_req(struct smbdirect_socket *sc) */ static int manage_credits_prior_sending(struct smbdirect_socket *sc) { + int missing; + int available; int new_credits; if (atomic_read(&sc->recv_io.credits.count) >= sc->recv_io.credits.target) return 0; - new_credits = atomic_read(&sc->recv_io.posted.count); - if (new_credits == 0) + missing = (int)sc->recv_io.credits.target - atomic_read(&sc->recv_io.credits.count); + available = atomic_xchg(&sc->recv_io.credits.available, 0); + new_credits = (u16)min3(U16_MAX, missing, available); + if (new_credits <= 0) { + /* + * If credits are available, but not granted + * we need to re-add them again. + */ + if (available) + atomic_add(available, &sc->recv_io.credits.available); return 0; + } - new_credits -= atomic_read(&sc->recv_io.credits.count); - if (new_credits <= 0) - return 0; + if (new_credits < available) { + /* + * Readd the remaining available again. + */ + available -= new_credits; + atomic_add(available, &sc->recv_io.credits.available); + } + /* + * Remember we granted the credits + */ + atomic_add(new_credits, &sc->recv_io.credits.count); return new_credits; } @@ -1075,12 +1180,27 @@ static int manage_keep_alive_before_sending(struct smbdirect_socket *sc) return 0; } +static int smbd_ib_post_send(struct smbdirect_socket *sc, + struct ib_send_wr *wr) +{ + int ret; + + atomic_inc(&sc->send_io.pending.count); + ret = ib_post_send(sc->ib.qp, wr, NULL); + if (ret) { + pr_err("failed to post send: %d\n", ret); + smbd_disconnect_rdma_connection(sc); + ret = -EAGAIN; + } + return ret; +} + /* Post the send request */ static int smbd_post_send(struct smbdirect_socket *sc, - struct smbdirect_send_io *request) + struct smbdirect_send_batch *batch, + struct smbdirect_send_io *request) { - struct ib_send_wr send_wr; - int rc, i; + int i; for (i = 0; i < request->num_sge; i++) { log_rdma_send(INFO, @@ -1094,79 +1214,245 @@ static int smbd_post_send(struct smbdirect_socket *sc, } request->cqe.done = send_done; + request->wr.next = NULL; + request->wr.sg_list = request->sge; + request->wr.num_sge = request->num_sge; + request->wr.opcode = IB_WR_SEND; + + if (batch) { + request->wr.wr_cqe = NULL; + request->wr.send_flags = 0; + if (!list_empty(&batch->msg_list)) { + struct smbdirect_send_io *last; + + last = list_last_entry(&batch->msg_list, + struct smbdirect_send_io, + sibling_list); + last->wr.next = &request->wr; + } + list_add_tail(&request->sibling_list, &batch->msg_list); + batch->wr_cnt++; + return 0; + } - send_wr.next = NULL; - send_wr.wr_cqe = &request->cqe; - send_wr.sg_list = request->sge; - send_wr.num_sge = request->num_sge; - send_wr.opcode = IB_WR_SEND; - send_wr.send_flags = IB_SEND_SIGNALED; + request->wr.wr_cqe = &request->cqe; + request->wr.send_flags = IB_SEND_SIGNALED; + return smbd_ib_post_send(sc, &request->wr); +} - rc = ib_post_send(sc->ib.qp, &send_wr, NULL); - if (rc) { - log_rdma_send(ERR, "ib_post_send failed rc=%d\n", rc); - smbd_disconnect_rdma_connection(sc); - rc = -EAGAIN; +static void smbd_send_batch_init(struct smbdirect_send_batch *batch, + bool need_invalidate_rkey, + unsigned int remote_key) +{ + INIT_LIST_HEAD(&batch->msg_list); + batch->wr_cnt = 0; + batch->need_invalidate_rkey = need_invalidate_rkey; + batch->remote_key = remote_key; + batch->credit = 0; +} + +static int smbd_send_batch_flush(struct smbdirect_socket *sc, + struct smbdirect_send_batch *batch, + bool is_last) +{ + struct smbdirect_send_io *first, *last; + int ret = 0; + + if (list_empty(&batch->msg_list)) + goto release_credit; + + first = list_first_entry(&batch->msg_list, + struct smbdirect_send_io, + sibling_list); + last = list_last_entry(&batch->msg_list, + struct smbdirect_send_io, + sibling_list); + + if (batch->need_invalidate_rkey) { + first->wr.opcode = IB_WR_SEND_WITH_INV; + first->wr.ex.invalidate_rkey = batch->remote_key; + batch->need_invalidate_rkey = false; + batch->remote_key = 0; } - return rc; + last->wr.send_flags = IB_SEND_SIGNALED; + last->wr.wr_cqe = &last->cqe; + + /* + * Remove last from batch->msg_list + * and splice the rest of batch->msg_list + * to last->sibling_list. + * + * batch->msg_list is a valid empty list + * at the end. + */ + list_del_init(&last->sibling_list); + list_splice_tail_init(&batch->msg_list, &last->sibling_list); + batch->wr_cnt = 0; + + ret = smbd_ib_post_send(sc, &first->wr); + if (ret) { + struct smbdirect_send_io *sibling, *next; + + list_for_each_entry_safe(sibling, next, &last->sibling_list, sibling_list) { + list_del_init(&sibling->sibling_list); + smbd_free_send_io(sibling); + } + smbd_free_send_io(last); + } + +release_credit: + if (is_last && !ret && batch->credit) { + atomic_add(batch->credit, &sc->send_io.bcredits.count); + batch->credit = 0; + wake_up(&sc->send_io.bcredits.wait_queue); + } + + return ret; +} + +static int wait_for_credits(struct smbdirect_socket *sc, + wait_queue_head_t *waitq, atomic_t *total_credits, + int needed) +{ + int ret; + + do { + if (atomic_sub_return(needed, total_credits) >= 0) + return 0; + + atomic_add(needed, total_credits); + ret = wait_event_interruptible(*waitq, + atomic_read(total_credits) >= needed || + sc->status != SMBDIRECT_SOCKET_CONNECTED); + + if (sc->status != SMBDIRECT_SOCKET_CONNECTED) + return -ENOTCONN; + else if (ret < 0) + return ret; + } while (true); +} + +static int wait_for_send_bcredit(struct smbdirect_socket *sc, + struct smbdirect_send_batch *batch) +{ + int ret; + + if (batch->credit) + return 0; + + ret = wait_for_credits(sc, + &sc->send_io.bcredits.wait_queue, + &sc->send_io.bcredits.count, + 1); + if (ret) + return ret; + + batch->credit = 1; + return 0; +} + +static int wait_for_send_lcredit(struct smbdirect_socket *sc, + struct smbdirect_send_batch *batch) +{ + if (batch && (atomic_read(&sc->send_io.lcredits.count) <= 1)) { + int ret; + + ret = smbd_send_batch_flush(sc, batch, false); + if (ret) + return ret; + } + + return wait_for_credits(sc, + &sc->send_io.lcredits.wait_queue, + &sc->send_io.lcredits.count, + 1); +} + +static int wait_for_send_credits(struct smbdirect_socket *sc, + struct smbdirect_send_batch *batch) +{ + if (batch && + (batch->wr_cnt >= 16 || atomic_read(&sc->send_io.credits.count) <= 1)) { + int ret; + + ret = smbd_send_batch_flush(sc, batch, false); + if (ret) + return ret; + } + + return wait_for_credits(sc, + &sc->send_io.credits.wait_queue, + &sc->send_io.credits.count, + 1); } static int smbd_post_send_iter(struct smbdirect_socket *sc, + struct smbdirect_send_batch *batch, struct iov_iter *iter, int *_remaining_data_length) { struct smbdirect_socket_parameters *sp = &sc->parameters; - int i, rc; + int rc; int header_length; int data_length; struct smbdirect_send_io *request; struct smbdirect_data_transfer *packet; int new_credits = 0; + struct smbdirect_send_batch _batch; -wait_lcredit: - /* Wait for local send credits */ - rc = wait_event_interruptible(sc->send_io.lcredits.wait_queue, - atomic_read(&sc->send_io.lcredits.count) > 0 || - sc->status != SMBDIRECT_SOCKET_CONNECTED); - if (rc) - goto err_wait_lcredit; + if (!batch) { + smbd_send_batch_init(&_batch, false, 0); + batch = &_batch; + } - if (sc->status != SMBDIRECT_SOCKET_CONNECTED) { - log_outgoing(ERR, "disconnected not sending on wait_credit\n"); + rc = wait_for_send_bcredit(sc, batch); + if (rc) { + log_outgoing(ERR, "disconnected not sending on wait_bcredit\n"); rc = -EAGAIN; - goto err_wait_lcredit; - } - if (unlikely(atomic_dec_return(&sc->send_io.lcredits.count) < 0)) { - atomic_inc(&sc->send_io.lcredits.count); - goto wait_lcredit; + goto err_wait_bcredit; } -wait_credit: - /* Wait for send credits. A SMBD packet needs one credit */ - rc = wait_event_interruptible(sc->send_io.credits.wait_queue, - atomic_read(&sc->send_io.credits.count) > 0 || - sc->status != SMBDIRECT_SOCKET_CONNECTED); - if (rc) - goto err_wait_credit; + rc = wait_for_send_lcredit(sc, batch); + if (rc) { + log_outgoing(ERR, "disconnected not sending on wait_lcredit\n"); + rc = -EAGAIN; + goto err_wait_lcredit; + } - if (sc->status != SMBDIRECT_SOCKET_CONNECTED) { + rc = wait_for_send_credits(sc, batch); + if (rc) { log_outgoing(ERR, "disconnected not sending on wait_credit\n"); rc = -EAGAIN; goto err_wait_credit; } - if (unlikely(atomic_dec_return(&sc->send_io.credits.count) < 0)) { - atomic_inc(&sc->send_io.credits.count); - goto wait_credit; + + new_credits = manage_credits_prior_sending(sc); + if (new_credits == 0 && + atomic_read(&sc->send_io.credits.count) == 0 && + atomic_read(&sc->recv_io.credits.count) == 0) { + queue_work(sc->workqueue, &sc->recv_io.posted.refill_work); + rc = wait_event_interruptible(sc->send_io.credits.wait_queue, + atomic_read(&sc->send_io.credits.count) >= 1 || + atomic_read(&sc->recv_io.credits.available) >= 1 || + sc->status != SMBDIRECT_SOCKET_CONNECTED); + if (sc->status != SMBDIRECT_SOCKET_CONNECTED) + rc = -ENOTCONN; + if (rc < 0) { + log_outgoing(ERR, "disconnected not sending on last credit\n"); + rc = -EAGAIN; + goto err_wait_credit; + } + + new_credits = manage_credits_prior_sending(sc); } - request = mempool_alloc(sc->send_io.mem.pool, GFP_KERNEL); - if (!request) { - rc = -ENOMEM; + request = smbd_alloc_send_io(sc); + if (IS_ERR(request)) { + rc = PTR_ERR(request); goto err_alloc; } - request->socket = sc; memset(request->sge, 0, sizeof(request->sge)); /* Map the packet to DMA */ @@ -1215,9 +1501,6 @@ static int smbd_post_send_iter(struct smbdirect_socket *sc, /* Fill in the packet header */ packet->credits_requested = cpu_to_le16(sp->send_credit_target); - - new_credits = manage_credits_prior_sending(sc); - atomic_add(new_credits, &sc->recv_io.credits.count); packet->credits_granted = cpu_to_le16(new_credits); packet->flags = 0; @@ -1240,33 +1523,27 @@ static int smbd_post_send_iter(struct smbdirect_socket *sc, le32_to_cpu(packet->data_length), le32_to_cpu(packet->remaining_data_length)); - /* - * Now that we got a local and a remote credit - * we add us as pending - */ - atomic_inc(&sc->send_io.pending.count); + rc = smbd_post_send(sc, batch, request); + if (!rc) { + /* + * From here request is moved to batch + * and we should not free it explicitly. + */ - rc = smbd_post_send(sc, request); - if (!rc) - return 0; + if (batch != &_batch) + return 0; - if (atomic_dec_and_test(&sc->send_io.pending.count)) - wake_up(&sc->send_io.pending.zero_wait_queue); + rc = smbd_send_batch_flush(sc, batch, true); + if (!rc) + return 0; - wake_up(&sc->send_io.pending.dec_wait_queue); + goto err_flush; + } err_dma: - for (i = 0; i < request->num_sge; i++) - if (request->sge[i].addr) - ib_dma_unmap_single(sc->ib.dev, - request->sge[i].addr, - request->sge[i].length, - DMA_TO_DEVICE); - mempool_free(request, sc->send_io.mem.pool); - - /* roll back the granted receive credits */ - atomic_sub(new_credits, &sc->recv_io.credits.count); + smbd_free_send_io(request); +err_flush: err_alloc: atomic_inc(&sc->send_io.credits.count); wake_up(&sc->send_io.credits.wait_queue); @@ -1276,6 +1553,11 @@ static int smbd_post_send_iter(struct smbdirect_socket *sc, wake_up(&sc->send_io.lcredits.wait_queue); err_wait_lcredit: + atomic_add(batch->credit, &sc->send_io.bcredits.count); + batch->credit = 0; + wake_up(&sc->send_io.bcredits.wait_queue); + +err_wait_bcredit: return rc; } @@ -1289,10 +1571,11 @@ static int smbd_post_send_empty(struct smbdirect_socket *sc) int remaining_data_length = 0; sc->statistics.send_empty++; - return smbd_post_send_iter(sc, NULL, &remaining_data_length); + return smbd_post_send_iter(sc, NULL, NULL, &remaining_data_length); } static int smbd_post_send_full_iter(struct smbdirect_socket *sc, + struct smbdirect_send_batch *batch, struct iov_iter *iter, int *_remaining_data_length) { @@ -1305,7 +1588,7 @@ static int smbd_post_send_full_iter(struct smbdirect_socket *sc, */ while (iov_iter_count(iter) > 0) { - rc = smbd_post_send_iter(sc, iter, _remaining_data_length); + rc = smbd_post_send_iter(sc, batch, iter, _remaining_data_length); if (rc < 0) break; } @@ -2227,8 +2510,10 @@ int smbd_send(struct TCP_Server_Info *server, struct smbdirect_socket_parameters *sp = &sc->parameters; struct smb_rqst *rqst; struct iov_iter iter; + struct smbdirect_send_batch batch; unsigned int remaining_data_length, klen; int rc, i, rqst_idx; + int error = 0; if (sc->status != SMBDIRECT_SOCKET_CONNECTED) return -EAGAIN; @@ -2253,6 +2538,7 @@ int smbd_send(struct TCP_Server_Info *server, num_rqst, remaining_data_length); rqst_idx = 0; + smbd_send_batch_init(&batch, false, 0); do { rqst = &rqst_array[rqst_idx]; @@ -2271,20 +2557,28 @@ int smbd_send(struct TCP_Server_Info *server, klen += rqst->rq_iov[i].iov_len; iov_iter_kvec(&iter, ITER_SOURCE, rqst->rq_iov, rqst->rq_nvec, klen); - rc = smbd_post_send_full_iter(sc, &iter, &remaining_data_length); - if (rc < 0) + rc = smbd_post_send_full_iter(sc, &batch, &iter, &remaining_data_length); + if (rc < 0) { + error = rc; break; + } if (iov_iter_count(&rqst->rq_iter) > 0) { /* And then the data pages if there are any */ - rc = smbd_post_send_full_iter(sc, &rqst->rq_iter, + rc = smbd_post_send_full_iter(sc, &batch, &rqst->rq_iter, &remaining_data_length); - if (rc < 0) + if (rc < 0) { + error = rc; break; + } } } while (++rqst_idx < num_rqst); + rc = smbd_send_batch_flush(sc, &batch, true); + if (unlikely(!rc && error)) + rc = error; + /* * As an optimization, we don't wait for individual I/O to finish * before sending the next one. diff --git a/fs/smb/client/trace.h b/fs/smb/client/trace.h index a584a774311329..1e7167a23f5c1f 100644 --- a/fs/smb/client/trace.h +++ b/fs/smb/client/trace.h @@ -177,6 +177,7 @@ EM(netfs_trace_tcon_ref_get_cached_laundromat, "GET Ch-Lau") \ EM(netfs_trace_tcon_ref_get_cached_lease_break, "GET Ch-Lea") \ EM(netfs_trace_tcon_ref_get_cancelled_close, "GET Cn-Cls") \ + EM(netfs_trace_tcon_ref_get_close_defer_files, "GET Cl-Def") \ EM(netfs_trace_tcon_ref_get_dfs_refer, "GET DfsRef") \ EM(netfs_trace_tcon_ref_get_find, "GET Find ") \ EM(netfs_trace_tcon_ref_get_find_sess_tcon, "GET FndSes") \ @@ -188,7 +189,9 @@ EM(netfs_trace_tcon_ref_put_cancelled_close, "PUT Cn-Cls") \ EM(netfs_trace_tcon_ref_put_cancelled_close_fid, "PUT Cn-Fid") \ EM(netfs_trace_tcon_ref_put_cancelled_mid, "PUT Cn-Mid") \ + EM(netfs_trace_tcon_ref_put_close_defer_files, "PUT Cl-Def") \ EM(netfs_trace_tcon_ref_put_mnt_ctx, "PUT MntCtx") \ + EM(netfs_trace_tcon_ref_put_dfs_refer, "PUT DfsRfr") \ EM(netfs_trace_tcon_ref_put_reconnect_server, "PUT Reconn") \ EM(netfs_trace_tcon_ref_put_tlink, "PUT Tlink ") \ EM(netfs_trace_tcon_ref_see_cancelled_close, "SEE Cn-Cls") \ diff --git a/fs/smb/client/transport.c b/fs/smb/client/transport.c index 3b34c3f4da2df7..67aee82e988605 100644 --- a/fs/smb/client/transport.c +++ b/fs/smb/client/transport.c @@ -808,16 +808,21 @@ cifs_cancelled_callback(struct TCP_Server_Info *server, struct mid_q_entry *mid) } /* - * Return a channel (master if none) of @ses that can be used to send - * regular requests. + * cifs_pick_channel - pick an eligible channel for network operations * - * If we are currently binding a new channel (negprot/sess.setup), - * return the new incomplete channel. + * @ses: session reference + * + * Select an eligible channel (not terminating and not marked as needing + * reconnect), preferring the least loaded one. If no eligible channel is + * found, fall back to the primary channel (index 0). + * + * Return: TCP_Server_Info pointer for the chosen channel, or NULL if @ses is + * NULL. */ struct TCP_Server_Info *cifs_pick_channel(struct cifs_ses *ses) { uint index = 0; - unsigned int min_in_flight = UINT_MAX, max_in_flight = 0; + unsigned int min_in_flight = UINT_MAX; struct TCP_Server_Info *server = NULL; int i, start, cur; @@ -847,14 +852,8 @@ struct TCP_Server_Info *cifs_pick_channel(struct cifs_ses *ses) min_in_flight = server->in_flight; index = cur; } - if (server->in_flight > max_in_flight) - max_in_flight = server->in_flight; } - /* if all channels are equally loaded, fall back to round-robin */ - if (min_in_flight == max_in_flight) - index = (uint)start % ses->chan_count; - server = ses->chans[index].server; spin_unlock(&ses->chan_lock); diff --git a/fs/smb/common/smbdirect/smbdirect_socket.h b/fs/smb/common/smbdirect/smbdirect_socket.h index ee4c2726771a34..95265192bb01b2 100644 --- a/fs/smb/common/smbdirect/smbdirect_socket.h +++ b/fs/smb/common/smbdirect/smbdirect_socket.h @@ -162,6 +162,17 @@ struct smbdirect_socket { mempool_t *pool; } mem; + /* + * This is a coordination for smbdirect_send_batch. + * + * There's only one possible credit, which means + * only one instance is running at a time. + */ + struct { + atomic_t count; + wait_queue_head_t wait_queue; + } bcredits; + /* * The local credit state for ib_post_send() */ @@ -239,6 +250,7 @@ struct smbdirect_socket { */ struct { u16 target; + atomic_t available; atomic_t count; } credits; @@ -370,6 +382,9 @@ static __always_inline void smbdirect_socket_init(struct smbdirect_socket *sc) INIT_DELAYED_WORK(&sc->idle.timer_work, __smbdirect_socket_disabled_work); disable_delayed_work_sync(&sc->idle.timer_work); + atomic_set(&sc->send_io.bcredits.count, 0); + init_waitqueue_head(&sc->send_io.bcredits.wait_queue); + atomic_set(&sc->send_io.lcredits.count, 0); init_waitqueue_head(&sc->send_io.lcredits.wait_queue); @@ -387,6 +402,7 @@ static __always_inline void smbdirect_socket_init(struct smbdirect_socket *sc) INIT_WORK(&sc->recv_io.posted.refill_work, __smbdirect_socket_disabled_work); disable_work_sync(&sc->recv_io.posted.refill_work); + atomic_set(&sc->recv_io.credits.available, 0); atomic_set(&sc->recv_io.credits.count, 0); INIT_LIST_HEAD(&sc->recv_io.reassembly.list); @@ -483,6 +499,8 @@ struct smbdirect_send_batch { */ bool need_invalidate_rkey; u32 remote_key; + + int credit; }; struct smbdirect_recv_io { diff --git a/fs/smb/server/Kconfig b/fs/smb/server/Kconfig index 2775162c535c6b..12594879cb640f 100644 --- a/fs/smb/server/Kconfig +++ b/fs/smb/server/Kconfig @@ -13,6 +13,7 @@ config SMB_SERVER select CRYPTO_LIB_MD5 select CRYPTO_LIB_SHA256 select CRYPTO_LIB_SHA512 + select CRYPTO_LIB_UTILS select CRYPTO_CMAC select CRYPTO_AEAD2 select CRYPTO_CCM diff --git a/fs/smb/server/auth.c b/fs/smb/server/auth.c index 09af55b71153e5..36c4ad5663f4ad 100644 --- a/fs/smb/server/auth.c +++ b/fs/smb/server/auth.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include @@ -165,7 +166,8 @@ int ksmbd_auth_ntlmv2(struct ksmbd_conn *conn, struct ksmbd_session *sess, ntlmv2_rsp, CIFS_HMAC_MD5_HASH_SIZE, sess->sess_key); - if (memcmp(ntlmv2->ntlmv2_hash, ntlmv2_rsp, CIFS_HMAC_MD5_HASH_SIZE) != 0) + if (crypto_memneq(ntlmv2->ntlmv2_hash, ntlmv2_rsp, + CIFS_HMAC_MD5_HASH_SIZE)) return -EINVAL; return 0; } @@ -587,12 +589,8 @@ static int generate_smb3signingkey(struct ksmbd_session *sess, if (!(conn->dialect >= SMB30_PROT_ID && signing->binding)) memcpy(chann->smb3signingkey, key, SMB3_SIGN_KEY_SIZE); - ksmbd_debug(AUTH, "dumping generated AES signing keys\n"); + ksmbd_debug(AUTH, "generated SMB3 signing key\n"); ksmbd_debug(AUTH, "Session Id %llu\n", sess->id); - ksmbd_debug(AUTH, "Session Key %*ph\n", - SMB2_NTLMV2_SESSKEY_SIZE, sess->sess_key); - ksmbd_debug(AUTH, "Signing Key %*ph\n", - SMB3_SIGN_KEY_SIZE, key); return 0; } @@ -650,23 +648,9 @@ static void generate_smb3encryptionkey(struct ksmbd_conn *conn, ptwin->decryption.context, sess->smb3decryptionkey, SMB3_ENC_DEC_KEY_SIZE); - ksmbd_debug(AUTH, "dumping generated AES encryption keys\n"); + ksmbd_debug(AUTH, "generated SMB3 encryption/decryption keys\n"); ksmbd_debug(AUTH, "Cipher type %d\n", conn->cipher_type); ksmbd_debug(AUTH, "Session Id %llu\n", sess->id); - ksmbd_debug(AUTH, "Session Key %*ph\n", - SMB2_NTLMV2_SESSKEY_SIZE, sess->sess_key); - if (conn->cipher_type == SMB2_ENCRYPTION_AES256_CCM || - conn->cipher_type == SMB2_ENCRYPTION_AES256_GCM) { - ksmbd_debug(AUTH, "ServerIn Key %*ph\n", - SMB3_GCM256_CRYPTKEY_SIZE, sess->smb3encryptionkey); - ksmbd_debug(AUTH, "ServerOut Key %*ph\n", - SMB3_GCM256_CRYPTKEY_SIZE, sess->smb3decryptionkey); - } else { - ksmbd_debug(AUTH, "ServerIn Key %*ph\n", - SMB3_GCM128_CRYPTKEY_SIZE, sess->smb3encryptionkey); - ksmbd_debug(AUTH, "ServerOut Key %*ph\n", - SMB3_GCM128_CRYPTKEY_SIZE, sess->smb3decryptionkey); - } } void ksmbd_gen_smb30_encryptionkey(struct ksmbd_conn *conn, diff --git a/fs/smb/server/connection.c b/fs/smb/server/connection.c index 6cac48c8fbe8e3..7e58739c0d3e7c 100644 --- a/fs/smb/server/connection.c +++ b/fs/smb/server/connection.c @@ -39,6 +39,7 @@ void ksmbd_conn_free(struct ksmbd_conn *conn) xa_destroy(&conn->sessions); kvfree(conn->request_buf); kfree(conn->preauth_info); + kfree(conn->mechToken); if (atomic_dec_and_test(&conn->refcnt)) { conn->transport->ops->free_transport(conn->transport); kfree(conn); diff --git a/fs/smb/server/mgmt/user_session.c b/fs/smb/server/mgmt/user_session.c index 7d880ff34402e0..26cb87625f1c6b 100644 --- a/fs/smb/server/mgmt/user_session.c +++ b/fs/smb/server/mgmt/user_session.c @@ -32,12 +32,14 @@ static void free_channel_list(struct ksmbd_session *sess) struct channel *chann; unsigned long index; + down_write(&sess->chann_lock); xa_for_each(&sess->ksmbd_chann_list, index, chann) { xa_erase(&sess->ksmbd_chann_list, index); kfree(chann); } xa_destroy(&sess->ksmbd_chann_list); + up_write(&sess->chann_lock); } static void __session_rpc_close(struct ksmbd_session *sess, @@ -220,7 +222,9 @@ static int ksmbd_chann_del(struct ksmbd_conn *conn, struct ksmbd_session *sess) { struct channel *chann; + down_write(&sess->chann_lock); chann = xa_erase(&sess->ksmbd_chann_list, (long)conn); + up_write(&sess->chann_lock); if (!chann) return -ENOENT; @@ -454,6 +458,7 @@ static struct ksmbd_session *__session_create(int protocol) rwlock_init(&sess->tree_conns_lock); atomic_set(&sess->refcnt, 2); init_rwsem(&sess->rpc_lock); + init_rwsem(&sess->chann_lock); ret = __init_smb2_session(sess); if (ret) diff --git a/fs/smb/server/mgmt/user_session.h b/fs/smb/server/mgmt/user_session.h index c5749d6ec7151c..cba7f688f6b577 100644 --- a/fs/smb/server/mgmt/user_session.h +++ b/fs/smb/server/mgmt/user_session.h @@ -49,6 +49,7 @@ struct ksmbd_session { char sess_key[CIFS_KEY_SIZE]; struct hlist_node hlist; + struct rw_semaphore chann_lock; struct xarray ksmbd_chann_list; struct xarray tree_conns; struct ida tree_conn_ida; diff --git a/fs/smb/server/oplock.c b/fs/smb/server/oplock.c index a5967ac466049b..41a31e1d7b14a6 100644 --- a/fs/smb/server/oplock.c +++ b/fs/smb/server/oplock.c @@ -82,11 +82,19 @@ static void lease_del_list(struct oplock_info *opinfo) spin_unlock(&lb->lb_lock); } -static void lb_add(struct lease_table *lb) +static struct lease_table *alloc_lease_table(struct oplock_info *opinfo) { - write_lock(&lease_list_lock); - list_add(&lb->l_entry, &lease_table_list); - write_unlock(&lease_list_lock); + struct lease_table *lb; + + lb = kmalloc(sizeof(struct lease_table), KSMBD_DEFAULT_GFP); + if (!lb) + return NULL; + + memcpy(lb->client_guid, opinfo->conn->ClientGUID, + SMB2_CLIENT_GUID_SIZE); + INIT_LIST_HEAD(&lb->lease_list); + spin_lock_init(&lb->lb_lock); + return lb; } static int alloc_lease(struct oplock_info *opinfo, struct lease_ctx_info *lctx) @@ -120,7 +128,7 @@ static void free_lease(struct oplock_info *opinfo) kfree(lease); } -static void free_opinfo(struct oplock_info *opinfo) +static void __free_opinfo(struct oplock_info *opinfo) { if (opinfo->is_lease) free_lease(opinfo); @@ -129,6 +137,18 @@ static void free_opinfo(struct oplock_info *opinfo) kfree(opinfo); } +static void free_opinfo_rcu(struct rcu_head *rcu) +{ + struct oplock_info *opinfo = container_of(rcu, struct oplock_info, rcu); + + __free_opinfo(opinfo); +} + +static void free_opinfo(struct oplock_info *opinfo) +{ + call_rcu(&opinfo->rcu, free_opinfo_rcu); +} + struct oplock_info *opinfo_get(struct ksmbd_file *fp) { struct oplock_info *opinfo; @@ -176,9 +196,9 @@ void opinfo_put(struct oplock_info *opinfo) free_opinfo(opinfo); } -static void opinfo_add(struct oplock_info *opinfo) +static void opinfo_add(struct oplock_info *opinfo, struct ksmbd_file *fp) { - struct ksmbd_inode *ci = opinfo->o_fp->f_ci; + struct ksmbd_inode *ci = fp->f_ci; down_write(&ci->m_lock); list_add(&opinfo->op_entry, &ci->m_op_list); @@ -1030,34 +1050,27 @@ static void copy_lease(struct oplock_info *op1, struct oplock_info *op2) lease2->version = lease1->version; } -static int add_lease_global_list(struct oplock_info *opinfo) +static void add_lease_global_list(struct oplock_info *opinfo, + struct lease_table *new_lb) { struct lease_table *lb; - read_lock(&lease_list_lock); + write_lock(&lease_list_lock); list_for_each_entry(lb, &lease_table_list, l_entry) { if (!memcmp(lb->client_guid, opinfo->conn->ClientGUID, SMB2_CLIENT_GUID_SIZE)) { opinfo->o_lease->l_lb = lb; lease_add_list(opinfo); - read_unlock(&lease_list_lock); - return 0; + write_unlock(&lease_list_lock); + kfree(new_lb); + return; } } - read_unlock(&lease_list_lock); - - lb = kmalloc(sizeof(struct lease_table), KSMBD_DEFAULT_GFP); - if (!lb) - return -ENOMEM; - memcpy(lb->client_guid, opinfo->conn->ClientGUID, - SMB2_CLIENT_GUID_SIZE); - INIT_LIST_HEAD(&lb->lease_list); - spin_lock_init(&lb->lb_lock); - opinfo->o_lease->l_lb = lb; + opinfo->o_lease->l_lb = new_lb; lease_add_list(opinfo); - lb_add(lb); - return 0; + list_add(&new_lb->l_entry, &lease_table_list); + write_unlock(&lease_list_lock); } static void set_oplock_level(struct oplock_info *opinfo, int level, @@ -1123,10 +1136,12 @@ void smb_lazy_parent_lease_break_close(struct ksmbd_file *fp) rcu_read_lock(); opinfo = rcu_dereference(fp->f_opinfo); - rcu_read_unlock(); - if (!opinfo || !opinfo->is_lease || opinfo->o_lease->version != 2) + if (!opinfo || !opinfo->is_lease || opinfo->o_lease->version != 2) { + rcu_read_unlock(); return; + } + rcu_read_unlock(); p_ci = ksmbd_inode_lookup_lock(fp->filp->f_path.dentry->d_parent); if (!p_ci) @@ -1175,6 +1190,7 @@ int smb_grant_oplock(struct ksmbd_work *work, int req_op_level, u64 pid, int err = 0; struct oplock_info *opinfo = NULL, *prev_opinfo = NULL; struct ksmbd_inode *ci = fp->f_ci; + struct lease_table *new_lb = NULL; bool prev_op_has_lease; __le32 prev_op_state = 0; @@ -1277,20 +1293,37 @@ int smb_grant_oplock(struct ksmbd_work *work, int req_op_level, u64 pid, set_oplock_level(opinfo, req_op_level, lctx); out: - rcu_assign_pointer(fp->f_opinfo, opinfo); + /* + * Set o_fp before any publication so that concurrent readers + * (e.g. find_same_lease_key() on the lease list) that + * dereference opinfo->o_fp don't hit a NULL pointer. + * + * Keep the original publication order so concurrent opens can + * still observe the in-flight grant via ci->m_op_list, but make + * everything after opinfo_add() no-fail by preallocating any new + * lease_table first. + */ opinfo->o_fp = fp; - - opinfo_count_inc(fp); - opinfo_add(opinfo); if (opinfo->is_lease) { - err = add_lease_global_list(opinfo); - if (err) + new_lb = alloc_lease_table(opinfo); + if (!new_lb) { + err = -ENOMEM; goto err_out; + } } + opinfo_count_inc(fp); + opinfo_add(opinfo, fp); + + if (opinfo->is_lease) + add_lease_global_list(opinfo, new_lb); + + rcu_assign_pointer(fp->f_opinfo, opinfo); + return 0; err_out: - free_opinfo(opinfo); + kfree(new_lb); + opinfo_put(opinfo); return err; } diff --git a/fs/smb/server/oplock.h b/fs/smb/server/oplock.h index 9a56eaadd0dd8f..921e3199e4df43 100644 --- a/fs/smb/server/oplock.h +++ b/fs/smb/server/oplock.h @@ -69,8 +69,9 @@ struct oplock_info { struct lease *o_lease; struct list_head op_entry; struct list_head lease_entry; - wait_queue_head_t oplock_q; /* Other server threads */ - wait_queue_head_t oplock_brk; /* oplock breaking wait */ + wait_queue_head_t oplock_q; /* Other server threads */ + wait_queue_head_t oplock_brk; /* oplock breaking wait */ + struct rcu_head rcu; }; struct lease_break_info { diff --git a/fs/smb/server/server.c b/fs/smb/server/server.c index 554ae90df906db..d2410a3f163aec 100644 --- a/fs/smb/server/server.c +++ b/fs/smb/server/server.c @@ -126,21 +126,21 @@ static int __process_request(struct ksmbd_work *work, struct ksmbd_conn *conn, andx_again: if (command >= conn->max_cmds) { conn->ops->set_rsp_status(work, STATUS_INVALID_PARAMETER); - return SERVER_HANDLER_CONTINUE; + return SERVER_HANDLER_ABORT; } cmds = &conn->cmds[command]; if (!cmds->proc) { ksmbd_debug(SMB, "*** not implemented yet cmd = %x\n", command); conn->ops->set_rsp_status(work, STATUS_NOT_IMPLEMENTED); - return SERVER_HANDLER_CONTINUE; + return SERVER_HANDLER_ABORT; } if (work->sess && conn->ops->is_sign_req(work, command)) { ret = conn->ops->check_sign_req(work); if (!ret) { conn->ops->set_rsp_status(work, STATUS_ACCESS_DENIED); - return SERVER_HANDLER_CONTINUE; + return SERVER_HANDLER_ABORT; } } diff --git a/fs/smb/server/smb2pdu.c b/fs/smb/server/smb2pdu.c index 2fcd0d4d1fb0de..a75fe467a4f0c9 100644 --- a/fs/smb/server/smb2pdu.c +++ b/fs/smb/server/smb2pdu.c @@ -4,6 +4,7 @@ * Copyright (C) 2018 Samsung Electronics Co., Ltd. */ +#include #include #include #include @@ -79,7 +80,13 @@ static inline bool check_session_id(struct ksmbd_conn *conn, u64 id) struct channel *lookup_chann_list(struct ksmbd_session *sess, struct ksmbd_conn *conn) { - return xa_load(&sess->ksmbd_chann_list, (long)conn); + struct channel *chann; + + down_read(&sess->chann_lock); + chann = xa_load(&sess->ksmbd_chann_list, (long)conn); + up_read(&sess->chann_lock); + + return chann; } /** @@ -118,6 +125,8 @@ int smb2_get_ksmbd_tcon(struct ksmbd_work *work) pr_err("The first operation in the compound does not have tcon\n"); return -EINVAL; } + if (work->tcon->t_state != TREE_CONNECTED) + return -ENOENT; if (tree_id != UINT_MAX && work->tcon->id != tree_id) { pr_err("tree id(%u) is different with id(%u) in first operation\n", tree_id, work->tcon->id); @@ -1558,8 +1567,10 @@ static int ntlm_authenticate(struct ksmbd_work *work, return -ENOMEM; chann->conn = conn; + down_write(&sess->chann_lock); old = xa_store(&sess->ksmbd_chann_list, (long)conn, chann, KSMBD_DEFAULT_GFP); + up_write(&sess->chann_lock); if (xa_is_err(old)) { kfree(chann); return xa_err(old); @@ -1651,8 +1662,10 @@ static int krb5_authenticate(struct ksmbd_work *work, return -ENOMEM; chann->conn = conn; + down_write(&sess->chann_lock); old = xa_store(&sess->ksmbd_chann_list, (long)conn, chann, KSMBD_DEFAULT_GFP); + up_write(&sess->chann_lock); if (xa_is_err(old)) { kfree(chann); return xa_err(old); @@ -1901,7 +1914,7 @@ int smb2_sess_setup(struct ksmbd_work *work) else if (rc) rsp->hdr.Status = STATUS_LOGON_FAILURE; - if (conn->use_spnego && conn->mechToken) { + if (conn->mechToken) { kfree(conn->mechToken); conn->mechToken = NULL; } @@ -1925,8 +1938,14 @@ int smb2_sess_setup(struct ksmbd_work *work) if (sess->user && sess->user->flags & KSMBD_USER_FLAG_DELAY_SESSION) try_delay = true; - sess->last_active = jiffies; - sess->state = SMB2_SESSION_EXPIRED; + /* + * For binding requests, session belongs to another + * connection. Do not expire it. + */ + if (!(req->Flags & SMB2_SESSION_REQ_FLAG_BINDING)) { + sess->last_active = jiffies; + sess->state = SMB2_SESSION_EXPIRED; + } ksmbd_user_session_put(sess); work->sess = NULL; if (try_delay) { @@ -1936,6 +1955,7 @@ int smb2_sess_setup(struct ksmbd_work *work) } } smb2_set_err_rsp(work); + conn->binding = false; } else { unsigned int iov_len; @@ -2816,7 +2836,11 @@ static int parse_durable_handle_context(struct ksmbd_work *work, goto out; } - dh_info->fp->conn = conn; + if (dh_info->fp->conn) { + ksmbd_put_durable_fd(dh_info->fp); + err = -EBADF; + goto out; + } dh_info->reconnected = true; goto out; } @@ -3000,13 +3024,14 @@ int smb2_open(struct ksmbd_work *work) goto err_out2; } + fp = dh_info.fp; + if (ksmbd_override_fsids(work)) { rc = -ENOMEM; ksmbd_put_durable_fd(dh_info.fp); goto err_out2; } - fp = dh_info.fp; file_info = FILE_OPENED; rc = ksmbd_vfs_getattr(&fp->filp->f_path, &stat); @@ -3376,20 +3401,24 @@ int smb2_open(struct ksmbd_work *work) KSMBD_SHARE_FLAG_ACL_XATTR)) { struct smb_fattr fattr; struct smb_ntsd *pntsd; - int pntsd_size, ace_num = 0; + int pntsd_size; + size_t scratch_len; ksmbd_acls_fattr(&fattr, idmap, inode); - if (fattr.cf_acls) - ace_num = fattr.cf_acls->a_count; - if (fattr.cf_dacls) - ace_num += fattr.cf_dacls->a_count; - - pntsd = kmalloc(sizeof(struct smb_ntsd) + - sizeof(struct smb_sid) * 3 + - sizeof(struct smb_acl) + - sizeof(struct smb_ace) * ace_num * 2, - KSMBD_DEFAULT_GFP); + scratch_len = smb_acl_sec_desc_scratch_len(&fattr, + NULL, 0, + OWNER_SECINFO | GROUP_SECINFO | + DACL_SECINFO); + if (!scratch_len || scratch_len == SIZE_MAX) { + rc = -EFBIG; + posix_acl_release(fattr.cf_acls); + posix_acl_release(fattr.cf_dacls); + goto err_out; + } + + pntsd = kvzalloc(scratch_len, KSMBD_DEFAULT_GFP); if (!pntsd) { + rc = -ENOMEM; posix_acl_release(fattr.cf_acls); posix_acl_release(fattr.cf_dacls); goto err_out; @@ -3404,7 +3433,7 @@ int smb2_open(struct ksmbd_work *work) posix_acl_release(fattr.cf_acls); posix_acl_release(fattr.cf_dacls); if (rc) { - kfree(pntsd); + kvfree(pntsd); goto err_out; } @@ -3414,7 +3443,7 @@ int smb2_open(struct ksmbd_work *work) pntsd, pntsd_size, false); - kfree(pntsd); + kvfree(pntsd); if (rc) pr_err("failed to store ntacl in xattr : %d\n", rc); @@ -3604,10 +3633,8 @@ int smb2_open(struct ksmbd_work *work) reconnected_fp: rsp->StructureSize = cpu_to_le16(89); - rcu_read_lock(); - opinfo = rcu_dereference(fp->f_opinfo); + opinfo = opinfo_get(fp); rsp->OplockLevel = opinfo != NULL ? opinfo->level : 0; - rcu_read_unlock(); rsp->Flags = 0; rsp->CreateAction = cpu_to_le32(file_info); rsp->CreationTime = cpu_to_le64(fp->create_time); @@ -3648,6 +3675,7 @@ int smb2_open(struct ksmbd_work *work) next_ptr = &lease_ccontext->Next; next_off = conn->vals->create_lease_size; } + opinfo_put(opinfo); if (maximal_access_ctxt) { struct create_context *mxac_ccontext; @@ -4427,8 +4455,9 @@ int smb2_query_dir(struct ksmbd_work *work) d_info.wptr = (char *)rsp->Buffer; d_info.rptr = (char *)rsp->Buffer; d_info.out_buf_len = - smb2_calc_max_out_buf_len(work, 8, - le32_to_cpu(req->OutputBufferLength)); + smb2_calc_max_out_buf_len(work, + offsetof(struct smb2_query_directory_rsp, Buffer), + le32_to_cpu(req->OutputBufferLength)); if (d_info.out_buf_len < 0) { rc = -EINVAL; goto err_out; @@ -4686,6 +4715,11 @@ static int smb2_get_ea(struct ksmbd_work *work, struct ksmbd_file *fp, ea_req = (struct smb2_ea_info_req *)((char *)req + le16_to_cpu(req->InputBufferOffset)); + + if (le32_to_cpu(req->InputBufferLength) < + offsetof(struct smb2_ea_info_req, name) + + ea_req->EaNameLength) + return -EINVAL; } else { /* need to send all EAs, if no specific EA is requested*/ if (le32_to_cpu(req->Flags) & SL_RETURN_SINGLE_ENTRY) @@ -4695,8 +4729,9 @@ static int smb2_get_ea(struct ksmbd_work *work, struct ksmbd_file *fp, } buf_free_len = - smb2_calc_max_out_buf_len(work, 8, - le32_to_cpu(req->OutputBufferLength)); + smb2_calc_max_out_buf_len(work, + offsetof(struct smb2_query_info_rsp, Buffer), + le32_to_cpu(req->OutputBufferLength)); if (buf_free_len < 0) return -EINVAL; @@ -4913,7 +4948,8 @@ static int get_file_all_info(struct ksmbd_work *work, int conv_len; char *filename; u64 time; - int ret; + int ret, buf_free_len, filename_len; + struct smb2_query_info_req *req = ksmbd_req_buf_next(work); if (!(fp->daccess & FILE_READ_ATTRIBUTES_LE)) { ksmbd_debug(SMB, "no right to read the attributes : 0x%x\n", @@ -4925,6 +4961,16 @@ static int get_file_all_info(struct ksmbd_work *work, if (IS_ERR(filename)) return PTR_ERR(filename); + filename_len = strlen(filename); + buf_free_len = smb2_calc_max_out_buf_len(work, + offsetof(struct smb2_query_info_rsp, Buffer) + + offsetof(struct smb2_file_all_info, FileName), + le32_to_cpu(req->OutputBufferLength)); + if (buf_free_len < (filename_len + 1) * 2) { + kfree(filename); + return -EINVAL; + } + ret = vfs_getattr(&fp->filp->f_path, &stat, STATX_BASIC_STATS, AT_STATX_SYNC_AS_STAT); if (ret) { @@ -4968,7 +5014,8 @@ static int get_file_all_info(struct ksmbd_work *work, file_info->Mode = fp->coption; file_info->AlignmentRequirement = 0; conv_len = smbConvertToUTF16((__le16 *)file_info->FileName, filename, - PATH_MAX, conn->local_nls, 0); + min(filename_len, PATH_MAX), + conn->local_nls, 0); conv_len *= 2; file_info->FileNameLength = cpu_to_le32(conv_len); rsp->OutputBufferLength = @@ -5022,8 +5069,9 @@ static int get_file_stream_info(struct ksmbd_work *work, file_info = (struct smb2_file_stream_info *)rsp->Buffer; buf_free_len = - smb2_calc_max_out_buf_len(work, 8, - le32_to_cpu(req->OutputBufferLength)); + smb2_calc_max_out_buf_len(work, + offsetof(struct smb2_query_info_rsp, Buffer), + le32_to_cpu(req->OutputBufferLength)); if (buf_free_len < 0) goto out; @@ -5332,8 +5380,9 @@ static int smb2_get_info_file(struct ksmbd_work *work, if (test_share_config_flag(work->tcon->share_conf, KSMBD_SHARE_FLAG_PIPE)) { /* smb2 info file called for pipe */ - return smb2_get_info_file_pipe(work->sess, req, rsp, + rc = smb2_get_info_file_pipe(work->sess, req, rsp, work->response_buf); + goto iov_pin_out; } if (work->next_smb2_rcv_hdr_off) { @@ -5433,6 +5482,12 @@ static int smb2_get_info_file(struct ksmbd_work *work, rc = buffer_check_err(le32_to_cpu(req->OutputBufferLength), rsp, work->response_buf); ksmbd_fd_put(work, fp); + +iov_pin_out: + if (!rc) + rc = ksmbd_iov_pin_rsp(work, (void *)rsp, + offsetof(struct smb2_query_info_rsp, Buffer) + + le32_to_cpu(rsp->OutputBufferLength)); return rc; } @@ -5440,7 +5495,6 @@ static int smb2_get_info_filesystem(struct ksmbd_work *work, struct smb2_query_info_req *req, struct smb2_query_info_rsp *rsp) { - struct ksmbd_session *sess = work->sess; struct ksmbd_conn *conn = work->conn; struct ksmbd_share_config *share = work->tcon->share_conf; int fsinfoclass = 0; @@ -5577,10 +5631,11 @@ static int smb2_get_info_filesystem(struct ksmbd_work *work, info = (struct object_id_info *)(rsp->Buffer); - if (!user_guest(sess->user)) - memcpy(info->objid, user_passkey(sess->user), 16); + if (path.mnt->mnt_sb->s_uuid_len == 16) + memcpy(info->objid, path.mnt->mnt_sb->s_uuid.b, + path.mnt->mnt_sb->s_uuid_len); else - memset(info->objid, 0, 16); + memcpy(info->objid, &stfs.f_fsid, sizeof(stfs.f_fsid)); info->extended_info.magic = cpu_to_le32(EXTENDED_INFO_MAGIC); info->extended_info.version = cpu_to_le32(1); @@ -5659,6 +5714,11 @@ static int smb2_get_info_filesystem(struct ksmbd_work *work, rc = buffer_check_err(le32_to_cpu(req->OutputBufferLength), rsp, work->response_buf); path_put(&path); + + if (!rc) + rc = ksmbd_iov_pin_rsp(work, (void *)rsp, + offsetof(struct smb2_query_info_rsp, Buffer) + + le32_to_cpu(rsp->OutputBufferLength)); return rc; } @@ -5668,13 +5728,14 @@ static int smb2_get_info_sec(struct ksmbd_work *work, { struct ksmbd_file *fp; struct mnt_idmap *idmap; - struct smb_ntsd *pntsd = (struct smb_ntsd *)rsp->Buffer, *ppntsd = NULL; + struct smb_ntsd *pntsd = NULL, *ppntsd = NULL; struct smb_fattr fattr = {{0}}; struct inode *inode; __u32 secdesclen = 0; unsigned int id = KSMBD_NO_FID, pid = KSMBD_NO_FID; int addition_info = le32_to_cpu(req->AdditionalInformation); - int rc = 0, ppntsd_size = 0; + int rc = 0, ppntsd_size = 0, max_len; + size_t scratch_len = 0; if (addition_info & ~(OWNER_SECINFO | GROUP_SECINFO | DACL_SECINFO | PROTECTED_DACL_SECINFO | @@ -5682,6 +5743,11 @@ static int smb2_get_info_sec(struct ksmbd_work *work, ksmbd_debug(SMB, "Unsupported addition info: 0x%x)\n", addition_info); + pntsd = kzalloc(ALIGN(sizeof(struct smb_ntsd), 8), + KSMBD_DEFAULT_GFP); + if (!pntsd) + return -ENOMEM; + pntsd->revision = cpu_to_le16(1); pntsd->type = cpu_to_le16(SELF_RELATIVE | DACL_PROTECTED); pntsd->osidoffset = 0; @@ -5690,9 +5756,7 @@ static int smb2_get_info_sec(struct ksmbd_work *work, pntsd->dacloffset = 0; secdesclen = sizeof(struct smb_ntsd); - rsp->OutputBufferLength = cpu_to_le32(secdesclen); - - return 0; + goto iov_pin; } if (work->next_smb2_rcv_hdr_off) { @@ -5724,18 +5788,58 @@ static int smb2_get_info_sec(struct ksmbd_work *work, &ppntsd); /* Check if sd buffer size exceeds response buffer size */ - if (smb2_resp_buf_len(work, 8) > ppntsd_size) - rc = build_sec_desc(idmap, pntsd, ppntsd, ppntsd_size, - addition_info, &secdesclen, &fattr); + max_len = smb2_calc_max_out_buf_len(work, + offsetof(struct smb2_query_info_rsp, Buffer), + le32_to_cpu(req->OutputBufferLength)); + if (max_len < 0) { + rc = -EINVAL; + goto release_acl; + } + + scratch_len = smb_acl_sec_desc_scratch_len(&fattr, ppntsd, + ppntsd_size, addition_info); + if (!scratch_len || scratch_len == SIZE_MAX) { + rc = -EFBIG; + goto release_acl; + } + + pntsd = kvzalloc(scratch_len, KSMBD_DEFAULT_GFP); + if (!pntsd) { + rc = -ENOMEM; + goto release_acl; + } + + rc = build_sec_desc(idmap, pntsd, ppntsd, ppntsd_size, + addition_info, &secdesclen, &fattr); + +release_acl: posix_acl_release(fattr.cf_acls); posix_acl_release(fattr.cf_dacls); kfree(ppntsd); ksmbd_fd_put(work, fp); + + if (!rc && ALIGN(secdesclen, 8) > scratch_len) + rc = -EFBIG; if (rc) - return rc; + goto err_out; +iov_pin: rsp->OutputBufferLength = cpu_to_le32(secdesclen); - return 0; + rc = buffer_check_err(le32_to_cpu(req->OutputBufferLength), + rsp, work->response_buf); + if (rc) + goto err_out; + + rc = ksmbd_iov_pin_rsp_read(work, (void *)rsp, + offsetof(struct smb2_query_info_rsp, Buffer), + pntsd, secdesclen); +err_out: + if (rc) { + rsp->OutputBufferLength = 0; + kvfree(pntsd); + } + + return rc; } /** @@ -5759,6 +5863,9 @@ int smb2_query_info(struct ksmbd_work *work) goto err_out; } + rsp->StructureSize = cpu_to_le16(9); + rsp->OutputBufferOffset = cpu_to_le16(72); + switch (req->InfoType) { case SMB2_O_INFO_FILE: ksmbd_debug(SMB, "GOT SMB2_O_INFO_FILE\n"); @@ -5779,14 +5886,6 @@ int smb2_query_info(struct ksmbd_work *work) } ksmbd_revert_fsids(work); - if (!rc) { - rsp->StructureSize = cpu_to_le16(9); - rsp->OutputBufferOffset = cpu_to_le16(72); - rc = ksmbd_iov_pin_rsp(work, (void *)rsp, - offsetof(struct smb2_query_info_rsp, Buffer) + - le32_to_cpu(rsp->OutputBufferLength)); - } - err_out: if (rc < 0) { if (rc == -EACCES) @@ -5797,6 +5896,8 @@ int smb2_query_info(struct ksmbd_work *work) rsp->hdr.Status = STATUS_UNEXPECTED_IO_ERROR; else if (rc == -ENOMEM) rsp->hdr.Status = STATUS_INSUFFICIENT_RESOURCES; + else if (rc == -EINVAL && rsp->hdr.Status == 0) + rsp->hdr.Status = STATUS_INVALID_PARAMETER; else if (rc == -EOPNOTSUPP || rsp->hdr.Status == 0) rsp->hdr.Status = STATUS_INVALID_INFO_CLASS; smb2_set_err_rsp(work); @@ -6104,14 +6205,14 @@ static int smb2_create_link(struct ksmbd_work *work, rc = -EINVAL; ksmbd_debug(SMB, "cannot delete %s\n", link_name); - goto out; } } else { rc = -EEXIST; ksmbd_debug(SMB, "link already exists\n"); - goto out; } ksmbd_vfs_kern_path_end_removing(&path); + if (rc) + goto out; } rc = ksmbd_vfs_link(work, target_name, link_name); if (rc) @@ -7567,14 +7668,15 @@ int smb2_lock(struct ksmbd_work *work) rc = vfs_lock_file(filp, smb_lock->cmd, flock, NULL); skip: if (smb_lock->flags & SMB2_LOCKFLAG_UNLOCK) { + locks_free_lock(flock); + kfree(smb_lock); if (!rc) { ksmbd_debug(SMB, "File unlocked\n"); } else if (rc == -ENOENT) { rsp->hdr.Status = STATUS_NOT_LOCKED; + err = rc; goto out; } - locks_free_lock(flock); - kfree(smb_lock); } else { if (rc == FILE_LOCK_DEFERRED) { void **argv; @@ -7643,6 +7745,9 @@ int smb2_lock(struct ksmbd_work *work) spin_unlock(&work->conn->llist_lock); ksmbd_debug(SMB, "successful in taking lock\n"); } else { + locks_free_lock(flock); + kfree(smb_lock); + err = rc; goto out; } } @@ -7673,13 +7778,17 @@ int smb2_lock(struct ksmbd_work *work) struct file_lock *rlock = NULL; rlock = smb_flock_init(filp); - rlock->c.flc_type = F_UNLCK; - rlock->fl_start = smb_lock->start; - rlock->fl_end = smb_lock->end; + if (rlock) { + rlock->c.flc_type = F_UNLCK; + rlock->fl_start = smb_lock->start; + rlock->fl_end = smb_lock->end; - rc = vfs_lock_file(filp, F_SETLK, rlock, NULL); - if (rc) - pr_err("rollback unlock fail : %d\n", rc); + rc = vfs_lock_file(filp, F_SETLK, rlock, NULL); + if (rc) + pr_err("rollback unlock fail : %d\n", rc); + } else { + pr_err("rollback unlock alloc failed\n"); + } list_del(&smb_lock->llist); spin_lock(&work->conn->llist_lock); @@ -7689,7 +7798,8 @@ int smb2_lock(struct ksmbd_work *work) spin_unlock(&work->conn->llist_lock); locks_free_lock(smb_lock->fl); - locks_free_lock(rlock); + if (rlock) + locks_free_lock(rlock); kfree(smb_lock); } out2: @@ -8172,8 +8282,9 @@ int smb2_ioctl(struct ksmbd_work *work) buffer = (char *)req + le32_to_cpu(req->InputOffset); cnt_code = le32_to_cpu(req->CtlCode); - ret = smb2_calc_max_out_buf_len(work, 48, - le32_to_cpu(req->MaxOutputResponse)); + ret = smb2_calc_max_out_buf_len(work, + offsetof(struct smb2_ioctl_rsp, Buffer), + le32_to_cpu(req->MaxOutputResponse)); if (ret < 0) { rsp->hdr.Status = STATUS_INVALID_PARAMETER; goto out; @@ -8869,7 +8980,7 @@ int smb2_check_sign_req(struct ksmbd_work *work) ksmbd_sign_smb2_pdu(work->conn, work->sess->sess_key, iov, 1, signature); - if (memcmp(signature, signature_req, SMB2_SIGNATURE_SIZE)) { + if (crypto_memneq(signature, signature_req, SMB2_SIGNATURE_SIZE)) { pr_err("bad smb2 signature\n"); return 0; } @@ -8957,7 +9068,7 @@ int smb3_check_sign_req(struct ksmbd_work *work) if (ksmbd_sign_smb3_pdu(conn, signing_key, iov, 1, signature)) return 0; - if (memcmp(signature, signature_req, SMB2_SIGNATURE_SIZE)) { + if (crypto_memneq(signature, signature_req, SMB2_SIGNATURE_SIZE)) { pr_err("bad smb2 signature\n"); return 0; } diff --git a/fs/smb/server/smbacl.c b/fs/smb/server/smbacl.c index 05598d994a6866..04c96534575b2f 100644 --- a/fs/smb/server/smbacl.c +++ b/fs/smb/server/smbacl.c @@ -451,7 +451,8 @@ static void parse_dacl(struct mnt_idmap *idmap, ppace[i]->access_req = smb_map_generic_desired_access(ppace[i]->access_req); - if (!(compare_sids(&ppace[i]->sid, &sid_unix_NFS_mode))) { + if (ppace[i]->sid.num_subauth >= 3 && + !(compare_sids(&ppace[i]->sid, &sid_unix_NFS_mode))) { fattr->cf_mode = le32_to_cpu(ppace[i]->sid.sub_auth[2]); break; @@ -915,6 +916,49 @@ int parse_sec_desc(struct mnt_idmap *idmap, struct smb_ntsd *pntsd, return 0; } +size_t smb_acl_sec_desc_scratch_len(struct smb_fattr *fattr, + struct smb_ntsd *ppntsd, int ppntsd_size, int addition_info) +{ + size_t len = sizeof(struct smb_ntsd); + size_t tmp; + + if (addition_info & OWNER_SECINFO) + len += sizeof(struct smb_sid); + if (addition_info & GROUP_SECINFO) + len += sizeof(struct smb_sid); + if (!(addition_info & DACL_SECINFO)) + return len; + + len += sizeof(struct smb_acl); + if (ppntsd && ppntsd_size > 0) { + unsigned int dacl_offset = le32_to_cpu(ppntsd->dacloffset); + + if (dacl_offset < ppntsd_size && + check_add_overflow(len, ppntsd_size - dacl_offset, &len)) + return 0; + } + + if (fattr->cf_acls) { + if (check_mul_overflow((size_t)fattr->cf_acls->a_count, + 2 * sizeof(struct smb_ace), &tmp) || + check_add_overflow(len, tmp, &len)) + return 0; + } else { + /* default/minimum DACL */ + if (check_add_overflow(len, 5 * sizeof(struct smb_ace), &len)) + return 0; + } + + if (fattr->cf_dacls) { + if (check_mul_overflow((size_t)fattr->cf_dacls->a_count, + sizeof(struct smb_ace), &tmp) || + check_add_overflow(len, tmp, &len)) + return 0; + } + + return len; +} + /* Convert permission bits from mode to equivalent CIFS ACL */ int build_sec_desc(struct mnt_idmap *idmap, struct smb_ntsd *pntsd, struct smb_ntsd *ppntsd, diff --git a/fs/smb/server/smbacl.h b/fs/smb/server/smbacl.h index 355adaee39b871..ab21ba2cd4df35 100644 --- a/fs/smb/server/smbacl.h +++ b/fs/smb/server/smbacl.h @@ -101,6 +101,8 @@ int set_info_sec(struct ksmbd_conn *conn, struct ksmbd_tree_connect *tcon, bool type_check, bool get_write); void id_to_sid(unsigned int cid, uint sidtype, struct smb_sid *ssid); void ksmbd_init_domain(u32 *sub_auth); +size_t smb_acl_sec_desc_scratch_len(struct smb_fattr *fattr, + struct smb_ntsd *ppntsd, int ppntsd_size, int addition_info); static inline uid_t posix_acl_uid_translate(struct mnt_idmap *idmap, struct posix_acl_entry *pace) diff --git a/fs/smb/server/transport_rdma.c b/fs/smb/server/transport_rdma.c index e4273932e7e497..fa56b572699367 100644 --- a/fs/smb/server/transport_rdma.c +++ b/fs/smb/server/transport_rdma.c @@ -221,6 +221,7 @@ static void smb_direct_disconnect_wake_up_all(struct smbdirect_socket *sc) * in order to notice the broken connection. */ wake_up_all(&sc->status_wait); + wake_up_all(&sc->send_io.bcredits.wait_queue); wake_up_all(&sc->send_io.lcredits.wait_queue); wake_up_all(&sc->send_io.credits.wait_queue); wake_up_all(&sc->send_io.pending.zero_wait_queue); @@ -644,6 +645,7 @@ static void recv_done(struct ib_cq *cq, struct ib_wc *wc) struct smbdirect_data_transfer *data_transfer = (struct smbdirect_data_transfer *)recvmsg->packet; u32 remaining_data_length, data_offset, data_length; + int current_recv_credits; u16 old_recv_credit_target; if (wc->byte_len < @@ -682,7 +684,7 @@ static void recv_done(struct ib_cq *cq, struct ib_wc *wc) } atomic_dec(&sc->recv_io.posted.count); - atomic_dec(&sc->recv_io.credits.count); + current_recv_credits = atomic_dec_return(&sc->recv_io.credits.count); old_recv_credit_target = sc->recv_io.credits.target; sc->recv_io.credits.target = @@ -702,7 +704,8 @@ static void recv_done(struct ib_cq *cq, struct ib_wc *wc) wake_up(&sc->send_io.credits.wait_queue); if (data_length) { - if (sc->recv_io.credits.target > old_recv_credit_target) + if (current_recv_credits <= (sc->recv_io.credits.target / 4) || + sc->recv_io.credits.target > old_recv_credit_target) queue_work(sc->workqueue, &sc->recv_io.posted.refill_work); enqueue_reassembly(sc, recvmsg, (int)data_length); @@ -1028,6 +1031,17 @@ static void smb_direct_post_recv_credits(struct work_struct *work) } } + atomic_add(credits, &sc->recv_io.credits.available); + + /* + * If the last send credit is waiting for credits + * it can grant we need to wake it up + */ + if (credits && + atomic_read(&sc->send_io.bcredits.count) == 0 && + atomic_read(&sc->send_io.credits.count) == 0) + wake_up(&sc->send_io.credits.wait_queue); + if (credits) queue_work(sc->workqueue, &sc->idle.immediate_work); } @@ -1045,6 +1059,31 @@ static void send_done(struct ib_cq *cq, struct ib_wc *wc) ib_wc_status_msg(wc->status), wc->status, wc->opcode); + if (unlikely(!(sendmsg->wr.send_flags & IB_SEND_SIGNALED))) { + /* + * This happens when smbdirect_send_io is a sibling + * before the final message, it is signaled on + * error anyway, so we need to skip + * smbdirect_connection_free_send_io here, + * otherwise is will destroy the memory + * of the siblings too, which will cause + * use after free problems for the others + * triggered from ib_drain_qp(). + */ + if (wc->status != IB_WC_SUCCESS) + goto skip_free; + + /* + * This should not happen! + * But we better just close the + * connection... + */ + pr_err("unexpected send completion wc->status=%s (%d) wc->opcode=%d\n", + ib_wc_status_msg(wc->status), wc->status, wc->opcode); + smb_direct_disconnect_rdma_connection(sc); + return; + } + /* * Free possible siblings and then the main send_io */ @@ -1058,6 +1097,7 @@ static void send_done(struct ib_cq *cq, struct ib_wc *wc) lcredits += 1; if (wc->status != IB_WC_SUCCESS || wc->opcode != IB_WC_SEND) { +skip_free: pr_err("Send error. status='%s (%d)', opcode=%d\n", ib_wc_status_msg(wc->status), wc->status, wc->opcode); @@ -1074,19 +1114,37 @@ static void send_done(struct ib_cq *cq, struct ib_wc *wc) static int manage_credits_prior_sending(struct smbdirect_socket *sc) { + int missing; + int available; int new_credits; if (atomic_read(&sc->recv_io.credits.count) >= sc->recv_io.credits.target) return 0; - new_credits = atomic_read(&sc->recv_io.posted.count); - if (new_credits == 0) + missing = (int)sc->recv_io.credits.target - atomic_read(&sc->recv_io.credits.count); + available = atomic_xchg(&sc->recv_io.credits.available, 0); + new_credits = (u16)min3(U16_MAX, missing, available); + if (new_credits <= 0) { + /* + * If credits are available, but not granted + * we need to re-add them again. + */ + if (available) + atomic_add(available, &sc->recv_io.credits.available); return 0; + } - new_credits -= atomic_read(&sc->recv_io.credits.count); - if (new_credits <= 0) - return 0; + if (new_credits < available) { + /* + * Readd the remaining available again. + */ + available -= new_credits; + atomic_add(available, &sc->recv_io.credits.available); + } + /* + * Remember we granted the credits + */ atomic_add(new_credits, &sc->recv_io.credits.count); return new_credits; } @@ -1130,6 +1188,7 @@ static void smb_direct_send_ctx_init(struct smbdirect_send_batch *send_ctx, send_ctx->wr_cnt = 0; send_ctx->need_invalidate_rkey = need_invalidate_rkey; send_ctx->remote_key = remote_key; + send_ctx->credit = 0; } static int smb_direct_flush_send_list(struct smbdirect_socket *sc, @@ -1137,10 +1196,10 @@ static int smb_direct_flush_send_list(struct smbdirect_socket *sc, bool is_last) { struct smbdirect_send_io *first, *last; - int ret; + int ret = 0; if (list_empty(&send_ctx->msg_list)) - return 0; + goto release_credit; first = list_first_entry(&send_ctx->msg_list, struct smbdirect_send_io, @@ -1182,6 +1241,13 @@ static int smb_direct_flush_send_list(struct smbdirect_socket *sc, smb_direct_free_sendmsg(sc, last); } +release_credit: + if (is_last && !ret && send_ctx->credit) { + atomic_add(send_ctx->credit, &sc->send_io.bcredits.count); + send_ctx->credit = 0; + wake_up(&sc->send_io.bcredits.wait_queue); + } + return ret; } @@ -1207,6 +1273,25 @@ static int wait_for_credits(struct smbdirect_socket *sc, } while (true); } +static int wait_for_send_bcredit(struct smbdirect_socket *sc, + struct smbdirect_send_batch *send_ctx) +{ + int ret; + + if (send_ctx->credit) + return 0; + + ret = wait_for_credits(sc, + &sc->send_io.bcredits.wait_queue, + &sc->send_io.bcredits.count, + 1); + if (ret) + return ret; + + send_ctx->credit = 1; + return 0; +} + static int wait_for_send_lcredit(struct smbdirect_socket *sc, struct smbdirect_send_batch *send_ctx) { @@ -1256,6 +1341,7 @@ static int calc_rw_credits(struct smbdirect_socket *sc, static int smb_direct_create_header(struct smbdirect_socket *sc, int size, int remaining_data_length, + int new_credits, struct smbdirect_send_io **sendmsg_out) { struct smbdirect_socket_parameters *sp = &sc->parameters; @@ -1271,7 +1357,7 @@ static int smb_direct_create_header(struct smbdirect_socket *sc, /* Fill in the packet header */ packet = (struct smbdirect_data_transfer *)sendmsg->packet; packet->credits_requested = cpu_to_le16(sp->send_credit_target); - packet->credits_granted = cpu_to_le16(manage_credits_prior_sending(sc)); + packet->credits_granted = cpu_to_le16(new_credits); packet->flags = 0; if (manage_keep_alive_before_sending(sc)) @@ -1408,6 +1494,17 @@ static int smb_direct_post_send_data(struct smbdirect_socket *sc, struct smbdirect_send_io *msg; int data_length; struct scatterlist sg[SMBDIRECT_SEND_IO_MAX_SGE - 1]; + struct smbdirect_send_batch _send_ctx; + int new_credits; + + if (!send_ctx) { + smb_direct_send_ctx_init(&_send_ctx, false, 0); + send_ctx = &_send_ctx; + } + + ret = wait_for_send_bcredit(sc, send_ctx); + if (ret) + goto bcredit_failed; ret = wait_for_send_lcredit(sc, send_ctx); if (ret) @@ -1417,12 +1514,29 @@ static int smb_direct_post_send_data(struct smbdirect_socket *sc, if (ret) goto credit_failed; + new_credits = manage_credits_prior_sending(sc); + if (new_credits == 0 && + atomic_read(&sc->send_io.credits.count) == 0 && + atomic_read(&sc->recv_io.credits.count) == 0) { + queue_work(sc->workqueue, &sc->recv_io.posted.refill_work); + ret = wait_event_interruptible(sc->send_io.credits.wait_queue, + atomic_read(&sc->send_io.credits.count) >= 1 || + atomic_read(&sc->recv_io.credits.available) >= 1 || + sc->status != SMBDIRECT_SOCKET_CONNECTED); + if (sc->status != SMBDIRECT_SOCKET_CONNECTED) + ret = -ENOTCONN; + if (ret < 0) + goto credit_failed; + + new_credits = manage_credits_prior_sending(sc); + } + data_length = 0; for (i = 0; i < niov; i++) data_length += iov[i].iov_len; ret = smb_direct_create_header(sc, data_length, remaining_data_length, - &msg); + new_credits, &msg); if (ret) goto header_failed; @@ -1460,14 +1574,30 @@ static int smb_direct_post_send_data(struct smbdirect_socket *sc, ret = post_sendmsg(sc, send_ctx, msg); if (ret) goto err; + + /* + * From here msg is moved to send_ctx + * and we should not free it explicitly. + */ + + if (send_ctx == &_send_ctx) { + ret = smb_direct_flush_send_list(sc, send_ctx, true); + if (ret) + goto flush_failed; + } + return 0; err: smb_direct_free_sendmsg(sc, msg); +flush_failed: header_failed: atomic_inc(&sc->send_io.credits.count); credit_failed: atomic_inc(&sc->send_io.lcredits.count); lcredit_failed: + atomic_add(send_ctx->credit, &sc->send_io.bcredits.count); + send_ctx->credit = 0; +bcredit_failed: return ret; } @@ -1939,6 +2069,7 @@ static int smb_direct_send_negotiate_response(struct smbdirect_socket *sc, resp->max_fragmented_size = cpu_to_le32(sp->max_fragmented_recv_size); + atomic_set(&sc->send_io.bcredits.count, 1); sc->recv_io.expected = SMBDIRECT_EXPECT_DATA_TRANSFER; sc->status = SMBDIRECT_SOCKET_CONNECTED; } @@ -2402,9 +2533,9 @@ static int smb_direct_prepare(struct ksmbd_transport *t) goto put; req = (struct smbdirect_negotiate_req *)recvmsg->packet; - sp->max_recv_size = min_t(int, sp->max_recv_size, + sp->max_recv_size = min_t(u32, sp->max_recv_size, le32_to_cpu(req->preferred_send_size)); - sp->max_send_size = min_t(int, sp->max_send_size, + sp->max_send_size = min_t(u32, sp->max_send_size, le32_to_cpu(req->max_receive_size)); sp->max_fragmented_send_size = le32_to_cpu(req->max_fragmented_size); diff --git a/fs/smb/server/transport_tcp.c b/fs/smb/server/transport_tcp.c index 4bb07937d7efeb..2436dabada957a 100644 --- a/fs/smb/server/transport_tcp.c +++ b/fs/smb/server/transport_tcp.c @@ -40,6 +40,7 @@ static const struct ksmbd_transport_ops ksmbd_tcp_transport_ops; static void tcp_stop_kthread(struct task_struct *kthread); static struct interface *alloc_iface(char *ifname); +static void ksmbd_tcp_disconnect(struct ksmbd_transport *t); #define KSMBD_TRANS(t) (&(t)->transport) #define TCP_TRANS(t) ((struct tcp_transport *)container_of(t, \ @@ -202,7 +203,7 @@ static int ksmbd_tcp_new_connection(struct socket *client_sk) if (IS_ERR(handler)) { pr_err("cannot start conn thread\n"); rc = PTR_ERR(handler); - free_transport(t); + ksmbd_tcp_disconnect(KSMBD_TRANS(t)); } return rc; } diff --git a/fs/squashfs/cache.c b/fs/squashfs/cache.c index 181260e72680c2..92fb857d2c7610 100644 --- a/fs/squashfs/cache.c +++ b/fs/squashfs/cache.c @@ -344,6 +344,9 @@ int squashfs_read_metadata(struct super_block *sb, void *buffer, if (unlikely(length < 0)) return -EIO; + if (unlikely(*offset < 0 || *offset >= SQUASHFS_METADATA_SIZE)) + return -EIO; + while (length) { entry = squashfs_cache_get(sb, msblk->block_cache, *block, 0); if (entry->error) { diff --git a/fs/tests/exec_kunit.c b/fs/tests/exec_kunit.c index 7c77d039680bbe..1c32cac098cf51 100644 --- a/fs/tests/exec_kunit.c +++ b/fs/tests/exec_kunit.c @@ -87,9 +87,6 @@ static const struct bprm_stack_limits_result bprm_stack_limits_results[] = { .argc = 0, .envc = ARG_MAX / sizeof(void *) - 1 }, .expected_argmin = ULONG_MAX - sizeof(void *) }, /* Raising rlim_stack / 4 to _STK_LIM / 4 * 3 will see more space. */ - { { .p = ULONG_MAX, .rlim_stack.rlim_cur = 4 * (_STK_LIM / 4 * 3), - .argc = 0, .envc = 0 }, - .expected_argmin = ULONG_MAX - (_STK_LIM / 4 * 3) + sizeof(void *) }, { { .p = ULONG_MAX, .rlim_stack.rlim_cur = 4 * (_STK_LIM / 4 * 3), .argc = 0, .envc = 0 }, .expected_argmin = ULONG_MAX - (_STK_LIM / 4 * 3) + sizeof(void *) }, @@ -97,12 +94,6 @@ static const struct bprm_stack_limits_result bprm_stack_limits_results[] = { { { .p = ULONG_MAX, .rlim_stack.rlim_cur = 4 * (_STK_LIM / 4 * 3 + sizeof(void *)), .argc = 0, .envc = 0 }, .expected_argmin = ULONG_MAX - (_STK_LIM / 4 * 3) + sizeof(void *) }, - { { .p = ULONG_MAX, .rlim_stack.rlim_cur = 4 * (_STK_LIM / 4 * + sizeof(void *)), - .argc = 0, .envc = 0 }, - .expected_argmin = ULONG_MAX - (_STK_LIM / 4 * 3) + sizeof(void *) }, - { { .p = ULONG_MAX, .rlim_stack.rlim_cur = 4 * _STK_LIM, - .argc = 0, .envc = 0 }, - .expected_argmin = ULONG_MAX - (_STK_LIM / 4 * 3) + sizeof(void *) }, { { .p = ULONG_MAX, .rlim_stack.rlim_cur = 4 * _STK_LIM, .argc = 0, .envc = 0 }, .expected_argmin = ULONG_MAX - (_STK_LIM / 4 * 3) + sizeof(void *) }, diff --git a/fs/xfs/libxfs/xfs_attr.c b/fs/xfs/libxfs/xfs_attr.c index 8c04acd30d489c..b88e65c7e45def 100644 --- a/fs/xfs/libxfs/xfs_attr.c +++ b/fs/xfs/libxfs/xfs_attr.c @@ -50,7 +50,6 @@ STATIC int xfs_attr_shortform_addname(xfs_da_args_t *args); */ STATIC int xfs_attr_leaf_get(xfs_da_args_t *args); STATIC int xfs_attr_leaf_removename(xfs_da_args_t *args); -STATIC int xfs_attr_leaf_hasname(struct xfs_da_args *args, struct xfs_buf **bp); /* * Internal routines when attribute list is more than one block. @@ -979,11 +978,12 @@ xfs_attr_lookup( return error; if (xfs_attr_is_leaf(dp)) { - error = xfs_attr_leaf_hasname(args, &bp); - - if (bp) - xfs_trans_brelse(args->trans, bp); - + error = xfs_attr3_leaf_read(args->trans, args->dp, args->owner, + 0, &bp); + if (error) + return error; + error = xfs_attr3_leaf_lookup_int(bp, args); + xfs_trans_brelse(args->trans, bp); return error; } @@ -1222,27 +1222,6 @@ xfs_attr_shortform_addname( * External routines when attribute list is one block *========================================================================*/ -/* - * Return EEXIST if attr is found, or ENOATTR if not - */ -STATIC int -xfs_attr_leaf_hasname( - struct xfs_da_args *args, - struct xfs_buf **bp) -{ - int error = 0; - - error = xfs_attr3_leaf_read(args->trans, args->dp, args->owner, 0, bp); - if (error) - return error; - - error = xfs_attr3_leaf_lookup_int(*bp, args); - if (error != -ENOATTR && error != -EEXIST) - xfs_trans_brelse(args->trans, *bp); - - return error; -} - /* * Remove a name from the leaf attribute list structure * @@ -1253,25 +1232,22 @@ STATIC int xfs_attr_leaf_removename( struct xfs_da_args *args) { - struct xfs_inode *dp; - struct xfs_buf *bp; + struct xfs_inode *dp = args->dp; int error, forkoff; + struct xfs_buf *bp; trace_xfs_attr_leaf_removename(args); - /* - * Remove the attribute. - */ - dp = args->dp; - - error = xfs_attr_leaf_hasname(args, &bp); - if (error == -ENOATTR) { + error = xfs_attr3_leaf_read(args->trans, args->dp, args->owner, 0, &bp); + if (error) + return error; + error = xfs_attr3_leaf_lookup_int(bp, args); + if (error != -EEXIST) { xfs_trans_brelse(args->trans, bp); - if (args->op_flags & XFS_DA_OP_RECOVERY) + if (error == -ENOATTR && (args->op_flags & XFS_DA_OP_RECOVERY)) return 0; return error; - } else if (error != -EEXIST) - return error; + } xfs_attr3_leaf_remove(bp, args); @@ -1295,23 +1271,20 @@ xfs_attr_leaf_removename( * Returns 0 on successful retrieval, otherwise an error. */ STATIC int -xfs_attr_leaf_get(xfs_da_args_t *args) +xfs_attr_leaf_get( + struct xfs_da_args *args) { - struct xfs_buf *bp; - int error; + struct xfs_buf *bp; + int error; trace_xfs_attr_leaf_get(args); - error = xfs_attr_leaf_hasname(args, &bp); - - if (error == -ENOATTR) { - xfs_trans_brelse(args->trans, bp); - return error; - } else if (error != -EEXIST) + error = xfs_attr3_leaf_read(args->trans, args->dp, args->owner, 0, &bp); + if (error) return error; - - - error = xfs_attr3_leaf_getvalue(bp, args); + error = xfs_attr3_leaf_lookup_int(bp, args); + if (error == -EEXIST) + error = xfs_attr3_leaf_getvalue(bp, args); xfs_trans_brelse(args->trans, bp); return error; } diff --git a/fs/xfs/libxfs/xfs_attr_leaf.c b/fs/xfs/libxfs/xfs_attr_leaf.c index 91c1b30ebaab31..7a9fe22c2b69ba 100644 --- a/fs/xfs/libxfs/xfs_attr_leaf.c +++ b/fs/xfs/libxfs/xfs_attr_leaf.c @@ -1316,6 +1316,28 @@ xfs_attr3_leaf_create( return 0; } +/* + * Reinitialize an existing attr fork block as an empty leaf, and attach + * the buffer to tp. + */ +int +xfs_attr3_leaf_init( + struct xfs_trans *tp, + struct xfs_inode *dp, + xfs_dablk_t blkno) +{ + struct xfs_buf *bp = NULL; + struct xfs_da_args args = { + .trans = tp, + .dp = dp, + .owner = dp->i_ino, + .geo = dp->i_mount->m_attr_geo, + }; + + ASSERT(tp != NULL); + + return xfs_attr3_leaf_create(&args, blkno, &bp); +} /* * Split the leaf node, rebalance, then add the new entry. * @@ -1476,6 +1498,7 @@ xfs_attr3_leaf_add_work( struct xfs_attr_leaf_name_local *name_loc; struct xfs_attr_leaf_name_remote *name_rmt; struct xfs_mount *mp; + int old_end, new_end; int tmp; int i; @@ -1568,17 +1591,49 @@ xfs_attr3_leaf_add_work( if (be16_to_cpu(entry->nameidx) < ichdr->firstused) ichdr->firstused = be16_to_cpu(entry->nameidx); - ASSERT(ichdr->firstused >= ichdr->count * sizeof(xfs_attr_leaf_entry_t) - + xfs_attr3_leaf_hdr_size(leaf)); - tmp = (ichdr->count - 1) * sizeof(xfs_attr_leaf_entry_t) - + xfs_attr3_leaf_hdr_size(leaf); + new_end = ichdr->count * sizeof(struct xfs_attr_leaf_entry) + + xfs_attr3_leaf_hdr_size(leaf); + old_end = new_end - sizeof(struct xfs_attr_leaf_entry); + + ASSERT(ichdr->firstused >= new_end); for (i = 0; i < XFS_ATTR_LEAF_MAPSIZE; i++) { - if (ichdr->freemap[i].base == tmp) { - ichdr->freemap[i].base += sizeof(xfs_attr_leaf_entry_t); + int diff = 0; + + if (ichdr->freemap[i].base == old_end) { + /* + * This freemap entry starts at the old end of the + * leaf entry array, so we need to adjust its base + * upward to accomodate the larger array. + */ + diff = sizeof(struct xfs_attr_leaf_entry); + } else if (ichdr->freemap[i].size > 0 && + ichdr->freemap[i].base < new_end) { + /* + * This freemap entry starts in the space claimed by + * the new leaf entry. Adjust its base upward to + * reflect that. + */ + diff = new_end - ichdr->freemap[i].base; + } + + if (diff) { + ichdr->freemap[i].base += diff; ichdr->freemap[i].size -= - min_t(uint16_t, ichdr->freemap[i].size, - sizeof(xfs_attr_leaf_entry_t)); + min_t(uint16_t, ichdr->freemap[i].size, diff); + } + + /* + * Don't leave zero-length freemaps with nonzero base lying + * around, because we don't want the code in _remove that + * matches on base address to get confused and create + * overlapping freemaps. If we end up with no freemap entries + * then the next _add will compact the leaf block and + * regenerate the freemaps. + */ + if (ichdr->freemap[i].size == 0 && ichdr->freemap[i].base > 0) { + ichdr->freemap[i].base = 0; + ichdr->holes = 1; } } ichdr->usedbytes += xfs_attr_leaf_entsize(leaf, args->index); diff --git a/fs/xfs/libxfs/xfs_attr_leaf.h b/fs/xfs/libxfs/xfs_attr_leaf.h index 589f810eedc0d8..deb62b544ac542 100644 --- a/fs/xfs/libxfs/xfs_attr_leaf.h +++ b/fs/xfs/libxfs/xfs_attr_leaf.h @@ -86,6 +86,9 @@ int xfs_attr3_leaf_list_int(struct xfs_buf *bp, /* * Routines used for shrinking the Btree. */ + +int xfs_attr3_leaf_init(struct xfs_trans *tp, struct xfs_inode *dp, + xfs_dablk_t blkno); int xfs_attr3_leaf_toosmall(struct xfs_da_state *state, int *retval); void xfs_attr3_leaf_unbalance(struct xfs_da_state *state, struct xfs_da_state_blk *drop_blk, diff --git a/fs/xfs/libxfs/xfs_da_btree.c b/fs/xfs/libxfs/xfs_da_btree.c index 90f7fc219fccc8..d85bca22d68574 100644 --- a/fs/xfs/libxfs/xfs_da_btree.c +++ b/fs/xfs/libxfs/xfs_da_btree.c @@ -1506,21 +1506,20 @@ xfs_da3_fixhashpath( } /* - * Remove an entry from an intermediate node. + * Internal implementation to remove an entry from an intermediate node. */ STATIC void -xfs_da3_node_remove( - struct xfs_da_state *state, - struct xfs_da_state_blk *drop_blk) +__xfs_da3_node_remove( + struct xfs_trans *tp, + struct xfs_inode *dp, + struct xfs_da_geometry *geo, + struct xfs_da_state_blk *drop_blk) { struct xfs_da_intnode *node; struct xfs_da3_icnode_hdr nodehdr; struct xfs_da_node_entry *btree; int index; int tmp; - struct xfs_inode *dp = state->args->dp; - - trace_xfs_da_node_remove(state->args); node = drop_blk->bp->b_addr; xfs_da3_node_hdr_from_disk(dp->i_mount, &nodehdr, node); @@ -1536,17 +1535,17 @@ xfs_da3_node_remove( tmp = nodehdr.count - index - 1; tmp *= (uint)sizeof(xfs_da_node_entry_t); memmove(&btree[index], &btree[index + 1], tmp); - xfs_trans_log_buf(state->args->trans, drop_blk->bp, + xfs_trans_log_buf(tp, drop_blk->bp, XFS_DA_LOGRANGE(node, &btree[index], tmp)); index = nodehdr.count - 1; } memset(&btree[index], 0, sizeof(xfs_da_node_entry_t)); - xfs_trans_log_buf(state->args->trans, drop_blk->bp, + xfs_trans_log_buf(tp, drop_blk->bp, XFS_DA_LOGRANGE(node, &btree[index], sizeof(btree[index]))); nodehdr.count -= 1; xfs_da3_node_hdr_to_disk(dp->i_mount, node, &nodehdr); - xfs_trans_log_buf(state->args->trans, drop_blk->bp, - XFS_DA_LOGRANGE(node, &node->hdr, state->args->geo->node_hdr_size)); + xfs_trans_log_buf(tp, drop_blk->bp, + XFS_DA_LOGRANGE(node, &node->hdr, geo->node_hdr_size)); /* * Copy the last hash value from the block to propagate upwards. @@ -1554,6 +1553,38 @@ xfs_da3_node_remove( drop_blk->hashval = be32_to_cpu(btree[index - 1].hashval); } +/* + * Remove an entry from an intermediate node. + */ +STATIC void +xfs_da3_node_remove( + struct xfs_da_state *state, + struct xfs_da_state_blk *drop_blk) +{ + trace_xfs_da_node_remove(state->args); + + __xfs_da3_node_remove(state->args->trans, state->args->dp, + state->args->geo, drop_blk); +} + +/* + * Remove an entry from an intermediate attr node at the specified index. + */ +void +xfs_attr3_node_entry_remove( + struct xfs_trans *tp, + struct xfs_inode *dp, + struct xfs_buf *bp, + int index) +{ + struct xfs_da_state_blk blk = { + .index = index, + .bp = bp, + }; + + __xfs_da3_node_remove(tp, dp, dp->i_mount->m_attr_geo, &blk); +} + /* * Unbalance the elements between two intermediate nodes, * move all Btree elements from one node into another. diff --git a/fs/xfs/libxfs/xfs_da_btree.h b/fs/xfs/libxfs/xfs_da_btree.h index 354d5d65043e43..afcf2d3c7a21c0 100644 --- a/fs/xfs/libxfs/xfs_da_btree.h +++ b/fs/xfs/libxfs/xfs_da_btree.h @@ -184,6 +184,8 @@ int xfs_da3_split(xfs_da_state_t *state); int xfs_da3_join(xfs_da_state_t *state); void xfs_da3_fixhashpath(struct xfs_da_state *state, struct xfs_da_state_path *path_to_to_fix); +void xfs_attr3_node_entry_remove(struct xfs_trans *tp, struct xfs_inode *dp, + struct xfs_buf *bp, int index); /* * Routines used for finding things in the Btree. diff --git a/fs/xfs/libxfs/xfs_defer.c b/fs/xfs/libxfs/xfs_defer.c index 5b377cbbb1f7e0..e8db2f6149e076 100644 --- a/fs/xfs/libxfs/xfs_defer.c +++ b/fs/xfs/libxfs/xfs_defer.c @@ -809,7 +809,7 @@ xfs_defer_can_append( /* Paused items cannot absorb more work */ if (dfp->dfp_flags & XFS_DEFER_PAUSED) - return NULL; + return false; /* Already full? */ if (ops->max_items && dfp->dfp_count >= ops->max_items) diff --git a/fs/xfs/scrub/agheader_repair.c b/fs/xfs/scrub/agheader_repair.c index cd6f0223879f49..6e3fef36d6614a 100644 --- a/fs/xfs/scrub/agheader_repair.c +++ b/fs/xfs/scrub/agheader_repair.c @@ -837,8 +837,12 @@ xrep_agi_buf_cleanup( { struct xrep_agi *ragi = buf; - xfarray_destroy(ragi->iunlink_prev); - xfarray_destroy(ragi->iunlink_next); + if (ragi->iunlink_prev) + xfarray_destroy(ragi->iunlink_prev); + ragi->iunlink_prev = NULL; + if (ragi->iunlink_next) + xfarray_destroy(ragi->iunlink_next); + ragi->iunlink_next = NULL; xagino_bitmap_destroy(&ragi->iunlink_bmp); } @@ -1708,7 +1712,6 @@ xrep_agi( { struct xrep_agi *ragi; struct xfs_mount *mp = sc->mp; - char *descr; unsigned int i; int error; @@ -1742,17 +1745,13 @@ xrep_agi( xagino_bitmap_init(&ragi->iunlink_bmp); sc->buf_cleanup = xrep_agi_buf_cleanup; - descr = xchk_xfile_ag_descr(sc, "iunlinked next pointers"); - error = xfarray_create(descr, 0, sizeof(xfs_agino_t), - &ragi->iunlink_next); - kfree(descr); + error = xfarray_create("iunlinked next pointers", 0, + sizeof(xfs_agino_t), &ragi->iunlink_next); if (error) return error; - descr = xchk_xfile_ag_descr(sc, "iunlinked prev pointers"); - error = xfarray_create(descr, 0, sizeof(xfs_agino_t), - &ragi->iunlink_prev); - kfree(descr); + error = xfarray_create("iunlinked prev pointers", 0, + sizeof(xfs_agino_t), &ragi->iunlink_prev); if (error) return error; diff --git a/fs/xfs/scrub/alloc_repair.c b/fs/xfs/scrub/alloc_repair.c index bed6a09aa79112..35035d02a23163 100644 --- a/fs/xfs/scrub/alloc_repair.c +++ b/fs/xfs/scrub/alloc_repair.c @@ -850,7 +850,6 @@ xrep_allocbt( struct xrep_abt *ra; struct xfs_mount *mp = sc->mp; unsigned int busy_gen; - char *descr; int error; /* We require the rmapbt to rebuild anything. */ @@ -876,11 +875,9 @@ xrep_allocbt( } /* Set up enough storage to handle maximally fragmented free space. */ - descr = xchk_xfile_ag_descr(sc, "free space records"); - error = xfarray_create(descr, mp->m_sb.sb_agblocks / 2, + error = xfarray_create("free space records", mp->m_sb.sb_agblocks / 2, sizeof(struct xfs_alloc_rec_incore), &ra->free_records); - kfree(descr); if (error) goto out_ra; @@ -926,7 +923,22 @@ xrep_revalidate_allocbt( if (error) goto out; + /* + * If the bnobt is still corrupt, we've failed to repair the filesystem + * and should just bail out. + * + * If the bnobt fails cross-examination with the cntbt, the scan will + * free the cntbt cursor, so we need to mark the repair incomplete + * and avoid walking off the end of the NULL cntbt cursor. + */ + if (sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT) + goto out; + sc->sm->sm_type = XFS_SCRUB_TYPE_CNTBT; + if (!sc->sa.cnt_cur) { + xchk_set_incomplete(sc); + goto out; + } error = xchk_allocbt(sc); out: sc->sm->sm_type = old_type; diff --git a/fs/xfs/scrub/attr.c b/fs/xfs/scrub/attr.c index 708334f9b2bd13..a0878fdbcf3866 100644 --- a/fs/xfs/scrub/attr.c +++ b/fs/xfs/scrub/attr.c @@ -287,32 +287,6 @@ xchk_xattr_set_map( return ret; } -/* - * Check the leaf freemap from the usage bitmap. Returns false if the - * attr freemap has problems or points to used space. - */ -STATIC bool -xchk_xattr_check_freemap( - struct xfs_scrub *sc, - struct xfs_attr3_icleaf_hdr *leafhdr) -{ - struct xchk_xattr_buf *ab = sc->buf; - unsigned int mapsize = sc->mp->m_attr_geo->blksize; - int i; - - /* Construct bitmap of freemap contents. */ - bitmap_zero(ab->freemap, mapsize); - for (i = 0; i < XFS_ATTR_LEAF_MAPSIZE; i++) { - if (!xchk_xattr_set_map(sc, ab->freemap, - leafhdr->freemap[i].base, - leafhdr->freemap[i].size)) - return false; - } - - /* Look for bits that are set in freemap and are marked in use. */ - return !bitmap_intersects(ab->freemap, ab->usedmap, mapsize); -} - /* * Check this leaf entry's relations to everything else. * Returns the number of bytes used for the name/value data. @@ -364,7 +338,10 @@ xchk_xattr_entry( rentry = xfs_attr3_leaf_name_remote(leaf, idx); namesize = xfs_attr_leaf_entsize_remote(rentry->namelen); name_end = (char *)rentry + namesize; - if (rentry->namelen == 0 || rentry->valueblk == 0) + if (rentry->namelen == 0) + xchk_da_set_corrupt(ds, level); + if (rentry->valueblk == 0 && + !(ent->flags & XFS_ATTR_INCOMPLETE)) xchk_da_set_corrupt(ds, level); } if (name_end > buf_end) @@ -403,6 +380,7 @@ xchk_xattr_block( *last_checked = blk->blkno; bitmap_zero(ab->usedmap, mp->m_attr_geo->blksize); + bitmap_zero(ab->freemap, mp->m_attr_geo->blksize); /* Check all the padding. */ if (xfs_has_crc(ds->sc->mp)) { @@ -449,6 +427,9 @@ xchk_xattr_block( if ((char *)&entries[leafhdr.count] > (char *)leaf + leafhdr.firstused) xchk_da_set_corrupt(ds, level); + if (ds->sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT) + goto out; + buf_end = (char *)bp->b_addr + mp->m_attr_geo->blksize; for (i = 0, ent = entries; i < leafhdr.count; ent++, i++) { /* Mark the leaf entry itself. */ @@ -467,7 +448,29 @@ xchk_xattr_block( goto out; } - if (!xchk_xattr_check_freemap(ds->sc, &leafhdr)) + /* Construct bitmap of freemap contents. */ + for (i = 0; i < XFS_ATTR_LEAF_MAPSIZE; i++) { + if (!xchk_xattr_set_map(ds->sc, ab->freemap, + leafhdr.freemap[i].base, + leafhdr.freemap[i].size)) + xchk_da_set_corrupt(ds, level); + + /* + * freemap entries with zero length and nonzero base can cause + * problems with older kernels, so we mark these for preening + * even though there's no inconsistency. + */ + if (leafhdr.freemap[i].size == 0 && + leafhdr.freemap[i].base > 0) + xchk_da_set_preen(ds, level); + + if (ds->sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT) + goto out; + } + + /* Look for bits that are set in freemap and are marked in use. */ + if (bitmap_intersects(ab->freemap, ab->usedmap, + mp->m_attr_geo->blksize)) xchk_da_set_corrupt(ds, level); if (leafhdr.usedbytes != usedbytes) diff --git a/fs/xfs/scrub/attr_repair.c b/fs/xfs/scrub/attr_repair.c index 09d63aa10314b0..dd24044c44efd3 100644 --- a/fs/xfs/scrub/attr_repair.c +++ b/fs/xfs/scrub/attr_repair.c @@ -1516,8 +1516,10 @@ xrep_xattr_teardown( xfblob_destroy(rx->pptr_names); if (rx->pptr_recs) xfarray_destroy(rx->pptr_recs); - xfblob_destroy(rx->xattr_blobs); - xfarray_destroy(rx->xattr_records); + if (rx->xattr_blobs) + xfblob_destroy(rx->xattr_blobs); + if (rx->xattr_records) + xfarray_destroy(rx->xattr_records); mutex_destroy(&rx->lock); kfree(rx); } @@ -1529,7 +1531,6 @@ xrep_xattr_setup_scan( struct xrep_xattr **rxp) { struct xrep_xattr *rx; - char *descr; int max_len; int error; @@ -1555,35 +1556,26 @@ xrep_xattr_setup_scan( goto out_rx; /* Set up some staging for salvaged attribute keys and values */ - descr = xchk_xfile_ino_descr(sc, "xattr keys"); - error = xfarray_create(descr, 0, sizeof(struct xrep_xattr_key), + error = xfarray_create("xattr keys", 0, sizeof(struct xrep_xattr_key), &rx->xattr_records); - kfree(descr); if (error) goto out_rx; - descr = xchk_xfile_ino_descr(sc, "xattr names"); - error = xfblob_create(descr, &rx->xattr_blobs); - kfree(descr); + error = xfblob_create("xattr names", &rx->xattr_blobs); if (error) goto out_keys; if (xfs_has_parent(sc->mp)) { ASSERT(sc->flags & XCHK_FSGATES_DIRENTS); - descr = xchk_xfile_ino_descr(sc, - "xattr retained parent pointer entries"); - error = xfarray_create(descr, 0, + error = xfarray_create("xattr parent pointer entries", 0, sizeof(struct xrep_xattr_pptr), &rx->pptr_recs); - kfree(descr); if (error) goto out_values; - descr = xchk_xfile_ino_descr(sc, - "xattr retained parent pointer names"); - error = xfblob_create(descr, &rx->pptr_names); - kfree(descr); + error = xfblob_create("xattr parent pointer names", + &rx->pptr_names); if (error) goto out_pprecs; diff --git a/fs/xfs/scrub/bmap_repair.c b/fs/xfs/scrub/bmap_repair.c index 1084213b8e9b88..747cd9389b491d 100644 --- a/fs/xfs/scrub/bmap_repair.c +++ b/fs/xfs/scrub/bmap_repair.c @@ -923,7 +923,6 @@ xrep_bmap( bool allow_unwritten) { struct xrep_bmap *rb; - char *descr; xfs_extnum_t max_bmbt_recs; bool large_extcount; int error = 0; @@ -945,11 +944,8 @@ xrep_bmap( /* Set up enough storage to handle the max records for this fork. */ large_extcount = xfs_has_large_extent_counts(sc->mp); max_bmbt_recs = xfs_iext_max_nextents(large_extcount, whichfork); - descr = xchk_xfile_ino_descr(sc, "%s fork mapping records", - whichfork == XFS_DATA_FORK ? "data" : "attr"); - error = xfarray_create(descr, max_bmbt_recs, + error = xfarray_create("fork mapping records", max_bmbt_recs, sizeof(struct xfs_bmbt_rec), &rb->bmap_records); - kfree(descr); if (error) goto out_rb; diff --git a/fs/xfs/scrub/btree.c b/fs/xfs/scrub/btree.c index cd6f0ff382a7c8..b497f6a474c778 100644 --- a/fs/xfs/scrub/btree.c +++ b/fs/xfs/scrub/btree.c @@ -42,6 +42,8 @@ __xchk_btree_process_error( break; case -EFSBADCRC: case -EFSCORRUPTED: + case -EIO: + case -ENODATA: /* Note the badness but don't abort. */ sc->sm->sm_flags |= errflag; *error = 0; @@ -370,12 +372,15 @@ xchk_btree_check_block_owner( { xfs_agnumber_t agno; xfs_agblock_t agbno; + bool is_bnobt, is_rmapbt; bool init_sa; int error = 0; if (!bs->cur) return 0; + is_bnobt = xfs_btree_is_bno(bs->cur->bc_ops); + is_rmapbt = xfs_btree_is_rmap(bs->cur->bc_ops); agno = xfs_daddr_to_agno(bs->cur->bc_mp, daddr); agbno = xfs_daddr_to_agbno(bs->cur->bc_mp, daddr); @@ -398,11 +403,11 @@ xchk_btree_check_block_owner( * have to nullify it (to shut down further block owner checks) if * self-xref encounters problems. */ - if (!bs->sc->sa.bno_cur && xfs_btree_is_bno(bs->cur->bc_ops)) + if (!bs->sc->sa.bno_cur && is_bnobt) bs->cur = NULL; xchk_xref_is_only_owned_by(bs->sc, agbno, 1, bs->oinfo); - if (!bs->sc->sa.rmap_cur && xfs_btree_is_rmap(bs->cur->bc_ops)) + if (!bs->sc->sa.rmap_cur && is_rmapbt) bs->cur = NULL; out_free: diff --git a/fs/xfs/scrub/common.c b/fs/xfs/scrub/common.c index 7bfa37c99480f0..ebabf3b620a2cf 100644 --- a/fs/xfs/scrub/common.c +++ b/fs/xfs/scrub/common.c @@ -103,6 +103,8 @@ __xchk_process_error( break; case -EFSBADCRC: case -EFSCORRUPTED: + case -EIO: + case -ENODATA: /* Note the badness but don't abort. */ sc->sm->sm_flags |= errflag; *error = 0; @@ -177,6 +179,8 @@ __xchk_fblock_process_error( break; case -EFSBADCRC: case -EFSCORRUPTED: + case -EIO: + case -ENODATA: /* Note the badness but don't abort. */ sc->sm->sm_flags |= errflag; *error = 0; @@ -1395,6 +1399,9 @@ xchk_metadata_inode_subtype( int error; sub = xchk_scrub_create_subord(sc, scrub_type); + if (!sub) + return -ENOMEM; + error = sub->sc.ops->scrub(&sub->sc); xchk_scrub_free_subord(sub); return error; diff --git a/fs/xfs/scrub/common.h b/fs/xfs/scrub/common.h index ddbc065c798cd1..f2ecc68538f0c3 100644 --- a/fs/xfs/scrub/common.h +++ b/fs/xfs/scrub/common.h @@ -246,31 +246,6 @@ static inline bool xchk_could_repair(const struct xfs_scrub *sc) int xchk_metadata_inode_forks(struct xfs_scrub *sc); -/* - * Helper macros to allocate and format xfile description strings. - * Callers must kfree the pointer returned. - */ -#define xchk_xfile_descr(sc, fmt, ...) \ - kasprintf(XCHK_GFP_FLAGS, "XFS (%s): " fmt, \ - (sc)->mp->m_super->s_id, ##__VA_ARGS__) -#define xchk_xfile_ag_descr(sc, fmt, ...) \ - kasprintf(XCHK_GFP_FLAGS, "XFS (%s): AG 0x%x " fmt, \ - (sc)->mp->m_super->s_id, \ - (sc)->sa.pag ? \ - pag_agno((sc)->sa.pag) : (sc)->sm->sm_agno, \ - ##__VA_ARGS__) -#define xchk_xfile_ino_descr(sc, fmt, ...) \ - kasprintf(XCHK_GFP_FLAGS, "XFS (%s): inode 0x%llx " fmt, \ - (sc)->mp->m_super->s_id, \ - (sc)->ip ? (sc)->ip->i_ino : (sc)->sm->sm_ino, \ - ##__VA_ARGS__) -#define xchk_xfile_rtgroup_descr(sc, fmt, ...) \ - kasprintf(XCHK_GFP_FLAGS, "XFS (%s): rtgroup 0x%x " fmt, \ - (sc)->mp->m_super->s_id, \ - (sc)->sa.pag ? \ - rtg_rgno((sc)->sr.rtg) : (sc)->sm->sm_agno, \ - ##__VA_ARGS__) - /* * Setting up a hook to wait for intents to drain is costly -- we have to take * the CPU hotplug lock and force an i-cache flush on all CPUs once to set it diff --git a/fs/xfs/scrub/dabtree.c b/fs/xfs/scrub/dabtree.c index 056de4819f866d..a6a5d3a75d994e 100644 --- a/fs/xfs/scrub/dabtree.c +++ b/fs/xfs/scrub/dabtree.c @@ -45,6 +45,8 @@ xchk_da_process_error( break; case -EFSBADCRC: case -EFSCORRUPTED: + case -EIO: + case -ENODATA: /* Note the badness but don't abort. */ sc->sm->sm_flags |= XFS_SCRUB_OFLAG_CORRUPT; *error = 0; diff --git a/fs/xfs/scrub/dir.c b/fs/xfs/scrub/dir.c index c877bde71e6280..4f849d98cbdd22 100644 --- a/fs/xfs/scrub/dir.c +++ b/fs/xfs/scrub/dir.c @@ -1102,22 +1102,17 @@ xchk_directory( sd->xname.name = sd->namebuf; if (xfs_has_parent(sc->mp)) { - char *descr; - /* * Set up some staging memory for dirents that we can't check * due to locking contention. */ - descr = xchk_xfile_ino_descr(sc, "slow directory entries"); - error = xfarray_create(descr, 0, sizeof(struct xchk_dirent), - &sd->dir_entries); - kfree(descr); + error = xfarray_create("slow directory entries", 0, + sizeof(struct xchk_dirent), &sd->dir_entries); if (error) goto out_sd; - descr = xchk_xfile_ino_descr(sc, "slow directory entry names"); - error = xfblob_create(descr, &sd->dir_names); - kfree(descr); + error = xfblob_create("slow directory entry names", + &sd->dir_names); if (error) goto out_entries; } diff --git a/fs/xfs/scrub/dir_repair.c b/fs/xfs/scrub/dir_repair.c index 8d3b550990b58a..e0fa5e6ca1fe8d 100644 --- a/fs/xfs/scrub/dir_repair.c +++ b/fs/xfs/scrub/dir_repair.c @@ -172,8 +172,12 @@ xrep_dir_teardown( struct xrep_dir *rd = sc->buf; xrep_findparent_scan_teardown(&rd->pscan); - xfblob_destroy(rd->dir_names); - xfarray_destroy(rd->dir_entries); + if (rd->dir_names) + xfblob_destroy(rd->dir_names); + rd->dir_names = NULL; + if (rd->dir_entries) + xfarray_destroy(rd->dir_entries); + rd->dir_entries = NULL; } /* Set up for a directory repair. */ @@ -1784,20 +1788,15 @@ xrep_dir_setup_scan( struct xrep_dir *rd) { struct xfs_scrub *sc = rd->sc; - char *descr; int error; /* Set up some staging memory for salvaging dirents. */ - descr = xchk_xfile_ino_descr(sc, "directory entries"); - error = xfarray_create(descr, 0, sizeof(struct xrep_dirent), - &rd->dir_entries); - kfree(descr); + error = xfarray_create("directory entries", 0, + sizeof(struct xrep_dirent), &rd->dir_entries); if (error) return error; - descr = xchk_xfile_ino_descr(sc, "directory entry names"); - error = xfblob_create(descr, &rd->dir_names); - kfree(descr); + error = xfblob_create("directory entry names", &rd->dir_names); if (error) goto out_xfarray; diff --git a/fs/xfs/scrub/dirtree.c b/fs/xfs/scrub/dirtree.c index 3a9cdf8738b6db..3e0bbe75c44cff 100644 --- a/fs/xfs/scrub/dirtree.c +++ b/fs/xfs/scrub/dirtree.c @@ -81,8 +81,12 @@ xchk_dirtree_buf_cleanup( kfree(path); } - xfblob_destroy(dl->path_names); - xfarray_destroy(dl->path_steps); + if (dl->path_names) + xfblob_destroy(dl->path_names); + dl->path_names = NULL; + if (dl->path_steps) + xfarray_destroy(dl->path_steps); + dl->path_steps = NULL; mutex_destroy(&dl->lock); } @@ -92,7 +96,6 @@ xchk_setup_dirtree( struct xfs_scrub *sc) { struct xchk_dirtree *dl; - char *descr; int error; xchk_fsgates_enable(sc, XCHK_FSGATES_DIRENTS); @@ -116,16 +119,12 @@ xchk_setup_dirtree( mutex_init(&dl->lock); - descr = xchk_xfile_ino_descr(sc, "dirtree path steps"); - error = xfarray_create(descr, 0, sizeof(struct xchk_dirpath_step), - &dl->path_steps); - kfree(descr); + error = xfarray_create("dirtree path steps", 0, + sizeof(struct xchk_dirpath_step), &dl->path_steps); if (error) goto out_dl; - descr = xchk_xfile_ino_descr(sc, "dirtree path names"); - error = xfblob_create(descr, &dl->path_names); - kfree(descr); + error = xfblob_create("dirtree path names", &dl->path_names); if (error) goto out_steps; diff --git a/fs/xfs/scrub/ialloc_repair.c b/fs/xfs/scrub/ialloc_repair.c index 14e48d3f1912bf..f28459f58832f4 100644 --- a/fs/xfs/scrub/ialloc_repair.c +++ b/fs/xfs/scrub/ialloc_repair.c @@ -797,7 +797,6 @@ xrep_iallocbt( { struct xrep_ibt *ri; struct xfs_mount *mp = sc->mp; - char *descr; xfs_agino_t first_agino, last_agino; int error = 0; @@ -816,11 +815,9 @@ xrep_iallocbt( /* Set up enough storage to handle an AG with nothing but inodes. */ xfs_agino_range(mp, pag_agno(sc->sa.pag), &first_agino, &last_agino); last_agino /= XFS_INODES_PER_CHUNK; - descr = xchk_xfile_ag_descr(sc, "inode index records"); - error = xfarray_create(descr, last_agino, + error = xfarray_create("inode index records", last_agino, sizeof(struct xfs_inobt_rec_incore), &ri->inode_records); - kfree(descr); if (error) goto out_ri; @@ -866,10 +863,24 @@ xrep_revalidate_iallocbt( if (error) goto out; - if (xfs_has_finobt(sc->mp)) { - sc->sm->sm_type = XFS_SCRUB_TYPE_FINOBT; - error = xchk_iallocbt(sc); + /* + * If the inobt is still corrupt, we've failed to repair the filesystem + * and should just bail out. + * + * If the inobt fails cross-examination with the finobt, the scan will + * free the finobt cursor, so we need to mark the repair incomplete + * and avoid walking off the end of the NULL finobt cursor. + */ + if (!xfs_has_finobt(sc->mp) || + (sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT)) + goto out; + + sc->sm->sm_type = XFS_SCRUB_TYPE_FINOBT; + if (!sc->sa.fino_cur) { + xchk_set_incomplete(sc); + goto out; } + error = xchk_iallocbt(sc); out: sc->sm->sm_type = old_type; diff --git a/fs/xfs/scrub/nlinks.c b/fs/xfs/scrub/nlinks.c index 091c79e432e592..dec3b9b47453ea 100644 --- a/fs/xfs/scrub/nlinks.c +++ b/fs/xfs/scrub/nlinks.c @@ -971,7 +971,8 @@ xchk_nlinks_teardown_scan( xfs_dir_hook_del(xnc->sc->mp, &xnc->dhook); - xfarray_destroy(xnc->nlinks); + if (xnc->nlinks) + xfarray_destroy(xnc->nlinks); xnc->nlinks = NULL; xchk_iscan_teardown(&xnc->collect_iscan); @@ -990,7 +991,6 @@ xchk_nlinks_setup_scan( struct xchk_nlink_ctrs *xnc) { struct xfs_mount *mp = sc->mp; - char *descr; unsigned long long max_inos; xfs_agnumber_t last_agno = mp->m_sb.sb_agcount - 1; xfs_agino_t first_agino, last_agino; @@ -1007,10 +1007,9 @@ xchk_nlinks_setup_scan( */ xfs_agino_range(mp, last_agno, &first_agino, &last_agino); max_inos = XFS_AGINO_TO_INO(mp, last_agno, last_agino) + 1; - descr = xchk_xfile_descr(sc, "file link counts"); - error = xfarray_create(descr, min(XFS_MAXINUMBER + 1, max_inos), + error = xfarray_create("file link counts", + min(XFS_MAXINUMBER + 1, max_inos), sizeof(struct xchk_nlink), &xnc->nlinks); - kfree(descr); if (error) goto out_teardown; diff --git a/fs/xfs/scrub/orphanage.c b/fs/xfs/scrub/orphanage.c index 4e550a1d5353b7..3fa78bfe5f663d 100644 --- a/fs/xfs/scrub/orphanage.c +++ b/fs/xfs/scrub/orphanage.c @@ -442,6 +442,11 @@ xrep_adoption_check_dcache( return 0; d_child = try_lookup_noperm(&qname, d_orphanage); + if (IS_ERR(d_child)) { + dput(d_orphanage); + return PTR_ERR(d_child); + } + if (d_child) { trace_xrep_adoption_check_child(sc->mp, d_child); @@ -479,7 +484,7 @@ xrep_adoption_zap_dcache( return; d_child = try_lookup_noperm(&qname, d_orphanage); - while (d_child != NULL) { + while (!IS_ERR_OR_NULL(d_child)) { trace_xrep_adoption_invalidate_child(sc->mp, d_child); ASSERT(d_is_negative(d_child)); diff --git a/fs/xfs/scrub/parent.c b/fs/xfs/scrub/parent.c index 11d5de10fd567b..23c195d14494e5 100644 --- a/fs/xfs/scrub/parent.c +++ b/fs/xfs/scrub/parent.c @@ -755,7 +755,6 @@ xchk_parent_pptr( struct xfs_scrub *sc) { struct xchk_pptrs *pp; - char *descr; int error; pp = kvzalloc(sizeof(struct xchk_pptrs), XCHK_GFP_FLAGS); @@ -768,16 +767,12 @@ xchk_parent_pptr( * Set up some staging memory for parent pointers that we can't check * due to locking contention. */ - descr = xchk_xfile_ino_descr(sc, "slow parent pointer entries"); - error = xfarray_create(descr, 0, sizeof(struct xchk_pptr), - &pp->pptr_entries); - kfree(descr); + error = xfarray_create("slow parent pointer entries", 0, + sizeof(struct xchk_pptr), &pp->pptr_entries); if (error) goto out_pp; - descr = xchk_xfile_ino_descr(sc, "slow parent pointer names"); - error = xfblob_create(descr, &pp->pptr_names); - kfree(descr); + error = xfblob_create("slow parent pointer names", &pp->pptr_names); if (error) goto out_entries; diff --git a/fs/xfs/scrub/parent_repair.c b/fs/xfs/scrub/parent_repair.c index 2949feda627175..897902c54178d4 100644 --- a/fs/xfs/scrub/parent_repair.c +++ b/fs/xfs/scrub/parent_repair.c @@ -1497,7 +1497,6 @@ xrep_parent_setup_scan( struct xrep_parent *rp) { struct xfs_scrub *sc = rp->sc; - char *descr; struct xfs_da_geometry *geo = sc->mp->m_attr_geo; int max_len; int error; @@ -1525,32 +1524,22 @@ xrep_parent_setup_scan( goto out_xattr_name; /* Set up some staging memory for logging parent pointer updates. */ - descr = xchk_xfile_ino_descr(sc, "parent pointer entries"); - error = xfarray_create(descr, 0, sizeof(struct xrep_pptr), - &rp->pptr_recs); - kfree(descr); + error = xfarray_create("parent pointer entries", 0, + sizeof(struct xrep_pptr), &rp->pptr_recs); if (error) goto out_xattr_value; - descr = xchk_xfile_ino_descr(sc, "parent pointer names"); - error = xfblob_create(descr, &rp->pptr_names); - kfree(descr); + error = xfblob_create("parent pointer names", &rp->pptr_names); if (error) goto out_recs; /* Set up some storage for copying attrs before the mapping exchange */ - descr = xchk_xfile_ino_descr(sc, - "parent pointer retained xattr entries"); - error = xfarray_create(descr, 0, sizeof(struct xrep_parent_xattr), - &rp->xattr_records); - kfree(descr); + error = xfarray_create("parent pointer xattr entries", 0, + sizeof(struct xrep_parent_xattr), &rp->xattr_records); if (error) goto out_names; - descr = xchk_xfile_ino_descr(sc, - "parent pointer retained xattr values"); - error = xfblob_create(descr, &rp->xattr_blobs); - kfree(descr); + error = xfblob_create("parent pointer xattr values", &rp->xattr_blobs); if (error) goto out_attr_keys; diff --git a/fs/xfs/scrub/quota.c b/fs/xfs/scrub/quota.c index 5c5374c44c5a7c..088701116d4362 100644 --- a/fs/xfs/scrub/quota.c +++ b/fs/xfs/scrub/quota.c @@ -171,8 +171,10 @@ xchk_quota_item( error = xchk_quota_item_bmap(sc, dq, offset); xchk_iunlock(sc, XFS_ILOCK_SHARED); - if (!xchk_fblock_process_error(sc, XFS_DATA_FORK, offset, &error)) + if (!xchk_fblock_process_error(sc, XFS_DATA_FORK, offset, &error)) { + mutex_unlock(&dq->q_qlock); return error; + } /* * Warn if the hard limits are larger than the fs. diff --git a/fs/xfs/scrub/quotacheck.c b/fs/xfs/scrub/quotacheck.c index d412a8359784ee..3b2f4ccde2ec09 100644 --- a/fs/xfs/scrub/quotacheck.c +++ b/fs/xfs/scrub/quotacheck.c @@ -741,7 +741,6 @@ xqcheck_setup_scan( struct xfs_scrub *sc, struct xqcheck *xqc) { - char *descr; struct xfs_quotainfo *qi = sc->mp->m_quotainfo; unsigned long long max_dquots = XFS_DQ_ID_MAX + 1ULL; int error; @@ -756,28 +755,22 @@ xqcheck_setup_scan( error = -ENOMEM; if (xfs_this_quota_on(sc->mp, XFS_DQTYPE_USER)) { - descr = xchk_xfile_descr(sc, "user dquot records"); - error = xfarray_create(descr, max_dquots, + error = xfarray_create("user dquot records", max_dquots, sizeof(struct xqcheck_dquot), &xqc->ucounts); - kfree(descr); if (error) goto out_teardown; } if (xfs_this_quota_on(sc->mp, XFS_DQTYPE_GROUP)) { - descr = xchk_xfile_descr(sc, "group dquot records"); - error = xfarray_create(descr, max_dquots, + error = xfarray_create("group dquot records", max_dquots, sizeof(struct xqcheck_dquot), &xqc->gcounts); - kfree(descr); if (error) goto out_teardown; } if (xfs_this_quota_on(sc->mp, XFS_DQTYPE_PROJ)) { - descr = xchk_xfile_descr(sc, "project dquot records"); - error = xfarray_create(descr, max_dquots, + error = xfarray_create("project dquot records", max_dquots, sizeof(struct xqcheck_dquot), &xqc->pcounts); - kfree(descr); if (error) goto out_teardown; } diff --git a/fs/xfs/scrub/refcount_repair.c b/fs/xfs/scrub/refcount_repair.c index 9c8cb5332da042..360fd7354880a7 100644 --- a/fs/xfs/scrub/refcount_repair.c +++ b/fs/xfs/scrub/refcount_repair.c @@ -123,13 +123,7 @@ int xrep_setup_ag_refcountbt( struct xfs_scrub *sc) { - char *descr; - int error; - - descr = xchk_xfile_ag_descr(sc, "rmap record bag"); - error = xrep_setup_xfbtree(sc, descr); - kfree(descr); - return error; + return xrep_setup_xfbtree(sc, "rmap record bag"); } /* Check for any obvious conflicts with this shared/CoW staging extent. */ @@ -704,7 +698,6 @@ xrep_refcountbt( { struct xrep_refc *rr; struct xfs_mount *mp = sc->mp; - char *descr; int error; /* We require the rmapbt to rebuild anything. */ @@ -717,11 +710,9 @@ xrep_refcountbt( rr->sc = sc; /* Set up enough storage to handle one refcount record per block. */ - descr = xchk_xfile_ag_descr(sc, "reference count records"); - error = xfarray_create(descr, mp->m_sb.sb_agblocks, + error = xfarray_create("reference count records", mp->m_sb.sb_agblocks, sizeof(struct xfs_refcount_irec), &rr->refcount_records); - kfree(descr); if (error) goto out_rr; diff --git a/fs/xfs/scrub/repair.c b/fs/xfs/scrub/repair.c index efd5a7ccdf624a..4d45d39e67f11e 100644 --- a/fs/xfs/scrub/repair.c +++ b/fs/xfs/scrub/repair.c @@ -1136,6 +1136,9 @@ xrep_metadata_inode_subtype( * setup/teardown routines. */ sub = xchk_scrub_create_subord(sc, scrub_type); + if (!sub) + return -ENOMEM; + error = sub->sc.ops->scrub(&sub->sc); if (error) goto out; diff --git a/fs/xfs/scrub/rmap_repair.c b/fs/xfs/scrub/rmap_repair.c index 17d4a38d735cb8..cfd1cf403b37eb 100644 --- a/fs/xfs/scrub/rmap_repair.c +++ b/fs/xfs/scrub/rmap_repair.c @@ -164,14 +164,11 @@ xrep_setup_ag_rmapbt( struct xfs_scrub *sc) { struct xrep_rmap *rr; - char *descr; int error; xchk_fsgates_enable(sc, XCHK_FSGATES_RMAP); - descr = xchk_xfile_ag_descr(sc, "reverse mapping records"); - error = xrep_setup_xfbtree(sc, descr); - kfree(descr); + error = xrep_setup_xfbtree(sc, "reverse mapping records"); if (error) return error; diff --git a/fs/xfs/scrub/rtbitmap_repair.c b/fs/xfs/scrub/rtbitmap_repair.c index 203a1a97c5026e..41d6736a529d02 100644 --- a/fs/xfs/scrub/rtbitmap_repair.c +++ b/fs/xfs/scrub/rtbitmap_repair.c @@ -43,7 +43,6 @@ xrep_setup_rtbitmap( struct xchk_rtbitmap *rtb) { struct xfs_mount *mp = sc->mp; - char *descr; unsigned long long blocks = mp->m_sb.sb_rbmblocks; int error; @@ -52,9 +51,8 @@ xrep_setup_rtbitmap( return error; /* Create an xfile to hold our reconstructed bitmap. */ - descr = xchk_xfile_rtgroup_descr(sc, "bitmap file"); - error = xfile_create(descr, blocks * mp->m_sb.sb_blocksize, &sc->xfile); - kfree(descr); + error = xfile_create("realtime bitmap file", + blocks * mp->m_sb.sb_blocksize, &sc->xfile); if (error) return error; diff --git a/fs/xfs/scrub/rtrefcount_repair.c b/fs/xfs/scrub/rtrefcount_repair.c index 983362447826de..b35e39cce7ad5a 100644 --- a/fs/xfs/scrub/rtrefcount_repair.c +++ b/fs/xfs/scrub/rtrefcount_repair.c @@ -128,13 +128,7 @@ int xrep_setup_rtrefcountbt( struct xfs_scrub *sc) { - char *descr; - int error; - - descr = xchk_xfile_ag_descr(sc, "rmap record bag"); - error = xrep_setup_xfbtree(sc, descr); - kfree(descr); - return error; + return xrep_setup_xfbtree(sc, "realtime rmap record bag"); } /* Check for any obvious conflicts with this shared/CoW staging extent. */ @@ -704,7 +698,6 @@ xrep_rtrefcountbt( { struct xrep_rtrefc *rr; struct xfs_mount *mp = sc->mp; - char *descr; int error; /* We require the rmapbt to rebuild anything. */ @@ -722,11 +715,9 @@ xrep_rtrefcountbt( rr->sc = sc; /* Set up enough storage to handle one refcount record per rt extent. */ - descr = xchk_xfile_ag_descr(sc, "reference count records"); - error = xfarray_create(descr, mp->m_sb.sb_rextents, - sizeof(struct xfs_refcount_irec), + error = xfarray_create("realtime reference count records", + mp->m_sb.sb_rextents, sizeof(struct xfs_refcount_irec), &rr->refcount_records); - kfree(descr); if (error) goto out_rr; diff --git a/fs/xfs/scrub/rtrmap_repair.c b/fs/xfs/scrub/rtrmap_repair.c index 7561941a337a1f..749977a66e40ff 100644 --- a/fs/xfs/scrub/rtrmap_repair.c +++ b/fs/xfs/scrub/rtrmap_repair.c @@ -103,14 +103,11 @@ xrep_setup_rtrmapbt( struct xfs_scrub *sc) { struct xrep_rtrmap *rr; - char *descr; int error; xchk_fsgates_enable(sc, XCHK_FSGATES_RMAP); - descr = xchk_xfile_rtgroup_descr(sc, "reverse mapping records"); - error = xrep_setup_xfbtree(sc, descr); - kfree(descr); + error = xrep_setup_xfbtree(sc, "realtime reverse mapping records"); if (error) return error; diff --git a/fs/xfs/scrub/rtsummary.c b/fs/xfs/scrub/rtsummary.c index 4ac679c1bd29cd..fb78cff2ac3a16 100644 --- a/fs/xfs/scrub/rtsummary.c +++ b/fs/xfs/scrub/rtsummary.c @@ -43,7 +43,6 @@ xchk_setup_rtsummary( struct xfs_scrub *sc) { struct xfs_mount *mp = sc->mp; - char *descr; struct xchk_rtsummary *rts; int error; @@ -70,10 +69,8 @@ xchk_setup_rtsummary( * Create an xfile to construct a new rtsummary file. The xfile allows * us to avoid pinning kernel memory for this purpose. */ - descr = xchk_xfile_descr(sc, "realtime summary file"); - error = xfile_create(descr, XFS_FSB_TO_B(mp, mp->m_rsumblocks), - &sc->xfile); - kfree(descr); + error = xfile_create("realtime summary file", + XFS_FSB_TO_B(mp, mp->m_rsumblocks), &sc->xfile); if (error) return error; diff --git a/fs/xfs/scrub/scrub.c b/fs/xfs/scrub/scrub.c index 3c3b0d25006ff4..c312f0a672e65f 100644 --- a/fs/xfs/scrub/scrub.c +++ b/fs/xfs/scrub/scrub.c @@ -634,7 +634,7 @@ xchk_scrub_create_subord( sub = kzalloc(sizeof(*sub), XCHK_GFP_FLAGS); if (!sub) - return ERR_PTR(-ENOMEM); + return NULL; sub->old_smtype = sc->sm->sm_type; sub->old_smflags = sc->sm->sm_flags; diff --git a/fs/xfs/scrub/trace.h b/fs/xfs/scrub/trace.h index 39ea651cbb7510..286c5f5e054449 100644 --- a/fs/xfs/scrub/trace.h +++ b/fs/xfs/scrub/trace.h @@ -972,20 +972,12 @@ TRACE_EVENT(xfile_create, TP_STRUCT__entry( __field(dev_t, dev) __field(unsigned long, ino) - __array(char, pathname, MAXNAMELEN) ), TP_fast_assign( - char *path; - __entry->ino = file_inode(xf->file)->i_ino; - path = file_path(xf->file, __entry->pathname, MAXNAMELEN); - if (IS_ERR(path)) - strncpy(__entry->pathname, "(unknown)", - sizeof(__entry->pathname)); ), - TP_printk("xfino 0x%lx path '%s'", - __entry->ino, - __entry->pathname) + TP_printk("xfino 0x%lx", + __entry->ino) ); TRACE_EVENT(xfile_destroy, diff --git a/fs/xfs/xfs_attr_inactive.c b/fs/xfs/xfs_attr_inactive.c index 319004bf089fa3..2305ddad8a13a5 100644 --- a/fs/xfs/xfs_attr_inactive.c +++ b/fs/xfs/xfs_attr_inactive.c @@ -140,7 +140,7 @@ xfs_attr3_node_inactive( xfs_daddr_t parent_blkno, child_blkno; struct xfs_buf *child_bp; struct xfs_da3_icnode_hdr ichdr; - int error, i; + int error; /* * Since this code is recursive (gasp!) we must protect ourselves. @@ -152,7 +152,7 @@ xfs_attr3_node_inactive( return -EFSCORRUPTED; } - xfs_da3_node_hdr_from_disk(dp->i_mount, &ichdr, bp->b_addr); + xfs_da3_node_hdr_from_disk(mp, &ichdr, bp->b_addr); parent_blkno = xfs_buf_daddr(bp); if (!ichdr.count) { xfs_trans_brelse(*trans, bp); @@ -167,7 +167,7 @@ xfs_attr3_node_inactive( * over the leaves removing all of them. If this is higher up * in the tree, recurse downward. */ - for (i = 0; i < ichdr.count; i++) { + while (ichdr.count > 0) { /* * Read the subsidiary block to see what we have to work with. * Don't do this in a transaction. This is a depth-first @@ -218,29 +218,32 @@ xfs_attr3_node_inactive( xfs_trans_binval(*trans, child_bp); child_bp = NULL; + error = xfs_da3_node_read_mapped(*trans, dp, + parent_blkno, &bp, XFS_ATTR_FORK); + if (error) + return error; + /* - * If we're not done, re-read the parent to get the next - * child block number. + * Remove entry from parent node, prevents being indexed to. */ - if (i + 1 < ichdr.count) { - struct xfs_da3_icnode_hdr phdr; + xfs_attr3_node_entry_remove(*trans, dp, bp, 0); + + xfs_da3_node_hdr_from_disk(mp, &ichdr, bp->b_addr); + bp = NULL; - error = xfs_da3_node_read_mapped(*trans, dp, - parent_blkno, &bp, XFS_ATTR_FORK); + if (ichdr.count > 0) { + /* + * If we're not done, get the next child block number. + */ + child_fsb = be32_to_cpu(ichdr.btree[0].before); + + /* + * Atomically commit the whole invalidate stuff. + */ + error = xfs_trans_roll_inode(trans, dp); if (error) return error; - xfs_da3_node_hdr_from_disk(dp->i_mount, &phdr, - bp->b_addr); - child_fsb = be32_to_cpu(phdr.btree[i + 1].before); - xfs_trans_brelse(*trans, bp); - bp = NULL; } - /* - * Atomically commit the whole invalidate stuff. - */ - error = xfs_trans_roll_inode(trans, dp); - if (error) - return error; } return 0; @@ -257,10 +260,8 @@ xfs_attr3_root_inactive( struct xfs_trans **trans, struct xfs_inode *dp) { - struct xfs_mount *mp = dp->i_mount; struct xfs_da_blkinfo *info; struct xfs_buf *bp; - xfs_daddr_t blkno; int error; /* @@ -272,7 +273,6 @@ xfs_attr3_root_inactive( error = xfs_da3_node_read(*trans, dp, 0, &bp, XFS_ATTR_FORK); if (error) return error; - blkno = xfs_buf_daddr(bp); /* * Invalidate the tree, even if the "tree" is only a single leaf block. @@ -283,10 +283,26 @@ xfs_attr3_root_inactive( case cpu_to_be16(XFS_DA_NODE_MAGIC): case cpu_to_be16(XFS_DA3_NODE_MAGIC): error = xfs_attr3_node_inactive(trans, dp, bp, 1); + /* + * Empty root node block are not allowed, convert it to leaf. + */ + if (!error) + error = xfs_attr3_leaf_init(*trans, dp, 0); + if (!error) + error = xfs_trans_roll_inode(trans, dp); break; case cpu_to_be16(XFS_ATTR_LEAF_MAGIC): case cpu_to_be16(XFS_ATTR3_LEAF_MAGIC): error = xfs_attr3_leaf_inactive(trans, dp, bp); + /* + * Reinit the leaf before truncating extents so that a crash + * mid-truncation leaves an empty leaf rather than one with + * entries that may reference freed remote value blocks. + */ + if (!error) + error = xfs_attr3_leaf_init(*trans, dp, 0); + if (!error) + error = xfs_trans_roll_inode(trans, dp); break; default: xfs_dirattr_mark_sick(dp, XFS_ATTR_FORK); @@ -295,21 +311,6 @@ xfs_attr3_root_inactive( xfs_trans_brelse(*trans, bp); break; } - if (error) - return error; - - /* - * Invalidate the incore copy of the root block. - */ - error = xfs_trans_get_buf(*trans, mp->m_ddev_targp, blkno, - XFS_FSB_TO_BB(mp, mp->m_attr_geo->fsbcount), 0, &bp); - if (error) - return error; - xfs_trans_binval(*trans, bp); /* remove from cache */ - /* - * Commit the invalidate and start the next transaction. - */ - error = xfs_trans_roll_inode(trans, dp); return error; } @@ -328,6 +329,7 @@ xfs_attr_inactive( { struct xfs_trans *trans; struct xfs_mount *mp; + struct xfs_buf *bp; int lock_mode = XFS_ILOCK_SHARED; int error = 0; @@ -363,10 +365,27 @@ xfs_attr_inactive( * removal below. */ if (dp->i_af.if_nextents > 0) { + /* + * Invalidate and truncate all blocks but leave the root block. + */ error = xfs_attr3_root_inactive(&trans, dp); if (error) goto out_cancel; + error = xfs_itruncate_extents(&trans, dp, XFS_ATTR_FORK, + XFS_FSB_TO_B(mp, mp->m_attr_geo->fsbcount)); + if (error) + goto out_cancel; + + /* + * Invalidate and truncate the root block and ensure that the + * operation is completed within a single transaction. + */ + error = xfs_da_get_buf(trans, dp, 0, &bp, XFS_ATTR_FORK); + if (error) + goto out_cancel; + + xfs_trans_binval(trans, bp); error = xfs_itruncate_extents(&trans, dp, XFS_ATTR_FORK, 0); if (error) goto out_cancel; diff --git a/fs/xfs/xfs_attr_item.c b/fs/xfs/xfs_attr_item.c index e8fa326ac995bc..8beaa474fd33cd 100644 --- a/fs/xfs/xfs_attr_item.c +++ b/fs/xfs/xfs_attr_item.c @@ -656,7 +656,6 @@ xfs_attri_recover_work( break; } if (error) { - xfs_irele(ip); XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, attrp, sizeof(*attrp)); return ERR_PTR(-EFSCORRUPTED); @@ -1050,8 +1049,8 @@ xlog_recover_attri_commit_pass2( break; case XFS_ATTRI_OP_FLAGS_SET: case XFS_ATTRI_OP_FLAGS_REPLACE: - /* Log item, attr name, attr value */ - if (item->ri_total != 3) { + /* Log item, attr name, optional attr value */ + if (item->ri_total != 2 + !!attri_formatp->alfi_value_len) { XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, attri_formatp, len); return -EFSCORRUPTED; diff --git a/fs/xfs/xfs_bmap_item.c b/fs/xfs/xfs_bmap_item.c index 80f0c4bcc483fb..bcc89d65de7fbf 100644 --- a/fs/xfs/xfs_bmap_item.c +++ b/fs/xfs/xfs_bmap_item.c @@ -247,7 +247,7 @@ xfs_bmap_update_diff_items( struct xfs_bmap_intent *ba = bi_entry(a); struct xfs_bmap_intent *bb = bi_entry(b); - return ba->bi_owner->i_ino - bb->bi_owner->i_ino; + return cmp_int(ba->bi_owner->i_ino, bb->bi_owner->i_ino); } /* Log bmap updates in the intent item. */ diff --git a/fs/xfs/xfs_dquot.c b/fs/xfs/xfs_dquot.c index 612ca682a51333..d00f0a466b5829 100644 --- a/fs/xfs/xfs_dquot.c +++ b/fs/xfs/xfs_dquot.c @@ -1439,9 +1439,15 @@ xfs_qm_dqflush( return 0; out_abort: + /* + * Shut down the log before removing the dquot item from the AIL. + * Otherwise, the log tail may advance past this item's LSN while + * log writes are still in progress, making these unflushed changes + * unrecoverable on the next mount. + */ + xfs_force_shutdown(mp, SHUTDOWN_CORRUPT_INCORE); dqp->q_flags &= ~XFS_DQFLAG_DIRTY; xfs_trans_ail_delete(lip, 0); - xfs_force_shutdown(mp, SHUTDOWN_CORRUPT_INCORE); xfs_dqfunlock(dqp); return error; } diff --git a/fs/xfs/xfs_dquot_item.c b/fs/xfs/xfs_dquot_item.c index b374cd9f190034..4754ea8ab59ab2 100644 --- a/fs/xfs/xfs_dquot_item.c +++ b/fs/xfs/xfs_dquot_item.c @@ -126,6 +126,7 @@ xfs_qm_dquot_logitem_push( struct xfs_dq_logitem *qlip = DQUOT_ITEM(lip); struct xfs_dquot *dqp = qlip->qli_dquot; struct xfs_buf *bp; + struct xfs_ail *ailp = lip->li_ailp; uint rval = XFS_ITEM_SUCCESS; int error; @@ -154,7 +155,7 @@ xfs_qm_dquot_logitem_push( goto out_unlock; } - spin_unlock(&lip->li_ailp->ail_lock); + spin_unlock(&ailp->ail_lock); error = xfs_dquot_use_attached_buf(dqp, &bp); if (error == -EAGAIN) { @@ -173,9 +174,13 @@ xfs_qm_dquot_logitem_push( rval = XFS_ITEM_FLUSHING; } xfs_buf_relse(bp); + /* + * The buffer no longer protects the log item from reclaim, so + * do not reference lip after this point. + */ out_relock_ail: - spin_lock(&lip->li_ailp->ail_lock); + spin_lock(&ailp->ail_lock); out_unlock: mutex_unlock(&dqp->q_qlock); return rval; diff --git a/fs/xfs/xfs_inode.c b/fs/xfs/xfs_inode.c index f1f88e48fe223d..c98af0ffc31579 100644 --- a/fs/xfs/xfs_inode.c +++ b/fs/xfs/xfs_inode.c @@ -1048,7 +1048,8 @@ xfs_itruncate_extents_flags( xfs_assert_ilocked(ip, XFS_ILOCK_EXCL); if (icount_read(VFS_I(ip))) xfs_assert_ilocked(ip, XFS_IOLOCK_EXCL); - ASSERT(new_size <= XFS_ISIZE(ip)); + if (whichfork == XFS_DATA_FORK) + ASSERT(new_size <= XFS_ISIZE(ip)); ASSERT(tp->t_flags & XFS_TRANS_PERM_LOG_RES); ASSERT(ip->i_itemp != NULL); ASSERT(ip->i_itemp->ili_lock_flags == 0); diff --git a/fs/xfs/xfs_inode_item.c b/fs/xfs/xfs_inode_item.c index 2eb0c6011a2e8a..ef7dd1c46d7a9c 100644 --- a/fs/xfs/xfs_inode_item.c +++ b/fs/xfs/xfs_inode_item.c @@ -749,6 +749,7 @@ xfs_inode_item_push( struct xfs_inode_log_item *iip = INODE_ITEM(lip); struct xfs_inode *ip = iip->ili_inode; struct xfs_buf *bp = lip->li_buf; + struct xfs_ail *ailp = lip->li_ailp; uint rval = XFS_ITEM_SUCCESS; int error; @@ -774,7 +775,7 @@ xfs_inode_item_push( if (!xfs_buf_trylock(bp)) return XFS_ITEM_LOCKED; - spin_unlock(&lip->li_ailp->ail_lock); + spin_unlock(&ailp->ail_lock); /* * We need to hold a reference for flushing the cluster buffer as it may @@ -798,7 +799,11 @@ xfs_inode_item_push( rval = XFS_ITEM_LOCKED; } - spin_lock(&lip->li_ailp->ail_lock); + /* + * The buffer no longer protects the log item from reclaim, so + * do not reference lip after this point. + */ + spin_lock(&ailp->ail_lock); return rval; } diff --git a/fs/xfs/xfs_log.c b/fs/xfs/xfs_log.c index d4544ccafea5d5..7668557018c647 100644 --- a/fs/xfs/xfs_log.c +++ b/fs/xfs/xfs_log.c @@ -1399,6 +1399,8 @@ xlog_alloc_log( if (xfs_has_logv2(mp) && mp->m_sb.sb_logsunit > 1) log->l_iclog_roundoff = mp->m_sb.sb_logsunit; + else if (mp->m_sb.sb_logsectsize > 0) + log->l_iclog_roundoff = mp->m_sb.sb_logsectsize; else log->l_iclog_roundoff = BBSIZE; diff --git a/fs/xfs/xfs_mount.c b/fs/xfs/xfs_mount.c index 0953f6ae94abc8..8848aea99c87f4 100644 --- a/fs/xfs/xfs_mount.c +++ b/fs/xfs/xfs_mount.c @@ -607,8 +607,9 @@ xfs_unmount_check( * have been retrying in the background. This will prevent never-ending * retries in AIL pushing from hanging the unmount. * - * Finally, we can push the AIL to clean all the remaining dirty objects, then - * reclaim the remaining inodes that are still in memory at this point in time. + * Stop inodegc and background reclaim before pushing the AIL so that they + * are not running while the AIL is being flushed. Then push the AIL to + * clean all the remaining dirty objects and reclaim the remaining inodes. */ static void xfs_unmount_flush_inodes( @@ -620,9 +621,9 @@ xfs_unmount_flush_inodes( xfs_set_unmounting(mp); - xfs_ail_push_all_sync(mp->m_ail); xfs_inodegc_stop(mp); cancel_delayed_work_sync(&mp->m_reclaim_work); + xfs_ail_push_all_sync(mp->m_ail); xfs_reclaim_inodes(mp); xfs_health_unmount(mp); } diff --git a/fs/xfs/xfs_notify_failure.c b/fs/xfs/xfs_notify_failure.c index b1767288994206..0700a723f38e73 100644 --- a/fs/xfs/xfs_notify_failure.c +++ b/fs/xfs/xfs_notify_failure.c @@ -293,7 +293,7 @@ xfs_dax_notify_dev_failure( error = xfs_alloc_read_agf(pag, tp, 0, &agf_bp); if (error) { - xfs_perag_put(pag); + xfs_perag_rele(pag); break; } @@ -329,7 +329,7 @@ xfs_dax_notify_dev_failure( if (rtg) xfs_rtgroup_unlock(rtg, XFS_RTGLOCK_RMAP); if (error) { - xfs_group_put(xg); + xfs_group_rele(xg); break; } } diff --git a/fs/xfs/xfs_trace.h b/fs/xfs/xfs_trace.h index f70afbf3cb196b..14acdcac996522 100644 --- a/fs/xfs/xfs_trace.h +++ b/fs/xfs/xfs_trace.h @@ -56,6 +56,7 @@ #include struct xfs_agf; +struct xfs_ail; struct xfs_alloc_arg; struct xfs_attr_list_context; struct xfs_buf_log_item; @@ -1647,16 +1648,43 @@ TRACE_EVENT(xfs_log_force, DEFINE_EVENT(xfs_log_item_class, name, \ TP_PROTO(struct xfs_log_item *lip), \ TP_ARGS(lip)) -DEFINE_LOG_ITEM_EVENT(xfs_ail_push); -DEFINE_LOG_ITEM_EVENT(xfs_ail_pinned); -DEFINE_LOG_ITEM_EVENT(xfs_ail_locked); -DEFINE_LOG_ITEM_EVENT(xfs_ail_flushing); DEFINE_LOG_ITEM_EVENT(xfs_cil_whiteout_mark); DEFINE_LOG_ITEM_EVENT(xfs_cil_whiteout_skip); DEFINE_LOG_ITEM_EVENT(xfs_cil_whiteout_unpin); DEFINE_LOG_ITEM_EVENT(xlog_ail_insert_abort); DEFINE_LOG_ITEM_EVENT(xfs_trans_free_abort); +DECLARE_EVENT_CLASS(xfs_ail_push_class, + TP_PROTO(struct xfs_ail *ailp, uint type, unsigned long flags, xfs_lsn_t lsn), + TP_ARGS(ailp, type, flags, lsn), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(uint, type) + __field(unsigned long, flags) + __field(xfs_lsn_t, lsn) + ), + TP_fast_assign( + __entry->dev = ailp->ail_log->l_mp->m_super->s_dev; + __entry->type = type; + __entry->flags = flags; + __entry->lsn = lsn; + ), + TP_printk("dev %d:%d lsn %d/%d type %s flags %s", + MAJOR(__entry->dev), MINOR(__entry->dev), + CYCLE_LSN(__entry->lsn), BLOCK_LSN(__entry->lsn), + __print_symbolic(__entry->type, XFS_LI_TYPE_DESC), + __print_flags(__entry->flags, "|", XFS_LI_FLAGS)) +) + +#define DEFINE_AIL_PUSH_EVENT(name) \ +DEFINE_EVENT(xfs_ail_push_class, name, \ + TP_PROTO(struct xfs_ail *ailp, uint type, unsigned long flags, xfs_lsn_t lsn), \ + TP_ARGS(ailp, type, flags, lsn)) +DEFINE_AIL_PUSH_EVENT(xfs_ail_push); +DEFINE_AIL_PUSH_EVENT(xfs_ail_pinned); +DEFINE_AIL_PUSH_EVENT(xfs_ail_locked); +DEFINE_AIL_PUSH_EVENT(xfs_ail_flushing); + DECLARE_EVENT_CLASS(xfs_ail_class, TP_PROTO(struct xfs_log_item *lip, xfs_lsn_t old_lsn, xfs_lsn_t new_lsn), TP_ARGS(lip, old_lsn, new_lsn), @@ -5087,23 +5115,16 @@ TRACE_EVENT(xmbuf_create, TP_STRUCT__entry( __field(dev_t, dev) __field(unsigned long, ino) - __array(char, pathname, MAXNAMELEN) ), TP_fast_assign( - char *path; struct file *file = btp->bt_file; __entry->dev = btp->bt_mount->m_super->s_dev; __entry->ino = file_inode(file)->i_ino; - path = file_path(file, __entry->pathname, MAXNAMELEN); - if (IS_ERR(path)) - strncpy(__entry->pathname, "(unknown)", - sizeof(__entry->pathname)); ), - TP_printk("dev %d:%d xmino 0x%lx path '%s'", + TP_printk("dev %d:%d xmino 0x%lx", MAJOR(__entry->dev), MINOR(__entry->dev), - __entry->ino, - __entry->pathname) + __entry->ino) ); TRACE_EVENT(xmbuf_free, diff --git a/fs/xfs/xfs_trans_ail.c b/fs/xfs/xfs_trans_ail.c index 38983c6777df31..68386db8246467 100644 --- a/fs/xfs/xfs_trans_ail.c +++ b/fs/xfs/xfs_trans_ail.c @@ -365,6 +365,12 @@ xfsaild_resubmit_item( return XFS_ITEM_SUCCESS; } +/* + * Push a single log item from the AIL. + * + * @lip may have been released and freed by the time this function returns, + * so callers must not dereference the log item afterwards. + */ static inline uint xfsaild_push_item( struct xfs_ail *ailp, @@ -505,7 +511,10 @@ xfsaild_push( lsn = lip->li_lsn; while ((XFS_LSN_CMP(lip->li_lsn, ailp->ail_target) <= 0)) { - int lock_result; + int lock_result; + uint type = lip->li_type; + unsigned long flags = lip->li_flags; + xfs_lsn_t item_lsn = lip->li_lsn; if (test_bit(XFS_LI_FLUSHING, &lip->li_flags)) goto next_item; @@ -514,14 +523,17 @@ xfsaild_push( * Note that iop_push may unlock and reacquire the AIL lock. We * rely on the AIL cursor implementation to be able to deal with * the dropped lock. + * + * The log item may have been freed by the push, so it must not + * be accessed or dereferenced below this line. */ lock_result = xfsaild_push_item(ailp, lip); switch (lock_result) { case XFS_ITEM_SUCCESS: XFS_STATS_INC(mp, xs_push_ail_success); - trace_xfs_ail_push(lip); + trace_xfs_ail_push(ailp, type, flags, item_lsn); - ailp->ail_last_pushed_lsn = lsn; + ailp->ail_last_pushed_lsn = item_lsn; break; case XFS_ITEM_FLUSHING: @@ -537,22 +549,22 @@ xfsaild_push( * AIL is being flushed. */ XFS_STATS_INC(mp, xs_push_ail_flushing); - trace_xfs_ail_flushing(lip); + trace_xfs_ail_flushing(ailp, type, flags, item_lsn); flushing++; - ailp->ail_last_pushed_lsn = lsn; + ailp->ail_last_pushed_lsn = item_lsn; break; case XFS_ITEM_PINNED: XFS_STATS_INC(mp, xs_push_ail_pinned); - trace_xfs_ail_pinned(lip); + trace_xfs_ail_pinned(ailp, type, flags, item_lsn); stuck++; ailp->ail_log_flush++; break; case XFS_ITEM_LOCKED: XFS_STATS_INC(mp, xs_push_ail_locked); - trace_xfs_ail_locked(lip); + trace_xfs_ail_locked(ailp, type, flags, item_lsn); stuck++; break; diff --git a/include/acpi/ghes.h b/include/acpi/ghes.h index ebd21b05fe6ed1..93db60da5934e7 100644 --- a/include/acpi/ghes.h +++ b/include/acpi/ghes.h @@ -21,6 +21,7 @@ struct ghes { struct acpi_hest_generic_v2 *generic_v2; }; struct acpi_hest_generic_status *estatus; + unsigned int estatus_length; unsigned long flags; union { struct list_head list; diff --git a/include/acpi/pcc.h b/include/acpi/pcc.h index 9af3b502f83952..840bfc95bae332 100644 --- a/include/acpi/pcc.h +++ b/include/acpi/pcc.h @@ -17,35 +17,6 @@ struct pcc_mbox_chan { u32 latency; u32 max_access_rate; u16 min_turnaround_time; - - /* Set to true to indicate that the mailbox should manage - * writing the dat to the shared buffer. This differs from - * the case where the drivesr are writing to the buffer and - * using send_data only to ring the doorbell. If this flag - * is set, then the void * data parameter of send_data must - * point to a kernel-memory buffer formatted in accordance with - * the PCC specification. - * - * The active buffer management will include reading the - * notify_on_completion flag, and will then - * call mbox_chan_txdone when the acknowledgment interrupt is - * received. - */ - bool manage_writes; - - /* Optional callback that allows the driver - * to allocate the memory used for receiving - * messages. The return value is the location - * inside the buffer where the mailbox should write the data. - */ - void *(*rx_alloc)(struct mbox_client *cl, int size); -}; - -struct pcc_header { - u32 signature; - u32 flags; - u32 length; - u32 command; }; /* Generic Communications Channel Shared Memory Region */ diff --git a/include/asm-generic/audit_change_attr.h b/include/asm-generic/audit_change_attr.h index cc840537885fbc..ddd90bbe40dfcb 100644 --- a/include/asm-generic/audit_change_attr.h +++ b/include/asm-generic/audit_change_attr.h @@ -26,6 +26,9 @@ __NR_fremovexattr, __NR_fchownat, __NR_fchmodat, #endif +#ifdef __NR_fchmodat2 +__NR_fchmodat2, +#endif #ifdef __NR_chown32 __NR_chown32, __NR_fchown32, diff --git a/include/asm-generic/audit_read.h b/include/asm-generic/audit_read.h index 7bb7b5a83ae2e5..fb9991f53fb6f9 100644 --- a/include/asm-generic/audit_read.h +++ b/include/asm-generic/audit_read.h @@ -4,9 +4,15 @@ __NR_readlink, #endif __NR_quotactl, __NR_listxattr, +#ifdef __NR_listxattrat +__NR_listxattrat, +#endif __NR_llistxattr, __NR_flistxattr, __NR_getxattr, +#ifdef __NR_getxattrat +__NR_getxattrat, +#endif __NR_lgetxattr, __NR_fgetxattr, #ifdef __NR_readlinkat diff --git a/include/asm-generic/rqspinlock.h b/include/asm-generic/rqspinlock.h index 0f2dcbbfee2f0a..5c5cf2f7fc395a 100644 --- a/include/asm-generic/rqspinlock.h +++ b/include/asm-generic/rqspinlock.h @@ -191,7 +191,7 @@ static __always_inline int res_spin_lock(rqspinlock_t *lock) #else -#define res_spin_lock(lock) resilient_tas_spin_lock(lock) +#define res_spin_lock(lock) ({ grab_held_lock_entry(lock); resilient_tas_spin_lock(lock); }) #endif /* CONFIG_QUEUED_SPINLOCKS */ diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h index 8ca130af301fc5..7b38ec8dc8dd3b 100644 --- a/include/asm-generic/vmlinux.lds.h +++ b/include/asm-generic/vmlinux.lds.h @@ -848,12 +848,14 @@ /* Required sections not related to debugging. */ #define ELF_DETAILS \ - .modinfo : { *(.modinfo) . = ALIGN(8); } \ .comment 0 : { *(.comment) } \ .symtab 0 : { *(.symtab) } \ .strtab 0 : { *(.strtab) } \ .shstrtab 0 : { *(.shstrtab) } +#define MODINFO \ + .modinfo : { *(.modinfo) . = ALIGN(8); } + #ifdef CONFIG_GENERIC_BUG #define BUG_TABLE \ . = ALIGN(8); \ diff --git a/include/crypto/if_alg.h b/include/crypto/if_alg.h index 107b797c33ecf7..0cc8fa749f68d2 100644 --- a/include/crypto/if_alg.h +++ b/include/crypto/if_alg.h @@ -230,9 +230,8 @@ static inline bool af_alg_readable(struct sock *sk) return PAGE_SIZE <= af_alg_rcvbuf(sk); } -unsigned int af_alg_count_tsgl(struct sock *sk, size_t bytes, size_t offset); -void af_alg_pull_tsgl(struct sock *sk, size_t used, struct scatterlist *dst, - size_t dst_offset); +unsigned int af_alg_count_tsgl(struct sock *sk, size_t bytes); +void af_alg_pull_tsgl(struct sock *sk, size_t used, struct scatterlist *dst); void af_alg_wmem_wakeup(struct sock *sk); int af_alg_wait_for_data(struct sock *sk, unsigned flags, unsigned min); int af_alg_sendmsg(struct socket *sock, struct msghdr *msg, size_t size, diff --git a/include/cxl/event.h b/include/cxl/event.h index 6fd90f9cc2034f..4d7d1036ea9cb7 100644 --- a/include/cxl/event.h +++ b/include/cxl/event.h @@ -320,4 +320,14 @@ static inline int cxl_cper_prot_err_kfifo_get(struct cxl_cper_prot_err_work_data } #endif +#ifdef CONFIG_ACPI_APEI_PCIEAER +int cxl_cper_sec_prot_err_valid(struct cxl_cper_sec_prot_err *prot_err); +#else +static inline int +cxl_cper_sec_prot_err_valid(struct cxl_cper_sec_prot_err *prot_err) +{ + return -EOPNOTSUPP; +} +#endif + #endif /* _LINUX_CXL_EVENT_H */ diff --git a/include/drm/drm_atomic.h b/include/drm/drm_atomic.h index 43783891d3594b..ff1fecd30478a9 100644 --- a/include/drm/drm_atomic.h +++ b/include/drm/drm_atomic.h @@ -30,7 +30,6 @@ #include #include -#include /** * struct drm_crtc_commit - track modeset commits on a CRTC @@ -707,6 +706,14 @@ drm_atomic_get_plane_state(struct drm_atomic_state *state, struct drm_colorop_state * drm_atomic_get_colorop_state(struct drm_atomic_state *state, struct drm_colorop *colorop); + +struct drm_colorop_state * +drm_atomic_get_old_colorop_state(struct drm_atomic_state *state, + struct drm_colorop *colorop); +struct drm_colorop_state * +drm_atomic_get_new_colorop_state(struct drm_atomic_state *state, + struct drm_colorop *colorop); + struct drm_connector_state * __must_check drm_atomic_get_connector_state(struct drm_atomic_state *state, struct drm_connector *connector); @@ -803,36 +810,6 @@ drm_atomic_get_new_plane_state(const struct drm_atomic_state *state, return state->planes[drm_plane_index(plane)].new_state; } -/** - * drm_atomic_get_old_colorop_state - get colorop state, if it exists - * @state: global atomic state object - * @colorop: colorop to grab - * - * This function returns the old colorop state for the given colorop, or - * NULL if the colorop is not part of the global atomic state. - */ -static inline struct drm_colorop_state * -drm_atomic_get_old_colorop_state(struct drm_atomic_state *state, - struct drm_colorop *colorop) -{ - return state->colorops[drm_colorop_index(colorop)].old_state; -} - -/** - * drm_atomic_get_new_colorop_state - get colorop state, if it exists - * @state: global atomic state object - * @colorop: colorop to grab - * - * This function returns the new colorop state for the given colorop, or - * NULL if the colorop is not part of the global atomic state. - */ -static inline struct drm_colorop_state * -drm_atomic_get_new_colorop_state(struct drm_atomic_state *state, - struct drm_colorop *colorop) -{ - return state->colorops[drm_colorop_index(colorop)].new_state; -} - /** * drm_atomic_get_old_connector_state - get connector state, if it exists * @state: global atomic state object diff --git a/include/drm/drm_gem_shmem_helper.h b/include/drm/drm_gem_shmem_helper.h index 589f7bfe7506eb..8d56970d7eed1e 100644 --- a/include/drm/drm_gem_shmem_helper.h +++ b/include/drm/drm_gem_shmem_helper.h @@ -303,4 +303,15 @@ struct drm_gem_object *drm_gem_shmem_prime_import_no_map(struct drm_device *dev, .gem_prime_import = drm_gem_shmem_prime_import_no_map, \ .dumb_create = drm_gem_shmem_dumb_create +/* + * Kunit helpers + */ + +#if IS_ENABLED(CONFIG_KUNIT) +int drm_gem_shmem_vmap(struct drm_gem_shmem_object *shmem, struct iosys_map *map); +void drm_gem_shmem_vunmap(struct drm_gem_shmem_object *shmem, struct iosys_map *map); +int drm_gem_shmem_madvise(struct drm_gem_shmem_object *shmem, int madv); +int drm_gem_shmem_purge(struct drm_gem_shmem_object *shmem); +#endif + #endif /* __DRM_GEM_SHMEM_HELPER_H__ */ diff --git a/include/drm/drm_gpuvm.h b/include/drm/drm_gpuvm.h index fdfc575b260360..75c48345677b0f 100644 --- a/include/drm/drm_gpuvm.h +++ b/include/drm/drm_gpuvm.h @@ -57,10 +57,19 @@ enum drm_gpuva_flags { */ DRM_GPUVA_SPARSE = (1 << 1), + /** + * @DRM_GPUVA_REPEAT: + * + * Flag indicating that the &drm_gpuva is a mapping of a GEM + * object with a certain range that is repeated multiple times to + * fill the virtual address range. + */ + DRM_GPUVA_REPEAT = (1 << 2), + /** * @DRM_GPUVA_USERBITS: user defined bits */ - DRM_GPUVA_USERBITS = (1 << 2), + DRM_GPUVA_USERBITS = (1 << 3), }; /** @@ -112,6 +121,18 @@ struct drm_gpuva { */ u64 offset; + /* + * @gem.range: the range of the GEM that is mapped + * + * When dealing with normal mappings, this must be zero. + * When flags has DRM_GPUVA_REPEAT set, this field must be + * smaller than va.range and va.range must be a multiple of + * gem.range. + * This is a u32 not a u64 because we expect repeated mappings + * to be pointing to relatively small portions of a GEM object. + */ + u32 range; + /** * @gem.obj: the mapped &drm_gem_object */ @@ -882,11 +903,27 @@ struct drm_gpuva_op_map { */ u64 offset; + /* + * @gem.range: the range of the GEM that is mapped + * + * When dealing with normal mappings, this must be zero. + * When flags has DRM_GPUVA_REPEAT set, it must be a multiple + * of va.range. This is a u32 not a u64 because we expect + * repeated mappings to be pointing to a relatively small + * portion of a GEM object. + */ + u32 range; + /** * @gem.obj: the &drm_gem_object to map */ struct drm_gem_object *obj; } gem; + + /** + * @flags: requested flags for the &drm_gpuva for this mapping + */ + enum drm_gpuva_flags flags; }; /** @@ -1123,6 +1160,7 @@ void drm_gpuva_ops_free(struct drm_gpuvm *gpuvm, static inline void drm_gpuva_init_from_op(struct drm_gpuva *va, struct drm_gpuva_op_map *op) { + va->flags = op->flags; va->va.addr = op->va.addr; va->va.range = op->va.range; va->gem.obj = op->gem.obj; @@ -1248,6 +1286,16 @@ struct drm_gpuvm_ops { * used. */ int (*sm_step_unmap)(struct drm_gpuva_op *op, void *priv); + + /** + * @sm_can_merge_flags: called during &drm_gpuvm_sm_map + * + * This callback is called to determine whether two va ranges can be merged, + * based on their flags. + * + * If NULL, va ranges can only be merged if their flags are equal. + */ + bool (*sm_can_merge_flags)(enum drm_gpuva_flags a, enum drm_gpuva_flags b); }; int drm_gpuvm_sm_map(struct drm_gpuvm *gpuvm, void *priv, @@ -1255,6 +1303,7 @@ int drm_gpuvm_sm_map(struct drm_gpuvm *gpuvm, void *priv, int drm_gpuvm_sm_unmap(struct drm_gpuvm *gpuvm, void *priv, u64 addr, u64 range); +int drm_gpuvm_bo_unmap(struct drm_gpuvm_bo *bo, void *priv); int drm_gpuvm_sm_map_exec_lock(struct drm_gpuvm *gpuvm, struct drm_exec *exec, unsigned int num_fences, diff --git a/include/drm/drm_of.h b/include/drm/drm_of.h index 7f0256dae3f13d..f3e55ea2174c09 100644 --- a/include/drm/drm_of.h +++ b/include/drm/drm_of.h @@ -5,6 +5,7 @@ #include #include #if IS_ENABLED(CONFIG_OF) && IS_ENABLED(CONFIG_DRM_PANEL_BRIDGE) +#include #include #endif @@ -173,6 +174,8 @@ static inline int drm_of_panel_bridge_remove(const struct device_node *np, bridge = of_drm_find_bridge(remote); drm_panel_bridge_remove(bridge); + of_node_put(remote); + return 0; #else return -EINVAL; diff --git a/include/drm/gpu_scheduler.h b/include/drm/gpu_scheduler.h index fb88301b3c45c7..e2d646070cd316 100644 --- a/include/drm/gpu_scheduler.h +++ b/include/drm/gpu_scheduler.h @@ -301,6 +301,11 @@ struct drm_sched_fence { * @lock: the lock used by the scheduled and the finished fences. */ spinlock_t lock; + /** + * @sched_name: the name of the scheduler that owns this fence. We + * keep a copy here since fences can outlive their scheduler. + */ + char sched_name[16]; /** * @owner: job owner for debugging */ diff --git a/include/drm/intel/intel_lb_mei_interface.h b/include/drm/intel/intel_lb_mei_interface.h index d65be2cba2ab9f..0850738a30fc75 100644 --- a/include/drm/intel/intel_lb_mei_interface.h +++ b/include/drm/intel/intel_lb_mei_interface.h @@ -53,7 +53,8 @@ enum intel_lb_status { */ struct intel_lb_component_ops { /** - * push_payload - Sends a payload to the authentication firmware + * @push_payload: Sends a payload to the authentication firmware + * * @dev: Device struct corresponding to the mei device * @type: Payload type (see &enum intel_lb_type) * @flags: Payload flags bitmap (e.g. %INTEL_LB_FLAGS_IS_PERSISTENT) diff --git a/include/hyperv/hvgdk_mini.h b/include/hyperv/hvgdk_mini.h index 30fbbde81c5c43..9c523ee57a358d 100644 --- a/include/hyperv/hvgdk_mini.h +++ b/include/hyperv/hvgdk_mini.h @@ -1528,4 +1528,10 @@ struct hv_mmio_write_input { u8 data[HV_HYPERCALL_MMIO_MAX_DATA_LENGTH]; } __packed; +enum hv_intercept_access_type { + HV_INTERCEPT_ACCESS_READ = 0, + HV_INTERCEPT_ACCESS_WRITE = 1, + HV_INTERCEPT_ACCESS_EXECUTE = 2 +}; + #endif /* _HV_HVGDK_MINI_H */ diff --git a/include/hyperv/hvhdk.h b/include/hyperv/hvhdk.h index 08965970c17dfc..84ebe56f1f8dbd 100644 --- a/include/hyperv/hvhdk.h +++ b/include/hyperv/hvhdk.h @@ -770,7 +770,7 @@ struct hv_x64_intercept_message_header { u32 vp_index; u8 instruction_length:4; u8 cr8:4; /* Only set for exo partitions */ - u8 intercept_access_type; + u8 intercept_access_type; /* enum hv_intercept_access_type */ union hv_x64_vp_execution_state execution_state; struct hv_x64_segment_register cs_segment; u64 rip; @@ -816,7 +816,7 @@ union hv_arm64_vp_execution_state { struct hv_arm64_intercept_message_header { u32 vp_index; u8 instruction_length; - u8 intercept_access_type; + u8 intercept_access_type; /* enum hv_intercept_access_type */ union hv_arm64_vp_execution_state execution_state; u64 pc; u64 cpsr; diff --git a/include/kunit/run-in-irq-context.h b/include/kunit/run-in-irq-context.h index c89b1b1b12dd58..bfe60d6cf28d86 100644 --- a/include/kunit/run-in-irq-context.h +++ b/include/kunit/run-in-irq-context.h @@ -12,16 +12,16 @@ #include #include -#define KUNIT_IRQ_TEST_HRTIMER_INTERVAL us_to_ktime(5) - struct kunit_irq_test_state { bool (*func)(void *test_specific_state); void *test_specific_state; bool task_func_reported_failure; bool hardirq_func_reported_failure; bool softirq_func_reported_failure; + atomic_t task_func_calls; atomic_t hardirq_func_calls; atomic_t softirq_func_calls; + ktime_t interval; struct hrtimer timer; struct work_struct bh_work; }; @@ -30,14 +30,25 @@ static enum hrtimer_restart kunit_irq_test_timer_func(struct hrtimer *timer) { struct kunit_irq_test_state *state = container_of(timer, typeof(*state), timer); + int task_calls, hardirq_calls, softirq_calls; WARN_ON_ONCE(!in_hardirq()); - atomic_inc(&state->hardirq_func_calls); + task_calls = atomic_read(&state->task_func_calls); + hardirq_calls = atomic_inc_return(&state->hardirq_func_calls); + softirq_calls = atomic_read(&state->softirq_func_calls); + + /* + * If the timer is firing too often for the softirq or task to ever have + * a chance to run, increase the timer interval. This is needed on very + * slow systems. + */ + if (hardirq_calls >= 20 && (softirq_calls == 0 || task_calls == 0)) + state->interval = ktime_add_ns(state->interval, 250); if (!state->func(state->test_specific_state)) state->hardirq_func_reported_failure = true; - hrtimer_forward_now(&state->timer, KUNIT_IRQ_TEST_HRTIMER_INTERVAL); + hrtimer_forward_now(&state->timer, state->interval); queue_work(system_bh_wq, &state->bh_work); return HRTIMER_RESTART; } @@ -86,10 +97,14 @@ static inline void kunit_run_irq_test(struct kunit *test, bool (*func)(void *), struct kunit_irq_test_state state = { .func = func, .test_specific_state = test_specific_state, + /* + * Start with a 5us timer interval. If the system can't keep + * up, kunit_irq_test_timer_func() will increase it. + */ + .interval = us_to_ktime(5), }; unsigned long end_jiffies; - int hardirq_calls, softirq_calls; - bool allctx = false; + int task_calls, hardirq_calls, softirq_calls; /* * Set up a hrtimer (the way we access hardirq context) and a work @@ -104,21 +119,18 @@ static inline void kunit_run_irq_test(struct kunit *test, bool (*func)(void *), * and hardirq), or 1 second, whichever comes first. */ end_jiffies = jiffies + HZ; - hrtimer_start(&state.timer, KUNIT_IRQ_TEST_HRTIMER_INTERVAL, - HRTIMER_MODE_REL_HARD); - for (int task_calls = 0, calls = 0; - ((calls < max_iterations) || !allctx) && - !time_after(jiffies, end_jiffies); - task_calls++) { + hrtimer_start(&state.timer, state.interval, HRTIMER_MODE_REL_HARD); + do { if (!func(test_specific_state)) state.task_func_reported_failure = true; + task_calls = atomic_inc_return(&state.task_func_calls); hardirq_calls = atomic_read(&state.hardirq_func_calls); softirq_calls = atomic_read(&state.softirq_func_calls); - calls = task_calls + hardirq_calls + softirq_calls; - allctx = (task_calls > 0) && (hardirq_calls > 0) && - (softirq_calls > 0); - } + } while ((task_calls + hardirq_calls + softirq_calls < max_iterations || + (task_calls == 0 || hardirq_calls == 0 || + softirq_calls == 0)) && + !time_after(jiffies, end_jiffies)); /* Cancel the timer and work. */ hrtimer_cancel(&state.timer); diff --git a/include/linux/ata.h b/include/linux/ata.h index 54b416e2699590..c9013e472aa3d5 100644 --- a/include/linux/ata.h +++ b/include/linux/ata.h @@ -29,7 +29,6 @@ enum { ATA_MAX_SECTORS_128 = 128, ATA_MAX_SECTORS = 256, ATA_MAX_SECTORS_1024 = 1024, - ATA_MAX_SECTORS_8191 = 8191, ATA_MAX_SECTORS_LBA48 = 65535,/* avoid count to be 0000h */ ATA_MAX_SECTORS_TAPE = 65535, ATA_MAX_TRIM_RNUM = 64, /* 512-byte payload / (6-byte LBA + 2-byte range per entry) */ diff --git a/include/linux/audit.h b/include/linux/audit.h index 536f8ee8da818c..b8d8029c6c480c 100644 --- a/include/linux/audit.h +++ b/include/linux/audit.h @@ -128,12 +128,6 @@ enum audit_nfcfgop { extern int __init audit_register_class(int class, unsigned *list); extern int audit_classify_syscall(int abi, unsigned syscall); extern int audit_classify_arch(int arch); -/* only for compat system calls */ -extern unsigned compat_write_class[]; -extern unsigned compat_read_class[]; -extern unsigned compat_dir_class[]; -extern unsigned compat_chattr_class[]; -extern unsigned compat_signal_class[]; /* audit_names->type values */ #define AUDIT_TYPE_UNKNOWN 0 /* we don't know yet */ diff --git a/include/linux/audit_arch.h b/include/linux/audit_arch.h index 0e34d673ef1712..2b8153791e6a5d 100644 --- a/include/linux/audit_arch.h +++ b/include/linux/audit_arch.h @@ -23,4 +23,11 @@ enum auditsc_class_t { extern int audit_classify_compat_syscall(int abi, unsigned syscall); +/* only for compat system calls */ +extern unsigned compat_write_class[]; +extern unsigned compat_read_class[]; +extern unsigned compat_dir_class[]; +extern unsigned compat_chattr_class[]; +extern unsigned compat_signal_class[]; + #endif diff --git a/include/linux/auxvec.h b/include/linux/auxvec.h index 407f7005e6d60a..8bcb9b72626286 100644 --- a/include/linux/auxvec.h +++ b/include/linux/auxvec.h @@ -4,6 +4,6 @@ #include -#define AT_VECTOR_SIZE_BASE 22 /* NEW_AUX_ENT entries in auxiliary table */ +#define AT_VECTOR_SIZE_BASE 24 /* NEW_AUX_ENT entries in auxiliary table */ /* number of "#define AT_.*" above, minus {AT_NULL, AT_IGNORE, AT_NOTELF} */ #endif /* _LINUX_AUXVEC_H */ diff --git a/include/linux/bpf.h b/include/linux/bpf.h index e5be698256d15a..7b2e51216e736a 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -3243,6 +3243,11 @@ static inline void bpf_prog_report_arena_violation(bool write, unsigned long add } #endif /* CONFIG_BPF_SYSCALL */ +static inline bool bpf_net_capable(void) +{ + return capable(CAP_NET_ADMIN) || capable(CAP_SYS_ADMIN); +} + static __always_inline int bpf_probe_read_kernel_common(void *dst, u32 size, const void *unsafe_ptr) { diff --git a/include/linux/bpf_mprog.h b/include/linux/bpf_mprog.h index 929225f7b09594..0b9f4caeeb0a32 100644 --- a/include/linux/bpf_mprog.h +++ b/include/linux/bpf_mprog.h @@ -340,4 +340,14 @@ static inline bool bpf_mprog_supported(enum bpf_prog_type type) return false; } } + +static inline bool bpf_mprog_detach_empty(enum bpf_prog_type type) +{ + switch (type) { + case BPF_PROG_TYPE_SCHED_CLS: + return bpf_net_capable(); + default: + return false; + } +} #endif /* __BPF_MPROG_H */ diff --git a/include/linux/capability.h b/include/linux/capability.h index 1fb08922552c79..37db92b3d6f89e 100644 --- a/include/linux/capability.h +++ b/include/linux/capability.h @@ -203,6 +203,12 @@ static inline bool checkpoint_restore_ns_capable(struct user_namespace *ns) ns_capable(ns, CAP_SYS_ADMIN); } +static inline bool checkpoint_restore_ns_capable_noaudit(struct user_namespace *ns) +{ + return ns_capable_noaudit(ns, CAP_CHECKPOINT_RESTORE) || + ns_capable_noaudit(ns, CAP_SYS_ADMIN); +} + /* audit system wants to get cap info from files as well */ int get_vfs_caps_from_disk(struct mnt_idmap *idmap, const struct dentry *dentry, diff --git a/include/linux/cgroup-defs.h b/include/linux/cgroup-defs.h index f7cc60de00583f..2bff3e2be0d3b3 100644 --- a/include/linux/cgroup-defs.h +++ b/include/linux/cgroup-defs.h @@ -609,6 +609,9 @@ struct cgroup { /* used to wait for offlining of csses */ wait_queue_head_t offline_waitq; + /* used by cgroup_rmdir() to wait for dying tasks to leave */ + wait_queue_head_t dying_populated_waitq; + /* used to schedule release agent */ struct work_struct release_agent_work; diff --git a/include/linux/clk.h b/include/linux/clk.h index b607482ca77e98..64ff118ffb1a1d 100644 --- a/include/linux/clk.h +++ b/include/linux/clk.h @@ -228,6 +228,23 @@ int devm_clk_rate_exclusive_get(struct device *dev, struct clk *clk); */ void clk_rate_exclusive_put(struct clk *clk); +/** + * clk_save_context - save clock context for poweroff + * + * Saves the context of the clock register for powerstates in which the + * contents of the registers will be lost. Occurs deep within the suspend + * code so locking is not necessary. + */ +int clk_save_context(void); + +/** + * clk_restore_context - restore clock context after poweroff + * + * This occurs with all clocks enabled. Occurs deep within the resume code + * so locking is not necessary. + */ +void clk_restore_context(void); + #else static inline int clk_notifier_register(struct clk *clk, @@ -293,6 +310,13 @@ static inline int devm_clk_rate_exclusive_get(struct device *dev, struct clk *cl static inline void clk_rate_exclusive_put(struct clk *clk) {} +static inline int clk_save_context(void) +{ + return 0; +} + +static inline void clk_restore_context(void) {} + #endif #ifdef CONFIG_HAVE_CLK_PREPARE @@ -933,23 +957,6 @@ struct clk *clk_get_parent(struct clk *clk); */ struct clk *clk_get_sys(const char *dev_id, const char *con_id); -/** - * clk_save_context - save clock context for poweroff - * - * Saves the context of the clock register for powerstates in which the - * contents of the registers will be lost. Occurs deep within the suspend - * code so locking is not necessary. - */ -int clk_save_context(void); - -/** - * clk_restore_context - restore clock context after poweroff - * - * This occurs with all clocks enabled. Occurs deep within the resume code - * so locking is not necessary. - */ -void clk_restore_context(void); - #else /* !CONFIG_HAVE_CLK */ static inline struct clk *clk_get(struct device *dev, const char *id) @@ -1129,13 +1136,6 @@ static inline struct clk *clk_get_sys(const char *dev_id, const char *con_id) return NULL; } -static inline int clk_save_context(void) -{ - return 0; -} - -static inline void clk_restore_context(void) {} - #endif /* clk_prepare_enable helps cases using clk_enable in non-atomic context. */ diff --git a/include/linux/compiler-clang.h b/include/linux/compiler-clang.h index 7edf1a07b53505..e1123dd284862b 100644 --- a/include/linux/compiler-clang.h +++ b/include/linux/compiler-clang.h @@ -153,4 +153,4 @@ * Bindgen uses LLVM even if our C compiler is GCC, so we cannot * rely on the auto-detected CONFIG_CC_HAS_TYPEOF_UNQUAL. */ -#define CC_HAS_TYPEOF_UNQUAL (__clang_major__ >= 19) +#define CC_HAS_TYPEOF_UNQUAL (__clang_major__ > 19 || (__clang_major__ == 19 && __clang_minor__ > 0)) diff --git a/include/linux/compiler_types.h b/include/linux/compiler_types.h index d3318a3c257775..86111a189a8747 100644 --- a/include/linux/compiler_types.h +++ b/include/linux/compiler_types.h @@ -303,6 +303,22 @@ struct ftrace_likely_data { # define __no_kasan_or_inline __always_inline #endif +#ifdef CONFIG_KCSAN +/* + * Type qualifier to mark variables where all data-racy accesses should be + * ignored by KCSAN. Note, the implementation simply marks these variables as + * volatile, since KCSAN will treat such accesses as "marked". + * + * Defined here because defining __data_racy as volatile for KCSAN objects only + * causes problems in BPF Type Format (BTF) generation since struct members + * of core kernel data structs will be volatile in some objects and not in + * others. Instead define it globally for KCSAN kernels. + */ +# define __data_racy volatile +#else +# define __data_racy +#endif + #ifdef __SANITIZE_THREAD__ /* * Clang still emits instrumentation for __tsan_func_{entry,exit}() and builtin @@ -314,16 +330,9 @@ struct ftrace_likely_data { * disable all instrumentation. See Kconfig.kcsan where this is mandatory. */ # define __no_kcsan __no_sanitize_thread __disable_sanitizer_instrumentation -/* - * Type qualifier to mark variables where all data-racy accesses should be - * ignored by KCSAN. Note, the implementation simply marks these variables as - * volatile, since KCSAN will treat such accesses as "marked". - */ -# define __data_racy volatile # define __no_sanitize_or_inline __no_kcsan notrace __maybe_unused #else # define __no_kcsan -# define __data_racy #endif #ifdef __SANITIZE_MEMORY__ diff --git a/include/linux/console_struct.h b/include/linux/console_struct.h index 13b35637bd5aee..d5ca855116df45 100644 --- a/include/linux/console_struct.h +++ b/include/linux/console_struct.h @@ -160,6 +160,7 @@ struct vc_data { struct uni_pagedict **uni_pagedict_loc; /* [!] Location of uni_pagedict variable for this console */ u32 **vc_uni_lines; /* unicode screen content */ u16 *vc_saved_screen; + u32 **vc_saved_uni_lines; unsigned int vc_saved_cols; unsigned int vc_saved_rows; /* additional information is in vt_kern.h */ diff --git a/include/linux/cper.h b/include/linux/cper.h index 5b1236d8c65bb7..440b35e459e538 100644 --- a/include/linux/cper.h +++ b/include/linux/cper.h @@ -595,7 +595,8 @@ void cper_mem_err_pack(const struct cper_sec_mem_err *, const char *cper_mem_err_unpack(struct trace_seq *, struct cper_mem_err_compact *); void cper_print_proc_arm(const char *pfx, - const struct cper_sec_proc_arm *proc); + const struct cper_sec_proc_arm *proc, + u32 length); void cper_print_proc_ia(const char *pfx, const struct cper_sec_proc_ia *proc); int cper_mem_err_location(struct cper_mem_err_compact *mem, char *msg); diff --git a/include/linux/cpuset.h b/include/linux/cpuset.h index a98d3330385c29..631577384677a4 100644 --- a/include/linux/cpuset.h +++ b/include/linux/cpuset.h @@ -174,7 +174,7 @@ static inline void set_mems_allowed(nodemask_t nodemask) task_unlock(current); } -extern bool cpuset_node_allowed(struct cgroup *cgroup, int nid); +extern void cpuset_nodes_allowed(struct cgroup *cgroup, nodemask_t *mask); #else /* !CONFIG_CPUSETS */ static inline bool cpusets_enabled(void) { return false; } @@ -301,9 +301,9 @@ static inline bool read_mems_allowed_retry(unsigned int seq) return false; } -static inline bool cpuset_node_allowed(struct cgroup *cgroup, int nid) +static inline void cpuset_nodes_allowed(struct cgroup *cgroup, nodemask_t *mask) { - return true; + nodes_copy(*mask, node_states[N_MEMORY]); } #endif /* !CONFIG_CPUSETS */ diff --git a/include/linux/damon.h b/include/linux/damon.h index 3813373a9200c4..e44ec9d76771af 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -15,7 +15,7 @@ #include /* Minimal region size. Every damon_region is aligned by this. */ -#define DAMON_MIN_REGION PAGE_SIZE +#define DAMON_MIN_REGION_SZ PAGE_SIZE /* Max priority score for DAMON-based operation schemes */ #define DAMOS_MAX_SCORE (99) @@ -769,7 +769,7 @@ struct damon_attrs { * * @ops: Set of monitoring operations for given use cases. * @addr_unit: Scale factor for core to ops address conversion. - * @min_sz_region: Minimum region size. + * @min_region_sz: Minimum region size. * @adaptive_targets: Head of monitoring targets (&damon_target) list. * @schemes: Head of schemes (&damos) list. */ @@ -806,13 +806,20 @@ struct damon_ctx { struct damos_walk_control *walk_control; struct mutex walk_control_lock; + /* + * indicate if this may be corrupted. Currentonly this is set only for + * damon_commit_ctx() failure. + */ + bool maybe_corrupted; + /* public: */ + /* Working thread of the given DAMON context */ struct task_struct *kdamond; struct mutex kdamond_lock; struct damon_operations ops; unsigned long addr_unit; - unsigned long min_sz_region; + unsigned long min_region_sz; struct list_head adaptive_targets; struct list_head schemes; @@ -901,7 +908,7 @@ static inline void damon_insert_region(struct damon_region *r, void damon_add_region(struct damon_region *r, struct damon_target *t); void damon_destroy_region(struct damon_region *r, struct damon_target *t); int damon_set_regions(struct damon_target *t, struct damon_addr_range *ranges, - unsigned int nr_ranges, unsigned long min_sz_region); + unsigned int nr_ranges, unsigned long min_region_sz); void damon_update_region_access_rate(struct damon_region *r, bool accessed, struct damon_attrs *attrs); @@ -968,7 +975,7 @@ int damos_walk(struct damon_ctx *ctx, struct damos_walk_control *control); int damon_set_region_biggest_system_ram_default(struct damon_target *t, unsigned long *start, unsigned long *end, - unsigned long min_sz_region); + unsigned long min_region_sz); #endif /* CONFIG_DAMON */ diff --git a/include/linux/device.h b/include/linux/device.h index 0be95294b6e615..e65d564f01cd7c 100644 --- a/include/linux/device.h +++ b/include/linux/device.h @@ -483,6 +483,8 @@ struct device_physical_location { * on. This shrinks the "Board Support Packages" (BSPs) and * minimizes board-specific #ifdefs in drivers. * @driver_data: Private pointer for driver specific info. + * @driver_override: Driver name to force a match. Do not touch directly; use + * device_set_driver_override() instead. * @links: Links to suppliers and consumers of this device. * @power: For device power management. * See Documentation/driver-api/pm/devices.rst for details. @@ -576,6 +578,10 @@ struct device { core doesn't touch it */ void *driver_data; /* Driver data, set and get with dev_set_drvdata/dev_get_drvdata */ + struct { + const char *name; + spinlock_t lock; + } driver_override; struct mutex mutex; /* mutex to synchronize calls to * its driver. */ @@ -701,6 +707,54 @@ struct device_link { #define kobj_to_dev(__kobj) container_of_const(__kobj, struct device, kobj) +int __device_set_driver_override(struct device *dev, const char *s, size_t len); + +/** + * device_set_driver_override() - Helper to set or clear driver override. + * @dev: Device to change + * @s: NUL-terminated string, new driver name to force a match, pass empty + * string to clear it ("" or "\n", where the latter is only for sysfs + * interface). + * + * Helper to set or clear driver override of a device. + * + * Returns: 0 on success or a negative error code on failure. + */ +static inline int device_set_driver_override(struct device *dev, const char *s) +{ + return __device_set_driver_override(dev, s, s ? strlen(s) : 0); +} + +/** + * device_has_driver_override() - Check if a driver override has been set. + * @dev: device to check + * + * Returns true if a driver override has been set for this device. + */ +static inline bool device_has_driver_override(struct device *dev) +{ + guard(spinlock)(&dev->driver_override.lock); + return !!dev->driver_override.name; +} + +/** + * device_match_driver_override() - Match a driver against the device's driver_override. + * @dev: device to check + * @drv: driver to match against + * + * Returns > 0 if a driver override is set and matches the given driver, 0 if a + * driver override is set but does not match, or < 0 if a driver override is not + * set at all. + */ +static inline int device_match_driver_override(struct device *dev, + const struct device_driver *drv) +{ + guard(spinlock)(&dev->driver_override.lock); + if (dev->driver_override.name) + return !strcmp(dev->driver_override.name, drv->name); + return -1; +} + /** * device_iommu_mapped - Returns true when the device DMA is translated * by an IOMMU diff --git a/include/linux/device/bus.h b/include/linux/device/bus.h index 99b1002b3e318b..f047b40a30b746 100644 --- a/include/linux/device/bus.h +++ b/include/linux/device/bus.h @@ -63,6 +63,9 @@ struct fwnode_handle; * this bus. * @pm: Power management operations of this bus, callback the specific * device driver's pm-ops. + * @driver_override: Set to true if this bus supports the driver_override + * mechanism, which allows userspace to force a specific + * driver to bind to a device via a sysfs attribute. * @need_parent_lock: When probing or removing a device on this bus, the * device core should lock the device's parent. * @@ -104,6 +107,7 @@ struct bus_type { const struct dev_pm_ops *pm; + bool driver_override; bool need_parent_lock; }; diff --git a/include/linux/device_cgroup.h b/include/linux/device_cgroup.h index 0864773a57e8fe..822085bc2d202c 100644 --- a/include/linux/device_cgroup.h +++ b/include/linux/device_cgroup.h @@ -21,7 +21,7 @@ static inline int devcgroup_inode_permission(struct inode *inode, int mask) if (likely(!S_ISBLK(inode->i_mode) && !S_ISCHR(inode->i_mode))) return 0; - if (likely(!inode->i_rdev)) + if (!inode->i_rdev) return 0; if (S_ISBLK(inode->i_mode)) diff --git a/include/linux/dma-mapping.h b/include/linux/dma-mapping.h index aa36a0d1d9df6a..3e63046b899bca 100644 --- a/include/linux/dma-mapping.h +++ b/include/linux/dma-mapping.h @@ -78,6 +78,13 @@ */ #define DMA_ATTR_MMIO (1UL << 10) +/* + * DMA_ATTR_CPU_CACHE_CLEAN: Indicates the CPU will not dirty any cacheline + * overlapping this buffer while it is mapped for DMA. All mappings sharing + * a cacheline must have this attribute for this to be considered safe. + */ +#define DMA_ATTR_CPU_CACHE_CLEAN (1UL << 11) + /* * A dma_addr_t can hold any valid DMA or bus address for the platform. It can * be given to a device to use as a DMA source or target. It is specific to a @@ -240,8 +247,8 @@ static inline void *dma_alloc_attrs(struct device *dev, size_t size, { return NULL; } -static void dma_free_attrs(struct device *dev, size_t size, void *cpu_addr, - dma_addr_t dma_handle, unsigned long attrs) +static inline void dma_free_attrs(struct device *dev, size_t size, + void *cpu_addr, dma_addr_t dma_handle, unsigned long attrs) { } static inline void *dmam_alloc_attrs(struct device *dev, size_t size, diff --git a/include/linux/etherdevice.h b/include/linux/etherdevice.h index 9a1eacf35d3708..df8f88f63a7063 100644 --- a/include/linux/etherdevice.h +++ b/include/linux/etherdevice.h @@ -42,7 +42,8 @@ extern const struct header_ops eth_header_ops; int eth_header(struct sk_buff *skb, struct net_device *dev, unsigned short type, const void *daddr, const void *saddr, unsigned len); -int eth_header_parse(const struct sk_buff *skb, unsigned char *haddr); +int eth_header_parse(const struct sk_buff *skb, const struct net_device *dev, + unsigned char *haddr); int eth_header_cache(const struct neighbour *neigh, struct hh_cache *hh, __be16 type); void eth_header_cache_update(struct hh_cache *hh, const struct net_device *dev, diff --git a/include/linux/exportfs.h b/include/linux/exportfs.h index f0cf2714ec52dd..9bd93d6bd9a49e 100644 --- a/include/linux/exportfs.h +++ b/include/linux/exportfs.h @@ -317,6 +317,15 @@ static inline bool exportfs_can_decode_fh(const struct export_operations *nop) return nop && nop->fh_to_dentry; } +static inline bool exportfs_may_export(const struct export_operations *nop) +{ + /* + * Do not allow nfs export for filesystems with custom ->open() or + * ->permission() ops, which nfsd does not respect (e.g. pidfs, nsfs). + */ + return exportfs_can_decode_fh(nop) && !nop->open && !nop->permission; +} + static inline bool exportfs_can_encode_fh(const struct export_operations *nop, int fh_flags) { diff --git a/include/linux/f2fs_fs.h b/include/linux/f2fs_fs.h index a7880787cad366..dc41722fcc9dde 100644 --- a/include/linux/f2fs_fs.h +++ b/include/linux/f2fs_fs.h @@ -17,7 +17,6 @@ #define F2FS_LOG_SECTORS_PER_BLOCK (PAGE_SHIFT - 9) /* log number for sector/blk */ #define F2FS_BLKSIZE PAGE_SIZE /* support only block == page */ #define F2FS_BLKSIZE_BITS PAGE_SHIFT /* bits for F2FS_BLKSIZE */ -#define F2FS_SUM_BLKSIZE 4096 /* only support 4096 byte sum block */ #define F2FS_MAX_EXTENSION 64 /* # of extension entries */ #define F2FS_EXTENSION_LEN 8 /* max size of extension */ @@ -442,10 +441,8 @@ struct f2fs_sit_block { * from node's page's beginning to get a data block address. * ex) data_blkaddr = (block_t)(nodepage_start_address + ofs_in_node) */ -#define ENTRIES_IN_SUM (F2FS_SUM_BLKSIZE / 8) #define SUMMARY_SIZE (7) /* sizeof(struct f2fs_summary) */ #define SUM_FOOTER_SIZE (5) /* sizeof(struct summary_footer) */ -#define SUM_ENTRY_SIZE (SUMMARY_SIZE * ENTRIES_IN_SUM) /* a summary entry for a block in a segment */ struct f2fs_summary { @@ -468,22 +465,6 @@ struct summary_footer { __le32 check_sum; /* summary checksum */ } __packed; -#define SUM_JOURNAL_SIZE (F2FS_SUM_BLKSIZE - SUM_FOOTER_SIZE -\ - SUM_ENTRY_SIZE) -#define NAT_JOURNAL_ENTRIES ((SUM_JOURNAL_SIZE - 2) /\ - sizeof(struct nat_journal_entry)) -#define NAT_JOURNAL_RESERVED ((SUM_JOURNAL_SIZE - 2) %\ - sizeof(struct nat_journal_entry)) -#define SIT_JOURNAL_ENTRIES ((SUM_JOURNAL_SIZE - 2) /\ - sizeof(struct sit_journal_entry)) -#define SIT_JOURNAL_RESERVED ((SUM_JOURNAL_SIZE - 2) %\ - sizeof(struct sit_journal_entry)) - -/* Reserved area should make size of f2fs_extra_info equals to - * that of nat_journal and sit_journal. - */ -#define EXTRA_INFO_RESERVED (SUM_JOURNAL_SIZE - 2 - 8) - /* * frequently updated NAT/SIT entries can be stored in the spare area in * summary blocks @@ -498,9 +479,16 @@ struct nat_journal_entry { struct f2fs_nat_entry ne; } __packed; +/* + * The nat_journal structure is a placeholder whose actual size varies depending + * on the use of packed_ssa. Therefore, it must always be accessed only through + * specific sets of macros and fields, and size calculations should use + * size-related macros instead of sizeof(). + * Relevant macros: sbi->nat_journal_entries, nat_in_journal(), + * nid_in_journal(), MAX_NAT_JENTRIES(). + */ struct nat_journal { - struct nat_journal_entry entries[NAT_JOURNAL_ENTRIES]; - __u8 reserved[NAT_JOURNAL_RESERVED]; + struct nat_journal_entry entries[0]; } __packed; struct sit_journal_entry { @@ -508,14 +496,21 @@ struct sit_journal_entry { struct f2fs_sit_entry se; } __packed; +/* + * The sit_journal structure is a placeholder whose actual size varies depending + * on the use of packed_ssa. Therefore, it must always be accessed only through + * specific sets of macros and fields, and size calculations should use + * size-related macros instead of sizeof(). + * Relevant macros: sbi->sit_journal_entries, sit_in_journal(), + * segno_in_journal(), MAX_SIT_JENTRIES(). + */ struct sit_journal { - struct sit_journal_entry entries[SIT_JOURNAL_ENTRIES]; - __u8 reserved[SIT_JOURNAL_RESERVED]; + struct sit_journal_entry entries[0]; } __packed; struct f2fs_extra_info { __le64 kbytes_written; - __u8 reserved[EXTRA_INFO_RESERVED]; + __u8 reserved[]; } __packed; struct f2fs_journal { @@ -531,11 +526,33 @@ struct f2fs_journal { }; } __packed; -/* Block-sized summary block structure */ +/* + * Block-sized summary block structure + * + * The f2fs_summary_block structure is a placeholder whose actual size varies + * depending on the use of packed_ssa. Therefore, it must always be accessed + * only through specific sets of macros and fields, and size calculations should + * use size-related macros instead of sizeof(). + * Relevant macros: sbi->sum_blocksize, sbi->entries_in_sum, + * sbi->sum_entry_size, sum_entries(), sum_journal(), sum_footer(). + * + * Summary Block Layout + * + * +-----------------------+ <--- Block Start + * | struct f2fs_summary | + * | entries[0] | + * | ... | + * | entries[N-1] | + * +-----------------------+ + * | struct f2fs_journal | + * +-----------------------+ + * | struct summary_footer | + * +-----------------------+ <--- Block End + */ struct f2fs_summary_block { - struct f2fs_summary entries[ENTRIES_IN_SUM]; - struct f2fs_journal journal; - struct summary_footer footer; + struct f2fs_summary entries[0]; + // struct f2fs_journal journal; + // struct summary_footer footer; } __packed; /* diff --git a/include/linux/fb.h b/include/linux/fb.h index 05cc251035da98..c3302d51354660 100644 --- a/include/linux/fb.h +++ b/include/linux/fb.h @@ -497,7 +497,6 @@ struct fb_info { #if defined(CONFIG_FB_DEVICE) struct device *dev; /* This is this fb device */ #endif - int class_flag; /* private sysfs flags */ #ifdef CONFIG_FB_TILEBLITTING struct fb_tile_ops *tileops; /* Tile Blitting */ #endif diff --git a/include/linux/filter.h b/include/linux/filter.h index fd54fed8f95f3b..7452817d707d19 100644 --- a/include/linux/filter.h +++ b/include/linux/filter.h @@ -1375,24 +1375,13 @@ static inline bool bpf_jit_kallsyms_enabled(void) return false; } -int __bpf_address_lookup(unsigned long addr, unsigned long *size, - unsigned long *off, char *sym); +int bpf_address_lookup(unsigned long addr, unsigned long *size, + unsigned long *off, char *sym); bool is_bpf_text_address(unsigned long addr); int bpf_get_kallsym(unsigned int symnum, unsigned long *value, char *type, char *sym); struct bpf_prog *bpf_prog_ksym_find(unsigned long addr); -static inline int -bpf_address_lookup(unsigned long addr, unsigned long *size, - unsigned long *off, char **modname, char *sym) -{ - int ret = __bpf_address_lookup(addr, size, off, sym); - - if (ret && modname) - *modname = NULL; - return ret; -} - void bpf_prog_kallsyms_add(struct bpf_prog *fp); void bpf_prog_kallsyms_del(struct bpf_prog *fp); @@ -1431,8 +1420,8 @@ static inline bool bpf_jit_kallsyms_enabled(void) } static inline int -__bpf_address_lookup(unsigned long addr, unsigned long *size, - unsigned long *off, char *sym) +bpf_address_lookup(unsigned long addr, unsigned long *size, + unsigned long *off, char *sym) { return 0; } @@ -1453,13 +1442,6 @@ static inline struct bpf_prog *bpf_prog_ksym_find(unsigned long addr) return NULL; } -static inline int -bpf_address_lookup(unsigned long addr, unsigned long *size, - unsigned long *off, char **modname, char *sym) -{ - return 0; -} - static inline void bpf_prog_kallsyms_add(struct bpf_prog *fp) { } diff --git a/include/linux/firmware/intel/stratix10-svc-client.h b/include/linux/firmware/intel/stratix10-svc-client.h index d290060f4c73d1..91013161e9db98 100644 --- a/include/linux/firmware/intel/stratix10-svc-client.h +++ b/include/linux/firmware/intel/stratix10-svc-client.h @@ -68,12 +68,12 @@ * timeout value used in Stratix10 FPGA manager driver. * timeout value used in RSU driver */ -#define SVC_RECONFIG_REQUEST_TIMEOUT_MS 300 -#define SVC_RECONFIG_BUFFER_TIMEOUT_MS 720 -#define SVC_RSU_REQUEST_TIMEOUT_MS 300 +#define SVC_RECONFIG_REQUEST_TIMEOUT_MS 5000 +#define SVC_RECONFIG_BUFFER_TIMEOUT_MS 5000 +#define SVC_RSU_REQUEST_TIMEOUT_MS 2000 #define SVC_FCS_REQUEST_TIMEOUT_MS 2000 #define SVC_COMPLETED_TIMEOUT_MS 30000 -#define SVC_HWMON_REQUEST_TIMEOUT_MS 300 +#define SVC_HWMON_REQUEST_TIMEOUT_MS 2000 struct stratix10_svc_chan; diff --git a/include/linux/firmware/thead/thead,th1520-aon.h b/include/linux/firmware/thead/thead,th1520-aon.h index dae132b66873a8..d81f5f6f5b905a 100644 --- a/include/linux/firmware/thead/thead,th1520-aon.h +++ b/include/linux/firmware/thead/thead,th1520-aon.h @@ -97,80 +97,6 @@ struct th1520_aon_rpc_ack_common { #define RPC_GET_SVC_FLAG_ACK_TYPE(MESG) (((MESG)->svc & 0x40) >> 6) #define RPC_SET_SVC_FLAG_ACK_TYPE(MESG, ACK) ((MESG)->svc |= (ACK) << 6) -#define RPC_SET_BE64(MESG, OFFSET, SET_DATA) \ - do { \ - u8 *data = (u8 *)(MESG); \ - u64 _offset = (OFFSET); \ - u64 _set_data = (SET_DATA); \ - data[_offset + 7] = _set_data & 0xFF; \ - data[_offset + 6] = (_set_data & 0xFF00) >> 8; \ - data[_offset + 5] = (_set_data & 0xFF0000) >> 16; \ - data[_offset + 4] = (_set_data & 0xFF000000) >> 24; \ - data[_offset + 3] = (_set_data & 0xFF00000000) >> 32; \ - data[_offset + 2] = (_set_data & 0xFF0000000000) >> 40; \ - data[_offset + 1] = (_set_data & 0xFF000000000000) >> 48; \ - data[_offset + 0] = (_set_data & 0xFF00000000000000) >> 56; \ - } while (0) - -#define RPC_SET_BE32(MESG, OFFSET, SET_DATA) \ - do { \ - u8 *data = (u8 *)(MESG); \ - u64 _offset = (OFFSET); \ - u64 _set_data = (SET_DATA); \ - data[_offset + 3] = (_set_data) & 0xFF; \ - data[_offset + 2] = (_set_data & 0xFF00) >> 8; \ - data[_offset + 1] = (_set_data & 0xFF0000) >> 16; \ - data[_offset + 0] = (_set_data & 0xFF000000) >> 24; \ - } while (0) - -#define RPC_SET_BE16(MESG, OFFSET, SET_DATA) \ - do { \ - u8 *data = (u8 *)(MESG); \ - u64 _offset = (OFFSET); \ - u64 _set_data = (SET_DATA); \ - data[_offset + 1] = (_set_data) & 0xFF; \ - data[_offset + 0] = (_set_data & 0xFF00) >> 8; \ - } while (0) - -#define RPC_SET_U8(MESG, OFFSET, SET_DATA) \ - do { \ - u8 *data = (u8 *)(MESG); \ - data[OFFSET] = (SET_DATA) & 0xFF; \ - } while (0) - -#define RPC_GET_BE64(MESG, OFFSET, PTR) \ - do { \ - u8 *data = (u8 *)(MESG); \ - u64 _offset = (OFFSET); \ - *(u32 *)(PTR) = \ - (data[_offset + 7] | data[_offset + 6] << 8 | \ - data[_offset + 5] << 16 | data[_offset + 4] << 24 | \ - data[_offset + 3] << 32 | data[_offset + 2] << 40 | \ - data[_offset + 1] << 48 | data[_offset + 0] << 56); \ - } while (0) - -#define RPC_GET_BE32(MESG, OFFSET, PTR) \ - do { \ - u8 *data = (u8 *)(MESG); \ - u64 _offset = (OFFSET); \ - *(u32 *)(PTR) = \ - (data[_offset + 3] | data[_offset + 2] << 8 | \ - data[_offset + 1] << 16 | data[_offset + 0] << 24); \ - } while (0) - -#define RPC_GET_BE16(MESG, OFFSET, PTR) \ - do { \ - u8 *data = (u8 *)(MESG); \ - u64 _offset = (OFFSET); \ - *(u16 *)(PTR) = (data[_offset + 1] | data[_offset + 0] << 8); \ - } while (0) - -#define RPC_GET_U8(MESG, OFFSET, PTR) \ - do { \ - u8 *data = (u8 *)(MESG); \ - *(u8 *)(PTR) = (data[OFFSET]); \ - } while (0) - /* * Defines for SC PM Power Mode */ diff --git a/include/linux/fs/super_types.h b/include/linux/fs/super_types.h index 6bd3009e09b3b8..72ef2ae24abc42 100644 --- a/include/linux/fs/super_types.h +++ b/include/linux/fs/super_types.h @@ -332,5 +332,6 @@ struct super_block { #define SB_I_NOUMASK 0x00001000 /* VFS does not apply umask */ #define SB_I_NOIDMAP 0x00002000 /* No idmapped mounts on this superblock */ #define SB_I_ALLOW_HSM 0x00004000 /* Allow HSM events on this superblock */ +#define SB_I_NO_DATA_INTEGRITY 0x00008000 /* fs cannot guarantee data persistence on sync */ #endif /* _LINUX_FS_SUPER_TYPES_H */ diff --git a/include/linux/ftrace.h b/include/linux/ftrace.h index a3a8989e3268da..d029afcd22a5de 100644 --- a/include/linux/ftrace.h +++ b/include/linux/ftrace.h @@ -87,11 +87,13 @@ struct ftrace_hash; defined(CONFIG_DYNAMIC_FTRACE) int ftrace_mod_address_lookup(unsigned long addr, unsigned long *size, - unsigned long *off, char **modname, char *sym); + unsigned long *off, char **modname, + const unsigned char **modbuildid, char *sym); #else static inline int ftrace_mod_address_lookup(unsigned long addr, unsigned long *size, - unsigned long *off, char **modname, char *sym) + unsigned long *off, char **modname, + const unsigned char **modbuildid, char *sym) { return 0; } @@ -359,7 +361,6 @@ enum { FTRACE_OPS_FL_DIRECT = BIT(17), FTRACE_OPS_FL_SUBOP = BIT(18), FTRACE_OPS_FL_GRAPH = BIT(19), - FTRACE_OPS_FL_JMP = BIT(20), }; #ifndef CONFIG_DYNAMIC_FTRACE_WITH_ARGS @@ -1063,10 +1064,17 @@ static inline bool is_ftrace_trampoline(unsigned long addr) #ifdef CONFIG_FUNCTION_GRAPH_TRACER #ifndef ftrace_graph_func -#define ftrace_graph_func ftrace_stub -#define FTRACE_OPS_GRAPH_STUB FTRACE_OPS_FL_STUB +# define ftrace_graph_func ftrace_stub +# define FTRACE_OPS_GRAPH_STUB FTRACE_OPS_FL_STUB +/* + * The function graph is called every time the function tracer is called. + * It must always test the ops hash and cannot just directly call + * the handler. + */ +# define FGRAPH_NO_DIRECT 1 #else -#define FTRACE_OPS_GRAPH_STUB 0 +# define FTRACE_OPS_GRAPH_STUB 0 +# define FGRAPH_NO_DIRECT 0 #endif #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ diff --git a/include/linux/ftrace_regs.h b/include/linux/ftrace_regs.h index 15627ceea9bcc7..386fa48c4a9575 100644 --- a/include/linux/ftrace_regs.h +++ b/include/linux/ftrace_regs.h @@ -33,6 +33,31 @@ struct ftrace_regs; #define ftrace_regs_get_frame_pointer(fregs) \ frame_pointer(&arch_ftrace_regs(fregs)->regs) +static __always_inline void +ftrace_partial_regs_update(struct ftrace_regs *fregs, struct pt_regs *regs) { } + +#else + +/* + * ftrace_partial_regs_update - update the original ftrace_regs from regs + * @fregs: The ftrace_regs to update from @regs + * @regs: The partial regs from ftrace_partial_regs() that was updated + * + * Some architectures have the partial regs living in the ftrace_regs + * structure, whereas other architectures need to make a different copy + * of the @regs. If a partial @regs is retrieved by ftrace_partial_regs() and + * if the code using @regs updates a field (like the instruction pointer or + * stack pointer) it may need to propagate that change to the original @fregs + * it retrieved the partial @regs from. Use this function to guarantee that + * update happens. + */ +static __always_inline void +ftrace_partial_regs_update(struct ftrace_regs *fregs, struct pt_regs *regs) +{ + ftrace_regs_set_instruction_pointer(fregs, instruction_pointer(regs)); + ftrace_regs_set_return_value(fregs, regs_return_value(regs)); +} + #endif /* HAVE_ARCH_FTRACE_REGS */ /* This can be overridden by the architectures */ diff --git a/include/linux/fwnode.h b/include/linux/fwnode.h index 097be89487bf5c..53e46648131d91 100644 --- a/include/linux/fwnode.h +++ b/include/linux/fwnode.h @@ -229,5 +229,6 @@ int fwnode_link_add(struct fwnode_handle *con, struct fwnode_handle *sup, void fwnode_links_purge(struct fwnode_handle *fwnode); void fw_devlink_purge_absent_suppliers(struct fwnode_handle *fwnode); bool fw_devlink_is_strict(void); +int fw_devlink_count_absent_consumers(struct fwnode_handle *fwnode); #endif diff --git a/include/linux/hid.h b/include/linux/hid.h index dce862cafbbd35..5952d95a53e8c5 100644 --- a/include/linux/hid.h +++ b/include/linux/hid.h @@ -625,7 +625,9 @@ struct hid_input { enum hid_type { HID_TYPE_OTHER = 0, HID_TYPE_USBMOUSE, - HID_TYPE_USBNONE + HID_TYPE_USBNONE, + HID_TYPE_SPI_KEYBOARD, + HID_TYPE_SPI_MOUSE, }; enum hid_battery_status { @@ -786,6 +788,8 @@ struct hid_descriptor { .bus = BUS_BLUETOOTH, .vendor = (ven), .product = (prod) #define HID_I2C_DEVICE(ven, prod) \ .bus = BUS_I2C, .vendor = (ven), .product = (prod) +#define HID_SPI_DEVICE(ven, prod) \ + .bus = BUS_SPI, .vendor = (ven), .product = (prod) #define HID_REPORT_ID(rep) \ .report_type = (rep) diff --git a/include/linux/hisi_acc_qm.h b/include/linux/hisi_acc_qm.h index ca1ec437a3ca3b..59f9858049586a 100644 --- a/include/linux/hisi_acc_qm.h +++ b/include/linux/hisi_acc_qm.h @@ -447,12 +447,16 @@ struct hisi_qp_ops { int (*fill_sqe)(void *sqe, void *q_parm, void *d_parm); }; +struct instance_backlog { + struct list_head list; + spinlock_t lock; +}; + struct hisi_qp { u32 qp_id; u16 sq_depth; u16 cq_depth; u8 alg_type; - u8 req_type; struct qm_dma qdma; void *sqe; @@ -462,7 +466,6 @@ struct hisi_qp { struct hisi_qp_status qp_status; struct hisi_qp_ops *hw_ops; - void *qp_ctx; void (*req_cb)(struct hisi_qp *qp, void *data); void (*event_cb)(struct hisi_qp *qp); @@ -471,6 +474,10 @@ struct hisi_qp { bool is_in_kernel; u16 pasid; struct uacce_queue *uacce_q; + + spinlock_t qp_lock; + struct instance_backlog backlog; + const void **msg; }; static inline int vfs_num_set(const char *val, const struct kernel_param *kp) @@ -575,7 +582,7 @@ struct hisi_acc_sgl_pool *hisi_acc_create_sgl_pool(struct device *dev, void hisi_acc_free_sgl_pool(struct device *dev, struct hisi_acc_sgl_pool *pool); int hisi_qm_alloc_qps_node(struct hisi_qm_list *qm_list, int qp_num, - u8 alg_type, int node, struct hisi_qp **qps); + u8 *alg_type, int node, struct hisi_qp **qps); void hisi_qm_free_qps(struct hisi_qp **qps, int qp_num); void hisi_qm_dev_shutdown(struct pci_dev *pdev); void hisi_qm_wait_task_finish(struct hisi_qm *qm, struct hisi_qm_list *qm_list); diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index e51b8ef0cebd94..986372cd5c14b0 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h @@ -796,6 +796,23 @@ static inline unsigned huge_page_shift(struct hstate *h) return h->order + PAGE_SHIFT; } +/** + * hugetlb_linear_page_index() - linear_page_index() but in hugetlb + * page size granularity. + * @vma: the hugetlb VMA + * @address: the virtual address within the VMA + * + * Return: the page offset within the mapping in huge page units. + */ +static inline pgoff_t hugetlb_linear_page_index(struct vm_area_struct *vma, + unsigned long address) +{ + struct hstate *h = hstate_vma(vma); + + return ((address - vma->vm_start) >> huge_page_shift(h)) + + (vma->vm_pgoff >> huge_page_order(h)); +} + static inline bool order_is_gigantic(unsigned int order) { return order > MAX_PAGE_ORDER; diff --git a/include/linux/hw_random.h b/include/linux/hw_random.h index b424555753b11f..b77bc55a4cf356 100644 --- a/include/linux/hw_random.h +++ b/include/linux/hw_random.h @@ -15,6 +15,7 @@ #include #include #include +#include /** * struct hwrng - Hardware Random Number Generator driver @@ -48,6 +49,7 @@ struct hwrng { /* internal. */ struct list_head list; struct kref ref; + struct work_struct cleanup_work; struct completion cleanup_done; struct completion dying; }; diff --git a/include/linux/if_ether.h b/include/linux/if_ether.h index 61b7335aa037c7..ca9afa824aa4fa 100644 --- a/include/linux/if_ether.h +++ b/include/linux/if_ether.h @@ -40,7 +40,8 @@ static inline struct ethhdr *inner_eth_hdr(const struct sk_buff *skb) return (struct ethhdr *)skb_inner_mac_header(skb); } -int eth_header_parse(const struct sk_buff *skb, unsigned char *haddr); +int eth_header_parse(const struct sk_buff *skb, const struct net_device *dev, + unsigned char *haddr); extern ssize_t sysfs_format_mac(char *buf, const unsigned char *addr, int len); diff --git a/include/linux/iio/iio.h b/include/linux/iio/iio.h index 872ebdf0dd77a1..82ab8d9a8bf575 100644 --- a/include/linux/iio/iio.h +++ b/include/linux/iio/iio.h @@ -816,6 +816,18 @@ static inline void *iio_device_get_drvdata(const struct iio_dev *indio_dev) #define IIO_DECLARE_DMA_BUFFER_WITH_TS(type, name, count) \ __IIO_DECLARE_BUFFER_WITH_TS(type, name, count) __aligned(IIO_DMA_MINALIGN) +/** + * IIO_DECLARE_QUATERNION() - Declare a quaternion element + * @type: element type of the individual vectors + * @name: identifier name + * + * Quaternions are a vector composed of 4 elements (W, X, Y, Z). Use this macro + * to declare a quaternion element in a struct to ensure proper alignment in + * an IIO buffer. + */ +#define IIO_DECLARE_QUATERNION(type, name) \ + type name[4] __aligned(sizeof(type) * 4) + struct iio_dev *iio_device_alloc(struct device *parent, int sizeof_priv); /* The information at the returned address is guaranteed to be cacheline aligned */ diff --git a/include/linux/ima.h b/include/linux/ima.h index 8e29cb4e6a01da..abf8923f8fc51e 100644 --- a/include/linux/ima.h +++ b/include/linux/ima.h @@ -69,6 +69,7 @@ static inline int ima_measure_critical_data(const char *event_label, #ifdef CONFIG_HAVE_IMA_KEXEC int __init ima_free_kexec_buffer(void); int __init ima_get_kexec_buffer(void **addr, size_t *size); +int ima_validate_range(phys_addr_t phys, size_t size); #endif #ifdef CONFIG_IMA_SECURE_AND_OR_TRUSTED_BOOT diff --git a/include/linux/indirect_call_wrapper.h b/include/linux/indirect_call_wrapper.h index 35227d47cfc98c..dc272b514a01b8 100644 --- a/include/linux/indirect_call_wrapper.h +++ b/include/linux/indirect_call_wrapper.h @@ -16,22 +16,26 @@ */ #define INDIRECT_CALL_1(f, f1, ...) \ ({ \ - likely(f == f1) ? f1(__VA_ARGS__) : f(__VA_ARGS__); \ + typeof(f) __f1 = (f); \ + likely(__f1 == f1) ? f1(__VA_ARGS__) : __f1(__VA_ARGS__); \ }) #define INDIRECT_CALL_2(f, f2, f1, ...) \ ({ \ - likely(f == f2) ? f2(__VA_ARGS__) : \ - INDIRECT_CALL_1(f, f1, __VA_ARGS__); \ + typeof(f) __f2 = (f); \ + likely(__f2 == f2) ? f2(__VA_ARGS__) : \ + INDIRECT_CALL_1(__f2, f1, __VA_ARGS__); \ }) #define INDIRECT_CALL_3(f, f3, f2, f1, ...) \ ({ \ - likely(f == f3) ? f3(__VA_ARGS__) : \ - INDIRECT_CALL_2(f, f2, f1, __VA_ARGS__); \ + typeof(f) __f3 = (f); \ + likely(__f3 == f3) ? f3(__VA_ARGS__) : \ + INDIRECT_CALL_2(__f3, f2, f1, __VA_ARGS__); \ }) #define INDIRECT_CALL_4(f, f4, f3, f2, f1, ...) \ ({ \ - likely(f == f4) ? f4(__VA_ARGS__) : \ - INDIRECT_CALL_3(f, f3, f2, f1, __VA_ARGS__); \ + typeof(f) __f4 = (f); \ + likely(__f4 == f4) ? f4(__VA_ARGS__) : \ + INDIRECT_CALL_3(__f4, f3, f2, f1, __VA_ARGS__); \ }) #define INDIRECT_CALLABLE_DECLARE(f) f diff --git a/include/linux/inetdevice.h b/include/linux/inetdevice.h index 5730ba6b1cfaff..dccbeb25f70141 100644 --- a/include/linux/inetdevice.h +++ b/include/linux/inetdevice.h @@ -38,11 +38,11 @@ struct in_device { struct ip_mc_list *mc_tomb; unsigned long mr_v1_seen; unsigned long mr_v2_seen; - unsigned long mr_maxdelay; unsigned long mr_qi; /* Query Interval */ unsigned long mr_qri; /* Query Response Interval */ unsigned char mr_qrv; /* Query Robustness Variable */ unsigned char mr_gq_running; + u32 mr_maxdelay; u32 mr_ifc_count; struct timer_list mr_gq_timer; /* general query timer */ struct timer_list mr_ifc_timer; /* interface change timer */ diff --git a/include/linux/input/adp5589.h b/include/linux/input/adp5589.h deleted file mode 100644 index 0e4742c8c81e3c..00000000000000 --- a/include/linux/input/adp5589.h +++ /dev/null @@ -1,180 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Analog Devices ADP5589/ADP5585 I/O Expander and QWERTY Keypad Controller - * - * Copyright 2010-2011 Analog Devices Inc. - */ - -#ifndef _ADP5589_H -#define _ADP5589_H - -/* - * ADP5589 specific GPI and Keymap defines - */ - -#define ADP5589_KEYMAPSIZE 88 - -#define ADP5589_GPI_PIN_ROW0 97 -#define ADP5589_GPI_PIN_ROW1 98 -#define ADP5589_GPI_PIN_ROW2 99 -#define ADP5589_GPI_PIN_ROW3 100 -#define ADP5589_GPI_PIN_ROW4 101 -#define ADP5589_GPI_PIN_ROW5 102 -#define ADP5589_GPI_PIN_ROW6 103 -#define ADP5589_GPI_PIN_ROW7 104 -#define ADP5589_GPI_PIN_COL0 105 -#define ADP5589_GPI_PIN_COL1 106 -#define ADP5589_GPI_PIN_COL2 107 -#define ADP5589_GPI_PIN_COL3 108 -#define ADP5589_GPI_PIN_COL4 109 -#define ADP5589_GPI_PIN_COL5 110 -#define ADP5589_GPI_PIN_COL6 111 -#define ADP5589_GPI_PIN_COL7 112 -#define ADP5589_GPI_PIN_COL8 113 -#define ADP5589_GPI_PIN_COL9 114 -#define ADP5589_GPI_PIN_COL10 115 -#define GPI_LOGIC1 116 -#define GPI_LOGIC2 117 - -#define ADP5589_GPI_PIN_ROW_BASE ADP5589_GPI_PIN_ROW0 -#define ADP5589_GPI_PIN_ROW_END ADP5589_GPI_PIN_ROW7 -#define ADP5589_GPI_PIN_COL_BASE ADP5589_GPI_PIN_COL0 -#define ADP5589_GPI_PIN_COL_END ADP5589_GPI_PIN_COL10 - -#define ADP5589_GPI_PIN_BASE ADP5589_GPI_PIN_ROW_BASE -#define ADP5589_GPI_PIN_END ADP5589_GPI_PIN_COL_END - -#define ADP5589_GPIMAPSIZE_MAX (ADP5589_GPI_PIN_END - ADP5589_GPI_PIN_BASE + 1) - -/* - * ADP5585 specific GPI and Keymap defines - */ - -#define ADP5585_KEYMAPSIZE 30 - -#define ADP5585_GPI_PIN_ROW0 37 -#define ADP5585_GPI_PIN_ROW1 38 -#define ADP5585_GPI_PIN_ROW2 39 -#define ADP5585_GPI_PIN_ROW3 40 -#define ADP5585_GPI_PIN_ROW4 41 -#define ADP5585_GPI_PIN_ROW5 42 -#define ADP5585_GPI_PIN_COL0 43 -#define ADP5585_GPI_PIN_COL1 44 -#define ADP5585_GPI_PIN_COL2 45 -#define ADP5585_GPI_PIN_COL3 46 -#define ADP5585_GPI_PIN_COL4 47 -#define GPI_LOGIC 48 - -#define ADP5585_GPI_PIN_ROW_BASE ADP5585_GPI_PIN_ROW0 -#define ADP5585_GPI_PIN_ROW_END ADP5585_GPI_PIN_ROW5 -#define ADP5585_GPI_PIN_COL_BASE ADP5585_GPI_PIN_COL0 -#define ADP5585_GPI_PIN_COL_END ADP5585_GPI_PIN_COL4 - -#define ADP5585_GPI_PIN_BASE ADP5585_GPI_PIN_ROW_BASE -#define ADP5585_GPI_PIN_END ADP5585_GPI_PIN_COL_END - -#define ADP5585_GPIMAPSIZE_MAX (ADP5585_GPI_PIN_END - ADP5585_GPI_PIN_BASE + 1) - -struct adp5589_gpi_map { - unsigned short pin; - unsigned short sw_evt; -}; - -/* scan_cycle_time */ -#define ADP5589_SCAN_CYCLE_10ms 0 -#define ADP5589_SCAN_CYCLE_20ms 1 -#define ADP5589_SCAN_CYCLE_30ms 2 -#define ADP5589_SCAN_CYCLE_40ms 3 - -/* RESET_CFG */ -#define RESET_PULSE_WIDTH_500us 0 -#define RESET_PULSE_WIDTH_1ms 1 -#define RESET_PULSE_WIDTH_2ms 2 -#define RESET_PULSE_WIDTH_10ms 3 - -#define RESET_TRIG_TIME_0ms (0 << 2) -#define RESET_TRIG_TIME_1000ms (1 << 2) -#define RESET_TRIG_TIME_1500ms (2 << 2) -#define RESET_TRIG_TIME_2000ms (3 << 2) -#define RESET_TRIG_TIME_2500ms (4 << 2) -#define RESET_TRIG_TIME_3000ms (5 << 2) -#define RESET_TRIG_TIME_3500ms (6 << 2) -#define RESET_TRIG_TIME_4000ms (7 << 2) - -#define RESET_PASSTHRU_EN (1 << 5) -#define RESET1_POL_HIGH (1 << 6) -#define RESET1_POL_LOW (0 << 6) -#define RESET2_POL_HIGH (1 << 7) -#define RESET2_POL_LOW (0 << 7) - -/* ADP5589 Mask Bits: - * C C C C C C C C C C C | R R R R R R R R - * 1 9 8 7 6 5 4 3 2 1 0 | 7 6 5 4 3 2 1 0 - * 0 - * ---------------- BIT ------------------ - * 1 1 1 1 1 1 1 1 1 0 0 | 0 0 0 0 0 0 0 0 - * 8 7 6 5 4 3 2 1 0 9 8 | 7 6 5 4 3 2 1 0 - */ - -#define ADP_ROW(x) (1 << (x)) -#define ADP_COL(x) (1 << (x + 8)) -#define ADP5589_ROW_MASK 0xFF -#define ADP5589_COL_MASK 0xFF -#define ADP5589_COL_SHIFT 8 -#define ADP5589_MAX_ROW_NUM 7 -#define ADP5589_MAX_COL_NUM 10 - -/* ADP5585 Mask Bits: - * C C C C C | R R R R R R - * 4 3 2 1 0 | 5 4 3 2 1 0 - * - * ---- BIT -- ----------- - * 1 0 0 0 0 | 0 0 0 0 0 0 - * 0 9 8 7 6 | 5 4 3 2 1 0 - */ - -#define ADP5585_ROW_MASK 0x3F -#define ADP5585_COL_MASK 0x1F -#define ADP5585_ROW_SHIFT 0 -#define ADP5585_COL_SHIFT 6 -#define ADP5585_MAX_ROW_NUM 5 -#define ADP5585_MAX_COL_NUM 4 - -#define ADP5585_ROW(x) (1 << ((x) & ADP5585_ROW_MASK)) -#define ADP5585_COL(x) (1 << (((x) & ADP5585_COL_MASK) + ADP5585_COL_SHIFT)) - -/* Put one of these structures in i2c_board_info platform_data */ - -struct adp5589_kpad_platform_data { - unsigned keypad_en_mask; /* Keypad (Rows/Columns) enable mask */ - const unsigned short *keymap; /* Pointer to keymap */ - unsigned short keymapsize; /* Keymap size */ - bool repeat; /* Enable key repeat */ - bool en_keylock; /* Enable key lock feature (ADP5589 only)*/ - unsigned char unlock_key1; /* Unlock Key 1 (ADP5589 only) */ - unsigned char unlock_key2; /* Unlock Key 2 (ADP5589 only) */ - unsigned char unlock_timer; /* Time in seconds [0..7] between the two unlock keys 0=disable (ADP5589 only) */ - unsigned char scan_cycle_time; /* Time between consecutive scan cycles */ - unsigned char reset_cfg; /* Reset config */ - unsigned short reset1_key_1; /* Reset Key 1 */ - unsigned short reset1_key_2; /* Reset Key 2 */ - unsigned short reset1_key_3; /* Reset Key 3 */ - unsigned short reset2_key_1; /* Reset Key 1 */ - unsigned short reset2_key_2; /* Reset Key 2 */ - unsigned debounce_dis_mask; /* Disable debounce mask */ - unsigned pull_dis_mask; /* Disable all pull resistors mask */ - unsigned pullup_en_100k; /* Pull-Up 100k Enable Mask */ - unsigned pullup_en_300k; /* Pull-Up 300k Enable Mask */ - unsigned pulldown_en_300k; /* Pull-Down 300k Enable Mask */ - const struct adp5589_gpi_map *gpimap; - unsigned short gpimapsize; - const struct adp5589_gpio_platform_data *gpio_data; -}; - -struct i2c_client; /* forward declaration */ - -struct adp5589_gpio_platform_data { - int gpio_start; /* GPIO Chip base # */ -}; - -#endif diff --git a/include/linux/intel_rapl.h b/include/linux/intel_rapl.h index f479ef5b3341cf..fa1f328d671204 100644 --- a/include/linux/intel_rapl.h +++ b/include/linux/intel_rapl.h @@ -152,7 +152,7 @@ struct rapl_if_priv { union rapl_reg reg_unit; union rapl_reg regs[RAPL_DOMAIN_MAX][RAPL_DOMAIN_REG_MAX]; int limits[RAPL_DOMAIN_MAX]; - int (*read_raw)(int id, struct reg_action *ra, bool atomic); + int (*read_raw)(int id, struct reg_action *ra, bool pmu_ctx); int (*write_raw)(int id, struct reg_action *ra); void *defaults; void *rpi; diff --git a/include/linux/interrupt.h b/include/linux/interrupt.h index 266f2b39213a01..b2bb878abd113c 100644 --- a/include/linux/interrupt.h +++ b/include/linux/interrupt.h @@ -228,7 +228,7 @@ static inline int __must_check devm_request_irq(struct device *dev, unsigned int irq, irq_handler_t handler, unsigned long irqflags, const char *devname, void *dev_id) { - return devm_request_threaded_irq(dev, irq, handler, NULL, irqflags, + return devm_request_threaded_irq(dev, irq, handler, NULL, irqflags | IRQF_COND_ONESHOT, devname, dev_id); } diff --git a/include/linux/io-pgtable.h b/include/linux/io-pgtable.h index 7a1516011ccf72..5fc33ac318dec2 100644 --- a/include/linux/io-pgtable.h +++ b/include/linux/io-pgtable.h @@ -176,7 +176,7 @@ struct io_pgtable_cfg { } arm_mali_lpae_cfg; struct { - u64 ttbr[4]; + void *ttbr[4]; u32 n_ttbrs; u32 n_levels; } apple_dart_cfg; diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h index a3e8ddc9b380f8..f69b31303764ec 100644 --- a/include/linux/io_uring_types.h +++ b/include/linux/io_uring_types.h @@ -371,6 +371,7 @@ struct io_ring_ctx { * regularly bounce b/w CPUs. */ struct { + struct io_rings __rcu *rings_rcu; struct llist_head work_llist; struct llist_head retry_llist; unsigned long check_cq; @@ -444,6 +445,9 @@ struct io_ring_ctx { struct list_head defer_list; unsigned nr_drained; + /* protected by ->completion_lock */ + unsigned nr_req_allocated; + #ifdef CONFIG_NET_RX_BUSY_POLL struct list_head napi_list; /* track busy poll napi_id */ spinlock_t napi_lock; /* napi_list lock */ @@ -456,10 +460,6 @@ struct io_ring_ctx { DECLARE_HASHTABLE(napi_ht, 4); #endif - /* protected by ->completion_lock */ - unsigned evfd_last_cq_tail; - unsigned nr_req_allocated; - /* * Protection for resize vs mmap races - both the mmap and resize * side will need to grab this lock, to prevent either side from @@ -524,6 +524,7 @@ enum { REQ_F_BL_NO_RECYCLE_BIT, REQ_F_BUFFERS_COMMIT_BIT, REQ_F_BUF_NODE_BIT, + REQ_F_BUF_MORE_BIT, REQ_F_HAS_METADATA_BIT, REQ_F_IMPORT_BUFFER_BIT, REQ_F_SQE_COPIED_BIT, @@ -609,6 +610,8 @@ enum { REQ_F_BUFFERS_COMMIT = IO_REQ_FLAG(REQ_F_BUFFERS_COMMIT_BIT), /* buf node is valid */ REQ_F_BUF_NODE = IO_REQ_FLAG(REQ_F_BUF_NODE_BIT), + /* incremental buffer consumption, more space available */ + REQ_F_BUF_MORE = IO_REQ_FLAG(REQ_F_BUF_MORE_BIT), /* request has read/write metadata assigned */ REQ_F_HAS_METADATA = IO_REQ_FLAG(REQ_F_HAS_METADATA_BIT), /* diff --git a/include/linux/iommu.h b/include/linux/iommu.h index 8c66284a91a8b0..6e593b7dcc76c5 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -289,12 +289,18 @@ enum iommu_resv_type { IOMMU_RESV_MSI, /* Software-managed MSI translation window */ IOMMU_RESV_SW_MSI, + /* + * Memory regions which must be mapped with the specified mapping + * at all times. + */ + IOMMU_RESV_TRANSLATED, }; /** * struct iommu_resv_region - descriptor for a reserved memory region * @list: Linked list pointers * @start: System physical start address of the region + * @start: Device virtual start address of the region for IOMMU_RESV_TRANSLATED * @length: Length of the region in bytes * @prot: IOMMU Protection flags (READ/WRITE/...) * @type: Type of the reserved region @@ -303,6 +309,7 @@ enum iommu_resv_type { struct iommu_resv_region { struct list_head list; phys_addr_t start; + dma_addr_t dva; size_t length; int prot; enum iommu_resv_type type; @@ -838,6 +845,7 @@ struct iommu_fault_param { * @pci_32bit_workaround: Limit DMA allocations to 32-bit IOVAs * @require_direct: device requires IOMMU_RESV_DIRECT regions * @shadow_on_flush: IOTLB flushes are used to sync shadow tables + * @require_translated: device requires IOMMU_RESV_TRANSLATED regions * * TODO: migrate other per device data pointers under iommu_dev_data, e.g. * struct iommu_group *iommu_group; @@ -853,6 +861,7 @@ struct dev_iommu { u32 pci_32bit_workaround:1; u32 require_direct:1; u32 shadow_on_flush:1; + u32 require_translated:1; }; int iommu_device_register(struct iommu_device *iommu, @@ -937,6 +946,9 @@ extern bool iommu_default_passthrough(void); extern struct iommu_resv_region * iommu_alloc_resv_region(phys_addr_t start, size_t length, int prot, enum iommu_resv_type type, gfp_t gfp); +extern struct iommu_resv_region * +iommu_alloc_resv_region_tr(phys_addr_t start, dma_addr_t dva_start, size_t length, + int prot, enum iommu_resv_type type, gfp_t gfp); extern int iommu_get_group_resv_regions(struct iommu_group *group, struct list_head *head); diff --git a/include/linux/irqchip/arm-gic-v3.h b/include/linux/irqchip/arm-gic-v3.h index 70c0948f978eb0..0225121f301380 100644 --- a/include/linux/irqchip/arm-gic-v3.h +++ b/include/linux/irqchip/arm-gic-v3.h @@ -394,6 +394,7 @@ #define GITS_TYPER_VLPIS (1UL << 1) #define GITS_TYPER_ITT_ENTRY_SIZE_SHIFT 4 #define GITS_TYPER_ITT_ENTRY_SIZE GENMASK_ULL(7, 4) +#define GITS_TYPER_IDBITS GENMASK_ULL(12, 8) #define GITS_TYPER_IDBITS_SHIFT 8 #define GITS_TYPER_DEVBITS_SHIFT 13 #define GITS_TYPER_DEVBITS GENMASK_ULL(17, 13) diff --git a/include/linux/kthread.h b/include/linux/kthread.h index 8d27403888ce93..68d4b31d8989e0 100644 --- a/include/linux/kthread.h +++ b/include/linux/kthread.h @@ -7,6 +7,24 @@ struct mm_struct; +/* opaque kthread data */ +struct kthread; + +/* + * When "(p->flags & PF_KTHREAD)" is set the task is a kthread and will + * always remain a kthread. For kthreads p->worker_private always + * points to a struct kthread. For tasks that are not kthreads + * p->worker_private is used to point to other things. + * + * Return NULL for any task that is not a kthread. + */ +static inline struct kthread *tsk_is_kthread(struct task_struct *p) +{ + if (p->flags & PF_KTHREAD) + return p->worker_private; + return NULL; +} + __printf(4, 5) struct task_struct *kthread_create_on_node(int (*threadfn)(void *data), void *data, @@ -98,8 +116,9 @@ void *kthread_probe_data(struct task_struct *k); int kthread_park(struct task_struct *k); void kthread_unpark(struct task_struct *k); void kthread_parkme(void); -void kthread_exit(long result) __noreturn; +#define kthread_exit(result) do_exit(result) void kthread_complete_and_exit(struct completion *, long) __noreturn; +void kthread_do_exit(struct kthread *, long); int kthreadd(void *unused); extern struct task_struct *kthreadd_task; diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index d93f75b05ae227..7501735045e104 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -320,7 +320,8 @@ static inline bool kvm_vcpu_can_poll(ktime_t cur, ktime_t stop) struct kvm_mmio_fragment { gpa_t gpa; void *data; - unsigned len; + u64 val; + unsigned int len; }; struct kvm_vcpu { @@ -1030,6 +1031,13 @@ static inline struct kvm_vcpu *kvm_get_vcpu_by_id(struct kvm *kvm, int id) return NULL; } +static inline bool kvm_is_vcpu_creation_in_progress(struct kvm *kvm) +{ + lockdep_assert_held(&kvm->lock); + + return kvm->created_vcpus != atomic_read(&kvm->online_vcpus); +} + void kvm_destroy_vcpus(struct kvm *kvm); int kvm_trylock_all_vcpus(struct kvm *kvm); @@ -1927,56 +1935,43 @@ enum kvm_stat_kind { struct kvm_stat_data { struct kvm *kvm; - const struct _kvm_stats_desc *desc; + const struct kvm_stats_desc *desc; enum kvm_stat_kind kind; }; -struct _kvm_stats_desc { - struct kvm_stats_desc desc; - char name[KVM_STATS_NAME_SIZE]; -}; - -#define STATS_DESC_COMMON(type, unit, base, exp, sz, bsz) \ - .flags = type | unit | base | \ - BUILD_BUG_ON_ZERO(type & ~KVM_STATS_TYPE_MASK) | \ - BUILD_BUG_ON_ZERO(unit & ~KVM_STATS_UNIT_MASK) | \ - BUILD_BUG_ON_ZERO(base & ~KVM_STATS_BASE_MASK), \ - .exponent = exp, \ - .size = sz, \ +#define STATS_DESC_COMMON(type, unit, base, exp, sz, bsz) \ + .flags = type | unit | base | \ + BUILD_BUG_ON_ZERO(type & ~KVM_STATS_TYPE_MASK) | \ + BUILD_BUG_ON_ZERO(unit & ~KVM_STATS_UNIT_MASK) | \ + BUILD_BUG_ON_ZERO(base & ~KVM_STATS_BASE_MASK), \ + .exponent = exp, \ + .size = sz, \ .bucket_size = bsz -#define VM_GENERIC_STATS_DESC(stat, type, unit, base, exp, sz, bsz) \ - { \ - { \ - STATS_DESC_COMMON(type, unit, base, exp, sz, bsz), \ - .offset = offsetof(struct kvm_vm_stat, generic.stat) \ - }, \ - .name = #stat, \ - } -#define VCPU_GENERIC_STATS_DESC(stat, type, unit, base, exp, sz, bsz) \ - { \ - { \ - STATS_DESC_COMMON(type, unit, base, exp, sz, bsz), \ - .offset = offsetof(struct kvm_vcpu_stat, generic.stat) \ - }, \ - .name = #stat, \ - } -#define VM_STATS_DESC(stat, type, unit, base, exp, sz, bsz) \ - { \ - { \ - STATS_DESC_COMMON(type, unit, base, exp, sz, bsz), \ - .offset = offsetof(struct kvm_vm_stat, stat) \ - }, \ - .name = #stat, \ - } -#define VCPU_STATS_DESC(stat, type, unit, base, exp, sz, bsz) \ - { \ - { \ - STATS_DESC_COMMON(type, unit, base, exp, sz, bsz), \ - .offset = offsetof(struct kvm_vcpu_stat, stat) \ - }, \ - .name = #stat, \ - } +#define VM_GENERIC_STATS_DESC(stat, type, unit, base, exp, sz, bsz) \ +{ \ + STATS_DESC_COMMON(type, unit, base, exp, sz, bsz), \ + .offset = offsetof(struct kvm_vm_stat, generic.stat), \ + .name = #stat, \ +} +#define VCPU_GENERIC_STATS_DESC(stat, type, unit, base, exp, sz, bsz) \ +{ \ + STATS_DESC_COMMON(type, unit, base, exp, sz, bsz), \ + .offset = offsetof(struct kvm_vcpu_stat, generic.stat), \ + .name = #stat, \ +} +#define VM_STATS_DESC(stat, type, unit, base, exp, sz, bsz) \ +{ \ + STATS_DESC_COMMON(type, unit, base, exp, sz, bsz), \ + .offset = offsetof(struct kvm_vm_stat, stat), \ + .name = #stat, \ +} +#define VCPU_STATS_DESC(stat, type, unit, base, exp, sz, bsz) \ +{ \ + STATS_DESC_COMMON(type, unit, base, exp, sz, bsz), \ + .offset = offsetof(struct kvm_vcpu_stat, stat), \ + .name = #stat, \ +} /* SCOPE: VM, VM_GENERIC, VCPU, VCPU_GENERIC */ #define STATS_DESC(SCOPE, stat, type, unit, base, exp, sz, bsz) \ SCOPE##_STATS_DESC(stat, type, unit, base, exp, sz, bsz) @@ -2053,7 +2048,7 @@ struct _kvm_stats_desc { STATS_DESC_IBOOLEAN(VCPU_GENERIC, blocking) ssize_t kvm_stats_read(char *id, const struct kvm_stats_header *header, - const struct _kvm_stats_desc *desc, + const struct kvm_stats_desc *desc, void *stats, size_t size_stats, char __user *user_buffer, size_t size, loff_t *offset); @@ -2098,9 +2093,9 @@ static inline void kvm_stats_log_hist_update(u64 *data, size_t size, u64 value) extern const struct kvm_stats_header kvm_vm_stats_header; -extern const struct _kvm_stats_desc kvm_vm_stats_desc[]; +extern const struct kvm_stats_desc kvm_vm_stats_desc[]; extern const struct kvm_stats_header kvm_vcpu_stats_header; -extern const struct _kvm_stats_desc kvm_vcpu_stats_desc[]; +extern const struct kvm_stats_desc kvm_vcpu_stats_desc[]; #ifdef CONFIG_KVM_GENERIC_MMU_NOTIFIER static inline int mmu_invalidate_retry(struct kvm *kvm, unsigned long mmu_seq) diff --git a/include/linux/leafops.h b/include/linux/leafops.h index a9ff94b744f22c..05673d3529e754 100644 --- a/include/linux/leafops.h +++ b/include/linux/leafops.h @@ -363,6 +363,23 @@ static inline unsigned long softleaf_to_pfn(softleaf_t entry) return swp_offset(entry) & SWP_PFN_MASK; } +static inline void softleaf_migration_sync(softleaf_t entry, + struct folio *folio) +{ + /* + * Ensure we do not race with split, which might alter tail pages into new + * folios and thus result in observing an unlocked folio. + * This matches the write barrier in __split_folio_to_order(). + */ + smp_rmb(); + + /* + * Any use of migration entries may only occur while the + * corresponding page is locked + */ + VM_WARN_ON_ONCE(!folio_test_locked(folio)); +} + /** * softleaf_to_page() - Obtains struct page for PFN encoded within leaf entry. * @entry: Leaf entry, softleaf_has_pfn(@entry) must return true. @@ -374,11 +391,8 @@ static inline struct page *softleaf_to_page(softleaf_t entry) struct page *page = pfn_to_page(softleaf_to_pfn(entry)); VM_WARN_ON_ONCE(!softleaf_has_pfn(entry)); - /* - * Any use of migration entries may only occur while the - * corresponding page is locked - */ - VM_WARN_ON_ONCE(softleaf_is_migration(entry) && !PageLocked(page)); + if (softleaf_is_migration(entry)) + softleaf_migration_sync(entry, page_folio(page)); return page; } @@ -394,12 +408,8 @@ static inline struct folio *softleaf_to_folio(softleaf_t entry) struct folio *folio = pfn_folio(softleaf_to_pfn(entry)); VM_WARN_ON_ONCE(!softleaf_has_pfn(entry)); - /* - * Any use of migration entries may only occur while the - * corresponding folio is locked. - */ - VM_WARN_ON_ONCE(softleaf_is_migration(entry) && - !folio_test_locked(folio)); + if (softleaf_is_migration(entry)) + softleaf_migration_sync(entry, folio); return folio; } diff --git a/include/linux/leds-expresswire.h b/include/linux/leds-expresswire.h index a422921f4159f2..7f8c4795f69fa0 100644 --- a/include/linux/leds-expresswire.h +++ b/include/linux/leds-expresswire.h @@ -30,9 +30,6 @@ struct expresswire_common_props { void expresswire_power_off(struct expresswire_common_props *props); void expresswire_enable(struct expresswire_common_props *props); -void expresswire_start(struct expresswire_common_props *props); -void expresswire_end(struct expresswire_common_props *props); -void expresswire_set_bit(struct expresswire_common_props *props, bool bit); void expresswire_write_u8(struct expresswire_common_props *props, u8 val); #endif /* _LEDS_EXPRESSWIRE_H */ diff --git a/include/linux/libata.h b/include/linux/libata.h index 39534fafa36ae1..a5c66eb708f466 100644 --- a/include/linux/libata.h +++ b/include/linux/libata.h @@ -75,7 +75,7 @@ enum ata_quirks { __ATA_QUIRK_NO_DMA_LOG, /* Do not use DMA for log read */ __ATA_QUIRK_NOTRIM, /* Do not use TRIM */ __ATA_QUIRK_MAX_SEC_1024, /* Limit max sects to 1024 */ - __ATA_QUIRK_MAX_SEC_8191, /* Limit max sects to 8191 */ + __ATA_QUIRK_MAX_SEC, /* Limit max sectors */ __ATA_QUIRK_MAX_TRIM_128M, /* Limit max trim size to 128M */ __ATA_QUIRK_NO_NCQ_ON_ATI, /* Disable NCQ on ATI chipset */ __ATA_QUIRK_NO_LPM_ON_ATI, /* Disable LPM on ATI chipset */ @@ -116,7 +116,7 @@ enum { ATA_QUIRK_NO_DMA_LOG = (1U << __ATA_QUIRK_NO_DMA_LOG), ATA_QUIRK_NOTRIM = (1U << __ATA_QUIRK_NOTRIM), ATA_QUIRK_MAX_SEC_1024 = (1U << __ATA_QUIRK_MAX_SEC_1024), - ATA_QUIRK_MAX_SEC_8191 = (1U << __ATA_QUIRK_MAX_SEC_8191), + ATA_QUIRK_MAX_SEC = (1U << __ATA_QUIRK_MAX_SEC), ATA_QUIRK_MAX_TRIM_128M = (1U << __ATA_QUIRK_MAX_TRIM_128M), ATA_QUIRK_NO_NCQ_ON_ATI = (1U << __ATA_QUIRK_NO_NCQ_ON_ATI), ATA_QUIRK_NO_LPM_ON_ATI = (1U << __ATA_QUIRK_NO_LPM_ON_ATI), @@ -903,6 +903,9 @@ struct ata_port { u64 qc_active; int nr_active_links; /* #links with active qcs */ + struct work_struct deferred_qc_work; + struct ata_queued_cmd *deferred_qc; + struct ata_link link; /* host default link */ struct ata_link *slave_link; /* see ata_slave_link_init() */ diff --git a/include/linux/liveupdate.h b/include/linux/liveupdate.h index a7f6ee5b677184..5ccdcc462af2be 100644 --- a/include/linux/liveupdate.h +++ b/include/linux/liveupdate.h @@ -20,8 +20,11 @@ struct file; /** * struct liveupdate_file_op_args - Arguments for file operation callbacks. * @handler: The file handler being called. - * @retrieved: The retrieve status for the 'can_finish / finish' - * operation. + * @retrieve_status: The retrieve status for the 'can_finish / finish' + * operation. A value of 0 means the retrieve has not been + * attempted, a positive value means the retrieve was + * successful, and a negative value means the retrieve failed, + * and the value is the error code of the call. * @file: The file object. For retrieve: [OUT] The callback sets * this to the new file. For other ops: [IN] The caller sets * this to the file being operated on. @@ -37,7 +40,7 @@ struct file; */ struct liveupdate_file_op_args { struct liveupdate_file_handler *handler; - bool retrieved; + int retrieve_status; struct file *file; u64 serialized_data; void *private_data; diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h index 0651865a4564fa..412db7663357e3 100644 --- a/include/linux/memcontrol.h +++ b/include/linux/memcontrol.h @@ -1744,7 +1744,7 @@ static inline void count_objcg_events(struct obj_cgroup *objcg, rcu_read_unlock(); } -bool mem_cgroup_node_allowed(struct mem_cgroup *memcg, int nid); +void mem_cgroup_node_filter_allowed(struct mem_cgroup *memcg, nodemask_t *mask); void mem_cgroup_show_protected_memory(struct mem_cgroup *memcg); @@ -1815,9 +1815,9 @@ static inline ino_t page_cgroup_ino(struct page *page) return 0; } -static inline bool mem_cgroup_node_allowed(struct mem_cgroup *memcg, int nid) +static inline void mem_cgroup_node_filter_allowed(struct mem_cgroup *memcg, + nodemask_t *mask) { - return true; } static inline void mem_cgroup_show_protected_memory(struct mem_cgroup *memcg) diff --git a/include/linux/memory_ordering_model.h b/include/linux/memory_ordering_model.h new file mode 100644 index 00000000000000..267a12ca66307e --- /dev/null +++ b/include/linux/memory_ordering_model.h @@ -0,0 +1,11 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_MEMORY_ORDERING_MODEL_H +#define __ASM_MEMORY_ORDERING_MODEL_H + +/* Arch hooks to implement the PR_{GET_SET}_MEM_MODEL prctls */ + +struct task_struct; +int arch_prctl_mem_model_get(struct task_struct *t); +int arch_prctl_mem_model_set(struct task_struct *t, unsigned long val); + +#endif diff --git a/include/linux/mempolicy.h b/include/linux/mempolicy.h index 0fe96f3ab3ef02..65c732d440d2f4 100644 --- a/include/linux/mempolicy.h +++ b/include/linux/mempolicy.h @@ -55,6 +55,7 @@ struct mempolicy { nodemask_t cpuset_mems_allowed; /* relative to these nodes */ nodemask_t user_nodemask; /* nodemask passed by user */ } w; + struct rcu_head rcu; }; /* diff --git a/include/linux/mfd/tps65219.h b/include/linux/mfd/tps65219.h index 55234e771ba735..3abf937191d0c2 100644 --- a/include/linux/mfd/tps65219.h +++ b/include/linux/mfd/tps65219.h @@ -149,6 +149,8 @@ enum pmic_id { #define TPS65215_ENABLE_LDO2_EN_MASK BIT(5) #define TPS65214_ENABLE_LDO1_EN_MASK BIT(5) #define TPS65219_ENABLE_LDO4_EN_MASK BIT(6) +/* Register Unlock */ +#define TPS65214_LOCK_ACCESS_CMD 0x5a /* power ON-OFF sequence slot */ #define TPS65219_BUCKS_LDOS_SEQUENCE_OFF_SLOT_MASK GENMASK(3, 0) #define TPS65219_BUCKS_LDOS_SEQUENCE_ON_SLOT_MASK GENMASK(7, 4) diff --git a/include/linux/mfd/wm8350/core.h b/include/linux/mfd/wm8350/core.h index 5f70d3b5d1b1a4..097ef4dfcdac8c 100644 --- a/include/linux/mfd/wm8350/core.h +++ b/include/linux/mfd/wm8350/core.h @@ -667,7 +667,7 @@ static inline int wm8350_register_irq(struct wm8350 *wm8350, int irq, return -ENODEV; return request_threaded_irq(irq + wm8350->irq_base, NULL, - handler, flags, name, data); + handler, flags | IRQF_ONESHOT, name, data); } static inline void wm8350_free_irq(struct wm8350 *wm8350, int irq, void *data) diff --git a/include/linux/migrate.h b/include/linux/migrate.h index 26ca00c325d91a..d5af2b7f577b33 100644 --- a/include/linux/migrate.h +++ b/include/linux/migrate.h @@ -65,7 +65,7 @@ bool isolate_folio_to_list(struct folio *folio, struct list_head *list); int migrate_huge_page_move_mapping(struct address_space *mapping, struct folio *dst, struct folio *src); -void migration_entry_wait_on_locked(softleaf_t entry, spinlock_t *ptl) +void softleaf_entry_wait_on_locked(softleaf_t entry, spinlock_t *ptl) __releases(ptl); void folio_migrate_flags(struct folio *newfolio, struct folio *folio); int folio_migrate_mapping(struct address_space *mapping, @@ -97,6 +97,14 @@ static inline int set_movable_ops(const struct movable_operations *ops, enum pag return -ENOSYS; } +static inline void softleaf_entry_wait_on_locked(softleaf_t entry, spinlock_t *ptl) + __releases(ptl) +{ + WARN_ON_ONCE(1); + + spin_unlock(ptl); +} + #endif /* CONFIG_MIGRATION */ #ifdef CONFIG_NUMA_BALANCING diff --git a/include/linux/mlx5/driver.h b/include/linux/mlx5/driver.h index 1c54aa6f74fbc6..1967d1c79139bb 100644 --- a/include/linux/mlx5/driver.h +++ b/include/linux/mlx5/driver.h @@ -1281,12 +1281,12 @@ static inline bool mlx5_rl_is_supported(struct mlx5_core_dev *dev) static inline int mlx5_core_is_mp_slave(struct mlx5_core_dev *dev) { return MLX5_CAP_GEN(dev, affiliate_nic_vport_criteria) && - MLX5_CAP_GEN(dev, num_vhca_ports) <= 1; + MLX5_CAP_GEN_MAX(dev, num_vhca_ports) <= 1; } static inline int mlx5_core_is_mp_master(struct mlx5_core_dev *dev) { - return MLX5_CAP_GEN(dev, num_vhca_ports) > 1; + return MLX5_CAP_GEN_MAX(dev, num_vhca_ports) > 1; } static inline int mlx5_core_mp_enabled(struct mlx5_core_dev *dev) diff --git a/include/linux/mm.h b/include/linux/mm.h index f0d5be9dc7368c..05dbfdb6dd1e04 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -3304,26 +3304,21 @@ static inline bool ptlock_init(struct ptdesc *ptdesc) { return true; } static inline void ptlock_free(struct ptdesc *ptdesc) {} #endif /* defined(CONFIG_SPLIT_PTE_PTLOCKS) */ -static inline unsigned long ptdesc_nr_pages(const struct ptdesc *ptdesc) -{ - return compound_nr(ptdesc_page(ptdesc)); -} - static inline void __pagetable_ctor(struct ptdesc *ptdesc) { - pg_data_t *pgdat = NODE_DATA(memdesc_nid(ptdesc->pt_flags)); + struct folio *folio = ptdesc_folio(ptdesc); - __SetPageTable(ptdesc_page(ptdesc)); - mod_node_page_state(pgdat, NR_PAGETABLE, ptdesc_nr_pages(ptdesc)); + __folio_set_pgtable(folio); + lruvec_stat_add_folio(folio, NR_PAGETABLE); } static inline void pagetable_dtor(struct ptdesc *ptdesc) { - pg_data_t *pgdat = NODE_DATA(memdesc_nid(ptdesc->pt_flags)); + struct folio *folio = ptdesc_folio(ptdesc); ptlock_free(ptdesc); - __ClearPageTable(ptdesc_page(ptdesc)); - mod_node_page_state(pgdat, NR_PAGETABLE, -ptdesc_nr_pages(ptdesc)); + __folio_clear_pgtable(folio); + lruvec_stat_sub_folio(folio, NR_PAGETABLE); } static inline void pagetable_dtor_free(struct ptdesc *ptdesc) diff --git a/include/linux/mmap_lock.h b/include/linux/mmap_lock.h index d53f72dba7feea..81fcfde3563dd9 100644 --- a/include/linux/mmap_lock.h +++ b/include/linux/mmap_lock.h @@ -345,7 +345,7 @@ static inline void mmap_write_lock_nested(struct mm_struct *mm, int subclass) __mmap_lock_trace_acquire_returned(mm, true, true); } -static inline int mmap_write_lock_killable(struct mm_struct *mm) +static inline int __must_check mmap_write_lock_killable(struct mm_struct *mm) { int ret; @@ -392,7 +392,7 @@ static inline void mmap_read_lock(struct mm_struct *mm) __mmap_lock_trace_acquire_returned(mm, false, true); } -static inline int mmap_read_lock_killable(struct mm_struct *mm) +static inline int __must_check mmap_read_lock_killable(struct mm_struct *mm) { int ret; @@ -402,7 +402,7 @@ static inline int mmap_read_lock_killable(struct mm_struct *mm) return ret; } -static inline bool mmap_read_trylock(struct mm_struct *mm) +static inline bool __must_check mmap_read_trylock(struct mm_struct *mm) { bool ret; diff --git a/include/linux/mmc/host.h b/include/linux/mmc/host.h index e0e2c265e5d101..ba84f02c2a101e 100644 --- a/include/linux/mmc/host.h +++ b/include/linux/mmc/host.h @@ -486,14 +486,12 @@ struct mmc_host { struct mmc_ios ios; /* current io bus settings */ + bool claimed; /* host exclusively claimed */ + /* group bitfields together to minimize padding */ unsigned int use_spi_crc:1; - unsigned int claimed:1; /* host exclusively claimed */ unsigned int doing_init_tune:1; /* initial tuning in progress */ - unsigned int can_retune:1; /* re-tuning can be used */ unsigned int doing_retune:1; /* re-tuning in progress */ - unsigned int retune_now:1; /* do re-tuning at next req */ - unsigned int retune_paused:1; /* re-tuning is temporarily disabled */ unsigned int retune_crc_disable:1; /* don't trigger retune upon crc */ unsigned int can_dma_map_merge:1; /* merging can be used */ unsigned int vqmmc_enabled:1; /* vqmmc regulator is enabled */ @@ -508,6 +506,9 @@ struct mmc_host { int rescan_disable; /* disable card detection */ int rescan_entered; /* used with nonremovable devices */ + bool can_retune; /* re-tuning can be used */ + bool retune_now; /* do re-tuning at next req */ + bool retune_paused; /* re-tuning is temporarily disabled */ int need_retune; /* re-tuning is needed */ int hold_retune; /* hold off re-tuning */ unsigned int retune_period; /* re-tuning period in secs */ diff --git a/include/linux/module.h b/include/linux/module.h index d80c3ea5747266..ac254525014cd6 100644 --- a/include/linux/module.h +++ b/include/linux/module.h @@ -748,6 +748,15 @@ static inline void __module_get(struct module *module) __mod ? __mod->name : "kernel"; \ }) +static inline const unsigned char *module_buildid(struct module *mod) +{ +#ifdef CONFIG_STACKTRACE_BUILD_ID + return mod->build_id; +#else + return NULL; +#endif +} + /* Dereference module function descriptor */ void *dereference_module_function_descriptor(struct module *mod, void *ptr); diff --git a/include/linux/mtd/spinand.h b/include/linux/mtd/spinand.h index ce76f5c632e179..049d55c38d5228 100644 --- a/include/linux/mtd/spinand.h +++ b/include/linux/mtd/spinand.h @@ -287,7 +287,7 @@ struct spinand_device; /** * struct spinand_id - SPI NAND id structure - * @data: buffer containing the id bytes. Currently 4 bytes large, but can + * @data: buffer containing the id bytes. Currently 5 bytes large, but can * be extended if required * @len: ID length */ diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index d99b0fbc1942ad..846afec74703b4 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -311,7 +311,9 @@ struct header_ops { int (*create) (struct sk_buff *skb, struct net_device *dev, unsigned short type, const void *daddr, const void *saddr, unsigned int len); - int (*parse)(const struct sk_buff *skb, unsigned char *haddr); + int (*parse)(const struct sk_buff *skb, + const struct net_device *dev, + unsigned char *haddr); int (*cache)(const struct neighbour *neigh, struct hh_cache *hh, __be16 type); void (*cache_update)(struct hh_cache *hh, const struct net_device *dev, @@ -1831,6 +1833,8 @@ enum netdev_reg_state { * * @mpls_features: Mask of features inheritable by MPLS * @gso_partial_features: value(s) from NETIF_F_GSO\* + * @mangleid_features: Mask of features requiring MANGLEID, will be + * disabled together with the latter. * * @ifindex: interface index * @group: The group the device belongs to @@ -2153,6 +2157,7 @@ struct net_device { unsigned long state; unsigned int flags; unsigned short hard_header_len; + enum netdev_stat_type pcpu_stat_type:8; netdev_features_t features; struct inet6_dev __rcu *ip6_ptr; __cacheline_group_end(net_device_read_txrx); @@ -2219,6 +2224,7 @@ struct net_device { netdev_features_t vlan_features; netdev_features_t hw_enc_features; netdev_features_t mpls_features; + netdev_features_t mangleid_features; unsigned int min_mtu; unsigned int max_mtu; @@ -2401,8 +2407,6 @@ struct net_device { void *ml_priv; enum netdev_ml_priv_type ml_priv_type; - enum netdev_stat_type pcpu_stat_type:8; - #if IS_ENABLED(CONFIG_GARP) struct garp_port __rcu *garp_port; #endif @@ -3443,7 +3447,7 @@ static inline int dev_parse_header(const struct sk_buff *skb, if (!dev->header_ops || !dev->header_ops->parse) return 0; - return dev->header_ops->parse(skb, haddr); + return dev->header_ops->parse(skb, dev, haddr); } static inline __be16 dev_parse_header_protocol(const struct sk_buff *skb) @@ -3573,17 +3577,49 @@ struct page_pool_bh { }; DECLARE_PER_CPU(struct page_pool_bh, system_page_pool); +#define XMIT_RECURSION_LIMIT 8 + #ifndef CONFIG_PREEMPT_RT static inline int dev_recursion_level(void) { return this_cpu_read(softnet_data.xmit.recursion); } + +static inline bool dev_xmit_recursion(void) +{ + return unlikely(__this_cpu_read(softnet_data.xmit.recursion) > + XMIT_RECURSION_LIMIT); +} + +static inline void dev_xmit_recursion_inc(void) +{ + __this_cpu_inc(softnet_data.xmit.recursion); +} + +static inline void dev_xmit_recursion_dec(void) +{ + __this_cpu_dec(softnet_data.xmit.recursion); +} #else static inline int dev_recursion_level(void) { return current->net_xmit.recursion; } +static inline bool dev_xmit_recursion(void) +{ + return unlikely(current->net_xmit.recursion > XMIT_RECURSION_LIMIT); +} + +static inline void dev_xmit_recursion_inc(void) +{ + current->net_xmit.recursion++; +} + +static inline void dev_xmit_recursion_dec(void) +{ + current->net_xmit.recursion--; +} #endif void __netif_schedule(struct Qdisc *q); @@ -4708,7 +4744,7 @@ static inline u32 netif_msg_init(int debug_value, int default_msg_enable_bits) static inline void __netif_tx_lock(struct netdev_queue *txq, int cpu) { spin_lock(&txq->_xmit_lock); - /* Pairs with READ_ONCE() in __dev_queue_xmit() */ + /* Pairs with READ_ONCE() in netif_tx_owned() */ WRITE_ONCE(txq->xmit_lock_owner, cpu); } @@ -4726,7 +4762,7 @@ static inline void __netif_tx_release(struct netdev_queue *txq) static inline void __netif_tx_lock_bh(struct netdev_queue *txq) { spin_lock_bh(&txq->_xmit_lock); - /* Pairs with READ_ONCE() in __dev_queue_xmit() */ + /* Pairs with READ_ONCE() in netif_tx_owned() */ WRITE_ONCE(txq->xmit_lock_owner, smp_processor_id()); } @@ -4735,7 +4771,7 @@ static inline bool __netif_tx_trylock(struct netdev_queue *txq) bool ok = spin_trylock(&txq->_xmit_lock); if (likely(ok)) { - /* Pairs with READ_ONCE() in __dev_queue_xmit() */ + /* Pairs with READ_ONCE() in netif_tx_owned() */ WRITE_ONCE(txq->xmit_lock_owner, smp_processor_id()); } return ok; @@ -4743,14 +4779,14 @@ static inline bool __netif_tx_trylock(struct netdev_queue *txq) static inline void __netif_tx_unlock(struct netdev_queue *txq) { - /* Pairs with READ_ONCE() in __dev_queue_xmit() */ + /* Pairs with READ_ONCE() in netif_tx_owned() */ WRITE_ONCE(txq->xmit_lock_owner, -1); spin_unlock(&txq->_xmit_lock); } static inline void __netif_tx_unlock_bh(struct netdev_queue *txq) { - /* Pairs with READ_ONCE() in __dev_queue_xmit() */ + /* Pairs with READ_ONCE() in netif_tx_owned() */ WRITE_ONCE(txq->xmit_lock_owner, -1); spin_unlock_bh(&txq->_xmit_lock); } @@ -4843,6 +4879,23 @@ static inline void netif_tx_disable(struct net_device *dev) local_bh_enable(); } +#ifndef CONFIG_PREEMPT_RT +static inline bool netif_tx_owned(struct netdev_queue *txq, unsigned int cpu) +{ + /* Other cpus might concurrently change txq->xmit_lock_owner + * to -1 or to their cpu id, but not to our id. + */ + return READ_ONCE(txq->xmit_lock_owner) == cpu; +} + +#else +static inline bool netif_tx_owned(struct netdev_queue *txq, unsigned int cpu) +{ + return rt_mutex_owner(&txq->_xmit_lock.lock) == current; +} + +#endif + static inline void netif_addr_lock(struct net_device *dev) { unsigned char nest_level = 0; diff --git a/include/linux/netfilter/ipset/ip_set.h b/include/linux/netfilter/ipset/ip_set.h index e9f4f845d760af..b98331572ad298 100644 --- a/include/linux/netfilter/ipset/ip_set.h +++ b/include/linux/netfilter/ipset/ip_set.h @@ -309,7 +309,7 @@ enum { /* register and unregister set references */ extern ip_set_id_t ip_set_get_byname(struct net *net, - const char *name, struct ip_set **set); + const struct nlattr *name, struct ip_set **set); extern void ip_set_put_byindex(struct net *net, ip_set_id_t index); extern void ip_set_name_byindex(struct net *net, ip_set_id_t index, char *name); extern ip_set_id_t ip_set_nfnl_get_byindex(struct net *net, ip_set_id_t index); diff --git a/include/linux/netfs.h b/include/linux/netfs.h index 72ee7d210a7445..ba17ac5bf356ae 100644 --- a/include/linux/netfs.h +++ b/include/linux/netfs.h @@ -140,7 +140,6 @@ struct netfs_io_stream { void (*issue_write)(struct netfs_io_subrequest *subreq); /* Collection tracking */ struct list_head subrequests; /* Contributory I/O operations */ - struct netfs_io_subrequest *front; /* Op being collected */ unsigned long long collected_to; /* Position we've collected results to */ size_t transferred; /* The amount transferred from this stream */ unsigned short error; /* Aggregate error for the stream */ diff --git a/include/linux/ns_common.h b/include/linux/ns_common.h index 825f5865bfc5ae..c8e227a3f9e22b 100644 --- a/include/linux/ns_common.h +++ b/include/linux/ns_common.h @@ -55,6 +55,8 @@ static __always_inline bool is_ns_init_id(const struct ns_common *ns) #define ns_common_free(__ns) __ns_common_free(to_ns_common((__ns))) +bool may_see_all_namespaces(void); + static __always_inline __must_check int __ns_ref_active_read(const struct ns_common *ns) { return atomic_read(&ns->__ns_ref_active); diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h index ec442af3f88613..31a848485ad9d9 100644 --- a/include/linux/pagemap.h +++ b/include/linux/pagemap.h @@ -210,7 +210,6 @@ enum mapping_flags { AS_WRITEBACK_MAY_DEADLOCK_ON_RECLAIM = 9, AS_KERNEL_FILE = 10, /* mapping for a fake kernel file that shouldn't account usage to user cgroups */ - AS_NO_DATA_INTEGRITY = 11, /* no data integrity guarantees */ /* Bits 16-25 are used for FOLIO_ORDER */ AS_FOLIO_ORDER_BITS = 5, AS_FOLIO_ORDER_MIN = 16, @@ -346,16 +345,6 @@ static inline bool mapping_writeback_may_deadlock_on_reclaim(const struct addres return test_bit(AS_WRITEBACK_MAY_DEADLOCK_ON_RECLAIM, &mapping->flags); } -static inline void mapping_set_no_data_integrity(struct address_space *mapping) -{ - set_bit(AS_NO_DATA_INTEGRITY, &mapping->flags); -} - -static inline bool mapping_no_data_integrity(const struct address_space *mapping) -{ - return test_bit(AS_NO_DATA_INTEGRITY, &mapping->flags); -} - static inline gfp_t mapping_gfp_mask(const struct address_space *mapping) { return mapping->gfp_mask; diff --git a/include/linux/pci-epc.h b/include/linux/pci-epc.h index 4286bfdbfdfad2..c021c7af175fd0 100644 --- a/include/linux/pci-epc.h +++ b/include/linux/pci-epc.h @@ -223,6 +223,13 @@ struct pci_epc_bar_desc { /** * struct pci_epc_features - features supported by a EPC device per function * @linkup_notifier: indicate if the EPC device can notify EPF driver on link up + * @dynamic_inbound_mapping: indicate if the EPC device supports updating + * inbound mappings for an already configured BAR + * (i.e. allow calling pci_epc_set_bar() again + * without first calling pci_epc_clear_bar()) + * @subrange_mapping: indicate if the EPC device can map inbound subranges for a + * BAR. This feature depends on @dynamic_inbound_mapping + * feature. * @msi_capable: indicate if the endpoint function has MSI capability * @msix_capable: indicate if the endpoint function has MSI-X capability * @intx_capable: indicate if the endpoint can raise INTx interrupts @@ -231,6 +238,8 @@ struct pci_epc_bar_desc { */ struct pci_epc_features { unsigned int linkup_notifier : 1; + unsigned int dynamic_inbound_mapping : 1; + unsigned int subrange_mapping : 1; unsigned int msi_capable : 1; unsigned int msix_capable : 1; unsigned int intx_capable : 1; diff --git a/include/linux/pci-epf.h b/include/linux/pci-epf.h index 48f68c4dcfa5e2..7737a7c03260d9 100644 --- a/include/linux/pci-epf.h +++ b/include/linux/pci-epf.h @@ -110,6 +110,22 @@ struct pci_epf_driver { #define to_pci_epf_driver(drv) container_of_const((drv), struct pci_epf_driver, driver) +/** + * struct pci_epf_bar_submap - BAR subrange for inbound mapping + * @phys_addr: target physical/DMA address for this subrange + * @size: the size of the subrange to be mapped + * + * When pci_epf_bar.num_submap is >0, pci_epf_bar.submap describes the + * complete BAR layout. This allows an EPC driver to program multiple + * inbound translation windows for a single BAR when supported by the + * controller. The array order defines the BAR layout (submap[0] at offset + * 0, and each immediately follows the previous one). + */ +struct pci_epf_bar_submap { + dma_addr_t phys_addr; + size_t size; +}; + /** * struct pci_epf_bar - represents the BAR of EPF device * @phys_addr: physical address that should be mapped to the BAR @@ -119,6 +135,9 @@ struct pci_epf_driver { * requirement * @barno: BAR number * @flags: flags that are set for the BAR + * @num_submap: number of entries in @submap + * @submap: array of subrange descriptors allocated by the caller. See + * struct pci_epf_bar_submap for the semantics in detail. */ struct pci_epf_bar { dma_addr_t phys_addr; @@ -127,6 +146,10 @@ struct pci_epf_bar { size_t mem_size; enum pci_barno barno; int flags; + + /* Optional sub-range mapping */ + unsigned int num_submap; + struct pci_epf_bar_submap *submap; }; /** diff --git a/include/linux/pci.h b/include/linux/pci.h index b5cc0c2b99065d..e958ff74433569 100644 --- a/include/linux/pci.h +++ b/include/linux/pci.h @@ -406,6 +406,7 @@ struct pci_dev { user sysfs */ unsigned int clear_retrain_link:1; /* Need to clear Retrain Link bit manually */ + unsigned int no_bw_notif:1; /* BW notifications may cause issues */ unsigned int d3hot_delay; /* D3hot->D0 transition time in ms */ unsigned int d3cold_delay; /* D3cold->D0 transition time in ms */ diff --git a/include/linux/pci_ids.h b/include/linux/pci_ids.h index a9a089566b7cbe..f2849ff1830b11 100644 --- a/include/linux/pci_ids.h +++ b/include/linux/pci_ids.h @@ -3143,6 +3143,7 @@ #define PCI_DEVICE_ID_INTEL_HDA_CML_S 0xa3f0 #define PCI_DEVICE_ID_INTEL_HDA_LNL_P 0xa828 #define PCI_DEVICE_ID_INTEL_S21152BB 0xb152 +#define PCI_DEVICE_ID_INTEL_HDA_NVL 0xd328 #define PCI_DEVICE_ID_INTEL_HDA_BMG 0xe2f7 #define PCI_DEVICE_ID_INTEL_HDA_PTL_H 0xe328 #define PCI_DEVICE_ID_INTEL_HDA_PTL 0xe428 diff --git a/include/linux/pinctrl/pinconf-generic.h b/include/linux/pinctrl/pinconf-generic.h index 1be4032071c23b..89277808ea6141 100644 --- a/include/linux/pinctrl/pinconf-generic.h +++ b/include/linux/pinctrl/pinconf-generic.h @@ -250,9 +250,4 @@ static inline int pinconf_generic_dt_node_to_map_all(struct pinctrl_dev *pctldev return pinconf_generic_dt_node_to_map(pctldev, np_config, map, num_maps, PIN_MAP_TYPE_INVALID); } - -int pinconf_generic_dt_node_to_map_pinmux(struct pinctrl_dev *pctldev, - struct device_node *np, - struct pinctrl_map **map, - unsigned int *num_maps); #endif /* __LINUX_PINCTRL_PINCONF_GENERIC_H */ diff --git a/include/linux/platform_device.h b/include/linux/platform_device.h index 813da101b5bf8e..ed1d50d1c3c15c 100644 --- a/include/linux/platform_device.h +++ b/include/linux/platform_device.h @@ -31,11 +31,6 @@ struct platform_device { struct resource *resource; const struct platform_device_id *id_entry; - /* - * Driver name to force a match. Do not set directly, because core - * frees it. Use driver_set_override() to set or clear it. - */ - const char *driver_override; /* MFD cell pointer */ struct mfd_cell *mfd_cell; diff --git a/include/linux/pm.h b/include/linux/pm.h index 98a899858eceee..afcaaa37a81266 100644 --- a/include/linux/pm.h +++ b/include/linux/pm.h @@ -681,10 +681,10 @@ struct dev_pm_info { struct list_head entry; struct completion completion; struct wakeup_source *wakeup; + bool work_in_progress; /* Owned by the PM core */ bool wakeup_path:1; bool syscore:1; bool no_pm_callbacks:1; /* Owned by the PM core */ - bool work_in_progress:1; /* Owned by the PM core */ bool smart_suspend:1; /* Owned by the PM core */ bool must_resume:1; /* Owned by the PM core */ bool may_skip_resume:1; /* Set by subsystems */ diff --git a/include/linux/pm_domain.h b/include/linux/pm_domain.h index 93ba0143ca476a..309385f9b4c8f9 100644 --- a/include/linux/pm_domain.h +++ b/include/linux/pm_domain.h @@ -121,6 +121,13 @@ struct dev_pm_domain_list { * powered-off until the ->sync_state() callback is * invoked. This flag informs genpd to allow a * power-off without waiting for ->sync_state(). + * GENPD_FLAG_DEFER_OFF: Defer powerdown if there are any consumer + * device fwlinks indicating that some consumer + * devices have not yet probed. This is useful + * for power domains which are active at boot and + * must not be shut down until all consumers + * complete their probe sequence. + */ #define GENPD_FLAG_PM_CLK (1U << 0) #define GENPD_FLAG_IRQ_SAFE (1U << 1) @@ -133,6 +140,7 @@ struct dev_pm_domain_list { #define GENPD_FLAG_DEV_NAME_FW (1U << 8) #define GENPD_FLAG_NO_SYNC_STATE (1U << 9) #define GENPD_FLAG_NO_STAY_ON (1U << 10) +#define GENPD_FLAG_DEFER_OFF (1U << 11) enum gpd_status { GENPD_STATE_ON = 0, /* PM domain is on */ diff --git a/include/linux/psp.h b/include/linux/psp.h index 92e60aeef21e13..b337dcce1e9916 100644 --- a/include/linux/psp.h +++ b/include/linux/psp.h @@ -18,6 +18,7 @@ * and should include an appropriate local definition in their source file. */ #define PSP_CMDRESP_STS GENMASK(15, 0) +#define PSP_TEE_STS_RING_BUSY 0x0000000d /* Ring already initialized */ #define PSP_CMDRESP_CMD GENMASK(23, 16) #define PSP_CMDRESP_RESERVED GENMASK(29, 24) #define PSP_CMDRESP_RECOVERY BIT(30) diff --git a/include/linux/ring_buffer.h b/include/linux/ring_buffer.h index 876358cfe1b125..d862fa610270bf 100644 --- a/include/linux/ring_buffer.h +++ b/include/linux/ring_buffer.h @@ -248,6 +248,7 @@ int trace_rb_cpu_prepare(unsigned int cpu, struct hlist_node *node); int ring_buffer_map(struct trace_buffer *buffer, int cpu, struct vm_area_struct *vma); +void ring_buffer_map_dup(struct trace_buffer *buffer, int cpu); int ring_buffer_unmap(struct trace_buffer *buffer, int cpu); int ring_buffer_map_get_reader(struct trace_buffer *buffer, int cpu); #endif /* _LINUX_RING_BUFFER_H */ diff --git a/include/linux/rseq_types.h b/include/linux/rseq_types.h index ef0811379c5401..a612959c5b17f2 100644 --- a/include/linux/rseq_types.h +++ b/include/linux/rseq_types.h @@ -103,10 +103,12 @@ struct rseq_data { }; * @active: MM CID is active for the task * @cid: The CID associated to the task either permanently or * borrowed from the CPU + * @node: Queued in the per MM MMCID list */ struct sched_mm_cid { unsigned int active; unsigned int cid; + struct hlist_node node; }; /** @@ -127,6 +129,7 @@ struct mm_cid_pcpu { * @work: Regular work to handle the affinity mode change case * @lock: Spinlock to protect against affinity setting which can't take @mutex * @mutex: Mutex to serialize forks and exits related to this mm + * @user_list: List of the MM CID users of a MM * @nr_cpus_allowed: The number of CPUs in the per MM allowed CPUs map. The map * is growth only. * @users: The number of tasks sharing this MM. Separate from mm::mm_users @@ -147,13 +150,14 @@ struct mm_mm_cid { raw_spinlock_t lock; struct mutex mutex; + struct hlist_head user_list; /* Low frequency modified */ unsigned int nr_cpus_allowed; unsigned int users; unsigned int pcpu_thrs; unsigned int update_deferred; -}____cacheline_aligned_in_smp; +} ____cacheline_aligned; #else /* CONFIG_SCHED_MM_CID */ struct mm_mm_cid { }; struct sched_mm_cid { }; diff --git a/include/linux/sched.h b/include/linux/sched.h index 5f00b5ed0f3b7d..0719862970a28c 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -574,6 +574,7 @@ struct sched_entity { u64 deadline; u64 min_vruntime; u64 min_slice; + u64 max_slice; struct list_head group_node; unsigned char on_rq; @@ -2312,7 +2313,6 @@ static __always_inline void alloc_tag_restore(struct alloc_tag *tag, struct allo #ifdef CONFIG_SCHED_MM_CID void sched_mm_cid_before_execve(struct task_struct *t); void sched_mm_cid_after_execve(struct task_struct *t); -void sched_mm_cid_fork(struct task_struct *t); void sched_mm_cid_exit(struct task_struct *t); static __always_inline int task_mm_cid(struct task_struct *t) { @@ -2321,7 +2321,6 @@ static __always_inline int task_mm_cid(struct task_struct *t) #else static inline void sched_mm_cid_before_execve(struct task_struct *t) { } static inline void sched_mm_cid_after_execve(struct task_struct *t) { } -static inline void sched_mm_cid_fork(struct task_struct *t) { } static inline void sched_mm_cid_exit(struct task_struct *t) { } static __always_inline int task_mm_cid(struct task_struct *t) { diff --git a/include/linux/security.h b/include/linux/security.h index 83a646d72f6f8f..ee88dd2d2d1f71 100644 --- a/include/linux/security.h +++ b/include/linux/security.h @@ -145,6 +145,7 @@ enum lockdown_reason { LOCKDOWN_BPF_WRITE_USER, LOCKDOWN_DBG_WRITE_KERNEL, LOCKDOWN_RTAS_ERROR_INJECTION, + LOCKDOWN_XEN_USER_ACTIONS, LOCKDOWN_INTEGRITY_MAX, LOCKDOWN_KCORE, LOCKDOWN_KPROBES, diff --git a/include/linux/seq_file.h b/include/linux/seq_file.h index d6ebf0596510c9..2fb266ea69fa4d 100644 --- a/include/linux/seq_file.h +++ b/include/linux/seq_file.h @@ -181,7 +181,6 @@ int seq_open_private(struct file *, const struct seq_operations *, int); int seq_release_private(struct inode *, struct file *); #ifdef CONFIG_BINARY_PRINTF -__printf(2, 0) void seq_bprintf(struct seq_file *m, const char *f, const u32 *binary); #endif diff --git a/include/linux/seqlock.h b/include/linux/seqlock.h index 221123660e710e..f827cc3cb1460d 100644 --- a/include/linux/seqlock.h +++ b/include/linux/seqlock.h @@ -1303,15 +1303,14 @@ __scoped_seqlock_next(struct ss_tmp *sst, seqlock_t *lock, enum ss_state target) __scoped_seqlock_next(&_s, _seqlock, _target)) /** - * scoped_seqlock_read (lock, ss_state) - execute the read side critical - * section without manual sequence - * counter handling or calls to other - * helpers - * @lock: pointer to seqlock_t protecting the data - * @ss_state: one of {ss_lock, ss_lock_irqsave, ss_lockless} indicating - * the type of critical read section - * - * Example: + * scoped_seqlock_read() - execute the read-side critical section + * without manual sequence counter handling + * or calls to other helpers + * @_seqlock: pointer to seqlock_t protecting the data + * @_target: an enum ss_state: one of {ss_lock, ss_lock_irqsave, ss_lockless} + * indicating the type of critical read section + * + * Example:: * * scoped_seqlock_read (&lock, ss_lock) { * // read-side critical section diff --git a/include/linux/serial_8250.h b/include/linux/serial_8250.h index 01efdce0fda078..a95b2d143d2489 100644 --- a/include/linux/serial_8250.h +++ b/include/linux/serial_8250.h @@ -195,6 +195,7 @@ void serial8250_do_set_mctrl(struct uart_port *port, unsigned int mctrl); void serial8250_do_set_divisor(struct uart_port *port, unsigned int baud, unsigned int quot); int fsl8250_handle_irq(struct uart_port *port); +void serial8250_handle_irq_locked(struct uart_port *port, unsigned int iir); int serial8250_handle_irq(struct uart_port *port, unsigned int iir); u16 serial8250_rx_chars(struct uart_8250_port *up, u16 lsr); void serial8250_read_char(struct uart_8250_port *up, u16 lsr); diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index 112e48970338fa..13c6eca3bbc693 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -5093,6 +5093,7 @@ static inline bool skb_has_extensions(struct sk_buff *skb) return unlikely(skb->active_extensions); } #else +static inline void __skb_ext_put(struct skb_ext *ext) {} static inline void skb_ext_put(struct sk_buff *skb) {} static inline void skb_ext_reset(struct sk_buff *skb) {} static inline void skb_ext_del(struct sk_buff *skb, int unused) {} diff --git a/include/linux/skmsg.h b/include/linux/skmsg.h index 49847888c287ab..829b281d6c9c27 100644 --- a/include/linux/skmsg.h +++ b/include/linux/skmsg.h @@ -97,6 +97,8 @@ struct sk_psock { struct sk_buff_head ingress_skb; struct list_head ingress_msg; spinlock_t ingress_lock; + /** @msg_tot_len: Total bytes queued in ingress_msg list. */ + u32 msg_tot_len; unsigned long state; struct list_head link; spinlock_t link_lock; @@ -141,6 +143,8 @@ int sk_msg_memcopy_from_iter(struct sock *sk, struct iov_iter *from, struct sk_msg *msg, u32 bytes); int sk_msg_recvmsg(struct sock *sk, struct sk_psock *psock, struct msghdr *msg, int len, int flags); +int __sk_msg_recvmsg(struct sock *sk, struct sk_psock *psock, struct msghdr *msg, + int len, int flags, int *copied_from_self); bool sk_msg_is_readable(struct sock *sk); static inline void sk_msg_check_to_free(struct sk_msg *msg, u32 i, u32 bytes) @@ -319,6 +323,27 @@ static inline void sock_drop(struct sock *sk, struct sk_buff *skb) kfree_skb(skb); } +static inline u32 sk_psock_get_msg_len_nolock(struct sk_psock *psock) +{ + /* Used by ioctl to read msg_tot_len only; lock-free for performance */ + return READ_ONCE(psock->msg_tot_len); +} + +static inline void sk_psock_msg_len_add_locked(struct sk_psock *psock, int diff) +{ + /* Use WRITE_ONCE to ensure correct read in sk_psock_get_msg_len_nolock(). + * ingress_lock should be held to prevent concurrent updates to msg_tot_len + */ + WRITE_ONCE(psock->msg_tot_len, psock->msg_tot_len + diff); +} + +static inline void sk_psock_msg_len_add(struct sk_psock *psock, int diff) +{ + spin_lock_bh(&psock->ingress_lock); + sk_psock_msg_len_add_locked(psock, diff); + spin_unlock_bh(&psock->ingress_lock); +} + static inline bool sk_psock_queue_msg(struct sk_psock *psock, struct sk_msg *msg) { @@ -327,6 +352,7 @@ static inline bool sk_psock_queue_msg(struct sk_psock *psock, spin_lock_bh(&psock->ingress_lock); if (sk_psock_test_state(psock, SK_PSOCK_TX_ENABLED)) { list_add_tail(&msg->list, &psock->ingress_msg); + sk_psock_msg_len_add_locked(psock, msg->sg.size); ret = true; } else { sk_msg_free(psock->sk, msg); @@ -343,18 +369,25 @@ static inline struct sk_msg *sk_psock_dequeue_msg(struct sk_psock *psock) spin_lock_bh(&psock->ingress_lock); msg = list_first_entry_or_null(&psock->ingress_msg, struct sk_msg, list); - if (msg) + if (msg) { list_del(&msg->list); + sk_psock_msg_len_add_locked(psock, -msg->sg.size); + } spin_unlock_bh(&psock->ingress_lock); return msg; } +static inline struct sk_msg *sk_psock_peek_msg_locked(struct sk_psock *psock) +{ + return list_first_entry_or_null(&psock->ingress_msg, struct sk_msg, list); +} + static inline struct sk_msg *sk_psock_peek_msg(struct sk_psock *psock) { struct sk_msg *msg; spin_lock_bh(&psock->ingress_lock); - msg = list_first_entry_or_null(&psock->ingress_msg, struct sk_msg, list); + msg = sk_psock_peek_msg_locked(psock); spin_unlock_bh(&psock->ingress_lock); return msg; } @@ -521,6 +554,39 @@ static inline bool sk_psock_strp_enabled(struct sk_psock *psock) return !!psock->saved_data_ready; } +/* for tcp only, sk is locked */ +static inline ssize_t sk_psock_msg_inq(struct sock *sk) +{ + struct sk_psock *psock; + ssize_t inq = 0; + + psock = sk_psock_get(sk); + if (likely(psock)) { + inq = sk_psock_get_msg_len_nolock(psock); + sk_psock_put(sk, psock); + } + return inq; +} + +/* for udp only, sk is not locked */ +static inline ssize_t sk_msg_first_len(struct sock *sk) +{ + struct sk_psock *psock; + struct sk_msg *msg; + ssize_t inq = 0; + + psock = sk_psock_get(sk); + if (likely(psock)) { + spin_lock_bh(&psock->ingress_lock); + msg = sk_psock_peek_msg_locked(psock); + if (msg) + inq = msg->sg.size; + spin_unlock_bh(&psock->ingress_lock); + sk_psock_put(sk, psock); + } + return inq; +} + #if IS_ENABLED(CONFIG_NET_SOCK_MSG) #define BPF_F_STRPARSER (1UL << 1) diff --git a/include/linux/soc/apple/dockchannel.h b/include/linux/soc/apple/dockchannel.h new file mode 100644 index 00000000000000..0b7093935ddf47 --- /dev/null +++ b/include/linux/soc/apple/dockchannel.h @@ -0,0 +1,26 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* + * Apple Dockchannel devices + * Copyright (C) The Asahi Linux Contributors + */ +#ifndef _LINUX_APPLE_DOCKCHANNEL_H_ +#define _LINUX_APPLE_DOCKCHANNEL_H_ + +#include +#include +#include + +#if IS_ENABLED(CONFIG_APPLE_DOCKCHANNEL) + +struct dockchannel; + +struct dockchannel *dockchannel_init(struct platform_device *pdev); + +int dockchannel_send(struct dockchannel *dockchannel, const void *buf, size_t count); +int dockchannel_recv(struct dockchannel *dockchannel, void *buf, size_t count); +int dockchannel_await(struct dockchannel *dockchannel, + void (*callback)(void *cookie, size_t avail), + void *cookie, size_t count); + +#endif +#endif diff --git a/drivers/soc/apple/mailbox.h b/include/linux/soc/apple/mailbox.h similarity index 100% rename from drivers/soc/apple/mailbox.h rename to include/linux/soc/apple/mailbox.h diff --git a/include/linux/soc/apple/rtkit.h b/include/linux/soc/apple/rtkit.h index 736f530180179b..22e1d3bb35ef0c 100644 --- a/include/linux/soc/apple/rtkit.h +++ b/include/linux/soc/apple/rtkit.h @@ -78,6 +78,13 @@ struct apple_rtkit; struct apple_rtkit *devm_apple_rtkit_init(struct device *dev, void *cookie, const char *mbox_name, int mbox_idx, const struct apple_rtkit_ops *ops); +/* + * Frees internal RTKit state allocated by devm_apple_rtkit_init(). + * + * @dev: Pointer to the device node this coprocessor is assocated with + * @rtk: Internal RTKit state initialized by devm_apple_rtkit_init() + */ +void devm_apple_rtkit_free(struct device *dev, struct apple_rtkit *rtk); /* * Non-devm version of devm_apple_rtkit_init. Must be freed with @@ -172,4 +179,12 @@ int apple_rtkit_send_message(struct apple_rtkit *rtk, u8 ep, u64 message, */ int apple_rtkit_poll(struct apple_rtkit *rtk); +/* + * Checks if an endpoint with a given index exists + * + * @rtk: RTKit reference + * @ep: endpoint to check for + */ +bool apple_rtkit_has_endpoint(struct apple_rtkit *rtk, u8 ep); + #endif /* _LINUX_APPLE_RTKIT_H_ */ diff --git a/include/linux/soc/apple/tunable.h b/include/linux/soc/apple/tunable.h new file mode 100644 index 00000000000000..531ca814cd0235 --- /dev/null +++ b/include/linux/soc/apple/tunable.h @@ -0,0 +1,62 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* + * Apple Silicon hardware tunable support + * + * Each tunable is a list with each entry containing a offset into the MMIO + * region, a mask of bits to be cleared and a set of bits to be set. These + * tunables are passed along by the previous boot stages and vary from device + * to device such that they cannot be hardcoded in the individual drivers. + * + * Copyright (C) The Asahi Linux Contributors + */ + +#ifndef _LINUX_SOC_APPLE_TUNABLE_H_ +#define _LINUX_SOC_APPLE_TUNABLE_H_ + +#include +#include + +/** + * Struct to store an Apple Silicon hardware tunable. + * + * Each tunable is a list with each entry containing a offset into the MMIO + * region, a mask of bits to be cleared and a set of bits to be set. These + * tunables are passed along by the previous boot stages and vary from device + * to device such that they cannot be hardcoded in the individual drivers. + * + * @param sz Number of [offset, mask, value] tuples stored in values. + * @param values [offset, mask, value] array. + */ +struct apple_tunable { + size_t sz; + struct { + u32 offset; + u32 mask; + u32 value; + } values[] __counted_by(sz); +}; + +/** + * Parse an array of hardware tunables from the device tree. + * + * @dev: Device node used for devm_kzalloc internally. + * @np: Device node which contains the tunable array. + * @name: Name of the device tree property which contains the tunables. + * @res: Resource to which the tunables will be applied, used for bound checking + * + * @return: devres allocated struct on success or PTR_ERR on failure. + */ +struct apple_tunable *devm_apple_tunable_parse(struct device *dev, + struct device_node *np, + const char *name, + struct resource *res); + +/** + * Apply a previously loaded hardware tunable. + * + * @param regs: MMIO to which the tunable will be applied. + * @param tunable: Pointer to the tunable. + */ +void apple_tunable_apply(void __iomem *regs, struct apple_tunable *tunable); + +#endif diff --git a/include/linux/soc/qcom/pdr.h b/include/linux/soc/qcom/pdr.h index 83a8ea612e69a3..2b7691e47c2a95 100644 --- a/include/linux/soc/qcom/pdr.h +++ b/include/linux/soc/qcom/pdr.h @@ -5,6 +5,7 @@ #include #define SERVREG_NAME_LENGTH 64 +#define SERVREG_PFR_LENGTH 256 struct pdr_service; struct pdr_handle; diff --git a/include/linux/soc/qcom/ubwc.h b/include/linux/soc/qcom/ubwc.h index 0a4edfe3d96d4f..f052e241736c4b 100644 --- a/include/linux/soc/qcom/ubwc.h +++ b/include/linux/soc/qcom/ubwc.h @@ -8,6 +8,7 @@ #define __QCOM_UBWC_H__ #include +#include #include struct qcom_ubwc_cfg_data { diff --git a/include/linux/spi/spi.h b/include/linux/spi/spi.h index cb2c2df3108999..fe9dd430cc03a3 100644 --- a/include/linux/spi/spi.h +++ b/include/linux/spi/spi.h @@ -156,10 +156,6 @@ extern void spi_transfer_cs_change_delay_exec(struct spi_message *msg, * @modalias: Name of the driver to use with this device, or an alias * for that name. This appears in the sysfs "modalias" attribute * for driver coldplugging, and in uevents used for hotplugging - * @driver_override: If the name of a driver is written to this attribute, then - * the device will bind to the named driver and only the named driver. - * Do not set directly, because core frees it; use driver_set_override() to - * set or clear it. * @pcpu_statistics: statistics for the spi_device * @word_delay: delay to be inserted between consecutive * words of a transfer @@ -217,7 +213,6 @@ struct spi_device { void *controller_state; void *controller_data; char modalias[SPI_NAME_SIZE]; - const char *driver_override; /* The statistics */ struct spi_statistics __percpu *pcpu_statistics; diff --git a/include/linux/srcutiny.h b/include/linux/srcutiny.h index e0698024667a70..313a0e17f22feb 100644 --- a/include/linux/srcutiny.h +++ b/include/linux/srcutiny.h @@ -11,6 +11,7 @@ #ifndef _LINUX_SRCU_TINY_H #define _LINUX_SRCU_TINY_H +#include #include struct srcu_struct { @@ -24,18 +25,21 @@ struct srcu_struct { struct rcu_head *srcu_cb_head; /* Pending callbacks: Head. */ struct rcu_head **srcu_cb_tail; /* Pending callbacks: Tail. */ struct work_struct srcu_work; /* For driving grace periods. */ + struct irq_work srcu_irq_work; /* Defer schedule_work() to irq work. */ #ifdef CONFIG_DEBUG_LOCK_ALLOC struct lockdep_map dep_map; #endif /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */ }; void srcu_drive_gp(struct work_struct *wp); +void srcu_tiny_irq_work(struct irq_work *irq_work); #define __SRCU_STRUCT_INIT(name, __ignored, ___ignored, ____ignored) \ { \ .srcu_wq = __SWAIT_QUEUE_HEAD_INITIALIZER(name.srcu_wq), \ .srcu_cb_tail = &name.srcu_cb_head, \ .srcu_work = __WORK_INITIALIZER(name.srcu_work, srcu_drive_gp), \ + .srcu_irq_work = { .func = srcu_tiny_irq_work }, \ __SRCU_DEP_MAP_INIT(name) \ } diff --git a/include/linux/stmmac.h b/include/linux/stmmac.h index f1054b9c2d8aca..0c26ccfeeb8d86 100644 --- a/include/linux/stmmac.h +++ b/include/linux/stmmac.h @@ -299,7 +299,6 @@ struct plat_stmmacenet_data { int int_snapshot_num; int msi_mac_vec; int msi_wol_vec; - int msi_lpi_vec; int msi_sfty_ce_vec; int msi_sfty_ue_vec; int msi_rx_base_vec; diff --git a/include/linux/string.h b/include/linux/string.h index 1b564c36d721b9..b850bd91b3d884 100644 --- a/include/linux/string.h +++ b/include/linux/string.h @@ -336,8 +336,8 @@ int __sysfs_match_string(const char * const *array, size_t n, const char *s); #define sysfs_match_string(_a, _s) __sysfs_match_string(_a, ARRAY_SIZE(_a), _s) #ifdef CONFIG_BINARY_PRINTF -__printf(3, 0) int vbin_printf(u32 *bin_buf, size_t size, const char *fmt, va_list args); -__printf(3, 0) int bstr_printf(char *buf, size_t size, const char *fmt, const u32 *bin_buf); +int vbin_printf(u32 *bin_buf, size_t size, const char *fmt, va_list args); +int bstr_printf(char *buf, size_t size, const char *fmt, const u32 *bin_buf); #endif extern ssize_t memory_read_from_buffer(void *to, size_t count, loff_t *ppos, diff --git a/include/linux/sunrpc/xdrgen/_builtins.h b/include/linux/sunrpc/xdrgen/_builtins.h index 66ca3ece951ab9..a5ab75d2db0446 100644 --- a/include/linux/sunrpc/xdrgen/_builtins.h +++ b/include/linux/sunrpc/xdrgen/_builtins.h @@ -188,12 +188,10 @@ xdrgen_decode_string(struct xdr_stream *xdr, string *ptr, u32 maxlen) return false; if (unlikely(maxlen && len > maxlen)) return false; - if (len != 0) { - p = xdr_inline_decode(xdr, len); - if (unlikely(!p)) - return false; - ptr->data = (unsigned char *)p; - } + p = xdr_inline_decode(xdr, len); + if (unlikely(!p)) + return false; + ptr->data = (unsigned char *)p; ptr->len = len; return true; } @@ -219,12 +217,10 @@ xdrgen_decode_opaque(struct xdr_stream *xdr, opaque *ptr, u32 maxlen) return false; if (unlikely(maxlen && len > maxlen)) return false; - if (len != 0) { - p = xdr_inline_decode(xdr, len); - if (unlikely(!p)) - return false; - ptr->data = (u8 *)p; - } + p = xdr_inline_decode(xdr, len); + if (unlikely(!p)) + return false; + ptr->data = (u8 *)p; ptr->len = len; return true; } diff --git a/include/linux/tnum.h b/include/linux/tnum.h index c52b862dad45be..ca2cfec8de08ac 100644 --- a/include/linux/tnum.h +++ b/include/linux/tnum.h @@ -63,6 +63,11 @@ struct tnum tnum_union(struct tnum t1, struct tnum t2); /* Return @a with all but the lowest @size bytes cleared */ struct tnum tnum_cast(struct tnum a, u8 size); +/* Swap the bytes of a tnum */ +struct tnum tnum_bswap16(struct tnum a); +struct tnum tnum_bswap32(struct tnum a); +struct tnum tnum_bswap64(struct tnum a); + /* Returns true if @a is a known constant */ static inline bool tnum_is_const(struct tnum a) { @@ -126,4 +131,7 @@ static inline bool tnum_subreg_is_const(struct tnum a) return !(tnum_subreg(a)).mask; } +/* Returns the smallest member of t larger than z */ +u64 tnum_step(struct tnum t, u64 z); + #endif /* _LINUX_TNUM_H */ diff --git a/include/linux/trace_events.h b/include/linux/trace_events.h index 3690221ba3d80d..f925034e402dc4 100644 --- a/include/linux/trace_events.h +++ b/include/linux/trace_events.h @@ -683,6 +683,11 @@ static inline void hist_poll_wakeup(void) #define hist_poll_wait(file, wait) \ poll_wait(file, &hist_poll_wq, wait) + +#else +static inline void hist_poll_wakeup(void) +{ +} #endif #define __TRACE_EVENT_FLAGS(name, value) \ diff --git a/include/linux/u64_stats_sync.h b/include/linux/u64_stats_sync.h index 457879938fc198..3366090a86bd2f 100644 --- a/include/linux/u64_stats_sync.h +++ b/include/linux/u64_stats_sync.h @@ -89,6 +89,11 @@ static inline void u64_stats_add(u64_stats_t *p, unsigned long val) local64_add(val, &p->v); } +static inline void u64_stats_sub(u64_stats_t *p, s64 val) +{ + local64_sub(val, &p->v); +} + static inline void u64_stats_inc(u64_stats_t *p) { local64_inc(&p->v); @@ -130,6 +135,11 @@ static inline void u64_stats_add(u64_stats_t *p, unsigned long val) p->v += val; } +static inline void u64_stats_sub(u64_stats_t *p, s64 val) +{ + p->v -= val; +} + static inline void u64_stats_inc(u64_stats_t *p) { p->v++; diff --git a/include/linux/uaccess.h b/include/linux/uaccess.h index 1f3804245c0667..fa7125c0e103dc 100644 --- a/include/linux/uaccess.h +++ b/include/linux/uaccess.h @@ -331,16 +331,21 @@ static inline size_t probe_subpage_writeable(char __user *uaddr, size_t size) #endif /* CONFIG_ARCH_HAS_SUBPAGE_FAULTS */ -#ifndef ARCH_HAS_NOCACHE_UACCESS +#ifndef ARCH_HAS_NONTEMPORAL_UACCESS static inline __must_check unsigned long -__copy_from_user_inatomic_nocache(void *to, const void __user *from, +copy_from_user_inatomic_nontemporal(void *to, const void __user *from, unsigned long n) { + if (can_do_masked_user_access()) + from = mask_user_address(from); + else + if (!access_ok(from, n)) + return n; return __copy_from_user_inatomic(to, from, n); } -#endif /* ARCH_HAS_NOCACHE_UACCESS */ +#endif /* ARCH_HAS_NONTEMPORAL_UACCESS */ extern __must_check int check_zeroed_user(const void __user *from, size_t size); @@ -647,36 +652,22 @@ static inline void user_access_restore(unsigned long flags) { } /* Define RW variant so the below _mode macro expansion works */ #define masked_user_rw_access_begin(u) masked_user_access_begin(u) #define user_rw_access_begin(u, s) user_access_begin(u, s) -#define user_rw_access_end() user_access_end() /* Scoped user access */ -#define USER_ACCESS_GUARD(_mode) \ -static __always_inline void __user * \ -class_user_##_mode##_begin(void __user *ptr) \ -{ \ - return ptr; \ -} \ - \ -static __always_inline void \ -class_user_##_mode##_end(void __user *ptr) \ -{ \ - user_##_mode##_access_end(); \ -} \ - \ -DEFINE_CLASS(user_ ##_mode## _access, void __user *, \ - class_user_##_mode##_end(_T), \ - class_user_##_mode##_begin(ptr), void __user *ptr) \ - \ -static __always_inline class_user_##_mode##_access_t \ -class_user_##_mode##_access_ptr(void __user *scope) \ -{ \ - return scope; \ -} -USER_ACCESS_GUARD(read) -USER_ACCESS_GUARD(write) -USER_ACCESS_GUARD(rw) -#undef USER_ACCESS_GUARD +/* Cleanup wrapper functions */ +static __always_inline void __scoped_user_read_access_end(const void *p) +{ + user_read_access_end(); +}; +static __always_inline void __scoped_user_write_access_end(const void *p) +{ + user_write_access_end(); +}; +static __always_inline void __scoped_user_rw_access_end(const void *p) +{ + user_access_end(); +}; /** * __scoped_user_access_begin - Start a scoped user access @@ -750,13 +741,13 @@ USER_ACCESS_GUARD(rw) * * Don't use directly. Use scoped_masked_user_$MODE_access() instead. */ -#define __scoped_user_access(mode, uptr, size, elbl) \ -for (bool done = false; !done; done = true) \ - for (void __user *_tmpptr = __scoped_user_access_begin(mode, uptr, size, elbl); \ - !done; done = true) \ - for (CLASS(user_##mode##_access, scope)(_tmpptr); !done; done = true) \ - /* Force modified pointer usage within the scope */ \ - for (const typeof(uptr) uptr = _tmpptr; !done; done = true) +#define __scoped_user_access(mode, uptr, size, elbl) \ +for (bool done = false; !done; done = true) \ + for (auto _tmpptr = __scoped_user_access_begin(mode, uptr, size, elbl); \ + !done; done = true) \ + /* Force modified pointer usage within the scope */ \ + for (const auto uptr __cleanup(__scoped_user_##mode##_access_end) = \ + _tmpptr; !done; done = true) /** * scoped_user_read_access_size - Start a scoped user read access with given size diff --git a/include/linux/uprobes.h b/include/linux/uprobes.h index ee3d36eda45dd2..f548fea2adec8f 100644 --- a/include/linux/uprobes.h +++ b/include/linux/uprobes.h @@ -242,6 +242,7 @@ extern void arch_uprobe_clear_state(struct mm_struct *mm); extern void arch_uprobe_init_state(struct mm_struct *mm); extern void handle_syscall_uprobe(struct pt_regs *regs, unsigned long bp_vaddr); extern void arch_uprobe_optimize(struct arch_uprobe *auprobe, unsigned long vaddr); +extern unsigned long arch_uprobe_get_xol_area(void); #else /* !CONFIG_UPROBES */ struct uprobes_state { }; diff --git a/include/linux/usb.h b/include/linux/usb.h index e85105939af8ee..2511f1e5b114dc 100644 --- a/include/linux/usb.h +++ b/include/linux/usb.h @@ -21,6 +21,7 @@ #include /* for struct completion */ #include /* for current && schedule_timeout */ #include /* for struct mutex */ +#include /* for spinlock_t */ #include /* for runtime PM */ struct usb_device; @@ -636,8 +637,9 @@ struct usb3_lpm_parameters { * @do_remote_wakeup: remote wakeup should be enabled * @reset_resume: needs reset instead of resume * @port_is_suspended: the upstream port is suspended (L2 or U3) - * @offload_at_suspend: offload activities during suspend is enabled. + * @offload_pm_locked: prevents offload_usage changes during PM transitions. * @offload_usage: number of offload activities happening on this usb device. + * @offload_lock: protects offload_usage and offload_pm_locked * @slot_id: Slot ID assigned by xHCI * @l1_params: best effor service latency for USB2 L1 LPM state, and L1 timeout. * @u1_params: exit latencies for USB3 U1 LPM state, and hub-initiated timeout. @@ -726,8 +728,9 @@ struct usb_device { unsigned do_remote_wakeup:1; unsigned reset_resume:1; unsigned port_is_suspended:1; - unsigned offload_at_suspend:1; + unsigned offload_pm_locked:1; int offload_usage; + spinlock_t offload_lock; enum usb_link_tunnel_mode tunnel_mode; struct device_link *usb4_link; @@ -849,6 +852,7 @@ static inline void usb_mark_last_busy(struct usb_device *udev) int usb_offload_get(struct usb_device *udev); int usb_offload_put(struct usb_device *udev); bool usb_offload_check(struct usb_device *udev); +void usb_offload_set_pm_locked(struct usb_device *udev, bool locked); #else static inline int usb_offload_get(struct usb_device *udev) @@ -857,6 +861,8 @@ static inline int usb_offload_put(struct usb_device *udev) { return 0; } static inline bool usb_offload_check(struct usb_device *udev) { return false; } +static inline void usb_offload_set_pm_locked(struct usb_device *udev, bool locked) +{ } #endif extern int usb_disable_lpm(struct usb_device *udev); @@ -1863,14 +1869,18 @@ void usb_free_noncoherent(struct usb_device *dev, size_t size, * SYNCHRONOUS CALL SUPPORT * *-------------------------------------------------------------------*/ +/* Maximum value allowed for timeout in synchronous routines below */ +#define USB_MAX_SYNCHRONOUS_TIMEOUT 60000 /* ms */ + extern int usb_control_msg(struct usb_device *dev, unsigned int pipe, __u8 request, __u8 requesttype, __u16 value, __u16 index, void *data, __u16 size, int timeout); extern int usb_interrupt_msg(struct usb_device *usb_dev, unsigned int pipe, void *data, int len, int *actual_length, int timeout); extern int usb_bulk_msg(struct usb_device *usb_dev, unsigned int pipe, - void *data, int len, int *actual_length, - int timeout); + void *data, int len, int *actual_length, int timeout); +extern int usb_bulk_msg_killable(struct usb_device *usb_dev, unsigned int pipe, + void *data, int len, int *actual_length, int timeout); /* wrappers around usb_control_msg() for the most common standard requests */ int usb_control_msg_send(struct usb_device *dev, __u8 endpoint, __u8 request, diff --git a/include/linux/usb/quirks.h b/include/linux/usb/quirks.h index 2f7bd2fdc6164b..b3cc7beab4a3ce 100644 --- a/include/linux/usb/quirks.h +++ b/include/linux/usb/quirks.h @@ -78,4 +78,7 @@ /* skip BOS descriptor request */ #define USB_QUIRK_NO_BOS BIT(17) +/* Device claims zero configurations, forcing to 1 */ +#define USB_QUIRK_FORCE_ONE_CONFIG BIT(18) + #endif /* __LINUX_USB_QUIRKS_H */ diff --git a/include/linux/usb/r8152.h b/include/linux/usb/r8152.h index 2ca60828f28bb6..1502b2a355f980 100644 --- a/include/linux/usb/r8152.h +++ b/include/linux/usb/r8152.h @@ -32,6 +32,7 @@ #define VENDOR_ID_DLINK 0x2001 #define VENDOR_ID_DELL 0x413c #define VENDOR_ID_ASUS 0x0b05 +#define VENDOR_ID_TRENDNET 0x20f4 #if IS_REACHABLE(CONFIG_USB_RTL8152) extern u8 rtl8152_get_version(struct usb_interface *intf); diff --git a/include/linux/usb/usbnet.h b/include/linux/usb/usbnet.h index 2945923a8a9584..7296fdf3d88b01 100644 --- a/include/linux/usb/usbnet.h +++ b/include/linux/usb/usbnet.h @@ -132,6 +132,7 @@ struct driver_info { #define FLAG_MULTI_PACKET 0x2000 #define FLAG_RX_ASSEMBLE 0x4000 /* rx packets may span >1 frames */ #define FLAG_NOARP 0x8000 /* device can't do ARP */ +#define FLAG_NOMAXMTU 0x10000 /* allow max_mtu above hard_mtu */ /* init device ... can sleep, or cause probe() failure */ int (*bind)(struct usbnet *, struct usb_interface *); diff --git a/include/linux/virtio_net.h b/include/linux/virtio_net.h index 75dabb763c6504..f36d21b5bc19e2 100644 --- a/include/linux/virtio_net.h +++ b/include/linux/virtio_net.h @@ -207,6 +207,39 @@ static inline int virtio_net_hdr_to_skb(struct sk_buff *skb, return __virtio_net_hdr_to_skb(skb, hdr, little_endian, hdr->gso_type); } +/* This function must be called after virtio_net_hdr_from_skb(). */ +static inline void __virtio_net_set_hdrlen(const struct sk_buff *skb, + struct virtio_net_hdr *hdr, + bool little_endian) +{ + u16 hdr_len; + + hdr_len = skb_transport_offset(skb); + + if (hdr->gso_type == VIRTIO_NET_HDR_GSO_UDP_L4) + hdr_len += sizeof(struct udphdr); + else + hdr_len += tcp_hdrlen(skb); + + hdr->hdr_len = __cpu_to_virtio16(little_endian, hdr_len); +} + +/* This function must be called after virtio_net_hdr_from_skb(). */ +static inline void __virtio_net_set_tnl_hdrlen(const struct sk_buff *skb, + struct virtio_net_hdr *hdr) +{ + u16 hdr_len; + + hdr_len = skb_inner_transport_offset(skb); + + if (hdr->gso_type == VIRTIO_NET_HDR_GSO_UDP_L4) + hdr_len += sizeof(struct udphdr); + else + hdr_len += inner_tcp_hdrlen(skb); + + hdr->hdr_len = __cpu_to_virtio16(true, hdr_len); +} + static inline int virtio_net_hdr_from_skb(const struct sk_buff *skb, struct virtio_net_hdr *hdr, bool little_endian, @@ -385,7 +418,8 @@ virtio_net_hdr_tnl_from_skb(const struct sk_buff *skb, bool tnl_hdr_negotiated, bool little_endian, int vlan_hlen, - bool has_data_valid) + bool has_data_valid, + bool feature_hdrlen) { struct virtio_net_hdr *hdr = (struct virtio_net_hdr *)vhdr; unsigned int inner_nh, outer_th; @@ -394,9 +428,17 @@ virtio_net_hdr_tnl_from_skb(const struct sk_buff *skb, tnl_gso_type = skb_shinfo(skb)->gso_type & (SKB_GSO_UDP_TUNNEL | SKB_GSO_UDP_TUNNEL_CSUM); - if (!tnl_gso_type) - return virtio_net_hdr_from_skb(skb, hdr, little_endian, - has_data_valid, vlan_hlen); + if (!tnl_gso_type) { + ret = virtio_net_hdr_from_skb(skb, hdr, little_endian, + has_data_valid, vlan_hlen); + if (ret) + return ret; + + if (feature_hdrlen && hdr->hdr_len) + __virtio_net_set_hdrlen(skb, hdr, little_endian); + + return ret; + } /* Tunnel support not negotiated but skb ask for it. */ if (!tnl_hdr_negotiated) @@ -414,6 +456,9 @@ virtio_net_hdr_tnl_from_skb(const struct sk_buff *skb, if (ret) return ret; + if (feature_hdrlen && hdr->hdr_len) + __virtio_net_set_tnl_hdrlen(skb, hdr); + if (skb->protocol == htons(ETH_P_IPV6)) hdr->gso_type |= VIRTIO_NET_HDR_GSO_UDP_TUNNEL_IPV6; else diff --git a/include/linux/xarray.h b/include/linux/xarray.h index be850174e802e6..64f2a5e06cebcc 100644 --- a/include/linux/xarray.h +++ b/include/linux/xarray.h @@ -563,6 +563,8 @@ void *__xa_erase(struct xarray *, unsigned long index); void *__xa_store(struct xarray *, unsigned long index, void *entry, gfp_t); void *__xa_cmpxchg(struct xarray *, unsigned long index, void *old, void *entry, gfp_t); +void *__xa_cmpxchg_raw(struct xarray *, unsigned long index, void *old, + void *entry, gfp_t); int __must_check __xa_insert(struct xarray *, unsigned long index, void *entry, gfp_t); int __must_check __xa_alloc(struct xarray *, u32 *id, void *entry, diff --git a/include/media/dvb_vb2.h b/include/media/dvb_vb2.h index 8cb88452cd6c28..0fbbfc65157e64 100644 --- a/include/media/dvb_vb2.h +++ b/include/media/dvb_vb2.h @@ -124,7 +124,7 @@ static inline int dvb_vb2_release(struct dvb_vb2_ctx *ctx) return 0; }; #define dvb_vb2_is_streaming(ctx) (0) -#define dvb_vb2_fill_buffer(ctx, file, wait, flags) (0) +#define dvb_vb2_fill_buffer(ctx, file, wait, flags, flush) (0) static inline __poll_t dvb_vb2_poll(struct dvb_vb2_ctx *ctx, struct file *file, @@ -166,10 +166,12 @@ int dvb_vb2_is_streaming(struct dvb_vb2_ctx *ctx); * @buffer_flags: * pointer to buffer flags as defined by &enum dmx_buffer_flags. * can be NULL. + * @flush: flush the buffer, even if it isn't full. */ int dvb_vb2_fill_buffer(struct dvb_vb2_ctx *ctx, const unsigned char *src, int len, - enum dmx_buffer_flags *buffer_flags); + enum dmx_buffer_flags *buffer_flags, + bool flush); /** * dvb_vb2_poll - Wrapper to vb2_core_streamon() for Digital TV diff --git a/include/media/v4l2-mem2mem.h b/include/media/v4l2-mem2mem.h index bf6a09a04dcf84..31de25d792b98f 100644 --- a/include/media/v4l2-mem2mem.h +++ b/include/media/v4l2-mem2mem.h @@ -547,6 +547,27 @@ v4l2_m2m_register_media_controller(struct v4l2_m2m_dev *m2m_dev, */ void v4l2_m2m_release(struct v4l2_m2m_dev *m2m_dev); +/** + * v4l2_m2m_get() - take a reference to the m2m_dev structure + * + * @m2m_dev: opaque pointer to the internal data to handle M2M context + * + * This is used to share the M2M device across multiple devices. This + * can be used to avoid scheduling two hardware nodes concurrently. + */ +void v4l2_m2m_get(struct v4l2_m2m_dev *m2m_dev); + +/** + * v4l2_m2m_put() - remove a reference to the m2m_dev structure + * + * @m2m_dev: opaque pointer to the internal data to handle M2M context + * + * Once the M2M device has no more references, v4l2_m2m_release() will be + * called automatically. Users of this method should never call + * v4l2_m2m_release() directly. See v4l2_m2m_get() for more details. + */ +void v4l2_m2m_put(struct v4l2_m2m_dev *m2m_dev); + /** * v4l2_m2m_ctx_init() - allocate and initialize a m2m context * diff --git a/include/net/act_api.h b/include/net/act_api.h index 91a24b5e0b93e4..2ba40eb45aad2f 100644 --- a/include/net/act_api.h +++ b/include/net/act_api.h @@ -70,6 +70,7 @@ struct tc_action { #define TCA_ACT_FLAGS_REPLACE (1U << (TCA_ACT_FLAGS_USER_BITS + 2)) #define TCA_ACT_FLAGS_NO_RTNL (1U << (TCA_ACT_FLAGS_USER_BITS + 3)) #define TCA_ACT_FLAGS_AT_INGRESS (1U << (TCA_ACT_FLAGS_USER_BITS + 4)) +#define TCA_ACT_FLAGS_AT_INGRESS_OR_CLSACT (1U << (TCA_ACT_FLAGS_USER_BITS + 5)) /* Update lastuse only if needed, to avoid dirtying a cache line. * We use a temp variable to avoid fetching jiffies twice. diff --git a/include/net/bluetooth/bluetooth.h b/include/net/bluetooth/bluetooth.h index d46ed9011ee5d0..20ce829473f0a0 100644 --- a/include/net/bluetooth/bluetooth.h +++ b/include/net/bluetooth/bluetooth.h @@ -130,21 +130,30 @@ struct bt_voice { #define BT_RCVMTU 13 #define BT_PHY 14 -#define BT_PHY_BR_1M_1SLOT 0x00000001 -#define BT_PHY_BR_1M_3SLOT 0x00000002 -#define BT_PHY_BR_1M_5SLOT 0x00000004 -#define BT_PHY_EDR_2M_1SLOT 0x00000008 -#define BT_PHY_EDR_2M_3SLOT 0x00000010 -#define BT_PHY_EDR_2M_5SLOT 0x00000020 -#define BT_PHY_EDR_3M_1SLOT 0x00000040 -#define BT_PHY_EDR_3M_3SLOT 0x00000080 -#define BT_PHY_EDR_3M_5SLOT 0x00000100 -#define BT_PHY_LE_1M_TX 0x00000200 -#define BT_PHY_LE_1M_RX 0x00000400 -#define BT_PHY_LE_2M_TX 0x00000800 -#define BT_PHY_LE_2M_RX 0x00001000 -#define BT_PHY_LE_CODED_TX 0x00002000 -#define BT_PHY_LE_CODED_RX 0x00004000 +#define BT_PHY_BR_1M_1SLOT BIT(0) +#define BT_PHY_BR_1M_3SLOT BIT(1) +#define BT_PHY_BR_1M_5SLOT BIT(2) +#define BT_PHY_EDR_2M_1SLOT BIT(3) +#define BT_PHY_EDR_2M_3SLOT BIT(4) +#define BT_PHY_EDR_2M_5SLOT BIT(5) +#define BT_PHY_EDR_3M_1SLOT BIT(6) +#define BT_PHY_EDR_3M_3SLOT BIT(7) +#define BT_PHY_EDR_3M_5SLOT BIT(8) +#define BT_PHY_LE_1M_TX BIT(9) +#define BT_PHY_LE_1M_RX BIT(10) +#define BT_PHY_LE_2M_TX BIT(11) +#define BT_PHY_LE_2M_RX BIT(12) +#define BT_PHY_LE_CODED_TX BIT(13) +#define BT_PHY_LE_CODED_RX BIT(14) + +#define BT_PHY_BREDR_MASK (BT_PHY_BR_1M_1SLOT | BT_PHY_BR_1M_3SLOT | \ + BT_PHY_BR_1M_5SLOT | BT_PHY_EDR_2M_1SLOT | \ + BT_PHY_EDR_2M_3SLOT | BT_PHY_EDR_2M_5SLOT | \ + BT_PHY_EDR_3M_1SLOT | BT_PHY_EDR_3M_3SLOT | \ + BT_PHY_EDR_3M_5SLOT) +#define BT_PHY_LE_MASK (BT_PHY_LE_1M_TX | BT_PHY_LE_1M_RX | \ + BT_PHY_LE_2M_TX | BT_PHY_LE_2M_RX | \ + BT_PHY_LE_CODED_TX | BT_PHY_LE_CODED_RX) #define BT_MODE 15 @@ -448,6 +457,7 @@ struct l2cap_ctrl { }; struct hci_dev; +struct hci_conn; typedef void (*hci_req_complete_t)(struct hci_dev *hdev, u8 status, u16 opcode); typedef void (*hci_req_complete_skb_t)(struct hci_dev *hdev, u8 status, @@ -460,6 +470,9 @@ void hci_req_cmd_complete(struct hci_dev *hdev, u16 opcode, u8 status, int hci_ethtool_ts_info(unsigned int index, int sk_proto, struct kernel_ethtool_ts_info *ts_info); +int hci_conn_setsockopt(struct hci_conn *conn, struct sock *sk, int level, + int optname, sockptr_t optval, unsigned int optlen); + #define HCI_REQ_START BIT(0) #define HCI_REQ_SKB BIT(1) diff --git a/include/net/bluetooth/hci.h b/include/net/bluetooth/hci.h index a27cd3626b8726..a2beda3b0071d1 100644 --- a/include/net/bluetooth/hci.h +++ b/include/net/bluetooth/hci.h @@ -1883,6 +1883,15 @@ struct hci_cp_le_set_default_phy { #define HCI_LE_SET_PHY_2M 0x02 #define HCI_LE_SET_PHY_CODED 0x04 +#define HCI_OP_LE_SET_PHY 0x2032 +struct hci_cp_le_set_phy { + __le16 handle; + __u8 all_phys; + __u8 tx_phys; + __u8 rx_phys; + __le16 phy_opts; +} __packed; + #define HCI_OP_LE_SET_EXT_SCAN_PARAMS 0x2041 struct hci_cp_le_set_ext_scan_params { __u8 own_addr_type; diff --git a/include/net/bluetooth/hci_core.h b/include/net/bluetooth/hci_core.h index 4263e71a23efb2..a53b318dc7dc6f 100644 --- a/include/net/bluetooth/hci_core.h +++ b/include/net/bluetooth/hci_core.h @@ -642,6 +642,10 @@ struct hci_dev { bool aosp_quality_report; #endif +#if IS_ENABLED(CONFIG_BT_BRCMEXT) + bool brcm_capable; +#endif + int (*open)(struct hci_dev *hdev); int (*close)(struct hci_dev *hdev); int (*flush)(struct hci_dev *hdev); @@ -730,6 +734,8 @@ struct hci_conn { __u16 le_per_adv_data_offset; __u8 le_adv_phy; __u8 le_adv_sec_phy; + __u8 le_tx_def_phys; + __u8 le_rx_def_phys; __u8 le_tx_phy; __u8 le_rx_phy; __s8 rssi; @@ -1789,6 +1795,13 @@ static inline void hci_set_aosp_capable(struct hci_dev *hdev) #endif } +static inline void hci_set_brcm_capable(struct hci_dev *hdev) +{ +#if IS_ENABLED(CONFIG_BT_BRCMEXT) + hdev->brcm_capable = true; +#endif +} + static inline void hci_devcd_setup(struct hci_dev *hdev) { #ifdef CONFIG_DEV_COREDUMP @@ -2334,6 +2347,7 @@ void *hci_sent_cmd_data(struct hci_dev *hdev, __u16 opcode); void *hci_recv_event_data(struct hci_dev *hdev, __u8 event); u32 hci_conn_get_phy(struct hci_conn *conn); +int hci_conn_set_phy(struct hci_conn *conn, u32 phys); /* ----- HCI Sockets ----- */ void hci_send_to_sock(struct hci_dev *hdev, struct sk_buff *skb); diff --git a/include/net/bluetooth/hci_sync.h b/include/net/bluetooth/hci_sync.h index 56076bbc981d92..73e494b2591de3 100644 --- a/include/net/bluetooth/hci_sync.h +++ b/include/net/bluetooth/hci_sync.h @@ -191,3 +191,6 @@ int hci_connect_big_sync(struct hci_dev *hdev, struct hci_conn *conn); int hci_past_sync(struct hci_conn *conn, struct hci_conn *le); int hci_le_read_remote_features(struct hci_conn *conn); + +int hci_acl_change_pkt_type(struct hci_conn *conn, u16 pkt_type); +int hci_le_set_phy(struct hci_conn *conn, u8 tx_phys, u8 rx_phys); diff --git a/include/net/bluetooth/l2cap.h b/include/net/bluetooth/l2cap.h index 00e182a22720a6..5172afee549433 100644 --- a/include/net/bluetooth/l2cap.h +++ b/include/net/bluetooth/l2cap.h @@ -284,9 +284,9 @@ struct l2cap_conn_rsp { #define L2CAP_CR_LE_BAD_KEY_SIZE 0x0007 #define L2CAP_CR_LE_ENCRYPTION 0x0008 #define L2CAP_CR_LE_INVALID_SCID 0x0009 -#define L2CAP_CR_LE_SCID_IN_USE 0X000A -#define L2CAP_CR_LE_UNACCEPT_PARAMS 0X000B -#define L2CAP_CR_LE_INVALID_PARAMS 0X000C +#define L2CAP_CR_LE_SCID_IN_USE 0x000A +#define L2CAP_CR_LE_UNACCEPT_PARAMS 0x000B +#define L2CAP_CR_LE_INVALID_PARAMS 0x000C /* connect/create channel status */ #define L2CAP_CS_NO_INFO 0x0000 @@ -493,6 +493,8 @@ struct l2cap_ecred_reconf_req { #define L2CAP_RECONF_SUCCESS 0x0000 #define L2CAP_RECONF_INVALID_MTU 0x0001 #define L2CAP_RECONF_INVALID_MPS 0x0002 +#define L2CAP_RECONF_INVALID_CID 0x0003 +#define L2CAP_RECONF_INVALID_PARAMS 0x0004 struct l2cap_ecred_reconf_rsp { __le16 result; @@ -655,8 +657,8 @@ struct l2cap_conn { struct sk_buff *rx_skb; __u32 rx_len; + struct ida tx_ida; __u8 tx_ident; - struct mutex ident_lock; struct sk_buff_head pending_rx; struct work_struct pending_rx_work; diff --git a/include/net/bonding.h b/include/net/bonding.h index 46207840355709..99c1bdadcd11a9 100644 --- a/include/net/bonding.h +++ b/include/net/bonding.h @@ -698,6 +698,7 @@ void bond_debug_register(struct bonding *bond); void bond_debug_unregister(struct bonding *bond); void bond_debug_reregister(struct bonding *bond); const char *bond_mode_name(int mode); +bool __bond_xdp_check(int mode, int xmit_policy); bool bond_xdp_check(struct bonding *bond, int mode); void bond_setup(struct net_device *bond_dev); unsigned int bond_get_num_tx_queues(void); diff --git a/include/net/cfg80211.h b/include/net/cfg80211.h index 2900202588a547..39a04776705eb4 100644 --- a/include/net/cfg80211.h +++ b/include/net/cfg80211.h @@ -10147,9 +10147,9 @@ cfg80211_6ghz_power_type(u8 control, u32 client_flags) case IEEE80211_6GHZ_CTRL_REG_LPI_AP: case IEEE80211_6GHZ_CTRL_REG_INDOOR_LPI_AP: case IEEE80211_6GHZ_CTRL_REG_AP_ROLE_NOT_RELEVANT: + case IEEE80211_6GHZ_CTRL_REG_INDOOR_SP_AP_OLD: return IEEE80211_REG_LPI_AP; case IEEE80211_6GHZ_CTRL_REG_SP_AP: - case IEEE80211_6GHZ_CTRL_REG_INDOOR_SP_AP_OLD: return IEEE80211_REG_SP_AP; case IEEE80211_6GHZ_CTRL_REG_VLP_AP: return IEEE80211_REG_VLP_AP; diff --git a/include/net/codel_impl.h b/include/net/codel_impl.h index 78a27ac730700b..b2c359c6dd1b84 100644 --- a/include/net/codel_impl.h +++ b/include/net/codel_impl.h @@ -158,6 +158,7 @@ static struct sk_buff *codel_dequeue(void *ctx, bool drop; if (!skb) { + vars->first_above_time = 0; vars->dropping = false; return skb; } diff --git a/include/net/inet6_hashtables.h b/include/net/inet6_hashtables.h index 282e29237d9367..c16de5b7963fdb 100644 --- a/include/net/inet6_hashtables.h +++ b/include/net/inet6_hashtables.h @@ -175,7 +175,7 @@ static inline bool inet6_match(const struct net *net, const struct sock *sk, { if (!net_eq(sock_net(sk), net) || sk->sk_family != AF_INET6 || - sk->sk_portpair != ports || + READ_ONCE(sk->sk_portpair) != ports || !ipv6_addr_equal(&sk->sk_v6_daddr, saddr) || !ipv6_addr_equal(&sk->sk_v6_rcv_saddr, daddr)) return false; diff --git a/include/net/inet_connection_sock.h b/include/net/inet_connection_sock.h index ecb362025c4e51..5cb3056d6ddc7c 100644 --- a/include/net/inet_connection_sock.h +++ b/include/net/inet_connection_sock.h @@ -42,7 +42,9 @@ struct inet_connection_sock_af_ops { struct request_sock *req, struct dst_entry *dst, struct request_sock *req_unhash, - bool *own_req); + bool *own_req, + void (*opt_child_init)(struct sock *newsk, + const struct sock *sk)); u16 net_header_len; int (*setsockopt)(struct sock *sk, int level, int optname, sockptr_t optval, unsigned int optlen); diff --git a/include/net/inet_ecn.h b/include/net/inet_ecn.h index ea32393464a291..827b87a95dab3f 100644 --- a/include/net/inet_ecn.h +++ b/include/net/inet_ecn.h @@ -51,11 +51,25 @@ static inline __u8 INET_ECN_encapsulate(__u8 outer, __u8 inner) return outer; } +/* Apply either ECT(0) or ECT(1) */ +static inline void __INET_ECN_xmit(struct sock *sk, bool use_ect_1) +{ + __u8 ect = use_ect_1 ? INET_ECN_ECT_1 : INET_ECN_ECT_0; + + /* Mask the complete byte in case the connection alternates between + * ECT(0) and ECT(1). + */ + inet_sk(sk)->tos &= ~INET_ECN_MASK; + inet_sk(sk)->tos |= ect; + if (inet6_sk(sk)) { + inet6_sk(sk)->tclass &= ~INET_ECN_MASK; + inet6_sk(sk)->tclass |= ect; + } +} + static inline void INET_ECN_xmit(struct sock *sk) { - inet_sk(sk)->tos |= INET_ECN_ECT_0; - if (inet6_sk(sk) != NULL) - inet6_sk(sk)->tclass |= INET_ECN_ECT_0; + __INET_ECN_xmit(sk, false); } static inline void INET_ECN_dontxmit(struct sock *sk) diff --git a/include/net/inet_hashtables.h b/include/net/inet_hashtables.h index ac05a52d9e1382..6d936e9f2fd32c 100644 --- a/include/net/inet_hashtables.h +++ b/include/net/inet_hashtables.h @@ -264,6 +264,20 @@ inet_bhashfn_portaddr(const struct inet_hashinfo *hinfo, const struct sock *sk, return &hinfo->bhash2[hash & (hinfo->bhash_size - 1)]; } +static inline bool inet_use_hash2_on_bind(const struct sock *sk) +{ +#if IS_ENABLED(CONFIG_IPV6) + if (sk->sk_family == AF_INET6) { + if (ipv6_addr_any(&sk->sk_v6_rcv_saddr)) + return false; + + if (!ipv6_addr_v4mapped(&sk->sk_v6_rcv_saddr)) + return true; + } +#endif + return sk->sk_rcv_saddr != htonl(INADDR_ANY); +} + struct inet_bind_hashbucket * inet_bhash2_addr_any_hashbucket(const struct sock *sk, const struct net *net, int port); @@ -345,7 +359,7 @@ static inline bool inet_match(const struct net *net, const struct sock *sk, int dif, int sdif) { if (!net_eq(sock_net(sk), net) || - sk->sk_portpair != ports || + READ_ONCE(sk->sk_portpair) != ports || sk->sk_addrpair != cookie) return false; diff --git a/include/net/ioam6.h b/include/net/ioam6.h index 2cbbee6e806aab..a75912fe247e66 100644 --- a/include/net/ioam6.h +++ b/include/net/ioam6.h @@ -60,6 +60,8 @@ void ioam6_fill_trace_data(struct sk_buff *skb, struct ioam6_trace_hdr *trace, bool is_input); +u8 ioam6_trace_compute_nodelen(u32 trace_type); + int ioam6_init(void); void ioam6_exit(void); diff --git a/include/net/ip.h b/include/net/ip.h index 69d5cef4600405..7f9abd457e0181 100644 --- a/include/net/ip.h +++ b/include/net/ip.h @@ -101,7 +101,7 @@ static inline void ipcm_init_sk(struct ipcm_cookie *ipcm, ipcm->oif = READ_ONCE(inet->sk.sk_bound_dev_if); ipcm->addr = inet->inet_saddr; - ipcm->protocol = inet->inet_num; + ipcm->protocol = READ_ONCE(inet->inet_num); } #define IPCB(skb) ((struct inet_skb_parm*)((skb)->cb)) diff --git a/include/net/ip6_fib.h b/include/net/ip6_fib.h index 88b0dd4d8e094d..9f8b6814a96a04 100644 --- a/include/net/ip6_fib.h +++ b/include/net/ip6_fib.h @@ -507,12 +507,14 @@ void fib6_rt_update(struct net *net, struct fib6_info *rt, void inet6_rt_notify(int event, struct fib6_info *rt, struct nl_info *info, unsigned int flags); +void fib6_age_exceptions(struct fib6_info *rt, struct fib6_gc_args *gc_args, + unsigned long now); void fib6_run_gc(unsigned long expires, struct net *net, bool force); - void fib6_gc_cleanup(void); int fib6_init(void); +#if IS_ENABLED(CONFIG_IPV6) /* Add the route to the gc list if it is not already there * * The callers should hold f6i->fib6_table->tb6_lock. @@ -545,6 +547,23 @@ static inline void fib6_remove_gc_list(struct fib6_info *f6i) hlist_del_init(&f6i->gc_link); } +static inline void fib6_may_remove_gc_list(struct net *net, + struct fib6_info *f6i) +{ + struct fib6_gc_args gc_args; + + if (hlist_unhashed(&f6i->gc_link)) + return; + + gc_args.timeout = READ_ONCE(net->ipv6.sysctl.ip6_rt_gc_interval); + gc_args.more = 0; + + rcu_read_lock(); + fib6_age_exceptions(f6i, &gc_args, jiffies); + rcu_read_unlock(); +} +#endif + struct ipv6_route_iter { struct seq_net_private p; struct fib6_walker w; diff --git a/include/net/ip6_tunnel.h b/include/net/ip6_tunnel.h index 120db286581125..359b595f1df936 100644 --- a/include/net/ip6_tunnel.h +++ b/include/net/ip6_tunnel.h @@ -156,6 +156,18 @@ static inline void ip6tunnel_xmit(struct sock *sk, struct sk_buff *skb, { int pkt_len, err; + if (unlikely(dev_recursion_level() > IP_TUNNEL_RECURSION_LIMIT)) { + if (dev) { + net_crit_ratelimited("Dead loop on virtual device %s, fix it urgently!\n", + dev->name); + DEV_STATS_INC(dev, tx_errors); + } + kfree_skb(skb); + return; + } + + dev_xmit_recursion_inc(); + memset(skb->cb, 0, sizeof(struct inet6_skb_parm)); IP6CB(skb)->flags = ip6cb_flags; pkt_len = skb->len - skb_inner_network_offset(skb); @@ -166,6 +178,8 @@ static inline void ip6tunnel_xmit(struct sock *sk, struct sk_buff *skb, pkt_len = -1; iptunnel_xmit_stats(dev, pkt_len); } + + dev_xmit_recursion_dec(); } #endif #endif diff --git a/include/net/ip_fib.h b/include/net/ip_fib.h index b4495c38e0a01d..318593743b6e15 100644 --- a/include/net/ip_fib.h +++ b/include/net/ip_fib.h @@ -559,7 +559,7 @@ static inline u32 fib_multipath_hash_from_keys(const struct net *net, siphash_aligned_key_t hash_key; u32 mp_seed; - mp_seed = READ_ONCE(net->ipv4.sysctl_fib_multipath_hash_seed).mp_seed; + mp_seed = READ_ONCE(net->ipv4.sysctl_fib_multipath_hash_seed.mp_seed); fib_multipath_hash_construct_key(&hash_key, mp_seed); return flow_hash_from_keys_seed(keys, &hash_key); diff --git a/include/net/ip_tunnels.h b/include/net/ip_tunnels.h index 4021e6a73e32b8..d708b66e55cda5 100644 --- a/include/net/ip_tunnels.h +++ b/include/net/ip_tunnels.h @@ -27,6 +27,13 @@ #include #endif +/* Recursion limit for tunnel xmit to detect routing loops. + * Unlike XMIT_RECURSION_LIMIT (8) used in the no-qdisc path, tunnel + * recursion involves route lookups and full IP output, consuming much + * more stack per level, so a lower limit is needed. + */ +#define IP_TUNNEL_RECURSION_LIMIT 5 + /* Keep error state on tunnel for 30 sec */ #define IPTUNNEL_ERR_TIMEO (30*HZ) @@ -658,13 +665,29 @@ static inline int iptunnel_pull_offloads(struct sk_buff *skb) static inline void iptunnel_xmit_stats(struct net_device *dev, int pkt_len) { if (pkt_len > 0) { - struct pcpu_sw_netstats *tstats = get_cpu_ptr(dev->tstats); - - u64_stats_update_begin(&tstats->syncp); - u64_stats_add(&tstats->tx_bytes, pkt_len); - u64_stats_inc(&tstats->tx_packets); - u64_stats_update_end(&tstats->syncp); - put_cpu_ptr(tstats); + if (dev->pcpu_stat_type == NETDEV_PCPU_STAT_DSTATS) { + struct pcpu_dstats *dstats = get_cpu_ptr(dev->dstats); + + u64_stats_update_begin(&dstats->syncp); + u64_stats_add(&dstats->tx_bytes, pkt_len); + u64_stats_inc(&dstats->tx_packets); + u64_stats_update_end(&dstats->syncp); + put_cpu_ptr(dstats); + return; + } + if (dev->pcpu_stat_type == NETDEV_PCPU_STAT_TSTATS) { + struct pcpu_sw_netstats *tstats = get_cpu_ptr(dev->tstats); + + u64_stats_update_begin(&tstats->syncp); + u64_stats_add(&tstats->tx_bytes, pkt_len); + u64_stats_inc(&tstats->tx_packets); + u64_stats_update_end(&tstats->syncp); + put_cpu_ptr(tstats); + return; + } + pr_err_once("iptunnel_xmit_stats pcpu_stat_type=%d\n", + dev->pcpu_stat_type); + WARN_ON_ONCE(1); return; } diff --git a/include/net/ipv6.h b/include/net/ipv6.h index 74fbf1ad8065a6..e759a00dbde195 100644 --- a/include/net/ipv6.h +++ b/include/net/ipv6.h @@ -1010,11 +1010,11 @@ static inline int ip6_default_np_autolabel(struct net *net) #if IS_ENABLED(CONFIG_IPV6) static inline int ip6_multipath_hash_policy(const struct net *net) { - return net->ipv6.sysctl.multipath_hash_policy; + return READ_ONCE(net->ipv6.sysctl.multipath_hash_policy); } static inline u32 ip6_multipath_hash_fields(const struct net *net) { - return net->ipv6.sysctl.multipath_hash_fields; + return READ_ONCE(net->ipv6.sysctl.multipath_hash_fields); } #else static inline int ip6_multipath_hash_policy(const struct net *net) @@ -1280,12 +1280,15 @@ int ipv6_sock_mc_drop(struct sock *sk, int ifindex, static inline int ip6_sock_set_v6only(struct sock *sk) { - if (inet_sk(sk)->inet_num) - return -EINVAL; + int ret = 0; + lock_sock(sk); - sk->sk_ipv6only = true; + if (inet_sk(sk)->inet_num) + ret = -EINVAL; + else + sk->sk_ipv6only = true; release_sock(sk); - return 0; + return ret; } static inline void ip6_sock_set_recverr(struct sock *sk) diff --git a/include/net/mac80211.h b/include/net/mac80211.h index c2e49542626c8e..706f87c6d905a5 100644 --- a/include/net/mac80211.h +++ b/include/net/mac80211.h @@ -7291,7 +7291,9 @@ void ieee80211_report_wowlan_wakeup(struct ieee80211_vif *vif, * @band: the band to transmit on * @sta: optional pointer to get the station to send the frame to * - * Return: %true if the skb was prepared, %false otherwise + * Return: %true if the skb was prepared, %false otherwise. + * On failure, the skb is freed by this function; callers must not + * free it again. * * Note: must be called under RCU lock */ diff --git a/include/net/netfilter/nf_conntrack_core.h b/include/net/netfilter/nf_conntrack_core.h index 3384859a892101..8883575adcc1e7 100644 --- a/include/net/netfilter/nf_conntrack_core.h +++ b/include/net/netfilter/nf_conntrack_core.h @@ -83,6 +83,11 @@ void nf_conntrack_lock(spinlock_t *lock); extern spinlock_t nf_conntrack_expect_lock; +static inline void lockdep_nfct_expect_lock_held(void) +{ + lockdep_assert_held(&nf_conntrack_expect_lock); +} + /* ctnetlink code shared by both ctnetlink and nf_conntrack_bpf */ static inline void __nf_ct_set_timeout(struct nf_conn *ct, u64 timeout) diff --git a/include/net/netfilter/nf_conntrack_count.h b/include/net/netfilter/nf_conntrack_count.h index 52a06de41aa0f9..cf0166520cf338 100644 --- a/include/net/netfilter/nf_conntrack_count.h +++ b/include/net/netfilter/nf_conntrack_count.h @@ -13,6 +13,7 @@ struct nf_conncount_list { u32 last_gc; /* jiffies at most recent gc */ struct list_head head; /* connections with the same filtering key */ unsigned int count; /* length of list */ + unsigned int last_gc_count; /* length of list at most recent gc */ }; struct nf_conncount_data *nf_conncount_init(struct net *net, unsigned int keylen); diff --git a/include/net/netfilter/nf_conntrack_expect.h b/include/net/netfilter/nf_conntrack_expect.h index 165e7a03b8e9dc..e9a8350e7ccfb0 100644 --- a/include/net/netfilter/nf_conntrack_expect.h +++ b/include/net/netfilter/nf_conntrack_expect.h @@ -22,10 +22,16 @@ struct nf_conntrack_expect { /* Hash member */ struct hlist_node hnode; + /* Network namespace */ + possible_net_t net; + /* We expect this tuple, with the following mask */ struct nf_conntrack_tuple tuple; struct nf_conntrack_tuple_mask mask; +#ifdef CONFIG_NF_CONNTRACK_ZONES + struct nf_conntrack_zone zone; +#endif /* Usage count. */ refcount_t use; @@ -40,7 +46,7 @@ struct nf_conntrack_expect { struct nf_conntrack_expect *this); /* Helper to assign to new connection */ - struct nf_conntrack_helper *helper; + struct nf_conntrack_helper __rcu *helper; /* The conntrack of the master connection */ struct nf_conn *master; @@ -62,7 +68,17 @@ struct nf_conntrack_expect { static inline struct net *nf_ct_exp_net(struct nf_conntrack_expect *exp) { - return nf_ct_net(exp->master); + return read_pnet(&exp->net); +} + +static inline bool nf_ct_exp_zone_equal_any(const struct nf_conntrack_expect *a, + const struct nf_conntrack_zone *b) +{ +#ifdef CONFIG_NF_CONNTRACK_ZONES + return a->zone.id == b->id; +#else + return true; +#endif } #define NF_CT_EXP_POLICY_NAME_LEN 16 diff --git a/include/net/netfilter/nf_conntrack_timeout.h b/include/net/netfilter/nf_conntrack_timeout.h index 9fdaba911de64d..3a66d4abb6d688 100644 --- a/include/net/netfilter/nf_conntrack_timeout.h +++ b/include/net/netfilter/nf_conntrack_timeout.h @@ -14,6 +14,7 @@ struct nf_ct_timeout { __u16 l3num; const struct nf_conntrack_l4proto *l4proto; + struct rcu_head rcu; char data[]; }; diff --git a/include/net/netfilter/nf_queue.h b/include/net/netfilter/nf_queue.h index 4aeffddb75861e..d17035d14d96cf 100644 --- a/include/net/netfilter/nf_queue.h +++ b/include/net/netfilter/nf_queue.h @@ -6,11 +6,13 @@ #include #include #include +#include #include /* Each queued (to userspace) skbuff has one of these. */ struct nf_queue_entry { struct list_head list; + struct rhash_head hash_node; struct sk_buff *skb; unsigned int id; unsigned int hook_index; /* index in hook_entries->hook[] */ @@ -19,6 +21,7 @@ struct nf_queue_entry { struct net_device *physout; #endif struct nf_hook_state state; + bool nf_ct_is_unconfirmed; u16 size; /* sizeof(entry) + saved route keys */ /* extra space to store route keys */ diff --git a/include/net/netfilter/nf_tables.h b/include/net/netfilter/nf_tables.h index 0e266c2d0e7f07..4dc080f7f27c65 100644 --- a/include/net/netfilter/nf_tables.h +++ b/include/net/netfilter/nf_tables.h @@ -317,11 +317,13 @@ static inline void *nft_elem_priv_cast(const struct nft_elem_priv *priv) * @NFT_ITER_UNSPEC: unspecified, to catch errors * @NFT_ITER_READ: read-only iteration over set elements * @NFT_ITER_UPDATE: iteration under mutex to update set element state + * @NFT_ITER_UPDATE_CLONE: clone set before iteration under mutex to update element */ enum nft_iter_type { NFT_ITER_UNSPEC, NFT_ITER_READ, NFT_ITER_UPDATE, + NFT_ITER_UPDATE_CLONE, }; struct nft_set; @@ -452,6 +454,7 @@ struct nft_set_ext; * @init: initialize private data of new set instance * @destroy: destroy private data of set instance * @gc_init: initialize garbage collection + * @abort_skip_removal: skip removal of elements from abort path * @elemsize: element private size * * Operations lookup, update and delete have simpler interfaces, are faster @@ -509,6 +512,7 @@ struct nft_set_ops { const struct nft_set *set); void (*gc_init)(const struct nft_set *set); + bool abort_skip_removal; unsigned int elemsize; }; @@ -871,6 +875,8 @@ struct nft_elem_priv *nft_set_elem_init(const struct nft_set *set, u64 timeout, u64 expiration, gfp_t gfp); int nft_set_elem_expr_clone(const struct nft_ctx *ctx, struct nft_set *set, struct nft_expr *expr_array[]); +void nft_set_elem_expr_destroy(const struct nft_ctx *ctx, + struct nft_set_elem_expr *elem_expr); void nft_set_elem_destroy(const struct nft_set *set, const struct nft_elem_priv *elem_priv, bool destroy_expr); @@ -1856,6 +1862,11 @@ struct nft_trans_gc { struct rcu_head rcu; }; +static inline int nft_trans_gc_space(const struct nft_trans_gc *trans) +{ + return NFT_TRANS_GC_BATCHCOUNT - trans->count; +} + static inline void nft_ctx_update(struct nft_ctx *ctx, const struct nft_trans *trans) { diff --git a/include/net/netns/ipv4.h b/include/net/netns/ipv4.h index 2dbd46fc4734b7..8e971c7bf16464 100644 --- a/include/net/netns/ipv4.h +++ b/include/net/netns/ipv4.h @@ -88,6 +88,12 @@ struct netns_ipv4 { int sysctl_tcp_rcvbuf_low_rtt; __cacheline_group_end(netns_ipv4_read_rx); + /* ICMP rate limiter hot cache line. */ + __cacheline_group_begin_aligned(icmp); + atomic_t icmp_global_credit; + u32 icmp_global_stamp; + __cacheline_group_end_aligned(icmp); + struct inet_timewait_death_row tcp_death_row; struct udp_table *udp_table; @@ -141,8 +147,7 @@ struct netns_ipv4 { int sysctl_icmp_ratemask; int sysctl_icmp_msgs_per_sec; int sysctl_icmp_msgs_burst; - atomic_t icmp_global_credit; - u32 icmp_global_stamp; + u32 ip_rt_min_pmtu; int ip_rt_mtu_expires; int ip_rt_min_advmss; diff --git a/include/net/netns/mpls.h b/include/net/netns/mpls.h index 6682e51513efa5..2073cbac2afb51 100644 --- a/include/net/netns/mpls.h +++ b/include/net/netns/mpls.h @@ -17,6 +17,7 @@ struct netns_mpls { size_t platform_labels; struct mpls_route __rcu * __rcu *platform_label; struct mutex platform_mutex; + seqcount_mutex_t platform_label_seq; struct ctl_table_header *ctl; }; diff --git a/include/net/page_pool/types.h b/include/net/page_pool/types.h index 1509a536cb855a..fb4f03ccd61567 100644 --- a/include/net/page_pool/types.h +++ b/include/net/page_pool/types.h @@ -246,7 +246,7 @@ struct page_pool { /* User-facing fields, protected by page_pools_lock */ struct { struct hlist_node list; - u64 detach_time; + ktime_t detach_time; u32 id; } user; }; diff --git a/include/net/sch_generic.h b/include/net/sch_generic.h index c3a7268b567e0a..c3d657359a3d2d 100644 --- a/include/net/sch_generic.h +++ b/include/net/sch_generic.h @@ -716,6 +716,34 @@ void qdisc_destroy(struct Qdisc *qdisc); void qdisc_put(struct Qdisc *qdisc); void qdisc_put_unlocked(struct Qdisc *qdisc); void qdisc_tree_reduce_backlog(struct Qdisc *qdisc, int n, int len); + +static inline void dev_reset_queue(struct net_device *dev, + struct netdev_queue *dev_queue, + void *_unused) +{ + struct Qdisc *qdisc; + bool nolock; + + qdisc = rtnl_dereference(dev_queue->qdisc_sleeping); + if (!qdisc) + return; + + nolock = qdisc->flags & TCQ_F_NOLOCK; + + if (nolock) + spin_lock_bh(&qdisc->seqlock); + spin_lock_bh(qdisc_lock(qdisc)); + + qdisc_reset(qdisc); + + spin_unlock_bh(qdisc_lock(qdisc)); + if (nolock) { + clear_bit(__QDISC_STATE_MISSED, &qdisc->state); + clear_bit(__QDISC_STATE_DRAINING, &qdisc->state); + spin_unlock_bh(&qdisc->seqlock); + } +} + #ifdef CONFIG_NET_SCHED int qdisc_offload_dump_helper(struct Qdisc *q, enum tc_setup_type type, void *type_data); @@ -778,13 +806,23 @@ static inline bool skb_skip_tc_classify(struct sk_buff *skb) static inline void qdisc_reset_all_tx_gt(struct net_device *dev, unsigned int i) { struct Qdisc *qdisc; + bool nolock; for (; i < dev->num_tx_queues; i++) { qdisc = rtnl_dereference(netdev_get_tx_queue(dev, i)->qdisc); if (qdisc) { + nolock = qdisc->flags & TCQ_F_NOLOCK; + + if (nolock) + spin_lock_bh(&qdisc->seqlock); spin_lock_bh(qdisc_lock(qdisc)); qdisc_reset(qdisc); spin_unlock_bh(qdisc_lock(qdisc)); + if (nolock) { + clear_bit(__QDISC_STATE_MISSED, &qdisc->state); + clear_bit(__QDISC_STATE_DRAINING, &qdisc->state); + spin_unlock_bh(&qdisc->seqlock); + } } } } @@ -1419,6 +1457,11 @@ void mini_qdisc_pair_init(struct mini_Qdisc_pair *miniqp, struct Qdisc *qdisc, void mini_qdisc_pair_block_init(struct mini_Qdisc_pair *miniqp, struct tcf_block *block); +static inline bool mini_qdisc_pair_inited(struct mini_Qdisc_pair *miniqp) +{ + return !!miniqp->p_miniq; +} + void mq_change_real_num_tx(struct Qdisc *sch, unsigned int new_real_tx); int sch_frag_xmit_hook(struct sk_buff *skb, int (*xmit)(struct sk_buff *skb)); diff --git a/include/net/secure_seq.h b/include/net/secure_seq.h index cddebafb9f779e..6f996229167b3c 100644 --- a/include/net/secure_seq.h +++ b/include/net/secure_seq.h @@ -5,16 +5,47 @@ #include struct net; +extern struct net init_net; + +union tcp_seq_and_ts_off { + struct { + u32 seq; + u32 ts_off; + }; + u64 hash64; +}; u64 secure_ipv4_port_ephemeral(__be32 saddr, __be32 daddr, __be16 dport); u64 secure_ipv6_port_ephemeral(const __be32 *saddr, const __be32 *daddr, __be16 dport); -u32 secure_tcp_seq(__be32 saddr, __be32 daddr, - __be16 sport, __be16 dport); -u32 secure_tcp_ts_off(const struct net *net, __be32 saddr, __be32 daddr); -u32 secure_tcpv6_seq(const __be32 *saddr, const __be32 *daddr, - __be16 sport, __be16 dport); -u32 secure_tcpv6_ts_off(const struct net *net, - const __be32 *saddr, const __be32 *daddr); +union tcp_seq_and_ts_off +secure_tcp_seq_and_ts_off(const struct net *net, __be32 saddr, __be32 daddr, + __be16 sport, __be16 dport); + +static inline u32 secure_tcp_seq(__be32 saddr, __be32 daddr, + __be16 sport, __be16 dport) +{ + union tcp_seq_and_ts_off ts; + + ts = secure_tcp_seq_and_ts_off(&init_net, saddr, daddr, + sport, dport); + + return ts.seq; +} + +union tcp_seq_and_ts_off +secure_tcpv6_seq_and_ts_off(const struct net *net, const __be32 *saddr, + const __be32 *daddr, + __be16 sport, __be16 dport); + +static inline u32 secure_tcpv6_seq(const __be32 *saddr, const __be32 *daddr, + __be16 sport, __be16 dport) +{ + union tcp_seq_and_ts_off ts; + + ts = secure_tcpv6_seq_and_ts_off(&init_net, saddr, daddr, + sport, dport); + return ts.seq; +} #endif /* _NET_SECURE_SEQ */ diff --git a/include/net/sock.h b/include/net/sock.h index aafe8bdb2c0f93..ff65c3a67efa21 100644 --- a/include/net/sock.h +++ b/include/net/sock.h @@ -2089,7 +2089,7 @@ static inline int sk_rx_queue_get(const struct sock *sk) static inline void sk_set_socket(struct sock *sk, struct socket *sock) { - sk->sk_socket = sock; + WRITE_ONCE(sk->sk_socket, sock); if (sock) { WRITE_ONCE(sk->sk_uid, SOCK_INODE(sock)->i_uid); WRITE_ONCE(sk->sk_ino, SOCK_INODE(sock)->i_ino); diff --git a/include/net/tc_act/tc_gate.h b/include/net/tc_act/tc_gate.h index c1a67149c6b625..5223c00279d5a2 100644 --- a/include/net/tc_act/tc_gate.h +++ b/include/net/tc_act/tc_gate.h @@ -32,6 +32,7 @@ struct tcf_gate_params { s32 tcfg_clockid; size_t num_entries; struct list_head entries; + struct rcu_head rcu; }; #define GATE_ACT_GATE_OPEN BIT(0) @@ -39,7 +40,7 @@ struct tcf_gate_params { struct tcf_gate { struct tc_action common; - struct tcf_gate_params param; + struct tcf_gate_params __rcu *param; u8 current_gate_status; ktime_t current_close_time; u32 current_entry_octets; @@ -51,47 +52,65 @@ struct tcf_gate { #define to_gate(a) ((struct tcf_gate *)a) +static inline struct tcf_gate_params *tcf_gate_params_locked(const struct tc_action *a) +{ + struct tcf_gate *gact = to_gate(a); + + return rcu_dereference_protected(gact->param, + lockdep_is_held(&gact->tcf_lock)); +} + static inline s32 tcf_gate_prio(const struct tc_action *a) { + struct tcf_gate_params *p; s32 tcfg_prio; - tcfg_prio = to_gate(a)->param.tcfg_priority; + p = tcf_gate_params_locked(a); + tcfg_prio = p->tcfg_priority; return tcfg_prio; } static inline u64 tcf_gate_basetime(const struct tc_action *a) { + struct tcf_gate_params *p; u64 tcfg_basetime; - tcfg_basetime = to_gate(a)->param.tcfg_basetime; + p = tcf_gate_params_locked(a); + tcfg_basetime = p->tcfg_basetime; return tcfg_basetime; } static inline u64 tcf_gate_cycletime(const struct tc_action *a) { + struct tcf_gate_params *p; u64 tcfg_cycletime; - tcfg_cycletime = to_gate(a)->param.tcfg_cycletime; + p = tcf_gate_params_locked(a); + tcfg_cycletime = p->tcfg_cycletime; return tcfg_cycletime; } static inline u64 tcf_gate_cycletimeext(const struct tc_action *a) { + struct tcf_gate_params *p; u64 tcfg_cycletimeext; - tcfg_cycletimeext = to_gate(a)->param.tcfg_cycletime_ext; + p = tcf_gate_params_locked(a); + tcfg_cycletimeext = p->tcfg_cycletime_ext; return tcfg_cycletimeext; } static inline u32 tcf_gate_num_entries(const struct tc_action *a) { + struct tcf_gate_params *p; u32 num_entries; - num_entries = to_gate(a)->param.num_entries; + p = tcf_gate_params_locked(a); + num_entries = p->num_entries; return num_entries; } @@ -105,7 +124,7 @@ static inline struct action_gate_entry u32 num_entries; int i = 0; - p = &to_gate(a)->param; + p = tcf_gate_params_locked(a); num_entries = p->num_entries; list_for_each_entry(entry, &p->entries, list) diff --git a/include/net/tc_act/tc_ife.h b/include/net/tc_act/tc_ife.h index c7f24a2da1cad4..24d4d5a62b3c23 100644 --- a/include/net/tc_act/tc_ife.h +++ b/include/net/tc_act/tc_ife.h @@ -13,15 +13,13 @@ struct tcf_ife_params { u8 eth_src[ETH_ALEN]; u16 eth_type; u16 flags; - + struct list_head metalist; struct rcu_head rcu; }; struct tcf_ife_info { struct tc_action common; struct tcf_ife_params __rcu *params; - /* list of metaids allowed */ - struct list_head metalist; }; #define to_ife(a) ((struct tcf_ife_info *)a) diff --git a/include/net/tcp.h b/include/net/tcp.h index 0deb5e9dd91146..e15e1d0e6f4e20 100644 --- a/include/net/tcp.h +++ b/include/net/tcp.h @@ -43,6 +43,7 @@ #include #include #include +#include #include #include @@ -533,7 +534,9 @@ struct sock *tcp_v4_syn_recv_sock(const struct sock *sk, struct sk_buff *skb, struct request_sock *req, struct dst_entry *dst, struct request_sock *req_unhash, - bool *own_req); + bool *own_req, + void (*opt_child_init)(struct sock *newsk, + const struct sock *sk)); int tcp_v4_do_rcv(struct sock *sk, struct sk_buff *skb); int tcp_v4_connect(struct sock *sk, struct sockaddr_unsized *uaddr, int addr_len); int tcp_connect(struct sock *sk); @@ -1203,7 +1206,15 @@ enum tcp_ca_ack_event_flags { #define TCP_CONG_NON_RESTRICTED BIT(0) /* Requires ECN/ECT set on all packets */ #define TCP_CONG_NEEDS_ECN BIT(1) -#define TCP_CONG_MASK (TCP_CONG_NON_RESTRICTED | TCP_CONG_NEEDS_ECN) +/* Require successfully negotiated AccECN capability */ +#define TCP_CONG_NEEDS_ACCECN BIT(2) +/* Use ECT(1) instead of ECT(0) while the CA is uninitialized */ +#define TCP_CONG_ECT_1_NEGOTIATION BIT(3) +/* Cannot fallback to RFC3168 during AccECN negotiation */ +#define TCP_CONG_NO_FALLBACK_RFC3168 BIT(4) +#define TCP_CONG_MASK (TCP_CONG_NON_RESTRICTED | TCP_CONG_NEEDS_ECN | \ + TCP_CONG_NEEDS_ACCECN | TCP_CONG_ECT_1_NEGOTIATION | \ + TCP_CONG_NO_FALLBACK_RFC3168) union tcp_cc_info; @@ -1335,6 +1346,27 @@ static inline bool tcp_ca_needs_ecn(const struct sock *sk) return icsk->icsk_ca_ops->flags & TCP_CONG_NEEDS_ECN; } +static inline bool tcp_ca_needs_accecn(const struct sock *sk) +{ + const struct inet_connection_sock *icsk = inet_csk(sk); + + return icsk->icsk_ca_ops->flags & TCP_CONG_NEEDS_ACCECN; +} + +static inline bool tcp_ca_ect_1_negotiation(const struct sock *sk) +{ + const struct inet_connection_sock *icsk = inet_csk(sk); + + return icsk->icsk_ca_ops->flags & TCP_CONG_ECT_1_NEGOTIATION; +} + +static inline bool tcp_ca_no_fallback_rfc3168(const struct sock *sk) +{ + const struct inet_connection_sock *icsk = inet_csk(sk); + + return icsk->icsk_ca_ops->flags & TCP_CONG_NO_FALLBACK_RFC3168; +} + static inline void tcp_ca_event(struct sock *sk, const enum tcp_ca_event event) { const struct inet_connection_sock *icsk = inet_csk(sk); @@ -2406,8 +2438,9 @@ struct tcp_request_sock_ops { struct flowi *fl, struct request_sock *req, u32 tw_isn); - u32 (*init_seq)(const struct sk_buff *skb); - u32 (*init_ts_off)(const struct net *net, const struct sk_buff *skb); + union tcp_seq_and_ts_off (*init_seq_and_ts_off)( + const struct net *net, + const struct sk_buff *skb); int (*send_synack)(const struct sock *sk, struct dst_entry *dst, struct flowi *fl, struct request_sock *req, struct tcp_fastopen_cookie *foc, diff --git a/include/net/tcp_ecn.h b/include/net/tcp_ecn.h index f13e5cd2b1ac3c..a709fb1756eb75 100644 --- a/include/net/tcp_ecn.h +++ b/include/net/tcp_ecn.h @@ -31,6 +31,12 @@ enum tcp_accecn_option { TCP_ACCECN_OPTION_FULL = 2, }; +/* Apply either ECT(0) or ECT(1) based on TCP_CONG_ECT_1_NEGOTIATION flag */ +static inline void INET_ECN_xmit_ect_1_negotiation(struct sock *sk) +{ + __INET_ECN_xmit(sk, tcp_ca_ect_1_negotiation(sk)); +} + static inline void tcp_ecn_queue_cwr(struct tcp_sock *tp) { /* Do not set CWR if in AccECN mode! */ @@ -467,6 +473,26 @@ static inline u8 tcp_accecn_option_init(const struct sk_buff *skb, return TCP_ACCECN_OPT_COUNTER_SEEN; } +static inline void tcp_ecn_rcv_synack_accecn(struct sock *sk, + const struct sk_buff *skb, u8 dsf) +{ + struct tcp_sock *tp = tcp_sk(sk); + + tcp_ecn_mode_set(tp, TCP_ECN_MODE_ACCECN); + tp->syn_ect_rcv = dsf & INET_ECN_MASK; + /* Demand Accurate ECN option in response to the SYN on the SYN/ACK + * and the TCP server will try to send one more packet with an AccECN + * Option at a later point during the connection. + */ + if (tp->rx_opt.accecn && + tp->saw_accecn_opt < TCP_ACCECN_OPT_COUNTER_SEEN) { + u8 saw_opt = tcp_accecn_option_init(skb, tp->rx_opt.accecn); + + tcp_accecn_saw_opt_fail_recv(tp, saw_opt); + tp->accecn_opt_demand = 2; + } +} + /* See Table 2 of the AccECN draft */ static inline void tcp_ecn_rcv_synack(struct sock *sk, const struct sk_buff *skb, const struct tcphdr *th, u8 ip_dsfield) @@ -489,32 +515,32 @@ static inline void tcp_ecn_rcv_synack(struct sock *sk, const struct sk_buff *skb tcp_ecn_mode_set(tp, TCP_ECN_DISABLED); break; case 0x1: - case 0x5: /* +========+========+============+=============+ * | A | B | SYN/ACK | Feedback | * | | | B->A | Mode of A | * | | | AE CWR ECE | | * +========+========+============+=============+ - * | AccECN | Nonce | 1 0 1 | (Reserved) | * | AccECN | ECN | 0 0 1 | Classic ECN | * | Nonce | AccECN | 0 0 1 | Classic ECN | * | ECN | AccECN | 0 0 1 | Classic ECN | * +========+========+============+=============+ */ - if (tcp_ecn_mode_pending(tp)) - /* Downgrade from AccECN, or requested initially */ + if (tcp_ca_no_fallback_rfc3168(sk)) + tcp_ecn_mode_set(tp, TCP_ECN_DISABLED); + else tcp_ecn_mode_set(tp, TCP_ECN_MODE_RFC3168); break; - default: - tcp_ecn_mode_set(tp, TCP_ECN_MODE_ACCECN); - tp->syn_ect_rcv = ip_dsfield & INET_ECN_MASK; - if (tp->rx_opt.accecn && - tp->saw_accecn_opt < TCP_ACCECN_OPT_COUNTER_SEEN) { - u8 saw_opt = tcp_accecn_option_init(skb, tp->rx_opt.accecn); - - tcp_accecn_saw_opt_fail_recv(tp, saw_opt); - tp->accecn_opt_demand = 2; + case 0x5: + if (tcp_ecn_mode_pending(tp)) { + tcp_ecn_rcv_synack_accecn(sk, skb, ip_dsfield); + if (INET_ECN_is_ce(ip_dsfield)) { + tp->received_ce++; + tp->received_ce_pending++; + } } + break; + default: + tcp_ecn_rcv_synack_accecn(sk, skb, ip_dsfield); if (INET_ECN_is_ce(ip_dsfield) && tcp_accecn_validate_syn_feedback(sk, ace, tp->syn_ect_snt)) { @@ -525,9 +551,11 @@ static inline void tcp_ecn_rcv_synack(struct sock *sk, const struct sk_buff *skb } } -static inline void tcp_ecn_rcv_syn(struct tcp_sock *tp, const struct tcphdr *th, +static inline void tcp_ecn_rcv_syn(struct sock *sk, const struct tcphdr *th, const struct sk_buff *skb) { + struct tcp_sock *tp = tcp_sk(sk); + if (tcp_ecn_mode_pending(tp)) { if (!tcp_accecn_syn_requested(th)) { /* Downgrade to classic ECN feedback */ @@ -539,7 +567,8 @@ static inline void tcp_ecn_rcv_syn(struct tcp_sock *tp, const struct tcphdr *th, tcp_ecn_mode_set(tp, TCP_ECN_MODE_ACCECN); } } - if (tcp_ecn_mode_rfc3168(tp) && (!th->ece || !th->cwr)) + if (tcp_ecn_mode_rfc3168(tp) && + (!th->ece || !th->cwr || tcp_ca_no_fallback_rfc3168(sk))) tcp_ecn_mode_set(tp, TCP_ECN_DISABLED); } @@ -561,7 +590,7 @@ static inline void tcp_ecn_send_synack(struct sock *sk, struct sk_buff *skb) TCP_SKB_CB(skb)->tcp_flags &= ~TCPHDR_ECE; else if (tcp_ca_needs_ecn(sk) || tcp_bpf_ca_needs_ecn(sk)) - INET_ECN_xmit(sk); + INET_ECN_xmit_ect_1_negotiation(sk); if (tp->ecn_flags & TCP_ECN_MODE_ACCECN) { TCP_SKB_CB(skb)->tcp_flags &= ~TCPHDR_ACE; @@ -579,7 +608,8 @@ static inline void tcp_ecn_send_syn(struct sock *sk, struct sk_buff *skb) bool use_ecn, use_accecn; u8 tcp_ecn = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_ecn); - use_accecn = tcp_ecn == TCP_ECN_IN_ACCECN_OUT_ACCECN; + use_accecn = tcp_ecn == TCP_ECN_IN_ACCECN_OUT_ACCECN || + tcp_ca_needs_accecn(sk); use_ecn = tcp_ecn == TCP_ECN_IN_ECN_OUT_ECN || tcp_ecn == TCP_ECN_IN_ACCECN_OUT_ECN || tcp_ca_needs_ecn(sk) || bpf_needs_ecn || use_accecn; @@ -595,7 +625,7 @@ static inline void tcp_ecn_send_syn(struct sock *sk, struct sk_buff *skb) if (use_ecn) { if (tcp_ca_needs_ecn(sk) || bpf_needs_ecn) - INET_ECN_xmit(sk); + INET_ECN_xmit_ect_1_negotiation(sk); TCP_SKB_CB(skb)->tcp_flags |= TCPHDR_ECE | TCPHDR_CWR; if (use_accecn) { diff --git a/include/net/udp_tunnel.h b/include/net/udp_tunnel.h index 9acef2fbd2fdcf..d97ee26ba4f66f 100644 --- a/include/net/udp_tunnel.h +++ b/include/net/udp_tunnel.h @@ -47,7 +47,7 @@ int udp_sock_create6(struct net *net, struct udp_port_cfg *cfg, static inline int udp_sock_create6(struct net *net, struct udp_port_cfg *cfg, struct socket **sockp) { - return 0; + return -EPFNOSUPPORT; } #endif diff --git a/include/net/xdp_sock.h b/include/net/xdp_sock.h index 23e8861e8b25e6..ebac60a3d8a17b 100644 --- a/include/net/xdp_sock.h +++ b/include/net/xdp_sock.h @@ -14,7 +14,7 @@ #include #include -#define XDP_UMEM_SG_FLAG (1 << 1) +#define XDP_UMEM_SG_FLAG BIT(3) struct net_device; struct xsk_queue; diff --git a/include/net/xdp_sock_drv.h b/include/net/xdp_sock_drv.h index 242e34f771cca6..46797645a0c241 100644 --- a/include/net/xdp_sock_drv.h +++ b/include/net/xdp_sock_drv.h @@ -41,16 +41,42 @@ static inline u32 xsk_pool_get_headroom(struct xsk_buff_pool *pool) return XDP_PACKET_HEADROOM + pool->headroom; } +static inline u32 xsk_pool_get_tailroom(bool mbuf) +{ + return mbuf ? SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) : 0; +} + static inline u32 xsk_pool_get_chunk_size(struct xsk_buff_pool *pool) { return pool->chunk_size; } -static inline u32 xsk_pool_get_rx_frame_size(struct xsk_buff_pool *pool) +static inline u32 __xsk_pool_get_rx_frame_size(struct xsk_buff_pool *pool) { return xsk_pool_get_chunk_size(pool) - xsk_pool_get_headroom(pool); } +static inline u32 xsk_pool_get_rx_frame_size(struct xsk_buff_pool *pool) +{ + u32 frame_size = __xsk_pool_get_rx_frame_size(pool); + struct xdp_umem *umem = pool->umem; + bool mbuf; + + /* Reserve tailroom only for zero-copy pools that opted into + * multi-buffer. The reserved area is used for skb_shared_info, + * matching the XDP core's xdp_data_hard_end() layout. + */ + mbuf = pool->dev && (umem->flags & XDP_UMEM_SG_FLAG); + frame_size -= xsk_pool_get_tailroom(mbuf); + + return ALIGN_DOWN(frame_size, 128); +} + +static inline u32 xsk_pool_get_rx_frag_step(struct xsk_buff_pool *pool) +{ + return pool->unaligned ? 0 : xsk_pool_get_chunk_size(pool); +} + static inline void xsk_pool_set_rxq_info(struct xsk_buff_pool *pool, struct xdp_rxq_info *rxq) { @@ -122,7 +148,7 @@ static inline void xsk_buff_free(struct xdp_buff *xdp) goto out; list_for_each_entry_safe(pos, tmp, xskb_list, list_node) { - list_del(&pos->list_node); + list_del_init(&pos->list_node); xp_free(pos); } @@ -157,7 +183,7 @@ static inline struct xdp_buff *xsk_buff_get_frag(const struct xdp_buff *first) frag = list_first_entry_or_null(&xskb->pool->xskb_list, struct xdp_buff_xsk, list_node); if (frag) { - list_del(&frag->list_node); + list_del_init(&frag->list_node); ret = &frag->xdp; } @@ -168,7 +194,7 @@ static inline void xsk_buff_del_frag(struct xdp_buff *xdp) { struct xdp_buff_xsk *xskb = container_of(xdp, struct xdp_buff_xsk, xdp); - list_del(&xskb->list_node); + list_del_init(&xskb->list_node); } static inline struct xdp_buff *xsk_buff_get_head(struct xdp_buff *first) @@ -337,6 +363,11 @@ static inline u32 xsk_pool_get_rx_frame_size(struct xsk_buff_pool *pool) return 0; } +static inline u32 xsk_pool_get_rx_frag_step(struct xsk_buff_pool *pool) +{ + return 0; +} + static inline void xsk_pool_set_rxq_info(struct xsk_buff_pool *pool, struct xdp_rxq_info *rxq) { diff --git a/include/rdma/rw.h b/include/rdma/rw.h index d606cac482338b..9a8f4b76ce588d 100644 --- a/include/rdma/rw.h +++ b/include/rdma/rw.h @@ -66,6 +66,8 @@ int rdma_rw_ctx_post(struct rdma_rw_ctx *ctx, struct ib_qp *qp, u32 port_num, unsigned int rdma_rw_mr_factor(struct ib_device *device, u32 port_num, unsigned int maxpages); +unsigned int rdma_rw_max_send_wr(struct ib_device *dev, u32 port_num, + unsigned int max_rdma_ctxs, u32 create_flags); void rdma_rw_init_qp(struct ib_device *dev, struct ib_qp_init_attr *attr); int rdma_rw_init_mrs(struct ib_qp *qp, struct ib_qp_init_attr *attr); void rdma_rw_cleanup_mrs(struct ib_qp *qp); diff --git a/include/sound/control.h b/include/sound/control.h index e07f6b960641ff..9be6546bf787de 100644 --- a/include/sound/control.h +++ b/include/sound/control.h @@ -14,9 +14,12 @@ #define snd_kcontrol_chip(kcontrol) ((kcontrol)->private_data) struct snd_kcontrol; +struct snd_ctl_file; typedef int (snd_kcontrol_info_t) (struct snd_kcontrol * kcontrol, struct snd_ctl_elem_info * uinfo); typedef int (snd_kcontrol_get_t) (struct snd_kcontrol * kcontrol, struct snd_ctl_elem_value * ucontrol); typedef int (snd_kcontrol_put_t) (struct snd_kcontrol * kcontrol, struct snd_ctl_elem_value * ucontrol); +typedef int (snd_kcontrol_lock_t) (struct snd_kcontrol * kcontrol, struct snd_ctl_file *owner); +typedef void (snd_kcontrol_unlock_t) (struct snd_kcontrol * kcontrol); typedef int (snd_kcontrol_tlv_rw_t)(struct snd_kcontrol *kcontrol, int op_flag, /* SNDRV_CTL_TLV_OP_XXX */ unsigned int size, @@ -55,6 +58,8 @@ struct snd_kcontrol_new { snd_kcontrol_info_t *info; snd_kcontrol_get_t *get; snd_kcontrol_put_t *put; + snd_kcontrol_lock_t *lock; + snd_kcontrol_unlock_t *unlock; union { snd_kcontrol_tlv_rw_t *c; const unsigned int *p; @@ -74,6 +79,8 @@ struct snd_kcontrol { snd_kcontrol_info_t *info; snd_kcontrol_get_t *get; snd_kcontrol_put_t *put; + snd_kcontrol_lock_t *lock; + snd_kcontrol_unlock_t *unlock; union { snd_kcontrol_tlv_rw_t *c; const unsigned int *p; diff --git a/include/sound/cs35l56.h b/include/sound/cs35l56.h index 5928af539c4681..d0ae1ae2ae2a0c 100644 --- a/include/sound/cs35l56.h +++ b/include/sound/cs35l56.h @@ -374,6 +374,7 @@ extern const char * const cs35l56_cal_set_status_text[3]; extern const char * const cs35l56_tx_input_texts[CS35L56_NUM_INPUT_SRC]; extern const unsigned int cs35l56_tx_input_values[CS35L56_NUM_INPUT_SRC]; +int cs35l56_set_asp_patch(struct cs35l56_base *cs35l56_base); int cs35l56_set_patch(struct cs35l56_base *cs35l56_base); int cs35l56_mbox_send(struct cs35l56_base *cs35l56_base, unsigned int command); int cs35l56_firmware_shutdown(struct cs35l56_base *cs35l56_base); diff --git a/include/sound/cs42l42.h b/include/sound/cs42l42.h index 1bd8eee54f6665..b3657965d49109 100644 --- a/include/sound/cs42l42.h +++ b/include/sound/cs42l42.h @@ -62,6 +62,10 @@ #define CS42L42_INTERNAL_FS_MASK (1 << CS42L42_INTERNAL_FS_SHIFT) #define CS42L42_SFTRAMP_RATE (CS42L42_PAGE_10 + 0x0A) +#define CS42L42_SFTRAMP_ASR_RATE_MASK GENMASK(7, 4) +#define CS42L42_SFTRAMP_ASR_RATE_SHIFT 4 +#define CS42L42_SFTRAMP_DSR_RATE_MASK GENMASK(3, 0) +#define CS42L42_SFTRAMP_DSR_RATE_SHIFT 0 #define CS42L42_SLOW_START_ENABLE (CS42L42_PAGE_10 + 0x0B) #define CS42L42_SLOW_START_EN_MASK GENMASK(6, 4) #define CS42L42_SLOW_START_EN_SHIFT 4 diff --git a/include/sound/pcm.h b/include/sound/pcm.h index a7860c047503a8..a0c987c9bd1fee 100644 --- a/include/sound/pcm.h +++ b/include/sound/pcm.h @@ -1073,6 +1073,7 @@ int snd_interval_ranges(struct snd_interval *i, unsigned int count, int snd_interval_ratnum(struct snd_interval *i, unsigned int rats_count, const struct snd_ratnum *rats, unsigned int *nump, unsigned int *denp); +int snd_interval_rate_bits(struct snd_interval *i, unsigned int rate_bits); void _snd_pcm_hw_params_any(struct snd_pcm_hw_params *params); void _snd_pcm_hw_param_setempty(struct snd_pcm_hw_params *params, snd_pcm_hw_param_t var); diff --git a/include/sound/sdca_interrupts.h b/include/sound/sdca_interrupts.h index 8f13417d129aba..109e7826ce38c6 100644 --- a/include/sound/sdca_interrupts.h +++ b/include/sound/sdca_interrupts.h @@ -69,6 +69,8 @@ struct sdca_interrupt_info { int sdca_irq_request(struct device *dev, struct sdca_interrupt_info *interrupt_info, int sdca_irq, const char *name, irq_handler_t handler, void *data); +void sdca_irq_free(struct device *dev, struct sdca_interrupt_info *interrupt_info, + int sdca_irq, const char *name, void *data); int sdca_irq_data_populate(struct device *dev, struct regmap *function_regmap, struct snd_soc_component *component, struct sdca_function_data *function, @@ -81,6 +83,9 @@ int sdca_irq_populate_early(struct device *dev, struct regmap *function_regmap, int sdca_irq_populate(struct sdca_function_data *function, struct snd_soc_component *component, struct sdca_interrupt_info *info); +void sdca_irq_cleanup(struct device *dev, + struct sdca_function_data *function, + struct sdca_interrupt_info *info); struct sdca_interrupt_info *sdca_irq_allocate(struct device *dev, struct regmap *regmap, int irq); diff --git a/include/sound/sdca_jack.h b/include/sound/sdca_jack.h new file mode 100644 index 00000000000000..3ec22046d3ebc2 --- /dev/null +++ b/include/sound/sdca_jack.h @@ -0,0 +1,32 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * The MIPI SDCA specification is available for public downloads at + * https://www.mipi.org/mipi-sdca-v1-0-download + * + * Copyright (C) 2025 Cirrus Logic, Inc. and + * Cirrus Logic International Semiconductor Ltd. + */ + +#ifndef __SDCA_JACK_H__ +#define __SDCA_JACK_H__ + +struct sdca_interrupt; +struct snd_kcontrol; +struct snd_soc_jack; + +/** + * struct jack_state - Jack state structure to keep data between interrupts + * @kctl: Pointer to the ALSA control attached to this jack + * @jack: Pointer to the ASoC jack struct for this jack + */ +struct jack_state { + struct snd_kcontrol *kctl; + struct snd_soc_jack *jack; +}; + +int sdca_jack_alloc_state(struct sdca_interrupt *interrupt); +int sdca_jack_process(struct sdca_interrupt *interrupt); +int sdca_jack_set_jack(struct sdca_interrupt_info *info, struct snd_soc_jack *jack); +int sdca_jack_report(struct sdca_interrupt *interrupt); + +#endif // __SDCA_JACK_H__ diff --git a/include/sound/soc-card.h b/include/sound/soc-card.h index ecc02e955279fd..ef46cac97d9968 100644 --- a/include/sound/soc-card.h +++ b/include/sound/soc-card.h @@ -44,7 +44,7 @@ int snd_soc_card_resume_post(struct snd_soc_card *card); int snd_soc_card_probe(struct snd_soc_card *card); int snd_soc_card_late_probe(struct snd_soc_card *card); -void snd_soc_card_fixup_controls(struct snd_soc_card *card); +int snd_soc_card_fixup_controls(struct snd_soc_card *card); int snd_soc_card_remove(struct snd_soc_card *card); int snd_soc_card_set_bias_level(struct snd_soc_card *card, diff --git a/include/sound/soc.h b/include/sound/soc.h index aa0fe6b802934b..8e2f1b8b8886f5 100644 --- a/include/sound/soc.h +++ b/include/sound/soc.h @@ -575,8 +575,14 @@ int snd_soc_get_volsw_sx(struct snd_kcontrol *kcontrol, struct snd_ctl_elem_value *ucontrol); int snd_soc_put_volsw_sx(struct snd_kcontrol *kcontrol, struct snd_ctl_elem_value *ucontrol); +bool snd_soc_control_matches(struct snd_kcontrol *kcontrol, + const char *pattern); int snd_soc_limit_volume(struct snd_soc_card *card, const char *name, int max); +int snd_soc_deactivate_kctl(struct snd_soc_card *card, + const char *name, int active); +int snd_soc_set_enum_kctl(struct snd_soc_card *card, + const char *name, const char *strval); int snd_soc_bytes_info(struct snd_kcontrol *kcontrol, struct snd_ctl_elem_info *uinfo); int snd_soc_bytes_get(struct snd_kcontrol *kcontrol, @@ -1003,7 +1009,7 @@ struct snd_soc_card { int (*probe)(struct snd_soc_card *card); int (*late_probe)(struct snd_soc_card *card); - void (*fixup_controls)(struct snd_soc_card *card); + int (*fixup_controls)(struct snd_soc_card *card); int (*remove)(struct snd_soc_card *card); /* the pre and post PM functions are used to do any PM work before and diff --git a/include/trace/events/btrfs.h b/include/trace/events/btrfs.h index 125bdc166bfed7..0864700f76e0a1 100644 --- a/include/trace/events/btrfs.h +++ b/include/trace/events/btrfs.h @@ -769,12 +769,15 @@ TRACE_EVENT(btrfs_sync_file, ), TP_fast_assign( - const struct dentry *dentry = file->f_path.dentry; - const struct inode *inode = d_inode(dentry); + struct dentry *dentry = file_dentry(file); + struct inode *inode = file_inode(file); + struct dentry *parent = dget_parent(dentry); + struct inode *parent_inode = d_inode(parent); - TP_fast_assign_fsid(btrfs_sb(file->f_path.dentry->d_sb)); + dput(parent); + TP_fast_assign_fsid(btrfs_sb(inode->i_sb)); __entry->ino = btrfs_ino(BTRFS_I(inode)); - __entry->parent = btrfs_ino(BTRFS_I(d_inode(dentry->d_parent))); + __entry->parent = btrfs_ino(BTRFS_I(parent_inode)); __entry->datasync = datasync; __entry->root_objectid = btrfs_root_id(BTRFS_I(inode)->root); ), diff --git a/include/trace/events/dma.h b/include/trace/events/dma.h index 33e99e792f1aa2..69cb3805ee81ca 100644 --- a/include/trace/events/dma.h +++ b/include/trace/events/dma.h @@ -32,7 +32,8 @@ TRACE_DEFINE_ENUM(DMA_NONE); { DMA_ATTR_ALLOC_SINGLE_PAGES, "ALLOC_SINGLE_PAGES" }, \ { DMA_ATTR_NO_WARN, "NO_WARN" }, \ { DMA_ATTR_PRIVILEGED, "PRIVILEGED" }, \ - { DMA_ATTR_MMIO, "MMIO" }) + { DMA_ATTR_MMIO, "MMIO" }, \ + { DMA_ATTR_CPU_CACHE_CLEAN, "CACHE_CLEAN" }) DECLARE_EVENT_CLASS(dma_map, TP_PROTO(struct device *dev, phys_addr_t phys_addr, dma_addr_t dma_addr, diff --git a/include/trace/events/kmem.h b/include/trace/events/kmem.h index 7f93e754da5c24..cd7920c81f85ba 100644 --- a/include/trace/events/kmem.h +++ b/include/trace/events/kmem.h @@ -440,7 +440,13 @@ TRACE_EVENT(rss_stat, TP_fast_assign( __entry->mm_id = mm_ptr_to_hash(mm); - __entry->curr = !!(current->mm == mm); + /* + * curr is true if the mm matches the current task's mm_struct. + * Since kthreads (PF_KTHREAD) have no mm_struct of their own + * but can borrow one via kthread_use_mm(), we must filter them + * out to avoid incorrectly attributing the RSS update to them. + */ + __entry->curr = current->mm == mm && !(current->flags & PF_KTHREAD); __entry->member = member; __entry->size = (percpu_counter_sum_positive(&mm->rss_stat[member]) << PAGE_SHIFT); diff --git a/include/trace/events/netfs.h b/include/trace/events/netfs.h index 64a382fbc31a84..cbe28211106c52 100644 --- a/include/trace/events/netfs.h +++ b/include/trace/events/netfs.h @@ -57,6 +57,7 @@ EM(netfs_rreq_trace_done, "DONE ") \ EM(netfs_rreq_trace_end_copy_to_cache, "END-C2C") \ EM(netfs_rreq_trace_free, "FREE ") \ + EM(netfs_rreq_trace_intr, "INTR ") \ EM(netfs_rreq_trace_ki_complete, "KI-CMPL") \ EM(netfs_rreq_trace_recollect, "RECLLCT") \ EM(netfs_rreq_trace_redirty, "REDIRTY") \ @@ -169,7 +170,8 @@ EM(netfs_sreq_trace_put_oom, "PUT OOM ") \ EM(netfs_sreq_trace_put_wip, "PUT WIP ") \ EM(netfs_sreq_trace_put_work, "PUT WORK ") \ - E_(netfs_sreq_trace_put_terminated, "PUT TERM ") + EM(netfs_sreq_trace_put_terminated, "PUT TERM ") \ + E_(netfs_sreq_trace_see_failed, "SEE FAILED ") #define netfs_folio_traces \ EM(netfs_folio_is_uptodate, "mod-uptodate") \ @@ -738,19 +740,19 @@ TRACE_EVENT(netfs_collect_stream, __field(unsigned int, wreq) __field(unsigned char, stream) __field(unsigned long long, collected_to) - __field(unsigned long long, front) + __field(unsigned long long, issued_to) ), TP_fast_assign( __entry->wreq = wreq->debug_id; __entry->stream = stream->stream_nr; __entry->collected_to = stream->collected_to; - __entry->front = stream->front ? stream->front->start : UINT_MAX; + __entry->issued_to = atomic64_read(&wreq->issued_to); ), - TP_printk("R=%08x[%x:] cto=%llx frn=%llx", + TP_printk("R=%08x[%x:] cto=%llx ito=%llx", __entry->wreq, __entry->stream, - __entry->collected_to, __entry->front) + __entry->collected_to, __entry->issued_to) ); TRACE_EVENT(netfs_folioq, diff --git a/include/trace/events/rxrpc.h b/include/trace/events/rxrpc.h index 869f97c9bf733e..578b8038b21178 100644 --- a/include/trace/events/rxrpc.h +++ b/include/trace/events/rxrpc.h @@ -185,6 +185,7 @@ EM(rxrpc_skb_put_input, "PUT input ") \ EM(rxrpc_skb_put_jumbo_subpacket, "PUT jumbo-sub") \ EM(rxrpc_skb_put_oob, "PUT oob ") \ + EM(rxrpc_skb_put_old_response, "PUT old-resp ") \ EM(rxrpc_skb_put_purge, "PUT purge ") \ EM(rxrpc_skb_put_purge_oob, "PUT purge-oob") \ EM(rxrpc_skb_put_response, "PUT response ") \ @@ -347,7 +348,7 @@ EM(rxrpc_call_see_release, "SEE release ") \ EM(rxrpc_call_see_userid_exists, "SEE u-exists") \ EM(rxrpc_call_see_waiting_call, "SEE q-conn ") \ - E_(rxrpc_call_see_zap, "SEE zap ") + E_(rxrpc_call_see_still_live, "SEE !still-l") #define rxrpc_txqueue_traces \ EM(rxrpc_txqueue_await_reply, "AWR") \ @@ -520,6 +521,7 @@ #define rxrpc_req_ack_traces \ EM(rxrpc_reqack_ack_lost, "ACK-LOST ") \ EM(rxrpc_reqack_app_stall, "APP-STALL ") \ + EM(rxrpc_reqack_jumbo_win, "JUMBO-WIN ") \ EM(rxrpc_reqack_more_rtt, "MORE-RTT ") \ EM(rxrpc_reqack_no_srv_last, "NO-SRVLAST") \ EM(rxrpc_reqack_old_rtt, "OLD-RTT ") \ diff --git a/include/trace/events/task.h b/include/trace/events/task.h index 4f0759634306c7..b9a129eb54d9eb 100644 --- a/include/trace/events/task.h +++ b/include/trace/events/task.h @@ -38,19 +38,22 @@ TRACE_EVENT(task_rename, TP_ARGS(task, comm), TP_STRUCT__entry( + __field( pid_t, pid) __array( char, oldcomm, TASK_COMM_LEN) __array( char, newcomm, TASK_COMM_LEN) __field( short, oom_score_adj) ), TP_fast_assign( + __entry->pid = task->pid; memcpy(entry->oldcomm, task->comm, TASK_COMM_LEN); strscpy(entry->newcomm, comm, TASK_COMM_LEN); __entry->oom_score_adj = task->signal->oom_score_adj; ), - TP_printk("oldcomm=%s newcomm=%s oom_score_adj=%hd", - __entry->oldcomm, __entry->newcomm, __entry->oom_score_adj) + TP_printk("pid=%d oldcomm=%s newcomm=%s oom_score_adj=%hd", + __entry->pid, __entry->oldcomm, + __entry->newcomm, __entry->oom_score_adj) ); /** diff --git a/include/uapi/drm/amdgpu_drm.h b/include/uapi/drm/amdgpu_drm.h index f80aa4c9d88fdc..ba828108662975 100644 --- a/include/uapi/drm/amdgpu_drm.h +++ b/include/uapi/drm/amdgpu_drm.h @@ -105,8 +105,6 @@ extern "C" { * * %AMDGPU_GEM_DOMAIN_DOORBELL Doorbell. It is an MMIO region for * signalling user mode queues. - * - * %AMDGPU_GEM_DOMAIN_MMIO_REMAP MMIO remap page (special mapping for HDP flushing). */ #define AMDGPU_GEM_DOMAIN_CPU 0x1 #define AMDGPU_GEM_DOMAIN_GTT 0x2 @@ -115,15 +113,13 @@ extern "C" { #define AMDGPU_GEM_DOMAIN_GWS 0x10 #define AMDGPU_GEM_DOMAIN_OA 0x20 #define AMDGPU_GEM_DOMAIN_DOORBELL 0x40 -#define AMDGPU_GEM_DOMAIN_MMIO_REMAP 0x80 #define AMDGPU_GEM_DOMAIN_MASK (AMDGPU_GEM_DOMAIN_CPU | \ AMDGPU_GEM_DOMAIN_GTT | \ AMDGPU_GEM_DOMAIN_VRAM | \ AMDGPU_GEM_DOMAIN_GDS | \ AMDGPU_GEM_DOMAIN_GWS | \ AMDGPU_GEM_DOMAIN_OA | \ - AMDGPU_GEM_DOMAIN_DOORBELL | \ - AMDGPU_GEM_DOMAIN_MMIO_REMAP) + AMDGPU_GEM_DOMAIN_DOORBELL) /* Flag that CPU access will be required for the case of VRAM domain */ #define AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED (1 << 0) diff --git a/include/uapi/drm/drm_fourcc.h b/include/uapi/drm/drm_fourcc.h index e527b24bd824b1..c89aede3cb1201 100644 --- a/include/uapi/drm/drm_fourcc.h +++ b/include/uapi/drm/drm_fourcc.h @@ -401,8 +401,8 @@ extern "C" { * implementation can multiply the values by 2^6=64. For that reason the padding * must only contain zeros. * index 0 = Y plane, [15:0] z:Y [6:10] little endian - * index 1 = Cr plane, [15:0] z:Cr [6:10] little endian - * index 2 = Cb plane, [15:0] z:Cb [6:10] little endian + * index 1 = Cb plane, [15:0] z:Cb [6:10] little endian + * index 2 = Cr plane, [15:0] z:Cr [6:10] little endian */ #define DRM_FORMAT_S010 fourcc_code('S', '0', '1', '0') /* 2x2 subsampled Cb (1) and Cr (2) planes 10 bits per channel */ #define DRM_FORMAT_S210 fourcc_code('S', '2', '1', '0') /* 2x1 subsampled Cb (1) and Cr (2) planes 10 bits per channel */ @@ -414,8 +414,8 @@ extern "C" { * implementation can multiply the values by 2^4=16. For that reason the padding * must only contain zeros. * index 0 = Y plane, [15:0] z:Y [4:12] little endian - * index 1 = Cr plane, [15:0] z:Cr [4:12] little endian - * index 2 = Cb plane, [15:0] z:Cb [4:12] little endian + * index 1 = Cb plane, [15:0] z:Cb [4:12] little endian + * index 2 = Cr plane, [15:0] z:Cr [4:12] little endian */ #define DRM_FORMAT_S012 fourcc_code('S', '0', '1', '2') /* 2x2 subsampled Cb (1) and Cr (2) planes 12 bits per channel */ #define DRM_FORMAT_S212 fourcc_code('S', '2', '1', '2') /* 2x1 subsampled Cb (1) and Cr (2) planes 12 bits per channel */ @@ -424,8 +424,8 @@ extern "C" { /* * 3 plane YCbCr * index 0 = Y plane, [15:0] Y little endian - * index 1 = Cr plane, [15:0] Cr little endian - * index 2 = Cb plane, [15:0] Cb little endian + * index 1 = Cb plane, [15:0] Cb little endian + * index 2 = Cr plane, [15:0] Cr little endian */ #define DRM_FORMAT_S016 fourcc_code('S', '0', '1', '6') /* 2x2 subsampled Cb (1) and Cr (2) planes 16 bits per channel */ #define DRM_FORMAT_S216 fourcc_code('S', '2', '1', '6') /* 2x1 subsampled Cb (1) and Cr (2) planes 16 bits per channel */ diff --git a/include/uapi/linux/dma-buf.h b/include/uapi/linux/dma-buf.h index 5a6fda66d9adf0..e827c9d20c5d35 100644 --- a/include/uapi/linux/dma-buf.h +++ b/include/uapi/linux/dma-buf.h @@ -20,6 +20,7 @@ #ifndef _DMA_BUF_UAPI_H_ #define _DMA_BUF_UAPI_H_ +#include #include /** diff --git a/include/uapi/linux/hyperv.h b/include/uapi/linux/hyperv.h index aaa502a7bff46c..1749b35ab2c21d 100644 --- a/include/uapi/linux/hyperv.h +++ b/include/uapi/linux/hyperv.h @@ -362,7 +362,7 @@ struct hv_kvp_exchg_msg_value { __u8 value[HV_KVP_EXCHANGE_MAX_VALUE_SIZE]; __u32 value_u32; __u64 value_u64; - }; + } __attribute__((packed)); } __attribute__((packed)); struct hv_kvp_msg_enumerate { diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h index dddb781b0507dc..bcc8532986171e 100644 --- a/include/uapi/linux/kvm.h +++ b/include/uapi/linux/kvm.h @@ -11,9 +11,14 @@ #include #include #include +#include #include #include +#ifdef __KERNEL__ +#include +#endif + #define KVM_API_VERSION 12 /* @@ -528,7 +533,7 @@ struct kvm_coalesced_mmio { struct kvm_coalesced_mmio_ring { __u32 first, last; - struct kvm_coalesced_mmio coalesced_mmio[]; + __DECLARE_FLEX_ARRAY(struct kvm_coalesced_mmio, coalesced_mmio); }; #define KVM_COALESCED_MMIO_MAX \ @@ -578,7 +583,7 @@ struct kvm_clear_dirty_log { /* for KVM_SET_SIGNAL_MASK */ struct kvm_signal_mask { __u32 len; - __u8 sigset[]; + __DECLARE_FLEX_ARRAY(__u8, sigset); }; /* for KVM_TPR_ACCESS_REPORTING */ @@ -1036,7 +1041,7 @@ struct kvm_irq_routing_entry { struct kvm_irq_routing { __u32 nr; __u32 flags; - struct kvm_irq_routing_entry entries[]; + __DECLARE_FLEX_ARRAY(struct kvm_irq_routing_entry, entries); }; #define KVM_IRQFD_FLAG_DEASSIGN (1 << 0) @@ -1127,7 +1132,7 @@ struct kvm_dirty_tlb { struct kvm_reg_list { __u64 n; /* number of regs */ - __u64 reg[]; + __DECLARE_FLEX_ARRAY(__u64, reg); }; struct kvm_one_reg { @@ -1579,7 +1584,11 @@ struct kvm_stats_desc { __u16 size; __u32 offset; __u32 bucket_size; - char name[]; +#ifdef __KERNEL__ + char name[KVM_STATS_NAME_SIZE]; +#else + __DECLARE_FLEX_ARRAY(char, name); +#endif }; #define KVM_GET_STATS_FD _IO(KVMIO, 0xce) diff --git a/include/uapi/linux/netfilter/nf_conntrack_common.h b/include/uapi/linux/netfilter/nf_conntrack_common.h index 26071021e986f6..56b6b60a814f5e 100644 --- a/include/uapi/linux/netfilter/nf_conntrack_common.h +++ b/include/uapi/linux/netfilter/nf_conntrack_common.h @@ -159,5 +159,9 @@ enum ip_conntrack_expect_events { #define NF_CT_EXPECT_INACTIVE 0x2 #define NF_CT_EXPECT_USERSPACE 0x4 +#ifdef __KERNEL__ +#define NF_CT_EXPECT_MASK (NF_CT_EXPECT_PERMANENT | NF_CT_EXPECT_INACTIVE | \ + NF_CT_EXPECT_USERSPACE) +#endif #endif /* _UAPI_NF_CONNTRACK_COMMON_H */ diff --git a/include/uapi/linux/netfilter_bridge.h b/include/uapi/linux/netfilter_bridge.h index 1610fdbab98dfc..ad520d3e9df8f7 100644 --- a/include/uapi/linux/netfilter_bridge.h +++ b/include/uapi/linux/netfilter_bridge.h @@ -5,6 +5,10 @@ /* bridge-specific defines for netfilter. */ +#ifndef __KERNEL__ +#include /* for __UAPI_DEF_ETHHDR if defined */ +#endif + #include #include #include diff --git a/include/uapi/linux/nfs.h b/include/uapi/linux/nfs.h index 71c7196d32817d..e629c495353456 100644 --- a/include/uapi/linux/nfs.h +++ b/include/uapi/linux/nfs.h @@ -55,7 +55,7 @@ NFSERR_NODEV = 19, /* v2 v3 v4 */ NFSERR_NOTDIR = 20, /* v2 v3 v4 */ NFSERR_ISDIR = 21, /* v2 v3 v4 */ - NFSERR_INVAL = 22, /* v2 v3 v4 */ + NFSERR_INVAL = 22, /* v3 v4 */ NFSERR_FBIG = 27, /* v2 v3 v4 */ NFSERR_NOSPC = 28, /* v2 v3 v4 */ NFSERR_ROFS = 30, /* v2 v3 v4 */ diff --git a/include/uapi/linux/pci_regs.h b/include/uapi/linux/pci_regs.h index 3add74ae259483..48b0616ddbbbd2 100644 --- a/include/uapi/linux/pci_regs.h +++ b/include/uapi/linux/pci_regs.h @@ -707,7 +707,7 @@ #define PCI_EXP_LNKCTL2_HASD 0x0020 /* HW Autonomous Speed Disable */ #define PCI_EXP_LNKSTA2 0x32 /* Link Status 2 */ #define PCI_EXP_LNKSTA2_FLIT 0x0400 /* Flit Mode Status */ -#define PCI_CAP_EXP_ENDPOINT_SIZEOF_V2 0x32 /* end of v2 EPs w/ link */ +#define PCI_CAP_EXP_ENDPOINT_SIZEOF_V2 0x34 /* end of v2 EPs w/ link */ #define PCI_EXP_SLTCAP2 0x34 /* Slot Capabilities 2 */ #define PCI_EXP_SLTCAP2_IBPD 0x00000001 /* In-band PD Disable Supported */ #define PCI_EXP_SLTCTL2 0x38 /* Slot Control 2 */ diff --git a/include/uapi/linux/prctl.h b/include/uapi/linux/prctl.h index 51c4e8c82b1e98..d58fce1f4ffafb 100644 --- a/include/uapi/linux/prctl.h +++ b/include/uapi/linux/prctl.h @@ -6,6 +6,11 @@ /* Values to pass as first argument to prctl() */ +#define PR_GET_MEM_MODEL 0x6d4d444c +#define PR_SET_MEM_MODEL 0x4d4d444c +# define PR_SET_MEM_MODEL_DEFAULT 0 +# define PR_SET_MEM_MODEL_TSO 1 + #define PR_SET_PDEATHSIG 1 /* Second arg is a signal */ #define PR_GET_PDEATHSIG 2 /* Second arg is a ptr to return the signal */ diff --git a/include/uapi/linux/vbox_vmmdev_types.h b/include/uapi/linux/vbox_vmmdev_types.h index 6073858d52a2ed..11f3627c3729b5 100644 --- a/include/uapi/linux/vbox_vmmdev_types.h +++ b/include/uapi/linux/vbox_vmmdev_types.h @@ -236,7 +236,7 @@ struct vmmdev_hgcm_function_parameter32 { /** Relative to the request header. */ __u32 offset; } page_list; - } u; + } __packed u; } __packed; VMMDEV_ASSERT_SIZE(vmmdev_hgcm_function_parameter32, 4 + 8); @@ -251,7 +251,7 @@ struct vmmdev_hgcm_function_parameter64 { union { __u64 phys_addr; __u64 linear_addr; - } u; + } __packed u; } __packed pointer; struct { /** Size of the buffer described by the page list. */ diff --git a/include/ufs/ufshcd.h b/include/ufs/ufshcd.h index 19154228780b22..8e15c3a70ed413 100644 --- a/include/ufs/ufshcd.h +++ b/include/ufs/ufshcd.h @@ -1342,17 +1342,13 @@ static inline void *ufshcd_get_variant(struct ufs_hba *hba) return hba->priv; } -#ifdef CONFIG_PM extern int ufshcd_runtime_suspend(struct device *dev); extern int ufshcd_runtime_resume(struct device *dev); -#endif -#ifdef CONFIG_PM_SLEEP extern int ufshcd_system_suspend(struct device *dev); extern int ufshcd_system_resume(struct device *dev); extern int ufshcd_system_freeze(struct device *dev); extern int ufshcd_system_thaw(struct device *dev); extern int ufshcd_system_restore(struct device *dev); -#endif extern int ufshcd_dme_reset(struct ufs_hba *hba); extern int ufshcd_dme_enable(struct ufs_hba *hba); diff --git a/include/xen/xen.h b/include/xen/xen.h index 61854e3f283776..f280c5dcf92369 100644 --- a/include/xen/xen.h +++ b/include/xen/xen.h @@ -69,11 +69,13 @@ extern u64 xen_saved_max_mem_size; #endif #ifdef CONFIG_XEN_UNPOPULATED_ALLOC +extern unsigned long xen_unpopulated_pages; int xen_alloc_unpopulated_pages(unsigned int nr_pages, struct page **pages); void xen_free_unpopulated_pages(unsigned int nr_pages, struct page **pages); #include int arch_xen_unpopulated_init(struct resource **res); #else +#define xen_unpopulated_pages 0UL #include static inline int xen_alloc_unpopulated_pages(unsigned int nr_pages, struct page **pages) diff --git a/io_uring/cancel.h b/io_uring/cancel.h index 6783961ede1bfc..1b201a0943030e 100644 --- a/io_uring/cancel.h +++ b/io_uring/cancel.h @@ -6,10 +6,8 @@ struct io_cancel_data { struct io_ring_ctx *ctx; - union { - u64 data; - struct file *file; - }; + u64 data; + struct file *file; u8 opcode; u32 flags; int seq; diff --git a/io_uring/cmd_net.c b/io_uring/cmd_net.c index 19d3ce2bd20ad0..17d499f68fe6db 100644 --- a/io_uring/cmd_net.c +++ b/io_uring/cmd_net.c @@ -145,7 +145,7 @@ static int io_uring_cmd_getsockname(struct socket *sock, return -EINVAL; uaddr = u64_to_user_ptr(READ_ONCE(sqe->addr)); - ulen = u64_to_user_ptr(sqe->addr3); + ulen = u64_to_user_ptr(READ_ONCE(sqe->addr3)); peer = READ_ONCE(sqe->optlen); if (peer > 1) return -EINVAL; @@ -159,16 +159,19 @@ int io_uring_cmd_sock(struct io_uring_cmd *cmd, unsigned int issue_flags) struct proto *prot = READ_ONCE(sk->sk_prot); int ret, arg = 0; - if (!prot || !prot->ioctl) - return -EOPNOTSUPP; - switch (cmd->cmd_op) { case SOCKET_URING_OP_SIOCINQ: + if (!prot || !prot->ioctl) + return -EOPNOTSUPP; + ret = prot->ioctl(sk, SIOCINQ, &arg); if (ret) return ret; return arg; case SOCKET_URING_OP_SIOCOUTQ: + if (!prot || !prot->ioctl) + return -EOPNOTSUPP; + ret = prot->ioctl(sk, SIOCOUTQ, &arg); if (ret) return ret; diff --git a/io_uring/eventfd.c b/io_uring/eventfd.c index 78f8ab7db104f0..ab789e1ebe91e8 100644 --- a/io_uring/eventfd.c +++ b/io_uring/eventfd.c @@ -76,11 +76,15 @@ void io_eventfd_signal(struct io_ring_ctx *ctx, bool cqe_event) { bool skip = false; struct io_ev_fd *ev_fd; - - if (READ_ONCE(ctx->rings->cq_flags) & IORING_CQ_EVENTFD_DISABLED) - return; + struct io_rings *rings; guard(rcu)(); + + rings = rcu_dereference(ctx->rings_rcu); + if (!rings) + return; + if (READ_ONCE(rings->cq_flags) & IORING_CQ_EVENTFD_DISABLED) + return; ev_fd = rcu_dereference(ctx->io_ev_fd); /* * Check again if ev_fd exists in case an io_eventfd_unregister call diff --git a/io_uring/fdinfo.c b/io_uring/fdinfo.c index 80178b69e05a25..c2d3e45544bb4e 100644 --- a/io_uring/fdinfo.c +++ b/io_uring/fdinfo.c @@ -119,12 +119,14 @@ static void __io_uring_show_fdinfo(struct io_ring_ctx *ctx, struct seq_file *m) sq_idx); break; } - if ((++sq_head & sq_mask) == 0) { + if (sq_idx == sq_mask) { seq_printf(m, "%5u: corrupted sqe, wrapping 128B entry\n", sq_idx); break; } + sq_head++; + i++; sqe128 = true; } seq_printf(m, "%5u: opcode:%s, fd:%d, flags:%x, off:%llu, " diff --git a/io_uring/filetable.c b/io_uring/filetable.c index 794ef95df293c5..cb1838c9fc3776 100644 --- a/io_uring/filetable.c +++ b/io_uring/filetable.c @@ -22,6 +22,10 @@ static int io_file_bitmap_get(struct io_ring_ctx *ctx) if (!table->bitmap) return -ENFILE; + if (table->alloc_hint < ctx->file_alloc_start || + table->alloc_hint >= ctx->file_alloc_end) + table->alloc_hint = ctx->file_alloc_start; + do { ret = find_next_zero_bit(table->bitmap, nr, table->alloc_hint); if (ret != nr) diff --git a/io_uring/io-wq.c b/io_uring/io-wq.c index 2fa7d3601edb0c..f42d573b5b1734 100644 --- a/io_uring/io-wq.c +++ b/io_uring/io-wq.c @@ -34,6 +34,7 @@ enum { enum { IO_WQ_BIT_EXIT = 0, /* wq exiting */ + IO_WQ_BIT_EXIT_ON_IDLE = 1, /* allow all workers to exit on idle */ }; enum { @@ -706,9 +707,13 @@ static int io_wq_worker(void *data) raw_spin_lock(&acct->workers_lock); /* * Last sleep timed out. Exit if we're not the last worker, - * or if someone modified our affinity. + * or if someone modified our affinity. If wq is marked + * idle-exit, drop the worker as well. This is used to avoid + * keeping io-wq workers around for tasks that no longer have + * any active io_uring instances. */ - if (last_timeout && (exit_mask || acct->nr_workers > 1)) { + if ((last_timeout && (exit_mask || acct->nr_workers > 1)) || + test_bit(IO_WQ_BIT_EXIT_ON_IDLE, &wq->state)) { acct->nr_workers--; raw_spin_unlock(&acct->workers_lock); __set_current_state(TASK_RUNNING); @@ -963,6 +968,24 @@ static bool io_wq_worker_wake(struct io_worker *worker, void *data) return false; } +void io_wq_set_exit_on_idle(struct io_wq *wq, bool enable) +{ + if (!wq->task) + return; + + if (!enable) { + clear_bit(IO_WQ_BIT_EXIT_ON_IDLE, &wq->state); + return; + } + + if (test_and_set_bit(IO_WQ_BIT_EXIT_ON_IDLE, &wq->state)) + return; + + rcu_read_lock(); + io_wq_for_each_worker(wq, io_wq_worker_wake, NULL); + rcu_read_unlock(); +} + static void io_run_cancel(struct io_wq_work *work, struct io_wq *wq) { do { diff --git a/io_uring/io-wq.h b/io_uring/io-wq.h index 774abab54732ef..94b14742b70328 100644 --- a/io_uring/io-wq.h +++ b/io_uring/io-wq.h @@ -41,6 +41,7 @@ struct io_wq_data { struct io_wq *io_wq_create(unsigned bounded, struct io_wq_data *data); void io_wq_exit_start(struct io_wq *wq); void io_wq_put_and_exit(struct io_wq *wq); +void io_wq_set_exit_on_idle(struct io_wq *wq, bool enable); void io_wq_enqueue(struct io_wq *wq, struct io_wq_work *work); void io_wq_hash_work(struct io_wq_work *work, void *val); diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index b7a077c11c21a9..84fb1f7b0d818a 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -138,7 +138,7 @@ static void io_queue_sqe(struct io_kiocb *req, unsigned int extra_flags); static void __io_req_caches_free(struct io_ring_ctx *ctx); -static __read_mostly DEFINE_STATIC_KEY_FALSE(io_key_has_sqarray); +static __read_mostly DEFINE_STATIC_KEY_DEFERRED_FALSE(io_key_has_sqarray, HZ); struct kmem_cache *req_cachep; static struct workqueue_struct *iou_wq __ro_after_init; @@ -189,12 +189,15 @@ static void io_poison_req(struct io_kiocb *req) static inline unsigned int __io_cqring_events(struct io_ring_ctx *ctx) { - return ctx->cached_cq_tail - READ_ONCE(ctx->rings->cq.head); + struct io_rings *rings = io_get_rings(ctx); + return ctx->cached_cq_tail - READ_ONCE(rings->cq.head); } static inline unsigned int __io_cqring_events_user(struct io_ring_ctx *ctx) { - return READ_ONCE(ctx->rings->cq.tail) - READ_ONCE(ctx->rings->cq.head); + struct io_rings *rings = io_get_rings(ctx); + + return READ_ONCE(rings->cq.tail) - READ_ONCE(rings->cq.head); } static inline void req_fail_link_node(struct io_kiocb *req, int res) @@ -1202,6 +1205,21 @@ void tctx_task_work(struct callback_head *cb) WARN_ON_ONCE(ret); } +/* + * Sets IORING_SQ_TASKRUN in the sq_flags shared with userspace, using the + * RCU protected rings pointer to be safe against concurrent ring resizing. + */ +static void io_ctx_mark_taskrun(struct io_ring_ctx *ctx) +{ + lockdep_assert_in_rcu_read_lock(); + + if (ctx->flags & IORING_SETUP_TASKRUN_FLAG) { + struct io_rings *rings = rcu_dereference(ctx->rings_rcu); + + atomic_or(IORING_SQ_TASKRUN, &rings->sq_flags); + } +} + static void io_req_local_work_add(struct io_kiocb *req, unsigned flags) { struct io_ring_ctx *ctx = req->ctx; @@ -1256,8 +1274,7 @@ static void io_req_local_work_add(struct io_kiocb *req, unsigned flags) */ if (!head) { - if (ctx->flags & IORING_SETUP_TASKRUN_FLAG) - atomic_or(IORING_SQ_TASKRUN, &ctx->rings->sq_flags); + io_ctx_mark_taskrun(ctx); if (ctx->has_evfd) io_eventfd_signal(ctx, false); } @@ -1281,6 +1298,10 @@ static void io_req_normal_work_add(struct io_kiocb *req) if (!llist_add(&req->io_task_work.node, &tctx->task_list)) return; + /* + * Doesn't need to use ->rings_rcu, as resizing isn't supported for + * !DEFER_TASKRUN. + */ if (ctx->flags & IORING_SETUP_TASKRUN_FLAG) atomic_or(IORING_SQ_TASKRUN, &ctx->rings->sq_flags); @@ -2152,7 +2173,7 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req, * well as 2 contiguous entries. */ if (!(ctx->flags & IORING_SETUP_SQE_MIXED) || *left < 2 || - !(ctx->cached_sq_head & (ctx->sq_entries - 1))) + (unsigned)(sqe - ctx->sq_sqes) >= ctx->sq_entries - 1) return io_init_fail_req(req, -EINVAL); /* * A 128b operation on a mixed SQ uses two entries, so we have @@ -2375,7 +2396,7 @@ static bool io_get_sqe(struct io_ring_ctx *ctx, const struct io_uring_sqe **sqe) unsigned mask = ctx->sq_entries - 1; unsigned head = ctx->cached_sq_head++ & mask; - if (static_branch_unlikely(&io_key_has_sqarray) && + if (static_branch_unlikely(&io_key_has_sqarray.key) && (!(ctx->flags & IORING_SETUP_NO_SQARRAY))) { head = READ_ONCE(ctx->sq_array[head]); if (unlikely(head >= ctx->sq_entries)) { @@ -2518,12 +2539,15 @@ static enum hrtimer_restart io_cqring_min_timer_wakeup(struct hrtimer *timer) if (io_has_work(ctx)) goto out_wake; /* got events since we started waiting, min timeout is done */ - if (iowq->cq_min_tail != READ_ONCE(ctx->rings->cq.tail)) - goto out_wake; - /* if we have any events and min timeout expired, we're done */ - if (io_cqring_events(ctx)) - goto out_wake; + scoped_guard(rcu) { + struct io_rings *rings = io_get_rings(ctx); + if (iowq->cq_min_tail != READ_ONCE(rings->cq.tail)) + goto out_wake; + /* if we have any events and min timeout expired, we're done */ + if (io_cqring_events(ctx)) + goto out_wake; + } /* * If using deferred task_work running and application is waiting on * more than one request, ensure we reset it now where we are switching @@ -2634,9 +2658,9 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, u32 flags, struct ext_arg *ext_arg) { struct io_wait_queue iowq; - struct io_rings *rings = ctx->rings; + struct io_rings *rings; ktime_t start_time; - int ret; + int ret, nr_wait; min_events = min_t(int, min_events, ctx->cq_entries); @@ -2649,15 +2673,23 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, u32 flags, if (unlikely(test_bit(IO_CHECK_CQ_OVERFLOW_BIT, &ctx->check_cq))) io_cqring_do_overflow_flush(ctx); - if (__io_cqring_events_user(ctx) >= min_events) + + rcu_read_lock(); + rings = io_get_rings(ctx); + if (__io_cqring_events_user(ctx) >= min_events) { + rcu_read_unlock(); return 0; + } init_waitqueue_func_entry(&iowq.wq, io_wake_function); iowq.wq.private = current; INIT_LIST_HEAD(&iowq.wq.entry); iowq.ctx = ctx; - iowq.cq_tail = READ_ONCE(ctx->rings->cq.head) + min_events; - iowq.cq_min_tail = READ_ONCE(ctx->rings->cq.tail); + iowq.cq_tail = READ_ONCE(rings->cq.head) + min_events; + iowq.cq_min_tail = READ_ONCE(rings->cq.tail); + nr_wait = (int) iowq.cq_tail - READ_ONCE(rings->cq.tail); + rcu_read_unlock(); + rings = NULL; iowq.nr_timeouts = atomic_read(&ctx->cq_timeouts); iowq.hit_timeout = 0; iowq.min_timeout = ext_arg->min_time; @@ -2688,14 +2720,6 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, u32 flags, trace_io_uring_cqring_wait(ctx, min_events); do { unsigned long check_cq; - int nr_wait; - - /* if min timeout has been hit, don't reset wait count */ - if (!iowq.hit_timeout) - nr_wait = (int) iowq.cq_tail - - READ_ONCE(ctx->rings->cq.tail); - else - nr_wait = 1; if (ctx->flags & IORING_SETUP_DEFER_TASKRUN) { atomic_set(&ctx->cq_wait_nr, nr_wait); @@ -2746,13 +2770,22 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, u32 flags, break; } cond_resched(); + + /* if min timeout has been hit, don't reset wait count */ + if (!iowq.hit_timeout) + scoped_guard(rcu) + nr_wait = (int) iowq.cq_tail - + READ_ONCE(io_get_rings(ctx)->cq.tail); + else + nr_wait = 1; } while (1); if (!(ctx->flags & IORING_SETUP_DEFER_TASKRUN)) finish_wait(&ctx->cq_wait, &iowq.wq); restore_saved_sigmask_unless(ret == -EINTR); - return READ_ONCE(rings->cq.head) == READ_ONCE(rings->cq.tail) ? ret : 0; + guard(rcu)(); + return READ_ONCE(io_get_rings(ctx)->cq.head) == READ_ONCE(io_get_rings(ctx)->cq.tail) ? ret : 0; } static void io_rings_free(struct io_ring_ctx *ctx) @@ -2760,6 +2793,7 @@ static void io_rings_free(struct io_ring_ctx *ctx) io_free_region(ctx->user, &ctx->sq_region); io_free_region(ctx->user, &ctx->ring_region); ctx->rings = NULL; + RCU_INIT_POINTER(ctx->rings_rcu, NULL); ctx->sq_sqes = NULL; } @@ -2867,7 +2901,7 @@ static __cold void io_ring_ctx_free(struct io_ring_ctx *ctx) io_rings_free(ctx); if (!(ctx->flags & IORING_SETUP_NO_SQARRAY)) - static_branch_dec(&io_key_has_sqarray); + static_branch_slow_dec_deferred(&io_key_has_sqarray); percpu_ref_exit(&ctx->refs); free_uid(ctx->user); @@ -2935,7 +2969,9 @@ static __poll_t io_uring_poll(struct file *file, poll_table *wait) */ poll_wait(file, &ctx->poll_wq, wait); - if (!io_sqring_full(ctx)) + rcu_read_lock(); + + if (!__io_sqring_full(ctx)) mask |= EPOLLOUT | EPOLLWRNORM; /* @@ -2955,6 +2991,7 @@ static __poll_t io_uring_poll(struct file *file, poll_table *wait) if (__io_cqring_events_user(ctx) || io_has_work(ctx)) mask |= EPOLLIN | EPOLLRDNORM; + rcu_read_unlock(); return mask; } @@ -3256,7 +3293,11 @@ SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit, ctx = file->private_data; ret = -EBADFD; - if (unlikely(ctx->flags & IORING_SETUP_R_DISABLED)) + /* + * Keep IORING_SETUP_R_DISABLED check before submitter_task load + * in io_uring_add_tctx_node() -> __io_uring_add_tctx_node_from_submit() + */ + if (unlikely(smp_load_acquire(&ctx->flags) & IORING_SETUP_R_DISABLED)) goto out; /* @@ -3385,6 +3426,7 @@ static __cold int io_allocate_scq_urings(struct io_ring_ctx *ctx, if (ret) return ret; ctx->rings = rings = io_region_get_ptr(&ctx->ring_region); + rcu_assign_pointer(ctx->rings_rcu, rings); if (!(ctx->flags & IORING_SETUP_NO_SQARRAY)) ctx->sq_array = (u32 *)((char *)rings + rl->sq_array_offset); @@ -3596,7 +3638,7 @@ static __cold int io_uring_create(struct io_ctx_config *config) ctx->clock_offset = 0; if (!(ctx->flags & IORING_SETUP_NO_SQARRAY)) - static_branch_inc(&io_key_has_sqarray); + static_branch_deferred_inc(&io_key_has_sqarray); if ((ctx->flags & IORING_SETUP_DEFER_TASKRUN) && !(ctx->flags & IORING_SETUP_IOPOLL) && diff --git a/io_uring/io_uring.h b/io_uring/io_uring.h index a790c16854d32d..6ee49991cec8b2 100644 --- a/io_uring/io_uring.h +++ b/io_uring/io_uring.h @@ -132,16 +132,28 @@ struct io_wait_queue { #endif }; +static inline struct io_rings *io_get_rings(struct io_ring_ctx *ctx) +{ + return rcu_dereference_check(ctx->rings_rcu, + lockdep_is_held(&ctx->uring_lock) || + lockdep_is_held(&ctx->completion_lock)); +} + static inline bool io_should_wake(struct io_wait_queue *iowq) { struct io_ring_ctx *ctx = iowq->ctx; - int dist = READ_ONCE(ctx->rings->cq.tail) - (int) iowq->cq_tail; + struct io_rings *rings; + int dist; + + guard(rcu)(); + rings = io_get_rings(ctx); /* * Wake up if we have enough events, or if a timeout occurred since we * started waiting. For timeouts, we always want to return to userspace, * regardless of event count. */ + dist = READ_ONCE(rings->cq.tail) - (int) iowq->cq_tail; return dist >= 0 || atomic_read(&ctx->cq_timeouts) != iowq->nr_timeouts; } @@ -432,9 +444,9 @@ static inline void io_cqring_wake(struct io_ring_ctx *ctx) __io_wq_wake(&ctx->cq_wait); } -static inline bool io_sqring_full(struct io_ring_ctx *ctx) +static inline bool __io_sqring_full(struct io_ring_ctx *ctx) { - struct io_rings *r = ctx->rings; + struct io_rings *r = io_get_rings(ctx); /* * SQPOLL must use the actual sqring head, as using the cached_sq_head @@ -446,9 +458,15 @@ static inline bool io_sqring_full(struct io_ring_ctx *ctx) return READ_ONCE(r->sq.tail) - READ_ONCE(r->sq.head) == ctx->sq_entries; } -static inline unsigned int io_sqring_entries(struct io_ring_ctx *ctx) +static inline bool io_sqring_full(struct io_ring_ctx *ctx) { - struct io_rings *rings = ctx->rings; + guard(rcu)(); + return __io_sqring_full(ctx); +} + +static inline unsigned int __io_sqring_entries(struct io_ring_ctx *ctx) +{ + struct io_rings *rings = io_get_rings(ctx); unsigned int entries; /* make sure SQ entry isn't read before tail */ @@ -509,6 +527,12 @@ static inline void io_tw_lock(struct io_ring_ctx *ctx, io_tw_token_t tw) lockdep_assert_held(&ctx->uring_lock); } +static inline unsigned int io_sqring_entries(struct io_ring_ctx *ctx) +{ + guard(rcu)(); + return __io_sqring_entries(ctx); +} + /* * Don't complete immediately but use deferred completion infrastructure. * Protected by ->uring_lock and can only be used either with @@ -595,6 +619,12 @@ static inline bool io_file_can_poll(struct io_kiocb *req) return false; } +static inline bool io_is_uring_cmd(const struct io_kiocb *req) +{ + return req->opcode == IORING_OP_URING_CMD || + req->opcode == IORING_OP_URING_CMD128; +} + static inline ktime_t io_get_time(struct io_ring_ctx *ctx) { if (ctx->clockid == CLOCK_MONOTONIC) diff --git a/io_uring/kbuf.c b/io_uring/kbuf.c index 796d131107ddb9..f72f38d22d2b38 100644 --- a/io_uring/kbuf.c +++ b/io_uring/kbuf.c @@ -34,6 +34,10 @@ struct io_provide_buf { static bool io_kbuf_inc_commit(struct io_buffer_list *bl, int len) { + /* No data consumed, return false early to avoid consuming the buffer */ + if (!len) + return false; + while (len) { struct io_uring_buf *buf; u32 buf_len, this_len; @@ -111,9 +115,18 @@ bool io_kbuf_recycle_legacy(struct io_kiocb *req, unsigned issue_flags) buf = req->kbuf; bl = io_buffer_get_list(ctx, buf->bgid); - list_add(&buf->list, &bl->buf_list); - bl->nbufs++; + /* + * If the buffer list was upgraded to a ring-based one, or removed, + * while the request was in-flight in io-wq, drop it. + */ + if (bl && !(bl->flags & IOBL_BUF_RING)) { + list_add(&buf->list, &bl->buf_list); + bl->nbufs++; + } else { + kfree(buf); + } req->flags &= ~REQ_F_BUFFER_SELECTED; + req->kbuf = NULL; io_ring_submit_unlock(ctx, issue_flags); return true; @@ -171,7 +184,7 @@ static bool io_should_commit(struct io_kiocb *req, unsigned int issue_flags) return true; /* uring_cmd commits kbuf upfront, no need to auto-commit */ - if (!io_file_can_poll(req) && req->opcode != IORING_OP_URING_CMD) + if (!io_file_can_poll(req) && !io_is_uring_cmd(req)) return true; return false; } @@ -203,7 +216,8 @@ static struct io_br_sel io_ring_buffer_select(struct io_kiocb *req, size_t *len, sel.addr = u64_to_user_ptr(READ_ONCE(buf->addr)); if (io_should_commit(req, issue_flags)) { - io_kbuf_commit(req, sel.buf_list, *len, 1); + if (!io_kbuf_commit(req, sel.buf_list, *len, 1)) + req->flags |= REQ_F_BUF_MORE; sel.buf_list = NULL; } return sel; @@ -336,7 +350,8 @@ int io_buffers_select(struct io_kiocb *req, struct buf_sel_arg *arg, */ if (ret > 0) { req->flags |= REQ_F_BUFFERS_COMMIT | REQ_F_BL_NO_RECYCLE; - io_kbuf_commit(req, sel->buf_list, arg->out_len, ret); + if (!io_kbuf_commit(req, sel->buf_list, arg->out_len, ret)) + req->flags |= REQ_F_BUF_MORE; } } else { ret = io_provided_buffers_select(req, &arg->out_len, sel->buf_list, arg->iovs); @@ -382,8 +397,10 @@ static inline bool __io_put_kbuf_ring(struct io_kiocb *req, if (bl) ret = io_kbuf_commit(req, bl, len, nr); + if (ret && (req->flags & REQ_F_BUF_MORE)) + ret = false; - req->flags &= ~REQ_F_BUFFER_RING; + req->flags &= ~(REQ_F_BUFFER_RING | REQ_F_BUF_MORE); return ret; } @@ -669,8 +686,9 @@ int io_register_pbuf_ring(struct io_ring_ctx *ctx, void __user *arg) bl->buf_ring = br; if (reg.flags & IOU_PBUF_RING_INC) bl->flags |= IOBL_INC; - io_buffer_add_list(ctx, bl, reg.bgid); - return 0; + ret = io_buffer_add_list(ctx, bl, reg.bgid); + if (!ret) + return 0; fail: io_free_region(ctx->user, &bl->region); kfree(bl); diff --git a/io_uring/msg_ring.c b/io_uring/msg_ring.c index 7063ea7964e754..87b4d306cf1b6a 100644 --- a/io_uring/msg_ring.c +++ b/io_uring/msg_ring.c @@ -125,7 +125,11 @@ static int __io_msg_ring_data(struct io_ring_ctx *target_ctx, return -EINVAL; if (!(msg->flags & IORING_MSG_RING_FLAGS_PASS) && msg->dst_fd) return -EINVAL; - if (target_ctx->flags & IORING_SETUP_R_DISABLED) + /* + * Keep IORING_SETUP_R_DISABLED check before submitter_task load + * in io_msg_data_remote() -> io_msg_remote_post() + */ + if (smp_load_acquire(&target_ctx->flags) & IORING_SETUP_R_DISABLED) return -EBADFD; if (io_msg_need_remote(target_ctx)) @@ -245,7 +249,11 @@ static int io_msg_send_fd(struct io_kiocb *req, unsigned int issue_flags) return -EINVAL; if (target_ctx == ctx) return -EINVAL; - if (target_ctx->flags & IORING_SETUP_R_DISABLED) + /* + * Keep IORING_SETUP_R_DISABLED check before submitter_task load + * in io_msg_fd_remote() + */ + if (smp_load_acquire(&target_ctx->flags) & IORING_SETUP_R_DISABLED) return -EBADFD; if (!msg->src_file) { int ret = io_msg_grab_file(req, issue_flags); diff --git a/io_uring/net.c b/io_uring/net.c index 519ea055b76197..014ebdae2c0dd6 100644 --- a/io_uring/net.c +++ b/io_uring/net.c @@ -375,6 +375,8 @@ static int io_send_setup(struct io_kiocb *req, const struct io_uring_sqe *sqe) kmsg->msg.msg_namelen = addr_len; } if (sr->flags & IORING_RECVSEND_FIXED_BUF) { + if (sr->flags & IORING_SEND_VECTORIZED) + return -EINVAL; req->flags |= REQ_F_IMPORT_BUFFER; return 0; } @@ -419,6 +421,8 @@ int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) sr->done_io = 0; sr->len = READ_ONCE(sqe->len); + if (unlikely(sr->len < 0)) + return -EINVAL; sr->flags = READ_ONCE(sqe->ioprio); if (sr->flags & ~SENDMSG_FLAGS) return -EINVAL; @@ -515,7 +519,11 @@ static inline bool io_send_finish(struct io_kiocb *req, cflags = io_put_kbufs(req, sel->val, sel->buf_list, io_bundle_nbufs(kmsg, sel->val)); - if (bundle_finished || req->flags & REQ_F_BL_EMPTY) + /* + * Don't start new bundles if the buffer list is empty, or if the + * current operation needed to go through polling to complete. + */ + if (bundle_finished || req->flags & (REQ_F_BL_EMPTY | REQ_F_POLLED)) goto finish; /* @@ -785,6 +793,8 @@ int io_recvmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr)); sr->len = READ_ONCE(sqe->len); + if (unlikely(sr->len < 0)) + return -EINVAL; sr->flags = READ_ONCE(sqe->ioprio); if (sr->flags & ~RECVMSG_FLAGS) return -EINVAL; diff --git a/io_uring/openclose.c b/io_uring/openclose.c index 15dde9bd6ff670..606ce0664e6a45 100644 --- a/io_uring/openclose.c +++ b/io_uring/openclose.c @@ -336,31 +336,34 @@ static int io_pipe_fixed(struct io_kiocb *req, struct file **files, { struct io_pipe *p = io_kiocb_to_cmd(req, struct io_pipe); struct io_ring_ctx *ctx = req->ctx; + bool alloc_slot; int ret, fds[2] = { -1, -1 }; int slot = p->file_slot; if (p->flags & O_CLOEXEC) return -EINVAL; + alloc_slot = slot == IORING_FILE_INDEX_ALLOC; + io_ring_submit_lock(ctx, issue_flags); ret = __io_fixed_fd_install(ctx, files[0], slot); if (ret < 0) goto err; - fds[0] = ret; + fds[0] = alloc_slot ? ret : slot - 1; files[0] = NULL; /* * If a specific slot is given, next one will be used for * the write side. */ - if (slot != IORING_FILE_INDEX_ALLOC) + if (!alloc_slot) slot++; ret = __io_fixed_fd_install(ctx, files[1], slot); if (ret < 0) goto err; - fds[1] = ret; + fds[1] = alloc_slot ? ret : slot - 1; files[1] = NULL; io_ring_submit_unlock(ctx, issue_flags); diff --git a/io_uring/poll.c b/io_uring/poll.c index aac4b3b881fb71..488c08593b6444 100644 --- a/io_uring/poll.c +++ b/io_uring/poll.c @@ -272,6 +272,7 @@ static int io_poll_check_events(struct io_kiocb *req, io_tw_token_t tw) atomic_andnot(IO_POLL_RETRY_FLAG, &req->poll_refs); v &= ~IO_POLL_RETRY_FLAG; } + v &= IO_POLL_REF_MASK; } /* the mask was stashed in __io_poll_execute */ @@ -304,8 +305,13 @@ static int io_poll_check_events(struct io_kiocb *req, io_tw_token_t tw) return IOU_POLL_REMOVE_POLL_USE_RES; } } else { - int ret = io_poll_issue(req, tw); + int ret; + /* multiple refs and HUP, ensure we loop once more */ + if ((req->cqe.res & (POLLHUP | POLLRDHUP)) && v != 1) + v--; + + ret = io_poll_issue(req, tw); if (ret == IOU_COMPLETE) return IOU_POLL_REMOVE_POLL_USE_RES; else if (ret == IOU_REQUEUE) @@ -321,7 +327,6 @@ static int io_poll_check_events(struct io_kiocb *req, io_tw_token_t tw) * Release all references, retry if someone tried to restart * task_work while we were executing it. */ - v &= IO_POLL_REF_MASK; } while (atomic_sub_return(v, &req->poll_refs) & IO_POLL_REF_MASK); io_napi_add(req); diff --git a/io_uring/register.c b/io_uring/register.c index 3d3822ff3fd9e1..ff9d75ffbd15c0 100644 --- a/io_uring/register.c +++ b/io_uring/register.c @@ -193,7 +193,8 @@ static int io_register_enable_rings(struct io_ring_ctx *ctx) if (ctx->restrictions.registered) ctx->restricted = 1; - ctx->flags &= ~IORING_SETUP_R_DISABLED; + /* Keep submitter_task store before clearing IORING_SETUP_R_DISABLED */ + smp_store_release(&ctx->flags, ctx->flags & ~IORING_SETUP_R_DISABLED); if (ctx->sq_data && wq_has_sleeper(&ctx->sq_data->wait)) wake_up(&ctx->sq_data->wait); return 0; @@ -544,7 +545,15 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg) ctx->sq_entries = p->sq_entries; ctx->cq_entries = p->cq_entries; + /* + * Just mark any flag we may have missed and that the application + * should act on unconditionally. Worst case it'll be an extra + * syscall. + */ + atomic_or(IORING_SQ_TASKRUN | IORING_SQ_NEED_WAKEUP, &n.rings->sq_flags); ctx->rings = n.rings; + rcu_assign_pointer(ctx->rings_rcu, n.rings); + ctx->sq_sqes = n.sq_sqes; swap_old(ctx, o, n, ring_region); swap_old(ctx, o, n, sq_region); @@ -553,6 +562,10 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg) out: spin_unlock(&ctx->completion_lock); mutex_unlock(&ctx->mmap_lock); + + /* Wait for concurrent io_ctx_mark_taskrun() */ + if (to_free == &o) + synchronize_rcu_expedited(); io_register_free_rings(ctx, to_free); if (ctx->sq_data) diff --git a/io_uring/rsrc.c b/io_uring/rsrc.c index 41c89f5c616da2..81446f9649ae91 100644 --- a/io_uring/rsrc.c +++ b/io_uring/rsrc.c @@ -1084,6 +1084,10 @@ static int io_import_fixed(int ddir, struct iov_iter *iter, return ret; if (!(imu->dir & (1 << ddir))) return -EFAULT; + if (unlikely(!len)) { + iov_iter_bvec(iter, ddir, NULL, 0, 0); + return 0; + } offset = buf_addr - imu->ubuf; diff --git a/io_uring/rw.c b/io_uring/rw.c index 28555bc85ba0f9..01367ac09531a9 100644 --- a/io_uring/rw.c +++ b/io_uring/rw.c @@ -1253,7 +1253,7 @@ static int io_uring_classic_poll(struct io_kiocb *req, struct io_comp_batch *iob { struct file *file = req->file; - if (req->opcode == IORING_OP_URING_CMD) { + if (io_is_uring_cmd(req)) { struct io_uring_cmd *ioucmd; ioucmd = io_kiocb_to_cmd(req, struct io_uring_cmd); @@ -1376,7 +1376,7 @@ int io_do_iopoll(struct io_ring_ctx *ctx, bool force_nonspin) break; nr_events++; req->cqe.flags = io_put_kbuf(req, req->cqe.res, NULL); - if (req->opcode != IORING_OP_URING_CMD) + if (!io_is_uring_cmd(req)) io_req_rw_cleanup(req, 0); } if (unlikely(!nr_events)) diff --git a/io_uring/sync.c b/io_uring/sync.c index cea2d381ffd2a4..ab7fa1cd7dd639 100644 --- a/io_uring/sync.c +++ b/io_uring/sync.c @@ -62,6 +62,8 @@ int io_fsync_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) return -EINVAL; sync->off = READ_ONCE(sqe->off); + if (sync->off < 0) + return -EINVAL; sync->len = READ_ONCE(sqe->len); req->flags |= REQ_F_FORCE_ASYNC; return 0; diff --git a/io_uring/tctx.c b/io_uring/tctx.c index 6d6f44215ec807..91f4b830b77b9e 100644 --- a/io_uring/tctx.c +++ b/io_uring/tctx.c @@ -122,6 +122,14 @@ int __io_uring_add_tctx_node(struct io_ring_ctx *ctx) return ret; } } + + /* + * Re-activate io-wq keepalive on any new io_uring usage. The wq may have + * been marked for idle-exit when the task temporarily had no active + * io_uring instances. + */ + if (tctx->io_wq) + io_wq_set_exit_on_idle(tctx->io_wq, false); if (!xa_load(&tctx->xa, (unsigned long)ctx)) { node = kmalloc(sizeof(*node), GFP_KERNEL); if (!node) @@ -183,6 +191,9 @@ __cold void io_uring_del_tctx_node(unsigned long index) if (tctx->last == node->ctx) tctx->last = NULL; kfree(node); + + if (xa_empty(&tctx->xa) && tctx->io_wq) + io_wq_set_exit_on_idle(tctx->io_wq, true); } __cold void io_uring_clean_tctx(struct io_uring_task *tctx) diff --git a/io_uring/timeout.c b/io_uring/timeout.c index d8fbbaf31cf35d..84dda24f3eb249 100644 --- a/io_uring/timeout.c +++ b/io_uring/timeout.c @@ -130,7 +130,7 @@ __cold void io_flush_timeouts(struct io_ring_ctx *ctx) u32 seq; raw_spin_lock_irq(&ctx->timeout_lock); - seq = ctx->cached_cq_tail - atomic_read(&ctx->cq_timeouts); + seq = READ_ONCE(ctx->cached_cq_tail) - atomic_read(&ctx->cq_timeouts); list_for_each_entry_safe(timeout, tmp, &ctx->timeout_list, list) { struct io_kiocb *req = cmd_to_io_kiocb(timeout); diff --git a/io_uring/zcrx.c b/io_uring/zcrx.c index 3d398283cf3405..262ac732cf9669 100644 --- a/io_uring/zcrx.c +++ b/io_uring/zcrx.c @@ -288,6 +288,9 @@ static int io_zcrx_map_area(struct io_zcrx_ifq *ifq, struct io_zcrx_area *area) } ret = io_populate_area_dma(ifq, area); + if (ret && !area->mem.is_dmabuf) + dma_unmap_sgtable(ifq->dev, &area->mem.page_sg_table, + DMA_FROM_DEVICE, IO_DMA_ATTR); if (ret == 0) area->is_mapped = true; return ret; @@ -334,10 +337,14 @@ static inline atomic_t *io_get_user_counter(struct net_iov *niov) static bool io_zcrx_put_niov_uref(struct net_iov *niov) { atomic_t *uref = io_get_user_counter(niov); + int old; + + old = atomic_read(uref); + do { + if (unlikely(old == 0)) + return false; + } while (!atomic_try_cmpxchg(uref, &old, old - 1)); - if (unlikely(!atomic_read(uref))) - return false; - atomic_dec(uref); return true; } @@ -512,9 +519,6 @@ static void io_close_queue(struct io_zcrx_ifq *ifq) .mp_priv = ifq, }; - if (ifq->if_rxq == -1) - return; - scoped_guard(mutex, &ifq->pp_lock) { netdev = ifq->netdev; netdev_tracker = ifq->netdev_tracker; @@ -522,7 +526,8 @@ static void io_close_queue(struct io_zcrx_ifq *ifq) } if (netdev) { - net_mp_close_rxq(netdev, ifq->if_rxq, &p); + if (ifq->if_rxq != -1) + net_mp_close_rxq(netdev, ifq->if_rxq, &p); netdev_put(netdev, &netdev_tracker); } ifq->if_rxq = -1; @@ -676,6 +681,8 @@ static int import_zcrx(struct io_ring_ctx *ctx, return -EINVAL; if (reg->if_rxq || reg->rq_entries || reg->area_ptr || reg->region_ptr) return -EINVAL; + if (reg->flags & ~ZCRX_REG_IMPORT) + return -EINVAL; fd = reg->if_idx; CLASS(fd, f)(fd); @@ -830,13 +837,12 @@ int io_register_zcrx_ifq(struct io_ring_ctx *ctx, } return 0; netdev_put_unlock: - netdev_put(ifq->netdev, &ifq->netdev_tracker); netdev_unlock(ifq->netdev); err: scoped_guard(mutex, &ctx->mmap_lock) xa_erase(&ctx->zcrx_ctxs, id); ifq_free: - io_zcrx_ifq_free(ifq); + zcrx_unregister(ifq); return ret; } @@ -892,11 +898,12 @@ static inline bool io_parse_rqe(struct io_uring_zcrx_rqe *rqe, struct io_zcrx_ifq *ifq, struct net_iov **ret_niov) { + __u64 off = READ_ONCE(rqe->off); unsigned niov_idx, area_idx; struct io_zcrx_area *area; - area_idx = rqe->off >> IORING_ZCRX_AREA_SHIFT; - niov_idx = (rqe->off & ~IORING_ZCRX_AREA_MASK) >> ifq->niov_shift; + area_idx = off >> IORING_ZCRX_AREA_SHIFT; + niov_idx = (off & ~IORING_ZCRX_AREA_MASK) >> ifq->niov_shift; if (unlikely(rqe->__pad || area_idx)) return false; diff --git a/ipc/ipc_sysctl.c b/ipc/ipc_sysctl.c index 15b17e86e198cf..9b087ebeb643b7 100644 --- a/ipc/ipc_sysctl.c +++ b/ipc/ipc_sysctl.c @@ -214,7 +214,7 @@ static int ipc_permissions(struct ctl_table_header *head, const struct ctl_table if (((table->data == &ns->ids[IPC_SEM_IDS].next_id) || (table->data == &ns->ids[IPC_MSG_IDS].next_id) || (table->data == &ns->ids[IPC_SHM_IDS].next_id)) && - checkpoint_restore_ns_capable(ns->user_ns)) + checkpoint_restore_ns_capable_noaudit(ns->user_ns)) mode = 0666; else #endif diff --git a/kernel/bpf/bpf_insn_array.c b/kernel/bpf/bpf_insn_array.c index c96630cb75bf7a..37b43102953eeb 100644 --- a/kernel/bpf/bpf_insn_array.c +++ b/kernel/bpf/bpf_insn_array.c @@ -126,7 +126,7 @@ static int insn_array_map_direct_value_addr(const struct bpf_map *map, u64 *imm, return -EINVAL; /* from BPF's point of view, this map is a jump table */ - *imm = (unsigned long)insn_array->ips + off; + *imm = (unsigned long)insn_array->ips; return 0; } diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 0de8fc8a0e0b32..75a5df36f91700 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -1676,7 +1676,16 @@ static void btf_free_id(struct btf *btf) * of the _bh() version. */ spin_lock_irqsave(&btf_idr_lock, flags); - idr_remove(&btf_idr, btf->id); + if (btf->id) { + idr_remove(&btf_idr, btf->id); + /* + * Clear the id here to make this function idempotent, since it will get + * called a couple of times for module BTFs: on module unload, and then + * the final btf_put(). btf_alloc_id() starts IDs with 1, so we can use + * 0 as sentinel value. + */ + WRITE_ONCE(btf->id, 0); + } spin_unlock_irqrestore(&btf_idr_lock, flags); } @@ -7995,7 +8004,7 @@ static void bpf_btf_show_fdinfo(struct seq_file *m, struct file *filp) { const struct btf *btf = filp->private_data; - seq_printf(m, "btf_id:\t%u\n", btf->id); + seq_printf(m, "btf_id:\t%u\n", READ_ONCE(btf->id)); } #endif @@ -8077,7 +8086,7 @@ int btf_get_info_by_fd(const struct btf *btf, if (copy_from_user(&info, uinfo, info_copy)) return -EFAULT; - info.id = btf->id; + info.id = READ_ONCE(btf->id); ubtf = u64_to_user_ptr(info.btf); btf_copy = min_t(u32, btf->data_size, info.btf_size); if (copy_to_user(ubtf, btf->data, btf_copy)) @@ -8140,7 +8149,7 @@ int btf_get_fd_by_id(u32 id) u32 btf_obj_id(const struct btf *btf) { - return btf->id; + return READ_ONCE(btf->id); } bool btf_is_kernel(const struct btf *btf) @@ -8262,6 +8271,13 @@ static int btf_module_notify(struct notifier_block *nb, unsigned long op, if (btf_mod->module != module) continue; + /* + * For modules, we do the freeing of BTF IDR as soon as + * module goes away to disable BTF discovery, since the + * btf_try_get_module() on such BTFs will fail. This may + * be called again on btf_put(), but it's ok to do so. + */ + btf_free_id(btf_mod->btf); list_del(&btf_mod->list); if (btf_mod->sysfs_attr) sysfs_remove_bin_file(btf_kobj, btf_mod->sysfs_attr); diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index 1b9b18e5b03cb0..5a56bc2ab900d6 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -713,8 +713,8 @@ static struct bpf_ksym *bpf_ksym_find(unsigned long addr) return n ? container_of(n, struct bpf_ksym, tnode) : NULL; } -int __bpf_address_lookup(unsigned long addr, unsigned long *size, - unsigned long *off, char *sym) +int bpf_address_lookup(unsigned long addr, unsigned long *size, + unsigned long *off, char *sym) { struct bpf_ksym *ksym; int ret = 0; @@ -1419,6 +1419,27 @@ static int bpf_jit_blind_insn(const struct bpf_insn *from, *to++ = BPF_ALU64_IMM(BPF_XOR, BPF_REG_AX, imm_rnd); *to++ = BPF_STX_MEM(from->code, from->dst_reg, BPF_REG_AX, from->off); break; + + case BPF_ST | BPF_PROBE_MEM32 | BPF_DW: + case BPF_ST | BPF_PROBE_MEM32 | BPF_W: + case BPF_ST | BPF_PROBE_MEM32 | BPF_H: + case BPF_ST | BPF_PROBE_MEM32 | BPF_B: + *to++ = BPF_ALU64_IMM(BPF_MOV, BPF_REG_AX, imm_rnd ^ + from->imm); + *to++ = BPF_ALU64_IMM(BPF_XOR, BPF_REG_AX, imm_rnd); + /* + * Cannot use BPF_STX_MEM() macro here as it + * hardcodes BPF_MEM mode, losing PROBE_MEM32 + * and breaking arena addressing in the JIT. + */ + *to++ = (struct bpf_insn) { + .code = BPF_STX | BPF_PROBE_MEM32 | + BPF_SIZE(from->code), + .dst_reg = from->dst_reg, + .src_reg = BPF_REG_AX, + .off = from->off, + }; + break; } out: return to - to_buff; @@ -1733,6 +1754,12 @@ bool bpf_opcode_in_insntable(u8 code) } #ifndef CONFIG_BPF_JIT_ALWAYS_ON +/* Absolute value of s32 without undefined behavior for S32_MIN */ +static u32 abs_s32(s32 x) +{ + return x >= 0 ? (u32)x : -(u32)x; +} + /** * ___bpf_prog_run - run eBPF program on a given context * @regs: is the array of MAX_BPF_EXT_REG eBPF pseudo-registers @@ -1897,8 +1924,8 @@ static u64 ___bpf_prog_run(u64 *regs, const struct bpf_insn *insn) DST = do_div(AX, (u32) SRC); break; case 1: - AX = abs((s32)DST); - AX = do_div(AX, abs((s32)SRC)); + AX = abs_s32((s32)DST); + AX = do_div(AX, abs_s32((s32)SRC)); if ((s32)DST < 0) DST = (u32)-AX; else @@ -1925,8 +1952,8 @@ static u64 ___bpf_prog_run(u64 *regs, const struct bpf_insn *insn) DST = do_div(AX, (u32) IMM); break; case 1: - AX = abs((s32)DST); - AX = do_div(AX, abs((s32)IMM)); + AX = abs_s32((s32)DST); + AX = do_div(AX, abs_s32((s32)IMM)); if ((s32)DST < 0) DST = (u32)-AX; else @@ -1952,8 +1979,8 @@ static u64 ___bpf_prog_run(u64 *regs, const struct bpf_insn *insn) DST = (u32) AX; break; case 1: - AX = abs((s32)DST); - do_div(AX, abs((s32)SRC)); + AX = abs_s32((s32)DST); + do_div(AX, abs_s32((s32)SRC)); if (((s32)DST < 0) == ((s32)SRC < 0)) DST = (u32)AX; else @@ -1979,8 +2006,8 @@ static u64 ___bpf_prog_run(u64 *regs, const struct bpf_insn *insn) DST = (u32) AX; break; case 1: - AX = abs((s32)DST); - do_div(AX, abs((s32)IMM)); + AX = abs_s32((s32)DST); + do_div(AX, abs_s32((s32)IMM)); if (((s32)DST < 0) == ((s32)IMM < 0)) DST = (u32)AX; else diff --git a/kernel/bpf/cpumap.c b/kernel/bpf/cpumap.c index 703e5df1f4ef9d..306bf983780412 100644 --- a/kernel/bpf/cpumap.c +++ b/kernel/bpf/cpumap.c @@ -29,6 +29,7 @@ #include #include #include +#include #include #include #include @@ -52,6 +53,7 @@ struct xdp_bulk_queue { struct list_head flush_node; struct bpf_cpu_map_entry *obj; unsigned int count; + local_lock_t bq_lock; }; /* Struct for every remote "destination" CPU in map */ @@ -451,6 +453,7 @@ __cpu_map_entry_alloc(struct bpf_map *map, struct bpf_cpumap_val *value, for_each_possible_cpu(i) { bq = per_cpu_ptr(rcpu->bulkq, i); bq->obj = rcpu; + local_lock_init(&bq->bq_lock); } /* Alloc queue */ @@ -717,6 +720,8 @@ static void bq_flush_to_queue(struct xdp_bulk_queue *bq) struct ptr_ring *q; int i; + lockdep_assert_held(&bq->bq_lock); + if (unlikely(!bq->count)) return; @@ -744,11 +749,15 @@ static void bq_flush_to_queue(struct xdp_bulk_queue *bq) } /* Runs under RCU-read-side, plus in softirq under NAPI protection. - * Thus, safe percpu variable access. + * Thus, safe percpu variable access. PREEMPT_RT relies on + * local_lock_nested_bh() to serialise access to the per-CPU bq. */ static void bq_enqueue(struct bpf_cpu_map_entry *rcpu, struct xdp_frame *xdpf) { - struct xdp_bulk_queue *bq = this_cpu_ptr(rcpu->bulkq); + struct xdp_bulk_queue *bq; + + local_lock_nested_bh(&rcpu->bulkq->bq_lock); + bq = this_cpu_ptr(rcpu->bulkq); if (unlikely(bq->count == CPU_MAP_BULK_SIZE)) bq_flush_to_queue(bq); @@ -769,6 +778,8 @@ static void bq_enqueue(struct bpf_cpu_map_entry *rcpu, struct xdp_frame *xdpf) list_add(&bq->flush_node, flush_list); } + + local_unlock_nested_bh(&rcpu->bulkq->bq_lock); } int cpu_map_enqueue(struct bpf_cpu_map_entry *rcpu, struct xdp_frame *xdpf, @@ -805,7 +816,9 @@ void __cpu_map_flush(struct list_head *flush_list) struct xdp_bulk_queue *bq, *tmp; list_for_each_entry_safe(bq, tmp, flush_list, flush_node) { + local_lock_nested_bh(&bq->obj->bulkq->bq_lock); bq_flush_to_queue(bq); + local_unlock_nested_bh(&bq->obj->bulkq->bq_lock); /* If already running, costs spin_lock_irqsave + smb_mb */ wake_up_process(bq->obj->kthread); diff --git a/kernel/bpf/crypto.c b/kernel/bpf/crypto.c index 83c4d9943084b9..1d024fe7248acc 100644 --- a/kernel/bpf/crypto.c +++ b/kernel/bpf/crypto.c @@ -261,6 +261,12 @@ __bpf_kfunc void bpf_crypto_ctx_release(struct bpf_crypto_ctx *ctx) call_rcu(&ctx->rcu, crypto_free_cb); } +__bpf_kfunc void bpf_crypto_ctx_release_dtor(void *ctx) +{ + bpf_crypto_ctx_release(ctx); +} +CFI_NOSEAL(bpf_crypto_ctx_release_dtor); + static int bpf_crypto_crypt(const struct bpf_crypto_ctx *ctx, const struct bpf_dynptr_kern *src, const struct bpf_dynptr_kern *dst, @@ -368,7 +374,7 @@ static const struct btf_kfunc_id_set crypt_kfunc_set = { BTF_ID_LIST(bpf_crypto_dtor_ids) BTF_ID(struct, bpf_crypto_ctx) -BTF_ID(func, bpf_crypto_ctx_release) +BTF_ID(func, bpf_crypto_ctx_release_dtor) static int __init crypto_kfunc_init(void) { diff --git a/kernel/bpf/devmap.c b/kernel/bpf/devmap.c index 2625601de76e95..3d619d01088e3e 100644 --- a/kernel/bpf/devmap.c +++ b/kernel/bpf/devmap.c @@ -45,6 +45,7 @@ * types of devmap; only the lookup and insertion is different. */ #include +#include #include #include #include @@ -60,6 +61,7 @@ struct xdp_dev_bulk_queue { struct net_device *dev_rx; struct bpf_prog *xdp_prog; unsigned int count; + local_lock_t bq_lock; }; struct bpf_dtab_netdev { @@ -381,6 +383,8 @@ static void bq_xmit_all(struct xdp_dev_bulk_queue *bq, u32 flags) int to_send = cnt; int i; + lockdep_assert_held(&bq->bq_lock); + if (unlikely(!cnt)) return; @@ -425,10 +429,12 @@ void __dev_flush(struct list_head *flush_list) struct xdp_dev_bulk_queue *bq, *tmp; list_for_each_entry_safe(bq, tmp, flush_list, flush_node) { + local_lock_nested_bh(&bq->dev->xdp_bulkq->bq_lock); bq_xmit_all(bq, XDP_XMIT_FLUSH); bq->dev_rx = NULL; bq->xdp_prog = NULL; __list_del_clearprev(&bq->flush_node); + local_unlock_nested_bh(&bq->dev->xdp_bulkq->bq_lock); } } @@ -451,12 +457,16 @@ static void *__dev_map_lookup_elem(struct bpf_map *map, u32 key) /* Runs in NAPI, i.e., softirq under local_bh_disable(). Thus, safe percpu * variable access, and map elements stick around. See comment above - * xdp_do_flush() in filter.c. + * xdp_do_flush() in filter.c. PREEMPT_RT relies on local_lock_nested_bh() + * to serialise access to the per-CPU bq. */ static void bq_enqueue(struct net_device *dev, struct xdp_frame *xdpf, struct net_device *dev_rx, struct bpf_prog *xdp_prog) { - struct xdp_dev_bulk_queue *bq = this_cpu_ptr(dev->xdp_bulkq); + struct xdp_dev_bulk_queue *bq; + + local_lock_nested_bh(&dev->xdp_bulkq->bq_lock); + bq = this_cpu_ptr(dev->xdp_bulkq); if (unlikely(bq->count == DEV_MAP_BULK_SIZE)) bq_xmit_all(bq, 0); @@ -477,6 +487,8 @@ static void bq_enqueue(struct net_device *dev, struct xdp_frame *xdpf, } bq->q[bq->count++] = xdpf; + + local_unlock_nested_bh(&dev->xdp_bulkq->bq_lock); } static inline int __xdp_enqueue(struct net_device *dev, struct xdp_frame *xdpf, @@ -588,18 +600,22 @@ static inline bool is_ifindex_excluded(int *excluded, int num_excluded, int ifin } /* Get ifindex of each upper device. 'indexes' must be able to hold at - * least MAX_NEST_DEV elements. - * Returns the number of ifindexes added. + * least 'max' elements. + * Returns the number of ifindexes added, or -EOVERFLOW if there are too + * many upper devices. */ -static int get_upper_ifindexes(struct net_device *dev, int *indexes) +static int get_upper_ifindexes(struct net_device *dev, int *indexes, int max) { struct net_device *upper; struct list_head *iter; int n = 0; netdev_for_each_upper_dev_rcu(dev, upper, iter) { + if (n >= max) + return -EOVERFLOW; indexes[n++] = upper->ifindex; } + return n; } @@ -615,7 +631,11 @@ int dev_map_enqueue_multi(struct xdp_frame *xdpf, struct net_device *dev_rx, int err; if (exclude_ingress) { - num_excluded = get_upper_ifindexes(dev_rx, excluded_devices); + num_excluded = get_upper_ifindexes(dev_rx, excluded_devices, + ARRAY_SIZE(excluded_devices) - 1); + if (num_excluded < 0) + return num_excluded; + excluded_devices[num_excluded++] = dev_rx->ifindex; } @@ -733,7 +753,11 @@ int dev_map_redirect_multi(struct net_device *dev, struct sk_buff *skb, int err; if (exclude_ingress) { - num_excluded = get_upper_ifindexes(dev, excluded_devices); + num_excluded = get_upper_ifindexes(dev, excluded_devices, + ARRAY_SIZE(excluded_devices) - 1); + if (num_excluded < 0) + return num_excluded; + excluded_devices[num_excluded++] = dev->ifindex; } @@ -1115,8 +1139,13 @@ static int dev_map_notification(struct notifier_block *notifier, if (!netdev->xdp_bulkq) return NOTIFY_BAD; - for_each_possible_cpu(cpu) - per_cpu_ptr(netdev->xdp_bulkq, cpu)->dev = netdev; + for_each_possible_cpu(cpu) { + struct xdp_dev_bulk_queue *bq; + + bq = per_cpu_ptr(netdev->xdp_bulkq, cpu); + bq->dev = netdev; + local_lock_init(&bq->bq_lock); + } break; case NETDEV_UNREGISTER: /* This rcu_read_lock/unlock pair is needed because diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c index db72b96f9c8c85..f66284f8ec2cc9 100644 --- a/kernel/bpf/helpers.c +++ b/kernel/bpf/helpers.c @@ -1077,7 +1077,7 @@ const struct bpf_func_proto bpf_snprintf_proto = { .func = bpf_snprintf, .gpl_only = true, .ret_type = RET_INTEGER, - .arg1_type = ARG_PTR_TO_MEM_OR_NULL, + .arg1_type = ARG_PTR_TO_MEM_OR_NULL | MEM_WRITE, .arg2_type = ARG_CONST_SIZE_OR_ZERO, .arg3_type = ARG_PTR_TO_CONST_STR, .arg4_type = ARG_PTR_TO_MEM | PTR_MAYBE_NULL | MEM_RDONLY, diff --git a/kernel/bpf/rqspinlock.c b/kernel/bpf/rqspinlock.c index f7d0c8d4644edb..2fdfa828e3d35a 100644 --- a/kernel/bpf/rqspinlock.c +++ b/kernel/bpf/rqspinlock.c @@ -265,10 +265,11 @@ int __lockfunc resilient_tas_spin_lock(rqspinlock_t *lock) RES_INIT_TIMEOUT(ts); /* - * The fast path is not invoked for the TAS fallback, so we must grab - * the deadlock detection entry here. + * We are either called directly from res_spin_lock after grabbing the + * deadlock detection entry when queued spinlocks are disabled, or from + * resilient_queued_spin_lock_slowpath after grabbing the deadlock + * detection entry. No need to obtain it here. */ - grab_held_lock_entry(lock); /* * Since the waiting loop's time is dependent on the amount of diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 4ff82144f8853d..ce7db2f3be6f63 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -1366,11 +1366,6 @@ static int map_check_btf(struct bpf_map *map, struct bpf_token *token, return ret; } -static bool bpf_net_capable(void) -{ - return capable(CAP_NET_ADMIN) || capable(CAP_SYS_ADMIN); -} - #define BPF_MAP_CREATE_LAST_FIELD excl_prog_hash_size /* called via syscall */ static int map_create(union bpf_attr *attr, bpfptr_t uattr) @@ -2820,6 +2815,13 @@ static int bpf_prog_verify_signature(struct bpf_prog *prog, union bpf_attr *attr void *sig; int err = 0; + /* + * Don't attempt to use kmalloc_large or vmalloc for signatures. + * Practical signature for BPF program should be below this limit. + */ + if (attr->signature_size > KMALLOC_MAX_CACHE_SIZE) + return -EINVAL; + if (system_keyring_id_check(attr->keyring_id) == 0) key = bpf_lookup_system_key(attr->keyring_id); else @@ -4565,6 +4567,8 @@ static int bpf_prog_detach(const union bpf_attr *attr) prog = bpf_prog_get_type(attr->attach_bpf_fd, ptype); if (IS_ERR(prog)) return PTR_ERR(prog); + } else if (!bpf_mprog_detach_empty(ptype)) { + return -EPERM; } } else if (is_cgroup_prog_type(ptype, 0, false)) { if (attr->attach_flags || attr->relative_fd) @@ -5310,6 +5314,9 @@ static int bpf_map_get_info_by_fd(struct file *file, if (info.hash_size != SHA256_DIGEST_SIZE) return -EINVAL; + if (!READ_ONCE(map->frozen)) + return -EPERM; + err = map->ops->map_get_hash(map, SHA256_DIGEST_SIZE, map->sha); if (err != 0) return err; @@ -6407,7 +6414,7 @@ static const struct bpf_func_proto bpf_kallsyms_lookup_name_proto = { .func = bpf_kallsyms_lookup_name, .gpl_only = false, .ret_type = RET_INTEGER, - .arg1_type = ARG_PTR_TO_MEM, + .arg1_type = ARG_PTR_TO_MEM | MEM_RDONLY, .arg2_type = ARG_CONST_SIZE_OR_ZERO, .arg3_type = ARG_ANYTHING, .arg4_type = ARG_PTR_TO_FIXED_SIZE_MEM | MEM_UNINIT | MEM_WRITE | MEM_ALIGNED, diff --git a/kernel/bpf/tnum.c b/kernel/bpf/tnum.c index f8e70e9c3998d4..4abc359b3db012 100644 --- a/kernel/bpf/tnum.c +++ b/kernel/bpf/tnum.c @@ -8,6 +8,7 @@ */ #include #include +#include #define TNUM(_v, _m) (struct tnum){.value = _v, .mask = _m} /* A completely unknown value */ @@ -253,3 +254,74 @@ struct tnum tnum_const_subreg(struct tnum a, u32 value) { return tnum_with_subreg(a, tnum_const(value)); } + +struct tnum tnum_bswap16(struct tnum a) +{ + return TNUM(swab16(a.value & 0xFFFF), swab16(a.mask & 0xFFFF)); +} + +struct tnum tnum_bswap32(struct tnum a) +{ + return TNUM(swab32(a.value & 0xFFFFFFFF), swab32(a.mask & 0xFFFFFFFF)); +} + +struct tnum tnum_bswap64(struct tnum a) +{ + return TNUM(swab64(a.value), swab64(a.mask)); +} + +/* Given tnum t, and a number z such that tmin <= z < tmax, where tmin + * is the smallest member of the t (= t.value) and tmax is the largest + * member of t (= t.value | t.mask), returns the smallest member of t + * larger than z. + * + * For example, + * t = x11100x0 + * z = 11110001 (241) + * result = 11110010 (242) + * + * Note: if this function is called with z >= tmax, it just returns + * early with tmax; if this function is called with z < tmin, the + * algorithm already returns tmin. + */ +u64 tnum_step(struct tnum t, u64 z) +{ + u64 tmax, j, p, q, r, s, v, u, w, res; + u8 k; + + tmax = t.value | t.mask; + + /* if z >= largest member of t, return largest member of t */ + if (z >= tmax) + return tmax; + + /* if z < smallest member of t, return smallest member of t */ + if (z < t.value) + return t.value; + + /* keep t's known bits, and match all unknown bits to z */ + j = t.value | (z & t.mask); + + if (j > z) { + p = ~z & t.value & ~t.mask; + k = fls64(p); /* k is the most-significant 0-to-1 flip */ + q = U64_MAX << k; + r = q & z; /* positions > k matched to z */ + s = ~q & t.value; /* positions <= k matched to t.value */ + v = r | s; + res = v; + } else { + p = z & ~t.value & ~t.mask; + k = fls64(p); /* k is the most-significant 1-to-0 flip */ + q = U64_MAX << k; + r = q & t.mask & z; /* unknown positions > k, matched to z */ + s = q & ~t.mask; /* known positions > k, set to 1 */ + v = r | s; + /* add 1 to unknown positions > k to make value greater than z */ + u = v + (1ULL << k); + /* extract bits in unknown positions > k from u, rest from t.value */ + w = (u & t.mask) | t.value; + res = w; + } + return res; +} diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c index 976d89011b1570..47c70eb451f3aa 100644 --- a/kernel/bpf/trampoline.c +++ b/kernel/bpf/trampoline.c @@ -214,10 +214,15 @@ static int modify_fentry(struct bpf_trampoline *tr, u32 orig_flags, int ret; if (tr->func.ftrace_managed) { + unsigned long addr = (unsigned long) new_addr; + + if (bpf_trampoline_use_jmp(tr->flags)) + addr = ftrace_jmp_set(addr); + if (lock_direct_mutex) - ret = modify_ftrace_direct(tr->fops, (long)new_addr); + ret = modify_ftrace_direct(tr->fops, addr); else - ret = modify_ftrace_direct_nolock(tr->fops, (long)new_addr); + ret = modify_ftrace_direct_nolock(tr->fops, addr); } else { ret = bpf_trampoline_update_fentry(tr, orig_flags, old_addr, new_addr); @@ -240,10 +245,15 @@ static int register_fentry(struct bpf_trampoline *tr, void *new_addr) } if (tr->func.ftrace_managed) { + unsigned long addr = (unsigned long) new_addr; + + if (bpf_trampoline_use_jmp(tr->flags)) + addr = ftrace_jmp_set(addr); + ret = ftrace_set_filter_ip(tr->fops, (unsigned long)ip, 0, 1); if (ret) return ret; - ret = register_ftrace_direct(tr->fops, (long)new_addr); + ret = register_ftrace_direct(tr->fops, addr); } else { ret = bpf_trampoline_update_fentry(tr, 0, NULL, new_addr); } @@ -499,13 +509,6 @@ static int bpf_trampoline_update(struct bpf_trampoline *tr, bool lock_direct_mut if (err) goto out_free; -#ifdef CONFIG_DYNAMIC_FTRACE_WITH_JMP - if (bpf_trampoline_use_jmp(tr->flags)) - tr->fops->flags |= FTRACE_OPS_FL_JMP; - else - tr->fops->flags &= ~FTRACE_OPS_FL_JMP; -#endif - WARN_ON(tr->cur_image && total == 0); if (tr->cur_image) /* progs already running at this address */ @@ -533,15 +536,8 @@ static int bpf_trampoline_update(struct bpf_trampoline *tr, bool lock_direct_mut tr->cur_image = im; out: /* If any error happens, restore previous flags */ - if (err) { + if (err) tr->flags = orig_flags; -#ifdef CONFIG_DYNAMIC_FTRACE_WITH_JMP - if (bpf_trampoline_use_jmp(tr->flags)) - tr->fops->flags |= FTRACE_OPS_FL_JMP; - else - tr->fops->flags &= ~FTRACE_OPS_FL_JMP; -#endif - } kfree(tlinks); return err; @@ -800,10 +796,8 @@ int bpf_trampoline_link_cgroup_shim(struct bpf_prog *prog, mutex_lock(&tr->mutex); shim_link = cgroup_shim_find(tr, bpf_func); - if (shim_link) { + if (shim_link && !IS_ERR(bpf_link_inc_not_zero(&shim_link->link.link))) { /* Reusing existing shim attached by the other program. */ - bpf_link_inc(&shim_link->link.link); - mutex_unlock(&tr->mutex); bpf_trampoline_put(tr); /* bpf_trampoline_get above */ return 0; diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 3135643d569552..d1394e16d108c2 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -608,6 +608,13 @@ static bool is_atomic_load_insn(const struct bpf_insn *insn) insn->imm == BPF_LOAD_ACQ; } +static bool is_atomic_fetch_insn(const struct bpf_insn *insn) +{ + return BPF_CLASS(insn->code) == BPF_STX && + BPF_MODE(insn->code) == BPF_ATOMIC && + (insn->imm & BPF_FETCH); +} + static int __get_spi(s32 off) { return (-off - 1) / BPF_REG_SIZE; @@ -2358,6 +2365,9 @@ static void __update_reg32_bounds(struct bpf_reg_state *reg) static void __update_reg64_bounds(struct bpf_reg_state *reg) { + u64 tnum_next, tmax; + bool umin_in_tnum; + /* min signed is max(sign bit) | min(other bits) */ reg->smin_value = max_t(s64, reg->smin_value, reg->var_off.value | (reg->var_off.mask & S64_MIN)); @@ -2367,6 +2377,33 @@ static void __update_reg64_bounds(struct bpf_reg_state *reg) reg->umin_value = max(reg->umin_value, reg->var_off.value); reg->umax_value = min(reg->umax_value, reg->var_off.value | reg->var_off.mask); + + /* Check if u64 and tnum overlap in a single value */ + tnum_next = tnum_step(reg->var_off, reg->umin_value); + umin_in_tnum = (reg->umin_value & ~reg->var_off.mask) == reg->var_off.value; + tmax = reg->var_off.value | reg->var_off.mask; + if (umin_in_tnum && tnum_next > reg->umax_value) { + /* The u64 range and the tnum only overlap in umin. + * u64: ---[xxxxxx]----- + * tnum: --xx----------x- + */ + ___mark_reg_known(reg, reg->umin_value); + } else if (!umin_in_tnum && tnum_next == tmax) { + /* The u64 range and the tnum only overlap in the maximum value + * represented by the tnum, called tmax. + * u64: ---[xxxxxx]----- + * tnum: xx-----x-------- + */ + ___mark_reg_known(reg, tmax); + } else if (!umin_in_tnum && tnum_next <= reg->umax_value && + tnum_step(reg->var_off, tnum_next) > reg->umax_value) { + /* The u64 range and the tnum only overlap in between umin + * (excluded) and umax. + * u64: ---[xxxxxx]----- + * tnum: xx----x-------x- + */ + ___mark_reg_known(reg, tnum_next); + } } static void __update_reg_bounds(struct bpf_reg_state *reg) @@ -2460,6 +2497,30 @@ static void __reg32_deduce_bounds(struct bpf_reg_state *reg) if ((u32)reg->s32_min_value <= (u32)reg->s32_max_value) { reg->u32_min_value = max_t(u32, reg->s32_min_value, reg->u32_min_value); reg->u32_max_value = min_t(u32, reg->s32_max_value, reg->u32_max_value); + } else { + if (reg->u32_max_value < (u32)reg->s32_min_value) { + /* See __reg64_deduce_bounds() for detailed explanation. + * Refine ranges in the following situation: + * + * 0 U32_MAX + * | [xxxxxxxxxxxxxx u32 range xxxxxxxxxxxxxx] | + * |----------------------------|----------------------------| + * |xxxxx s32 range xxxxxxxxx] [xxxxxxx| + * 0 S32_MAX S32_MIN -1 + */ + reg->s32_min_value = (s32)reg->u32_min_value; + reg->u32_max_value = min_t(u32, reg->u32_max_value, reg->s32_max_value); + } else if ((u32)reg->s32_max_value < reg->u32_min_value) { + /* + * 0 U32_MAX + * | [xxxxxxxxxxxxxx u32 range xxxxxxxxxxxxxx] | + * |----------------------------|----------------------------| + * |xxxxxxxxx] [xxxxxxxxxxxx s32 range | + * 0 S32_MAX S32_MIN -1 + */ + reg->s32_max_value = (s32)reg->u32_max_value; + reg->u32_min_value = max_t(u32, reg->u32_min_value, reg->s32_min_value); + } } } @@ -4302,10 +4363,24 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx, * dreg still needs precision before this insn */ } - } else if (class == BPF_LDX || is_atomic_load_insn(insn)) { - if (!bt_is_reg_set(bt, dreg)) + } else if (class == BPF_LDX || + is_atomic_load_insn(insn) || + is_atomic_fetch_insn(insn)) { + u32 load_reg = dreg; + + /* + * Atomic fetch operation writes the old value into + * a register (sreg or r0) and if it was tracked for + * precision, propagate to the stack slot like we do + * in regular ldx. + */ + if (is_atomic_fetch_insn(insn)) + load_reg = insn->imm == BPF_CMPXCHG ? + BPF_REG_0 : sreg; + + if (!bt_is_reg_set(bt, load_reg)) return 0; - bt_clear_reg(bt, dreg); + bt_clear_reg(bt, load_reg); /* scalars can only be spilled into stack w/o losing precision. * Load from any other memory can be zero extended. @@ -7752,7 +7827,8 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regn } else if (reg->type == CONST_PTR_TO_MAP) { err = check_ptr_to_map_access(env, regs, regno, off, size, t, value_regno); - } else if (base_type(reg->type) == PTR_TO_BUF) { + } else if (base_type(reg->type) == PTR_TO_BUF && + !type_may_be_null(reg->type)) { bool rdonly_mem = type_is_rdonly_mem(reg->type); u32 *max_access; @@ -15305,21 +15381,17 @@ static void __scalar64_min_max_lsh(struct bpf_reg_state *dst_reg, u64 umin_val, u64 umax_val) { /* Special case <<32 because it is a common compiler pattern to sign - * extend subreg by doing <<32 s>>32. In this case if 32bit bounds are - * positive we know this shift will also be positive so we can track - * bounds correctly. Otherwise we lose all sign bit information except - * what we can pick up from var_off. Perhaps we can generalize this - * later to shifts of any length. + * extend subreg by doing <<32 s>>32. smin/smax assignments are correct + * because s32 bounds don't flip sign when shifting to the left by + * 32bits. */ - if (umin_val == 32 && umax_val == 32 && dst_reg->s32_max_value >= 0) + if (umin_val == 32 && umax_val == 32) { dst_reg->smax_value = (s64)dst_reg->s32_max_value << 32; - else - dst_reg->smax_value = S64_MAX; - - if (umin_val == 32 && umax_val == 32 && dst_reg->s32_min_value >= 0) dst_reg->smin_value = (s64)dst_reg->s32_min_value << 32; - else + } else { + dst_reg->smax_value = S64_MAX; dst_reg->smin_value = S64_MIN; + } /* If we might shift our top bit out, then we know nothing */ if (dst_reg->umax_value > 1ULL << (63 - umax_val)) { @@ -15462,6 +15534,55 @@ static void scalar_min_max_arsh(struct bpf_reg_state *dst_reg, __update_reg_bounds(dst_reg); } +static void scalar_byte_swap(struct bpf_reg_state *dst_reg, struct bpf_insn *insn) +{ + /* + * Byte swap operation - update var_off using tnum_bswap. + * Three cases: + * 1. bswap(16|32|64): opcode=0xd7 (BPF_END | BPF_ALU64 | BPF_TO_LE) + * unconditional swap + * 2. to_le(16|32|64): opcode=0xd4 (BPF_END | BPF_ALU | BPF_TO_LE) + * swap on big-endian, truncation or no-op on little-endian + * 3. to_be(16|32|64): opcode=0xdc (BPF_END | BPF_ALU | BPF_TO_BE) + * swap on little-endian, truncation or no-op on big-endian + */ + + bool alu64 = BPF_CLASS(insn->code) == BPF_ALU64; + bool to_le = BPF_SRC(insn->code) == BPF_TO_LE; + bool is_big_endian; +#ifdef CONFIG_CPU_BIG_ENDIAN + is_big_endian = true; +#else + is_big_endian = false; +#endif + /* Apply bswap if alu64 or switch between big-endian and little-endian machines */ + bool need_bswap = alu64 || (to_le == is_big_endian); + + /* + * If the register is mutated, manually reset its scalar ID to break + * any existing ties and avoid incorrect bounds propagation. + */ + if (need_bswap || insn->imm == 16 || insn->imm == 32) + dst_reg->id = 0; + + if (need_bswap) { + if (insn->imm == 16) + dst_reg->var_off = tnum_bswap16(dst_reg->var_off); + else if (insn->imm == 32) + dst_reg->var_off = tnum_bswap32(dst_reg->var_off); + else if (insn->imm == 64) + dst_reg->var_off = tnum_bswap64(dst_reg->var_off); + /* + * Byteswap scrambles the range, so we must reset bounds. + * Bounds will be re-derived from the new tnum later. + */ + __mark_reg_unbounded(dst_reg); + } + /* For bswap16/32, truncate dst register to match the swapped size */ + if (insn->imm == 16 || insn->imm == 32) + coerce_reg_to_size(dst_reg, insn->imm / 8); +} + static bool is_safe_to_compute_dst_reg_range(struct bpf_insn *insn, const struct bpf_reg_state *src_reg) { @@ -15488,6 +15609,7 @@ static bool is_safe_to_compute_dst_reg_range(struct bpf_insn *insn, case BPF_XOR: case BPF_OR: case BPF_MUL: + case BPF_END: return true; /* Shift operators range is only computable if shift dimension operand @@ -15503,6 +15625,35 @@ static bool is_safe_to_compute_dst_reg_range(struct bpf_insn *insn, } } +static int maybe_fork_scalars(struct bpf_verifier_env *env, struct bpf_insn *insn, + struct bpf_reg_state *dst_reg) +{ + struct bpf_verifier_state *branch; + struct bpf_reg_state *regs; + bool alu32; + + if (dst_reg->smin_value == -1 && dst_reg->smax_value == 0) + alu32 = false; + else if (dst_reg->s32_min_value == -1 && dst_reg->s32_max_value == 0) + alu32 = true; + else + return 0; + + branch = push_stack(env, env->insn_idx, env->insn_idx, false); + if (IS_ERR(branch)) + return PTR_ERR(branch); + + regs = branch->frame[branch->curframe]->regs; + if (alu32) { + __mark_reg32_known(®s[insn->dst_reg], 0); + __mark_reg32_known(dst_reg, -1ull); + } else { + __mark_reg_known(®s[insn->dst_reg], 0); + __mark_reg_known(dst_reg, -1ull); + } + return 0; +} + /* WARNING: This function does calculations on 64-bit values, but the actual * execution may occur on 32-bit values. Therefore, things like bitshifts * need extra checks in the 32-bit case. @@ -15565,11 +15716,21 @@ static int adjust_scalar_min_max_vals(struct bpf_verifier_env *env, scalar_min_max_mul(dst_reg, &src_reg); break; case BPF_AND: + if (tnum_is_const(src_reg.var_off)) { + ret = maybe_fork_scalars(env, insn, dst_reg); + if (ret) + return ret; + } dst_reg->var_off = tnum_and(dst_reg->var_off, src_reg.var_off); scalar32_min_max_and(dst_reg, &src_reg); scalar_min_max_and(dst_reg, &src_reg); break; case BPF_OR: + if (tnum_is_const(src_reg.var_off)) { + ret = maybe_fork_scalars(env, insn, dst_reg); + if (ret) + return ret; + } dst_reg->var_off = tnum_or(dst_reg->var_off, src_reg.var_off); scalar32_min_max_or(dst_reg, &src_reg); scalar_min_max_or(dst_reg, &src_reg); @@ -15597,12 +15758,23 @@ static int adjust_scalar_min_max_vals(struct bpf_verifier_env *env, else scalar_min_max_arsh(dst_reg, &src_reg); break; + case BPF_END: + scalar_byte_swap(dst_reg, insn); + break; default: break; } - /* ALU32 ops are zero extended into 64bit register */ - if (alu32) + /* + * ALU32 ops are zero extended into 64bit register. + * + * BPF_END is already handled inside the helper (truncation), + * so skip zext here to avoid unexpected zero extension. + * e.g., le64: opcode=(BPF_END|BPF_ALU|BPF_TO_LE), imm=0x40 + * This is a 64bit byte swap operation with alu32==true, + * but we should not zero extend the result. + */ + if (alu32 && opcode != BPF_END) zext_32_to_64(dst_reg); reg_bounds_sync(dst_reg); return 0; @@ -15782,7 +15954,7 @@ static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn) } /* check dest operand */ - if (opcode == BPF_NEG && + if ((opcode == BPF_NEG || opcode == BPF_END) && regs[insn->dst_reg].type == SCALAR_VALUE) { err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK); err = err ?: adjust_scalar_min_max_vals(env, insn, @@ -16776,17 +16948,24 @@ static void __collect_linked_regs(struct linked_regs *reg_set, struct bpf_reg_st * in verifier state, save R in linked_regs if R->id == id. * If there are too many Rs sharing same id, reset id for leftover Rs. */ -static void collect_linked_regs(struct bpf_verifier_state *vstate, u32 id, +static void collect_linked_regs(struct bpf_verifier_env *env, + struct bpf_verifier_state *vstate, + u32 id, struct linked_regs *linked_regs) { + struct bpf_insn_aux_data *aux = env->insn_aux_data; struct bpf_func_state *func; struct bpf_reg_state *reg; + u16 live_regs; int i, j; id = id & ~BPF_ADD_CONST; for (i = vstate->curframe; i >= 0; i--) { + live_regs = aux[frame_insn_idx(vstate, i)].live_regs_before; func = vstate->frame[i]; for (j = 0; j < BPF_REG_FP; j++) { + if (!(live_regs & BIT(j))) + continue; reg = &func->regs[j]; __collect_linked_regs(linked_regs, reg, id, i, j, true); } @@ -16827,6 +17006,7 @@ static void sync_linked_regs(struct bpf_verifier_state *vstate, struct bpf_reg_s } else { s32 saved_subreg_def = reg->subreg_def; s32 saved_off = reg->off; + u32 saved_id = reg->id; fake_reg.type = SCALAR_VALUE; __mark_reg_known(&fake_reg, (s32)reg->off - (s32)known_reg->off); @@ -16834,10 +17014,11 @@ static void sync_linked_regs(struct bpf_verifier_state *vstate, struct bpf_reg_s /* reg = known_reg; reg += delta */ copy_register_state(reg, known_reg); /* - * Must preserve off, id and add_const flag, + * Must preserve off, id and subreg_def flag, * otherwise another sync_linked_regs() will be incorrect. */ reg->off = saved_off; + reg->id = saved_id; reg->subreg_def = saved_subreg_def; scalar32_min_max_add(reg, &fake_reg); @@ -16992,9 +17173,9 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env, * if parent state is created. */ if (BPF_SRC(insn->code) == BPF_X && src_reg->type == SCALAR_VALUE && src_reg->id) - collect_linked_regs(this_branch, src_reg->id, &linked_regs); + collect_linked_regs(env, this_branch, src_reg->id, &linked_regs); if (dst_reg->type == SCALAR_VALUE && dst_reg->id) - collect_linked_regs(this_branch, dst_reg->id, &linked_regs); + collect_linked_regs(env, this_branch, dst_reg->id, &linked_regs); if (linked_regs.cnt > 1) { err = push_jmp_history(env, this_branch, 0, linked_regs_pack(&linked_regs)); if (err) @@ -19184,8 +19365,13 @@ static bool regsafe(struct bpf_verifier_env *env, struct bpf_reg_state *rold, * since someone could have accessed through (ptr - k), or * even done ptr -= k in a register, to get a safe access. */ - if (rold->range > rcur->range) + if (rold->range < 0 || rcur->range < 0) { + /* special case for [BEYOND|AT]_PKT_END */ + if (rold->range != rcur->range) + return false; + } else if (rold->range > rcur->range) { return false; + } /* If the offsets don't match, we can't trust our alignment; * nor can we be sure that we won't fall out of range. */ @@ -19835,8 +20021,10 @@ static int is_state_visited(struct bpf_verifier_env *env, int insn_idx) } } if (bpf_calls_callback(env, insn_idx)) { - if (states_equal(env, &sl->state, cur, RANGE_WITHIN)) + if (states_equal(env, &sl->state, cur, RANGE_WITHIN)) { + loop = true; goto hit; + } goto skip_inf_loop_check; } /* attempt to detect infinite loop to avoid unnecessary doomed work */ @@ -20182,7 +20370,8 @@ static int process_bpf_exit_full(struct bpf_verifier_env *env, * state when it exits. */ int err = check_resource_leak(env, exception_exit, - !env->cur_state->curframe, + exception_exit || !env->cur_state->curframe, + exception_exit ? "bpf_throw" : "BPF_EXIT instruction in main prog"); if (err) return err; @@ -20611,17 +20800,19 @@ static int do_check(struct bpf_verifier_env *env) * may skip a nospec patched-in after the jump. This can * currently never happen because nospec_result is only * used for the write-ops - * `*(size*)(dst_reg+off)=src_reg|imm32` which must - * never skip the following insn. Still, add a warning - * to document this in case nospec_result is used - * elsewhere in the future. + * `*(size*)(dst_reg+off)=src_reg|imm32` and helper + * calls. These must never skip the following insn + * (i.e., bpf_insn_successors()'s opcode_info.can_jump + * is false). Still, add a warning to document this in + * case nospec_result is used elsewhere in the future. * * All non-branch instructions have a single * fall-through edge. For these, nospec_result should * already work. */ - if (verifier_bug_if(BPF_CLASS(insn->code) == BPF_JMP || - BPF_CLASS(insn->code) == BPF_JMP32, env, + if (verifier_bug_if((BPF_CLASS(insn->code) == BPF_JMP || + BPF_CLASS(insn->code) == BPF_JMP32) && + BPF_OP(insn->code) != BPF_CALL, env, "speculation barrier after jump instruction may not have the desired effect")) return -EFAULT; process_bpf_exit: @@ -20679,29 +20870,29 @@ static int find_btf_percpu_datasec(struct btf *btf) } /* - * Add btf to the used_btfs array and return the index. (If the btf was - * already added, then just return the index.) Upon successful insertion - * increase btf refcnt, and, if present, also refcount the corresponding - * kernel module. + * Add btf to the env->used_btfs array. If needed, refcount the + * corresponding kernel module. To simplify caller's logic + * in case of error or if btf was added before the function + * decreases the btf refcount. */ static int __add_used_btf(struct bpf_verifier_env *env, struct btf *btf) { struct btf_mod_pair *btf_mod; + int ret = 0; int i; /* check whether we recorded this BTF (and maybe module) already */ for (i = 0; i < env->used_btf_cnt; i++) if (env->used_btfs[i].btf == btf) - return i; + goto ret_put; if (env->used_btf_cnt >= MAX_USED_BTFS) { verbose(env, "The total number of btfs per program has reached the limit of %u\n", MAX_USED_BTFS); - return -E2BIG; + ret = -E2BIG; + goto ret_put; } - btf_get(btf); - btf_mod = &env->used_btfs[env->used_btf_cnt]; btf_mod->btf = btf; btf_mod->module = NULL; @@ -20710,12 +20901,18 @@ static int __add_used_btf(struct bpf_verifier_env *env, struct btf *btf) if (btf_is_module(btf)) { btf_mod->module = btf_try_get_module(btf); if (!btf_mod->module) { - btf_put(btf); - return -ENXIO; + ret = -ENXIO; + goto ret_put; } } - return env->used_btf_cnt++; + env->used_btf_cnt++; + return 0; + +ret_put: + /* Either error or this BTF was already added */ + btf_put(btf); + return ret; } /* replace pseudo btf_id with kernel symbol address */ @@ -20812,9 +21009,7 @@ static int check_pseudo_btf_id(struct bpf_verifier_env *env, btf_fd = insn[1].imm; if (btf_fd) { - CLASS(fd, f)(btf_fd); - - btf = __btf_get_by_fd(f); + btf = btf_get_by_fd(btf_fd); if (IS_ERR(btf)) { verbose(env, "invalid module BTF object FD specified.\n"); return -EINVAL; @@ -20824,17 +21019,17 @@ static int check_pseudo_btf_id(struct bpf_verifier_env *env, verbose(env, "kernel is missing BTF, make sure CONFIG_DEBUG_INFO_BTF=y is specified in Kconfig.\n"); return -EINVAL; } + btf_get(btf_vmlinux); btf = btf_vmlinux; } err = __check_pseudo_btf_id(env, insn, aux, btf); - if (err) + if (err) { + btf_put(btf); return err; + } - err = __add_used_btf(env, btf); - if (err < 0) - return err; - return 0; + return __add_used_btf(env, btf); } static bool is_tracing_prog_type(enum bpf_prog_type type) @@ -24505,7 +24700,6 @@ BTF_ID(func, __x64_sys_exit_group) BTF_ID(func, do_exit) BTF_ID(func, do_group_exit) BTF_ID(func, kthread_complete_and_exit) -BTF_ID(func, kthread_exit) BTF_ID(func, make_task_dead) BTF_SET_END(noreturn_deny) @@ -24650,10 +24844,8 @@ static int add_fd_from_fd_array(struct bpf_verifier_env *env, int fd) btf = __btf_get_by_fd(f); if (!IS_ERR(btf)) { - err = __add_used_btf(env, btf); - if (err < 0) - return err; - return 0; + btf_get(btf); + return __add_used_btf(env, btf); } verbose(env, "fd %d is not pointing to valid bpf_map or btf\n", fd); @@ -24809,6 +25001,12 @@ static void compute_insn_live_regs(struct bpf_verifier_env *env, case BPF_JMP32: switch (code) { case BPF_JA: + def = 0; + if (BPF_SRC(insn->code) == BPF_X) + use = dst; + else + use = 0; + break; case BPF_JCOND: def = 0; use = 0; @@ -25076,15 +25274,18 @@ static int compute_scc(struct bpf_verifier_env *env) } /* * Assign SCC number only if component has two or more elements, - * or if component has a self reference. + * or if component has a self reference, or if instruction is a + * callback calling function (implicit loop). */ - assign_scc = stack[stack_sz - 1] != w; - for (j = 0; j < succ->cnt; ++j) { + assign_scc = stack[stack_sz - 1] != w; /* two or more elements? */ + for (j = 0; j < succ->cnt; ++j) { /* self reference? */ if (succ->items[j] == w) { assign_scc = true; break; } } + if (bpf_calls_callback(env, w)) /* implicit loop? */ + assign_scc = true; /* Pop component elements from stack */ do { t = stack[--stack_sz]; diff --git a/kernel/cgroup/cgroup.c b/kernel/cgroup/cgroup.c index 5f0d33b0491023..93701007649049 100644 --- a/kernel/cgroup/cgroup.c +++ b/kernel/cgroup/cgroup.c @@ -2126,6 +2126,7 @@ static void init_cgroup_housekeeping(struct cgroup *cgrp) #endif init_waitqueue_head(&cgrp->offline_waitq); + init_waitqueue_head(&cgrp->dying_populated_waitq); INIT_WORK(&cgrp->release_agent_work, cgroup1_release_agent); } @@ -2608,6 +2609,7 @@ static void cgroup_migrate_add_task(struct task_struct *task, mgctx->tset.nr_tasks++; + css_set_skip_task_iters(cset, task); list_move_tail(&task->cg_list, &cset->mg_tasks); if (list_empty(&cset->mg_node)) list_add_tail(&cset->mg_node, @@ -5108,6 +5110,12 @@ static void css_task_iter_advance(struct css_task_iter *it) return; task = list_entry(it->task_pos, struct task_struct, cg_list); + /* + * Hide tasks that are exiting but not yet removed. Keep zombie + * leaders with live threads visible. + */ + if ((task->flags & PF_EXITING) && !atomic_read(&task->signal->live)) + goto repeat; if (it->flags & CSS_TASK_ITER_PROCS) { /* if PROCS, skip over tasks which aren't group leaders */ @@ -6217,6 +6225,78 @@ static int cgroup_destroy_locked(struct cgroup *cgrp) return 0; }; +/** + * cgroup_drain_dying - wait for dying tasks to leave before rmdir + * @cgrp: the cgroup being removed + * + * cgroup.procs and cgroup.threads use css_task_iter which filters out + * PF_EXITING tasks so that userspace doesn't see tasks that have already been + * reaped via waitpid(). However, cgroup_has_tasks() - which tests whether the + * cgroup has non-empty css_sets - is only updated when dying tasks pass through + * cgroup_task_dead() in finish_task_switch(). This creates a window where + * cgroup.procs reads empty but cgroup_has_tasks() is still true, making rmdir + * fail with -EBUSY from cgroup_destroy_locked() even though userspace sees no + * tasks. + * + * This function aligns cgroup_has_tasks() with what userspace can observe. If + * cgroup_has_tasks() but the task iterator sees nothing (all remaining tasks are + * PF_EXITING), we wait for cgroup_task_dead() to finish processing them. As the + * window between PF_EXITING and cgroup_task_dead() is short, the wait is brief. + * + * This function only concerns itself with this cgroup's own dying tasks. + * Whether the cgroup has children is cgroup_destroy_locked()'s problem. + * + * Each cgroup_task_dead() kicks the waitqueue via cset->cgrp_links, and we + * retry the full check from scratch. + * + * Must be called with cgroup_mutex held. + */ +static int cgroup_drain_dying(struct cgroup *cgrp) + __releases(&cgroup_mutex) __acquires(&cgroup_mutex) +{ + struct css_task_iter it; + struct task_struct *task; + DEFINE_WAIT(wait); + + lockdep_assert_held(&cgroup_mutex); +retry: + if (!cgroup_has_tasks(cgrp)) + return 0; + + /* Same iterator as cgroup.threads - if any task is visible, it's busy */ + css_task_iter_start(&cgrp->self, 0, &it); + task = css_task_iter_next(&it); + css_task_iter_end(&it); + + if (task) + return -EBUSY; + + /* + * All remaining tasks are PF_EXITING and will pass through + * cgroup_task_dead() shortly. Wait for a kick and retry. + * + * cgroup_has_tasks() can't transition from false to true while we're + * holding cgroup_mutex, but the true to false transition happens + * under css_set_lock (via cgroup_task_dead()). We must retest and + * prepare_to_wait() under css_set_lock. Otherwise, the transition + * can happen between our first test and prepare_to_wait(), and we + * sleep with no one to wake us. + */ + spin_lock_irq(&css_set_lock); + if (!cgroup_has_tasks(cgrp)) { + spin_unlock_irq(&css_set_lock); + return 0; + } + prepare_to_wait(&cgrp->dying_populated_waitq, &wait, + TASK_UNINTERRUPTIBLE); + spin_unlock_irq(&css_set_lock); + mutex_unlock(&cgroup_mutex); + schedule(); + finish_wait(&cgrp->dying_populated_waitq, &wait); + mutex_lock(&cgroup_mutex); + goto retry; +} + int cgroup_rmdir(struct kernfs_node *kn) { struct cgroup *cgrp; @@ -6226,9 +6306,12 @@ int cgroup_rmdir(struct kernfs_node *kn) if (!cgrp) return 0; - ret = cgroup_destroy_locked(cgrp); - if (!ret) - TRACE_CGROUP_PATH(rmdir, cgrp); + ret = cgroup_drain_dying(cgrp); + if (!ret) { + ret = cgroup_destroy_locked(cgrp); + if (!ret) + TRACE_CGROUP_PATH(rmdir, cgrp); + } cgroup_kn_unlock(kn); return ret; @@ -6988,6 +7071,7 @@ void cgroup_task_exit(struct task_struct *tsk) static void do_cgroup_task_dead(struct task_struct *tsk) { + struct cgrp_cset_link *link; struct css_set *cset; unsigned long flags; @@ -7001,6 +7085,11 @@ static void do_cgroup_task_dead(struct task_struct *tsk) if (thread_group_leader(tsk) && atomic_read(&tsk->signal->live)) list_add_tail(&tsk->cg_list, &cset->dying_tasks); + /* kick cgroup_drain_dying() waiters, see cgroup_rmdir() */ + list_for_each_entry(link, &cset->cgrp_links, cgrp_link) + if (waitqueue_active(&link->cgrp->dying_populated_waitq)) + wake_up(&link->cgrp->dying_populated_waitq); + if (dl_task(tsk)) dec_dl_tasks_cs(tsk); diff --git a/kernel/cgroup/cpuset.c b/kernel/cgroup/cpuset.c index c06e2e96f79dcd..81b3165f1aaa15 100644 --- a/kernel/cgroup/cpuset.c +++ b/kernel/cgroup/cpuset.c @@ -603,33 +603,31 @@ static inline bool cpusets_are_exclusive(struct cpuset *cs1, struct cpuset *cs2) /** * cpus_excl_conflict - Check if two cpusets have exclusive CPU conflicts - * @cs1: first cpuset to check - * @cs2: second cpuset to check + * @trial: the trial cpuset to be checked + * @sibling: a sibling cpuset to be checked against + * @xcpus_changed: set if exclusive_cpus has been set * * Returns: true if CPU exclusivity conflict exists, false otherwise * * Conflict detection rules: * 1. If either cpuset is CPU exclusive, they must be mutually exclusive * 2. exclusive_cpus masks cannot intersect between cpusets - * 3. The allowed CPUs of one cpuset cannot be a subset of another's exclusive CPUs + * 3. The allowed CPUs of a sibling cpuset cannot be a subset of the new exclusive CPUs */ -static inline bool cpus_excl_conflict(struct cpuset *cs1, struct cpuset *cs2) +static inline bool cpus_excl_conflict(struct cpuset *trial, struct cpuset *sibling, + bool xcpus_changed) { /* If either cpuset is exclusive, check if they are mutually exclusive */ - if (is_cpu_exclusive(cs1) || is_cpu_exclusive(cs2)) - return !cpusets_are_exclusive(cs1, cs2); + if (is_cpu_exclusive(trial) || is_cpu_exclusive(sibling)) + return !cpusets_are_exclusive(trial, sibling); /* Exclusive_cpus cannot intersect */ - if (cpumask_intersects(cs1->exclusive_cpus, cs2->exclusive_cpus)) + if (cpumask_intersects(trial->exclusive_cpus, sibling->exclusive_cpus)) return true; - /* The cpus_allowed of one cpuset cannot be a subset of another cpuset's exclusive_cpus */ - if (!cpumask_empty(cs1->cpus_allowed) && - cpumask_subset(cs1->cpus_allowed, cs2->exclusive_cpus)) - return true; - - if (!cpumask_empty(cs2->cpus_allowed) && - cpumask_subset(cs2->cpus_allowed, cs1->exclusive_cpus)) + /* The cpus_allowed of a sibling cpuset cannot be a subset of the new exclusive_cpus */ + if (xcpus_changed && !cpumask_empty(sibling->cpus_allowed) && + cpumask_subset(sibling->cpus_allowed, trial->exclusive_cpus)) return true; return false; @@ -666,6 +664,7 @@ static int validate_change(struct cpuset *cur, struct cpuset *trial) { struct cgroup_subsys_state *css; struct cpuset *c, *par; + bool xcpus_changed; int ret = 0; rcu_read_lock(); @@ -722,10 +721,11 @@ static int validate_change(struct cpuset *cur, struct cpuset *trial) * overlap. exclusive_cpus cannot overlap with each other if set. */ ret = -EINVAL; + xcpus_changed = !cpumask_equal(cur->exclusive_cpus, trial->exclusive_cpus); cpuset_for_each_child(c, css, par) { if (c == cur) continue; - if (cpus_excl_conflict(trial, c)) + if (cpus_excl_conflict(trial, c, xcpus_changed)) goto out; if (mems_excl_conflict(trial, c)) goto out; @@ -1401,8 +1401,8 @@ static void partition_xcpus_del(int old_prs, struct cpuset *parent, isolated_cpus_update(old_prs, parent->partition_root_state, xcpus); - cpumask_and(xcpus, xcpus, cpu_active_mask); cpumask_or(parent->effective_cpus, parent->effective_cpus, xcpus); + cpumask_and(parent->effective_cpus, parent->effective_cpus, cpu_active_mask); } /* @@ -2350,7 +2350,7 @@ static void update_cpumasks_hier(struct cpuset *cs, struct tmpmasks *tmp, WARN_ON(!is_in_v2_mode() && !cpumask_equal(cp->cpus_allowed, cp->effective_cpus)); - cpuset_update_tasks_cpumask(cp, cp->effective_cpus); + cpuset_update_tasks_cpumask(cp, tmp->new_cpus); /* * On default hierarchy, inherit the CS_SCHED_LOAD_BALANCE @@ -4424,40 +4424,58 @@ bool cpuset_current_node_allowed(int node, gfp_t gfp_mask) return allowed; } -bool cpuset_node_allowed(struct cgroup *cgroup, int nid) +/** + * cpuset_nodes_allowed - return effective_mems mask from a cgroup cpuset. + * @cgroup: pointer to struct cgroup. + * @mask: pointer to struct nodemask_t to be returned. + * + * Returns effective_mems mask from a cgroup cpuset if it is cgroup v2 and + * has cpuset subsys. Otherwise, returns node_states[N_MEMORY]. + * + * This function intentionally avoids taking the cpuset_mutex or callback_lock + * when accessing effective_mems. This is because the obtained effective_mems + * is stale immediately after the query anyway (e.g., effective_mems is updated + * immediately after releasing the lock but before returning). + * + * As a result, returned @mask may be empty because cs->effective_mems can be + * rebound during this call. Besides, nodes in @mask are not guaranteed to be + * online due to hot plugins. Callers should check the mask for validity on + * return based on its subsequent use. + **/ +void cpuset_nodes_allowed(struct cgroup *cgroup, nodemask_t *mask) { struct cgroup_subsys_state *css; struct cpuset *cs; - bool allowed; /* * In v1, mem_cgroup and cpuset are unlikely in the same hierarchy * and mems_allowed is likely to be empty even if we could get to it, - * so return true to avoid taking a global lock on the empty check. + * so return directly to avoid taking a global lock on the empty check. */ - if (!cpuset_v2()) - return true; + if (!cgroup || !cpuset_v2()) { + nodes_copy(*mask, node_states[N_MEMORY]); + return; + } css = cgroup_get_e_css(cgroup, &cpuset_cgrp_subsys); - if (!css) - return true; + if (!css) { + nodes_copy(*mask, node_states[N_MEMORY]); + return; + } /* - * Normally, accessing effective_mems would require the cpuset_mutex - * or callback_lock - but node_isset is atomic and the reference - * taken via cgroup_get_e_css is sufficient to protect css. - * - * Since this interface is intended for use by migration paths, we - * relax locking here to avoid taking global locks - while accepting - * there may be rare scenarios where the result may be innaccurate. + * The reference taken via cgroup_get_e_css is sufficient to + * protect css, but it does not imply safe accesses to effective_mems. * - * Reclaim and migration are subject to these same race conditions, and - * cannot make strong isolation guarantees, so this is acceptable. + * Normally, accessing effective_mems would require the cpuset_mutex + * or callback_lock - but the correctness of this information is stale + * immediately after the query anyway. We do not acquire the lock + * during this process to save lock contention in exchange for racing + * against mems_allowed rebinds. */ cs = container_of(css, struct cpuset, css); - allowed = node_isset(nid, cs->effective_mems); + nodes_copy(*mask, cs->effective_mems); css_put(css); - return allowed; } /** diff --git a/kernel/configs/debug.config b/kernel/configs/debug.config index 9f6ab7dabf672e..0a6c1763d976ed 100644 --- a/kernel/configs/debug.config +++ b/kernel/configs/debug.config @@ -29,7 +29,6 @@ CONFIG_SECTION_MISMATCH_WARN_ONLY=y # CONFIG_UBSAN_ALIGNMENT is not set # CONFIG_UBSAN_DIV_ZERO is not set # CONFIG_UBSAN_TRAP is not set -# CONFIG_WARN_ALL_UNSEEDED_RANDOM is not set CONFIG_DEBUG_FS=y CONFIG_DEBUG_FS_ALLOW_ALL=y CONFIG_DEBUG_IRQFLAGS=y diff --git a/kernel/crash_dump_dm_crypt.c b/kernel/crash_dump_dm_crypt.c index 401423ba477da0..9501b0704f19f6 100644 --- a/kernel/crash_dump_dm_crypt.c +++ b/kernel/crash_dump_dm_crypt.c @@ -143,6 +143,7 @@ static int read_key_from_user_keying(struct dm_crypt_key *dm_key) { const struct user_key_payload *ukp; struct key *key; + int ret = 0; kexec_dprintk("Requesting logon key %s", dm_key->key_desc); key = request_key(&key_type_logon, dm_key->key_desc, NULL); @@ -152,20 +153,28 @@ static int read_key_from_user_keying(struct dm_crypt_key *dm_key) return PTR_ERR(key); } + down_read(&key->sem); ukp = user_key_payload_locked(key); - if (!ukp) - return -EKEYREVOKED; + if (!ukp) { + ret = -EKEYREVOKED; + goto out; + } if (ukp->datalen > KEY_SIZE_MAX) { pr_err("Key size %u exceeds maximum (%u)\n", ukp->datalen, KEY_SIZE_MAX); - return -EINVAL; + ret = -EINVAL; + goto out; } memcpy(dm_key->data, ukp->data, ukp->datalen); dm_key->key_size = ukp->datalen; - kexec_dprintk("Get dm crypt key (size=%u) %s: %8ph\n", dm_key->key_size, - dm_key->key_desc, dm_key->data); - return 0; + kexec_dprintk("Get dm crypt key (size=%u) %s\n", dm_key->key_size, + dm_key->key_desc); + +out: + up_read(&key->sem); + key_put(key); + return ret; } struct config_key { diff --git a/kernel/dma/debug.c b/kernel/dma/debug.c index 138ede653de400..21db331185911e 100644 --- a/kernel/dma/debug.c +++ b/kernel/dma/debug.c @@ -63,6 +63,7 @@ enum map_err_types { * @sg_mapped_ents: 'mapped_ents' from dma_map_sg * @paddr: physical start address of the mapping * @map_err_type: track whether dma_mapping_error() was checked + * @is_cache_clean: driver promises not to write to buffer while mapped * @stack_len: number of backtrace entries in @stack_entries * @stack_entries: stack of backtrace history */ @@ -76,7 +77,8 @@ struct dma_debug_entry { int sg_call_ents; int sg_mapped_ents; phys_addr_t paddr; - enum map_err_types map_err_type; + enum map_err_types map_err_type; + bool is_cache_clean; #ifdef CONFIG_STACKTRACE unsigned int stack_len; unsigned long stack_entries[DMA_DEBUG_STACKTRACE_ENTRIES]; @@ -451,7 +453,7 @@ static int active_cacheline_set_overlap(phys_addr_t cln, int overlap) return overlap; } -static void active_cacheline_inc_overlap(phys_addr_t cln) +static void active_cacheline_inc_overlap(phys_addr_t cln, bool is_cache_clean) { int overlap = active_cacheline_read_overlap(cln); @@ -460,7 +462,7 @@ static void active_cacheline_inc_overlap(phys_addr_t cln) /* If we overflowed the overlap counter then we're potentially * leaking dma-mappings. */ - WARN_ONCE(overlap > ACTIVE_CACHELINE_MAX_OVERLAP, + WARN_ONCE(!is_cache_clean && overlap > ACTIVE_CACHELINE_MAX_OVERLAP, pr_fmt("exceeded %d overlapping mappings of cacheline %pa\n"), ACTIVE_CACHELINE_MAX_OVERLAP, &cln); } @@ -472,12 +474,15 @@ static int active_cacheline_dec_overlap(phys_addr_t cln) return active_cacheline_set_overlap(cln, --overlap); } -static int active_cacheline_insert(struct dma_debug_entry *entry) +static int active_cacheline_insert(struct dma_debug_entry *entry, + bool *overlap_cache_clean) { phys_addr_t cln = to_cacheline_number(entry); unsigned long flags; int rc; + *overlap_cache_clean = false; + /* If the device is not writing memory then we don't have any * concerns about the cpu consuming stale data. This mitigates * legitimate usages of overlapping mappings. @@ -487,8 +492,16 @@ static int active_cacheline_insert(struct dma_debug_entry *entry) spin_lock_irqsave(&radix_lock, flags); rc = radix_tree_insert(&dma_active_cacheline, cln, entry); - if (rc == -EEXIST) - active_cacheline_inc_overlap(cln); + if (rc == -EEXIST) { + struct dma_debug_entry *existing; + + active_cacheline_inc_overlap(cln, entry->is_cache_clean); + existing = radix_tree_lookup(&dma_active_cacheline, cln); + /* A lookup failure here after we got -EEXIST is unexpected. */ + WARN_ON(!existing); + if (existing) + *overlap_cache_clean = existing->is_cache_clean; + } spin_unlock_irqrestore(&radix_lock, flags); return rc; @@ -583,19 +596,25 @@ DEFINE_SHOW_ATTRIBUTE(dump); */ static void add_dma_entry(struct dma_debug_entry *entry, unsigned long attrs) { + bool overlap_cache_clean; struct hash_bucket *bucket; unsigned long flags; int rc; + entry->is_cache_clean = !!(attrs & DMA_ATTR_CPU_CACHE_CLEAN); + bucket = get_hash_bucket(entry, &flags); hash_bucket_add(bucket, entry); put_hash_bucket(bucket, flags); - rc = active_cacheline_insert(entry); + rc = active_cacheline_insert(entry, &overlap_cache_clean); if (rc == -ENOMEM) { pr_err_once("cacheline tracking ENOMEM, dma-debug disabled\n"); global_disable = true; - } else if (rc == -EEXIST && !(attrs & DMA_ATTR_SKIP_CPU_SYNC) && + } else if (rc == -EEXIST && + !(attrs & DMA_ATTR_SKIP_CPU_SYNC) && + !(entry->is_cache_clean && overlap_cache_clean) && + dma_get_cache_alignment() >= L1_CACHE_BYTES && !(IS_ENABLED(CONFIG_DMA_BOUNCE_UNALIGNED_KMALLOC) && is_swiotlb_active(entry->dev))) { err_printk(entry->dev, entry, diff --git a/kernel/dma/direct.h b/kernel/dma/direct.h index da2fadf45bcd6c..62f0d9d0ba02ef 100644 --- a/kernel/dma/direct.h +++ b/kernel/dma/direct.h @@ -88,7 +88,7 @@ static inline dma_addr_t dma_direct_map_phys(struct device *dev, if (is_swiotlb_force_bounce(dev)) { if (attrs & DMA_ATTR_MMIO) - goto err_overflow; + return DMA_MAPPING_ERROR; return swiotlb_map(dev, phys, size, dir, attrs); } diff --git a/kernel/dma/swiotlb.c b/kernel/dma/swiotlb.c index a547c7693135b6..b4bc7ce01dadcc 100644 --- a/kernel/dma/swiotlb.c +++ b/kernel/dma/swiotlb.c @@ -30,6 +30,7 @@ #include #include #include +#include #include #include #include @@ -901,10 +902,19 @@ static void swiotlb_bounce(struct device *dev, phys_addr_t tlb_addr, size_t size local_irq_save(flags); page = pfn_to_page(pfn); - if (dir == DMA_TO_DEVICE) + if (dir == DMA_TO_DEVICE) { + /* + * Ideally, kmsan_check_highmem_page() + * could be used here to detect infoleaks, + * but callers may map uninitialized buffers + * that will be written by the device, + * causing false positives. + */ memcpy_from_page(vaddr, page, offset, sz); - else + } else { + kmsan_unpoison_memory(vaddr, sz); memcpy_to_page(page, offset, vaddr, sz); + } local_irq_restore(flags); size -= sz; @@ -913,8 +923,15 @@ static void swiotlb_bounce(struct device *dev, phys_addr_t tlb_addr, size_t size offset = 0; } } else if (dir == DMA_TO_DEVICE) { + /* + * Ideally, kmsan_check_memory() could be used here to detect + * infoleaks (uninitialized data being sent to device), but + * callers may map uninitialized buffers that will be written + * by the device, causing false positives. + */ memcpy(vaddr, phys_to_virt(orig_addr), size); } else { + kmsan_unpoison_memory(vaddr, size); memcpy(phys_to_virt(orig_addr), vaddr, size); } } diff --git a/kernel/events/core.c b/kernel/events/core.c index 8cca8009462481..39b35f280845b2 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -4017,7 +4017,8 @@ static int merge_sched_in(struct perf_event *event, void *data) if (*perf_event_fasync(event)) event->pending_kill = POLL_ERR; - perf_event_wakeup(event); + event->pending_wakeup = 1; + irq_work_queue(&event->pending_irq); } else { struct perf_cpu_pmu_context *cpc = this_cpc(event->pmu_ctx->pmu); @@ -4671,7 +4672,7 @@ static void __perf_event_read(void *info) struct perf_event *sub, *event = data->event; struct perf_event_context *ctx = event->ctx; struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context); - struct pmu *pmu = event->pmu; + struct pmu *pmu; /* * If this is a task context, we need to check whether it is @@ -4683,7 +4684,7 @@ static void __perf_event_read(void *info) if (ctx->task && cpuctx->task_ctx != ctx) return; - raw_spin_lock(&ctx->lock); + guard(raw_spinlock)(&ctx->lock); ctx_time_update_event(ctx, event); perf_event_update_time(event); @@ -4691,25 +4692,22 @@ static void __perf_event_read(void *info) perf_event_update_sibling_time(event); if (event->state != PERF_EVENT_STATE_ACTIVE) - goto unlock; + return; if (!data->group) { - pmu->read(event); + perf_pmu_read(event); data->ret = 0; - goto unlock; + return; } + pmu = event->pmu_ctx->pmu; pmu->start_txn(pmu, PERF_PMU_TXN_READ); - pmu->read(event); - + perf_pmu_read(event); for_each_sibling_event(sub, event) perf_pmu_read(sub); data->ret = pmu->commit_txn(pmu); - -unlock: - raw_spin_unlock(&ctx->lock); } static inline u64 perf_event_count(struct perf_event *event, bool self) @@ -5280,9 +5278,20 @@ attach_task_ctx_data(struct task_struct *task, struct kmem_cache *ctx_cache, return -ENOMEM; for (;;) { - if (try_cmpxchg((struct perf_ctx_data **)&task->perf_ctx_data, &old, cd)) { + if (try_cmpxchg(&task->perf_ctx_data, &old, cd)) { if (old) perf_free_ctx_data_rcu(old); + /* + * Above try_cmpxchg() pairs with try_cmpxchg() from + * detach_task_ctx_data() such that + * if we race with perf_event_exit_task(), we must + * observe PF_EXITING. + */ + if (task->flags & PF_EXITING) { + /* detach_task_ctx_data() may free it already */ + if (try_cmpxchg(&task->perf_ctx_data, &cd, NULL)) + perf_free_ctx_data_rcu(cd); + } return 0; } @@ -5328,6 +5337,8 @@ attach_global_ctx_data(struct kmem_cache *ctx_cache) /* Allocate everything */ scoped_guard (rcu) { for_each_process_thread(g, p) { + if (p->flags & PF_EXITING) + continue; cd = rcu_dereference(p->perf_ctx_data); if (cd && !cd->global) { cd->global = 1; @@ -7175,28 +7186,28 @@ static int perf_mmap(struct file *file, struct vm_area_struct *vma) ret = perf_mmap_aux(vma, event, nr_pages); if (ret) return ret; - } - /* - * Since pinned accounting is per vm we cannot allow fork() to copy our - * vma. - */ - vm_flags_set(vma, VM_DONTCOPY | VM_DONTEXPAND | VM_DONTDUMP); - vma->vm_ops = &perf_mmap_vmops; + /* + * Since pinned accounting is per vm we cannot allow fork() to copy our + * vma. + */ + vm_flags_set(vma, VM_DONTCOPY | VM_DONTEXPAND | VM_DONTDUMP); + vma->vm_ops = &perf_mmap_vmops; - mapped = get_mapped(event, event_mapped); - if (mapped) - mapped(event, vma->vm_mm); + mapped = get_mapped(event, event_mapped); + if (mapped) + mapped(event, vma->vm_mm); - /* - * Try to map it into the page table. On fail, invoke - * perf_mmap_close() to undo the above, as the callsite expects - * full cleanup in this case and therefore does not invoke - * vmops::close(). - */ - ret = map_range(event->rb, vma); - if (ret) - perf_mmap_close(vma); + /* + * Try to map it into the page table. On fail, invoke + * perf_mmap_close() to undo the above, as the callsite expects + * full cleanup in this case and therefore does not invoke + * vmops::close(). + */ + ret = map_range(event->rb, vma); + if (ret) + perf_mmap_close(vma); + } return ret; } @@ -10484,6 +10495,13 @@ int perf_event_overflow(struct perf_event *event, struct perf_sample_data *data, struct pt_regs *regs) { + /* + * Entry point from hardware PMI, interrupts should be disabled here. + * This serializes us against perf_event_remove_from_context() in + * things like perf_event_release_kernel(). + */ + lockdep_assert_irqs_disabled(); + return __perf_event_overflow(event, 1, data, regs); } @@ -10560,6 +10578,19 @@ static void perf_swevent_event(struct perf_event *event, u64 nr, { struct hw_perf_event *hwc = &event->hw; + /* + * This is: + * - software preempt + * - tracepoint preempt + * - tp_target_task irq (ctx->lock) + * - uprobes preempt/irq + * - kprobes preempt/irq + * - hw_breakpoint irq + * + * Any of these are sufficient to hold off RCU and thus ensure @event + * exists. + */ + lockdep_assert_preemption_disabled(); local64_add(nr, &event->count); if (!regs) @@ -10568,6 +10599,16 @@ static void perf_swevent_event(struct perf_event *event, u64 nr, if (!is_sampling_event(event)) return; + /* + * Serialize against event_function_call() IPIs like normal overflow + * event handling. Specifically, must not allow + * perf_event_release_kernel() -> perf_remove_from_context() to make + * progress and 'release' the event from under us. + */ + guard(irqsave)(); + if (event->state != PERF_EVENT_STATE_ACTIVE) + return; + if ((event->attr.sample_type & PERF_SAMPLE_PERIOD) && !event->attr.freq) { data->period = nr; return perf_swevent_overflow(event, 1, data, regs); @@ -11066,6 +11107,11 @@ void perf_tp_event(u16 event_type, u64 count, void *record, int entry_size, struct perf_sample_data data; struct perf_event *event; + /* + * Per being a tracepoint, this runs with preemption disabled. + */ + lockdep_assert_preemption_disabled(); + struct perf_raw_record raw = { .frag = { .size = entry_size, @@ -11398,6 +11444,11 @@ void perf_bp_event(struct perf_event *bp, void *data) struct perf_sample_data sample; struct pt_regs *regs = data; + /* + * Exception context, will have interrupts disabled. + */ + lockdep_assert_irqs_disabled(); + perf_sample_data_init(&sample, bp->attr.bp_addr, 0); if (!bp->hw.state && !perf_exclude_event(bp, regs)) @@ -11862,7 +11913,7 @@ static enum hrtimer_restart perf_swevent_hrtimer(struct hrtimer *hrtimer) if (regs && !perf_exclude_event(event, regs)) { if (!(event->attr.exclude_idle && is_idle_task(current))) - if (__perf_event_overflow(event, 1, &data, regs)) + if (perf_event_overflow(event, &data, regs)) ret = HRTIMER_NORESTART; } @@ -14294,8 +14345,11 @@ void perf_event_exit_task(struct task_struct *task) /* * Detach the perf_ctx_data for the system-wide event. + * + * Done without holding global_ctx_data_rwsem; typically + * attach_global_ctx_data() will skip over this task, but otherwise + * attach_task_ctx_data() will observe PF_EXITING. */ - guard(percpu_read)(&global_ctx_data_rwsem); detach_task_ctx_data(task); } @@ -14404,7 +14458,7 @@ inherit_event(struct perf_event *parent_event, get_ctx(child_ctx); child_event->ctx = child_ctx; - pmu_ctx = find_get_pmu_context(child_event->pmu, child_ctx, child_event); + pmu_ctx = find_get_pmu_context(parent_event->pmu_ctx->pmu, child_ctx, child_event); if (IS_ERR(pmu_ctx)) { free_event(child_event); return ERR_CAST(pmu_ctx); diff --git a/kernel/events/uprobes.c b/kernel/events/uprobes.c index d546d32390a81d..3ec996ca6de0d5 100644 --- a/kernel/events/uprobes.c +++ b/kernel/events/uprobes.c @@ -1138,7 +1138,7 @@ static bool filter_chain(struct uprobe *uprobe, struct mm_struct *mm) bool ret = false; down_read(&uprobe->consumer_rwsem); - list_for_each_entry_rcu(uc, &uprobe->consumers, cons_node, rcu_read_lock_trace_held()) { + list_for_each_entry(uc, &uprobe->consumers, cons_node) { ret = consumer_filter(uc, mm); if (ret) break; @@ -1694,6 +1694,12 @@ static const struct vm_special_mapping xol_mapping = { .mremap = xol_mremap, }; +unsigned long __weak arch_uprobe_get_xol_area(void) +{ + /* Try to map as high as possible, this is only a hint. */ + return get_unmapped_area(NULL, TASK_SIZE - PAGE_SIZE, PAGE_SIZE, 0, 0); +} + /* Slot allocation for XOL */ static int xol_add_vma(struct mm_struct *mm, struct xol_area *area) { @@ -1709,9 +1715,7 @@ static int xol_add_vma(struct mm_struct *mm, struct xol_area *area) } if (!area->vaddr) { - /* Try to map as high as possible, this is only a hint. */ - area->vaddr = get_unmapped_area(NULL, TASK_SIZE - PAGE_SIZE, - PAGE_SIZE, 0, 0); + area->vaddr = arch_uprobe_get_xol_area(); if (IS_ERR_VALUE(area->vaddr)) { ret = area->vaddr; goto fail; diff --git a/kernel/exit.c b/kernel/exit.c index 8a87021211ae72..ede3117fa7d413 100644 --- a/kernel/exit.c +++ b/kernel/exit.c @@ -896,11 +896,16 @@ static void synchronize_group_exit(struct task_struct *tsk, long code) void __noreturn do_exit(long code) { struct task_struct *tsk = current; + struct kthread *kthread; int group_dead; WARN_ON(irqs_disabled()); WARN_ON(tsk->plug); + kthread = tsk_is_kthread(tsk); + if (unlikely(kthread)) + kthread_do_exit(kthread, code); + kcov_task_exit(tsk); kmsan_task_exit(tsk); @@ -1013,6 +1018,7 @@ void __noreturn do_exit(long code) lockdep_free_task(tsk); do_task_dead(); } +EXPORT_SYMBOL(do_exit); void __noreturn make_task_dead(int signr) { diff --git a/kernel/fork.c b/kernel/fork.c index b1f3915d5f8ec8..5b45887435dcc0 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -999,6 +999,7 @@ static struct task_struct *dup_task_struct(struct task_struct *orig, int node) #ifdef CONFIG_SCHED_MM_CID tsk->mm_cid.cid = MM_CID_UNSET; tsk->mm_cid.active = 0; + INIT_HLIST_NODE(&tsk->mm_cid.node); #endif return tsk; @@ -1585,7 +1586,6 @@ static int copy_mm(u64 clone_flags, struct task_struct *tsk) tsk->mm = mm; tsk->active_mm = mm; - sched_mm_cid_fork(tsk); return 0; } @@ -2496,7 +2496,6 @@ __latent_entropy struct task_struct *copy_process( exit_nsproxy_namespaces(p); bad_fork_cleanup_mm: if (p->mm) { - sched_mm_cid_exit(p); mm_clear_owner(p->mm, p); mmput(p->mm); } @@ -3082,7 +3081,7 @@ static int unshare_fs(unsigned long unshare_flags, struct fs_struct **new_fsp) return 0; /* don't need lock here; in the worst case we'll do useless copy */ - if (fs->users == 1) + if (!(unshare_flags & CLONE_NEWNS) && fs->users == 1) return 0; *new_fsp = copy_fs_struct(fs); diff --git a/kernel/futex/core.c b/kernel/futex/core.c index cf7e610eac4297..31e83a09789e00 100644 --- a/kernel/futex/core.c +++ b/kernel/futex/core.c @@ -342,7 +342,7 @@ static int __futex_key_to_node(struct mm_struct *mm, unsigned long addr) if (!vma) return FUTEX_NO_NODE; - mpol = vma_policy(vma); + mpol = READ_ONCE(vma->vm_policy); if (!mpol) return FUTEX_NO_NODE; diff --git a/kernel/futex/pi.c b/kernel/futex/pi.c index dacb2330f1fbcc..64cb87d3a73e87 100644 --- a/kernel/futex/pi.c +++ b/kernel/futex/pi.c @@ -918,7 +918,7 @@ int fixup_pi_owner(u32 __user *uaddr, struct futex_q *q, int locked) int futex_lock_pi(u32 __user *uaddr, unsigned int flags, ktime_t *time, int trylock) { struct hrtimer_sleeper timeout, *to; - struct task_struct *exiting = NULL; + struct task_struct *exiting; struct rt_mutex_waiter rt_waiter; struct futex_q q = futex_q_init; DEFINE_WAKE_Q(wake_q); @@ -933,6 +933,7 @@ int futex_lock_pi(u32 __user *uaddr, unsigned int flags, ktime_t *time, int tryl to = futex_setup_timer(time, &timeout, flags, 0); retry: + exiting = NULL; ret = get_futex_key(uaddr, flags, &q.key, FUTEX_WRITE); if (unlikely(ret != 0)) goto out; diff --git a/kernel/futex/syscalls.c b/kernel/futex/syscalls.c index 880c9bf2f31504..99723189c8cf70 100644 --- a/kernel/futex/syscalls.c +++ b/kernel/futex/syscalls.c @@ -459,6 +459,14 @@ SYSCALL_DEFINE4(futex_requeue, if (ret) return ret; + /* + * For now mandate both flags are identical, like the sys_futex() + * interface has. If/when we merge the variable sized futex support, + * that patch can modify this test to allow a difference in size. + */ + if (futexes[0].w.flags != futexes[1].w.flags) + return -EINVAL; + cmpval = futexes[0].w.val; return futex_requeue(u64_to_user_ptr(futexes[0].w.uaddr), futexes[0].w.flags, diff --git a/kernel/irq/cpuhotplug.c b/kernel/irq/cpuhotplug.c index 755346ea981967..cd5689e383b003 100644 --- a/kernel/irq/cpuhotplug.c +++ b/kernel/irq/cpuhotplug.c @@ -177,9 +177,11 @@ void irq_migrate_all_off_this_cpu(void) bool affinity_broken; desc = irq_to_desc(irq); - scoped_guard(raw_spinlock, &desc->lock) + scoped_guard(raw_spinlock, &desc->lock) { affinity_broken = migrate_one_irq(desc); - + if (affinity_broken && desc->affinity_notify) + irq_affinity_schedule_notify_work(desc); + } if (affinity_broken) { pr_debug_ratelimited("IRQ %u: no longer affine to CPU%u\n", irq, smp_processor_id()); diff --git a/kernel/irq/internals.h b/kernel/irq/internals.h index 0164ca48da59e0..5568ed3a8b8520 100644 --- a/kernel/irq/internals.h +++ b/kernel/irq/internals.h @@ -135,6 +135,7 @@ extern bool irq_can_set_affinity_usr(unsigned int irq); extern int irq_do_set_affinity(struct irq_data *data, const struct cpumask *dest, bool force); +extern void irq_affinity_schedule_notify_work(struct irq_desc *desc); #ifdef CONFIG_SMP extern int irq_setup_affinity(struct irq_desc *desc); @@ -142,7 +143,6 @@ extern int irq_setup_affinity(struct irq_desc *desc); static inline int irq_setup_affinity(struct irq_desc *desc) { return 0; } #endif - #define for_each_action_of_desc(desc, act) \ for (act = desc->action; act; act = act->next) diff --git a/kernel/irq/manage.c b/kernel/irq/manage.c index 349ae7979da0e3..4873b0f73df96f 100644 --- a/kernel/irq/manage.c +++ b/kernel/irq/manage.c @@ -347,6 +347,21 @@ static bool irq_set_affinity_deactivated(struct irq_data *data, return true; } +/** + * irq_affinity_schedule_notify_work - Schedule work to notify about affinity change + * @desc: Interrupt descriptor whose affinity changed + */ +void irq_affinity_schedule_notify_work(struct irq_desc *desc) +{ + lockdep_assert_held(&desc->lock); + + kref_get(&desc->affinity_notify->kref); + if (!schedule_work(&desc->affinity_notify->work)) { + /* Work was already scheduled, drop our extra ref */ + kref_put(&desc->affinity_notify->kref, desc->affinity_notify->release); + } +} + int irq_set_affinity_locked(struct irq_data *data, const struct cpumask *mask, bool force) { @@ -367,14 +382,9 @@ int irq_set_affinity_locked(struct irq_data *data, const struct cpumask *mask, irq_copy_pending(desc, mask); } - if (desc->affinity_notify) { - kref_get(&desc->affinity_notify->kref); - if (!schedule_work(&desc->affinity_notify->work)) { - /* Work was already scheduled, drop our extra ref */ - kref_put(&desc->affinity_notify->kref, - desc->affinity_notify->release); - } - } + if (desc->affinity_notify) + irq_affinity_schedule_notify_work(desc); + irqd_set(data, IRQD_AFFINITY_SET); return ret; diff --git a/kernel/kallsyms.c b/kernel/kallsyms.c index 049e296f586ccd..e0813ca9469a3d 100644 --- a/kernel/kallsyms.c +++ b/kernel/kallsyms.c @@ -345,7 +345,7 @@ int kallsyms_lookup_size_offset(unsigned long addr, unsigned long *symbolsize, return 1; } return !!module_address_lookup(addr, symbolsize, offset, NULL, NULL, namebuf) || - !!__bpf_address_lookup(addr, symbolsize, offset, namebuf); + !!bpf_address_lookup(addr, symbolsize, offset, namebuf); } static int kallsyms_lookup_buildid(unsigned long addr, @@ -355,8 +355,21 @@ static int kallsyms_lookup_buildid(unsigned long addr, { int ret; - namebuf[KSYM_NAME_LEN - 1] = 0; + /* + * kallsyms_lookus() returns pointer to namebuf on success and + * NULL on error. But some callers ignore the return value. + * Instead they expect @namebuf filled either with valid + * or empty string. + */ namebuf[0] = 0; + /* + * Initialize the module-related return values. They are not set + * when the symbol is in vmlinux or it is a bpf address. + */ + if (modname) + *modname = NULL; + if (modbuildid) + *modbuildid = NULL; if (is_ksym_addr(addr)) { unsigned long pos; @@ -365,10 +378,6 @@ static int kallsyms_lookup_buildid(unsigned long addr, /* Grab name */ kallsyms_expand_symbol(get_symbol_offset(pos), namebuf, KSYM_NAME_LEN); - if (modname) - *modname = NULL; - if (modbuildid) - *modbuildid = NULL; return strlen(namebuf); } @@ -377,12 +386,11 @@ static int kallsyms_lookup_buildid(unsigned long addr, ret = module_address_lookup(addr, symbolsize, offset, modname, modbuildid, namebuf); if (!ret) - ret = bpf_address_lookup(addr, symbolsize, - offset, modname, namebuf); + ret = bpf_address_lookup(addr, symbolsize, offset, namebuf); if (!ret) - ret = ftrace_mod_address_lookup(addr, symbolsize, - offset, modname, namebuf); + ret = ftrace_mod_address_lookup(addr, symbolsize, offset, + modname, modbuildid, namebuf); return ret; } @@ -426,6 +434,37 @@ int lookup_symbol_name(unsigned long addr, char *symname) return lookup_module_symbol_name(addr, symname); } +#ifdef CONFIG_STACKTRACE_BUILD_ID + +static int append_buildid(char *buffer, const char *modname, + const unsigned char *buildid) +{ + if (!modname) + return 0; + + if (!buildid) { + pr_warn_once("Undefined buildid for the module %s\n", modname); + return 0; + } + + /* build ID should match length of sprintf */ +#ifdef CONFIG_MODULES + static_assert(sizeof(typeof_member(struct module, build_id)) == 20); +#endif + + return sprintf(buffer, " %20phN", buildid); +} + +#else /* CONFIG_STACKTRACE_BUILD_ID */ + +static int append_buildid(char *buffer, const char *modname, + const unsigned char *buildid) +{ + return 0; +} + +#endif /* CONFIG_STACKTRACE_BUILD_ID */ + /* Look up a kernel symbol and return it in a text buffer. */ static int __sprint_symbol(char *buffer, unsigned long address, int symbol_offset, int add_offset, int add_buildid) @@ -435,6 +474,9 @@ static int __sprint_symbol(char *buffer, unsigned long address, unsigned long offset, size; int len; + /* Prevent module removal until modname and modbuildid are printed */ + guard(rcu)(); + address += symbol_offset; len = kallsyms_lookup_buildid(address, &size, &offset, &modname, &buildid, buffer); @@ -448,15 +490,8 @@ static int __sprint_symbol(char *buffer, unsigned long address, if (modname) { len += sprintf(buffer + len, " [%s", modname); -#if IS_ENABLED(CONFIG_STACKTRACE_BUILD_ID) - if (add_buildid && buildid) { - /* build ID should match length of sprintf */ -#if IS_ENABLED(CONFIG_MODULES) - static_assert(sizeof(typeof_member(struct module, build_id)) == 20); -#endif - len += sprintf(buffer + len, " %20phN", buildid); - } -#endif + if (add_buildid) + len += append_buildid(buffer + len, modname, buildid); len += sprintf(buffer + len, "]"); } diff --git a/kernel/kexec_file.c b/kernel/kexec_file.c index eb62a97942428c..2bfbb2d144e69b 100644 --- a/kernel/kexec_file.c +++ b/kernel/kexec_file.c @@ -882,6 +882,60 @@ static int kexec_calculate_store_digests(struct kimage *image) } #ifdef CONFIG_ARCH_SUPPORTS_KEXEC_PURGATORY +/* + * kexec_purgatory_find_symbol - find a symbol in the purgatory + * @pi: Purgatory to search in. + * @name: Name of the symbol. + * + * Return: pointer to symbol in read-only symtab on success, NULL on error. + */ +static const Elf_Sym *kexec_purgatory_find_symbol(struct purgatory_info *pi, + const char *name) +{ + const Elf_Shdr *sechdrs; + const Elf_Ehdr *ehdr; + const Elf_Sym *syms; + const char *strtab; + int i, k; + + if (!pi->ehdr) + return NULL; + + ehdr = pi->ehdr; + sechdrs = (void *)ehdr + ehdr->e_shoff; + + for (i = 0; i < ehdr->e_shnum; i++) { + if (sechdrs[i].sh_type != SHT_SYMTAB) + continue; + + if (sechdrs[i].sh_link >= ehdr->e_shnum) + /* Invalid strtab section number */ + continue; + strtab = (void *)ehdr + sechdrs[sechdrs[i].sh_link].sh_offset; + syms = (void *)ehdr + sechdrs[i].sh_offset; + + /* Go through symbols for a match */ + for (k = 0; k < sechdrs[i].sh_size/sizeof(Elf_Sym); k++) { + if (ELF_ST_BIND(syms[k].st_info) != STB_GLOBAL) + continue; + + if (strcmp(strtab + syms[k].st_name, name) != 0) + continue; + + if (syms[k].st_shndx == SHN_UNDEF || + syms[k].st_shndx >= ehdr->e_shnum) { + pr_debug("Symbol: %s has bad section index %d.\n", + name, syms[k].st_shndx); + return NULL; + } + + /* Found the symbol we are looking for */ + return &syms[k]; + } + } + + return NULL; +} /* * kexec_purgatory_setup_kbuf - prepare buffer to load purgatory. * @pi: Purgatory to be loaded. @@ -960,6 +1014,10 @@ static int kexec_purgatory_setup_sechdrs(struct purgatory_info *pi, unsigned long offset; size_t sechdrs_size; Elf_Shdr *sechdrs; + const Elf_Sym *entry_sym; + u16 entry_shndx = 0; + unsigned long entry_off = 0; + bool start_fixed = false; int i; /* @@ -977,6 +1035,12 @@ static int kexec_purgatory_setup_sechdrs(struct purgatory_info *pi, bss_addr = kbuf->mem + kbuf->bufsz; kbuf->image->start = pi->ehdr->e_entry; + entry_sym = kexec_purgatory_find_symbol(pi, "purgatory_start"); + if (entry_sym) { + entry_shndx = entry_sym->st_shndx; + entry_off = entry_sym->st_value; + } + for (i = 0; i < pi->ehdr->e_shnum; i++) { unsigned long align; void *src, *dst; @@ -994,6 +1058,13 @@ static int kexec_purgatory_setup_sechdrs(struct purgatory_info *pi, offset = ALIGN(offset, align); + if (!start_fixed && entry_sym && i == entry_shndx && + (sechdrs[i].sh_flags & SHF_EXECINSTR) && + entry_off < sechdrs[i].sh_size) { + kbuf->image->start = kbuf->mem + offset + entry_off; + start_fixed = true; + } + /* * Check if the segment contains the entry point, if so, * calculate the value of image->start based on it. @@ -1004,13 +1075,14 @@ static int kexec_purgatory_setup_sechdrs(struct purgatory_info *pi, * is not set to the initial value, and warn the user so they * have a chance to fix their purgatory's linker script. */ - if (sechdrs[i].sh_flags & SHF_EXECINSTR && + if (!start_fixed && sechdrs[i].sh_flags & SHF_EXECINSTR && pi->ehdr->e_entry >= sechdrs[i].sh_addr && pi->ehdr->e_entry < (sechdrs[i].sh_addr + sechdrs[i].sh_size) && - !WARN_ON(kbuf->image->start != pi->ehdr->e_entry)) { + kbuf->image->start == pi->ehdr->e_entry) { kbuf->image->start -= sechdrs[i].sh_addr; kbuf->image->start += kbuf->mem + offset; + start_fixed = true; } src = (void *)pi->ehdr + sechdrs[i].sh_offset; @@ -1128,61 +1200,6 @@ int kexec_load_purgatory(struct kimage *image, struct kexec_buf *kbuf) return ret; } -/* - * kexec_purgatory_find_symbol - find a symbol in the purgatory - * @pi: Purgatory to search in. - * @name: Name of the symbol. - * - * Return: pointer to symbol in read-only symtab on success, NULL on error. - */ -static const Elf_Sym *kexec_purgatory_find_symbol(struct purgatory_info *pi, - const char *name) -{ - const Elf_Shdr *sechdrs; - const Elf_Ehdr *ehdr; - const Elf_Sym *syms; - const char *strtab; - int i, k; - - if (!pi->ehdr) - return NULL; - - ehdr = pi->ehdr; - sechdrs = (void *)ehdr + ehdr->e_shoff; - - for (i = 0; i < ehdr->e_shnum; i++) { - if (sechdrs[i].sh_type != SHT_SYMTAB) - continue; - - if (sechdrs[i].sh_link >= ehdr->e_shnum) - /* Invalid strtab section number */ - continue; - strtab = (void *)ehdr + sechdrs[sechdrs[i].sh_link].sh_offset; - syms = (void *)ehdr + sechdrs[i].sh_offset; - - /* Go through symbols for a match */ - for (k = 0; k < sechdrs[i].sh_size/sizeof(Elf_Sym); k++) { - if (ELF_ST_BIND(syms[k].st_info) != STB_GLOBAL) - continue; - - if (strcmp(strtab + syms[k].st_name, name) != 0) - continue; - - if (syms[k].st_shndx == SHN_UNDEF || - syms[k].st_shndx >= ehdr->e_shnum) { - pr_debug("Symbol: %s has bad section index %d.\n", - name, syms[k].st_shndx); - return NULL; - } - - /* Found the symbol we are looking for */ - return &syms[k]; - } - } - - return NULL; -} - void *kexec_purgatory_get_symbol_addr(struct kimage *image, const char *name) { struct purgatory_info *pi = &image->purgatory_info; diff --git a/kernel/kprobes.c b/kernel/kprobes.c index ab8f9fc1f0d172..87e6f4d61b95ba 100644 --- a/kernel/kprobes.c +++ b/kernel/kprobes.c @@ -1070,12 +1070,12 @@ static int __arm_kprobe_ftrace(struct kprobe *p, struct ftrace_ops *ops, lockdep_assert_held(&kprobe_mutex); ret = ftrace_set_filter_ip(ops, (unsigned long)p->addr, 0, 0); - if (WARN_ONCE(ret < 0, "Failed to arm kprobe-ftrace at %pS (error %d)\n", p->addr, ret)) + if (ret < 0) return ret; if (*cnt == 0) { ret = register_ftrace_function(ops); - if (WARN(ret < 0, "Failed to register kprobe-ftrace (error %d)\n", ret)) { + if (ret < 0) { /* * At this point, sinec ops is not registered, we should be sefe from * registering empty filter. @@ -1104,6 +1104,10 @@ static int __disarm_kprobe_ftrace(struct kprobe *p, struct ftrace_ops *ops, int ret; lockdep_assert_held(&kprobe_mutex); + if (unlikely(kprobe_ftrace_disabled)) { + /* Now ftrace is disabled forever, disarm is already done. */ + return 0; + } if (*cnt == 1) { ret = unregister_ftrace_function(ops); diff --git a/kernel/kthread.c b/kernel/kthread.c index 39511dd2abc97f..9f5d3efa7a2fd0 100644 --- a/kernel/kthread.c +++ b/kernel/kthread.c @@ -85,24 +85,6 @@ static inline struct kthread *to_kthread(struct task_struct *k) return k->worker_private; } -/* - * Variant of to_kthread() that doesn't assume @p is a kthread. - * - * When "(p->flags & PF_KTHREAD)" is set the task is a kthread and will - * always remain a kthread. For kthreads p->worker_private always - * points to a struct kthread. For tasks that are not kthreads - * p->worker_private is used to point to other things. - * - * Return NULL for any task that is not a kthread. - */ -static inline struct kthread *__to_kthread(struct task_struct *p) -{ - void *kthread = p->worker_private; - if (kthread && !(p->flags & PF_KTHREAD)) - kthread = NULL; - return kthread; -} - void get_kthread_comm(char *buf, size_t buf_size, struct task_struct *tsk) { struct kthread *kthread = to_kthread(tsk); @@ -193,7 +175,7 @@ EXPORT_SYMBOL_GPL(kthread_should_park); bool kthread_should_stop_or_park(void) { - struct kthread *kthread = __to_kthread(current); + struct kthread *kthread = tsk_is_kthread(current); if (!kthread) return false; @@ -234,7 +216,7 @@ EXPORT_SYMBOL_GPL(kthread_freezable_should_stop); */ void *kthread_func(struct task_struct *task) { - struct kthread *kthread = __to_kthread(task); + struct kthread *kthread = tsk_is_kthread(task); if (kthread) return kthread->threadfn; return NULL; @@ -266,7 +248,7 @@ EXPORT_SYMBOL_GPL(kthread_data); */ void *kthread_probe_data(struct task_struct *task) { - struct kthread *kthread = __to_kthread(task); + struct kthread *kthread = tsk_is_kthread(task); void *data = NULL; if (kthread) @@ -309,19 +291,8 @@ void kthread_parkme(void) } EXPORT_SYMBOL_GPL(kthread_parkme); -/** - * kthread_exit - Cause the current kthread return @result to kthread_stop(). - * @result: The integer value to return to kthread_stop(). - * - * While kthread_exit can be called directly, it exists so that - * functions which do some additional work in non-modular code such as - * module_put_and_kthread_exit can be implemented. - * - * Does not return. - */ -void __noreturn kthread_exit(long result) +void kthread_do_exit(struct kthread *kthread, long result) { - struct kthread *kthread = to_kthread(current); kthread->result = result; if (!list_empty(&kthread->hotplug_node)) { mutex_lock(&kthreads_hotplug_lock); @@ -333,9 +304,7 @@ void __noreturn kthread_exit(long result) kthread->preferred_affinity = NULL; } } - do_exit(0); } -EXPORT_SYMBOL(kthread_exit); /** * kthread_complete_and_exit - Exit the current kthread. @@ -680,7 +649,7 @@ void kthread_set_per_cpu(struct task_struct *k, int cpu) bool kthread_is_per_cpu(struct task_struct *p) { - struct kthread *kthread = __to_kthread(p); + struct kthread *kthread = tsk_is_kthread(p); if (!kthread) return false; diff --git a/kernel/liveupdate/kexec_handover.c b/kernel/liveupdate/kexec_handover.c index 90d411a59f76d9..fcbbfcd3365f6a 100644 --- a/kernel/liveupdate/kexec_handover.c +++ b/kernel/liveupdate/kexec_handover.c @@ -645,7 +645,7 @@ static void __init kho_reserve_scratch(void) scratch_size_update(); /* FIXME: deal with node hot-plug/remove */ - kho_scratch_cnt = num_online_nodes() + 2; + kho_scratch_cnt = nodes_weight(node_states[N_MEMORY]) + 2; size = kho_scratch_cnt * sizeof(*kho_scratch); kho_scratch = memblock_alloc(size, PAGE_SIZE); if (!kho_scratch) @@ -675,7 +675,11 @@ static void __init kho_reserve_scratch(void) kho_scratch[i].size = size; i++; - for_each_online_node(nid) { + /* + * Loop over nodes that have both memory and are online. Skip + * memoryless nodes, as we can not allocate scratch areas there. + */ + for_each_node_state(nid, N_MEMORY) { size = scratch_size_node(nid); addr = memblock_alloc_range_nid(size, CMA_MIN_ALIGNMENT_BYTES, 0, MEMBLOCK_ALLOC_ACCESSIBLE, diff --git a/kernel/liveupdate/luo_file.c b/kernel/liveupdate/luo_file.c index 9f7283379ebc06..6d0370107878a2 100644 --- a/kernel/liveupdate/luo_file.c +++ b/kernel/liveupdate/luo_file.c @@ -133,9 +133,12 @@ static LIST_HEAD(luo_file_handler_list); * state that is not preserved. Set by the handler's .preserve() * callback, and must be freed in the handler's .unpreserve() * callback. - * @retrieved: A flag indicating whether a user/kernel in the new kernel has + * @retrieve_status: Status code indicating whether a user/kernel in the new kernel has * successfully called retrieve() on this file. This prevents - * multiple retrieval attempts. + * multiple retrieval attempts. A value of 0 means a retrieve() + * has not been attempted, a positive value means the retrieve() + * was successful, and a negative value means the retrieve() + * failed, and the value is the error code of the call. * @mutex: A mutex that protects the fields of this specific instance * (e.g., @retrieved, @file), ensuring that operations like * retrieving or finishing a file are atomic. @@ -160,7 +163,7 @@ struct luo_file { struct file *file; u64 serialized_data; void *private_data; - bool retrieved; + int retrieve_status; struct mutex mutex; struct list_head list; u64 token; @@ -293,7 +296,6 @@ int luo_preserve_file(struct luo_file_set *file_set, u64 token, int fd) luo_file->file = file; luo_file->fh = fh; luo_file->token = token; - luo_file->retrieved = false; mutex_init(&luo_file->mutex); args.handler = fh; @@ -569,7 +571,12 @@ int luo_retrieve_file(struct luo_file_set *file_set, u64 token, return -ENOENT; guard(mutex)(&luo_file->mutex); - if (luo_file->retrieved) { + if (luo_file->retrieve_status < 0) { + /* Retrieve was attempted and it failed. Return the error code. */ + return luo_file->retrieve_status; + } + + if (luo_file->retrieve_status > 0) { /* * Someone is asking for this file again, so get a reference * for them. @@ -582,16 +589,19 @@ int luo_retrieve_file(struct luo_file_set *file_set, u64 token, args.handler = luo_file->fh; args.serialized_data = luo_file->serialized_data; err = luo_file->fh->ops->retrieve(&args); - if (!err) { - luo_file->file = args.file; - - /* Get reference so we can keep this file in LUO until finish */ - get_file(luo_file->file); - *filep = luo_file->file; - luo_file->retrieved = true; + if (err) { + /* Keep the error code for later use. */ + luo_file->retrieve_status = err; + return err; } - return err; + luo_file->file = args.file; + /* Get reference so we can keep this file in LUO until finish */ + get_file(luo_file->file); + *filep = luo_file->file; + luo_file->retrieve_status = 1; + + return 0; } static int luo_file_can_finish_one(struct luo_file_set *file_set, @@ -607,7 +617,7 @@ static int luo_file_can_finish_one(struct luo_file_set *file_set, args.handler = luo_file->fh; args.file = luo_file->file; args.serialized_data = luo_file->serialized_data; - args.retrieved = luo_file->retrieved; + args.retrieve_status = luo_file->retrieve_status; can_finish = luo_file->fh->ops->can_finish(&args); } @@ -624,7 +634,7 @@ static void luo_file_finish_one(struct luo_file_set *file_set, args.handler = luo_file->fh; args.file = luo_file->file; args.serialized_data = luo_file->serialized_data; - args.retrieved = luo_file->retrieved; + args.retrieve_status = luo_file->retrieve_status; luo_file->fh->ops->finish(&args); } @@ -779,7 +789,6 @@ int luo_file_deserialize(struct luo_file_set *file_set, luo_file->file = NULL; luo_file->serialized_data = file_ser[i].data; luo_file->token = file_ser[i].token; - luo_file->retrieved = false; mutex_init(&luo_file->mutex); list_add_tail(&luo_file->list, &file_set->files_list); } diff --git a/kernel/liveupdate/luo_session.c b/kernel/liveupdate/luo_session.c index dbdbc3bd7929dd..5c5c5f241c0db2 100644 --- a/kernel/liveupdate/luo_session.c +++ b/kernel/liveupdate/luo_session.c @@ -558,8 +558,13 @@ int luo_session_deserialize(void) } scoped_guard(mutex, &session->mutex) { - luo_file_deserialize(&session->file_set, - &sh->ser[i].file_set_ser); + err = luo_file_deserialize(&session->file_set, + &sh->ser[i].file_set_ser); + } + if (err) { + pr_warn("Failed to deserialize files for session [%s] %pe\n", + session->name, ERR_PTR(err)); + return err; } } diff --git a/kernel/locking/lockdep_internals.h b/kernel/locking/lockdep_internals.h index 0e5e6ffe91a3fe..91a802fea0aa0f 100644 --- a/kernel/locking/lockdep_internals.h +++ b/kernel/locking/lockdep_internals.h @@ -121,7 +121,7 @@ enum { #define MAX_LOCKDEP_CHAINS (1UL << MAX_LOCKDEP_CHAINS_BITS) -#define AVG_LOCKDEP_CHAIN_DEPTH 5 +#define AVG_LOCKDEP_CHAIN_DEPTH 10 #define MAX_LOCKDEP_CHAIN_HLOCKS (MAX_LOCKDEP_CHAINS * AVG_LOCKDEP_CHAIN_DEPTH) extern struct lock_chain lock_chains[]; diff --git a/kernel/module/kallsyms.c b/kernel/module/kallsyms.c index 00a60796327c06..0fc11e45df9b91 100644 --- a/kernel/module/kallsyms.c +++ b/kernel/module/kallsyms.c @@ -334,13 +334,8 @@ int module_address_lookup(unsigned long addr, if (mod) { if (modname) *modname = mod->name; - if (modbuildid) { -#if IS_ENABLED(CONFIG_STACKTRACE_BUILD_ID) - *modbuildid = mod->build_id; -#else - *modbuildid = NULL; -#endif - } + if (modbuildid) + *modbuildid = module_buildid(mod); sym = find_kallsyms_symbol(mod, addr, size, offset); diff --git a/kernel/module/main.c b/kernel/module/main.c index 710ee30b3beab9..21c5c0d14fa83a 100644 --- a/kernel/module/main.c +++ b/kernel/module/main.c @@ -1568,6 +1568,13 @@ static int simplify_symbols(struct module *mod, const struct load_info *info) break; default: + if (sym[i].st_shndx >= info->hdr->e_shnum) { + pr_err("%s: Symbol %s has an invalid section index %u (max %u)\n", + mod->name, name, sym[i].st_shndx, info->hdr->e_shnum - 1); + ret = -ENOEXEC; + break; + } + /* Divert to percpu allocation if a percpu var. */ if (sym[i].st_shndx == info->index.pcpu) secbase = (unsigned long)mod_percpu(mod); @@ -3544,12 +3551,6 @@ static int load_module(struct load_info *info, const char __user *uargs, mutex_unlock(&module_mutex); free_module: mod_stat_bump_invalid(info, flags); - /* Free lock-classes; relies on the preceding sync_rcu() */ - for_class_mod_mem_type(type, core_data) { - lockdep_free_key_range(mod->mem[type].base, - mod->mem[type].size); - } - module_memory_restore_rox(mod); module_deallocate(mod, info); free_copy: diff --git a/kernel/nscommon.c b/kernel/nscommon.c index bdc3c86231d38e..3166c1fd844afd 100644 --- a/kernel/nscommon.c +++ b/kernel/nscommon.c @@ -309,3 +309,9 @@ void __ns_ref_active_get(struct ns_common *ns) return; } } + +bool may_see_all_namespaces(void) +{ + return (task_active_pid_ns(current) == &init_pid_ns) && + ns_capable_noaudit(init_pid_ns.user_ns, CAP_SYS_ADMIN); +} diff --git a/kernel/nstree.c b/kernel/nstree.c index f36c59e6951dc0..6d12e5900ac015 100644 --- a/kernel/nstree.c +++ b/kernel/nstree.c @@ -515,32 +515,11 @@ static inline bool __must_check ns_requested(const struct klistns *kls, static inline bool __must_check may_list_ns(const struct klistns *kls, struct ns_common *ns) { - if (kls->user_ns) { - if (kls->userns_capable) - return true; - } else { - struct ns_common *owner; - struct user_namespace *user_ns; - - owner = ns_owner(ns); - if (owner) - user_ns = to_user_ns(owner); - else - user_ns = &init_user_ns; - if (ns_capable_noaudit(user_ns, CAP_SYS_ADMIN)) - return true; - } - - if (is_current_namespace(ns)) + if (kls->user_ns && kls->userns_capable) return true; - - if (ns->ns_type != CLONE_NEWUSER) - return false; - - if (ns_capable_noaudit(to_user_ns(ns), CAP_SYS_ADMIN)) + if (is_current_namespace(ns)) return true; - - return false; + return may_see_all_namespaces(); } static inline void ns_put(struct ns_common *ns) @@ -600,7 +579,7 @@ static ssize_t do_listns_userns(struct klistns *kls) ret = 0; head = &to_ns_common(kls->user_ns)->ns_owner_root.ns_list_head; - kls->userns_capable = ns_capable_noaudit(kls->user_ns, CAP_SYS_ADMIN); + kls->userns_capable = may_see_all_namespaces(); rcu_read_lock(); diff --git a/kernel/power/em_netlink.c b/kernel/power/em_netlink.c index 5a611d3950fd51..4d4fd29bd2be9c 100644 --- a/kernel/power/em_netlink.c +++ b/kernel/power/em_netlink.c @@ -109,6 +109,8 @@ int dev_energymodel_nl_get_perf_domains_doit(struct sk_buff *skb, id = nla_get_u32(info->attrs[DEV_ENERGYMODEL_A_PERF_DOMAIN_PERF_DOMAIN_ID]); pd = em_perf_domain_get_by_id(id); + if (!pd) + return -EINVAL; __em_nl_get_pd_size(pd, &msg_sz); msg = genlmsg_new(msg_sz, GFP_KERNEL); diff --git a/kernel/power/hibernate.c b/kernel/power/hibernate.c index af8d07bafe02a7..d04eae18ea688c 100644 --- a/kernel/power/hibernate.c +++ b/kernel/power/hibernate.c @@ -33,6 +33,7 @@ #include #include #include +#include #include #include "power.h" @@ -110,7 +111,8 @@ bool hibernation_available(void) { return nohibernate == 0 && !security_locked_down(LOCKDOWN_HIBERNATION) && - !secretmem_active() && !cxl_mem_active(); + !secretmem_active() && !cxl_mem_active() && + !of_machine_is_compatible("apple,arm-platform"); } /** diff --git a/kernel/power/main.c b/kernel/power/main.c index 03b2c5495c77ac..9ce75b1a23ed38 100644 --- a/kernel/power/main.c +++ b/kernel/power/main.c @@ -40,7 +40,7 @@ void pm_restore_gfp_mask(void) { WARN_ON(!mutex_is_locked(&system_transition_mutex)); - if (WARN_ON(!saved_gfp_count) || --saved_gfp_count) + if (!saved_gfp_count || --saved_gfp_count) return; gfp_allowed_mask = saved_gfp_mask; diff --git a/kernel/power/snapshot.c b/kernel/power/snapshot.c index 0a946932d5c17d..5706287e7230e5 100644 --- a/kernel/power/snapshot.c +++ b/kernel/power/snapshot.c @@ -2855,6 +2855,17 @@ int snapshot_write_finalize(struct snapshot_handle *handle) { int error; + /* + * Call snapshot_write_next() to drain any trailing zero pages, + * but make sure we're in the data page region first. + * This function can return PAGE_SIZE if the kernel was expecting + * another copy page. Return -ENODATA in that situation. + */ + if (handle->cur > nr_meta_pages + 1) { + error = snapshot_write_next(handle); + if (error) + return error > 0 ? -ENODATA : error; + } copy_last_highmem_page(); error = hibernate_restore_protect_page(handle->buffer); /* Do that only if we have loaded the image entirely */ diff --git a/kernel/rcu/rcutorture.c b/kernel/rcu/rcutorture.c index 07e51974b06bc6..83934402a287b5 100644 --- a/kernel/rcu/rcutorture.c +++ b/kernel/rcu/rcutorture.c @@ -1750,7 +1750,7 @@ rcu_torture_writer(void *arg) ulo[i] = cur_ops->get_comp_state(); gp_snap = cur_ops->start_gp_poll(); rcu_torture_writer_state = RTWS_POLL_WAIT; - if (cur_ops->exp_current && !torture_random(&rand) % 0xff) + if (cur_ops->exp_current && !(torture_random(&rand) & 0xff)) cur_ops->exp_current(); while (!cur_ops->poll_gp_state(gp_snap)) { gp_snap1 = cur_ops->get_gp_state(); @@ -1772,7 +1772,7 @@ rcu_torture_writer(void *arg) cur_ops->get_comp_state_full(&rgo[i]); cur_ops->start_gp_poll_full(&gp_snap_full); rcu_torture_writer_state = RTWS_POLL_WAIT_FULL; - if (cur_ops->exp_current && !torture_random(&rand) % 0xff) + if (cur_ops->exp_current && !(torture_random(&rand) & 0xff)) cur_ops->exp_current(); while (!cur_ops->poll_gp_state_full(&gp_snap_full)) { cur_ops->get_gp_state_full(&gp_snap1_full); diff --git a/kernel/rcu/srcutiny.c b/kernel/rcu/srcutiny.c index 3450c3751ef7ad..a2e2d516e51b9d 100644 --- a/kernel/rcu/srcutiny.c +++ b/kernel/rcu/srcutiny.c @@ -9,6 +9,7 @@ */ #include +#include #include #include #include @@ -41,6 +42,7 @@ static int init_srcu_struct_fields(struct srcu_struct *ssp) ssp->srcu_idx_max = 0; INIT_WORK(&ssp->srcu_work, srcu_drive_gp); INIT_LIST_HEAD(&ssp->srcu_work.entry); + init_irq_work(&ssp->srcu_irq_work, srcu_tiny_irq_work); return 0; } @@ -84,6 +86,7 @@ EXPORT_SYMBOL_GPL(init_srcu_struct); void cleanup_srcu_struct(struct srcu_struct *ssp) { WARN_ON(ssp->srcu_lock_nesting[0] || ssp->srcu_lock_nesting[1]); + irq_work_sync(&ssp->srcu_irq_work); flush_work(&ssp->srcu_work); WARN_ON(ssp->srcu_gp_running); WARN_ON(ssp->srcu_gp_waiting); @@ -177,6 +180,20 @@ void srcu_drive_gp(struct work_struct *wp) } EXPORT_SYMBOL_GPL(srcu_drive_gp); +/* + * Use an irq_work to defer schedule_work() to avoid acquiring the workqueue + * pool->lock while the caller might hold scheduler locks, causing lockdep + * splats due to workqueue_init() doing a wakeup. + */ +void srcu_tiny_irq_work(struct irq_work *irq_work) +{ + struct srcu_struct *ssp; + + ssp = container_of(irq_work, struct srcu_struct, srcu_irq_work); + schedule_work(&ssp->srcu_work); +} +EXPORT_SYMBOL_GPL(srcu_tiny_irq_work); + static void srcu_gp_start_if_needed(struct srcu_struct *ssp) { unsigned long cookie; @@ -189,7 +206,7 @@ static void srcu_gp_start_if_needed(struct srcu_struct *ssp) WRITE_ONCE(ssp->srcu_idx_max, cookie); if (!READ_ONCE(ssp->srcu_gp_running)) { if (likely(srcu_init_done)) - schedule_work(&ssp->srcu_work); + irq_work_queue(&ssp->srcu_irq_work); else if (list_empty(&ssp->srcu_work.entry)) list_add(&ssp->srcu_work.entry, &srcu_boot_list); } diff --git a/kernel/rcu/tree.h b/kernel/rcu/tree.h index b8bbe7960cda7a..2265b9c2906e1e 100644 --- a/kernel/rcu/tree.h +++ b/kernel/rcu/tree.h @@ -203,7 +203,7 @@ struct rcu_data { /* during and after the last grace */ /* period it is aware of. */ struct irq_work defer_qs_iw; /* Obtain later scheduler attention. */ - int defer_qs_iw_pending; /* Scheduler attention pending? */ + int defer_qs_pending; /* irqwork or softirq pending? */ struct work_struct strict_work; /* Schedule readers for strict GPs. */ /* 2) batch handling */ diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h index dbe2d02be824a0..95ad967adcf3cb 100644 --- a/kernel/rcu/tree_plugin.h +++ b/kernel/rcu/tree_plugin.h @@ -487,8 +487,8 @@ rcu_preempt_deferred_qs_irqrestore(struct task_struct *t, unsigned long flags) union rcu_special special; rdp = this_cpu_ptr(&rcu_data); - if (rdp->defer_qs_iw_pending == DEFER_QS_PENDING) - rdp->defer_qs_iw_pending = DEFER_QS_IDLE; + if (rdp->defer_qs_pending == DEFER_QS_PENDING) + rdp->defer_qs_pending = DEFER_QS_IDLE; /* * If RCU core is waiting for this CPU to exit its critical section, @@ -645,7 +645,7 @@ static void rcu_preempt_deferred_qs_handler(struct irq_work *iwp) * 5. Deferred QS reporting does not happen. */ if (rcu_preempt_depth() > 0) - WRITE_ONCE(rdp->defer_qs_iw_pending, DEFER_QS_IDLE); + WRITE_ONCE(rdp->defer_qs_pending, DEFER_QS_IDLE); } /* @@ -747,7 +747,10 @@ static void rcu_read_unlock_special(struct task_struct *t) // Using softirq, safe to awaken, and either the // wakeup is free or there is either an expedited // GP in flight or a potential need to deboost. - raise_softirq_irqoff(RCU_SOFTIRQ); + if (rdp->defer_qs_pending != DEFER_QS_PENDING) { + rdp->defer_qs_pending = DEFER_QS_PENDING; + raise_softirq_irqoff(RCU_SOFTIRQ); + } } else { // Enabling BH or preempt does reschedule, so... // Also if no expediting and no possible deboosting, @@ -755,11 +758,11 @@ static void rcu_read_unlock_special(struct task_struct *t) // tick enabled. set_need_resched_current(); if (IS_ENABLED(CONFIG_IRQ_WORK) && irqs_were_disabled && - needs_exp && rdp->defer_qs_iw_pending != DEFER_QS_PENDING && + needs_exp && rdp->defer_qs_pending != DEFER_QS_PENDING && cpu_online(rdp->cpu)) { // Get scheduler to re-evaluate and call hooks. // If !IRQ_WORK, FQS scan will eventually IPI. - rdp->defer_qs_iw_pending = DEFER_QS_PENDING; + rdp->defer_qs_pending = DEFER_QS_PENDING; irq_work_queue_on(&rdp->defer_qs_iw, rdp->cpu); } } diff --git a/kernel/rseq.c b/kernel/rseq.c index 395d8b002350a3..6cb5b7e51555d0 100644 --- a/kernel/rseq.c +++ b/kernel/rseq.c @@ -428,8 +428,9 @@ SYSCALL_DEFINE4(rseq, struct rseq __user *, rseq, u32, rseq_len, int, flags, u32 * auxiliary vector AT_RSEQ_ALIGN. If rseq_len is the original rseq * size, the required alignment is the original struct rseq alignment. * - * In order to be valid, rseq_len is either the original rseq size, or - * large enough to contain all supported fields, as communicated to + * The rseq_len is required to be greater or equal to the original rseq + * size. In order to be valid, rseq_len is either the original rseq size, + * or large enough to contain all supported fields, as communicated to * user-space through the ELF auxiliary vector AT_RSEQ_FEATURE_SIZE. */ if (rseq_len < ORIG_RSEQ_SIZE || diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 854984967fe295..011fe1b2ae9118 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -119,6 +119,9 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(sched_util_est_cfs_tp); EXPORT_TRACEPOINT_SYMBOL_GPL(sched_util_est_se_tp); EXPORT_TRACEPOINT_SYMBOL_GPL(sched_update_nr_running_tp); EXPORT_TRACEPOINT_SYMBOL_GPL(sched_compute_energy_tp); +EXPORT_TRACEPOINT_SYMBOL_GPL(sched_entry_tp); +EXPORT_TRACEPOINT_SYMBOL_GPL(sched_exit_tp); +EXPORT_TRACEPOINT_SYMBOL_GPL(sched_set_need_resched_tp); DEFINE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues); DEFINE_PER_CPU(struct rnd_state, sched_rnd_state); @@ -1136,6 +1139,7 @@ void __trace_set_need_resched(struct task_struct *curr, int tif) { trace_sched_set_need_resched_tp(curr, smp_processor_id(), tif); } +EXPORT_SYMBOL_GPL(__trace_set_need_resched); void resched_curr(struct rq *rq) { @@ -4704,8 +4708,11 @@ void sched_cancel_fork(struct task_struct *p) scx_cancel_fork(p); } +static void sched_mm_cid_fork(struct task_struct *t); + void sched_post_fork(struct task_struct *p) { + sched_mm_cid_fork(p); uclamp_post_fork(p); scx_post_fork(p); } @@ -9111,6 +9118,7 @@ void sched_move_task(struct task_struct *tsk, bool for_autogroup) { unsigned int queue_flags = DEQUEUE_SAVE | DEQUEUE_MOVE; bool resched = false; + bool queued = false; struct rq *rq; CLASS(task_rq_lock, rq_guard)(tsk); @@ -9122,10 +9130,13 @@ void sched_move_task(struct task_struct *tsk, bool for_autogroup) scx_cgroup_move_task(tsk); if (scope->running) resched = true; + queued = scope->queued; } if (resched) resched_curr(rq); + else if (queued) + wakeup_preempt(rq, tsk, 0); __balance_callbacks(rq, &rq_guard.rf); } @@ -10557,13 +10568,10 @@ static inline void mm_cid_transit_to_cpu(struct task_struct *t, struct mm_cid_pc } } -static bool mm_cid_fixup_task_to_cpu(struct task_struct *t, struct mm_struct *mm) +static void mm_cid_fixup_task_to_cpu(struct task_struct *t, struct mm_struct *mm) { /* Remote access to mm::mm_cid::pcpu requires rq_lock */ guard(task_rq_lock)(t); - /* If the task is not active it is not in the users count */ - if (!t->mm_cid.active) - return false; if (cid_on_task(t->mm_cid.cid)) { /* If running on the CPU, put the CID in transit mode, otherwise drop it */ if (task_rq(t)->curr == t) @@ -10571,69 +10579,43 @@ static bool mm_cid_fixup_task_to_cpu(struct task_struct *t, struct mm_struct *mm else mm_unset_cid_on_task(t); } - return true; } -static void mm_cid_do_fixup_tasks_to_cpus(struct mm_struct *mm) +static void mm_cid_fixup_tasks_to_cpus(void) { - struct task_struct *p, *t; - unsigned int users; + struct mm_struct *mm = current->mm; + struct task_struct *t; - /* - * This can obviously race with a concurrent affinity change, which - * increases the number of allowed CPUs for this mm, but that does - * not affect the mode and only changes the CID constraints. A - * possible switch back to per task mode happens either in the - * deferred handler function or in the next fork()/exit(). - * - * The caller has already transferred. The newly incoming task is - * already accounted for, but not yet visible. - */ - users = mm->mm_cid.users - 2; - if (!users) - return; + lockdep_assert_held(&mm->mm_cid.mutex); - guard(rcu)(); - for_other_threads(current, t) { - if (mm_cid_fixup_task_to_cpu(t, mm)) - users--; + hlist_for_each_entry(t, &mm->mm_cid.user_list, mm_cid.node) { + /* Current has already transferred before invoking the fixup. */ + if (t != current) + mm_cid_fixup_task_to_cpu(t, mm); } - if (!users) - return; - - /* Happens only for VM_CLONE processes. */ - for_each_process_thread(p, t) { - if (t == current || t->mm != mm) - continue; - if (mm_cid_fixup_task_to_cpu(t, mm)) { - if (--users == 0) - return; - } - } -} - -static void mm_cid_fixup_tasks_to_cpus(void) -{ - struct mm_struct *mm = current->mm; - - mm_cid_do_fixup_tasks_to_cpus(mm); mm_cid_complete_transit(mm, MM_CID_ONCPU); } static bool sched_mm_cid_add_user(struct task_struct *t, struct mm_struct *mm) { + lockdep_assert_held(&mm->mm_cid.lock); + t->mm_cid.active = 1; + hlist_add_head(&t->mm_cid.node, &mm->mm_cid.user_list); mm->mm_cid.users++; return mm_update_max_cids(mm); } -void sched_mm_cid_fork(struct task_struct *t) +static void sched_mm_cid_fork(struct task_struct *t) { struct mm_struct *mm = t->mm; bool percpu; - WARN_ON_ONCE(!mm || t->mm_cid.cid != MM_CID_UNSET); + if (!mm) + return; + + WARN_ON_ONCE(t->mm_cid.cid != MM_CID_UNSET); guard(mutex)(&mm->mm_cid.mutex); scoped_guard(raw_spinlock_irq, &mm->mm_cid.lock) { @@ -10672,12 +10654,13 @@ void sched_mm_cid_fork(struct task_struct *t) static bool sched_mm_cid_remove_user(struct task_struct *t) { + lockdep_assert_held(&t->mm->mm_cid.lock); + t->mm_cid.active = 0; - scoped_guard(preempt) { - /* Clear the transition bit */ - t->mm_cid.cid = cid_from_transit_cid(t->mm_cid.cid); - mm_unset_cid_on_task(t); - } + /* Clear the transition bit */ + t->mm_cid.cid = cid_from_transit_cid(t->mm_cid.cid); + mm_unset_cid_on_task(t); + hlist_del_init(&t->mm_cid.node); t->mm->mm_cid.users--; return mm_update_max_cids(t->mm); } @@ -10729,10 +10712,9 @@ void sched_mm_cid_exit(struct task_struct *t) return; /* * Mode change. The task has the CID unset - * already. The CPU CID is still valid and - * does not have MM_CID_TRANSIT set as the - * mode change has just taken effect under - * mm::mm_cid::lock. Drop it. + * already and dealt with an eventually set + * TRANSIT bit. If the CID is owned by the CPU + * then drop it. */ mm_drop_cid_on_cpu(mm, this_cpu_ptr(mm->mm_cid.pcpu)); } @@ -10821,11 +10803,13 @@ void mm_init_cid(struct mm_struct *mm, struct task_struct *p) mutex_init(&mm->mm_cid.mutex); mm->mm_cid.irq_work = IRQ_WORK_INIT_HARD(mm_cid_irq_work); INIT_WORK(&mm->mm_cid.work, mm_cid_work_fn); + INIT_HLIST_HEAD(&mm->mm_cid.user_list); cpumask_copy(mm_cpus_allowed(mm), &p->cpus_mask); bitmap_zero(mm_cidmask(mm), num_possible_cpus()); } #else /* CONFIG_SCHED_MM_CID */ static inline void mm_update_cpus_allowed(struct mm_struct *mm, const struct cpumask *affmsk) { } +static inline void sched_mm_cid_fork(struct task_struct *t) { } #endif /* !CONFIG_SCHED_MM_CID */ static DEFINE_PER_CPU(struct sched_change_ctx, sched_change_ctx); diff --git a/kernel/sched/deadline.c b/kernel/sched/deadline.c index 7bcde7114f1b65..3f804119720679 100644 --- a/kernel/sched/deadline.c +++ b/kernel/sched/deadline.c @@ -1027,7 +1027,7 @@ static void update_dl_entity(struct sched_dl_entity *dl_se) if (dl_time_before(dl_se->deadline, rq_clock(rq)) || dl_entity_overflow(dl_se, rq_clock(rq))) { - if (unlikely(!dl_is_implicit(dl_se) && + if (unlikely((!dl_is_implicit(dl_se) || dl_se->dl_defer) && !dl_time_before(dl_se->deadline, rq_clock(rq)) && !is_dl_boosted(dl_se))) { update_dl_revised_wakeup(dl_se, rq); @@ -3656,6 +3656,9 @@ static void __dl_clear_params(struct sched_dl_entity *dl_se) dl_se->dl_non_contending = 0; dl_se->dl_overrun = 0; dl_se->dl_server = 0; + dl_se->dl_defer = 0; + dl_se->dl_defer_running = 0; + dl_se->dl_defer_armed = 0; #ifdef CONFIG_RT_MUTEXES dl_se->pi_se = dl_se; diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c index 41caa22e0680a6..3504ec9bd73074 100644 --- a/kernel/sched/debug.c +++ b/kernel/sched/debug.c @@ -345,8 +345,8 @@ static ssize_t sched_fair_server_write(struct file *filp, const char __user *ubu long cpu = (long) ((struct seq_file *) filp->private_data)->private; struct rq *rq = cpu_rq(cpu); u64 runtime, period; + int retval = 0; size_t err; - int retval; u64 value; err = kstrtoull_from_user(ubuf, cnt, 10, &value); @@ -380,8 +380,6 @@ static ssize_t sched_fair_server_write(struct file *filp, const char __user *ubu dl_server_stop(&rq->fair_server); retval = dl_server_apply_params(&rq->fair_server, runtime, period, 0); - if (retval) - cnt = retval; if (!runtime) printk_deferred("Fair server disabled in CPU %d, system may crash due to starvation.\n", @@ -389,6 +387,9 @@ static ssize_t sched_fair_server_write(struct file *filp, const char __user *ubu if (rq->cfs.h_nr_queued) dl_server_start(&rq->fair_server); + + if (retval < 0) + return retval; } *ppos += cnt; @@ -797,6 +798,7 @@ static void print_rq(struct seq_file *m, struct rq *rq, int rq_cpu) void print_cfs_rq(struct seq_file *m, int cpu, struct cfs_rq *cfs_rq) { s64 left_vruntime = -1, zero_vruntime, right_vruntime = -1, left_deadline = -1, spread; + u64 avruntime; struct sched_entity *last, *first, *root; struct rq *rq = cpu_rq(cpu); unsigned long flags; @@ -820,6 +822,7 @@ void print_cfs_rq(struct seq_file *m, int cpu, struct cfs_rq *cfs_rq) if (last) right_vruntime = last->vruntime; zero_vruntime = cfs_rq->zero_vruntime; + avruntime = avg_vruntime(cfs_rq); raw_spin_rq_unlock_irqrestore(rq, flags); SEQ_printf(m, " .%-30s: %Ld.%06ld\n", "left_deadline", @@ -829,7 +832,7 @@ void print_cfs_rq(struct seq_file *m, int cpu, struct cfs_rq *cfs_rq) SEQ_printf(m, " .%-30s: %Ld.%06ld\n", "zero_vruntime", SPLIT_NS(zero_vruntime)); SEQ_printf(m, " .%-30s: %Ld.%06ld\n", "avg_vruntime", - SPLIT_NS(avg_vruntime(cfs_rq))); + SPLIT_NS(avruntime)); SEQ_printf(m, " .%-30s: %Ld.%06ld\n", "right_vruntime", SPLIT_NS(right_vruntime)); spread = right_vruntime - left_vruntime; diff --git a/kernel/sched/ext.c b/kernel/sched/ext.c index 0bb8fa927e9e9f..35fa2970abac2b 100644 --- a/kernel/sched/ext.c +++ b/kernel/sched/ext.c @@ -1097,21 +1097,12 @@ static void dispatch_enqueue(struct scx_sched *sch, struct scx_dispatch_q *dsq, } /* seq records the order tasks are queued, used by BPF DSQ iterator */ - dsq->seq++; + WRITE_ONCE(dsq->seq, dsq->seq + 1); p->scx.dsq_seq = dsq->seq; dsq_mod_nr(dsq, 1); p->scx.dsq = dsq; - /* - * scx.ddsp_dsq_id and scx.ddsp_enq_flags are only relevant on the - * direct dispatch path, but we clear them here because the direct - * dispatch verdict may be overridden on the enqueue path during e.g. - * bypass. - */ - p->scx.ddsp_dsq_id = SCX_DSQ_INVALID; - p->scx.ddsp_enq_flags = 0; - /* * We're transitioning out of QUEUEING or DISPATCHING. store_release to * match waiters' load_acquire. @@ -1277,12 +1268,34 @@ static void mark_direct_dispatch(struct scx_sched *sch, p->scx.ddsp_enq_flags = enq_flags; } +/* + * Clear @p direct dispatch state when leaving the scheduler. + * + * Direct dispatch state must be cleared in the following cases: + * - direct_dispatch(): cleared on the synchronous enqueue path, deferred + * dispatch keeps the state until consumed + * - process_ddsp_deferred_locals(): cleared after consuming deferred state, + * - do_enqueue_task(): cleared on enqueue fallbacks where the dispatch + * verdict is ignored (local/global/bypass) + * - dequeue_task_scx(): cleared after dispatch_dequeue(), covering deferred + * cancellation and holding_cpu races + * - scx_disable_task(): cleared for queued wakeup tasks, which are excluded by + * the scx_bypass() loop, so that stale state is not reused by a subsequent + * scheduler instance + */ +static inline void clear_direct_dispatch(struct task_struct *p) +{ + p->scx.ddsp_dsq_id = SCX_DSQ_INVALID; + p->scx.ddsp_enq_flags = 0; +} + static void direct_dispatch(struct scx_sched *sch, struct task_struct *p, u64 enq_flags) { struct rq *rq = task_rq(p); struct scx_dispatch_q *dsq = find_dsq_for_dispatch(sch, rq, p->scx.ddsp_dsq_id, p); + u64 ddsp_enq_flags; touch_core_sched_dispatch(rq, p); @@ -1323,8 +1336,10 @@ static void direct_dispatch(struct scx_sched *sch, struct task_struct *p, return; } - dispatch_enqueue(sch, dsq, p, - p->scx.ddsp_enq_flags | SCX_ENQ_CLEAR_OPSS); + ddsp_enq_flags = p->scx.ddsp_enq_flags; + clear_direct_dispatch(p); + + dispatch_enqueue(sch, dsq, p, ddsp_enq_flags | SCX_ENQ_CLEAR_OPSS); } static bool scx_rq_online(struct rq *rq) @@ -1433,6 +1448,7 @@ static void do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, */ touch_core_sched(rq, p); refill_task_slice_dfl(sch, p); + clear_direct_dispatch(p); dispatch_enqueue(sch, dsq, p, enq_flags); } @@ -1464,16 +1480,15 @@ static void clr_task_runnable(struct task_struct *p, bool reset_runnable_at) p->scx.flags |= SCX_TASK_RESET_RUNNABLE_AT; } -static void enqueue_task_scx(struct rq *rq, struct task_struct *p, int enq_flags) +static void enqueue_task_scx(struct rq *rq, struct task_struct *p, int core_enq_flags) { struct scx_sched *sch = scx_root; int sticky_cpu = p->scx.sticky_cpu; + u64 enq_flags = core_enq_flags | rq->scx.extra_enq_flags; if (enq_flags & ENQUEUE_WAKEUP) rq->scx.flags |= SCX_RQ_IN_WAKEUP; - enq_flags |= rq->scx.extra_enq_flags; - if (sticky_cpu >= 0) p->scx.sticky_cpu = -1; @@ -1601,6 +1616,7 @@ static bool dequeue_task_scx(struct rq *rq, struct task_struct *p, int deq_flags sub_nr_running(rq, 1); dispatch_dequeue(rq, p); + clear_direct_dispatch(p); return true; } @@ -2284,13 +2300,15 @@ static void process_ddsp_deferred_locals(struct rq *rq) struct task_struct, scx.dsq_list.node))) { struct scx_sched *sch = scx_root; struct scx_dispatch_q *dsq; + u64 dsq_id = p->scx.ddsp_dsq_id; + u64 enq_flags = p->scx.ddsp_enq_flags; list_del_init(&p->scx.dsq_list.node); + clear_direct_dispatch(p); - dsq = find_dsq_for_dispatch(sch, rq, p->scx.ddsp_dsq_id, p); + dsq = find_dsq_for_dispatch(sch, rq, dsq_id, p); if (!WARN_ON_ONCE(dsq->id != SCX_DSQ_LOCAL)) - dispatch_to_local_dsq(sch, rq, dsq, p, - p->scx.ddsp_enq_flags); + dispatch_to_local_dsq(sch, rq, dsq, p, enq_flags); } } @@ -2395,7 +2413,7 @@ static void put_prev_task_scx(struct rq *rq, struct task_struct *p, { struct scx_sched *sch = scx_root; - /* see kick_cpus_irq_workfn() */ + /* see kick_sync_wait_bal_cb() */ smp_store_release(&rq->scx.kick_sync, rq->scx.kick_sync + 1); update_curr_scx(rq); @@ -2438,6 +2456,48 @@ static void put_prev_task_scx(struct rq *rq, struct task_struct *p, switch_class(rq, next); } +static void kick_sync_wait_bal_cb(struct rq *rq) +{ + struct scx_kick_syncs __rcu *ks = __this_cpu_read(scx_kick_syncs); + unsigned long *ksyncs = rcu_dereference_sched(ks)->syncs; + bool waited; + s32 cpu; + + /* + * Drop rq lock and enable IRQs while waiting. IRQs must be enabled + * — a target CPU may be waiting for us to process an IPI (e.g. TLB + * flush) while we wait for its kick_sync to advance. + * + * Also, keep advancing our own kick_sync so that new kick_sync waits + * targeting us, which can start after we drop the lock, cannot form + * cyclic dependencies. + */ +retry: + waited = false; + for_each_cpu(cpu, rq->scx.cpus_to_sync) { + /* + * smp_load_acquire() pairs with smp_store_release() on + * kick_sync updates on the target CPUs. + */ + if (cpu == cpu_of(rq) || + smp_load_acquire(&cpu_rq(cpu)->scx.kick_sync) != ksyncs[cpu]) { + cpumask_clear_cpu(cpu, rq->scx.cpus_to_sync); + continue; + } + + raw_spin_rq_unlock_irq(rq); + while (READ_ONCE(cpu_rq(cpu)->scx.kick_sync) == ksyncs[cpu]) { + smp_store_release(&rq->scx.kick_sync, rq->scx.kick_sync + 1); + cpu_relax(); + } + raw_spin_rq_lock_irq(rq); + waited = true; + } + + if (waited) + goto retry; +} + static struct task_struct *first_local_task(struct rq *rq) { return list_first_entry_or_null(&rq->scx.local_dsq.list, @@ -2451,7 +2511,7 @@ do_pick_task_scx(struct rq *rq, struct rq_flags *rf, bool force_scx) bool keep_prev; struct task_struct *p; - /* see kick_cpus_irq_workfn() */ + /* see kick_sync_wait_bal_cb() */ smp_store_release(&rq->scx.kick_sync, rq->scx.kick_sync + 1); rq_modified_clear(rq); @@ -2461,6 +2521,17 @@ do_pick_task_scx(struct rq *rq, struct rq_flags *rf, bool force_scx) rq_repin_lock(rq, rf); maybe_queue_balance_callback(rq); + /* + * Defer to a balance callback which can drop rq lock and enable + * IRQs. Waiting directly in the pick path would deadlock against + * CPUs sending us IPIs (e.g. TLB flushes) while we wait for them. + */ + if (unlikely(rq->scx.kick_sync_pending)) { + rq->scx.kick_sync_pending = false; + queue_balance_callback(rq, &rq->scx.kick_sync_bal_cb, + kick_sync_wait_bal_cb); + } + /* * If any higher-priority sched class enqueued a runnable task on * this rq during balance_one(), abort and return RETRY_TASK, so @@ -2926,6 +2997,8 @@ static void scx_disable_task(struct task_struct *p) lockdep_assert_rq_held(rq); WARN_ON_ONCE(scx_get_task_state(p) != SCX_TASK_ENABLED); + clear_direct_dispatch(p); + if (SCX_HAS_OP(sch, disable)) SCX_CALL_OP_TASK(sch, SCX_KF_REST, disable, rq, p); scx_set_task_state(p, SCX_TASK_READY); @@ -3553,7 +3626,6 @@ static int scx_cgroup_init(struct scx_sched *sch) ret = SCX_CALL_OP_RET(sch, SCX_KF_UNLOCKED, cgroup_init, NULL, css->cgroup, &args); if (ret) { - css_put(css); scx_error(sch, "ops.cgroup_init() failed (%d)", ret); return ret; } @@ -4391,10 +4463,19 @@ static void scx_disable_workfn(struct kthread_work *work) scx_bypass(false); } +/* + * Claim the exit on @sch. The caller must ensure that the helper kthread work + * is kicked before the current task can be preempted. Once exit_kind is + * claimed, scx_error() can no longer trigger, so if the current task gets + * preempted and the BPF scheduler fails to schedule it back, the helper work + * will never be kicked and the whole system can wedge. + */ static bool scx_claim_exit(struct scx_sched *sch, enum scx_exit_kind kind) { int none = SCX_EXIT_NONE; + lockdep_assert_preemption_disabled(); + if (!atomic_try_cmpxchg(&sch->exit_kind, &none, kind)) return false; @@ -4417,6 +4498,7 @@ static void scx_disable(enum scx_exit_kind kind) rcu_read_lock(); sch = rcu_dereference(scx_root); if (sch) { + guard(preempt)(); scx_claim_exit(sch, kind); kthread_queue_work(sch->helper, &sch->disable_work); } @@ -4665,6 +4747,9 @@ static void scx_dump_state(struct scx_exit_info *ei, size_t dump_len) if (!cpumask_empty(rq->scx.cpus_to_wait)) dump_line(&ns, " cpus_to_wait : %*pb", cpumask_pr_args(rq->scx.cpus_to_wait)); + if (!cpumask_empty(rq->scx.cpus_to_sync)) + dump_line(&ns, " cpus_to_sync : %*pb", + cpumask_pr_args(rq->scx.cpus_to_sync)); used = seq_buf_used(&ns); if (SCX_HAS_OP(sch, dump_cpu)) { @@ -4739,6 +4824,8 @@ static bool scx_vexit(struct scx_sched *sch, { struct scx_exit_info *ei = sch->exit_info; + guard(preempt)(); + if (!scx_claim_exit(sch, kind)) return false; @@ -4924,20 +5011,30 @@ static int validate_ops(struct scx_sched *sch, const struct sched_ext_ops *ops) return 0; } -static int scx_enable(struct sched_ext_ops *ops, struct bpf_link *link) +/* + * scx_enable() is offloaded to a dedicated system-wide RT kthread to avoid + * starvation. During the READY -> ENABLED task switching loop, the calling + * thread's sched_class gets switched from fair to ext. As fair has higher + * priority than ext, the calling thread can be indefinitely starved under + * fair-class saturation, leading to a system hang. + */ +struct scx_enable_cmd { + struct kthread_work work; + struct sched_ext_ops *ops; + int ret; +}; + +static void scx_enable_workfn(struct kthread_work *work) { + struct scx_enable_cmd *cmd = + container_of(work, struct scx_enable_cmd, work); + struct sched_ext_ops *ops = cmd->ops; struct scx_sched *sch; struct scx_task_iter sti; struct task_struct *p; unsigned long timeout; int i, cpu, ret; - if (!cpumask_equal(housekeeping_cpumask(HK_TYPE_DOMAIN), - cpu_possible_mask)) { - pr_err("sched_ext: Not compatible with \"isolcpus=\" domain isolation\n"); - return -EINVAL; - } - mutex_lock(&scx_enable_mutex); if (scx_enable_state() != SCX_DISABLED) { @@ -5154,13 +5251,15 @@ static int scx_enable(struct sched_ext_ops *ops, struct bpf_link *link) atomic_long_inc(&scx_enable_seq); - return 0; + cmd->ret = 0; + return; err_free_ksyncs: free_kick_syncs(); err_unlock: mutex_unlock(&scx_enable_mutex); - return ret; + cmd->ret = ret; + return; err_disable_unlock_all: scx_cgroup_unlock(); @@ -5179,7 +5278,42 @@ static int scx_enable(struct sched_ext_ops *ops, struct bpf_link *link) */ scx_error(sch, "scx_enable() failed (%d)", ret); kthread_flush_work(&sch->disable_work); - return 0; + cmd->ret = 0; +} + +static int scx_enable(struct sched_ext_ops *ops, struct bpf_link *link) +{ + static struct kthread_worker *helper; + static DEFINE_MUTEX(helper_mutex); + struct scx_enable_cmd cmd; + + if (!cpumask_equal(housekeeping_cpumask(HK_TYPE_DOMAIN), + cpu_possible_mask)) { + pr_err("sched_ext: Not compatible with \"isolcpus=\" domain isolation\n"); + return -EINVAL; + } + + if (!READ_ONCE(helper)) { + mutex_lock(&helper_mutex); + if (!helper) { + struct kthread_worker *w = + kthread_run_worker(0, "scx_enable_helper"); + if (IS_ERR_OR_NULL(w)) { + mutex_unlock(&helper_mutex); + return -ENOMEM; + } + sched_set_fifo(w->task); + WRITE_ONCE(helper, w); + } + mutex_unlock(&helper_mutex); + } + + kthread_init_work(&cmd.work, scx_enable_workfn); + cmd.ops = ops; + + kthread_queue_work(READ_ONCE(helper), &cmd.work); + kthread_flush_work(&cmd.work); + return cmd.ret; } @@ -5514,11 +5648,11 @@ static bool kick_one_cpu(s32 cpu, struct rq *this_rq, unsigned long *ksyncs) if (cpumask_test_cpu(cpu, this_scx->cpus_to_wait)) { if (cur_class == &ext_sched_class) { + cpumask_set_cpu(cpu, this_scx->cpus_to_sync); ksyncs[cpu] = rq->scx.kick_sync; should_wait = true; - } else { - cpumask_clear_cpu(cpu, this_scx->cpus_to_wait); } + cpumask_clear_cpu(cpu, this_scx->cpus_to_wait); } resched_curr(rq); @@ -5573,27 +5707,15 @@ static void kick_cpus_irq_workfn(struct irq_work *irq_work) cpumask_clear_cpu(cpu, this_scx->cpus_to_kick_if_idle); } - if (!should_wait) - return; - - for_each_cpu(cpu, this_scx->cpus_to_wait) { - unsigned long *wait_kick_sync = &cpu_rq(cpu)->scx.kick_sync; - - /* - * Busy-wait until the task running at the time of kicking is no - * longer running. This can be used to implement e.g. core - * scheduling. - * - * smp_cond_load_acquire() pairs with store_releases in - * pick_task_scx() and put_prev_task_scx(). The former breaks - * the wait if SCX's scheduling path is entered even if the same - * task is picked subsequently. The latter is necessary to break - * the wait when $cpu is taken by a higher sched class. - */ - if (cpu != cpu_of(this_rq)) - smp_cond_load_acquire(wait_kick_sync, VAL != ksyncs[cpu]); - - cpumask_clear_cpu(cpu, this_scx->cpus_to_wait); + /* + * Can't wait in hardirq — kick_sync can't advance, deadlocking if + * CPUs wait for each other. Defer to kick_sync_wait_bal_cb(). + */ + if (should_wait) { + raw_spin_rq_lock(this_rq); + this_scx->kick_sync_pending = true; + resched_curr(this_rq); + raw_spin_rq_unlock(this_rq); } } @@ -5698,6 +5820,7 @@ void __init init_sched_ext_class(void) BUG_ON(!zalloc_cpumask_var_node(&rq->scx.cpus_to_kick_if_idle, GFP_KERNEL, n)); BUG_ON(!zalloc_cpumask_var_node(&rq->scx.cpus_to_preempt, GFP_KERNEL, n)); BUG_ON(!zalloc_cpumask_var_node(&rq->scx.cpus_to_wait, GFP_KERNEL, n)); + BUG_ON(!zalloc_cpumask_var_node(&rq->scx.cpus_to_sync, GFP_KERNEL, n)); rq->scx.deferred_irq_work = IRQ_WORK_INIT_HARD(deferred_irq_workfn); rq->scx.kick_cpus_irq_work = IRQ_WORK_INIT_HARD(kick_cpus_irq_workfn); diff --git a/kernel/sched/ext_idle.c b/kernel/sched/ext_idle.c index 3d9d404d5cd206..06f33f120a68ff 100644 --- a/kernel/sched/ext_idle.c +++ b/kernel/sched/ext_idle.c @@ -543,7 +543,7 @@ s32 scx_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, u64 wake_flags, * piled up on it even if there is an idle core elsewhere on * the system. */ - waker_node = cpu_to_node(cpu); + waker_node = scx_cpu_node_if_enabled(cpu); if (!(current->flags & PF_EXITING) && cpu_rq(cpu)->scx.local_dsq.nr == 0 && (!(flags & SCX_PICK_IDLE_IN_NODE) || (waker_node == node)) && @@ -861,25 +861,32 @@ static bool check_builtin_idle_enabled(struct scx_sched *sch) * code. * * We can't simply check whether @p->migration_disabled is set in a - * sched_ext callback, because migration is always disabled for the current - * task while running BPF code. + * sched_ext callback, because the BPF prolog (__bpf_prog_enter) may disable + * migration for the current task while running BPF code. * - * The prolog (__bpf_prog_enter) and epilog (__bpf_prog_exit) respectively - * disable and re-enable migration. For this reason, the current task - * inside a sched_ext callback is always a migration-disabled task. + * Since the BPF prolog calls migrate_disable() only when CONFIG_PREEMPT_RCU + * is enabled (via rcu_read_lock_dont_migrate()), migration_disabled == 1 for + * the current task is ambiguous only in that case: it could be from the BPF + * prolog rather than a real migrate_disable() call. * - * Therefore, when @p->migration_disabled == 1, check whether @p is the - * current task or not: if it is, then migration was not disabled before - * entering the callback, otherwise migration was disabled. + * Without CONFIG_PREEMPT_RCU, the BPF prolog never calls migrate_disable(), + * so migration_disabled == 1 always means the task is truly + * migration-disabled. + * + * Therefore, when migration_disabled == 1 and CONFIG_PREEMPT_RCU is enabled, + * check whether @p is the current task or not: if it is, then migration was + * not disabled before entering the callback, otherwise migration was disabled. * * Returns true if @p is migration-disabled, false otherwise. */ static bool is_bpf_migration_disabled(const struct task_struct *p) { - if (p->migration_disabled == 1) - return p != current; - else - return p->migration_disabled; + if (p->migration_disabled == 1) { + if (IS_ENABLED(CONFIG_PREEMPT_RCU)) + return p != current; + return true; + } + return p->migration_disabled; } static s32 select_cpu_from_kfunc(struct scx_sched *sch, struct task_struct *p, diff --git a/kernel/sched/ext_internal.h b/kernel/sched/ext_internal.h index 386c677e4c9a0c..11ebb744d8931e 100644 --- a/kernel/sched/ext_internal.h +++ b/kernel/sched/ext_internal.h @@ -74,7 +74,7 @@ enum scx_exit_flags { * info communication. The following flag indicates whether ops.init() * finished successfully. */ - SCX_EFLAG_INITIALIZED, + SCX_EFLAG_INITIALIZED = 1LLU << 0, }; /* diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 3eaeceda71b00f..2625a78c030019 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -524,10 +524,48 @@ void account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec); * Scheduling class tree data structure manipulation methods: */ +extern void __BUILD_BUG_vruntime_cmp(void); + +/* Use __builtin_strcmp() because of __HAVE_ARCH_STRCMP: */ + +#define vruntime_cmp(A, CMP_STR, B) ({ \ + int __res = 0; \ + \ + if (!__builtin_strcmp(CMP_STR, "<")) { \ + __res = ((s64)((A)-(B)) < 0); \ + } else if (!__builtin_strcmp(CMP_STR, "<=")) { \ + __res = ((s64)((A)-(B)) <= 0); \ + } else if (!__builtin_strcmp(CMP_STR, ">")) { \ + __res = ((s64)((A)-(B)) > 0); \ + } else if (!__builtin_strcmp(CMP_STR, ">=")) { \ + __res = ((s64)((A)-(B)) >= 0); \ + } else { \ + /* Unknown operator throws linker error: */ \ + __BUILD_BUG_vruntime_cmp(); \ + } \ + \ + __res; \ +}) + +extern void __BUILD_BUG_vruntime_op(void); + +#define vruntime_op(A, OP_STR, B) ({ \ + s64 __res = 0; \ + \ + if (!__builtin_strcmp(OP_STR, "-")) { \ + __res = (s64)((A)-(B)); \ + } else { \ + /* Unknown operator throws linker error: */ \ + __BUILD_BUG_vruntime_op(); \ + } \ + \ + __res; \ +}) + + static inline __maybe_unused u64 max_vruntime(u64 max_vruntime, u64 vruntime) { - s64 delta = (s64)(vruntime - max_vruntime); - if (delta > 0) + if (vruntime_cmp(vruntime, ">", max_vruntime)) max_vruntime = vruntime; return max_vruntime; @@ -535,8 +573,7 @@ static inline __maybe_unused u64 max_vruntime(u64 max_vruntime, u64 vruntime) static inline __maybe_unused u64 min_vruntime(u64 min_vruntime, u64 vruntime) { - s64 delta = (s64)(vruntime - min_vruntime); - if (delta < 0) + if (vruntime_cmp(vruntime, "<", min_vruntime)) min_vruntime = vruntime; return min_vruntime; @@ -549,12 +586,27 @@ static inline bool entity_before(const struct sched_entity *a, * Tiebreak on vruntime seems unnecessary since it can * hardly happen. */ - return (s64)(a->deadline - b->deadline) < 0; + return vruntime_cmp(a->deadline, "<", b->deadline); } +/* + * Per avg_vruntime() below, cfs_rq::zero_vruntime is only slightly stale + * and this value should be no more than two lag bounds. Which puts it in the + * general order of: + * + * (slice + TICK_NSEC) << NICE_0_LOAD_SHIFT + * + * which is around 44 bits in size (on 64bit); that is 20 for + * NICE_0_LOAD_SHIFT, another 20 for NSEC_PER_MSEC and then a handful for + * however many msec the actual slice+tick ends up begin. + * + * (disregarding the actual divide-by-weight part makes for the worst case + * weight of 2, which nicely cancels vs the fuzz in zero_vruntime not actually + * being the zero-lag point). + */ static inline s64 entity_key(struct cfs_rq *cfs_rq, struct sched_entity *se) { - return (s64)(se->vruntime - cfs_rq->zero_vruntime); + return vruntime_op(se->vruntime, "-", cfs_rq->zero_vruntime); } #define __node_2_se(node) \ @@ -607,8 +659,8 @@ static inline s64 entity_key(struct cfs_rq *cfs_rq, struct sched_entity *se) * Which we track using: * * v0 := cfs_rq->zero_vruntime - * \Sum (v_i - v0) * w_i := cfs_rq->avg_vruntime - * \Sum w_i := cfs_rq->avg_load + * \Sum (v_i - v0) * w_i := cfs_rq->sum_w_vruntime + * \Sum w_i := cfs_rq->sum_weight * * Since zero_vruntime closely tracks the per-task service, these * deltas: (v_i - v), will be in the order of the maximal (virtual) lag @@ -619,61 +671,85 @@ static inline s64 entity_key(struct cfs_rq *cfs_rq, struct sched_entity *se) * As measured, the max (key * weight) value was ~44 bits for a kernel build. */ static void -avg_vruntime_add(struct cfs_rq *cfs_rq, struct sched_entity *se) +sum_w_vruntime_add(struct cfs_rq *cfs_rq, struct sched_entity *se) { unsigned long weight = scale_load_down(se->load.weight); s64 key = entity_key(cfs_rq, se); - cfs_rq->avg_vruntime += key * weight; - cfs_rq->avg_load += weight; + cfs_rq->sum_w_vruntime += key * weight; + cfs_rq->sum_weight += weight; } static void -avg_vruntime_sub(struct cfs_rq *cfs_rq, struct sched_entity *se) +sum_w_vruntime_sub(struct cfs_rq *cfs_rq, struct sched_entity *se) { unsigned long weight = scale_load_down(se->load.weight); s64 key = entity_key(cfs_rq, se); - cfs_rq->avg_vruntime -= key * weight; - cfs_rq->avg_load -= weight; + cfs_rq->sum_w_vruntime -= key * weight; + cfs_rq->sum_weight -= weight; } static inline -void avg_vruntime_update(struct cfs_rq *cfs_rq, s64 delta) +void update_zero_vruntime(struct cfs_rq *cfs_rq, s64 delta) { /* - * v' = v + d ==> avg_vruntime' = avg_runtime - d*avg_load + * v' = v + d ==> sum_w_vruntime' = sum_w_vruntime - d*sum_weight */ - cfs_rq->avg_vruntime -= cfs_rq->avg_load * delta; + cfs_rq->sum_w_vruntime -= cfs_rq->sum_weight * delta; + cfs_rq->zero_vruntime += delta; } /* - * Specifically: avg_runtime() + 0 must result in entity_eligible() := true + * Specifically: avg_vruntime() + 0 must result in entity_eligible() := true * For this to be so, the result of this function must have a left bias. + * + * Called in: + * - place_entity() -- before enqueue + * - update_entity_lag() -- before dequeue + * - update_deadline() -- slice expiration + * + * This means it is one entry 'behind' but that puts it close enough to where + * the bound on entity_key() is at most two lag bounds. */ u64 avg_vruntime(struct cfs_rq *cfs_rq) { struct sched_entity *curr = cfs_rq->curr; - s64 avg = cfs_rq->avg_vruntime; - long load = cfs_rq->avg_load; + long weight = cfs_rq->sum_weight; + s64 delta = 0; - if (curr && curr->on_rq) { - unsigned long weight = scale_load_down(curr->load.weight); + if (curr && !curr->on_rq) + curr = NULL; - avg += entity_key(cfs_rq, curr) * weight; - load += weight; - } + if (weight) { + s64 runtime = cfs_rq->sum_w_vruntime; + + if (curr) { + unsigned long w = scale_load_down(curr->load.weight); + + runtime += entity_key(cfs_rq, curr) * w; + weight += w; + } - if (load) { /* sign flips effective floor / ceiling */ - if (avg < 0) - avg -= (load - 1); - avg = div_s64(avg, load); + if (runtime < 0) + runtime -= (weight - 1); + + delta = div_s64(runtime, weight); + } else if (curr) { + /* + * When there is but one element, it is the average. + */ + delta = curr->vruntime - cfs_rq->zero_vruntime; } - return cfs_rq->zero_vruntime + avg; + update_zero_vruntime(cfs_rq, delta); + + return cfs_rq->zero_vruntime; } +static inline u64 cfs_rq_max_slice(struct cfs_rq *cfs_rq); + /* * lag_i = S - s_i = w_i * (V - v_i) * @@ -687,17 +763,16 @@ u64 avg_vruntime(struct cfs_rq *cfs_rq) * EEVDF gives the following limit for a steady state system: * * -r_max < lag < max(r_max, q) - * - * XXX could add max_slice to the augmented data to track this. */ static void update_entity_lag(struct cfs_rq *cfs_rq, struct sched_entity *se) { + u64 max_slice = cfs_rq_max_slice(cfs_rq) + TICK_NSEC; s64 vlag, limit; WARN_ON_ONCE(!se->on_rq); vlag = avg_vruntime(cfs_rq) - se->vruntime; - limit = calc_delta_fair(max_t(u64, 2*se->slice, TICK_NSEC), se); + limit = calc_delta_fair(max_slice, se); se->vlag = clamp(vlag, -limit, limit); } @@ -722,8 +797,8 @@ static void update_entity_lag(struct cfs_rq *cfs_rq, struct sched_entity *se) static int vruntime_eligible(struct cfs_rq *cfs_rq, u64 vruntime) { struct sched_entity *curr = cfs_rq->curr; - s64 avg = cfs_rq->avg_vruntime; - long load = cfs_rq->avg_load; + s64 avg = cfs_rq->sum_w_vruntime; + long load = cfs_rq->sum_weight; if (curr && curr->on_rq) { unsigned long weight = scale_load_down(curr->load.weight); @@ -732,7 +807,7 @@ static int vruntime_eligible(struct cfs_rq *cfs_rq, u64 vruntime) load += weight; } - return avg >= (s64)(vruntime - cfs_rq->zero_vruntime) * load; + return avg >= vruntime_op(vruntime, "-", cfs_rq->zero_vruntime) * load; } int entity_eligible(struct cfs_rq *cfs_rq, struct sched_entity *se) @@ -740,16 +815,6 @@ int entity_eligible(struct cfs_rq *cfs_rq, struct sched_entity *se) return vruntime_eligible(cfs_rq, se->vruntime); } -static void update_zero_vruntime(struct cfs_rq *cfs_rq) -{ - u64 vruntime = avg_vruntime(cfs_rq); - s64 delta = (s64)(vruntime - cfs_rq->zero_vruntime); - - avg_vruntime_update(cfs_rq, delta); - - cfs_rq->zero_vruntime = vruntime; -} - static inline u64 cfs_rq_min_slice(struct cfs_rq *cfs_rq) { struct sched_entity *root = __pick_root_entity(cfs_rq); @@ -765,18 +830,32 @@ static inline u64 cfs_rq_min_slice(struct cfs_rq *cfs_rq) return min_slice; } +static inline u64 cfs_rq_max_slice(struct cfs_rq *cfs_rq) +{ + struct sched_entity *root = __pick_root_entity(cfs_rq); + struct sched_entity *curr = cfs_rq->curr; + u64 max_slice = 0ULL; + + if (curr && curr->on_rq) + max_slice = curr->slice; + + if (root) + max_slice = max(max_slice, root->max_slice); + + return max_slice; +} + static inline bool __entity_less(struct rb_node *a, const struct rb_node *b) { return entity_before(__node_2_se(a), __node_2_se(b)); } -#define vruntime_gt(field, lse, rse) ({ (s64)((lse)->field - (rse)->field) > 0; }) - static inline void __min_vruntime_update(struct sched_entity *se, struct rb_node *node) { if (node) { struct sched_entity *rse = __node_2_se(node); - if (vruntime_gt(min_vruntime, se, rse)) + + if (vruntime_cmp(se->min_vruntime, ">", rse->min_vruntime)) se->min_vruntime = rse->min_vruntime; } } @@ -790,6 +869,15 @@ static inline void __min_slice_update(struct sched_entity *se, struct rb_node *n } } +static inline void __max_slice_update(struct sched_entity *se, struct rb_node *node) +{ + if (node) { + struct sched_entity *rse = __node_2_se(node); + if (rse->max_slice > se->max_slice) + se->max_slice = rse->max_slice; + } +} + /* * se->min_vruntime = min(se->vruntime, {left,right}->min_vruntime) */ @@ -797,6 +885,7 @@ static inline bool min_vruntime_update(struct sched_entity *se, bool exit) { u64 old_min_vruntime = se->min_vruntime; u64 old_min_slice = se->min_slice; + u64 old_max_slice = se->max_slice; struct rb_node *node = &se->run_node; se->min_vruntime = se->vruntime; @@ -807,8 +896,13 @@ static inline bool min_vruntime_update(struct sched_entity *se, bool exit) __min_slice_update(se, node->rb_right); __min_slice_update(se, node->rb_left); + se->max_slice = se->slice; + __max_slice_update(se, node->rb_right); + __max_slice_update(se, node->rb_left); + return se->min_vruntime == old_min_vruntime && - se->min_slice == old_min_slice; + se->min_slice == old_min_slice && + se->max_slice == old_max_slice; } RB_DECLARE_CALLBACKS(static, min_vruntime_cb, struct sched_entity, @@ -819,8 +913,7 @@ RB_DECLARE_CALLBACKS(static, min_vruntime_cb, struct sched_entity, */ static void __enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) { - avg_vruntime_add(cfs_rq, se); - update_zero_vruntime(cfs_rq); + sum_w_vruntime_add(cfs_rq, se); se->min_vruntime = se->vruntime; se->min_slice = se->slice; rb_add_augmented_cached(&se->run_node, &cfs_rq->tasks_timeline, @@ -831,8 +924,7 @@ static void __dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) { rb_erase_augmented_cached(&se->run_node, &cfs_rq->tasks_timeline, &min_vruntime_cb); - avg_vruntime_sub(cfs_rq, se); - update_zero_vruntime(cfs_rq); + sum_w_vruntime_sub(cfs_rq, se); } struct sched_entity *__pick_root_entity(struct cfs_rq *cfs_rq) @@ -887,7 +979,7 @@ static inline void update_protect_slice(struct cfs_rq *cfs_rq, struct sched_enti static inline bool protect_slice(struct sched_entity *se) { - return ((s64)(se->vprot - se->vruntime) > 0); + return vruntime_cmp(se->vruntime, "<", se->vprot); } static inline void cancel_protect_slice(struct sched_entity *se) @@ -1024,7 +1116,7 @@ static void clear_buddies(struct cfs_rq *cfs_rq, struct sched_entity *se); */ static bool update_deadline(struct cfs_rq *cfs_rq, struct sched_entity *se) { - if ((s64)(se->vruntime - se->deadline) < 0) + if (vruntime_cmp(se->vruntime, "<", se->deadline)) return false; /* @@ -1039,6 +1131,7 @@ static bool update_deadline(struct cfs_rq *cfs_rq, struct sched_entity *se) * EEVDF: vd_i = ve_i + r_i / w_i */ se->deadline = se->vruntime + calc_delta_fair(se->slice, se); + avg_vruntime(cfs_rq); /* * The task has consumed its request, reschedule. @@ -3755,6 +3848,8 @@ static void reweight_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, unsigned long weight) { bool curr = cfs_rq->curr == se; + bool rel_vprot = false; + u64 vprot; if (se->on_rq) { /* commit outstanding execution time */ @@ -3762,6 +3857,11 @@ static void reweight_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, update_entity_lag(cfs_rq, se); se->deadline -= se->vruntime; se->rel_deadline = 1; + if (curr && protect_slice(se)) { + vprot = se->vprot - se->vruntime; + rel_vprot = true; + } + cfs_rq->nr_queued--; if (!curr) __dequeue_entity(cfs_rq, se); @@ -3777,6 +3877,9 @@ static void reweight_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, if (se->rel_deadline) se->deadline = div_s64(se->deadline * se->load.weight, weight); + if (rel_vprot) + vprot = div_s64(vprot * se->load.weight, weight); + update_load_set(&se->load, weight); do { @@ -3788,6 +3891,8 @@ static void reweight_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, enqueue_load_avg(cfs_rq, se); if (se->on_rq) { place_entity(cfs_rq, se, 0); + if (rel_vprot) + se->vprot = se->vruntime + vprot; update_load_add(&cfs_rq->load, se->load.weight); if (!curr) __enqueue_entity(cfs_rq, se); @@ -5175,7 +5280,7 @@ place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) * * vl_i = (W + w_i)*vl'_i / W */ - load = cfs_rq->avg_load; + load = cfs_rq->sum_weight; if (curr && curr->on_rq) load += scale_load_down(curr->load.weight); @@ -5427,7 +5532,7 @@ dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) } static void -set_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) +set_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, bool first) { clear_buddies(cfs_rq, se); @@ -5442,7 +5547,8 @@ set_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) __dequeue_entity(cfs_rq, se); update_load_avg(cfs_rq, se, UPDATE_TG); - set_protect_slice(cfs_rq, se); + if (first) + set_protect_slice(cfs_rq, se); } update_stats_curr_start(cfs_rq, se); @@ -8950,13 +9056,13 @@ pick_next_task_fair(struct rq *rq, struct task_struct *prev, struct rq_flags *rf pse = parent_entity(pse); } if (se_depth >= pse_depth) { - set_next_entity(cfs_rq_of(se), se); + set_next_entity(cfs_rq_of(se), se, true); se = parent_entity(se); } } put_prev_entity(cfs_rq, pse); - set_next_entity(cfs_rq, se); + set_next_entity(cfs_rq, se, true); __set_next_task_fair(rq, p, true); } @@ -9056,7 +9162,7 @@ static void yield_task_fair(struct rq *rq) */ if (entity_eligible(cfs_rq, se)) { se->vruntime = se->deadline; - se->deadline += calc_delta_fair(se->slice, se); + update_deadline(cfs_rq, se); } } @@ -13319,8 +13425,8 @@ bool cfs_prio_less(const struct task_struct *a, const struct task_struct *b, * zero_vruntime_fi, which would have been updated in prior calls * to se_fi_update(). */ - delta = (s64)(sea->vruntime - seb->vruntime) + - (s64)(cfs_rqb->zero_vruntime_fi - cfs_rqa->zero_vruntime_fi); + delta = vruntime_op(sea->vruntime, "-", seb->vruntime) + + vruntime_op(cfs_rqb->zero_vruntime_fi, "-", cfs_rqa->zero_vruntime_fi); return delta > 0; } @@ -13555,7 +13661,7 @@ static void set_next_task_fair(struct rq *rq, struct task_struct *p, bool first) for_each_sched_entity(se) { struct cfs_rq *cfs_rq = cfs_rq_of(se); - set_next_entity(cfs_rq, se); + set_next_entity(cfs_rq, se, first); /* ensure bandwidth has been allocated on our new cfs_rq */ account_cfs_rq_runtime(cfs_rq, 0); } diff --git a/kernel/sched/idle.c b/kernel/sched/idle.c index abf8f15d60c9e4..8e00d95fb33880 100644 --- a/kernel/sched/idle.c +++ b/kernel/sched/idle.c @@ -161,6 +161,14 @@ static int call_cpuidle(struct cpuidle_driver *drv, struct cpuidle_device *dev, return cpuidle_enter(drv, dev, next_state); } +static void idle_call_stop_or_retain_tick(bool stop_tick) +{ + if (stop_tick || tick_nohz_tick_stopped()) + tick_nohz_idle_stop_tick(); + else + tick_nohz_idle_retain_tick(); +} + /** * cpuidle_idle_call - the main idle function * @@ -170,7 +178,7 @@ static int call_cpuidle(struct cpuidle_driver *drv, struct cpuidle_device *dev, * set, and it returns with polling set. If it ever stops polling, it * must clear the polling bit. */ -static void cpuidle_idle_call(void) +static void cpuidle_idle_call(bool stop_tick) { struct cpuidle_device *dev = cpuidle_get_device(); struct cpuidle_driver *drv = cpuidle_get_cpu_driver(dev); @@ -186,7 +194,7 @@ static void cpuidle_idle_call(void) } if (cpuidle_not_available(drv, dev)) { - tick_nohz_idle_stop_tick(); + idle_call_stop_or_retain_tick(stop_tick); default_idle_call(); goto exit_idle; @@ -221,24 +229,35 @@ static void cpuidle_idle_call(void) next_state = cpuidle_find_deepest_state(drv, dev, max_latency_ns); call_cpuidle(drv, dev, next_state); - } else { - bool stop_tick = true; + } else if (drv->state_count > 1) { + /* + * stop_tick is expected to be true by default by cpuidle + * governors, which allows them to select idle states with + * target residency above the tick period length. + */ + stop_tick = true; /* * Ask the cpuidle framework to choose a convenient idle state. */ next_state = cpuidle_select(drv, dev, &stop_tick); - if (stop_tick || tick_nohz_tick_stopped()) - tick_nohz_idle_stop_tick(); - else - tick_nohz_idle_retain_tick(); + idle_call_stop_or_retain_tick(stop_tick); entered_state = call_cpuidle(drv, dev, next_state); /* * Give the governor an opportunity to reflect on the outcome */ cpuidle_reflect(dev, entered_state); + } else { + idle_call_stop_or_retain_tick(stop_tick); + + /* + * If there is only a single idle state (or none), there is + * nothing meaningful for the governor to choose. Skip the + * governor and always use state 0. + */ + call_cpuidle(drv, dev, 0); } exit_idle: @@ -259,6 +278,7 @@ static void cpuidle_idle_call(void) static void do_idle(void) { int cpu = smp_processor_id(); + bool got_tick = false; /* * Check if we need to update blocked load @@ -329,8 +349,9 @@ static void do_idle(void) tick_nohz_idle_restart_tick(); cpu_idle_poll(); } else { - cpuidle_idle_call(); + cpuidle_idle_call(got_tick); } + got_tick = tick_nohz_idle_got_tick(); arch_cpu_idle_exit(); } diff --git a/kernel/sched/rt.c b/kernel/sched/rt.c index f1867fe8e5c535..e0ff909050190b 100644 --- a/kernel/sched/rt.c +++ b/kernel/sched/rt.c @@ -2100,6 +2100,7 @@ static void push_rt_tasks(struct rq *rq) */ static int rto_next_cpu(struct root_domain *rd) { + int this_cpu = smp_processor_id(); int next; int cpu; @@ -2123,6 +2124,10 @@ static int rto_next_cpu(struct root_domain *rd) rd->rto_cpu = cpu; + /* Do not send IPI to self */ + if (cpu == this_cpu) + continue; + if (cpu < nr_cpu_ids) return cpu; diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index bd350e40859d89..8ae171de9e6cb3 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -678,8 +678,8 @@ struct cfs_rq { unsigned int h_nr_runnable; /* SCHED_{NORMAL,BATCH,IDLE} */ unsigned int h_nr_idle; /* SCHED_IDLE */ - s64 avg_vruntime; - u64 avg_load; + s64 sum_w_vruntime; + u64 sum_weight; u64 zero_vruntime; #ifdef CONFIG_SCHED_CORE @@ -803,9 +803,12 @@ struct scx_rq { cpumask_var_t cpus_to_kick_if_idle; cpumask_var_t cpus_to_preempt; cpumask_var_t cpus_to_wait; + cpumask_var_t cpus_to_sync; + bool kick_sync_pending; unsigned long kick_sync; local_t reenq_local_deferred; struct balance_callback deferred_bal_cb; + struct balance_callback kick_sync_bal_cb; struct irq_work deferred_irq_work; struct irq_work kick_cpus_irq_work; struct scx_dispatch_q bypass_dsq; @@ -3758,8 +3761,10 @@ static __always_inline void mm_unset_cid_on_task(struct task_struct *t) static __always_inline void mm_drop_cid_on_cpu(struct mm_struct *mm, struct mm_cid_pcpu *pcp) { /* Clear the ONCPU bit, but do not set UNSET in the per CPU storage */ - pcp->cid = cpu_cid_to_cid(pcp->cid); - mm_drop_cid(mm, pcp->cid); + if (cid_on_cpu(pcp->cid)) { + pcp->cid = cpu_cid_to_cid(pcp->cid); + mm_drop_cid(mm, pcp->cid); + } } static inline unsigned int __mm_get_cid(struct mm_struct *mm, unsigned int max_cids) diff --git a/kernel/sched/syscalls.c b/kernel/sched/syscalls.c index 6f10db3646e7f6..cadb0e9fe19b90 100644 --- a/kernel/sched/syscalls.c +++ b/kernel/sched/syscalls.c @@ -284,6 +284,35 @@ static bool check_same_owner(struct task_struct *p) uid_eq(cred->euid, pcred->uid)); } +#ifdef CONFIG_RT_MUTEXES +static inline void __setscheduler_dl_pi(int newprio, int policy, + struct task_struct *p, + struct sched_change_ctx *scope) +{ + /* + * In case a DEADLINE task (either proper or boosted) gets + * setscheduled to a lower priority class, check if it neeeds to + * inherit parameters from a potential pi_task. In that case make + * sure replenishment happens with the next enqueue. + */ + + if (dl_prio(newprio) && !dl_policy(policy)) { + struct task_struct *pi_task = rt_mutex_get_top_task(p); + + if (pi_task) { + p->dl.pi_se = pi_task->dl.pi_se; + scope->flags |= ENQUEUE_REPLENISH; + } + } +} +#else /* !CONFIG_RT_MUTEXES */ +static inline void __setscheduler_dl_pi(int newprio, int policy, + struct task_struct *p, + struct sched_change_ctx *scope) +{ +} +#endif /* !CONFIG_RT_MUTEXES */ + #ifdef CONFIG_UCLAMP_TASK static int uclamp_validate(struct task_struct *p, @@ -655,6 +684,7 @@ int __sched_setscheduler(struct task_struct *p, __setscheduler_params(p, attr); p->sched_class = next_class; p->prio = newprio; + __setscheduler_dl_pi(newprio, policy, p, scope); } __setscheduler_uclamp(p, attr); diff --git a/kernel/sys.c b/kernel/sys.c index 8b58eece4e580b..93e4d60b6fead8 100644 --- a/kernel/sys.c +++ b/kernel/sys.c @@ -45,6 +45,7 @@ #include #include #include +#include #include #include @@ -2515,6 +2516,16 @@ static int prctl_set_thp_disable(bool thp_disable, unsigned long flags, return 0; } +int __weak arch_prctl_mem_model_get(struct task_struct *t) +{ + return -EINVAL; +} + +int __weak arch_prctl_mem_model_set(struct task_struct *t, unsigned long val) +{ + return -EINVAL; +} + SYSCALL_DEFINE5(prctl, int, option, unsigned long, arg2, unsigned long, arg3, unsigned long, arg4, unsigned long, arg5) { @@ -2528,6 +2539,16 @@ SYSCALL_DEFINE5(prctl, int, option, unsigned long, arg2, unsigned long, arg3, error = 0; switch (option) { + case PR_GET_MEM_MODEL: + if (arg2 || arg3 || arg4 || arg5) + return -EINVAL; + error = arch_prctl_mem_model_get(me); + break; + case PR_SET_MEM_MODEL: + if (arg3 || arg4 || arg5) + return -EINVAL; + error = arch_prctl_mem_model_set(me, arg2); + break; case PR_SET_PDEATHSIG: if (!valid_signal(arg2)) { error = -EINVAL; diff --git a/kernel/sysctl.c b/kernel/sysctl.c index 2cd767b9680eb6..c9389b50b82646 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -895,7 +895,7 @@ int proc_do_large_bitmap(const struct ctl_table *table, int dir, unsigned long bitmap_len = table->maxlen; unsigned long *bitmap = *(unsigned long **) table->data; unsigned long *tmp_bitmap = NULL; - char tr_a[] = { '-', ',', '\n' }, tr_b[] = { ',', '\n', 0 }, c; + char tr_a[] = { '-', ',', '\n' }, tr_b[] = { ',', '\n', 0 }, c = 0; if (!bitmap || !bitmap_len || !left || (*ppos && SYSCTL_KERN_TO_USER(dir))) { *lenp = 0; diff --git a/kernel/time/alarmtimer.c b/kernel/time/alarmtimer.c index 069d93bfb0c75c..b64db405ba5c71 100644 --- a/kernel/time/alarmtimer.c +++ b/kernel/time/alarmtimer.c @@ -540,7 +540,7 @@ static s64 alarm_timer_forward(struct k_itimer *timr, ktime_t now) { struct alarm *alarm = &timr->it.alarm.alarmtimer; - return alarm_forward(alarm, timr->it_interval, now); + return alarm_forward(alarm, now, timr->it_interval); } /** diff --git a/kernel/time/hrtimer.c b/kernel/time/hrtimer.c index 0e4bc1ca15ff10..84c8ab2a0cebf3 100644 --- a/kernel/time/hrtimer.c +++ b/kernel/time/hrtimer.c @@ -1742,7 +1742,7 @@ static void __run_hrtimer(struct hrtimer_cpu_base *cpu_base, lockdep_assert_held(&cpu_base->lock); - debug_deactivate(timer); + debug_hrtimer_deactivate(timer); base->running = timer; /* diff --git a/kernel/time/jiffies.c b/kernel/time/jiffies.c index d31a6d40d38dc4..11d09cd8037c52 100644 --- a/kernel/time/jiffies.c +++ b/kernel/time/jiffies.c @@ -162,8 +162,6 @@ EXPORT_SYMBOL(proc_dointvec_jiffies); int proc_dointvec_userhz_jiffies(const struct ctl_table *table, int dir, void *buffer, size_t *lenp, loff_t *ppos) { - if (SYSCTL_USER_TO_KERN(dir) && USER_HZ < HZ) - return -EINVAL; return proc_dointvec_conv(table, dir, buffer, lenp, ppos, do_proc_int_conv_userhz_jiffies); } diff --git a/kernel/time/sched_clock.c b/kernel/time/sched_clock.c index f39111830ca363..f3aaef695b8cdb 100644 --- a/kernel/time/sched_clock.c +++ b/kernel/time/sched_clock.c @@ -215,7 +215,7 @@ void sched_clock_register(u64 (*read)(void), int bits, unsigned long rate) update_clock_read_data(&rd); - if (sched_clock_timer.function != NULL) { + if (ACCESS_PRIVATE(&sched_clock_timer, function) != NULL) { /* update timeout for clock wrap */ hrtimer_start(&sched_clock_timer, cd.wrap_kt, HRTIMER_MODE_REL_HARD); diff --git a/kernel/time/time.c b/kernel/time/time.c index 0ba8e3c50d6257..155cf7def9146d 100644 --- a/kernel/time/time.c +++ b/kernel/time/time.c @@ -702,7 +702,7 @@ EXPORT_SYMBOL(clock_t_to_jiffies); * * Return: jiffies_64 value converted to 64-bit "clock_t" (CLOCKS_PER_SEC) */ -u64 jiffies_64_to_clock_t(u64 x) +notrace u64 jiffies_64_to_clock_t(u64 x) { #if (TICK_NSEC % (NSEC_PER_SEC / USER_HZ)) == 0 # if HZ < USER_HZ diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c index 91fa2003351c9d..c07e562ee4c1ad 100644 --- a/kernel/time/timekeeping.c +++ b/kernel/time/timekeeping.c @@ -2653,7 +2653,8 @@ static int timekeeping_validate_timex(const struct __kernel_timex *txc, bool aux if (aux_clock) { /* Auxiliary clocks are similar to TAI and do not have leap seconds */ - if (txc->status & (STA_INS | STA_DEL)) + if (txc->modes & ADJ_STATUS && + txc->status & (STA_INS | STA_DEL)) return -EINVAL; /* No TAI offset setting */ @@ -2661,7 +2662,8 @@ static int timekeeping_validate_timex(const struct __kernel_timex *txc, bool aux return -EINVAL; /* No PPS support either */ - if (txc->status & (STA_PPSFREQ | STA_PPSTIME)) + if (txc->modes & ADJ_STATUS && + txc->status & (STA_PPSFREQ | STA_PPSTIME)) return -EINVAL; } diff --git a/kernel/trace/blktrace.c b/kernel/trace/blktrace.c index d031c8d80be4fd..0548e64b08f230 100644 --- a/kernel/trace/blktrace.c +++ b/kernel/trace/blktrace.c @@ -383,8 +383,6 @@ static void __blk_add_trace(struct blk_trace *bt, sector_t sector, int bytes, cpu = raw_smp_processor_id(); if (blk_tracer) { - tracing_record_cmdline(current); - buffer = blk_tr->array_buffer.buffer; trace_ctx = tracing_gen_ctx_flags(0); switch (bt->version) { @@ -419,6 +417,7 @@ static void __blk_add_trace(struct blk_trace *bt, sector_t sector, int bytes, if (!event) return; + tracing_record_cmdline(current); switch (bt->version) { case 1: record_blktrace_event(ring_buffer_event_data(event), @@ -793,7 +792,7 @@ int blk_trace_setup(struct request_queue *q, char *name, dev_t dev, return PTR_ERR(bt); } blk_trace_setup_finalize(q, name, 1, bt, &buts2); - strcpy(buts.name, buts2.name); + strscpy(buts.name, buts2.name, BLKTRACE_BDEV_SIZE); mutex_unlock(&q->debugfs_mutex); if (copy_to_user(arg, &buts, sizeof(buts))) { diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c index fe28d86f7c3576..42734975a06bc0 100644 --- a/kernel/trace/bpf_trace.c +++ b/kernel/trace/bpf_trace.c @@ -1022,7 +1022,7 @@ const struct bpf_func_proto bpf_snprintf_btf_proto = { .func = bpf_snprintf_btf, .gpl_only = false, .ret_type = RET_INTEGER, - .arg1_type = ARG_PTR_TO_MEM, + .arg1_type = ARG_PTR_TO_MEM | MEM_WRITE, .arg2_type = ARG_CONST_SIZE, .arg3_type = ARG_PTR_TO_MEM | MEM_RDONLY, .arg4_type = ARG_CONST_SIZE, @@ -1526,7 +1526,7 @@ static const struct bpf_func_proto bpf_read_branch_records_proto = { .gpl_only = true, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, - .arg2_type = ARG_PTR_TO_MEM_OR_NULL, + .arg2_type = ARG_PTR_TO_MEM_OR_NULL | MEM_WRITE, .arg3_type = ARG_CONST_SIZE_OR_ZERO, .arg4_type = ARG_ANYTHING, }; @@ -1661,7 +1661,7 @@ static const struct bpf_func_proto bpf_get_stack_proto_raw_tp = { .gpl_only = true, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, - .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, + .arg2_type = ARG_PTR_TO_UNINIT_MEM, .arg3_type = ARG_CONST_SIZE_OR_ZERO, .arg4_type = ARG_ANYTHING, }; @@ -2441,8 +2441,10 @@ static void bpf_kprobe_multi_show_fdinfo(const struct bpf_link *link, struct seq_file *seq) { struct bpf_kprobe_multi_link *kmulti_link; + bool has_cookies; kmulti_link = container_of(link, struct bpf_kprobe_multi_link, link); + has_cookies = !!kmulti_link->cookies; seq_printf(seq, "kprobe_cnt:\t%u\n" @@ -2454,7 +2456,7 @@ static void bpf_kprobe_multi_show_fdinfo(const struct bpf_link *link, for (int i = 0; i < kmulti_link->cnt; i++) { seq_printf(seq, "%llu\t %pS\n", - kmulti_link->cookies[i], + has_cookies ? kmulti_link->cookies[i] : 0, (void *)kmulti_link->addrs[i]); } } @@ -2564,6 +2566,7 @@ kprobe_multi_link_prog_run(struct bpf_kprobe_multi_link *link, old_run_ctx = bpf_set_run_ctx(&run_ctx.session_ctx.run_ctx); err = bpf_prog_run(link->link.prog, regs); bpf_reset_run_ctx(old_run_ctx); + ftrace_partial_regs_update(fregs, bpf_kprobe_multi_pt_regs_ptr()); rcu_read_unlock(); out: @@ -2736,6 +2739,10 @@ int bpf_kprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *pr if (!is_kprobe_multi(prog)) return -EINVAL; + /* kprobe_multi is not allowed to be sleepable. */ + if (prog->sleepable) + return -EINVAL; + /* Writing to context is not allowed for kprobes. */ if (prog->aux->kprobe_write_ctx) return -EINVAL; diff --git a/kernel/trace/fgraph.c b/kernel/trace/fgraph.c index cc48d16be43e0e..40d373d65f9b96 100644 --- a/kernel/trace/fgraph.c +++ b/kernel/trace/fgraph.c @@ -539,7 +539,11 @@ static struct fgraph_ops fgraph_stub = { static struct fgraph_ops *fgraph_direct_gops = &fgraph_stub; DEFINE_STATIC_CALL(fgraph_func, ftrace_graph_entry_stub); DEFINE_STATIC_CALL(fgraph_retfunc, ftrace_graph_ret_stub); +#if FGRAPH_NO_DIRECT +static DEFINE_STATIC_KEY_FALSE(fgraph_do_direct); +#else static DEFINE_STATIC_KEY_TRUE(fgraph_do_direct); +#endif /** * ftrace_graph_stop - set to permanently disable function graph tracing @@ -843,7 +847,7 @@ __ftrace_return_to_handler(struct ftrace_regs *fregs, unsigned long frame_pointe bitmap = get_bitmap_bits(current, offset); #ifdef CONFIG_HAVE_STATIC_CALL - if (static_branch_likely(&fgraph_do_direct)) { + if (!FGRAPH_NO_DIRECT && static_branch_likely(&fgraph_do_direct)) { if (test_bit(fgraph_direct_gops->idx, &bitmap)) static_call(fgraph_retfunc)(&trace, fgraph_direct_gops, fregs); } else @@ -1285,6 +1289,9 @@ static void ftrace_graph_enable_direct(bool enable_branch, struct fgraph_ops *go trace_func_graph_ret_t retfunc = NULL; int i; + if (FGRAPH_NO_DIRECT) + return; + if (gops) { func = gops->entryfunc; retfunc = gops->retfunc; @@ -1303,11 +1310,14 @@ static void ftrace_graph_enable_direct(bool enable_branch, struct fgraph_ops *go static_call_update(fgraph_func, func); static_call_update(fgraph_retfunc, retfunc); if (enable_branch) - static_branch_disable(&fgraph_do_direct); + static_branch_enable(&fgraph_do_direct); } static void ftrace_graph_disable_direct(bool disable_branch) { + if (FGRAPH_NO_DIRECT) + return; + if (disable_branch) static_branch_disable(&fgraph_do_direct); static_call_update(fgraph_func, ftrace_graph_entry_stub); diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c index aa758efc373149..e835d878038b21 100644 --- a/kernel/trace/ftrace.c +++ b/kernel/trace/ftrace.c @@ -6049,15 +6049,8 @@ int register_ftrace_direct(struct ftrace_ops *ops, unsigned long addr) if (ftrace_hash_empty(hash)) return -EINVAL; - /* This is a "raw" address, and this should never happen. */ - if (WARN_ON_ONCE(ftrace_is_jmp(addr))) - return -EINVAL; - mutex_lock(&direct_mutex); - if (ops->flags & FTRACE_OPS_FL_JMP) - addr = ftrace_jmp_set(addr); - /* Make sure requested entries are not already registered.. */ size = 1 << hash->size_bits; for (i = 0; i < size; i++) { @@ -6178,13 +6171,6 @@ __modify_ftrace_direct(struct ftrace_ops *ops, unsigned long addr) lockdep_assert_held_once(&direct_mutex); - /* This is a "raw" address, and this should never happen. */ - if (WARN_ON_ONCE(ftrace_is_jmp(addr))) - return -EINVAL; - - if (ops->flags & FTRACE_OPS_FL_JMP) - addr = ftrace_jmp_set(addr); - /* Enable the tmp_ops to have the same functions as the direct ops */ ftrace_ops_init(&tmp_ops); tmp_ops.func_hash = ops->func_hash; @@ -7753,7 +7739,8 @@ ftrace_func_address_lookup(struct ftrace_mod_map *mod_map, int ftrace_mod_address_lookup(unsigned long addr, unsigned long *size, - unsigned long *off, char **modname, char *sym) + unsigned long *off, char **modname, + const unsigned char **modbuildid, char *sym) { struct ftrace_mod_map *mod_map; int ret = 0; @@ -7765,6 +7752,8 @@ ftrace_mod_address_lookup(unsigned long addr, unsigned long *size, if (ret) { if (modname) *modname = mod_map->mod->name; + if (modbuildid) + *modbuildid = module_buildid(mod_map->mod); break; } } diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c index 630221b00838ef..380791abeec5a2 100644 --- a/kernel/trace/ring_buffer.c +++ b/kernel/trace/ring_buffer.c @@ -1848,6 +1848,7 @@ static int rb_read_data_buffer(struct buffer_data_page *dpage, int tail, int cpu struct ring_buffer_event *event; u64 ts, delta; int events = 0; + int len; int e; *delta_ptr = 0; @@ -1855,9 +1856,12 @@ static int rb_read_data_buffer(struct buffer_data_page *dpage, int tail, int cpu ts = dpage->time_stamp; - for (e = 0; e < tail; e += rb_event_length(event)) { + for (e = 0; e < tail; e += len) { event = (struct ring_buffer_event *)(dpage->data + e); + len = rb_event_length(event); + if (len <= 0 || len > tail - e) + return -1; switch (event->type_len) { @@ -1918,6 +1922,8 @@ static void rb_meta_validate_events(struct ring_buffer_per_cpu *cpu_buffer) if (!meta || !meta->head_buffer) return; + orig_head = head_page = cpu_buffer->head_page; + /* Do the reader page first */ ret = rb_validate_buffer(cpu_buffer->reader_page->page, cpu_buffer->cpu); if (ret < 0) { @@ -1928,7 +1934,6 @@ static void rb_meta_validate_events(struct ring_buffer_per_cpu *cpu_buffer) entry_bytes += local_read(&cpu_buffer->reader_page->page->commit); local_set(&cpu_buffer->reader_page->entries, ret); - orig_head = head_page = cpu_buffer->head_page; ts = head_page->page->time_stamp; /* @@ -2047,7 +2052,7 @@ static void rb_meta_validate_events(struct ring_buffer_per_cpu *cpu_buffer) entries += ret; entry_bytes += local_read(&head_page->page->commit); - local_set(&cpu_buffer->head_page->entries, ret); + local_set(&head_page->entries, ret); if (head_page == cpu_buffer->commit_page) break; @@ -7287,6 +7292,27 @@ int ring_buffer_map(struct trace_buffer *buffer, int cpu, return err; } +/* + * This is called when a VMA is duplicated (e.g., on fork()) to increment + * the user_mapped counter without remapping pages. + */ +void ring_buffer_map_dup(struct trace_buffer *buffer, int cpu) +{ + struct ring_buffer_per_cpu *cpu_buffer; + + if (WARN_ON(!cpumask_test_cpu(cpu, buffer->cpumask))) + return; + + cpu_buffer = buffer->buffers[cpu]; + + guard(mutex)(&cpu_buffer->mapping_lock); + + if (cpu_buffer->user_mapped) + __rb_inc_dec_mapped(cpu_buffer, true); + else + WARN(1, "Unexpected buffer stat, it should be mapped"); +} + int ring_buffer_unmap(struct trace_buffer *buffer, int cpu) { struct ring_buffer_per_cpu *cpu_buffer; diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c index 8bd4ec08fb361a..906be54df83fb5 100644 --- a/kernel/trace/trace.c +++ b/kernel/trace/trace.c @@ -567,7 +567,7 @@ static bool update_marker_trace(struct trace_array *tr, int enabled) lockdep_assert_held(&event_mutex); if (enabled) { - if (!list_empty(&tr->marker_list)) + if (tr->trace_flags & TRACE_ITER(COPY_MARKER)) return false; list_add_rcu(&tr->marker_list, &marker_copies); @@ -575,10 +575,10 @@ static bool update_marker_trace(struct trace_array *tr, int enabled) return true; } - if (list_empty(&tr->marker_list)) + if (!(tr->trace_flags & TRACE_ITER(COPY_MARKER))) return false; - list_del_init(&tr->marker_list); + list_del_rcu(&tr->marker_list); tr->trace_flags &= ~TRACE_ITER(COPY_MARKER); return true; } @@ -4881,6 +4881,8 @@ static int tracing_single_release_tr(struct inode *inode, struct file *file) return single_release(inode, file); } +static bool update_last_data_if_empty(struct trace_array *tr); + static int tracing_open(struct inode *inode, struct file *file) { struct trace_array *tr = inode->i_private; @@ -4905,6 +4907,8 @@ static int tracing_open(struct inode *inode, struct file *file) tracing_reset_online_cpus(trace_buf); else tracing_reset_cpu(trace_buf, cpu); + + update_last_data_if_empty(tr); } if (file->f_mode & FMODE_READ) { @@ -5971,6 +5975,7 @@ tracing_set_trace_read(struct file *filp, char __user *ubuf, int tracer_init(struct tracer *t, struct trace_array *tr) { tracing_reset_online_cpus(&tr->array_buffer); + update_last_data_if_empty(tr); return t->init(tr); } @@ -7540,6 +7545,23 @@ char *trace_user_fault_read(struct trace_user_buf_info *tinfo, */ do { + /* + * It is possible that something is trying to migrate this + * task. What happens then, is when preemption is enabled, + * the migration thread will preempt this task, try to + * migrate it, fail, then let it run again. That will + * cause this to loop again and never succeed. + * On failures, enabled and disable preemption with + * migration enabled, to allow the migration thread to + * migrate this task. + */ + if (trys) { + preempt_enable_notrace(); + preempt_disable_notrace(); + cpu = smp_processor_id(); + buffer = per_cpu_ptr(tinfo->tbuf, cpu)->buf; + } + /* * If for some reason, copy_from_user() always causes a context * switch, this would then cause an infinite loop. @@ -7789,6 +7811,7 @@ int tracing_set_clock(struct trace_array *tr, const char *clockstr) ring_buffer_set_clock(tr->max_buffer.buffer, trace_clocks[i].func); tracing_reset_online_cpus(&tr->max_buffer); #endif + update_last_data_if_empty(tr); if (tr->scratch && !(tr->flags & TRACE_ARRAY_FL_LAST_BOOT)) { struct trace_scratch *tscratch = tr->scratch; @@ -8993,6 +9016,18 @@ static inline int get_snapshot_map(struct trace_array *tr) { return 0; } static inline void put_snapshot_map(struct trace_array *tr) { } #endif +/* + * This is called when a VMA is duplicated (e.g., on fork()) to increment + * the user_mapped counter without remapping pages. + */ +static void tracing_buffers_mmap_open(struct vm_area_struct *vma) +{ + struct ftrace_buffer_info *info = vma->vm_file->private_data; + struct trace_iterator *iter = &info->iter; + + ring_buffer_map_dup(iter->array_buffer->buffer, iter->cpu_file); +} + static void tracing_buffers_mmap_close(struct vm_area_struct *vma) { struct ftrace_buffer_info *info = vma->vm_file->private_data; @@ -9012,6 +9047,7 @@ static int tracing_buffers_may_split(struct vm_area_struct *vma, unsigned long a } static const struct vm_operations_struct tracing_buffers_vmops = { + .open = tracing_buffers_mmap_open, .close = tracing_buffers_mmap_close, .may_split = tracing_buffers_may_split, }; @@ -9398,7 +9434,7 @@ tracing_init_tracefs_percpu(struct trace_array *tr, long cpu) trace_create_cpu_file("stats", TRACE_MODE_READ, d_cpu, tr, cpu, &tracing_stats_fops); - trace_create_cpu_file("buffer_size_kb", TRACE_MODE_READ, d_cpu, + trace_create_cpu_file("buffer_size_kb", TRACE_MODE_WRITE, d_cpu, tr, cpu, &tracing_entries_fops); if (tr->range_addr_start) @@ -10117,7 +10153,7 @@ static void setup_trace_scratch(struct trace_array *tr, } static int -allocate_trace_buffer(struct trace_array *tr, struct array_buffer *buf, int size) +allocate_trace_buffer(struct trace_array *tr, struct array_buffer *buf, unsigned long size) { enum ring_buffer_flags rb_flags; struct trace_scratch *tscratch; @@ -10172,7 +10208,7 @@ static void free_trace_buffer(struct array_buffer *buf) } } -static int allocate_trace_buffers(struct trace_array *tr, int size) +static int allocate_trace_buffers(struct trace_array *tr, unsigned long size) { int ret; @@ -10511,18 +10547,19 @@ static int __remove_instance(struct trace_array *tr) list_del(&tr->list); - /* Disable all the flags that were enabled coming in */ - for (i = 0; i < TRACE_FLAGS_MAX_SIZE; i++) { - if ((1ULL << i) & ZEROED_TRACE_FLAGS) - set_tracer_flag(tr, 1ULL << i, 0); - } - if (printk_trace == tr) update_printk_trace(&global_trace); + /* Must be done before disabling all the flags */ if (update_marker_trace(tr, 0)) synchronize_rcu(); + /* Disable all the flags that were enabled coming in */ + for (i = 0; i < TRACE_FLAGS_MAX_SIZE; i++) { + if ((1ULL << i) & ZEROED_TRACE_FLAGS) + set_tracer_flag(tr, 1ULL << i, 0); + } + tracing_set_nop(tr); clear_ftrace_function_probes(tr); event_trace_del_tracer(tr); @@ -11538,7 +11575,7 @@ __init static void enable_instances(void) __init static int tracer_alloc_buffers(void) { - int ring_buf_size; + unsigned long ring_buf_size; int ret = -ENOMEM; diff --git a/kernel/trace/trace_events.c b/kernel/trace/trace_events.c index 137b4d9bb116d9..1833c2a42aa924 100644 --- a/kernel/trace/trace_events.c +++ b/kernel/trace/trace_events.c @@ -1295,6 +1295,9 @@ static void remove_event_file_dir(struct trace_event_file *file) free_event_filter(file->filter); file->flags |= EVENT_FILE_FL_FREED; event_file_put(file); + + /* Wake up hist poll waiters to notice the EVENT_FILE_FL_FREED flag. */ + hist_poll_wakeup(); } /* @@ -3963,11 +3966,6 @@ void trace_put_event_file(struct trace_event_file *file) EXPORT_SYMBOL_GPL(trace_put_event_file); #ifdef CONFIG_DYNAMIC_FTRACE - -/* Avoid typos */ -#define ENABLE_EVENT_STR "enable_event" -#define DISABLE_EVENT_STR "disable_event" - struct event_probe_data { struct trace_event_file *file; unsigned long count; @@ -4343,7 +4341,11 @@ static char bootup_event_buf[COMMAND_LINE_SIZE] __initdata; static __init int setup_trace_event(char *str) { - strscpy(bootup_event_buf, str, COMMAND_LINE_SIZE); + if (bootup_event_buf[0] != '\0') + strlcat(bootup_event_buf, ",", COMMAND_LINE_SIZE); + + strlcat(bootup_event_buf, str, COMMAND_LINE_SIZE); + trace_set_ring_buffer_expanded(NULL); disable_tracing_selftest("running event tracing"); @@ -4514,26 +4516,22 @@ static __init int event_trace_memsetup(void) return 0; } -__init void -early_enable_events(struct trace_array *tr, char *buf, bool disable_first) +/* + * Helper function to enable or disable a comma-separated list of events + * from the bootup buffer. + */ +static __init void __early_set_events(struct trace_array *tr, char *buf, bool enable) { char *token; - int ret; - - while (true) { - token = strsep(&buf, ","); - - if (!token) - break; + while ((token = strsep(&buf, ","))) { if (*token) { - /* Restarting syscalls requires that we stop them first */ - if (disable_first) + if (enable) { + if (ftrace_set_clr_event(tr, token, 1)) + pr_warn("Failed to enable trace event: %s\n", token); + } else { ftrace_set_clr_event(tr, token, 0); - - ret = ftrace_set_clr_event(tr, token, 1); - if (ret) - pr_warn("Failed to enable trace event: %s\n", token); + } } /* Put back the comma to allow this to be called again */ @@ -4542,6 +4540,32 @@ early_enable_events(struct trace_array *tr, char *buf, bool disable_first) } } +/** + * early_enable_events - enable events from the bootup buffer + * @tr: The trace array to enable the events in + * @buf: The buffer containing the comma separated list of events + * @disable_first: If true, disable all events in @buf before enabling them + * + * This function enables events from the bootup buffer. If @disable_first + * is true, it will first disable all events in the buffer before enabling + * them. + * + * For syscall events, which rely on a global refcount to register the + * SYSCALL_WORK_SYSCALL_TRACEPOINT flag (especially for pid 1), we must + * ensure the refcount hits zero before re-enabling them. A simple + * "disable then enable" per-event is not enough if multiple syscalls are + * used, as the refcount will stay above zero. Thus, we need a two-phase + * approach: disable all, then enable all. + */ +__init void +early_enable_events(struct trace_array *tr, char *buf, bool disable_first) +{ + if (disable_first) + __early_set_events(tr, buf, false); + + __early_set_events(tr, buf, true); +} + static __init int event_trace_enable(void) { struct trace_array *tr = top_trace_array(); diff --git a/kernel/trace/trace_events_hist.c b/kernel/trace/trace_events_hist.c index c97bb2fda5c02a..864aa33712ce46 100644 --- a/kernel/trace/trace_events_hist.c +++ b/kernel/trace/trace_events_hist.c @@ -5778,7 +5778,7 @@ static __poll_t event_hist_poll(struct file *file, struct poll_table_struct *wai guard(mutex)(&event_mutex); - event_file = event_file_data(file); + event_file = event_file_file(file); if (!event_file) return EPOLLERR; @@ -5816,7 +5816,7 @@ static int event_hist_open(struct inode *inode, struct file *file) guard(mutex)(&event_mutex); - event_file = event_file_data(file); + event_file = event_file_file(file); if (!event_file) { ret = -ENODEV; goto err; @@ -6911,7 +6911,7 @@ static int event_hist_trigger_parse(struct event_command *cmd_ops, remove_hist_vars(hist_data); - kfree(trigger_data); + trigger_data_free(trigger_data); destroy_hist_data(hist_data); goto out; diff --git a/kernel/trace/trace_events_trigger.c b/kernel/trace/trace_events_trigger.c index 06b75bcfc7b8b0..036a656edd1740 100644 --- a/kernel/trace/trace_events_trigger.c +++ b/kernel/trace/trace_events_trigger.c @@ -22,6 +22,39 @@ static struct task_struct *trigger_kthread; static struct llist_head trigger_data_free_list; static DEFINE_MUTEX(trigger_data_kthread_mutex); +static int trigger_kthread_fn(void *ignore); + +static void trigger_create_kthread_locked(void) +{ + lockdep_assert_held(&trigger_data_kthread_mutex); + + if (!trigger_kthread) { + struct task_struct *kthread; + + kthread = kthread_create(trigger_kthread_fn, NULL, + "trigger_data_free"); + if (!IS_ERR(kthread)) + WRITE_ONCE(trigger_kthread, kthread); + } +} + +static void trigger_data_free_queued_locked(void) +{ + struct event_trigger_data *data, *tmp; + struct llist_node *llnodes; + + lockdep_assert_held(&trigger_data_kthread_mutex); + + llnodes = llist_del_all(&trigger_data_free_list); + if (!llnodes) + return; + + tracepoint_synchronize_unregister(); + + llist_for_each_entry_safe(data, tmp, llnodes, llist) + kfree(data); +} + /* Bulk garbage collection of event_trigger_data elements */ static int trigger_kthread_fn(void *ignore) { @@ -50,33 +83,56 @@ static int trigger_kthread_fn(void *ignore) void trigger_data_free(struct event_trigger_data *data) { + if (!data) + return; + if (data->cmd_ops->set_filter) data->cmd_ops->set_filter(NULL, data, NULL); + /* + * Boot-time trigger registration can fail before kthread creation + * works. Keep the deferred-free semantics during boot and let late + * init start the kthread to drain the list. + */ + if (system_state == SYSTEM_BOOTING && !trigger_kthread) { + llist_add(&data->llist, &trigger_data_free_list); + return; + } + if (unlikely(!trigger_kthread)) { guard(mutex)(&trigger_data_kthread_mutex); + + trigger_create_kthread_locked(); /* Check again after taking mutex */ if (!trigger_kthread) { - struct task_struct *kthread; - - kthread = kthread_create(trigger_kthread_fn, NULL, - "trigger_data_free"); - if (!IS_ERR(kthread)) - WRITE_ONCE(trigger_kthread, kthread); + llist_add(&data->llist, &trigger_data_free_list); + /* Drain the queued frees synchronously if creation failed. */ + trigger_data_free_queued_locked(); + return; } } - if (!trigger_kthread) { - /* Do it the slow way */ - tracepoint_synchronize_unregister(); - kfree(data); - return; - } - llist_add(&data->llist, &trigger_data_free_list); wake_up_process(trigger_kthread); } +static int __init trigger_data_free_init(void) +{ + guard(mutex)(&trigger_data_kthread_mutex); + + if (llist_empty(&trigger_data_free_list)) + return 0; + + trigger_create_kthread_locked(); + if (trigger_kthread) + wake_up_process(trigger_kthread); + else + trigger_data_free_queued_locked(); + + return 0; +} +late_initcall(trigger_data_free_init); + static inline void data_ops_trigger(struct event_trigger_data *data, struct trace_buffer *buffer, void *rec, struct ring_buffer_event *event) diff --git a/kernel/trace/trace_functions_graph.c b/kernel/trace/trace_functions_graph.c index 1de6f157362131..b9c81fbd98dc83 100644 --- a/kernel/trace/trace_functions_graph.c +++ b/kernel/trace/trace_functions_graph.c @@ -400,14 +400,19 @@ static void trace_graph_thresh_return(struct ftrace_graph_ret *trace, struct fgraph_ops *gops, struct ftrace_regs *fregs) { + unsigned long *task_var = fgraph_get_task_var(gops); struct fgraph_times *ftimes; struct trace_array *tr; + unsigned int trace_ctx; + u64 calltime, rettime; int size; + rettime = trace_clock_local(); + ftrace_graph_addr_finish(gops, trace); - if (trace_recursion_test(TRACE_GRAPH_NOTRACE_BIT)) { - trace_recursion_clear(TRACE_GRAPH_NOTRACE_BIT); + if (*task_var & TRACE_GRAPH_NOTRACE) { + *task_var &= ~TRACE_GRAPH_NOTRACE; return; } @@ -418,11 +423,13 @@ static void trace_graph_thresh_return(struct ftrace_graph_ret *trace, tr = gops->private; handle_nosleeptime(tr, trace, ftimes, size); - if (tracing_thresh && - (trace_clock_local() - ftimes->calltime < tracing_thresh)) + calltime = ftimes->calltime; + + if (tracing_thresh && (rettime - calltime < tracing_thresh)) return; - else - trace_graph_return(trace, gops, fregs); + + trace_ctx = tracing_gen_ctx(); + __trace_graph_return(tr, trace, trace_ctx, calltime, rettime); } static struct fgraph_ops funcgraph_ops = { diff --git a/kernel/trace/trace_hwlat.c b/kernel/trace/trace_hwlat.c index 2f7b94e98317cb..3fe274b84f1c2e 100644 --- a/kernel/trace/trace_hwlat.c +++ b/kernel/trace/trace_hwlat.c @@ -102,9 +102,9 @@ struct hwlat_sample { /* keep the global state somewhere. */ static struct hwlat_data { - struct mutex lock; /* protect changes */ + struct mutex lock; /* protect changes */ - u64 count; /* total since reset */ + atomic64_t count; /* total since reset */ u64 sample_window; /* total sampling window (on+off) */ u64 sample_width; /* active sampling portion of window */ @@ -193,8 +193,7 @@ void trace_hwlat_callback(bool enter) * get_sample - sample the CPU TSC and look for likely hardware latencies * * Used to repeatedly capture the CPU TSC (or similar), looking for potential - * hardware-induced latency. Called with interrupts disabled and with - * hwlat_data.lock held. + * hardware-induced latency. Called with interrupts disabled. */ static int get_sample(void) { @@ -204,6 +203,7 @@ static int get_sample(void) time_type start, t1, t2, last_t2; s64 diff, outer_diff, total, last_total = 0; u64 sample = 0; + u64 sample_width = READ_ONCE(hwlat_data.sample_width); u64 thresh = tracing_thresh; u64 outer_sample = 0; int ret = -1; @@ -267,7 +267,7 @@ static int get_sample(void) if (diff > sample) sample = diff; /* only want highest value */ - } while (total <= hwlat_data.sample_width); + } while (total <= sample_width); barrier(); /* finish the above in the view for NMIs */ trace_hwlat_callback_enabled = false; @@ -285,8 +285,7 @@ static int get_sample(void) if (kdata->nmi_total_ts) do_div(kdata->nmi_total_ts, NSEC_PER_USEC); - hwlat_data.count++; - s.seqnum = hwlat_data.count; + s.seqnum = atomic64_inc_return(&hwlat_data.count); s.duration = sample; s.outer_duration = outer_sample; s.nmi_total_ts = kdata->nmi_total_ts; @@ -832,7 +831,7 @@ static int hwlat_tracer_init(struct trace_array *tr) hwlat_trace = tr; - hwlat_data.count = 0; + atomic64_set(&hwlat_data.count, 0); tr->max_latency = 0; save_tracing_thresh = tracing_thresh; diff --git a/kernel/trace/trace_osnoise.c b/kernel/trace/trace_osnoise.c index 827104d00bc04a..f3b54bb834c47c 100644 --- a/kernel/trace/trace_osnoise.c +++ b/kernel/trace/trace_osnoise.c @@ -2073,8 +2073,8 @@ static void osnoise_hotplug_workfn(struct work_struct *dummy) if (!osnoise_has_registered_instances()) return; - guard(mutex)(&interface_lock); guard(cpus_read_lock)(); + guard(mutex)(&interface_lock); if (!cpu_online(cpu)) return; @@ -2237,11 +2237,11 @@ static ssize_t osnoise_options_write(struct file *filp, const char __user *ubuf, if (running) stop_per_cpu_kthreads(); - mutex_lock(&interface_lock); /* * avoid CPU hotplug operations that might read options. */ cpus_read_lock(); + mutex_lock(&interface_lock); retval = cnt; @@ -2257,8 +2257,8 @@ static ssize_t osnoise_options_write(struct file *filp, const char __user *ubuf, clear_bit(option, &osnoise_options); } - cpus_read_unlock(); mutex_unlock(&interface_lock); + cpus_read_unlock(); if (running) start_per_cpu_kthreads(); @@ -2345,16 +2345,16 @@ osnoise_cpus_write(struct file *filp, const char __user *ubuf, size_t count, if (running) stop_per_cpu_kthreads(); - mutex_lock(&interface_lock); /* * osnoise_cpumask is read by CPU hotplug operations. */ cpus_read_lock(); + mutex_lock(&interface_lock); cpumask_copy(&osnoise_cpumask, osnoise_cpumask_new); - cpus_read_unlock(); mutex_unlock(&interface_lock); + cpus_read_unlock(); if (running) start_per_cpu_kthreads(); diff --git a/kernel/trace/trace_probe.c b/kernel/trace/trace_probe.c index 2f571083ce9ec1..8dc495561c3f93 100644 --- a/kernel/trace/trace_probe.c +++ b/kernel/trace/trace_probe.c @@ -1069,7 +1069,7 @@ static int __parse_imm_string(char *str, char **pbuf, int offs) { size_t len = strlen(str); - if (str[len - 1] != '"') { + if (!len || str[len - 1] != '"') { trace_probe_log_err(offs + len, IMMSTR_NO_CLOSE); return -EINVAL; } diff --git a/kernel/ucount.c b/kernel/ucount.c index 586af49fc03e42..fc4a8f2d309651 100644 --- a/kernel/ucount.c +++ b/kernel/ucount.c @@ -47,7 +47,7 @@ static int set_permissions(struct ctl_table_header *head, int mode; /* Allow users with CAP_SYS_RESOURCE unrestrained access */ - if (ns_capable(user_ns, CAP_SYS_RESOURCE)) + if (ns_capable_noaudit(user_ns, CAP_SYS_RESOURCE)) mode = (table->mode & S_IRWXU) >> 6; else /* Allow all others at most read-only access */ diff --git a/kernel/watchdog.c b/kernel/watchdog.c index 366122f4a0f871..70eaa77242814f 100644 --- a/kernel/watchdog.c +++ b/kernel/watchdog.c @@ -550,7 +550,7 @@ static bool need_counting_irqs(void) u8 util; int tail = __this_cpu_read(cpustat_tail); - tail = (tail + NUM_HARDIRQ_REPORT - 1) % NUM_HARDIRQ_REPORT; + tail = (tail + NUM_SAMPLE_PERIODS - 1) % NUM_SAMPLE_PERIODS; util = __this_cpu_read(cpustat_util[tail][STATS_HARDIRQ]); return util > HARDIRQ_PERCENT_THRESH; } diff --git a/kernel/workqueue.c b/kernel/workqueue.c index 253311af47c6d3..5dcf00fca5c3eb 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -117,6 +117,8 @@ enum wq_internal_consts { MAYDAY_INTERVAL = HZ / 10, /* and then every 100ms */ CREATE_COOLDOWN = HZ, /* time to breath after fail */ + RESCUER_BATCH = 16, /* process items per turn */ + /* * Rescue workers are used only on emergencies and shared by * all cpus. Give MIN_NICE. @@ -286,6 +288,7 @@ struct pool_workqueue { struct list_head pending_node; /* LN: node on wq_node_nr_active->pending_pwqs */ struct list_head pwqs_node; /* WR: node on wq->pwqs */ struct list_head mayday_node; /* MD: node on wq->maydays */ + struct work_struct mayday_cursor; /* L: cursor on pool->worklist */ u64 stats[PWQ_NR_STATS]; @@ -1120,6 +1123,12 @@ static struct worker *find_worker_executing_work(struct worker_pool *pool, return NULL; } +static void mayday_cursor_func(struct work_struct *work) +{ + /* should not be processed, only for marking position */ + BUG(); +} + /** * move_linked_works - move linked works to a list * @work: start of series of works to be scheduled @@ -1182,6 +1191,16 @@ static bool assign_work(struct work_struct *work, struct worker *worker, lockdep_assert_held(&pool->lock); + /* The cursor work should not be processed */ + if (unlikely(work->func == mayday_cursor_func)) { + /* only worker_thread() can possibly take this branch */ + WARN_ON_ONCE(worker->rescue_wq); + if (nextp) + *nextp = list_next_entry(work, entry); + list_del_init(&work->entry); + return false; + } + /* * A single work shouldn't be executed concurrently by multiple workers. * __queue_work() ensures that @work doesn't jump to a different pool @@ -1830,8 +1849,20 @@ static void unplug_oldest_pwq(struct workqueue_struct *wq) raw_spin_lock_irq(&pwq->pool->lock); if (pwq->plugged) { pwq->plugged = false; - if (pwq_activate_first_inactive(pwq, true)) + if (pwq_activate_first_inactive(pwq, true)) { + /* + * While plugged, queueing skips activation which + * includes bumping the nr_active count and adding the + * pwq to nna->pending_pwqs if the count can't be + * obtained. We need to restore both for the pwq being + * unplugged. The first call activates the first + * inactive work item and the second, if there are more + * inactive, puts the pwq on pending_pwqs. + */ + pwq_activate_first_inactive(pwq, false); + kick_pool(pwq->pool); + } } raw_spin_unlock_irq(&pwq->pool->lock); } @@ -3440,22 +3471,30 @@ static int worker_thread(void *__worker) static bool assign_rescuer_work(struct pool_workqueue *pwq, struct worker *rescuer) { struct worker_pool *pool = pwq->pool; + struct work_struct *cursor = &pwq->mayday_cursor; struct work_struct *work, *n; /* need rescue? */ if (!pwq->nr_active || !need_to_create_worker(pool)) return false; - /* - * Slurp in all works issued via this workqueue and - * process'em. - */ - list_for_each_entry_safe(work, n, &pool->worklist, entry) { - if (get_work_pwq(work) == pwq && assign_work(work, rescuer, &n)) + /* search from the start or cursor if available */ + if (list_empty(&cursor->entry)) + work = list_first_entry(&pool->worklist, struct work_struct, entry); + else + work = list_next_entry(cursor, entry); + + /* find the next work item to rescue */ + list_for_each_entry_safe_from(work, n, &pool->worklist, entry) { + if (get_work_pwq(work) == pwq && assign_work(work, rescuer, &n)) { pwq->stats[PWQ_STAT_RESCUED]++; + /* put the cursor for next search */ + list_move_tail(&cursor->entry, &n->entry); + return true; + } } - return !list_empty(&rescuer->scheduled); + return false; } /** @@ -3512,6 +3551,7 @@ static int rescuer_thread(void *__rescuer) struct pool_workqueue *pwq = list_first_entry(&wq->maydays, struct pool_workqueue, mayday_node); struct worker_pool *pool = pwq->pool; + unsigned int count = 0; __set_current_state(TASK_RUNNING); list_del_init(&pwq->mayday_node); @@ -3524,19 +3564,16 @@ static int rescuer_thread(void *__rescuer) WARN_ON_ONCE(!list_empty(&rescuer->scheduled)); - if (assign_rescuer_work(pwq, rescuer)) { + while (assign_rescuer_work(pwq, rescuer)) { process_scheduled_works(rescuer); /* - * The above execution of rescued work items could - * have created more to rescue through - * pwq_activate_first_inactive() or chained - * queueing. Let's put @pwq back on mayday list so - * that such back-to-back work items, which may be - * being used to relieve memory pressure, don't - * incur MAYDAY_INTERVAL delay inbetween. + * If the per-turn work item limit is reached and other + * PWQs are in mayday, requeue mayday for this PWQ and + * let the rescuer handle the other PWQs first. */ - if (pwq->nr_active && need_to_create_worker(pool)) { + if (++count > RESCUER_BATCH && !list_empty(&pwq->wq->maydays) && + pwq->nr_active && need_to_create_worker(pool)) { raw_spin_lock(&wq_mayday_lock); /* * Queue iff somebody else hasn't queued it already. @@ -3546,9 +3583,14 @@ static int rescuer_thread(void *__rescuer) list_add_tail(&pwq->mayday_node, &wq->maydays); } raw_spin_unlock(&wq_mayday_lock); + break; } } + /* The cursor can not be left behind without the rescuer watching it. */ + if (!list_empty(&pwq->mayday_cursor.entry) && list_empty(&pwq->mayday_node)) + list_del_init(&pwq->mayday_cursor.entry); + /* * Leave this pool. Notify regular workers; otherwise, we end up * with 0 concurrency and stalling the execution. @@ -5167,6 +5209,19 @@ static void init_pwq(struct pool_workqueue *pwq, struct workqueue_struct *wq, INIT_LIST_HEAD(&pwq->pwqs_node); INIT_LIST_HEAD(&pwq->mayday_node); kthread_init_work(&pwq->release_work, pwq_release_workfn); + + /* + * Set the dummy cursor work with valid function and get_work_pwq(). + * + * The cursor work should only be in the pwq->pool->worklist, and + * should not be treated as a processable work item. + * + * WORK_STRUCT_PENDING and WORK_STRUCT_INACTIVE just make it less + * surprise for kernel debugging tools and reviewers. + */ + INIT_WORK(&pwq->mayday_cursor, mayday_cursor_func); + atomic_long_set(&pwq->mayday_cursor.data, (unsigned long)pwq | + WORK_STRUCT_PENDING | WORK_STRUCT_PWQ | WORK_STRUCT_INACTIVE); } /* sync @pwq with the current state of its associated wq and link it */ @@ -6211,7 +6266,7 @@ static void pr_cont_worker_id(struct worker *worker) { struct worker_pool *pool = worker->pool; - if (pool->flags & WQ_BH) + if (pool->flags & POOL_BH) pr_cont("bh%s", pool->attrs->nice == HIGHPRI_NICE_LEVEL ? "-hi" : ""); else diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index ba36939fda79bf..52c7a3a89f0880 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -723,6 +723,7 @@ source "mm/Kconfig.debug" config DEBUG_OBJECTS bool "Debug object operations" + depends on PREEMPT_COUNT || !DEFERRED_STRUCT_PAGE_INIT depends on DEBUG_KERNEL help If you say Y here, additional code will be inserted into the @@ -1273,7 +1274,7 @@ config BOOTPARAM_HUNG_TASK_PANIC high-availability systems that have uptime guarantees and where a hung tasks must be resolved ASAP. - Say N if unsure. + Say 0 if unsure. config DETECT_HUNG_TASK_BLOCKER bool "Dump Hung Tasks Blocker" @@ -1687,33 +1688,6 @@ config STACKTRACE It is also used by various kernel debugging features that require stack trace generation. -config WARN_ALL_UNSEEDED_RANDOM - bool "Warn for all uses of unseeded randomness" - default n - help - Some parts of the kernel contain bugs relating to their use of - cryptographically secure random numbers before it's actually possible - to generate those numbers securely. This setting ensures that these - flaws don't go unnoticed, by enabling a message, should this ever - occur. This will allow people with obscure setups to know when things - are going wrong, so that they might contact developers about fixing - it. - - Unfortunately, on some models of some architectures getting - a fully seeded CRNG is extremely difficult, and so this can - result in dmesg getting spammed for a surprisingly long - time. This is really bad from a security perspective, and - so architecture maintainers really need to do what they can - to get the CRNG seeded sooner after the system is booted. - However, since users cannot do anything actionable to - address this, by default this option is disabled. - - Say Y here if you want to receive warnings for all uses of - unseeded randomness. This will be of use primarily for - those developers interested in improving the security of - Linux kernels running on their architecture (or - subarchitecture). - config DEBUG_KOBJECT bool "kobject debugging" depends on DEBUG_KERNEL diff --git a/lib/bootconfig.c b/lib/bootconfig.c index 81f29c29f47b60..5d3802eba52a3b 100644 --- a/lib/bootconfig.c +++ b/lib/bootconfig.c @@ -316,7 +316,7 @@ int __init xbc_node_compose_key_after(struct xbc_node *root, depth ? "." : ""); if (ret < 0) return ret; - if (ret > size) { + if (ret >= size) { size = 0; } else { size -= ret; @@ -532,9 +532,9 @@ static char *skip_spaces_until_newline(char *p) static int __init __xbc_open_brace(char *p) { /* Push the last key as open brace */ - open_brace[brace_index++] = xbc_node_index(last_parent); if (brace_index >= XBC_DEPTH_MAX) return xbc_parse_error("Exceed max depth of braces", p); + open_brace[brace_index++] = xbc_node_index(last_parent); return 0; } @@ -712,7 +712,8 @@ static int __init xbc_parse_kv(char **k, char *v, int op) if (op == ':') { unsigned short nidx = child->next; - xbc_init_node(child, v, XBC_VALUE); + if (xbc_init_node(child, v, XBC_VALUE) < 0) + return xbc_parse_error("Failed to override value", v); child->next = nidx; /* keep subkeys */ goto array; } @@ -791,7 +792,7 @@ static int __init xbc_verify_tree(void) /* Brace closing */ if (brace_index) { - n = &xbc_nodes[open_brace[brace_index]]; + n = &xbc_nodes[open_brace[brace_index - 1]]; return xbc_parse_error("Brace is not closed", xbc_node_get_data(n)); } diff --git a/lib/bug.c b/lib/bug.c index 623c467a8b76c7..aab9e6a40c5f95 100644 --- a/lib/bug.c +++ b/lib/bug.c @@ -173,10 +173,8 @@ struct bug_entry *find_bug(unsigned long bugaddr) return module_find_bug(bugaddr); } -__diag_push(); -__diag_ignore(GCC, all, "-Wsuggest-attribute=format", - "Not a valid __printf() conversion candidate."); -static void __warn_printf(const char *fmt, struct pt_regs *regs) +static __printf(1, 0) +void __warn_printf(const char *fmt, struct pt_regs *regs) { if (!fmt) return; @@ -195,7 +193,6 @@ static void __warn_printf(const char *fmt, struct pt_regs *regs) printk("%s", fmt); } -__diag_pop(); static enum bug_trap_type __report_bug(struct bug_entry *bug, unsigned long bugaddr, struct pt_regs *regs) { diff --git a/lib/crypto/chacha-block-generic.c b/lib/crypto/chacha-block-generic.c index 77f68de71066a9..4a6d627580cb62 100644 --- a/lib/crypto/chacha-block-generic.c +++ b/lib/crypto/chacha-block-generic.c @@ -87,6 +87,8 @@ void chacha_block_generic(struct chacha_state *state, &out[i * sizeof(u32)]); state->x[12]++; + + chacha_zeroize_state(&permuted_state); } EXPORT_SYMBOL(chacha_block_generic); @@ -110,5 +112,7 @@ void hchacha_block_generic(const struct chacha_state *state, memcpy(&out[0], &permuted_state.x[0], 16); memcpy(&out[4], &permuted_state.x[12], 16); + + chacha_zeroize_state(&permuted_state); } EXPORT_SYMBOL(hchacha_block_generic); diff --git a/lib/debugobjects.c b/lib/debugobjects.c index 89a1d6745dc2c6..12f50de85b621a 100644 --- a/lib/debugobjects.c +++ b/lib/debugobjects.c @@ -398,9 +398,26 @@ static void fill_pool(void) atomic_inc(&cpus_allocating); while (pool_should_refill(&pool_global)) { + gfp_t gfp = __GFP_HIGH | __GFP_NOWARN; HLIST_HEAD(head); - if (!kmem_alloc_batch(&head, obj_cache, __GFP_HIGH | __GFP_NOWARN)) + /* + * Allow reclaim only in preemptible context and during + * early boot. If not preemptible, the caller might hold + * locks causing a deadlock in the allocator. + * + * If the reclaim flag is not set during early boot then + * allocations, which happen before deferred page + * initialization has completed, will fail. + * + * In preemptible context the flag is harmless and not a + * performance issue as that's usually invoked from slow + * path initialization context. + */ + if (preemptible() || system_state < SYSTEM_SCHEDULING) + gfp |= __GFP_KSWAPD_RECLAIM; + + if (!kmem_alloc_batch(&head, obj_cache, gfp)) break; guard(raw_spinlock_irqsave)(&pool_lock); diff --git a/lib/iov_iter.c b/lib/iov_iter.c index 896760bad455fc..3abbe7405be434 100644 --- a/lib/iov_iter.c +++ b/lib/iov_iter.c @@ -277,7 +277,7 @@ static __always_inline size_t copy_from_user_iter_nocache(void __user *iter_from, size_t progress, size_t len, void *to, void *priv2) { - return __copy_from_user_inatomic_nocache(to + progress, iter_from, len); + return copy_from_user_inatomic_nontemporal(to + progress, iter_from, len); } size_t _copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i) @@ -296,7 +296,7 @@ static __always_inline size_t copy_from_user_iter_flushcache(void __user *iter_from, size_t progress, size_t len, void *to, void *priv2) { - return __copy_from_user_flushcache(to + progress, iter_from, len); + return copy_from_user_flushcache(to + progress, iter_from, len); } static __always_inline diff --git a/lib/kstrtox.c b/lib/kstrtox.c index bdde40cd69d789..97be2a39f53710 100644 --- a/lib/kstrtox.c +++ b/lib/kstrtox.c @@ -340,8 +340,8 @@ EXPORT_SYMBOL(kstrtos8); * @s: input string * @res: result * - * This routine returns 0 iff the first character is one of 'YyTt1NnFf0', or - * [oO][NnFf] for "on" and "off". Otherwise it will return -EINVAL. Value + * This routine returns 0 iff the first character is one of 'EeYyTt1DdNnFf0', + * or [oO][NnFf] for "on" and "off". Otherwise it will return -EINVAL. Value * pointed to by res is updated upon finding a match. */ noinline diff --git a/lib/objpool.c b/lib/objpool.c index b998b720c7329d..d98fadf1de169f 100644 --- a/lib/objpool.c +++ b/lib/objpool.c @@ -142,7 +142,7 @@ int objpool_init(struct objpool_head *pool, int nr_objs, int object_size, pool->gfp = gfp & ~__GFP_ZERO; pool->context = context; pool->release = release; - slot_size = nr_cpu_ids * sizeof(struct objpool_slot); + slot_size = nr_cpu_ids * sizeof(struct objpool_slot *); pool->cpu_slots = kzalloc(slot_size, pool->gfp); if (!pool->cpu_slots) return -ENOMEM; diff --git a/lib/xarray.c b/lib/xarray.c index 9a8b4916540cf1..fe7f18d7194187 100644 --- a/lib/xarray.c +++ b/lib/xarray.c @@ -1738,9 +1738,6 @@ void *xa_store(struct xarray *xa, unsigned long index, void *entry, gfp_t gfp) } EXPORT_SYMBOL(xa_store); -static inline void *__xa_cmpxchg_raw(struct xarray *xa, unsigned long index, - void *old, void *entry, gfp_t gfp); - /** * __xa_cmpxchg() - Conditionally replace an entry in the XArray. * @xa: XArray. @@ -1767,7 +1764,29 @@ void *__xa_cmpxchg(struct xarray *xa, unsigned long index, } EXPORT_SYMBOL(__xa_cmpxchg); -static inline void *__xa_cmpxchg_raw(struct xarray *xa, unsigned long index, +/** + * __xa_cmpxchg_raw() - Conditionally replace an entry in the XArray. + * @xa: XArray. + * @index: Index into array. + * @old: Old value to test against. + * @entry: New value to place in array. + * @gfp: Memory allocation flags. + * + * You must already be holding the xa_lock when calling this function. + * It will drop the lock if needed to allocate memory, and then reacquire + * it afterwards. + * + * If the entry at @index is the same as @old, replace it with @entry. + * If the return value is equal to @old, then the exchange was successful. + * + * This function is the same as __xa_cmpxchg() except that it does not coerce + * XA_ZERO_ENTRY to NULL on egress. + * + * Context: Any context. Expects xa_lock to be held on entry. May + * release and reacquire xa_lock if @gfp flags permit. + * Return: The old value at this index or xa_err() if an error happened. + */ +void *__xa_cmpxchg_raw(struct xarray *xa, unsigned long index, void *old, void *entry, gfp_t gfp) { XA_STATE(xas, xa, index); @@ -1787,6 +1806,7 @@ static inline void *__xa_cmpxchg_raw(struct xarray *xa, unsigned long index, return xas_result(&xas, curr); } +EXPORT_SYMBOL(__xa_cmpxchg_raw); /** * __xa_insert() - Store this entry in the XArray if no entry is present. diff --git a/mm/backing-dev.c b/mm/backing-dev.c index c5740c6d37a2c4..d51c9c4d7b8df6 100644 --- a/mm/backing-dev.c +++ b/mm/backing-dev.c @@ -618,12 +618,13 @@ static void cgwb_release_workfn(struct work_struct *work) wb_shutdown(wb); css_put(wb->memcg_css); - css_put(wb->blkcg_css); - mutex_unlock(&wb->bdi->cgwb_release_mutex); /* triggers blkg destruction if no online users left */ blkcg_unpin_online(wb->blkcg_css); + css_put(wb->blkcg_css); + mutex_unlock(&wb->bdi->cgwb_release_mutex); + fprop_local_destroy_percpu(&wb->memcg_completions); spin_lock_irq(&cgwb_lock); diff --git a/mm/damon/core.c b/mm/damon/core.c index 84f80a20f2336e..0464ef163b7c56 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -197,7 +197,7 @@ static int damon_fill_regions_holes(struct damon_region *first, * @t: the given target. * @ranges: array of new monitoring target ranges. * @nr_ranges: length of @ranges. - * @min_sz_region: minimum region size. + * @min_region_sz: minimum region size. * * This function adds new regions to, or modify existing regions of a * monitoring target to fit in specific ranges. @@ -205,7 +205,7 @@ static int damon_fill_regions_holes(struct damon_region *first, * Return: 0 if success, or negative error code otherwise. */ int damon_set_regions(struct damon_target *t, struct damon_addr_range *ranges, - unsigned int nr_ranges, unsigned long min_sz_region) + unsigned int nr_ranges, unsigned long min_region_sz) { struct damon_region *r, *next; unsigned int i; @@ -242,16 +242,16 @@ int damon_set_regions(struct damon_target *t, struct damon_addr_range *ranges, /* no region intersects with this range */ newr = damon_new_region( ALIGN_DOWN(range->start, - min_sz_region), - ALIGN(range->end, min_sz_region)); + min_region_sz), + ALIGN(range->end, min_region_sz)); if (!newr) return -ENOMEM; damon_insert_region(newr, damon_prev_region(r), r, t); } else { /* resize intersecting regions to fit in this range */ first->ar.start = ALIGN_DOWN(range->start, - min_sz_region); - last->ar.end = ALIGN(range->end, min_sz_region); + min_region_sz); + last->ar.end = ALIGN(range->end, min_region_sz); /* fill possible holes in the range */ err = damon_fill_regions_holes(first, last, t); @@ -546,7 +546,7 @@ struct damon_ctx *damon_new_ctx(void) ctx->attrs.max_nr_regions = 1000; ctx->addr_unit = 1; - ctx->min_sz_region = DAMON_MIN_REGION; + ctx->min_region_sz = DAMON_MIN_REGION_SZ; INIT_LIST_HEAD(&ctx->adaptive_targets); INIT_LIST_HEAD(&ctx->schemes); @@ -1131,7 +1131,7 @@ static struct damon_target *damon_nth_target(int n, struct damon_ctx *ctx) * If @src has no region, @dst keeps current regions. */ static int damon_commit_target_regions(struct damon_target *dst, - struct damon_target *src, unsigned long src_min_sz_region) + struct damon_target *src, unsigned long src_min_region_sz) { struct damon_region *src_region; struct damon_addr_range *ranges; @@ -1148,7 +1148,7 @@ static int damon_commit_target_regions(struct damon_target *dst, i = 0; damon_for_each_region(src_region, src) ranges[i++] = src_region->ar; - err = damon_set_regions(dst, ranges, i, src_min_sz_region); + err = damon_set_regions(dst, ranges, i, src_min_region_sz); kfree(ranges); return err; } @@ -1156,11 +1156,11 @@ static int damon_commit_target_regions(struct damon_target *dst, static int damon_commit_target( struct damon_target *dst, bool dst_has_pid, struct damon_target *src, bool src_has_pid, - unsigned long src_min_sz_region) + unsigned long src_min_region_sz) { int err; - err = damon_commit_target_regions(dst, src, src_min_sz_region); + err = damon_commit_target_regions(dst, src, src_min_region_sz); if (err) return err; if (dst_has_pid) @@ -1187,7 +1187,7 @@ static int damon_commit_targets( err = damon_commit_target( dst_target, damon_target_has_pid(dst), src_target, damon_target_has_pid(src), - src->min_sz_region); + src->min_region_sz); if (err) return err; } else { @@ -1214,7 +1214,7 @@ static int damon_commit_targets( return -ENOMEM; err = damon_commit_target(new_target, false, src_target, damon_target_has_pid(src), - src->min_sz_region); + src->min_region_sz); if (err) { damon_destroy_target(new_target, NULL); return err; @@ -1241,6 +1241,10 @@ int damon_commit_ctx(struct damon_ctx *dst, struct damon_ctx *src) { int err; + dst->maybe_corrupted = true; + if (!is_power_of_2(src->min_region_sz)) + return -EINVAL; + err = damon_commit_schemes(dst, src); if (err) return err; @@ -1261,8 +1265,9 @@ int damon_commit_ctx(struct damon_ctx *dst, struct damon_ctx *src) } dst->ops = src->ops; dst->addr_unit = src->addr_unit; - dst->min_sz_region = src->min_sz_region; + dst->min_region_sz = src->min_region_sz; + dst->maybe_corrupted = false; return 0; } @@ -1294,8 +1299,8 @@ static unsigned long damon_region_sz_limit(struct damon_ctx *ctx) if (ctx->attrs.min_nr_regions) sz /= ctx->attrs.min_nr_regions; - if (sz < ctx->min_sz_region) - sz = ctx->min_sz_region; + if (sz < ctx->min_region_sz) + sz = ctx->min_region_sz; return sz; } @@ -1531,8 +1536,13 @@ int damos_walk(struct damon_ctx *ctx, struct damos_walk_control *control) } ctx->walk_control = control; mutex_unlock(&ctx->walk_control_lock); - if (!damon_is_running(ctx)) + if (!damon_is_running(ctx)) { + mutex_lock(&ctx->walk_control_lock); + if (ctx->walk_control == control) + ctx->walk_control = NULL; + mutex_unlock(&ctx->walk_control_lock); return -EINVAL; + } wait_for_completion(&control->completion); if (control->canceled) return -ECANCELED; @@ -1668,7 +1678,7 @@ static bool damos_valid_target(struct damon_ctx *c, struct damon_target *t, * @t: The target of the region. * @rp: The pointer to the region. * @s: The scheme to be applied. - * @min_sz_region: minimum region size. + * @min_region_sz: minimum region size. * * If a quota of a scheme has exceeded in a quota charge window, the scheme's * action would applied to only a part of the target access pattern fulfilling @@ -1686,7 +1696,8 @@ static bool damos_valid_target(struct damon_ctx *c, struct damon_target *t, * Return: true if the region should be entirely skipped, false otherwise. */ static bool damos_skip_charged_region(struct damon_target *t, - struct damon_region **rp, struct damos *s, unsigned long min_sz_region) + struct damon_region **rp, struct damos *s, + unsigned long min_region_sz) { struct damon_region *r = *rp; struct damos_quota *quota = &s->quota; @@ -1708,11 +1719,11 @@ static bool damos_skip_charged_region(struct damon_target *t, if (quota->charge_addr_from && r->ar.start < quota->charge_addr_from) { sz_to_skip = ALIGN_DOWN(quota->charge_addr_from - - r->ar.start, min_sz_region); + r->ar.start, min_region_sz); if (!sz_to_skip) { - if (damon_sz_region(r) <= min_sz_region) + if (damon_sz_region(r) <= min_region_sz) return true; - sz_to_skip = min_sz_region; + sz_to_skip = min_region_sz; } damon_split_region_at(t, r, sz_to_skip); r = damon_next_region(r); @@ -1738,7 +1749,7 @@ static void damos_update_stat(struct damos *s, static bool damos_filter_match(struct damon_ctx *ctx, struct damon_target *t, struct damon_region *r, struct damos_filter *filter, - unsigned long min_sz_region) + unsigned long min_region_sz) { bool matched = false; struct damon_target *ti; @@ -1755,8 +1766,8 @@ static bool damos_filter_match(struct damon_ctx *ctx, struct damon_target *t, matched = target_idx == filter->target_idx; break; case DAMOS_FILTER_TYPE_ADDR: - start = ALIGN_DOWN(filter->addr_range.start, min_sz_region); - end = ALIGN_DOWN(filter->addr_range.end, min_sz_region); + start = ALIGN_DOWN(filter->addr_range.start, min_region_sz); + end = ALIGN_DOWN(filter->addr_range.end, min_region_sz); /* inside the range */ if (start <= r->ar.start && r->ar.end <= end) { @@ -1792,7 +1803,7 @@ static bool damos_filter_out(struct damon_ctx *ctx, struct damon_target *t, s->core_filters_allowed = false; damos_for_each_core_filter(filter, s) { - if (damos_filter_match(ctx, t, r, filter, ctx->min_sz_region)) { + if (damos_filter_match(ctx, t, r, filter, ctx->min_region_sz)) { if (filter->allow) s->core_filters_allowed = true; return !filter->allow; @@ -1927,7 +1938,7 @@ static void damos_apply_scheme(struct damon_ctx *c, struct damon_target *t, if (c->ops.apply_scheme) { if (quota->esz && quota->charged_sz + sz > quota->esz) { sz = ALIGN_DOWN(quota->esz - quota->charged_sz, - c->min_sz_region); + c->min_region_sz); if (!sz) goto update_stat; damon_split_region_at(t, r, sz); @@ -1975,7 +1986,7 @@ static void damon_do_apply_schemes(struct damon_ctx *c, if (quota->esz && quota->charged_sz >= quota->esz) continue; - if (damos_skip_charged_region(t, &r, s, c->min_sz_region)) + if (damos_skip_charged_region(t, &r, s, c->min_region_sz)) continue; if (!damos_valid_target(c, t, r, s)) @@ -2424,7 +2435,7 @@ static void damon_split_region_at(struct damon_target *t, /* Split every region in the given target into 'nr_subs' regions */ static void damon_split_regions_of(struct damon_target *t, int nr_subs, - unsigned long min_sz_region) + unsigned long min_region_sz) { struct damon_region *r, *next; unsigned long sz_region, sz_sub = 0; @@ -2434,13 +2445,13 @@ static void damon_split_regions_of(struct damon_target *t, int nr_subs, sz_region = damon_sz_region(r); for (i = 0; i < nr_subs - 1 && - sz_region > 2 * min_sz_region; i++) { + sz_region > 2 * min_region_sz; i++) { /* * Randomly select size of left sub-region to be at * least 10 percent and at most 90% of original region */ sz_sub = ALIGN_DOWN(damon_rand(1, 10) * - sz_region / 10, min_sz_region); + sz_region / 10, min_region_sz); /* Do not allow blank region */ if (sz_sub == 0 || sz_sub >= sz_region) continue; @@ -2480,7 +2491,7 @@ static void kdamond_split_regions(struct damon_ctx *ctx) nr_subregions = 3; damon_for_each_target(t, ctx) - damon_split_regions_of(t, nr_subregions, ctx->min_sz_region); + damon_split_regions_of(t, nr_subregions, ctx->min_region_sz); last_nr_regions = nr_regions; } @@ -2601,10 +2612,11 @@ static void kdamond_call(struct damon_ctx *ctx, bool cancel) complete(&control->completion); } else if (control->canceled && control->dealloc_on_cancel) { kfree(control); - continue; } else { list_add(&control->list, &repeat_controls); } + if (!cancel && ctx->maybe_corrupted) + break; } control = list_first_entry_or_null(&repeat_controls, struct damon_call_control, list); @@ -2637,6 +2649,8 @@ static int kdamond_wait_activation(struct damon_ctx *ctx) kdamond_usleep(min_wait_time); kdamond_call(ctx, false); + if (ctx->maybe_corrupted) + return -EINVAL; damos_walk_cancel(ctx); } return -EBUSY; @@ -2722,6 +2736,8 @@ static int kdamond_fn(void *data) * kdamond_merge_regions() if possible, to reduce overhead */ kdamond_call(ctx, false); + if (ctx->maybe_corrupted) + break; if (!list_empty(&ctx->schemes)) kdamond_apply_schemes(ctx); else @@ -2850,7 +2866,7 @@ static bool damon_find_biggest_system_ram(unsigned long *start, * @t: The monitoring target to set the region. * @start: The pointer to the start address of the region. * @end: The pointer to the end address of the region. - * @min_sz_region: Minimum region size. + * @min_region_sz: Minimum region size. * * This function sets the region of @t as requested by @start and @end. If the * values of @start and @end are zero, however, this function finds the biggest @@ -2862,7 +2878,7 @@ static bool damon_find_biggest_system_ram(unsigned long *start, */ int damon_set_region_biggest_system_ram_default(struct damon_target *t, unsigned long *start, unsigned long *end, - unsigned long min_sz_region) + unsigned long min_region_sz) { struct damon_addr_range addr_range; @@ -2875,7 +2891,7 @@ int damon_set_region_biggest_system_ram_default(struct damon_target *t, addr_range.start = *start; addr_range.end = *end; - return damon_set_regions(t, &addr_range, 1, min_sz_region); + return damon_set_regions(t, &addr_range, 1, min_region_sz); } /* diff --git a/mm/damon/lru_sort.c b/mm/damon/lru_sort.c index 49b4bc294f4e1e..9cef1619527f49 100644 --- a/mm/damon/lru_sort.c +++ b/mm/damon/lru_sort.c @@ -212,7 +212,7 @@ static int damon_lru_sort_apply_parameters(void) if (!monitor_region_start && !monitor_region_end) addr_unit = 1; param_ctx->addr_unit = addr_unit; - param_ctx->min_sz_region = max(DAMON_MIN_REGION / addr_unit, 1); + param_ctx->min_region_sz = max(DAMON_MIN_REGION_SZ / addr_unit, 1); if (!damon_lru_sort_mon_attrs.sample_interval) { err = -EINVAL; @@ -243,7 +243,7 @@ static int damon_lru_sort_apply_parameters(void) err = damon_set_region_biggest_system_ram_default(param_target, &monitor_region_start, &monitor_region_end, - param_ctx->min_sz_region); + param_ctx->min_region_sz); if (err) goto out; err = damon_commit_ctx(ctx, param_ctx); diff --git a/mm/damon/reclaim.c b/mm/damon/reclaim.c index 36a582e09eaef1..c262ec6cb54586 100644 --- a/mm/damon/reclaim.c +++ b/mm/damon/reclaim.c @@ -208,7 +208,7 @@ static int damon_reclaim_apply_parameters(void) if (!monitor_region_start && !monitor_region_end) addr_unit = 1; param_ctx->addr_unit = addr_unit; - param_ctx->min_sz_region = max(DAMON_MIN_REGION / addr_unit, 1); + param_ctx->min_region_sz = max(DAMON_MIN_REGION_SZ / addr_unit, 1); if (!damon_reclaim_mon_attrs.aggr_interval) { err = -EINVAL; @@ -251,7 +251,7 @@ static int damon_reclaim_apply_parameters(void) err = damon_set_region_biggest_system_ram_default(param_target, &monitor_region_start, &monitor_region_end, - param_ctx->min_sz_region); + param_ctx->min_region_sz); if (err) goto out; err = damon_commit_ctx(ctx, param_ctx); diff --git a/mm/damon/stat.c b/mm/damon/stat.c index ed8e3629d31a43..723bdd673c105b 100644 --- a/mm/damon/stat.c +++ b/mm/damon/stat.c @@ -145,12 +145,59 @@ static int damon_stat_damon_call_fn(void *data) return 0; } +struct damon_stat_system_ram_range_walk_arg { + bool walked; + struct resource res; +}; + +static int damon_stat_system_ram_walk_fn(struct resource *res, void *arg) +{ + struct damon_stat_system_ram_range_walk_arg *a = arg; + + if (!a->walked) { + a->walked = true; + a->res.start = res->start; + } + a->res.end = res->end; + return 0; +} + +static unsigned long damon_stat_res_to_core_addr(resource_size_t ra, + unsigned long addr_unit) +{ + /* + * Use div_u64() for avoiding linking errors related with __udivdi3, + * __aeabi_uldivmod, or similar problems. This should also improve the + * performance optimization (read div_u64() comment for the detail). + */ + if (sizeof(ra) == 8 && sizeof(addr_unit) == 4) + return div_u64(ra, addr_unit); + return ra / addr_unit; +} + +static int damon_stat_set_monitoring_region(struct damon_target *t, + unsigned long addr_unit, unsigned long min_region_sz) +{ + struct damon_addr_range addr_range; + struct damon_stat_system_ram_range_walk_arg arg = {}; + + walk_system_ram_res(0, -1, &arg, damon_stat_system_ram_walk_fn); + if (!arg.walked) + return -EINVAL; + addr_range.start = damon_stat_res_to_core_addr( + arg.res.start, addr_unit); + addr_range.end = damon_stat_res_to_core_addr( + arg.res.end + 1, addr_unit); + if (addr_range.end <= addr_range.start) + return -EINVAL; + return damon_set_regions(t, &addr_range, 1, min_region_sz); +} + static struct damon_ctx *damon_stat_build_ctx(void) { struct damon_ctx *ctx; struct damon_attrs attrs; struct damon_target *target; - unsigned long start = 0, end = 0; ctx = damon_new_ctx(); if (!ctx) @@ -188,8 +235,8 @@ static struct damon_ctx *damon_stat_build_ctx(void) if (!target) goto free_out; damon_add_target(ctx, target); - if (damon_set_region_biggest_system_ram_default(target, &start, &end, - ctx->min_sz_region)) + if (damon_stat_set_monitoring_region(target, ctx->addr_unit, + ctx->min_region_sz)) goto free_out; return ctx; free_out: @@ -206,6 +253,12 @@ static int damon_stat_start(void) { int err; + if (damon_stat_context) { + if (damon_is_running(damon_stat_context)) + return -EAGAIN; + damon_destroy_ctx(damon_stat_context); + } + damon_stat_context = damon_stat_build_ctx(); if (!damon_stat_context) return -ENOMEM; @@ -222,6 +275,7 @@ static void damon_stat_stop(void) { damon_stop(&damon_stat_context, 1); damon_destroy_ctx(damon_stat_context); + damon_stat_context = NULL; } static int damon_stat_enabled_store( diff --git a/mm/damon/sysfs.c b/mm/damon/sysfs.c index 95fd9375a7d841..fc50edf3c42c88 100644 --- a/mm/damon/sysfs.c +++ b/mm/damon/sysfs.c @@ -1365,7 +1365,7 @@ static int damon_sysfs_set_attrs(struct damon_ctx *ctx, static int damon_sysfs_set_regions(struct damon_target *t, struct damon_sysfs_regions *sysfs_regions, - unsigned long min_sz_region) + unsigned long min_region_sz) { struct damon_addr_range *ranges = kmalloc_array(sysfs_regions->nr, sizeof(*ranges), GFP_KERNEL | __GFP_NOWARN); @@ -1387,7 +1387,7 @@ static int damon_sysfs_set_regions(struct damon_target *t, if (ranges[i - 1].end > ranges[i].start) goto out; } - err = damon_set_regions(t, ranges, sysfs_regions->nr, min_sz_region); + err = damon_set_regions(t, ranges, sysfs_regions->nr, min_region_sz); out: kfree(ranges); return err; @@ -1409,7 +1409,8 @@ static int damon_sysfs_add_target(struct damon_sysfs_target *sys_target, return -EINVAL; } t->obsolete = sys_target->obsolete; - return damon_sysfs_set_regions(t, sys_target->regions, ctx->min_sz_region); + return damon_sysfs_set_regions(t, sys_target->regions, + ctx->min_region_sz); } static int damon_sysfs_add_targets(struct damon_ctx *ctx, @@ -1469,8 +1470,8 @@ static int damon_sysfs_apply_inputs(struct damon_ctx *ctx, ctx->addr_unit = sys_ctx->addr_unit; /* addr_unit is respected by only DAMON_OPS_PADDR */ if (sys_ctx->ops_id == DAMON_OPS_PADDR) - ctx->min_sz_region = max( - DAMON_MIN_REGION / sys_ctx->addr_unit, 1); + ctx->min_region_sz = max( + DAMON_MIN_REGION_SZ / sys_ctx->addr_unit, 1); err = damon_sysfs_set_attrs(ctx, sys_ctx->attrs); if (err) return err; @@ -1525,8 +1526,10 @@ static int damon_sysfs_commit_input(void *data) if (IS_ERR(param_ctx)) return PTR_ERR(param_ctx); test_ctx = damon_sysfs_new_test_ctx(kdamond->damon_ctx); - if (!test_ctx) + if (!test_ctx) { + damon_destroy_ctx(param_ctx); return -ENOMEM; + } err = damon_commit_ctx(test_ctx, param_ctx); if (err) goto out; @@ -1619,9 +1622,12 @@ static int damon_sysfs_repeat_call_fn(void *data) if (!mutex_trylock(&damon_sysfs_lock)) return 0; + if (sysfs_kdamond->contexts->nr != 1) + goto out; damon_sysfs_upd_tuned_intervals(sysfs_kdamond); damon_sysfs_upd_schemes_stats(sysfs_kdamond); damon_sysfs_upd_schemes_effective_quotas(sysfs_kdamond); +out: mutex_unlock(&damon_sysfs_lock); return 0; } @@ -1667,7 +1673,8 @@ static int damon_sysfs_turn_damon_on(struct damon_sysfs_kdamond *kdamond) repeat_call_control->data = kdamond; repeat_call_control->repeat = true; repeat_call_control->dealloc_on_cancel = true; - damon_call(ctx, repeat_call_control); + if (damon_call(ctx, repeat_call_control)) + kfree(repeat_call_control); return err; } @@ -1749,6 +1756,9 @@ static int damon_sysfs_update_schemes_tried_regions( static int damon_sysfs_handle_cmd(enum damon_sysfs_cmd cmd, struct damon_sysfs_kdamond *kdamond) { + if (cmd != DAMON_SYSFS_CMD_OFF && kdamond->contexts->nr != 1) + return -EINVAL; + switch (cmd) { case DAMON_SYSFS_CMD_ON: return damon_sysfs_turn_damon_on(kdamond); diff --git a/mm/damon/tests/vaddr-kunit.h b/mm/damon/tests/vaddr-kunit.h index 30dc5459f1d2c0..cfae870178bfd5 100644 --- a/mm/damon/tests/vaddr-kunit.h +++ b/mm/damon/tests/vaddr-kunit.h @@ -147,7 +147,7 @@ static void damon_do_test_apply_three_regions(struct kunit *test, damon_add_region(r, t); } - damon_set_regions(t, three_regions, 3, DAMON_MIN_REGION); + damon_set_regions(t, three_regions, 3, DAMON_MIN_REGION_SZ); for (i = 0; i < nr_expected / 2; i++) { r = __nth_region_of(t, i); diff --git a/mm/damon/vaddr.c b/mm/damon/vaddr.c index 23ed738a0bd6f9..226a3f0c9b4a71 100644 --- a/mm/damon/vaddr.c +++ b/mm/damon/vaddr.c @@ -19,8 +19,8 @@ #include "ops-common.h" #ifdef CONFIG_DAMON_VADDR_KUNIT_TEST -#undef DAMON_MIN_REGION -#define DAMON_MIN_REGION 1 +#undef DAMON_MIN_REGION_SZ +#define DAMON_MIN_REGION_SZ 1 #endif /* @@ -78,7 +78,7 @@ static int damon_va_evenly_split_region(struct damon_target *t, orig_end = r->ar.end; sz_orig = damon_sz_region(r); - sz_piece = ALIGN_DOWN(sz_orig / nr_pieces, DAMON_MIN_REGION); + sz_piece = ALIGN_DOWN(sz_orig / nr_pieces, DAMON_MIN_REGION_SZ); if (!sz_piece) return -EINVAL; @@ -161,12 +161,12 @@ static int __damon_va_three_regions(struct mm_struct *mm, swap(first_gap, second_gap); /* Store the result */ - regions[0].start = ALIGN(start, DAMON_MIN_REGION); - regions[0].end = ALIGN(first_gap.start, DAMON_MIN_REGION); - regions[1].start = ALIGN(first_gap.end, DAMON_MIN_REGION); - regions[1].end = ALIGN(second_gap.start, DAMON_MIN_REGION); - regions[2].start = ALIGN(second_gap.end, DAMON_MIN_REGION); - regions[2].end = ALIGN(prev->vm_end, DAMON_MIN_REGION); + regions[0].start = ALIGN(start, DAMON_MIN_REGION_SZ); + regions[0].end = ALIGN(first_gap.start, DAMON_MIN_REGION_SZ); + regions[1].start = ALIGN(first_gap.end, DAMON_MIN_REGION_SZ); + regions[1].end = ALIGN(second_gap.start, DAMON_MIN_REGION_SZ); + regions[2].start = ALIGN(second_gap.end, DAMON_MIN_REGION_SZ); + regions[2].end = ALIGN(prev->vm_end, DAMON_MIN_REGION_SZ); return 0; } @@ -259,8 +259,8 @@ static void __damon_va_init_regions(struct damon_ctx *ctx, sz += regions[i].end - regions[i].start; if (ctx->attrs.min_nr_regions) sz /= ctx->attrs.min_nr_regions; - if (sz < DAMON_MIN_REGION) - sz = DAMON_MIN_REGION; + if (sz < DAMON_MIN_REGION_SZ) + sz = DAMON_MIN_REGION_SZ; /* Set the initial three regions of the target */ for (i = 0; i < 3; i++) { @@ -299,7 +299,7 @@ static void damon_va_update(struct damon_ctx *ctx) damon_for_each_target(t, ctx) { if (damon_va_three_regions(t, three_regions)) continue; - damon_set_regions(t, three_regions, 3, DAMON_MIN_REGION); + damon_set_regions(t, three_regions, 3, DAMON_MIN_REGION_SZ); } } diff --git a/mm/filemap.c b/mm/filemap.c index ebd75684cb0a7c..0f3c731549f56f 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -228,7 +228,8 @@ void __filemap_remove_folio(struct folio *folio, void *shadow) page_cache_delete(mapping, folio, shadow); } -void filemap_free_folio(struct address_space *mapping, struct folio *folio) +static void filemap_free_folio(const struct address_space *mapping, + struct folio *folio) { void (*free_folio)(struct folio *); @@ -1379,14 +1380,16 @@ static inline int folio_wait_bit_common(struct folio *folio, int bit_nr, #ifdef CONFIG_MIGRATION /** - * migration_entry_wait_on_locked - Wait for a migration entry to be removed - * @entry: migration swap entry. + * softleaf_entry_wait_on_locked - Wait for a migration entry or + * device_private entry to be removed. + * @entry: migration or device_private swap entry. * @ptl: already locked ptl. This function will drop the lock. * - * Wait for a migration entry referencing the given page to be removed. This is + * Wait for a migration entry referencing the given page, or device_private + * entry referencing a dvice_private page to be unlocked. This is * equivalent to folio_put_wait_locked(folio, TASK_UNINTERRUPTIBLE) except * this can be called without taking a reference on the page. Instead this - * should be called while holding the ptl for the migration entry referencing + * should be called while holding the ptl for @entry referencing * the page. * * Returns after unlocking the ptl. @@ -1394,7 +1397,7 @@ static inline int folio_wait_bit_common(struct folio *folio, int bit_nr, * This follows the same logic as folio_wait_bit_common() so see the comments * there. */ -void migration_entry_wait_on_locked(softleaf_t entry, spinlock_t *ptl) +void softleaf_entry_wait_on_locked(softleaf_t entry, spinlock_t *ptl) __releases(ptl) { struct wait_page_queue wait_page; @@ -1428,6 +1431,9 @@ void migration_entry_wait_on_locked(softleaf_t entry, spinlock_t *ptl) * If a migration entry exists for the page the migration path must hold * a valid reference to the page, and it must take the ptl to remove the * migration entry. So the page is valid until the ptl is dropped. + * Similarly any path attempting to drop the last reference to a + * device-private page needs to grab the ptl to remove the device-private + * entry. */ spin_unlock(ptl); @@ -3878,14 +3884,19 @@ vm_fault_t filemap_map_pages(struct vm_fault *vmf, unsigned int nr_pages = 0, folio_type; unsigned short mmap_miss = 0, mmap_miss_saved; + /* + * Recalculate end_pgoff based on file_end before calling + * next_uptodate_folio() to avoid races with concurrent + * truncation. + */ + file_end = DIV_ROUND_UP(i_size_read(mapping->host), PAGE_SIZE) - 1; + end_pgoff = min(end_pgoff, file_end); + rcu_read_lock(); folio = next_uptodate_folio(&xas, mapping, end_pgoff); if (!folio) goto out; - file_end = DIV_ROUND_UP(i_size_read(mapping->host), PAGE_SIZE) - 1; - end_pgoff = min(end_pgoff, file_end); - /* * Do not allow to map with PMD across i_size to preserve * SIGBUS semantics. diff --git a/mm/highmem.c b/mm/highmem.c index b5c8e4c2d5d49e..a33e4118395175 100644 --- a/mm/highmem.c +++ b/mm/highmem.c @@ -180,12 +180,13 @@ struct page *__kmap_to_page(void *vaddr) for (i = 0; i < kctrl->idx; i++) { unsigned long base_addr; int idx; + pte_t pteval = kctrl->pteval[i]; idx = arch_kmap_local_map_idx(i, pte_pfn(pteval)); base_addr = __fix_to_virt(FIX_KMAP_BEGIN + idx); if (base_addr == base) - return pte_page(kctrl->pteval[i]); + return pte_page(pteval); } } diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 40cf59301c21aa..3596168473642b 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -94,6 +94,9 @@ static inline bool file_thp_enabled(struct vm_area_struct *vma) inode = file_inode(vma->vm_file); + if (IS_ANON_FILE(inode)) + return false; + return !inode_is_open_for_write(inode) && S_ISREG(inode->i_mode); } @@ -2794,7 +2797,8 @@ int move_pages_huge_pmd(struct mm_struct *mm, pmd_t *dst_pmd, pmd_t *src_pmd, pm _dst_pmd = pmd_mkwrite(pmd_mkdirty(_dst_pmd), dst_vma); } else { src_pmdval = pmdp_huge_clear_flush(src_vma, src_addr, src_pmd); - _dst_pmd = folio_mk_pmd(src_folio, dst_vma->vm_page_prot); + _dst_pmd = move_soft_dirty_pmd(src_pmdval); + _dst_pmd = clear_uffd_wp_pmd(_dst_pmd); } set_pmd_at(mm, dst_addr, dst_pmd, _dst_pmd); @@ -3628,6 +3632,7 @@ static int __split_unmapped_folio(struct folio *folio, int new_order, const bool is_anon = folio_test_anon(folio); int old_order = folio_order(folio); int start_order = split_type == SPLIT_TYPE_UNIFORM ? new_order : old_order - 1; + struct folio *old_folio = folio; int split_order; /* @@ -3648,12 +3653,16 @@ static int __split_unmapped_folio(struct folio *folio, int new_order, * uniform split has xas_split_alloc() called before * irq is disabled to allocate enough memory, whereas * non-uniform split can handle ENOMEM. + * Use the to-be-split folio, so that a parallel + * folio_try_get() waits on it until xarray is updated + * with after-split folios and the original one is + * unfrozen. */ - if (split_type == SPLIT_TYPE_UNIFORM) - xas_split(xas, folio, old_order); - else { + if (split_type == SPLIT_TYPE_UNIFORM) { + xas_split(xas, old_folio, old_order); + } else { xas_set_order(xas, folio->index, split_order); - xas_try_split(xas, folio, old_order); + xas_try_split(xas, old_folio, old_order); if (xas_error(xas)) return xas_error(xas); } diff --git a/mm/hugetlb.c b/mm/hugetlb.c index a1832da0f62362..77e45dd50ba210 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -6717,6 +6717,15 @@ long hugetlb_reserve_pages(struct inode *inode, */ hugetlb_acct_memory(h, -gbl_resv); } + /* Restore used_hpages for pages that failed global reservation */ + if (gbl_reserve && spool) { + unsigned long flags; + + spin_lock_irqsave(&spool->lock, flags); + if (spool->max_hpages != -1) + spool->used_hpages -= gbl_reserve; + unlock_or_release_subpool(spool, flags); + } out_uncharge_cgroup: hugetlb_cgroup_uncharge_cgroup_rsvd(hstate_index(h), chg * pages_per_huge_page(h), h_cg); diff --git a/mm/internal.h b/mm/internal.h index f35dbcf99a86b0..d54ef4a8f2c568 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -471,7 +471,6 @@ unsigned find_lock_entries(struct address_space *mapping, pgoff_t *start, pgoff_t end, struct folio_batch *fbatch, pgoff_t *indices); unsigned find_get_entries(struct address_space *mapping, pgoff_t *start, pgoff_t end, struct folio_batch *fbatch, pgoff_t *indices); -void filemap_free_folio(struct address_space *mapping, struct folio *folio); int truncate_inode_folio(struct address_space *mapping, struct folio *folio); bool truncate_inode_partial_folio(struct folio *folio, loff_t start, loff_t end); diff --git a/mm/kasan/init.c b/mm/kasan/init.c index f084e7a5df1e5f..9c880f607c6a2d 100644 --- a/mm/kasan/init.c +++ b/mm/kasan/init.c @@ -292,7 +292,7 @@ static void kasan_free_pte(pte_t *pte_start, pmd_t *pmd) return; } - pte_free_kernel(&init_mm, (pte_t *)page_to_virt(pmd_page(*pmd))); + pte_free_kernel(&init_mm, pte_start); pmd_clear(pmd); } @@ -307,7 +307,7 @@ static void kasan_free_pmd(pmd_t *pmd_start, pud_t *pud) return; } - pmd_free(&init_mm, (pmd_t *)page_to_virt(pud_page(*pud))); + pmd_free(&init_mm, pmd_start); pud_clear(pud); } @@ -322,7 +322,7 @@ static void kasan_free_pud(pud_t *pud_start, p4d_t *p4d) return; } - pud_free(&init_mm, (pud_t *)page_to_virt(p4d_page(*p4d))); + pud_free(&init_mm, pud_start); p4d_clear(p4d); } @@ -337,7 +337,7 @@ static void kasan_free_p4d(p4d_t *p4d_start, pgd_t *pgd) return; } - p4d_free(&init_mm, (p4d_t *)page_to_virt(pgd_page(*pgd))); + p4d_free(&init_mm, p4d_start); pgd_clear(pgd); } diff --git a/mm/kfence/core.c b/mm/kfence/core.c index 4f79ec72075254..30959c97b881d4 100644 --- a/mm/kfence/core.c +++ b/mm/kfence/core.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -911,6 +912,20 @@ void __init kfence_alloc_pool_and_metadata(void) if (!kfence_sample_interval) return; + /* + * If KASAN hardware tags are enabled, disable KFENCE, because it + * does not support MTE yet. + */ + if (kasan_hw_tags_enabled()) { + pr_info("disabled as KASAN HW tags are enabled\n"); + if (__kfence_pool) { + memblock_free(__kfence_pool, KFENCE_POOL_SIZE); + __kfence_pool = NULL; + } + kfence_sample_interval = 0; + return; + } + /* * If the pool has already been initialized by arch, there is no need to * re-allocate the memory pool. @@ -984,14 +999,14 @@ static int kfence_init_late(void) #ifdef CONFIG_CONTIG_ALLOC struct page *pages; - pages = alloc_contig_pages(nr_pages_pool, GFP_KERNEL, first_online_node, - NULL); + pages = alloc_contig_pages(nr_pages_pool, GFP_KERNEL | __GFP_SKIP_KASAN, + first_online_node, NULL); if (!pages) return -ENOMEM; __kfence_pool = page_to_virt(pages); - pages = alloc_contig_pages(nr_pages_meta, GFP_KERNEL, first_online_node, - NULL); + pages = alloc_contig_pages(nr_pages_meta, GFP_KERNEL | __GFP_SKIP_KASAN, + first_online_node, NULL); if (pages) kfence_metadata_init = page_to_virt(pages); #else @@ -1001,11 +1016,13 @@ static int kfence_init_late(void) return -EINVAL; } - __kfence_pool = alloc_pages_exact(KFENCE_POOL_SIZE, GFP_KERNEL); + __kfence_pool = alloc_pages_exact(KFENCE_POOL_SIZE, + GFP_KERNEL | __GFP_SKIP_KASAN); if (!__kfence_pool) return -ENOMEM; - kfence_metadata_init = alloc_pages_exact(KFENCE_METADATA_SIZE, GFP_KERNEL); + kfence_metadata_init = alloc_pages_exact(KFENCE_METADATA_SIZE, + GFP_KERNEL | __GFP_SKIP_KASAN); #endif if (!kfence_metadata_init) diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 86f43b7e5f7104..a7b5192ad7d5a9 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -3052,7 +3052,7 @@ static void refill_obj_stock(struct obj_cgroup *objcg, unsigned int nr_bytes, if (!local_trylock(&obj_stock.lock)) { if (pgdat) - mod_objcg_mlstate(objcg, pgdat, idx, nr_bytes); + mod_objcg_mlstate(objcg, pgdat, idx, nr_acct); nr_pages = nr_bytes >> PAGE_SHIFT; nr_bytes = nr_bytes & (PAGE_SIZE - 1); atomic_add(nr_bytes, &objcg->nr_charged_bytes); @@ -5624,9 +5624,21 @@ subsys_initcall(mem_cgroup_swap_init); #endif /* CONFIG_SWAP */ -bool mem_cgroup_node_allowed(struct mem_cgroup *memcg, int nid) +void mem_cgroup_node_filter_allowed(struct mem_cgroup *memcg, nodemask_t *mask) { - return memcg ? cpuset_node_allowed(memcg->css.cgroup, nid) : true; + nodemask_t allowed; + + if (!memcg) + return; + + /* + * Since this interface is intended for use by migration paths, and + * reclaim and migration are subject to race conditions such as changes + * in effective_mems and hot-unpluging of nodes, inaccurate allowed + * mask is acceptable. + */ + cpuset_nodes_allowed(memcg->css.cgroup, &allowed); + nodes_and(*mask, *mask, allowed); } void mem_cgroup_show_protected_memory(struct mem_cgroup *memcg) diff --git a/mm/memfd_luo.c b/mm/memfd_luo.c index a34fccc23b6a42..02993d83d39c09 100644 --- a/mm/memfd_luo.c +++ b/mm/memfd_luo.c @@ -146,19 +146,56 @@ static int memfd_luo_preserve_folios(struct file *file, for (i = 0; i < nr_folios; i++) { struct memfd_luo_folio_ser *pfolio = &folios_ser[i]; struct folio *folio = folios[i]; - unsigned int flags = 0; err = kho_preserve_folio(folio); if (err) goto err_unpreserve; - if (folio_test_dirty(folio)) - flags |= MEMFD_LUO_FOLIO_DIRTY; - if (folio_test_uptodate(folio)) - flags |= MEMFD_LUO_FOLIO_UPTODATE; + folio_lock(folio); + + /* + * A dirty folio is one which has been written to. A clean folio + * is its opposite. Since a clean folio does not carry user + * data, it can be freed by page reclaim under memory pressure. + * + * Saving the dirty flag at prepare() time doesn't work since it + * can change later. Saving it at freeze() also won't work + * because the dirty bit is normally synced at unmap and there + * might still be a mapping of the file at freeze(). + * + * To see why this is a problem, say a folio is clean at + * preserve, but gets dirtied later. The pfolio flags will mark + * it as clean. After retrieve, the next kernel might try to + * reclaim this folio under memory pressure, losing user data. + * + * Unconditionally mark it dirty to avoid this problem. This + * comes at the cost of making clean folios un-reclaimable after + * live update. + */ + folio_mark_dirty(folio); + + /* + * If the folio is not uptodate, it was fallocated but never + * used. Saving this flag at prepare() doesn't work since it + * might change later when someone uses the folio. + * + * Since we have taken the performance penalty of allocating, + * zeroing, and pinning all the folios in the holes, take a bit + * more and zero all non-uptodate folios too. + * + * NOTE: For someone looking to improve preserve performance, + * this is a good place to look. + */ + if (!folio_test_uptodate(folio)) { + folio_zero_range(folio, 0, folio_size(folio)); + flush_dcache_folio(folio); + folio_mark_uptodate(folio); + } + + folio_unlock(folio); pfolio->pfn = folio_pfn(folio); - pfolio->flags = flags; + pfolio->flags = MEMFD_LUO_FOLIO_DIRTY | MEMFD_LUO_FOLIO_UPTODATE; pfolio->index = folio->index; } @@ -326,7 +363,12 @@ static void memfd_luo_finish(struct liveupdate_file_op_args *args) struct memfd_luo_folio_ser *folios_ser; struct memfd_luo_ser *ser; - if (args->retrieved) + /* + * If retrieve was successful, nothing to do. If it failed, retrieve() + * already cleaned up everything it could. So nothing to do there + * either. Only need to clean up when retrieve was not called. + */ + if (args->retrieve_status) return; ser = phys_to_virt(args->serialized_data); diff --git a/mm/memory.c b/mm/memory.c index da360a6eb8a48e..a4e6d92d7892cb 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -4684,7 +4684,8 @@ vm_fault_t do_swap_page(struct vm_fault *vmf) unlock_page(vmf->page); put_page(vmf->page); } else { - pte_unmap_unlock(vmf->pte, vmf->ptl); + pte_unmap(vmf->pte); + softleaf_entry_wait_on_locked(entry, vmf->ptl); } } else if (softleaf_is_hwpoison(entry)) { ret = VM_FAULT_HWPOISON; @@ -6774,11 +6775,16 @@ int follow_pfnmap_start(struct follow_pfnmap_args *args) pudp = pud_offset(p4dp, address); pud = pudp_get(pudp); - if (pud_none(pud)) + if (!pud_present(pud)) goto out; if (pud_leaf(pud)) { lock = pud_lock(mm, pudp); - if (!unlikely(pud_leaf(pud))) { + pud = pudp_get(pudp); + + if (unlikely(!pud_present(pud))) { + spin_unlock(lock); + goto out; + } else if (unlikely(!pud_leaf(pud))) { spin_unlock(lock); goto retry; } @@ -6790,9 +6796,16 @@ int follow_pfnmap_start(struct follow_pfnmap_args *args) pmdp = pmd_offset(pudp, address); pmd = pmdp_get_lockless(pmdp); + if (!pmd_present(pmd)) + goto out; if (pmd_leaf(pmd)) { lock = pmd_lock(mm, pmdp); - if (!unlikely(pmd_leaf(pmd))) { + pmd = pmdp_get(pmdp); + + if (unlikely(!pmd_present(pmd))) { + spin_unlock(lock); + goto out; + } else if (unlikely(!pmd_leaf(pmd))) { spin_unlock(lock); goto retry; } diff --git a/mm/memory_hotplug.c b/mm/memory_hotplug.c index a63ec679d86141..08767e689c25b4 100644 --- a/mm/memory_hotplug.c +++ b/mm/memory_hotplug.c @@ -1209,6 +1209,13 @@ int online_pages(unsigned long pfn, unsigned long nr_pages, if (node_arg.nid >= 0) node_set_state(nid, N_MEMORY); + /* + * Check whether we are adding normal memory to the node for the first + * time. + */ + if (!node_state(nid, N_NORMAL_MEMORY) && zone_idx(zone) <= ZONE_NORMAL) + node_set_state(nid, N_NORMAL_MEMORY); + if (need_zonelists_rebuild) build_all_zonelists(NULL); @@ -1908,6 +1915,8 @@ int offline_pages(unsigned long start_pfn, unsigned long nr_pages, unsigned long flags; char *reason; int ret; + unsigned long normal_pages = 0; + enum zone_type zt; /* * {on,off}lining is constrained to full memory sections (or more @@ -2055,6 +2064,17 @@ int offline_pages(unsigned long start_pfn, unsigned long nr_pages, /* reinitialise watermarks and update pcp limits */ init_per_zone_wmark_min(); + /* + * Check whether this operation removes the last normal memory from + * the node. We do this before clearing N_MEMORY to avoid the possible + * transient "!N_MEMORY && N_NORMAL_MEMORY" state. + */ + if (zone_idx(zone) <= ZONE_NORMAL) { + for (zt = 0; zt <= ZONE_NORMAL; zt++) + normal_pages += pgdat->node_zones[zt].present_pages; + if (!normal_pages) + node_clear_state(node, N_NORMAL_MEMORY); + } /* * Make sure to mark the node as memory-less before rebuilding the zone * list. Otherwise this node would still appear in the fallback lists. diff --git a/mm/mempolicy.c b/mm/mempolicy.c index 68a98ba5788211..74ebf38a7db1a4 100644 --- a/mm/mempolicy.c +++ b/mm/mempolicy.c @@ -488,7 +488,13 @@ void __mpol_put(struct mempolicy *pol) { if (!atomic_dec_and_test(&pol->refcnt)) return; - kmem_cache_free(policy_cache, pol); + /* + * Required to allow mmap_lock_speculative*() access, see for example + * futex_key_to_node_opt(). All accesses are serialized by mmap_lock, + * however the speculative lock section unbound by the normal lock + * boundaries, requiring RCU freeing. + */ + kfree_rcu(pol, rcu); } EXPORT_SYMBOL_FOR_MODULES(__mpol_put, "kvm"); @@ -1021,7 +1027,7 @@ static int vma_replace_policy(struct vm_area_struct *vma, } old = vma->vm_policy; - vma->vm_policy = new; /* protected by mmap_lock */ + WRITE_ONCE(vma->vm_policy, new); /* protected by mmap_lock */ mpol_put(old); return 0; diff --git a/mm/migrate.c b/mm/migrate.c index 4688b9e38cd2fd..cf6449b4202e49 100644 --- a/mm/migrate.c +++ b/mm/migrate.c @@ -499,7 +499,7 @@ void migration_entry_wait(struct mm_struct *mm, pmd_t *pmd, if (!softleaf_is_migration(entry)) goto out; - migration_entry_wait_on_locked(entry, ptl); + softleaf_entry_wait_on_locked(entry, ptl); return; out: spin_unlock(ptl); @@ -531,10 +531,10 @@ void migration_entry_wait_huge(struct vm_area_struct *vma, unsigned long addr, p * If migration entry existed, safe to release vma lock * here because the pgtable page won't be freed without the * pgtable lock released. See comment right above pgtable - * lock release in migration_entry_wait_on_locked(). + * lock release in softleaf_entry_wait_on_locked(). */ hugetlb_vma_unlock_read(vma); - migration_entry_wait_on_locked(entry, ptl); + softleaf_entry_wait_on_locked(entry, ptl); return; } @@ -552,7 +552,7 @@ void pmd_migration_entry_wait(struct mm_struct *mm, pmd_t *pmd) ptl = pmd_lock(mm, pmd); if (!pmd_is_migration_entry(*pmd)) goto unlock; - migration_entry_wait_on_locked(softleaf_from_pmd(*pmd), ptl); + softleaf_entry_wait_on_locked(softleaf_from_pmd(*pmd), ptl); return; unlock: spin_unlock(ptl); diff --git a/mm/migrate_device.c b/mm/migrate_device.c index 23379663b1e192..deab89fd454191 100644 --- a/mm/migrate_device.c +++ b/mm/migrate_device.c @@ -176,7 +176,7 @@ static int migrate_vma_collect_huge_pmd(pmd_t *pmdp, unsigned long start, } if (softleaf_is_migration(entry)) { - migration_entry_wait_on_locked(entry, ptl); + softleaf_entry_wait_on_locked(entry, ptl); spin_unlock(ptl); return -EAGAIN; } diff --git a/mm/mseal.c b/mm/mseal.c index ae442683c5c0dd..3d2f06046e90b3 100644 --- a/mm/mseal.c +++ b/mm/mseal.c @@ -56,7 +56,6 @@ static int mseal_apply(struct mm_struct *mm, unsigned long start, unsigned long end) { struct vm_area_struct *vma, *prev; - unsigned long curr_start = start; VMA_ITERATOR(vmi, mm, start); /* We know there are no gaps so this will be non-NULL. */ @@ -66,6 +65,7 @@ static int mseal_apply(struct mm_struct *mm, prev = vma; for_each_vma_range(vmi, vma, end) { + const unsigned long curr_start = MAX(vma->vm_start, start); const unsigned long curr_end = MIN(vma->vm_end, end); if (!(vma->vm_flags & VM_SEALED)) { @@ -79,7 +79,6 @@ static int mseal_apply(struct mm_struct *mm, } prev = vma; - curr_start = curr_end; } return 0; diff --git a/mm/numa_memblks.c b/mm/numa_memblks.c index 8f5735fda0a218..3f53464240e8d4 100644 --- a/mm/numa_memblks.c +++ b/mm/numa_memblks.c @@ -570,15 +570,16 @@ static int meminfo_to_nid(struct numa_meminfo *mi, u64 start) int phys_to_target_node(u64 start) { int nid = meminfo_to_nid(&numa_meminfo, start); + int reserved_nid = meminfo_to_nid(&numa_reserved_meminfo, start); /* - * Prefer online nodes, but if reserved memory might be - * hot-added continue the search with reserved ranges. + * Prefer online nodes unless the address is also described + * by reserved ranges, in which case use the reserved nid. */ - if (nid != NUMA_NO_NODE) + if (nid != NUMA_NO_NODE && reserved_nid == NUMA_NO_NODE) return nid; - return meminfo_to_nid(&numa_reserved_meminfo, start); + return reserved_nid; } EXPORT_SYMBOL_GPL(phys_to_target_node); diff --git a/mm/page_alloc.c b/mm/page_alloc.c index cbf758e27aa2c3..469ee8cb7b2ec3 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -1340,8 +1340,8 @@ static inline void pgalloc_tag_sub_pages(struct alloc_tag *tag, unsigned int nr) #endif /* CONFIG_MEM_ALLOC_PROFILING */ -__always_inline bool free_pages_prepare(struct page *page, - unsigned int order) +__always_inline bool __free_pages_prepare(struct page *page, + unsigned int order, fpi_t fpi_flags) { int bad = 0; bool skip_kasan_poison = should_skip_kasan_poison(page); @@ -1430,11 +1430,12 @@ __always_inline bool free_pages_prepare(struct page *page, page_cpupid_reset_last(page); page->flags.f &= ~PAGE_FLAGS_CHECK_AT_PREP; + page->private = 0; reset_page_owner(page, order); page_table_check_free(page, order); pgalloc_tag_sub(page, 1 << order); - if (!PageHighMem(page)) { + if (!PageHighMem(page) && !(fpi_flags & FPI_TRYLOCK)) { debug_check_no_locks_freed(page_address(page), PAGE_SIZE << order); debug_check_no_obj_freed(page_address(page), @@ -1473,6 +1474,11 @@ __always_inline bool free_pages_prepare(struct page *page, return true; } +bool free_pages_prepare(struct page *page, unsigned int order) +{ + return __free_pages_prepare(page, order, FPI_NONE); +} + /* * Frees a number of pages from the PCP lists * Assumes all pages on list are in same zone. @@ -1606,7 +1612,7 @@ static void __free_pages_ok(struct page *page, unsigned int order, unsigned long pfn = page_to_pfn(page); struct zone *zone = page_zone(page); - if (free_pages_prepare(page, order)) + if (__free_pages_prepare(page, order, fpi_flags)) free_one_page(zone, page, pfn, order, fpi_flags); } @@ -2970,7 +2976,7 @@ static void __free_frozen_pages(struct page *page, unsigned int order, return; } - if (!free_pages_prepare(page, order)) + if (!__free_pages_prepare(page, order, fpi_flags)) return; /* @@ -3027,7 +3033,7 @@ void free_unref_folios(struct folio_batch *folios) unsigned long pfn = folio_pfn(folio); unsigned int order = folio_order(folio); - if (!free_pages_prepare(&folio->page, order)) + if (!__free_pages_prepare(&folio->page, order, FPI_NONE)) continue; /* * Free orders not handled on the PCP directly to the @@ -4818,6 +4824,20 @@ __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order, compact_result == COMPACT_DEFERRED) goto nopage; + /* + * THP page faults may attempt local node only first, + * but are then allowed to only compact, not reclaim, + * see alloc_pages_mpol(). + * + * Compaction can fail for other reasons than those + * checked above and we don't want such THP allocations + * to put reclaim pressure on a single node in a + * situation where other nodes might have plenty of + * available memory. + */ + if (gfp_mask & __GFP_THISNODE) + goto nopage; + /* * Looks like reclaim/compaction is worth trying, but * sync compaction could be very expensive, so keep @@ -6921,7 +6941,8 @@ static int __alloc_contig_verify_gfp_mask(gfp_t gfp_mask, gfp_t *gfp_cc_mask) { const gfp_t reclaim_mask = __GFP_IO | __GFP_FS | __GFP_RECLAIM; const gfp_t action_mask = __GFP_COMP | __GFP_RETRY_MAYFAIL | __GFP_NOWARN | - __GFP_ZERO | __GFP_ZEROTAGS | __GFP_SKIP_ZERO; + __GFP_ZERO | __GFP_ZEROTAGS | __GFP_SKIP_ZERO | + __GFP_SKIP_KASAN; const gfp_t cc_action_mask = __GFP_RETRY_MAYFAIL | __GFP_NOWARN; /* diff --git a/mm/pagewalk.c b/mm/pagewalk.c index 90cc346a6ecf16..6b8905956d6abf 100644 --- a/mm/pagewalk.c +++ b/mm/pagewalk.c @@ -97,6 +97,7 @@ static int walk_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end, static int walk_pmd_range(pud_t *pud, unsigned long addr, unsigned long end, struct mm_walk *walk) { + pud_t pudval = pudp_get(pud); pmd_t *pmd; unsigned long next; const struct mm_walk_ops *ops = walk->ops; @@ -105,6 +106,24 @@ static int walk_pmd_range(pud_t *pud, unsigned long addr, unsigned long end, int err = 0; int depth = real_depth(3); + /* + * For PTE handling, pte_offset_map_lock() takes care of checking + * whether there actually is a page table. But it also has to be + * very careful about concurrent page table reclaim. + * + * Similarly, we have to be careful here - a PUD entry that points + * to a PMD table cannot go away, so we can just walk it. But if + * it's something else, we need to ensure we didn't race something, + * so need to retry. + * + * A pertinent example of this is a PUD refault after PUD split - + * we will need to split again or risk accessing invalid memory. + */ + if (!pud_present(pudval) || pud_leaf(pudval)) { + walk->action = ACTION_AGAIN; + return 0; + } + pmd = pmd_offset(pud, addr); do { again: @@ -218,12 +237,12 @@ static int walk_pud_range(p4d_t *p4d, unsigned long addr, unsigned long end, else if (pud_leaf(*pud) || !pud_present(*pud)) continue; /* Nothing to do. */ - if (pud_none(*pud)) - goto again; - err = walk_pmd_range(pud, addr, next, walk); if (err) break; + + if (walk->action == ACTION_AGAIN) + goto again; } while (pud++, addr = next, addr != end); return err; diff --git a/mm/rmap.c b/mm/rmap.c index 7b9879ef442d91..5b62b1da942e30 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -1843,7 +1843,14 @@ static inline unsigned int folio_unmap_pte_batch(struct folio *folio, if (pte_unused(pte)) return 1; - return folio_pte_batch(folio, pvmw->pte, pte, max_nr); + /* + * If unmap fails, we need to restore the ptes. To avoid accidentally + * upgrading write permissions for ptes that were not originally + * writable, and to avoid losing the soft-dirty bit, use the + * appropriate FPB flags. + */ + return folio_pte_batch_flags(folio, vma, pvmw->pte, &pte, max_nr, + FPB_RESPECT_WRITE | FPB_RESPECT_SOFT_DIRTY); } /* @@ -2331,11 +2338,17 @@ static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma, __maybe_unused pmd_t pmdval; if (flags & TTU_SPLIT_HUGE_PMD) { + /* + * split_huge_pmd_locked() might leave the + * folio mapped through PTEs. Retry the walk + * so we can detect this scenario and properly + * abort the walk. + */ split_huge_pmd_locked(vma, pvmw.address, pvmw.pmd, true); - ret = false; - page_vma_mapped_walk_done(&pvmw); - break; + flags &= ~TTU_SPLIT_HUGE_PMD; + page_vma_mapped_walk_restart(&pvmw); + continue; } #ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION pmdval = pmdp_get(pvmw.pmd); diff --git a/mm/slab_common.c b/mm/slab_common.c index eed7ea556cb1a4..ee994ec7f251ee 100644 --- a/mm/slab_common.c +++ b/mm/slab_common.c @@ -2133,8 +2133,11 @@ EXPORT_SYMBOL_GPL(kvfree_rcu_barrier); */ void kvfree_rcu_barrier_on_cache(struct kmem_cache *s) { - if (s->cpu_sheaves) + if (s->cpu_sheaves) { flush_rcu_sheaves_on_cache(s); + rcu_barrier(); + } + /* * TODO: Introduce a version of __kvfree_rcu_barrier() that works * on a specific slab cache. diff --git a/mm/slub.c b/mm/slub.c index cdc1e652ec52fe..92f891816bb86b 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -41,6 +41,7 @@ #include #include #include +#include #include #include #include @@ -857,7 +858,7 @@ static inline bool slab_update_freelist(struct kmem_cache *s, struct slab *slab, * request size in the meta data area, for better debug and sanity check. */ static inline void set_orig_size(struct kmem_cache *s, - void *object, unsigned int orig_size) + void *object, unsigned long orig_size) { void *p = kasan_reset_tag(object); @@ -867,10 +868,10 @@ static inline void set_orig_size(struct kmem_cache *s, p += get_info_end(s); p += sizeof(struct track) * 2; - *(unsigned int *)p = orig_size; + *(unsigned long *)p = orig_size; } -static inline unsigned int get_orig_size(struct kmem_cache *s, void *object) +static inline unsigned long get_orig_size(struct kmem_cache *s, void *object) { void *p = kasan_reset_tag(object); @@ -883,7 +884,7 @@ static inline unsigned int get_orig_size(struct kmem_cache *s, void *object) p += get_info_end(s); p += sizeof(struct track) * 2; - return *(unsigned int *)p; + return *(unsigned long *)p; } #ifdef CONFIG_SLUB_DEBUG @@ -1198,7 +1199,7 @@ static void print_trailer(struct kmem_cache *s, struct slab *slab, u8 *p) off += 2 * sizeof(struct track); if (slub_debug_orig_size(s)) - off += sizeof(unsigned int); + off += sizeof(unsigned long); off += kasan_metadata_size(s, false); @@ -1394,7 +1395,7 @@ static int check_pad_bytes(struct kmem_cache *s, struct slab *slab, u8 *p) off += 2 * sizeof(struct track); if (s->flags & SLAB_KMALLOC) - off += sizeof(unsigned int); + off += sizeof(unsigned long); } off += kasan_metadata_size(s, false); @@ -2095,6 +2096,42 @@ static inline void init_slab_obj_exts(struct slab *slab) slab->obj_exts = 0; } +/* + * Calculate the allocation size for slabobj_ext array. + * + * When memory allocation profiling is enabled, the obj_exts array + * could be allocated from the same slab cache it's being allocated for. + * This would prevent the slab from ever being freed because it would + * always contain at least one allocated object (its own obj_exts array). + * + * To avoid this, increase the allocation size when we detect the array + * may come from the same cache, forcing it to use a different cache. + */ +static inline size_t obj_exts_alloc_size(struct kmem_cache *s, + struct slab *slab, gfp_t gfp) +{ + size_t sz = sizeof(struct slabobj_ext) * slab->objects; + struct kmem_cache *obj_exts_cache; + + if (sz > KMALLOC_MAX_CACHE_SIZE) + return sz; + + if (!is_kmalloc_normal(s)) + return sz; + + obj_exts_cache = kmalloc_slab(sz, NULL, gfp, 0); + /* + * We can't simply compare s with obj_exts_cache, because random kmalloc + * caches have multiple caches per size, selected by caller address. + * Since caller address may differ between kmalloc_slab() and actual + * allocation, bump size when sizes are equal. + */ + if (s->object_size == obj_exts_cache->object_size) + return obj_exts_cache->object_size + 1; + + return sz; +} + int alloc_slab_obj_exts(struct slab *slab, struct kmem_cache *s, gfp_t gfp, bool new_slab) { @@ -2103,26 +2140,26 @@ int alloc_slab_obj_exts(struct slab *slab, struct kmem_cache *s, unsigned long new_exts; unsigned long old_exts; struct slabobj_ext *vec; + size_t sz; gfp &= ~OBJCGS_CLEAR_MASK; /* Prevent recursive extension vector allocation */ gfp |= __GFP_NO_OBJ_EXT; + sz = obj_exts_alloc_size(s, slab, gfp); + /* * Note that allow_spin may be false during early boot and its * restricted GFP_BOOT_MASK. Due to kmalloc_nolock() only supporting * architectures with cmpxchg16b, early obj_exts will be missing for * very early allocations on those. */ - if (unlikely(!allow_spin)) { - size_t sz = objects * sizeof(struct slabobj_ext); - + if (unlikely(!allow_spin)) vec = kmalloc_nolock(sz, __GFP_ZERO | __GFP_NO_OBJ_EXT, slab_nid(slab)); - } else { - vec = kcalloc_node(objects, sizeof(struct slabobj_ext), gfp, - slab_nid(slab)); - } + else + vec = kmalloc_node(sz, gfp | __GFP_ZERO, slab_nid(slab)); + if (!vec) { /* * Try to mark vectors which failed to allocate. @@ -2136,6 +2173,9 @@ int alloc_slab_obj_exts(struct slab *slab, struct kmem_cache *s, return -ENOMEM; } + VM_WARN_ON_ONCE(virt_to_slab(vec) != NULL && + virt_to_slab(vec)->slab_cache == s); + new_exts = (unsigned long)vec; if (unlikely(!allow_spin)) new_exts |= OBJEXTS_NOSPIN_ALLOC; @@ -2668,7 +2708,7 @@ static struct slab_sheaf *alloc_full_sheaf(struct kmem_cache *s, gfp_t gfp) if (!sheaf) return NULL; - if (refill_sheaf(s, sheaf, gfp | __GFP_NOMEMALLOC)) { + if (refill_sheaf(s, sheaf, gfp | __GFP_NOMEMALLOC | __GFP_NOWARN)) { free_empty_sheaf(s, sheaf); return NULL; } @@ -2690,19 +2730,19 @@ static void __kmem_cache_free_bulk(struct kmem_cache *s, size_t size, void **p); * object pointers are moved to a on-stack array under the lock. To bound the * stack usage, limit each batch to PCS_BATCH_MAX. * - * returns true if at least partially flushed + * Must be called with s->cpu_sheaves->lock locked, returns with the lock + * unlocked. + * + * Returns how many objects are remaining to be flushed */ -static bool sheaf_flush_main(struct kmem_cache *s) +static unsigned int __sheaf_flush_main_batch(struct kmem_cache *s) { struct slub_percpu_sheaves *pcs; unsigned int batch, remaining; void *objects[PCS_BATCH_MAX]; struct slab_sheaf *sheaf; - bool ret = false; -next_batch: - if (!local_trylock(&s->cpu_sheaves->lock)) - return ret; + lockdep_assert_held(this_cpu_ptr(&s->cpu_sheaves->lock)); pcs = this_cpu_ptr(s->cpu_sheaves); sheaf = pcs->main; @@ -2720,10 +2760,37 @@ static bool sheaf_flush_main(struct kmem_cache *s) stat_add(s, SHEAF_FLUSH, batch); - ret = true; + return remaining; +} + +static void sheaf_flush_main(struct kmem_cache *s) +{ + unsigned int remaining; - if (remaining) - goto next_batch; + do { + local_lock(&s->cpu_sheaves->lock); + + remaining = __sheaf_flush_main_batch(s); + + } while (remaining); +} + +/* + * Returns true if the main sheaf was at least partially flushed. + */ +static bool sheaf_try_flush_main(struct kmem_cache *s) +{ + unsigned int remaining; + bool ret = false; + + do { + if (!local_trylock(&s->cpu_sheaves->lock)) + return ret; + + ret = true; + remaining = __sheaf_flush_main_batch(s); + + } while (remaining); return ret; } @@ -3150,8 +3217,11 @@ static void *next_freelist_entry(struct kmem_cache *s, return (char *)start + idx; } +static DEFINE_PER_CPU(struct rnd_state, slab_rnd_state); + /* Shuffle the single linked freelist based on a random pre-computed sequence */ -static bool shuffle_freelist(struct kmem_cache *s, struct slab *slab) +static bool shuffle_freelist(struct kmem_cache *s, struct slab *slab, + bool allow_spin) { void *start; void *cur; @@ -3162,7 +3232,19 @@ static bool shuffle_freelist(struct kmem_cache *s, struct slab *slab) return false; freelist_count = oo_objects(s->oo); - pos = get_random_u32_below(freelist_count); + if (allow_spin) { + pos = get_random_u32_below(freelist_count); + } else { + struct rnd_state *state; + + /* + * An interrupt or NMI handler might interrupt and change + * the state in the middle, but that's safe. + */ + state = &get_cpu_var(slab_rnd_state); + pos = prandom_u32_state(state) % freelist_count; + put_cpu_var(slab_rnd_state); + } page_limit = slab->objects * s->size; start = fixup_red_left(s, slab_address(slab)); @@ -3189,7 +3271,8 @@ static inline int init_cache_random_seq(struct kmem_cache *s) return 0; } static inline void init_freelist_randomization(void) { } -static inline bool shuffle_freelist(struct kmem_cache *s, struct slab *slab) +static inline bool shuffle_freelist(struct kmem_cache *s, struct slab *slab, + bool allow_spin) { return false; } @@ -3274,7 +3357,7 @@ static struct slab *allocate_slab(struct kmem_cache *s, gfp_t flags, int node) setup_slab_debug(s, slab, start); - shuffle = shuffle_freelist(s, slab); + shuffle = shuffle_freelist(s, slab, allow_spin); if (!shuffle) { start = fixup_red_left(s, start); @@ -3564,6 +3647,7 @@ static struct slab *get_any_partial(struct kmem_cache *s, enum zone_type highest_zoneidx = gfp_zone(pc->flags); struct slab *slab; unsigned int cpuset_mems_cookie; + bool allow_spin = gfpflags_allow_spinning(pc->flags); /* * The defrag ratio allows a configuration of the tradeoffs between @@ -3588,7 +3672,15 @@ static struct slab *get_any_partial(struct kmem_cache *s, return NULL; do { - cpuset_mems_cookie = read_mems_allowed_begin(); + /* + * read_mems_allowed_begin() accesses current->mems_allowed_seq, + * a seqcount_spinlock_t that is not NMI-safe. Do not access + * current->mems_allowed_seq and avoid retry when GFP flags + * indicate spinning is not allowed. + */ + if (allow_spin) + cpuset_mems_cookie = read_mems_allowed_begin(); + zonelist = node_zonelist(mempolicy_slab_node(), pc->flags); for_each_zone_zonelist(zone, z, zonelist, highest_zoneidx) { struct kmem_cache_node *n; @@ -3610,7 +3702,7 @@ static struct slab *get_any_partial(struct kmem_cache *s, } } } - } while (read_mems_allowed_retry(cpuset_mems_cookie)); + } while (allow_spin && read_mems_allowed_retry(cpuset_mems_cookie)); #endif /* CONFIG_NUMA */ return NULL; } @@ -5020,7 +5112,7 @@ __pcs_replace_empty_main(struct kmem_cache *s, struct slub_percpu_sheaves *pcs, return NULL; if (empty) { - if (!refill_sheaf(s, empty, gfp | __GFP_NOMEMALLOC)) { + if (!refill_sheaf(s, empty, gfp | __GFP_NOMEMALLOC | __GFP_NOWARN)) { full = empty; } else { /* @@ -5323,9 +5415,14 @@ EXPORT_SYMBOL(kmem_cache_alloc_node_noprof); static int __prefill_sheaf_pfmemalloc(struct kmem_cache *s, struct slab_sheaf *sheaf, gfp_t gfp) { - int ret = 0; + gfp_t gfp_nomemalloc; + int ret; - ret = refill_sheaf(s, sheaf, gfp | __GFP_NOMEMALLOC); + gfp_nomemalloc = gfp | __GFP_NOMEMALLOC; + if (gfp_pfmemalloc_allowed(gfp)) + gfp_nomemalloc |= __GFP_NOWARN; + + ret = refill_sheaf(s, sheaf, gfp_nomemalloc); if (likely(!ret || !gfp_pfmemalloc_allowed(gfp))) return ret; @@ -6145,7 +6242,7 @@ __pcs_replace_full_main(struct kmem_cache *s, struct slub_percpu_sheaves *pcs) if (put_fail) stat(s, BARN_PUT_FAIL); - if (!sheaf_flush_main(s)) + if (!sheaf_try_flush_main(s)) return NULL; if (!local_trylock(&s->cpu_sheaves->lock)) @@ -6265,11 +6362,29 @@ static void rcu_free_sheaf(struct rcu_head *head) free_empty_sheaf(s, sheaf); } +/* + * kvfree_call_rcu() can be called while holding a raw_spinlock_t. Since + * __kfree_rcu_sheaf() may acquire a spinlock_t (sleeping lock on PREEMPT_RT), + * this would violate lock nesting rules. Therefore, kvfree_call_rcu() avoids + * this problem by bypassing the sheaves layer entirely on PREEMPT_RT. + * + * However, lockdep still complains that it is invalid to acquire spinlock_t + * while holding raw_spinlock_t, even on !PREEMPT_RT where spinlock_t is a + * spinning lock. Tell lockdep that acquiring spinlock_t is valid here + * by temporarily raising the wait-type to LD_WAIT_CONFIG. + */ +static DEFINE_WAIT_OVERRIDE_MAP(kfree_rcu_sheaf_map, LD_WAIT_CONFIG); + bool __kfree_rcu_sheaf(struct kmem_cache *s, void *obj) { struct slub_percpu_sheaves *pcs; struct slab_sheaf *rcu_sheaf; + if (WARN_ON_ONCE(IS_ENABLED(CONFIG_PREEMPT_RT))) + return false; + + lock_map_acquire_try(&kfree_rcu_sheaf_map); + if (!local_trylock(&s->cpu_sheaves->lock)) goto fail; @@ -6346,10 +6461,12 @@ bool __kfree_rcu_sheaf(struct kmem_cache *s, void *obj) local_unlock(&s->cpu_sheaves->lock); stat(s, FREE_RCU_SHEAF); + lock_map_release(&kfree_rcu_sheaf_map); return true; fail: stat(s, FREE_RCU_SHEAF_FAIL); + lock_map_release(&kfree_rcu_sheaf_map); return false; } @@ -7955,7 +8072,7 @@ static int calculate_sizes(struct kmem_cache_args *args, struct kmem_cache *s) /* Save the original kmalloc request size */ if (flags & SLAB_KMALLOC) - size += sizeof(unsigned int); + size += sizeof(unsigned long); } #endif @@ -8552,6 +8669,9 @@ void __init kmem_cache_init_late(void) { flushwq = alloc_workqueue("slub_flushwq", WQ_MEM_RECLAIM, 0); WARN_ON(!flushwq); +#ifdef CONFIG_SLAB_FREELIST_RANDOM + prandom_init_once(&slab_rnd_state); +#endif } struct kmem_cache * diff --git a/mm/truncate.c b/mm/truncate.c index 12467c1bd711eb..8617a12cb16956 100644 --- a/mm/truncate.c +++ b/mm/truncate.c @@ -622,6 +622,7 @@ static int folio_launder(struct address_space *mapping, struct folio *folio) int folio_unmap_invalidate(struct address_space *mapping, struct folio *folio, gfp_t gfp) { + void (*free_folio)(struct folio *); int ret; VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); @@ -648,9 +649,12 @@ int folio_unmap_invalidate(struct address_space *mapping, struct folio *folio, xa_unlock_irq(&mapping->i_pages); if (mapping_shrinkable(mapping)) inode_lru_list_add(mapping->host); + free_folio = mapping->a_ops->free_folio; spin_unlock(&mapping->host->i_lock); - filemap_free_folio(mapping, folio); + if (free_folio) + free_folio(folio); + folio_put_refs(folio, folio_nr_pages(folio)); return 1; failed: xa_unlock_irq(&mapping->i_pages); diff --git a/mm/userfaultfd.c b/mm/userfaultfd.c index e6dfd5f28acd79..a553e821dff6d0 100644 --- a/mm/userfaultfd.c +++ b/mm/userfaultfd.c @@ -573,7 +573,7 @@ static __always_inline ssize_t mfill_atomic_hugetlb( * in the case of shared pmds. fault mutex prevents * races with other faulting threads. */ - idx = linear_page_index(dst_vma, dst_addr); + idx = hugetlb_linear_page_index(dst_vma, dst_addr); mapping = dst_vma->vm_file->f_mapping; hash = hugetlb_fault_mutex_hash(mapping, idx); mutex_lock(&hugetlb_fault_mutex_table[hash]); diff --git a/mm/vma.c b/mm/vma.c index 7a908a964d18df..690e68931868d5 100644 --- a/mm/vma.c +++ b/mm/vma.c @@ -2774,6 +2774,13 @@ static unsigned long __mmap_region(struct file *file, unsigned long addr, if (map.charged) vm_unacct_memory(map.charged); abort_munmap: + /* + * This indicates that .mmap_prepare has set a new file, differing from + * desc->vm_file. But since we're aborting the operation, only the + * original file will be cleaned up. Ensure we clean up both. + */ + if (map.file_doesnt_need_get) + fput(map.file); vms_abort_munmap_vmas(&map.vms, &map.mas_detach); return error; } diff --git a/mm/vmalloc.c b/mm/vmalloc.c index e286c2d2068cbd..ea24ee957605e0 100644 --- a/mm/vmalloc.c +++ b/mm/vmalloc.c @@ -2268,11 +2268,14 @@ decay_va_pool_node(struct vmap_node *vn, bool full_decay) reclaim_list_global(&decay_list); } +#define KASAN_RELEASE_BATCH_SIZE 32 + static void kasan_release_vmalloc_node(struct vmap_node *vn) { struct vmap_area *va; unsigned long start, end; + unsigned int batch_count = 0; start = list_first_entry(&vn->purge_list, struct vmap_area, list)->va_start; end = list_last_entry(&vn->purge_list, struct vmap_area, list)->va_end; @@ -2282,6 +2285,11 @@ kasan_release_vmalloc_node(struct vmap_node *vn) kasan_release_vmalloc(va->va_start, va->va_end, va->va_start, va->va_end, KASAN_VMALLOC_PAGE_RANGE); + + if (need_resched() || (++batch_count >= KASAN_RELEASE_BATCH_SIZE)) { + cond_resched(); + batch_count = 0; + } } kasan_release_vmalloc(start, end, start, end, KASAN_VMALLOC_TLB_FLUSH); diff --git a/mm/vmscan.c b/mm/vmscan.c index 614ccf39fe3fae..7724b1a1a8b525 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -344,19 +344,21 @@ static void flush_reclaim_state(struct scan_control *sc) static bool can_demote(int nid, struct scan_control *sc, struct mem_cgroup *memcg) { - int demotion_nid; + struct pglist_data *pgdat = NODE_DATA(nid); + nodemask_t allowed_mask; - if (!numa_demotion_enabled) + if (!pgdat || !numa_demotion_enabled) return false; if (sc && sc->no_demotion) return false; - demotion_nid = next_demotion_node(nid); - if (demotion_nid == NUMA_NO_NODE) + node_get_allowed_targets(pgdat, &allowed_mask); + if (nodes_empty(allowed_mask)) return false; - /* If demotion node isn't in the cgroup's mems_allowed, fall back */ - return mem_cgroup_node_allowed(memcg, demotion_nid); + /* Filter out nodes that are not in cgroup's mems_allowed. */ + mem_cgroup_node_filter_allowed(memcg, &allowed_mask); + return !nodes_empty(allowed_mask); } static inline bool can_reclaim_anon_pages(struct mem_cgroup *memcg, @@ -1019,9 +1021,10 @@ static struct folio *alloc_demote_folio(struct folio *src, * Folios which are not demoted are left on @demote_folios. */ static unsigned int demote_folio_list(struct list_head *demote_folios, - struct pglist_data *pgdat) + struct pglist_data *pgdat, + struct mem_cgroup *memcg) { - int target_nid = next_demotion_node(pgdat->node_id); + int target_nid; unsigned int nr_succeeded; nodemask_t allowed_mask; @@ -1033,7 +1036,6 @@ static unsigned int demote_folio_list(struct list_head *demote_folios, */ .gfp_mask = (GFP_HIGHUSER_MOVABLE & ~__GFP_RECLAIM) | __GFP_NOMEMALLOC | GFP_NOWAIT, - .nid = target_nid, .nmask = &allowed_mask, .reason = MR_DEMOTION, }; @@ -1041,10 +1043,18 @@ static unsigned int demote_folio_list(struct list_head *demote_folios, if (list_empty(demote_folios)) return 0; - if (target_nid == NUMA_NO_NODE) + node_get_allowed_targets(pgdat, &allowed_mask); + mem_cgroup_node_filter_allowed(memcg, &allowed_mask); + if (nodes_empty(allowed_mask)) return 0; - node_get_allowed_targets(pgdat, &allowed_mask); + target_nid = next_demotion_node(pgdat->node_id); + if (target_nid == NUMA_NO_NODE) + /* No lower-tier nodes or nodes were hot-unplugged. */ + return 0; + if (!node_isset(target_nid, allowed_mask)) + target_nid = node_random(&allowed_mask); + mtc.nid = target_nid; /* Demotion ignores all cpuset and mempolicy settings */ migrate_pages(demote_folios, alloc_demote_folio, NULL, @@ -1566,7 +1576,7 @@ static unsigned int shrink_folio_list(struct list_head *folio_list, /* 'folio_list' is always empty here */ /* Migrate folios selected for demotion */ - nr_demoted = demote_folio_list(&demote_folios, pgdat); + nr_demoted = demote_folio_list(&demote_folios, pgdat, memcg); nr_reclaimed += nr_demoted; stat->nr_demoted += nr_demoted; /* Folios that could not be demoted are still in @demote_folios */ diff --git a/net/9p/trans_xen.c b/net/9p/trans_xen.c index 12f752a9233245..9bbfc20744f696 100644 --- a/net/9p/trans_xen.c +++ b/net/9p/trans_xen.c @@ -277,45 +277,52 @@ static void xen_9pfs_front_free(struct xen_9pfs_front_priv *priv) { int i, j; - write_lock(&xen_9pfs_lock); - list_del(&priv->list); - write_unlock(&xen_9pfs_lock); - - for (i = 0; i < XEN_9PFS_NUM_RINGS; i++) { - struct xen_9pfs_dataring *ring = &priv->rings[i]; - - cancel_work_sync(&ring->work); - - if (!priv->rings[i].intf) - break; - if (priv->rings[i].irq > 0) - unbind_from_irqhandler(priv->rings[i].irq, ring); - if (priv->rings[i].data.in) { - for (j = 0; - j < (1 << priv->rings[i].intf->ring_order); - j++) { - grant_ref_t ref; - - ref = priv->rings[i].intf->ref[j]; - gnttab_end_foreign_access(ref, NULL); - } - free_pages_exact(priv->rings[i].data.in, + if (priv->rings) { + for (i = 0; i < XEN_9PFS_NUM_RINGS; i++) { + struct xen_9pfs_dataring *ring = &priv->rings[i]; + + cancel_work_sync(&ring->work); + + if (!priv->rings[i].intf) + break; + if (priv->rings[i].irq > 0) + unbind_from_irqhandler(priv->rings[i].irq, ring); + if (priv->rings[i].data.in) { + for (j = 0; + j < (1 << priv->rings[i].intf->ring_order); + j++) { + grant_ref_t ref; + + ref = priv->rings[i].intf->ref[j]; + gnttab_end_foreign_access(ref, NULL); + } + free_pages_exact(priv->rings[i].data.in, 1UL << (priv->rings[i].intf->ring_order + XEN_PAGE_SHIFT)); + } + gnttab_end_foreign_access(priv->rings[i].ref, NULL); + free_page((unsigned long)priv->rings[i].intf); } - gnttab_end_foreign_access(priv->rings[i].ref, NULL); - free_page((unsigned long)priv->rings[i].intf); + kfree(priv->rings); } - kfree(priv->rings); kfree(priv->tag); kfree(priv); } static void xen_9pfs_front_remove(struct xenbus_device *dev) { - struct xen_9pfs_front_priv *priv = dev_get_drvdata(&dev->dev); + struct xen_9pfs_front_priv *priv; + write_lock(&xen_9pfs_lock); + priv = dev_get_drvdata(&dev->dev); + if (priv == NULL) { + write_unlock(&xen_9pfs_lock); + return; + } dev_set_drvdata(&dev->dev, NULL); + list_del(&priv->list); + write_unlock(&xen_9pfs_lock); + xen_9pfs_front_free(priv); } @@ -382,7 +389,7 @@ static int xen_9pfs_front_init(struct xenbus_device *dev) { int ret, i; struct xenbus_transaction xbt; - struct xen_9pfs_front_priv *priv = dev_get_drvdata(&dev->dev); + struct xen_9pfs_front_priv *priv; char *versions, *v; unsigned int max_rings, max_ring_order, len = 0; @@ -410,6 +417,10 @@ static int xen_9pfs_front_init(struct xenbus_device *dev) if (p9_xen_trans.maxsize > XEN_FLEX_RING_SIZE(max_ring_order)) p9_xen_trans.maxsize = XEN_FLEX_RING_SIZE(max_ring_order) / 2; + priv = kzalloc(sizeof(*priv), GFP_KERNEL); + if (!priv) + return -ENOMEM; + priv->dev = dev; priv->rings = kcalloc(XEN_9PFS_NUM_RINGS, sizeof(*priv->rings), GFP_KERNEL); if (!priv->rings) { @@ -468,6 +479,11 @@ static int xen_9pfs_front_init(struct xenbus_device *dev) goto error; } + write_lock(&xen_9pfs_lock); + dev_set_drvdata(&dev->dev, priv); + list_add_tail(&priv->list, &xen_9pfs_devs); + write_unlock(&xen_9pfs_lock); + xenbus_switch_state(dev, XenbusStateInitialised); return 0; @@ -482,19 +498,6 @@ static int xen_9pfs_front_init(struct xenbus_device *dev) static int xen_9pfs_front_probe(struct xenbus_device *dev, const struct xenbus_device_id *id) { - struct xen_9pfs_front_priv *priv = NULL; - - priv = kzalloc(sizeof(*priv), GFP_KERNEL); - if (!priv) - return -ENOMEM; - - priv->dev = dev; - dev_set_drvdata(&dev->dev, priv); - - write_lock(&xen_9pfs_lock); - list_add_tail(&priv->list, &xen_9pfs_devs); - write_unlock(&xen_9pfs_lock); - return 0; } diff --git a/net/atm/lec.c b/net/atm/lec.c index afb8d3eb218507..b6f764e524f7c6 100644 --- a/net/atm/lec.c +++ b/net/atm/lec.c @@ -154,10 +154,19 @@ static void lec_handle_bridge(struct sk_buff *skb, struct net_device *dev) /* 0x01 is topology change */ priv = netdev_priv(dev); - atm_force_charge(priv->lecd, skb2->truesize); - sk = sk_atm(priv->lecd); - skb_queue_tail(&sk->sk_receive_queue, skb2); - sk->sk_data_ready(sk); + struct atm_vcc *vcc; + + rcu_read_lock(); + vcc = rcu_dereference(priv->lecd); + if (vcc) { + atm_force_charge(vcc, skb2->truesize); + sk = sk_atm(vcc); + skb_queue_tail(&sk->sk_receive_queue, skb2); + sk->sk_data_ready(sk); + } else { + dev_kfree_skb(skb2); + } + rcu_read_unlock(); } } #endif /* IS_ENABLED(CONFIG_BRIDGE) */ @@ -216,7 +225,7 @@ static netdev_tx_t lec_start_xmit(struct sk_buff *skb, int is_rdesc; pr_debug("called\n"); - if (!priv->lecd) { + if (!rcu_access_pointer(priv->lecd)) { pr_info("%s:No lecd attached\n", dev->name); dev->stats.tx_errors++; netif_stop_queue(dev); @@ -449,10 +458,19 @@ static int lec_atm_send(struct atm_vcc *vcc, struct sk_buff *skb) break; skb2->len = sizeof(struct atmlec_msg); skb_copy_to_linear_data(skb2, mesg, sizeof(*mesg)); - atm_force_charge(priv->lecd, skb2->truesize); - sk = sk_atm(priv->lecd); - skb_queue_tail(&sk->sk_receive_queue, skb2); - sk->sk_data_ready(sk); + struct atm_vcc *vcc; + + rcu_read_lock(); + vcc = rcu_dereference(priv->lecd); + if (vcc) { + atm_force_charge(vcc, skb2->truesize); + sk = sk_atm(vcc); + skb_queue_tail(&sk->sk_receive_queue, skb2); + sk->sk_data_ready(sk); + } else { + dev_kfree_skb(skb2); + } + rcu_read_unlock(); } } #endif /* IS_ENABLED(CONFIG_BRIDGE) */ @@ -468,23 +486,16 @@ static int lec_atm_send(struct atm_vcc *vcc, struct sk_buff *skb) static void lec_atm_close(struct atm_vcc *vcc) { - struct sk_buff *skb; struct net_device *dev = (struct net_device *)vcc->proto_data; struct lec_priv *priv = netdev_priv(dev); - priv->lecd = NULL; + rcu_assign_pointer(priv->lecd, NULL); + synchronize_rcu(); /* Do something needful? */ netif_stop_queue(dev); lec_arp_destroy(priv); - if (skb_peek(&sk_atm(vcc)->sk_receive_queue)) - pr_info("%s closing with messages pending\n", dev->name); - while ((skb = skb_dequeue(&sk_atm(vcc)->sk_receive_queue))) { - atm_return(vcc, skb->truesize); - dev_kfree_skb(skb); - } - pr_info("%s: Shut down!\n", dev->name); module_put(THIS_MODULE); } @@ -510,12 +521,14 @@ send_to_lecd(struct lec_priv *priv, atmlec_msg_type type, const unsigned char *mac_addr, const unsigned char *atm_addr, struct sk_buff *data) { + struct atm_vcc *vcc; struct sock *sk; struct sk_buff *skb; struct atmlec_msg *mesg; - if (!priv || !priv->lecd) + if (!priv || !rcu_access_pointer(priv->lecd)) return -1; + skb = alloc_skb(sizeof(struct atmlec_msg), GFP_ATOMIC); if (!skb) return -1; @@ -532,18 +545,27 @@ send_to_lecd(struct lec_priv *priv, atmlec_msg_type type, if (atm_addr) memcpy(&mesg->content.normal.atm_addr, atm_addr, ATM_ESA_LEN); - atm_force_charge(priv->lecd, skb->truesize); - sk = sk_atm(priv->lecd); + rcu_read_lock(); + vcc = rcu_dereference(priv->lecd); + if (!vcc) { + rcu_read_unlock(); + kfree_skb(skb); + return -1; + } + + atm_force_charge(vcc, skb->truesize); + sk = sk_atm(vcc); skb_queue_tail(&sk->sk_receive_queue, skb); sk->sk_data_ready(sk); if (data != NULL) { pr_debug("about to send %d bytes of data\n", data->len); - atm_force_charge(priv->lecd, data->truesize); + atm_force_charge(vcc, data->truesize); skb_queue_tail(&sk->sk_receive_queue, data); sk->sk_data_ready(sk); } + rcu_read_unlock(); return 0; } @@ -618,7 +640,7 @@ static void lec_push(struct atm_vcc *vcc, struct sk_buff *skb) atm_return(vcc, skb->truesize); if (*(__be16 *) skb->data == htons(priv->lecid) || - !priv->lecd || !(dev->flags & IFF_UP)) { + !rcu_access_pointer(priv->lecd) || !(dev->flags & IFF_UP)) { /* * Probably looping back, or if lecd is missing, * lecd has gone down @@ -753,12 +775,12 @@ static int lecd_attach(struct atm_vcc *vcc, int arg) priv = netdev_priv(dev_lec[i]); } else { priv = netdev_priv(dev_lec[i]); - if (priv->lecd) + if (rcu_access_pointer(priv->lecd)) return -EADDRINUSE; } lec_arp_init(priv); priv->itfnum = i; /* LANE2 addition */ - priv->lecd = vcc; + rcu_assign_pointer(priv->lecd, vcc); vcc->dev = &lecatm_dev; vcc_insert_socket(sk_atm(vcc)); @@ -1260,24 +1282,28 @@ static void lec_arp_clear_vccs(struct lec_arp_table *entry) struct lec_vcc_priv *vpriv = LEC_VCC_PRIV(vcc); struct net_device *dev = (struct net_device *)vcc->proto_data; - vcc->pop = vpriv->old_pop; - if (vpriv->xoff) - netif_wake_queue(dev); - kfree(vpriv); - vcc->user_back = NULL; - vcc->push = entry->old_push; - vcc_release_async(vcc, -EPIPE); + if (vpriv) { + vcc->pop = vpriv->old_pop; + if (vpriv->xoff) + netif_wake_queue(dev); + kfree(vpriv); + vcc->user_back = NULL; + vcc->push = entry->old_push; + vcc_release_async(vcc, -EPIPE); + } entry->vcc = NULL; } if (entry->recv_vcc) { struct atm_vcc *vcc = entry->recv_vcc; struct lec_vcc_priv *vpriv = LEC_VCC_PRIV(vcc); - kfree(vpriv); - vcc->user_back = NULL; + if (vpriv) { + kfree(vpriv); + vcc->user_back = NULL; - entry->recv_vcc->push = entry->old_recv_push; - vcc_release_async(entry->recv_vcc, -EPIPE); + entry->recv_vcc->push = entry->old_recv_push; + vcc_release_async(entry->recv_vcc, -EPIPE); + } entry->recv_vcc = NULL; } } diff --git a/net/atm/lec.h b/net/atm/lec.h index be0e2667bd8c3f..ec85709bf81859 100644 --- a/net/atm/lec.h +++ b/net/atm/lec.h @@ -91,7 +91,7 @@ struct lec_priv { */ spinlock_t lec_arp_lock; struct atm_vcc *mcast_vcc; /* Default Multicast Send VCC */ - struct atm_vcc *lecd; + struct atm_vcc __rcu *lecd; struct delayed_work lec_arp_work; /* C10 */ unsigned int maximum_unknown_frame_count; /* diff --git a/net/atm/signaling.c b/net/atm/signaling.c index e70ae2c113f954..358fbe5e4d1d06 100644 --- a/net/atm/signaling.c +++ b/net/atm/signaling.c @@ -22,6 +22,36 @@ struct atm_vcc *sigd = NULL; +/* + * find_get_vcc - validate and get a reference to a vcc pointer + * @vcc: the vcc pointer to validate + * + * This function validates that @vcc points to a registered VCC in vcc_hash. + * If found, it increments the socket reference count and returns the vcc. + * The caller must call sock_put(sk_atm(vcc)) when done. + * + * Returns the vcc pointer if valid, NULL otherwise. + */ +static struct atm_vcc *find_get_vcc(struct atm_vcc *vcc) +{ + int i; + + read_lock(&vcc_sklist_lock); + for (i = 0; i < VCC_HTABLE_SIZE; i++) { + struct sock *s; + + sk_for_each(s, &vcc_hash[i]) { + if (atm_sk(s) == vcc) { + sock_hold(s); + read_unlock(&vcc_sklist_lock); + return vcc; + } + } + } + read_unlock(&vcc_sklist_lock); + return NULL; +} + static void sigd_put_skb(struct sk_buff *skb) { if (!sigd) { @@ -69,7 +99,14 @@ static int sigd_send(struct atm_vcc *vcc, struct sk_buff *skb) msg = (struct atmsvc_msg *) skb->data; WARN_ON(refcount_sub_and_test(skb->truesize, &sk_atm(vcc)->sk_wmem_alloc)); - vcc = *(struct atm_vcc **) &msg->vcc; + + vcc = find_get_vcc(*(struct atm_vcc **)&msg->vcc); + if (!vcc) { + pr_debug("invalid vcc pointer in msg\n"); + dev_kfree_skb(skb); + return -EINVAL; + } + pr_debug("%d (0x%lx)\n", (int)msg->type, (unsigned long)vcc); sk = sk_atm(vcc); @@ -100,7 +137,16 @@ static int sigd_send(struct atm_vcc *vcc, struct sk_buff *skb) clear_bit(ATM_VF_WAITING, &vcc->flags); break; case as_indicate: - vcc = *(struct atm_vcc **)&msg->listen_vcc; + /* Release the reference from msg->vcc, we'll use msg->listen_vcc instead */ + sock_put(sk); + + vcc = find_get_vcc(*(struct atm_vcc **)&msg->listen_vcc); + if (!vcc) { + pr_debug("invalid listen_vcc pointer in msg\n"); + dev_kfree_skb(skb); + return -EINVAL; + } + sk = sk_atm(vcc); pr_debug("as_indicate!!!\n"); lock_sock(sk); @@ -115,6 +161,8 @@ static int sigd_send(struct atm_vcc *vcc, struct sk_buff *skb) sk->sk_state_change(sk); as_indicate_complete: release_sock(sk); + /* Paired with find_get_vcc(msg->listen_vcc) above */ + sock_put(sk); return 0; case as_close: set_bit(ATM_VF_RELEASED, &vcc->flags); @@ -131,11 +179,15 @@ static int sigd_send(struct atm_vcc *vcc, struct sk_buff *skb) break; default: pr_alert("bad message type %d\n", (int)msg->type); + /* Paired with find_get_vcc(msg->vcc) above */ + sock_put(sk); return -EINVAL; } sk->sk_state_change(sk); out: dev_kfree_skb(skb); + /* Paired with find_get_vcc(msg->vcc) above */ + sock_put(sk); return 0; } diff --git a/net/batman-adv/bat_iv_ogm.c b/net/batman-adv/bat_iv_ogm.c index b75c2228e69a65..f28e9cbf8ad5f2 100644 --- a/net/batman-adv/bat_iv_ogm.c +++ b/net/batman-adv/bat_iv_ogm.c @@ -473,6 +473,9 @@ batadv_iv_ogm_can_aggregate(const struct batadv_ogm_packet *new_bat_ogm_packet, if (aggregated_bytes > max_bytes) return false; + if (skb_tailroom(forw_packet->skb) < packet_len) + return false; + if (packet_num >= BATADV_MAX_AGGREGATION_PACKETS) return false; diff --git a/net/batman-adv/bat_v_elp.c b/net/batman-adv/bat_v_elp.c index cb16c1ed2a58fc..fe832093d421fb 100644 --- a/net/batman-adv/bat_v_elp.c +++ b/net/batman-adv/bat_v_elp.c @@ -111,7 +111,15 @@ static bool batadv_v_elp_get_throughput(struct batadv_hardif_neigh_node *neigh, /* unsupported WiFi driver version */ goto default_throughput; - real_netdev = batadv_get_real_netdev(hard_iface->net_dev); + /* only use rtnl_trylock because the elp worker will be cancelled while + * the rntl_lock is held. the cancel_delayed_work_sync() would otherwise + * wait forever when the elp work_item was started and it is then also + * trying to rtnl_lock + */ + if (!rtnl_trylock()) + return false; + real_netdev = __batadv_get_real_netdev(hard_iface->net_dev); + rtnl_unlock(); if (!real_netdev) goto default_throughput; diff --git a/net/batman-adv/bridge_loop_avoidance.c b/net/batman-adv/bridge_loop_avoidance.c index 3dc791c15bf72e..648fa97ea913f5 100644 --- a/net/batman-adv/bridge_loop_avoidance.c +++ b/net/batman-adv/bridge_loop_avoidance.c @@ -2130,6 +2130,7 @@ batadv_bla_claim_dump_entry(struct sk_buff *msg, u32 portid, struct batadv_bla_claim *claim) { const u8 *primary_addr = primary_if->net_dev->dev_addr; + struct batadv_bla_backbone_gw *backbone_gw; u16 backbone_crc; bool is_own; void *hdr; @@ -2145,32 +2146,35 @@ batadv_bla_claim_dump_entry(struct sk_buff *msg, u32 portid, genl_dump_check_consistent(cb, hdr); - is_own = batadv_compare_eth(claim->backbone_gw->orig, - primary_addr); + backbone_gw = batadv_bla_claim_get_backbone_gw(claim); + + is_own = batadv_compare_eth(backbone_gw->orig, primary_addr); - spin_lock_bh(&claim->backbone_gw->crc_lock); - backbone_crc = claim->backbone_gw->crc; - spin_unlock_bh(&claim->backbone_gw->crc_lock); + spin_lock_bh(&backbone_gw->crc_lock); + backbone_crc = backbone_gw->crc; + spin_unlock_bh(&backbone_gw->crc_lock); if (is_own) if (nla_put_flag(msg, BATADV_ATTR_BLA_OWN)) { genlmsg_cancel(msg, hdr); - goto out; + goto put_backbone_gw; } if (nla_put(msg, BATADV_ATTR_BLA_ADDRESS, ETH_ALEN, claim->addr) || nla_put_u16(msg, BATADV_ATTR_BLA_VID, claim->vid) || nla_put(msg, BATADV_ATTR_BLA_BACKBONE, ETH_ALEN, - claim->backbone_gw->orig) || + backbone_gw->orig) || nla_put_u16(msg, BATADV_ATTR_BLA_CRC, backbone_crc)) { genlmsg_cancel(msg, hdr); - goto out; + goto put_backbone_gw; } genlmsg_end(msg, hdr); ret = 0; +put_backbone_gw: + batadv_backbone_gw_put(backbone_gw); out: return ret; } @@ -2448,6 +2452,7 @@ int batadv_bla_backbone_dump(struct sk_buff *msg, struct netlink_callback *cb) bool batadv_bla_check_claim(struct batadv_priv *bat_priv, u8 *addr, unsigned short vid) { + struct batadv_bla_backbone_gw *backbone_gw; struct batadv_bla_claim search_claim; struct batadv_bla_claim *claim = NULL; struct batadv_hard_iface *primary_if = NULL; @@ -2470,9 +2475,13 @@ bool batadv_bla_check_claim(struct batadv_priv *bat_priv, * return false. */ if (claim) { - if (!batadv_compare_eth(claim->backbone_gw->orig, + backbone_gw = batadv_bla_claim_get_backbone_gw(claim); + + if (!batadv_compare_eth(backbone_gw->orig, primary_if->net_dev->dev_addr)) ret = false; + + batadv_backbone_gw_put(backbone_gw); batadv_claim_put(claim); } diff --git a/net/batman-adv/hard-interface.c b/net/batman-adv/hard-interface.c index 5113f879736b54..1c488049d55463 100644 --- a/net/batman-adv/hard-interface.c +++ b/net/batman-adv/hard-interface.c @@ -204,7 +204,7 @@ static bool batadv_is_valid_iface(const struct net_device *net_dev) } /** - * batadv_get_real_netdevice() - check if the given netdev struct is a virtual + * __batadv_get_real_netdev() - check if the given netdev struct is a virtual * interface on top of another 'real' interface * @netdev: the device to check * @@ -214,7 +214,7 @@ static bool batadv_is_valid_iface(const struct net_device *net_dev) * Return: the 'real' net device or the original net device and NULL in case * of an error. */ -static struct net_device *batadv_get_real_netdevice(struct net_device *netdev) +struct net_device *__batadv_get_real_netdev(struct net_device *netdev) { struct batadv_hard_iface *hard_iface = NULL; struct net_device *real_netdev = NULL; @@ -267,7 +267,7 @@ struct net_device *batadv_get_real_netdev(struct net_device *net_device) struct net_device *real_netdev; rtnl_lock(); - real_netdev = batadv_get_real_netdevice(net_device); + real_netdev = __batadv_get_real_netdev(net_device); rtnl_unlock(); return real_netdev; @@ -336,7 +336,7 @@ static u32 batadv_wifi_flags_evaluate(struct net_device *net_device) if (batadv_is_cfg80211_netdev(net_device)) wifi_flags |= BATADV_HARDIF_WIFI_CFG80211_DIRECT; - real_netdev = batadv_get_real_netdevice(net_device); + real_netdev = __batadv_get_real_netdev(net_device); if (!real_netdev) return wifi_flags; diff --git a/net/batman-adv/hard-interface.h b/net/batman-adv/hard-interface.h index 9db8a310961ead..9ba8fb2bdceb48 100644 --- a/net/batman-adv/hard-interface.h +++ b/net/batman-adv/hard-interface.h @@ -67,6 +67,7 @@ enum batadv_hard_if_bcast { extern struct notifier_block batadv_hard_if_notifier; +struct net_device *__batadv_get_real_netdev(struct net_device *net_device); struct net_device *batadv_get_real_netdev(struct net_device *net_device); bool batadv_is_cfg80211_hardif(struct batadv_hard_iface *hard_iface); bool batadv_is_wifi_hardif(struct batadv_hard_iface *hard_iface); diff --git a/net/batman-adv/translation-table.c b/net/batman-adv/translation-table.c index 6e95e883c2bf08..05cddcf994f651 100644 --- a/net/batman-adv/translation-table.c +++ b/net/batman-adv/translation-table.c @@ -798,8 +798,8 @@ batadv_tt_prepare_tvlv_global_data(struct batadv_orig_node *orig_node, { u16 num_vlan = 0; u16 num_entries = 0; - u16 change_offset; - u16 tvlv_len; + u16 tvlv_len = 0; + unsigned int change_offset; struct batadv_tvlv_tt_vlan_data *tt_vlan; struct batadv_orig_node_vlan *vlan; u8 *tt_change_ptr; @@ -816,6 +816,11 @@ batadv_tt_prepare_tvlv_global_data(struct batadv_orig_node *orig_node, if (*tt_len < 0) *tt_len = batadv_tt_len(num_entries); + if (change_offset > U16_MAX || *tt_len > U16_MAX - change_offset) { + *tt_len = 0; + goto out; + } + tvlv_len = *tt_len; tvlv_len += change_offset; diff --git a/net/bluetooth/Kconfig b/net/bluetooth/Kconfig index 6b2b65a667008b..0f2a5fbcafc563 100644 --- a/net/bluetooth/Kconfig +++ b/net/bluetooth/Kconfig @@ -110,6 +110,13 @@ config BT_AOSPEXT This options enables support for the Android Open Source Project defined HCI vendor extensions. +config BT_BRCMEXT + bool "Enable Broadcom extensions" + depends on BT + help + This option enables support for the Broadcom defined HCI + vendor extensions. + config BT_DEBUGFS bool "Export Bluetooth internals in debugfs" depends on BT && DEBUG_FS diff --git a/net/bluetooth/Makefile b/net/bluetooth/Makefile index a7eede7616d856..b4c9013a46cec2 100644 --- a/net/bluetooth/Makefile +++ b/net/bluetooth/Makefile @@ -24,5 +24,6 @@ bluetooth-$(CONFIG_BT_LE) += iso.o bluetooth-$(CONFIG_BT_LEDS) += leds.o bluetooth-$(CONFIG_BT_MSFTEXT) += msft.o bluetooth-$(CONFIG_BT_AOSPEXT) += aosp.o +bluetooth-$(CONFIG_BT_BRCMEXT) += brcm.o bluetooth-$(CONFIG_BT_DEBUGFS) += hci_debugfs.o bluetooth-$(CONFIG_BT_SELFTEST) += selftest.o diff --git a/net/bluetooth/brcm.c b/net/bluetooth/brcm.c new file mode 100644 index 00000000000000..9aa0a265ab3d6b --- /dev/null +++ b/net/bluetooth/brcm.c @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2026 The Asahi Linux Contributors + */ + +#include +#include + +#include "brcm.h" + +int brcm_set_high_priority(struct hci_dev *hdev, u16 handle, bool enable) +{ + struct sk_buff *skb; + u8 cmd[3]; + + if (!hdev->brcm_capable) + return 0; + + cmd[0] = handle; + cmd[1] = handle >> 8; + cmd[2] = !!enable; + + skb = hci_cmd_sync(hdev, 0xfc57, sizeof(cmd), cmd, HCI_CMD_TIMEOUT); + if (IS_ERR(skb)) + return PTR_ERR(skb); + + kfree_skb(skb); + return 0; +} diff --git a/net/bluetooth/brcm.h b/net/bluetooth/brcm.h new file mode 100644 index 00000000000000..fdaee63bd1d23c --- /dev/null +++ b/net/bluetooth/brcm.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2026 The Asahi Linux Contributors + */ + +#if IS_ENABLED(CONFIG_BT_BRCMEXT) + +int brcm_set_high_priority(struct hci_dev *hdev, u16 handle, bool enable); + +#else + +static inline int brcm_set_high_priority(struct hci_dev *hdev, u16 handle, bool enable) +{ + return 0; +} + +#endif diff --git a/net/bluetooth/hci_conn.c b/net/bluetooth/hci_conn.c index c3f7828bf9d54a..a2dd9171ac6f8e 100644 --- a/net/bluetooth/hci_conn.c +++ b/net/bluetooth/hci_conn.c @@ -35,6 +35,7 @@ #include #include +#include "brcm.h" #include "smp.h" #include "eir.h" @@ -1002,12 +1003,18 @@ static struct hci_conn *__hci_conn_add(struct hci_dev *hdev, int type, switch (type) { case ACL_LINK: conn->pkt_type = hdev->pkt_type & ACL_PTYPE_MASK; + conn->link_policy = hdev->link_policy; conn->mtu = hdev->acl_mtu; break; case LE_LINK: /* conn->src should reflect the local identity address */ hci_copy_identity_address(hdev, &conn->src, &conn->src_type); conn->mtu = hdev->le_mtu ? hdev->le_mtu : hdev->acl_mtu; + /* Use the controller supported PHYS as default until the + * remote features are resolved. + */ + conn->le_tx_def_phys = hdev->le_tx_def_phys; + conn->le_rx_def_phys = hdev->le_tx_def_phys; break; case CIS_LINK: /* conn->src should reflect the local identity address */ @@ -1837,9 +1844,13 @@ static int set_cig_params_sync(struct hci_dev *hdev, void *data) u8 aux_num_cis = 0; u8 cis_id; + hci_dev_lock(hdev); + conn = hci_conn_hash_lookup_cig(hdev, cig_id); - if (!conn) + if (!conn) { + hci_dev_unlock(hdev); return 0; + } qos = &conn->iso_qos; pdu->cig_id = cig_id; @@ -1878,6 +1889,8 @@ static int set_cig_params_sync(struct hci_dev *hdev, void *data) } pdu->num_cis = aux_num_cis; + hci_dev_unlock(hdev); + if (!pdu->num_cis) return 0; @@ -1938,6 +1951,8 @@ static bool hci_le_set_cig_params(struct hci_conn *conn, struct bt_iso_qos *qos) return false; done: + conn->iso_qos = *qos; + if (hci_cmd_sync_queue(hdev, set_cig_params_sync, UINT_PTR(qos->ucast.cig), NULL) < 0) return false; @@ -2007,8 +2022,6 @@ struct hci_conn *hci_bind_cis(struct hci_dev *hdev, bdaddr_t *dst, } hci_conn_hold(cis); - - cis->iso_qos = *qos; cis->state = BT_BOUND; return cis; @@ -2614,8 +2627,8 @@ void hci_conn_enter_active_mode(struct hci_conn *conn, __u8 force_active) timer: if (hdev->idle_timeout > 0) - queue_delayed_work(hdev->workqueue, &conn->idle_work, - msecs_to_jiffies(hdev->idle_timeout)); + mod_delayed_work(hdev->workqueue, &conn->idle_work, + msecs_to_jiffies(hdev->idle_timeout)); } /* Drop all connection on the device */ @@ -2928,22 +2941,22 @@ u32 hci_conn_get_phy(struct hci_conn *conn) break; case LE_LINK: - if (conn->le_tx_phy & HCI_LE_SET_PHY_1M) + if (conn->le_tx_def_phys & HCI_LE_SET_PHY_1M) phys |= BT_PHY_LE_1M_TX; - if (conn->le_rx_phy & HCI_LE_SET_PHY_1M) + if (conn->le_rx_def_phys & HCI_LE_SET_PHY_1M) phys |= BT_PHY_LE_1M_RX; - if (conn->le_tx_phy & HCI_LE_SET_PHY_2M) + if (conn->le_tx_def_phys & HCI_LE_SET_PHY_2M) phys |= BT_PHY_LE_2M_TX; - if (conn->le_rx_phy & HCI_LE_SET_PHY_2M) + if (conn->le_rx_def_phys & HCI_LE_SET_PHY_2M) phys |= BT_PHY_LE_2M_RX; - if (conn->le_tx_phy & HCI_LE_SET_PHY_CODED) + if (conn->le_tx_def_phys & HCI_LE_SET_PHY_CODED) phys |= BT_PHY_LE_CODED_TX; - if (conn->le_rx_phy & HCI_LE_SET_PHY_CODED) + if (conn->le_rx_def_phys & HCI_LE_SET_PHY_CODED) phys |= BT_PHY_LE_CODED_RX; break; @@ -2952,6 +2965,137 @@ u32 hci_conn_get_phy(struct hci_conn *conn) return phys; } +static u16 bt_phy_pkt_type(struct hci_conn *conn, u32 phys) +{ + u16 pkt_type = conn->pkt_type; + + if (phys & BT_PHY_BR_1M_3SLOT) + pkt_type |= HCI_DM3 | HCI_DH3; + else + pkt_type &= ~(HCI_DM3 | HCI_DH3); + + if (phys & BT_PHY_BR_1M_5SLOT) + pkt_type |= HCI_DM5 | HCI_DH5; + else + pkt_type &= ~(HCI_DM5 | HCI_DH5); + + if (phys & BT_PHY_EDR_2M_1SLOT) + pkt_type &= ~HCI_2DH1; + else + pkt_type |= HCI_2DH1; + + if (phys & BT_PHY_EDR_2M_3SLOT) + pkt_type &= ~HCI_2DH3; + else + pkt_type |= HCI_2DH3; + + if (phys & BT_PHY_EDR_2M_5SLOT) + pkt_type &= ~HCI_2DH5; + else + pkt_type |= HCI_2DH5; + + if (phys & BT_PHY_EDR_3M_1SLOT) + pkt_type &= ~HCI_3DH1; + else + pkt_type |= HCI_3DH1; + + if (phys & BT_PHY_EDR_3M_3SLOT) + pkt_type &= ~HCI_3DH3; + else + pkt_type |= HCI_3DH3; + + if (phys & BT_PHY_EDR_3M_5SLOT) + pkt_type &= ~HCI_3DH5; + else + pkt_type |= HCI_3DH5; + + return pkt_type; +} + +static int bt_phy_le_phy(u32 phys, u8 *tx_phys, u8 *rx_phys) +{ + if (!tx_phys || !rx_phys) + return -EINVAL; + + *tx_phys = 0; + *rx_phys = 0; + + if (phys & BT_PHY_LE_1M_TX) + *tx_phys |= HCI_LE_SET_PHY_1M; + + if (phys & BT_PHY_LE_1M_RX) + *rx_phys |= HCI_LE_SET_PHY_1M; + + if (phys & BT_PHY_LE_2M_TX) + *tx_phys |= HCI_LE_SET_PHY_2M; + + if (phys & BT_PHY_LE_2M_RX) + *rx_phys |= HCI_LE_SET_PHY_2M; + + if (phys & BT_PHY_LE_CODED_TX) + *tx_phys |= HCI_LE_SET_PHY_CODED; + + if (phys & BT_PHY_LE_CODED_RX) + *rx_phys |= HCI_LE_SET_PHY_CODED; + + return 0; +} + +int hci_conn_set_phy(struct hci_conn *conn, u32 phys) +{ + u8 tx_phys, rx_phys; + + switch (conn->type) { + case SCO_LINK: + case ESCO_LINK: + return -EINVAL; + case ACL_LINK: + /* Only allow setting BR/EDR PHYs if link type is ACL */ + if (phys & ~BT_PHY_BREDR_MASK) + return -EINVAL; + + return hci_acl_change_pkt_type(conn, + bt_phy_pkt_type(conn, phys)); + case LE_LINK: + /* Only allow setting LE PHYs if link type is LE */ + if (phys & ~BT_PHY_LE_MASK) + return -EINVAL; + + if (bt_phy_le_phy(phys, &tx_phys, &rx_phys)) + return -EINVAL; + + return hci_le_set_phy(conn, tx_phys, rx_phys); + default: + return -EINVAL; + } +} + +int hci_conn_setsockopt(struct hci_conn *conn, struct sock *sk, int level, + int optname, sockptr_t optval, unsigned int optlen) { + int val; + bool old_high, new_high, changed; + + if (level != SOL_SOCKET) + return 0; + + if (optname != SO_PRIORITY) + return 0; + + if (optlen < sizeof(int)) + return -EINVAL; + + if (copy_from_sockptr(&val, optval, sizeof(val))) + return -EFAULT; + + old_high = sk->sk_priority >= TC_PRIO_INTERACTIVE; + new_high = val >= TC_PRIO_INTERACTIVE; + changed = old_high != new_high; + if (!changed) + return 0; + + return brcm_set_high_priority(conn->hdev, conn->handle, new_high); +} + static int abort_conn_sync(struct hci_dev *hdev, void *data) { struct hci_conn *conn = data; @@ -2983,7 +3127,7 @@ int hci_abort_conn(struct hci_conn *conn, u8 reason) * hci_connect_le serializes the connection attempts so only one * connection can be in BT_CONNECT at time. */ - if (conn->state == BT_CONNECT && hdev->req_status == HCI_REQ_PEND) { + if (conn->state == BT_CONNECT && READ_ONCE(hdev->req_status) == HCI_REQ_PEND) { switch (hci_skb_event(hdev->sent_cmd)) { case HCI_EV_CONN_COMPLETE: case HCI_EV_LE_CONN_COMPLETE: diff --git a/net/bluetooth/hci_core.c b/net/bluetooth/hci_core.c index 8ccec73dce45c2..0f86b81b39730a 100644 --- a/net/bluetooth/hci_core.c +++ b/net/bluetooth/hci_core.c @@ -4125,7 +4125,7 @@ static int hci_send_cmd_sync(struct hci_dev *hdev, struct sk_buff *skb) kfree_skb(skb); } - if (hdev->req_status == HCI_REQ_PEND && + if (READ_ONCE(hdev->req_status) == HCI_REQ_PEND && !hci_dev_test_and_set_flag(hdev, HCI_CMD_PENDING)) { kfree_skb(hdev->req_skb); hdev->req_skb = skb_clone(hdev->sent_cmd, GFP_KERNEL); diff --git a/net/bluetooth/hci_event.c b/net/bluetooth/hci_event.c index a9868f17ef40ff..90c5b67eeff1fc 100644 --- a/net/bluetooth/hci_event.c +++ b/net/bluetooth/hci_event.c @@ -80,6 +80,10 @@ static void *hci_le_ev_skb_pull(struct hci_dev *hdev, struct sk_buff *skb, return data; } +static void hci_store_wake_reason(struct hci_dev *hdev, + const bdaddr_t *bdaddr, u8 addr_type) + __must_hold(&hdev->lock); + static u8 hci_cc_inquiry_cancel(struct hci_dev *hdev, void *data, struct sk_buff *skb) { @@ -2869,6 +2873,31 @@ static void hci_cs_le_ext_create_conn(struct hci_dev *hdev, u8 status) hci_dev_unlock(hdev); } +static void hci_cs_le_set_phy(struct hci_dev *hdev, u8 status) +{ + struct hci_cp_le_set_phy *cp; + struct hci_conn *conn; + + bt_dev_dbg(hdev, "status 0x%2.2x", status); + + if (status) + return; + + cp = hci_sent_cmd_data(hdev, HCI_OP_LE_SET_PHY); + if (!cp) + return; + + hci_dev_lock(hdev); + + conn = hci_conn_hash_lookup_handle(hdev, __le16_to_cpu(cp->handle)); + if (conn) { + conn->le_tx_def_phys = cp->tx_phys; + conn->le_rx_def_phys = cp->rx_phys; + } + + hci_dev_unlock(hdev); +} + static void hci_cs_le_read_remote_features(struct hci_dev *hdev, u8 status) { struct hci_cp_le_read_remote_features *cp; @@ -3086,6 +3115,7 @@ static void hci_conn_complete_evt(struct hci_dev *hdev, void *data, bt_dev_dbg(hdev, "status 0x%2.2x", status); hci_dev_lock(hdev); + hci_store_wake_reason(hdev, &ev->bdaddr, BDADDR_BREDR); /* Check for existing connection: * @@ -3249,6 +3279,10 @@ static void hci_conn_request_evt(struct hci_dev *hdev, void *data, bt_dev_dbg(hdev, "bdaddr %pMR type 0x%x", &ev->bdaddr, ev->link_type); + hci_dev_lock(hdev); + hci_store_wake_reason(hdev, &ev->bdaddr, BDADDR_BREDR); + hci_dev_unlock(hdev); + /* Reject incoming connection from device with same BD ADDR against * CVE-2020-26555 */ @@ -4359,6 +4393,7 @@ static const struct hci_cs { HCI_CS(HCI_OP_LE_CREATE_CONN, hci_cs_le_create_conn), HCI_CS(HCI_OP_LE_READ_REMOTE_FEATURES, hci_cs_le_read_remote_features), HCI_CS(HCI_OP_LE_START_ENC, hci_cs_le_start_enc), + HCI_CS(HCI_OP_LE_SET_PHY, hci_cs_le_set_phy), HCI_CS(HCI_OP_LE_EXT_CREATE_CONN, hci_cs_le_ext_create_conn), HCI_CS(HCI_OP_LE_CREATE_CIS, hci_cs_le_create_cis), HCI_CS(HCI_OP_LE_CREATE_BIG, hci_cs_le_create_big), @@ -4995,6 +5030,7 @@ static void hci_sync_conn_complete_evt(struct hci_dev *hdev, void *data, bt_dev_dbg(hdev, "status 0x%2.2x", status); hci_dev_lock(hdev); + hci_store_wake_reason(hdev, &ev->bdaddr, BDADDR_BREDR); conn = hci_conn_hash_lookup_ba(hdev, ev->link_type, &ev->bdaddr); if (!conn) { @@ -5687,6 +5723,7 @@ static void le_conn_complete_evt(struct hci_dev *hdev, u8 status, int err; hci_dev_lock(hdev); + hci_store_wake_reason(hdev, bdaddr, bdaddr_type); /* All controllers implicitly stop advertising in the event of a * connection, so ensure that the state bit is cleared. @@ -5979,6 +6016,7 @@ static void hci_le_past_received_evt(struct hci_dev *hdev, void *data, bt_dev_dbg(hdev, "status 0x%2.2x", ev->status); hci_dev_lock(hdev); + hci_store_wake_reason(hdev, &ev->bdaddr, ev->bdaddr_type); hci_dev_clear_flag(hdev, HCI_PA_SYNC); @@ -6377,6 +6415,8 @@ static void hci_le_adv_report_evt(struct hci_dev *hdev, void *data, info->length + 1)) break; + hci_store_wake_reason(hdev, &info->bdaddr, info->bdaddr_type); + if (info->length <= max_adv_len(hdev)) { rssi = info->data[info->length]; process_adv_report(hdev, info->type, &info->bdaddr, @@ -6465,6 +6505,8 @@ static void hci_le_ext_adv_report_evt(struct hci_dev *hdev, void *data, info->length)) break; + hci_store_wake_reason(hdev, &info->bdaddr, info->bdaddr_type); + evt_type = __le16_to_cpu(info->type) & LE_EXT_ADV_EVT_TYPE_MASK; legacy_evt_type = ext_evt_type_to_legacy(hdev, evt_type); @@ -6510,6 +6552,7 @@ static void hci_le_pa_sync_established_evt(struct hci_dev *hdev, void *data, bt_dev_dbg(hdev, "status 0x%2.2x", ev->status); hci_dev_lock(hdev); + hci_store_wake_reason(hdev, &ev->bdaddr, ev->bdaddr_type); hci_dev_clear_flag(hdev, HCI_PA_SYNC); @@ -6607,8 +6650,20 @@ static void hci_le_remote_feat_complete_evt(struct hci_dev *hdev, void *data, conn = hci_conn_hash_lookup_handle(hdev, __le16_to_cpu(ev->handle)); if (conn) { - if (!ev->status) - memcpy(conn->features[0], ev->features, 8); + if (!ev->status) { + memcpy(conn->le_features, ev->features, 8); + + /* Update supported PHYs */ + if (!(conn->le_features[1] & HCI_LE_PHY_2M)) { + conn->le_tx_def_phys &= ~HCI_LE_SET_PHY_2M; + conn->le_rx_def_phys &= ~HCI_LE_SET_PHY_2M; + } + + if (!(conn->le_features[1] & HCI_LE_PHY_CODED)) { + conn->le_tx_def_phys &= ~HCI_LE_SET_PHY_CODED; + conn->le_rx_def_phys &= ~HCI_LE_SET_PHY_CODED; + } + } if (conn->state == BT_CONFIG) { __u8 status; @@ -6729,25 +6784,31 @@ static void hci_le_remote_conn_param_req_evt(struct hci_dev *hdev, void *data, latency = le16_to_cpu(ev->latency); timeout = le16_to_cpu(ev->timeout); + hci_dev_lock(hdev); + hcon = hci_conn_hash_lookup_handle(hdev, handle); - if (!hcon || hcon->state != BT_CONNECTED) - return send_conn_param_neg_reply(hdev, handle, - HCI_ERROR_UNKNOWN_CONN_ID); + if (!hcon || hcon->state != BT_CONNECTED) { + send_conn_param_neg_reply(hdev, handle, + HCI_ERROR_UNKNOWN_CONN_ID); + goto unlock; + } - if (max > hcon->le_conn_max_interval) - return send_conn_param_neg_reply(hdev, handle, - HCI_ERROR_INVALID_LL_PARAMS); + if (max > hcon->le_conn_max_interval) { + send_conn_param_neg_reply(hdev, handle, + HCI_ERROR_INVALID_LL_PARAMS); + goto unlock; + } - if (hci_check_conn_params(min, max, latency, timeout)) - return send_conn_param_neg_reply(hdev, handle, - HCI_ERROR_INVALID_LL_PARAMS); + if (hci_check_conn_params(min, max, latency, timeout)) { + send_conn_param_neg_reply(hdev, handle, + HCI_ERROR_INVALID_LL_PARAMS); + goto unlock; + } if (hcon->role == HCI_ROLE_MASTER) { struct hci_conn_params *params; u8 store_hint; - hci_dev_lock(hdev); - params = hci_conn_params_lookup(hdev, &hcon->dst, hcon->dst_type); if (params) { @@ -6760,8 +6821,6 @@ static void hci_le_remote_conn_param_req_evt(struct hci_dev *hdev, void *data, store_hint = 0x00; } - hci_dev_unlock(hdev); - mgmt_new_conn_param(hdev, &hcon->dst, hcon->dst_type, store_hint, min, max, latency, timeout); } @@ -6775,6 +6834,9 @@ static void hci_le_remote_conn_param_req_evt(struct hci_dev *hdev, void *data, cp.max_ce_len = 0; hci_send_cmd(hdev, HCI_OP_LE_CONN_PARAM_REQ_REPLY, sizeof(cp), &cp); + +unlock: + hci_dev_unlock(hdev); } static void hci_le_direct_adv_report_evt(struct hci_dev *hdev, void *data, @@ -6796,6 +6858,8 @@ static void hci_le_direct_adv_report_evt(struct hci_dev *hdev, void *data, for (i = 0; i < ev->num; i++) { struct hci_ev_le_direct_adv_info *info = &ev->info[i]; + hci_store_wake_reason(hdev, &info->bdaddr, info->bdaddr_type); + process_adv_report(hdev, info->type, &info->bdaddr, info->bdaddr_type, &info->direct_addr, info->direct_addr_type, HCI_ADV_PHY_1M, 0, @@ -7221,9 +7285,21 @@ static void hci_le_read_all_remote_features_evt(struct hci_dev *hdev, if (!conn) goto unlock; - if (!ev->status) + if (!ev->status) { memcpy(conn->le_features, ev->features, 248); + /* Update supported PHYs */ + if (!(conn->le_features[1] & HCI_LE_PHY_2M)) { + conn->le_tx_def_phys &= ~HCI_LE_SET_PHY_2M; + conn->le_rx_def_phys &= ~HCI_LE_SET_PHY_2M; + } + + if (!(conn->le_features[1] & HCI_LE_PHY_CODED)) { + conn->le_tx_def_phys &= ~HCI_LE_SET_PHY_CODED; + conn->le_rx_def_phys &= ~HCI_LE_SET_PHY_CODED; + } + } + if (conn->state == BT_CONFIG) { __u8 status; @@ -7452,73 +7528,29 @@ static bool hci_get_cmd_complete(struct hci_dev *hdev, u16 opcode, return true; } -static void hci_store_wake_reason(struct hci_dev *hdev, u8 event, - struct sk_buff *skb) +static void hci_store_wake_reason(struct hci_dev *hdev, + const bdaddr_t *bdaddr, u8 addr_type) + __must_hold(&hdev->lock) { - struct hci_ev_le_advertising_info *adv; - struct hci_ev_le_direct_adv_info *direct_adv; - struct hci_ev_le_ext_adv_info *ext_adv; - const struct hci_ev_conn_complete *conn_complete = (void *)skb->data; - const struct hci_ev_conn_request *conn_request = (void *)skb->data; - - hci_dev_lock(hdev); + lockdep_assert_held(&hdev->lock); /* If we are currently suspended and this is the first BT event seen, * save the wake reason associated with the event. */ if (!hdev->suspended || hdev->wake_reason) - goto unlock; + return; + + if (!bdaddr) { + hdev->wake_reason = MGMT_WAKE_REASON_UNEXPECTED; + return; + } /* Default to remote wake. Values for wake_reason are documented in the * Bluez mgmt api docs. */ hdev->wake_reason = MGMT_WAKE_REASON_REMOTE_WAKE; - - /* Once configured for remote wakeup, we should only wake up for - * reconnections. It's useful to see which device is waking us up so - * keep track of the bdaddr of the connection event that woke us up. - */ - if (event == HCI_EV_CONN_REQUEST) { - bacpy(&hdev->wake_addr, &conn_request->bdaddr); - hdev->wake_addr_type = BDADDR_BREDR; - } else if (event == HCI_EV_CONN_COMPLETE) { - bacpy(&hdev->wake_addr, &conn_complete->bdaddr); - hdev->wake_addr_type = BDADDR_BREDR; - } else if (event == HCI_EV_LE_META) { - struct hci_ev_le_meta *le_ev = (void *)skb->data; - u8 subevent = le_ev->subevent; - u8 *ptr = &skb->data[sizeof(*le_ev)]; - u8 num_reports = *ptr; - - if ((subevent == HCI_EV_LE_ADVERTISING_REPORT || - subevent == HCI_EV_LE_DIRECT_ADV_REPORT || - subevent == HCI_EV_LE_EXT_ADV_REPORT) && - num_reports) { - adv = (void *)(ptr + 1); - direct_adv = (void *)(ptr + 1); - ext_adv = (void *)(ptr + 1); - - switch (subevent) { - case HCI_EV_LE_ADVERTISING_REPORT: - bacpy(&hdev->wake_addr, &adv->bdaddr); - hdev->wake_addr_type = adv->bdaddr_type; - break; - case HCI_EV_LE_DIRECT_ADV_REPORT: - bacpy(&hdev->wake_addr, &direct_adv->bdaddr); - hdev->wake_addr_type = direct_adv->bdaddr_type; - break; - case HCI_EV_LE_EXT_ADV_REPORT: - bacpy(&hdev->wake_addr, &ext_adv->bdaddr); - hdev->wake_addr_type = ext_adv->bdaddr_type; - break; - } - } - } else { - hdev->wake_reason = MGMT_WAKE_REASON_UNEXPECTED; - } - -unlock: - hci_dev_unlock(hdev); + bacpy(&hdev->wake_addr, bdaddr); + hdev->wake_addr_type = addr_type; } #define HCI_EV_VL(_op, _func, _min_len, _max_len) \ @@ -7765,14 +7797,15 @@ void hci_event_packet(struct hci_dev *hdev, struct sk_buff *skb) skb_pull(skb, HCI_EVENT_HDR_SIZE); - /* Store wake reason if we're suspended */ - hci_store_wake_reason(hdev, event, skb); - bt_dev_dbg(hdev, "event 0x%2.2x", event); hci_event_func(hdev, event, skb, &opcode, &status, &req_complete, &req_complete_skb); + hci_dev_lock(hdev); + hci_store_wake_reason(hdev, NULL, 0); + hci_dev_unlock(hdev); + if (req_complete) { req_complete(hdev, status, opcode); } else if (req_complete_skb) { diff --git a/net/bluetooth/hci_sock.c b/net/bluetooth/hci_sock.c index 4e7bf63af9c5fb..0290dea081f621 100644 --- a/net/bluetooth/hci_sock.c +++ b/net/bluetooth/hci_sock.c @@ -2166,6 +2166,7 @@ static void hci_sock_destruct(struct sock *sk) mgmt_cleanup(sk); skb_queue_purge(&sk->sk_receive_queue); skb_queue_purge(&sk->sk_write_queue); + skb_queue_purge(&sk->sk_error_queue); } static const struct proto_ops hci_sock_ops = { diff --git a/net/bluetooth/hci_sync.c b/net/bluetooth/hci_sync.c index cbc3a75d732623..74339358d59940 100644 --- a/net/bluetooth/hci_sync.c +++ b/net/bluetooth/hci_sync.c @@ -25,11 +25,11 @@ static void hci_cmd_sync_complete(struct hci_dev *hdev, u8 result, u16 opcode, { bt_dev_dbg(hdev, "result 0x%2.2x", result); - if (hdev->req_status != HCI_REQ_PEND) + if (READ_ONCE(hdev->req_status) != HCI_REQ_PEND) return; hdev->req_result = result; - hdev->req_status = HCI_REQ_DONE; + WRITE_ONCE(hdev->req_status, HCI_REQ_DONE); /* Free the request command so it is not used as response */ kfree_skb(hdev->req_skb); @@ -167,20 +167,20 @@ struct sk_buff *__hci_cmd_sync_sk(struct hci_dev *hdev, u16 opcode, u32 plen, hci_cmd_sync_add(&req, opcode, plen, param, event, sk); - hdev->req_status = HCI_REQ_PEND; + WRITE_ONCE(hdev->req_status, HCI_REQ_PEND); err = hci_req_sync_run(&req); if (err < 0) return ERR_PTR(err); err = wait_event_interruptible_timeout(hdev->req_wait_q, - hdev->req_status != HCI_REQ_PEND, + READ_ONCE(hdev->req_status) != HCI_REQ_PEND, timeout); if (err == -ERESTARTSYS) return ERR_PTR(-EINTR); - switch (hdev->req_status) { + switch (READ_ONCE(hdev->req_status)) { case HCI_REQ_DONE: err = -bt_to_errno(hdev->req_result); break; @@ -194,7 +194,7 @@ struct sk_buff *__hci_cmd_sync_sk(struct hci_dev *hdev, u16 opcode, u32 plen, break; } - hdev->req_status = 0; + WRITE_ONCE(hdev->req_status, 0); hdev->req_result = 0; skb = hdev->req_rsp; hdev->req_rsp = NULL; @@ -665,9 +665,9 @@ void hci_cmd_sync_cancel(struct hci_dev *hdev, int err) { bt_dev_dbg(hdev, "err 0x%2.2x", err); - if (hdev->req_status == HCI_REQ_PEND) { + if (READ_ONCE(hdev->req_status) == HCI_REQ_PEND) { hdev->req_result = err; - hdev->req_status = HCI_REQ_CANCELED; + WRITE_ONCE(hdev->req_status, HCI_REQ_CANCELED); queue_work(hdev->workqueue, &hdev->cmd_sync_cancel_work); } @@ -683,12 +683,12 @@ void hci_cmd_sync_cancel_sync(struct hci_dev *hdev, int err) { bt_dev_dbg(hdev, "err 0x%2.2x", err); - if (hdev->req_status == HCI_REQ_PEND) { + if (READ_ONCE(hdev->req_status) == HCI_REQ_PEND) { /* req_result is __u32 so error must be positive to be properly * propagated. */ hdev->req_result = err < 0 ? -err : err; - hdev->req_status = HCI_REQ_CANCELED; + WRITE_ONCE(hdev->req_status, HCI_REQ_CANCELED); wake_up_interruptible(&hdev->req_wait_q); } @@ -780,7 +780,7 @@ int hci_cmd_sync_queue_once(struct hci_dev *hdev, hci_cmd_sync_work_func_t func, void *data, hci_cmd_sync_work_destroy_t destroy) { if (hci_cmd_sync_lookup_entry(hdev, func, data, destroy)) - return 0; + return -EEXIST; return hci_cmd_sync_queue(hdev, func, data, destroy); } @@ -801,8 +801,15 @@ int hci_cmd_sync_run(struct hci_dev *hdev, hci_cmd_sync_work_func_t func, return -ENETDOWN; /* If on cmd_sync_work then run immediately otherwise queue */ - if (current_work() == &hdev->cmd_sync_work) - return func(hdev, data); + if (current_work() == &hdev->cmd_sync_work) { + int err; + + err = func(hdev, data); + if (destroy) + destroy(hdev, data, err); + + return 0; + } return hci_cmd_sync_submit(hdev, func, data, destroy); } @@ -3255,6 +3262,8 @@ static int update_passive_scan_sync(struct hci_dev *hdev, void *data) int hci_update_passive_scan(struct hci_dev *hdev) { + int err; + /* Only queue if it would have any effect */ if (!test_bit(HCI_UP, &hdev->flags) || test_bit(HCI_INIT, &hdev->flags) || @@ -3264,8 +3273,9 @@ int hci_update_passive_scan(struct hci_dev *hdev) hci_dev_test_flag(hdev, HCI_UNREGISTER)) return 0; - return hci_cmd_sync_queue_once(hdev, update_passive_scan_sync, NULL, - NULL); + err = hci_cmd_sync_queue_once(hdev, update_passive_scan_sync, NULL, + NULL); + return (err == -EEXIST) ? 0 : err; } int hci_write_sc_support_sync(struct hci_dev *hdev, u8 val) @@ -4564,7 +4574,7 @@ static int hci_le_set_host_feature_sync(struct hci_dev *hdev) { struct hci_cp_le_set_host_feature cp; - if (!iso_capable(hdev)) + if (!cis_capable(hdev)) return 0; memset(&cp, 0, sizeof(cp)); @@ -6596,8 +6606,8 @@ static int hci_le_create_conn_sync(struct hci_dev *hdev, void *data) * state. */ if (hci_dev_test_flag(hdev, HCI_LE_SCAN)) { - hci_scan_disable_sync(hdev); hci_dev_set_flag(hdev, HCI_LE_SCAN_INTERRUPTED); + hci_scan_disable_sync(hdev); } /* Update random address, but set require_privacy to false so @@ -6897,8 +6907,6 @@ static int hci_acl_create_conn_sync(struct hci_dev *hdev, void *data) conn->attempt++; - conn->link_policy = hdev->link_policy; - memset(&cp, 0, sizeof(cp)); bacpy(&cp.bdaddr, &conn->dst); cp.pscan_rep_mode = 0x02; @@ -6929,8 +6937,11 @@ static int hci_acl_create_conn_sync(struct hci_dev *hdev, void *data) int hci_connect_acl_sync(struct hci_dev *hdev, struct hci_conn *conn) { - return hci_cmd_sync_queue_once(hdev, hci_acl_create_conn_sync, conn, - NULL); + int err; + + err = hci_cmd_sync_queue_once(hdev, hci_acl_create_conn_sync, conn, + NULL); + return (err == -EEXIST) ? 0 : err; } static void create_le_conn_complete(struct hci_dev *hdev, void *data, int err) @@ -6966,8 +6977,11 @@ static void create_le_conn_complete(struct hci_dev *hdev, void *data, int err) int hci_connect_le_sync(struct hci_dev *hdev, struct hci_conn *conn) { - return hci_cmd_sync_queue_once(hdev, hci_le_create_conn_sync, conn, - create_le_conn_complete); + int err; + + err = hci_cmd_sync_queue_once(hdev, hci_le_create_conn_sync, conn, + create_le_conn_complete); + return (err == -EEXIST) ? 0 : err; } int hci_cancel_connect_sync(struct hci_dev *hdev, struct hci_conn *conn) @@ -7174,8 +7188,11 @@ static int hci_le_pa_create_sync(struct hci_dev *hdev, void *data) int hci_connect_pa_sync(struct hci_dev *hdev, struct hci_conn *conn) { - return hci_cmd_sync_queue_once(hdev, hci_le_pa_create_sync, conn, - create_pa_complete); + int err; + + err = hci_cmd_sync_queue_once(hdev, hci_le_pa_create_sync, conn, + create_pa_complete); + return (err == -EEXIST) ? 0 : err; } static void create_big_complete(struct hci_dev *hdev, void *data, int err) @@ -7193,7 +7210,8 @@ static void create_big_complete(struct hci_dev *hdev, void *data, int err) static int hci_le_big_create_sync(struct hci_dev *hdev, void *data) { - DEFINE_FLEX(struct hci_cp_le_big_create_sync, cp, bis, num_bis, 0x11); + DEFINE_FLEX(struct hci_cp_le_big_create_sync, cp, bis, num_bis, + HCI_MAX_ISO_BIS); struct hci_conn *conn = data; struct bt_iso_qos *qos = &conn->iso_qos; int err; @@ -7237,8 +7255,11 @@ static int hci_le_big_create_sync(struct hci_dev *hdev, void *data) int hci_connect_big_sync(struct hci_dev *hdev, struct hci_conn *conn) { - return hci_cmd_sync_queue_once(hdev, hci_le_big_create_sync, conn, - create_big_complete); + int err; + + err = hci_cmd_sync_queue_once(hdev, hci_le_big_create_sync, conn, + create_big_complete); + return (err == -EEXIST) ? 0 : err; } struct past_data { @@ -7330,7 +7351,7 @@ int hci_past_sync(struct hci_conn *conn, struct hci_conn *le) if (err) kfree(data); - return err; + return (err == -EEXIST) ? 0 : err; } static void le_read_features_complete(struct hci_dev *hdev, void *data, int err) @@ -7339,10 +7360,8 @@ static void le_read_features_complete(struct hci_dev *hdev, void *data, int err) bt_dev_dbg(hdev, "err %d", err); - if (err == -ECANCELED) - return; - hci_conn_drop(conn); + hci_conn_put(conn); } static int hci_le_read_all_remote_features_sync(struct hci_dev *hdev, @@ -7409,13 +7428,100 @@ int hci_le_read_remote_features(struct hci_conn *conn) * role is possible. Otherwise just transition into the * connected state without requesting the remote features. */ - if (conn->out || (hdev->le_features[0] & HCI_LE_PERIPHERAL_FEATURES)) + if (conn->out || (hdev->le_features[0] & HCI_LE_PERIPHERAL_FEATURES)) { err = hci_cmd_sync_queue_once(hdev, hci_le_read_remote_features_sync, - hci_conn_hold(conn), + hci_conn_hold(hci_conn_get(conn)), le_read_features_complete); - else + if (err) { + hci_conn_drop(conn); + hci_conn_put(conn); + } + } else { err = -EOPNOTSUPP; + } - return err; + return (err == -EEXIST) ? 0 : err; +} + +static void pkt_type_changed(struct hci_dev *hdev, void *data, int err) +{ + struct hci_cp_change_conn_ptype *cp = data; + + bt_dev_dbg(hdev, "err %d", err); + + kfree(cp); +} + +static int hci_change_conn_ptype_sync(struct hci_dev *hdev, void *data) +{ + struct hci_cp_change_conn_ptype *cp = data; + + return __hci_cmd_sync_status_sk(hdev, HCI_OP_CHANGE_CONN_PTYPE, + sizeof(*cp), cp, + HCI_EV_PKT_TYPE_CHANGE, + HCI_CMD_TIMEOUT, NULL); +} + +int hci_acl_change_pkt_type(struct hci_conn *conn, u16 pkt_type) +{ + struct hci_dev *hdev = conn->hdev; + struct hci_cp_change_conn_ptype *cp; + int err; + + cp = kmalloc(sizeof(*cp), GFP_KERNEL); + if (!cp) + return -ENOMEM; + + cp->handle = cpu_to_le16(conn->handle); + cp->pkt_type = cpu_to_le16(pkt_type); + + err = hci_cmd_sync_queue_once(hdev, hci_change_conn_ptype_sync, cp, + pkt_type_changed); + if (err) + kfree(cp); + + return (err == -EEXIST) ? 0 : err; +} + +static void le_phy_update_complete(struct hci_dev *hdev, void *data, int err) +{ + struct hci_cp_le_set_phy *cp = data; + + bt_dev_dbg(hdev, "err %d", err); + + kfree(cp); +} + +static int hci_le_set_phy_sync(struct hci_dev *hdev, void *data) +{ + struct hci_cp_le_set_phy *cp = data; + + return __hci_cmd_sync_status_sk(hdev, HCI_OP_LE_SET_PHY, + sizeof(*cp), cp, + HCI_EV_LE_PHY_UPDATE_COMPLETE, + HCI_CMD_TIMEOUT, NULL); +} + +int hci_le_set_phy(struct hci_conn *conn, u8 tx_phys, u8 rx_phys) +{ + struct hci_dev *hdev = conn->hdev; + struct hci_cp_le_set_phy *cp; + int err; + + cp = kmalloc(sizeof(*cp), GFP_KERNEL); + if (!cp) + return -ENOMEM; + + memset(cp, 0, sizeof(*cp)); + cp->handle = cpu_to_le16(conn->handle); + cp->tx_phys = tx_phys; + cp->rx_phys = rx_phys; + + err = hci_cmd_sync_queue_once(hdev, hci_le_set_phy_sync, cp, + le_phy_update_complete); + if (err) + kfree(cp); + + return (err == -EEXIST) ? 0 : err; } diff --git a/net/bluetooth/hidp/core.c b/net/bluetooth/hidp/core.c index 6724adce615b62..e0e40038155009 100644 --- a/net/bluetooth/hidp/core.c +++ b/net/bluetooth/hidp/core.c @@ -986,7 +986,8 @@ static void session_free(struct kref *ref) skb_queue_purge(&session->intr_transmit); fput(session->intr_sock->file); fput(session->ctrl_sock->file); - l2cap_conn_put(session->conn); + if (session->conn) + l2cap_conn_put(session->conn); kfree(session); } @@ -1164,6 +1165,15 @@ static void hidp_session_remove(struct l2cap_conn *conn, down_write(&hidp_session_sem); + /* Drop L2CAP reference immediately to indicate that + * l2cap_unregister_user() shall not be called as it is already + * considered removed. + */ + if (session->conn) { + l2cap_conn_put(session->conn); + session->conn = NULL; + } + hidp_session_terminate(session); cancel_work_sync(&session->dev_init); @@ -1301,7 +1311,9 @@ static int hidp_session_thread(void *arg) * Instead, this call has the same semantics as if user-space tried to * delete the session. */ - l2cap_unregister_user(session->conn, &session->user); + if (session->conn) + l2cap_unregister_user(session->conn, &session->user); + hidp_session_put(session); module_put_and_kthread_exit(0); diff --git a/net/bluetooth/iso.c b/net/bluetooth/iso.c index e36d24a9098b9d..0f07f05c155770 100644 --- a/net/bluetooth/iso.c +++ b/net/bluetooth/iso.c @@ -746,6 +746,7 @@ static void iso_sock_destruct(struct sock *sk) skb_queue_purge(&sk->sk_receive_queue); skb_queue_purge(&sk->sk_write_queue); + skb_queue_purge(&sk->sk_error_queue); } static void iso_sock_cleanup_listen(struct sock *parent) diff --git a/net/bluetooth/l2cap_core.c b/net/bluetooth/l2cap_core.c index 07b493331fd781..29e23f20dc438d 100644 --- a/net/bluetooth/l2cap_core.c +++ b/net/bluetooth/l2cap_core.c @@ -924,26 +924,41 @@ int l2cap_chan_check_security(struct l2cap_chan *chan, bool initiator) initiator); } -static u8 l2cap_get_ident(struct l2cap_conn *conn) +static int l2cap_get_ident(struct l2cap_conn *conn) { - u8 id; + u8 max; + int ident; + /* LE link does not support tools like l2ping so use the full range */ + if (conn->hcon->type == LE_LINK) + max = 255; /* Get next available identificator. * 1 - 128 are used by kernel. * 129 - 199 are reserved. * 200 - 254 are used by utilities like l2ping, etc. */ + else + max = 128; + + /* Allocate ident using min as last used + 1 (cyclic) */ + ident = ida_alloc_range(&conn->tx_ida, READ_ONCE(conn->tx_ident) + 1, + max, GFP_ATOMIC); + /* Force min 1 to start over */ + if (ident <= 0) { + ident = ida_alloc_range(&conn->tx_ida, 1, max, GFP_ATOMIC); + if (ident <= 0) { + /* If all idents are in use, log an error, this is + * extremely unlikely to happen and would indicate a bug + * in the code that idents are not being freed properly. + */ + BT_ERR("Unable to allocate ident: %d", ident); + return 0; + } + } - mutex_lock(&conn->ident_lock); - - if (++conn->tx_ident > 128) - conn->tx_ident = 1; - - id = conn->tx_ident; - - mutex_unlock(&conn->ident_lock); + WRITE_ONCE(conn->tx_ident, ident); - return id; + return ident; } static void l2cap_send_acl(struct l2cap_conn *conn, struct sk_buff *skb, @@ -1686,17 +1701,15 @@ static void l2cap_info_timeout(struct work_struct *work) int l2cap_register_user(struct l2cap_conn *conn, struct l2cap_user *user) { - struct hci_dev *hdev = conn->hcon->hdev; int ret; /* We need to check whether l2cap_conn is registered. If it is not, we - * must not register the l2cap_user. l2cap_conn_del() is unregisters - * l2cap_conn objects, but doesn't provide its own locking. Instead, it - * relies on the parent hci_conn object to be locked. This itself relies - * on the hci_dev object to be locked. So we must lock the hci device - * here, too. */ + * must not register the l2cap_user. l2cap_conn_del() unregisters + * l2cap_conn objects under conn->lock, and we use the same lock here + * to protect access to conn->users and conn->hchan. + */ - hci_dev_lock(hdev); + mutex_lock(&conn->lock); if (!list_empty(&user->list)) { ret = -EINVAL; @@ -1717,16 +1730,14 @@ int l2cap_register_user(struct l2cap_conn *conn, struct l2cap_user *user) ret = 0; out_unlock: - hci_dev_unlock(hdev); + mutex_unlock(&conn->lock); return ret; } EXPORT_SYMBOL(l2cap_register_user); void l2cap_unregister_user(struct l2cap_conn *conn, struct l2cap_user *user) { - struct hci_dev *hdev = conn->hcon->hdev; - - hci_dev_lock(hdev); + mutex_lock(&conn->lock); if (list_empty(&user->list)) goto out_unlock; @@ -1735,7 +1746,7 @@ void l2cap_unregister_user(struct l2cap_conn *conn, struct l2cap_user *user) user->remove(conn, user); out_unlock: - hci_dev_unlock(hdev); + mutex_unlock(&conn->lock); } EXPORT_SYMBOL(l2cap_unregister_user); @@ -1760,6 +1771,9 @@ static void l2cap_conn_del(struct hci_conn *hcon, int err) BT_DBG("hcon %p conn %p, err %d", hcon, conn, err); + disable_delayed_work_sync(&conn->info_timer); + disable_delayed_work_sync(&conn->id_addr_timer); + mutex_lock(&conn->lock); kfree_skb(conn->rx_skb); @@ -1773,7 +1787,7 @@ static void l2cap_conn_del(struct hci_conn *hcon, int err) if (work_pending(&conn->pending_rx_work)) cancel_work_sync(&conn->pending_rx_work); - cancel_delayed_work_sync(&conn->id_addr_timer); + ida_destroy(&conn->tx_ida); l2cap_unregister_all_users(conn); @@ -1793,9 +1807,6 @@ static void l2cap_conn_del(struct hci_conn *hcon, int err) l2cap_chan_put(chan); } - if (conn->info_state & L2CAP_INFO_FEAT_MASK_REQ_SENT) - cancel_delayed_work_sync(&conn->info_timer); - hci_chan_del(conn->hchan); conn->hchan = NULL; @@ -2387,6 +2398,9 @@ static int l2cap_segment_sdu(struct l2cap_chan *chan, /* Remote device may have requested smaller PDUs */ pdu_len = min_t(size_t, pdu_len, chan->remote_mps); + if (!pdu_len) + return -EINVAL; + if (len <= pdu_len) { sar = L2CAP_SAR_UNSEGMENTED; sdu_len = 0; @@ -4322,14 +4336,16 @@ static inline int l2cap_config_req(struct l2cap_conn *conn, if (test_bit(CONF_INPUT_DONE, &chan->conf_state)) { set_default_fcs(chan); - if (chan->mode == L2CAP_MODE_ERTM || - chan->mode == L2CAP_MODE_STREAMING) - err = l2cap_ertm_init(chan); + if (chan->state != BT_CONNECTED) { + if (chan->mode == L2CAP_MODE_ERTM || + chan->mode == L2CAP_MODE_STREAMING) + err = l2cap_ertm_init(chan); - if (err < 0) - l2cap_send_disconn_req(chan, -err); - else - l2cap_chan_ready(chan); + if (err < 0) + l2cap_send_disconn_req(chan, -err); + else + l2cap_chan_ready(chan); + } goto unlock; } @@ -4622,7 +4638,8 @@ static inline int l2cap_information_rsp(struct l2cap_conn *conn, switch (type) { case L2CAP_IT_FEAT_MASK: - conn->feat_mask = get_unaligned_le32(rsp->data); + if (cmd_len >= sizeof(*rsp) + sizeof(u32)) + conn->feat_mask = get_unaligned_le32(rsp->data); if (conn->feat_mask & L2CAP_FEAT_FIXED_CHAN) { struct l2cap_info_req req; @@ -4641,7 +4658,8 @@ static inline int l2cap_information_rsp(struct l2cap_conn *conn, break; case L2CAP_IT_FIXED_CHAN: - conn->remote_fixed_chan = rsp->data[0]; + if (cmd_len >= sizeof(*rsp) + sizeof(rsp->data[0])) + conn->remote_fixed_chan = rsp->data[0]; conn->info_state |= L2CAP_INFO_FEAT_MASK_REQ_DONE; conn->info_ident = 0; @@ -4782,12 +4800,34 @@ static int l2cap_le_connect_rsp(struct l2cap_conn *conn, return err; } +static void l2cap_put_ident(struct l2cap_conn *conn, u8 code, u8 id) +{ + switch (code) { + case L2CAP_COMMAND_REJ: + case L2CAP_CONN_RSP: + case L2CAP_CONF_RSP: + case L2CAP_DISCONN_RSP: + case L2CAP_ECHO_RSP: + case L2CAP_INFO_RSP: + case L2CAP_CONN_PARAM_UPDATE_RSP: + case L2CAP_ECRED_CONN_RSP: + case L2CAP_ECRED_RECONF_RSP: + /* First do a lookup since the remote may send bogus ids that + * would make ida_free to generate warnings. + */ + if (ida_find_first_range(&conn->tx_ida, id, id) >= 0) + ida_free(&conn->tx_ida, id); + } +} + static inline int l2cap_bredr_sig_cmd(struct l2cap_conn *conn, struct l2cap_cmd_hdr *cmd, u16 cmd_len, u8 *data) { int err = 0; + l2cap_put_ident(conn, cmd->code, cmd->ident); + switch (cmd->code) { case L2CAP_COMMAND_REJ: l2cap_command_rej(conn, cmd, cmd_len, data); @@ -4900,6 +4940,13 @@ static int l2cap_le_connect_req(struct l2cap_conn *conn, goto response_unlock; } + /* Check if Key Size is sufficient for the security level */ + if (!l2cap_check_enc_key_size(conn->hcon, pchan)) { + result = L2CAP_CR_LE_BAD_KEY_SIZE; + chan = NULL; + goto response_unlock; + } + /* Check for valid dynamic CID range */ if (scid < L2CAP_CID_DYN_START || scid > L2CAP_CID_LE_DYN_END) { result = L2CAP_CR_LE_INVALID_SCID; @@ -5035,18 +5082,28 @@ static inline int l2cap_ecred_conn_req(struct l2cap_conn *conn, struct l2cap_chan *chan, *pchan; u16 mtu, mps; __le16 psm; - u8 result, len = 0; - int i, num_scid; + u8 result, rsp_len = 0; + int i, num_scid = 0; bool defer = false; if (!enable_ecred) return -EINVAL; + memset(pdu, 0, sizeof(*pdu)); + if (cmd_len < sizeof(*req) || (cmd_len - sizeof(*req)) % sizeof(u16)) { result = L2CAP_CR_LE_INVALID_PARAMS; goto response; } + /* Check if there are no pending channels with the same ident */ + __l2cap_chan_list_id(conn, cmd->ident, l2cap_ecred_list_defer, + &num_scid); + if (num_scid) { + result = L2CAP_CR_LE_INVALID_PARAMS; + goto response; + } + cmd_len -= sizeof(*req); num_scid = cmd_len / sizeof(u16); @@ -5055,11 +5112,14 @@ static inline int l2cap_ecred_conn_req(struct l2cap_conn *conn, goto response; } + /* Always respond with the same number of scids as in the request */ + rsp_len = cmd_len; + mtu = __le16_to_cpu(req->mtu); mps = __le16_to_cpu(req->mps); if (mtu < L2CAP_ECRED_MIN_MTU || mps < L2CAP_ECRED_MIN_MPS) { - result = L2CAP_CR_LE_UNACCEPT_PARAMS; + result = L2CAP_CR_LE_INVALID_PARAMS; goto response; } @@ -5079,8 +5139,6 @@ static inline int l2cap_ecred_conn_req(struct l2cap_conn *conn, BT_DBG("psm 0x%2.2x mtu %u mps %u", __le16_to_cpu(psm), mtu, mps); - memset(pdu, 0, sizeof(*pdu)); - /* Check if we have socket listening on psm */ pchan = l2cap_global_chan_by_psm(BT_LISTEN, psm, &conn->hcon->src, &conn->hcon->dst, LE_LINK); @@ -5093,7 +5151,16 @@ static inline int l2cap_ecred_conn_req(struct l2cap_conn *conn, if (!smp_sufficient_security(conn->hcon, pchan->sec_level, SMP_ALLOW_STK)) { - result = L2CAP_CR_LE_AUTHENTICATION; + result = pchan->sec_level == BT_SECURITY_MEDIUM ? + L2CAP_CR_LE_ENCRYPTION : L2CAP_CR_LE_AUTHENTICATION; + goto unlock; + } + + /* Check if the listening channel has set an output MTU then the + * requested MTU shall be less than or equal to that value. + */ + if (pchan->omtu && mtu < pchan->omtu) { + result = L2CAP_CR_LE_UNACCEPT_PARAMS; goto unlock; } @@ -5105,7 +5172,6 @@ static inline int l2cap_ecred_conn_req(struct l2cap_conn *conn, BT_DBG("scid[%d] 0x%4.4x", i, scid); pdu->dcid[i] = 0x0000; - len += sizeof(*pdu->dcid); /* Check for valid dynamic CID range */ if (scid < L2CAP_CID_DYN_START || scid > L2CAP_CID_LE_DYN_END) { @@ -5172,7 +5238,7 @@ static inline int l2cap_ecred_conn_req(struct l2cap_conn *conn, return 0; l2cap_send_cmd(conn, cmd->ident, L2CAP_ECRED_CONN_RSP, - sizeof(*pdu) + len, pdu); + sizeof(*pdu) + rsp_len, pdu); return 0; } @@ -5294,14 +5360,14 @@ static inline int l2cap_ecred_reconf_req(struct l2cap_conn *conn, struct l2cap_ecred_reconf_req *req = (void *) data; struct l2cap_ecred_reconf_rsp rsp; u16 mtu, mps, result; - struct l2cap_chan *chan; + struct l2cap_chan *chan[L2CAP_ECRED_MAX_CID] = {}; int i, num_scid; if (!enable_ecred) return -EINVAL; - if (cmd_len < sizeof(*req) || cmd_len - sizeof(*req) % sizeof(u16)) { - result = L2CAP_CR_LE_INVALID_PARAMS; + if (cmd_len < sizeof(*req) || (cmd_len - sizeof(*req)) % sizeof(u16)) { + result = L2CAP_RECONF_INVALID_CID; goto respond; } @@ -5311,42 +5377,69 @@ static inline int l2cap_ecred_reconf_req(struct l2cap_conn *conn, BT_DBG("mtu %u mps %u", mtu, mps); if (mtu < L2CAP_ECRED_MIN_MTU) { - result = L2CAP_RECONF_INVALID_MTU; + result = L2CAP_RECONF_INVALID_PARAMS; goto respond; } if (mps < L2CAP_ECRED_MIN_MPS) { - result = L2CAP_RECONF_INVALID_MPS; + result = L2CAP_RECONF_INVALID_PARAMS; goto respond; } cmd_len -= sizeof(*req); num_scid = cmd_len / sizeof(u16); + + if (num_scid > L2CAP_ECRED_MAX_CID) { + result = L2CAP_RECONF_INVALID_PARAMS; + goto respond; + } + result = L2CAP_RECONF_SUCCESS; + /* Check if each SCID, MTU and MPS are valid */ for (i = 0; i < num_scid; i++) { u16 scid; scid = __le16_to_cpu(req->scid[i]); - if (!scid) - return -EPROTO; + if (!scid) { + result = L2CAP_RECONF_INVALID_CID; + goto respond; + } - chan = __l2cap_get_chan_by_dcid(conn, scid); - if (!chan) - continue; + chan[i] = __l2cap_get_chan_by_dcid(conn, scid); + if (!chan[i]) { + result = L2CAP_RECONF_INVALID_CID; + goto respond; + } - /* If the MTU value is decreased for any of the included - * channels, then the receiver shall disconnect all - * included channels. + /* The MTU field shall be greater than or equal to the greatest + * current MTU size of these channels. */ - if (chan->omtu > mtu) { - BT_ERR("chan %p decreased MTU %u -> %u", chan, - chan->omtu, mtu); + if (chan[i]->omtu > mtu) { + BT_ERR("chan %p decreased MTU %u -> %u", chan[i], + chan[i]->omtu, mtu); result = L2CAP_RECONF_INVALID_MTU; + goto respond; } - chan->omtu = mtu; - chan->remote_mps = mps; + /* If more than one channel is being configured, the MPS field + * shall be greater than or equal to the current MPS size of + * each of these channels. If only one channel is being + * configured, the MPS field may be less than the current MPS + * of that channel. + */ + if (chan[i]->remote_mps >= mps && i) { + BT_ERR("chan %p decreased MPS %u -> %u", chan[i], + chan[i]->remote_mps, mps); + result = L2CAP_RECONF_INVALID_MPS; + goto respond; + } + } + + /* Commit the new MTU and MPS values after checking they are valid */ + for (i = 0; i < num_scid; i++) { + chan[i]->omtu = mtu; + chan[i]->remote_mps = mps; } respond: @@ -5363,7 +5456,7 @@ static inline int l2cap_ecred_reconf_rsp(struct l2cap_conn *conn, u8 *data) { struct l2cap_chan *chan, *tmp; - struct l2cap_ecred_conn_rsp *rsp = (void *) data; + struct l2cap_ecred_reconf_rsp *rsp = (void *)data; u16 result; if (cmd_len < sizeof(*rsp)) @@ -5371,7 +5464,7 @@ static inline int l2cap_ecred_reconf_rsp(struct l2cap_conn *conn, result = __le16_to_cpu(rsp->result); - BT_DBG("result 0x%4.4x", rsp->result); + BT_DBG("result 0x%4.4x", result); if (!result) return 0; @@ -5419,6 +5512,8 @@ static inline int l2cap_le_sig_cmd(struct l2cap_conn *conn, { int err = 0; + l2cap_put_ident(conn, cmd->code, cmd->ident); + switch (cmd->code) { case L2CAP_COMMAND_REJ: l2cap_le_command_rej(conn, cmd, cmd_len, data); @@ -6538,6 +6633,10 @@ static void l2cap_chan_le_send_credits(struct l2cap_chan *chan) struct l2cap_le_credits pkt; u16 return_credits = l2cap_le_rx_credits(chan); + if (chan->mode != L2CAP_MODE_LE_FLOWCTL && + chan->mode != L2CAP_MODE_EXT_FLOWCTL) + return; + if (chan->rx_credits >= return_credits) return; @@ -6599,8 +6698,10 @@ static int l2cap_ecred_data_rcv(struct l2cap_chan *chan, struct sk_buff *skb) return -ENOBUFS; } - if (chan->imtu < skb->len) { - BT_ERR("Too big LE L2CAP PDU"); + if (skb->len > chan->imtu) { + BT_ERR("Too big LE L2CAP PDU: len %u > %u", skb->len, + chan->imtu); + l2cap_send_disconn_req(chan, ECONNRESET); return -ENOBUFS; } @@ -6619,6 +6720,11 @@ static int l2cap_ecred_data_rcv(struct l2cap_chan *chan, struct sk_buff *skb) if (!chan->sdu) { u16 sdu_len; + if (!pskb_may_pull(skb, L2CAP_SDULEN_SIZE)) { + err = -EINVAL; + goto failed; + } + sdu_len = get_unaligned_le16(skb->data); skb_pull(skb, L2CAP_SDULEN_SIZE); @@ -6626,7 +6732,9 @@ static int l2cap_ecred_data_rcv(struct l2cap_chan *chan, struct sk_buff *skb) sdu_len, skb->len, chan->imtu); if (sdu_len > chan->imtu) { - BT_ERR("Too big LE L2CAP SDU length received"); + BT_ERR("Too big LE L2CAP SDU length: len %u > %u", + skb->len, sdu_len); + l2cap_send_disconn_req(chan, ECONNRESET); err = -EMSGSIZE; goto failed; } @@ -6662,6 +6770,7 @@ static int l2cap_ecred_data_rcv(struct l2cap_chan *chan, struct sk_buff *skb) if (chan->sdu->len + skb->len > chan->sdu_len) { BT_ERR("Too much LE L2CAP data received"); + l2cap_send_disconn_req(chan, ECONNRESET); err = -EINVAL; goto failed; } @@ -6907,13 +7016,13 @@ static struct l2cap_conn *l2cap_conn_add(struct hci_conn *hcon) hci_dev_test_flag(hcon->hdev, HCI_FORCE_BREDR_SMP))) conn->local_fixed_chan |= L2CAP_FC_SMP_BREDR; - mutex_init(&conn->ident_lock); mutex_init(&conn->lock); INIT_LIST_HEAD(&conn->chan_l); INIT_LIST_HEAD(&conn->users); INIT_DELAYED_WORK(&conn->info_timer, l2cap_info_timeout); + ida_init(&conn->tx_ida); skb_queue_head_init(&conn->pending_rx); INIT_WORK(&conn->pending_rx_work, process_pending_rx); diff --git a/net/bluetooth/l2cap_sock.c b/net/bluetooth/l2cap_sock.c index 9ee189c815d498..901d053baf3aaa 100644 --- a/net/bluetooth/l2cap_sock.c +++ b/net/bluetooth/l2cap_sock.c @@ -885,12 +885,22 @@ static int l2cap_sock_setsockopt(struct socket *sock, int level, int optname, struct bt_power pwr; struct l2cap_conn *conn; int err = 0; - u32 opt; + u32 opt, phys; u16 mtu; u8 mode; BT_DBG("sk %p", sk); + if (level == SOL_SOCKET) { + conn = chan->conn; + if (conn) + err = hci_conn_setsockopt(conn->hcon, sock->sk, level, + optname, optval, optlen); + if (err) + return err; + return sock_setsockopt(sock, level, optname, optval, optlen); + } + if (level == SOL_L2CAP) return l2cap_sock_setsockopt_old(sock, optname, optval, optlen); @@ -1029,10 +1039,17 @@ static int l2cap_sock_setsockopt(struct socket *sock, int level, int optname, break; } - /* Setting is not supported as it's the remote side that - * decides this. - */ - err = -EPERM; + /* Only allow setting output MTU when not connected */ + if (sk->sk_state == BT_CONNECTED) { + err = -EISCONN; + break; + } + + err = copy_safe_from_sockptr(&mtu, sizeof(mtu), optval, optlen); + if (err) + break; + + chan->omtu = mtu; break; case BT_RCVMTU: @@ -1059,6 +1076,24 @@ static int l2cap_sock_setsockopt(struct socket *sock, int level, int optname, break; + case BT_PHY: + if (sk->sk_state != BT_CONNECTED) { + err = -ENOTCONN; + break; + } + + err = copy_safe_from_sockptr(&phys, sizeof(phys), optval, + optlen); + if (err) + break; + + if (!chan->conn) + break; + + conn = chan->conn; + err = hci_conn_set_phy(conn->hcon, phys); + break; + case BT_MODE: if (!enable_ecred) { err = -ENOPROTOOPT; @@ -1674,6 +1709,9 @@ static void l2cap_sock_ready_cb(struct l2cap_chan *chan) struct sock *sk = chan->data; struct sock *parent; + if (!sk) + return; + lock_sock(sk); parent = bt_sk(sk)->parent; @@ -1799,6 +1837,7 @@ static void l2cap_sock_destruct(struct sock *sk) skb_queue_purge(&sk->sk_receive_queue); skb_queue_purge(&sk->sk_write_queue); + skb_queue_purge(&sk->sk_error_queue); } static void l2cap_skb_msg_name(struct sk_buff *skb, void *msg_name, @@ -1903,6 +1942,9 @@ static struct sock *l2cap_sock_alloc(struct net *net, struct socket *sock, INIT_LIST_HEAD(&l2cap_pi(sk)->rx_busy); + if (sock) + set_bit(SOCK_CUSTOM_SOCKOPT, &sock->flags); + chan = l2cap_chan_create(); if (!chan) { sk_free(sk); diff --git a/net/bluetooth/mgmt.c b/net/bluetooth/mgmt.c index 0e46f9e08b1067..86fd2009de0d2d 100644 --- a/net/bluetooth/mgmt.c +++ b/net/bluetooth/mgmt.c @@ -2195,10 +2195,7 @@ static void set_mesh_complete(struct hci_dev *hdev, void *data, int err) sk = cmd->sk; if (status) { - mgmt_cmd_status(cmd->sk, hdev->id, MGMT_OP_SET_MESH_RECEIVER, - status); - mgmt_pending_foreach(MGMT_OP_SET_MESH_RECEIVER, hdev, true, - cmd_status_rsp, &status); + mgmt_cmd_status(cmd->sk, hdev->id, cmd->opcode, status); goto done; } @@ -2481,6 +2478,7 @@ static int mesh_send(struct sock *sk, struct hci_dev *hdev, void *data, u16 len) struct mgmt_mesh_tx *mesh_tx; struct mgmt_cp_mesh_send *send = data; struct mgmt_rp_mesh_read_features rp; + u16 expected_len; bool sending; int err = 0; @@ -2488,12 +2486,19 @@ static int mesh_send(struct sock *sk, struct hci_dev *hdev, void *data, u16 len) !hci_dev_test_flag(hdev, HCI_MESH_EXPERIMENTAL)) return mgmt_cmd_status(sk, hdev->id, MGMT_OP_MESH_SEND, MGMT_STATUS_NOT_SUPPORTED); - if (!hci_dev_test_flag(hdev, HCI_LE_ENABLED) || - len <= MGMT_MESH_SEND_SIZE || - len > (MGMT_MESH_SEND_SIZE + 31)) + if (!hci_dev_test_flag(hdev, HCI_LE_ENABLED)) + return mgmt_cmd_status(sk, hdev->id, MGMT_OP_MESH_SEND, + MGMT_STATUS_REJECTED); + + if (!send->adv_data_len || send->adv_data_len > 31) return mgmt_cmd_status(sk, hdev->id, MGMT_OP_MESH_SEND, MGMT_STATUS_REJECTED); + expected_len = struct_size(send, adv_data, send->adv_data_len); + if (expected_len != len) + return mgmt_cmd_status(sk, hdev->id, MGMT_OP_MESH_SEND, + MGMT_STATUS_INVALID_PARAMS); + hci_dev_lock(hdev); memset(&rp, 0, sizeof(rp)); @@ -5358,7 +5363,7 @@ static void mgmt_add_adv_patterns_monitor_complete(struct hci_dev *hdev, * hci_adv_monitors_clear is about to be called which will take care of * freeing the adv_monitor instances. */ - if (status == -ECANCELED && !mgmt_pending_valid(hdev, cmd)) + if (status == -ECANCELED || !mgmt_pending_valid(hdev, cmd)) return; monitor = cmd->user_data; @@ -5377,7 +5382,7 @@ static void mgmt_add_adv_patterns_monitor_complete(struct hci_dev *hdev, mgmt_cmd_complete(cmd->sk, cmd->hdev->id, cmd->opcode, mgmt_status(status), &rp, sizeof(rp)); - mgmt_pending_remove(cmd); + mgmt_pending_free(cmd); hci_dev_unlock(hdev); bt_dev_dbg(hdev, "add monitor %d complete, status %d", @@ -7251,6 +7256,9 @@ static bool ltk_is_valid(struct mgmt_ltk_info *key) if (key->initiator != 0x00 && key->initiator != 0x01) return false; + if (key->enc_size > sizeof(key->val)) + return false; + switch (key->addr.type) { case BDADDR_LE_PUBLIC: return true; diff --git a/net/bluetooth/sco.c b/net/bluetooth/sco.c index 87ba90336e8034..a446844354a180 100644 --- a/net/bluetooth/sco.c +++ b/net/bluetooth/sco.c @@ -298,7 +298,7 @@ static int sco_chan_add(struct sco_conn *conn, struct sock *sk, int err = 0; sco_conn_lock(conn); - if (conn->sk) + if (conn->sk || sco_pi(sk)->conn) err = -EBUSY; else __sco_chan_add(conn, sk, parent); @@ -353,9 +353,20 @@ static int sco_connect(struct sock *sk) lock_sock(sk); + /* Recheck state after reacquiring the socket lock, as another + * thread may have changed it (e.g., closed the socket). + */ + if (sk->sk_state != BT_OPEN && sk->sk_state != BT_BOUND) { + release_sock(sk); + hci_conn_drop(hcon); + err = -EBADFD; + goto unlock; + } + err = sco_chan_add(conn, sk, NULL); if (err) { release_sock(sk); + hci_conn_drop(hcon); goto unlock; } @@ -401,7 +412,7 @@ static void sco_recv_frame(struct sco_conn *conn, struct sk_buff *skb) struct sock *sk; sco_conn_lock(conn); - sk = conn->sk; + sk = sco_sock_hold(conn); sco_conn_unlock(conn); if (!sk) @@ -410,11 +421,15 @@ static void sco_recv_frame(struct sco_conn *conn, struct sk_buff *skb) BT_DBG("sk %p len %u", sk, skb->len); if (sk->sk_state != BT_CONNECTED) - goto drop; + goto drop_put; - if (!sock_queue_rcv_skb(sk, skb)) + if (!sock_queue_rcv_skb(sk, skb)) { + sock_put(sk); return; + } +drop_put: + sock_put(sk); drop: kfree_skb(skb); } @@ -470,6 +485,7 @@ static void sco_sock_destruct(struct sock *sk) skb_queue_purge(&sk->sk_receive_queue); skb_queue_purge(&sk->sk_write_queue); + skb_queue_purge(&sk->sk_error_queue); } static void sco_sock_cleanup_listen(struct sock *parent) @@ -651,13 +667,18 @@ static int sco_sock_connect(struct socket *sock, struct sockaddr_unsized *addr, addr->sa_family != AF_BLUETOOTH) return -EINVAL; - if (sk->sk_state != BT_OPEN && sk->sk_state != BT_BOUND) + lock_sock(sk); + + if (sk->sk_state != BT_OPEN && sk->sk_state != BT_BOUND) { + release_sock(sk); return -EBADFD; + } - if (sk->sk_type != SOCK_SEQPACKET) - err = -EINVAL; + if (sk->sk_type != SOCK_SEQPACKET) { + release_sock(sk); + return -EINVAL; + } - lock_sock(sk); /* Set destination address and psm */ bacpy(&sco_pi(sk)->dst, &sa->sco_bdaddr); release_sock(sk); diff --git a/net/bluetooth/smp.c b/net/bluetooth/smp.c index 3a1ce04a7a5361..abf3ab7479ff2d 100644 --- a/net/bluetooth/smp.c +++ b/net/bluetooth/smp.c @@ -1018,10 +1018,7 @@ static u8 smp_random(struct smp_chan *smp) smp_s1(smp->tk, smp->prnd, smp->rrnd, stk); - if (hcon->pending_sec_level == BT_SECURITY_HIGH) - auth = 1; - else - auth = 0; + auth = test_bit(SMP_FLAG_MITM_AUTH, &smp->flags) ? 1 : 0; /* Even though there's no _RESPONDER suffix this is the * responder STK we're adding for later lookup (the initiator @@ -1826,7 +1823,7 @@ static u8 smp_cmd_pairing_req(struct l2cap_conn *conn, struct sk_buff *skb) if (sec_level > conn->hcon->pending_sec_level) conn->hcon->pending_sec_level = sec_level; - /* If we need MITM check that it can be achieved */ + /* If we need MITM check that it can be achieved. */ if (conn->hcon->pending_sec_level >= BT_SECURITY_HIGH) { u8 method; @@ -1834,6 +1831,10 @@ static u8 smp_cmd_pairing_req(struct l2cap_conn *conn, struct sk_buff *skb) req->io_capability); if (method == JUST_WORKS || method == JUST_CFM) return SMP_AUTH_REQUIREMENTS; + + /* Force MITM bit if it isn't set by the initiator. */ + auth |= SMP_AUTH_MITM; + rsp.auth_req |= SMP_AUTH_MITM; } key_size = min(req->max_key_size, rsp.max_key_size); @@ -2743,7 +2744,7 @@ static int smp_cmd_public_key(struct l2cap_conn *conn, struct sk_buff *skb) if (!test_bit(SMP_FLAG_DEBUG_KEY, &smp->flags) && !crypto_memneq(key, smp->local_pk, 64)) { bt_dev_err(hdev, "Remote and local public keys are identical"); - return SMP_UNSPECIFIED; + return SMP_DHKEY_CHECK_FAILED; } memcpy(smp->remote_pk, key, 64); diff --git a/net/bridge/br_arp_nd_proxy.c b/net/bridge/br_arp_nd_proxy.c index 1e2b51769eec81..6b5595868a39c0 100644 --- a/net/bridge/br_arp_nd_proxy.c +++ b/net/bridge/br_arp_nd_proxy.c @@ -251,12 +251,12 @@ struct nd_msg *br_is_nd_neigh_msg(const struct sk_buff *skb, struct nd_msg *msg) static void br_nd_send(struct net_bridge *br, struct net_bridge_port *p, struct sk_buff *request, struct neighbour *n, - __be16 vlan_proto, u16 vlan_tci, struct nd_msg *ns) + __be16 vlan_proto, u16 vlan_tci) { struct net_device *dev = request->dev; struct net_bridge_vlan_group *vg; + struct nd_msg *na, *ns; struct sk_buff *reply; - struct nd_msg *na; struct ipv6hdr *pip6; int na_olen = 8; /* opt hdr + ETH_ALEN for target */ int ns_olen; @@ -264,7 +264,7 @@ static void br_nd_send(struct net_bridge *br, struct net_bridge_port *p, u8 *daddr; u16 pvid; - if (!dev) + if (!dev || skb_linearize(request)) return; len = LL_RESERVED_SPACE(dev) + sizeof(struct ipv6hdr) + @@ -281,17 +281,21 @@ static void br_nd_send(struct net_bridge *br, struct net_bridge_port *p, skb_set_mac_header(reply, 0); daddr = eth_hdr(request)->h_source; + ns = (struct nd_msg *)(skb_network_header(request) + + sizeof(struct ipv6hdr)); /* Do we need option processing ? */ ns_olen = request->len - (skb_network_offset(request) + sizeof(struct ipv6hdr)) - sizeof(*ns); for (i = 0; i < ns_olen - 1; i += (ns->opt[i + 1] << 3)) { - if (!ns->opt[i + 1]) { + if (!ns->opt[i + 1] || i + (ns->opt[i + 1] << 3) > ns_olen) { kfree_skb(reply); return; } if (ns->opt[i] == ND_OPT_SOURCE_LL_ADDR) { - daddr = ns->opt + i + sizeof(struct nd_opt_hdr); + if ((ns->opt[i + 1] << 3) >= + sizeof(struct nd_opt_hdr) + ETH_ALEN) + daddr = ns->opt + i + sizeof(struct nd_opt_hdr); break; } } @@ -472,9 +476,9 @@ void br_do_suppress_nd(struct sk_buff *skb, struct net_bridge *br, if (vid != 0) br_nd_send(br, p, skb, n, skb->vlan_proto, - skb_vlan_tag_get(skb), msg); + skb_vlan_tag_get(skb)); else - br_nd_send(br, p, skb, n, 0, 0, msg); + br_nd_send(br, p, skb, n, 0, 0); replied = true; } diff --git a/net/bridge/br_cfm.c b/net/bridge/br_cfm.c index c2c1c7d44c615f..f4ca77d9b0e96e 100644 --- a/net/bridge/br_cfm.c +++ b/net/bridge/br_cfm.c @@ -576,7 +576,7 @@ static void mep_delete_implementation(struct net_bridge *br, /* Empty and free peer MEP list */ hlist_for_each_entry_safe(peer_mep, n_store, &mep->peer_mep_list, head) { - cancel_delayed_work_sync(&peer_mep->ccm_rx_dwork); + disable_delayed_work_sync(&peer_mep->ccm_rx_dwork); hlist_del_rcu(&peer_mep->head); kfree_rcu(peer_mep, rcu); } @@ -732,7 +732,7 @@ int br_cfm_cc_peer_mep_remove(struct net_bridge *br, const u32 instance, return -ENOENT; } - cc_peer_disable(peer_mep); + disable_delayed_work_sync(&peer_mep->ccm_rx_dwork); hlist_del_rcu(&peer_mep->head); kfree_rcu(peer_mep, rcu); diff --git a/net/bridge/br_device.c b/net/bridge/br_device.c index a818fdc22da9ad..525d4eccd194a7 100644 --- a/net/bridge/br_device.c +++ b/net/bridge/br_device.c @@ -74,7 +74,7 @@ netdev_tx_t br_dev_xmit(struct sk_buff *skb, struct net_device *dev) eth_hdr(skb)->h_proto == htons(ETH_P_RARP)) && br_opt_get(br, BROPT_NEIGH_SUPPRESS_ENABLED)) { br_do_proxy_suppress_arp(skb, br, vid, NULL); - } else if (IS_ENABLED(CONFIG_IPV6) && + } else if (ipv6_mod_enabled() && skb->protocol == htons(ETH_P_IPV6) && br_opt_get(br, BROPT_NEIGH_SUPPRESS_ENABLED) && pskb_may_pull(skb, sizeof(struct ipv6hdr) + diff --git a/net/bridge/br_fdb.c b/net/bridge/br_fdb.c index 0501ffcb8a3ddb..e2c17f620f009a 100644 --- a/net/bridge/br_fdb.c +++ b/net/bridge/br_fdb.c @@ -597,6 +597,9 @@ static void br_fdb_delete_locals_per_vlan_port(struct net_bridge *br, dev = br->dev; } + if (!vg) + return; + list_for_each_entry(v, &vg->vlan_list, vlist) br_fdb_find_delete_local(br, p, dev->dev_addr, v->vid); } @@ -630,6 +633,9 @@ static int br_fdb_insert_locals_per_vlan_port(struct net_bridge *br, dev = br->dev; } + if (!vg) + return 0; + list_for_each_entry(v, &vg->vlan_list, vlist) { if (!br_vlan_should_use(v)) continue; diff --git a/net/bridge/br_input.c b/net/bridge/br_input.c index 1405f1061a5493..2cbae0f9ae1f03 100644 --- a/net/bridge/br_input.c +++ b/net/bridge/br_input.c @@ -170,7 +170,7 @@ int br_handle_frame_finish(struct net *net, struct sock *sk, struct sk_buff *skb (skb->protocol == htons(ETH_P_ARP) || skb->protocol == htons(ETH_P_RARP))) { br_do_proxy_suppress_arp(skb, br, vid, p); - } else if (IS_ENABLED(CONFIG_IPV6) && + } else if (ipv6_mod_enabled() && skb->protocol == htons(ETH_P_IPV6) && br_opt_get(br, BROPT_NEIGH_SUPPRESS_ENABLED) && pskb_may_pull(skb, sizeof(struct ipv6hdr) + diff --git a/net/bridge/br_mrp_netlink.c b/net/bridge/br_mrp_netlink.c index ce6f63c77cc0ac..86f0e75d6e345f 100644 --- a/net/bridge/br_mrp_netlink.c +++ b/net/bridge/br_mrp_netlink.c @@ -196,7 +196,7 @@ static const struct nla_policy br_mrp_start_test_policy[IFLA_BRIDGE_MRP_START_TEST_MAX + 1] = { [IFLA_BRIDGE_MRP_START_TEST_UNSPEC] = { .type = NLA_REJECT }, [IFLA_BRIDGE_MRP_START_TEST_RING_ID] = { .type = NLA_U32 }, - [IFLA_BRIDGE_MRP_START_TEST_INTERVAL] = { .type = NLA_U32 }, + [IFLA_BRIDGE_MRP_START_TEST_INTERVAL] = NLA_POLICY_MIN(NLA_U32, 1), [IFLA_BRIDGE_MRP_START_TEST_MAX_MISS] = { .type = NLA_U32 }, [IFLA_BRIDGE_MRP_START_TEST_PERIOD] = { .type = NLA_U32 }, [IFLA_BRIDGE_MRP_START_TEST_MONITOR] = { .type = NLA_U32 }, @@ -316,7 +316,7 @@ static const struct nla_policy br_mrp_start_in_test_policy[IFLA_BRIDGE_MRP_START_IN_TEST_MAX + 1] = { [IFLA_BRIDGE_MRP_START_IN_TEST_UNSPEC] = { .type = NLA_REJECT }, [IFLA_BRIDGE_MRP_START_IN_TEST_IN_ID] = { .type = NLA_U32 }, - [IFLA_BRIDGE_MRP_START_IN_TEST_INTERVAL] = { .type = NLA_U32 }, + [IFLA_BRIDGE_MRP_START_IN_TEST_INTERVAL] = NLA_POLICY_MIN(NLA_U32, 1), [IFLA_BRIDGE_MRP_START_IN_TEST_MAX_MISS] = { .type = NLA_U32 }, [IFLA_BRIDGE_MRP_START_IN_TEST_PERIOD] = { .type = NLA_U32 }, }; diff --git a/net/bridge/br_multicast.c b/net/bridge/br_multicast.c index d55a4ab87837f7..e9a7e653040177 100644 --- a/net/bridge/br_multicast.c +++ b/net/bridge/br_multicast.c @@ -244,14 +244,11 @@ br_multicast_port_vid_to_port_ctx(struct net_bridge_port *port, u16 vid) lockdep_assert_held_once(&port->br->multicast_lock); - if (!br_opt_get(port->br, BROPT_MCAST_VLAN_SNOOPING_ENABLED)) - return NULL; - /* Take RCU to access the vlan. */ rcu_read_lock(); vlan = br_vlan_find(nbp_vlan_group_rcu(port), vid); - if (vlan && !br_multicast_port_ctx_vlan_disabled(&vlan->port_mcast_ctx)) + if (vlan) pmctx = &vlan->port_mcast_ctx; rcu_read_unlock(); @@ -701,7 +698,10 @@ br_multicast_port_ngroups_inc_one(struct net_bridge_mcast_port *pmctx, u32 max = READ_ONCE(pmctx->mdb_max_entries); u32 n = READ_ONCE(pmctx->mdb_n_entries); - if (max && n >= max) { + /* enforce the max limit when it's a port pmctx or a port-vlan pmctx + * with snooping enabled + */ + if (!br_multicast_port_ctx_vlan_disabled(pmctx) && max && n >= max) { NL_SET_ERR_MSG_FMT_MOD(extack, "%s is already in %u groups, and mcast_max_groups=%u", what, n, max); return -E2BIG; @@ -736,9 +736,7 @@ static int br_multicast_port_ngroups_inc(struct net_bridge_port *port, return err; } - /* Only count on the VLAN context if VID is given, and if snooping on - * that VLAN is enabled. - */ + /* Only count on the VLAN context if VID is given */ if (!group->vid) return 0; @@ -2011,6 +2009,18 @@ void br_multicast_port_ctx_init(struct net_bridge_port *port, timer_setup(&pmctx->ip6_own_query.timer, br_ip6_multicast_port_query_expired, 0); #endif + /* initialize mdb_n_entries if a new port vlan is being created */ + if (vlan) { + struct net_bridge_port_group *pg; + u32 n = 0; + + spin_lock_bh(&port->br->multicast_lock); + hlist_for_each_entry(pg, &port->mglist, mglist) + if (pg->key.addr.vid == vlan->vid) + n++; + WRITE_ONCE(pmctx->mdb_n_entries, n); + spin_unlock_bh(&port->br->multicast_lock); + } } void br_multicast_port_ctx_deinit(struct net_bridge_mcast_port *pmctx) @@ -2094,25 +2104,6 @@ static void __br_multicast_enable_port_ctx(struct net_bridge_mcast_port *pmctx) br_ip4_multicast_add_router(brmctx, pmctx); br_ip6_multicast_add_router(brmctx, pmctx); } - - if (br_multicast_port_ctx_is_vlan(pmctx)) { - struct net_bridge_port_group *pg; - u32 n = 0; - - /* The mcast_n_groups counter might be wrong. First, - * BR_VLFLAG_MCAST_ENABLED is toggled before temporary entries - * are flushed, thus mcast_n_groups after the toggle does not - * reflect the true values. And second, permanent entries added - * while BR_VLFLAG_MCAST_ENABLED was disabled, are not reflected - * either. Thus we have to refresh the counter. - */ - - hlist_for_each_entry(pg, &pmctx->port->mglist, mglist) { - if (pg->key.addr.vid == pmctx->vlan->vid) - n++; - } - WRITE_ONCE(pmctx->mdb_n_entries, n); - } } static void br_multicast_enable_port_ctx(struct net_bridge_mcast_port *pmctx) diff --git a/net/bridge/br_private.h b/net/bridge/br_private.h index b9b2981c484149..9b55d38ea9edbf 100644 --- a/net/bridge/br_private.h +++ b/net/bridge/br_private.h @@ -1344,6 +1344,16 @@ br_multicast_ctx_options_equal(const struct net_bridge_mcast *brmctx1, true; } +static inline bool +br_multicast_port_ctx_options_equal(const struct net_bridge_mcast_port *pmctx1, + const struct net_bridge_mcast_port *pmctx2) +{ + return br_multicast_ngroups_get(pmctx1) == + br_multicast_ngroups_get(pmctx2) && + br_multicast_ngroups_get_max(pmctx1) == + br_multicast_ngroups_get_max(pmctx2); +} + static inline bool br_multicast_ctx_matches_vlan_snooping(const struct net_bridge_mcast *brmctx) { diff --git a/net/bridge/br_vlan_options.c b/net/bridge/br_vlan_options.c index 8fa89b04ee942d..5514e1fc8d1faf 100644 --- a/net/bridge/br_vlan_options.c +++ b/net/bridge/br_vlan_options.c @@ -43,9 +43,29 @@ bool br_vlan_opts_eq_range(const struct net_bridge_vlan *v_curr, u8 range_mc_rtr = br_vlan_multicast_router(range_end); u8 curr_mc_rtr = br_vlan_multicast_router(v_curr); - return v_curr->state == range_end->state && - __vlan_tun_can_enter_range(v_curr, range_end) && - curr_mc_rtr == range_mc_rtr; + if (v_curr->state != range_end->state) + return false; + + if (!__vlan_tun_can_enter_range(v_curr, range_end)) + return false; + + if (curr_mc_rtr != range_mc_rtr) + return false; + + /* Check user-visible priv_flags that affect output */ + if ((v_curr->priv_flags ^ range_end->priv_flags) & + (BR_VLFLAG_NEIGH_SUPPRESS_ENABLED | BR_VLFLAG_MCAST_ENABLED)) + return false; + +#ifdef CONFIG_BRIDGE_IGMP_SNOOPING + if (!br_vlan_is_master(v_curr) && + !br_multicast_port_ctx_vlan_disabled(&v_curr->port_mcast_ctx) && + !br_multicast_port_ctx_options_equal(&v_curr->port_mcast_ctx, + &range_end->port_mcast_ctx)) + return false; +#endif + + return true; } bool br_vlan_opts_fill(struct sk_buff *skb, const struct net_bridge_vlan *v, diff --git a/net/can/af_can.c b/net/can/af_can.c index 770173d8db4281..a624c04ed5c631 100644 --- a/net/can/af_can.c +++ b/net/can/af_can.c @@ -469,7 +469,7 @@ int can_rx_register(struct net *net, struct net_device *dev, canid_t can_id, rcv->can_id = can_id; rcv->mask = mask; - rcv->matches = 0; + atomic_long_set(&rcv->matches, 0); rcv->func = func; rcv->data = data; rcv->ident = ident; @@ -573,7 +573,7 @@ EXPORT_SYMBOL(can_rx_unregister); static inline void deliver(struct sk_buff *skb, struct receiver *rcv) { rcv->func(skb, rcv->data); - rcv->matches++; + atomic_long_inc(&rcv->matches); } static int can_rcv_filter(struct can_dev_rcv_lists *dev_rcv_lists, struct sk_buff *skb) diff --git a/net/can/af_can.h b/net/can/af_can.h index 22f3352c77fece..87887014f5628e 100644 --- a/net/can/af_can.h +++ b/net/can/af_can.h @@ -52,7 +52,7 @@ struct receiver { struct hlist_node list; canid_t can_id; canid_t mask; - unsigned long matches; + atomic_long_t matches; void (*func)(struct sk_buff *skb, void *data); void *data; char *ident; diff --git a/net/can/bcm.c b/net/can/bcm.c index 7eba8ae01a5b1b..ba65e6e8a923a3 100644 --- a/net/can/bcm.c +++ b/net/can/bcm.c @@ -1170,6 +1170,7 @@ static int bcm_rx_setup(struct bcm_msg_head *msg_head, struct msghdr *msg, if (!op) return -ENOMEM; + spin_lock_init(&op->bcm_tx_lock); op->can_id = msg_head->can_id; op->nframes = msg_head->nframes; op->cfsiz = CFSIZ(msg_head->flags); diff --git a/net/can/gw.c b/net/can/gw.c index 55eccb1c7620c0..79fa58cb232ed1 100644 --- a/net/can/gw.c +++ b/net/can/gw.c @@ -374,10 +374,10 @@ static void cgw_csum_crc8_rel(struct canfd_frame *cf, return; if (from <= to) { - for (i = crc8->from_idx; i <= crc8->to_idx; i++) + for (i = from; i <= to; i++) crc = crc8->crctab[crc ^ cf->data[i]]; } else { - for (i = crc8->from_idx; i >= crc8->to_idx; i--) + for (i = from; i >= to; i--) crc = crc8->crctab[crc ^ cf->data[i]]; } @@ -396,7 +396,7 @@ static void cgw_csum_crc8_rel(struct canfd_frame *cf, break; } - cf->data[crc8->result_idx] = crc ^ crc8->final_xor_val; + cf->data[res] = crc ^ crc8->final_xor_val; } static void cgw_csum_crc8_pos(struct canfd_frame *cf, diff --git a/net/can/isotp.c b/net/can/isotp.c index ce588b85665a05..3811c4d0957aec 100644 --- a/net/can/isotp.c +++ b/net/can/isotp.c @@ -1230,12 +1230,6 @@ static int isotp_release(struct socket *sock) so->ifindex = 0; so->bound = 0; - if (so->rx.buf != so->rx.sbuf) - kfree(so->rx.buf); - - if (so->tx.buf != so->tx.sbuf) - kfree(so->tx.buf); - sock_orphan(sk); sock->sk = NULL; @@ -1604,6 +1598,21 @@ static int isotp_notifier(struct notifier_block *nb, unsigned long msg, return NOTIFY_DONE; } +static void isotp_sock_destruct(struct sock *sk) +{ + struct isotp_sock *so = isotp_sk(sk); + + /* do the standard CAN sock destruct work */ + can_sock_destruct(sk); + + /* free potential extended PDU buffers */ + if (so->rx.buf != so->rx.sbuf) + kfree(so->rx.buf); + + if (so->tx.buf != so->tx.sbuf) + kfree(so->tx.buf); +} + static int isotp_init(struct sock *sk) { struct isotp_sock *so = isotp_sk(sk); @@ -1648,6 +1657,9 @@ static int isotp_init(struct sock *sk) list_add_tail(&so->notifier, &isotp_notifier_list); spin_unlock(&isotp_notifier_lock); + /* re-assign default can_sock_destruct() reference */ + sk->sk_destruct = isotp_sock_destruct; + return 0; } diff --git a/net/can/proc.c b/net/can/proc.c index 0938bf7dd646ac..de4d05ae345977 100644 --- a/net/can/proc.c +++ b/net/can/proc.c @@ -196,7 +196,8 @@ static void can_print_rcvlist(struct seq_file *m, struct hlist_head *rx_list, " %-5s %03x %08x %pK %pK %8ld %s\n"; seq_printf(m, fmt, DNAME(dev), r->can_id, r->mask, - r->func, r->data, r->matches, r->ident); + r->func, r->data, atomic_long_read(&r->matches), + r->ident); } } diff --git a/net/can/raw.c b/net/can/raw.c index 12293363413ce6..d7c557802cf4b4 100644 --- a/net/can/raw.c +++ b/net/can/raw.c @@ -360,6 +360,14 @@ static int raw_notifier(struct notifier_block *nb, unsigned long msg, return NOTIFY_DONE; } +static void raw_sock_destruct(struct sock *sk) +{ + struct raw_sock *ro = raw_sk(sk); + + free_percpu(ro->uniq); + can_sock_destruct(sk); +} + static int raw_init(struct sock *sk) { struct raw_sock *ro = raw_sk(sk); @@ -386,6 +394,8 @@ static int raw_init(struct sock *sk) if (unlikely(!ro->uniq)) return -ENOMEM; + sk->sk_destruct = raw_sock_destruct; + /* set notifier */ spin_lock(&raw_notifier_lock); list_add_tail(&ro->notifier, &raw_notifier_list); @@ -435,7 +445,6 @@ static int raw_release(struct socket *sock) ro->bound = 0; ro->dev = NULL; ro->count = 0; - free_percpu(ro->uniq); sock_orphan(sk); sock->sk = NULL; diff --git a/net/ceph/auth.c b/net/ceph/auth.c index d38c9eadbe2f18..0d75679c6a7ed4 100644 --- a/net/ceph/auth.c +++ b/net/ceph/auth.c @@ -205,9 +205,9 @@ int ceph_handle_auth_reply(struct ceph_auth_client *ac, s32 result; u64 global_id; void *payload, *payload_end; - int payload_len; + u32 payload_len; char *result_msg; - int result_msg_len; + u32 result_msg_len; int ret = -EINVAL; mutex_lock(&ac->mutex); @@ -217,10 +217,12 @@ int ceph_handle_auth_reply(struct ceph_auth_client *ac, result = ceph_decode_32(&p); global_id = ceph_decode_64(&p); payload_len = ceph_decode_32(&p); + ceph_decode_need(&p, end, payload_len, bad); payload = p; p += payload_len; ceph_decode_need(&p, end, sizeof(u32), bad); result_msg_len = ceph_decode_32(&p); + ceph_decode_need(&p, end, result_msg_len, bad); result_msg = p; p += result_msg_len; if (p != end) diff --git a/net/ceph/crypto.c b/net/ceph/crypto.c index 01b2ce1e8fc06e..5601732cf4faa8 100644 --- a/net/ceph/crypto.c +++ b/net/ceph/crypto.c @@ -37,9 +37,6 @@ static int set_secret(struct ceph_crypto_key *key, void *buf) return -ENOTSUPP; } - if (!key->len) - return -EINVAL; - key->key = kmemdup(buf, key->len, GFP_NOIO); if (!key->key) { ret = -ENOMEM; @@ -83,6 +80,11 @@ int ceph_crypto_key_decode(struct ceph_crypto_key *key, void **p, void *end) ceph_decode_copy(p, &key->created, sizeof(key->created)); key->len = ceph_decode_16(p); ceph_decode_need(p, end, key->len, bad); + if (key->len > CEPH_MAX_KEY_LEN) { + pr_err("secret too big %d\n", key->len); + return -EINVAL; + } + ret = set_secret(key, *p); memzero_explicit(*p, key->len); *p += key->len; diff --git a/net/ceph/crypto.h b/net/ceph/crypto.h index 23de29fc613cf1..a20bad6d1e964a 100644 --- a/net/ceph/crypto.h +++ b/net/ceph/crypto.h @@ -5,7 +5,7 @@ #include #include -#define CEPH_KEY_LEN 16 +#define CEPH_MAX_KEY_LEN 16 #define CEPH_MAX_CON_SECRET_LEN 64 /* diff --git a/net/ceph/messenger_v2.c b/net/ceph/messenger_v2.c index c9d50c0dcd33a6..4653330374e4e7 100644 --- a/net/ceph/messenger_v2.c +++ b/net/ceph/messenger_v2.c @@ -392,7 +392,7 @@ static int head_onwire_len(int ctrl_len, bool secure) int head_len; int rem_len; - BUG_ON(ctrl_len < 0 || ctrl_len > CEPH_MSG_MAX_CONTROL_LEN); + BUG_ON(ctrl_len < 1 || ctrl_len > CEPH_MSG_MAX_CONTROL_LEN); if (secure) { head_len = CEPH_PREAMBLE_SECURE_LEN; @@ -401,9 +401,7 @@ static int head_onwire_len(int ctrl_len, bool secure) head_len += padded_len(rem_len) + CEPH_GCM_TAG_LEN; } } else { - head_len = CEPH_PREAMBLE_PLAIN_LEN; - if (ctrl_len) - head_len += ctrl_len + CEPH_CRC_LEN; + head_len = CEPH_PREAMBLE_PLAIN_LEN + ctrl_len + CEPH_CRC_LEN; } return head_len; } @@ -528,11 +526,16 @@ static int decode_preamble(void *p, struct ceph_frame_desc *desc) desc->fd_aligns[i] = ceph_decode_16(&p); } - if (desc->fd_lens[0] < 0 || + /* + * This would fire for FRAME_TAG_WAIT (it has one empty + * segment), but we should never get it as client. + */ + if (desc->fd_lens[0] < 1 || desc->fd_lens[0] > CEPH_MSG_MAX_CONTROL_LEN) { pr_err("bad control segment length %d\n", desc->fd_lens[0]); return -EINVAL; } + if (desc->fd_lens[1] < 0 || desc->fd_lens[1] > CEPH_MSG_MAX_FRONT_LEN) { pr_err("bad front segment length %d\n", desc->fd_lens[1]); @@ -549,10 +552,6 @@ static int decode_preamble(void *p, struct ceph_frame_desc *desc) return -EINVAL; } - /* - * This would fire for FRAME_TAG_WAIT (it has one empty - * segment), but we should never get it as client. - */ if (!desc->fd_lens[desc->fd_seg_cnt - 1]) { pr_err("last segment empty, segment count %d\n", desc->fd_seg_cnt); @@ -2360,7 +2359,7 @@ static int process_auth_reply_more(struct ceph_connection *con, */ static int process_auth_done(struct ceph_connection *con, void *p, void *end) { - u8 session_key_buf[CEPH_KEY_LEN + 16]; + u8 session_key_buf[CEPH_MAX_KEY_LEN + 16]; u8 con_secret_buf[CEPH_MAX_CON_SECRET_LEN + 16]; u8 *session_key = PTR_ALIGN(&session_key_buf[0], 16); u8 *con_secret = PTR_ALIGN(&con_secret_buf[0], 16); @@ -2833,12 +2832,15 @@ static int process_message_header(struct ceph_connection *con, void *p, void *end) { struct ceph_frame_desc *desc = &con->v2.in_desc; - struct ceph_msg_header2 *hdr2 = p; + struct ceph_msg_header2 *hdr2; struct ceph_msg_header hdr; int skip; int ret; u64 seq; + ceph_decode_need(&p, end, sizeof(*hdr2), bad); + hdr2 = p; + /* verify seq# */ seq = le64_to_cpu(hdr2->seq); if ((s64)seq - (s64)con->in_seq < 1) { @@ -2869,6 +2871,10 @@ static int process_message_header(struct ceph_connection *con, WARN_ON(!con->in_msg); WARN_ON(con->in_msg->con != con); return 1; + +bad: + pr_err("failed to decode message header\n"); + return -EINVAL; } static int process_message(struct ceph_connection *con) @@ -2898,6 +2904,11 @@ static int __handle_control(struct ceph_connection *con, void *p) if (con->v2.in_desc.fd_tag != FRAME_TAG_MESSAGE) return process_control(con, p, end); + if (con->state != CEPH_CON_S_OPEN) { + con->error_msg = "protocol error, unexpected message"; + return -EINVAL; + } + ret = process_message_header(con, p, end); if (ret < 0) return ret; diff --git a/net/ceph/mon_client.c b/net/ceph/mon_client.c index fa8dd2a20f7d28..94a7a82ca47561 100644 --- a/net/ceph/mon_client.c +++ b/net/ceph/mon_client.c @@ -72,8 +72,8 @@ static struct ceph_monmap *ceph_monmap_decode(void **p, void *end, bool msgr2) struct ceph_monmap *monmap = NULL; struct ceph_fsid fsid; u32 struct_len; - int blob_len; - int num_mon; + u32 blob_len; + u32 num_mon; u8 struct_v; u32 epoch; int ret; @@ -112,7 +112,7 @@ static struct ceph_monmap *ceph_monmap_decode(void **p, void *end, bool msgr2) } ceph_decode_32_safe(p, end, num_mon, e_inval); - dout("%s fsid %pU epoch %u num_mon %d\n", __func__, &fsid, epoch, + dout("%s fsid %pU epoch %u num_mon %u\n", __func__, &fsid, epoch, num_mon); if (num_mon > CEPH_MAX_MON) goto e_inval; diff --git a/net/core/dev.c b/net/core/dev.c index ccef685023c299..384250c3a519ca 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -738,7 +738,7 @@ static struct net_device_path *dev_fwd_path(struct net_device_path_stack *stack) { int k = stack->num_paths++; - if (WARN_ON_ONCE(k >= NET_DEVICE_PATH_STACK_MAX)) + if (k >= NET_DEVICE_PATH_STACK_MAX) return NULL; return &stack->path[k]; @@ -3763,6 +3763,22 @@ static netdev_features_t dflt_features_check(struct sk_buff *skb, return vlan_features_check(skb, features); } +static bool skb_gso_has_extension_hdr(const struct sk_buff *skb) +{ + if (!skb->encapsulation) + return ((skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6 || + (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4 && + vlan_get_protocol(skb) == htons(ETH_P_IPV6))) && + skb_transport_header_was_set(skb) && + skb_network_header_len(skb) != sizeof(struct ipv6hdr)); + else + return (!skb_inner_network_header_was_set(skb) || + ((skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6 || + (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4 && + inner_ip_hdr(skb)->version == 6)) && + skb_inner_network_header_len(skb) != sizeof(struct ipv6hdr))); +} + static netdev_features_t gso_features_check(const struct sk_buff *skb, struct net_device *dev, netdev_features_t features) @@ -3799,22 +3815,23 @@ static netdev_features_t gso_features_check(const struct sk_buff *skb, * segmentation-offloads.rst). */ if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4) { - struct iphdr *iph = skb->encapsulation ? - inner_ip_hdr(skb) : ip_hdr(skb); + const struct iphdr *iph; + struct iphdr _iph; + int nhoff = skb->encapsulation ? + skb_inner_network_offset(skb) : + skb_network_offset(skb); + + iph = skb_header_pointer(skb, nhoff, sizeof(_iph), &_iph); - if (!(iph->frag_off & htons(IP_DF))) - features &= ~NETIF_F_TSO_MANGLEID; + if (!iph || !(iph->frag_off & htons(IP_DF))) + features &= ~dev->mangleid_features; } /* NETIF_F_IPV6_CSUM does not support IPv6 extension headers, * so neither does TSO that depends on it. */ if (features & NETIF_F_IPV6_CSUM && - (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6 || - (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4 && - vlan_get_protocol(skb) == htons(ETH_P_IPV6))) && - skb_transport_header_was_set(skb) && - skb_network_header_len(skb) != sizeof(struct ipv6hdr) && + skb_gso_has_extension_hdr(skb) && !ipv6_has_hopopt_jumbo(skb)) features &= ~(NETIF_F_IPV6_CSUM | NETIF_F_TSO6 | NETIF_F_GSO_UDP_L4); @@ -3983,7 +4000,7 @@ static struct sk_buff *validate_xmit_unreadable_skb(struct sk_buff *skb, if (shinfo->nr_frags > 0) { niov = netmem_to_net_iov(skb_frag_netmem(&shinfo->frags[0])); if (net_is_devmem_iov(niov) && - net_devmem_iov_binding(niov)->dev != dev) + READ_ONCE(net_devmem_iov_binding(niov)->dev) != dev) goto out_free; } @@ -4814,10 +4831,9 @@ int __dev_queue_xmit(struct sk_buff *skb, struct net_device *sb_dev) if (dev->flags & IFF_UP) { int cpu = smp_processor_id(); /* ok because BHs are off */ - /* Other cpus might concurrently change txq->xmit_lock_owner - * to -1 or to their cpu id, but not to our id. - */ - if (READ_ONCE(txq->xmit_lock_owner) != cpu) { + if (!netif_tx_owned(txq, cpu)) { + bool is_list = false; + if (dev_xmit_recursion()) goto recursion_alert; @@ -4828,17 +4844,28 @@ int __dev_queue_xmit(struct sk_buff *skb, struct net_device *sb_dev) HARD_TX_LOCK(dev, txq, cpu); if (!netif_xmit_stopped(txq)) { + is_list = !!skb->next; + dev_xmit_recursion_inc(); skb = dev_hard_start_xmit(skb, dev, txq, &rc); dev_xmit_recursion_dec(); - if (dev_xmit_complete(rc)) { - HARD_TX_UNLOCK(dev, txq); - goto out; - } + + /* GSO segments a single SKB into + * a list of frames. TCP expects error + * to mean none of the data was sent. + */ + if (is_list) + rc = NETDEV_TX_OK; } HARD_TX_UNLOCK(dev, txq); + if (!skb) /* xmit completed */ + goto out; + net_crit_ratelimited("Virtual device %s asks to queue packet!\n", dev->name); + /* NETDEV_TX_BUSY or queue was stopped */ + if (!is_list) + rc = -ENETDOWN; } else { /* Recursion is detected! It is possible, * unfortunately @@ -4846,10 +4873,10 @@ int __dev_queue_xmit(struct sk_buff *skb, struct net_device *sb_dev) recursion_alert: net_crit_ratelimited("Dead loop on virtual device %s, fix it urgently!\n", dev->name); + rc = -ENETDOWN; } } - rc = -ENETDOWN; rcu_read_unlock_bh(); dev_core_stats_tx_dropped_inc(dev); @@ -4988,8 +5015,7 @@ static bool rps_flow_is_active(struct rps_dev_flow *rflow, static struct rps_dev_flow * set_rps_cpu(struct net_device *dev, struct sk_buff *skb, - struct rps_dev_flow *rflow, u16 next_cpu, u32 hash, - u32 flow_id) + struct rps_dev_flow *rflow, u16 next_cpu, u32 hash) { if (next_cpu < nr_cpu_ids) { u32 head; @@ -5000,6 +5026,7 @@ set_rps_cpu(struct net_device *dev, struct sk_buff *skb, struct rps_dev_flow *tmp_rflow; unsigned int tmp_cpu; u16 rxq_index; + u32 flow_id; int rc; /* Should we steer this flow to a different hardware queue? */ @@ -5015,6 +5042,7 @@ set_rps_cpu(struct net_device *dev, struct sk_buff *skb, if (!flow_table) goto out; + flow_id = rfs_slot(hash, flow_table); tmp_rflow = &flow_table->flows[flow_id]; tmp_cpu = READ_ONCE(tmp_rflow->cpu); @@ -5062,7 +5090,6 @@ static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb, struct rps_dev_flow_table *flow_table; struct rps_map *map; int cpu = -1; - u32 flow_id; u32 tcpu; u32 hash; @@ -5109,8 +5136,7 @@ static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb, /* OK, now we know there is a match, * we can look at the local (per receive queue) flow table */ - flow_id = rfs_slot(hash, flow_table); - rflow = &flow_table->flows[flow_id]; + rflow = &flow_table->flows[rfs_slot(hash, flow_table)]; tcpu = rflow->cpu; /* @@ -5129,8 +5155,7 @@ static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb, ((int)(READ_ONCE(per_cpu(softnet_data, tcpu).input_queue_head) - rflow->last_qtail)) >= 0)) { tcpu = next_cpu; - rflow = set_rps_cpu(dev, skb, rflow, next_cpu, hash, - flow_id); + rflow = set_rps_cpu(dev, skb, rflow, next_cpu, hash); } if (tcpu < nr_cpu_ids && cpu_online(tcpu)) { @@ -7780,11 +7805,12 @@ static int napi_thread_wait(struct napi_struct *napi) return -1; } -static void napi_threaded_poll_loop(struct napi_struct *napi, bool busy_poll) +static void napi_threaded_poll_loop(struct napi_struct *napi, + unsigned long *busy_poll_last_qs) { + unsigned long last_qs = busy_poll_last_qs ? *busy_poll_last_qs : jiffies; struct bpf_net_context __bpf_net_ctx, *bpf_net_ctx; struct softnet_data *sd; - unsigned long last_qs = jiffies; for (;;) { bool repoll = false; @@ -7813,12 +7839,12 @@ static void napi_threaded_poll_loop(struct napi_struct *napi, bool busy_poll) /* When busy poll is enabled, the old packets are not flushed in * napi_complete_done. So flush them here. */ - if (busy_poll) + if (busy_poll_last_qs) gro_flush_normal(&napi->gro, HZ >= 1000); local_bh_enable(); /* Call cond_resched here to avoid watchdog warnings. */ - if (repoll || busy_poll) { + if (repoll || busy_poll_last_qs) { rcu_softirq_qs_periodic(last_qs); cond_resched(); } @@ -7826,11 +7852,15 @@ static void napi_threaded_poll_loop(struct napi_struct *napi, bool busy_poll) if (!repoll) break; } + + if (busy_poll_last_qs) + *busy_poll_last_qs = last_qs; } static int napi_threaded_poll(void *data) { struct napi_struct *napi = data; + unsigned long last_qs = jiffies; bool want_busy_poll; bool in_busy_poll; unsigned long val; @@ -7848,7 +7878,7 @@ static int napi_threaded_poll(void *data) assign_bit(NAPI_STATE_IN_BUSY_POLL, &napi->state, want_busy_poll); - napi_threaded_poll_loop(napi, want_busy_poll); + napi_threaded_poll_loop(napi, want_busy_poll ? &last_qs : NULL); } return 0; @@ -11386,6 +11416,9 @@ int register_netdevice(struct net_device *dev) if (dev->hw_enc_features & NETIF_F_TSO) dev->hw_enc_features |= NETIF_F_TSO_MANGLEID; + /* TSO_MANGLEID belongs in mangleid_features by definition */ + dev->mangleid_features |= NETIF_F_TSO_MANGLEID; + /* Make NETIF_F_HIGHDMA inheritable to VLAN devices. */ dev->vlan_features |= NETIF_F_HIGHDMA; @@ -13159,7 +13192,7 @@ static void run_backlog_napi(unsigned int cpu) { struct softnet_data *sd = per_cpu_ptr(&softnet_data, cpu); - napi_threaded_poll_loop(&sd->backlog, false); + napi_threaded_poll_loop(&sd->backlog, NULL); } static void backlog_napi_setup(unsigned int cpu) diff --git a/net/core/dev.h b/net/core/dev.h index da18536cbd3575..49173702e15e18 100644 --- a/net/core/dev.h +++ b/net/core/dev.h @@ -361,41 +361,6 @@ static inline void napi_assert_will_not_race(const struct napi_struct *napi) void kick_defer_list_purge(unsigned int cpu); -#define XMIT_RECURSION_LIMIT 8 - -#ifndef CONFIG_PREEMPT_RT -static inline bool dev_xmit_recursion(void) -{ - return unlikely(__this_cpu_read(softnet_data.xmit.recursion) > - XMIT_RECURSION_LIMIT); -} - -static inline void dev_xmit_recursion_inc(void) -{ - __this_cpu_inc(softnet_data.xmit.recursion); -} - -static inline void dev_xmit_recursion_dec(void) -{ - __this_cpu_dec(softnet_data.xmit.recursion); -} -#else -static inline bool dev_xmit_recursion(void) -{ - return unlikely(current->net_xmit.recursion > XMIT_RECURSION_LIMIT); -} - -static inline void dev_xmit_recursion_inc(void) -{ - current->net_xmit.recursion++; -} - -static inline void dev_xmit_recursion_dec(void) -{ - current->net_xmit.recursion--; -} -#endif - int dev_set_hwtstamp_phylib(struct net_device *dev, struct kernel_hwtstamp_config *cfg, struct netlink_ext_ack *extack); diff --git a/net/core/devmem.c b/net/core/devmem.c index ec4217d6c0b4fd..e9c5d750918002 100644 --- a/net/core/devmem.c +++ b/net/core/devmem.c @@ -387,7 +387,8 @@ struct net_devmem_dmabuf_binding *net_devmem_get_binding(struct sock *sk, * net_device. */ dst_dev = dst_dev_rcu(dst); - if (unlikely(!dst_dev) || unlikely(dst_dev != binding->dev)) { + if (unlikely(!dst_dev) || + unlikely(dst_dev != READ_ONCE(binding->dev))) { err = -ENODEV; goto out_unlock; } @@ -504,7 +505,8 @@ static void mp_dmabuf_devmem_uninstall(void *mp_priv, xa_erase(&binding->bound_rxqs, xa_idx); if (xa_empty(&binding->bound_rxqs)) { mutex_lock(&binding->lock); - binding->dev = NULL; + ASSERT_EXCLUSIVE_WRITER(binding->dev); + WRITE_ONCE(binding->dev, NULL); mutex_unlock(&binding->lock); } break; diff --git a/net/core/filter.c b/net/core/filter.c index 029e560e32ce3e..8bbf24c15413eb 100644 --- a/net/core/filter.c +++ b/net/core/filter.c @@ -4137,7 +4137,7 @@ static const struct bpf_func_proto bpf_xdp_store_bytes_proto = { .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_ANYTHING, - .arg3_type = ARG_PTR_TO_UNINIT_MEM, + .arg3_type = ARG_PTR_TO_MEM | MEM_RDONLY, .arg4_type = ARG_CONST_SIZE, }; @@ -4151,12 +4151,14 @@ static int bpf_xdp_frags_increase_tail(struct xdp_buff *xdp, int offset) struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(xdp); skb_frag_t *frag = &sinfo->frags[sinfo->nr_frags - 1]; struct xdp_rxq_info *rxq = xdp->rxq; - unsigned int tailroom; + int tailroom; if (!rxq->frag_size || rxq->frag_size > xdp->frame_sz) return -EOPNOTSUPP; - tailroom = rxq->frag_size - skb_frag_size(frag) - skb_frag_off(frag); + tailroom = rxq->frag_size - skb_frag_size(frag) - + skb_frag_off(frag) % rxq->frag_size; + WARN_ON_ONCE(tailroom < 0); if (unlikely(offset > tailroom)) return -EINVAL; @@ -6401,7 +6403,7 @@ static const struct bpf_func_proto bpf_xdp_fib_lookup_proto = { .gpl_only = true, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, - .arg2_type = ARG_PTR_TO_MEM, + .arg2_type = ARG_PTR_TO_MEM | MEM_WRITE, .arg3_type = ARG_CONST_SIZE, .arg4_type = ARG_ANYTHING, }; @@ -6456,7 +6458,7 @@ static const struct bpf_func_proto bpf_skb_fib_lookup_proto = { .gpl_only = true, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, - .arg2_type = ARG_PTR_TO_MEM, + .arg2_type = ARG_PTR_TO_MEM | MEM_WRITE, .arg3_type = ARG_CONST_SIZE, .arg4_type = ARG_ANYTHING, }; @@ -8010,9 +8012,9 @@ static const struct bpf_func_proto bpf_tcp_raw_gen_syncookie_ipv4_proto = { .gpl_only = true, /* __cookie_v4_init_sequence() is GPL */ .pkt_access = true, .ret_type = RET_INTEGER, - .arg1_type = ARG_PTR_TO_FIXED_SIZE_MEM, + .arg1_type = ARG_PTR_TO_FIXED_SIZE_MEM | MEM_RDONLY, .arg1_size = sizeof(struct iphdr), - .arg2_type = ARG_PTR_TO_MEM, + .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, .arg3_type = ARG_CONST_SIZE_OR_ZERO, }; @@ -8042,9 +8044,9 @@ static const struct bpf_func_proto bpf_tcp_raw_gen_syncookie_ipv6_proto = { .gpl_only = true, /* __cookie_v6_init_sequence() is GPL */ .pkt_access = true, .ret_type = RET_INTEGER, - .arg1_type = ARG_PTR_TO_FIXED_SIZE_MEM, + .arg1_type = ARG_PTR_TO_FIXED_SIZE_MEM | MEM_RDONLY, .arg1_size = sizeof(struct ipv6hdr), - .arg2_type = ARG_PTR_TO_MEM, + .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, .arg3_type = ARG_CONST_SIZE_OR_ZERO, }; @@ -8062,9 +8064,9 @@ static const struct bpf_func_proto bpf_tcp_raw_check_syncookie_ipv4_proto = { .gpl_only = true, /* __cookie_v4_check is GPL */ .pkt_access = true, .ret_type = RET_INTEGER, - .arg1_type = ARG_PTR_TO_FIXED_SIZE_MEM, + .arg1_type = ARG_PTR_TO_FIXED_SIZE_MEM | MEM_RDONLY, .arg1_size = sizeof(struct iphdr), - .arg2_type = ARG_PTR_TO_FIXED_SIZE_MEM, + .arg2_type = ARG_PTR_TO_FIXED_SIZE_MEM | MEM_RDONLY, .arg2_size = sizeof(struct tcphdr), }; @@ -8086,9 +8088,9 @@ static const struct bpf_func_proto bpf_tcp_raw_check_syncookie_ipv6_proto = { .gpl_only = true, /* __cookie_v6_check is GPL */ .pkt_access = true, .ret_type = RET_INTEGER, - .arg1_type = ARG_PTR_TO_FIXED_SIZE_MEM, + .arg1_type = ARG_PTR_TO_FIXED_SIZE_MEM | MEM_RDONLY, .arg1_size = sizeof(struct ipv6hdr), - .arg2_type = ARG_PTR_TO_FIXED_SIZE_MEM, + .arg2_type = ARG_PTR_TO_FIXED_SIZE_MEM | MEM_RDONLY, .arg2_size = sizeof(struct tcphdr), }; #endif /* CONFIG_SYN_COOKIES */ diff --git a/net/core/gro.c b/net/core/gro.c index 482fa7d7f5981a..ef61695fbdbb67 100644 --- a/net/core/gro.c +++ b/net/core/gro.c @@ -417,7 +417,7 @@ static void gro_pull_from_frag0(struct sk_buff *skb, int grow) { struct skb_shared_info *pinfo = skb_shinfo(skb); - BUG_ON(skb->end - skb->tail < grow); + DEBUG_NET_WARN_ON_ONCE(skb->end - skb->tail < grow); memcpy(skb_tail_pointer(skb), NAPI_GRO_CB(skb)->frag0, grow); diff --git a/net/core/neighbour.c b/net/core/neighbour.c index 96a3b1a93252a5..e4ee0c02fb4435 100644 --- a/net/core/neighbour.c +++ b/net/core/neighbour.c @@ -821,7 +821,8 @@ int pneigh_create(struct neigh_table *tbl, struct net *net, update: WRITE_ONCE(n->flags, flags); n->permanent = permanent; - WRITE_ONCE(n->protocol, protocol); + if (protocol) + WRITE_ONCE(n->protocol, protocol); out: mutex_unlock(&tbl->phash_lock); return err; diff --git a/net/core/netpoll.c b/net/core/netpoll.c index 09f72f10813cc6..5af14f14a36231 100644 --- a/net/core/netpoll.c +++ b/net/core/netpoll.c @@ -132,7 +132,7 @@ static int netif_local_xmit_active(struct net_device *dev) for (i = 0; i < dev->num_tx_queues; i++) { struct netdev_queue *txq = netdev_get_tx_queue(dev, i); - if (READ_ONCE(txq->xmit_lock_owner) == smp_processor_id()) + if (netif_tx_owned(txq, smp_processor_id())) return 1; } diff --git a/net/core/page_pool_user.c b/net/core/page_pool_user.c index c82a95beceff84..ee5060d8eec0eb 100644 --- a/net/core/page_pool_user.c +++ b/net/core/page_pool_user.c @@ -245,7 +245,7 @@ page_pool_nl_fill(struct sk_buff *rsp, const struct page_pool *pool, goto err_cancel; if (pool->user.detach_time && nla_put_uint(rsp, NETDEV_A_PAGE_POOL_DETACH_TIME, - pool->user.detach_time)) + ktime_divns(pool->user.detach_time, NSEC_PER_SEC))) goto err_cancel; if (pool->mp_ops && pool->mp_ops->nl_fill(pool->mp_priv, rsp, NULL)) @@ -337,7 +337,7 @@ int page_pool_list(struct page_pool *pool) void page_pool_detached(struct page_pool *pool) { mutex_lock(&page_pools_lock); - pool->user.detach_time = ktime_get_boottime_seconds(); + pool->user.detach_time = ktime_get_boottime(); netdev_nl_page_pool_event(pool, NETDEV_CMD_PAGE_POOL_CHANGE_NTF); mutex_unlock(&page_pools_lock); } diff --git a/net/core/rtnetlink.c b/net/core/rtnetlink.c index b1ed55141d8a7a..c2ada5107dff0b 100644 --- a/net/core/rtnetlink.c +++ b/net/core/rtnetlink.c @@ -629,6 +629,9 @@ int rtnl_link_register(struct rtnl_link_ops *ops) unlock: mutex_unlock(&link_ops_mutex); + if (err) + cleanup_srcu_struct(&ops->srcu); + return err; } EXPORT_SYMBOL_GPL(rtnl_link_register); @@ -707,11 +710,14 @@ static size_t rtnl_link_get_slave_info_data_size(const struct net_device *dev) goto out; ops = master_dev->rtnl_link_ops; - if (!ops || !ops->get_slave_size) + if (!ops) + goto out; + size += nla_total_size(strlen(ops->kind) + 1); /* IFLA_INFO_SLAVE_KIND */ + if (!ops->get_slave_size) goto out; /* IFLA_INFO_SLAVE_DATA + nested data */ - size = nla_total_size(sizeof(struct nlattr)) + - ops->get_slave_size(master_dev, dev); + size += nla_total_size(sizeof(struct nlattr)) + + ops->get_slave_size(master_dev, dev); out: rcu_read_unlock(); @@ -1267,6 +1273,21 @@ static size_t rtnl_dpll_pin_size(const struct net_device *dev) return size; } +static size_t rtnl_dev_parent_size(const struct net_device *dev) +{ + size_t size = 0; + + /* IFLA_PARENT_DEV_NAME */ + if (dev->dev.parent) + size += nla_total_size(strlen(dev_name(dev->dev.parent)) + 1); + + /* IFLA_PARENT_DEV_BUS_NAME */ + if (dev->dev.parent && dev->dev.parent->bus) + size += nla_total_size(strlen(dev->dev.parent->bus->name) + 1); + + return size; +} + static noinline size_t if_nlmsg_size(const struct net_device *dev, u32 ext_filter_mask) { @@ -1328,6 +1349,7 @@ static noinline size_t if_nlmsg_size(const struct net_device *dev, + nla_total_size(8) /* IFLA_MAX_PACING_OFFLOAD_HORIZON */ + nla_total_size(2) /* IFLA_HEADROOM */ + nla_total_size(2) /* IFLA_TAILROOM */ + + rtnl_dev_parent_size(dev) + 0; if (!(ext_filter_mask & RTEXT_FILTER_SKIP_STATS)) @@ -3872,28 +3894,42 @@ static int rtnl_newlink_create(struct sk_buff *skb, struct ifinfomsg *ifm, goto out; } -static struct net *rtnl_get_peer_net(const struct rtnl_link_ops *ops, +static struct net *rtnl_get_peer_net(struct sk_buff *skb, + const struct rtnl_link_ops *ops, struct nlattr *tbp[], struct nlattr *data[], struct netlink_ext_ack *extack) { - struct nlattr *tb[IFLA_MAX + 1]; + struct nlattr *tb[IFLA_MAX + 1], **attrs; + struct net *net; int err; - if (!data || !data[ops->peer_type]) - return rtnl_link_get_net_ifla(tbp); - - err = rtnl_nla_parse_ifinfomsg(tb, data[ops->peer_type], extack); - if (err < 0) - return ERR_PTR(err); - - if (ops->validate) { - err = ops->validate(tb, NULL, extack); + if (!data || !data[ops->peer_type]) { + attrs = tbp; + } else { + err = rtnl_nla_parse_ifinfomsg(tb, data[ops->peer_type], extack); if (err < 0) return ERR_PTR(err); + + if (ops->validate) { + err = ops->validate(tb, NULL, extack); + if (err < 0) + return ERR_PTR(err); + } + + attrs = tb; } - return rtnl_link_get_net_ifla(tb); + net = rtnl_link_get_net_ifla(attrs); + if (IS_ERR_OR_NULL(net)) + return net; + + if (!netlink_ns_capable(skb, net->user_ns, CAP_NET_ADMIN)) { + put_net(net); + return ERR_PTR(-EPERM); + } + + return net; } static int __rtnl_newlink(struct sk_buff *skb, struct nlmsghdr *nlh, @@ -4032,7 +4068,7 @@ static int rtnl_newlink(struct sk_buff *skb, struct nlmsghdr *nlh, } if (ops->peer_type) { - peer_net = rtnl_get_peer_net(ops, tb, data, extack); + peer_net = rtnl_get_peer_net(skb, ops, tb, data, extack); if (IS_ERR(peer_net)) { ret = PTR_ERR(peer_net); goto put_ops; diff --git a/net/core/secure_seq.c b/net/core/secure_seq.c index 9a39656804513d..6a6f2cda5aaef8 100644 --- a/net/core/secure_seq.c +++ b/net/core/secure_seq.c @@ -20,7 +20,6 @@ #include static siphash_aligned_key_t net_secret; -static siphash_aligned_key_t ts_secret; #define EPHEMERAL_PORT_SHUFFLE_PERIOD (10 * HZ) @@ -28,11 +27,6 @@ static __always_inline void net_secret_init(void) { net_get_random_once(&net_secret, sizeof(net_secret)); } - -static __always_inline void ts_secret_init(void) -{ - net_get_random_once(&ts_secret, sizeof(ts_secret)); -} #endif #ifdef CONFIG_INET @@ -53,28 +47,9 @@ static u32 seq_scale(u32 seq) #endif #if IS_ENABLED(CONFIG_IPV6) -u32 secure_tcpv6_ts_off(const struct net *net, - const __be32 *saddr, const __be32 *daddr) -{ - const struct { - struct in6_addr saddr; - struct in6_addr daddr; - } __aligned(SIPHASH_ALIGNMENT) combined = { - .saddr = *(struct in6_addr *)saddr, - .daddr = *(struct in6_addr *)daddr, - }; - - if (READ_ONCE(net->ipv4.sysctl_tcp_timestamps) != 1) - return 0; - - ts_secret_init(); - return siphash(&combined, offsetofend(typeof(combined), daddr), - &ts_secret); -} -EXPORT_IPV6_MOD(secure_tcpv6_ts_off); - -u32 secure_tcpv6_seq(const __be32 *saddr, const __be32 *daddr, - __be16 sport, __be16 dport) +union tcp_seq_and_ts_off +secure_tcpv6_seq_and_ts_off(const struct net *net, const __be32 *saddr, + const __be32 *daddr, __be16 sport, __be16 dport) { const struct { struct in6_addr saddr; @@ -87,14 +62,20 @@ u32 secure_tcpv6_seq(const __be32 *saddr, const __be32 *daddr, .sport = sport, .dport = dport }; - u32 hash; + union tcp_seq_and_ts_off st; net_secret_init(); - hash = siphash(&combined, offsetofend(typeof(combined), dport), - &net_secret); - return seq_scale(hash); + + st.hash64 = siphash(&combined, offsetofend(typeof(combined), dport), + &net_secret); + + if (READ_ONCE(net->ipv4.sysctl_tcp_timestamps) != 1) + st.ts_off = 0; + + st.seq = seq_scale(st.seq); + return st; } -EXPORT_SYMBOL(secure_tcpv6_seq); +EXPORT_SYMBOL(secure_tcpv6_seq_and_ts_off); u64 secure_ipv6_port_ephemeral(const __be32 *saddr, const __be32 *daddr, __be16 dport) @@ -118,33 +99,30 @@ EXPORT_SYMBOL(secure_ipv6_port_ephemeral); #endif #ifdef CONFIG_INET -u32 secure_tcp_ts_off(const struct net *net, __be32 saddr, __be32 daddr) -{ - if (READ_ONCE(net->ipv4.sysctl_tcp_timestamps) != 1) - return 0; - - ts_secret_init(); - return siphash_2u32((__force u32)saddr, (__force u32)daddr, - &ts_secret); -} - /* secure_tcp_seq_and_tsoff(a, b, 0, d) == secure_ipv4_port_ephemeral(a, b, d), * but fortunately, `sport' cannot be 0 in any circumstances. If this changes, * it would be easy enough to have the former function use siphash_4u32, passing * the arguments as separate u32. */ -u32 secure_tcp_seq(__be32 saddr, __be32 daddr, - __be16 sport, __be16 dport) +union tcp_seq_and_ts_off +secure_tcp_seq_and_ts_off(const struct net *net, __be32 saddr, __be32 daddr, + __be16 sport, __be16 dport) { - u32 hash; + u32 ports = (__force u32)sport << 16 | (__force u32)dport; + union tcp_seq_and_ts_off st; net_secret_init(); - hash = siphash_3u32((__force u32)saddr, (__force u32)daddr, - (__force u32)sport << 16 | (__force u32)dport, - &net_secret); - return seq_scale(hash); + + st.hash64 = siphash_3u32((__force u32)saddr, (__force u32)daddr, + ports, &net_secret); + + if (READ_ONCE(net->ipv4.sysctl_tcp_timestamps) != 1) + st.ts_off = 0; + + st.seq = seq_scale(st.seq); + return st; } -EXPORT_SYMBOL_GPL(secure_tcp_seq); +EXPORT_SYMBOL_GPL(secure_tcp_seq_and_ts_off); u64 secure_ipv4_port_ephemeral(__be32 saddr, __be32 daddr, __be16 dport) { diff --git a/net/core/skbuff.c b/net/core/skbuff.c index 61746c2b95f63e..a95dc0638836dd 100644 --- a/net/core/skbuff.c +++ b/net/core/skbuff.c @@ -1062,10 +1062,7 @@ static int skb_pp_frag_ref(struct sk_buff *skb) static void skb_kfree_head(void *head, unsigned int end_offset) { - if (end_offset == SKB_SMALL_HEAD_HEADROOM) - kmem_cache_free(net_hotdata.skb_small_head_cache, head); - else - kfree(head); + kfree(head); } static void skb_free_head(struct sk_buff *skb) @@ -5555,15 +5552,28 @@ static void __skb_complete_tx_timestamp(struct sk_buff *skb, static bool skb_may_tx_timestamp(struct sock *sk, bool tsonly) { - bool ret; + struct socket *sock; + struct file *file; + bool ret = false; if (likely(tsonly || READ_ONCE(sock_net(sk)->core.sysctl_tstamp_allow_data))) return true; - read_lock_bh(&sk->sk_callback_lock); - ret = sk->sk_socket && sk->sk_socket->file && - file_ns_capable(sk->sk_socket->file, &init_user_ns, CAP_NET_RAW); - read_unlock_bh(&sk->sk_callback_lock); + /* The sk pointer remains valid as long as the skb is. The sk_socket and + * file pointer may become NULL if the socket is closed. Both structures + * (including file->cred) are RCU freed which means they can be accessed + * within a RCU read section. + */ + rcu_read_lock(); + sock = READ_ONCE(sk->sk_socket); + if (!sock) + goto out; + file = READ_ONCE(sock->file); + if (!file) + goto out; + ret = file_ns_capable(file, &init_user_ns, CAP_NET_RAW); +out: + rcu_read_unlock(); return ret; } @@ -7231,10 +7241,15 @@ void skb_attempt_defer_free(struct sk_buff *skb) { struct skb_defer_node *sdn; unsigned long defer_count; - int cpu = skb->alloc_cpu; unsigned int defer_max; bool kick; + int cpu; + + /* zero copy notifications should not be delayed. */ + if (skb_zcopy(skb)) + goto nodefer; + cpu = skb->alloc_cpu; if (cpu == raw_smp_processor_id() || WARN_ON_ONCE(cpu >= nr_cpu_ids) || !cpu_online(cpu)) { diff --git a/net/core/skmsg.c b/net/core/skmsg.c index 2ac7731e1e0a74..35a6acbf9a5795 100644 --- a/net/core/skmsg.c +++ b/net/core/skmsg.c @@ -409,22 +409,26 @@ int sk_msg_memcopy_from_iter(struct sock *sk, struct iov_iter *from, } EXPORT_SYMBOL_GPL(sk_msg_memcopy_from_iter); -/* Receive sk_msg from psock->ingress_msg to @msg. */ -int sk_msg_recvmsg(struct sock *sk, struct sk_psock *psock, struct msghdr *msg, - int len, int flags) +int __sk_msg_recvmsg(struct sock *sk, struct sk_psock *psock, struct msghdr *msg, + int len, int flags, int *copied_from_self) { struct iov_iter *iter = &msg->msg_iter; int peek = flags & MSG_PEEK; struct sk_msg *msg_rx; int i, copied = 0; + bool from_self; msg_rx = sk_psock_peek_msg(psock); + if (copied_from_self) + *copied_from_self = 0; + while (copied != len) { struct scatterlist *sge; if (unlikely(!msg_rx)) break; + from_self = msg_rx->sk == sk; i = msg_rx->sg.start; do { struct page *page; @@ -443,6 +447,9 @@ int sk_msg_recvmsg(struct sock *sk, struct sk_psock *psock, struct msghdr *msg, } copied += copy; + if (from_self && copied_from_self) + *copied_from_self += copy; + if (likely(!peek)) { sge->offset += copy; sge->length -= copy; @@ -451,6 +458,7 @@ int sk_msg_recvmsg(struct sock *sk, struct sk_psock *psock, struct msghdr *msg, atomic_sub(copy, &sk->sk_rmem_alloc); } msg_rx->sg.size -= copy; + sk_psock_msg_len_add(psock, -copy); if (!sge->length) { sk_msg_iter_var_next(i); @@ -487,6 +495,13 @@ int sk_msg_recvmsg(struct sock *sk, struct sk_psock *psock, struct msghdr *msg, out: return copied; } + +/* Receive sk_msg from psock->ingress_msg to @msg. */ +int sk_msg_recvmsg(struct sock *sk, struct sk_psock *psock, struct msghdr *msg, + int len, int flags) +{ + return __sk_msg_recvmsg(sk, psock, msg, len, flags, NULL); +} EXPORT_SYMBOL_GPL(sk_msg_recvmsg); bool sk_msg_is_readable(struct sock *sk) @@ -616,6 +631,12 @@ static int sk_psock_skb_ingress_self(struct sk_psock *psock, struct sk_buff *skb if (unlikely(!msg)) return -EAGAIN; skb_set_owner_r(skb, sk); + + /* This is used in tcp_bpf_recvmsg_parser() to determine whether the + * data originates from the socket's own protocol stack. No need to + * refcount sk because msg's lifetime is bound to sk via the ingress_msg. + */ + msg->sk = sk; err = sk_psock_skb_ingress_enqueue(skb, off, len, psock, sk, msg, take_ref); if (err < 0) kfree(msg); @@ -801,9 +822,11 @@ static void __sk_psock_purge_ingress_msg(struct sk_psock *psock) list_del(&msg->list); if (!msg->skb) atomic_sub(msg->sg.size, &psock->sk->sk_rmem_alloc); + sk_psock_msg_len_add(psock, -msg->sg.size); sk_msg_free(psock->sk, msg); kfree(msg); } + WARN_ON_ONCE(psock->msg_tot_len); } static void __sk_psock_zap_ingress(struct sk_psock *psock) @@ -909,6 +932,7 @@ int sk_psock_msg_verdict(struct sock *sk, struct sk_psock *psock, sk_msg_compute_data_pointers(msg); msg->sk = sk; ret = bpf_prog_run_pin_on_cpu(prog, msg); + msg->sk = NULL; ret = sk_psock_map_verd(ret, msg->sk_redir); psock->apply_bytes = msg->apply_bytes; if (ret == __SK_REDIRECT) { @@ -1181,8 +1205,8 @@ void sk_psock_start_strp(struct sock *sk, struct sk_psock *psock) return; psock->saved_data_ready = sk->sk_data_ready; - sk->sk_data_ready = sk_psock_strp_data_ready; - sk->sk_write_space = sk_psock_write_space; + WRITE_ONCE(sk->sk_data_ready, sk_psock_strp_data_ready); + WRITE_ONCE(sk->sk_write_space, sk_psock_write_space); } void sk_psock_stop_strp(struct sock *sk, struct sk_psock *psock) @@ -1192,8 +1216,8 @@ void sk_psock_stop_strp(struct sock *sk, struct sk_psock *psock) if (!psock->saved_data_ready) return; - sk->sk_data_ready = psock->saved_data_ready; - psock->saved_data_ready = NULL; + WRITE_ONCE(sk->sk_data_ready, psock->saved_data_ready); + WRITE_ONCE(psock->saved_data_ready, NULL); strp_stop(&psock->strp); } @@ -1243,17 +1267,20 @@ static int sk_psock_verdict_recv(struct sock *sk, struct sk_buff *skb) static void sk_psock_verdict_data_ready(struct sock *sk) { - struct socket *sock = sk->sk_socket; - const struct proto_ops *ops; + const struct proto_ops *ops = NULL; + struct socket *sock; int copied; trace_sk_data_ready(sk); - if (unlikely(!sock)) - return; - ops = READ_ONCE(sock->ops); + rcu_read_lock(); + sock = READ_ONCE(sk->sk_socket); + if (likely(sock)) + ops = READ_ONCE(sock->ops); + rcu_read_unlock(); if (!ops || !ops->read_skb) return; + copied = ops->read_skb(sk, sk_psock_verdict_recv); if (copied >= 0) { struct sk_psock *psock; @@ -1272,8 +1299,8 @@ void sk_psock_start_verdict(struct sock *sk, struct sk_psock *psock) return; psock->saved_data_ready = sk->sk_data_ready; - sk->sk_data_ready = sk_psock_verdict_data_ready; - sk->sk_write_space = sk_psock_write_space; + WRITE_ONCE(sk->sk_data_ready, sk_psock_verdict_data_ready); + WRITE_ONCE(sk->sk_write_space, sk_psock_write_space); } void sk_psock_stop_verdict(struct sock *sk, struct sk_psock *psock) @@ -1284,6 +1311,6 @@ void sk_psock_stop_verdict(struct sock *sk, struct sk_psock *psock) if (!psock->saved_data_ready) return; - sk->sk_data_ready = psock->saved_data_ready; + WRITE_ONCE(sk->sk_data_ready, psock->saved_data_ready); psock->saved_data_ready = NULL; } diff --git a/net/devlink/health.c b/net/devlink/health.c index 136a67c36a20dd..0798c82096bdc8 100644 --- a/net/devlink/health.c +++ b/net/devlink/health.c @@ -1327,7 +1327,7 @@ void devlink_fmsg_dump_skb(struct devlink_fmsg *fmsg, const struct sk_buff *skb) if (sk) { devlink_fmsg_pair_nest_start(fmsg, "sk"); devlink_fmsg_obj_nest_start(fmsg); - devlink_fmsg_put(fmsg, "family", sk->sk_type); + devlink_fmsg_put(fmsg, "family", sk->sk_family); devlink_fmsg_put(fmsg, "type", sk->sk_type); devlink_fmsg_put(fmsg, "proto", sk->sk_protocol); devlink_fmsg_obj_nest_end(fmsg); diff --git a/net/ethernet/eth.c b/net/ethernet/eth.c index 13a63b48b7eeb8..d9faadbe9b6c86 100644 --- a/net/ethernet/eth.c +++ b/net/ethernet/eth.c @@ -193,14 +193,11 @@ __be16 eth_type_trans(struct sk_buff *skb, struct net_device *dev) } EXPORT_SYMBOL(eth_type_trans); -/** - * eth_header_parse - extract hardware address from packet - * @skb: packet to extract header from - * @haddr: destination buffer - */ -int eth_header_parse(const struct sk_buff *skb, unsigned char *haddr) +int eth_header_parse(const struct sk_buff *skb, const struct net_device *dev, + unsigned char *haddr) { const struct ethhdr *eth = eth_hdr(skb); + memcpy(haddr, eth->h_source, ETH_ALEN); return ETH_ALEN; } diff --git a/net/hsr/hsr_device.c b/net/hsr/hsr_device.c index d1bfc49b5f017b..fd2fea25eff0dd 100644 --- a/net/hsr/hsr_device.c +++ b/net/hsr/hsr_device.c @@ -532,8 +532,8 @@ static void hsr_change_rx_flags(struct net_device *dev, int change) static int hsr_ndo_vlan_rx_add_vid(struct net_device *dev, __be16 proto, u16 vid) { - bool is_slave_a_added = false; - bool is_slave_b_added = false; + struct net_device *slave_a_dev = NULL; + struct net_device *slave_b_dev = NULL; struct hsr_port *port; struct hsr_priv *hsr; int ret = 0; @@ -549,33 +549,35 @@ static int hsr_ndo_vlan_rx_add_vid(struct net_device *dev, switch (port->type) { case HSR_PT_SLAVE_A: if (ret) { - /* clean up Slave-B */ netdev_err(dev, "add vid failed for Slave-A\n"); - if (is_slave_b_added) - vlan_vid_del(port->dev, proto, vid); - return ret; + goto unwind; } - - is_slave_a_added = true; + slave_a_dev = port->dev; break; - case HSR_PT_SLAVE_B: if (ret) { - /* clean up Slave-A */ netdev_err(dev, "add vid failed for Slave-B\n"); - if (is_slave_a_added) - vlan_vid_del(port->dev, proto, vid); - return ret; + goto unwind; } - - is_slave_b_added = true; + slave_b_dev = port->dev; break; default: + if (ret) + goto unwind; break; } } return 0; + +unwind: + if (slave_a_dev) + vlan_vid_del(slave_a_dev, proto, vid); + + if (slave_b_dev) + vlan_vid_del(slave_b_dev, proto, vid); + + return ret; } static int hsr_ndo_vlan_rx_kill_vid(struct net_device *dev, diff --git a/net/ipv4/Kconfig b/net/ipv4/Kconfig index b71c22475c515f..df922f9f528916 100644 --- a/net/ipv4/Kconfig +++ b/net/ipv4/Kconfig @@ -748,6 +748,7 @@ config TCP_SIGPOOL config TCP_AO bool "TCP: Authentication Option (RFC5925)" select CRYPTO + select CRYPTO_LIB_UTILS select TCP_SIGPOOL depends on 64BIT && IPV6 != m # seq-number extension needs WRITE_ONCE(u64) help @@ -761,6 +762,7 @@ config TCP_AO config TCP_MD5SIG bool "TCP: MD5 Signature Option support (RFC2385)" select CRYPTO_LIB_MD5 + select CRYPTO_LIB_UTILS help RFC2385 specifies a method of giving MD5 protection to TCP sessions. Its main (only?) use is to protect BGP sessions between core routers diff --git a/net/ipv4/esp4.c b/net/ipv4/esp4.c index 2c922afadb8f6b..6dfc0bcdef6542 100644 --- a/net/ipv4/esp4.c +++ b/net/ipv4/esp4.c @@ -235,10 +235,13 @@ static void esp_output_done(void *data, int err) xfrm_dev_resume(skb); } else { if (!err && - x->encap && x->encap->encap_type == TCP_ENCAP_ESPINTCP) - esp_output_tail_tcp(x, skb); - else + x->encap && x->encap->encap_type == TCP_ENCAP_ESPINTCP) { + err = esp_output_tail_tcp(x, skb); + if (err != -EINPROGRESS) + kfree_skb(skb); + } else { xfrm_output_resume(skb_to_full_sk(skb), skb, err); + } } } diff --git a/net/ipv4/fib_lookup.h b/net/ipv4/fib_lookup.h index f9b9e26c32c193..0b72796dd1ad38 100644 --- a/net/ipv4/fib_lookup.h +++ b/net/ipv4/fib_lookup.h @@ -28,8 +28,10 @@ struct fib_alias { /* Don't write on fa_state unless needed, to keep it shared on all cpus */ static inline void fib_alias_accessed(struct fib_alias *fa) { - if (!(fa->fa_state & FA_S_ACCESSED)) - fa->fa_state |= FA_S_ACCESSED; + u8 fa_state = READ_ONCE(fa->fa_state); + + if (!(fa_state & FA_S_ACCESSED)) + WRITE_ONCE(fa->fa_state, fa_state | FA_S_ACCESSED); } /* Exported by fib_semantics.c */ diff --git a/net/ipv4/fib_trie.c b/net/ipv4/fib_trie.c index 7e2c17fec3fc46..1308213791f19d 100644 --- a/net/ipv4/fib_trie.c +++ b/net/ipv4/fib_trie.c @@ -1280,7 +1280,7 @@ int fib_table_insert(struct net *net, struct fib_table *tb, new_fa->fa_dscp = fa->fa_dscp; new_fa->fa_info = fi; new_fa->fa_type = cfg->fc_type; - state = fa->fa_state; + state = READ_ONCE(fa->fa_state); new_fa->fa_state = state & ~FA_S_ACCESSED; new_fa->fa_slen = fa->fa_slen; new_fa->tb_id = tb->tb_id; @@ -1745,7 +1745,7 @@ int fib_table_delete(struct net *net, struct fib_table *tb, fib_remove_alias(t, tp, l, fa_to_delete); - if (fa_to_delete->fa_state & FA_S_ACCESSED) + if (READ_ONCE(fa_to_delete->fa_state) & FA_S_ACCESSED) rt_cache_flush(cfg->fc_nlinfo.nl_net); fib_release_info(fa_to_delete->fa_info); diff --git a/net/ipv4/icmp.c b/net/ipv4/icmp.c index 4abbec2f47ef58..11bda6c9eaa44a 100644 --- a/net/ipv4/icmp.c +++ b/net/ipv4/icmp.c @@ -248,7 +248,8 @@ bool icmp_global_allow(struct net *net) if (delta < HZ / 50) return false; - incr = READ_ONCE(net->ipv4.sysctl_icmp_msgs_per_sec) * delta / HZ; + incr = READ_ONCE(net->ipv4.sysctl_icmp_msgs_per_sec); + incr = div_u64((u64)incr * delta, HZ); if (!incr) return false; @@ -554,6 +555,21 @@ static struct rtable *icmp_route_lookup(struct net *net, struct flowi4 *fl4, /* steal dst entry from skb_in, don't drop refcnt */ skb_dstref_steal(skb_in); skb_dstref_restore(skb_in, orefdst); + + /* + * At this point, fl4_dec.daddr should NOT be local (we + * checked fl4_dec.saddr above). However, a race condition + * may occur if the address is added to the interface + * concurrently. In that case, ip_route_input() returns a + * LOCAL route with dst.output=ip_rt_bug, which must not + * be used for output. + */ + if (!err && rt2 && rt2->rt_type == RTN_LOCAL) { + net_warn_ratelimited("detected local route for %pI4 during ICMP sending, src %pI4\n", + &fl4_dec.daddr, &fl4_dec.saddr); + dst_release(&rt2->dst); + err = -EINVAL; + } } if (err) @@ -1031,24 +1047,32 @@ static void icmp_socket_deliver(struct sk_buff *skb, u32 info) /* Checkin full IP header plus 8 bytes of protocol to * avoid additional coding at protocol handlers. */ - if (!pskb_may_pull(skb, iph->ihl * 4 + 8)) { - __ICMP_INC_STATS(dev_net_rcu(skb->dev), ICMP_MIB_INERRORS); - return; - } + if (!pskb_may_pull(skb, iph->ihl * 4 + 8)) + goto out; + + /* IPPROTO_RAW sockets are not supposed to receive anything. */ + if (protocol == IPPROTO_RAW) + goto out; raw_icmp_error(skb, protocol, info); ipprot = rcu_dereference(inet_protos[protocol]); if (ipprot && ipprot->err_handler) ipprot->err_handler(skb, info); + return; + +out: + __ICMP_INC_STATS(dev_net_rcu(skb->dev), ICMP_MIB_INERRORS); } static bool icmp_tag_validation(int proto) { + const struct net_protocol *ipprot; bool ok; rcu_read_lock(); - ok = rcu_dereference(inet_protos[proto])->icmp_strict_tag_validation; + ipprot = rcu_dereference(inet_protos[proto]); + ok = ipprot ? ipprot->icmp_strict_tag_validation : false; rcu_read_unlock(); return ok; } @@ -1309,6 +1333,13 @@ bool icmp_build_probe(struct sk_buff *skb, struct icmphdr *icmphdr) if (iio->ident.addr.ctype3_hdr.addrlen != sizeof(struct in6_addr)) goto send_mal_query; dev = ipv6_stub->ipv6_dev_find(net, &iio->ident.addr.ip_addr.ipv6_addr, dev); + /* + * If IPv6 identifier lookup is unavailable, silently + * discard the request instead of misreporting NO_IF. + */ + if (IS_ERR(dev)) + return false; + dev_hold(dev); break; #endif diff --git a/net/ipv4/igmp.c b/net/ipv4/igmp.c index 7182f1419c2a4d..0adc993c211d7a 100644 --- a/net/ipv4/igmp.c +++ b/net/ipv4/igmp.c @@ -227,7 +227,7 @@ static void igmp_start_timer(struct ip_mc_list *im, int max_delay) static void igmp_gq_start_timer(struct in_device *in_dev) { - int tv = get_random_u32_below(in_dev->mr_maxdelay); + int tv = get_random_u32_below(READ_ONCE(in_dev->mr_maxdelay)); unsigned long exp = jiffies + tv + 2; if (in_dev->mr_gq_running && @@ -1009,7 +1009,7 @@ static bool igmp_heard_query(struct in_device *in_dev, struct sk_buff *skb, max_delay = IGMPV3_MRC(ih3->code)*(HZ/IGMP_TIMER_SCALE); if (!max_delay) max_delay = 1; /* can't mod w/ 0 */ - in_dev->mr_maxdelay = max_delay; + WRITE_ONCE(in_dev->mr_maxdelay, max_delay); /* RFC3376, 4.1.6. QRV and 4.1.7. QQIC, when the most recently * received value was zero, use the default or statically diff --git a/net/ipv4/inet_connection_sock.c b/net/ipv4/inet_connection_sock.c index 97d57c52b9ad95..d587c5df843890 100644 --- a/net/ipv4/inet_connection_sock.c +++ b/net/ipv4/inet_connection_sock.c @@ -153,20 +153,6 @@ bool inet_sk_get_local_port_range(const struct sock *sk, int *low, int *high) } EXPORT_SYMBOL(inet_sk_get_local_port_range); -static bool inet_use_bhash2_on_bind(const struct sock *sk) -{ -#if IS_ENABLED(CONFIG_IPV6) - if (sk->sk_family == AF_INET6) { - if (ipv6_addr_any(&sk->sk_v6_rcv_saddr)) - return false; - - if (!ipv6_addr_v4mapped(&sk->sk_v6_rcv_saddr)) - return true; - } -#endif - return sk->sk_rcv_saddr != htonl(INADDR_ANY); -} - static bool inet_bind_conflict(const struct sock *sk, struct sock *sk2, kuid_t uid, bool relax, bool reuseport_cb_ok, bool reuseport_ok) @@ -258,7 +244,7 @@ static int inet_csk_bind_conflict(const struct sock *sk, * checks separately because their spinlocks have to be acquired/released * independently of each other, to prevent possible deadlocks */ - if (inet_use_bhash2_on_bind(sk)) + if (inet_use_hash2_on_bind(sk)) return tb2 && inet_bhash2_conflict(sk, tb2, uid, relax, reuseport_cb_ok, reuseport_ok); @@ -375,7 +361,7 @@ inet_csk_find_open_port(const struct sock *sk, struct inet_bind_bucket **tb_ret, head = &hinfo->bhash[inet_bhashfn(net, port, hinfo->bhash_size)]; spin_lock_bh(&head->lock); - if (inet_use_bhash2_on_bind(sk)) { + if (inet_use_hash2_on_bind(sk)) { if (inet_bhash2_addr_any_conflict(sk, port, l3mdev, relax, false)) goto next_port; } @@ -561,7 +547,7 @@ int inet_csk_get_port(struct sock *sk, unsigned short snum) check_bind_conflict = false; } - if (check_bind_conflict && inet_use_bhash2_on_bind(sk)) { + if (check_bind_conflict && inet_use_hash2_on_bind(sk)) { if (inet_bhash2_addr_any_conflict(sk, port, l3mdev, true, true)) goto fail_unlock; } diff --git a/net/ipv4/inet_hashtables.c b/net/ipv4/inet_hashtables.c index f5826ec4bcaa86..46817b4c141b61 100644 --- a/net/ipv4/inet_hashtables.c +++ b/net/ipv4/inet_hashtables.c @@ -200,7 +200,7 @@ static bool inet_bind2_bucket_addr_match(const struct inet_bind2_bucket *tb2, void inet_bind_hash(struct sock *sk, struct inet_bind_bucket *tb, struct inet_bind2_bucket *tb2, unsigned short port) { - inet_sk(sk)->inet_num = port; + WRITE_ONCE(inet_sk(sk)->inet_num, port); inet_csk(sk)->icsk_bind_hash = tb; inet_csk(sk)->icsk_bind2_hash = tb2; sk_add_bind_node(sk, &tb2->owners); @@ -224,7 +224,7 @@ static void __inet_put_port(struct sock *sk) spin_lock(&head->lock); tb = inet_csk(sk)->icsk_bind_hash; inet_csk(sk)->icsk_bind_hash = NULL; - inet_sk(sk)->inet_num = 0; + WRITE_ONCE(inet_sk(sk)->inet_num, 0); sk->sk_userlocks &= ~SOCK_CONNECT_BIND; spin_lock(&head2->lock); @@ -352,7 +352,7 @@ static inline int compute_score(struct sock *sk, const struct net *net, { int score = -1; - if (net_eq(sock_net(sk), net) && sk->sk_num == hnum && + if (net_eq(sock_net(sk), net) && READ_ONCE(sk->sk_num) == hnum && !ipv6_only_sock(sk)) { if (sk->sk_rcv_saddr != daddr) return -1; @@ -1206,7 +1206,7 @@ int __inet_hash_connect(struct inet_timewait_death_row *death_row, sk->sk_hash = 0; inet_sk(sk)->inet_sport = 0; - inet_sk(sk)->inet_num = 0; + WRITE_ONCE(inet_sk(sk)->inet_num, 0); if (tw) inet_twsk_bind_unhash(tw, hinfo); diff --git a/net/ipv4/ip_gre.c b/net/ipv4/ip_gre.c index e13244729ad8d5..35f0baa99d4092 100644 --- a/net/ipv4/ip_gre.c +++ b/net/ipv4/ip_gre.c @@ -919,7 +919,8 @@ static int ipgre_header(struct sk_buff *skb, struct net_device *dev, return -(t->hlen + sizeof(*iph)); } -static int ipgre_header_parse(const struct sk_buff *skb, unsigned char *haddr) +static int ipgre_header_parse(const struct sk_buff *skb, const struct net_device *dev, + unsigned char *haddr) { const struct iphdr *iph = (const struct iphdr *) skb_mac_header(skb); memcpy(haddr, &iph->saddr, 4); diff --git a/net/ipv4/ip_tunnel_core.c b/net/ipv4/ip_tunnel_core.c index 2e61ac1371289a..5683c328990f49 100644 --- a/net/ipv4/ip_tunnel_core.c +++ b/net/ipv4/ip_tunnel_core.c @@ -58,6 +58,19 @@ void iptunnel_xmit(struct sock *sk, struct rtable *rt, struct sk_buff *skb, struct iphdr *iph; int err; + if (unlikely(dev_recursion_level() > IP_TUNNEL_RECURSION_LIMIT)) { + if (dev) { + net_crit_ratelimited("Dead loop on virtual device %s, fix it urgently!\n", + dev->name); + DEV_STATS_INC(dev, tx_errors); + } + ip_rt_put(rt); + kfree_skb(skb); + return; + } + + dev_xmit_recursion_inc(); + skb_scrub_packet(skb, xnet); skb_clear_hash_if_not_l4(skb); @@ -88,6 +101,8 @@ void iptunnel_xmit(struct sock *sk, struct rtable *rt, struct sk_buff *skb, pkt_len = 0; iptunnel_xmit_stats(dev, pkt_len); } + + dev_xmit_recursion_dec(); } EXPORT_SYMBOL_GPL(iptunnel_xmit); diff --git a/net/ipv4/nexthop.c b/net/ipv4/nexthop.c index 7b9d70f9b31c7a..c958b8edfe5401 100644 --- a/net/ipv4/nexthop.c +++ b/net/ipv4/nexthop.c @@ -905,8 +905,7 @@ static int nla_put_nh_group(struct sk_buff *skb, struct nexthop *nh, goto nla_put_failure; if (op_flags & NHA_OP_FLAG_DUMP_STATS && - (nla_put_u32(skb, NHA_HW_STATS_ENABLE, nhg->hw_stats) || - nla_put_nh_group_stats(skb, nh, op_flags))) + nla_put_nh_group_stats(skb, nh, op_flags)) goto nla_put_failure; return 0; @@ -1007,16 +1006,32 @@ static size_t nh_nlmsg_size_grp_res(struct nh_group *nhg) nla_total_size_64bit(8);/* NHA_RES_GROUP_UNBALANCED_TIME */ } -static size_t nh_nlmsg_size_grp(struct nexthop *nh) +static size_t nh_nlmsg_size_grp(struct nexthop *nh, u32 op_flags) { struct nh_group *nhg = rtnl_dereference(nh->nh_grp); size_t sz = sizeof(struct nexthop_grp) * nhg->num_nh; size_t tot = nla_total_size(sz) + - nla_total_size(2); /* NHA_GROUP_TYPE */ + nla_total_size(2) + /* NHA_GROUP_TYPE */ + nla_total_size(0); /* NHA_FDB */ if (nhg->resilient) tot += nh_nlmsg_size_grp_res(nhg); + if (op_flags & NHA_OP_FLAG_DUMP_STATS) { + tot += nla_total_size(0) + /* NHA_GROUP_STATS */ + nla_total_size(4); /* NHA_HW_STATS_ENABLE */ + tot += nhg->num_nh * + (nla_total_size(0) + /* NHA_GROUP_STATS_ENTRY */ + nla_total_size(4) + /* NHA_GROUP_STATS_ENTRY_ID */ + nla_total_size_64bit(8)); /* NHA_GROUP_STATS_ENTRY_PACKETS */ + + if (op_flags & NHA_OP_FLAG_DUMP_HW_STATS) { + tot += nhg->num_nh * + nla_total_size_64bit(8); /* NHA_GROUP_STATS_ENTRY_PACKETS_HW */ + tot += nla_total_size(4); /* NHA_HW_STATS_USED */ + } + } + return tot; } @@ -1051,14 +1066,14 @@ static size_t nh_nlmsg_size_single(struct nexthop *nh) return sz; } -static size_t nh_nlmsg_size(struct nexthop *nh) +static size_t nh_nlmsg_size(struct nexthop *nh, u32 op_flags) { size_t sz = NLMSG_ALIGN(sizeof(struct nhmsg)); sz += nla_total_size(4); /* NHA_ID */ if (nh->is_group) - sz += nh_nlmsg_size_grp(nh) + + sz += nh_nlmsg_size_grp(nh, op_flags) + nla_total_size(4) + /* NHA_OP_FLAGS */ 0; else @@ -1074,7 +1089,7 @@ static void nexthop_notify(int event, struct nexthop *nh, struct nl_info *info) struct sk_buff *skb; int err = -ENOBUFS; - skb = nlmsg_new(nh_nlmsg_size(nh), gfp_any()); + skb = nlmsg_new(nh_nlmsg_size(nh, 0), gfp_any()); if (!skb) goto errout; @@ -2005,7 +2020,8 @@ static void nh_hthr_group_rebalance(struct nh_group *nhg) } static void remove_nh_grp_entry(struct net *net, struct nh_grp_entry *nhge, - struct nl_info *nlinfo) + struct nl_info *nlinfo, + struct list_head *deferred_free) { struct nh_grp_entry *nhges, *new_nhges; struct nexthop *nhp = nhge->nh_parent; @@ -2065,8 +2081,8 @@ static void remove_nh_grp_entry(struct net *net, struct nh_grp_entry *nhge, rcu_assign_pointer(nhp->nh_grp, newg); list_del(&nhge->nh_list); - free_percpu(nhge->stats); nexthop_put(nhge->nh); + list_add(&nhge->nh_list, deferred_free); /* Removal of a NH from a resilient group is notified through * bucket notifications. @@ -2086,6 +2102,7 @@ static void remove_nexthop_from_groups(struct net *net, struct nexthop *nh, struct nl_info *nlinfo) { struct nh_grp_entry *nhge, *tmp; + LIST_HEAD(deferred_free); /* If there is nothing to do, let's avoid the costly call to * synchronize_net() @@ -2094,10 +2111,16 @@ static void remove_nexthop_from_groups(struct net *net, struct nexthop *nh, return; list_for_each_entry_safe(nhge, tmp, &nh->grp_list, nh_list) - remove_nh_grp_entry(net, nhge, nlinfo); + remove_nh_grp_entry(net, nhge, nlinfo, &deferred_free); /* make sure all see the newly published array before releasing rtnl */ synchronize_net(); + + /* Now safe to free percpu stats — all RCU readers have finished */ + list_for_each_entry_safe(nhge, tmp, &deferred_free, nh_list) { + list_del(&nhge->nh_list); + free_percpu(nhge->stats); + } } static void remove_nexthop_group(struct nexthop *nh, struct nl_info *nlinfo) @@ -3372,15 +3395,15 @@ static int rtm_get_nexthop(struct sk_buff *in_skb, struct nlmsghdr *nlh, if (err) return err; - err = -ENOBUFS; - skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL); - if (!skb) - goto out; - err = -ENOENT; nh = nexthop_find_by_id(net, id); if (!nh) - goto errout_free; + goto out; + + err = -ENOBUFS; + skb = nlmsg_new(nh_nlmsg_size(nh, op_flags), GFP_KERNEL); + if (!skb) + goto out; err = nh_fill_node(skb, nh, RTM_NEWNEXTHOP, NETLINK_CB(in_skb).portid, nlh->nlmsg_seq, 0, op_flags); diff --git a/net/ipv4/ping.c b/net/ipv4/ping.c index cfbd563498e85e..0fec4e56456671 100644 --- a/net/ipv4/ping.c +++ b/net/ipv4/ping.c @@ -148,7 +148,7 @@ void ping_unhash(struct sock *sk) pr_debug("ping_unhash(isk=%p,isk->num=%u)\n", isk, isk->inet_num); spin_lock(&ping_table.lock); if (sk_del_node_init_rcu(sk)) { - isk->inet_num = 0; + WRITE_ONCE(isk->inet_num, 0); isk->inet_sport = 0; sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1); } @@ -181,31 +181,35 @@ static struct sock *ping_lookup(struct net *net, struct sk_buff *skb, u16 ident) } sk_for_each_rcu(sk, hslot) { + int bound_dev_if; + if (!net_eq(sock_net(sk), net)) continue; isk = inet_sk(sk); pr_debug("iterate\n"); - if (isk->inet_num != ident) + if (READ_ONCE(isk->inet_num) != ident) continue; + bound_dev_if = READ_ONCE(sk->sk_bound_dev_if); if (skb->protocol == htons(ETH_P_IP) && sk->sk_family == AF_INET) { + __be32 rcv_saddr = READ_ONCE(isk->inet_rcv_saddr); + pr_debug("found: %p: num=%d, daddr=%pI4, dif=%d\n", sk, - (int) isk->inet_num, &isk->inet_rcv_saddr, - sk->sk_bound_dev_if); + ident, &rcv_saddr, + bound_dev_if); - if (isk->inet_rcv_saddr && - isk->inet_rcv_saddr != ip_hdr(skb)->daddr) + if (rcv_saddr && rcv_saddr != ip_hdr(skb)->daddr) continue; #if IS_ENABLED(CONFIG_IPV6) } else if (skb->protocol == htons(ETH_P_IPV6) && sk->sk_family == AF_INET6) { pr_debug("found: %p: num=%d, daddr=%pI6c, dif=%d\n", sk, - (int) isk->inet_num, + ident, &sk->sk_v6_rcv_saddr, - sk->sk_bound_dev_if); + bound_dev_if); if (!ipv6_addr_any(&sk->sk_v6_rcv_saddr) && !ipv6_addr_equal(&sk->sk_v6_rcv_saddr, @@ -216,8 +220,8 @@ static struct sock *ping_lookup(struct net *net, struct sk_buff *skb, u16 ident) continue; } - if (sk->sk_bound_dev_if && sk->sk_bound_dev_if != dif && - sk->sk_bound_dev_if != sdif) + if (bound_dev_if && bound_dev_if != dif && + bound_dev_if != sdif) continue; goto exit; @@ -392,7 +396,9 @@ static void ping_set_saddr(struct sock *sk, struct sockaddr_unsized *saddr) if (saddr->sa_family == AF_INET) { struct inet_sock *isk = inet_sk(sk); struct sockaddr_in *addr = (struct sockaddr_in *) saddr; - isk->inet_rcv_saddr = isk->inet_saddr = addr->sin_addr.s_addr; + + isk->inet_saddr = addr->sin_addr.s_addr; + WRITE_ONCE(isk->inet_rcv_saddr, addr->sin_addr.s_addr); #if IS_ENABLED(CONFIG_IPV6) } else if (saddr->sa_family == AF_INET6) { struct sockaddr_in6 *addr = (struct sockaddr_in6 *) saddr; @@ -849,7 +855,8 @@ int ping_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, int flags, struct sk_buff *skb; int copied, err; - pr_debug("ping_recvmsg(sk=%p,sk->num=%u)\n", isk, isk->inet_num); + pr_debug("ping_recvmsg(sk=%p,sk->num=%u)\n", isk, + READ_ONCE(isk->inet_num)); err = -EOPNOTSUPP; if (flags & MSG_OOB) diff --git a/net/ipv4/syncookies.c b/net/ipv4/syncookies.c index 569befcf021ba5..fc3affd9c8014b 100644 --- a/net/ipv4/syncookies.c +++ b/net/ipv4/syncookies.c @@ -203,7 +203,7 @@ struct sock *tcp_get_cookie_sock(struct sock *sk, struct sk_buff *skb, bool own_req; child = icsk->icsk_af_ops->syn_recv_sock(sk, skb, req, dst, - NULL, &own_req); + NULL, &own_req, NULL); if (child) { refcount_set(&req->rsk_refcnt, 1); sock_rps_save_rxhash(child, skb); @@ -378,9 +378,14 @@ static struct request_sock *cookie_tcp_check(struct net *net, struct sock *sk, tcp_parse_options(net, skb, &tcp_opt, 0, NULL); if (tcp_opt.saw_tstamp && tcp_opt.rcv_tsecr) { - tsoff = secure_tcp_ts_off(net, - ip_hdr(skb)->daddr, - ip_hdr(skb)->saddr); + union tcp_seq_and_ts_off st; + + st = secure_tcp_seq_and_ts_off(net, + ip_hdr(skb)->daddr, + ip_hdr(skb)->saddr, + tcp_hdr(skb)->dest, + tcp_hdr(skb)->source); + tsoff = st.ts_off; tcp_opt.rcv_tsecr -= tsoff; } diff --git a/net/ipv4/sysctl_net_ipv4.c b/net/ipv4/sysctl_net_ipv4.c index a1a50a5c80dc11..a96875e32050af 100644 --- a/net/ipv4/sysctl_net_ipv4.c +++ b/net/ipv4/sysctl_net_ipv4.c @@ -486,7 +486,8 @@ static void proc_fib_multipath_hash_set_seed(struct net *net, u32 user_seed) proc_fib_multipath_hash_rand_seed), }; - WRITE_ONCE(net->ipv4.sysctl_fib_multipath_hash_seed, new); + WRITE_ONCE(net->ipv4.sysctl_fib_multipath_hash_seed.user_seed, new.user_seed); + WRITE_ONCE(net->ipv4.sysctl_fib_multipath_hash_seed.mp_seed, new.mp_seed); } static int proc_fib_multipath_hash_seed(const struct ctl_table *table, int write, @@ -500,7 +501,7 @@ static int proc_fib_multipath_hash_seed(const struct ctl_table *table, int write int ret; mphs = &net->ipv4.sysctl_fib_multipath_hash_seed; - user_seed = mphs->user_seed; + user_seed = READ_ONCE(mphs->user_seed); tmp = *table; tmp.data = &user_seed; diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c index d5319ebe24525e..7fbaccf4c0ad6d 100644 --- a/net/ipv4/tcp.c +++ b/net/ipv4/tcp.c @@ -244,6 +244,7 @@ #define pr_fmt(fmt) "TCP: " fmt #include +#include #include #include #include @@ -501,6 +502,9 @@ static void tcp_tx_timestamp(struct sock *sk, struct sockcm_cookie *sockc) struct sk_buff *skb = tcp_write_queue_tail(sk); u32 tsflags = sockc->tsflags; + if (unlikely(!skb)) + skb = skb_rb_last(&sk->tcp_rtx_queue); + if (tsflags && skb) { struct skb_shared_info *shinfo = skb_shinfo(skb); struct tcp_skb_cb *tcb = TCP_SKB_CB(skb); @@ -1394,7 +1398,7 @@ int tcp_sendmsg_locked(struct sock *sk, struct msghdr *msg, size_t size) err = sk_stream_error(sk, flags, err); /* make sure we wake any epoll edge trigger waiter */ if (unlikely(tcp_rtx_and_write_queues_empty(sk) && err == -EAGAIN)) { - sk->sk_write_space(sk); + READ_ONCE(sk->sk_write_space)(sk); tcp_chrono_stop(sk, TCP_CHRONO_SNDBUF_LIMITED); } if (binding) @@ -4128,7 +4132,7 @@ int do_tcp_setsockopt(struct sock *sk, int level, int optname, break; case TCP_NOTSENT_LOWAT: WRITE_ONCE(tp->notsent_lowat, val); - sk->sk_write_space(sk); + READ_ONCE(sk->sk_write_space)(sk); break; case TCP_INQ: if (val > 1 || val < 0) @@ -4909,7 +4913,7 @@ tcp_inbound_md5_hash(const struct sock *sk, const struct sk_buff *skb, tcp_v4_md5_hash_skb(newhash, key, NULL, skb); else tp->af_specific->calc_md5_hash(newhash, key, NULL, skb); - if (memcmp(hash_location, newhash, 16) != 0) { + if (crypto_memneq(hash_location, newhash, 16)) { NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPMD5FAILURE); trace_tcp_hash_md5_mismatch(sk, skb); return SKB_DROP_REASON_TCP_MD5FAILURE; diff --git a/net/ipv4/tcp_ao.c b/net/ipv4/tcp_ao.c index 34b8450829d0d6..849a69c1f497f5 100644 --- a/net/ipv4/tcp_ao.c +++ b/net/ipv4/tcp_ao.c @@ -10,6 +10,7 @@ #define pr_fmt(fmt) "TCP: " fmt #include +#include #include #include @@ -922,7 +923,7 @@ tcp_ao_verify_hash(const struct sock *sk, const struct sk_buff *skb, /* XXX: make it per-AF callback? */ tcp_ao_hash_skb(family, hash_buf, key, sk, skb, traffic_key, (phash - (u8 *)th), sne); - if (memcmp(phash, hash_buf, maclen)) { + if (crypto_memneq(phash, hash_buf, maclen)) { NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPAOBAD); atomic64_inc(&info->counters.pkt_bad); atomic64_inc(&key->pkt_bad); diff --git a/net/ipv4/tcp_bpf.c b/net/ipv4/tcp_bpf.c index a268e1595b22aa..d3d6a47af52701 100644 --- a/net/ipv4/tcp_bpf.c +++ b/net/ipv4/tcp_bpf.c @@ -10,6 +10,7 @@ #include #include +#include void tcp_eat_skb(struct sock *sk, struct sk_buff *skb) { @@ -226,6 +227,7 @@ static int tcp_bpf_recvmsg_parser(struct sock *sk, int peek = flags & MSG_PEEK; struct sk_psock *psock; struct tcp_sock *tcp; + int copied_from_self = 0; int copied = 0; u32 seq; @@ -262,7 +264,7 @@ static int tcp_bpf_recvmsg_parser(struct sock *sk, } msg_bytes_ready: - copied = sk_msg_recvmsg(sk, psock, msg, len, flags); + copied = __sk_msg_recvmsg(sk, psock, msg, len, flags, &copied_from_self); /* The typical case for EFAULT is the socket was gracefully * shutdown with a FIN pkt. So check here the other case is * some error on copy_page_to_iter which would be unexpected. @@ -277,7 +279,7 @@ static int tcp_bpf_recvmsg_parser(struct sock *sk, goto out; } } - seq += copied; + seq += copied_from_self; if (!copied) { long timeo; int data; @@ -331,6 +333,24 @@ static int tcp_bpf_recvmsg_parser(struct sock *sk, return copied; } +static int tcp_bpf_ioctl(struct sock *sk, int cmd, int *karg) +{ + bool slow; + + if (cmd != SIOCINQ) + return tcp_ioctl(sk, cmd, karg); + + /* works similar as tcp_ioctl */ + if (sk->sk_state == TCP_LISTEN) + return -EINVAL; + + slow = lock_sock_fast(sk); + *karg = sk_psock_msg_inq(sk); + unlock_sock_fast(sk, slow); + + return 0; +} + static int tcp_bpf_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, int flags, int *addr_len) { @@ -609,6 +629,7 @@ static void tcp_bpf_rebuild_protos(struct proto prot[TCP_BPF_NUM_CFGS], prot[TCP_BPF_BASE].close = sock_map_close; prot[TCP_BPF_BASE].recvmsg = tcp_bpf_recvmsg; prot[TCP_BPF_BASE].sock_is_readable = sk_msg_is_readable; + prot[TCP_BPF_BASE].ioctl = tcp_bpf_ioctl; prot[TCP_BPF_TX] = prot[TCP_BPF_BASE]; prot[TCP_BPF_TX].sendmsg = tcp_bpf_sendmsg; @@ -704,7 +725,7 @@ int tcp_bpf_update_proto(struct sock *sk, struct sk_psock *psock, bool restore) WRITE_ONCE(sk->sk_prot->unhash, psock->saved_unhash); tcp_update_ulp(sk, psock->sk_proto, psock->saved_write_space); } else { - sk->sk_write_space = psock->saved_write_space; + WRITE_ONCE(sk->sk_write_space, psock->saved_write_space); /* Pairs with lockless read in sk_clone_lock() */ sock_replace_proto(sk, psock->sk_proto); } diff --git a/net/ipv4/tcp_cong.c b/net/ipv4/tcp_cong.c index df758adbb445f2..e9f6c77e063163 100644 --- a/net/ipv4/tcp_cong.c +++ b/net/ipv4/tcp_cong.c @@ -16,6 +16,7 @@ #include #include #include +#include #include static DEFINE_SPINLOCK(tcp_cong_list_lock); @@ -227,7 +228,7 @@ void tcp_assign_congestion_control(struct sock *sk) memset(icsk->icsk_ca_priv, 0, sizeof(icsk->icsk_ca_priv)); if (ca->flags & TCP_CONG_NEEDS_ECN) - INET_ECN_xmit(sk); + INET_ECN_xmit_ect_1_negotiation(sk); else INET_ECN_dontxmit(sk); } @@ -257,7 +258,7 @@ static void tcp_reinit_congestion_control(struct sock *sk, memset(icsk->icsk_ca_priv, 0, sizeof(icsk->icsk_ca_priv)); if (ca->flags & TCP_CONG_NEEDS_ECN) - INET_ECN_xmit(sk); + INET_ECN_xmit_ect_1_negotiation(sk); else INET_ECN_dontxmit(sk); diff --git a/net/ipv4/tcp_diag.c b/net/ipv4/tcp_diag.c index d83efd91f461c8..7935702e394b26 100644 --- a/net/ipv4/tcp_diag.c +++ b/net/ipv4/tcp_diag.c @@ -509,7 +509,7 @@ static void tcp_diag_dump(struct sk_buff *skb, struct netlink_callback *cb, if (r->sdiag_family != AF_UNSPEC && sk->sk_family != r->sdiag_family) goto next_normal; - if (r->id.idiag_sport != htons(sk->sk_num) && + if (r->id.idiag_sport != htons(READ_ONCE(sk->sk_num)) && r->id.idiag_sport) goto next_normal; if (r->id.idiag_dport != sk->sk_dport && diff --git a/net/ipv4/tcp_fastopen.c b/net/ipv4/tcp_fastopen.c index 7d945a527daf09..444306af444ae0 100644 --- a/net/ipv4/tcp_fastopen.c +++ b/net/ipv4/tcp_fastopen.c @@ -247,7 +247,7 @@ static struct sock *tcp_fastopen_create_child(struct sock *sk, bool own_req; child = inet_csk(sk)->icsk_af_ops->syn_recv_sock(sk, skb, req, NULL, - NULL, &own_req); + NULL, &own_req, NULL); if (!child) return NULL; diff --git a/net/ipv4/tcp_input.c b/net/ipv4/tcp_input.c index 198f8a0d37be04..3e95b36fa2736a 100644 --- a/net/ipv4/tcp_input.c +++ b/net/ipv4/tcp_input.c @@ -4611,15 +4611,24 @@ static enum skb_drop_reason tcp_disordered_ack_check(const struct sock *sk, */ static enum skb_drop_reason tcp_sequence(const struct sock *sk, - u32 seq, u32 end_seq) + u32 seq, u32 end_seq, + const struct tcphdr *th) { const struct tcp_sock *tp = tcp_sk(sk); + u32 seq_limit; if (before(end_seq, tp->rcv_wup)) return SKB_DROP_REASON_TCP_OLD_SEQUENCE; - if (after(end_seq, tp->rcv_nxt + tcp_receive_window(tp))) { - if (after(seq, tp->rcv_nxt + tcp_receive_window(tp))) + seq_limit = tp->rcv_nxt + tcp_receive_window(tp); + if (unlikely(after(end_seq, seq_limit))) { + /* Some stacks are known to handle FIN incorrectly; allow the + * FIN to extend beyond the window and check it in detail later. + */ + if (!after(end_seq - th->fin, seq_limit)) + return SKB_NOT_DROPPED_YET; + + if (after(seq, seq_limit)) return SKB_DROP_REASON_TCP_INVALID_SEQUENCE; /* Only accept this packet if receive queue is empty. */ @@ -5107,25 +5116,11 @@ static void tcp_ofo_queue(struct sock *sk) static bool tcp_prune_ofo_queue(struct sock *sk, const struct sk_buff *in_skb); static int tcp_prune_queue(struct sock *sk, const struct sk_buff *in_skb); -/* Check if this incoming skb can be added to socket receive queues - * while satisfying sk->sk_rcvbuf limit. - * - * In theory we should use skb->truesize, but this can cause problems - * when applications use too small SO_RCVBUF values. - * When LRO / hw gro is used, the socket might have a high tp->scaling_ratio, - * allowing RWIN to be close to available space. - * Whenever the receive queue gets full, we can receive a small packet - * filling RWIN, but with a high skb->truesize, because most NIC use 4K page - * plus sk_buff metadata even when receiving less than 1500 bytes of payload. - * - * Note that we use skb->len to decide to accept or drop this packet, - * but sk->sk_rmem_alloc is the sum of all skb->truesize. - */ static bool tcp_can_ingest(const struct sock *sk, const struct sk_buff *skb) { unsigned int rmem = atomic_read(&sk->sk_rmem_alloc); - return rmem + skb->len <= sk->sk_rcvbuf; + return rmem <= sk->sk_rcvbuf; } static int tcp_try_rmem_schedule(struct sock *sk, const struct sk_buff *skb, @@ -5158,7 +5153,7 @@ static void tcp_data_queue_ofo(struct sock *sk, struct sk_buff *skb) if (unlikely(tcp_try_rmem_schedule(sk, skb, skb->truesize))) { NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPOFODROP); - sk->sk_data_ready(sk); + READ_ONCE(sk->sk_data_ready)(sk); tcp_drop_reason(sk, skb, SKB_DROP_REASON_PROTO_MEM); return; } @@ -5368,7 +5363,7 @@ int tcp_send_rcvq(struct sock *sk, struct msghdr *msg, size_t size) void tcp_data_ready(struct sock *sk) { if (tcp_epollin_ready(sk, sk->sk_rcvlowat) || sock_flag(sk, SOCK_DONE)) - sk->sk_data_ready(sk); + READ_ONCE(sk->sk_data_ready)(sk); } static void tcp_data_queue(struct sock *sk, struct sk_buff *skb) @@ -5424,7 +5419,7 @@ static void tcp_data_queue(struct sock *sk, struct sk_buff *skb) inet_csk(sk)->icsk_ack.pending |= (ICSK_ACK_NOMEM | ICSK_ACK_NOW); inet_csk_schedule_ack(sk); - sk->sk_data_ready(sk); + READ_ONCE(sk->sk_data_ready)(sk); if (skb_queue_len(&sk->sk_receive_queue) && skb->len) { reason = SKB_DROP_REASON_PROTO_MEM; @@ -5866,7 +5861,9 @@ static void tcp_new_space(struct sock *sk) tp->snd_cwnd_stamp = tcp_jiffies32; } - INDIRECT_CALL_1(sk->sk_write_space, sk_stream_write_space, sk); + INDIRECT_CALL_1(READ_ONCE(sk->sk_write_space), + sk_stream_write_space, + sk); } /* Caller made space either from: @@ -6082,7 +6079,7 @@ static void tcp_urg(struct sock *sk, struct sk_buff *skb, const struct tcphdr *t BUG(); WRITE_ONCE(tp->urg_data, TCP_URG_VALID | tmp); if (!sock_flag(sk, SOCK_DEAD)) - sk->sk_data_ready(sk); + READ_ONCE(sk->sk_data_ready)(sk); } } } @@ -6145,7 +6142,8 @@ static bool tcp_validate_incoming(struct sock *sk, struct sk_buff *skb, step1: /* Step 1: check sequence number */ - reason = tcp_sequence(sk, TCP_SKB_CB(skb)->seq, TCP_SKB_CB(skb)->end_seq); + reason = tcp_sequence(sk, TCP_SKB_CB(skb)->seq, + TCP_SKB_CB(skb)->end_seq, th); if (reason) { /* RFC793, page 37: "In all states except SYN-SENT, all reset * (RST) segments are validated by checking their SEQ-fields." @@ -6843,7 +6841,7 @@ static int tcp_rcv_synsent_state_process(struct sock *sk, struct sk_buff *skb, tp->snd_wl1 = TCP_SKB_CB(skb)->seq; tp->max_window = tp->snd_wnd; - tcp_ecn_rcv_syn(tp, th, skb); + tcp_ecn_rcv_syn(sk, th, skb); tcp_mtup_init(sk); tcp_sync_mss(sk, icsk->icsk_pmtu_cookie); @@ -7248,7 +7246,8 @@ static void tcp_ecn_create_request(struct request_sock *req, u32 ecn_ok_dst; if (tcp_accecn_syn_requested(th) && - READ_ONCE(net->ipv4.sysctl_tcp_ecn) >= 3) { + (READ_ONCE(net->ipv4.sysctl_tcp_ecn) >= 3 || + tcp_ca_needs_accecn(listen_sk))) { inet_rsk(req)->ecn_ok = 1; tcp_rsk(req)->accecn_ok = 1; tcp_rsk(req)->syn_ect_rcv = TCP_SKB_CB(skb)->ip_dsfield & @@ -7412,6 +7411,7 @@ int tcp_conn_request(struct request_sock_ops *rsk_ops, const struct tcp_sock *tp = tcp_sk(sk); struct net *net = sock_net(sk); struct sock *fastopen_sk = NULL; + union tcp_seq_and_ts_off st; struct request_sock *req; bool want_cookie = false; struct dst_entry *dst; @@ -7481,9 +7481,12 @@ int tcp_conn_request(struct request_sock_ops *rsk_ops, if (!dst) goto drop_and_free; + if (tmp_opt.tstamp_ok || (!want_cookie && !isn)) + st = af_ops->init_seq_and_ts_off(net, skb); + if (tmp_opt.tstamp_ok) { tcp_rsk(req)->req_usec_ts = dst_tcp_usec_ts(dst); - tcp_rsk(req)->ts_off = af_ops->init_ts_off(net, skb); + tcp_rsk(req)->ts_off = st.ts_off; } if (!want_cookie && !isn) { int max_syn_backlog = READ_ONCE(net->ipv4.sysctl_max_syn_backlog); @@ -7505,7 +7508,7 @@ int tcp_conn_request(struct request_sock_ops *rsk_ops, goto drop_and_release; } - isn = af_ops->init_seq(skb); + isn = st.seq; } tcp_ecn_create_request(req, skb, sk, dst); @@ -7546,7 +7549,7 @@ int tcp_conn_request(struct request_sock_ops *rsk_ops, sock_put(fastopen_sk); goto drop_and_free; } - sk->sk_data_ready(sk); + READ_ONCE(sk->sk_data_ready)(sk); bh_unlock_sock(fastopen_sk); sock_put(fastopen_sk); } else { diff --git a/net/ipv4/tcp_ipv4.c b/net/ipv4/tcp_ipv4.c index f8a9596e8f4d41..750e3ac90587f8 100644 --- a/net/ipv4/tcp_ipv4.c +++ b/net/ipv4/tcp_ipv4.c @@ -88,6 +88,7 @@ #include #include +#include #include @@ -104,17 +105,14 @@ static DEFINE_PER_CPU(struct sock_bh_locked, ipv4_tcp_sk) = { static DEFINE_MUTEX(tcp_exit_batch_mutex); -static u32 tcp_v4_init_seq(const struct sk_buff *skb) +static union tcp_seq_and_ts_off +tcp_v4_init_seq_and_ts_off(const struct net *net, const struct sk_buff *skb) { - return secure_tcp_seq(ip_hdr(skb)->daddr, - ip_hdr(skb)->saddr, - tcp_hdr(skb)->dest, - tcp_hdr(skb)->source); -} - -static u32 tcp_v4_init_ts_off(const struct net *net, const struct sk_buff *skb) -{ - return secure_tcp_ts_off(net, ip_hdr(skb)->daddr, ip_hdr(skb)->saddr); + return secure_tcp_seq_and_ts_off(net, + ip_hdr(skb)->daddr, + ip_hdr(skb)->saddr, + tcp_hdr(skb)->dest, + tcp_hdr(skb)->source); } int tcp_twsk_unique(struct sock *sk, struct sock *sktw, void *twp) @@ -326,15 +324,16 @@ int tcp_v4_connect(struct sock *sk, struct sockaddr_unsized *uaddr, int addr_len rt = NULL; if (likely(!tp->repair)) { + union tcp_seq_and_ts_off st; + + st = secure_tcp_seq_and_ts_off(net, + inet->inet_saddr, + inet->inet_daddr, + inet->inet_sport, + usin->sin_port); if (!tp->write_seq) - WRITE_ONCE(tp->write_seq, - secure_tcp_seq(inet->inet_saddr, - inet->inet_daddr, - inet->inet_sport, - usin->sin_port)); - WRITE_ONCE(tp->tsoffset, - secure_tcp_ts_off(net, inet->inet_saddr, - inet->inet_daddr)); + WRITE_ONCE(tp->write_seq, st.seq); + WRITE_ONCE(tp->tsoffset, st.ts_off); } atomic_set(&inet->inet_id, get_random_u16()); @@ -840,7 +839,7 @@ static void tcp_v4_send_reset(const struct sock *sk, struct sk_buff *skb, goto out; tcp_v4_md5_hash_skb(newhash, key, NULL, skb); - if (memcmp(md5_hash_location, newhash, 16) != 0) + if (crypto_memneq(md5_hash_location, newhash, 16)) goto out; } @@ -1677,8 +1676,7 @@ const struct tcp_request_sock_ops tcp_request_sock_ipv4_ops = { .cookie_init_seq = cookie_v4_init_sequence, #endif .route_req = tcp_v4_route_req, - .init_seq = tcp_v4_init_seq, - .init_ts_off = tcp_v4_init_ts_off, + .init_seq_and_ts_off = tcp_v4_init_seq_and_ts_off, .send_synack = tcp_v4_send_synack, }; @@ -1706,7 +1704,9 @@ struct sock *tcp_v4_syn_recv_sock(const struct sock *sk, struct sk_buff *skb, struct request_sock *req, struct dst_entry *dst, struct request_sock *req_unhash, - bool *own_req) + bool *own_req, + void (*opt_child_init)(struct sock *newsk, + const struct sock *sk)) { struct inet_request_sock *ireq; bool found_dup_sk = false; @@ -1758,6 +1758,10 @@ struct sock *tcp_v4_syn_recv_sock(const struct sock *sk, struct sk_buff *skb, } sk_setup_caps(newsk, dst); +#if IS_ENABLED(CONFIG_IPV6) + if (opt_child_init) + opt_child_init(newsk, sk); +#endif tcp_ca_openreq_child(newsk, dst); tcp_sync_mss(newsk, dst_mtu(dst)); diff --git a/net/ipv4/tcp_minisocks.c b/net/ipv4/tcp_minisocks.c index bd5462154f970b..12f69cc285577f 100644 --- a/net/ipv4/tcp_minisocks.c +++ b/net/ipv4/tcp_minisocks.c @@ -485,9 +485,10 @@ static void tcp_ecn_openreq_child(struct sock *sk, tp->accecn_opt_demand = 1; tcp_ecn_received_counters_payload(sk, skb); } else { - tcp_ecn_mode_set(tp, inet_rsk(req)->ecn_ok ? - TCP_ECN_MODE_RFC3168 : - TCP_ECN_DISABLED); + if (inet_rsk(req)->ecn_ok && !tcp_ca_no_fallback_rfc3168(sk)) + tcp_ecn_mode_set(tp, TCP_ECN_MODE_RFC3168); + else + tcp_ecn_mode_set(tp, TCP_ECN_DISABLED); } } @@ -908,7 +909,7 @@ struct sock *tcp_check_req(struct sock *sk, struct sk_buff *skb, * socket is created, wait for troubles. */ child = inet_csk(sk)->icsk_af_ops->syn_recv_sock(sk, skb, req, NULL, - req, &own_req); + req, &own_req, NULL); if (!child) goto listen_overflow; @@ -987,7 +988,7 @@ enum skb_drop_reason tcp_child_process(struct sock *parent, struct sock *child, reason = tcp_rcv_state_process(child, skb); /* Wakeup parent, send SIGIO */ if (state == TCP_SYN_RECV && child->sk_state != state) - parent->sk_data_ready(parent); + READ_ONCE(parent->sk_data_ready)(parent); } else { /* Alas, it is possible again, because we do lookup * in main socket hash table and lock on listening diff --git a/net/ipv4/udp.c b/net/ipv4/udp.c index ee63af0ef42ccb..bbb076c6042b2b 100644 --- a/net/ipv4/udp.c +++ b/net/ipv4/udp.c @@ -287,7 +287,7 @@ int udp_lib_get_port(struct sock *sk, unsigned short snum, } else { hslot = udp_hashslot(udptable, net, snum); spin_lock_bh(&hslot->lock); - if (hslot->count > 10) { + if (inet_use_hash2_on_bind(sk) && hslot->count > 10) { int exist; unsigned int slot2 = udp_sk(sk)->udp_portaddr_hash ^ snum; @@ -1786,7 +1786,7 @@ int __udp_enqueue_schedule_skb(struct sock *sk, struct sk_buff *skb) * using prepare_to_wait_exclusive(). */ while (nb) { - INDIRECT_CALL_1(sk->sk_data_ready, + INDIRECT_CALL_1(READ_ONCE(sk->sk_data_ready), sock_def_readable, sk); nb--; } @@ -2268,7 +2268,6 @@ void udp_lib_rehash(struct sock *sk, u16 newhash, u16 newhash4) udp_sk(sk)->udp_port_hash); hslot2 = udp_hashslot2(udptable, udp_sk(sk)->udp_portaddr_hash); nhslot2 = udp_hashslot2(udptable, newhash); - udp_sk(sk)->udp_portaddr_hash = newhash; if (hslot2 != nhslot2 || rcu_access_pointer(sk->sk_reuseport_cb)) { @@ -2302,19 +2301,25 @@ void udp_lib_rehash(struct sock *sk, u16 newhash, u16 newhash4) if (udp_hashed4(sk)) { spin_lock_bh(&hslot->lock); - udp_rehash4(udptable, sk, newhash4); - if (hslot2 != nhslot2) { - spin_lock(&hslot2->lock); - udp_hash4_dec(hslot2); - spin_unlock(&hslot2->lock); - - spin_lock(&nhslot2->lock); - udp_hash4_inc(nhslot2); - spin_unlock(&nhslot2->lock); + if (inet_rcv_saddr_any(sk)) { + udp_unhash4(udptable, sk); + } else { + udp_rehash4(udptable, sk, newhash4); + if (hslot2 != nhslot2) { + spin_lock(&hslot2->lock); + udp_hash4_dec(hslot2); + spin_unlock(&hslot2->lock); + + spin_lock(&nhslot2->lock); + udp_hash4_inc(nhslot2); + spin_unlock(&nhslot2->lock); + } } spin_unlock_bh(&hslot->lock); } + + udp_sk(sk)->udp_portaddr_hash = newhash; } } EXPORT_IPV6_MOD(udp_lib_rehash); diff --git a/net/ipv4/udp_bpf.c b/net/ipv4/udp_bpf.c index 0735d820e413f3..779a3a03762f1e 100644 --- a/net/ipv4/udp_bpf.c +++ b/net/ipv4/udp_bpf.c @@ -5,6 +5,7 @@ #include #include #include +#include #include "udp_impl.h" @@ -111,12 +112,26 @@ enum { static DEFINE_SPINLOCK(udpv6_prot_lock); static struct proto udp_bpf_prots[UDP_BPF_NUM_PROTS]; +static int udp_bpf_ioctl(struct sock *sk, int cmd, int *karg) +{ + if (cmd != SIOCINQ) + return udp_ioctl(sk, cmd, karg); + + /* Since we don't hold a lock, sk_receive_queue may contain data. + * BPF might only be processing this data at the moment. We only + * care about the data in the ingress_msg here. + */ + *karg = sk_msg_first_len(sk); + return 0; +} + static void udp_bpf_rebuild_protos(struct proto *prot, const struct proto *base) { - *prot = *base; - prot->close = sock_map_close; - prot->recvmsg = udp_bpf_recvmsg; - prot->sock_is_readable = sk_msg_is_readable; + *prot = *base; + prot->close = sock_map_close; + prot->recvmsg = udp_bpf_recvmsg; + prot->sock_is_readable = sk_msg_is_readable; + prot->ioctl = udp_bpf_ioctl; } static void udp_bpf_check_v6_needs_rebuild(struct proto *ops) @@ -143,7 +158,7 @@ int udp_bpf_update_proto(struct sock *sk, struct sk_psock *psock, bool restore) int family = sk->sk_family == AF_INET ? UDP_BPF_IPV4 : UDP_BPF_IPV6; if (restore) { - sk->sk_write_space = psock->saved_write_space; + WRITE_ONCE(sk->sk_write_space, psock->saved_write_space); sock_replace_proto(sk, psock->sk_proto); return 0; } diff --git a/net/ipv4/udplite.c b/net/ipv4/udplite.c index d3e621a11a1aa4..826e9e79eb19ca 100644 --- a/net/ipv4/udplite.c +++ b/net/ipv4/udplite.c @@ -20,10 +20,9 @@ EXPORT_SYMBOL(udplite_table); /* Designate sk as UDP-Lite socket */ static int udplite_sk_init(struct sock *sk) { - udp_init_sock(sk); pr_warn_once("UDP-Lite is deprecated and scheduled to be removed in 2025, " "please contact the netdev mailing list\n"); - return 0; + return udp_init_sock(sk); } static int udplite_rcv(struct sk_buff *skb) diff --git a/net/ipv4/xfrm4_input.c b/net/ipv4/xfrm4_input.c index f28cfd88eaf593..c2eac844bcdb4e 100644 --- a/net/ipv4/xfrm4_input.c +++ b/net/ipv4/xfrm4_input.c @@ -50,6 +50,7 @@ int xfrm4_transport_finish(struct sk_buff *skb, int async) { struct xfrm_offload *xo = xfrm_offload(skb); struct iphdr *iph = ip_hdr(skb); + struct net_device *dev = skb->dev; iph->protocol = XFRM_MODE_SKB_CB(skb)->protocol; @@ -73,8 +74,10 @@ int xfrm4_transport_finish(struct sk_buff *skb, int async) } NF_HOOK(NFPROTO_IPV4, NF_INET_PRE_ROUTING, - dev_net(skb->dev), NULL, skb, skb->dev, NULL, + dev_net(dev), NULL, skb, dev, NULL, xfrm4_rcv_encap_finish); + if (async) + dev_put(dev); return 0; } diff --git a/net/ipv6/addrconf.c b/net/ipv6/addrconf.c index 27ab9d7adc6495..272dd1a0acd0e5 100644 --- a/net/ipv6/addrconf.c +++ b/net/ipv6/addrconf.c @@ -2863,7 +2863,7 @@ void addrconf_prefix_rcv(struct net_device *dev, u8 *opt, int len, bool sllao) fib6_add_gc_list(rt); } else { fib6_clean_expires(rt); - fib6_remove_gc_list(rt); + fib6_may_remove_gc_list(net, rt); } spin_unlock_bh(&table->tb6_lock); @@ -3621,12 +3621,12 @@ static void addrconf_permanent_addr(struct net *net, struct net_device *dev) if ((ifp->flags & IFA_F_PERMANENT) && fixup_permanent_addr(net, idev, ifp) < 0) { write_unlock_bh(&idev->lock); - in6_ifa_hold(ifp); - ipv6_del_addr(ifp); - write_lock_bh(&idev->lock); net_info_ratelimited("%s: Failed to add prefix route for address %pI6c; dropping\n", idev->dev->name, &ifp->addr); + in6_ifa_hold(ifp); + ipv6_del_addr(ifp); + write_lock_bh(&idev->lock); } } @@ -4836,7 +4836,7 @@ static int modify_prefix_route(struct net *net, struct inet6_ifaddr *ifp, if (!(flags & RTF_EXPIRES)) { fib6_clean_expires(f6i); - fib6_remove_gc_list(f6i); + fib6_may_remove_gc_list(net, f6i); } else { fib6_set_expires(f6i, expires); fib6_add_gc_list(f6i); diff --git a/net/ipv6/af_inet6.c b/net/ipv6/af_inet6.c index b705751eb73c6b..56d453a598ec66 100644 --- a/net/ipv6/af_inet6.c +++ b/net/ipv6/af_inet6.c @@ -224,8 +224,8 @@ static int inet6_create(struct net *net, struct socket *sock, int protocol, inet6_set_bit(MC6_LOOP, sk); inet6_set_bit(MC6_ALL, sk); np->pmtudisc = IPV6_PMTUDISC_WANT; - inet6_assign_bit(REPFLOW, sk, net->ipv6.sysctl.flowlabel_reflect & - FLOWLABEL_REFLECT_ESTABLISHED); + inet6_assign_bit(REPFLOW, sk, READ_ONCE(net->ipv6.sysctl.flowlabel_reflect) & + FLOWLABEL_REFLECT_ESTABLISHED); sk->sk_ipv6only = net->ipv6.sysctl.bindv6only; sk->sk_txrehash = READ_ONCE(net->core.sysctl_txrehash); @@ -955,7 +955,7 @@ static int __net_init inet6_net_init(struct net *net) int err = 0; net->ipv6.sysctl.bindv6only = 0; - net->ipv6.sysctl.icmpv6_time = 1*HZ; + net->ipv6.sysctl.icmpv6_time = HZ / 10; net->ipv6.sysctl.icmpv6_echo_ignore_all = 0; net->ipv6.sysctl.icmpv6_echo_ignore_multicast = 0; net->ipv6.sysctl.icmpv6_echo_ignore_anycast = 0; diff --git a/net/ipv6/datagram.c b/net/ipv6/datagram.c index 83e03176819ce9..022069c7d6edc6 100644 --- a/net/ipv6/datagram.c +++ b/net/ipv6/datagram.c @@ -762,6 +762,7 @@ int ip6_datagram_send_ctl(struct net *net, struct sock *sk, { struct in6_pktinfo *src_info; struct cmsghdr *cmsg; + struct ipv6_rt_hdr *orthdr; struct ipv6_rt_hdr *rthdr; struct ipv6_opt_hdr *hdr; struct ipv6_txoptions *opt = ipc6->opt; @@ -923,9 +924,13 @@ int ip6_datagram_send_ctl(struct net *net, struct sock *sk, goto exit_f; } if (cmsg->cmsg_type == IPV6_DSTOPTS) { + if (opt->dst1opt) + opt->opt_flen -= ipv6_optlen(opt->dst1opt); opt->opt_flen += len; opt->dst1opt = hdr; } else { + if (opt->dst0opt) + opt->opt_nflen -= ipv6_optlen(opt->dst0opt); opt->opt_nflen += len; opt->dst0opt = hdr; } @@ -968,12 +973,17 @@ int ip6_datagram_send_ctl(struct net *net, struct sock *sk, goto exit_f; } + orthdr = opt->srcrt; + if (orthdr) + opt->opt_nflen -= ((orthdr->hdrlen + 1) << 3); opt->opt_nflen += len; opt->srcrt = rthdr; if (cmsg->cmsg_type == IPV6_2292RTHDR && opt->dst1opt) { int dsthdrlen = ((opt->dst1opt->hdrlen+1)<<3); + if (opt->dst0opt) + opt->opt_nflen -= ipv6_optlen(opt->dst0opt); opt->opt_nflen += dsthdrlen; opt->dst0opt = opt->dst1opt; opt->dst1opt = NULL; diff --git a/net/ipv6/esp6.c b/net/ipv6/esp6.c index e75da98f528387..9f75313734f8cd 100644 --- a/net/ipv6/esp6.c +++ b/net/ipv6/esp6.c @@ -271,10 +271,13 @@ static void esp_output_done(void *data, int err) xfrm_dev_resume(skb); } else { if (!err && - x->encap && x->encap->encap_type == TCP_ENCAP_ESPINTCP) - esp_output_tail_tcp(x, skb); - else + x->encap && x->encap->encap_type == TCP_ENCAP_ESPINTCP) { + err = esp_output_tail_tcp(x, skb); + if (err != -EINPROGRESS) + kfree_skb(skb); + } else { xfrm_output_resume(skb_to_full_sk(skb), skb, err); + } } } diff --git a/net/ipv6/exthdrs.c b/net/ipv6/exthdrs.c index a23eb8734e151d..1d509b6d16bbd2 100644 --- a/net/ipv6/exthdrs.c +++ b/net/ipv6/exthdrs.c @@ -314,7 +314,7 @@ static int ipv6_destopt_rcv(struct sk_buff *skb) } extlen = (skb_transport_header(skb)[1] + 1) << 3; - if (extlen > net->ipv6.sysctl.max_dst_opts_len) + if (extlen > READ_ONCE(net->ipv6.sysctl.max_dst_opts_len)) goto fail_and_free; opt->lastopt = opt->dst1 = skb_network_header_len(skb); @@ -322,7 +322,8 @@ static int ipv6_destopt_rcv(struct sk_buff *skb) dstbuf = opt->dst1; #endif - if (ip6_parse_tlv(false, skb, net->ipv6.sysctl.max_dst_opts_cnt)) { + if (ip6_parse_tlv(false, skb, + READ_ONCE(net->ipv6.sysctl.max_dst_opts_cnt))) { skb->transport_header += extlen; opt = IP6CB(skb); #if IS_ENABLED(CONFIG_IPV6_MIP6) @@ -378,6 +379,10 @@ static int ipv6_srh_rcv(struct sk_buff *skb) hdr = (struct ipv6_sr_hdr *)skb_transport_header(skb); idev = __in6_dev_get(skb->dev); + if (!idev) { + kfree_skb(skb); + return -1; + } accept_seg6 = min(READ_ONCE(net->ipv6.devconf_all->seg6_enabled), READ_ONCE(idev->cnf.seg6_enabled)); @@ -930,6 +935,11 @@ static bool ipv6_hop_ioam(struct sk_buff *skb, int optoff) if (hdr->opt_len < 2 + sizeof(*trace) + trace->remlen * 4) goto drop; + /* Inconsistent Pre-allocated Trace header */ + if (trace->nodelen != + ioam6_trace_compute_nodelen(be32_to_cpu(trace->type_be32))) + goto drop; + /* Ignore if the IOAM namespace is unknown */ ns = ioam6_namespace(dev_net(skb->dev), trace->namespace_id); if (!ns) @@ -1049,11 +1059,12 @@ int ipv6_parse_hopopts(struct sk_buff *skb) } extlen = (skb_transport_header(skb)[1] + 1) << 3; - if (extlen > net->ipv6.sysctl.max_hbh_opts_len) + if (extlen > READ_ONCE(net->ipv6.sysctl.max_hbh_opts_len)) goto fail_and_free; opt->flags |= IP6SKB_HOPBYHOP; - if (ip6_parse_tlv(true, skb, net->ipv6.sysctl.max_hbh_opts_cnt)) { + if (ip6_parse_tlv(true, skb, + READ_ONCE(net->ipv6.sysctl.max_hbh_opts_cnt))) { skb->transport_header += extlen; opt = IP6CB(skb); opt->nhoff = sizeof(struct ipv6hdr); diff --git a/net/ipv6/icmp.c b/net/ipv6/icmp.c index 9d37e7711bc2b8..98447582527180 100644 --- a/net/ipv6/icmp.c +++ b/net/ipv6/icmp.c @@ -217,14 +217,9 @@ static bool icmpv6_xrlim_allow(struct sock *sk, u8 type, } else if (dev && (dev->flags & IFF_LOOPBACK)) { res = true; } else { - struct rt6_info *rt = dst_rt6_info(dst); - int tmo = net->ipv6.sysctl.icmpv6_time; + int tmo = READ_ONCE(net->ipv6.sysctl.icmpv6_time); struct inet_peer *peer; - /* Give more bandwidth to wider prefixes. */ - if (rt->rt6i_dst.plen < 128) - tmo >>= ((128 - rt->rt6i_dst.plen)>>5); - peer = inet_getpeer_v6(net->ipv6.peers, &fl6->daddr); res = inet_peer_xrlim_allow(peer, tmo); } @@ -876,6 +871,9 @@ int ip6_err_gen_icmpv6_unreach(struct sk_buff *skb, int nhs, int type, if (!skb2) return 1; + /* Remove debris left by IPv4 stack. */ + memset(IP6CB(skb2), 0, sizeof(*IP6CB(skb2))); + skb_dst_drop(skb2); skb_pull(skb2, nhs); skb_reset_network_header(skb2); @@ -958,7 +956,8 @@ static enum skb_drop_reason icmpv6_echo_reply(struct sk_buff *skb) tmp_hdr.icmp6_type = type; memset(&fl6, 0, sizeof(fl6)); - if (net->ipv6.sysctl.flowlabel_reflect & FLOWLABEL_REFLECT_ICMPV6_ECHO_REPLIES) + if (READ_ONCE(net->ipv6.sysctl.flowlabel_reflect) & + FLOWLABEL_REFLECT_ICMPV6_ECHO_REPLIES) fl6.flowlabel = ip6_flowlabel(ipv6_hdr(skb)); fl6.flowi6_proto = IPPROTO_ICMPV6; @@ -1066,6 +1065,12 @@ enum skb_drop_reason icmpv6_notify(struct sk_buff *skb, u8 type, if (reason != SKB_NOT_DROPPED_YET) goto out; + if (nexthdr == IPPROTO_RAW) { + /* Add a more specific reason later ? */ + reason = SKB_DROP_REASON_NOT_SPECIFIED; + goto out; + } + /* BUGGG_FUTURE: we should try to parse exthdrs in this packet. Without this we will not able f.e. to make source routed pmtu discovery. diff --git a/net/ipv6/inet6_hashtables.c b/net/ipv6/inet6_hashtables.c index 5e1da088d8e119..182d38e6d6d8da 100644 --- a/net/ipv6/inet6_hashtables.c +++ b/net/ipv6/inet6_hashtables.c @@ -95,7 +95,8 @@ static inline int compute_score(struct sock *sk, const struct net *net, { int score = -1; - if (net_eq(sock_net(sk), net) && inet_sk(sk)->inet_num == hnum && + if (net_eq(sock_net(sk), net) && + READ_ONCE(inet_sk(sk)->inet_num) == hnum && sk->sk_family == PF_INET6) { if (!ipv6_addr_equal(&sk->sk_v6_rcv_saddr, daddr)) return -1; diff --git a/net/ipv6/ioam6.c b/net/ipv6/ioam6.c index 9553a320008134..b91de51ffa9ea3 100644 --- a/net/ipv6/ioam6.c +++ b/net/ipv6/ioam6.c @@ -690,13 +690,29 @@ struct ioam6_namespace *ioam6_namespace(struct net *net, __be16 id) return rhashtable_lookup_fast(&nsdata->namespaces, &id, rht_ns_params); } +#define IOAM6_MASK_SHORT_FIELDS 0xff1ffc00 +#define IOAM6_MASK_WIDE_FIELDS 0x00e00000 + +u8 ioam6_trace_compute_nodelen(u32 trace_type) +{ + u8 nodelen = hweight32(trace_type & IOAM6_MASK_SHORT_FIELDS) + * (sizeof(__be32) / 4); + + nodelen += hweight32(trace_type & IOAM6_MASK_WIDE_FIELDS) + * (sizeof(__be64) / 4); + + return nodelen; +} + static void __ioam6_fill_trace_data(struct sk_buff *skb, struct ioam6_namespace *ns, struct ioam6_trace_hdr *trace, struct ioam6_schema *sc, - u8 sclen, bool is_input) + unsigned int sclen, bool is_input) { - struct net_device *dev = skb_dst_dev(skb); + /* Note: skb_dst_dev_rcu() can't be NULL at this point. */ + struct net_device *dev = skb_dst_dev_rcu(skb); + struct inet6_dev *i_skb_dev, *idev; struct timespec64 ts; ktime_t tstamp; u64 raw64; @@ -707,13 +723,16 @@ static void __ioam6_fill_trace_data(struct sk_buff *skb, data = trace->data + trace->remlen * 4 - trace->nodelen * 4 - sclen * 4; + i_skb_dev = skb->dev ? __in6_dev_get(skb->dev) : NULL; + idev = __in6_dev_get(dev); + /* hop_lim and node_id */ if (trace->type.bit0) { byte = ipv6_hdr(skb)->hop_limit; if (is_input) byte--; - raw32 = dev_net(dev)->ipv6.sysctl.ioam6_id; + raw32 = READ_ONCE(dev_net(dev)->ipv6.sysctl.ioam6_id); *(__be32 *)data = cpu_to_be32((byte << 24) | raw32); data += sizeof(__be32); @@ -721,18 +740,18 @@ static void __ioam6_fill_trace_data(struct sk_buff *skb, /* ingress_if_id and egress_if_id */ if (trace->type.bit1) { - if (!skb->dev) + if (!i_skb_dev) raw16 = IOAM6_U16_UNAVAILABLE; else - raw16 = (__force u16)READ_ONCE(__in6_dev_get(skb->dev)->cnf.ioam6_id); + raw16 = (__force u16)READ_ONCE(i_skb_dev->cnf.ioam6_id); *(__be16 *)data = cpu_to_be16(raw16); data += sizeof(__be16); - if (dev->flags & IFF_LOOPBACK) + if ((dev->flags & IFF_LOOPBACK) || !idev) raw16 = IOAM6_U16_UNAVAILABLE; else - raw16 = (__force u16)READ_ONCE(__in6_dev_get(dev)->cnf.ioam6_id); + raw16 = (__force u16)READ_ONCE(idev->cnf.ioam6_id); *(__be16 *)data = cpu_to_be16(raw16); data += sizeof(__be16); @@ -784,12 +803,16 @@ static void __ioam6_fill_trace_data(struct sk_buff *skb, struct Qdisc *qdisc; __u32 qlen, backlog; - if (dev->flags & IFF_LOOPBACK) { + if (dev->flags & IFF_LOOPBACK || + skb_get_queue_mapping(skb) >= dev->num_tx_queues) { *(__be32 *)data = cpu_to_be32(IOAM6_U32_UNAVAILABLE); } else { queue = skb_get_tx_queue(dev, skb); qdisc = rcu_dereference(queue->qdisc); + + spin_lock_bh(qdisc_lock(qdisc)); qdisc_qstats_qlen_backlog(qdisc, &qlen, &backlog); + spin_unlock_bh(qdisc_lock(qdisc)); *(__be32 *)data = cpu_to_be32(backlog); } @@ -808,7 +831,7 @@ static void __ioam6_fill_trace_data(struct sk_buff *skb, if (is_input) byte--; - raw64 = dev_net(dev)->ipv6.sysctl.ioam6_id_wide; + raw64 = READ_ONCE(dev_net(dev)->ipv6.sysctl.ioam6_id_wide); *(__be64 *)data = cpu_to_be64(((u64)byte << 56) | raw64); data += sizeof(__be64); @@ -816,18 +839,18 @@ static void __ioam6_fill_trace_data(struct sk_buff *skb, /* ingress_if_id and egress_if_id (wide) */ if (trace->type.bit9) { - if (!skb->dev) + if (!i_skb_dev) raw32 = IOAM6_U32_UNAVAILABLE; else - raw32 = READ_ONCE(__in6_dev_get(skb->dev)->cnf.ioam6_id_wide); + raw32 = READ_ONCE(i_skb_dev->cnf.ioam6_id_wide); *(__be32 *)data = cpu_to_be32(raw32); data += sizeof(__be32); - if (dev->flags & IFF_LOOPBACK) + if ((dev->flags & IFF_LOOPBACK) || !idev) raw32 = IOAM6_U32_UNAVAILABLE; else - raw32 = READ_ONCE(__in6_dev_get(dev)->cnf.ioam6_id_wide); + raw32 = READ_ONCE(idev->cnf.ioam6_id_wide); *(__be32 *)data = cpu_to_be32(raw32); data += sizeof(__be32); @@ -925,7 +948,7 @@ void ioam6_fill_trace_data(struct sk_buff *skb, bool is_input) { struct ioam6_schema *sc; - u8 sclen = 0; + unsigned int sclen = 0; /* Skip if Overflow flag is set */ diff --git a/net/ipv6/ioam6_iptunnel.c b/net/ipv6/ioam6_iptunnel.c index 1fe7894f14dd95..b9f6d892a566c0 100644 --- a/net/ipv6/ioam6_iptunnel.c +++ b/net/ipv6/ioam6_iptunnel.c @@ -22,9 +22,6 @@ #include #include -#define IOAM6_MASK_SHORT_FIELDS 0xff100000 -#define IOAM6_MASK_WIDE_FIELDS 0xe00000 - struct ioam6_lwt_encap { struct ipv6_hopopt_hdr eh; u8 pad[2]; /* 2-octet padding for 4n-alignment */ @@ -93,13 +90,8 @@ static bool ioam6_validate_trace_hdr(struct ioam6_trace_hdr *trace) trace->type.bit21 | trace->type.bit23) return false; - trace->nodelen = 0; fields = be32_to_cpu(trace->type_be32); - - trace->nodelen += hweight32(fields & IOAM6_MASK_SHORT_FIELDS) - * (sizeof(__be32) / 4); - trace->nodelen += hweight32(fields & IOAM6_MASK_WIDE_FIELDS) - * (sizeof(__be64) / 4); + trace->nodelen = ioam6_trace_compute_nodelen(fields); return true; } diff --git a/net/ipv6/ip6_fib.c b/net/ipv6/ip6_fib.c index c6439e30e892af..ffa77335983334 100644 --- a/net/ipv6/ip6_fib.c +++ b/net/ipv6/ip6_fib.c @@ -1133,13 +1133,13 @@ static int fib6_add_rt2node(struct fib6_node *fn, struct fib6_info *rt, return -EEXIST; if (!(rt->fib6_flags & RTF_EXPIRES)) { fib6_clean_expires(iter); - fib6_remove_gc_list(iter); + fib6_may_remove_gc_list(info->nl_net, iter); } else { fib6_set_expires(iter, rt->expires); fib6_add_gc_list(iter); } if (!(rt->fib6_flags & (RTF_ADDRCONF | RTF_PREFIX_RT)) && - !iter->fib6_nh->fib_nh_gw_family) { + (iter->nh || !iter->fib6_nh->fib_nh_gw_family)) { iter->fib6_flags &= ~RTF_ADDRCONF; iter->fib6_flags &= ~RTF_PREFIX_RT; } @@ -2348,6 +2348,17 @@ static void fib6_flush_trees(struct net *net) /* * Garbage collection */ +void fib6_age_exceptions(struct fib6_info *rt, struct fib6_gc_args *gc_args, + unsigned long now) +{ + bool may_expire = rt->fib6_flags & RTF_EXPIRES && rt->expires; + int old_more = gc_args->more; + + rt6_age_exceptions(rt, gc_args, now); + + if (!may_expire && old_more == gc_args->more) + fib6_remove_gc_list(rt); +} static int fib6_age(struct fib6_info *rt, struct fib6_gc_args *gc_args) { @@ -2370,7 +2381,7 @@ static int fib6_age(struct fib6_info *rt, struct fib6_gc_args *gc_args) * Note, that clones are aged out * only if they are not in use now. */ - rt6_age_exceptions(rt, gc_args, now); + fib6_age_exceptions(rt, gc_args, now); return 0; } diff --git a/net/ipv6/ip6_flowlabel.c b/net/ipv6/ip6_flowlabel.c index 60d0be47a9f318..8aa29b3d3daca5 100644 --- a/net/ipv6/ip6_flowlabel.c +++ b/net/ipv6/ip6_flowlabel.c @@ -133,11 +133,6 @@ static void fl_release(struct ip6_flowlabel *fl) if (time_after(ttd, fl->expires)) fl->expires = ttd; ttd = fl->expires; - if (fl->opt && fl->share == IPV6_FL_S_EXCL) { - struct ipv6_txoptions *opt = fl->opt; - fl->opt = NULL; - kfree(opt); - } if (!timer_pending(&ip6_fl_gc_timer) || time_after(ip6_fl_gc_timer.expires, ttd)) mod_timer(&ip6_fl_gc_timer, ttd); diff --git a/net/ipv6/ip6_tunnel.c b/net/ipv6/ip6_tunnel.c index c1f39735a23676..9e2449db0bdf2b 100644 --- a/net/ipv6/ip6_tunnel.c +++ b/net/ipv6/ip6_tunnel.c @@ -601,11 +601,16 @@ ip4ip6_err(struct sk_buff *skb, struct inet6_skb_parm *opt, if (!skb2) return 0; + /* Remove debris left by IPv6 stack. */ + memset(IPCB(skb2), 0, sizeof(*IPCB(skb2))); + skb_dst_drop(skb2); skb_pull(skb2, offset); skb_reset_network_header(skb2); eiph = ip_hdr(skb2); + if (eiph->version != 4 || eiph->ihl < 5) + goto out; /* Try to guess incoming interface */ rt = ip_route_output_ports(dev_net(skb->dev), &fl4, NULL, eiph->saddr, diff --git a/net/ipv6/ndisc.c b/net/ipv6/ndisc.c index f6a5d8c73af972..186e60c792145c 100644 --- a/net/ipv6/ndisc.c +++ b/net/ipv6/ndisc.c @@ -1209,6 +1209,9 @@ static void ndisc_ra_useropt(struct sk_buff *ra, struct nd_opt_hdr *opt) ndmsg->nduseropt_icmp_type = icmp6h->icmp6_type; ndmsg->nduseropt_icmp_code = icmp6h->icmp6_code; ndmsg->nduseropt_opts_len = opt->nd_opt_len << 3; + ndmsg->nduseropt_pad1 = 0; + ndmsg->nduseropt_pad2 = 0; + ndmsg->nduseropt_pad3 = 0; memcpy(ndmsg + 1, opt, opt->nd_opt_len << 3); diff --git a/net/ipv6/netfilter/ip6t_eui64.c b/net/ipv6/netfilter/ip6t_eui64.c index d704f7ed300c24..da69a27e8332c1 100644 --- a/net/ipv6/netfilter/ip6t_eui64.c +++ b/net/ipv6/netfilter/ip6t_eui64.c @@ -22,8 +22,7 @@ eui64_mt6(const struct sk_buff *skb, struct xt_action_param *par) unsigned char eui64[8]; if (!(skb_mac_header(skb) >= skb->head && - skb_mac_header(skb) + ETH_HLEN <= skb->data) && - par->fragoff != 0) { + skb_mac_header(skb) + ETH_HLEN <= skb->data)) { par->hotdrop = true; return false; } diff --git a/net/ipv6/netfilter/ip6t_rt.c b/net/ipv6/netfilter/ip6t_rt.c index 4ad8b2032f1f92..5561bd9cea8185 100644 --- a/net/ipv6/netfilter/ip6t_rt.c +++ b/net/ipv6/netfilter/ip6t_rt.c @@ -157,6 +157,10 @@ static int rt_mt6_check(const struct xt_mtchk_param *par) pr_debug("unknown flags %X\n", rtinfo->invflags); return -EINVAL; } + if (rtinfo->addrnr > IP6T_RT_HOPS) { + pr_debug("too many addresses specified\n"); + return -EINVAL; + } if ((rtinfo->flags & (IP6T_RT_RES | IP6T_RT_FST_MASK)) && (!(rtinfo->flags & IP6T_RT_TYP) || (rtinfo->rt_type != 0) || diff --git a/net/ipv6/route.c b/net/ipv6/route.c index e3a260a5564ba8..446f4de7d6a227 100644 --- a/net/ipv6/route.c +++ b/net/ipv6/route.c @@ -1033,7 +1033,7 @@ int rt6_route_rcv(struct net_device *dev, u8 *opt, int len, if (!addrconf_finite_timeout(lifetime)) { fib6_clean_expires(rt); - fib6_remove_gc_list(rt); + fib6_may_remove_gc_list(net, rt); } else { fib6_set_expires(rt, jiffies + HZ * lifetime); fib6_add_gc_list(rt); @@ -1063,7 +1063,8 @@ static struct net_device *ip6_rt_get_dev_rcu(const struct fib6_result *res) */ if (netif_is_l3_slave(dev) && !rt6_need_strict(&res->f6i->fib6_dst.addr)) - dev = l3mdev_master_dev_rcu(dev); + dev = l3mdev_master_dev_rcu(dev) ? : + dev_net(dev)->loopback_dev; else if (!netif_is_l3_master(dev)) dev = dev_net(dev)->loopback_dev; /* last case is netif_is_l3_master(dev) is true in which @@ -2895,7 +2896,7 @@ static void rt6_do_update_pmtu(struct rt6_info *rt, u32 mtu) dst_metric_set(&rt->dst, RTAX_MTU, mtu); rt->rt6i_flags |= RTF_MODIFIED; - rt6_update_expires(rt, net->ipv6.sysctl.ip6_rt_mtu_expires); + rt6_update_expires(rt, READ_ONCE(net->ipv6.sysctl.ip6_rt_mtu_expires)); } static bool rt6_cache_allowed_for_pmtu(const struct rt6_info *rt) @@ -3256,8 +3257,8 @@ static unsigned int ip6_default_advmss(const struct dst_entry *dst) rcu_read_lock(); net = dst_dev_net_rcu(dst); - if (mtu < net->ipv6.sysctl.ip6_rt_min_advmss) - mtu = net->ipv6.sysctl.ip6_rt_min_advmss; + mtu = max_t(unsigned int, mtu, + READ_ONCE(net->ipv6.sysctl.ip6_rt_min_advmss)); rcu_read_unlock(); @@ -3359,10 +3360,10 @@ struct dst_entry *icmp6_dst_alloc(struct net_device *dev, static void ip6_dst_gc(struct dst_ops *ops) { struct net *net = container_of(ops, struct net, ipv6.ip6_dst_ops); - int rt_min_interval = net->ipv6.sysctl.ip6_rt_gc_min_interval; - int rt_elasticity = net->ipv6.sysctl.ip6_rt_gc_elasticity; - int rt_gc_timeout = net->ipv6.sysctl.ip6_rt_gc_timeout; - unsigned long rt_last_gc = net->ipv6.ip6_rt_last_gc; + int rt_min_interval = READ_ONCE(net->ipv6.sysctl.ip6_rt_gc_min_interval); + int rt_elasticity = READ_ONCE(net->ipv6.sysctl.ip6_rt_gc_elasticity); + int rt_gc_timeout = READ_ONCE(net->ipv6.sysctl.ip6_rt_gc_timeout); + unsigned long rt_last_gc = READ_ONCE(net->ipv6.ip6_rt_last_gc); unsigned int val; int entries; @@ -3583,7 +3584,6 @@ int fib6_nh_init(struct net *net, struct fib6_nh *fib6_nh, netdevice_tracker *dev_tracker = &fib6_nh->fib_nh_dev_tracker; struct net_device *dev = NULL; struct inet6_dev *idev = NULL; - int addr_type; int err; fib6_nh->fib_nh_family = AF_INET6; @@ -3625,11 +3625,10 @@ int fib6_nh_init(struct net *net, struct fib6_nh *fib6_nh, fib6_nh->fib_nh_weight = 1; - /* We cannot add true routes via loopback here, - * they would result in kernel looping; promote them to reject routes + /* Reset the nexthop device to the loopback device in case of reject + * routes. */ - addr_type = ipv6_addr_type(&cfg->fc_dst); - if (fib6_is_reject(cfg->fc_flags, dev, addr_type)) { + if (cfg->fc_flags & RTF_REJECT) { /* hold loopback dev/idev if we haven't done so. */ if (dev != net->loopback_dev) { if (dev) { @@ -5008,7 +5007,7 @@ void rt6_sync_down_dev(struct net_device *dev, unsigned long event) }; struct net *net = dev_net(dev); - if (net->ipv6.sysctl.skip_notify_on_dev_down) + if (READ_ONCE(net->ipv6.sysctl.skip_notify_on_dev_down)) fib6_clean_all_skip_notify(net, fib6_ifdown, &arg); else fib6_clean_all(net, fib6_ifdown, &arg); @@ -6408,6 +6407,7 @@ void fib6_rt_update(struct net *net, struct fib6_info *rt, void fib6_info_hw_flags_set(struct net *net, struct fib6_info *f6i, bool offload, bool trap, bool offload_failed) { + u8 fib_notify_on_flag_change; struct sk_buff *skb; int err; @@ -6419,8 +6419,9 @@ void fib6_info_hw_flags_set(struct net *net, struct fib6_info *f6i, WRITE_ONCE(f6i->offload, offload); WRITE_ONCE(f6i->trap, trap); + fib_notify_on_flag_change = READ_ONCE(net->ipv6.sysctl.fib_notify_on_flag_change); /* 2 means send notifications only if offload_failed was changed. */ - if (net->ipv6.sysctl.fib_notify_on_flag_change == 2 && + if (fib_notify_on_flag_change == 2 && READ_ONCE(f6i->offload_failed) == offload_failed) return; @@ -6432,7 +6433,7 @@ void fib6_info_hw_flags_set(struct net *net, struct fib6_info *f6i, */ return; - if (!net->ipv6.sysctl.fib_notify_on_flag_change) + if (!fib_notify_on_flag_change) return; skb = nlmsg_new(rt6_nlmsg_size(f6i), GFP_KERNEL); @@ -6529,7 +6530,7 @@ static int ipv6_sysctl_rtcache_flush(const struct ctl_table *ctl, int write, return ret; net = (struct net *)ctl->extra1; - delay = net->ipv6.sysctl.flush_delay; + delay = READ_ONCE(net->ipv6.sysctl.flush_delay); fib6_run_gc(delay <= 0 ? 0 : (unsigned long)delay, net, delay > 0); return 0; } diff --git a/net/ipv6/seg6_hmac.c b/net/ipv6/seg6_hmac.c index ee6bac0160acea..e6964c6b0d3810 100644 --- a/net/ipv6/seg6_hmac.c +++ b/net/ipv6/seg6_hmac.c @@ -184,6 +184,8 @@ bool seg6_hmac_validate_skb(struct sk_buff *skb) int require_hmac; idev = __in6_dev_get(skb->dev); + if (!idev) + return false; srh = (struct ipv6_sr_hdr *)skb_transport_header(skb); diff --git a/net/ipv6/seg6_iptunnel.c b/net/ipv6/seg6_iptunnel.c index 3e1b9991131a23..d6a0f7df90807f 100644 --- a/net/ipv6/seg6_iptunnel.c +++ b/net/ipv6/seg6_iptunnel.c @@ -48,7 +48,8 @@ static size_t seg6_lwt_headroom(struct seg6_iptunnel_encap *tuninfo) } struct seg6_lwt { - struct dst_cache cache; + struct dst_cache cache_input; + struct dst_cache cache_output; struct seg6_iptunnel_encap tuninfo[]; }; @@ -488,7 +489,7 @@ static int seg6_input_core(struct net *net, struct sock *sk, slwt = seg6_lwt_lwtunnel(lwtst); local_bh_disable(); - dst = dst_cache_get(&slwt->cache); + dst = dst_cache_get(&slwt->cache_input); local_bh_enable(); err = seg6_do_srh(skb, dst); @@ -504,7 +505,7 @@ static int seg6_input_core(struct net *net, struct sock *sk, /* cache only if we don't create a dst reference loop */ if (!dst->error && lwtst != dst->lwtstate) { local_bh_disable(); - dst_cache_set_ip6(&slwt->cache, dst, + dst_cache_set_ip6(&slwt->cache_input, dst, &ipv6_hdr(skb)->saddr); local_bh_enable(); } @@ -564,7 +565,7 @@ static int seg6_output_core(struct net *net, struct sock *sk, slwt = seg6_lwt_lwtunnel(orig_dst->lwtstate); local_bh_disable(); - dst = dst_cache_get(&slwt->cache); + dst = dst_cache_get(&slwt->cache_output); local_bh_enable(); err = seg6_do_srh(skb, dst); @@ -591,7 +592,7 @@ static int seg6_output_core(struct net *net, struct sock *sk, /* cache only if we don't create a dst reference loop */ if (orig_dst->lwtstate != dst->lwtstate) { local_bh_disable(); - dst_cache_set_ip6(&slwt->cache, dst, &fl6.saddr); + dst_cache_set_ip6(&slwt->cache_output, dst, &fl6.saddr); local_bh_enable(); } @@ -701,11 +702,13 @@ static int seg6_build_state(struct net *net, struct nlattr *nla, slwt = seg6_lwt_lwtunnel(newts); - err = dst_cache_init(&slwt->cache, GFP_ATOMIC); - if (err) { - kfree(newts); - return err; - } + err = dst_cache_init(&slwt->cache_input, GFP_ATOMIC); + if (err) + goto err_free_newts; + + err = dst_cache_init(&slwt->cache_output, GFP_ATOMIC); + if (err) + goto err_destroy_input; memcpy(&slwt->tuninfo, tuninfo, tuninfo_len); @@ -720,11 +723,20 @@ static int seg6_build_state(struct net *net, struct nlattr *nla, *ts = newts; return 0; + +err_destroy_input: + dst_cache_destroy(&slwt->cache_input); +err_free_newts: + kfree(newts); + return err; } static void seg6_destroy_state(struct lwtunnel_state *lwt) { - dst_cache_destroy(&seg6_lwt_lwtunnel(lwt)->cache); + struct seg6_lwt *slwt = seg6_lwt_lwtunnel(lwt); + + dst_cache_destroy(&slwt->cache_input); + dst_cache_destroy(&slwt->cache_output); } static int seg6_fill_encap_info(struct sk_buff *skb, diff --git a/net/ipv6/syncookies.c b/net/ipv6/syncookies.c index 7e007f013ec827..4f6f0d751d6c53 100644 --- a/net/ipv6/syncookies.c +++ b/net/ipv6/syncookies.c @@ -151,9 +151,14 @@ static struct request_sock *cookie_tcp_check(struct net *net, struct sock *sk, tcp_parse_options(net, skb, &tcp_opt, 0, NULL); if (tcp_opt.saw_tstamp && tcp_opt.rcv_tsecr) { - tsoff = secure_tcpv6_ts_off(net, - ipv6_hdr(skb)->daddr.s6_addr32, - ipv6_hdr(skb)->saddr.s6_addr32); + union tcp_seq_and_ts_off st; + + st = secure_tcpv6_seq_and_ts_off(net, + ipv6_hdr(skb)->daddr.s6_addr32, + ipv6_hdr(skb)->saddr.s6_addr32, + tcp_hdr(skb)->dest, + tcp_hdr(skb)->source); + tsoff = st.ts_off; tcp_opt.rcv_tsecr -= tsoff; } diff --git a/net/ipv6/tcp_ipv6.c b/net/ipv6/tcp_ipv6.c index 280fe59785598e..1ba201e69a3da3 100644 --- a/net/ipv6/tcp_ipv6.c +++ b/net/ipv6/tcp_ipv6.c @@ -68,6 +68,7 @@ #include #include +#include #include @@ -104,18 +105,14 @@ static void inet6_sk_rx_dst_set(struct sock *sk, const struct sk_buff *skb) } } -static u32 tcp_v6_init_seq(const struct sk_buff *skb) +static union tcp_seq_and_ts_off +tcp_v6_init_seq_and_ts_off(const struct net *net, const struct sk_buff *skb) { - return secure_tcpv6_seq(ipv6_hdr(skb)->daddr.s6_addr32, - ipv6_hdr(skb)->saddr.s6_addr32, - tcp_hdr(skb)->dest, - tcp_hdr(skb)->source); -} - -static u32 tcp_v6_init_ts_off(const struct net *net, const struct sk_buff *skb) -{ - return secure_tcpv6_ts_off(net, ipv6_hdr(skb)->daddr.s6_addr32, - ipv6_hdr(skb)->saddr.s6_addr32); + return secure_tcpv6_seq_and_ts_off(net, + ipv6_hdr(skb)->daddr.s6_addr32, + ipv6_hdr(skb)->saddr.s6_addr32, + tcp_hdr(skb)->dest, + tcp_hdr(skb)->source); } static int tcp_v6_pre_connect(struct sock *sk, struct sockaddr_unsized *uaddr, @@ -318,14 +315,16 @@ static int tcp_v6_connect(struct sock *sk, struct sockaddr_unsized *uaddr, sk_set_txhash(sk); if (likely(!tp->repair)) { + union tcp_seq_and_ts_off st; + + st = secure_tcpv6_seq_and_ts_off(net, + np->saddr.s6_addr32, + sk->sk_v6_daddr.s6_addr32, + inet->inet_sport, + inet->inet_dport); if (!tp->write_seq) - WRITE_ONCE(tp->write_seq, - secure_tcpv6_seq(np->saddr.s6_addr32, - sk->sk_v6_daddr.s6_addr32, - inet->inet_sport, - inet->inet_dport)); - tp->tsoffset = secure_tcpv6_ts_off(net, np->saddr.s6_addr32, - sk->sk_v6_daddr.s6_addr32); + WRITE_ONCE(tp->write_seq, st.seq); + tp->tsoffset = st.ts_off; } if (tcp_fastopen_defer_connect(sk, &err)) @@ -814,8 +813,7 @@ const struct tcp_request_sock_ops tcp_request_sock_ipv6_ops = { .cookie_init_seq = cookie_v6_init_sequence, #endif .route_req = tcp_v6_route_req, - .init_seq = tcp_v6_init_seq, - .init_ts_off = tcp_v6_init_ts_off, + .init_seq_and_ts_off = tcp_v6_init_seq_and_ts_off, .send_synack = tcp_v6_send_synack, }; @@ -1046,7 +1044,7 @@ static void tcp_v6_send_reset(const struct sock *sk, struct sk_buff *skb, key.type = TCP_KEY_MD5; tcp_v6_md5_hash_skb(newhash, key.md5_key, NULL, skb); - if (memcmp(md5_hash_location, newhash, 16) != 0) + if (crypto_memneq(md5_hash_location, newhash, 16)) goto out; } #endif @@ -1085,7 +1083,8 @@ static void tcp_v6_send_reset(const struct sock *sk, struct sk_buff *skb, txhash = inet_twsk(sk)->tw_txhash; } } else { - if (net->ipv6.sysctl.flowlabel_reflect & FLOWLABEL_REFLECT_TCP_RESET) + if (READ_ONCE(net->ipv6.sysctl.flowlabel_reflect) & + FLOWLABEL_REFLECT_TCP_RESET) label = ip6_flowlabel(ipv6h); } @@ -1309,11 +1308,48 @@ static void tcp_v6_restore_cb(struct sk_buff *skb) sizeof(struct inet6_skb_parm)); } +/* Called from tcp_v4_syn_recv_sock() for v6_mapped children. */ +static void tcp_v6_mapped_child_init(struct sock *newsk, const struct sock *sk) +{ + struct inet_sock *newinet = inet_sk(newsk); + struct ipv6_pinfo *newnp; + + newinet->pinet6 = newnp = tcp_inet6_sk(newsk); + newinet->ipv6_fl_list = NULL; + + memcpy(newnp, tcp_inet6_sk(sk), sizeof(struct ipv6_pinfo)); + + newnp->saddr = newsk->sk_v6_rcv_saddr; + + inet_csk(newsk)->icsk_af_ops = &ipv6_mapped; + if (sk_is_mptcp(newsk)) + mptcpv6_handle_mapped(newsk, true); + newsk->sk_backlog_rcv = tcp_v4_do_rcv; +#if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO) + tcp_sk(newsk)->af_specific = &tcp_sock_ipv6_mapped_specific; +#endif + + newnp->ipv6_mc_list = NULL; + newnp->ipv6_ac_list = NULL; + newnp->pktoptions = NULL; + newnp->opt = NULL; + + /* tcp_v4_syn_recv_sock() has initialized newinet->mc_{index,ttl} */ + newnp->mcast_oif = newinet->mc_index; + newnp->mcast_hops = newinet->mc_ttl; + + newnp->rcv_flowinfo = 0; + if (inet6_test_bit(REPFLOW, sk)) + newnp->flow_label = 0; +} + static struct sock *tcp_v6_syn_recv_sock(const struct sock *sk, struct sk_buff *skb, struct request_sock *req, struct dst_entry *dst, struct request_sock *req_unhash, - bool *own_req) + bool *own_req, + void (*opt_child_init)(struct sock *newsk, + const struct sock *sk)) { struct inet_request_sock *ireq; struct ipv6_pinfo *newnp; @@ -1329,61 +1365,10 @@ static struct sock *tcp_v6_syn_recv_sock(const struct sock *sk, struct sk_buff * #endif struct flowi6 fl6; - if (skb->protocol == htons(ETH_P_IP)) { - /* - * v6 mapped - */ - - newsk = tcp_v4_syn_recv_sock(sk, skb, req, dst, - req_unhash, own_req); - - if (!newsk) - return NULL; - - newinet = inet_sk(newsk); - newinet->pinet6 = tcp_inet6_sk(newsk); - newinet->ipv6_fl_list = NULL; - - newnp = tcp_inet6_sk(newsk); - newtp = tcp_sk(newsk); - - memcpy(newnp, np, sizeof(struct ipv6_pinfo)); - - newnp->saddr = newsk->sk_v6_rcv_saddr; - - inet_csk(newsk)->icsk_af_ops = &ipv6_mapped; - if (sk_is_mptcp(newsk)) - mptcpv6_handle_mapped(newsk, true); - newsk->sk_backlog_rcv = tcp_v4_do_rcv; -#if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO) - newtp->af_specific = &tcp_sock_ipv6_mapped_specific; -#endif - - newnp->ipv6_mc_list = NULL; - newnp->ipv6_ac_list = NULL; - newnp->pktoptions = NULL; - newnp->opt = NULL; - newnp->mcast_oif = inet_iif(skb); - newnp->mcast_hops = ip_hdr(skb)->ttl; - newnp->rcv_flowinfo = 0; - if (inet6_test_bit(REPFLOW, sk)) - newnp->flow_label = 0; - - /* - * No need to charge this sock to the relevant IPv6 refcnt debug socks count - * here, tcp_create_openreq_child now does this for us, see the comment in - * that function for the gory details. -acme - */ - - /* It is tricky place. Until this moment IPv4 tcp - worked with IPv6 icsk.icsk_af_ops. - Sync it now. - */ - tcp_sync_mss(newsk, inet_csk(newsk)->icsk_pmtu_cookie); - - return newsk; - } - + if (skb->protocol == htons(ETH_P_IP)) + return tcp_v4_syn_recv_sock(sk, skb, req, dst, + req_unhash, own_req, + tcp_v6_mapped_child_init); ireq = inet_rsk(req); if (sk_acceptq_is_full(sk)) diff --git a/net/ipv6/udplite.c b/net/ipv6/udplite.c index 2cec542437f74e..e867721cda4d3e 100644 --- a/net/ipv6/udplite.c +++ b/net/ipv6/udplite.c @@ -16,10 +16,9 @@ static int udplitev6_sk_init(struct sock *sk) { - udpv6_init_sock(sk); pr_warn_once("UDP-Lite is deprecated and scheduled to be removed in 2025, " "please contact the netdev mailing list\n"); - return 0; + return udpv6_init_sock(sk); } static int udplitev6_rcv(struct sk_buff *skb) diff --git a/net/ipv6/xfrm6_input.c b/net/ipv6/xfrm6_input.c index 9005fc156a20e6..699a001ac16629 100644 --- a/net/ipv6/xfrm6_input.c +++ b/net/ipv6/xfrm6_input.c @@ -43,6 +43,7 @@ static int xfrm6_transport_finish2(struct net *net, struct sock *sk, int xfrm6_transport_finish(struct sk_buff *skb, int async) { struct xfrm_offload *xo = xfrm_offload(skb); + struct net_device *dev = skb->dev; int nhlen = -skb_network_offset(skb); skb_network_header(skb)[IP6CB(skb)->nhoff] = @@ -68,8 +69,10 @@ int xfrm6_transport_finish(struct sk_buff *skb, int async) } NF_HOOK(NFPROTO_IPV6, NF_INET_PRE_ROUTING, - dev_net(skb->dev), NULL, skb, skb->dev, NULL, + dev_net(dev), NULL, skb, dev, NULL, xfrm6_transport_finish2); + if (async) + dev_put(dev); return 0; } diff --git a/net/ipv6/xfrm6_policy.c b/net/ipv6/xfrm6_policy.c index 1f19b6f14484c6..125ea9a5b8a082 100644 --- a/net/ipv6/xfrm6_policy.c +++ b/net/ipv6/xfrm6_policy.c @@ -57,6 +57,7 @@ static int xfrm6_get_saddr(xfrm_address_t *saddr, struct dst_entry *dst; struct net_device *dev; struct inet6_dev *idev; + int err; dst = xfrm6_dst_lookup(params); if (IS_ERR(dst)) @@ -68,9 +69,11 @@ static int xfrm6_get_saddr(xfrm_address_t *saddr, return -EHOSTUNREACH; } dev = idev->dev; - ipv6_dev_get_saddr(dev_net(dev), dev, ¶ms->daddr->in6, 0, - &saddr->in6); + err = ipv6_dev_get_saddr(dev_net(dev), dev, ¶ms->daddr->in6, 0, + &saddr->in6); dst_release(dst); + if (err) + return -EHOSTUNREACH; return 0; } diff --git a/net/kcm/kcmsock.c b/net/kcm/kcmsock.c index 5dd7e0509a48fc..3912e75079f5eb 100644 --- a/net/kcm/kcmsock.c +++ b/net/kcm/kcmsock.c @@ -628,7 +628,7 @@ static int kcm_write_msgs(struct kcm_sock *kcm) skb = txm->frag_skb; } - if (WARN_ON(!skb_shinfo(skb)->nr_frags) || + if (WARN_ON_ONCE(!skb_shinfo(skb)->nr_frags) || WARN_ON_ONCE(!skb_frag_page(&skb_shinfo(skb)->frags[0]))) { ret = -EINVAL; goto out; @@ -749,7 +749,7 @@ static int kcm_sendmsg(struct socket *sock, struct msghdr *msg, size_t len) { struct sock *sk = sock->sk; struct kcm_sock *kcm = kcm_sk(sk); - struct sk_buff *skb = NULL, *head = NULL; + struct sk_buff *skb = NULL, *head = NULL, *frag_prev = NULL; size_t copy, copied = 0; long timeo = sock_sndtimeo(sk, msg->msg_flags & MSG_DONTWAIT); int eor = (sock->type == SOCK_DGRAM) ? @@ -824,6 +824,7 @@ static int kcm_sendmsg(struct socket *sock, struct msghdr *msg, size_t len) else skb->next = tskb; + frag_prev = skb; skb = tskb; skb->ip_summed = CHECKSUM_UNNECESSARY; continue; @@ -933,6 +934,22 @@ static int kcm_sendmsg(struct socket *sock, struct msghdr *msg, size_t len) out_error: kcm_push(kcm); + /* When MAX_SKB_FRAGS was reached, a new skb was allocated and + * linked into the frag_list before data copy. If the copy + * subsequently failed, this skb has zero frags. Remove it from + * the frag_list to prevent kcm_write_msgs from later hitting + * WARN_ON(!skb_shinfo(skb)->nr_frags). + */ + if (frag_prev && !skb_shinfo(skb)->nr_frags) { + if (head == frag_prev) + skb_shinfo(head)->frag_list = NULL; + else + frag_prev->next = NULL; + kfree_skb(skb); + /* Update skb as it may be saved in partial_message via goto */ + skb = frag_prev; + } + if (sock->type == SOCK_SEQPACKET) { /* Wrote some bytes before encountering an * error, return partial success. diff --git a/net/key/af_key.c b/net/key/af_key.c index 571200433aa90c..a6a9a40717ee83 100644 --- a/net/key/af_key.c +++ b/net/key/af_key.c @@ -757,6 +757,22 @@ static unsigned int pfkey_sockaddr_fill(const xfrm_address_t *xaddr, __be16 port return 0; } +static unsigned int pfkey_sockaddr_fill_zero_tail(const xfrm_address_t *xaddr, + __be16 port, + struct sockaddr *sa, + unsigned short family) +{ + unsigned int prefixlen; + int sockaddr_len = pfkey_sockaddr_len(family); + int sockaddr_size = pfkey_sockaddr_size(family); + + prefixlen = pfkey_sockaddr_fill(xaddr, port, sa, family); + if (sockaddr_size > sockaddr_len) + memset((u8 *)sa + sockaddr_len, 0, sockaddr_size - sockaddr_len); + + return prefixlen; +} + static struct sk_buff *__pfkey_xfrm_state2msg(const struct xfrm_state *x, int add_keys, int hsc) { @@ -3206,9 +3222,9 @@ static int pfkey_send_acquire(struct xfrm_state *x, struct xfrm_tmpl *t, struct addr->sadb_address_proto = 0; addr->sadb_address_reserved = 0; addr->sadb_address_prefixlen = - pfkey_sockaddr_fill(&x->props.saddr, 0, - (struct sockaddr *) (addr + 1), - x->props.family); + pfkey_sockaddr_fill_zero_tail(&x->props.saddr, 0, + (struct sockaddr *)(addr + 1), + x->props.family); if (!addr->sadb_address_prefixlen) BUG(); @@ -3221,9 +3237,9 @@ static int pfkey_send_acquire(struct xfrm_state *x, struct xfrm_tmpl *t, struct addr->sadb_address_proto = 0; addr->sadb_address_reserved = 0; addr->sadb_address_prefixlen = - pfkey_sockaddr_fill(&x->id.daddr, 0, - (struct sockaddr *) (addr + 1), - x->props.family); + pfkey_sockaddr_fill_zero_tail(&x->id.daddr, 0, + (struct sockaddr *)(addr + 1), + x->props.family); if (!addr->sadb_address_prefixlen) BUG(); @@ -3421,9 +3437,9 @@ static int pfkey_send_new_mapping(struct xfrm_state *x, xfrm_address_t *ipaddr, addr->sadb_address_proto = 0; addr->sadb_address_reserved = 0; addr->sadb_address_prefixlen = - pfkey_sockaddr_fill(&x->props.saddr, 0, - (struct sockaddr *) (addr + 1), - x->props.family); + pfkey_sockaddr_fill_zero_tail(&x->props.saddr, 0, + (struct sockaddr *)(addr + 1), + x->props.family); if (!addr->sadb_address_prefixlen) BUG(); @@ -3443,9 +3459,9 @@ static int pfkey_send_new_mapping(struct xfrm_state *x, xfrm_address_t *ipaddr, addr->sadb_address_proto = 0; addr->sadb_address_reserved = 0; addr->sadb_address_prefixlen = - pfkey_sockaddr_fill(ipaddr, 0, - (struct sockaddr *) (addr + 1), - x->props.family); + pfkey_sockaddr_fill_zero_tail(ipaddr, 0, + (struct sockaddr *)(addr + 1), + x->props.family); if (!addr->sadb_address_prefixlen) BUG(); @@ -3474,15 +3490,15 @@ static int set_sadb_address(struct sk_buff *skb, int sasize, int type, switch (type) { case SADB_EXT_ADDRESS_SRC: addr->sadb_address_prefixlen = sel->prefixlen_s; - pfkey_sockaddr_fill(&sel->saddr, 0, - (struct sockaddr *)(addr + 1), - sel->family); + pfkey_sockaddr_fill_zero_tail(&sel->saddr, 0, + (struct sockaddr *)(addr + 1), + sel->family); break; case SADB_EXT_ADDRESS_DST: addr->sadb_address_prefixlen = sel->prefixlen_d; - pfkey_sockaddr_fill(&sel->daddr, 0, - (struct sockaddr *)(addr + 1), - sel->family); + pfkey_sockaddr_fill_zero_tail(&sel->daddr, 0, + (struct sockaddr *)(addr + 1), + sel->family); break; default: return -EINVAL; @@ -3518,7 +3534,7 @@ static int set_sadb_kmaddress(struct sk_buff *skb, const struct xfrm_kmaddress * static int set_ipsecrequest(struct sk_buff *skb, uint8_t proto, uint8_t mode, int level, - uint32_t reqid, uint8_t family, + uint32_t reqid, sa_family_t family, const xfrm_address_t *src, const xfrm_address_t *dst) { struct sadb_x_ipsecrequest *rq; @@ -3583,12 +3599,17 @@ static int pfkey_send_migrate(const struct xfrm_selector *sel, u8 dir, u8 type, /* ipsecrequests */ for (i = 0, mp = m; i < num_bundles; i++, mp++) { - /* old locator pair */ - size_pol += sizeof(struct sadb_x_ipsecrequest) + - pfkey_sockaddr_pair_size(mp->old_family); - /* new locator pair */ - size_pol += sizeof(struct sadb_x_ipsecrequest) + - pfkey_sockaddr_pair_size(mp->new_family); + int pair_size; + + pair_size = pfkey_sockaddr_pair_size(mp->old_family); + if (!pair_size) + return -EINVAL; + size_pol += sizeof(struct sadb_x_ipsecrequest) + pair_size; + + pair_size = pfkey_sockaddr_pair_size(mp->new_family); + if (!pair_size) + return -EINVAL; + size_pol += sizeof(struct sadb_x_ipsecrequest) + pair_size; } size += sizeof(struct sadb_msg) + size_pol; diff --git a/net/l2tp/l2tp_core.c b/net/l2tp/l2tp_core.c index f9b0f666600f1c..336e447897bd6a 100644 --- a/net/l2tp/l2tp_core.c +++ b/net/l2tp/l2tp_core.c @@ -1290,6 +1290,11 @@ static int l2tp_xmit_core(struct l2tp_session *session, struct sk_buff *skb, uns uh->source = inet->inet_sport; uh->dest = inet->inet_dport; udp_len = uhlen + session->hdr_len + data_len; + if (udp_len > U16_MAX) { + kfree_skb(skb); + ret = NET_XMIT_DROP; + goto out_unlock; + } uh->len = htons(udp_len); /* Calculate UDP checksum if configured to do so */ diff --git a/net/mac80211/cfg.c b/net/mac80211/cfg.c index c81091a5cc3a33..e480b48e8365de 100644 --- a/net/mac80211/cfg.c +++ b/net/mac80211/cfg.c @@ -1889,12 +1889,6 @@ static int ieee80211_stop_ap(struct wiphy *wiphy, struct net_device *dev, __sta_info_flush(sdata, true, link_id, NULL); - ieee80211_remove_link_keys(link, &keys); - if (!list_empty(&keys)) { - synchronize_net(); - ieee80211_free_key_list(local, &keys); - } - ieee80211_stop_mbssid(sdata); RCU_INIT_POINTER(link_conf->tx_bss_conf, NULL); @@ -1906,6 +1900,12 @@ static int ieee80211_stop_ap(struct wiphy *wiphy, struct net_device *dev, ieee80211_link_info_change_notify(sdata, link, BSS_CHANGED_BEACON_ENABLED); + ieee80211_remove_link_keys(link, &keys); + if (!list_empty(&keys)) { + synchronize_net(); + ieee80211_free_key_list(local, &keys); + } + if (sdata->wdev.links[link_id].cac_started) { chandef = link_conf->chanreq.oper; wiphy_delayed_work_cancel(wiphy, &link->dfs_cac_timer_work); diff --git a/net/mac80211/chan.c b/net/mac80211/chan.c index d8c5f11afc157b..1b4bab698141a7 100644 --- a/net/mac80211/chan.c +++ b/net/mac80211/chan.c @@ -561,14 +561,16 @@ static void ieee80211_chan_bw_change(struct ieee80211_local *local, rcu_read_lock(); list_for_each_entry_rcu(sta, &local->sta_list, list) { - struct ieee80211_sub_if_data *sdata = sta->sdata; + struct ieee80211_sub_if_data *sdata; enum ieee80211_sta_rx_bandwidth new_sta_bw; unsigned int link_id; if (!ieee80211_sdata_running(sta->sdata)) continue; - for (link_id = 0; link_id < ARRAY_SIZE(sta->sdata->link); link_id++) { + sdata = get_bss_sdata(sta->sdata); + + for (link_id = 0; link_id < ARRAY_SIZE(sdata->link); link_id++) { struct ieee80211_link_data *link = rcu_dereference(sdata->link[link_id]); struct ieee80211_bss_conf *link_conf; diff --git a/net/mac80211/debugfs.c b/net/mac80211/debugfs.c index d02f07368c5119..687a66cd49433f 100644 --- a/net/mac80211/debugfs.c +++ b/net/mac80211/debugfs.c @@ -320,7 +320,6 @@ static ssize_t aql_enable_read(struct file *file, char __user *user_buf, static ssize_t aql_enable_write(struct file *file, const char __user *user_buf, size_t count, loff_t *ppos) { - bool aql_disabled = static_key_false(&aql_disable.key); char buf[3]; size_t len; @@ -335,15 +334,12 @@ static ssize_t aql_enable_write(struct file *file, const char __user *user_buf, if (len > 0 && buf[len - 1] == '\n') buf[len - 1] = 0; - if (buf[0] == '0' && buf[1] == '\0') { - if (!aql_disabled) - static_branch_inc(&aql_disable); - } else if (buf[0] == '1' && buf[1] == '\0') { - if (aql_disabled) - static_branch_dec(&aql_disable); - } else { + if (buf[0] == '0' && buf[1] == '\0') + static_branch_enable(&aql_disable); + else if (buf[0] == '1' && buf[1] == '\0') + static_branch_disable(&aql_disable); + else return -EINVAL; - } return count; } diff --git a/net/mac80211/link.c b/net/mac80211/link.c index 1e05845872afc8..b659497680b516 100644 --- a/net/mac80211/link.c +++ b/net/mac80211/link.c @@ -281,6 +281,7 @@ static int ieee80211_vif_update_links(struct ieee80211_sub_if_data *sdata, struct ieee80211_bss_conf *old[IEEE80211_MLD_MAX_NUM_LINKS]; struct ieee80211_link_data *old_data[IEEE80211_MLD_MAX_NUM_LINKS]; bool use_deflink = old_links == 0; /* set for error case */ + bool non_sta = sdata->vif.type != NL80211_IFTYPE_STATION; lockdep_assert_wiphy(sdata->local->hw.wiphy); @@ -337,6 +338,7 @@ static int ieee80211_vif_update_links(struct ieee80211_sub_if_data *sdata, link = links[link_id]; ieee80211_link_init(sdata, link_id, &link->data, &link->conf); ieee80211_link_setup(&link->data); + ieee80211_set_wmm_default(&link->data, true, non_sta); } if (new_links == 0) diff --git a/net/mac80211/mesh.c b/net/mac80211/mesh.c index 68901f1def0ddb..d7f691325746c2 100644 --- a/net/mac80211/mesh.c +++ b/net/mac80211/mesh.c @@ -79,6 +79,9 @@ bool mesh_matches_local(struct ieee80211_sub_if_data *sdata, * - MDA enabled * - Power management control on fc */ + if (!ie->mesh_config) + return false; + if (!(ifmsh->mesh_id_len == ie->mesh_id_len && memcmp(ifmsh->mesh_id, ie->mesh_id, ie->mesh_id_len) == 0 && (ifmsh->mesh_pp_id == ie->mesh_config->meshconf_psel) && @@ -1636,6 +1639,9 @@ static void mesh_rx_csa_frame(struct ieee80211_sub_if_data *sdata, if (!mesh_matches_local(sdata, elems)) goto free; + if (!elems->mesh_chansw_params_ie) + goto free; + ifmsh->chsw_ttl = elems->mesh_chansw_params_ie->mesh_ttl; if (!--ifmsh->chsw_ttl) fwd_csa = false; diff --git a/net/mac80211/mlme.c b/net/mac80211/mlme.c index 73f57b9e0ebf76..63346ee15069a1 100644 --- a/net/mac80211/mlme.c +++ b/net/mac80211/mlme.c @@ -6975,6 +6975,9 @@ static void ieee80211_ml_reconfiguration(struct ieee80211_sub_if_data *sdata, control = le16_to_cpu(prof->control); link_id = control & IEEE80211_MLE_STA_RECONF_CONTROL_LINK_ID; + if (link_id >= IEEE80211_MLD_MAX_NUM_LINKS) + continue; + removed_links |= BIT(link_id); /* the MAC address should not be included, but handle it */ diff --git a/net/mac80211/sta_info.c b/net/mac80211/sta_info.c index 1a995bc301b19a..b0d9bb830f2931 100644 --- a/net/mac80211/sta_info.c +++ b/net/mac80211/sta_info.c @@ -2759,7 +2759,9 @@ static void sta_set_link_sinfo(struct sta_info *sta, } link_sinfo->inactive_time = - jiffies_to_msecs(jiffies - ieee80211_sta_last_active(sta, link_id)); + jiffies_delta_to_msecs(jiffies - + ieee80211_sta_last_active(sta, + link_id)); if (!(link_sinfo->filled & (BIT_ULL(NL80211_STA_INFO_TX_BYTES64) | BIT_ULL(NL80211_STA_INFO_TX_BYTES)))) { @@ -2992,7 +2994,8 @@ void sta_set_sinfo(struct sta_info *sta, struct station_info *sinfo, sinfo->connected_time = ktime_get_seconds() - sta->last_connected; sinfo->assoc_at = sta->assoc_at; sinfo->inactive_time = - jiffies_to_msecs(jiffies - ieee80211_sta_last_active(sta, -1)); + jiffies_delta_to_msecs(jiffies - + ieee80211_sta_last_active(sta, -1)); if (!(sinfo->filled & (BIT_ULL(NL80211_STA_INFO_TX_BYTES64) | BIT_ULL(NL80211_STA_INFO_TX_BYTES)))) { diff --git a/net/mac80211/tdls.c b/net/mac80211/tdls.c index dbbfe2d6842fbe..1dca2fae05a521 100644 --- a/net/mac80211/tdls.c +++ b/net/mac80211/tdls.c @@ -1449,7 +1449,7 @@ int ieee80211_tdls_oper(struct wiphy *wiphy, struct net_device *dev, } sta = sta_info_get(sdata, peer); - if (!sta) + if (!sta || !sta->sta.tdls) return -ENOLINK; iee80211_tdls_recalc_chanctx(sdata, sta); diff --git a/net/mac80211/tx.c b/net/mac80211/tx.c index 1b55e83404135a..0692fbb6c489e8 100644 --- a/net/mac80211/tx.c +++ b/net/mac80211/tx.c @@ -1898,8 +1898,10 @@ bool ieee80211_tx_prepare_skb(struct ieee80211_hw *hw, struct ieee80211_tx_data tx; struct sk_buff *skb2; - if (ieee80211_tx_prepare(sdata, &tx, NULL, skb) == TX_DROP) + if (ieee80211_tx_prepare(sdata, &tx, NULL, skb) == TX_DROP) { + kfree_skb(skb); return false; + } info->band = band; info->control.vif = vif; diff --git a/net/mac802154/iface.c b/net/mac802154/iface.c index 9e4631fade90c9..000be60d958034 100644 --- a/net/mac802154/iface.c +++ b/net/mac802154/iface.c @@ -469,7 +469,9 @@ static int mac802154_header_create(struct sk_buff *skb, } static int -mac802154_header_parse(const struct sk_buff *skb, unsigned char *haddr) +mac802154_header_parse(const struct sk_buff *skb, + const struct net_device *dev, + unsigned char *haddr) { struct ieee802154_hdr hdr; diff --git a/net/mctp/device.c b/net/mctp/device.c index 4d404edd7446e1..04c5570bacff69 100644 --- a/net/mctp/device.c +++ b/net/mctp/device.c @@ -70,6 +70,7 @@ static int mctp_fill_addrinfo(struct sk_buff *skb, return -EMSGSIZE; hdr = nlmsg_data(nlh); + memset(hdr, 0, sizeof(*hdr)); hdr->ifa_family = AF_MCTP; hdr->ifa_prefixlen = 0; hdr->ifa_flags = 0; diff --git a/net/mctp/neigh.c b/net/mctp/neigh.c index 05b899f22d902b..fc85f0e6930143 100644 --- a/net/mctp/neigh.c +++ b/net/mctp/neigh.c @@ -218,6 +218,7 @@ static int mctp_fill_neigh(struct sk_buff *skb, u32 portid, u32 seq, int event, return -EMSGSIZE; hdr = nlmsg_data(nlh); + memset(hdr, 0, sizeof(*hdr)); hdr->ndm_family = AF_MCTP; hdr->ndm_ifindex = dev->ifindex; hdr->ndm_state = 0; // TODO other state bits? diff --git a/net/mctp/route.c b/net/mctp/route.c index 2ac4011a953fff..7a94b58f00978e 100644 --- a/net/mctp/route.c +++ b/net/mctp/route.c @@ -359,6 +359,7 @@ static void mctp_flow_prepare_output(struct sk_buff *skb, struct mctp_dev *dev) { struct mctp_sk_key *key; struct mctp_flow *flow; + unsigned long flags; flow = skb_ext_find(skb, SKB_EXT_MCTP); if (!flow) @@ -366,12 +367,14 @@ static void mctp_flow_prepare_output(struct sk_buff *skb, struct mctp_dev *dev) key = flow->key; - if (key->dev) { + spin_lock_irqsave(&key->lock, flags); + + if (!key->dev) + mctp_dev_set_key(dev, key); + else WARN_ON(key->dev != dev); - return; - } - mctp_dev_set_key(dev, key); + spin_unlock_irqrestore(&key->lock, flags); } #else static void mctp_skb_set_flow(struct sk_buff *skb, struct mctp_sk_key *key) {} @@ -1643,6 +1646,7 @@ static int mctp_fill_rtinfo(struct sk_buff *skb, struct mctp_route *rt, return -EMSGSIZE; hdr = nlmsg_data(nlh); + memset(hdr, 0, sizeof(*hdr)); hdr->rtm_family = AF_MCTP; /* we use the _len fields as a number of EIDs, rather than diff --git a/net/mpls/af_mpls.c b/net/mpls/af_mpls.c index 580aac112dd219..d77bbe49698863 100644 --- a/net/mpls/af_mpls.c +++ b/net/mpls/af_mpls.c @@ -83,14 +83,30 @@ static struct mpls_route *mpls_route_input(struct net *net, unsigned int index) return mpls_dereference(net, platform_label[index]); } +static struct mpls_route __rcu **mpls_platform_label_rcu(struct net *net, size_t *platform_labels) +{ + struct mpls_route __rcu **platform_label; + unsigned int sequence; + + do { + sequence = read_seqcount_begin(&net->mpls.platform_label_seq); + platform_label = rcu_dereference(net->mpls.platform_label); + *platform_labels = net->mpls.platform_labels; + } while (read_seqcount_retry(&net->mpls.platform_label_seq, sequence)); + + return platform_label; +} + static struct mpls_route *mpls_route_input_rcu(struct net *net, unsigned int index) { struct mpls_route __rcu **platform_label; + size_t platform_labels; + + platform_label = mpls_platform_label_rcu(net, &platform_labels); - if (index >= net->mpls.platform_labels) + if (index >= platform_labels) return NULL; - platform_label = rcu_dereference(net->mpls.platform_label); return rcu_dereference(platform_label[index]); } @@ -2240,8 +2256,7 @@ static int mpls_dump_routes(struct sk_buff *skb, struct netlink_callback *cb) if (index < MPLS_LABEL_FIRST_UNRESERVED) index = MPLS_LABEL_FIRST_UNRESERVED; - platform_label = rcu_dereference(net->mpls.platform_label); - platform_labels = net->mpls.platform_labels; + platform_label = mpls_platform_label_rcu(net, &platform_labels); if (filter.filter_set) flags |= NLM_F_DUMP_FILTERED; @@ -2645,8 +2660,12 @@ static int resize_platform_label_table(struct net *net, size_t limit) } /* Update the global pointers */ + local_bh_disable(); + write_seqcount_begin(&net->mpls.platform_label_seq); net->mpls.platform_labels = limit; rcu_assign_pointer(net->mpls.platform_label, labels); + write_seqcount_end(&net->mpls.platform_label_seq); + local_bh_enable(); mutex_unlock(&net->mpls.platform_mutex); @@ -2728,6 +2747,8 @@ static __net_init int mpls_net_init(struct net *net) int i; mutex_init(&net->mpls.platform_mutex); + seqcount_mutex_init(&net->mpls.platform_label_seq, &net->mpls.platform_mutex); + net->mpls.platform_labels = 0; net->mpls.platform_label = NULL; net->mpls.ip_ttl_propagate = 1; @@ -2854,6 +2875,7 @@ static int __init mpls_init(void) rtnl_af_unregister(&mpls_af_ops); out_unregister_dev_type: dev_remove_pack(&mpls_packet_type); + unregister_netdevice_notifier(&mpls_dev_notifier); out_unregister_pernet: unregister_pernet_subsys(&mpls_net_ops); goto out; diff --git a/net/mptcp/pm.c b/net/mptcp/pm.c index e2040c327af676..f5e1a204007aa7 100644 --- a/net/mptcp/pm.c +++ b/net/mptcp/pm.c @@ -212,9 +212,24 @@ void mptcp_pm_send_ack(struct mptcp_sock *msk, spin_lock_bh(&msk->pm.lock); } -void mptcp_pm_addr_send_ack(struct mptcp_sock *msk) +static bool subflow_in_rm_list(const struct mptcp_subflow_context *subflow, + const struct mptcp_rm_list *rm_list) +{ + u8 i, id = subflow_get_local_id(subflow); + + for (i = 0; i < rm_list->nr; i++) { + if (rm_list->ids[i] == id) + return true; + } + + return false; +} + +static void +mptcp_pm_addr_send_ack_avoid_list(struct mptcp_sock *msk, + const struct mptcp_rm_list *rm_list) { - struct mptcp_subflow_context *subflow, *alt = NULL; + struct mptcp_subflow_context *subflow, *stale = NULL, *same_id = NULL; msk_owned_by_me(msk); lockdep_assert_held(&msk->pm.lock); @@ -224,19 +239,35 @@ void mptcp_pm_addr_send_ack(struct mptcp_sock *msk) return; mptcp_for_each_subflow(msk, subflow) { - if (__mptcp_subflow_active(subflow)) { - if (!subflow->stale) { - mptcp_pm_send_ack(msk, subflow, false, false); - return; - } + if (!__mptcp_subflow_active(subflow)) + continue; - if (!alt) - alt = subflow; + if (unlikely(subflow->stale)) { + if (!stale) + stale = subflow; + } else if (unlikely(rm_list && + subflow_in_rm_list(subflow, rm_list))) { + if (!same_id) + same_id = subflow; + } else { + goto send_ack; } } - if (alt) - mptcp_pm_send_ack(msk, alt, false, false); + if (same_id) + subflow = same_id; + else if (stale) + subflow = stale; + else + return; + +send_ack: + mptcp_pm_send_ack(msk, subflow, false, false); +} + +void mptcp_pm_addr_send_ack(struct mptcp_sock *msk) +{ + mptcp_pm_addr_send_ack_avoid_list(msk, NULL); } int mptcp_pm_mp_prio_send_ack(struct mptcp_sock *msk, @@ -470,7 +501,7 @@ int mptcp_pm_remove_addr(struct mptcp_sock *msk, const struct mptcp_rm_list *rm_ msk->pm.rm_list_tx = *rm_list; rm_addr |= BIT(MPTCP_RM_ADDR_SIGNAL); WRITE_ONCE(msk->pm.addr_signal, rm_addr); - mptcp_pm_addr_send_ack(msk); + mptcp_pm_addr_send_ack_avoid_list(msk, rm_list); return 0; } diff --git a/net/mptcp/pm_kernel.c b/net/mptcp/pm_kernel.c index b26675054b0dc7..38547fbadd35b5 100644 --- a/net/mptcp/pm_kernel.c +++ b/net/mptcp/pm_kernel.c @@ -418,6 +418,15 @@ static void mptcp_pm_create_subflow_or_signal_addr(struct mptcp_sock *msk) } exit: + /* If an endpoint has both the signal and subflow flags, but it is not + * possible to create subflows -- the 'while' loop body above never + * executed -- then still mark the endp as used, which is somehow the + * case. This avoids issues later when removing the endpoint and calling + * __mark_subflow_endp_available(), which expects the increment here. + */ + if (signal_and_subflow && local.addr.id != msk->mpc_endpoint_id) + msk->pm.local_addr_used++; + mptcp_pm_nl_check_work_pending(msk); } @@ -711,7 +720,7 @@ static void __mptcp_pm_release_addr_entry(struct mptcp_pm_addr_entry *entry) static int mptcp_pm_nl_append_new_local_addr(struct pm_nl_pernet *pernet, struct mptcp_pm_addr_entry *entry, - bool needs_id, bool replace) + bool replace) { struct mptcp_pm_addr_entry *cur, *del_entry = NULL; int ret = -EINVAL; @@ -770,7 +779,7 @@ static int mptcp_pm_nl_append_new_local_addr(struct pm_nl_pernet *pernet, } } - if (!entry->addr.id && needs_id) { + if (!entry->addr.id) { find_next: entry->addr.id = find_next_zero_bit(pernet->id_bitmap, MPTCP_PM_MAX_ADDR_ID + 1, @@ -781,7 +790,7 @@ static int mptcp_pm_nl_append_new_local_addr(struct pm_nl_pernet *pernet, } } - if (!entry->addr.id && needs_id) + if (!entry->addr.id) goto out; __set_bit(entry->addr.id, pernet->id_bitmap); @@ -829,7 +838,7 @@ static struct lock_class_key mptcp_keys[2]; static int mptcp_pm_nl_create_listen_socket(struct sock *sk, struct mptcp_pm_addr_entry *entry) { - bool is_ipv6 = sk->sk_family == AF_INET6; + bool is_ipv6 = entry->addr.family == AF_INET6; int addrlen = sizeof(struct sockaddr_in); struct sockaddr_storage addr; struct sock *newsk, *ssk; @@ -914,7 +923,7 @@ int mptcp_pm_nl_get_local_id(struct mptcp_sock *msk, return -ENOMEM; entry->addr.port = 0; - ret = mptcp_pm_nl_append_new_local_addr(pernet, entry, true, false); + ret = mptcp_pm_nl_append_new_local_addr(pernet, entry, false); if (ret < 0) kfree(entry); @@ -968,18 +977,6 @@ static int mptcp_nl_add_subflow_or_signal_addr(struct net *net, return 0; } -static bool mptcp_pm_has_addr_attr_id(const struct nlattr *attr, - struct genl_info *info) -{ - struct nlattr *tb[MPTCP_PM_ADDR_ATTR_MAX + 1]; - - if (!nla_parse_nested_deprecated(tb, MPTCP_PM_ADDR_ATTR_MAX, attr, - mptcp_pm_address_nl_policy, info->extack) && - tb[MPTCP_PM_ADDR_ATTR_ID]) - return true; - return false; -} - /* Add an MPTCP endpoint */ int mptcp_pm_nl_add_addr_doit(struct sk_buff *skb, struct genl_info *info) { @@ -1028,9 +1025,7 @@ int mptcp_pm_nl_add_addr_doit(struct sk_buff *skb, struct genl_info *info) goto out_free; } } - ret = mptcp_pm_nl_append_new_local_addr(pernet, entry, - !mptcp_pm_has_addr_attr_id(attr, info), - true); + ret = mptcp_pm_nl_append_new_local_addr(pernet, entry, true); if (ret < 0) { GENL_SET_ERR_MSG_FMT(info, "too many addresses or duplicate one: %d", ret); goto out_free; @@ -1056,10 +1051,8 @@ static bool mptcp_pm_remove_anno_addr(struct mptcp_sock *msk, ret = mptcp_remove_anno_list_by_saddr(msk, addr); if (ret || force) { spin_lock_bh(&msk->pm.lock); - if (ret) { - __set_bit(addr->id, msk->pm.id_avail_bitmap); + if (ret) msk->pm.add_addr_signaled--; - } mptcp_pm_remove_addr(msk, &list); spin_unlock_bh(&msk->pm.lock); } @@ -1097,17 +1090,15 @@ static int mptcp_nl_remove_subflow_and_signal_addr(struct net *net, !(entry->flags & MPTCP_PM_ADDR_FLAG_IMPLICIT)); list.ids[0] = mptcp_endp_get_local_id(msk, addr); - if (remove_subflow) { - spin_lock_bh(&msk->pm.lock); - mptcp_pm_rm_subflow(msk, &list); - spin_unlock_bh(&msk->pm.lock); - } - if (entry->flags & MPTCP_PM_ADDR_FLAG_SUBFLOW) { - spin_lock_bh(&msk->pm.lock); + spin_lock_bh(&msk->pm.lock); + if (remove_subflow) + mptcp_pm_rm_subflow(msk, &list); + if (entry->flags & MPTCP_PM_ADDR_FLAG_SUBFLOW) __mark_subflow_endp_available(msk, list.ids[0]); - spin_unlock_bh(&msk->pm.lock); - } + else /* mark endp ID as available, e.g. Signal or MPC endp */ + __set_bit(addr->id, msk->pm.id_avail_bitmap); + spin_unlock_bh(&msk->pm.lock); if (msk->mpc_endpoint_id == entry->addr.id) msk->mpc_endpoint_id = 0; diff --git a/net/mptcp/protocol.c b/net/mptcp/protocol.c index 8d323366741810..80cb723ba1f309 100644 --- a/net/mptcp/protocol.c +++ b/net/mptcp/protocol.c @@ -224,9 +224,6 @@ static bool mptcp_rcvbuf_grow(struct sock *sk, u32 newval) do_div(grow, oldval); rcvwin += grow << 1; - if (!RB_EMPTY_ROOT(&msk->out_of_order_queue)) - rcvwin += MPTCP_SKB_CB(msk->ooo_last_skb)->end_seq - msk->ack_seq; - cap = READ_ONCE(net->ipv4.sysctl_tcp_rmem[2]); rcvbuf = min_t(u32, mptcp_space_from_win(sk, rcvwin), cap); @@ -350,9 +347,6 @@ static void mptcp_data_queue_ofo(struct mptcp_sock *msk, struct sk_buff *skb) end: skb_condense(skb); skb_set_owner_r(skb, sk); - /* do not grow rcvbuf for not-yet-accepted or orphaned sockets. */ - if (sk->sk_socket) - mptcp_rcvbuf_grow(sk, msk->rcvq_space.space); } static void mptcp_init_skb(struct sock *ssk, struct sk_buff *skb, int offset, @@ -1995,10 +1989,21 @@ static int mptcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t len) static void mptcp_rcv_space_adjust(struct mptcp_sock *msk, int copied); +static void mptcp_eat_recv_skb(struct sock *sk, struct sk_buff *skb) +{ + /* avoid the indirect call, we know the destructor is sock_rfree */ + skb->destructor = NULL; + skb->sk = NULL; + atomic_sub(skb->truesize, &sk->sk_rmem_alloc); + sk_mem_uncharge(sk, skb->truesize); + __skb_unlink(skb, &sk->sk_receive_queue); + skb_attempt_defer_free(skb); +} + static int __mptcp_recvmsg_mskq(struct sock *sk, struct msghdr *msg, size_t len, int flags, int copied_total, struct scm_timestamping_internal *tss, - int *cmsg_flags) + int *cmsg_flags, struct sk_buff **last) { struct mptcp_sock *msk = mptcp_sk(sk); struct sk_buff *skb, *tmp; @@ -2015,6 +2020,7 @@ static int __mptcp_recvmsg_mskq(struct sock *sk, struct msghdr *msg, /* skip already peeked skbs */ if (total_data_len + data_len <= copied_total) { total_data_len += data_len; + *last = skb; continue; } @@ -2049,13 +2055,9 @@ static int __mptcp_recvmsg_mskq(struct sock *sk, struct msghdr *msg, break; } - /* avoid the indirect call, we know the destructor is sock_rfree */ - skb->destructor = NULL; - skb->sk = NULL; - atomic_sub(skb->truesize, &sk->sk_rmem_alloc); - sk_mem_uncharge(sk, skb->truesize); - __skb_unlink(skb, &sk->sk_receive_queue); - skb_attempt_defer_free(skb); + mptcp_eat_recv_skb(sk, skb); + } else { + *last = skb; } if (copied >= len) @@ -2088,8 +2090,8 @@ static void mptcp_rcv_space_adjust(struct mptcp_sock *msk, int copied) msk->rcvq_space.copied += copied; - mstamp = div_u64(tcp_clock_ns(), NSEC_PER_USEC); - time = tcp_stamp_us_delta(mstamp, msk->rcvq_space.time); + mstamp = mptcp_stamp(); + time = tcp_stamp_us_delta(mstamp, READ_ONCE(msk->rcvq_space.time)); rtt_us = msk->rcvq_space.rtt_us; if (rtt_us && time < (rtt_us >> 3)) @@ -2270,10 +2272,12 @@ static int mptcp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, cmsg_flags = MPTCP_CMSG_INQ; while (copied < len) { + struct sk_buff *last = NULL; int err, bytes_read; bytes_read = __mptcp_recvmsg_mskq(sk, msg, len - copied, flags, - copied, &tss, &cmsg_flags); + copied, &tss, &cmsg_flags, + &last); if (unlikely(bytes_read < 0)) { if (!copied) copied = bytes_read; @@ -2325,7 +2329,7 @@ static int mptcp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, pr_debug("block timeout %ld\n", timeo); mptcp_cleanup_rbuf(msk, copied); - err = sk_wait_data(sk, &timeo, NULL); + err = sk_wait_data(sk, &timeo, last); if (err < 0) { err = copied ? : err; goto out_err; @@ -3549,6 +3553,7 @@ struct sock *mptcp_sk_clone_init(const struct sock *sk, __mptcp_propagate_sndbuf(nsk, ssk); mptcp_rcv_space_init(msk, ssk); + msk->rcvq_space.time = mptcp_stamp(); if (mp_opt->suboptions & OPTION_MPTCP_MPC_ACK) __mptcp_subflow_fully_established(msk, subflow, mp_opt); @@ -3566,8 +3571,6 @@ void mptcp_rcv_space_init(struct mptcp_sock *msk, const struct sock *ssk) msk->rcvq_space.copied = 0; msk->rcvq_space.rtt_us = 0; - msk->rcvq_space.time = tp->tcp_mstamp; - /* initial rcv_space offering made to peer */ msk->rcvq_space.space = min_t(u32, tp->rcv_wnd, TCP_INIT_CWND * tp->advmss); @@ -3763,6 +3766,7 @@ void mptcp_finish_connect(struct sock *ssk) * accessing the field below */ WRITE_ONCE(msk->local_key, subflow->local_key); + WRITE_ONCE(msk->rcvq_space.time, mptcp_stamp()); mptcp_pm_new_connection(msk, ssk, 0); } @@ -4452,6 +4456,8 @@ int __init mptcp_proto_v6_init(void) { int err; + mptcp_subflow_v6_init(); + mptcp_v6_prot = mptcp_prot; strscpy(mptcp_v6_prot.name, "MPTCPv6", sizeof(mptcp_v6_prot.name)); mptcp_v6_prot.slab = NULL; diff --git a/net/mptcp/protocol.h b/net/mptcp/protocol.h index 66e9735007912e..b10453adf99776 100644 --- a/net/mptcp/protocol.h +++ b/net/mptcp/protocol.h @@ -875,6 +875,7 @@ static inline void mptcp_subflow_tcp_fallback(struct sock *sk, void __init mptcp_proto_init(void); #if IS_ENABLED(CONFIG_MPTCP_IPV6) int __init mptcp_proto_v6_init(void); +void __init mptcp_subflow_v6_init(void); #endif struct sock *mptcp_sk_clone_init(const struct sock *sk, @@ -915,6 +916,11 @@ static inline bool mptcp_is_fully_established(struct sock *sk) READ_ONCE(mptcp_sk(sk)->fully_established); } +static inline u64 mptcp_stamp(void) +{ + return div_u64(tcp_clock_ns(), NSEC_PER_USEC); +} + void mptcp_rcv_space_init(struct mptcp_sock *msk, const struct sock *ssk); void mptcp_data_ready(struct sock *sk, struct sock *ssk); bool mptcp_finish_join(struct sock *sk); diff --git a/net/mptcp/subflow.c b/net/mptcp/subflow.c index 96d54cb2cd93f4..10458e2e9b86b8 100644 --- a/net/mptcp/subflow.c +++ b/net/mptcp/subflow.c @@ -810,7 +810,9 @@ static struct sock *subflow_syn_recv_sock(const struct sock *sk, struct request_sock *req, struct dst_entry *dst, struct request_sock *req_unhash, - bool *own_req) + bool *own_req, + void (*opt_child_init)(struct sock *newsk, + const struct sock *sk)) { struct mptcp_subflow_context *listener = mptcp_subflow_ctx(sk); struct mptcp_subflow_request_sock *subflow_req; @@ -857,7 +859,7 @@ static struct sock *subflow_syn_recv_sock(const struct sock *sk, create_child: child = listener->icsk_af_ops->syn_recv_sock(sk, skb, req, dst, - req_unhash, own_req); + req_unhash, own_req, opt_child_init); if (child && *own_req) { struct mptcp_subflow_context *ctx = mptcp_subflow_ctx(child); @@ -2165,7 +2167,15 @@ void __init mptcp_subflow_init(void) tcp_prot_override.psock_update_sk_prot = NULL; #endif + mptcp_diag_subflow_init(&subflow_ulp_ops); + + if (tcp_register_ulp(&subflow_ulp_ops) != 0) + panic("MPTCP: failed to register subflows to ULP\n"); +} + #if IS_ENABLED(CONFIG_MPTCP_IPV6) +void __init mptcp_subflow_v6_init(void) +{ /* In struct mptcp_subflow_request_sock, we assume the TCP request sock * structures for v4 and v6 have the same size. It should not changed in * the future but better to make sure to be warned if it is no longer @@ -2204,10 +2214,5 @@ void __init mptcp_subflow_init(void) /* Disable sockmap processing for subflows */ tcpv6_prot_override.psock_update_sk_prot = NULL; #endif -#endif - - mptcp_diag_subflow_init(&subflow_ulp_ops); - - if (tcp_register_ulp(&subflow_ulp_ops) != 0) - panic("MPTCP: failed to register subflows to ULP\n"); } +#endif diff --git a/net/ncsi/ncsi-aen.c b/net/ncsi/ncsi-aen.c index 62fb1031763d14..040a31557201bc 100644 --- a/net/ncsi/ncsi-aen.c +++ b/net/ncsi/ncsi-aen.c @@ -224,7 +224,8 @@ int ncsi_aen_handler(struct ncsi_dev_priv *ndp, struct sk_buff *skb) if (!nah) { netdev_warn(ndp->ndev.dev, "Invalid AEN (0x%x) received\n", h->type); - return -ENOENT; + ret = -ENOENT; + goto out; } ret = ncsi_validate_aen_pkt(h, nah->payload); diff --git a/net/ncsi/ncsi-rsp.c b/net/ncsi/ncsi-rsp.c index 271ec6c3929e85..fbd84bc8026a39 100644 --- a/net/ncsi/ncsi-rsp.c +++ b/net/ncsi/ncsi-rsp.c @@ -1176,8 +1176,10 @@ int ncsi_rcv_rsp(struct sk_buff *skb, struct net_device *dev, /* Find the NCSI device */ nd = ncsi_find_dev(orig_dev); ndp = nd ? TO_NCSI_DEV_PRIV(nd) : NULL; - if (!ndp) - return -ENODEV; + if (!ndp) { + ret = -ENODEV; + goto err_free_skb; + } /* Check if it is AEN packet */ hdr = (struct ncsi_pkt_hdr *)skb_network_header(skb); @@ -1199,7 +1201,8 @@ int ncsi_rcv_rsp(struct sk_buff *skb, struct net_device *dev, if (!nrh) { netdev_err(nd->dev, "Received unrecognized packet (0x%x)\n", hdr->type); - return -ENOENT; + ret = -ENOENT; + goto err_free_skb; } /* Associate with the request */ @@ -1207,7 +1210,8 @@ int ncsi_rcv_rsp(struct sk_buff *skb, struct net_device *dev, nr = &ndp->requests[hdr->id]; if (!nr->used) { spin_unlock_irqrestore(&ndp->lock, flags); - return -ENODEV; + ret = -ENODEV; + goto err_free_skb; } nr->rsp = skb; @@ -1261,4 +1265,8 @@ int ncsi_rcv_rsp(struct sk_buff *skb, struct net_device *dev, out: ncsi_free_request(nr); return ret; + +err_free_skb: + kfree_skb(skb); + return ret; } diff --git a/net/netfilter/ipset/ip_set_core.c b/net/netfilter/ipset/ip_set_core.c index cc20e6d56807c6..a4e1d7951b2c60 100644 --- a/net/netfilter/ipset/ip_set_core.c +++ b/net/netfilter/ipset/ip_set_core.c @@ -821,7 +821,7 @@ EXPORT_SYMBOL_GPL(ip_set_del); * */ ip_set_id_t -ip_set_get_byname(struct net *net, const char *name, struct ip_set **set) +ip_set_get_byname(struct net *net, const struct nlattr *name, struct ip_set **set) { ip_set_id_t i, index = IPSET_INVALID_ID; struct ip_set *s; @@ -830,7 +830,7 @@ ip_set_get_byname(struct net *net, const char *name, struct ip_set **set) rcu_read_lock(); for (i = 0; i < inst->ip_set_max; i++) { s = rcu_dereference(inst->ip_set_list)[i]; - if (s && STRNCMP(s->name, name)) { + if (s && nla_strcmp(name, s->name) == 0) { __ip_set_get(s); index = i; *set = s; diff --git a/net/netfilter/ipset/ip_set_hash_gen.h b/net/netfilter/ipset/ip_set_hash_gen.h index 5e4453e9ef8e73..4e56269efef28e 100644 --- a/net/netfilter/ipset/ip_set_hash_gen.h +++ b/net/netfilter/ipset/ip_set_hash_gen.h @@ -1099,7 +1099,7 @@ mtype_del(struct ip_set *set, void *value, const struct ip_set_ext *ext, if (!test_bit(i, n->used)) k++; } - if (n->pos == 0 && k == 0) { + if (k == n->pos) { t->hregion[r].ext_size -= ext_size(n->size, dsize); rcu_assign_pointer(hbucket(t, key), NULL); kfree_rcu(n, rcu); diff --git a/net/netfilter/ipset/ip_set_list_set.c b/net/netfilter/ipset/ip_set_list_set.c index 13c7a08aa868c0..34bb84d7b174c9 100644 --- a/net/netfilter/ipset/ip_set_list_set.c +++ b/net/netfilter/ipset/ip_set_list_set.c @@ -367,7 +367,7 @@ list_set_uadt(struct ip_set *set, struct nlattr *tb[], ret = ip_set_get_extensions(set, tb, &ext); if (ret) return ret; - e.id = ip_set_get_byname(map->net, nla_data(tb[IPSET_ATTR_NAME]), &s); + e.id = ip_set_get_byname(map->net, tb[IPSET_ATTR_NAME], &s); if (e.id == IPSET_INVALID_ID) return -IPSET_ERR_NAME; /* "Loop detection" */ @@ -389,7 +389,7 @@ list_set_uadt(struct ip_set *set, struct nlattr *tb[], if (tb[IPSET_ATTR_NAMEREF]) { e.refid = ip_set_get_byname(map->net, - nla_data(tb[IPSET_ATTR_NAMEREF]), + tb[IPSET_ATTR_NAMEREF], &s); if (e.refid == IPSET_INVALID_ID) { ret = -IPSET_ERR_NAMEREF; diff --git a/net/netfilter/ipvs/ip_vs_ctl.c b/net/netfilter/ipvs/ip_vs_ctl.c index 06870289437748..ce217a25a6af76 100644 --- a/net/netfilter/ipvs/ip_vs_ctl.c +++ b/net/netfilter/ipvs/ip_vs_ctl.c @@ -1452,7 +1452,6 @@ ip_vs_add_service(struct netns_ipvs *ipvs, struct ip_vs_service_user_kern *u, ret = ip_vs_bind_scheduler(svc, sched); if (ret) goto out_err; - sched = NULL; } ret = ip_vs_start_estimator(ipvs, &svc->stats); diff --git a/net/netfilter/ipvs/ip_vs_proto_sctp.c b/net/netfilter/ipvs/ip_vs_proto_sctp.c index 83e452916403d5..63c78a1f3918a7 100644 --- a/net/netfilter/ipvs/ip_vs_proto_sctp.c +++ b/net/netfilter/ipvs/ip_vs_proto_sctp.c @@ -10,7 +10,8 @@ #include static int -sctp_csum_check(int af, struct sk_buff *skb, struct ip_vs_protocol *pp); +sctp_csum_check(int af, struct sk_buff *skb, struct ip_vs_protocol *pp, + unsigned int sctphoff); static int sctp_conn_schedule(struct netns_ipvs *ipvs, int af, struct sk_buff *skb, @@ -108,7 +109,7 @@ sctp_snat_handler(struct sk_buff *skb, struct ip_vs_protocol *pp, int ret; /* Some checks before mangling */ - if (!sctp_csum_check(cp->af, skb, pp)) + if (!sctp_csum_check(cp->af, skb, pp, sctphoff)) return 0; /* Call application helper if needed */ @@ -156,7 +157,7 @@ sctp_dnat_handler(struct sk_buff *skb, struct ip_vs_protocol *pp, int ret; /* Some checks before mangling */ - if (!sctp_csum_check(cp->af, skb, pp)) + if (!sctp_csum_check(cp->af, skb, pp, sctphoff)) return 0; /* Call application helper if needed */ @@ -185,19 +186,12 @@ sctp_dnat_handler(struct sk_buff *skb, struct ip_vs_protocol *pp, } static int -sctp_csum_check(int af, struct sk_buff *skb, struct ip_vs_protocol *pp) +sctp_csum_check(int af, struct sk_buff *skb, struct ip_vs_protocol *pp, + unsigned int sctphoff) { - unsigned int sctphoff; struct sctphdr *sh; __le32 cmp, val; -#ifdef CONFIG_IP_VS_IPV6 - if (af == AF_INET6) - sctphoff = sizeof(struct ipv6hdr); - else -#endif - sctphoff = ip_hdrlen(skb); - sh = (struct sctphdr *)(skb->data + sctphoff); cmp = sh->checksum; val = sctp_compute_cksum(skb, sctphoff); diff --git a/net/netfilter/ipvs/ip_vs_proto_tcp.c b/net/netfilter/ipvs/ip_vs_proto_tcp.c index f68a1533ee455e..8cc0a8ce624112 100644 --- a/net/netfilter/ipvs/ip_vs_proto_tcp.c +++ b/net/netfilter/ipvs/ip_vs_proto_tcp.c @@ -28,7 +28,8 @@ #include static int -tcp_csum_check(int af, struct sk_buff *skb, struct ip_vs_protocol *pp); +tcp_csum_check(int af, struct sk_buff *skb, struct ip_vs_protocol *pp, + unsigned int tcphoff); static int tcp_conn_schedule(struct netns_ipvs *ipvs, int af, struct sk_buff *skb, @@ -165,7 +166,7 @@ tcp_snat_handler(struct sk_buff *skb, struct ip_vs_protocol *pp, int ret; /* Some checks before mangling */ - if (!tcp_csum_check(cp->af, skb, pp)) + if (!tcp_csum_check(cp->af, skb, pp, tcphoff)) return 0; /* Call application helper if needed */ @@ -243,7 +244,7 @@ tcp_dnat_handler(struct sk_buff *skb, struct ip_vs_protocol *pp, int ret; /* Some checks before mangling */ - if (!tcp_csum_check(cp->af, skb, pp)) + if (!tcp_csum_check(cp->af, skb, pp, tcphoff)) return 0; /* @@ -300,17 +301,9 @@ tcp_dnat_handler(struct sk_buff *skb, struct ip_vs_protocol *pp, static int -tcp_csum_check(int af, struct sk_buff *skb, struct ip_vs_protocol *pp) +tcp_csum_check(int af, struct sk_buff *skb, struct ip_vs_protocol *pp, + unsigned int tcphoff) { - unsigned int tcphoff; - -#ifdef CONFIG_IP_VS_IPV6 - if (af == AF_INET6) - tcphoff = sizeof(struct ipv6hdr); - else -#endif - tcphoff = ip_hdrlen(skb); - switch (skb->ip_summed) { case CHECKSUM_NONE: skb->csum = skb_checksum(skb, tcphoff, skb->len - tcphoff, 0); @@ -321,7 +314,7 @@ tcp_csum_check(int af, struct sk_buff *skb, struct ip_vs_protocol *pp) if (csum_ipv6_magic(&ipv6_hdr(skb)->saddr, &ipv6_hdr(skb)->daddr, skb->len - tcphoff, - ipv6_hdr(skb)->nexthdr, + IPPROTO_TCP, skb->csum)) { IP_VS_DBG_RL_PKT(0, af, pp, skb, 0, "Failed checksum for"); diff --git a/net/netfilter/ipvs/ip_vs_proto_udp.c b/net/netfilter/ipvs/ip_vs_proto_udp.c index 0f0107c80dd235..f9de632e38cdd6 100644 --- a/net/netfilter/ipvs/ip_vs_proto_udp.c +++ b/net/netfilter/ipvs/ip_vs_proto_udp.c @@ -24,7 +24,8 @@ #include static int -udp_csum_check(int af, struct sk_buff *skb, struct ip_vs_protocol *pp); +udp_csum_check(int af, struct sk_buff *skb, struct ip_vs_protocol *pp, + unsigned int udphoff); static int udp_conn_schedule(struct netns_ipvs *ipvs, int af, struct sk_buff *skb, @@ -154,7 +155,7 @@ udp_snat_handler(struct sk_buff *skb, struct ip_vs_protocol *pp, int ret; /* Some checks before mangling */ - if (!udp_csum_check(cp->af, skb, pp)) + if (!udp_csum_check(cp->af, skb, pp, udphoff)) return 0; /* @@ -237,7 +238,7 @@ udp_dnat_handler(struct sk_buff *skb, struct ip_vs_protocol *pp, int ret; /* Some checks before mangling */ - if (!udp_csum_check(cp->af, skb, pp)) + if (!udp_csum_check(cp->af, skb, pp, udphoff)) return 0; /* @@ -296,17 +297,10 @@ udp_dnat_handler(struct sk_buff *skb, struct ip_vs_protocol *pp, static int -udp_csum_check(int af, struct sk_buff *skb, struct ip_vs_protocol *pp) +udp_csum_check(int af, struct sk_buff *skb, struct ip_vs_protocol *pp, + unsigned int udphoff) { struct udphdr _udph, *uh; - unsigned int udphoff; - -#ifdef CONFIG_IP_VS_IPV6 - if (af == AF_INET6) - udphoff = sizeof(struct ipv6hdr); - else -#endif - udphoff = ip_hdrlen(skb); uh = skb_header_pointer(skb, udphoff, sizeof(_udph), &_udph); if (uh == NULL) @@ -324,7 +318,7 @@ udp_csum_check(int af, struct sk_buff *skb, struct ip_vs_protocol *pp) if (csum_ipv6_magic(&ipv6_hdr(skb)->saddr, &ipv6_hdr(skb)->daddr, skb->len - udphoff, - ipv6_hdr(skb)->nexthdr, + IPPROTO_UDP, skb->csum)) { IP_VS_DBG_RL_PKT(0, af, pp, skb, 0, "Failed checksum for"); diff --git a/net/netfilter/ipvs/ip_vs_xmit.c b/net/netfilter/ipvs/ip_vs_xmit.c index 64c697212578ae..124f779424b0ff 100644 --- a/net/netfilter/ipvs/ip_vs_xmit.c +++ b/net/netfilter/ipvs/ip_vs_xmit.c @@ -294,6 +294,12 @@ static inline bool decrement_ttl(struct netns_ipvs *ipvs, return true; } +/* rt has device that is down */ +static bool rt_dev_is_down(const struct net_device *dev) +{ + return dev && !netif_running(dev); +} + /* Get route to destination or remote server */ static int __ip_vs_get_out_rt(struct netns_ipvs *ipvs, int skb_af, struct sk_buff *skb, @@ -309,9 +315,11 @@ __ip_vs_get_out_rt(struct netns_ipvs *ipvs, int skb_af, struct sk_buff *skb, if (dest) { dest_dst = __ip_vs_dst_check(dest); - if (likely(dest_dst)) + if (likely(dest_dst)) { rt = dst_rtable(dest_dst->dst_cache); - else { + if (ret_saddr) + *ret_saddr = dest_dst->dst_saddr.ip; + } else { dest_dst = ip_vs_dest_dst_alloc(); spin_lock_bh(&dest->dst_lock); if (!dest_dst) { @@ -327,14 +335,22 @@ __ip_vs_get_out_rt(struct netns_ipvs *ipvs, int skb_af, struct sk_buff *skb, ip_vs_dest_dst_free(dest_dst); goto err_unreach; } - __ip_vs_dst_set(dest, dest_dst, &rt->dst, 0); + /* It is forbidden to attach dest->dest_dst if + * device is going down. + */ + if (!rt_dev_is_down(dst_dev_rcu(&rt->dst))) + __ip_vs_dst_set(dest, dest_dst, &rt->dst, 0); + else + noref = 0; spin_unlock_bh(&dest->dst_lock); IP_VS_DBG(10, "new dst %pI4, src %pI4, refcnt=%d\n", &dest->addr.ip, &dest_dst->dst_saddr.ip, rcuref_read(&rt->dst.__rcuref)); + if (ret_saddr) + *ret_saddr = dest_dst->dst_saddr.ip; + if (!noref) + ip_vs_dest_dst_free(dest_dst); } - if (ret_saddr) - *ret_saddr = dest_dst->dst_saddr.ip; } else { noref = 0; @@ -471,9 +487,11 @@ __ip_vs_get_out_rt_v6(struct netns_ipvs *ipvs, int skb_af, struct sk_buff *skb, if (dest) { dest_dst = __ip_vs_dst_check(dest); - if (likely(dest_dst)) + if (likely(dest_dst)) { rt = dst_rt6_info(dest_dst->dst_cache); - else { + if (ret_saddr) + *ret_saddr = dest_dst->dst_saddr.in6; + } else { u32 cookie; dest_dst = ip_vs_dest_dst_alloc(); @@ -494,14 +512,22 @@ __ip_vs_get_out_rt_v6(struct netns_ipvs *ipvs, int skb_af, struct sk_buff *skb, } rt = dst_rt6_info(dst); cookie = rt6_get_cookie(rt); - __ip_vs_dst_set(dest, dest_dst, &rt->dst, cookie); + /* It is forbidden to attach dest->dest_dst if + * device is going down. + */ + if (!rt_dev_is_down(dst_dev_rcu(&rt->dst))) + __ip_vs_dst_set(dest, dest_dst, &rt->dst, cookie); + else + noref = 0; spin_unlock_bh(&dest->dst_lock); IP_VS_DBG(10, "new dst %pI6, src %pI6, refcnt=%d\n", &dest->addr.in6, &dest_dst->dst_saddr.in6, rcuref_read(&rt->dst.__rcuref)); + if (ret_saddr) + *ret_saddr = dest_dst->dst_saddr.in6; + if (!noref) + ip_vs_dest_dst_free(dest_dst); } - if (ret_saddr) - *ret_saddr = dest_dst->dst_saddr.in6; } else { noref = 0; dst = __ip_vs_route_output_v6(net, daddr, ret_saddr, do_xfrm, diff --git a/net/netfilter/nf_bpf_link.c b/net/netfilter/nf_bpf_link.c index 46e667a50d988e..248840dbca1b26 100644 --- a/net/netfilter/nf_bpf_link.c +++ b/net/netfilter/nf_bpf_link.c @@ -170,7 +170,7 @@ static int bpf_nf_link_update(struct bpf_link *link, struct bpf_prog *new_prog, static const struct bpf_link_ops bpf_nf_link_lops = { .release = bpf_nf_link_release, - .dealloc = bpf_nf_link_dealloc, + .dealloc_deferred = bpf_nf_link_dealloc, .detach = bpf_nf_link_detach, .show_fdinfo = bpf_nf_link_show_info, .fill_link_info = bpf_nf_link_fill_link_info, diff --git a/net/netfilter/nf_conncount.c b/net/netfilter/nf_conncount.c index 8487808c87614f..14e62b3263cd94 100644 --- a/net/netfilter/nf_conncount.c +++ b/net/netfilter/nf_conncount.c @@ -34,8 +34,9 @@ #define CONNCOUNT_SLOTS 256U -#define CONNCOUNT_GC_MAX_NODES 8 -#define MAX_KEYLEN 5 +#define CONNCOUNT_GC_MAX_NODES 8 +#define CONNCOUNT_GC_MAX_COLLECT 64 +#define MAX_KEYLEN 5 /* we will save the tuples of all connections we care about */ struct nf_conncount_tuple { @@ -178,16 +179,28 @@ static int __nf_conncount_add(struct net *net, return -ENOENT; if (ct && nf_ct_is_confirmed(ct)) { - err = -EEXIST; - goto out_put; + /* local connections are confirmed in postrouting so confirmation + * might have happened before hitting connlimit + */ + if (skb->skb_iif != LOOPBACK_IFINDEX) { + err = -EEXIST; + goto out_put; + } + + /* this is likely a local connection, skip optimization to avoid + * adding duplicates from a 'packet train' + */ + goto check_connections; } - if ((u32)jiffies == list->last_gc) + if ((u32)jiffies == list->last_gc && + (list->count - list->last_gc_count) < CONNCOUNT_GC_MAX_COLLECT) goto add_new_node; +check_connections: /* check the saved connections */ list_for_each_entry_safe(conn, conn_n, &list->head, node) { - if (collect > CONNCOUNT_GC_MAX_NODES) + if (collect > CONNCOUNT_GC_MAX_COLLECT) break; found = find_or_evict(net, list, conn); @@ -230,6 +243,7 @@ static int __nf_conncount_add(struct net *net, nf_ct_put(found_ct); } list->last_gc = (u32)jiffies; + list->last_gc_count = list->count; add_new_node: if (WARN_ON_ONCE(list->count > INT_MAX)) { @@ -277,6 +291,7 @@ void nf_conncount_list_init(struct nf_conncount_list *list) spin_lock_init(&list->list_lock); INIT_LIST_HEAD(&list->head); list->count = 0; + list->last_gc_count = 0; list->last_gc = (u32)jiffies; } EXPORT_SYMBOL_GPL(nf_conncount_list_init); @@ -316,13 +331,14 @@ static bool __nf_conncount_gc_list(struct net *net, } nf_ct_put(found_ct); - if (collected > CONNCOUNT_GC_MAX_NODES) + if (collected > CONNCOUNT_GC_MAX_COLLECT) break; } if (!list->count) ret = true; list->last_gc = (u32)jiffies; + list->last_gc_count = list->count; return ret; } diff --git a/net/netfilter/nf_conntrack_broadcast.c b/net/netfilter/nf_conntrack_broadcast.c index a7552a46d6acf5..4f39bf7c843f2d 100644 --- a/net/netfilter/nf_conntrack_broadcast.c +++ b/net/netfilter/nf_conntrack_broadcast.c @@ -21,6 +21,7 @@ int nf_conntrack_broadcast_help(struct sk_buff *skb, unsigned int timeout) { const struct nf_conntrack_helper *helper; + struct net *net = read_pnet(&ct->ct_net); struct nf_conntrack_expect *exp; struct iphdr *iph = ip_hdr(skb); struct rtable *rt = skb_rtable(skb); @@ -70,8 +71,11 @@ int nf_conntrack_broadcast_help(struct sk_buff *skb, exp->expectfn = NULL; exp->flags = NF_CT_EXPECT_PERMANENT; exp->class = NF_CT_EXPECT_CLASS_DEFAULT; - exp->helper = NULL; - + rcu_assign_pointer(exp->helper, helper); + write_pnet(&exp->net, net); +#ifdef CONFIG_NF_CONNTRACK_ZONES + exp->zone = ct->zone; +#endif nf_ct_expect_related(exp, 0); nf_ct_expect_put(exp); diff --git a/net/netfilter/nf_conntrack_ecache.c b/net/netfilter/nf_conntrack_ecache.c index 81baf20826046e..9df159448b8979 100644 --- a/net/netfilter/nf_conntrack_ecache.c +++ b/net/netfilter/nf_conntrack_ecache.c @@ -247,6 +247,8 @@ void nf_ct_expect_event_report(enum ip_conntrack_expect_events event, struct nf_ct_event_notifier *notify; struct nf_conntrack_ecache *e; + lockdep_nfct_expect_lock_held(); + rcu_read_lock(); notify = rcu_dereference(net->ct.nf_conntrack_event_cb); if (!notify) diff --git a/net/netfilter/nf_conntrack_expect.c b/net/netfilter/nf_conntrack_expect.c index cfc2daa3fc7f34..24d0576d84b7f6 100644 --- a/net/netfilter/nf_conntrack_expect.c +++ b/net/netfilter/nf_conntrack_expect.c @@ -51,6 +51,7 @@ void nf_ct_unlink_expect_report(struct nf_conntrack_expect *exp, struct net *net = nf_ct_exp_net(exp); struct nf_conntrack_net *cnet; + lockdep_nfct_expect_lock_held(); WARN_ON(!master_help); WARN_ON(timer_pending(&exp->timeout)); @@ -112,12 +113,14 @@ nf_ct_exp_equal(const struct nf_conntrack_tuple *tuple, const struct net *net) { return nf_ct_tuple_mask_cmp(tuple, &i->tuple, &i->mask) && - net_eq(net, nf_ct_net(i->master)) && - nf_ct_zone_equal_any(i->master, zone); + net_eq(net, read_pnet(&i->net)) && + nf_ct_exp_zone_equal_any(i, zone); } bool nf_ct_remove_expect(struct nf_conntrack_expect *exp) { + lockdep_nfct_expect_lock_held(); + if (timer_delete(&exp->timeout)) { nf_ct_unlink_expect(exp); nf_ct_expect_put(exp); @@ -177,6 +180,8 @@ nf_ct_find_expectation(struct net *net, struct nf_conntrack_expect *i, *exp = NULL; unsigned int h; + lockdep_nfct_expect_lock_held(); + if (!cnet->expect_count) return NULL; @@ -309,12 +314,20 @@ struct nf_conntrack_expect *nf_ct_expect_alloc(struct nf_conn *me) } EXPORT_SYMBOL_GPL(nf_ct_expect_alloc); +/* This function can only be used from packet path, where accessing + * master's helper is safe, because the packet holds a reference on + * the conntrack object. Never use it from control plane. + */ void nf_ct_expect_init(struct nf_conntrack_expect *exp, unsigned int class, u_int8_t family, const union nf_inet_addr *saddr, const union nf_inet_addr *daddr, u_int8_t proto, const __be16 *src, const __be16 *dst) { + struct nf_conntrack_helper *helper = NULL; + struct nf_conn *ct = exp->master; + struct net *net = read_pnet(&ct->ct_net); + struct nf_conn_help *help; int len; if (family == AF_INET) @@ -325,7 +338,16 @@ void nf_ct_expect_init(struct nf_conntrack_expect *exp, unsigned int class, exp->flags = 0; exp->class = class; exp->expectfn = NULL; - exp->helper = NULL; + + help = nfct_help(ct); + if (help) + helper = rcu_dereference(help->helper); + + rcu_assign_pointer(exp->helper, helper); + write_pnet(&exp->net, net); +#ifdef CONFIG_NF_CONNTRACK_ZONES + exp->zone = ct->zone; +#endif exp->tuple.src.l3num = family; exp->tuple.dst.protonum = proto; @@ -442,6 +464,8 @@ static inline int __nf_ct_expect_check(struct nf_conntrack_expect *expect, unsigned int h; int ret = 0; + lockdep_nfct_expect_lock_held(); + if (!master_help) { ret = -ESHUTDOWN; goto out; @@ -498,8 +522,9 @@ int nf_ct_expect_related_report(struct nf_conntrack_expect *expect, nf_ct_expect_insert(expect); - spin_unlock_bh(&nf_conntrack_expect_lock); nf_ct_expect_event_report(IPEXP_NEW, expect, portid, report); + spin_unlock_bh(&nf_conntrack_expect_lock); + return 0; out: spin_unlock_bh(&nf_conntrack_expect_lock); @@ -627,11 +652,15 @@ static int exp_seq_show(struct seq_file *s, void *v) { struct nf_conntrack_expect *expect; struct nf_conntrack_helper *helper; + struct net *net = seq_file_net(s); struct hlist_node *n = v; char *delim = ""; expect = hlist_entry(n, struct nf_conntrack_expect, hnode); + if (!net_eq(nf_ct_exp_net(expect), net)) + return 0; + if (expect->timeout.function) seq_printf(s, "%ld ", timer_pending(&expect->timeout) ? (long)(expect->timeout.expires - jiffies)/HZ : 0); @@ -654,7 +683,7 @@ static int exp_seq_show(struct seq_file *s, void *v) if (expect->flags & NF_CT_EXPECT_USERSPACE) seq_printf(s, "%sUSERSPACE", delim); - helper = rcu_dereference(nfct_help(expect->master)->helper); + helper = rcu_dereference(expect->helper); if (helper) { seq_printf(s, "%s%s", expect->flags ? " " : "", helper->name); if (helper->expect_policy[expect->class].name[0]) diff --git a/net/netfilter/nf_conntrack_h323_asn1.c b/net/netfilter/nf_conntrack_h323_asn1.c index 540d97715bd23d..7b1497ed97d269 100644 --- a/net/netfilter/nf_conntrack_h323_asn1.c +++ b/net/netfilter/nf_conntrack_h323_asn1.c @@ -331,6 +331,8 @@ static int decode_int(struct bitstr *bs, const struct field_t *f, if (nf_h323_error_boundary(bs, 0, 2)) return H323_ERROR_BOUND; len = get_bits(bs, 2) + 1; + if (nf_h323_error_boundary(bs, len, 0)) + return H323_ERROR_BOUND; BYTE_ALIGN(bs); if (base && (f->attr & DECODE)) { /* timeToLive */ unsigned int v = get_uint(bs, len) + f->lb; @@ -796,7 +798,7 @@ static int decode_choice(struct bitstr *bs, const struct field_t *f, if (ext || (son->attr & OPEN)) { BYTE_ALIGN(bs); - if (nf_h323_error_boundary(bs, len, 0)) + if (nf_h323_error_boundary(bs, 2, 0)) return H323_ERROR_BOUND; len = get_len(bs); if (nf_h323_error_boundary(bs, len, 0)) @@ -922,6 +924,8 @@ int DecodeQ931(unsigned char *buf, size_t sz, Q931 *q931) break; p++; len--; + if (len <= 0) + break; return DecodeH323_UserInformation(buf, p, len, &q931->UUIE); } diff --git a/net/netfilter/nf_conntrack_h323_main.c b/net/netfilter/nf_conntrack_h323_main.c index 14f73872f64778..bd7e9e13e4f684 100644 --- a/net/netfilter/nf_conntrack_h323_main.c +++ b/net/netfilter/nf_conntrack_h323_main.c @@ -642,7 +642,7 @@ static int expect_h245(struct sk_buff *skb, struct nf_conn *ct, &ct->tuplehash[!dir].tuple.src.u3, &ct->tuplehash[!dir].tuple.dst.u3, IPPROTO_TCP, NULL, &port); - exp->helper = &nf_conntrack_helper_h245; + rcu_assign_pointer(exp->helper, &nf_conntrack_helper_h245); nathook = rcu_dereference(nfct_h323_nat_hook); if (memcmp(&ct->tuplehash[dir].tuple.src.u3, @@ -766,7 +766,7 @@ static int expect_callforwarding(struct sk_buff *skb, nf_ct_expect_init(exp, NF_CT_EXPECT_CLASS_DEFAULT, nf_ct_l3num(ct), &ct->tuplehash[!dir].tuple.src.u3, &addr, IPPROTO_TCP, NULL, &port); - exp->helper = nf_conntrack_helper_q931; + rcu_assign_pointer(exp->helper, nf_conntrack_helper_q931); nathook = rcu_dereference(nfct_h323_nat_hook); if (memcmp(&ct->tuplehash[dir].tuple.src.u3, @@ -1186,13 +1186,13 @@ static struct nf_conntrack_expect *find_expect(struct nf_conn *ct, { struct net *net = nf_ct_net(ct); struct nf_conntrack_expect *exp; - struct nf_conntrack_tuple tuple; + struct nf_conntrack_tuple tuple = { + .src.l3num = nf_ct_l3num(ct), + .dst.protonum = IPPROTO_TCP, + .dst.u.tcp.port = port, + }; - memset(&tuple.src.u3, 0, sizeof(tuple.src.u3)); - tuple.src.u.tcp.port = 0; memcpy(&tuple.dst.u3, addr, sizeof(tuple.dst.u3)); - tuple.dst.u.tcp.port = port; - tuple.dst.protonum = IPPROTO_TCP; exp = __nf_ct_expect_find(net, nf_ct_zone(ct), &tuple); if (exp && exp->master == ct) @@ -1233,7 +1233,7 @@ static int expect_q931(struct sk_buff *skb, struct nf_conn *ct, &ct->tuplehash[!dir].tuple.src.u3 : NULL, &ct->tuplehash[!dir].tuple.dst.u3, IPPROTO_TCP, NULL, &port); - exp->helper = nf_conntrack_helper_q931; + rcu_assign_pointer(exp->helper, nf_conntrack_helper_q931); exp->flags = NF_CT_EXPECT_PERMANENT; /* Accept multiple calls */ nathook = rcu_dereference(nfct_h323_nat_hook); @@ -1305,7 +1305,7 @@ static int process_gcf(struct sk_buff *skb, struct nf_conn *ct, nf_ct_expect_init(exp, NF_CT_EXPECT_CLASS_DEFAULT, nf_ct_l3num(ct), &ct->tuplehash[!dir].tuple.src.u3, &addr, IPPROTO_UDP, NULL, &port); - exp->helper = nf_conntrack_helper_ras; + rcu_assign_pointer(exp->helper, nf_conntrack_helper_ras); if (nf_ct_expect_related(exp, 0) == 0) { pr_debug("nf_ct_ras: expect RAS "); @@ -1522,7 +1522,7 @@ static int process_acf(struct sk_buff *skb, struct nf_conn *ct, &ct->tuplehash[!dir].tuple.src.u3, &addr, IPPROTO_TCP, NULL, &port); exp->flags = NF_CT_EXPECT_PERMANENT; - exp->helper = nf_conntrack_helper_q931; + rcu_assign_pointer(exp->helper, nf_conntrack_helper_q931); if (nf_ct_expect_related(exp, 0) == 0) { pr_debug("nf_ct_ras: expect Q.931 "); @@ -1576,7 +1576,7 @@ static int process_lcf(struct sk_buff *skb, struct nf_conn *ct, &ct->tuplehash[!dir].tuple.src.u3, &addr, IPPROTO_TCP, NULL, &port); exp->flags = NF_CT_EXPECT_PERMANENT; - exp->helper = nf_conntrack_helper_q931; + rcu_assign_pointer(exp->helper, nf_conntrack_helper_q931); if (nf_ct_expect_related(exp, 0) == 0) { pr_debug("nf_ct_ras: expect Q.931 "); diff --git a/net/netfilter/nf_conntrack_helper.c b/net/netfilter/nf_conntrack_helper.c index ceb48c3ca0a439..a715304a53d8c2 100644 --- a/net/netfilter/nf_conntrack_helper.c +++ b/net/netfilter/nf_conntrack_helper.c @@ -395,14 +395,10 @@ EXPORT_SYMBOL_GPL(nf_conntrack_helper_register); static bool expect_iter_me(struct nf_conntrack_expect *exp, void *data) { - struct nf_conn_help *help = nfct_help(exp->master); const struct nf_conntrack_helper *me = data; const struct nf_conntrack_helper *this; - if (exp->helper == me) - return true; - - this = rcu_dereference_protected(help->helper, + this = rcu_dereference_protected(exp->helper, lockdep_is_held(&nf_conntrack_expect_lock)); return this == me; } @@ -419,8 +415,13 @@ void nf_conntrack_helper_unregister(struct nf_conntrack_helper *me) */ synchronize_rcu(); - nf_ct_expect_iterate_destroy(expect_iter_me, NULL); + nf_ct_expect_iterate_destroy(expect_iter_me, me); nf_ct_iterate_destroy(unhelp, me); + + /* nf_ct_iterate_destroy() does an unconditional synchronize_rcu() as + * last step, this ensures rcu readers of exp->helper are done. + * No need for another synchronize_rcu() here. + */ } EXPORT_SYMBOL_GPL(nf_conntrack_helper_unregister); diff --git a/net/netfilter/nf_conntrack_netlink.c b/net/netfilter/nf_conntrack_netlink.c index 3a04665adf9927..fbe9e3f1036f82 100644 --- a/net/netfilter/nf_conntrack_netlink.c +++ b/net/netfilter/nf_conntrack_netlink.c @@ -909,8 +909,8 @@ struct ctnetlink_filter { }; static const struct nla_policy cta_filter_nla_policy[CTA_FILTER_MAX + 1] = { - [CTA_FILTER_ORIG_FLAGS] = { .type = NLA_U32 }, - [CTA_FILTER_REPLY_FLAGS] = { .type = NLA_U32 }, + [CTA_FILTER_ORIG_FLAGS] = NLA_POLICY_MASK(NLA_U32, CTA_FILTER_F_ALL), + [CTA_FILTER_REPLY_FLAGS] = NLA_POLICY_MASK(NLA_U32, CTA_FILTER_F_ALL), }; static int ctnetlink_parse_filter(const struct nlattr *attr, @@ -924,17 +924,11 @@ static int ctnetlink_parse_filter(const struct nlattr *attr, if (ret) return ret; - if (tb[CTA_FILTER_ORIG_FLAGS]) { + if (tb[CTA_FILTER_ORIG_FLAGS]) filter->orig_flags = nla_get_u32(tb[CTA_FILTER_ORIG_FLAGS]); - if (filter->orig_flags & ~CTA_FILTER_F_ALL) - return -EOPNOTSUPP; - } - if (tb[CTA_FILTER_REPLY_FLAGS]) { + if (tb[CTA_FILTER_REPLY_FLAGS]) filter->reply_flags = nla_get_u32(tb[CTA_FILTER_REPLY_FLAGS]); - if (filter->reply_flags & ~CTA_FILTER_F_ALL) - return -EOPNOTSUPP; - } return 0; } @@ -2633,7 +2627,7 @@ static const struct nla_policy exp_nla_policy[CTA_EXPECT_MAX+1] = { [CTA_EXPECT_HELP_NAME] = { .type = NLA_NUL_STRING, .len = NF_CT_HELPER_NAME_LEN - 1 }, [CTA_EXPECT_ZONE] = { .type = NLA_U16 }, - [CTA_EXPECT_FLAGS] = { .type = NLA_U32 }, + [CTA_EXPECT_FLAGS] = NLA_POLICY_MASK(NLA_BE32, NF_CT_EXPECT_MASK), [CTA_EXPECT_CLASS] = { .type = NLA_U32 }, [CTA_EXPECT_NAT] = { .type = NLA_NESTED }, [CTA_EXPECT_FN] = { .type = NLA_NUL_STRING }, @@ -2641,7 +2635,6 @@ static const struct nla_policy exp_nla_policy[CTA_EXPECT_MAX+1] = { static struct nf_conntrack_expect * ctnetlink_alloc_expect(const struct nlattr *const cda[], struct nf_conn *ct, - struct nf_conntrack_helper *helper, struct nf_conntrack_tuple *tuple, struct nf_conntrack_tuple *mask); @@ -2870,7 +2863,6 @@ ctnetlink_glue_attach_expect(const struct nlattr *attr, struct nf_conn *ct, { struct nlattr *cda[CTA_EXPECT_MAX+1]; struct nf_conntrack_tuple tuple, mask; - struct nf_conntrack_helper *helper = NULL; struct nf_conntrack_expect *exp; int err; @@ -2884,17 +2876,8 @@ ctnetlink_glue_attach_expect(const struct nlattr *attr, struct nf_conn *ct, if (err < 0) return err; - if (cda[CTA_EXPECT_HELP_NAME]) { - const char *helpname = nla_data(cda[CTA_EXPECT_HELP_NAME]); - - helper = __nf_conntrack_helper_find(helpname, nf_ct_l3num(ct), - nf_ct_protonum(ct)); - if (helper == NULL) - return -EOPNOTSUPP; - } - exp = ctnetlink_alloc_expect((const struct nlattr * const *)cda, ct, - helper, &tuple, &mask); + &tuple, &mask); if (IS_ERR(exp)) return PTR_ERR(exp); @@ -3011,7 +2994,7 @@ ctnetlink_exp_dump_expect(struct sk_buff *skb, { struct nf_conn *master = exp->master; long timeout = ((long)exp->timeout.expires - (long)jiffies) / HZ; - struct nf_conn_help *help; + struct nf_conntrack_helper *helper; #if IS_ENABLED(CONFIG_NF_NAT) struct nlattr *nest_parms; struct nf_conntrack_tuple nat_tuple = {}; @@ -3056,15 +3039,12 @@ ctnetlink_exp_dump_expect(struct sk_buff *skb, nla_put_be32(skb, CTA_EXPECT_FLAGS, htonl(exp->flags)) || nla_put_be32(skb, CTA_EXPECT_CLASS, htonl(exp->class))) goto nla_put_failure; - help = nfct_help(master); - if (help) { - struct nf_conntrack_helper *helper; - helper = rcu_dereference(help->helper); - if (helper && - nla_put_string(skb, CTA_EXPECT_HELP_NAME, helper->name)) - goto nla_put_failure; - } + helper = rcu_dereference(exp->helper); + if (helper && + nla_put_string(skb, CTA_EXPECT_HELP_NAME, helper->name)) + goto nla_put_failure; + expfn = nf_ct_helper_expectfn_find_by_symbol(exp->expectfn); if (expfn != NULL && nla_put_string(skb, CTA_EXPECT_FN, expfn->name)) @@ -3211,7 +3191,7 @@ ctnetlink_exp_ct_dump_table(struct sk_buff *skb, struct netlink_callback *cb) { struct nfgenmsg *nfmsg = nlmsg_data(cb->nlh); struct nf_conn *ct = cb->data; - struct nf_conn_help *help = nfct_help(ct); + struct nf_conn_help *help; u_int8_t l3proto = nfmsg->nfgen_family; unsigned long last_id = cb->args[1]; struct nf_conntrack_expect *exp; @@ -3219,6 +3199,10 @@ ctnetlink_exp_ct_dump_table(struct sk_buff *skb, struct netlink_callback *cb) if (cb->args[0]) return 0; + help = nfct_help(ct); + if (!help) + return 0; + rcu_read_lock(); restart: @@ -3248,6 +3232,24 @@ ctnetlink_exp_ct_dump_table(struct sk_buff *skb, struct netlink_callback *cb) return skb->len; } +static int ctnetlink_dump_exp_ct_start(struct netlink_callback *cb) +{ + struct nf_conn *ct = cb->data; + + if (!refcount_inc_not_zero(&ct->ct_general.use)) + return -ENOENT; + return 0; +} + +static int ctnetlink_dump_exp_ct_done(struct netlink_callback *cb) +{ + struct nf_conn *ct = cb->data; + + if (ct) + nf_ct_put(ct); + return 0; +} + static int ctnetlink_dump_exp_ct(struct net *net, struct sock *ctnl, struct sk_buff *skb, const struct nlmsghdr *nlh, @@ -3263,6 +3265,8 @@ static int ctnetlink_dump_exp_ct(struct net *net, struct sock *ctnl, struct nf_conntrack_zone zone; struct netlink_dump_control c = { .dump = ctnetlink_exp_ct_dump_table, + .start = ctnetlink_dump_exp_ct_start, + .done = ctnetlink_dump_exp_ct_done, }; err = ctnetlink_parse_tuple(cda, &tuple, CTA_EXPECT_MASTER, @@ -3333,31 +3337,37 @@ static int ctnetlink_get_expect(struct sk_buff *skb, if (err < 0) return err; + skb2 = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL); + if (!skb2) + return -ENOMEM; + + spin_lock_bh(&nf_conntrack_expect_lock); exp = nf_ct_expect_find_get(info->net, &zone, &tuple); - if (!exp) + if (!exp) { + spin_unlock_bh(&nf_conntrack_expect_lock); + kfree_skb(skb2); return -ENOENT; + } if (cda[CTA_EXPECT_ID]) { __be32 id = nla_get_be32(cda[CTA_EXPECT_ID]); if (id != nf_expect_get_id(exp)) { nf_ct_expect_put(exp); + spin_unlock_bh(&nf_conntrack_expect_lock); + kfree_skb(skb2); return -ENOENT; } } - skb2 = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL); - if (!skb2) { - nf_ct_expect_put(exp); - return -ENOMEM; - } - rcu_read_lock(); err = ctnetlink_exp_fill_info(skb2, NETLINK_CB(skb).portid, info->nlh->nlmsg_seq, IPCTNL_MSG_EXP_NEW, exp); rcu_read_unlock(); nf_ct_expect_put(exp); + spin_unlock_bh(&nf_conntrack_expect_lock); + if (err <= 0) { kfree_skb(skb2); return -ENOMEM; @@ -3369,12 +3379,9 @@ static int ctnetlink_get_expect(struct sk_buff *skb, static bool expect_iter_name(struct nf_conntrack_expect *exp, void *data) { struct nf_conntrack_helper *helper; - const struct nf_conn_help *m_help; const char *name = data; - m_help = nfct_help(exp->master); - - helper = rcu_dereference(m_help->helper); + helper = rcu_dereference(exp->helper); if (!helper) return false; @@ -3407,22 +3414,26 @@ static int ctnetlink_del_expect(struct sk_buff *skb, if (err < 0) return err; + spin_lock_bh(&nf_conntrack_expect_lock); + /* bump usage count to 2 */ exp = nf_ct_expect_find_get(info->net, &zone, &tuple); - if (!exp) + if (!exp) { + spin_unlock_bh(&nf_conntrack_expect_lock); return -ENOENT; + } if (cda[CTA_EXPECT_ID]) { __be32 id = nla_get_be32(cda[CTA_EXPECT_ID]); if (id != nf_expect_get_id(exp)) { nf_ct_expect_put(exp); + spin_unlock_bh(&nf_conntrack_expect_lock); return -ENOENT; } } /* after list removal, usage count == 1 */ - spin_lock_bh(&nf_conntrack_expect_lock); if (timer_delete(&exp->timeout)) { nf_ct_unlink_expect_report(exp, NETLINK_CB(skb).portid, nlmsg_report(info->nlh)); @@ -3464,7 +3475,7 @@ ctnetlink_change_expect(struct nf_conntrack_expect *x, #if IS_ENABLED(CONFIG_NF_NAT) static const struct nla_policy exp_nat_nla_policy[CTA_EXPECT_NAT_MAX+1] = { - [CTA_EXPECT_NAT_DIR] = { .type = NLA_U32 }, + [CTA_EXPECT_NAT_DIR] = NLA_POLICY_MAX(NLA_BE32, IP_CT_DIR_REPLY), [CTA_EXPECT_NAT_TUPLE] = { .type = NLA_NESTED }, }; #endif @@ -3505,20 +3516,25 @@ ctnetlink_parse_expect_nat(const struct nlattr *attr, static struct nf_conntrack_expect * ctnetlink_alloc_expect(const struct nlattr * const cda[], struct nf_conn *ct, - struct nf_conntrack_helper *helper, struct nf_conntrack_tuple *tuple, struct nf_conntrack_tuple *mask) { - u_int32_t class = 0; + struct net *net = read_pnet(&ct->ct_net); + struct nf_conntrack_helper *helper; struct nf_conntrack_expect *exp; struct nf_conn_help *help; + u32 class = 0; int err; help = nfct_help(ct); if (!help) return ERR_PTR(-EOPNOTSUPP); - if (cda[CTA_EXPECT_CLASS] && helper) { + helper = rcu_dereference(help->helper); + if (!helper) + return ERR_PTR(-EOPNOTSUPP); + + if (cda[CTA_EXPECT_CLASS]) { class = ntohl(nla_get_be32(cda[CTA_EXPECT_CLASS])); if (class > helper->expect_class_max) return ERR_PTR(-EINVAL); @@ -3548,7 +3564,11 @@ ctnetlink_alloc_expect(const struct nlattr * const cda[], struct nf_conn *ct, exp->class = class; exp->master = ct; - exp->helper = helper; + write_pnet(&exp->net, net); +#ifdef CONFIG_NF_CONNTRACK_ZONES + exp->zone = ct->zone; +#endif + rcu_assign_pointer(exp->helper, helper); exp->tuple = *tuple; exp->mask.src.u3 = mask->src.u3; exp->mask.src.u.all = mask->src.u.all; @@ -3558,6 +3578,12 @@ ctnetlink_alloc_expect(const struct nlattr * const cda[], struct nf_conn *ct, exp, nf_ct_l3num(ct)); if (err < 0) goto err_out; +#if IS_ENABLED(CONFIG_NF_NAT) + } else { + memset(&exp->saved_addr, 0, sizeof(exp->saved_addr)); + memset(&exp->saved_proto, 0, sizeof(exp->saved_proto)); + exp->dir = 0; +#endif } return exp; err_out: @@ -3573,7 +3599,6 @@ ctnetlink_create_expect(struct net *net, { struct nf_conntrack_tuple tuple, mask, master_tuple; struct nf_conntrack_tuple_hash *h = NULL; - struct nf_conntrack_helper *helper = NULL; struct nf_conntrack_expect *exp; struct nf_conn *ct; int err; @@ -3599,33 +3624,7 @@ ctnetlink_create_expect(struct net *net, ct = nf_ct_tuplehash_to_ctrack(h); rcu_read_lock(); - if (cda[CTA_EXPECT_HELP_NAME]) { - const char *helpname = nla_data(cda[CTA_EXPECT_HELP_NAME]); - - helper = __nf_conntrack_helper_find(helpname, u3, - nf_ct_protonum(ct)); - if (helper == NULL) { - rcu_read_unlock(); -#ifdef CONFIG_MODULES - if (request_module("nfct-helper-%s", helpname) < 0) { - err = -EOPNOTSUPP; - goto err_ct; - } - rcu_read_lock(); - helper = __nf_conntrack_helper_find(helpname, u3, - nf_ct_protonum(ct)); - if (helper) { - err = -EAGAIN; - goto err_rcu; - } - rcu_read_unlock(); -#endif - err = -EOPNOTSUPP; - goto err_ct; - } - } - - exp = ctnetlink_alloc_expect(cda, ct, helper, &tuple, &mask); + exp = ctnetlink_alloc_expect(cda, ct, &tuple, &mask); if (IS_ERR(exp)) { err = PTR_ERR(exp); goto err_rcu; @@ -3635,8 +3634,8 @@ ctnetlink_create_expect(struct net *net, nf_ct_expect_put(exp); err_rcu: rcu_read_unlock(); -err_ct: nf_ct_put(ct); + return err; } diff --git a/net/netfilter/nf_conntrack_proto_generic.c b/net/netfilter/nf_conntrack_proto_generic.c index e831637bc8ca8f..cb260eb3d012c3 100644 --- a/net/netfilter/nf_conntrack_proto_generic.c +++ b/net/netfilter/nf_conntrack_proto_generic.c @@ -67,6 +67,7 @@ void nf_conntrack_generic_init_net(struct net *net) const struct nf_conntrack_l4proto nf_conntrack_l4proto_generic = { .l4proto = 255, + .allow_clash = true, #ifdef CONFIG_NF_CONNTRACK_TIMEOUT .ctnl_timeout = { .nlattr_to_obj = generic_timeout_nlattr_to_obj, diff --git a/net/netfilter/nf_conntrack_proto_sctp.c b/net/netfilter/nf_conntrack_proto_sctp.c index 7c6f7c9f73320d..645d2c43ebf7af 100644 --- a/net/netfilter/nf_conntrack_proto_sctp.c +++ b/net/netfilter/nf_conntrack_proto_sctp.c @@ -582,7 +582,8 @@ static int sctp_to_nlattr(struct sk_buff *skb, struct nlattr *nla, } static const struct nla_policy sctp_nla_policy[CTA_PROTOINFO_SCTP_MAX+1] = { - [CTA_PROTOINFO_SCTP_STATE] = { .type = NLA_U8 }, + [CTA_PROTOINFO_SCTP_STATE] = NLA_POLICY_MAX(NLA_U8, + SCTP_CONNTRACK_HEARTBEAT_SENT), [CTA_PROTOINFO_SCTP_VTAG_ORIGINAL] = { .type = NLA_U32 }, [CTA_PROTOINFO_SCTP_VTAG_REPLY] = { .type = NLA_U32 }, }; diff --git a/net/netfilter/nf_conntrack_proto_tcp.c b/net/netfilter/nf_conntrack_proto_tcp.c index 0c1d086e96cb3f..b67426c2189b2d 100644 --- a/net/netfilter/nf_conntrack_proto_tcp.c +++ b/net/netfilter/nf_conntrack_proto_tcp.c @@ -1385,9 +1385,9 @@ static int tcp_to_nlattr(struct sk_buff *skb, struct nlattr *nla, } static const struct nla_policy tcp_nla_policy[CTA_PROTOINFO_TCP_MAX+1] = { - [CTA_PROTOINFO_TCP_STATE] = { .type = NLA_U8 }, - [CTA_PROTOINFO_TCP_WSCALE_ORIGINAL] = { .type = NLA_U8 }, - [CTA_PROTOINFO_TCP_WSCALE_REPLY] = { .type = NLA_U8 }, + [CTA_PROTOINFO_TCP_STATE] = NLA_POLICY_MAX(NLA_U8, TCP_CONNTRACK_SYN_SENT2), + [CTA_PROTOINFO_TCP_WSCALE_ORIGINAL] = NLA_POLICY_MAX(NLA_U8, TCP_MAX_WSCALE), + [CTA_PROTOINFO_TCP_WSCALE_REPLY] = NLA_POLICY_MAX(NLA_U8, TCP_MAX_WSCALE), [CTA_PROTOINFO_TCP_FLAGS_ORIGINAL] = { .len = sizeof(struct nf_ct_tcp_flags) }, [CTA_PROTOINFO_TCP_FLAGS_REPLY] = { .len = sizeof(struct nf_ct_tcp_flags) }, }; @@ -1414,10 +1414,6 @@ static int nlattr_to_tcp(struct nlattr *cda[], struct nf_conn *ct) if (err < 0) return err; - if (tb[CTA_PROTOINFO_TCP_STATE] && - nla_get_u8(tb[CTA_PROTOINFO_TCP_STATE]) >= TCP_CONNTRACK_MAX) - return -EINVAL; - spin_lock_bh(&ct->lock); if (tb[CTA_PROTOINFO_TCP_STATE]) ct->proto.tcp.state = nla_get_u8(tb[CTA_PROTOINFO_TCP_STATE]); diff --git a/net/netfilter/nf_conntrack_sip.c b/net/netfilter/nf_conntrack_sip.c index ca748f8dbff130..939502ff7c8713 100644 --- a/net/netfilter/nf_conntrack_sip.c +++ b/net/netfilter/nf_conntrack_sip.c @@ -924,7 +924,7 @@ static int set_expected_rtp_rtcp(struct sk_buff *skb, unsigned int protoff, exp = __nf_ct_expect_find(net, nf_ct_zone(ct), &tuple); if (!exp || exp->master == ct || - nfct_help(exp->master)->helper != nfct_help(ct)->helper || + exp->helper != nfct_help(ct)->helper || exp->class != class) break; #if IS_ENABLED(CONFIG_NF_NAT) @@ -1040,6 +1040,7 @@ static int process_sdp(struct sk_buff *skb, unsigned int protoff, unsigned int port; const struct sdp_media_type *t; int ret = NF_ACCEPT; + bool have_rtp_addr = false; hooks = rcu_dereference(nf_nat_sip_hooks); @@ -1056,8 +1057,11 @@ static int process_sdp(struct sk_buff *skb, unsigned int protoff, caddr_len = 0; if (ct_sip_parse_sdp_addr(ct, *dptr, sdpoff, *datalen, SDP_HDR_CONNECTION, SDP_HDR_MEDIA, - &matchoff, &matchlen, &caddr) > 0) + &matchoff, &matchlen, &caddr) > 0) { caddr_len = matchlen; + memcpy(&rtp_addr, &caddr, sizeof(rtp_addr)); + have_rtp_addr = true; + } mediaoff = sdpoff; for (i = 0; i < ARRAY_SIZE(sdp_media_types); ) { @@ -1091,9 +1095,11 @@ static int process_sdp(struct sk_buff *skb, unsigned int protoff, &matchoff, &matchlen, &maddr) > 0) { maddr_len = matchlen; memcpy(&rtp_addr, &maddr, sizeof(rtp_addr)); - } else if (caddr_len) + have_rtp_addr = true; + } else if (caddr_len) { memcpy(&rtp_addr, &caddr, sizeof(rtp_addr)); - else { + have_rtp_addr = true; + } else { nf_ct_helper_log(skb, ct, "cannot parse SDP message"); return NF_DROP; } @@ -1125,7 +1131,7 @@ static int process_sdp(struct sk_buff *skb, unsigned int protoff, /* Update session connection and owner addresses */ hooks = rcu_dereference(nf_nat_sip_hooks); - if (hooks && ct->status & IPS_NAT_MASK) + if (hooks && ct->status & IPS_NAT_MASK && have_rtp_addr) ret = hooks->sdp_session(skb, protoff, dataoff, dptr, datalen, sdpoff, &rtp_addr); @@ -1297,7 +1303,7 @@ static int process_register_request(struct sk_buff *skb, unsigned int protoff, nf_ct_expect_init(exp, SIP_EXPECT_SIGNALLING, nf_ct_l3num(ct), saddr, &daddr, proto, NULL, &port); exp->timeout.expires = sip_timeout * HZ; - exp->helper = helper; + rcu_assign_pointer(exp->helper, helper); exp->flags = NF_CT_EXPECT_PERMANENT | NF_CT_EXPECT_INACTIVE; hooks = rcu_dereference(nf_nat_sip_hooks); @@ -1534,11 +1540,12 @@ static int sip_help_tcp(struct sk_buff *skb, unsigned int protoff, { struct tcphdr *th, _tcph; unsigned int dataoff, datalen; - unsigned int matchoff, matchlen, clen; + unsigned int matchoff, matchlen; unsigned int msglen, origlen; const char *dptr, *end; s16 diff, tdiff = 0; int ret = NF_ACCEPT; + unsigned long clen; bool term; if (ctinfo != IP_CT_ESTABLISHED && @@ -1573,6 +1580,9 @@ static int sip_help_tcp(struct sk_buff *skb, unsigned int protoff, if (dptr + matchoff == end) break; + if (clen > datalen) + break; + term = false; for (; end + strlen("\r\n\r\n") <= dptr + datalen; end++) { if (end[0] == '\r' && end[1] == '\n' && diff --git a/net/netfilter/nf_flow_table_ip.c b/net/netfilter/nf_flow_table_ip.c index 78883343e5d686..458895e9e1f85a 100644 --- a/net/netfilter/nf_flow_table_ip.c +++ b/net/netfilter/nf_flow_table_ip.c @@ -576,6 +576,7 @@ static int nf_flow_encap_push(struct sk_buff *skb, switch (tuple->encap[i].proto) { case htons(ETH_P_8021Q): case htons(ETH_P_8021AD): + skb_reset_mac_header(skb); if (skb_vlan_push(skb, tuple->encap[i].proto, tuple->encap[i].id) < 0) return -1; diff --git a/net/netfilter/nf_flow_table_offload.c b/net/netfilter/nf_flow_table_offload.c index d8f7bfd60ac66b..77e46eae2025da 100644 --- a/net/netfilter/nf_flow_table_offload.c +++ b/net/netfilter/nf_flow_table_offload.c @@ -13,6 +13,8 @@ #include #include +#define NF_FLOW_RULE_ACTION_MAX 24 + static struct workqueue_struct *nf_flow_offload_add_wq; static struct workqueue_struct *nf_flow_offload_del_wq; static struct workqueue_struct *nf_flow_offload_stats_wq; @@ -215,7 +217,12 @@ static void flow_offload_mangle(struct flow_action_entry *entry, static inline struct flow_action_entry * flow_action_entry_next(struct nf_flow_rule *flow_rule) { - int i = flow_rule->rule->action.num_entries++; + int i; + + if (unlikely(flow_rule->rule->action.num_entries >= NF_FLOW_RULE_ACTION_MAX)) + return NULL; + + i = flow_rule->rule->action.num_entries++; return &flow_rule->rule->action.entries[i]; } @@ -233,6 +240,9 @@ static int flow_offload_eth_src(struct net *net, u32 mask, val; u16 val16; + if (!entry0 || !entry1) + return -E2BIG; + this_tuple = &flow->tuplehash[dir].tuple; switch (this_tuple->xmit_type) { @@ -283,6 +293,9 @@ static int flow_offload_eth_dst(struct net *net, u8 nud_state; u16 val16; + if (!entry0 || !entry1) + return -E2BIG; + this_tuple = &flow->tuplehash[dir].tuple; switch (this_tuple->xmit_type) { @@ -324,16 +337,19 @@ static int flow_offload_eth_dst(struct net *net, return 0; } -static void flow_offload_ipv4_snat(struct net *net, - const struct flow_offload *flow, - enum flow_offload_tuple_dir dir, - struct nf_flow_rule *flow_rule) +static int flow_offload_ipv4_snat(struct net *net, + const struct flow_offload *flow, + enum flow_offload_tuple_dir dir, + struct nf_flow_rule *flow_rule) { struct flow_action_entry *entry = flow_action_entry_next(flow_rule); u32 mask = ~htonl(0xffffffff); __be32 addr; u32 offset; + if (!entry) + return -E2BIG; + switch (dir) { case FLOW_OFFLOAD_DIR_ORIGINAL: addr = flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].tuple.dst_v4.s_addr; @@ -344,23 +360,27 @@ static void flow_offload_ipv4_snat(struct net *net, offset = offsetof(struct iphdr, daddr); break; default: - return; + return -EOPNOTSUPP; } flow_offload_mangle(entry, FLOW_ACT_MANGLE_HDR_TYPE_IP4, offset, &addr, &mask); + return 0; } -static void flow_offload_ipv4_dnat(struct net *net, - const struct flow_offload *flow, - enum flow_offload_tuple_dir dir, - struct nf_flow_rule *flow_rule) +static int flow_offload_ipv4_dnat(struct net *net, + const struct flow_offload *flow, + enum flow_offload_tuple_dir dir, + struct nf_flow_rule *flow_rule) { struct flow_action_entry *entry = flow_action_entry_next(flow_rule); u32 mask = ~htonl(0xffffffff); __be32 addr; u32 offset; + if (!entry) + return -E2BIG; + switch (dir) { case FLOW_OFFLOAD_DIR_ORIGINAL: addr = flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].tuple.src_v4.s_addr; @@ -371,14 +391,15 @@ static void flow_offload_ipv4_dnat(struct net *net, offset = offsetof(struct iphdr, saddr); break; default: - return; + return -EOPNOTSUPP; } flow_offload_mangle(entry, FLOW_ACT_MANGLE_HDR_TYPE_IP4, offset, &addr, &mask); + return 0; } -static void flow_offload_ipv6_mangle(struct nf_flow_rule *flow_rule, +static int flow_offload_ipv6_mangle(struct nf_flow_rule *flow_rule, unsigned int offset, const __be32 *addr, const __be32 *mask) { @@ -387,15 +408,20 @@ static void flow_offload_ipv6_mangle(struct nf_flow_rule *flow_rule, for (i = 0; i < sizeof(struct in6_addr) / sizeof(u32); i++) { entry = flow_action_entry_next(flow_rule); + if (!entry) + return -E2BIG; + flow_offload_mangle(entry, FLOW_ACT_MANGLE_HDR_TYPE_IP6, offset + i * sizeof(u32), &addr[i], mask); } + + return 0; } -static void flow_offload_ipv6_snat(struct net *net, - const struct flow_offload *flow, - enum flow_offload_tuple_dir dir, - struct nf_flow_rule *flow_rule) +static int flow_offload_ipv6_snat(struct net *net, + const struct flow_offload *flow, + enum flow_offload_tuple_dir dir, + struct nf_flow_rule *flow_rule) { u32 mask = ~htonl(0xffffffff); const __be32 *addr; @@ -411,16 +437,16 @@ static void flow_offload_ipv6_snat(struct net *net, offset = offsetof(struct ipv6hdr, daddr); break; default: - return; + return -EOPNOTSUPP; } - flow_offload_ipv6_mangle(flow_rule, offset, addr, &mask); + return flow_offload_ipv6_mangle(flow_rule, offset, addr, &mask); } -static void flow_offload_ipv6_dnat(struct net *net, - const struct flow_offload *flow, - enum flow_offload_tuple_dir dir, - struct nf_flow_rule *flow_rule) +static int flow_offload_ipv6_dnat(struct net *net, + const struct flow_offload *flow, + enum flow_offload_tuple_dir dir, + struct nf_flow_rule *flow_rule) { u32 mask = ~htonl(0xffffffff); const __be32 *addr; @@ -436,10 +462,10 @@ static void flow_offload_ipv6_dnat(struct net *net, offset = offsetof(struct ipv6hdr, saddr); break; default: - return; + return -EOPNOTSUPP; } - flow_offload_ipv6_mangle(flow_rule, offset, addr, &mask); + return flow_offload_ipv6_mangle(flow_rule, offset, addr, &mask); } static int flow_offload_l4proto(const struct flow_offload *flow) @@ -461,15 +487,18 @@ static int flow_offload_l4proto(const struct flow_offload *flow) return type; } -static void flow_offload_port_snat(struct net *net, - const struct flow_offload *flow, - enum flow_offload_tuple_dir dir, - struct nf_flow_rule *flow_rule) +static int flow_offload_port_snat(struct net *net, + const struct flow_offload *flow, + enum flow_offload_tuple_dir dir, + struct nf_flow_rule *flow_rule) { struct flow_action_entry *entry = flow_action_entry_next(flow_rule); u32 mask, port; u32 offset; + if (!entry) + return -E2BIG; + switch (dir) { case FLOW_OFFLOAD_DIR_ORIGINAL: port = ntohs(flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].tuple.dst_port); @@ -484,22 +513,26 @@ static void flow_offload_port_snat(struct net *net, mask = ~htonl(0xffff); break; default: - return; + return -EOPNOTSUPP; } flow_offload_mangle(entry, flow_offload_l4proto(flow), offset, &port, &mask); + return 0; } -static void flow_offload_port_dnat(struct net *net, - const struct flow_offload *flow, - enum flow_offload_tuple_dir dir, - struct nf_flow_rule *flow_rule) +static int flow_offload_port_dnat(struct net *net, + const struct flow_offload *flow, + enum flow_offload_tuple_dir dir, + struct nf_flow_rule *flow_rule) { struct flow_action_entry *entry = flow_action_entry_next(flow_rule); u32 mask, port; u32 offset; + if (!entry) + return -E2BIG; + switch (dir) { case FLOW_OFFLOAD_DIR_ORIGINAL: port = ntohs(flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].tuple.src_port); @@ -514,20 +547,24 @@ static void flow_offload_port_dnat(struct net *net, mask = ~htonl(0xffff0000); break; default: - return; + return -EOPNOTSUPP; } flow_offload_mangle(entry, flow_offload_l4proto(flow), offset, &port, &mask); + return 0; } -static void flow_offload_ipv4_checksum(struct net *net, - const struct flow_offload *flow, - struct nf_flow_rule *flow_rule) +static int flow_offload_ipv4_checksum(struct net *net, + const struct flow_offload *flow, + struct nf_flow_rule *flow_rule) { u8 protonum = flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.l4proto; struct flow_action_entry *entry = flow_action_entry_next(flow_rule); + if (!entry) + return -E2BIG; + entry->id = FLOW_ACTION_CSUM; entry->csum_flags = TCA_CSUM_UPDATE_FLAG_IPV4HDR; @@ -539,12 +576,14 @@ static void flow_offload_ipv4_checksum(struct net *net, entry->csum_flags |= TCA_CSUM_UPDATE_FLAG_UDP; break; } + + return 0; } -static void flow_offload_redirect(struct net *net, - const struct flow_offload *flow, - enum flow_offload_tuple_dir dir, - struct nf_flow_rule *flow_rule) +static int flow_offload_redirect(struct net *net, + const struct flow_offload *flow, + enum flow_offload_tuple_dir dir, + struct nf_flow_rule *flow_rule) { const struct flow_offload_tuple *this_tuple, *other_tuple; struct flow_action_entry *entry; @@ -562,21 +601,28 @@ static void flow_offload_redirect(struct net *net, ifindex = other_tuple->iifidx; break; default: - return; + return -EOPNOTSUPP; } dev = dev_get_by_index(net, ifindex); if (!dev) - return; + return -ENODEV; entry = flow_action_entry_next(flow_rule); + if (!entry) { + dev_put(dev); + return -E2BIG; + } + entry->id = FLOW_ACTION_REDIRECT; entry->dev = dev; + + return 0; } -static void flow_offload_encap_tunnel(const struct flow_offload *flow, - enum flow_offload_tuple_dir dir, - struct nf_flow_rule *flow_rule) +static int flow_offload_encap_tunnel(const struct flow_offload *flow, + enum flow_offload_tuple_dir dir, + struct nf_flow_rule *flow_rule) { const struct flow_offload_tuple *this_tuple; struct flow_action_entry *entry; @@ -584,7 +630,7 @@ static void flow_offload_encap_tunnel(const struct flow_offload *flow, this_tuple = &flow->tuplehash[dir].tuple; if (this_tuple->xmit_type == FLOW_OFFLOAD_XMIT_DIRECT) - return; + return 0; dst = this_tuple->dst_cache; if (dst && dst->lwtstate) { @@ -593,15 +639,19 @@ static void flow_offload_encap_tunnel(const struct flow_offload *flow, tun_info = lwt_tun_info(dst->lwtstate); if (tun_info && (tun_info->mode & IP_TUNNEL_INFO_TX)) { entry = flow_action_entry_next(flow_rule); + if (!entry) + return -E2BIG; entry->id = FLOW_ACTION_TUNNEL_ENCAP; entry->tunnel = tun_info; } } + + return 0; } -static void flow_offload_decap_tunnel(const struct flow_offload *flow, - enum flow_offload_tuple_dir dir, - struct nf_flow_rule *flow_rule) +static int flow_offload_decap_tunnel(const struct flow_offload *flow, + enum flow_offload_tuple_dir dir, + struct nf_flow_rule *flow_rule) { const struct flow_offload_tuple *other_tuple; struct flow_action_entry *entry; @@ -609,7 +659,7 @@ static void flow_offload_decap_tunnel(const struct flow_offload *flow, other_tuple = &flow->tuplehash[!dir].tuple; if (other_tuple->xmit_type == FLOW_OFFLOAD_XMIT_DIRECT) - return; + return 0; dst = other_tuple->dst_cache; if (dst && dst->lwtstate) { @@ -618,9 +668,13 @@ static void flow_offload_decap_tunnel(const struct flow_offload *flow, tun_info = lwt_tun_info(dst->lwtstate); if (tun_info && (tun_info->mode & IP_TUNNEL_INFO_TX)) { entry = flow_action_entry_next(flow_rule); + if (!entry) + return -E2BIG; entry->id = FLOW_ACTION_TUNNEL_DECAP; } } + + return 0; } static int @@ -632,8 +686,9 @@ nf_flow_rule_route_common(struct net *net, const struct flow_offload *flow, const struct flow_offload_tuple *tuple; int i; - flow_offload_decap_tunnel(flow, dir, flow_rule); - flow_offload_encap_tunnel(flow, dir, flow_rule); + if (flow_offload_decap_tunnel(flow, dir, flow_rule) < 0 || + flow_offload_encap_tunnel(flow, dir, flow_rule) < 0) + return -1; if (flow_offload_eth_src(net, flow, dir, flow_rule) < 0 || flow_offload_eth_dst(net, flow, dir, flow_rule) < 0) @@ -649,6 +704,8 @@ nf_flow_rule_route_common(struct net *net, const struct flow_offload *flow, if (tuple->encap[i].proto == htons(ETH_P_8021Q)) { entry = flow_action_entry_next(flow_rule); + if (!entry) + return -1; entry->id = FLOW_ACTION_VLAN_POP; } } @@ -662,6 +719,8 @@ nf_flow_rule_route_common(struct net *net, const struct flow_offload *flow, continue; entry = flow_action_entry_next(flow_rule); + if (!entry) + return -1; switch (other_tuple->encap[i].proto) { case htons(ETH_P_PPP_SES): @@ -687,18 +746,22 @@ int nf_flow_rule_route_ipv4(struct net *net, struct flow_offload *flow, return -1; if (test_bit(NF_FLOW_SNAT, &flow->flags)) { - flow_offload_ipv4_snat(net, flow, dir, flow_rule); - flow_offload_port_snat(net, flow, dir, flow_rule); + if (flow_offload_ipv4_snat(net, flow, dir, flow_rule) < 0 || + flow_offload_port_snat(net, flow, dir, flow_rule) < 0) + return -1; } if (test_bit(NF_FLOW_DNAT, &flow->flags)) { - flow_offload_ipv4_dnat(net, flow, dir, flow_rule); - flow_offload_port_dnat(net, flow, dir, flow_rule); + if (flow_offload_ipv4_dnat(net, flow, dir, flow_rule) < 0 || + flow_offload_port_dnat(net, flow, dir, flow_rule) < 0) + return -1; } if (test_bit(NF_FLOW_SNAT, &flow->flags) || test_bit(NF_FLOW_DNAT, &flow->flags)) - flow_offload_ipv4_checksum(net, flow, flow_rule); + if (flow_offload_ipv4_checksum(net, flow, flow_rule) < 0) + return -1; - flow_offload_redirect(net, flow, dir, flow_rule); + if (flow_offload_redirect(net, flow, dir, flow_rule) < 0) + return -1; return 0; } @@ -712,22 +775,23 @@ int nf_flow_rule_route_ipv6(struct net *net, struct flow_offload *flow, return -1; if (test_bit(NF_FLOW_SNAT, &flow->flags)) { - flow_offload_ipv6_snat(net, flow, dir, flow_rule); - flow_offload_port_snat(net, flow, dir, flow_rule); + if (flow_offload_ipv6_snat(net, flow, dir, flow_rule) < 0 || + flow_offload_port_snat(net, flow, dir, flow_rule) < 0) + return -1; } if (test_bit(NF_FLOW_DNAT, &flow->flags)) { - flow_offload_ipv6_dnat(net, flow, dir, flow_rule); - flow_offload_port_dnat(net, flow, dir, flow_rule); + if (flow_offload_ipv6_dnat(net, flow, dir, flow_rule) < 0 || + flow_offload_port_dnat(net, flow, dir, flow_rule) < 0) + return -1; } - flow_offload_redirect(net, flow, dir, flow_rule); + if (flow_offload_redirect(net, flow, dir, flow_rule) < 0) + return -1; return 0; } EXPORT_SYMBOL_GPL(nf_flow_rule_route_ipv6); -#define NF_FLOW_RULE_ACTION_MAX 16 - static struct nf_flow_rule * nf_flow_offload_rule_alloc(struct net *net, const struct flow_offload_work *offload, diff --git a/net/netfilter/nf_tables_api.c b/net/netfilter/nf_tables_api.c index be92750e2af3ad..0349787e18465b 100644 --- a/net/netfilter/nf_tables_api.c +++ b/net/netfilter/nf_tables_api.c @@ -828,10 +828,14 @@ static void nft_map_catchall_deactivate(const struct nft_ctx *ctx, nft_set_elem_change_active(ctx->net, set, ext); nft_setelem_data_deactivate(ctx->net, set, catchall->elem); - break; } } +/* Use NFT_ITER_UPDATE iterator even if this may be called from the preparation + * phase, the set clone might already exist from a previous command, or it might + * be a set that is going away and does not require a clone. The netns and + * netlink release paths also need to work on the live set. + */ static void nft_map_deactivate(const struct nft_ctx *ctx, struct nft_set *set) { struct nft_set_iter iter = { @@ -2822,6 +2826,7 @@ static int nf_tables_addchain(struct nft_ctx *ctx, u8 family, u8 policy, err_register_hook: nft_chain_del(chain); + synchronize_rcu(); err_chain_add: nft_trans_destroy(trans); err_trans: @@ -3900,23 +3905,6 @@ static int nf_tables_dump_rules(struct sk_buff *skb, return skb->len; } -static int nf_tables_dumpreset_rules(struct sk_buff *skb, - struct netlink_callback *cb) -{ - struct nftables_pernet *nft_net = nft_pernet(sock_net(skb->sk)); - int ret; - - /* Mutex is held is to prevent that two concurrent dump-and-reset calls - * do not underrun counters and quotas. The commit_mutex is used for - * the lack a better lock, this is not transaction path. - */ - mutex_lock(&nft_net->commit_mutex); - ret = nf_tables_dump_rules(skb, cb); - mutex_unlock(&nft_net->commit_mutex); - - return ret; -} - static int nf_tables_dump_rules_start(struct netlink_callback *cb) { struct nft_rule_dump_ctx *ctx = (void *)cb->ctx; @@ -3936,16 +3924,10 @@ static int nf_tables_dump_rules_start(struct netlink_callback *cb) return -ENOMEM; } } - return 0; -} - -static int nf_tables_dumpreset_rules_start(struct netlink_callback *cb) -{ - struct nft_rule_dump_ctx *ctx = (void *)cb->ctx; - - ctx->reset = true; + if (NFNL_MSG_TYPE(cb->nlh->nlmsg_type) == NFT_MSG_GETRULE_RESET) + ctx->reset = true; - return nf_tables_dump_rules_start(cb); + return 0; } static int nf_tables_dump_rules_done(struct netlink_callback *cb) @@ -4011,6 +3993,8 @@ static int nf_tables_getrule(struct sk_buff *skb, const struct nfnl_info *info, u32 portid = NETLINK_CB(skb).portid; struct net *net = info->net; struct sk_buff *skb2; + bool reset = false; + char *buf; if (info->nlh->nlmsg_flags & NLM_F_DUMP) { struct netlink_dump_control c = { @@ -4024,47 +4008,16 @@ static int nf_tables_getrule(struct sk_buff *skb, const struct nfnl_info *info, return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c); } - skb2 = nf_tables_getrule_single(portid, info, nla, false); - if (IS_ERR(skb2)) - return PTR_ERR(skb2); - - return nfnetlink_unicast(skb2, net, portid); -} - -static int nf_tables_getrule_reset(struct sk_buff *skb, - const struct nfnl_info *info, - const struct nlattr * const nla[]) -{ - struct nftables_pernet *nft_net = nft_pernet(info->net); - u32 portid = NETLINK_CB(skb).portid; - struct net *net = info->net; - struct sk_buff *skb2; - char *buf; - - if (info->nlh->nlmsg_flags & NLM_F_DUMP) { - struct netlink_dump_control c = { - .start= nf_tables_dumpreset_rules_start, - .dump = nf_tables_dumpreset_rules, - .done = nf_tables_dump_rules_done, - .module = THIS_MODULE, - .data = (void *)nla, - }; - - return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c); - } - - if (!try_module_get(THIS_MODULE)) - return -EINVAL; - rcu_read_unlock(); - mutex_lock(&nft_net->commit_mutex); - skb2 = nf_tables_getrule_single(portid, info, nla, true); - mutex_unlock(&nft_net->commit_mutex); - rcu_read_lock(); - module_put(THIS_MODULE); + if (NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_GETRULE_RESET) + reset = true; + skb2 = nf_tables_getrule_single(portid, info, nla, reset); if (IS_ERR(skb2)) return PTR_ERR(skb2); + if (!reset) + return nfnetlink_unicast(skb2, net, portid); + buf = kasprintf(GFP_ATOMIC, "%.*s:%u", nla_len(nla[NFTA_RULE_TABLE]), (char *)nla_data(nla[NFTA_RULE_TABLE]), @@ -5919,7 +5872,6 @@ static void nft_map_catchall_activate(const struct nft_ctx *ctx, nft_clear(ctx->net, ext); nft_setelem_data_activate(ctx->net, set, catchall->elem); - break; } } @@ -6323,6 +6275,10 @@ static int nf_tables_dump_set(struct sk_buff *skb, struct netlink_callback *cb) nla_nest_end(skb, nest); nlmsg_end(skb, nlh); + if (dump_ctx->reset && args.iter.count > args.iter.skip) + audit_log_nft_set_reset(table, cb->seq, + args.iter.count - args.iter.skip); + rcu_read_unlock(); if (args.iter.err && args.iter.err != -EMSGSIZE) @@ -6338,26 +6294,6 @@ static int nf_tables_dump_set(struct sk_buff *skb, struct netlink_callback *cb) return -ENOSPC; } -static int nf_tables_dumpreset_set(struct sk_buff *skb, - struct netlink_callback *cb) -{ - struct nftables_pernet *nft_net = nft_pernet(sock_net(skb->sk)); - struct nft_set_dump_ctx *dump_ctx = cb->data; - int ret, skip = cb->args[0]; - - mutex_lock(&nft_net->commit_mutex); - - ret = nf_tables_dump_set(skb, cb); - - if (cb->args[0] > skip) - audit_log_nft_set_reset(dump_ctx->ctx.table, cb->seq, - cb->args[0] - skip); - - mutex_unlock(&nft_net->commit_mutex); - - return ret; -} - static int nf_tables_dump_set_start(struct netlink_callback *cb) { struct nft_set_dump_ctx *dump_ctx = cb->data; @@ -6601,8 +6537,13 @@ static int nf_tables_getsetelem(struct sk_buff *skb, { struct netlink_ext_ack *extack = info->extack; struct nft_set_dump_ctx dump_ctx; + int rem, err = 0, nelems = 0; + struct net *net = info->net; struct nlattr *attr; - int rem, err = 0; + bool reset = false; + + if (NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_GETSETELEM_RESET) + reset = true; if (info->nlh->nlmsg_flags & NLM_F_DUMP) { struct netlink_dump_control c = { @@ -6612,7 +6553,7 @@ static int nf_tables_getsetelem(struct sk_buff *skb, .module = THIS_MODULE, }; - err = nft_set_dump_ctx_init(&dump_ctx, skb, info, nla, false); + err = nft_set_dump_ctx_init(&dump_ctx, skb, info, nla, reset); if (err) return err; @@ -6623,75 +6564,21 @@ static int nf_tables_getsetelem(struct sk_buff *skb, if (!nla[NFTA_SET_ELEM_LIST_ELEMENTS]) return -EINVAL; - err = nft_set_dump_ctx_init(&dump_ctx, skb, info, nla, false); + err = nft_set_dump_ctx_init(&dump_ctx, skb, info, nla, reset); if (err) return err; nla_for_each_nested(attr, nla[NFTA_SET_ELEM_LIST_ELEMENTS], rem) { - err = nft_get_set_elem(&dump_ctx.ctx, dump_ctx.set, attr, false); - if (err < 0) { - NL_SET_BAD_ATTR(extack, attr); - break; - } - } - - return err; -} - -static int nf_tables_getsetelem_reset(struct sk_buff *skb, - const struct nfnl_info *info, - const struct nlattr * const nla[]) -{ - struct nftables_pernet *nft_net = nft_pernet(info->net); - struct netlink_ext_ack *extack = info->extack; - struct nft_set_dump_ctx dump_ctx; - int rem, err = 0, nelems = 0; - struct nlattr *attr; - - if (info->nlh->nlmsg_flags & NLM_F_DUMP) { - struct netlink_dump_control c = { - .start = nf_tables_dump_set_start, - .dump = nf_tables_dumpreset_set, - .done = nf_tables_dump_set_done, - .module = THIS_MODULE, - }; - - err = nft_set_dump_ctx_init(&dump_ctx, skb, info, nla, true); - if (err) - return err; - - c.data = &dump_ctx; - return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c); - } - - if (!nla[NFTA_SET_ELEM_LIST_ELEMENTS]) - return -EINVAL; - - if (!try_module_get(THIS_MODULE)) - return -EINVAL; - rcu_read_unlock(); - mutex_lock(&nft_net->commit_mutex); - rcu_read_lock(); - - err = nft_set_dump_ctx_init(&dump_ctx, skb, info, nla, true); - if (err) - goto out_unlock; - - nla_for_each_nested(attr, nla[NFTA_SET_ELEM_LIST_ELEMENTS], rem) { - err = nft_get_set_elem(&dump_ctx.ctx, dump_ctx.set, attr, true); + err = nft_get_set_elem(&dump_ctx.ctx, dump_ctx.set, attr, reset); if (err < 0) { NL_SET_BAD_ATTR(extack, attr); break; } nelems++; } - audit_log_nft_set_reset(dump_ctx.ctx.table, nft_base_seq(info->net), nelems); - -out_unlock: - rcu_read_unlock(); - mutex_unlock(&nft_net->commit_mutex); - rcu_read_lock(); - module_put(THIS_MODULE); + if (reset) + audit_log_nft_set_reset(dump_ctx.ctx.table, nft_base_seq(net), + nelems); return err; } @@ -6857,8 +6744,8 @@ static void __nft_set_elem_expr_destroy(const struct nft_ctx *ctx, } } -static void nft_set_elem_expr_destroy(const struct nft_ctx *ctx, - struct nft_set_elem_expr *elem_expr) +void nft_set_elem_expr_destroy(const struct nft_ctx *ctx, + struct nft_set_elem_expr *elem_expr) { struct nft_expr *expr; u32 size; @@ -7285,6 +7172,7 @@ static int nft_add_set_elem(struct nft_ctx *ctx, struct nft_set *set, struct nft_data_desc desc; enum nft_registers dreg; struct nft_trans *trans; + bool set_full = false; u64 expiration; u64 timeout; int err, i; @@ -7571,10 +7459,18 @@ static int nft_add_set_elem(struct nft_ctx *ctx, struct nft_set *set, if (err < 0) goto err_elem_free; + if (!(flags & NFT_SET_ELEM_CATCHALL)) { + unsigned int max = nft_set_maxsize(set), nelems; + + nelems = atomic_inc_return(&set->nelems); + if (nelems > max) + set_full = true; + } + trans = nft_trans_elem_alloc(ctx, NFT_MSG_NEWSETELEM, set); if (trans == NULL) { err = -ENOMEM; - goto err_elem_free; + goto err_set_size; } ext->genmask = nft_genmask_cur(ctx->net); @@ -7626,7 +7522,7 @@ static int nft_add_set_elem(struct nft_ctx *ctx, struct nft_set *set, ue->priv = elem_priv; nft_trans_commit_list_add_elem(ctx->net, trans); - goto err_elem_free; + goto err_set_size; } } } @@ -7635,27 +7531,25 @@ static int nft_add_set_elem(struct nft_ctx *ctx, struct nft_set *set, * and an existing one. */ err = -EEXIST; + } else if (err == -ECANCELED) { + /* ECANCELED reports an existing nul-element in + * interval sets. + */ + err = 0; } goto err_element_clash; } - if (!(flags & NFT_SET_ELEM_CATCHALL)) { - unsigned int max = nft_set_maxsize(set); - - if (!atomic_add_unless(&set->nelems, 1, max)) { - err = -ENFILE; - goto err_set_full; - } - } - nft_trans_container_elem(trans)->elems[0].priv = elem.priv; nft_trans_commit_list_add_elem(ctx->net, trans); - return 0; -err_set_full: - nft_setelem_remove(ctx->net, set, elem.priv); + return set_full ? -ENFILE : 0; + err_element_clash: kfree(trans); +err_set_size: + if (!(flags & NFT_SET_ELEM_CATCHALL)) + atomic_dec(&set->nelems); err_elem_free: nf_tables_set_elem_destroy(ctx, set, elem.priv); err_parse_data: @@ -7806,7 +7700,8 @@ static bool nft_trans_elems_new_abort(const struct nft_ctx *ctx, continue; } - if (!te->set->ops->abort || nft_setelem_is_catchall(te->set, te->elems[i].priv)) + if (!te->set->ops->abort_skip_removal || + nft_setelem_is_catchall(te->set, te->elems[i].priv)) nft_setelem_remove(ctx->net, te->set, te->elems[i].priv); if (!nft_setelem_is_catchall(te->set, te->elems[i].priv)) @@ -7999,9 +7894,12 @@ static int nft_set_catchall_flush(const struct nft_ctx *ctx, static int nft_set_flush(struct nft_ctx *ctx, struct nft_set *set, u8 genmask) { + /* The set backend might need to clone the set, do it now from the + * preparation phase, use NFT_ITER_UPDATE_CLONE iterator type. + */ struct nft_set_iter iter = { .genmask = genmask, - .type = NFT_ITER_UPDATE, + .type = NFT_ITER_UPDATE_CLONE, .fn = nft_setelem_flush, }; @@ -8544,19 +8442,6 @@ static int nf_tables_dump_obj(struct sk_buff *skb, struct netlink_callback *cb) return skb->len; } -static int nf_tables_dumpreset_obj(struct sk_buff *skb, - struct netlink_callback *cb) -{ - struct nftables_pernet *nft_net = nft_pernet(sock_net(skb->sk)); - int ret; - - mutex_lock(&nft_net->commit_mutex); - ret = nf_tables_dump_obj(skb, cb); - mutex_unlock(&nft_net->commit_mutex); - - return ret; -} - static int nf_tables_dump_obj_start(struct netlink_callback *cb) { struct nft_obj_dump_ctx *ctx = (void *)cb->ctx; @@ -8573,16 +8458,10 @@ static int nf_tables_dump_obj_start(struct netlink_callback *cb) if (nla[NFTA_OBJ_TYPE]) ctx->type = ntohl(nla_get_be32(nla[NFTA_OBJ_TYPE])); - return 0; -} - -static int nf_tables_dumpreset_obj_start(struct netlink_callback *cb) -{ - struct nft_obj_dump_ctx *ctx = (void *)cb->ctx; - - ctx->reset = true; + if (NFNL_MSG_TYPE(cb->nlh->nlmsg_type) == NFT_MSG_GETOBJ_RESET) + ctx->reset = true; - return nf_tables_dump_obj_start(cb); + return 0; } static int nf_tables_dump_obj_done(struct netlink_callback *cb) @@ -8644,42 +8523,16 @@ nf_tables_getobj_single(u32 portid, const struct nfnl_info *info, static int nf_tables_getobj(struct sk_buff *skb, const struct nfnl_info *info, const struct nlattr * const nla[]) { - u32 portid = NETLINK_CB(skb).portid; - struct sk_buff *skb2; - - if (info->nlh->nlmsg_flags & NLM_F_DUMP) { - struct netlink_dump_control c = { - .start = nf_tables_dump_obj_start, - .dump = nf_tables_dump_obj, - .done = nf_tables_dump_obj_done, - .module = THIS_MODULE, - .data = (void *)nla, - }; - - return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c); - } - - skb2 = nf_tables_getobj_single(portid, info, nla, false); - if (IS_ERR(skb2)) - return PTR_ERR(skb2); - - return nfnetlink_unicast(skb2, info->net, portid); -} - -static int nf_tables_getobj_reset(struct sk_buff *skb, - const struct nfnl_info *info, - const struct nlattr * const nla[]) -{ - struct nftables_pernet *nft_net = nft_pernet(info->net); u32 portid = NETLINK_CB(skb).portid; struct net *net = info->net; struct sk_buff *skb2; + bool reset = false; char *buf; if (info->nlh->nlmsg_flags & NLM_F_DUMP) { struct netlink_dump_control c = { - .start = nf_tables_dumpreset_obj_start, - .dump = nf_tables_dumpreset_obj, + .start = nf_tables_dump_obj_start, + .dump = nf_tables_dump_obj, .done = nf_tables_dump_obj_done, .module = THIS_MODULE, .data = (void *)nla, @@ -8688,18 +8541,16 @@ static int nf_tables_getobj_reset(struct sk_buff *skb, return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c); } - if (!try_module_get(THIS_MODULE)) - return -EINVAL; - rcu_read_unlock(); - mutex_lock(&nft_net->commit_mutex); - skb2 = nf_tables_getobj_single(portid, info, nla, true); - mutex_unlock(&nft_net->commit_mutex); - rcu_read_lock(); - module_put(THIS_MODULE); + if (NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_GETOBJ_RESET) + reset = true; + skb2 = nf_tables_getobj_single(portid, info, nla, reset); if (IS_ERR(skb2)) return PTR_ERR(skb2); + if (!reset) + return nfnetlink_unicast(skb2, net, NETLINK_CB(skb).portid); + buf = kasprintf(GFP_ATOMIC, "%.*s:%u", nla_len(nla[NFTA_OBJ_TABLE]), (char *)nla_data(nla[NFTA_OBJ_TABLE]), @@ -9352,6 +9203,7 @@ static int nf_tables_newflowtable(struct sk_buff *skb, return 0; err_flowtable_hooks: + synchronize_rcu(); nft_trans_destroy(trans); err_flowtable_trans: nft_hooks_destroy(&flowtable->hook_list); @@ -9822,7 +9674,7 @@ static int nft_flowtable_event(unsigned long event, struct net_device *dev, break; case NETDEV_REGISTER: /* NOP if not matching or already registered */ - if (!match || (changename && ops)) + if (!match || ops) continue; ops = kzalloc(sizeof(struct nf_hook_ops), @@ -10017,7 +9869,7 @@ static const struct nfnl_callback nf_tables_cb[NFT_MSG_MAX] = { .policy = nft_rule_policy, }, [NFT_MSG_GETRULE_RESET] = { - .call = nf_tables_getrule_reset, + .call = nf_tables_getrule, .type = NFNL_CB_RCU, .attr_count = NFTA_RULE_MAX, .policy = nft_rule_policy, @@ -10071,7 +9923,7 @@ static const struct nfnl_callback nf_tables_cb[NFT_MSG_MAX] = { .policy = nft_set_elem_list_policy, }, [NFT_MSG_GETSETELEM_RESET] = { - .call = nf_tables_getsetelem_reset, + .call = nf_tables_getsetelem, .type = NFNL_CB_RCU, .attr_count = NFTA_SET_ELEM_LIST_MAX, .policy = nft_set_elem_list_policy, @@ -10117,7 +9969,7 @@ static const struct nfnl_callback nf_tables_cb[NFT_MSG_MAX] = { .policy = nft_obj_policy, }, [NFT_MSG_GETOBJ_RESET] = { - .call = nf_tables_getobj_reset, + .call = nf_tables_getobj, .type = NFNL_CB_RCU, .attr_count = NFTA_OBJ_MAX, .policy = nft_obj_policy, @@ -10627,11 +10479,6 @@ static void nft_trans_gc_queue_work(struct nft_trans_gc *trans) schedule_work(&trans_gc_work); } -static int nft_trans_gc_space(struct nft_trans_gc *trans) -{ - return NFT_TRANS_GC_BATCHCOUNT - trans->count; -} - struct nft_trans_gc *nft_trans_gc_queue_async(struct nft_trans_gc *gc, unsigned int gc_seq, gfp_t gfp) { @@ -11536,6 +11383,13 @@ static int nf_tables_abort(struct net *net, struct sk_buff *skb, ret = __nf_tables_abort(net, action); nft_gc_seq_end(nft_net, gc_seq); + if (action == NFNL_ABORT_NONE) { + struct nft_table *table; + + list_for_each_entry(table, &nft_net->tables, list) + table->validate_state = NFT_VALIDATE_SKIP; + } + WARN_ON_ONCE(!list_empty(&nft_net->commit_list)); /* module autoload needs to happen after GC sequence update because it @@ -11813,8 +11667,6 @@ static int nft_verdict_init(const struct nft_ctx *ctx, struct nft_data *data, switch (data->verdict.code) { case NF_ACCEPT: case NF_DROP: - case NF_QUEUE: - break; case NFT_CONTINUE: case NFT_BREAK: case NFT_RETURN: @@ -11849,6 +11701,11 @@ static int nft_verdict_init(const struct nft_ctx *ctx, struct nft_data *data, data->verdict.chain = chain; break; + case NF_QUEUE: + /* The nft_queue expression is used for this purpose, an + * immediate NF_QUEUE verdict should not ever be seen here. + */ + fallthrough; default: return -EINVAL; } diff --git a/net/netfilter/nfnetlink_cthelper.c b/net/netfilter/nfnetlink_cthelper.c index 97248963a7d3b5..71a248cca746ae 100644 --- a/net/netfilter/nfnetlink_cthelper.c +++ b/net/netfilter/nfnetlink_cthelper.c @@ -603,10 +603,10 @@ nfnl_cthelper_dump_table(struct sk_buff *skb, struct netlink_callback *cb) goto out; } } - } - if (cb->args[1]) { - cb->args[1] = 0; - goto restart; + if (cb->args[1]) { + cb->args[1] = 0; + goto restart; + } } out: rcu_read_unlock(); diff --git a/net/netfilter/nfnetlink_log.c b/net/netfilter/nfnetlink_log.c index bfcb9cd335bff5..b1f3eda85989cd 100644 --- a/net/netfilter/nfnetlink_log.c +++ b/net/netfilter/nfnetlink_log.c @@ -361,10 +361,10 @@ static void __nfulnl_send(struct nfulnl_instance *inst) { if (inst->qlen > 1) { - struct nlmsghdr *nlh = nlmsg_put(inst->skb, 0, 0, - NLMSG_DONE, - sizeof(struct nfgenmsg), - 0); + struct nlmsghdr *nlh = nfnl_msg_put(inst->skb, 0, 0, + NLMSG_DONE, 0, + AF_UNSPEC, NFNETLINK_V0, + htons(inst->group_num)); if (WARN_ONCE(!nlh, "bad nlskb size: %u, tailroom %d\n", inst->skb->len, skb_tailroom(inst->skb))) { kfree_skb(inst->skb); @@ -647,15 +647,11 @@ __build_packet_message(struct nfnl_log_net *log, if (data_len) { struct nlattr *nla; - int size = nla_attr_size(data_len); - if (skb_tailroom(inst->skb) < nla_total_size(data_len)) + nla = nla_reserve(inst->skb, NFULA_PAYLOAD, data_len); + if (!nla) goto nla_put_failure; - nla = skb_put(inst->skb, nla_total_size(data_len)); - nla->nla_type = NFULA_PAYLOAD; - nla->nla_len = size; - if (skb_copy_bits(skb, 0, nla_data(nla), data_len)) BUG(); } @@ -730,7 +726,7 @@ nfulnl_log_packet(struct net *net, + nla_total_size(plen) /* prefix */ + nla_total_size(sizeof(struct nfulnl_msg_packet_hw)) + nla_total_size(sizeof(struct nfulnl_msg_packet_timestamp)) - + nla_total_size(sizeof(struct nfgenmsg)); /* NLMSG_DONE */ + + nlmsg_total_size(sizeof(struct nfgenmsg)); /* NLMSG_DONE */ if (in && skb_mac_header_was_set(skb)) { size += nla_total_size(skb->dev->hard_header_len) diff --git a/net/netfilter/nfnetlink_osf.c b/net/netfilter/nfnetlink_osf.c index c0fc431991e88d..9fc9544d4bc53d 100644 --- a/net/netfilter/nfnetlink_osf.c +++ b/net/netfilter/nfnetlink_osf.c @@ -302,7 +302,9 @@ static int nfnl_osf_add_callback(struct sk_buff *skb, { struct nf_osf_user_finger *f; struct nf_osf_finger *kf = NULL, *sf; + unsigned int tot_opt_len = 0; int err = 0; + int i; if (!capable(CAP_NET_ADMIN)) return -EPERM; @@ -318,6 +320,17 @@ static int nfnl_osf_add_callback(struct sk_buff *skb, if (f->opt_num > ARRAY_SIZE(f->opt)) return -EINVAL; + for (i = 0; i < f->opt_num; i++) { + if (!f->opt[i].length || f->opt[i].length > MAX_IPOPTLEN) + return -EINVAL; + if (f->opt[i].kind == OSFOPT_MSS && f->opt[i].length < 4) + return -EINVAL; + + tot_opt_len += f->opt[i].length; + if (tot_opt_len > MAX_IPOPTLEN) + return -EINVAL; + } + if (!memchr(f->genre, 0, MAXGENRELEN) || !memchr(f->subtype, 0, MAXGENRELEN) || !memchr(f->version, 0, MAXGENRELEN)) diff --git a/net/netfilter/nfnetlink_queue.c b/net/netfilter/nfnetlink_queue.c index 8b7b39d8a10913..fe5942535245db 100644 --- a/net/netfilter/nfnetlink_queue.c +++ b/net/netfilter/nfnetlink_queue.c @@ -30,6 +30,8 @@ #include #include #include +#include +#include #include #include #include @@ -47,6 +49,8 @@ #endif #define NFQNL_QMAX_DEFAULT 1024 +#define NFQNL_HASH_MIN 8 +#define NFQNL_HASH_MAX 32768 /* We're using struct nlattr which has 16bit nla_len. Note that nla_len * includes the header length. Thus, the maximum packet length that we @@ -58,7 +62,8 @@ struct nfqnl_instance { struct hlist_node hlist; /* global list of queues */ - struct rcu_head rcu; + struct rhashtable nfqnl_packet_map; + struct rcu_work rwork; u32 peer_portid; unsigned int queue_maxlen; @@ -82,6 +87,7 @@ struct nfqnl_instance { typedef int (*nfqnl_cmpfn)(struct nf_queue_entry *, unsigned long); +static struct workqueue_struct *nfq_cleanup_wq __read_mostly; static unsigned int nfnl_queue_net_id __read_mostly; #define INSTANCE_BUCKETS 16 @@ -100,6 +106,15 @@ static inline u_int8_t instance_hashfn(u_int16_t queue_num) return ((queue_num >> 8) ^ queue_num) % INSTANCE_BUCKETS; } +static const struct rhashtable_params nfqnl_rhashtable_params = { + .head_offset = offsetof(struct nf_queue_entry, hash_node), + .key_offset = offsetof(struct nf_queue_entry, id), + .key_len = sizeof(u32), + .automatic_shrinking = true, + .min_size = NFQNL_HASH_MIN, + .max_size = NFQNL_HASH_MAX, +}; + static struct nfqnl_instance * instance_lookup(struct nfnl_queue_net *q, u_int16_t queue_num) { @@ -121,17 +136,9 @@ instance_create(struct nfnl_queue_net *q, u_int16_t queue_num, u32 portid) unsigned int h; int err; - spin_lock(&q->instances_lock); - if (instance_lookup(q, queue_num)) { - err = -EEXIST; - goto out_unlock; - } - - inst = kzalloc(sizeof(*inst), GFP_ATOMIC); - if (!inst) { - err = -ENOMEM; - goto out_unlock; - } + inst = kzalloc(sizeof(*inst), GFP_KERNEL_ACCOUNT); + if (!inst) + return ERR_PTR(-ENOMEM); inst->queue_num = queue_num; inst->peer_portid = portid; @@ -141,9 +148,19 @@ instance_create(struct nfnl_queue_net *q, u_int16_t queue_num, u32 portid) spin_lock_init(&inst->lock); INIT_LIST_HEAD(&inst->queue_list); + err = rhashtable_init(&inst->nfqnl_packet_map, &nfqnl_rhashtable_params); + if (err < 0) + goto out_free; + + spin_lock(&q->instances_lock); + if (instance_lookup(q, queue_num)) { + err = -EEXIST; + goto out_unlock; + } + if (!try_module_get(THIS_MODULE)) { err = -EAGAIN; - goto out_free; + goto out_unlock; } h = instance_hashfn(queue_num); @@ -153,25 +170,29 @@ instance_create(struct nfnl_queue_net *q, u_int16_t queue_num, u32 portid) return inst; -out_free: - kfree(inst); out_unlock: spin_unlock(&q->instances_lock); + rhashtable_destroy(&inst->nfqnl_packet_map); +out_free: + kfree(inst); return ERR_PTR(err); } static void nfqnl_flush(struct nfqnl_instance *queue, nfqnl_cmpfn cmpfn, unsigned long data); -static void -instance_destroy_rcu(struct rcu_head *head) +static void instance_destroy_work(struct work_struct *work) { - struct nfqnl_instance *inst = container_of(head, struct nfqnl_instance, - rcu); + struct nfqnl_instance *inst; + inst = container_of(to_rcu_work(work), struct nfqnl_instance, + rwork); rcu_read_lock(); nfqnl_flush(inst, NULL, 0); rcu_read_unlock(); + + rhashtable_destroy(&inst->nfqnl_packet_map); + kfree(inst); module_put(THIS_MODULE); } @@ -180,7 +201,9 @@ static void __instance_destroy(struct nfqnl_instance *inst) { hlist_del_rcu(&inst->hlist); - call_rcu(&inst->rcu, instance_destroy_rcu); + + INIT_RCU_WORK(&inst->rwork, instance_destroy_work); + queue_rcu_work(nfq_cleanup_wq, &inst->rwork); } static void @@ -191,16 +214,27 @@ instance_destroy(struct nfnl_queue_net *q, struct nfqnl_instance *inst) spin_unlock(&q->instances_lock); } -static inline void +static int __enqueue_entry(struct nfqnl_instance *queue, struct nf_queue_entry *entry) { - list_add_tail(&entry->list, &queue->queue_list); - queue->queue_total++; + int err; + + err = rhashtable_insert_fast(&queue->nfqnl_packet_map, &entry->hash_node, + nfqnl_rhashtable_params); + if (unlikely(err)) + return err; + + list_add_tail(&entry->list, &queue->queue_list); + queue->queue_total++; + + return 0; } static void __dequeue_entry(struct nfqnl_instance *queue, struct nf_queue_entry *entry) { + rhashtable_remove_fast(&queue->nfqnl_packet_map, &entry->hash_node, + nfqnl_rhashtable_params); list_del(&entry->list); queue->queue_total--; } @@ -208,16 +242,11 @@ __dequeue_entry(struct nfqnl_instance *queue, struct nf_queue_entry *entry) static struct nf_queue_entry * find_dequeue_entry(struct nfqnl_instance *queue, unsigned int id) { - struct nf_queue_entry *entry = NULL, *i; + struct nf_queue_entry *entry; spin_lock_bh(&queue->lock); - - list_for_each_entry(i, &queue->queue_list, list) { - if (i->id == id) { - entry = i; - break; - } - } + entry = rhashtable_lookup_fast(&queue->nfqnl_packet_map, &id, + nfqnl_rhashtable_params); if (entry) __dequeue_entry(queue, entry); @@ -369,6 +398,34 @@ static void nf_reinject(struct nf_queue_entry *entry, unsigned int verdict) nf_queue_entry_free(entry); } +/* return true if the entry has an unconfirmed conntrack attached that isn't owned by us + * exclusively. + */ +static bool nf_ct_drop_unconfirmed(const struct nf_queue_entry *entry, bool *is_unconfirmed) +{ +#if IS_ENABLED(CONFIG_NF_CONNTRACK) + struct nf_conn *ct = (void *)skb_nfct(entry->skb); + + if (!ct || nf_ct_is_confirmed(ct)) + return false; + + if (is_unconfirmed) + *is_unconfirmed = true; + + /* in some cases skb_clone() can occur after initial conntrack + * pickup, but conntrack assumes exclusive skb->_nfct ownership for + * unconfirmed entries. + * + * This happens for br_netfilter and with ip multicast routing. + * This can't be solved with serialization here because one clone + * could have been queued for local delivery or could be transmitted + * in parallel on another CPU. + */ + return refcount_read(&ct->ct_general.use) > 1; +#endif + return false; +} + static void nfqnl_reinject(struct nf_queue_entry *entry, unsigned int verdict) { const struct nf_ct_hook *ct_hook; @@ -396,6 +453,24 @@ static void nfqnl_reinject(struct nf_queue_entry *entry, unsigned int verdict) break; } } + + if (verdict != NF_DROP && entry->nf_ct_is_unconfirmed) { + /* If first queued segment was already reinjected then + * there is a good chance the ct entry is now confirmed. + * + * Handle the rare cases: + * - out-of-order verdict + * - threaded userspace reinjecting in parallel + * - first segment was dropped + * + * In all of those cases we can't handle this packet + * because we can't be sure that another CPU won't modify + * nf_conn->ext in parallel which isn't allowed. + */ + if (nf_ct_drop_unconfirmed(entry, NULL)) + verdict = NF_DROP; + } + nf_reinject(entry, verdict); } @@ -407,8 +482,7 @@ nfqnl_flush(struct nfqnl_instance *queue, nfqnl_cmpfn cmpfn, unsigned long data) spin_lock_bh(&queue->lock); list_for_each_entry_safe(entry, next, &queue->queue_list, list) { if (!cmpfn || cmpfn(entry, data)) { - list_del(&entry->list); - queue->queue_total--; + __dequeue_entry(queue, entry); nfqnl_reinject(entry, NF_DROP); } } @@ -826,49 +900,6 @@ nfqnl_build_packet_message(struct net *net, struct nfqnl_instance *queue, return NULL; } -static bool nf_ct_drop_unconfirmed(const struct nf_queue_entry *entry) -{ -#if IS_ENABLED(CONFIG_NF_CONNTRACK) - static const unsigned long flags = IPS_CONFIRMED | IPS_DYING; - struct nf_conn *ct = (void *)skb_nfct(entry->skb); - unsigned long status; - unsigned int use; - - if (!ct) - return false; - - status = READ_ONCE(ct->status); - if ((status & flags) == IPS_DYING) - return true; - - if (status & IPS_CONFIRMED) - return false; - - /* in some cases skb_clone() can occur after initial conntrack - * pickup, but conntrack assumes exclusive skb->_nfct ownership for - * unconfirmed entries. - * - * This happens for br_netfilter and with ip multicast routing. - * We can't be solved with serialization here because one clone could - * have been queued for local delivery. - */ - use = refcount_read(&ct->ct_general.use); - if (likely(use == 1)) - return false; - - /* Can't decrement further? Exclusive ownership. */ - if (!refcount_dec_not_one(&ct->ct_general.use)) - return false; - - skb_set_nfct(entry->skb, 0); - /* No nf_ct_put(): we already decremented .use and it cannot - * drop down to 0. - */ - return true; -#endif - return false; -} - static int __nfqnl_enqueue_packet(struct net *net, struct nfqnl_instance *queue, struct nf_queue_entry *entry) @@ -885,26 +916,23 @@ __nfqnl_enqueue_packet(struct net *net, struct nfqnl_instance *queue, } spin_lock_bh(&queue->lock); - if (nf_ct_drop_unconfirmed(entry)) - goto err_out_free_nskb; + if (queue->queue_total >= queue->queue_maxlen) + goto err_out_queue_drop; - if (queue->queue_total >= queue->queue_maxlen) { - if (queue->flags & NFQA_CFG_F_FAIL_OPEN) { - failopen = 1; - err = 0; - } else { - queue->queue_dropped++; - net_warn_ratelimited("nf_queue: full at %d entries, dropping packets(s)\n", - queue->queue_total); - } - goto err_out_free_nskb; - } entry->id = ++queue->id_sequence; *packet_id_ptr = htonl(entry->id); + /* Insert into hash BEFORE unicast. If failure don't send to userspace. */ + err = __enqueue_entry(queue, entry); + if (unlikely(err)) + goto err_out_queue_drop; + /* nfnetlink_unicast will either free the nskb or add it to a socket */ err = nfnetlink_unicast(nskb, net, queue->peer_portid); if (err < 0) { + /* Unicast failed - remove entry we just inserted */ + __dequeue_entry(queue, entry); + if (queue->flags & NFQA_CFG_F_FAIL_OPEN) { failopen = 1; err = 0; @@ -914,12 +942,22 @@ __nfqnl_enqueue_packet(struct net *net, struct nfqnl_instance *queue, goto err_out_unlock; } - __enqueue_entry(queue, entry); - spin_unlock_bh(&queue->lock); return 0; -err_out_free_nskb: +err_out_queue_drop: + if (queue->flags & NFQA_CFG_F_FAIL_OPEN) { + failopen = 1; + err = 0; + } else { + queue->queue_dropped++; + + if (queue->queue_total >= queue->queue_maxlen) + net_warn_ratelimited("nf_queue: full at %d entries, dropping packets(s)\n", + queue->queue_total); + else + net_warn_ratelimited("nf_queue: hash insert failed: %d\n", err); + } kfree_skb(nskb); err_out_unlock: spin_unlock_bh(&queue->lock); @@ -998,9 +1036,10 @@ __nfqnl_enqueue_packet_gso(struct net *net, struct nfqnl_instance *queue, static int nfqnl_enqueue_packet(struct nf_queue_entry *entry, unsigned int queuenum) { - unsigned int queued; - struct nfqnl_instance *queue; struct sk_buff *skb, *segs, *nskb; + bool ct_is_unconfirmed = false; + struct nfqnl_instance *queue; + unsigned int queued; int err = -ENOBUFS; struct net *net = entry->state.net; struct nfnl_queue_net *q = nfnl_queue_pernet(net); @@ -1024,6 +1063,15 @@ nfqnl_enqueue_packet(struct nf_queue_entry *entry, unsigned int queuenum) break; } + /* Check if someone already holds another reference to + * unconfirmed ct. If so, we cannot queue the skb: + * concurrent modifications of nf_conn->ext are not + * allowed and we can't know if another CPU isn't + * processing the same nf_conn entry in parallel. + */ + if (nf_ct_drop_unconfirmed(entry, &ct_is_unconfirmed)) + return -EINVAL; + if (!skb_is_gso(skb) || ((queue->flags & NFQA_CFG_F_GSO) && !skb_is_gso_sctp(skb))) return __nfqnl_enqueue_packet(net, queue, entry); @@ -1037,7 +1085,23 @@ nfqnl_enqueue_packet(struct nf_queue_entry *entry, unsigned int queuenum) goto out_err; queued = 0; err = 0; + skb_list_walk_safe(segs, segs, nskb) { + if (ct_is_unconfirmed && queued > 0) { + /* skb_gso_segment() increments the ct refcount. + * This is a problem for unconfirmed (not in hash) + * entries, those can race when reinjections happen + * in parallel. + * + * Annotate this for all queued entries except the + * first one. + * + * As long as the first one is reinjected first it + * will do the confirmation for us. + */ + entry->nf_ct_is_unconfirmed = ct_is_unconfirmed; + } + if (err == 0) err = __nfqnl_enqueue_packet_gso(net, queue, segs, entry); @@ -1445,8 +1509,10 @@ static int nfqnl_recv_verdict(struct sk_buff *skb, const struct nfnl_info *info, if (entry->state.pf == PF_BRIDGE) { err = nfqa_parse_bridge(entry, nfqa); - if (err < 0) + if (err < 0) { + nfqnl_reinject(entry, NF_DROP); return err; + } } if (nfqa[NFQA_PAYLOAD]) { @@ -1498,7 +1564,8 @@ static int nfqnl_recv_config(struct sk_buff *skb, const struct nfnl_info *info, struct nfqnl_msg_config_cmd *cmd = NULL; struct nfqnl_instance *queue; __u32 flags = 0, mask = 0; - int ret = 0; + + WARN_ON_ONCE(!lockdep_nfnl_is_held(NFNL_SUBSYS_QUEUE)); if (nfqa[NFQA_CFG_CMD]) { cmd = nla_data(nfqa[NFQA_CFG_CMD]); @@ -1544,47 +1611,44 @@ static int nfqnl_recv_config(struct sk_buff *skb, const struct nfnl_info *info, } } + /* Lookup queue under RCU. After peer_portid check (or for new queue + * in BIND case), the queue is owned by the socket sending this message. + * A socket cannot simultaneously send a message and close, so while + * processing this CONFIG message, nfqnl_rcv_nl_event() (triggered by + * socket close) cannot destroy this queue. Safe to use without RCU. + */ rcu_read_lock(); queue = instance_lookup(q, queue_num); if (queue && queue->peer_portid != NETLINK_CB(skb).portid) { - ret = -EPERM; - goto err_out_unlock; + rcu_read_unlock(); + return -EPERM; } + rcu_read_unlock(); if (cmd != NULL) { switch (cmd->command) { case NFQNL_CFG_CMD_BIND: - if (queue) { - ret = -EBUSY; - goto err_out_unlock; - } - queue = instance_create(q, queue_num, - NETLINK_CB(skb).portid); - if (IS_ERR(queue)) { - ret = PTR_ERR(queue); - goto err_out_unlock; - } + if (queue) + return -EBUSY; + queue = instance_create(q, queue_num, NETLINK_CB(skb).portid); + if (IS_ERR(queue)) + return PTR_ERR(queue); break; case NFQNL_CFG_CMD_UNBIND: - if (!queue) { - ret = -ENODEV; - goto err_out_unlock; - } + if (!queue) + return -ENODEV; instance_destroy(q, queue); - goto err_out_unlock; + return 0; case NFQNL_CFG_CMD_PF_BIND: case NFQNL_CFG_CMD_PF_UNBIND: break; default: - ret = -ENOTSUPP; - goto err_out_unlock; + return -EOPNOTSUPP; } } - if (!queue) { - ret = -ENODEV; - goto err_out_unlock; - } + if (!queue) + return -ENODEV; if (nfqa[NFQA_CFG_PARAMS]) { struct nfqnl_msg_config_params *params = @@ -1609,9 +1673,7 @@ static int nfqnl_recv_config(struct sk_buff *skb, const struct nfnl_info *info, spin_unlock_bh(&queue->lock); } -err_out_unlock: - rcu_read_unlock(); - return ret; + return 0; } static const struct nfnl_callback nfqnl_cb[NFQNL_MSG_MAX] = { @@ -1781,35 +1843,38 @@ static int __init nfnetlink_queue_init(void) { int status; + nfq_cleanup_wq = alloc_ordered_workqueue("nfq_workqueue", 0); + if (!nfq_cleanup_wq) + return -ENOMEM; + status = register_pernet_subsys(&nfnl_queue_net_ops); - if (status < 0) { - pr_err("failed to register pernet ops\n"); - goto out; - } + if (status < 0) + goto cleanup_pernet_subsys; - netlink_register_notifier(&nfqnl_rtnl_notifier); - status = nfnetlink_subsys_register(&nfqnl_subsys); - if (status < 0) { - pr_err("failed to create netlink socket\n"); - goto cleanup_netlink_notifier; - } + status = netlink_register_notifier(&nfqnl_rtnl_notifier); + if (status < 0) + goto cleanup_rtnl_notifier; status = register_netdevice_notifier(&nfqnl_dev_notifier); - if (status < 0) { - pr_err("failed to register netdevice notifier\n"); - goto cleanup_netlink_subsys; - } + if (status < 0) + goto cleanup_dev_notifier; + + status = nfnetlink_subsys_register(&nfqnl_subsys); + if (status < 0) + goto cleanup_nfqnl_subsys; nf_register_queue_handler(&nfqh); return status; -cleanup_netlink_subsys: - nfnetlink_subsys_unregister(&nfqnl_subsys); -cleanup_netlink_notifier: +cleanup_nfqnl_subsys: + unregister_netdevice_notifier(&nfqnl_dev_notifier); +cleanup_dev_notifier: netlink_unregister_notifier(&nfqnl_rtnl_notifier); +cleanup_rtnl_notifier: unregister_pernet_subsys(&nfnl_queue_net_ops); -out: +cleanup_pernet_subsys: + destroy_workqueue(nfq_cleanup_wq); return status; } @@ -1820,7 +1885,7 @@ static void __exit nfnetlink_queue_fini(void) nfnetlink_subsys_unregister(&nfqnl_subsys); netlink_unregister_notifier(&nfqnl_rtnl_notifier); unregister_pernet_subsys(&nfnl_queue_net_ops); - + destroy_workqueue(nfq_cleanup_wq); rcu_barrier(); /* Wait for completion of call_rcu()'s */ } diff --git a/net/netfilter/nft_chain_filter.c b/net/netfilter/nft_chain_filter.c index b16185e9a6dd72..041426e3bdbf16 100644 --- a/net/netfilter/nft_chain_filter.c +++ b/net/netfilter/nft_chain_filter.c @@ -344,7 +344,7 @@ static int nft_netdev_event(unsigned long event, struct net_device *dev, break; case NETDEV_REGISTER: /* NOP if not matching or already registered */ - if (!match || (changename && ops)) + if (!match || ops) continue; ops = kmemdup(&basechain->ops, diff --git a/net/netfilter/nft_compat.c b/net/netfilter/nft_compat.c index 72711d62fddfa4..08f620311b03f1 100644 --- a/net/netfilter/nft_compat.c +++ b/net/netfilter/nft_compat.c @@ -134,7 +134,8 @@ static void nft_target_eval_bridge(const struct nft_expr *expr, } static const struct nla_policy nft_target_policy[NFTA_TARGET_MAX + 1] = { - [NFTA_TARGET_NAME] = { .type = NLA_NUL_STRING }, + [NFTA_TARGET_NAME] = { .type = NLA_NUL_STRING, + .len = XT_EXTENSION_MAXNAMELEN, }, [NFTA_TARGET_REV] = NLA_POLICY_MAX(NLA_BE32, 255), [NFTA_TARGET_INFO] = { .type = NLA_BINARY }, }; @@ -434,7 +435,8 @@ static void nft_match_eval(const struct nft_expr *expr, } static const struct nla_policy nft_match_policy[NFTA_MATCH_MAX + 1] = { - [NFTA_MATCH_NAME] = { .type = NLA_NUL_STRING }, + [NFTA_MATCH_NAME] = { .type = NLA_NUL_STRING, + .len = XT_EXTENSION_MAXNAMELEN }, [NFTA_MATCH_REV] = NLA_POLICY_MAX(NLA_BE32, 255), [NFTA_MATCH_INFO] = { .type = NLA_BINARY }, }; @@ -693,7 +695,12 @@ static int nfnl_compat_get_rcu(struct sk_buff *skb, name = nla_data(tb[NFTA_COMPAT_NAME]); rev = ntohl(nla_get_be32(tb[NFTA_COMPAT_REV])); - target = ntohl(nla_get_be32(tb[NFTA_COMPAT_TYPE])); + /* x_tables api checks for 'target == 1' to mean target, + * everything else means 'match'. + * In x_tables world, the number is set by kernel, not + * userspace. + */ + target = nla_get_be32(tb[NFTA_COMPAT_TYPE]) == htonl(1); switch(family) { case AF_INET: diff --git a/net/netfilter/nft_counter.c b/net/netfilter/nft_counter.c index cc732532949630..169ae93688bcc5 100644 --- a/net/netfilter/nft_counter.c +++ b/net/netfilter/nft_counter.c @@ -32,6 +32,9 @@ struct nft_counter_percpu_priv { static DEFINE_PER_CPU(struct u64_stats_sync, nft_counter_sync); +/* control plane only: sync fetch+reset */ +static DEFINE_SPINLOCK(nft_counter_lock); + static inline void nft_counter_do_eval(struct nft_counter_percpu_priv *priv, struct nft_regs *regs, const struct nft_pktinfo *pkt) @@ -117,8 +120,8 @@ static void nft_counter_reset(struct nft_counter_percpu_priv *priv, nft_sync = this_cpu_ptr(&nft_counter_sync); u64_stats_update_begin(nft_sync); - u64_stats_add(&this_cpu->packets, -total->packets); - u64_stats_add(&this_cpu->bytes, -total->bytes); + u64_stats_sub(&this_cpu->packets, total->packets); + u64_stats_sub(&this_cpu->bytes, total->bytes); u64_stats_update_end(nft_sync); local_bh_enable(); @@ -148,13 +151,25 @@ static void nft_counter_fetch(struct nft_counter_percpu_priv *priv, } } +static void nft_counter_fetch_and_reset(struct nft_counter_percpu_priv *priv, + struct nft_counter_tot *total) +{ + spin_lock(&nft_counter_lock); + nft_counter_fetch(priv, total); + nft_counter_reset(priv, total); + spin_unlock(&nft_counter_lock); +} + static int nft_counter_do_dump(struct sk_buff *skb, struct nft_counter_percpu_priv *priv, bool reset) { struct nft_counter_tot total; - nft_counter_fetch(priv, &total); + if (unlikely(reset)) + nft_counter_fetch_and_reset(priv, &total); + else + nft_counter_fetch(priv, &total); if (nla_put_be64(skb, NFTA_COUNTER_BYTES, cpu_to_be64(total.bytes), NFTA_COUNTER_PAD) || @@ -162,9 +177,6 @@ static int nft_counter_do_dump(struct sk_buff *skb, NFTA_COUNTER_PAD)) goto nla_put_failure; - if (reset) - nft_counter_reset(priv, &total); - return 0; nla_put_failure: diff --git a/net/netfilter/nft_ct.c b/net/netfilter/nft_ct.c index 6f2ae7cad73108..4f52ca4c48d514 100644 --- a/net/netfilter/nft_ct.c +++ b/net/netfilter/nft_ct.c @@ -23,6 +23,7 @@ #include #include #include +#include "nf_internals.h" struct nft_ct_helper_obj { struct nf_conntrack_helper *helper4; @@ -543,6 +544,7 @@ static void __nft_ct_set_destroy(const struct nft_ctx *ctx, struct nft_ct *priv) #endif #ifdef CONFIG_NF_CONNTRACK_ZONES case NFT_CT_ZONE: + nf_queue_nf_hook_drop(ctx->net); mutex_lock(&nft_ct_pcpu_mutex); if (--nft_ct_pcpu_template_refcnt == 0) nft_ct_tmpl_put_pcpu(); @@ -1016,9 +1018,10 @@ static void nft_ct_timeout_obj_destroy(const struct nft_ctx *ctx, struct nft_ct_timeout_obj *priv = nft_obj_data(obj); struct nf_ct_timeout *timeout = priv->timeout; + nf_queue_nf_hook_drop(ctx->net); nf_ct_untimeout(ctx->net, timeout); nf_ct_netns_put(ctx->net, ctx->family); - kfree(priv->timeout); + kfree_rcu(priv->timeout, rcu); } static int nft_ct_timeout_obj_dump(struct sk_buff *skb, @@ -1148,6 +1151,7 @@ static void nft_ct_helper_obj_destroy(const struct nft_ctx *ctx, { struct nft_ct_helper_obj *priv = nft_obj_data(obj); + nf_queue_nf_hook_drop(ctx->net); if (priv->helper4) nf_conntrack_helper_put(priv->helper4); if (priv->helper6) diff --git a/net/netfilter/nft_dynset.c b/net/netfilter/nft_dynset.c index 7807d812966464..9123277be03ced 100644 --- a/net/netfilter/nft_dynset.c +++ b/net/netfilter/nft_dynset.c @@ -30,18 +30,26 @@ static int nft_dynset_expr_setup(const struct nft_dynset *priv, const struct nft_set_ext *ext) { struct nft_set_elem_expr *elem_expr = nft_set_ext_expr(ext); + struct nft_ctx ctx = { + .net = read_pnet(&priv->set->net), + .family = priv->set->table->family, + }; struct nft_expr *expr; int i; for (i = 0; i < priv->num_exprs; i++) { expr = nft_setelem_expr_at(elem_expr, elem_expr->size); if (nft_expr_clone(expr, priv->expr_array[i], GFP_ATOMIC) < 0) - return -1; + goto err_out; elem_expr->size += priv->expr_array[i]->ops->size; } return 0; +err_out: + nft_set_elem_expr_destroy(&ctx, elem_expr); + + return -1; } struct nft_elem_priv *nft_dynset_new(struct nft_set *set, diff --git a/net/netfilter/nft_quota.c b/net/netfilter/nft_quota.c index df0798da2329b9..cb6c0e04ff6755 100644 --- a/net/netfilter/nft_quota.c +++ b/net/netfilter/nft_quota.c @@ -140,11 +140,16 @@ static int nft_quota_do_dump(struct sk_buff *skb, struct nft_quota *priv, u64 consumed, consumed_cap, quota; u32 flags = priv->flags; - /* Since we inconditionally increment consumed quota for each packet + /* Since we unconditionally increment consumed quota for each packet * that we see, don't go over the quota boundary in what we send to * userspace. */ - consumed = atomic64_read(priv->consumed); + if (reset) { + consumed = atomic64_xchg(priv->consumed, 0); + clear_bit(NFT_QUOTA_DEPLETED_BIT, &priv->flags); + } else { + consumed = atomic64_read(priv->consumed); + } quota = atomic64_read(&priv->quota); if (consumed >= quota) { consumed_cap = quota; @@ -160,10 +165,6 @@ static int nft_quota_do_dump(struct sk_buff *skb, struct nft_quota *priv, nla_put_be32(skb, NFTA_QUOTA_FLAGS, htonl(flags))) goto nla_put_failure; - if (reset) { - atomic64_sub(consumed, priv->consumed); - clear_bit(NFT_QUOTA_DEPLETED_BIT, &priv->flags); - } return 0; nla_put_failure: diff --git a/net/netfilter/nft_set_hash.c b/net/netfilter/nft_set_hash.c index ba01ce75d6dea9..b0e571c8e3f38a 100644 --- a/net/netfilter/nft_set_hash.c +++ b/net/netfilter/nft_set_hash.c @@ -374,6 +374,7 @@ static void nft_rhash_walk(const struct nft_ctx *ctx, struct nft_set *set, { switch (iter->type) { case NFT_ITER_UPDATE: + case NFT_ITER_UPDATE_CLONE: /* only relevant for netlink dumps which use READ type */ WARN_ON_ONCE(iter->skip != 0); @@ -619,15 +620,20 @@ static struct nft_elem_priv * nft_hash_get(const struct net *net, const struct nft_set *set, const struct nft_set_elem *elem, unsigned int flags) { + const u32 *key = (const u32 *)&elem->key.val; struct nft_hash *priv = nft_set_priv(set); u8 genmask = nft_genmask_cur(net); struct nft_hash_elem *he; u32 hash; - hash = jhash(elem->key.val.data, set->klen, priv->seed); + if (set->klen == 4) + hash = jhash_1word(*key, priv->seed); + else + hash = jhash(key, set->klen, priv->seed); + hash = reciprocal_scale(hash, priv->buckets); hlist_for_each_entry_rcu(he, &priv->table[hash], node) { - if (!memcmp(nft_set_ext_key(&he->ext), elem->key.val.data, set->klen) && + if (!memcmp(nft_set_ext_key(&he->ext), key, set->klen) && nft_set_elem_active(&he->ext, genmask)) return &he->priv; } diff --git a/net/netfilter/nft_set_pipapo.c b/net/netfilter/nft_set_pipapo.c index 6d77a5f0088ad0..394b78a00a6a5a 100644 --- a/net/netfilter/nft_set_pipapo.c +++ b/net/netfilter/nft_set_pipapo.c @@ -1641,6 +1641,7 @@ static void pipapo_drop(struct nft_pipapo_match *m, int i; nft_pipapo_for_each_field(f, i, m) { + bool last = i == m->field_count - 1; int g; for (g = 0; g < f->groups; g++) { @@ -1660,7 +1661,7 @@ static void pipapo_drop(struct nft_pipapo_match *m, } pipapo_unmap(f->mt, f->rules, rulemap[i].to, rulemap[i].n, - rulemap[i + 1].n, i == m->field_count - 1); + last ? 0 : rulemap[i + 1].n, last); if (pipapo_resize(f, f->rules, f->rules - rulemap[i].n)) { /* We can ignore this, a failure to shrink tables down * doesn't make tables invalid. @@ -1681,11 +1682,11 @@ static void nft_pipapo_gc_deactivate(struct net *net, struct nft_set *set, } /** - * pipapo_gc() - Drop expired entries from set, destroy start and end elements + * pipapo_gc_scan() - Drop expired entries from set and link them to gc list * @set: nftables API set representation * @m: Matching data */ -static void pipapo_gc(struct nft_set *set, struct nft_pipapo_match *m) +static void pipapo_gc_scan(struct nft_set *set, struct nft_pipapo_match *m) { struct nft_pipapo *priv = nft_set_priv(set); struct net *net = read_pnet(&set->net); @@ -1698,6 +1699,8 @@ static void pipapo_gc(struct nft_set *set, struct nft_pipapo_match *m) if (!gc) return; + list_add(&gc->list, &priv->gc_head); + while ((rules_f0 = pipapo_rules_same_key(m->f, first_rule))) { union nft_pipapo_map_bucket rulemap[NFT_PIPAPO_MAX_FIELDS]; const struct nft_pipapo_field *f; @@ -1725,9 +1728,13 @@ static void pipapo_gc(struct nft_set *set, struct nft_pipapo_match *m) * NFT_SET_ELEM_DEAD_BIT. */ if (__nft_set_elem_expired(&e->ext, tstamp)) { - gc = nft_trans_gc_queue_sync(gc, GFP_KERNEL); - if (!gc) - return; + if (!nft_trans_gc_space(gc)) { + gc = nft_trans_gc_alloc(set, 0, GFP_KERNEL); + if (!gc) + return; + + list_add(&gc->list, &priv->gc_head); + } nft_pipapo_gc_deactivate(net, set, e); pipapo_drop(m, rulemap); @@ -1741,10 +1748,30 @@ static void pipapo_gc(struct nft_set *set, struct nft_pipapo_match *m) } } - gc = nft_trans_gc_catchall_sync(gc); + priv->last_gc = jiffies; +} + +/** + * pipapo_gc_queue() - Free expired elements + * @set: nftables API set representation + */ +static void pipapo_gc_queue(struct nft_set *set) +{ + struct nft_pipapo *priv = nft_set_priv(set); + struct nft_trans_gc *gc, *next; + + /* always do a catchall cycle: */ + gc = nft_trans_gc_alloc(set, 0, GFP_KERNEL); if (gc) { + gc = nft_trans_gc_catchall_sync(gc); + if (gc) + nft_trans_gc_queue_sync_done(gc); + } + + /* always purge queued gc elements. */ + list_for_each_entry_safe(gc, next, &priv->gc_head, list) { + list_del(&gc->list); nft_trans_gc_queue_sync_done(gc); - priv->last_gc = jiffies; } } @@ -1798,6 +1825,10 @@ static void pipapo_reclaim_match(struct rcu_head *rcu) * * We also need to create a new working copy for subsequent insertions and * deletions. + * + * After the live copy has been replaced by the clone, we can safely queue + * expired elements that have been collected by pipapo_gc_scan() for + * memory reclaim. */ static void nft_pipapo_commit(struct nft_set *set) { @@ -1808,7 +1839,7 @@ static void nft_pipapo_commit(struct nft_set *set) return; if (time_after_eq(jiffies, priv->last_gc + nft_set_gc_interval(set))) - pipapo_gc(set, priv->clone); + pipapo_gc_scan(set, priv->clone); old = rcu_replace_pointer(priv->match, priv->clone, nft_pipapo_transaction_mutex_held(set)); @@ -1816,6 +1847,8 @@ static void nft_pipapo_commit(struct nft_set *set) if (old) call_rcu(&old->rcu, pipapo_reclaim_match); + + pipapo_gc_queue(set); } static void nft_pipapo_abort(const struct nft_set *set) @@ -2145,13 +2178,20 @@ static void nft_pipapo_walk(const struct nft_ctx *ctx, struct nft_set *set, const struct nft_pipapo_match *m; switch (iter->type) { - case NFT_ITER_UPDATE: + case NFT_ITER_UPDATE_CLONE: m = pipapo_maybe_clone(set); if (!m) { iter->err = -ENOMEM; return; } - + nft_pipapo_do_walk(ctx, set, m, iter); + break; + case NFT_ITER_UPDATE: + if (priv->clone) + m = priv->clone; + else + m = rcu_dereference_protected(priv->match, + nft_pipapo_transaction_mutex_held(set)); nft_pipapo_do_walk(ctx, set, m, iter); break; case NFT_ITER_READ: @@ -2273,6 +2313,7 @@ static int nft_pipapo_init(const struct nft_set *set, f->mt = NULL; } + INIT_LIST_HEAD(&priv->gc_head); rcu_assign_pointer(priv->match, m); return 0; @@ -2322,6 +2363,8 @@ static void nft_pipapo_destroy(const struct nft_ctx *ctx, struct nft_pipapo *priv = nft_set_priv(set); struct nft_pipapo_match *m; + WARN_ON_ONCE(!list_empty(&priv->gc_head)); + m = rcu_dereference_protected(priv->match, true); if (priv->clone) { @@ -2370,6 +2413,7 @@ const struct nft_set_type nft_set_pipapo_type = { .gc_init = nft_pipapo_gc_init, .commit = nft_pipapo_commit, .abort = nft_pipapo_abort, + .abort_skip_removal = true, .elemsize = offsetof(struct nft_pipapo_elem, ext), }, }; @@ -2394,6 +2438,7 @@ const struct nft_set_type nft_set_pipapo_avx2_type = { .gc_init = nft_pipapo_gc_init, .commit = nft_pipapo_commit, .abort = nft_pipapo_abort, + .abort_skip_removal = true, .elemsize = offsetof(struct nft_pipapo_elem, ext), }, }; diff --git a/net/netfilter/nft_set_pipapo.h b/net/netfilter/nft_set_pipapo.h index eaab422aa56ab2..9aee9a9eaeb759 100644 --- a/net/netfilter/nft_set_pipapo.h +++ b/net/netfilter/nft_set_pipapo.h @@ -156,12 +156,14 @@ struct nft_pipapo_match { * @clone: Copy where pending insertions and deletions are kept * @width: Total bytes to be matched for one packet, including padding * @last_gc: Timestamp of last garbage collection run, jiffies + * @gc_head: list of nft_trans_gc to queue up for mem reclaim */ struct nft_pipapo { struct nft_pipapo_match __rcu *match; struct nft_pipapo_match *clone; int width; unsigned long last_gc; + struct list_head gc_head; }; struct nft_pipapo_elem; diff --git a/net/netfilter/nft_set_pipapo_avx2.c b/net/netfilter/nft_set_pipapo_avx2.c index 7ff90325c97fa8..6395982e4d95c5 100644 --- a/net/netfilter/nft_set_pipapo_avx2.c +++ b/net/netfilter/nft_set_pipapo_avx2.c @@ -242,7 +242,7 @@ static int nft_pipapo_avx2_lookup_4b_2(unsigned long *map, unsigned long *fill, b = nft_pipapo_avx2_refill(i_ul, &map[i_ul], fill, f->mt, last); if (last) - return b; + ret = b; if (unlikely(ret == -1)) ret = b / XSAVE_YMM_SIZE; @@ -319,7 +319,7 @@ static int nft_pipapo_avx2_lookup_4b_4(unsigned long *map, unsigned long *fill, b = nft_pipapo_avx2_refill(i_ul, &map[i_ul], fill, f->mt, last); if (last) - return b; + ret = b; if (unlikely(ret == -1)) ret = b / XSAVE_YMM_SIZE; @@ -414,7 +414,7 @@ static int nft_pipapo_avx2_lookup_4b_8(unsigned long *map, unsigned long *fill, b = nft_pipapo_avx2_refill(i_ul, &map[i_ul], fill, f->mt, last); if (last) - return b; + ret = b; if (unlikely(ret == -1)) ret = b / XSAVE_YMM_SIZE; @@ -505,7 +505,7 @@ static int nft_pipapo_avx2_lookup_4b_12(unsigned long *map, unsigned long *fill, b = nft_pipapo_avx2_refill(i_ul, &map[i_ul], fill, f->mt, last); if (last) - return b; + ret = b; if (unlikely(ret == -1)) ret = b / XSAVE_YMM_SIZE; @@ -641,7 +641,7 @@ static int nft_pipapo_avx2_lookup_4b_32(unsigned long *map, unsigned long *fill, b = nft_pipapo_avx2_refill(i_ul, &map[i_ul], fill, f->mt, last); if (last) - return b; + ret = b; if (unlikely(ret == -1)) ret = b / XSAVE_YMM_SIZE; @@ -699,7 +699,7 @@ static int nft_pipapo_avx2_lookup_8b_1(unsigned long *map, unsigned long *fill, b = nft_pipapo_avx2_refill(i_ul, &map[i_ul], fill, f->mt, last); if (last) - return b; + ret = b; if (unlikely(ret == -1)) ret = b / XSAVE_YMM_SIZE; @@ -764,7 +764,7 @@ static int nft_pipapo_avx2_lookup_8b_2(unsigned long *map, unsigned long *fill, b = nft_pipapo_avx2_refill(i_ul, &map[i_ul], fill, f->mt, last); if (last) - return b; + ret = b; if (unlikely(ret == -1)) ret = b / XSAVE_YMM_SIZE; @@ -839,7 +839,7 @@ static int nft_pipapo_avx2_lookup_8b_4(unsigned long *map, unsigned long *fill, b = nft_pipapo_avx2_refill(i_ul, &map[i_ul], fill, f->mt, last); if (last) - return b; + ret = b; if (unlikely(ret == -1)) ret = b / XSAVE_YMM_SIZE; @@ -925,7 +925,7 @@ static int nft_pipapo_avx2_lookup_8b_6(unsigned long *map, unsigned long *fill, b = nft_pipapo_avx2_refill(i_ul, &map[i_ul], fill, f->mt, last); if (last) - return b; + ret = b; if (unlikely(ret == -1)) ret = b / XSAVE_YMM_SIZE; @@ -1019,7 +1019,7 @@ static int nft_pipapo_avx2_lookup_8b_16(unsigned long *map, unsigned long *fill, b = nft_pipapo_avx2_refill(i_ul, &map[i_ul], fill, f->mt, last); if (last) - return b; + ret = b; if (unlikely(ret == -1)) ret = b / XSAVE_YMM_SIZE; diff --git a/net/netfilter/nft_set_rbtree.c b/net/netfilter/nft_set_rbtree.c index ca594161b8402e..154bf2772e27d8 100644 --- a/net/netfilter/nft_set_rbtree.c +++ b/net/netfilter/nft_set_rbtree.c @@ -10,21 +10,41 @@ #include #include #include +#include #include #include #include #include +struct nft_array_interval { + struct nft_set_ext *from; + struct nft_set_ext *to; +}; + +struct nft_array { + u32 max_intervals; + u32 num_intervals; + struct nft_array_interval *intervals; + struct rcu_head rcu_head; +}; + struct nft_rbtree { struct rb_root root; rwlock_t lock; - seqcount_rwlock_t count; + struct nft_array __rcu *array; + struct nft_array *array_next; + unsigned long start_rbe_cookie; unsigned long last_gc; + struct list_head expired; + u64 last_tstamp; }; struct nft_rbtree_elem { struct nft_elem_priv priv; - struct rb_node node; + union { + struct rb_node node; + struct list_head list; + }; struct nft_set_ext ext; }; @@ -39,6 +59,13 @@ static bool nft_rbtree_interval_start(const struct nft_rbtree_elem *rbe) return !nft_rbtree_interval_end(rbe); } +static bool nft_rbtree_interval_null(const struct nft_set *set, + const struct nft_rbtree_elem *rbe) +{ + return (!memchr_inv(nft_set_ext_key(&rbe->ext), 0, set->klen) && + nft_rbtree_interval_end(rbe)); +} + static int nft_rbtree_cmp(const struct nft_set *set, const struct nft_rbtree_elem *e1, const struct nft_rbtree_elem *e2) @@ -47,67 +74,33 @@ static int nft_rbtree_cmp(const struct nft_set *set, set->klen); } -static bool nft_rbtree_elem_expired(const struct nft_rbtree_elem *rbe) -{ - return nft_set_elem_expired(&rbe->ext); -} +struct nft_array_lookup_ctx { + const u32 *key; + u32 klen; +}; -static const struct nft_set_ext * -__nft_rbtree_lookup(const struct net *net, const struct nft_set *set, - const u32 *key, unsigned int seq) +static int nft_array_lookup_cmp(const void *pkey, const void *entry) { - struct nft_rbtree *priv = nft_set_priv(set); - const struct nft_rbtree_elem *rbe, *interval = NULL; - u8 genmask = nft_genmask_cur(net); - const struct rb_node *parent; - int d; + const struct nft_array_interval *interval = entry; + const struct nft_array_lookup_ctx *ctx = pkey; + int a, b; - parent = rcu_dereference_raw(priv->root.rb_node); - while (parent != NULL) { - if (read_seqcount_retry(&priv->count, seq)) - return NULL; - - rbe = rb_entry(parent, struct nft_rbtree_elem, node); + if (!interval->from) + return 1; - d = memcmp(nft_set_ext_key(&rbe->ext), key, set->klen); - if (d < 0) { - parent = rcu_dereference_raw(parent->rb_left); - if (interval && - !nft_rbtree_cmp(set, rbe, interval) && - nft_rbtree_interval_end(rbe) && - nft_rbtree_interval_start(interval)) - continue; - if (nft_set_elem_active(&rbe->ext, genmask) && - !nft_rbtree_elem_expired(rbe)) - interval = rbe; - } else if (d > 0) - parent = rcu_dereference_raw(parent->rb_right); - else { - if (!nft_set_elem_active(&rbe->ext, genmask)) { - parent = rcu_dereference_raw(parent->rb_left); - continue; - } - - if (nft_rbtree_elem_expired(rbe)) - return NULL; - - if (nft_rbtree_interval_end(rbe)) { - if (nft_set_is_anonymous(set)) - return NULL; - parent = rcu_dereference_raw(parent->rb_left); - interval = NULL; - continue; - } + a = memcmp(ctx->key, nft_set_ext_key(interval->from), ctx->klen); + if (!interval->to) + b = -1; + else + b = memcmp(ctx->key, nft_set_ext_key(interval->to), ctx->klen); - return &rbe->ext; - } - } + if (a >= 0 && b < 0) + return 0; - if (set->flags & NFT_SET_INTERVAL && interval != NULL && - nft_rbtree_interval_start(interval)) - return &interval->ext; + if (a < 0) + return -1; - return NULL; + return 1; } INDIRECT_CALLABLE_SCOPE @@ -116,83 +109,57 @@ nft_rbtree_lookup(const struct net *net, const struct nft_set *set, const u32 *key) { struct nft_rbtree *priv = nft_set_priv(set); - unsigned int seq = read_seqcount_begin(&priv->count); - const struct nft_set_ext *ext; - - ext = __nft_rbtree_lookup(net, set, key, seq); - if (ext || !read_seqcount_retry(&priv->count, seq)) - return ext; - - read_lock_bh(&priv->lock); - seq = read_seqcount_begin(&priv->count); - ext = __nft_rbtree_lookup(net, set, key, seq); - read_unlock_bh(&priv->lock); - - return ext; + struct nft_array *array = rcu_dereference(priv->array); + const struct nft_array_interval *interval; + struct nft_array_lookup_ctx ctx = { + .key = key, + .klen = set->klen, + }; + + if (!array) + return NULL; + + interval = bsearch(&ctx, array->intervals, array->num_intervals, + sizeof(struct nft_array_interval), + nft_array_lookup_cmp); + if (!interval || nft_set_elem_expired(interval->from)) + return NULL; + + return interval->from; } -static bool __nft_rbtree_get(const struct net *net, const struct nft_set *set, - const u32 *key, struct nft_rbtree_elem **elem, - unsigned int seq, unsigned int flags, u8 genmask) -{ - struct nft_rbtree_elem *rbe, *interval = NULL; - struct nft_rbtree *priv = nft_set_priv(set); - const struct rb_node *parent; - const void *this; - int d; - - parent = rcu_dereference_raw(priv->root.rb_node); - while (parent != NULL) { - if (read_seqcount_retry(&priv->count, seq)) - return false; - - rbe = rb_entry(parent, struct nft_rbtree_elem, node); - - this = nft_set_ext_key(&rbe->ext); - d = memcmp(this, key, set->klen); - if (d < 0) { - parent = rcu_dereference_raw(parent->rb_left); - if (!(flags & NFT_SET_ELEM_INTERVAL_END)) - interval = rbe; - } else if (d > 0) { - parent = rcu_dereference_raw(parent->rb_right); - if (flags & NFT_SET_ELEM_INTERVAL_END) - interval = rbe; - } else { - if (!nft_set_elem_active(&rbe->ext, genmask)) { - parent = rcu_dereference_raw(parent->rb_left); - continue; - } +struct nft_array_get_ctx { + const u32 *key; + unsigned int flags; + u32 klen; +}; - if (nft_set_elem_expired(&rbe->ext)) - return false; +static int nft_array_get_cmp(const void *pkey, const void *entry) +{ + const struct nft_array_interval *interval = entry; + const struct nft_array_get_ctx *ctx = pkey; + int a, b; - if (!nft_set_ext_exists(&rbe->ext, NFT_SET_EXT_FLAGS) || - (*nft_set_ext_flags(&rbe->ext) & NFT_SET_ELEM_INTERVAL_END) == - (flags & NFT_SET_ELEM_INTERVAL_END)) { - *elem = rbe; - return true; - } + if (!interval->from) + return 1; - if (nft_rbtree_interval_end(rbe)) - interval = NULL; + a = memcmp(ctx->key, nft_set_ext_key(interval->from), ctx->klen); + if (!interval->to) + b = -1; + else + b = memcmp(ctx->key, nft_set_ext_key(interval->to), ctx->klen); - parent = rcu_dereference_raw(parent->rb_left); - } + if (a >= 0) { + if (ctx->flags & NFT_SET_ELEM_INTERVAL_END && b <= 0) + return 0; + else if (b < 0) + return 0; } - if (set->flags & NFT_SET_INTERVAL && interval != NULL && - nft_set_elem_active(&interval->ext, genmask) && - !nft_set_elem_expired(&interval->ext) && - ((!nft_rbtree_interval_end(interval) && - !(flags & NFT_SET_ELEM_INTERVAL_END)) || - (nft_rbtree_interval_end(interval) && - (flags & NFT_SET_ELEM_INTERVAL_END)))) { - *elem = interval; - return true; - } + if (a < 0) + return -1; - return false; + return 1; } static struct nft_elem_priv * @@ -200,34 +167,41 @@ nft_rbtree_get(const struct net *net, const struct nft_set *set, const struct nft_set_elem *elem, unsigned int flags) { struct nft_rbtree *priv = nft_set_priv(set); - unsigned int seq = read_seqcount_begin(&priv->count); - struct nft_rbtree_elem *rbe = ERR_PTR(-ENOENT); - const u32 *key = (const u32 *)&elem->key.val; - u8 genmask = nft_genmask_cur(net); - bool ret; - - ret = __nft_rbtree_get(net, set, key, &rbe, seq, flags, genmask); - if (ret || !read_seqcount_retry(&priv->count, seq)) - return &rbe->priv; - - read_lock_bh(&priv->lock); - seq = read_seqcount_begin(&priv->count); - ret = __nft_rbtree_get(net, set, key, &rbe, seq, flags, genmask); - read_unlock_bh(&priv->lock); - - if (!ret) + struct nft_array *array = rcu_dereference(priv->array); + const struct nft_array_interval *interval; + struct nft_array_get_ctx ctx = { + .key = (const u32 *)&elem->key.val, + .flags = flags, + .klen = set->klen, + }; + struct nft_rbtree_elem *rbe; + + if (!array) return ERR_PTR(-ENOENT); + interval = bsearch(&ctx, array->intervals, array->num_intervals, + sizeof(struct nft_array_interval), nft_array_get_cmp); + if (!interval || nft_set_elem_expired(interval->from)) + return ERR_PTR(-ENOENT); + + if (flags & NFT_SET_ELEM_INTERVAL_END) + rbe = container_of(interval->to, struct nft_rbtree_elem, ext); + else + rbe = container_of(interval->from, struct nft_rbtree_elem, ext); + return &rbe->priv; } -static void nft_rbtree_gc_elem_remove(struct net *net, struct nft_set *set, - struct nft_rbtree *priv, - struct nft_rbtree_elem *rbe) +static void nft_rbtree_gc_elem_move(struct net *net, struct nft_set *set, + struct nft_rbtree *priv, + struct nft_rbtree_elem *rbe) { lockdep_assert_held_write(&priv->lock); nft_setelem_data_deactivate(net, set, &rbe->priv); rb_erase(&rbe->node, &priv->root); + + /* collected later on in commit callback */ + list_add(&rbe->list, &priv->expired); } static const struct nft_rbtree_elem * @@ -238,11 +212,6 @@ nft_rbtree_gc_elem(const struct nft_set *__set, struct nft_rbtree *priv, struct rb_node *prev = rb_prev(&rbe->node); struct net *net = read_pnet(&set->net); struct nft_rbtree_elem *rbe_prev; - struct nft_trans_gc *gc; - - gc = nft_trans_gc_alloc(set, 0, GFP_ATOMIC); - if (!gc) - return ERR_PTR(-ENOMEM); /* search for end interval coming before this element. * end intervals don't carry a timeout extension, they @@ -260,28 +229,10 @@ nft_rbtree_gc_elem(const struct nft_set *__set, struct nft_rbtree *priv, rbe_prev = NULL; if (prev) { rbe_prev = rb_entry(prev, struct nft_rbtree_elem, node); - nft_rbtree_gc_elem_remove(net, set, priv, rbe_prev); - - /* There is always room in this trans gc for this element, - * memory allocation never actually happens, hence, the warning - * splat in such case. No need to set NFT_SET_ELEM_DEAD_BIT, - * this is synchronous gc which never fails. - */ - gc = nft_trans_gc_queue_sync(gc, GFP_ATOMIC); - if (WARN_ON_ONCE(!gc)) - return ERR_PTR(-ENOMEM); - - nft_trans_gc_elem_add(gc, rbe_prev); + nft_rbtree_gc_elem_move(net, set, priv, rbe_prev); } - nft_rbtree_gc_elem_remove(net, set, priv, rbe); - gc = nft_trans_gc_queue_sync(gc, GFP_ATOMIC); - if (WARN_ON_ONCE(!gc)) - return ERR_PTR(-ENOMEM); - - nft_trans_gc_elem_add(gc, rbe); - - nft_trans_gc_queue_sync_done(gc); + nft_rbtree_gc_elem_move(net, set, priv, rbe); return rbe_prev; } @@ -302,16 +253,97 @@ static bool nft_rbtree_update_first(const struct nft_set *set, return false; } +/* Only for anonymous sets which do not allow updates, all element are active. */ +static struct nft_rbtree_elem *nft_rbtree_prev_active(struct nft_rbtree_elem *rbe) +{ + struct rb_node *node; + + node = rb_prev(&rbe->node); + if (!node) + return NULL; + + return rb_entry(node, struct nft_rbtree_elem, node); +} + +static struct nft_rbtree_elem * +__nft_rbtree_next_active(struct rb_node *node, u8 genmask) +{ + struct nft_rbtree_elem *next_rbe; + + while (node) { + next_rbe = rb_entry(node, struct nft_rbtree_elem, node); + if (!nft_set_elem_active(&next_rbe->ext, genmask)) { + node = rb_next(node); + continue; + } + + return next_rbe; + } + + return NULL; +} + +static struct nft_rbtree_elem * +nft_rbtree_next_active(struct nft_rbtree_elem *rbe, u8 genmask) +{ + return __nft_rbtree_next_active(rb_next(&rbe->node), genmask); +} + +static void nft_rbtree_maybe_reset_start_cookie(struct nft_rbtree *priv, + u64 tstamp) +{ + if (priv->last_tstamp != tstamp) { + priv->start_rbe_cookie = 0; + priv->last_tstamp = tstamp; + } +} + +static void nft_rbtree_set_start_cookie(struct nft_rbtree *priv, + const struct nft_rbtree_elem *rbe) +{ + priv->start_rbe_cookie = (unsigned long)rbe; +} + +static bool nft_rbtree_cmp_start_cookie(struct nft_rbtree *priv, + const struct nft_rbtree_elem *rbe) +{ + return priv->start_rbe_cookie == (unsigned long)rbe; +} + +static bool nft_rbtree_insert_same_interval(const struct net *net, + struct nft_rbtree *priv, + struct nft_rbtree_elem *rbe) +{ + u8 genmask = nft_genmask_next(net); + struct nft_rbtree_elem *next_rbe; + + if (!priv->start_rbe_cookie) + return true; + + next_rbe = nft_rbtree_next_active(rbe, genmask); + if (next_rbe) { + /* Closest start element differs from last element added. */ + if (nft_rbtree_interval_start(next_rbe) && + nft_rbtree_cmp_start_cookie(priv, next_rbe)) { + priv->start_rbe_cookie = 0; + return true; + } + } + + priv->start_rbe_cookie = 0; + + return false; +} + static int __nft_rbtree_insert(const struct net *net, const struct nft_set *set, struct nft_rbtree_elem *new, - struct nft_elem_priv **elem_priv) + struct nft_elem_priv **elem_priv, u64 tstamp) { - struct nft_rbtree_elem *rbe, *rbe_le = NULL, *rbe_ge = NULL; + struct nft_rbtree_elem *rbe, *rbe_le = NULL, *rbe_ge = NULL, *rbe_prev; struct rb_node *node, *next, *parent, **p, *first = NULL; struct nft_rbtree *priv = nft_set_priv(set); u8 cur_genmask = nft_genmask_cur(net); u8 genmask = nft_genmask_next(net); - u64 tstamp = nft_net_tstamp(net); int d; /* Descend the tree to search for an existing element greater than the @@ -417,12 +449,18 @@ static int __nft_rbtree_insert(const struct net *net, const struct nft_set *set, } } + if (nft_rbtree_interval_null(set, new)) + priv->start_rbe_cookie = 0; + else if (nft_rbtree_interval_start(new) && priv->start_rbe_cookie) + priv->start_rbe_cookie = 0; + /* - new start element matching existing start element: full overlap * reported as -EEXIST, cleared by caller if NLM_F_EXCL is not given. */ if (rbe_ge && !nft_rbtree_cmp(set, new, rbe_ge) && nft_rbtree_interval_start(rbe_ge) == nft_rbtree_interval_start(new)) { *elem_priv = &rbe_ge->priv; + nft_rbtree_set_start_cookie(priv, rbe_ge); return -EEXIST; } @@ -431,18 +469,37 @@ static int __nft_rbtree_insert(const struct net *net, const struct nft_set *set, */ if (rbe_le && !nft_rbtree_cmp(set, new, rbe_le) && nft_rbtree_interval_end(rbe_le) == nft_rbtree_interval_end(new)) { + /* - ignore null interval, otherwise NLM_F_CREATE bogusly + * reports EEXIST. + */ + if (nft_rbtree_interval_null(set, new)) + return -ECANCELED; + *elem_priv = &rbe_le->priv; + + /* - start and end element belong to the same interval. */ + if (!nft_rbtree_insert_same_interval(net, priv, rbe_le)) + return -ENOTEMPTY; + return -EEXIST; } /* - new start element with existing closest, less or equal key value * being a start element: partial overlap, reported as -ENOTEMPTY. * Anonymous sets allow for two consecutive start element since they - * are constant, skip them to avoid bogus overlap reports. + * are constant, but validate that this new start element does not + * sit in between an existing start and end elements: partial overlap, + * reported as -ENOTEMPTY. */ - if (!nft_set_is_anonymous(set) && rbe_le && - nft_rbtree_interval_start(rbe_le) && nft_rbtree_interval_start(new)) - return -ENOTEMPTY; + if (rbe_le && + nft_rbtree_interval_start(rbe_le) && nft_rbtree_interval_start(new)) { + if (!nft_set_is_anonymous(set)) + return -ENOTEMPTY; + + rbe_prev = nft_rbtree_prev_active(rbe_le); + if (rbe_prev && nft_rbtree_interval_end(rbe_prev)) + return -ENOTEMPTY; + } /* - new end element with existing closest, less or equal key value * being a end element: partial overlap, reported as -ENOTEMPTY. @@ -481,14 +538,159 @@ static int __nft_rbtree_insert(const struct net *net, const struct nft_set *set, return 0; } +static int nft_array_intervals_alloc(struct nft_array *array, u32 max_intervals) +{ + struct nft_array_interval *intervals; + + intervals = kvcalloc(max_intervals, sizeof(struct nft_array_interval), + GFP_KERNEL_ACCOUNT); + if (!intervals) + return -ENOMEM; + + if (array->intervals) + kvfree(array->intervals); + + array->intervals = intervals; + array->max_intervals = max_intervals; + + return 0; +} + +static struct nft_array *nft_array_alloc(u32 max_intervals) +{ + struct nft_array *array; + + array = kzalloc(sizeof(*array), GFP_KERNEL_ACCOUNT); + if (!array) + return NULL; + + if (nft_array_intervals_alloc(array, max_intervals) < 0) { + kfree(array); + return NULL; + } + + return array; +} + +/* Similar to nft_rbtree_{u,k}size to hide details to userspace, but consider + * packed representation coming from userspace for anonymous sets too. + */ +static u32 nft_array_elems(const struct nft_set *set) +{ + u32 nelems = atomic_read(&set->nelems) - set->ndeact; + + /* Adjacent intervals are represented with a single start element in + * anonymous sets, use the current element counter as is. + */ + if (nft_set_is_anonymous(set)) + return nelems; + + /* Add extra room for never matching interval at the beginning and open + * interval at the end which only use a single element to represent it. + * The conversion to array will compact intervals, this allows reduce + * memory consumption. + */ + return (nelems / 2) + 2; +} + +#define NFT_ARRAY_INITIAL_SIZE 1024 +#define NFT_ARRAY_INITIAL_ANON_SIZE 16 +#define NFT_ARRAY_INITIAL_ANON_THRESH (8192U / sizeof(struct nft_array_interval)) + +static int nft_array_may_resize(const struct nft_set *set, bool flush) +{ + u32 initial_intervals, max_intervals, new_max_intervals, delta; + u32 shrinked_max_intervals, nelems = nft_array_elems(set); + struct nft_rbtree *priv = nft_set_priv(set); + struct nft_array *array; + + if (nft_set_is_anonymous(set)) + initial_intervals = NFT_ARRAY_INITIAL_ANON_SIZE; + else + initial_intervals = NFT_ARRAY_INITIAL_SIZE; + + if (priv->array_next) { + max_intervals = priv->array_next->max_intervals; + new_max_intervals = priv->array_next->max_intervals; + } else { + if (priv->array) { + max_intervals = priv->array->max_intervals; + new_max_intervals = priv->array->max_intervals; + } else { + max_intervals = 0; + new_max_intervals = initial_intervals; + } + } + + if (nft_set_is_anonymous(set)) + goto maybe_grow; + + if (flush) { + /* Set flush just started, nelems still report elements.*/ + nelems = 0; + new_max_intervals = NFT_ARRAY_INITIAL_SIZE; + goto realloc_array; + } + + if (check_add_overflow(new_max_intervals, new_max_intervals, + &shrinked_max_intervals)) + return -EOVERFLOW; + + shrinked_max_intervals = DIV_ROUND_UP(shrinked_max_intervals, 3); + + if (shrinked_max_intervals > NFT_ARRAY_INITIAL_SIZE && + nelems < shrinked_max_intervals) { + new_max_intervals = shrinked_max_intervals; + goto realloc_array; + } +maybe_grow: + if (nelems > new_max_intervals) { + if (nft_set_is_anonymous(set) && + new_max_intervals < NFT_ARRAY_INITIAL_ANON_THRESH) { + new_max_intervals <<= 1; + } else { + delta = new_max_intervals >> 1; + if (check_add_overflow(new_max_intervals, delta, + &new_max_intervals)) + return -EOVERFLOW; + } + } + +realloc_array: + if (WARN_ON_ONCE(nelems > new_max_intervals)) + return -ENOMEM; + + if (priv->array_next) { + if (max_intervals == new_max_intervals) + return 0; + + if (nft_array_intervals_alloc(priv->array_next, new_max_intervals) < 0) + return -ENOMEM; + } else { + array = nft_array_alloc(new_max_intervals); + if (!array) + return -ENOMEM; + + priv->array_next = array; + } + + return 0; +} + static int nft_rbtree_insert(const struct net *net, const struct nft_set *set, const struct nft_set_elem *elem, struct nft_elem_priv **elem_priv) { struct nft_rbtree_elem *rbe = nft_elem_priv_cast(elem->priv); struct nft_rbtree *priv = nft_set_priv(set); + u64 tstamp = nft_net_tstamp(net); int err; + nft_rbtree_maybe_reset_start_cookie(priv, tstamp); + + if (nft_array_may_resize(set, false) < 0) + return -ENOMEM; + do { if (fatal_signal_pending(current)) return -EINTR; @@ -496,9 +698,7 @@ static int nft_rbtree_insert(const struct net *net, const struct nft_set *set, cond_resched(); write_lock_bh(&priv->lock); - write_seqcount_begin(&priv->count); - err = __nft_rbtree_insert(net, set, rbe, elem_priv); - write_seqcount_end(&priv->count); + err = __nft_rbtree_insert(net, set, rbe, elem_priv, tstamp); write_unlock_bh(&priv->lock); } while (err == -EAGAIN); @@ -508,9 +708,7 @@ static int nft_rbtree_insert(const struct net *net, const struct nft_set *set, static void nft_rbtree_erase(struct nft_rbtree *priv, struct nft_rbtree_elem *rbe) { write_lock_bh(&priv->lock); - write_seqcount_begin(&priv->count); rb_erase(&rbe->node, &priv->root); - write_seqcount_end(&priv->count); write_unlock_bh(&priv->lock); } @@ -533,6 +731,48 @@ static void nft_rbtree_activate(const struct net *net, nft_clear(net, &rbe->ext); } +static struct nft_rbtree_elem * +nft_rbtree_next_inactive(struct nft_rbtree_elem *rbe, u8 genmask) +{ + struct nft_rbtree_elem *next_rbe; + struct rb_node *node; + + node = rb_next(&rbe->node); + if (node) { + next_rbe = rb_entry(node, struct nft_rbtree_elem, node); + if (nft_rbtree_interval_start(next_rbe) && + !nft_set_elem_active(&next_rbe->ext, genmask)) + return next_rbe; + } + + return NULL; +} + +static bool nft_rbtree_deactivate_same_interval(const struct net *net, + struct nft_rbtree *priv, + struct nft_rbtree_elem *rbe) +{ + u8 genmask = nft_genmask_next(net); + struct nft_rbtree_elem *next_rbe; + + if (!priv->start_rbe_cookie) + return true; + + next_rbe = nft_rbtree_next_inactive(rbe, genmask); + if (next_rbe) { + /* Closest start element differs from last element added. */ + if (nft_rbtree_interval_start(next_rbe) && + nft_rbtree_cmp_start_cookie(priv, next_rbe)) { + priv->start_rbe_cookie = 0; + return true; + } + } + + priv->start_rbe_cookie = 0; + + return false; +} + static void nft_rbtree_flush(const struct net *net, const struct nft_set *set, struct nft_elem_priv *elem_priv) @@ -547,12 +787,21 @@ nft_rbtree_deactivate(const struct net *net, const struct nft_set *set, const struct nft_set_elem *elem) { struct nft_rbtree_elem *rbe, *this = nft_elem_priv_cast(elem->priv); - const struct nft_rbtree *priv = nft_set_priv(set); + struct nft_rbtree *priv = nft_set_priv(set); const struct rb_node *parent = priv->root.rb_node; u8 genmask = nft_genmask_next(net); u64 tstamp = nft_net_tstamp(net); int d; + nft_rbtree_maybe_reset_start_cookie(priv, tstamp); + + if (nft_rbtree_interval_start(this) || + nft_rbtree_interval_null(set, this)) + priv->start_rbe_cookie = 0; + + if (nft_array_may_resize(set, false) < 0) + return NULL; + while (parent != NULL) { rbe = rb_entry(parent, struct nft_rbtree_elem, node); @@ -577,6 +826,12 @@ nft_rbtree_deactivate(const struct net *net, const struct nft_set *set, parent = parent->rb_left; continue; } + + if (nft_rbtree_interval_start(rbe)) + nft_rbtree_set_start_cookie(priv, rbe); + else if (!nft_rbtree_deactivate_same_interval(net, priv, rbe)) + return NULL; + nft_rbtree_flush(net, set, &rbe->priv); return &rbe->priv; } @@ -613,8 +868,15 @@ static void nft_rbtree_walk(const struct nft_ctx *ctx, struct nft_rbtree *priv = nft_set_priv(set); switch (iter->type) { + case NFT_ITER_UPDATE_CLONE: + if (nft_array_may_resize(set, true) < 0) { + iter->err = -ENOMEM; + break; + } + fallthrough; case NFT_ITER_UPDATE: lockdep_assert_held(&nft_pernet(ctx->net)->commit_mutex); + nft_rbtree_do_walk(ctx, set, iter); break; case NFT_ITER_READ: @@ -629,29 +891,13 @@ static void nft_rbtree_walk(const struct nft_ctx *ctx, } } -static void nft_rbtree_gc_remove(struct net *net, struct nft_set *set, - struct nft_rbtree *priv, - struct nft_rbtree_elem *rbe) -{ - nft_setelem_data_deactivate(net, set, &rbe->priv); - nft_rbtree_erase(priv, rbe); -} - -static void nft_rbtree_gc(struct nft_set *set) +static void nft_rbtree_gc_scan(struct nft_set *set) { struct nft_rbtree *priv = nft_set_priv(set); struct nft_rbtree_elem *rbe, *rbe_end = NULL; struct net *net = read_pnet(&set->net); u64 tstamp = nft_net_tstamp(net); struct rb_node *node, *next; - struct nft_trans_gc *gc; - - set = nft_set_container_of(priv); - net = read_pnet(&set->net); - - gc = nft_trans_gc_alloc(set, 0, GFP_KERNEL); - if (!gc) - return; for (node = rb_first(&priv->root); node ; node = next) { next = rb_next(node); @@ -669,34 +915,46 @@ static void nft_rbtree_gc(struct nft_set *set) if (!__nft_set_elem_expired(&rbe->ext, tstamp)) continue; - gc = nft_trans_gc_queue_sync(gc, GFP_KERNEL); - if (!gc) - goto try_later; - /* end element needs to be removed first, it has * no timeout extension. */ + write_lock_bh(&priv->lock); if (rbe_end) { - nft_rbtree_gc_remove(net, set, priv, rbe_end); - nft_trans_gc_elem_add(gc, rbe_end); + nft_rbtree_gc_elem_move(net, set, priv, rbe_end); rbe_end = NULL; } - gc = nft_trans_gc_queue_sync(gc, GFP_KERNEL); - if (!gc) - goto try_later; - - nft_rbtree_gc_remove(net, set, priv, rbe); - nft_trans_gc_elem_add(gc, rbe); + nft_rbtree_gc_elem_move(net, set, priv, rbe); + write_unlock_bh(&priv->lock); } -try_later: + priv->last_gc = jiffies; +} + +static void nft_rbtree_gc_queue(struct nft_set *set) +{ + struct nft_rbtree *priv = nft_set_priv(set); + struct nft_rbtree_elem *rbe, *rbe_end; + struct nft_trans_gc *gc; + + if (list_empty(&priv->expired)) + return; - if (gc) { - gc = nft_trans_gc_catchall_sync(gc); - nft_trans_gc_queue_sync_done(gc); - priv->last_gc = jiffies; + gc = nft_trans_gc_alloc(set, 0, GFP_KERNEL); + if (!gc) + return; + + list_for_each_entry_safe(rbe, rbe_end, &priv->expired, list) { + list_del(&rbe->list); + nft_trans_gc_elem_add(gc, rbe); + + gc = nft_trans_gc_queue_sync(gc, GFP_KERNEL); + if (!gc) + return; } + + gc = nft_trans_gc_catchall_sync(gc); + nft_trans_gc_queue_sync_done(gc); } static u64 nft_rbtree_privsize(const struct nlattr * const nla[], @@ -714,24 +972,45 @@ static int nft_rbtree_init(const struct nft_set *set, BUILD_BUG_ON(offsetof(struct nft_rbtree_elem, priv) != 0); rwlock_init(&priv->lock); - seqcount_rwlock_init(&priv->count, &priv->lock); priv->root = RB_ROOT; + INIT_LIST_HEAD(&priv->expired); + + priv->array = NULL; + priv->array_next = NULL; return 0; } +static void __nft_array_free(struct nft_array *array) +{ + kvfree(array->intervals); + kfree(array); +} + static void nft_rbtree_destroy(const struct nft_ctx *ctx, const struct nft_set *set) { struct nft_rbtree *priv = nft_set_priv(set); - struct nft_rbtree_elem *rbe; + struct nft_rbtree_elem *rbe, *next; + struct nft_array *array; struct rb_node *node; + list_for_each_entry_safe(rbe, next, &priv->expired, list) { + list_del(&rbe->list); + nf_tables_set_elem_destroy(ctx, set, &rbe->priv); + } + while ((node = priv->root.rb_node) != NULL) { rb_erase(node, &priv->root); rbe = rb_entry(node, struct nft_rbtree_elem, node); nf_tables_set_elem_destroy(ctx, set, &rbe->priv); } + + array = rcu_dereference_protected(priv->array, true); + if (array) + __nft_array_free(array); + if (priv->array_next) + __nft_array_free(priv->array_next); } static bool nft_rbtree_estimate(const struct nft_set_desc *desc, u32 features, @@ -752,12 +1031,105 @@ static bool nft_rbtree_estimate(const struct nft_set_desc *desc, u32 features, return true; } +static void nft_array_free_rcu(struct rcu_head *rcu_head) +{ + struct nft_array *array = container_of(rcu_head, struct nft_array, rcu_head); + + __nft_array_free(array); +} + static void nft_rbtree_commit(struct nft_set *set) { struct nft_rbtree *priv = nft_set_priv(set); + struct nft_rbtree_elem *rbe, *prev_rbe; + struct nft_array *old; + u32 num_intervals = 0; + struct rb_node *node; + /* No changes, skip, eg. elements updates only. */ + if (!priv->array_next) + return; + + /* GC can be performed if the binary search blob is going + * to be rebuilt. It has to be done in two phases: first + * scan tree and move all expired elements to the expired + * list. + * + * Then, after blob has been re-built and published to other + * CPUs, queue collected entries for freeing. + */ if (time_after_eq(jiffies, priv->last_gc + nft_set_gc_interval(set))) - nft_rbtree_gc(set); + nft_rbtree_gc_scan(set); + + /* Reverse walk to create an array from smaller to largest interval. */ + node = rb_last(&priv->root); + if (node) + prev_rbe = rb_entry(node, struct nft_rbtree_elem, node); + else + prev_rbe = NULL; + + while (prev_rbe) { + rbe = prev_rbe; + + if (nft_rbtree_interval_start(rbe)) + priv->array_next->intervals[num_intervals].from = &rbe->ext; + else if (nft_rbtree_interval_end(rbe)) + priv->array_next->intervals[num_intervals++].to = &rbe->ext; + + if (num_intervals >= priv->array_next->max_intervals) { + pr_warn_once("malformed interval set from userspace?"); + goto err_out; + } + + node = rb_prev(node); + if (!node) + break; + + prev_rbe = rb_entry(node, struct nft_rbtree_elem, node); + + /* For anonymous sets, when adjacent ranges are found, + * the end element is not added to the set to pack the set + * representation. Use next start element to complete this + * interval. + */ + if (nft_rbtree_interval_start(rbe) && + nft_rbtree_interval_start(prev_rbe) && + priv->array_next->intervals[num_intervals].from) + priv->array_next->intervals[num_intervals++].to = &prev_rbe->ext; + + if (num_intervals >= priv->array_next->max_intervals) { + pr_warn_once("malformed interval set from userspace?"); + goto err_out; + } + } + + if (priv->array_next->intervals[num_intervals].from) + num_intervals++; +err_out: + priv->array_next->num_intervals = num_intervals; + old = rcu_replace_pointer(priv->array, priv->array_next, + lockdep_is_held(&nft_pernet(read_pnet(&set->net))->commit_mutex)); + priv->array_next = NULL; + if (old) + call_rcu(&old->rcu_head, nft_array_free_rcu); + + /* New blob is public, queue collected entries for freeing. + * call_rcu ensures elements stay around until readers are done. + */ + nft_rbtree_gc_queue(set); +} + +static void nft_rbtree_abort(const struct nft_set *set) +{ + struct nft_rbtree *priv = nft_set_priv(set); + struct nft_array *array_next; + + if (!priv->array_next) + return; + + array_next = priv->array_next; + priv->array_next = NULL; + __nft_array_free(array_next); } static void nft_rbtree_gc_init(const struct nft_set *set) @@ -821,6 +1193,7 @@ const struct nft_set_type nft_set_rbtree_type = { .flush = nft_rbtree_flush, .activate = nft_rbtree_activate, .commit = nft_rbtree_commit, + .abort = nft_rbtree_abort, .gc_init = nft_rbtree_gc_init, .lookup = nft_rbtree_lookup, .walk = nft_rbtree_walk, diff --git a/net/netfilter/x_tables.c b/net/netfilter/x_tables.c index 48105ea3df1522..1ca4fa9d249b8f 100644 --- a/net/netfilter/x_tables.c +++ b/net/netfilter/x_tables.c @@ -501,6 +501,17 @@ int xt_check_match(struct xt_mtchk_param *par, par->match->table, par->table); return -EINVAL; } + + /* NFPROTO_UNSPEC implies NF_INET_* hooks which do not overlap with + * NF_ARP_IN,OUT,FORWARD, allow explicit extensions with NFPROTO_ARP + * support. + */ + if (par->family == NFPROTO_ARP && + par->match->family != NFPROTO_ARP) { + pr_info_ratelimited("%s_tables: %s match: not valid for this family\n", + xt_prefix[par->family], par->match->name); + return -EINVAL; + } if (par->match->hooks && (par->hook_mask & ~par->match->hooks) != 0) { char used[64], allow[64]; @@ -1016,6 +1027,18 @@ int xt_check_target(struct xt_tgchk_param *par, par->target->table, par->table); return -EINVAL; } + + /* NFPROTO_UNSPEC implies NF_INET_* hooks which do not overlap with + * NF_ARP_IN,OUT,FORWARD, allow explicit extensions with NFPROTO_ARP + * support. + */ + if (par->family == NFPROTO_ARP && + par->target->family != NFPROTO_ARP) { + pr_info_ratelimited("%s_tables: %s target: not valid for this family\n", + xt_prefix[par->family], par->target->name); + return -EINVAL; + } + if (par->target->hooks && (par->hook_mask & ~par->target->hooks) != 0) { char used[64], allow[64]; diff --git a/net/netfilter/xt_CT.c b/net/netfilter/xt_CT.c index 3ba94c34297cf5..498f5871c84a0e 100644 --- a/net/netfilter/xt_CT.c +++ b/net/netfilter/xt_CT.c @@ -16,6 +16,7 @@ #include #include #include +#include "nf_internals.h" static inline int xt_ct_target(struct sk_buff *skb, struct nf_conn *ct) { @@ -283,6 +284,9 @@ static void xt_ct_tg_destroy(const struct xt_tgdtor_param *par, struct nf_conn_help *help; if (ct) { + if (info->helper[0] || info->timeout[0]) + nf_queue_nf_hook_drop(par->net); + help = nfct_help(ct); xt_ct_put_helper(help); diff --git a/net/netfilter/xt_IDLETIMER.c b/net/netfilter/xt_IDLETIMER.c index d73957592c9d9b..bb7af92ac82a44 100644 --- a/net/netfilter/xt_IDLETIMER.c +++ b/net/netfilter/xt_IDLETIMER.c @@ -318,6 +318,12 @@ static int idletimer_tg_checkentry(const struct xt_tgchk_param *par) info->timer = __idletimer_tg_find_by_label(info->label); if (info->timer) { + if (info->timer->timer_type & XT_IDLETIMER_ALARM) { + pr_debug("Adding/Replacing rule with same label and different timer type is not allowed\n"); + mutex_unlock(&list_mutex); + return -EINVAL; + } + info->timer->refcnt++; mod_timer(&info->timer->timer, secs_to_jiffies(info->timeout) + jiffies); diff --git a/net/netfilter/xt_cgroup.c b/net/netfilter/xt_cgroup.c index c437fbd59ec130..43d2ae2be628dc 100644 --- a/net/netfilter/xt_cgroup.c +++ b/net/netfilter/xt_cgroup.c @@ -65,6 +65,9 @@ static int cgroup_mt_check_v1(const struct xt_mtchk_param *par) info->priv = NULL; if (info->has_path) { + if (strnlen(info->path, sizeof(info->path)) >= sizeof(info->path)) + return -ENAMETOOLONG; + cgrp = cgroup_get_from_path(info->path); if (IS_ERR(cgrp)) { pr_info_ratelimited("invalid path, errno=%ld\n", @@ -102,6 +105,9 @@ static int cgroup_mt_check_v2(const struct xt_mtchk_param *par) info->priv = NULL; if (info->has_path) { + if (strnlen(info->path, sizeof(info->path)) >= sizeof(info->path)) + return -ENAMETOOLONG; + cgrp = cgroup_get_from_path(info->path); if (IS_ERR(cgrp)) { pr_info_ratelimited("invalid path, errno=%ld\n", diff --git a/net/netfilter/xt_dccp.c b/net/netfilter/xt_dccp.c index e5a13ecbe67a01..037ab93e25d0af 100644 --- a/net/netfilter/xt_dccp.c +++ b/net/netfilter/xt_dccp.c @@ -62,10 +62,10 @@ dccp_find_option(u_int8_t option, return true; } - if (op[i] < 2) + if (op[i] < 2 || i == optlen - 1) i++; else - i += op[i+1]?:1; + i += op[i + 1] ? : 1; } spin_unlock_bh(&dccp_buflock); diff --git a/net/netfilter/xt_multiport.c b/net/netfilter/xt_multiport.c index 44a00f5acde8ae..a1691ff405d3c4 100644 --- a/net/netfilter/xt_multiport.c +++ b/net/netfilter/xt_multiport.c @@ -105,6 +105,28 @@ multiport_mt(const struct sk_buff *skb, struct xt_action_param *par) return ports_match_v1(multiinfo, ntohs(pptr[0]), ntohs(pptr[1])); } +static bool +multiport_valid_ranges(const struct xt_multiport_v1 *multiinfo) +{ + unsigned int i; + + for (i = 0; i < multiinfo->count; i++) { + if (!multiinfo->pflags[i]) + continue; + + if (++i >= multiinfo->count) + return false; + + if (multiinfo->pflags[i]) + return false; + + if (multiinfo->ports[i - 1] > multiinfo->ports[i]) + return false; + } + + return true; +} + static inline bool check(u_int16_t proto, u_int8_t ip_invflags, @@ -127,8 +149,10 @@ static int multiport_mt_check(const struct xt_mtchk_param *par) const struct ipt_ip *ip = par->entryinfo; const struct xt_multiport_v1 *multiinfo = par->matchinfo; - return check(ip->proto, ip->invflags, multiinfo->flags, - multiinfo->count) ? 0 : -EINVAL; + if (!check(ip->proto, ip->invflags, multiinfo->flags, multiinfo->count)) + return -EINVAL; + + return multiport_valid_ranges(multiinfo) ? 0 : -EINVAL; } static int multiport_mt6_check(const struct xt_mtchk_param *par) @@ -136,8 +160,10 @@ static int multiport_mt6_check(const struct xt_mtchk_param *par) const struct ip6t_ip6 *ip = par->entryinfo; const struct xt_multiport_v1 *multiinfo = par->matchinfo; - return check(ip->proto, ip->invflags, multiinfo->flags, - multiinfo->count) ? 0 : -EINVAL; + if (!check(ip->proto, ip->invflags, multiinfo->flags, multiinfo->count)) + return -EINVAL; + + return multiport_valid_ranges(multiinfo) ? 0 : -EINVAL; } static struct xt_match multiport_mt_reg[] __read_mostly = { diff --git a/net/netfilter/xt_rateest.c b/net/netfilter/xt_rateest.c index 72324bd976af8c..b1d736c15fcbe5 100644 --- a/net/netfilter/xt_rateest.c +++ b/net/netfilter/xt_rateest.c @@ -91,6 +91,11 @@ static int xt_rateest_mt_checkentry(const struct xt_mtchk_param *par) goto err1; } + if (strnlen(info->name1, sizeof(info->name1)) >= sizeof(info->name1)) + return -ENAMETOOLONG; + if (strnlen(info->name2, sizeof(info->name2)) >= sizeof(info->name2)) + return -ENAMETOOLONG; + ret = -ENOENT; est1 = xt_rateest_lookup(par->net, info->name1); if (!est1) diff --git a/net/netfilter/xt_tcpmss.c b/net/netfilter/xt_tcpmss.c index 37704ab0179923..0d32d4841cb325 100644 --- a/net/netfilter/xt_tcpmss.c +++ b/net/netfilter/xt_tcpmss.c @@ -61,7 +61,7 @@ tcpmss_mt(const struct sk_buff *skb, struct xt_action_param *par) return (mssval >= info->mss_min && mssval <= info->mss_max) ^ info->invert; } - if (op[i] < 2) + if (op[i] < 2 || i == optlen - 1) i++; else i += op[i+1] ? : 1; diff --git a/net/netfilter/xt_tcpudp.c b/net/netfilter/xt_tcpudp.c index e8991130a3de0c..f76cf18f1a2445 100644 --- a/net/netfilter/xt_tcpudp.c +++ b/net/netfilter/xt_tcpudp.c @@ -59,8 +59,10 @@ tcp_find_option(u_int8_t option, for (i = 0; i < optlen; ) { if (op[i] == option) return !invert; - if (op[i] < 2) i++; - else i += op[i+1]?:1; + if (op[i] < 2 || i == optlen - 1) + i++; + else + i += op[i + 1] ? : 1; } return invert; diff --git a/net/netfilter/xt_time.c b/net/netfilter/xt_time.c index 6aa12d0f54e23c..61de85e02a40fb 100644 --- a/net/netfilter/xt_time.c +++ b/net/netfilter/xt_time.c @@ -227,13 +227,13 @@ time_mt(const struct sk_buff *skb, struct xt_action_param *par) localtime_2(¤t_time, stamp); - if (!(info->weekdays_match & (1 << current_time.weekday))) + if (!(info->weekdays_match & (1U << current_time.weekday))) return false; /* Do not spend time computing monthday if all days match anyway */ if (info->monthdays_match != XT_TIME_ALL_MONTHDAYS) { localtime_3(¤t_time, stamp); - if (!(info->monthdays_match & (1 << current_time.monthday))) + if (!(info->monthdays_match & (1U << current_time.monthday))) return false; } diff --git a/net/nfc/digital_technology.c b/net/nfc/digital_technology.c index 3adf4589852aff..e29dd10f280ed7 100644 --- a/net/nfc/digital_technology.c +++ b/net/nfc/digital_technology.c @@ -424,6 +424,12 @@ static void digital_in_recv_sdd_res(struct nfc_digital_dev *ddev, void *arg, size = 4; } + if (target->nfcid1_len + size > NFC_NFCID1_MAXSIZE) { + PROTOCOL_ERR("4.7.2.1"); + rc = -EPROTO; + goto exit; + } + memcpy(target->nfcid1 + target->nfcid1_len, sdd_res->nfcid1 + offset, size); target->nfcid1_len += size; diff --git a/net/nfc/hci/llc_shdlc.c b/net/nfc/hci/llc_shdlc.c index 4fc37894860c9e..08c8aa1530d8a0 100644 --- a/net/nfc/hci/llc_shdlc.c +++ b/net/nfc/hci/llc_shdlc.c @@ -762,6 +762,14 @@ static void llc_shdlc_deinit(struct nfc_llc *llc) { struct llc_shdlc *shdlc = nfc_llc_get_data(llc); + timer_shutdown_sync(&shdlc->connect_timer); + timer_shutdown_sync(&shdlc->t1_timer); + timer_shutdown_sync(&shdlc->t2_timer); + shdlc->t1_active = false; + shdlc->t2_active = false; + + cancel_work_sync(&shdlc->sm_work); + skb_queue_purge(&shdlc->rcv_q); skb_queue_purge(&shdlc->send_q); skb_queue_purge(&shdlc->ack_pending_q); diff --git a/net/nfc/llcp_core.c b/net/nfc/llcp_core.c index 444a3774c8e806..da8d3add0018f3 100644 --- a/net/nfc/llcp_core.c +++ b/net/nfc/llcp_core.c @@ -1091,6 +1091,7 @@ static void nfc_llcp_recv_hdlc(struct nfc_llcp_local *local, if (sk->sk_state == LLCP_CLOSED) { release_sock(sk); nfc_llcp_sock_put(llcp_sock); + return; } /* Pass the payload upstream */ @@ -1182,6 +1183,7 @@ static void nfc_llcp_recv_disc(struct nfc_llcp_local *local, if (sk->sk_state == LLCP_CLOSED) { release_sock(sk); nfc_llcp_sock_put(llcp_sock); + return; } if (sk->sk_state == LLCP_CONNECTED) { diff --git a/net/nfc/nci/core.c b/net/nfc/nci/core.c index e419e020a70a33..25ba4cbb00e1ea 100644 --- a/net/nfc/nci/core.c +++ b/net/nfc/nci/core.c @@ -567,6 +567,10 @@ static int nci_close_device(struct nci_dev *ndev) flush_workqueue(ndev->cmd_wq); timer_delete_sync(&ndev->cmd_timer); timer_delete_sync(&ndev->data_timer); + if (test_bit(NCI_DATA_EXCHANGE, &ndev->flags)) + nci_data_exchange_complete(ndev, NULL, + ndev->cur_conn_id, + -ENODEV); mutex_unlock(&ndev->req_lock); return 0; } @@ -575,8 +579,7 @@ static int nci_close_device(struct nci_dev *ndev) skb_queue_purge(&ndev->rx_q); skb_queue_purge(&ndev->tx_q); - /* Flush RX and TX wq */ - flush_workqueue(ndev->rx_wq); + /* Flush TX wq, RX wq flush can't be under the lock */ flush_workqueue(ndev->tx_wq); /* Reset device */ @@ -588,22 +591,30 @@ static int nci_close_device(struct nci_dev *ndev) msecs_to_jiffies(NCI_RESET_TIMEOUT)); /* After this point our queues are empty - * and no works are scheduled. + * rx work may be running but will see that NCI_UP was cleared */ ndev->ops->close(ndev); clear_bit(NCI_INIT, &ndev->flags); - /* Flush cmd wq */ + /* Flush cmd and tx wq */ flush_workqueue(ndev->cmd_wq); timer_delete_sync(&ndev->cmd_timer); + timer_delete_sync(&ndev->data_timer); + + if (test_bit(NCI_DATA_EXCHANGE, &ndev->flags)) + nci_data_exchange_complete(ndev, NULL, ndev->cur_conn_id, + -ENODEV); /* Clear flags except NCI_UNREG */ ndev->flags &= BIT(NCI_UNREG); mutex_unlock(&ndev->req_lock); + /* rx_work may take req_lock via nci_deactivate_target */ + flush_workqueue(ndev->rx_wq); + return 0; } @@ -1035,18 +1046,23 @@ static int nci_transceive(struct nfc_dev *nfc_dev, struct nfc_target *target, struct nci_conn_info *conn_info; conn_info = ndev->rf_conn_info; - if (!conn_info) + if (!conn_info) { + kfree_skb(skb); return -EPROTO; + } pr_debug("target_idx %d, len %d\n", target->idx, skb->len); if (!ndev->target_active_prot) { pr_err("unable to exchange data, no active target\n"); + kfree_skb(skb); return -EINVAL; } - if (test_and_set_bit(NCI_DATA_EXCHANGE, &ndev->flags)) + if (test_and_set_bit(NCI_DATA_EXCHANGE, &ndev->flags)) { + kfree_skb(skb); return -EBUSY; + } /* store cb and context to be used on receiving data */ conn_info->data_exchange_cb = cb; @@ -1482,10 +1498,20 @@ static bool nci_valid_size(struct sk_buff *skb) unsigned int hdr_size = NCI_CTRL_HDR_SIZE; if (skb->len < hdr_size || - !nci_plen(skb->data) || skb->len < hdr_size + nci_plen(skb->data)) { return false; } + + if (!nci_plen(skb->data)) { + /* Allow zero length in proprietary notifications (0x20 - 0x3F). */ + if (nci_opcode_oid(nci_opcode(skb->data)) >= 0x20 && + nci_mt(skb->data) == NCI_MT_NTF_PKT) + return true; + + /* Disallow zero length otherwise. */ + return false; + } + return true; } diff --git a/net/nfc/nci/data.c b/net/nfc/nci/data.c index 78f4131af3cf3c..5f98c73db5afde 100644 --- a/net/nfc/nci/data.c +++ b/net/nfc/nci/data.c @@ -33,7 +33,8 @@ void nci_data_exchange_complete(struct nci_dev *ndev, struct sk_buff *skb, conn_info = nci_get_conn_info_by_conn_id(ndev, conn_id); if (!conn_info) { kfree_skb(skb); - goto exit; + clear_bit(NCI_DATA_EXCHANGE, &ndev->flags); + return; } cb = conn_info->data_exchange_cb; @@ -45,6 +46,12 @@ void nci_data_exchange_complete(struct nci_dev *ndev, struct sk_buff *skb, timer_delete_sync(&ndev->data_timer); clear_bit(NCI_DATA_EXCHANGE_TO, &ndev->flags); + /* Mark the exchange as done before calling the callback. + * The callback (e.g. rawsock_data_exchange_complete) may + * want to immediately queue another data exchange. + */ + clear_bit(NCI_DATA_EXCHANGE, &ndev->flags); + if (cb) { /* forward skb to nfc core */ cb(cb_context, skb, err); @@ -54,9 +61,6 @@ void nci_data_exchange_complete(struct nci_dev *ndev, struct sk_buff *skb, /* no waiting callback, free skb */ kfree_skb(skb); } - -exit: - clear_bit(NCI_DATA_EXCHANGE, &ndev->flags); } /* ----------------- NCI TX Data ----------------- */ diff --git a/net/nfc/nci/ntf.c b/net/nfc/nci/ntf.c index 418b84e2b2605f..c96512bb865315 100644 --- a/net/nfc/nci/ntf.c +++ b/net/nfc/nci/ntf.c @@ -58,7 +58,7 @@ static int nci_core_conn_credits_ntf_packet(struct nci_dev *ndev, struct nci_conn_info *conn_info; int i; - if (skb->len < sizeof(struct nci_core_conn_credit_ntf)) + if (skb->len < offsetofend(struct nci_core_conn_credit_ntf, num_entries)) return -EINVAL; ntf = (struct nci_core_conn_credit_ntf *)skb->data; @@ -68,6 +68,10 @@ static int nci_core_conn_credits_ntf_packet(struct nci_dev *ndev, if (ntf->num_entries > NCI_MAX_NUM_CONN) ntf->num_entries = NCI_MAX_NUM_CONN; + if (skb->len < offsetofend(struct nci_core_conn_credit_ntf, num_entries) + + ntf->num_entries * sizeof(struct conn_credit_entry)) + return -EINVAL; + /* update the credits */ for (i = 0; i < ntf->num_entries; i++) { ntf->conn_entries[i].conn_id = @@ -138,23 +142,48 @@ static int nci_core_conn_intf_error_ntf_packet(struct nci_dev *ndev, static const __u8 * nci_extract_rf_params_nfca_passive_poll(struct nci_dev *ndev, struct rf_tech_specific_params_nfca_poll *nfca_poll, - const __u8 *data) + const __u8 *data, ssize_t data_len) { + /* Check if we have enough data for sens_res (2 bytes) */ + if (data_len < 2) + return ERR_PTR(-EINVAL); + nfca_poll->sens_res = __le16_to_cpu(*((__le16 *)data)); data += 2; + data_len -= 2; + + /* Check if we have enough data for nfcid1_len (1 byte) */ + if (data_len < 1) + return ERR_PTR(-EINVAL); nfca_poll->nfcid1_len = min_t(__u8, *data++, NFC_NFCID1_MAXSIZE); + data_len--; pr_debug("sens_res 0x%x, nfcid1_len %d\n", nfca_poll->sens_res, nfca_poll->nfcid1_len); + /* Check if we have enough data for nfcid1 */ + if (data_len < nfca_poll->nfcid1_len) + return ERR_PTR(-EINVAL); + memcpy(nfca_poll->nfcid1, data, nfca_poll->nfcid1_len); data += nfca_poll->nfcid1_len; + data_len -= nfca_poll->nfcid1_len; + + /* Check if we have enough data for sel_res_len (1 byte) */ + if (data_len < 1) + return ERR_PTR(-EINVAL); nfca_poll->sel_res_len = *data++; + data_len--; + + if (nfca_poll->sel_res_len != 0) { + /* Check if we have enough data for sel_res (1 byte) */ + if (data_len < 1) + return ERR_PTR(-EINVAL); - if (nfca_poll->sel_res_len != 0) nfca_poll->sel_res = *data++; + } pr_debug("sel_res_len %d, sel_res 0x%x\n", nfca_poll->sel_res_len, @@ -166,12 +195,21 @@ nci_extract_rf_params_nfca_passive_poll(struct nci_dev *ndev, static const __u8 * nci_extract_rf_params_nfcb_passive_poll(struct nci_dev *ndev, struct rf_tech_specific_params_nfcb_poll *nfcb_poll, - const __u8 *data) + const __u8 *data, ssize_t data_len) { + /* Check if we have enough data for sensb_res_len (1 byte) */ + if (data_len < 1) + return ERR_PTR(-EINVAL); + nfcb_poll->sensb_res_len = min_t(__u8, *data++, NFC_SENSB_RES_MAXSIZE); + data_len--; pr_debug("sensb_res_len %d\n", nfcb_poll->sensb_res_len); + /* Check if we have enough data for sensb_res */ + if (data_len < nfcb_poll->sensb_res_len) + return ERR_PTR(-EINVAL); + memcpy(nfcb_poll->sensb_res, data, nfcb_poll->sensb_res_len); data += nfcb_poll->sensb_res_len; @@ -181,14 +219,29 @@ nci_extract_rf_params_nfcb_passive_poll(struct nci_dev *ndev, static const __u8 * nci_extract_rf_params_nfcf_passive_poll(struct nci_dev *ndev, struct rf_tech_specific_params_nfcf_poll *nfcf_poll, - const __u8 *data) + const __u8 *data, ssize_t data_len) { + /* Check if we have enough data for bit_rate (1 byte) */ + if (data_len < 1) + return ERR_PTR(-EINVAL); + nfcf_poll->bit_rate = *data++; + data_len--; + + /* Check if we have enough data for sensf_res_len (1 byte) */ + if (data_len < 1) + return ERR_PTR(-EINVAL); + nfcf_poll->sensf_res_len = min_t(__u8, *data++, NFC_SENSF_RES_MAXSIZE); + data_len--; pr_debug("bit_rate %d, sensf_res_len %d\n", nfcf_poll->bit_rate, nfcf_poll->sensf_res_len); + /* Check if we have enough data for sensf_res */ + if (data_len < nfcf_poll->sensf_res_len) + return ERR_PTR(-EINVAL); + memcpy(nfcf_poll->sensf_res, data, nfcf_poll->sensf_res_len); data += nfcf_poll->sensf_res_len; @@ -198,22 +251,49 @@ nci_extract_rf_params_nfcf_passive_poll(struct nci_dev *ndev, static const __u8 * nci_extract_rf_params_nfcv_passive_poll(struct nci_dev *ndev, struct rf_tech_specific_params_nfcv_poll *nfcv_poll, - const __u8 *data) + const __u8 *data, ssize_t data_len) { + /* Skip 1 byte (reserved) */ + if (data_len < 1) + return ERR_PTR(-EINVAL); + ++data; + data_len--; + + /* Check if we have enough data for dsfid (1 byte) */ + if (data_len < 1) + return ERR_PTR(-EINVAL); + nfcv_poll->dsfid = *data++; + data_len--; + + /* Check if we have enough data for uid (8 bytes) */ + if (data_len < NFC_ISO15693_UID_MAXSIZE) + return ERR_PTR(-EINVAL); + memcpy(nfcv_poll->uid, data, NFC_ISO15693_UID_MAXSIZE); data += NFC_ISO15693_UID_MAXSIZE; + return data; } static const __u8 * nci_extract_rf_params_nfcf_passive_listen(struct nci_dev *ndev, struct rf_tech_specific_params_nfcf_listen *nfcf_listen, - const __u8 *data) + const __u8 *data, ssize_t data_len) { + /* Check if we have enough data for local_nfcid2_len (1 byte) */ + if (data_len < 1) + return ERR_PTR(-EINVAL); + nfcf_listen->local_nfcid2_len = min_t(__u8, *data++, NFC_NFCID2_MAXSIZE); + data_len--; + + /* Check if we have enough data for local_nfcid2 */ + if (data_len < nfcf_listen->local_nfcid2_len) + return ERR_PTR(-EINVAL); + memcpy(nfcf_listen->local_nfcid2, data, nfcf_listen->local_nfcid2_len); data += nfcf_listen->local_nfcid2_len; @@ -364,7 +444,7 @@ static int nci_rf_discover_ntf_packet(struct nci_dev *ndev, const __u8 *data; bool add_target = true; - if (skb->len < sizeof(struct nci_rf_discover_ntf)) + if (skb->len < offsetofend(struct nci_rf_discover_ntf, rf_tech_specific_params_len)) return -EINVAL; data = skb->data; @@ -380,26 +460,42 @@ static int nci_rf_discover_ntf_packet(struct nci_dev *ndev, pr_debug("rf_tech_specific_params_len %d\n", ntf.rf_tech_specific_params_len); + if (skb->len < (data - skb->data) + + ntf.rf_tech_specific_params_len + sizeof(ntf.ntf_type)) + return -EINVAL; + if (ntf.rf_tech_specific_params_len > 0) { switch (ntf.rf_tech_and_mode) { case NCI_NFC_A_PASSIVE_POLL_MODE: data = nci_extract_rf_params_nfca_passive_poll(ndev, - &(ntf.rf_tech_specific_params.nfca_poll), data); + &(ntf.rf_tech_specific_params.nfca_poll), data, + ntf.rf_tech_specific_params_len); + if (IS_ERR(data)) + return PTR_ERR(data); break; case NCI_NFC_B_PASSIVE_POLL_MODE: data = nci_extract_rf_params_nfcb_passive_poll(ndev, - &(ntf.rf_tech_specific_params.nfcb_poll), data); + &(ntf.rf_tech_specific_params.nfcb_poll), data, + ntf.rf_tech_specific_params_len); + if (IS_ERR(data)) + return PTR_ERR(data); break; case NCI_NFC_F_PASSIVE_POLL_MODE: data = nci_extract_rf_params_nfcf_passive_poll(ndev, - &(ntf.rf_tech_specific_params.nfcf_poll), data); + &(ntf.rf_tech_specific_params.nfcf_poll), data, + ntf.rf_tech_specific_params_len); + if (IS_ERR(data)) + return PTR_ERR(data); break; case NCI_NFC_V_PASSIVE_POLL_MODE: data = nci_extract_rf_params_nfcv_passive_poll(ndev, - &(ntf.rf_tech_specific_params.nfcv_poll), data); + &(ntf.rf_tech_specific_params.nfcv_poll), data, + ntf.rf_tech_specific_params_len); + if (IS_ERR(data)) + return PTR_ERR(data); break; default: @@ -596,7 +692,7 @@ static int nci_rf_intf_activated_ntf_packet(struct nci_dev *ndev, const __u8 *data; int err = NCI_STATUS_OK; - if (skb->len < sizeof(struct nci_rf_intf_activated_ntf)) + if (skb->len < offsetofend(struct nci_rf_intf_activated_ntf, rf_tech_specific_params_len)) return -EINVAL; data = skb->data; @@ -628,26 +724,41 @@ static int nci_rf_intf_activated_ntf_packet(struct nci_dev *ndev, if (ntf.rf_interface == NCI_RF_INTERFACE_NFCEE_DIRECT) goto listen; + if (skb->len < (data - skb->data) + ntf.rf_tech_specific_params_len) + return -EINVAL; + if (ntf.rf_tech_specific_params_len > 0) { switch (ntf.activation_rf_tech_and_mode) { case NCI_NFC_A_PASSIVE_POLL_MODE: data = nci_extract_rf_params_nfca_passive_poll(ndev, - &(ntf.rf_tech_specific_params.nfca_poll), data); + &(ntf.rf_tech_specific_params.nfca_poll), data, + ntf.rf_tech_specific_params_len); + if (IS_ERR(data)) + return -EINVAL; break; case NCI_NFC_B_PASSIVE_POLL_MODE: data = nci_extract_rf_params_nfcb_passive_poll(ndev, - &(ntf.rf_tech_specific_params.nfcb_poll), data); + &(ntf.rf_tech_specific_params.nfcb_poll), data, + ntf.rf_tech_specific_params_len); + if (IS_ERR(data)) + return -EINVAL; break; case NCI_NFC_F_PASSIVE_POLL_MODE: data = nci_extract_rf_params_nfcf_passive_poll(ndev, - &(ntf.rf_tech_specific_params.nfcf_poll), data); + &(ntf.rf_tech_specific_params.nfcf_poll), data, + ntf.rf_tech_specific_params_len); + if (IS_ERR(data)) + return -EINVAL; break; case NCI_NFC_V_PASSIVE_POLL_MODE: data = nci_extract_rf_params_nfcv_passive_poll(ndev, - &(ntf.rf_tech_specific_params.nfcv_poll), data); + &(ntf.rf_tech_specific_params.nfcv_poll), data, + ntf.rf_tech_specific_params_len); + if (IS_ERR(data)) + return -EINVAL; break; case NCI_NFC_A_PASSIVE_LISTEN_MODE: @@ -657,7 +768,9 @@ static int nci_rf_intf_activated_ntf_packet(struct nci_dev *ndev, case NCI_NFC_F_PASSIVE_LISTEN_MODE: data = nci_extract_rf_params_nfcf_passive_listen(ndev, &(ntf.rf_tech_specific_params.nfcf_listen), - data); + data, ntf.rf_tech_specific_params_len); + if (IS_ERR(data)) + return -EINVAL; break; default: @@ -668,6 +781,13 @@ static int nci_rf_intf_activated_ntf_packet(struct nci_dev *ndev, } } + if (skb->len < (data - skb->data) + + sizeof(ntf.data_exch_rf_tech_and_mode) + + sizeof(ntf.data_exch_tx_bit_rate) + + sizeof(ntf.data_exch_rx_bit_rate) + + sizeof(ntf.activation_params_len)) + return -EINVAL; + ntf.data_exch_rf_tech_and_mode = *data++; ntf.data_exch_tx_bit_rate = *data++; ntf.data_exch_rx_bit_rate = *data++; @@ -679,6 +799,9 @@ static int nci_rf_intf_activated_ntf_packet(struct nci_dev *ndev, pr_debug("data_exch_rx_bit_rate 0x%x\n", ntf.data_exch_rx_bit_rate); pr_debug("activation_params_len %d\n", ntf.activation_params_len); + if (skb->len < (data - skb->data) + ntf.activation_params_len) + return -EINVAL; + if (ntf.activation_params_len > 0) { switch (ntf.rf_interface) { case NCI_RF_INTERFACE_ISO_DEP: diff --git a/net/nfc/rawsock.c b/net/nfc/rawsock.c index b049022399aea0..f7d7a599fade77 100644 --- a/net/nfc/rawsock.c +++ b/net/nfc/rawsock.c @@ -67,6 +67,17 @@ static int rawsock_release(struct socket *sock) if (sock->type == SOCK_RAW) nfc_sock_unlink(&raw_sk_list, sk); + if (sk->sk_state == TCP_ESTABLISHED) { + /* Prevent rawsock_tx_work from starting new transmits and + * wait for any in-progress work to finish. This must happen + * before the socket is orphaned to avoid a race where + * rawsock_tx_work runs after the NCI device has been freed. + */ + sk->sk_shutdown |= SEND_SHUTDOWN; + cancel_work_sync(&nfc_rawsock(sk)->tx_work); + rawsock_write_queue_purge(sk); + } + sock_orphan(sk); sock_put(sk); diff --git a/net/openvswitch/flow_netlink.c b/net/openvswitch/flow_netlink.c index 2d536901309ea9..2dc4a6c2aecec5 100644 --- a/net/openvswitch/flow_netlink.c +++ b/net/openvswitch/flow_netlink.c @@ -2953,6 +2953,8 @@ static int validate_set(const struct nlattr *a, case OVS_KEY_ATTR_MPLS: if (!eth_p_mpls(eth_type)) return -EINVAL; + if (key_len != sizeof(struct ovs_key_mpls)) + return -EINVAL; break; case OVS_KEY_ATTR_SCTP: diff --git a/net/openvswitch/vport-netdev.c b/net/openvswitch/vport-netdev.c index 6574f9bcdc0268..12055af832dc08 100644 --- a/net/openvswitch/vport-netdev.c +++ b/net/openvswitch/vport-netdev.c @@ -151,11 +151,15 @@ static void vport_netdev_free(struct rcu_head *rcu) void ovs_netdev_detach_dev(struct vport *vport) { ASSERT_RTNL(); - vport->dev->priv_flags &= ~IFF_OVS_DATAPATH; netdev_rx_handler_unregister(vport->dev); netdev_upper_dev_unlink(vport->dev, netdev_master_upper_dev_get(vport->dev)); dev_set_promiscuity(vport->dev, -1); + + /* paired with smp_mb() in netdev_destroy() */ + smp_wmb(); + + vport->dev->priv_flags &= ~IFF_OVS_DATAPATH; } static void netdev_destroy(struct vport *vport) @@ -174,6 +178,9 @@ static void netdev_destroy(struct vport *vport) rtnl_unlock(); } + /* paired with smp_wmb() in ovs_netdev_detach_dev() */ + smp_mb(); + call_rcu(&vport->rcu, vport_netdev_free); } @@ -189,8 +196,6 @@ void ovs_netdev_tunnel_destroy(struct vport *vport) */ if (vport->dev->reg_state == NETREG_REGISTERED) rtnl_delete_link(vport->dev, 0, NULL); - netdev_put(vport->dev, &vport->dev_tracker); - vport->dev = NULL; rtnl_unlock(); call_rcu(&vport->rcu, vport_netdev_free); diff --git a/net/packet/af_packet.c b/net/packet/af_packet.c index 494d628d10a510..070f7eba6b837a 100644 --- a/net/packet/af_packet.c +++ b/net/packet/af_packet.c @@ -3135,6 +3135,7 @@ static int packet_release(struct socket *sock) spin_lock(&po->bind_lock); unregister_prot_hook(sk, false); + WRITE_ONCE(po->num, 0); packet_cached_dev_reset(po); if (po->prot_hook.dev) { diff --git a/net/phonet/af_phonet.c b/net/phonet/af_phonet.c index 238a9638d2b0f6..d89225d6bfd3bf 100644 --- a/net/phonet/af_phonet.c +++ b/net/phonet/af_phonet.c @@ -129,9 +129,12 @@ static int pn_header_create(struct sk_buff *skb, struct net_device *dev, return 1; } -static int pn_header_parse(const struct sk_buff *skb, unsigned char *haddr) +static int pn_header_parse(const struct sk_buff *skb, + const struct net_device *dev, + unsigned char *haddr) { const u8 *media = skb_mac_header(skb); + *haddr = *media; return 1; } diff --git a/net/psp/Kconfig b/net/psp/Kconfig index 371e8771f3bd38..84d6b0f2546067 100644 --- a/net/psp/Kconfig +++ b/net/psp/Kconfig @@ -6,6 +6,7 @@ config INET_PSP bool "PSP Security Protocol support" depends on INET select SKB_DECRYPTED + select SKB_EXTENSIONS select SOCK_VALIDATE_XMIT help Enable kernel support for the PSP Security Protocol (PSP). diff --git a/net/psp/psp_main.c b/net/psp/psp_main.c index a8534124f62669..066222eb56c4af 100644 --- a/net/psp/psp_main.c +++ b/net/psp/psp_main.c @@ -166,9 +166,46 @@ static void psp_write_headers(struct net *net, struct sk_buff *skb, __be32 spi, { struct udphdr *uh = udp_hdr(skb); struct psphdr *psph = (struct psphdr *)(uh + 1); + const struct sock *sk = skb->sk; uh->dest = htons(PSP_DEFAULT_UDP_PORT); - uh->source = udp_flow_src_port(net, skb, 0, 0, false); + + /* A bit of theory: Selection of the source port. + * + * We need some entropy, so that multiple flows use different + * source ports for better RSS spreading at the receiver. + * + * We also need that all packets belonging to one TCP flow + * use the same source port through their duration, + * so that all these packets land in the same receive queue. + * + * udp_flow_src_port() is using sk_txhash, inherited from + * skb_set_hash_from_sk() call in __tcp_transmit_skb(). + * This field is subject to reshuffling, thanks to + * sk_rethink_txhash() calls in various TCP functions. + * + * Instead, use sk->sk_hash which is constant through + * the whole flow duration. + */ + if (likely(sk)) { + u32 hash = sk->sk_hash; + int min, max; + + /* These operations are cheap, no need to cache the result + * in another socket field. + */ + inet_get_local_port_range(net, &min, &max); + /* Since this is being sent on the wire obfuscate hash a bit + * to minimize possibility that any useful information to an + * attacker is leaked. Only upper 16 bits are relevant in the + * computation for 16 bit port value because we use a + * reciprocal divide. + */ + hash ^= hash << 16; + uh->source = htons((((u64)hash * (max - min)) >> 32) + min); + } else { + uh->source = udp_flow_src_port(net, skb, 0, 0, false); + } uh->check = 0; uh->len = htons(udp_len); diff --git a/net/qrtr/af_qrtr.c b/net/qrtr/af_qrtr.c index dab839f61ee93b..26609feff4f800 100644 --- a/net/qrtr/af_qrtr.c +++ b/net/qrtr/af_qrtr.c @@ -118,7 +118,7 @@ static DEFINE_XARRAY_ALLOC(qrtr_ports); * @ep: endpoint * @ref: reference count for node * @nid: node id - * @qrtr_tx_flow: tree of qrtr_tx_flow, keyed by node << 32 | port + * @qrtr_tx_flow: xarray of qrtr_tx_flow, keyed by node << 32 | port * @qrtr_tx_lock: lock for qrtr_tx_flow inserts * @rx_queue: receive queue * @item: list item for broadcast list @@ -129,7 +129,7 @@ struct qrtr_node { struct kref ref; unsigned int nid; - struct radix_tree_root qrtr_tx_flow; + struct xarray qrtr_tx_flow; struct mutex qrtr_tx_lock; /* for qrtr_tx_flow */ struct sk_buff_head rx_queue; @@ -172,6 +172,7 @@ static void __qrtr_node_release(struct kref *kref) struct qrtr_tx_flow *flow; unsigned long flags; void __rcu **slot; + unsigned long index; spin_lock_irqsave(&qrtr_nodes_lock, flags); /* If the node is a bridge for other nodes, there are possibly @@ -189,11 +190,9 @@ static void __qrtr_node_release(struct kref *kref) skb_queue_purge(&node->rx_queue); /* Free tx flow counters */ - radix_tree_for_each_slot(slot, &node->qrtr_tx_flow, &iter, 0) { - flow = *slot; - radix_tree_iter_delete(&node->qrtr_tx_flow, &iter, slot); + xa_for_each(&node->qrtr_tx_flow, index, flow) kfree(flow); - } + xa_destroy(&node->qrtr_tx_flow); kfree(node); } @@ -228,9 +227,7 @@ static void qrtr_tx_resume(struct qrtr_node *node, struct sk_buff *skb) key = remote_node << 32 | remote_port; - rcu_read_lock(); - flow = radix_tree_lookup(&node->qrtr_tx_flow, key); - rcu_read_unlock(); + flow = xa_load(&node->qrtr_tx_flow, key); if (flow) { spin_lock(&flow->resume_tx.lock); flow->pending = 0; @@ -269,12 +266,13 @@ static int qrtr_tx_wait(struct qrtr_node *node, int dest_node, int dest_port, return 0; mutex_lock(&node->qrtr_tx_lock); - flow = radix_tree_lookup(&node->qrtr_tx_flow, key); + flow = xa_load(&node->qrtr_tx_flow, key); if (!flow) { flow = kzalloc(sizeof(*flow), GFP_KERNEL); if (flow) { init_waitqueue_head(&flow->resume_tx); - if (radix_tree_insert(&node->qrtr_tx_flow, key, flow)) { + if (xa_err(xa_store(&node->qrtr_tx_flow, key, flow, + GFP_KERNEL))) { kfree(flow); flow = NULL; } @@ -326,9 +324,7 @@ static void qrtr_tx_flow_failed(struct qrtr_node *node, int dest_node, unsigned long key = (u64)dest_node << 32 | dest_port; struct qrtr_tx_flow *flow; - rcu_read_lock(); - flow = radix_tree_lookup(&node->qrtr_tx_flow, key); - rcu_read_unlock(); + flow = xa_load(&node->qrtr_tx_flow, key); if (flow) { spin_lock_irq(&flow->resume_tx.lock); flow->tx_failed = 1; @@ -599,7 +595,7 @@ int qrtr_endpoint_register(struct qrtr_endpoint *ep, unsigned int nid) node->nid = QRTR_EP_NID_AUTO; node->ep = ep; - INIT_RADIX_TREE(&node->qrtr_tx_flow, GFP_KERNEL); + xa_init(&node->qrtr_tx_flow); mutex_init(&node->qrtr_tx_lock); qrtr_node_assign(node, nid); @@ -627,6 +623,7 @@ void qrtr_endpoint_unregister(struct qrtr_endpoint *ep) struct qrtr_tx_flow *flow; struct sk_buff *skb; unsigned long flags; + unsigned long index; void __rcu **slot; mutex_lock(&node->ep_lock); @@ -649,10 +646,8 @@ void qrtr_endpoint_unregister(struct qrtr_endpoint *ep) /* Wake up any transmitters waiting for resume-tx from the node */ mutex_lock(&node->qrtr_tx_lock); - radix_tree_for_each_slot(slot, &node->qrtr_tx_flow, &iter, 0) { - flow = *slot; + xa_for_each(&node->qrtr_tx_flow, index, flow) wake_up_interruptible_all(&flow->resume_tx); - } mutex_unlock(&node->qrtr_tx_lock); qrtr_node_release(node); diff --git a/net/qrtr/mhi.c b/net/qrtr/mhi.c index 69f53625a049de..80e341d2f8a452 100644 --- a/net/qrtr/mhi.c +++ b/net/qrtr/mhi.c @@ -24,13 +24,25 @@ static void qcom_mhi_qrtr_dl_callback(struct mhi_device *mhi_dev, struct qrtr_mhi_dev *qdev = dev_get_drvdata(&mhi_dev->dev); int rc; - if (!qdev || mhi_res->transaction_status) + if (!qdev || (mhi_res->transaction_status && mhi_res->transaction_status != -ENOTCONN)) return; + /* Channel got reset. So just free the buffer */ + if (mhi_res->transaction_status == -ENOTCONN) { + devm_kfree(&mhi_dev->dev, mhi_res->buf_addr); + return; + } + rc = qrtr_endpoint_post(&qdev->ep, mhi_res->buf_addr, mhi_res->bytes_xferd); if (rc == -EINVAL) dev_err(qdev->dev, "invalid ipcrouter packet\n"); + + /* Done with the buffer, now recycle it for future use */ + rc = mhi_queue_buf(mhi_dev, DMA_FROM_DEVICE, mhi_res->buf_addr, + mhi_dev->mhi_cntrl->buffer_len, MHI_EOT); + if (rc) + dev_err(&mhi_dev->dev, "Failed to recycle the buffer: %d\n", rc); } /* From QRTR to MHI */ @@ -72,6 +84,29 @@ static int qcom_mhi_qrtr_send(struct qrtr_endpoint *ep, struct sk_buff *skb) return rc; } +static int qcom_mhi_qrtr_queue_dl_buffers(struct mhi_device *mhi_dev) +{ + u32 free_desc; + void *buf; + int ret; + + free_desc = mhi_get_free_desc_count(mhi_dev, DMA_FROM_DEVICE); + while (free_desc--) { + buf = devm_kmalloc(&mhi_dev->dev, mhi_dev->mhi_cntrl->buffer_len, GFP_KERNEL); + if (!buf) + return -ENOMEM; + + ret = mhi_queue_buf(mhi_dev, DMA_FROM_DEVICE, buf, mhi_dev->mhi_cntrl->buffer_len, + MHI_EOT); + if (ret) { + dev_err(&mhi_dev->dev, "Failed to queue buffer: %d\n", ret); + return ret; + } + } + + return 0; +} + static int qcom_mhi_qrtr_probe(struct mhi_device *mhi_dev, const struct mhi_device_id *id) { @@ -87,20 +122,30 @@ static int qcom_mhi_qrtr_probe(struct mhi_device *mhi_dev, qdev->ep.xmit = qcom_mhi_qrtr_send; dev_set_drvdata(&mhi_dev->dev, qdev); - rc = qrtr_endpoint_register(&qdev->ep, QRTR_EP_NID_AUTO); - if (rc) - return rc; /* start channels */ - rc = mhi_prepare_for_transfer_autoqueue(mhi_dev); - if (rc) { - qrtr_endpoint_unregister(&qdev->ep); + rc = mhi_prepare_for_transfer(mhi_dev); + if (rc) return rc; - } + + rc = qrtr_endpoint_register(&qdev->ep, QRTR_EP_NID_AUTO); + if (rc) + goto err_unprepare; + + rc = qcom_mhi_qrtr_queue_dl_buffers(mhi_dev); + if (rc) + goto err_unregister; dev_dbg(qdev->dev, "Qualcomm MHI QRTR driver probed\n"); return 0; + +err_unregister: + qrtr_endpoint_unregister(&qdev->ep); +err_unprepare: + mhi_unprepare_from_transfer(mhi_dev); + + return rc; } static void qcom_mhi_qrtr_remove(struct mhi_device *mhi_dev) @@ -151,11 +196,13 @@ static int __maybe_unused qcom_mhi_qrtr_pm_resume_early(struct device *dev) if (state == MHI_STATE_M3) return 0; - rc = mhi_prepare_for_transfer_autoqueue(mhi_dev); - if (rc) + rc = mhi_prepare_for_transfer(mhi_dev); + if (rc) { dev_err(dev, "failed to prepare for autoqueue transfer %d\n", rc); + return rc; + } - return rc; + return qcom_mhi_qrtr_queue_dl_buffers(mhi_dev); } static const struct dev_pm_ops qcom_mhi_qrtr_pm_ops = { diff --git a/net/rds/connection.c b/net/rds/connection.c index 68bc88cce84ec0..dbfea6fa112602 100644 --- a/net/rds/connection.c +++ b/net/rds/connection.c @@ -382,6 +382,8 @@ void rds_conn_shutdown(struct rds_conn_path *cp) if (!rds_conn_path_transition(cp, RDS_CONN_UP, RDS_CONN_DISCONNECTING) && !rds_conn_path_transition(cp, RDS_CONN_ERROR, + RDS_CONN_DISCONNECTING) && + !rds_conn_path_transition(cp, RDS_CONN_RESETTING, RDS_CONN_DISCONNECTING)) { rds_conn_path_error(cp, "shutdown called in state %d\n", @@ -427,6 +429,8 @@ void rds_conn_shutdown(struct rds_conn_path *cp) * to the conn hash, so we never trigger a reconnect on this * conn - the reconnect is always triggered by the active peer. */ cancel_delayed_work_sync(&cp->cp_conn_w); + + clear_bit(RDS_RECONNECT_PENDING, &cp->cp_flags); rcu_read_lock(); if (!hlist_unhashed(&conn->c_hash_node)) { rcu_read_unlock(); diff --git a/net/rds/ib_rdma.c b/net/rds/ib_rdma.c index 6585164c705953..dd08ccc4246da1 100644 --- a/net/rds/ib_rdma.c +++ b/net/rds/ib_rdma.c @@ -604,8 +604,13 @@ void *rds_ib_get_mr(struct scatterlist *sg, unsigned long nents, return ibmr; } - if (conn) + if (conn) { ic = conn->c_transport_data; + if (!ic || !ic->i_cm_id || !ic->i_cm_id->qp) { + ret = -ENODEV; + goto out; + } + } if (!rds_ibdev->mr_8k_pool || !rds_ibdev->mr_1m_pool) { ret = -ENODEV; diff --git a/net/rds/send.c b/net/rds/send.c index 0b3d0ef2f008b6..071c5dca969a23 100644 --- a/net/rds/send.c +++ b/net/rds/send.c @@ -1382,9 +1382,11 @@ int rds_sendmsg(struct socket *sock, struct msghdr *msg, size_t payload_len) else queue_delayed_work(rds_wq, &cpath->cp_send_w, 1); rcu_read_unlock(); + + if (ret) + goto out; } - if (ret) - goto out; + rds_message_put(rm); for (ind = 0; ind < vct.indx; ind++) diff --git a/net/rds/tcp.c b/net/rds/tcp.c index 3cc2f303bf7865..b66dfcc3efaa0f 100644 --- a/net/rds/tcp.c +++ b/net/rds/tcp.c @@ -495,18 +495,24 @@ bool rds_tcp_tune(struct socket *sock) struct rds_tcp_net *rtn; tcp_sock_set_nodelay(sock->sk); - lock_sock(sk); /* TCP timer functions might access net namespace even after * a process which created this net namespace terminated. */ if (!sk->sk_net_refcnt) { - if (!maybe_get_net(net)) { - release_sock(sk); + if (!maybe_get_net(net)) return false; - } + /* + * sk_net_refcnt_upgrade() must be called before lock_sock() + * because it does a GFP_KERNEL allocation, which can trigger + * fs_reclaim and create a circular lock dependency with the + * socket lock. The fields it modifies (sk_net_refcnt, + * ns_tracker) are not accessed by any concurrent code path + * at this point. + */ sk_net_refcnt_upgrade(sk); put_net(net); } + lock_sock(sk); rtn = net_generic(net, rds_tcp_netid); if (rtn->sndbuf_size > 0) { sk->sk_sndbuf = rtn->sndbuf_size; diff --git a/net/rds/tcp_listen.c b/net/rds/tcp_listen.c index 820d3e20de1952..27b6107ddc28d2 100644 --- a/net/rds/tcp_listen.c +++ b/net/rds/tcp_listen.c @@ -59,9 +59,6 @@ void rds_tcp_keepalive(struct socket *sock) * socket and force a reconneect from smaller -> larger ip addr. The reason * we special case cp_index 0 is to allow the rds probe ping itself to itself * get through efficiently. - * Since reconnects are only initiated from the node with the numerically - * smaller ip address, we recycle conns in RDS_CONN_ERROR on the passive side - * by moving them to CONNECTING in this function. */ static struct rds_tcp_connection *rds_tcp_accept_one_path(struct rds_connection *conn) @@ -86,8 +83,6 @@ struct rds_tcp_connection *rds_tcp_accept_one_path(struct rds_connection *conn) struct rds_conn_path *cp = &conn->c_path[i]; if (rds_conn_path_transition(cp, RDS_CONN_DOWN, - RDS_CONN_CONNECTING) || - rds_conn_path_transition(cp, RDS_CONN_ERROR, RDS_CONN_CONNECTING)) { return cp->cp_transport_data; } diff --git a/net/rfkill/core.c b/net/rfkill/core.c index 7d3e82e4c2fce0..868a8586dc1705 100644 --- a/net/rfkill/core.c +++ b/net/rfkill/core.c @@ -73,11 +73,14 @@ struct rfkill_int_event { struct rfkill_event_ext ev; }; +/* Max rfkill events that can be "in-flight" for one data source */ +#define MAX_RFKILL_EVENT 1000 struct rfkill_data { struct list_head list; struct list_head events; struct mutex mtx; wait_queue_head_t read_wait; + u32 event_count; bool input_handler; u8 max_size; }; @@ -255,10 +258,12 @@ static void rfkill_global_led_trigger_unregister(void) } #endif /* CONFIG_RFKILL_LEDS */ -static void rfkill_fill_event(struct rfkill_event_ext *ev, - struct rfkill *rfkill, - enum rfkill_operation op) +static int rfkill_fill_event(struct rfkill_int_event *int_ev, + struct rfkill *rfkill, + struct rfkill_data *data, + enum rfkill_operation op) { + struct rfkill_event_ext *ev = &int_ev->ev; unsigned long flags; ev->idx = rfkill->idx; @@ -271,6 +276,15 @@ static void rfkill_fill_event(struct rfkill_event_ext *ev, RFKILL_BLOCK_SW_PREV)); ev->hard_block_reasons = rfkill->hard_block_reasons; spin_unlock_irqrestore(&rfkill->lock, flags); + + scoped_guard(mutex, &data->mtx) { + if (data->event_count++ > MAX_RFKILL_EVENT) { + data->event_count--; + return -ENOSPC; + } + list_add_tail(&int_ev->list, &data->events); + } + return 0; } static void rfkill_send_events(struct rfkill *rfkill, enum rfkill_operation op) @@ -282,10 +296,10 @@ static void rfkill_send_events(struct rfkill *rfkill, enum rfkill_operation op) ev = kzalloc(sizeof(*ev), GFP_KERNEL); if (!ev) continue; - rfkill_fill_event(&ev->ev, rfkill, op); - mutex_lock(&data->mtx); - list_add_tail(&ev->list, &data->events); - mutex_unlock(&data->mtx); + if (rfkill_fill_event(ev, rfkill, data, op)) { + kfree(ev); + continue; + } wake_up_interruptible(&data->read_wait); } } @@ -1186,10 +1200,8 @@ static int rfkill_fop_open(struct inode *inode, struct file *file) if (!ev) goto free; rfkill_sync(rfkill); - rfkill_fill_event(&ev->ev, rfkill, RFKILL_OP_ADD); - mutex_lock(&data->mtx); - list_add_tail(&ev->list, &data->events); - mutex_unlock(&data->mtx); + if (rfkill_fill_event(ev, rfkill, data, RFKILL_OP_ADD)) + kfree(ev); } list_add(&data->list, &rfkill_fds); mutex_unlock(&rfkill_global_mutex); @@ -1259,6 +1271,7 @@ static ssize_t rfkill_fop_read(struct file *file, char __user *buf, ret = -EFAULT; list_del(&ev->list); + data->event_count--; kfree(ev); out: mutex_unlock(&data->mtx); diff --git a/net/rose/af_rose.c b/net/rose/af_rose.c index c0f5a515a8ce57..de18af4e406607 100644 --- a/net/rose/af_rose.c +++ b/net/rose/af_rose.c @@ -811,6 +811,11 @@ static int rose_connect(struct socket *sock, struct sockaddr_unsized *uaddr, int goto out_release; } + if (sk->sk_state == TCP_SYN_SENT) { + err = -EALREADY; + goto out_release; + } + sk->sk_state = TCP_CLOSE; sock->state = SS_UNCONNECTED; diff --git a/net/rxrpc/af_rxrpc.c b/net/rxrpc/af_rxrpc.c index 0c2c68c4b07e48..32ec91fa938fbd 100644 --- a/net/rxrpc/af_rxrpc.c +++ b/net/rxrpc/af_rxrpc.c @@ -267,12 +267,13 @@ static int rxrpc_listen(struct socket *sock, int backlog) * Lookup or create a remote transport endpoint record for the specified * address. * - * Return: The peer record found with a reference, %NULL if no record is found - * or a negative error code if the address is invalid or unsupported. + * Return: The peer record found with a reference or a negative error code if + * the address is invalid or unsupported. */ struct rxrpc_peer *rxrpc_kernel_lookup_peer(struct socket *sock, struct sockaddr_rxrpc *srx, gfp_t gfp) { + struct rxrpc_peer *peer; struct rxrpc_sock *rx = rxrpc_sk(sock->sk); int ret; @@ -280,7 +281,8 @@ struct rxrpc_peer *rxrpc_kernel_lookup_peer(struct socket *sock, if (ret < 0) return ERR_PTR(ret); - return rxrpc_lookup_peer(rx->local, srx, gfp); + peer = rxrpc_lookup_peer(rx->local, srx, gfp); + return peer ?: ERR_PTR(-ENOMEM); } EXPORT_SYMBOL(rxrpc_kernel_lookup_peer); @@ -652,9 +654,6 @@ static int rxrpc_setsockopt(struct socket *sock, int level, int optname, goto success; case RXRPC_SECURITY_KEY: - ret = -EINVAL; - if (rx->key) - goto error; ret = -EISCONN; if (rx->sk.sk_state != RXRPC_UNBOUND) goto error; @@ -662,9 +661,6 @@ static int rxrpc_setsockopt(struct socket *sock, int level, int optname, goto error; case RXRPC_SECURITY_KEYRING: - ret = -EINVAL; - if (rx->key) - goto error; ret = -EISCONN; if (rx->sk.sk_state != RXRPC_UNBOUND) goto error; diff --git a/net/rxrpc/ar-internal.h b/net/rxrpc/ar-internal.h index 36d6ca0d1089e1..96ecb83c907153 100644 --- a/net/rxrpc/ar-internal.h +++ b/net/rxrpc/ar-internal.h @@ -117,7 +117,7 @@ struct rxrpc_net { atomic_t stat_tx_jumbo[10]; atomic_t stat_rx_jumbo[10]; - atomic_t stat_why_req_ack[8]; + atomic_t stat_why_req_ack[9]; atomic_t stat_io_loop; }; diff --git a/net/rxrpc/call_object.c b/net/rxrpc/call_object.c index 918f41d97a2f93..f035f486c13973 100644 --- a/net/rxrpc/call_object.c +++ b/net/rxrpc/call_object.c @@ -654,11 +654,9 @@ void rxrpc_put_call(struct rxrpc_call *call, enum rxrpc_call_trace why) if (dead) { ASSERTCMP(__rxrpc_call_state(call), ==, RXRPC_CALL_COMPLETE); - if (!list_empty(&call->link)) { - spin_lock(&rxnet->call_lock); - list_del_init(&call->link); - spin_unlock(&rxnet->call_lock); - } + spin_lock(&rxnet->call_lock); + list_del_rcu(&call->link); + spin_unlock(&rxnet->call_lock); rxrpc_cleanup_call(call); } @@ -694,6 +692,7 @@ static void rxrpc_destroy_call(struct work_struct *work) rxrpc_put_bundle(call->bundle, rxrpc_bundle_put_call); rxrpc_put_peer(call->peer, rxrpc_peer_put_call); rxrpc_put_local(call->local, rxrpc_local_put_call); + key_put(call->key); call_rcu(&call->rcu, rxrpc_rcu_free_call); } @@ -730,24 +729,20 @@ void rxrpc_destroy_all_calls(struct rxrpc_net *rxnet) _enter(""); if (!list_empty(&rxnet->calls)) { - spin_lock(&rxnet->call_lock); + int shown = 0; - while (!list_empty(&rxnet->calls)) { - call = list_entry(rxnet->calls.next, - struct rxrpc_call, link); - _debug("Zapping call %p", call); + spin_lock(&rxnet->call_lock); - rxrpc_see_call(call, rxrpc_call_see_zap); - list_del_init(&call->link); + list_for_each_entry(call, &rxnet->calls, link) { + rxrpc_see_call(call, rxrpc_call_see_still_live); pr_err("Call %p still in use (%d,%s,%lx,%lx)!\n", call, refcount_read(&call->ref), rxrpc_call_states[__rxrpc_call_state(call)], call->flags, call->events); - spin_unlock(&rxnet->call_lock); - cond_resched(); - spin_lock(&rxnet->call_lock); + if (++shown >= 10) + break; } spin_unlock(&rxnet->call_lock); diff --git a/net/rxrpc/conn_event.c b/net/rxrpc/conn_event.c index 98ad9b51ca2cdc..9a41ec708aeb9f 100644 --- a/net/rxrpc/conn_event.c +++ b/net/rxrpc/conn_event.c @@ -247,6 +247,7 @@ static int rxrpc_process_event(struct rxrpc_connection *conn, struct sk_buff *skb) { struct rxrpc_skb_priv *sp = rxrpc_skb(skb); + bool secured = false; int ret; if (conn->state == RXRPC_CONN_ABORTED) @@ -262,6 +263,13 @@ static int rxrpc_process_event(struct rxrpc_connection *conn, return ret; case RXRPC_PACKET_TYPE_RESPONSE: + spin_lock_irq(&conn->state_lock); + if (conn->state != RXRPC_CONN_SERVICE_CHALLENGING) { + spin_unlock_irq(&conn->state_lock); + return 0; + } + spin_unlock_irq(&conn->state_lock); + ret = conn->security->verify_response(conn, skb); if (ret < 0) return ret; @@ -272,11 +280,13 @@ static int rxrpc_process_event(struct rxrpc_connection *conn, return ret; spin_lock_irq(&conn->state_lock); - if (conn->state == RXRPC_CONN_SERVICE_CHALLENGING) + if (conn->state == RXRPC_CONN_SERVICE_CHALLENGING) { conn->state = RXRPC_CONN_SERVICE; + secured = true; + } spin_unlock_irq(&conn->state_lock); - if (conn->state == RXRPC_CONN_SERVICE) { + if (secured) { /* Offload call state flipping to the I/O thread. As * we've already received the packet, put it on the * front of the queue. @@ -557,11 +567,11 @@ void rxrpc_post_response(struct rxrpc_connection *conn, struct sk_buff *skb) spin_lock_irq(&local->lock); old = conn->tx_response; if (old) { - struct rxrpc_skb_priv *osp = rxrpc_skb(skb); + struct rxrpc_skb_priv *osp = rxrpc_skb(old); /* Always go with the response to the most recent challenge. */ if (after(sp->resp.challenge_serial, osp->resp.challenge_serial)) - conn->tx_response = old; + conn->tx_response = skb; else old = skb; } else { @@ -569,4 +579,5 @@ void rxrpc_post_response(struct rxrpc_connection *conn, struct sk_buff *skb) } spin_unlock_irq(&local->lock); rxrpc_poke_conn(conn, rxrpc_conn_get_poke_response); + rxrpc_free_skb(old, rxrpc_skb_put_old_response); } diff --git a/net/rxrpc/input_rack.c b/net/rxrpc/input_rack.c index 13c371261e0a58..9eb109ffba56e8 100644 --- a/net/rxrpc/input_rack.c +++ b/net/rxrpc/input_rack.c @@ -413,6 +413,6 @@ void rxrpc_rack_timer_expired(struct rxrpc_call *call, ktime_t overran_by) break; //case RXRPC_CALL_RACKTIMER_ZEROWIN: default: - pr_warn("Unexpected rack timer %u", call->rack_timer_mode); + pr_warn("Unexpected rack timer %u", mode); } } diff --git a/net/rxrpc/io_thread.c b/net/rxrpc/io_thread.c index e939ecf417c4b5..69795693192521 100644 --- a/net/rxrpc/io_thread.c +++ b/net/rxrpc/io_thread.c @@ -419,7 +419,8 @@ static int rxrpc_input_packet_on_conn(struct rxrpc_connection *conn, if (sp->hdr.callNumber > chan->call_id) { if (rxrpc_to_client(sp)) { - rxrpc_put_call(call, rxrpc_call_put_input); + if (call) + rxrpc_put_call(call, rxrpc_call_put_input); return rxrpc_protocol_error(skb, rxrpc_eproto_unexpected_implicit_end); } diff --git a/net/rxrpc/key.c b/net/rxrpc/key.c index 9fdc1f031c9dae..e0c29ebe6b6d3b 100644 --- a/net/rxrpc/key.c +++ b/net/rxrpc/key.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -72,7 +73,7 @@ static int rxrpc_preparse_xdr_rxkad(struct key_preparsed_payload *prep, return -EKEYREJECTED; plen = sizeof(*token) + sizeof(*token->kad) + tktlen; - prep->quotalen = datalen + plen; + prep->quotalen += datalen + plen; plen -= sizeof(*token); token = kzalloc(sizeof(*token), GFP_KERNEL); @@ -171,7 +172,7 @@ static int rxrpc_preparse_xdr_yfs_rxgk(struct key_preparsed_payload *prep, size_t plen; const __be32 *ticket, *key; s64 tmp; - u32 tktlen, keylen; + size_t raw_keylen, raw_tktlen, keylen, tktlen; _enter(",{%x,%x,%x,%x},%x", ntohl(xdr[0]), ntohl(xdr[1]), ntohl(xdr[2]), ntohl(xdr[3]), @@ -181,32 +182,36 @@ static int rxrpc_preparse_xdr_yfs_rxgk(struct key_preparsed_payload *prep, goto reject; key = xdr + (6 * 2 + 1); - keylen = ntohl(key[-1]); - _debug("keylen: %x", keylen); - keylen = round_up(keylen, 4); + raw_keylen = ntohl(key[-1]); + _debug("keylen: %zx", raw_keylen); + if (raw_keylen > AFSTOKEN_GK_KEY_MAX) + goto reject; + keylen = round_up(raw_keylen, 4); if ((6 * 2 + 2) * 4 + keylen > toklen) goto reject; ticket = xdr + (6 * 2 + 1 + (keylen / 4) + 1); - tktlen = ntohl(ticket[-1]); - _debug("tktlen: %x", tktlen); - tktlen = round_up(tktlen, 4); + raw_tktlen = ntohl(ticket[-1]); + _debug("tktlen: %zx", raw_tktlen); + if (raw_tktlen > AFSTOKEN_GK_TOKEN_MAX) + goto reject; + tktlen = round_up(raw_tktlen, 4); if ((6 * 2 + 2) * 4 + keylen + tktlen != toklen) { - kleave(" = -EKEYREJECTED [%x!=%x, %x,%x]", + kleave(" = -EKEYREJECTED [%zx!=%x, %zx,%zx]", (6 * 2 + 2) * 4 + keylen + tktlen, toklen, keylen, tktlen); goto reject; } plen = sizeof(*token) + sizeof(*token->rxgk) + tktlen + keylen; - prep->quotalen = datalen + plen; + prep->quotalen += datalen + plen; plen -= sizeof(*token); token = kzalloc(sizeof(*token), GFP_KERNEL); if (!token) goto nomem; - token->rxgk = kzalloc(sizeof(*token->rxgk) + keylen, GFP_KERNEL); + token->rxgk = kzalloc(struct_size_t(struct rxgk_key, _key, raw_keylen), GFP_KERNEL); if (!token->rxgk) goto nomem_token; @@ -221,9 +226,9 @@ static int rxrpc_preparse_xdr_yfs_rxgk(struct key_preparsed_payload *prep, token->rxgk->enctype = tmp = xdr_dec64(xdr + 5 * 2); if (tmp < 0 || tmp > UINT_MAX) goto reject_token; - token->rxgk->key.len = ntohl(key[-1]); + token->rxgk->key.len = raw_keylen; token->rxgk->key.data = token->rxgk->_key; - token->rxgk->ticket.len = ntohl(ticket[-1]); + token->rxgk->ticket.len = raw_tktlen; if (token->rxgk->endtime != 0) { expiry = rxrpc_s64_to_time64(token->rxgk->endtime); @@ -236,8 +241,7 @@ static int rxrpc_preparse_xdr_yfs_rxgk(struct key_preparsed_payload *prep, memcpy(token->rxgk->key.data, key, token->rxgk->key.len); /* Pad the ticket so that we can use it directly in XDR */ - token->rxgk->ticket.data = kzalloc(round_up(token->rxgk->ticket.len, 4), - GFP_KERNEL); + token->rxgk->ticket.data = kzalloc(tktlen, GFP_KERNEL); if (!token->rxgk->ticket.data) goto nomem_yrxgk; memcpy(token->rxgk->ticket.data, ticket, token->rxgk->ticket.len); @@ -274,6 +278,7 @@ static int rxrpc_preparse_xdr_yfs_rxgk(struct key_preparsed_payload *prep, nomem: return -ENOMEM; reject_token: + kfree(token->rxgk); kfree(token); reject: return -EKEYREJECTED; @@ -460,6 +465,7 @@ static int rxrpc_preparse(struct key_preparsed_payload *prep) memcpy(&kver, prep->data, sizeof(kver)); prep->data += sizeof(kver); prep->datalen -= sizeof(kver); + prep->quotalen = 0; _debug("KEY I/F VERSION: %u", kver); @@ -497,7 +503,7 @@ static int rxrpc_preparse(struct key_preparsed_payload *prep) goto error; plen = sizeof(*token->kad) + v1->ticket_length; - prep->quotalen = plen + sizeof(*token); + prep->quotalen += plen + sizeof(*token); ret = -ENOMEM; token = kzalloc(sizeof(*token), GFP_KERNEL); @@ -616,7 +622,7 @@ int rxrpc_request_key(struct rxrpc_sock *rx, sockptr_t optval, int optlen) _enter(""); - if (optlen <= 0 || optlen > PAGE_SIZE - 1 || rx->securities) + if (optlen <= 0 || optlen > PAGE_SIZE - 1 || rx->key) return -EINVAL; description = memdup_sockptr_nul(optval, optlen); diff --git a/net/rxrpc/output.c b/net/rxrpc/output.c index d70db367e358db..870e59bf06af2b 100644 --- a/net/rxrpc/output.c +++ b/net/rxrpc/output.c @@ -479,6 +479,8 @@ static size_t rxrpc_prepare_data_subpacket(struct rxrpc_call *call, why = rxrpc_reqack_old_rtt; else if (!last && !after(READ_ONCE(call->send_top), txb->seq)) why = rxrpc_reqack_app_stall; + else if (call->tx_winsize <= (2 * req->n) || call->cong_cwnd <= (2 * req->n)) + why = rxrpc_reqack_jumbo_win; else goto dont_set_request_ack; diff --git a/net/rxrpc/proc.c b/net/rxrpc/proc.c index 59292f7f9205e7..e9a27fa7b25d8b 100644 --- a/net/rxrpc/proc.c +++ b/net/rxrpc/proc.c @@ -10,6 +10,10 @@ #include #include "ar-internal.h" +#define RXRPC_PROC_ADDRBUF_SIZE \ + (sizeof("[xxxx:xxxx:xxxx:xxxx:xxxx:xxxx:255.255.255.255]") + \ + sizeof(":12345")) + static const char *const rxrpc_conn_states[RXRPC_CONN__NR_STATES] = { [RXRPC_CONN_UNUSED] = "Unused ", [RXRPC_CONN_CLIENT_UNSECURED] = "ClUnsec ", @@ -53,7 +57,7 @@ static int rxrpc_call_seq_show(struct seq_file *seq, void *v) struct rxrpc_net *rxnet = rxrpc_net(seq_file_net(seq)); enum rxrpc_call_state state; rxrpc_seq_t tx_bottom; - char lbuff[50], rbuff[50]; + char lbuff[RXRPC_PROC_ADDRBUF_SIZE], rbuff[RXRPC_PROC_ADDRBUF_SIZE]; long timeout = 0; if (v == &rxnet->calls) { @@ -69,11 +73,11 @@ static int rxrpc_call_seq_show(struct seq_file *seq, void *v) local = call->local; if (local) - sprintf(lbuff, "%pISpc", &local->srx.transport); + scnprintf(lbuff, sizeof(lbuff), "%pISpc", &local->srx.transport); else strcpy(lbuff, "no_local"); - sprintf(rbuff, "%pISpc", &call->dest_srx.transport); + scnprintf(rbuff, sizeof(rbuff), "%pISpc", &call->dest_srx.transport); state = rxrpc_call_state(call); if (state != RXRPC_CALL_SERVER_PREALLOC) @@ -142,7 +146,7 @@ static int rxrpc_connection_seq_show(struct seq_file *seq, void *v) struct rxrpc_connection *conn; struct rxrpc_net *rxnet = rxrpc_net(seq_file_net(seq)); const char *state; - char lbuff[50], rbuff[50]; + char lbuff[RXRPC_PROC_ADDRBUF_SIZE], rbuff[RXRPC_PROC_ADDRBUF_SIZE]; if (v == &rxnet->conn_proc_list) { seq_puts(seq, @@ -161,8 +165,8 @@ static int rxrpc_connection_seq_show(struct seq_file *seq, void *v) goto print; } - sprintf(lbuff, "%pISpc", &conn->local->srx.transport); - sprintf(rbuff, "%pISpc", &conn->peer->srx.transport); + scnprintf(lbuff, sizeof(lbuff), "%pISpc", &conn->local->srx.transport); + scnprintf(rbuff, sizeof(rbuff), "%pISpc", &conn->peer->srx.transport); print: state = rxrpc_is_conn_aborted(conn) ? rxrpc_call_completions[conn->completion] : @@ -228,7 +232,7 @@ static int rxrpc_bundle_seq_show(struct seq_file *seq, void *v) { struct rxrpc_bundle *bundle; struct rxrpc_net *rxnet = rxrpc_net(seq_file_net(seq)); - char lbuff[50], rbuff[50]; + char lbuff[RXRPC_PROC_ADDRBUF_SIZE], rbuff[RXRPC_PROC_ADDRBUF_SIZE]; if (v == &rxnet->bundle_proc_list) { seq_puts(seq, @@ -242,8 +246,8 @@ static int rxrpc_bundle_seq_show(struct seq_file *seq, void *v) bundle = list_entry(v, struct rxrpc_bundle, proc_link); - sprintf(lbuff, "%pISpc", &bundle->local->srx.transport); - sprintf(rbuff, "%pISpc", &bundle->peer->srx.transport); + scnprintf(lbuff, sizeof(lbuff), "%pISpc", &bundle->local->srx.transport); + scnprintf(rbuff, sizeof(rbuff), "%pISpc", &bundle->peer->srx.transport); seq_printf(seq, "UDP %-47.47s %-47.47s %4x %3u %3d" " %c%c%c %08x | %08x %08x %08x %08x %08x\n", @@ -279,7 +283,7 @@ static int rxrpc_peer_seq_show(struct seq_file *seq, void *v) { struct rxrpc_peer *peer; time64_t now; - char lbuff[50], rbuff[50]; + char lbuff[RXRPC_PROC_ADDRBUF_SIZE], rbuff[RXRPC_PROC_ADDRBUF_SIZE]; if (v == SEQ_START_TOKEN) { seq_puts(seq, @@ -290,9 +294,9 @@ static int rxrpc_peer_seq_show(struct seq_file *seq, void *v) peer = list_entry(v, struct rxrpc_peer, hash_link); - sprintf(lbuff, "%pISpc", &peer->local->srx.transport); + scnprintf(lbuff, sizeof(lbuff), "%pISpc", &peer->local->srx.transport); - sprintf(rbuff, "%pISpc", &peer->srx.transport); + scnprintf(rbuff, sizeof(rbuff), "%pISpc", &peer->srx.transport); now = ktime_get_seconds(); seq_printf(seq, @@ -401,7 +405,7 @@ const struct seq_operations rxrpc_peer_seq_ops = { static int rxrpc_local_seq_show(struct seq_file *seq, void *v) { struct rxrpc_local *local; - char lbuff[50]; + char lbuff[RXRPC_PROC_ADDRBUF_SIZE]; if (v == SEQ_START_TOKEN) { seq_puts(seq, @@ -412,7 +416,7 @@ static int rxrpc_local_seq_show(struct seq_file *seq, void *v) local = hlist_entry(v, struct rxrpc_local, link); - sprintf(lbuff, "%pISpc", &local->srx.transport); + scnprintf(lbuff, sizeof(lbuff), "%pISpc", &local->srx.transport); seq_printf(seq, "UDP %-47.47s %3u %3u %3u\n", @@ -518,11 +522,12 @@ int rxrpc_stats_show(struct seq_file *seq, void *v) atomic_read(&rxnet->stat_rx_acks[RXRPC_ACK_IDLE]), atomic_read(&rxnet->stat_rx_acks[0])); seq_printf(seq, - "Why-Req-A: acklost=%u mrtt=%u ortt=%u stall=%u\n", + "Why-Req-A: acklost=%u mrtt=%u ortt=%u stall=%u jwin=%u\n", atomic_read(&rxnet->stat_why_req_ack[rxrpc_reqack_ack_lost]), atomic_read(&rxnet->stat_why_req_ack[rxrpc_reqack_more_rtt]), atomic_read(&rxnet->stat_why_req_ack[rxrpc_reqack_old_rtt]), - atomic_read(&rxnet->stat_why_req_ack[rxrpc_reqack_app_stall])); + atomic_read(&rxnet->stat_why_req_ack[rxrpc_reqack_app_stall]), + atomic_read(&rxnet->stat_why_req_ack[rxrpc_reqack_jumbo_win])); seq_printf(seq, "Why-Req-A: nolast=%u retx=%u slows=%u smtxw=%u\n", atomic_read(&rxnet->stat_why_req_ack[rxrpc_reqack_no_srv_last]), diff --git a/net/rxrpc/rxgk.c b/net/rxrpc/rxgk.c index 43cbf9efd89f18..c39f5066d8e865 100644 --- a/net/rxrpc/rxgk.c +++ b/net/rxrpc/rxgk.c @@ -1085,6 +1085,9 @@ static int rxgk_do_verify_authenticator(struct rxrpc_connection *conn, _enter(""); + if ((end - p) * sizeof(__be32) < 24) + return rxrpc_abort_conn(conn, skb, RXGK_NOTAUTH, -EPROTO, + rxgk_abort_resp_short_auth); if (memcmp(p, conn->rxgk.nonce, 20) != 0) return rxrpc_abort_conn(conn, skb, RXGK_NOTAUTH, -EPROTO, rxgk_abort_resp_bad_nonce); @@ -1098,7 +1101,7 @@ static int rxgk_do_verify_authenticator(struct rxrpc_connection *conn, p += xdr_round_up(app_len) / sizeof(__be32); if (end - p < 4) return rxrpc_abort_conn(conn, skb, RXGK_NOTAUTH, -EPROTO, - rxgk_abort_resp_short_applen); + rxgk_abort_resp_short_auth); level = ntohl(*p++); epoch = ntohl(*p++); @@ -1164,7 +1167,8 @@ static int rxgk_verify_authenticator(struct rxrpc_connection *conn, } p = auth; - ret = rxgk_do_verify_authenticator(conn, krb5, skb, p, p + auth_len); + ret = rxgk_do_verify_authenticator(conn, krb5, skb, p, + p + auth_len / sizeof(*p)); error: kfree(auth); return ret; @@ -1208,7 +1212,8 @@ static int rxgk_verify_response(struct rxrpc_connection *conn, token_offset = offset; token_len = ntohl(rhdr.token_len); - if (xdr_round_up(token_len) + sizeof(__be32) > len) + if (token_len > len || + xdr_round_up(token_len) + sizeof(__be32) > len) goto short_packet; trace_rxrpc_rx_response(conn, sp->hdr.serial, 0, sp->hdr.cksum, token_len); @@ -1223,7 +1228,7 @@ static int rxgk_verify_response(struct rxrpc_connection *conn, auth_offset = offset; auth_len = ntohl(xauth_len); - if (auth_len < len) + if (auth_len > len) goto short_packet; if (auth_len & 3) goto inconsistent; @@ -1268,16 +1273,18 @@ static int rxgk_verify_response(struct rxrpc_connection *conn, if (ret < 0) { rxrpc_abort_conn(conn, skb, RXGK_SEALEDINCON, ret, rxgk_abort_resp_auth_dec); - goto out; + goto out_gk; } ret = rxgk_verify_authenticator(conn, krb5, skb, auth_offset, auth_len); if (ret < 0) - goto out; + goto out_gk; conn->key = key; key = NULL; ret = 0; +out_gk: + rxgk_put(gk); out: key_put(key); _leave(" = %d", ret); diff --git a/net/rxrpc/rxkad.c b/net/rxrpc/rxkad.c index a756855a0a62d7..4a3c630941499a 100644 --- a/net/rxrpc/rxkad.c +++ b/net/rxrpc/rxkad.c @@ -197,6 +197,7 @@ static int rxkad_prime_packet_security(struct rxrpc_connection *conn, struct rxrpc_crypt iv; __be32 *tmpbuf; size_t tmpsize = 4 * sizeof(__be32); + int ret; _enter(""); @@ -225,13 +226,13 @@ static int rxkad_prime_packet_security(struct rxrpc_connection *conn, skcipher_request_set_sync_tfm(req, ci); skcipher_request_set_callback(req, 0, NULL, NULL); skcipher_request_set_crypt(req, &sg, &sg, tmpsize, iv.x); - crypto_skcipher_encrypt(req); + ret = crypto_skcipher_encrypt(req); skcipher_request_free(req); memcpy(&conn->rxkad.csum_iv, tmpbuf + 2, sizeof(conn->rxkad.csum_iv)); kfree(tmpbuf); - _leave(" = 0"); - return 0; + _leave(" = %d", ret); + return ret; } /* @@ -264,6 +265,7 @@ static int rxkad_secure_packet_auth(const struct rxrpc_call *call, struct scatterlist sg; size_t pad; u16 check; + int ret; _enter(""); @@ -286,11 +288,11 @@ static int rxkad_secure_packet_auth(const struct rxrpc_call *call, skcipher_request_set_sync_tfm(req, call->conn->rxkad.cipher); skcipher_request_set_callback(req, 0, NULL, NULL); skcipher_request_set_crypt(req, &sg, &sg, 8, iv.x); - crypto_skcipher_encrypt(req); + ret = crypto_skcipher_encrypt(req); skcipher_request_zero(req); - _leave(" = 0"); - return 0; + _leave(" = %d", ret); + return ret; } /* @@ -345,7 +347,7 @@ static int rxkad_secure_packet(struct rxrpc_call *call, struct rxrpc_txbuf *txb) union { __be32 buf[2]; } crypto __aligned(8); - u32 x, y; + u32 x, y = 0; int ret; _enter("{%d{%x}},{#%u},%u,", @@ -376,8 +378,10 @@ static int rxkad_secure_packet(struct rxrpc_call *call, struct rxrpc_txbuf *txb) skcipher_request_set_sync_tfm(req, call->conn->rxkad.cipher); skcipher_request_set_callback(req, 0, NULL, NULL); skcipher_request_set_crypt(req, &sg, &sg, 8, iv.x); - crypto_skcipher_encrypt(req); + ret = crypto_skcipher_encrypt(req); skcipher_request_zero(req); + if (ret < 0) + goto out; y = ntohl(crypto.buf[1]); y = (y >> 16) & 0xffff; @@ -413,6 +417,7 @@ static int rxkad_secure_packet(struct rxrpc_call *call, struct rxrpc_txbuf *txb) memset(p + txb->pkt_len, 0, gap); } +out: skcipher_request_free(req); _leave(" = %d [set %x]", ret, y); return ret; @@ -453,8 +458,10 @@ static int rxkad_verify_packet_1(struct rxrpc_call *call, struct sk_buff *skb, skcipher_request_set_sync_tfm(req, call->conn->rxkad.cipher); skcipher_request_set_callback(req, 0, NULL, NULL); skcipher_request_set_crypt(req, sg, sg, 8, iv.x); - crypto_skcipher_decrypt(req); + ret = crypto_skcipher_decrypt(req); skcipher_request_zero(req); + if (ret < 0) + return ret; /* Extract the decrypted packet length */ if (skb_copy_bits(skb, sp->offset, &sechdr, sizeof(sechdr)) < 0) @@ -531,10 +538,14 @@ static int rxkad_verify_packet_2(struct rxrpc_call *call, struct sk_buff *skb, skcipher_request_set_sync_tfm(req, call->conn->rxkad.cipher); skcipher_request_set_callback(req, 0, NULL, NULL); skcipher_request_set_crypt(req, sg, sg, sp->len, iv.x); - crypto_skcipher_decrypt(req); + ret = crypto_skcipher_decrypt(req); skcipher_request_zero(req); if (sg != _sg) kfree(sg); + if (ret < 0) { + WARN_ON_ONCE(ret != -ENOMEM); + return ret; + } /* Extract the decrypted packet length */ if (skb_copy_bits(skb, sp->offset, &sechdr, sizeof(sechdr)) < 0) @@ -602,8 +613,10 @@ static int rxkad_verify_packet(struct rxrpc_call *call, struct sk_buff *skb) skcipher_request_set_sync_tfm(req, call->conn->rxkad.cipher); skcipher_request_set_callback(req, 0, NULL, NULL); skcipher_request_set_crypt(req, &sg, &sg, 8, iv.x); - crypto_skcipher_encrypt(req); + ret = crypto_skcipher_encrypt(req); skcipher_request_zero(req); + if (ret < 0) + goto out; y = ntohl(crypto.buf[1]); cksum = (y >> 16) & 0xffff; @@ -958,6 +971,7 @@ static int rxkad_decrypt_ticket(struct rxrpc_connection *conn, struct in_addr addr; unsigned int life; time64_t issue, now; + int ret; bool little_endian; u8 *p, *q, *name, *end; @@ -977,8 +991,11 @@ static int rxkad_decrypt_ticket(struct rxrpc_connection *conn, sg_init_one(&sg[0], ticket, ticket_len); skcipher_request_set_callback(req, 0, NULL, NULL); skcipher_request_set_crypt(req, sg, sg, ticket_len, iv.x); - crypto_skcipher_decrypt(req); + ret = crypto_skcipher_decrypt(req); skcipher_request_free(req); + if (ret < 0) + return rxrpc_abort_conn(conn, skb, RXKADBADTICKET, -EPROTO, + rxkad_abort_resp_tkt_short); p = ticket; end = p + ticket_len; @@ -1073,21 +1090,23 @@ static int rxkad_decrypt_ticket(struct rxrpc_connection *conn, /* * decrypt the response packet */ -static void rxkad_decrypt_response(struct rxrpc_connection *conn, - struct rxkad_response *resp, - const struct rxrpc_crypt *session_key) +static int rxkad_decrypt_response(struct rxrpc_connection *conn, + struct rxkad_response *resp, + const struct rxrpc_crypt *session_key) { struct skcipher_request *req = rxkad_ci_req; struct scatterlist sg[1]; struct rxrpc_crypt iv; + int ret; _enter(",,%08x%08x", ntohl(session_key->n[0]), ntohl(session_key->n[1])); mutex_lock(&rxkad_ci_mutex); - if (crypto_sync_skcipher_setkey(rxkad_ci, session_key->x, - sizeof(*session_key)) < 0) - BUG(); + ret = crypto_sync_skcipher_setkey(rxkad_ci, session_key->x, + sizeof(*session_key)); + if (ret < 0) + goto unlock; memcpy(&iv, session_key, sizeof(iv)); @@ -1096,12 +1115,14 @@ static void rxkad_decrypt_response(struct rxrpc_connection *conn, skcipher_request_set_sync_tfm(req, rxkad_ci); skcipher_request_set_callback(req, 0, NULL, NULL); skcipher_request_set_crypt(req, sg, sg, sizeof(resp->encrypted), iv.x); - crypto_skcipher_decrypt(req); + ret = crypto_skcipher_decrypt(req); skcipher_request_zero(req); +unlock: mutex_unlock(&rxkad_ci_mutex); _leave(""); + return ret; } /* @@ -1194,7 +1215,9 @@ static int rxkad_verify_response(struct rxrpc_connection *conn, /* use the session key from inside the ticket to decrypt the * response */ - rxkad_decrypt_response(conn, response, &session_key); + ret = rxkad_decrypt_response(conn, response, &session_key); + if (ret < 0) + goto temporary_error_free_ticket; if (ntohl(response->encrypted.epoch) != conn->proto.epoch || ntohl(response->encrypted.cid) != conn->proto.cid || diff --git a/net/rxrpc/sendmsg.c b/net/rxrpc/sendmsg.c index ebbb78b842de80..39a6b21d6b8f7b 100644 --- a/net/rxrpc/sendmsg.c +++ b/net/rxrpc/sendmsg.c @@ -637,7 +637,7 @@ rxrpc_new_client_call_for_sendmsg(struct rxrpc_sock *rx, struct msghdr *msg, memset(&cp, 0, sizeof(cp)); cp.local = rx->local; cp.peer = peer; - cp.key = rx->key; + cp.key = key; cp.security_level = rx->min_sec_level; cp.exclusive = rx->exclusive | p->exclusive; cp.upgrade = p->upgrade; diff --git a/net/rxrpc/server_key.c b/net/rxrpc/server_key.c index 36b05fd842a7b1..27491f1e127366 100644 --- a/net/rxrpc/server_key.c +++ b/net/rxrpc/server_key.c @@ -125,6 +125,9 @@ int rxrpc_server_keyring(struct rxrpc_sock *rx, sockptr_t optval, int optlen) _enter(""); + if (rx->securities) + return -EINVAL; + if (optlen <= 0 || optlen > PAGE_SIZE - 1) return -EINVAL; diff --git a/net/sched/act_csum.c b/net/sched/act_csum.c index 0939e6b2ba4d19..3a377604ad3437 100644 --- a/net/sched/act_csum.c +++ b/net/sched/act_csum.c @@ -604,8 +604,12 @@ TC_INDIRECT_SCOPE int tcf_csum_act(struct sk_buff *skb, protocol = skb->protocol; orig_vlan_tag_present = true; } else { - struct vlan_hdr *vlan = (struct vlan_hdr *)skb->data; + struct vlan_hdr *vlan; + if (!pskb_may_pull(skb, VLAN_HLEN)) + goto drop; + + vlan = (struct vlan_hdr *)skb->data; protocol = vlan->h_vlan_encapsulated_proto; skb_pull(skb, VLAN_HLEN); skb_reset_network_header(skb); diff --git a/net/sched/act_ct.c b/net/sched/act_ct.c index 2b6ac7069dc168..2945a13c509088 100644 --- a/net/sched/act_ct.c +++ b/net/sched/act_ct.c @@ -1358,6 +1358,12 @@ static int tcf_ct_init(struct net *net, struct nlattr *nla, return -EINVAL; } + if (bind && !(flags & TCA_ACT_FLAGS_AT_INGRESS_OR_CLSACT)) { + NL_SET_ERR_MSG_MOD(extack, + "Attaching ct to a non ingress/clsact qdisc is unsupported"); + return -EOPNOTSUPP; + } + err = nla_parse_nested(tb, TCA_CT_MAX, nla, ct_policy, extack); if (err < 0) return err; diff --git a/net/sched/act_gate.c b/net/sched/act_gate.c index c1f75f2727576f..d09013ae1892af 100644 --- a/net/sched/act_gate.c +++ b/net/sched/act_gate.c @@ -32,9 +32,12 @@ static ktime_t gate_get_time(struct tcf_gate *gact) return KTIME_MAX; } -static void gate_get_start_time(struct tcf_gate *gact, ktime_t *start) +static void tcf_gate_params_free_rcu(struct rcu_head *head); + +static void gate_get_start_time(struct tcf_gate *gact, + const struct tcf_gate_params *param, + ktime_t *start) { - struct tcf_gate_params *param = &gact->param; ktime_t now, base, cycle; u64 n; @@ -69,12 +72,14 @@ static enum hrtimer_restart gate_timer_func(struct hrtimer *timer) { struct tcf_gate *gact = container_of(timer, struct tcf_gate, hitimer); - struct tcf_gate_params *p = &gact->param; struct tcfg_gate_entry *next; + struct tcf_gate_params *p; ktime_t close_time, now; spin_lock(&gact->tcf_lock); + p = rcu_dereference_protected(gact->param, + lockdep_is_held(&gact->tcf_lock)); next = gact->next_entry; /* cycle start, clear pending bit, clear total octets */ @@ -225,6 +230,35 @@ static void release_entry_list(struct list_head *entries) } } +static int tcf_gate_copy_entries(struct tcf_gate_params *dst, + const struct tcf_gate_params *src, + struct netlink_ext_ack *extack) +{ + struct tcfg_gate_entry *entry; + int i = 0; + + list_for_each_entry(entry, &src->entries, list) { + struct tcfg_gate_entry *new; + + new = kzalloc(sizeof(*new), GFP_ATOMIC); + if (!new) { + NL_SET_ERR_MSG(extack, "Not enough memory for entry"); + return -ENOMEM; + } + + new->index = entry->index; + new->gate_state = entry->gate_state; + new->interval = entry->interval; + new->ipv = entry->ipv; + new->maxoctets = entry->maxoctets; + list_add_tail(&new->list, &dst->entries); + i++; + } + + dst->num_entries = i; + return 0; +} + static int parse_gate_list(struct nlattr *list_attr, struct tcf_gate_params *sched, struct netlink_ext_ack *extack) @@ -270,24 +304,44 @@ static int parse_gate_list(struct nlattr *list_attr, return err; } -static void gate_setup_timer(struct tcf_gate *gact, u64 basetime, - enum tk_offsets tko, s32 clockid, - bool do_init) +static bool gate_timer_needs_cancel(u64 basetime, u64 old_basetime, + enum tk_offsets tko, + enum tk_offsets old_tko, + s32 clockid, s32 old_clockid) { - if (!do_init) { - if (basetime == gact->param.tcfg_basetime && - tko == gact->tk_offset && - clockid == gact->param.tcfg_clockid) - return; + return basetime != old_basetime || + clockid != old_clockid || + tko != old_tko; +} - spin_unlock_bh(&gact->tcf_lock); - hrtimer_cancel(&gact->hitimer); - spin_lock_bh(&gact->tcf_lock); +static int gate_clock_resolve(s32 clockid, enum tk_offsets *tko, + struct netlink_ext_ack *extack) +{ + switch (clockid) { + case CLOCK_REALTIME: + *tko = TK_OFFS_REAL; + return 0; + case CLOCK_MONOTONIC: + *tko = TK_OFFS_MAX; + return 0; + case CLOCK_BOOTTIME: + *tko = TK_OFFS_BOOT; + return 0; + case CLOCK_TAI: + *tko = TK_OFFS_TAI; + return 0; + default: + NL_SET_ERR_MSG(extack, "Invalid 'clockid'"); + return -EINVAL; } - gact->param.tcfg_basetime = basetime; - gact->param.tcfg_clockid = clockid; - gact->tk_offset = tko; - hrtimer_setup(&gact->hitimer, gate_timer_func, clockid, HRTIMER_MODE_ABS_SOFT); +} + +static void gate_setup_timer(struct tcf_gate *gact, s32 clockid, + enum tk_offsets tko) +{ + WRITE_ONCE(gact->tk_offset, tko); + hrtimer_setup(&gact->hitimer, gate_timer_func, clockid, + HRTIMER_MODE_ABS_SOFT); } static int tcf_gate_init(struct net *net, struct nlattr *nla, @@ -296,15 +350,22 @@ static int tcf_gate_init(struct net *net, struct nlattr *nla, struct netlink_ext_ack *extack) { struct tc_action_net *tn = net_generic(net, act_gate_ops.net_id); - enum tk_offsets tk_offset = TK_OFFS_TAI; + u64 cycletime = 0, basetime = 0, cycletime_ext = 0; + struct tcf_gate_params *p = NULL, *old_p = NULL; + enum tk_offsets old_tk_offset = TK_OFFS_TAI; + const struct tcf_gate_params *cur_p = NULL; bool bind = flags & TCA_ACT_FLAGS_BIND; struct nlattr *tb[TCA_GATE_MAX + 1]; + enum tk_offsets tko = TK_OFFS_TAI; struct tcf_chain *goto_ch = NULL; - u64 cycletime = 0, basetime = 0; - struct tcf_gate_params *p; + s32 timer_clockid = CLOCK_TAI; + bool use_old_entries = false; + s32 old_clockid = CLOCK_TAI; + bool need_cancel = false; s32 clockid = CLOCK_TAI; struct tcf_gate *gact; struct tc_gate *parm; + u64 old_basetime = 0; int ret = 0, err; u32 gflags = 0; s32 prio = -1; @@ -321,26 +382,8 @@ static int tcf_gate_init(struct net *net, struct nlattr *nla, if (!tb[TCA_GATE_PARMS]) return -EINVAL; - if (tb[TCA_GATE_CLOCKID]) { + if (tb[TCA_GATE_CLOCKID]) clockid = nla_get_s32(tb[TCA_GATE_CLOCKID]); - switch (clockid) { - case CLOCK_REALTIME: - tk_offset = TK_OFFS_REAL; - break; - case CLOCK_MONOTONIC: - tk_offset = TK_OFFS_MAX; - break; - case CLOCK_BOOTTIME: - tk_offset = TK_OFFS_BOOT; - break; - case CLOCK_TAI: - tk_offset = TK_OFFS_TAI; - break; - default: - NL_SET_ERR_MSG(extack, "Invalid 'clockid'"); - return -EINVAL; - } - } parm = nla_data(tb[TCA_GATE_PARMS]); index = parm->index; @@ -366,6 +409,60 @@ static int tcf_gate_init(struct net *net, struct nlattr *nla, return -EEXIST; } + gact = to_gate(*a); + + err = tcf_action_check_ctrlact(parm->action, tp, &goto_ch, extack); + if (err < 0) + goto release_idr; + + p = kzalloc(sizeof(*p), GFP_KERNEL); + if (!p) { + err = -ENOMEM; + goto chain_put; + } + INIT_LIST_HEAD(&p->entries); + + use_old_entries = !tb[TCA_GATE_ENTRY_LIST]; + if (!use_old_entries) { + err = parse_gate_list(tb[TCA_GATE_ENTRY_LIST], p, extack); + if (err < 0) + goto err_free; + use_old_entries = !err; + } + + if (ret == ACT_P_CREATED && use_old_entries) { + NL_SET_ERR_MSG(extack, "The entry list is empty"); + err = -EINVAL; + goto err_free; + } + + if (ret != ACT_P_CREATED) { + rcu_read_lock(); + cur_p = rcu_dereference(gact->param); + + old_basetime = cur_p->tcfg_basetime; + old_clockid = cur_p->tcfg_clockid; + old_tk_offset = READ_ONCE(gact->tk_offset); + + basetime = old_basetime; + cycletime_ext = cur_p->tcfg_cycletime_ext; + prio = cur_p->tcfg_priority; + gflags = cur_p->tcfg_flags; + + if (!tb[TCA_GATE_CLOCKID]) + clockid = old_clockid; + + err = 0; + if (use_old_entries) { + err = tcf_gate_copy_entries(p, cur_p, extack); + if (!err && !tb[TCA_GATE_CYCLE_TIME]) + cycletime = cur_p->tcfg_cycletime; + } + rcu_read_unlock(); + if (err) + goto err_free; + } + if (tb[TCA_GATE_PRIORITY]) prio = nla_get_s32(tb[TCA_GATE_PRIORITY]); @@ -375,25 +472,26 @@ static int tcf_gate_init(struct net *net, struct nlattr *nla, if (tb[TCA_GATE_FLAGS]) gflags = nla_get_u32(tb[TCA_GATE_FLAGS]); - gact = to_gate(*a); - if (ret == ACT_P_CREATED) - INIT_LIST_HEAD(&gact->param.entries); + if (tb[TCA_GATE_CYCLE_TIME]) + cycletime = nla_get_u64(tb[TCA_GATE_CYCLE_TIME]); - err = tcf_action_check_ctrlact(parm->action, tp, &goto_ch, extack); - if (err < 0) - goto release_idr; + if (tb[TCA_GATE_CYCLE_TIME_EXT]) + cycletime_ext = nla_get_u64(tb[TCA_GATE_CYCLE_TIME_EXT]); - spin_lock_bh(&gact->tcf_lock); - p = &gact->param; + err = gate_clock_resolve(clockid, &tko, extack); + if (err) + goto err_free; + timer_clockid = clockid; - if (tb[TCA_GATE_CYCLE_TIME]) - cycletime = nla_get_u64(tb[TCA_GATE_CYCLE_TIME]); + need_cancel = ret != ACT_P_CREATED && + gate_timer_needs_cancel(basetime, old_basetime, + tko, old_tk_offset, + timer_clockid, old_clockid); - if (tb[TCA_GATE_ENTRY_LIST]) { - err = parse_gate_list(tb[TCA_GATE_ENTRY_LIST], p, extack); - if (err < 0) - goto chain_put; - } + if (need_cancel) + hrtimer_cancel(&gact->hitimer); + + spin_lock_bh(&gact->tcf_lock); if (!cycletime) { struct tcfg_gate_entry *entry; @@ -402,22 +500,20 @@ static int tcf_gate_init(struct net *net, struct nlattr *nla, list_for_each_entry(entry, &p->entries, list) cycle = ktime_add_ns(cycle, entry->interval); cycletime = cycle; - if (!cycletime) { - err = -EINVAL; - goto chain_put; - } } p->tcfg_cycletime = cycletime; + p->tcfg_cycletime_ext = cycletime_ext; - if (tb[TCA_GATE_CYCLE_TIME_EXT]) - p->tcfg_cycletime_ext = - nla_get_u64(tb[TCA_GATE_CYCLE_TIME_EXT]); - - gate_setup_timer(gact, basetime, tk_offset, clockid, - ret == ACT_P_CREATED); + if (need_cancel || ret == ACT_P_CREATED) + gate_setup_timer(gact, timer_clockid, tko); p->tcfg_priority = prio; p->tcfg_flags = gflags; - gate_get_start_time(gact, &start); + p->tcfg_basetime = basetime; + p->tcfg_clockid = timer_clockid; + gate_get_start_time(gact, p, &start); + + old_p = rcu_replace_pointer(gact->param, p, + lockdep_is_held(&gact->tcf_lock)); gact->current_close_time = start; gact->current_gate_status = GATE_ACT_GATE_OPEN | GATE_ACT_PENDING; @@ -434,11 +530,15 @@ static int tcf_gate_init(struct net *net, struct nlattr *nla, if (goto_ch) tcf_chain_put_by_act(goto_ch); + if (old_p) + call_rcu(&old_p->rcu, tcf_gate_params_free_rcu); + return ret; +err_free: + release_entry_list(&p->entries); + kfree(p); chain_put: - spin_unlock_bh(&gact->tcf_lock); - if (goto_ch) tcf_chain_put_by_act(goto_ch); release_idr: @@ -446,21 +546,29 @@ static int tcf_gate_init(struct net *net, struct nlattr *nla, * without taking tcf_lock. */ if (ret == ACT_P_CREATED) - gate_setup_timer(gact, gact->param.tcfg_basetime, - gact->tk_offset, gact->param.tcfg_clockid, - true); + gate_setup_timer(gact, timer_clockid, tko); + tcf_idr_release(*a, bind); return err; } +static void tcf_gate_params_free_rcu(struct rcu_head *head) +{ + struct tcf_gate_params *p = container_of(head, struct tcf_gate_params, rcu); + + release_entry_list(&p->entries); + kfree(p); +} + static void tcf_gate_cleanup(struct tc_action *a) { struct tcf_gate *gact = to_gate(a); struct tcf_gate_params *p; - p = &gact->param; hrtimer_cancel(&gact->hitimer); - release_entry_list(&p->entries); + p = rcu_dereference_protected(gact->param, 1); + if (p) + call_rcu(&p->rcu, tcf_gate_params_free_rcu); } static int dumping_entry(struct sk_buff *skb, @@ -509,10 +617,9 @@ static int tcf_gate_dump(struct sk_buff *skb, struct tc_action *a, struct nlattr *entry_list; struct tcf_t t; - spin_lock_bh(&gact->tcf_lock); - opt.action = gact->tcf_action; - - p = &gact->param; + rcu_read_lock(); + opt.action = READ_ONCE(gact->tcf_action); + p = rcu_dereference(gact->param); if (nla_put(skb, TCA_GATE_PARMS, sizeof(opt), &opt)) goto nla_put_failure; @@ -552,12 +659,12 @@ static int tcf_gate_dump(struct sk_buff *skb, struct tc_action *a, tcf_tm_dump(&t, &gact->tcf_tm); if (nla_put_64bit(skb, TCA_GATE_TM, sizeof(t), &t, TCA_GATE_PAD)) goto nla_put_failure; - spin_unlock_bh(&gact->tcf_lock); + rcu_read_unlock(); return skb->len; nla_put_failure: - spin_unlock_bh(&gact->tcf_lock); + rcu_read_unlock(); nlmsg_trim(skb, b); return -1; } diff --git a/net/sched/act_ife.c b/net/sched/act_ife.c index 8e8f6af731d51c..4ad01d4e820db8 100644 --- a/net/sched/act_ife.c +++ b/net/sched/act_ife.c @@ -293,8 +293,8 @@ static int load_metaops_and_vet(u32 metaid, void *val, int len, bool rtnl_held) /* called when adding new meta information */ static int __add_metainfo(const struct tcf_meta_ops *ops, - struct tcf_ife_info *ife, u32 metaid, void *metaval, - int len, bool atomic, bool exists) + struct tcf_ife_params *p, u32 metaid, void *metaval, + int len, bool atomic) { struct tcf_meta_info *mi = NULL; int ret = 0; @@ -313,45 +313,40 @@ static int __add_metainfo(const struct tcf_meta_ops *ops, } } - if (exists) - spin_lock_bh(&ife->tcf_lock); - list_add_tail(&mi->metalist, &ife->metalist); - if (exists) - spin_unlock_bh(&ife->tcf_lock); + list_add_tail(&mi->metalist, &p->metalist); return ret; } static int add_metainfo_and_get_ops(const struct tcf_meta_ops *ops, - struct tcf_ife_info *ife, u32 metaid, - bool exists) + struct tcf_ife_params *p, u32 metaid) { int ret; if (!try_module_get(ops->owner)) return -ENOENT; - ret = __add_metainfo(ops, ife, metaid, NULL, 0, true, exists); + ret = __add_metainfo(ops, p, metaid, NULL, 0, true); if (ret) module_put(ops->owner); return ret; } -static int add_metainfo(struct tcf_ife_info *ife, u32 metaid, void *metaval, - int len, bool exists) +static int add_metainfo(struct tcf_ife_params *p, u32 metaid, void *metaval, + int len) { const struct tcf_meta_ops *ops = find_ife_oplist(metaid); int ret; if (!ops) return -ENOENT; - ret = __add_metainfo(ops, ife, metaid, metaval, len, false, exists); + ret = __add_metainfo(ops, p, metaid, metaval, len, false); if (ret) /*put back what find_ife_oplist took */ module_put(ops->owner); return ret; } -static int use_all_metadata(struct tcf_ife_info *ife, bool exists) +static int use_all_metadata(struct tcf_ife_params *p) { struct tcf_meta_ops *o; int rc = 0; @@ -359,7 +354,7 @@ static int use_all_metadata(struct tcf_ife_info *ife, bool exists) read_lock(&ife_mod_lock); list_for_each_entry(o, &ifeoplist, list) { - rc = add_metainfo_and_get_ops(o, ife, o->metaid, exists); + rc = add_metainfo_and_get_ops(o, p, o->metaid); if (rc == 0) installed += 1; } @@ -371,7 +366,7 @@ static int use_all_metadata(struct tcf_ife_info *ife, bool exists) return -EINVAL; } -static int dump_metalist(struct sk_buff *skb, struct tcf_ife_info *ife) +static int dump_metalist(struct sk_buff *skb, struct tcf_ife_params *p) { struct tcf_meta_info *e; struct nlattr *nest; @@ -379,14 +374,14 @@ static int dump_metalist(struct sk_buff *skb, struct tcf_ife_info *ife) int total_encoded = 0; /*can only happen on decode */ - if (list_empty(&ife->metalist)) + if (list_empty(&p->metalist)) return 0; nest = nla_nest_start_noflag(skb, TCA_IFE_METALST); if (!nest) goto out_nlmsg_trim; - list_for_each_entry(e, &ife->metalist, metalist) { + list_for_each_entry(e, &p->metalist, metalist) { if (!e->ops->get(skb, e)) total_encoded += 1; } @@ -403,13 +398,11 @@ static int dump_metalist(struct sk_buff *skb, struct tcf_ife_info *ife) return -1; } -/* under ife->tcf_lock */ -static void _tcf_ife_cleanup(struct tc_action *a) +static void __tcf_ife_cleanup(struct tcf_ife_params *p) { - struct tcf_ife_info *ife = to_ife(a); struct tcf_meta_info *e, *n; - list_for_each_entry_safe(e, n, &ife->metalist, metalist) { + list_for_each_entry_safe(e, n, &p->metalist, metalist) { list_del(&e->metalist); if (e->metaval) { if (e->ops->release) @@ -422,18 +415,23 @@ static void _tcf_ife_cleanup(struct tc_action *a) } } +static void tcf_ife_cleanup_params(struct rcu_head *head) +{ + struct tcf_ife_params *p = container_of(head, struct tcf_ife_params, + rcu); + + __tcf_ife_cleanup(p); + kfree(p); +} + static void tcf_ife_cleanup(struct tc_action *a) { struct tcf_ife_info *ife = to_ife(a); struct tcf_ife_params *p; - spin_lock_bh(&ife->tcf_lock); - _tcf_ife_cleanup(a); - spin_unlock_bh(&ife->tcf_lock); - p = rcu_dereference_protected(ife->params, 1); if (p) - kfree_rcu(p, rcu); + call_rcu(&p->rcu, tcf_ife_cleanup_params); } static int load_metalist(struct nlattr **tb, bool rtnl_held) @@ -455,8 +453,7 @@ static int load_metalist(struct nlattr **tb, bool rtnl_held) return 0; } -static int populate_metalist(struct tcf_ife_info *ife, struct nlattr **tb, - bool exists, bool rtnl_held) +static int populate_metalist(struct tcf_ife_params *p, struct nlattr **tb) { int len = 0; int rc = 0; @@ -468,7 +465,7 @@ static int populate_metalist(struct tcf_ife_info *ife, struct nlattr **tb, val = nla_data(tb[i]); len = nla_len(tb[i]); - rc = add_metainfo(ife, i, val, len, exists); + rc = add_metainfo(p, i, val, len); if (rc) return rc; } @@ -523,6 +520,7 @@ static int tcf_ife_init(struct net *net, struct nlattr *nla, p = kzalloc(sizeof(*p), GFP_KERNEL); if (!p) return -ENOMEM; + INIT_LIST_HEAD(&p->metalist); if (tb[TCA_IFE_METALST]) { err = nla_parse_nested_deprecated(tb2, IFE_META_MAX, @@ -567,8 +565,6 @@ static int tcf_ife_init(struct net *net, struct nlattr *nla, } ife = to_ife(*a); - if (ret == ACT_P_CREATED) - INIT_LIST_HEAD(&ife->metalist); err = tcf_action_check_ctrlact(parm->action, tp, &goto_ch, extack); if (err < 0) @@ -600,8 +596,7 @@ static int tcf_ife_init(struct net *net, struct nlattr *nla, } if (tb[TCA_IFE_METALST]) { - err = populate_metalist(ife, tb2, exists, - !(flags & TCA_ACT_FLAGS_NO_RTNL)); + err = populate_metalist(p, tb2); if (err) goto metadata_parse_err; } else { @@ -610,7 +605,7 @@ static int tcf_ife_init(struct net *net, struct nlattr *nla, * as we can. You better have at least one else we are * going to bail out */ - err = use_all_metadata(ife, exists); + err = use_all_metadata(p); if (err) goto metadata_parse_err; } @@ -626,13 +621,14 @@ static int tcf_ife_init(struct net *net, struct nlattr *nla, if (goto_ch) tcf_chain_put_by_act(goto_ch); if (p) - kfree_rcu(p, rcu); + call_rcu(&p->rcu, tcf_ife_cleanup_params); return ret; metadata_parse_err: if (goto_ch) tcf_chain_put_by_act(goto_ch); release_idr: + __tcf_ife_cleanup(p); kfree(p); tcf_idr_release(*a, bind); return err; @@ -679,7 +675,7 @@ static int tcf_ife_dump(struct sk_buff *skb, struct tc_action *a, int bind, if (nla_put(skb, TCA_IFE_TYPE, 2, &p->eth_type)) goto nla_put_failure; - if (dump_metalist(skb, ife)) { + if (dump_metalist(skb, p)) { /*ignore failure to dump metalist */ pr_info("Failed to dump metalist\n"); } @@ -693,13 +689,13 @@ static int tcf_ife_dump(struct sk_buff *skb, struct tc_action *a, int bind, return -1; } -static int find_decode_metaid(struct sk_buff *skb, struct tcf_ife_info *ife, +static int find_decode_metaid(struct sk_buff *skb, struct tcf_ife_params *p, u16 metaid, u16 mlen, void *mdata) { struct tcf_meta_info *e; /* XXX: use hash to speed up */ - list_for_each_entry(e, &ife->metalist, metalist) { + list_for_each_entry_rcu(e, &p->metalist, metalist) { if (metaid == e->metaid) { if (e->ops) { /* We check for decode presence already */ @@ -716,10 +712,13 @@ static int tcf_ife_decode(struct sk_buff *skb, const struct tc_action *a, { struct tcf_ife_info *ife = to_ife(a); int action = ife->tcf_action; + struct tcf_ife_params *p; u8 *ifehdr_end; u8 *tlv_data; u16 metalen; + p = rcu_dereference_bh(ife->params); + bstats_update(this_cpu_ptr(ife->common.cpu_bstats), skb); tcf_lastuse_update(&ife->tcf_tm); @@ -745,7 +744,7 @@ static int tcf_ife_decode(struct sk_buff *skb, const struct tc_action *a, return TC_ACT_SHOT; } - if (find_decode_metaid(skb, ife, mtype, dlen, curr_data)) { + if (find_decode_metaid(skb, p, mtype, dlen, curr_data)) { /* abuse overlimits to count when we receive metadata * but dont have an ops for it */ @@ -769,12 +768,12 @@ static int tcf_ife_decode(struct sk_buff *skb, const struct tc_action *a, /*XXX: check if we can do this at install time instead of current * send data path **/ -static int ife_get_sz(struct sk_buff *skb, struct tcf_ife_info *ife) +static int ife_get_sz(struct sk_buff *skb, struct tcf_ife_params *p) { - struct tcf_meta_info *e, *n; + struct tcf_meta_info *e; int tot_run_sz = 0, run_sz = 0; - list_for_each_entry_safe(e, n, &ife->metalist, metalist) { + list_for_each_entry_rcu(e, &p->metalist, metalist) { if (e->ops->check_presence) { run_sz = e->ops->check_presence(skb, e); tot_run_sz += run_sz; @@ -795,7 +794,7 @@ static int tcf_ife_encode(struct sk_buff *skb, const struct tc_action *a, OUTERHDR:TOTMETALEN:{TLVHDR:Metadatum:TLVHDR..}:ORIGDATA where ORIGDATA = original ethernet header ... */ - u16 metalen = ife_get_sz(skb, ife); + u16 metalen = ife_get_sz(skb, p); int hdrm = metalen + skb->dev->hard_header_len + IFE_METAHDRLEN; unsigned int skboff = 0; int new_len = skb->len + hdrm; @@ -833,25 +832,21 @@ static int tcf_ife_encode(struct sk_buff *skb, const struct tc_action *a, if (!ife_meta) goto drop; - spin_lock(&ife->tcf_lock); - /* XXX: we dont have a clever way of telling encode to * not repeat some of the computations that are done by * ops->presence_check... */ - list_for_each_entry(e, &ife->metalist, metalist) { + list_for_each_entry_rcu(e, &p->metalist, metalist) { if (e->ops->encode) { err = e->ops->encode(skb, (void *)(ife_meta + skboff), e); } if (err < 0) { /* too corrupt to keep around if overwritten */ - spin_unlock(&ife->tcf_lock); goto drop; } skboff += err; } - spin_unlock(&ife->tcf_lock); oethh = (struct ethhdr *)skb->data; if (!is_zero_ether_addr(p->eth_src)) diff --git a/net/sched/act_skbedit.c b/net/sched/act_skbedit.c index 8c1d1554f6575d..5450c1293eb508 100644 --- a/net/sched/act_skbedit.c +++ b/net/sched/act_skbedit.c @@ -126,7 +126,7 @@ static int tcf_skbedit_init(struct net *net, struct nlattr *nla, struct tcf_skbedit *d; u32 flags = 0, *priority = NULL, *mark = NULL, *mask = NULL; u16 *queue_mapping = NULL, *ptype = NULL; - u16 mapping_mod = 1; + u32 mapping_mod = 1; bool exists = false; int ret = 0, err; u32 index; @@ -194,6 +194,10 @@ static int tcf_skbedit_init(struct net *net, struct nlattr *nla, } mapping_mod = *queue_mapping_max - *queue_mapping + 1; + if (mapping_mod > U16_MAX) { + NL_SET_ERR_MSG_MOD(extack, "The range of queue_mapping is invalid."); + return -EINVAL; + } flags |= SKBEDIT_F_TXQ_SKBHASH; } if (*pure_flags & SKBEDIT_F_INHERITDSFIELD) diff --git a/net/sched/bpf_qdisc.c b/net/sched/bpf_qdisc.c index adcb618a2bfca0..e9bea9890777d4 100644 --- a/net/sched/bpf_qdisc.c +++ b/net/sched/bpf_qdisc.c @@ -202,6 +202,12 @@ __bpf_kfunc void bpf_kfree_skb(struct sk_buff *skb) kfree_skb(skb); } +__bpf_kfunc void bpf_kfree_skb_dtor(void *skb) +{ + bpf_kfree_skb(skb); +} +CFI_NOSEAL(bpf_kfree_skb_dtor); + /* bpf_qdisc_skb_drop - Drop an skb by adding it to a deferred free list. * @skb: The skb whose reference to be released and dropped. * @to_free_list: The list of skbs to be dropped. @@ -449,7 +455,7 @@ static struct bpf_struct_ops bpf_Qdisc_ops = { .owner = THIS_MODULE, }; -BTF_ID_LIST_SINGLE(bpf_sk_buff_dtor_ids, func, bpf_kfree_skb) +BTF_ID_LIST_SINGLE(bpf_sk_buff_dtor_ids, func, bpf_kfree_skb_dtor) static int __init bpf_qdisc_kfunc_init(void) { diff --git a/net/sched/cls_api.c b/net/sched/cls_api.c index ebca4b926dcf76..9edaff15052f3e 100644 --- a/net/sched/cls_api.c +++ b/net/sched/cls_api.c @@ -2228,6 +2228,11 @@ static bool is_qdisc_ingress(__u32 classid) return (TC_H_MIN(classid) == TC_H_MIN(TC_H_MIN_INGRESS)); } +static bool is_ingress_or_clsact(struct tcf_block *block, struct Qdisc *q) +{ + return tcf_block_shared(block) || (q && !!(q->flags & TCQ_F_INGRESS)); +} + static int tc_new_tfilter(struct sk_buff *skb, struct nlmsghdr *n, struct netlink_ext_ack *extack) { @@ -2420,6 +2425,8 @@ static int tc_new_tfilter(struct sk_buff *skb, struct nlmsghdr *n, flags |= TCA_ACT_FLAGS_NO_RTNL; if (is_qdisc_ingress(parent)) flags |= TCA_ACT_FLAGS_AT_INGRESS; + if (is_ingress_or_clsact(block, q)) + flags |= TCA_ACT_FLAGS_AT_INGRESS_OR_CLSACT; err = tp->ops->change(net, skb, tp, cl, t->tcm_handle, tca, &fh, flags, extack); if (err == 0) { @@ -2962,6 +2969,7 @@ static int tc_chain_fill_node(const struct tcf_proto_ops *tmplt_ops, tcm->tcm__pad1 = 0; tcm->tcm__pad2 = 0; tcm->tcm_handle = 0; + tcm->tcm_info = 0; if (block->q) { tcm->tcm_ifindex = qdisc_dev(block->q)->ifindex; tcm->tcm_parent = block->q->handle; diff --git a/net/sched/cls_flow.c b/net/sched/cls_flow.c index 5693b41b093f3f..edf1252c1fde75 100644 --- a/net/sched/cls_flow.c +++ b/net/sched/cls_flow.c @@ -503,8 +503,16 @@ static int flow_change(struct net *net, struct sk_buff *in_skb, } if (TC_H_MAJ(baseclass) == 0) { - struct Qdisc *q = tcf_block_q(tp->chain->block); + struct tcf_block *block = tp->chain->block; + struct Qdisc *q; + if (tcf_block_shared(block)) { + NL_SET_ERR_MSG(extack, + "Must specify baseclass when attaching flow filter to block"); + goto err2; + } + + q = tcf_block_q(block); baseclass = TC_H_MAKE(q->handle, baseclass); } if (TC_H_MIN(baseclass) == 0) diff --git a/net/sched/cls_fw.c b/net/sched/cls_fw.c index cdddc869522848..83a7372ea15c2a 100644 --- a/net/sched/cls_fw.c +++ b/net/sched/cls_fw.c @@ -247,8 +247,18 @@ static int fw_change(struct net *net, struct sk_buff *in_skb, struct nlattr *tb[TCA_FW_MAX + 1]; int err; - if (!opt) - return handle ? -EINVAL : 0; /* Succeed if it is old method. */ + if (!opt) { + if (handle) + return -EINVAL; + + if (tcf_block_shared(tp->chain->block)) { + NL_SET_ERR_MSG(extack, + "Must specify mark when attaching fw filter to block"); + return -EINVAL; + } + + return 0; /* Succeed if it is old method. */ + } err = nla_parse_nested_deprecated(tb, TCA_FW_MAX, opt, fw_policy, NULL); diff --git a/net/sched/sch_ets.c b/net/sched/sch_ets.c index 306e046276d465..a4b07b661b7756 100644 --- a/net/sched/sch_ets.c +++ b/net/sched/sch_ets.c @@ -115,12 +115,12 @@ static void ets_offload_change(struct Qdisc *sch) struct ets_sched *q = qdisc_priv(sch); struct tc_ets_qopt_offload qopt; unsigned int w_psum_prev = 0; - unsigned int q_psum = 0; - unsigned int q_sum = 0; unsigned int quantum; unsigned int w_psum; unsigned int weight; unsigned int i; + u64 q_psum = 0; + u64 q_sum = 0; if (!tc_can_offload(dev) || !dev->netdev_ops->ndo_setup_tc) return; @@ -138,8 +138,12 @@ static void ets_offload_change(struct Qdisc *sch) for (i = 0; i < q->nbands; i++) { quantum = q->classes[i].quantum; - q_psum += quantum; - w_psum = quantum ? q_psum * 100 / q_sum : 0; + if (quantum) { + q_psum += quantum; + w_psum = div64_u64(q_psum * 100, q_sum); + } else { + w_psum = 0; + } weight = w_psum - w_psum_prev; w_psum_prev = w_psum; diff --git a/net/sched/sch_fq.c b/net/sched/sch_fq.c index 6e5f2f4f241546..b570128ae10a61 100644 --- a/net/sched/sch_fq.c +++ b/net/sched/sch_fq.c @@ -829,6 +829,7 @@ static void fq_reset(struct Qdisc *sch) for (idx = 0; idx < FQ_BANDS; idx++) { q->band_flows[idx].new_flows.first = NULL; q->band_flows[idx].old_flows.first = NULL; + q->band_pkt_count[idx] = 0; } q->delayed = RB_ROOT; q->flows = 0; diff --git a/net/sched/sch_generic.c b/net/sched/sch_generic.c index 852e603c17551e..8b07d194c4c35f 100644 --- a/net/sched/sch_generic.c +++ b/net/sched/sch_generic.c @@ -1290,33 +1290,6 @@ static void dev_deactivate_queue(struct net_device *dev, } } -static void dev_reset_queue(struct net_device *dev, - struct netdev_queue *dev_queue, - void *_unused) -{ - struct Qdisc *qdisc; - bool nolock; - - qdisc = rtnl_dereference(dev_queue->qdisc_sleeping); - if (!qdisc) - return; - - nolock = qdisc->flags & TCQ_F_NOLOCK; - - if (nolock) - spin_lock_bh(&qdisc->seqlock); - spin_lock_bh(qdisc_lock(qdisc)); - - qdisc_reset(qdisc); - - spin_unlock_bh(qdisc_lock(qdisc)); - if (nolock) { - clear_bit(__QDISC_STATE_MISSED, &qdisc->state); - clear_bit(__QDISC_STATE_DRAINING, &qdisc->state); - spin_unlock_bh(&qdisc->seqlock); - } -} - static bool some_qdisc_is_busy(struct net_device *dev) { unsigned int i; diff --git a/net/sched/sch_hfsc.c b/net/sched/sch_hfsc.c index d8fd35da32a7c6..57221522fe56d2 100644 --- a/net/sched/sch_hfsc.c +++ b/net/sched/sch_hfsc.c @@ -555,7 +555,7 @@ static void rtsc_min(struct runtime_sc *rtsc, struct internal_sc *isc, u64 x, u64 y) { u64 y1, y2, dx, dy; - u32 dsm; + u64 dsm; if (isc->sm1 <= isc->sm2) { /* service curve is convex */ @@ -598,7 +598,7 @@ rtsc_min(struct runtime_sc *rtsc, struct internal_sc *isc, u64 x, u64 y) */ dx = (y1 - y) << SM_SHIFT; dsm = isc->sm1 - isc->sm2; - do_div(dx, dsm); + dx = div64_u64(dx, dsm); /* * check if (x, y1) belongs to the 1st segment of rtsc. * if so, add the offset. diff --git a/net/sched/sch_ingress.c b/net/sched/sch_ingress.c index cc6051d4f2ef88..c3e18bae8fbfc7 100644 --- a/net/sched/sch_ingress.c +++ b/net/sched/sch_ingress.c @@ -113,14 +113,15 @@ static void ingress_destroy(struct Qdisc *sch) { struct ingress_sched_data *q = qdisc_priv(sch); struct net_device *dev = qdisc_dev(sch); - struct bpf_mprog_entry *entry = rtnl_dereference(dev->tcx_ingress); + struct bpf_mprog_entry *entry; if (sch->parent != TC_H_INGRESS) return; tcf_block_put_ext(q->block, sch, &q->block_info); - if (entry) { + if (mini_qdisc_pair_inited(&q->miniqp)) { + entry = rtnl_dereference(dev->tcx_ingress); tcx_miniq_dec(entry); if (!tcx_entry_is_active(entry)) { tcx_entry_update(dev, NULL, true); @@ -290,10 +291,9 @@ static int clsact_init(struct Qdisc *sch, struct nlattr *opt, static void clsact_destroy(struct Qdisc *sch) { + struct bpf_mprog_entry *ingress_entry, *egress_entry; struct clsact_sched_data *q = qdisc_priv(sch); struct net_device *dev = qdisc_dev(sch); - struct bpf_mprog_entry *ingress_entry = rtnl_dereference(dev->tcx_ingress); - struct bpf_mprog_entry *egress_entry = rtnl_dereference(dev->tcx_egress); if (sch->parent != TC_H_CLSACT) return; @@ -301,7 +301,8 @@ static void clsact_destroy(struct Qdisc *sch) tcf_block_put_ext(q->ingress_block, sch, &q->ingress_block_info); tcf_block_put_ext(q->egress_block, sch, &q->egress_block_info); - if (ingress_entry) { + if (mini_qdisc_pair_inited(&q->miniqp_ingress)) { + ingress_entry = rtnl_dereference(dev->tcx_ingress); tcx_miniq_dec(ingress_entry); if (!tcx_entry_is_active(ingress_entry)) { tcx_entry_update(dev, NULL, true); @@ -309,7 +310,8 @@ static void clsact_destroy(struct Qdisc *sch) } } - if (egress_entry) { + if (mini_qdisc_pair_inited(&q->miniqp_egress)) { + egress_entry = rtnl_dereference(dev->tcx_egress); tcx_miniq_dec(egress_entry); if (!tcx_entry_is_active(egress_entry)) { tcx_entry_update(dev, NULL, false); diff --git a/net/sched/sch_netem.c b/net/sched/sch_netem.c index 32a5f33040461f..3356d62ad0548e 100644 --- a/net/sched/sch_netem.c +++ b/net/sched/sch_netem.c @@ -519,8 +519,9 @@ static int netem_enqueue(struct sk_buff *skb, struct Qdisc *sch, goto finish_segs; } - skb->data[get_random_u32_below(skb_headlen(skb))] ^= - 1<data[get_random_u32_below(skb_headlen(skb))] ^= + 1 << get_random_u32_below(8); } if (unlikely(q->t_len >= sch->limit)) { diff --git a/net/sched/sch_teql.c b/net/sched/sch_teql.c index 6e4bdaa876ed68..ec4039a201a2c2 100644 --- a/net/sched/sch_teql.c +++ b/net/sched/sch_teql.c @@ -146,15 +146,12 @@ teql_destroy(struct Qdisc *sch) master->slaves = NEXT_SLAVE(q); if (q == master->slaves) { struct netdev_queue *txq; - spinlock_t *root_lock; txq = netdev_get_tx_queue(master->dev, 0); master->slaves = NULL; - root_lock = qdisc_root_sleeping_lock(rtnl_dereference(txq->qdisc)); - spin_lock_bh(root_lock); - qdisc_reset(rtnl_dereference(txq->qdisc)); - spin_unlock_bh(root_lock); + dev_reset_queue(master->dev, + txq, NULL); } } skb_queue_purge(&dat->q); @@ -315,6 +312,7 @@ static netdev_tx_t teql_master_xmit(struct sk_buff *skb, struct net_device *dev) if (__netif_tx_trylock(slave_txq)) { unsigned int length = qdisc_pkt_len(skb); + skb->dev = slave; if (!netif_xmit_frozen_or_stopped(slave_txq) && netdev_start_xmit(skb, slave, slave_txq, false) == NETDEV_TX_OK) { diff --git a/net/shaper/shaper.c b/net/shaper/shaper.c index 7101a48bce5454..be9999ab62e391 100644 --- a/net/shaper/shaper.c +++ b/net/shaper/shaper.c @@ -36,29 +36,26 @@ static struct net_shaper_binding *net_shaper_binding_from_ctx(void *ctx) return &((struct net_shaper_nl_ctx *)ctx)->binding; } -static void net_shaper_lock(struct net_shaper_binding *binding) +static struct net_shaper_hierarchy * +net_shaper_hierarchy(struct net_shaper_binding *binding) { - switch (binding->type) { - case NET_SHAPER_BINDING_TYPE_NETDEV: - netdev_lock(binding->netdev); - break; - } -} + /* Pairs with WRITE_ONCE() in net_shaper_hierarchy_setup. */ + if (binding->type == NET_SHAPER_BINDING_TYPE_NETDEV) + return READ_ONCE(binding->netdev->net_shaper_hierarchy); -static void net_shaper_unlock(struct net_shaper_binding *binding) -{ - switch (binding->type) { - case NET_SHAPER_BINDING_TYPE_NETDEV: - netdev_unlock(binding->netdev); - break; - } + /* No other type supported yet. */ + return NULL; } static struct net_shaper_hierarchy * -net_shaper_hierarchy(struct net_shaper_binding *binding) +net_shaper_hierarchy_rcu(struct net_shaper_binding *binding) { - /* Pairs with WRITE_ONCE() in net_shaper_hierarchy_setup. */ - if (binding->type == NET_SHAPER_BINDING_TYPE_NETDEV) + /* Readers look up the device and take a ref, then take RCU lock + * later at which point netdev may have been unregistered and flushed. + * READ_ONCE() pairs with WRITE_ONCE() in net_shaper_hierarchy_setup. + */ + if (binding->type == NET_SHAPER_BINDING_TYPE_NETDEV && + READ_ONCE(binding->netdev->reg_state) <= NETREG_REGISTERED) return READ_ONCE(binding->netdev->net_shaper_hierarchy); /* No other type supported yet. */ @@ -204,12 +201,49 @@ static int net_shaper_ctx_setup(const struct genl_info *info, int type, return 0; } +/* Like net_shaper_ctx_setup(), but for "write" handlers (never for dumps!) + * Acquires the lock protecting the hierarchy (instance lock for netdev). + */ +static int net_shaper_ctx_setup_lock(const struct genl_info *info, int type, + struct net_shaper_nl_ctx *ctx) +{ + struct net *ns = genl_info_net(info); + struct net_device *dev; + int ifindex; + + if (GENL_REQ_ATTR_CHECK(info, type)) + return -EINVAL; + + ifindex = nla_get_u32(info->attrs[type]); + dev = netdev_get_by_index_lock(ns, ifindex); + if (!dev) { + NL_SET_BAD_ATTR(info->extack, info->attrs[type]); + return -ENOENT; + } + + if (!dev->netdev_ops->net_shaper_ops) { + NL_SET_BAD_ATTR(info->extack, info->attrs[type]); + netdev_unlock(dev); + return -EOPNOTSUPP; + } + + ctx->binding.type = NET_SHAPER_BINDING_TYPE_NETDEV; + ctx->binding.netdev = dev; + return 0; +} + static void net_shaper_ctx_cleanup(struct net_shaper_nl_ctx *ctx) { if (ctx->binding.type == NET_SHAPER_BINDING_TYPE_NETDEV) netdev_put(ctx->binding.netdev, &ctx->dev_tracker); } +static void net_shaper_ctx_cleanup_unlock(struct net_shaper_nl_ctx *ctx) +{ + if (ctx->binding.type == NET_SHAPER_BINDING_TYPE_NETDEV) + netdev_unlock(ctx->binding.netdev); +} + static u32 net_shaper_handle_to_index(const struct net_shaper_handle *handle) { return FIELD_PREP(NET_SHAPER_SCOPE_MASK, handle->scope) | @@ -251,9 +285,10 @@ static struct net_shaper * net_shaper_lookup(struct net_shaper_binding *binding, const struct net_shaper_handle *handle) { - struct net_shaper_hierarchy *hierarchy = net_shaper_hierarchy(binding); u32 index = net_shaper_handle_to_index(handle); + struct net_shaper_hierarchy *hierarchy; + hierarchy = net_shaper_hierarchy_rcu(binding); if (!hierarchy || xa_get_mark(&hierarchy->shapers, index, NET_SHAPER_NOT_VALID)) return NULL; @@ -262,7 +297,7 @@ net_shaper_lookup(struct net_shaper_binding *binding, } /* Allocate on demand the per device shaper's hierarchy container. - * Called under the net shaper lock + * Called under the lock protecting the hierarchy (instance lock for netdev) */ static struct net_shaper_hierarchy * net_shaper_hierarchy_setup(struct net_shaper_binding *binding) @@ -681,6 +716,22 @@ void net_shaper_nl_post_doit(const struct genl_split_ops *ops, net_shaper_generic_post(info); } +int net_shaper_nl_pre_doit_write(const struct genl_split_ops *ops, + struct sk_buff *skb, struct genl_info *info) +{ + struct net_shaper_nl_ctx *ctx = (struct net_shaper_nl_ctx *)info->ctx; + + BUILD_BUG_ON(sizeof(*ctx) > sizeof(info->ctx)); + + return net_shaper_ctx_setup_lock(info, NET_SHAPER_A_IFINDEX, ctx); +} + +void net_shaper_nl_post_doit_write(const struct genl_split_ops *ops, + struct sk_buff *skb, struct genl_info *info) +{ + net_shaper_ctx_cleanup_unlock((struct net_shaper_nl_ctx *)info->ctx); +} + int net_shaper_nl_pre_dumpit(struct netlink_callback *cb) { struct net_shaper_nl_ctx *ctx = (struct net_shaper_nl_ctx *)cb->ctx; @@ -759,11 +810,7 @@ int net_shaper_nl_get_doit(struct sk_buff *skb, struct genl_info *info) if (ret) goto free_msg; - ret = genlmsg_reply(msg, info); - if (ret) - goto free_msg; - - return 0; + return genlmsg_reply(msg, info); free_msg: nlmsg_free(msg); @@ -782,17 +829,19 @@ int net_shaper_nl_get_dumpit(struct sk_buff *skb, /* Don't error out dumps performed before any set operation. */ binding = net_shaper_binding_from_ctx(ctx); - hierarchy = net_shaper_hierarchy(binding); - if (!hierarchy) - return 0; rcu_read_lock(); + hierarchy = net_shaper_hierarchy_rcu(binding); + if (!hierarchy) + goto out_unlock; + for (; (shaper = xa_find(&hierarchy->shapers, &ctx->start_index, U32_MAX, XA_PRESENT)); ctx->start_index++) { ret = net_shaper_fill_one(skb, binding, shaper, info); if (ret) break; } +out_unlock: rcu_read_unlock(); return ret; @@ -810,45 +859,38 @@ int net_shaper_nl_set_doit(struct sk_buff *skb, struct genl_info *info) binding = net_shaper_binding_from_ctx(info->ctx); - net_shaper_lock(binding); ret = net_shaper_parse_info(binding, info->attrs, info, &shaper, &exists); if (ret) - goto unlock; + return ret; if (!exists) net_shaper_default_parent(&shaper.handle, &shaper.parent); hierarchy = net_shaper_hierarchy_setup(binding); - if (!hierarchy) { - ret = -ENOMEM; - goto unlock; - } + if (!hierarchy) + return -ENOMEM; /* The 'set' operation can't create node-scope shapers. */ handle = shaper.handle; if (handle.scope == NET_SHAPER_SCOPE_NODE && - !net_shaper_lookup(binding, &handle)) { - ret = -ENOENT; - goto unlock; - } + !net_shaper_lookup(binding, &handle)) + return -ENOENT; ret = net_shaper_pre_insert(binding, &handle, info->extack); if (ret) - goto unlock; + return ret; ops = net_shaper_ops(binding); ret = ops->set(binding, &shaper, info->extack); if (ret) { net_shaper_rollback(binding); - goto unlock; + return ret; } net_shaper_commit(binding, 1, &shaper); -unlock: - net_shaper_unlock(binding); - return ret; + return 0; } static int __net_shaper_delete(struct net_shaper_binding *binding, @@ -1077,35 +1119,26 @@ int net_shaper_nl_delete_doit(struct sk_buff *skb, struct genl_info *info) binding = net_shaper_binding_from_ctx(info->ctx); - net_shaper_lock(binding); ret = net_shaper_parse_handle(info->attrs[NET_SHAPER_A_HANDLE], info, &handle); if (ret) - goto unlock; + return ret; hierarchy = net_shaper_hierarchy(binding); - if (!hierarchy) { - ret = -ENOENT; - goto unlock; - } + if (!hierarchy) + return -ENOENT; shaper = net_shaper_lookup(binding, &handle); - if (!shaper) { - ret = -ENOENT; - goto unlock; - } + if (!shaper) + return -ENOENT; if (handle.scope == NET_SHAPER_SCOPE_NODE) { ret = net_shaper_pre_del_node(binding, shaper, info->extack); if (ret) - goto unlock; + return ret; } - ret = __net_shaper_delete(binding, shaper, info->extack); - -unlock: - net_shaper_unlock(binding); - return ret; + return __net_shaper_delete(binding, shaper, info->extack); } static int net_shaper_group_send_reply(struct net_shaper_binding *binding, @@ -1154,21 +1187,17 @@ int net_shaper_nl_group_doit(struct sk_buff *skb, struct genl_info *info) if (!net_shaper_ops(binding)->group) return -EOPNOTSUPP; - net_shaper_lock(binding); leaves_count = net_shaper_list_len(info, NET_SHAPER_A_LEAVES); if (!leaves_count) { NL_SET_BAD_ATTR(info->extack, info->attrs[NET_SHAPER_A_LEAVES]); - ret = -EINVAL; - goto unlock; + return -EINVAL; } leaves = kcalloc(leaves_count, sizeof(struct net_shaper) + sizeof(struct net_shaper *), GFP_KERNEL); - if (!leaves) { - ret = -ENOMEM; - goto unlock; - } + if (!leaves) + return -ENOMEM; old_nodes = (void *)&leaves[leaves_count]; ret = net_shaper_parse_node(binding, info->attrs, info, &node); @@ -1245,9 +1274,6 @@ int net_shaper_nl_group_doit(struct sk_buff *skb, struct genl_info *info) free_leaves: kfree(leaves); - -unlock: - net_shaper_unlock(binding); return ret; free_msg: @@ -1314,10 +1340,7 @@ int net_shaper_nl_cap_get_doit(struct sk_buff *skb, struct genl_info *info) if (ret) goto free_msg; - ret = genlmsg_reply(msg, info); - if (ret) - goto free_msg; - return 0; + return genlmsg_reply(msg, info); free_msg: nlmsg_free(msg); @@ -1360,14 +1383,12 @@ static void net_shaper_flush(struct net_shaper_binding *binding) if (!hierarchy) return; - net_shaper_lock(binding); xa_lock(&hierarchy->shapers); xa_for_each(&hierarchy->shapers, index, cur) { __xa_erase(&hierarchy->shapers, index); kfree(cur); } xa_unlock(&hierarchy->shapers); - net_shaper_unlock(binding); kfree(hierarchy); } diff --git a/net/shaper/shaper_nl_gen.c b/net/shaper/shaper_nl_gen.c index e8cccc4c118035..9b29be3ef19a85 100644 --- a/net/shaper/shaper_nl_gen.c +++ b/net/shaper/shaper_nl_gen.c @@ -99,27 +99,27 @@ static const struct genl_split_ops net_shaper_nl_ops[] = { }, { .cmd = NET_SHAPER_CMD_SET, - .pre_doit = net_shaper_nl_pre_doit, + .pre_doit = net_shaper_nl_pre_doit_write, .doit = net_shaper_nl_set_doit, - .post_doit = net_shaper_nl_post_doit, + .post_doit = net_shaper_nl_post_doit_write, .policy = net_shaper_set_nl_policy, .maxattr = NET_SHAPER_A_IFINDEX, .flags = GENL_ADMIN_PERM | GENL_CMD_CAP_DO, }, { .cmd = NET_SHAPER_CMD_DELETE, - .pre_doit = net_shaper_nl_pre_doit, + .pre_doit = net_shaper_nl_pre_doit_write, .doit = net_shaper_nl_delete_doit, - .post_doit = net_shaper_nl_post_doit, + .post_doit = net_shaper_nl_post_doit_write, .policy = net_shaper_delete_nl_policy, .maxattr = NET_SHAPER_A_IFINDEX, .flags = GENL_ADMIN_PERM | GENL_CMD_CAP_DO, }, { .cmd = NET_SHAPER_CMD_GROUP, - .pre_doit = net_shaper_nl_pre_doit, + .pre_doit = net_shaper_nl_pre_doit_write, .doit = net_shaper_nl_group_doit, - .post_doit = net_shaper_nl_post_doit, + .post_doit = net_shaper_nl_post_doit_write, .policy = net_shaper_group_nl_policy, .maxattr = NET_SHAPER_A_LEAVES, .flags = GENL_ADMIN_PERM | GENL_CMD_CAP_DO, diff --git a/net/shaper/shaper_nl_gen.h b/net/shaper/shaper_nl_gen.h index ec41c90431a4c9..42c46c52c77513 100644 --- a/net/shaper/shaper_nl_gen.h +++ b/net/shaper/shaper_nl_gen.h @@ -18,12 +18,17 @@ extern const struct nla_policy net_shaper_leaf_info_nl_policy[NET_SHAPER_A_WEIGH int net_shaper_nl_pre_doit(const struct genl_split_ops *ops, struct sk_buff *skb, struct genl_info *info); +int net_shaper_nl_pre_doit_write(const struct genl_split_ops *ops, + struct sk_buff *skb, struct genl_info *info); int net_shaper_nl_cap_pre_doit(const struct genl_split_ops *ops, struct sk_buff *skb, struct genl_info *info); void net_shaper_nl_post_doit(const struct genl_split_ops *ops, struct sk_buff *skb, struct genl_info *info); void +net_shaper_nl_post_doit_write(const struct genl_split_ops *ops, + struct sk_buff *skb, struct genl_info *info); +void net_shaper_nl_cap_post_doit(const struct genl_split_ops *ops, struct sk_buff *skb, struct genl_info *info); int net_shaper_nl_pre_dumpit(struct netlink_callback *cb); diff --git a/net/smc/af_smc.c b/net/smc/af_smc.c index f97f77b041d974..765f26aaca93de 100644 --- a/net/smc/af_smc.c +++ b/net/smc/af_smc.c @@ -124,12 +124,21 @@ static struct sock *smc_tcp_syn_recv_sock(const struct sock *sk, struct request_sock *req, struct dst_entry *dst, struct request_sock *req_unhash, - bool *own_req) + bool *own_req, + void (*opt_child_init)(struct sock *newsk, + const struct sock *sk)) { struct smc_sock *smc; struct sock *child; - smc = smc_clcsock_user_data(sk); + rcu_read_lock(); + smc = smc_clcsock_user_data_rcu(sk); + if (!smc || !refcount_inc_not_zero(&smc->sk.sk_refcnt)) { + rcu_read_unlock(); + smc = NULL; + goto drop; + } + rcu_read_unlock(); if (READ_ONCE(sk->sk_ack_backlog) + atomic_read(&smc->queued_smc_hs) > sk->sk_max_ack_backlog) @@ -142,7 +151,7 @@ static struct sock *smc_tcp_syn_recv_sock(const struct sock *sk, /* passthrough to original syn recv sock fct */ child = smc->ori_af_ops->syn_recv_sock(sk, skb, req, dst, req_unhash, - own_req); + own_req, opt_child_init); /* child must not inherit smc or its ops */ if (child) { rcu_assign_sk_user_data(child, NULL); @@ -151,11 +160,14 @@ static struct sock *smc_tcp_syn_recv_sock(const struct sock *sk, if (inet_csk(child)->icsk_af_ops == inet_csk(sk)->icsk_af_ops) inet_csk(child)->icsk_af_ops = smc->ori_af_ops; } + sock_put(&smc->sk); return child; drop: dst_release(dst); tcp_listendrop(sk); + if (smc) + sock_put(&smc->sk); return NULL; } @@ -252,7 +264,7 @@ static void smc_fback_restore_callbacks(struct smc_sock *smc) struct sock *clcsk = smc->clcsock->sk; write_lock_bh(&clcsk->sk_callback_lock); - clcsk->sk_user_data = NULL; + rcu_assign_sk_user_data(clcsk, NULL); smc_clcsock_restore_cb(&clcsk->sk_state_change, &smc->clcsk_state_change); smc_clcsock_restore_cb(&clcsk->sk_data_ready, &smc->clcsk_data_ready); @@ -900,7 +912,7 @@ static void smc_fback_replace_callbacks(struct smc_sock *smc) struct sock *clcsk = smc->clcsock->sk; write_lock_bh(&clcsk->sk_callback_lock); - clcsk->sk_user_data = (void *)((uintptr_t)smc | SK_USER_DATA_NOCOPY); + __rcu_assign_sk_user_data_with_flags(clcsk, smc, SK_USER_DATA_NOCOPY); smc_clcsock_replace_cb(&clcsk->sk_state_change, smc_fback_state_change, &smc->clcsk_state_change); @@ -2663,8 +2675,8 @@ int smc_listen(struct socket *sock, int backlog) * smc-specific sk_data_ready function */ write_lock_bh(&smc->clcsock->sk->sk_callback_lock); - smc->clcsock->sk->sk_user_data = - (void *)((uintptr_t)smc | SK_USER_DATA_NOCOPY); + __rcu_assign_sk_user_data_with_flags(smc->clcsock->sk, smc, + SK_USER_DATA_NOCOPY); smc_clcsock_replace_cb(&smc->clcsock->sk->sk_data_ready, smc_clcsock_data_ready, &smc->clcsk_data_ready); write_unlock_bh(&smc->clcsock->sk->sk_callback_lock); @@ -2685,10 +2697,11 @@ int smc_listen(struct socket *sock, int backlog) write_lock_bh(&smc->clcsock->sk->sk_callback_lock); smc_clcsock_restore_cb(&smc->clcsock->sk->sk_data_ready, &smc->clcsk_data_ready); - smc->clcsock->sk->sk_user_data = NULL; + rcu_assign_sk_user_data(smc->clcsock->sk, NULL); write_unlock_bh(&smc->clcsock->sk->sk_callback_lock); goto out; } + sock_set_flag(sk, SOCK_RCU_FREE); sk->sk_max_ack_backlog = backlog; sk->sk_ack_backlog = 0; sk->sk_state = SMC_LISTEN; @@ -3357,11 +3370,10 @@ int smc_create_clcsk(struct net *net, struct sock *sk, int family) return 0; } -static int __smc_create(struct net *net, struct socket *sock, int protocol, - int kern, struct socket *clcsock) +static int smc_create(struct net *net, struct socket *sock, int protocol, + int kern) { int family = (protocol == SMCPROTO_SMC6) ? PF_INET6 : PF_INET; - struct smc_sock *smc; struct sock *sk; int rc; @@ -3380,15 +3392,7 @@ static int __smc_create(struct net *net, struct socket *sock, int protocol, if (!sk) goto out; - /* create internal TCP socket for CLC handshake and fallback */ - smc = smc_sk(sk); - - rc = 0; - if (clcsock) - smc->clcsock = clcsock; - else - rc = smc_create_clcsk(net, sk, family); - + rc = smc_create_clcsk(net, sk, family); if (rc) { sk_common_release(sk); sock->sk = NULL; @@ -3397,76 +3401,12 @@ static int __smc_create(struct net *net, struct socket *sock, int protocol, return rc; } -static int smc_create(struct net *net, struct socket *sock, int protocol, - int kern) -{ - return __smc_create(net, sock, protocol, kern, NULL); -} - static const struct net_proto_family smc_sock_family_ops = { .family = PF_SMC, .owner = THIS_MODULE, .create = smc_create, }; -static int smc_ulp_init(struct sock *sk) -{ - struct socket *tcp = sk->sk_socket; - struct net *net = sock_net(sk); - struct socket *smcsock; - int protocol, ret; - - /* only TCP can be replaced */ - if (tcp->type != SOCK_STREAM || sk->sk_protocol != IPPROTO_TCP || - (sk->sk_family != AF_INET && sk->sk_family != AF_INET6)) - return -ESOCKTNOSUPPORT; - /* don't handle wq now */ - if (tcp->state != SS_UNCONNECTED || !tcp->file || tcp->wq.fasync_list) - return -ENOTCONN; - - if (sk->sk_family == AF_INET) - protocol = SMCPROTO_SMC; - else - protocol = SMCPROTO_SMC6; - - smcsock = sock_alloc(); - if (!smcsock) - return -ENFILE; - - smcsock->type = SOCK_STREAM; - __module_get(THIS_MODULE); /* tried in __tcp_ulp_find_autoload */ - ret = __smc_create(net, smcsock, protocol, 1, tcp); - if (ret) { - sock_release(smcsock); /* module_put() which ops won't be NULL */ - return ret; - } - - /* replace tcp socket to smc */ - smcsock->file = tcp->file; - smcsock->file->private_data = smcsock; - smcsock->file->f_inode = SOCK_INODE(smcsock); /* replace inode when sock_close */ - smcsock->file->f_path.dentry->d_inode = SOCK_INODE(smcsock); /* dput() in __fput */ - tcp->file = NULL; - - return ret; -} - -static void smc_ulp_clone(const struct request_sock *req, struct sock *newsk, - const gfp_t priority) -{ - struct inet_connection_sock *icsk = inet_csk(newsk); - - /* don't inherit ulp ops to child when listen */ - icsk->icsk_ulp_ops = NULL; -} - -static struct tcp_ulp_ops smc_ulp_ops __read_mostly = { - .name = "smc", - .owner = THIS_MODULE, - .init = smc_ulp_init, - .clone = smc_ulp_clone, -}; - unsigned int smc_net_id; static __net_init int smc_net_init(struct net *net) @@ -3589,16 +3529,10 @@ static int __init smc_init(void) pr_err("%s: ib_register fails with %d\n", __func__, rc); goto out_sock; } - - rc = tcp_register_ulp(&smc_ulp_ops); - if (rc) { - pr_err("%s: tcp_ulp_register fails with %d\n", __func__, rc); - goto out_ib; - } rc = smc_inet_init(); if (rc) { pr_err("%s: smc_inet_init fails with %d\n", __func__, rc); - goto out_ulp; + goto out_ib; } rc = bpf_smc_hs_ctrl_init(); if (rc) { @@ -3610,8 +3544,6 @@ static int __init smc_init(void) return 0; out_inet: smc_inet_exit(); -out_ulp: - tcp_unregister_ulp(&smc_ulp_ops); out_ib: smc_ib_unregister_client(); out_sock: @@ -3647,7 +3579,6 @@ static void __exit smc_exit(void) { static_branch_disable(&tcp_have_smc); smc_inet_exit(); - tcp_unregister_ulp(&smc_ulp_ops); sock_unregister(PF_SMC); smc_core_exit(); smc_ib_unregister_client(); @@ -3672,7 +3603,6 @@ MODULE_AUTHOR("Ursula Braun "); MODULE_DESCRIPTION("smc socket address family"); MODULE_LICENSE("GPL"); MODULE_ALIAS_NETPROTO(PF_SMC); -MODULE_ALIAS_TCP_ULP("smc"); /* 256 for IPPROTO_SMC and 1 for SOCK_STREAM */ MODULE_ALIAS_NET_PF_PROTO_TYPE(PF_INET, 256, 1); #if IS_ENABLED(CONFIG_IPV6) diff --git a/net/smc/smc.h b/net/smc/smc.h index 9e6af72784baa8..52145df83f6e7b 100644 --- a/net/smc/smc.h +++ b/net/smc/smc.h @@ -346,6 +346,11 @@ static inline struct smc_sock *smc_clcsock_user_data(const struct sock *clcsk) ((uintptr_t)clcsk->sk_user_data & ~SK_USER_DATA_NOCOPY); } +static inline struct smc_sock *smc_clcsock_user_data_rcu(const struct sock *clcsk) +{ + return (struct smc_sock *)rcu_dereference_sk_user_data(clcsk); +} + /* save target_cb in saved_cb, and replace target_cb with new_cb */ static inline void smc_clcsock_replace_cb(void (**target_cb)(struct sock *), void (*new_cb)(struct sock *), diff --git a/net/smc/smc_close.c b/net/smc/smc_close.c index 10219f55aad14d..bb0313ef5f7c1b 100644 --- a/net/smc/smc_close.c +++ b/net/smc/smc_close.c @@ -218,7 +218,7 @@ int smc_close_active(struct smc_sock *smc) write_lock_bh(&smc->clcsock->sk->sk_callback_lock); smc_clcsock_restore_cb(&smc->clcsock->sk->sk_data_ready, &smc->clcsk_data_ready); - smc->clcsock->sk->sk_user_data = NULL; + rcu_assign_sk_user_data(smc->clcsock->sk, NULL); write_unlock_bh(&smc->clcsock->sk->sk_callback_lock); rc = kernel_sock_shutdown(smc->clcsock, SHUT_RDWR); } diff --git a/net/smc/smc_rx.c b/net/smc/smc_rx.c index e7f1134453ef40..4a3d7b405132e8 100644 --- a/net/smc/smc_rx.c +++ b/net/smc/smc_rx.c @@ -135,9 +135,16 @@ static void smc_rx_pipe_buf_release(struct pipe_inode_info *pipe, sock_put(sk); } +static bool smc_rx_pipe_buf_get(struct pipe_inode_info *pipe, + struct pipe_buffer *buf) +{ + /* smc_spd_priv in buf->private is not shareable; disallow cloning. */ + return false; +} + static const struct pipe_buf_operations smc_pipe_ops = { .release = smc_rx_pipe_buf_release, - .get = generic_pipe_buf_get + .get = smc_rx_pipe_buf_get, }; static void smc_rx_spd_release(struct splice_pipe_desc *spd, diff --git a/net/socket.c b/net/socket.c index 136b98c54fb375..05952188127f5b 100644 --- a/net/socket.c +++ b/net/socket.c @@ -674,7 +674,7 @@ static void __sock_release(struct socket *sock, struct inode *inode) iput(SOCK_INODE(sock)); return; } - sock->file = NULL; + WRITE_ONCE(sock->file, NULL); } /** diff --git a/net/sunrpc/auth_gss/auth_gss.c b/net/sunrpc/auth_gss/auth_gss.c index 5c095cb8cb201e..bb3c3db2713b1f 100644 --- a/net/sunrpc/auth_gss/auth_gss.c +++ b/net/sunrpc/auth_gss/auth_gss.c @@ -39,6 +39,8 @@ static const struct rpc_authops authgss_ops; static const struct rpc_credops gss_credops; static const struct rpc_credops gss_nullops; +static void gss_free_callback(struct kref *kref); + #define GSS_RETRY_EXPIRED 5 static unsigned int gss_expired_cred_retry_delay = GSS_RETRY_EXPIRED; @@ -551,6 +553,7 @@ gss_alloc_msg(struct gss_auth *gss_auth, } return gss_msg; err_put_pipe_version: + kref_put(&gss_auth->kref, gss_free_callback); put_pipe_version(gss_auth->net); err_free_msg: kfree(gss_msg); diff --git a/net/sunrpc/auth_gss/gss_rpc_xdr.c b/net/sunrpc/auth_gss/gss_rpc_xdr.c index 7d2cdc2bd374e2..f320c0a8e60493 100644 --- a/net/sunrpc/auth_gss/gss_rpc_xdr.c +++ b/net/sunrpc/auth_gss/gss_rpc_xdr.c @@ -320,29 +320,47 @@ static int gssx_dec_status(struct xdr_stream *xdr, /* status->minor_status */ p = xdr_inline_decode(xdr, 8); - if (unlikely(p == NULL)) - return -ENOSPC; + if (unlikely(p == NULL)) { + err = -ENOSPC; + goto out_free_mech; + } p = xdr_decode_hyper(p, &status->minor_status); /* status->major_status_string */ err = gssx_dec_buffer(xdr, &status->major_status_string); if (err) - return err; + goto out_free_mech; /* status->minor_status_string */ err = gssx_dec_buffer(xdr, &status->minor_status_string); if (err) - return err; + goto out_free_major_status_string; /* status->server_ctx */ err = gssx_dec_buffer(xdr, &status->server_ctx); if (err) - return err; + goto out_free_minor_status_string; /* we assume we have no options for now, so simply consume them */ /* status->options */ err = dummy_dec_opt_array(xdr, &status->options); + if (err) + goto out_free_server_ctx; + return 0; + +out_free_server_ctx: + kfree(status->server_ctx.data); + status->server_ctx.data = NULL; +out_free_minor_status_string: + kfree(status->minor_status_string.data); + status->minor_status_string.data = NULL; +out_free_major_status_string: + kfree(status->major_status_string.data); + status->major_status_string.data = NULL; +out_free_mech: + kfree(status->mech.data); + status->mech.data = NULL; return err; } @@ -505,28 +523,35 @@ static int gssx_dec_name(struct xdr_stream *xdr, /* name->name_type */ err = gssx_dec_buffer(xdr, &dummy_netobj); if (err) - return err; + goto out_free_display_name; /* name->exported_name */ err = gssx_dec_buffer(xdr, &dummy_netobj); if (err) - return err; + goto out_free_display_name; /* name->exported_composite_name */ err = gssx_dec_buffer(xdr, &dummy_netobj); if (err) - return err; + goto out_free_display_name; /* we assume we have no attributes for now, so simply consume them */ /* name->name_attributes */ err = dummy_dec_nameattr_array(xdr, &dummy_name_attr_array); if (err) - return err; + goto out_free_display_name; /* we assume we have no options for now, so simply consume them */ /* name->extensions */ err = dummy_dec_opt_array(xdr, &dummy_option_array); + if (err) + goto out_free_display_name; + return 0; + +out_free_display_name: + kfree(name->display_name.data); + name->display_name.data = NULL; return err; } @@ -649,32 +674,34 @@ static int gssx_dec_ctx(struct xdr_stream *xdr, /* ctx->state */ err = gssx_dec_buffer(xdr, &ctx->state); if (err) - return err; + goto out_free_exported_context_token; /* ctx->need_release */ err = gssx_dec_bool(xdr, &ctx->need_release); if (err) - return err; + goto out_free_state; /* ctx->mech */ err = gssx_dec_buffer(xdr, &ctx->mech); if (err) - return err; + goto out_free_state; /* ctx->src_name */ err = gssx_dec_name(xdr, &ctx->src_name); if (err) - return err; + goto out_free_mech; /* ctx->targ_name */ err = gssx_dec_name(xdr, &ctx->targ_name); if (err) - return err; + goto out_free_src_name; /* ctx->lifetime */ p = xdr_inline_decode(xdr, 8+8); - if (unlikely(p == NULL)) - return -ENOSPC; + if (unlikely(p == NULL)) { + err = -ENOSPC; + goto out_free_targ_name; + } p = xdr_decode_hyper(p, &ctx->lifetime); /* ctx->ctx_flags */ @@ -683,17 +710,36 @@ static int gssx_dec_ctx(struct xdr_stream *xdr, /* ctx->locally_initiated */ err = gssx_dec_bool(xdr, &ctx->locally_initiated); if (err) - return err; + goto out_free_targ_name; /* ctx->open */ err = gssx_dec_bool(xdr, &ctx->open); if (err) - return err; + goto out_free_targ_name; /* we assume we have no options for now, so simply consume them */ /* ctx->options */ err = dummy_dec_opt_array(xdr, &ctx->options); + if (err) + goto out_free_targ_name; + + return 0; +out_free_targ_name: + kfree(ctx->targ_name.display_name.data); + ctx->targ_name.display_name.data = NULL; +out_free_src_name: + kfree(ctx->src_name.display_name.data); + ctx->src_name.display_name.data = NULL; +out_free_mech: + kfree(ctx->mech.data); + ctx->mech.data = NULL; +out_free_state: + kfree(ctx->state.data); + ctx->state.data = NULL; +out_free_exported_context_token: + kfree(ctx->exported_context_token.data); + ctx->exported_context_token.data = NULL; return err; } diff --git a/net/sunrpc/cache.c b/net/sunrpc/cache.c index 131090f31e6a83..6f6e0d4928afd6 100644 --- a/net/sunrpc/cache.c +++ b/net/sunrpc/cache.c @@ -1061,14 +1061,25 @@ static int cache_release(struct inode *inode, struct file *filp, struct cache_reader *rp = filp->private_data; if (rp) { + struct cache_request *rq = NULL; + spin_lock(&queue_lock); if (rp->offset) { struct cache_queue *cq; - for (cq= &rp->q; &cq->list != &cd->queue; - cq = list_entry(cq->list.next, struct cache_queue, list)) + for (cq = &rp->q; &cq->list != &cd->queue; + cq = list_entry(cq->list.next, + struct cache_queue, list)) if (!cq->reader) { - container_of(cq, struct cache_request, q) - ->readers--; + struct cache_request *cr = + container_of(cq, + struct cache_request, q); + cr->readers--; + if (cr->readers == 0 && + !test_bit(CACHE_PENDING, + &cr->item->flags)) { + list_del(&cr->q.list); + rq = cr; + } break; } rp->offset = 0; @@ -1076,9 +1087,14 @@ static int cache_release(struct inode *inode, struct file *filp, list_del(&rp->q.list); spin_unlock(&queue_lock); + if (rq) { + cache_put(rq->item, cd); + kfree(rq->buf); + kfree(rq); + } + filp->private_data = NULL; kfree(rp); - } if (filp->f_mode & FMODE_WRITE) { atomic_dec(&cd->writers); diff --git a/net/sunrpc/xprtrdma/svc_rdma_transport.c b/net/sunrpc/xprtrdma/svc_rdma_transport.c index b7b318ad25c42c..9b623849723ed0 100644 --- a/net/sunrpc/xprtrdma/svc_rdma_transport.c +++ b/net/sunrpc/xprtrdma/svc_rdma_transport.c @@ -462,7 +462,10 @@ static struct svc_xprt *svc_rdma_accept(struct svc_xprt *xprt) newxprt->sc_max_bc_requests = 2; } - /* Arbitrary estimate of the needed number of rdma_rw contexts. + /* Estimate the needed number of rdma_rw contexts. The maximum + * Read and Write chunks have one segment each. Each request + * can involve one Read chunk and either a Write chunk or Reply + * chunk; thus a factor of three. */ maxpayload = min(xprt->xpt_server->sv_max_payload, RPCSVC_MAXPAYLOAD_RDMA); @@ -470,7 +473,8 @@ static struct svc_xprt *svc_rdma_accept(struct svc_xprt *xprt) rdma_rw_mr_factor(dev, newxprt->sc_port_num, maxpayload >> PAGE_SHIFT); - newxprt->sc_sq_depth = rq_depth + ctxts; + newxprt->sc_sq_depth = rq_depth + + rdma_rw_max_send_wr(dev, newxprt->sc_port_num, ctxts, 0); if (newxprt->sc_sq_depth > dev->attrs.max_qp_wr) newxprt->sc_sq_depth = dev->attrs.max_qp_wr; atomic_set(&newxprt->sc_sq_avail, newxprt->sc_sq_depth); diff --git a/net/sunrpc/xprtrdma/verbs.c b/net/sunrpc/xprtrdma/verbs.c index 63262ef0c2e3a9..8abbd9c4045a49 100644 --- a/net/sunrpc/xprtrdma/verbs.c +++ b/net/sunrpc/xprtrdma/verbs.c @@ -1362,7 +1362,7 @@ void rpcrdma_post_recvs(struct rpcrdma_xprt *r_xprt, int needed) needed += RPCRDMA_MAX_RECV_BATCH; if (atomic_inc_return(&ep->re_receiving) > 1) - goto out; + goto out_dec; /* fast path: all needed reps can be found on the free list */ wr = NULL; @@ -1385,7 +1385,7 @@ void rpcrdma_post_recvs(struct rpcrdma_xprt *r_xprt, int needed) ++count; } if (!wr) - goto out; + goto out_dec; rc = ib_post_recv(ep->re_id->qp, wr, (const struct ib_recv_wr **)&bad_wr); @@ -1400,9 +1400,10 @@ void rpcrdma_post_recvs(struct rpcrdma_xprt *r_xprt, int needed) --count; } } + +out_dec: if (atomic_dec_return(&ep->re_receiving) > 0) complete(&ep->re_done); - out: trace_xprtrdma_post_recvs(r_xprt, count); ep->re_receive_count += count; diff --git a/net/tipc/crypto.c b/net/tipc/crypto.c index 970db62bd029b2..a3f9ca28c3d536 100644 --- a/net/tipc/crypto.c +++ b/net/tipc/crypto.c @@ -460,7 +460,7 @@ static void tipc_aead_users_dec(struct tipc_aead __rcu *aead, int lim) rcu_read_lock(); tmp = rcu_dereference(aead); if (tmp) - atomic_add_unless(&rcu_dereference(aead)->users, -1, lim); + atomic_add_unless(&tmp->users, -1, lim); rcu_read_unlock(); } diff --git a/net/tipc/group.c b/net/tipc/group.c index 3e137d8c9d2ff5..215f2a7d84588a 100644 --- a/net/tipc/group.c +++ b/net/tipc/group.c @@ -746,6 +746,7 @@ void tipc_group_proto_rcv(struct tipc_group *grp, bool *usr_wakeup, u32 port = msg_origport(hdr); struct tipc_member *m, *pm; u16 remitted, in_flight; + u16 acked; if (!grp) return; @@ -798,7 +799,10 @@ void tipc_group_proto_rcv(struct tipc_group *grp, bool *usr_wakeup, case GRP_ACK_MSG: if (!m) return; - m->bc_acked = msg_grp_bc_acked(hdr); + acked = msg_grp_bc_acked(hdr); + if (less_eq(acked, m->bc_acked)) + return; + m->bc_acked = acked; if (--grp->bc_ackers) return; list_del_init(&m->small_win); diff --git a/net/tipc/name_table.c b/net/tipc/name_table.c index e74940eab3a479..7f42fb6a8481fe 100644 --- a/net/tipc/name_table.c +++ b/net/tipc/name_table.c @@ -348,7 +348,8 @@ static bool tipc_service_insert_publ(struct net *net, /* Return if the publication already exists */ list_for_each_entry(_p, &sr->all_publ, all_publ) { - if (_p->key == key && (!_p->sk.node || _p->sk.node == node)) { + if (_p->key == key && _p->sk.ref == p->sk.ref && + (!_p->sk.node || _p->sk.node == node)) { pr_debug("Failed to bind duplicate %u,%u,%u/%u:%u/%u\n", p->sr.type, p->sr.lower, p->sr.upper, node, p->sk.ref, key); @@ -388,7 +389,8 @@ static struct publication *tipc_service_remove_publ(struct service_range *r, u32 node = sk->node; list_for_each_entry(p, &r->all_publ, all_publ) { - if (p->key != key || (node && node != p->sk.node)) + if (p->key != key || p->sk.ref != sk->ref || + (node && node != p->sk.node)) continue; list_del(&p->all_publ); list_del(&p->local_publ); diff --git a/net/tipc/socket.c b/net/tipc/socket.c index 817b07d95a9148..a7e6f5a9fd51eb 100644 --- a/net/tipc/socket.c +++ b/net/tipc/socket.c @@ -2233,6 +2233,8 @@ static bool tipc_sk_filter_connect(struct tipc_sock *tsk, struct sk_buff *skb, if (skb_queue_empty(&sk->sk_write_queue)) break; get_random_bytes(&delay, 2); + if (tsk->conn_timeout < 4) + tsk->conn_timeout = 4; delay %= (tsk->conn_timeout / 4); delay = msecs_to_jiffies(delay + 100); sk_reset_timer(sk, &sk->sk_timer, jiffies + delay); diff --git a/net/tls/tls_sw.c b/net/tls/tls_sw.c index 9937d4c810f2bd..7f10c18a6eea26 100644 --- a/net/tls/tls_sw.c +++ b/net/tls/tls_sw.c @@ -246,6 +246,7 @@ static int tls_decrypt_async_wait(struct tls_sw_context_rx *ctx) crypto_wait_req(-EINPROGRESS, &ctx->async_wait); atomic_inc(&ctx->decrypt_pending); + __skb_queue_purge(&ctx->async_hold); return ctx->async_wait.err; } @@ -583,6 +584,16 @@ static int tls_do_encryption(struct sock *sk, if (rc == -EBUSY) { rc = tls_encrypt_async_wait(ctx); rc = rc ?: -EINPROGRESS; + /* + * The async callback tls_encrypt_done() has already + * decremented encrypt_pending and restored the sge on + * both success and error. Skip the synchronous cleanup + * below on error, just remove the record and return. + */ + if (rc != -EINPROGRESS) { + list_del(&rec->list); + return rc; + } } if (!rc || rc != -EINPROGRESS) { atomic_dec(&ctx->encrypt_pending); @@ -2225,7 +2236,6 @@ int tls_sw_recvmsg(struct sock *sk, /* Wait for all previously submitted records to be decrypted */ ret = tls_decrypt_async_wait(ctx); - __skb_queue_purge(&ctx->async_hold); if (ret) { if (err >= 0 || err == -EINPROGRESS) @@ -2533,7 +2543,7 @@ void tls_sw_cancel_work_tx(struct tls_context *tls_ctx) set_bit(BIT_TX_CLOSING, &ctx->tx_bitmask); set_bit(BIT_TX_SCHEDULED, &ctx->tx_bitmask); - cancel_delayed_work_sync(&ctx->tx_work.work); + disable_delayed_work_sync(&ctx->tx_work.work); } void tls_sw_release_resources_tx(struct sock *sk) diff --git a/net/unix/af_unix.c b/net/unix/af_unix.c index d0511225799ba3..3db79e83d2114d 100644 --- a/net/unix/af_unix.c +++ b/net/unix/af_unix.c @@ -1650,10 +1650,9 @@ static int unix_stream_connect(struct socket *sock, struct sockaddr_unsized *uad timeo = sock_sndtimeo(sk, flags & O_NONBLOCK); - /* First of all allocate resources. - * If we will make it after state is locked, - * we will have to recheck all again in any case. - */ + err = prepare_peercred(&peercred); + if (err) + goto out; /* create new sock for complete connection */ newsk = unix_create1(net, NULL, 0, sock->type); @@ -1662,10 +1661,6 @@ static int unix_stream_connect(struct socket *sock, struct sockaddr_unsized *uad goto out; } - err = prepare_peercred(&peercred); - if (err) - goto out; - /* Allocate skb for sending to listening sock */ skb = sock_wmalloc(newsk, 1, 0, GFP_KERNEL); if (!skb) { @@ -1790,7 +1785,7 @@ static int unix_stream_connect(struct socket *sock, struct sockaddr_unsized *uad __skb_queue_tail(&other->sk_receive_queue, skb); spin_unlock(&other->sk_receive_queue.lock); unix_state_unlock(other); - other->sk_data_ready(other); + READ_ONCE(other->sk_data_ready)(other); sock_put(other); return 0; @@ -1963,6 +1958,8 @@ static void unix_detach_fds(struct scm_cookie *scm, struct sk_buff *skb) static void unix_peek_fds(struct scm_cookie *scm, struct sk_buff *skb) { scm->fp = scm_fp_dup(UNIXCB(skb).fp); + + unix_peek_fpl(scm->fp); } static void unix_destruct_scm(struct sk_buff *skb) @@ -2283,7 +2280,7 @@ static int unix_dgram_sendmsg(struct socket *sock, struct msghdr *msg, scm_stat_add(other, skb); skb_queue_tail(&other->sk_receive_queue, skb); unix_state_unlock(other); - other->sk_data_ready(other); + READ_ONCE(other->sk_data_ready)(other); sock_put(other); scm_destroy(&scm); return len; @@ -2356,7 +2353,7 @@ static int queue_oob(struct sock *sk, struct msghdr *msg, struct sock *other, sk_send_sigurg(other); unix_state_unlock(other); - other->sk_data_ready(other); + READ_ONCE(other->sk_data_ready)(other); return 0; out_unlock: @@ -2482,7 +2479,7 @@ static int unix_stream_sendmsg(struct socket *sock, struct msghdr *msg, spin_unlock(&other->sk_receive_queue.lock); unix_state_unlock(other); - other->sk_data_ready(other); + READ_ONCE(other->sk_data_ready)(other); sent += size; } diff --git a/net/unix/af_unix.h b/net/unix/af_unix.h index c4f1b2da363def..8119dbeef3a3c6 100644 --- a/net/unix/af_unix.h +++ b/net/unix/af_unix.h @@ -29,6 +29,7 @@ void unix_del_edges(struct scm_fp_list *fpl); void unix_update_edges(struct unix_sock *receiver); int unix_prepare_fpl(struct scm_fp_list *fpl); void unix_destroy_fpl(struct scm_fp_list *fpl); +void unix_peek_fpl(struct scm_fp_list *fpl); void unix_schedule_gc(struct user_struct *user); /* SOCK_DIAG */ diff --git a/net/unix/diag.c b/net/unix/diag.c index ca34730261510c..c9c1e51c441969 100644 --- a/net/unix/diag.c +++ b/net/unix/diag.c @@ -28,18 +28,23 @@ static int sk_diag_dump_name(struct sock *sk, struct sk_buff *nlskb) static int sk_diag_dump_vfs(struct sock *sk, struct sk_buff *nlskb) { - struct dentry *dentry = unix_sk(sk)->path.dentry; + struct unix_diag_vfs uv; + struct dentry *dentry; + bool have_vfs = false; + unix_state_lock(sk); + dentry = unix_sk(sk)->path.dentry; if (dentry) { - struct unix_diag_vfs uv = { - .udiag_vfs_ino = d_backing_inode(dentry)->i_ino, - .udiag_vfs_dev = dentry->d_sb->s_dev, - }; - - return nla_put(nlskb, UNIX_DIAG_VFS, sizeof(uv), &uv); + uv.udiag_vfs_ino = d_backing_inode(dentry)->i_ino; + uv.udiag_vfs_dev = dentry->d_sb->s_dev; + have_vfs = true; } + unix_state_unlock(sk); - return 0; + if (!have_vfs) + return 0; + + return nla_put(nlskb, UNIX_DIAG_VFS, sizeof(uv), &uv); } static int sk_diag_dump_peer(struct sock *sk, struct sk_buff *nlskb) diff --git a/net/unix/garbage.c b/net/unix/garbage.c index 25f65817faab93..aaa5f5bf51cad5 100644 --- a/net/unix/garbage.c +++ b/net/unix/garbage.c @@ -318,6 +318,25 @@ void unix_destroy_fpl(struct scm_fp_list *fpl) unix_free_vertices(fpl); } +static bool gc_in_progress; +static seqcount_t unix_peek_seq = SEQCNT_ZERO(unix_peek_seq); + +void unix_peek_fpl(struct scm_fp_list *fpl) +{ + static DEFINE_SPINLOCK(unix_peek_lock); + + if (!fpl || !fpl->count_unix) + return; + + if (!READ_ONCE(gc_in_progress)) + return; + + /* Invalidate the final refcnt check in unix_vertex_dead(). */ + spin_lock(&unix_peek_lock); + raw_write_seqcount_barrier(&unix_peek_seq); + spin_unlock(&unix_peek_lock); +} + static bool unix_vertex_dead(struct unix_vertex *vertex) { struct unix_edge *edge; @@ -351,6 +370,36 @@ static bool unix_vertex_dead(struct unix_vertex *vertex) return true; } +static LIST_HEAD(unix_visited_vertices); +static unsigned long unix_vertex_grouped_index = UNIX_VERTEX_INDEX_MARK2; + +static bool unix_scc_dead(struct list_head *scc, bool fast) +{ + struct unix_vertex *vertex; + bool scc_dead = true; + unsigned int seq; + + seq = read_seqcount_begin(&unix_peek_seq); + + list_for_each_entry_reverse(vertex, scc, scc_entry) { + /* Don't restart DFS from this vertex. */ + list_move_tail(&vertex->entry, &unix_visited_vertices); + + /* Mark vertex as off-stack for __unix_walk_scc(). */ + if (!fast) + vertex->index = unix_vertex_grouped_index; + + if (scc_dead) + scc_dead = unix_vertex_dead(vertex); + } + + /* If MSG_PEEK intervened, defer this SCC to the next round. */ + if (read_seqcount_retry(&unix_peek_seq, seq)) + return false; + + return scc_dead; +} + static void unix_collect_skb(struct list_head *scc, struct sk_buff_head *hitlist) { struct unix_vertex *vertex; @@ -404,9 +453,6 @@ static bool unix_scc_cyclic(struct list_head *scc) return false; } -static LIST_HEAD(unix_visited_vertices); -static unsigned long unix_vertex_grouped_index = UNIX_VERTEX_INDEX_MARK2; - static unsigned long __unix_walk_scc(struct unix_vertex *vertex, unsigned long *last_index, struct sk_buff_head *hitlist) @@ -474,9 +520,7 @@ static unsigned long __unix_walk_scc(struct unix_vertex *vertex, } if (vertex->index == vertex->scc_index) { - struct unix_vertex *v; struct list_head scc; - bool scc_dead = true; /* SCC finalised. * @@ -485,18 +529,7 @@ static unsigned long __unix_walk_scc(struct unix_vertex *vertex, */ __list_cut_position(&scc, &vertex_stack, &vertex->scc_entry); - list_for_each_entry_reverse(v, &scc, scc_entry) { - /* Don't restart DFS from this vertex in unix_walk_scc(). */ - list_move_tail(&v->entry, &unix_visited_vertices); - - /* Mark vertex as off-stack. */ - v->index = unix_vertex_grouped_index; - - if (scc_dead) - scc_dead = unix_vertex_dead(v); - } - - if (scc_dead) { + if (unix_scc_dead(&scc, false)) { unix_collect_skb(&scc, hitlist); } else { if (unix_vertex_max_scc_index < vertex->scc_index) @@ -550,19 +583,11 @@ static void unix_walk_scc_fast(struct sk_buff_head *hitlist) while (!list_empty(&unix_unvisited_vertices)) { struct unix_vertex *vertex; struct list_head scc; - bool scc_dead = true; vertex = list_first_entry(&unix_unvisited_vertices, typeof(*vertex), entry); list_add(&scc, &vertex->scc_entry); - list_for_each_entry_reverse(vertex, &scc, scc_entry) { - list_move_tail(&vertex->entry, &unix_visited_vertices); - - if (scc_dead) - scc_dead = unix_vertex_dead(vertex); - } - - if (scc_dead) { + if (unix_scc_dead(&scc, true)) { cyclic_sccs--; unix_collect_skb(&scc, hitlist); } @@ -577,8 +602,6 @@ static void unix_walk_scc_fast(struct sk_buff_head *hitlist) cyclic_sccs ? UNIX_GRAPH_CYCLIC : UNIX_GRAPH_NOT_CYCLIC); } -static bool gc_in_progress; - static void unix_gc(struct work_struct *work) { struct sk_buff_head hitlist; diff --git a/net/vmw_vsock/vmci_transport.c b/net/vmw_vsock/vmci_transport.c index 7eccd6708d6649..aca3132689cf18 100644 --- a/net/vmw_vsock/vmci_transport.c +++ b/net/vmw_vsock/vmci_transport.c @@ -161,7 +161,7 @@ vmci_transport_packet_init(struct vmci_transport_packet *pkt, case VMCI_TRANSPORT_PACKET_TYPE_WAITING_READ: case VMCI_TRANSPORT_PACKET_TYPE_WAITING_WRITE: - memcpy(&pkt->u.wait, wait, sizeof(pkt->u.wait)); + pkt->u.wait = *wait; break; case VMCI_TRANSPORT_PACKET_TYPE_REQUEST2: diff --git a/net/wireless/core.c b/net/wireless/core.c index 9a420d627d3ce0..381e329e02a4cf 100644 --- a/net/wireless/core.c +++ b/net/wireless/core.c @@ -661,12 +661,8 @@ int wiphy_verify_iface_combinations(struct wiphy *wiphy, c->limits[j].max > 1)) return -EINVAL; - /* Only a single NAN can be allowed, avoid this - * check for multi-radio global combination, since it - * hold the capabilities of all radio combinations. - */ - if (!combined_radio && - WARN_ON(types & BIT(NL80211_IFTYPE_NAN) && + /* Only a single NAN can be allowed */ + if (WARN_ON(types & BIT(NL80211_IFTYPE_NAN) && c->limits[j].max > 1)) return -EINVAL; @@ -1214,6 +1210,7 @@ void wiphy_unregister(struct wiphy *wiphy) /* this has nothing to do now but make sure it's gone */ cancel_work_sync(&rdev->wiphy_work); + cancel_work_sync(&rdev->rfkill_block); cancel_work_sync(&rdev->conn_work); flush_work(&rdev->event_work); cancel_delayed_work_sync(&rdev->dfs_update_channels_wk); @@ -1415,8 +1412,10 @@ void cfg80211_leave(struct cfg80211_registered_device *rdev, cfg80211_leave_ocb(rdev, dev); break; case NL80211_IFTYPE_P2P_DEVICE: + cfg80211_stop_p2p_device(rdev, wdev); + break; case NL80211_IFTYPE_NAN: - /* cannot happen, has no netdev */ + cfg80211_stop_nan(rdev, wdev); break; case NL80211_IFTYPE_AP_VLAN: case NL80211_IFTYPE_MONITOR: diff --git a/net/wireless/pmsr.c b/net/wireless/pmsr.c index a117f5093ca29f..13801cf35e9fca 100644 --- a/net/wireless/pmsr.c +++ b/net/wireless/pmsr.c @@ -647,6 +647,7 @@ void cfg80211_pmsr_wdev_down(struct wireless_dev *wdev) } spin_unlock_bh(&wdev->pmsr_lock); + cancel_work_sync(&wdev->pmsr_free_wk); if (found) cfg80211_pmsr_process_abort(wdev); diff --git a/net/wireless/radiotap.c b/net/wireless/radiotap.c index 326faea38ca381..c85eaa583a4666 100644 --- a/net/wireless/radiotap.c +++ b/net/wireless/radiotap.c @@ -239,14 +239,14 @@ int ieee80211_radiotap_iterator_next( default: if (!iterator->current_namespace || iterator->_arg_index >= iterator->current_namespace->n_bits) { - if (iterator->current_namespace == &radiotap_ns) - return -ENOENT; align = 0; } else { align = iterator->current_namespace->align_size[iterator->_arg_index].align; size = iterator->current_namespace->align_size[iterator->_arg_index].size; } if (!align) { + if (iterator->current_namespace == &radiotap_ns) + return -ENOENT; /* skip all subsequent data */ iterator->_arg = iterator->_next_ns_data; /* give up on this namespace */ diff --git a/net/wireless/scan.c b/net/wireless/scan.c index 7546647752fd8f..eb0e77813d466f 100644 --- a/net/wireless/scan.c +++ b/net/wireless/scan.c @@ -1959,7 +1959,7 @@ cfg80211_update_known_bss(struct cfg80211_registered_device *rdev, ether_addr_copy(known->parent_bssid, new->parent_bssid); known->pub.max_bssid_indicator = new->pub.max_bssid_indicator; known->pub.bssid_index = new->pub.bssid_index; - known->pub.use_for &= new->pub.use_for; + known->pub.use_for = new->pub.use_for; known->pub.cannot_use_reasons = new->pub.cannot_use_reasons; known->bss_source = new->bss_source; diff --git a/net/wireless/wext-compat.c b/net/wireless/wext-compat.c index 1241fda78a68c4..680500fa57cfdf 100644 --- a/net/wireless/wext-compat.c +++ b/net/wireless/wext-compat.c @@ -684,7 +684,7 @@ static int cfg80211_wext_siwencodeext(struct net_device *dev, idx = erq->flags & IW_ENCODE_INDEX; if (cipher == WLAN_CIPHER_SUITE_AES_CMAC) { - if (idx < 4 || idx > 5) { + if (idx < 5 || idx > 6) { idx = wdev->wext.default_mgmt_key; if (idx < 0) return -EINVAL; diff --git a/net/x25/x25_in.c b/net/x25/x25_in.c index b981a4828d08c2..e47ebd8acd21bf 100644 --- a/net/x25/x25_in.c +++ b/net/x25/x25_in.c @@ -34,6 +34,10 @@ static int x25_queue_rx_frame(struct sock *sk, struct sk_buff *skb, int more) struct sk_buff *skbo, *skbn = skb; struct x25_sock *x25 = x25_sk(sk); + /* make sure we don't overflow */ + if (x25->fraglen + skb->len > USHRT_MAX) + return 1; + if (more) { x25->fraglen += skb->len; skb_queue_tail(&x25->fragment_queue, skb); @@ -44,10 +48,9 @@ static int x25_queue_rx_frame(struct sock *sk, struct sk_buff *skb, int more) if (x25->fraglen > 0) { /* End of fragment */ int len = x25->fraglen + skb->len; - if ((skbn = alloc_skb(len, GFP_ATOMIC)) == NULL){ - kfree_skb(skb); + skbn = alloc_skb(len, GFP_ATOMIC); + if (!skbn) return 1; - } skb_queue_tail(&x25->fragment_queue, skb); diff --git a/net/x25/x25_subr.c b/net/x25/x25_subr.c index 0285aaa1e93c17..159708d9ad20cb 100644 --- a/net/x25/x25_subr.c +++ b/net/x25/x25_subr.c @@ -40,6 +40,7 @@ void x25_clear_queues(struct sock *sk) skb_queue_purge(&x25->interrupt_in_queue); skb_queue_purge(&x25->interrupt_out_queue); skb_queue_purge(&x25->fragment_queue); + x25->fraglen = 0; } diff --git a/net/xdp/xdp_umem.c b/net/xdp/xdp_umem.c index 9f76ca591d54fb..9ec7bd948acc74 100644 --- a/net/xdp/xdp_umem.c +++ b/net/xdp/xdp_umem.c @@ -202,7 +202,8 @@ static int xdp_umem_reg(struct xdp_umem *umem, struct xdp_umem_reg *mr) if (!unaligned_chunks && chunks_rem) return -EINVAL; - if (headroom >= chunk_size - XDP_PACKET_HEADROOM) + if (headroom > chunk_size - XDP_PACKET_HEADROOM - + SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) - 128) return -EINVAL; if (mr->flags & XDP_UMEM_TX_METADATA_LEN) { diff --git a/net/xdp/xsk.c b/net/xdp/xsk.c index f093c3453f64ce..4a1cc44ab305af 100644 --- a/net/xdp/xsk.c +++ b/net/xdp/xsk.c @@ -167,26 +167,32 @@ static int xsk_rcv_zc(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len) struct xdp_buff_xsk *pos, *tmp; struct list_head *xskb_list; u32 contd = 0; + u32 num_desc; int err; - if (frags) - contd = XDP_PKT_CONTD; + if (likely(!frags)) { + err = __xsk_rcv_zc(xs, xskb, len, contd); + if (err) + goto err; + return 0; + } - err = __xsk_rcv_zc(xs, xskb, len, contd); - if (err) + contd = XDP_PKT_CONTD; + num_desc = xdp_get_shared_info_from_buff(xdp)->nr_frags + 1; + if (xskq_prod_nb_free(xs->rx, num_desc) < num_desc) { + xs->rx_queue_full++; + err = -ENOBUFS; goto err; - if (likely(!frags)) - return 0; + } + __xsk_rcv_zc(xs, xskb, len, contd); xskb_list = &xskb->pool->xskb_list; list_for_each_entry_safe(pos, tmp, xskb_list, list_node) { if (list_is_singular(xskb_list)) contd = 0; len = pos->xdp.data_end - pos->xdp.data; - err = __xsk_rcv_zc(xs, pos, len, contd); - if (err) - goto err; - list_del(&pos->list_node); + __xsk_rcv_zc(xs, pos, len, contd); + list_del_init(&pos->list_node); } return 0; @@ -233,7 +239,7 @@ static u32 xsk_copy_xdp(void *to, void **from, u32 to_len, static int __xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len) { - u32 frame_size = xsk_pool_get_rx_frame_size(xs->pool); + u32 frame_size = __xsk_pool_get_rx_frame_size(xs->pool); void *copy_from = xsk_copy_xdp_start(xdp), *copy_to; u32 from_len, meta_len, rem, num_desc; struct xdp_buff_xsk *xskb; @@ -332,7 +338,7 @@ static int xsk_rcv_check(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len) if (xs->dev != xdp->rxq->dev || xs->queue_id != xdp->rxq->queue_index) return -EINVAL; - if (len > xsk_pool_get_rx_frame_size(xs->pool) && !xs->sg) { + if (len > __xsk_pool_get_rx_frame_size(xs->pool) && !xs->sg) { xs->rx_dropped++; return -ENOSPC; } diff --git a/net/xdp/xsk_buff_pool.c b/net/xdp/xsk_buff_pool.c index 51526034c42acb..1f96bdf1e7a607 100644 --- a/net/xdp/xsk_buff_pool.c +++ b/net/xdp/xsk_buff_pool.c @@ -10,6 +10,8 @@ #include "xdp_umem.h" #include "xsk.h" +#define ETH_PAD_LEN (ETH_HLEN + 2 * VLAN_HLEN + ETH_FCS_LEN) + void xp_add_xsk(struct xsk_buff_pool *pool, struct xdp_sock *xs) { if (!xs->tx) @@ -158,8 +160,12 @@ static void xp_disable_drv_zc(struct xsk_buff_pool *pool) int xp_assign_dev(struct xsk_buff_pool *pool, struct net_device *netdev, u16 queue_id, u16 flags) { + u32 needed = netdev->mtu + ETH_PAD_LEN; + u32 segs = netdev->xdp_zc_max_segs; + bool mbuf = flags & XDP_USE_SG; bool force_zc, force_copy; struct netdev_bpf bpf; + u32 frame_size; int err = 0; ASSERT_RTNL(); @@ -179,7 +185,7 @@ int xp_assign_dev(struct xsk_buff_pool *pool, if (err) return err; - if (flags & XDP_USE_SG) + if (mbuf) pool->umem->flags |= XDP_UMEM_SG_FLAG; if (flags & XDP_USE_NEED_WAKEUP) @@ -201,8 +207,24 @@ int xp_assign_dev(struct xsk_buff_pool *pool, goto err_unreg_pool; } - if (netdev->xdp_zc_max_segs == 1 && (flags & XDP_USE_SG)) { - err = -EOPNOTSUPP; + if (mbuf) { + if (segs == 1) { + err = -EOPNOTSUPP; + goto err_unreg_pool; + } + } else { + segs = 1; + } + + /* open-code xsk_pool_get_rx_frame_size() as pool->dev is not + * set yet at this point; we are before getting down to driver + */ + frame_size = __xsk_pool_get_rx_frame_size(pool) - + xsk_pool_get_tailroom(mbuf); + frame_size = ALIGN_DOWN(frame_size, 128); + + if (needed > frame_size * segs) { + err = -EINVAL; goto err_unreg_pool; } @@ -252,6 +274,10 @@ int xp_assign_dev_shared(struct xsk_buff_pool *pool, struct xdp_sock *umem_xs, return -EINVAL; flags = umem->zc ? XDP_ZEROCOPY : XDP_COPY; + + if (umem->flags & XDP_UMEM_SG_FLAG) + flags |= XDP_USE_SG; + if (umem_xs->pool->uses_need_wakeup) flags |= XDP_USE_NEED_WAKEUP; diff --git a/net/xfrm/espintcp.c b/net/xfrm/espintcp.c index bf744ac9d5a73e..8709df716e98ea 100644 --- a/net/xfrm/espintcp.c +++ b/net/xfrm/espintcp.c @@ -536,7 +536,7 @@ static void espintcp_close(struct sock *sk, long timeout) sk->sk_prot = &tcp_prot; barrier(); - cancel_work_sync(&ctx->work); + disable_work_sync(&ctx->work); strp_done(&ctx->strp); skb_queue_purge(&ctx->out_queue); diff --git a/net/xfrm/xfrm_device.c b/net/xfrm/xfrm_device.c index 52ae0e034d29e2..550457e4c4f01d 100644 --- a/net/xfrm/xfrm_device.c +++ b/net/xfrm/xfrm_device.c @@ -544,6 +544,14 @@ static int xfrm_dev_down(struct net_device *dev) return NOTIFY_DONE; } +static int xfrm_dev_unregister(struct net_device *dev) +{ + xfrm_dev_state_flush(dev_net(dev), dev, true); + xfrm_dev_policy_flush(dev_net(dev), dev, true); + + return NOTIFY_DONE; +} + static int xfrm_dev_event(struct notifier_block *this, unsigned long event, void *ptr) { struct net_device *dev = netdev_notifier_info_to_dev(ptr); @@ -556,8 +564,10 @@ static int xfrm_dev_event(struct notifier_block *this, unsigned long event, void return xfrm_api_check(dev); case NETDEV_DOWN: - case NETDEV_UNREGISTER: return xfrm_dev_down(dev); + + case NETDEV_UNREGISTER: + return xfrm_dev_unregister(dev); } return NOTIFY_DONE; } diff --git a/net/xfrm/xfrm_input.c b/net/xfrm/xfrm_input.c index 4ed346e682c7eb..5d49323695eb73 100644 --- a/net/xfrm/xfrm_input.c +++ b/net/xfrm/xfrm_input.c @@ -503,7 +503,6 @@ int xfrm_input(struct sk_buff *skb, int nexthdr, __be32 spi, int encap_type) /* An encap_type of -1 indicates async resumption. */ if (encap_type == -1) { async = 1; - dev_put(skb->dev); seq = XFRM_SKB_CB(skb)->seq.input.low; spin_lock(&x->lock); goto resume; @@ -656,8 +655,11 @@ int xfrm_input(struct sk_buff *skb, int nexthdr, __be32 spi, int encap_type) dev_hold(skb->dev); nexthdr = x->type->input(x, skb); - if (nexthdr == -EINPROGRESS) + if (nexthdr == -EINPROGRESS) { + if (async) + dev_put(skb->dev); return 0; + } dev_put(skb->dev); spin_lock(&x->lock); @@ -692,9 +694,11 @@ int xfrm_input(struct sk_buff *skb, int nexthdr, __be32 spi, int encap_type) XFRM_MODE_SKB_CB(skb)->protocol = nexthdr; err = xfrm_inner_mode_input(x, skb); - if (err == -EINPROGRESS) + if (err == -EINPROGRESS) { + if (async) + dev_put(skb->dev); return 0; - else if (err) { + } else if (err) { XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEMODEERROR); goto drop; } @@ -731,6 +735,8 @@ int xfrm_input(struct sk_buff *skb, int nexthdr, __be32 spi, int encap_type) sp->olen = 0; if (skb_valid_dst(skb)) skb_dst_drop(skb); + if (async) + dev_put(skb->dev); gro_cells_receive(&gro_cells, skb); return 0; } else { @@ -750,6 +756,8 @@ int xfrm_input(struct sk_buff *skb, int nexthdr, __be32 spi, int encap_type) sp->olen = 0; if (skb_valid_dst(skb)) skb_dst_drop(skb); + if (async) + dev_put(skb->dev); gro_cells_receive(&gro_cells, skb); return err; } @@ -760,6 +768,8 @@ int xfrm_input(struct sk_buff *skb, int nexthdr, __be32 spi, int encap_type) drop_unlock: spin_unlock(&x->lock); drop: + if (async) + dev_put(skb->dev); xfrm_rcv_cb(skb, family, x && x->type ? x->type->proto : nexthdr, -1); kfree_skb(skb); return 0; diff --git a/net/xfrm/xfrm_iptfs.c b/net/xfrm/xfrm_iptfs.c index 3b6d7284fc70a3..7cd97c1dcd1178 100644 --- a/net/xfrm/xfrm_iptfs.c +++ b/net/xfrm/xfrm_iptfs.c @@ -901,6 +901,12 @@ static u32 iptfs_reassem_cont(struct xfrm_iptfs_data *xtfs, u64 seq, iptfs_skb_can_add_frags(newskb, fragwalk, data, copylen)) { iptfs_skb_add_frags(newskb, fragwalk, data, copylen); } else { + if (skb_linearize(newskb)) { + XFRM_INC_STATS(xs_net(xtfs->x), + LINUX_MIB_XFRMINBUFFERERROR); + goto abandon; + } + /* copy fragment data into newskb */ if (skb_copy_seq_read(st, data, skb_put(newskb, copylen), copylen)) { @@ -991,6 +997,11 @@ static bool __input_process_payload(struct xfrm_state *x, u32 data, iplen = be16_to_cpu(iph->tot_len); iphlen = iph->ihl << 2; + if (iplen < iphlen || iphlen < sizeof(*iph)) { + XFRM_INC_STATS(net, + LINUX_MIB_XFRMINHDRERROR); + goto done; + } protocol = cpu_to_be16(ETH_P_IP); XFRM_MODE_SKB_CB(skbseq->root_skb)->tos = iph->tos; } else if (iph->version == 0x6) { @@ -2653,9 +2664,6 @@ static int iptfs_clone_state(struct xfrm_state *x, struct xfrm_state *orig) if (!xtfs) return -ENOMEM; - x->mode_data = xtfs; - xtfs->x = x; - xtfs->ra_newskb = NULL; if (xtfs->cfg.reorder_win_size) { xtfs->w_saved = kcalloc(xtfs->cfg.reorder_win_size, @@ -2666,6 +2674,9 @@ static int iptfs_clone_state(struct xfrm_state *x, struct xfrm_state *orig) } } + x->mode_data = xtfs; + xtfs->x = x; + return 0; } diff --git a/net/xfrm/xfrm_nat_keepalive.c b/net/xfrm/xfrm_nat_keepalive.c index ebf95d48e86c14..1856beee0149bb 100644 --- a/net/xfrm/xfrm_nat_keepalive.c +++ b/net/xfrm/xfrm_nat_keepalive.c @@ -261,7 +261,7 @@ int __net_init xfrm_nat_keepalive_net_init(struct net *net) int xfrm_nat_keepalive_net_fini(struct net *net) { - cancel_delayed_work_sync(&net->xfrm.nat_keepalive_work); + disable_delayed_work_sync(&net->xfrm.nat_keepalive_work); return 0; } diff --git a/net/xfrm/xfrm_policy.c b/net/xfrm/xfrm_policy.c index 62486f86697521..29c94ee0ceb256 100644 --- a/net/xfrm/xfrm_policy.c +++ b/net/xfrm/xfrm_policy.c @@ -3801,8 +3801,8 @@ int __xfrm_policy_check(struct sock *sk, int dir, struct sk_buff *skb, struct xfrm_tmpl *tp[XFRM_MAX_DEPTH]; struct xfrm_tmpl *stp[XFRM_MAX_DEPTH]; struct xfrm_tmpl **tpp = tp; + int i, k = 0; int ti = 0; - int i, k; sp = skb_sec_path(skb); if (!sp) @@ -3828,6 +3828,12 @@ int __xfrm_policy_check(struct sock *sk, int dir, struct sk_buff *skb, tpp = stp; } + if (pol->xdo.type == XFRM_DEV_OFFLOAD_PACKET && sp == &dummy) + /* This policy template was already checked by HW + * and secpath was removed in __xfrm_policy_check2. + */ + goto out; + /* For each tunnel xfrm, find the first matching tmpl. * For each tmpl before that, find corresponding xfrm. * Order is _important_. Later we will implement @@ -3837,7 +3843,7 @@ int __xfrm_policy_check(struct sock *sk, int dir, struct sk_buff *skb, * verified to allow them to be skipped in future policy * checks (e.g. nested tunnels). */ - for (i = xfrm_nr-1, k = 0; i >= 0; i--) { + for (i = xfrm_nr - 1; i >= 0; i--) { k = xfrm_policy_ok(tpp[i], sp, k, family, if_id); if (k < 0) { if (k < -1) @@ -3853,6 +3859,7 @@ int __xfrm_policy_check(struct sock *sk, int dir, struct sk_buff *skb, goto reject; } +out: xfrm_pols_put(pols, npols); sp->verified_cnt = k; @@ -4275,12 +4282,16 @@ static void xfrm_policy_fini(struct net *net) unsigned int sz; int dir; + disable_work_sync(&net->xfrm.policy_hthresh.work); + flush_work(&net->xfrm.policy_hash_work); #ifdef CONFIG_XFRM_SUB_POLICY xfrm_policy_flush(net, XFRM_POLICY_TYPE_SUB, false); #endif xfrm_policy_flush(net, XFRM_POLICY_TYPE_MAIN, false); + synchronize_rcu(); + WARN_ON(!list_empty(&net->xfrm.policy_all)); for (dir = 0; dir < XFRM_POLICY_MAX; dir++) { @@ -4517,9 +4528,6 @@ static struct xfrm_policy *xfrm_migrate_policy_find(const struct xfrm_selector * pol = xfrm_policy_lookup_bytype(net, type, &fl, sel->family, dir, if_id); if (IS_ERR_OR_NULL(pol)) goto out_unlock; - - if (!xfrm_pol_hold_rcu(pol)) - pol = NULL; out_unlock: rcu_read_unlock(); return pol; diff --git a/net/xfrm/xfrm_state.c b/net/xfrm/xfrm_state.c index 98b362d518363b..a00c4fe1ab0ce2 100644 --- a/net/xfrm/xfrm_state.c +++ b/net/xfrm/xfrm_state.c @@ -2264,6 +2264,7 @@ int xfrm_state_update(struct xfrm_state *x) err = 0; x->km.state = XFRM_STATE_DEAD; + xfrm_dev_state_delete(x); __xfrm_state_put(x); } diff --git a/net/xfrm/xfrm_user.c b/net/xfrm/xfrm_user.c index 403b5ecac2c544..b3f69c0760d4c6 100644 --- a/net/xfrm/xfrm_user.c +++ b/net/xfrm/xfrm_user.c @@ -1850,6 +1850,7 @@ static int xfrm_alloc_userspi(struct sk_buff *skb, struct nlmsghdr *nlh, pcpu_num = nla_get_u32(attrs[XFRMA_SA_PCPU]); if (pcpu_num >= num_possible_cpus()) { err = -EINVAL; + NL_SET_ERR_MSG(extack, "pCPU number too big"); goto out_noput; } } @@ -2667,7 +2668,8 @@ static inline unsigned int xfrm_aevent_msgsize(struct xfrm_state *x) + nla_total_size(4) /* XFRM_AE_RTHR */ + nla_total_size(4) /* XFRM_AE_ETHR */ + nla_total_size(sizeof(x->dir)) /* XFRMA_SA_DIR */ - + nla_total_size(4); /* XFRMA_SA_PCPU */ + + nla_total_size(4) /* XFRMA_SA_PCPU */ + + nla_total_size(sizeof(x->if_id)); /* XFRMA_IF_ID */ } static int build_aevent(struct sk_buff *skb, struct xfrm_state *x, const struct km_event *c) @@ -2779,7 +2781,12 @@ static int xfrm_get_ae(struct sk_buff *skb, struct nlmsghdr *nlh, c.portid = nlh->nlmsg_pid; err = build_aevent(r_skb, x, &c); - BUG_ON(err < 0); + if (err < 0) { + spin_unlock_bh(&x->lock); + xfrm_state_put(x); + kfree_skb(r_skb); + return err; + } err = nlmsg_unicast(net->xfrm.nlsk, r_skb, NETLINK_CB(skb).portid); spin_unlock_bh(&x->lock); @@ -3001,8 +3008,10 @@ static int xfrm_add_acquire(struct sk_buff *skb, struct nlmsghdr *nlh, if (attrs[XFRMA_SA_PCPU]) { x->pcpu_num = nla_get_u32(attrs[XFRMA_SA_PCPU]); err = -EINVAL; - if (x->pcpu_num >= num_possible_cpus()) + if (x->pcpu_num >= num_possible_cpus()) { + NL_SET_ERR_MSG(extack, "pCPU number too big"); goto free_state; + } } err = verify_newpolicy_info(&ua->policy, extack); @@ -3673,7 +3682,7 @@ static inline unsigned int xfrm_sa_len(struct xfrm_state *x) } if (x->if_id) l += nla_total_size(sizeof(x->if_id)); - if (x->pcpu_num) + if (x->pcpu_num != UINT_MAX) l += nla_total_size(sizeof(x->pcpu_num)); /* Must count x->lastused as it may become non-zero behind our back. */ @@ -3948,6 +3957,8 @@ static int build_polexpire(struct sk_buff *skb, struct xfrm_policy *xp, return err; } upe->hard = !!hard; + /* clear the padding bytes */ + memset_after(upe, 0, hard); nlmsg_end(skb, nlh); return 0; @@ -4105,6 +4116,7 @@ static int build_report(struct sk_buff *skb, u8 proto, return -EMSGSIZE; ur = nlmsg_data(nlh); + memset(ur, 0, sizeof(*ur)); ur->proto = proto; memcpy(&ur->sel, sel, sizeof(ur->sel)); @@ -4152,6 +4164,7 @@ static int build_mapping(struct sk_buff *skb, struct xfrm_state *x, um = nlmsg_data(nlh); + memset(&um->id, 0, sizeof(um->id)); memcpy(&um->id.daddr, &x->id.daddr, sizeof(um->id.daddr)); um->id.spi = x->id.spi; um->id.family = x->props.family; diff --git a/rust/Makefile b/rust/Makefile index 4dcc2eff51cb26..066c194c23b915 100644 --- a/rust/Makefile +++ b/rust/Makefile @@ -509,11 +509,9 @@ quiet_cmd_rustc_procmacrolibrary = $(RUSTC_OR_CLIPPY_QUIET) PL $@ cmd_rustc_procmacrolibrary = \ $(if $(skip_clippy),$(RUSTC),$(RUSTC_OR_CLIPPY)) \ $(filter-out $(skip_flags),$(rust_common_flags) $(rustc_target_flags)) \ - --emit=dep-info,link --crate-type rlib -O \ + --emit=dep-info=$(depfile) --emit=link=$@ --crate-type rlib -O \ --out-dir $(objtree)/$(obj) -L$(objtree)/$(obj) \ - --crate-name $(patsubst lib%.rlib,%,$(notdir $@)) $<; \ - mv $(objtree)/$(obj)/$(patsubst lib%.rlib,%,$(notdir $@)).d $(depfile); \ - sed -i '/^\#/d' $(depfile) + --crate-name $(patsubst lib%.rlib,%,$(notdir $@)) $< $(obj)/libproc_macro2.rlib: private skip_clippy = 1 $(obj)/libproc_macro2.rlib: private rustc_target_flags = $(proc_macro2-flags) @@ -552,6 +550,8 @@ $(obj)/$(libpin_init_internal_name): private rustc_target_flags = --cfg kernel $(obj)/$(libpin_init_internal_name): $(src)/pin-init/internal/src/lib.rs FORCE +$(call if_changed_dep,rustc_procmacro) +# `rustc` requires `-Zunstable-options` to use custom target specifications +# since Rust 1.95.0 (https://github.com/rust-lang/rust/pull/151534). quiet_cmd_rustc_library = $(if $(skip_clippy),RUSTC,$(RUSTC_OR_CLIPPY_QUIET)) L $@ cmd_rustc_library = \ OBJTREE=$(abspath $(objtree)) \ @@ -562,6 +562,7 @@ quiet_cmd_rustc_library = $(if $(skip_clippy),RUSTC,$(RUSTC_OR_CLIPPY_QUIET)) L --crate-type rlib -L$(objtree)/$(obj) \ --crate-name $(patsubst %.o,%,$(notdir $@)) $< \ --sysroot=/dev/null \ + -Zunstable-options \ $(if $(rustc_objcopy),;$(OBJCOPY) $(rustc_objcopy) $@) \ $(cmd_objtool) diff --git a/rust/bindgen_parameters b/rust/bindgen_parameters index fd2fd1c3cb9a51..eeb8c15e709c38 100644 --- a/rust/bindgen_parameters +++ b/rust/bindgen_parameters @@ -12,9 +12,15 @@ # Packed type cannot transitively contain a `#[repr(align)]` type. --opaque-type alt_instr +--opaque-type snd_codec_options +--opaque-type snd_codec +--opaque-type snd_compr_params --opaque-type x86_msi_data --opaque-type x86_msi_addr_lo +# Packed types cannot have larger alignment than the maximal natural aligment of menbers +--opaque-type snd_dec_flac + # `try` is a reserved keyword since Rust 2018; solved in `bindgen` v0.59.2, # commit 2aed6b021680 ("context: Escape the try keyword properly"). --opaque-type kunit_try_catch diff --git a/rust/bindings/bindings_helper.h b/rust/bindings/bindings_helper.h index a067038b4b422b..025dae9fc05534 100644 --- a/rust/bindings/bindings_helper.h +++ b/rust/bindings/bindings_helper.h @@ -31,9 +31,14 @@ #include #include #include +#include #include #include +#include +#include #include +#include +#include #include #include #include @@ -48,9 +53,13 @@ #include #include #include +#include #include #include +#include +#include #include +#include #include #include #include @@ -59,13 +68,20 @@ #include #include #include +#include +#include +#include #include +#include #include #include #include #include #include +#include +#include #include +#include #include #include #include @@ -80,12 +96,17 @@ #include #include #include +#include +#include #include #include #include #include #include #include +#include +#include +#include #include /* @@ -111,8 +132,16 @@ const gfp_t RUST_CONST_HELPER___GFP_ZERO = __GFP_ZERO; const gfp_t RUST_CONST_HELPER___GFP_HIGHMEM = ___GFP_HIGHMEM; const gfp_t RUST_CONST_HELPER___GFP_NOWARN = ___GFP_NOWARN; const blk_features_t RUST_CONST_HELPER_BLK_FEAT_ROTATIONAL = BLK_FEAT_ROTATIONAL; +const uint32_t RUST_CONST_HELPER_DRM_EXEC_INTERRUPTIBLE_WAIT = DRM_EXEC_INTERRUPTIBLE_WAIT; const fop_flags_t RUST_CONST_HELPER_FOP_UNSIGNED_OFFSET = FOP_UNSIGNED_OFFSET; +const u64 BINDINGS_SNDRV_PCM_FMTBIT_FLOAT_LE = SNDRV_PCM_FMTBIT_FLOAT_LE; + +const u32 BINDINGS_IIO_CHAN_INFO_RAW = IIO_CHAN_INFO_RAW; +const u32 BINDINGS_IIO_CHAN_INFO_PROCESSED = IIO_CHAN_INFO_PROCESSED; +const u32 BINDINGS_IIO_ANGL = IIO_ANGL; +const u32 BINDINGS_IIO_LIGHT = IIO_LIGHT; + const xa_mark_t RUST_CONST_HELPER_XA_PRESENT = XA_PRESENT; const gfp_t RUST_CONST_HELPER_XA_FLAGS_ALLOC = XA_FLAGS_ALLOC; diff --git a/rust/helpers/device.c b/rust/helpers/device.c index 9a4316bafedfbc..41c3f4df8909e8 100644 --- a/rust/helpers/device.c +++ b/rust/helpers/device.c @@ -25,3 +25,13 @@ void rust_helper_dev_set_drvdata(struct device *dev, void *data) { dev_set_drvdata(dev, data); } + +void rust_helper_device_lock(struct device *dev) +{ + device_lock(dev); +} + +void rust_helper_device_unlock(struct device *dev) +{ + device_unlock(dev); +} diff --git a/rust/helpers/dma-fence.c b/rust/helpers/dma-fence.c new file mode 100644 index 00000000000000..6491016262934b --- /dev/null +++ b/rust/helpers/dma-fence.c @@ -0,0 +1,33 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include + +#ifdef CONFIG_DMA_SHARED_BUFFER + +void rust_helper_dma_fence_get(struct dma_fence *fence) +{ + dma_fence_get(fence); +} + +void rust_helper_dma_fence_put(struct dma_fence *fence) +{ + dma_fence_put(fence); +} + +struct dma_fence_chain *rust_helper_dma_fence_chain_alloc(void) +{ + return dma_fence_chain_alloc(); +} + +void rust_helper_dma_fence_chain_free(struct dma_fence_chain *chain) +{ + dma_fence_chain_free(chain); +} + +void rust_helper_dma_fence_set_error(struct dma_fence *fence, int error) +{ + dma_fence_set_error(fence, error); +} + +#endif diff --git a/rust/helpers/dma-mapping.c b/rust/helpers/dma-mapping.c new file mode 100644 index 00000000000000..0d795b1b0738dc --- /dev/null +++ b/rust/helpers/dma-mapping.c @@ -0,0 +1,8 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +int rust_helper_dma_mapping_error(struct device *dev, dma_addr_t dma_addr) +{ + return dma_mapping_error(dev, dma_addr); +} diff --git a/rust/helpers/dma-resv.c b/rust/helpers/dma-resv.c new file mode 100644 index 00000000000000..05501cb814513b --- /dev/null +++ b/rust/helpers/dma-resv.c @@ -0,0 +1,13 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +int rust_helper_dma_resv_lock(struct dma_resv *obj, struct ww_acquire_ctx *ctx) +{ + return dma_resv_lock(obj, ctx); +} + +void rust_helper_dma_resv_unlock(struct dma_resv *obj) +{ + dma_resv_unlock(obj); +} diff --git a/rust/helpers/drm.c b/rust/helpers/drm.c index 450b406c6f2739..a4e997d0b47320 100644 --- a/rust/helpers/drm.c +++ b/rust/helpers/drm.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 #include +#include #include #ifdef CONFIG_DRM @@ -20,4 +21,49 @@ __u64 rust_helper_drm_vma_node_offset_addr(struct drm_vma_offset_node *node) return drm_vma_node_offset_addr(node); } -#endif +#ifdef CONFIG_DRM_GEM_SHMEM_HELPER +void rust_helper_drm_gem_shmem_object_free(struct drm_gem_object *obj) +{ + return drm_gem_shmem_object_free(obj); +} + +void rust_helper_drm_gem_shmem_object_print_info(struct drm_printer *p, unsigned int indent, + const struct drm_gem_object *obj) +{ + drm_gem_shmem_object_print_info(p, indent, obj); +} + +int rust_helper_drm_gem_shmem_object_pin(struct drm_gem_object *obj) +{ + return drm_gem_shmem_object_pin(obj); +} + +void rust_helper_drm_gem_shmem_object_unpin(struct drm_gem_object *obj) +{ + drm_gem_shmem_object_unpin(obj); +} + +struct sg_table *rust_helper_drm_gem_shmem_object_get_sg_table(struct drm_gem_object *obj) +{ + return drm_gem_shmem_object_get_sg_table(obj); +} + +int rust_helper_drm_gem_shmem_object_vmap(struct drm_gem_object *obj, + struct iosys_map *map) +{ + return drm_gem_shmem_object_vmap(obj, map); +} + +void rust_helper_drm_gem_shmem_object_vunmap(struct drm_gem_object *obj, + struct iosys_map *map) +{ + drm_gem_shmem_object_vunmap(obj, map); +} + +int rust_helper_drm_gem_shmem_object_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma) +{ + return drm_gem_shmem_object_mmap(obj, vma); +} + +#endif /* CONFIG_DRM_GEM_SHMEM_HELPER */ +#endif /* CONFIG_DRM */ diff --git a/rust/helpers/drm_gpuvm.c b/rust/helpers/drm_gpuvm.c new file mode 100644 index 00000000000000..1ed5b5841f68c2 --- /dev/null +++ b/rust/helpers/drm_gpuvm.c @@ -0,0 +1,39 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +#ifdef CONFIG_DRM +#ifdef CONFIG_DRM_GPUVM + +struct drm_gpuvm *rust_helper_drm_gpuvm_get(struct drm_gpuvm *obj) +{ + return drm_gpuvm_get(obj); +} + +void rust_helper_drm_gpuvm_exec_unlock(struct drm_gpuvm_exec *vm_exec) +{ + return drm_gpuvm_exec_unlock(vm_exec); +} + +void rust_helper_drm_gpuva_init_from_op(struct drm_gpuva *va, struct drm_gpuva_op_map *op) +{ + drm_gpuva_init_from_op(va, op); +} + +struct drm_gpuvm_bo *rust_helper_drm_gpuvm_bo_get(struct drm_gpuvm_bo *vm_bo) +{ + return drm_gpuvm_bo_get(vm_bo); +} + +bool rust_helper_drm_gpuvm_immediate_mode(struct drm_gpuvm *gpuvm) +{ + return drm_gpuvm_immediate_mode(gpuvm); +} + +bool rust_helper_drm_gpuvm_is_extobj(struct drm_gpuvm *gpuvm, struct drm_gem_object *obj) +{ + return drm_gpuvm_is_extobj(gpuvm, obj); +} + +#endif +#endif diff --git a/rust/helpers/drm_syncobj.c b/rust/helpers/drm_syncobj.c new file mode 100644 index 00000000000000..9e14c989edfd72 --- /dev/null +++ b/rust/helpers/drm_syncobj.c @@ -0,0 +1,22 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +#ifdef CONFIG_DRM + +void rust_helper_drm_syncobj_get(struct drm_syncobj *obj) +{ + drm_syncobj_get(obj); +} + +void rust_helper_drm_syncobj_put(struct drm_syncobj *obj) +{ + drm_syncobj_put(obj); +} + +struct dma_fence *rust_helper_drm_syncobj_fence_get(struct drm_syncobj *syncobj) +{ + return drm_syncobj_fence_get(syncobj); +} + +#endif diff --git a/rust/helpers/helpers.c b/rust/helpers/helpers.c index 79c72762ad9c4b..b663807262416a 100644 --- a/rust/helpers/helpers.c +++ b/rust/helpers/helpers.c @@ -25,11 +25,17 @@ #include "cred.c" #include "device.c" #include "dma.c" +#include "dma-fence.c" +#include "dma-mapping.c" +#include "dma-resv.c" #include "drm.c" +#include "drm_gpuvm.c" +#include "drm_syncobj.c" #include "err.c" #include "irq.c" #include "fs.c" #include "io.c" +#include "iosys_map.c" #include "jump_label.c" #include "kunit.c" #include "maple_tree.c" diff --git a/rust/helpers/io.c b/rust/helpers/io.c index c475913c69e647..6b1b05ab977b0c 100644 --- a/rust/helpers/io.c +++ b/rust/helpers/io.c @@ -18,6 +18,16 @@ void rust_helper_iounmap(void __iomem *addr) iounmap(addr); } +void rust_helper_memcpy_fromio(void *to, const void __iomem *from, long count) +{ + memcpy_fromio(to, from, count); +} + +void rust_helper_memcpy_toio(void __iomem *to, const void *from, size_t count) +{ + memcpy_toio(to, from, count); +} + u8 rust_helper_readb(const void __iomem *addr) { return readb(addr); diff --git a/rust/helpers/iosys_map.c b/rust/helpers/iosys_map.c new file mode 100644 index 00000000000000..f99598367147ff --- /dev/null +++ b/rust/helpers/iosys_map.c @@ -0,0 +1,21 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +void rust_helper_iosys_map_memcpy_to(struct iosys_map *dst, size_t dst_offset, + const void *src, size_t len) +{ + iosys_map_memcpy_to(dst, dst_offset, src, len); +} + +void rust_helper_iosys_map_memcpy_from(void *dst, const struct iosys_map *src, + size_t src_offset, size_t len) +{ + iosys_map_memcpy_from(dst, src, src_offset, len); +} + +void rust_helper_iosys_map_memset(struct iosys_map *dst, size_t offset, + int value, size_t len) +{ + iosys_map_memset(dst, offset, value, len); +} diff --git a/rust/helpers/of.c b/rust/helpers/of.c index 86b51167c913f9..8c3958795357c0 100644 --- a/rust/helpers/of.c +++ b/rust/helpers/of.c @@ -1,8 +1,29 @@ // SPDX-License-Identifier: GPL-2.0 #include +#include bool rust_helper_is_of_node(const struct fwnode_handle *fwnode) { return is_of_node(fwnode); } + +const struct of_device_id *rust_helper_of_match_device( + const struct of_device_id *matches, const struct device *dev) +{ + return of_match_device(matches, dev); +} + +#ifdef CONFIG_OF +bool rust_helper_of_node_is_root(const struct device_node *np) +{ + return of_node_is_root(np); +} +#endif + +struct device_node *rust_helper_of_parse_phandle(const struct device_node *np, + const char *phandle_name, + int index) +{ + return of_parse_phandle(np, phandle_name, index); +} diff --git a/rust/helpers/page.c b/rust/helpers/page.c index 7144de5a61dbdb..1e15bb4d525b59 100644 --- a/rust/helpers/page.c +++ b/rust/helpers/page.c @@ -1,5 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 +#include #include #include #include @@ -25,3 +26,28 @@ int rust_helper_page_to_nid(const struct page *page) return page_to_nid(page); } #endif + +struct page *rust_helper_phys_to_page(phys_addr_t phys) +{ + return phys_to_page(phys); +} + +phys_addr_t rust_helper_page_to_phys(struct page *page) +{ + return page_to_phys(page); +} + +unsigned long rust_helper_phys_to_pfn(phys_addr_t phys) +{ + return __phys_to_pfn(phys); +} + +struct page *rust_helper_pfn_to_page(unsigned long pfn) +{ + return pfn_to_page(pfn); +} + +bool rust_helper_pfn_valid(unsigned long pfn) +{ + return pfn_valid(pfn); +} diff --git a/rust/helpers/xarray.c b/rust/helpers/xarray.c index 60b299f11451d2..b6c078e6a343c2 100644 --- a/rust/helpers/xarray.c +++ b/rust/helpers/xarray.c @@ -2,6 +2,11 @@ #include +void *rust_helper_xa_zero_entry(void) +{ + return XA_ZERO_ENTRY; +} + int rust_helper_xa_err(void *entry) { return xa_err(entry); diff --git a/rust/kernel/addr.rs b/rust/kernel/addr.rs new file mode 100644 index 00000000000000..06aff10a033235 --- /dev/null +++ b/rust/kernel/addr.rs @@ -0,0 +1,15 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Kernel core address types. + +use bindings; +use core::ffi; + +/// A physical memory address (which may be wider than the CPU pointer size) +pub type PhysicalAddr = bindings::phys_addr_t; +/// A DMA memory address (which may be narrower than `PhysicalAddr` on some systems) +pub type DmaAddr = bindings::dma_addr_t; +/// A physical resource size, typically the same width as `PhysicalAddr` +pub type ResourceSize = bindings::resource_size_t; +/// A raw page frame number, not to be confused with the C `pfn_t` which also encodes flags. +pub type Pfn = ffi::c_ulong; diff --git a/rust/kernel/alloc.rs b/rust/kernel/alloc.rs index e38720349dcf7e..719927971f5796 100644 --- a/rust/kernel/alloc.rs +++ b/rust/kernel/alloc.rs @@ -18,6 +18,8 @@ pub use self::kvec::KVec; pub use self::kvec::VVec; pub use self::kvec::Vec; +use crate::types::declare_flags_type; + /// Indicates an allocation error. #[derive(Copy, Clone, PartialEq, Eq, Debug)] pub struct AllocError; @@ -25,45 +27,13 @@ pub struct AllocError; use crate::error::{code::EINVAL, Result}; use core::{alloc::Layout, ptr::NonNull}; -/// Flags to be used when allocating memory. -/// -/// They can be combined with the operators `|`, `&`, and `!`. -/// -/// Values can be used from the [`flags`] module. -#[derive(Clone, Copy, PartialEq)] -pub struct Flags(u32); - -impl Flags { - /// Get the raw representation of this flag. - pub(crate) fn as_raw(self) -> u32 { - self.0 - } - - /// Check whether `flags` is contained in `self`. - pub fn contains(self, flags: Flags) -> bool { - (self & flags) == flags - } -} - -impl core::ops::BitOr for Flags { - type Output = Self; - fn bitor(self, rhs: Self) -> Self::Output { - Self(self.0 | rhs.0) - } -} - -impl core::ops::BitAnd for Flags { - type Output = Self; - fn bitand(self, rhs: Self) -> Self::Output { - Self(self.0 & rhs.0) - } -} - -impl core::ops::Not for Flags { - type Output = Self; - fn not(self) -> Self::Output { - Self(!self.0) - } +declare_flags_type! { + /// Flags to be used when allocating memory. + /// + /// They can be combined with the operators `|`, `&`, and `!`. + /// + /// Values can be used from the [`flags`] module. + pub struct Flags(u32); } /// Allocation flags. diff --git a/rust/kernel/alloc/allocator.rs b/rust/kernel/alloc/allocator.rs index 63bfb91b36712a..1a53412c458dd3 100644 --- a/rust/kernel/alloc/allocator.rs +++ b/rust/kernel/alloc/allocator.rs @@ -1,4 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 +// FIXME +#![allow(clippy::undocumented_unsafe_blocks)] //! Allocator support. //! diff --git a/rust/kernel/alloc/kbox.rs b/rust/kernel/alloc/kbox.rs index 622b3529edfcbc..2684598cde4cbe 100644 --- a/rust/kernel/alloc/kbox.rs +++ b/rust/kernel/alloc/kbox.rs @@ -682,6 +682,16 @@ where } } +impl AsRef for Box +where + T: ?Sized, + A: Allocator, +{ + fn as_ref(&self) -> &T { + self + } +} + /// # Examples /// /// ``` diff --git a/rust/kernel/alloc/kvec.rs b/rust/kernel/alloc/kvec.rs index ac8d6f763ae81d..6cc48330515dd1 100644 --- a/rust/kernel/alloc/kvec.rs +++ b/rust/kernel/alloc/kvec.rs @@ -19,12 +19,15 @@ use core::{ ops::DerefMut, ops::Index, ops::IndexMut, + ops::{Range, RangeBounds}, ptr, ptr::NonNull, slice, slice::SliceIndex, }; +mod drain; +use self::drain::Drain; mod errors; pub use self::errors::{InsertError, PushError, RemoveError}; @@ -192,6 +195,19 @@ where self.len } + /// Forcefully sets `self.len` to `new_len`. + /// + /// # Safety + /// + /// - `new_len` must be less than or equal to [`Self::capacity`]. + /// - If `new_len` is greater than `self.len`, all elements within the interval + /// [`self.len`,`new_len`) must be initialized. + #[inline] + pub unsafe fn set_len(&mut self, new_len: usize) { + debug_assert!(new_len <= self.capacity()); + self.len = new_len; + } + /// Increments `self.len` by `additional`. /// /// # Safety @@ -733,6 +749,76 @@ where } self.truncate(num_kept); } + + /// Removes the specified range from the vector in bulk, returning all + /// removed elements as an iterator. If the iterator is dropped before + /// being fully consumed, it drops the remaining removed elements. + /// + /// The returned iterator keeps a mutable borrow on the vector to optimize + /// its implementation. + /// + /// # Panics + /// + /// Panics if the starting point is greater than the end point or if + /// the end point is greater than the length of the vector. + /// + /// # Leaking + /// + /// If the returned iterator goes out of scope without being dropped (due to + /// [`mem::forget`], for example), the vector may have lost and leaked + /// elements arbitrarily, including elements outside the range. + /// + /// # Examples + /// + /// ``` + /// let mut v = vec![1, 2, 3]; + /// let u: Vec<_> = v.drain(1..).collect(); + /// assert_eq!(v, &[1]); + /// assert_eq!(u, &[2, 3]); + /// + /// // A full range clears the vector, like `clear()` does + /// v.drain(..); + /// assert_eq!(v, &[]); + /// ``` + pub fn drain(&mut self, range: R) -> Drain<'_, T, A> + where + R: RangeBounds, + { + let len = self.len(); + let Range { start, end } = slice::range(range, ..len); + + unsafe { + // set self.vec length's to start, to be safe in case Drain is leaked + self.set_len(start); + let range_slice = slice::from_raw_parts(self.as_ptr().add(start), end - start); + Drain { + tail_start: end, + tail_len: len - end, + iter: range_slice.iter(), + vec: NonNull::from(self), + } + } + } + /// Removes an element from the vector and returns it. + /// + /// The removed element is replaced by the last element of the vector. + /// + /// This does not preserve ordering of the remaining elements, but is *O*(1). + /// If you need to preserve the element order, use [`remove`] instead. + pub fn swap_remove(&mut self, index: usize) -> T { + if index > self.len() { + panic!("Index out of range"); + } + // SAFETY: index is in range + // self.len() - 1 is in range since at last 1 element exists + unsafe { + let old = ptr::read(self.as_ptr().add(index)); + let last = ptr::read(self.as_ptr().add(self.len() - 1)); + ptr::write(self.as_mut_ptr().add(index), last); + self.dec_len(1); + old + } + } } impl Vec { @@ -1399,3 +1485,51 @@ mod tests { } } } + +// #[stable(feature = "array_try_from_vec", since = "1.48.0")] +impl TryFrom> for [T; N] { + type Error = Vec; + + /// Gets the entire contents of the `Vec` as an array, + /// if its size exactly matches that of the requested array. + /// + /// # Examples + /// + /// ``` + /// assert_eq!(vec![1, 2, 3].try_into(), Ok([1, 2, 3])); + /// assert_eq!(>::new().try_into(), Ok([])); + /// ``` + /// + /// If the length doesn't match, the input comes back in `Err`: + /// ``` + /// let r: Result<[i32; 4], _> = (0..10).collect::>().try_into(); + /// assert_eq!(r, Err(vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9])); + /// ``` + /// + /// If you're fine with just getting a prefix of the `Vec`, + /// you can call [`.truncate(N)`](Vec::truncate) first. + /// ``` + /// let mut v = String::from("hello world").into_bytes(); + /// v.sort(); + /// v.truncate(2); + /// let [a, b]: [_; 2] = v.try_into().unwrap(); + /// assert_eq!(a, b' '); + /// assert_eq!(b, b'd'); + /// ``` + fn try_from(mut vec: Vec) -> Result<[T; N], Vec> { + if vec.len() != N { + return Err(vec); + } + + // SAFETY: `.set_len(0)` is always sound. + unsafe { vec.dec_len(vec.len()) }; + + // SAFETY: A `Vec`'s pointer is always aligned properly, and + // the alignment the array needs is the same as the items. + // We checked earlier that we have sufficient items. + // The items will not double-drop as the `set_len` + // tells the `Vec` not to also drop them. + let array = unsafe { ptr::read(vec.as_ptr() as *const [T; N]) }; + Ok(array) + } +} diff --git a/rust/kernel/alloc/kvec/drain.rs b/rust/kernel/alloc/kvec/drain.rs new file mode 100644 index 00000000000000..035878fd112843 --- /dev/null +++ b/rust/kernel/alloc/kvec/drain.rs @@ -0,0 +1,181 @@ +//! Rust standard library vendored code. +//! +//! The contents of this file come from the Rust standard library, hosted in +//! the repository, licensed under +//! "Apache-2.0 OR MIT" and adapted for kernel use. For copyright details, +//! see . +#![allow(clippy::undocumented_unsafe_blocks)] + +use core::fmt; +use core::iter::FusedIterator; +use core::mem::{self, SizedTypeProperties}; +use core::ptr::{self, NonNull}; +use core::slice::{self}; + +use super::{Allocator, Vec}; + +/// A draining iterator for `Vec`. +/// +/// This `struct` is created by [`Vec::drain`]. +/// See its documentation for more. +/// +/// # Example +/// +/// ``` +/// let mut v = vec![0, 1, 2]; +/// let iter: std::vec::Drain<'_, _> = v.drain(..); +/// ``` +// #[stable(feature = "drain", since = "1.6.0")] +pub struct Drain< + 'a, + T, + A: Allocator, + // #[unstable(feature = "allocator_api", issue = "32838")] A: Allocator + 'a = Global, +> { + /// Index of tail to preserve + pub(super) tail_start: usize, + /// Length of tail + pub(super) tail_len: usize, + /// Current remaining range to remove + pub(super) iter: slice::Iter<'a, T>, + pub(super) vec: NonNull>, +} + +// #[stable(feature = "collection_debug", since = "1.17.0")] +impl fmt::Debug for Drain<'_, T, A> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_tuple("Drain").field(&self.iter.as_slice()).finish() + } +} + +impl<'a, T, A: Allocator> Drain<'a, T, A> { + /// Returns the remaining items of this iterator as a slice. + /// + /// # Examples + /// + /// ``` + /// let mut vec = vec!['a', 'b', 'c']; + /// let mut drain = vec.drain(..); + /// assert_eq!(drain.as_slice(), &['a', 'b', 'c']); + /// let _ = drain.next().unwrap(); + /// assert_eq!(drain.as_slice(), &['b', 'c']); + /// ``` + #[must_use] + // #[stable(feature = "vec_drain_as_slice", since = "1.46.0")] + pub fn as_slice(&self) -> &[T] { + self.iter.as_slice() + } +} + +// #[stable(feature = "vec_drain_as_slice", since = "1.46.0")] +impl<'a, T, A: Allocator> AsRef<[T]> for Drain<'a, T, A> { + fn as_ref(&self) -> &[T] { + self.as_slice() + } +} + +// #[stable(feature = "drain", since = "1.6.0")] +unsafe impl Sync for Drain<'_, T, A> {} +// #[stable(feature = "drain", since = "1.6.0")] +unsafe impl Send for Drain<'_, T, A> {} + +// #[stable(feature = "drain", since = "1.6.0")] +impl Iterator for Drain<'_, T, A> { + type Item = T; + + #[inline] + fn next(&mut self) -> Option { + self.iter + .next() + .map(|elt| unsafe { ptr::read(elt as *const _) }) + } + + fn size_hint(&self) -> (usize, Option) { + self.iter.size_hint() + } +} + +// #[stable(feature = "drain", since = "1.6.0")] +impl DoubleEndedIterator for Drain<'_, T, A> { + #[inline] + fn next_back(&mut self) -> Option { + self.iter + .next_back() + .map(|elt| unsafe { ptr::read(elt as *const _) }) + } +} + +// #[stable(feature = "drain", since = "1.6.0")] +impl Drop for Drain<'_, T, A> { + fn drop(&mut self) { + /// Moves back the un-`Drain`ed elements to restore the original `Vec`. + struct DropGuard<'r, 'a, T, A: Allocator>(&'r mut Drain<'a, T, A>); + + impl<'r, 'a, T, A: Allocator> Drop for DropGuard<'r, 'a, T, A> { + fn drop(&mut self) { + if self.0.tail_len > 0 { + unsafe { + let source_vec = self.0.vec.as_mut(); + // memmove back untouched tail, update to new length + let start = source_vec.len(); + let tail = self.0.tail_start; + if tail != start { + let src = source_vec.as_ptr().add(tail); + let dst = source_vec.as_mut_ptr().add(start); + ptr::copy(src, dst, self.0.tail_len); + } + source_vec.set_len(start + self.0.tail_len); + } + } + } + } + + let iter = mem::take(&mut self.iter); + let drop_len = iter.len(); + + let mut vec = self.vec; + + if T::IS_ZST { + // ZSTs have no identity, so we don't need to move them around, we only need to drop the correct amount. + // this can be achieved by manipulating the Vec length instead of moving values out from `iter`. + unsafe { + let vec = vec.as_mut(); + let old_len = vec.len(); + vec.set_len(old_len + drop_len + self.tail_len); + vec.truncate(old_len + self.tail_len); + } + + return; + } + + // ensure elements are moved back into their appropriate places, even when drop_in_place panics + let _guard = DropGuard(self); + + if drop_len == 0 { + return; + } + + // as_slice() must only be called when iter.len() is > 0 because + // it also gets touched by vec::Splice which may turn it into a dangling pointer + // which would make it and the vec pointer point to different allocations which would + // lead to invalid pointer arithmetic below. + let drop_ptr = iter.as_slice().as_ptr(); + + unsafe { + // drop_ptr comes from a slice::Iter which only gives us a &[T] but for drop_in_place + // a pointer with mutable provenance is necessary. Therefore we must reconstruct + // it from the original vec but also avoid creating a &mut to the front since that could + // invalidate raw pointers to it which some unsafe code might rely on. + let vec_ptr = vec.as_mut().as_mut_ptr(); + #[cfg(not(version("1.87")))] + let drop_offset = drop_ptr.sub_ptr(vec_ptr); + #[cfg(version("1.87"))] + let drop_offset = drop_ptr.offset_from_unsigned(vec_ptr); + let to_drop = ptr::slice_from_raw_parts_mut(vec_ptr.add(drop_offset), drop_len); + ptr::drop_in_place(to_drop); + } + } +} + +// #[stable(feature = "fused", since = "1.26.0")] +impl FusedIterator for Drain<'_, T, A> {} diff --git a/rust/kernel/cpufreq.rs b/rust/kernel/cpufreq.rs index f968fbd2289054..0879a79485f8e8 100644 --- a/rust/kernel/cpufreq.rs +++ b/rust/kernel/cpufreq.rs @@ -1015,6 +1015,8 @@ impl Registration { ..pin_init::zeroed() }; + // Always inline to optimize out error path of `build_assert`. + #[inline(always)] const fn copy_name(name: &'static CStr) -> [c_char; CPUFREQ_NAME_LEN] { let src = name.to_bytes_with_nul(); let mut dst = [0; CPUFREQ_NAME_LEN]; diff --git a/rust/kernel/devcoredump.rs b/rust/kernel/devcoredump.rs new file mode 100644 index 00000000000000..a4a42d862f63b5 --- /dev/null +++ b/rust/kernel/devcoredump.rs @@ -0,0 +1,79 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Device coredump support. +//! +//! C header: [`include/linux/devcoredump.h`](../../../../include/linux/devcoredump.h) + +use crate::{ + alloc, bindings, device, error::from_result, prelude::Result, time::Jiffies, + types::ForeignOwnable, ThisModule, +}; + +use core::ops::Deref; + +/// The default timeout for device coredumps. +pub const DEFAULT_TIMEOUT: Jiffies = bindings::DEVCD_TIMEOUT as Jiffies; + +/// Trait to implement reading from a device coredump. +/// +/// Users must implement this trait to provide device coredump support. +pub trait DevCoreDump { + /// Returns the IOVA (virtual address) of the buffer from RTKit's point of view, or an error if + /// unavailable. + fn read(&self, buf: &mut [u8], offset: usize) -> Result; +} + +unsafe extern "C" fn read_callback< + 'a, + T: ForeignOwnable: Deref>, + D: DevCoreDump, +>( + buffer: *mut crate::ffi::c_char, + offset: bindings::loff_t, + count: usize, + data: *mut crate::ffi::c_void, + _datalen: usize, +) -> isize { + // SAFETY: This pointer came from into_foreign() below. + let coredump = unsafe { T::borrow(data.cast()) }; + // SAFETY: The caller guarantees `buffer` points to at least `count` bytes. + let buf = unsafe { core::slice::from_raw_parts_mut(buffer, count) }; + + from_result(|| Ok(coredump.read(buf, offset.try_into()?)?.try_into()?)) +} + +unsafe extern "C" fn free_callback< + 'a, + T: ForeignOwnable: Deref>, + D: DevCoreDump, +>( + data: *mut crate::ffi::c_void, +) { + // SAFETY: This pointer came from into_foreign() below. + unsafe { + T::from_foreign(data.cast()); + } +} + +/// Registers a coredump for the given device. +pub fn dev_coredump<'a, T: ForeignOwnable: Deref>, D: DevCoreDump>( + dev: &device::Device, + module: &'static ThisModule, + coredump: T, + gfp: alloc::Flags, + timeout: Jiffies, +) { + // SAFETY: Call upholds dev_coredumpm lifetime requirements. + unsafe { + bindings::dev_coredumpm_timeout( + dev.as_raw(), + module.0, + coredump.into_foreign() as *mut _, + 0, + gfp.as_raw(), + Some(read_callback::<'a, T, D>), + Some(free_callback::<'a, T, D>), + timeout, + ) + } +} diff --git a/rust/kernel/device.rs b/rust/kernel/device.rs index 031720bf5d8ca2..e49eac46891a7c 100644 --- a/rust/kernel/device.rs +++ b/rust/kernel/device.rs @@ -4,17 +4,28 @@ //! //! C header: [`include/linux/device.h`](srctree/include/linux/device.h) +use crate::of; use crate::{ - bindings, fmt, + bindings, + fmt, prelude::*, sync::aref::ARef, - types::{ForeignOwnable, Opaque}, + types::{ + ForeignOwnable, + Opaque, // + }, // +}; +use core::{ + any::TypeId, + marker::PhantomData, + ptr, // }; -use core::{any::TypeId, marker::PhantomData, ptr}; #[cfg(CONFIG_PRINTK)] use crate::c_str; +use crate::types::NotThreadSafe; + pub mod property; // Assert that we can `read()` / `write()` a `TypeId` instance from / into `struct driver_type`. @@ -330,13 +341,13 @@ impl Device { impl Device { /// Obtain the raw `struct device *`. - pub(crate) fn as_raw(&self) -> *mut bindings::device { + pub fn as_raw(&self) -> *mut bindings::device { self.0.get() } /// Returns a reference to the parent device, if any. #[cfg_attr(not(CONFIG_AUXILIARY_BUS), expect(dead_code))] - pub(crate) fn parent(&self) -> Option<&Device> { + pub fn parent(&self) -> Option<&Device> { // SAFETY: // - By the type invariant `self.as_raw()` is always valid. // - The parent device is only ever set at device creation. @@ -353,6 +364,13 @@ impl Device { } } + /// Returns the driver_data pointer. + pub fn get_drvdata(&self) -> *mut T { + // SAFETY: dev_get_drvdata returns a field of the device, + // pointer to which is valid by type invariant + unsafe { bindings::dev_get_drvdata(self.as_raw()) as *mut T } + } + /// Convert a raw C `struct device` pointer to a `&'a Device`. /// /// # Safety @@ -366,6 +384,13 @@ impl Device { unsafe { &*ptr.cast() } } + /// Gets the OpenFirmware node attached to this device + pub fn of_node(&self) -> Option { + let ptr = self.0.get(); + // SAFETY: This is safe as long as of_node is NULL or valid. + unsafe { of::Node::get_from_raw((*ptr).of_node) } + } + /// Prints an emergency-level message (level 0) prefixed with device information. /// /// More details are available from [`dev_emerg`]. @@ -484,6 +509,58 @@ impl Device { // defined as a `#[repr(transparent)]` wrapper around `fwnode_handle`. Some(unsafe { &*fwnode_handle.cast() }) } + + /// Locks the [`Device`] for exclusive access. + pub fn lock(&self) -> Guard<'_, Ctx> { + // SAFETY: `self` is always valid by the type invariant. + unsafe { bindings::device_lock(self.as_raw()) }; + + Guard { + dev: self, + _not_send: NotThreadSafe, + } + } + + /// ensure Device is bound + pub fn is_bound(&self) -> Option> { + let guard = self.lock(); + if !unsafe { bindings::device_is_bound(self.as_raw()) } { + return None; + } + Some(guard) + } + + /// excute closure while the device is bound + pub fn while_bound_with(&self, f: F) -> Result + where + F: FnOnce(&Device) -> Result, + { + let _guard = self.lock(); + if unsafe { !bindings::device_is_bound(self.as_raw()) } { + return Err(ENODEV); + } + let ptr: *const Self = self; + let ptr = ptr.cast::>(); + f(unsafe { &*ptr }) + } +} + +/// A lock guard. +/// +/// The lock is unlocked when the guard goes out of scope. +#[must_use = "the lock unlocks immediately when the guard is unused"] +pub struct Guard<'a, Ctx: DeviceContext = Normal> { + dev: &'a Device, + _not_send: NotThreadSafe, +} + +impl Drop for Guard<'_, Ctx> { + fn drop(&mut self) { + // SAFETY: + // - `self.xa.xa` is always valid by the type invariant. + // - The caller holds the lock, so it is safe to unlock it. + unsafe { bindings::device_unlock(self.dev.as_raw()) }; + } } // SAFETY: `Device` is a transparent wrapper of a type that doesn't depend on `Device`'s generic @@ -596,6 +673,13 @@ impl DeviceContext for Core {} impl DeviceContext for CoreInternal {} impl DeviceContext for Normal {} +impl AsRef> for Device { + #[inline] + fn as_ref(&self) -> &Device { + self + } +} + /// Convert device references to bus device references. /// /// Bus devices can implement this trait to allow abstractions to provide the bus device in @@ -715,7 +799,7 @@ macro_rules! impl_device_context_into_aref { macro_rules! dev_printk { ($method:ident, $dev:expr, $($f:tt)*) => { { - ($dev).$method($crate::prelude::fmt!($($f)*)); + $crate::device::Device::$method($dev.as_ref(), $crate::prelude::fmt!($($f)*)) } } } diff --git a/rust/kernel/device/property.rs b/rust/kernel/device/property.rs index 3a332a8c53a9eb..71bbce339949ee 100644 --- a/rust/kernel/device/property.rs +++ b/rust/kernel/device/property.rs @@ -58,7 +58,7 @@ impl FwNode { } /// Obtain the raw `struct fwnode_handle *`. - pub(crate) fn as_raw(&self) -> *mut bindings::fwnode_handle { + pub fn as_raw(&self) -> *mut bindings::fwnode_handle { self.0.get() } diff --git a/rust/kernel/devres.rs b/rust/kernel/devres.rs index 835d9c11948e7d..a9917ce71ded48 100644 --- a/rust/kernel/devres.rs +++ b/rust/kernel/devres.rs @@ -8,30 +8,24 @@ use crate::{ alloc::Flags, bindings, - device::{Bound, Device}, - error::{to_result, Error, Result}, - ffi::c_void, + device::{ + Bound, + Device, // + }, + error::to_result, prelude::*, - revocable::{Revocable, RevocableGuard}, - sync::{aref::ARef, rcu, Completion}, - types::{ForeignOwnable, Opaque, ScopeGuard}, + revocable::{ + Revocable, + RevocableGuard, // + }, + sync::{ + aref::ARef, + rcu, + Arc, // + }, + types::ForeignOwnable, }; -use pin_init::Wrapper; - -/// [`Devres`] inner data accessed from [`Devres::callback`]. -#[pin_data] -struct Inner { - #[pin] - data: Revocable, - /// Tracks whether [`Devres::callback`] has been completed. - #[pin] - devm: Completion, - /// Tracks whether revoking [`Self::data`] has been completed. - #[pin] - revoke: Completion, -} - /// This abstraction is meant to be used by subsystems to containerize [`Device`] bound resources to /// manage their lifetime. /// @@ -105,18 +99,13 @@ struct Inner { /// # fn no_run(dev: &Device) -> Result<(), Error> { /// // SAFETY: Invalid usage for example purposes. /// let iomem = unsafe { IoMem::<{ core::mem::size_of::() }>::new(0xBAAAAAAD)? }; -/// let devres = KBox::pin_init(Devres::new(dev, iomem), GFP_KERNEL)?; +/// let devres = Devres::new(dev, iomem)?; /// /// let res = devres.try_access().ok_or(ENXIO)?; /// res.write8(0x42, 0x0); /// # Ok(()) /// # } /// ``` -/// -/// # Invariants -/// -/// `Self::inner` is guaranteed to be initialized and is always accessed read-only. -#[pin_data(PinnedDrop)] pub struct Devres { dev: ARef, /// Pointer to [`Self::devres_callback`]. @@ -124,14 +113,7 @@ pub struct Devres { /// Has to be stored, since Rust does not guarantee to always return the same address for a /// function. However, the C API uses the address as a key. callback: unsafe extern "C" fn(*mut c_void), - /// Contains all the fields shared with [`Self::callback`]. - // TODO: Replace with `UnsafePinned`, once available. - // - // Subsequently, the `drop_in_place()` in `Devres::drop` and `Devres::new` as well as the - // explicit `Send` and `Sync' impls can be removed. - #[pin] - inner: Opaque>, - _add_action: (), + data: Arc>, } impl Devres { @@ -139,74 +121,48 @@ impl Devres { /// /// The `data` encapsulated within the returned `Devres` instance' `data` will be /// (revoked)[`Revocable`] once the device is detached. - pub fn new<'a, E>( - dev: &'a Device, - data: impl PinInit + 'a, - ) -> impl PinInit + 'a + pub fn new(dev: &Device, data: impl PinInit) -> Result where - T: 'a, Error: From, { - try_pin_init!(&this in Self { - dev: dev.into(), - callback: Self::devres_callback, - // INVARIANT: `inner` is properly initialized. - inner <- Opaque::pin_init(try_pin_init!(Inner { - devm <- Completion::new(), - revoke <- Completion::new(), - data <- Revocable::new(data), - })), - // TODO: Replace with "initializer code blocks" [1] once available. - // - // [1] https://github.com/Rust-for-Linux/pin-init/pull/69 - _add_action: { - // SAFETY: `this` is a valid pointer to uninitialized memory. - let inner = unsafe { &raw mut (*this.as_ptr()).inner }; + let callback = Self::devres_callback; + let data = Arc::pin_init(Revocable::new(data), GFP_KERNEL)?; + let devres_data = data.clone(); + + // SAFETY: + // - `dev.as_raw()` is a pointer to a valid bound device. + // - `data` is guaranteed to be a valid for the duration of the lifetime of `Self`. + // - `devm_add_action()` is guaranteed not to call `callback` for the entire lifetime of + // `dev`. + to_result(unsafe { + bindings::devm_add_action( + dev.as_raw(), + Some(callback), + Arc::as_ptr(&data).cast_mut().cast(), + ) + })?; - // SAFETY: - // - `dev.as_raw()` is a pointer to a valid bound device. - // - `inner` is guaranteed to be a valid for the duration of the lifetime of `Self`. - // - `devm_add_action()` is guaranteed not to call `callback` until `this` has been - // properly initialized, because we require `dev` (i.e. the *bound* device) to - // live at least as long as the returned `impl PinInit`. - to_result(unsafe { - bindings::devm_add_action(dev.as_raw(), Some(*callback), inner.cast()) - }).inspect_err(|_| { - let inner = Opaque::cast_into(inner); + // `devm_add_action()` was successful and has consumed the reference count. + core::mem::forget(devres_data); - // SAFETY: `inner` is a valid pointer to an `Inner` and valid for both reads - // and writes. - unsafe { core::ptr::drop_in_place(inner) }; - })?; - }, + Ok(Self { + dev: dev.into(), + callback, + data, }) } - fn inner(&self) -> &Inner { - // SAFETY: By the type invairants of `Self`, `inner` is properly initialized and always - // accessed read-only. - unsafe { &*self.inner.get() } - } - fn data(&self) -> &Revocable { - &self.inner().data + &self.data } #[allow(clippy::missing_safety_doc)] unsafe extern "C" fn devres_callback(ptr: *mut kernel::ffi::c_void) { - // SAFETY: In `Self::new` we've passed a valid pointer to `Inner` to `devm_add_action()`, - // hence `ptr` must be a valid pointer to `Inner`. - let inner = unsafe { &*ptr.cast::>() }; + // SAFETY: In `Self::new` we've passed a valid pointer of `Revocable` to + // `devm_add_action()`, hence `ptr` must be a valid pointer to `Revocable`. + let data = unsafe { Arc::from_raw(ptr.cast::>()) }; - // Ensure that `inner` can't be used anymore after we signal completion of this callback. - let inner = ScopeGuard::new_with_data(inner, |inner| inner.devm.complete_all()); - - if !inner.data.revoke() { - // If `revoke()` returns false, it means that `Devres::drop` already started revoking - // `data` for us. Hence we have to wait until `Devres::drop` signals that it - // completed revoking `data`. - inner.revoke.wait_for_completion(); - } + data.revoke(); } fn remove_action(&self) -> bool { @@ -218,7 +174,7 @@ impl Devres { bindings::devm_remove_action_nowarn( self.dev.as_raw(), Some(self.callback), - core::ptr::from_ref(self.inner()).cast_mut().cast(), + core::ptr::from_ref(self.data()).cast_mut().cast(), ) } == 0) } @@ -289,31 +245,19 @@ unsafe impl Send for Devres {} // SAFETY: `Devres` can be shared with any task, if `T: Sync`. unsafe impl Sync for Devres {} -#[pinned_drop] -impl PinnedDrop for Devres { - fn drop(self: Pin<&mut Self>) { +impl Drop for Devres { + fn drop(&mut self) { // SAFETY: When `drop` runs, it is guaranteed that nobody is accessing the revocable data // anymore, hence it is safe not to wait for the grace period to finish. if unsafe { self.data().revoke_nosync() } { // We revoked `self.data` before the devres action did, hence try to remove it. - if !self.remove_action() { - // We could not remove the devres action, which means that it now runs concurrently, - // hence signal that `self.data` has been revoked by us successfully. - self.inner().revoke.complete_all(); - - // Wait for `Self::devres_callback` to be done using this object. - self.inner().devm.wait_for_completion(); + if self.remove_action() { + // SAFETY: In `Self::new` we have taken an additional reference count of `self.data` + // for `devm_add_action()`. Since `remove_action()` was successful, we have to drop + // this additional reference count. + drop(unsafe { Arc::from_raw(Arc::as_ptr(&self.data)) }); } - } else { - // `Self::devres_callback` revokes `self.data` for us, hence wait for it to be done - // using this object. - self.inner().devm.wait_for_completion(); } - - // INVARIANT: At this point it is guaranteed that `inner` can't be accessed any more. - // - // SAFETY: `inner` is valid for dropping. - unsafe { core::ptr::drop_in_place(self.inner.get()) }; } } diff --git a/rust/kernel/dma_buf.rs b/rust/kernel/dma_buf.rs new file mode 100644 index 00000000000000..318518ff0b28f9 --- /dev/null +++ b/rust/kernel/dma_buf.rs @@ -0,0 +1,39 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! DMA buffer API +//! +//! C header: [`include/linux/dma-buf.h`](srctree/include/linux/dma-buf.h) + +use bindings; +use kernel::types::*; + +/// A DMA buffer object. +/// +/// # Invariants +/// +/// The data layout of this type is equivalent to that of `struct dma_buf`. +#[repr(transparent)] +pub struct DmaBuf(Opaque); + +// SAFETY: `struct dma_buf` is thread-safe +unsafe impl Send for DmaBuf {} +// SAFETY: `struct dma_buf` is thread-safe +unsafe impl Sync for DmaBuf {} + +impl DmaBuf { + /// Convert from a `*mut bindings::dma_buf` to a [`DmaBuf`]. + /// + /// # Safety + /// + /// The caller guarantees that `self_ptr` points to a valid initialized `struct dma_buf` for the + /// duration of the lifetime of `'a`, and promises to not violate rust's data aliasing rules + /// using the reference provided by this function. + pub(crate) unsafe fn as_ref<'a>(self_ptr: *mut bindings::dma_buf) -> &'a Self { + // SAFETY: Our data layout is equivalent to `dma_buf` . + unsafe { &*self_ptr.cast() } + } + + pub(crate) fn as_raw(&self) -> *mut bindings::dma_buf { + self.0.get() + } +} diff --git a/rust/kernel/dma_fence.rs b/rust/kernel/dma_fence.rs new file mode 100644 index 00000000000000..139451e75685f9 --- /dev/null +++ b/rust/kernel/dma_fence.rs @@ -0,0 +1,480 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! DMA fence abstraction. +//! +//! C header: [`include/linux/dma_fence.h`](../../include/linux/dma_fence.h) + +use crate::{ + bindings, + error::{to_result, Result}, + prelude::*, + sync::LockClassKey, + types::Opaque, +}; +use core::ops::{Deref, DerefMut}; +use core::ptr::addr_of_mut; +use core::sync::atomic::{AtomicU64, Ordering}; + +mod private { + /// Marker that a trait cannot be implemented outside of this mod + pub trait Sealed {} +} + +/// Any kind of DMA Fence Object +/// +/// # Invariants +/// raw() returns a valid pointer to a dma_fence and we own a reference to it. +pub trait RawDmaFence: private::Sealed { + /// Returns the raw `struct dma_fence` pointer. + fn raw(&self) -> *mut bindings::dma_fence; + + /// Returns the raw `struct dma_fence` pointer and consumes the object. + /// + /// The caller is responsible for dropping the reference. + fn into_raw(self) -> *mut bindings::dma_fence + where + Self: Sized, + { + let ptr = self.raw(); + core::mem::forget(self); + ptr + } + + /// Advances this fence to the chain node which will signal this sequence number. + /// If no sequence number is provided, this returns `self` again. + /// If the seqno has already been signaled, returns None. + fn chain_find_seqno(self, seqno: u64) -> Result> + where + Self: Sized, + { + let mut ptr = self.into_raw(); + + // SAFETY: This will safely fail if this DmaFence is not a chain. + // `ptr` is valid per the type invariant. + let ret = unsafe { bindings::dma_fence_chain_find_seqno(&mut ptr, seqno) }; + + if ret != 0 { + // SAFETY: This is either an owned reference or NULL, dma_fence_put can handle both. + unsafe { bindings::dma_fence_put(ptr) }; + Err(Error::from_errno(ret)) + } else if ptr.is_null() { + Ok(None) + } else { + // SAFETY: ptr is valid and non-NULL as checked above. + Ok(Some(unsafe { Fence::from_raw(ptr) })) + } + } + + /// Signal completion of this fence + fn signal(&self) -> Result { + // SAFETY: Safe to call on any valid dma_fence object + to_result(unsafe { bindings::dma_fence_signal(self.raw()) }) + } + + /// Set the error flag on this fence + fn set_error(&self, err: Error) { + // SAFETY: Safe to call on any valid dma_fence object + unsafe { bindings::dma_fence_set_error(self.raw(), err.to_errno()) }; + } +} + +/// A generic DMA Fence Object +/// +/// # Invariants +/// ptr is a valid pointer to a dma_fence and we own a reference to it. +pub struct Fence { + ptr: *mut bindings::dma_fence, +} + +impl Fence { + /// Create a new Fence object from a raw pointer to a dma_fence. + /// + /// # Safety + /// The caller must own a reference to the dma_fence, which is transferred to the new object. + pub(crate) unsafe fn from_raw(ptr: *mut bindings::dma_fence) -> Fence { + Fence { ptr } + } + + /// Create a new Fence object from a raw pointer to a dma_fence. + /// + /// # Safety + /// Takes a borrowed reference to the dma_fence, and increments the reference count. + pub(crate) unsafe fn get_raw(ptr: *mut bindings::dma_fence) -> Fence { + // SAFETY: Pointer is valid per the safety contract + unsafe { bindings::dma_fence_get(ptr) }; + Fence { ptr } + } + + /// Create a new Fence object from a RawDmaFence. + pub fn from_fence(fence: &dyn RawDmaFence) -> Fence { + // SAFETY: Pointer is valid per the RawDmaFence contract + unsafe { Self::get_raw(fence.raw()) } + } +} + +impl private::Sealed for Fence {} + +impl RawDmaFence for Fence { + fn raw(&self) -> *mut bindings::dma_fence { + self.ptr + } +} + +impl Drop for Fence { + fn drop(&mut self) { + // SAFETY: We own a reference to this syncobj. + unsafe { bindings::dma_fence_put(self.ptr) }; + } +} + +impl Clone for Fence { + fn clone(&self) -> Self { + // SAFETY: `ptr` is valid per the type invariant and we own a reference to it. + unsafe { + bindings::dma_fence_get(self.ptr); + Self::from_raw(self.ptr) + } + } +} + +// SAFETY: The API for these objects is thread safe +unsafe impl Sync for Fence {} +// SAFETY: The API for these objects is thread safe +unsafe impl Send for Fence {} + +/// Trait which must be implemented by driver-specific fence objects. +#[vtable] +pub trait FenceOps: Sized + Send + Sync { + /// Returns the driver name. This is a callback to allow drivers to compute the name at + /// runtime, without having it to store permanently for each fence, or build a cache of + /// some sort. + fn get_driver_name<'a>(self: &'a FenceObject) -> &'a CStr; + + /// Return the name of the context this fence belongs to. This is a callback to allow drivers + /// to compute the name at runtime, without having it to store permanently for each fence, or + /// build a cache of some sort. + fn get_timeline_name<'a>(self: &'a FenceObject) -> &'a CStr; + + /// Enable software signaling of fence. + fn enable_signaling(self: &FenceObject) -> bool { + false + } + + /// Peek whether the fence is signaled, as a fastpath optimization for e.g. dma_fence_wait() or + /// dma_fence_add_callback(). + fn signaled(self: &FenceObject) -> bool { + false + } +} + +unsafe extern "C" fn get_driver_name_cb( + fence: *mut bindings::dma_fence, +) -> *const crate::ffi::c_char { + // SAFETY: All of our fences are FenceObject. + let p = unsafe { crate::container_of!(fence, FenceObject, fence) as *mut FenceObject }; + + // SAFETY: The caller is responsible for passing a valid dma_fence subtype + T::get_driver_name(unsafe { &mut *p }).as_char_ptr() +} + +unsafe extern "C" fn get_timeline_name_cb( + fence: *mut bindings::dma_fence, +) -> *const crate::ffi::c_char { + // SAFETY: All of our fences are FenceObject. + let p = unsafe { crate::container_of!(fence, FenceObject, fence) as *mut FenceObject }; + + // SAFETY: The caller is responsible for passing a valid dma_fence subtype + T::get_timeline_name(unsafe { &mut *p }).as_char_ptr() +} + +unsafe extern "C" fn enable_signaling_cb(fence: *mut bindings::dma_fence) -> bool { + // SAFETY: All of our fences are FenceObject. + let p = unsafe { crate::container_of!(fence, FenceObject, fence) as *mut FenceObject }; + + // SAFETY: The caller is responsible for passing a valid dma_fence subtype + T::enable_signaling(unsafe { &mut *p }) +} + +unsafe extern "C" fn signaled_cb(fence: *mut bindings::dma_fence) -> bool { + // SAFETY: All of our fences are FenceObject. + let p = unsafe { crate::container_of!(fence, FenceObject, fence) as *mut FenceObject }; + + // SAFETY: The caller is responsible for passing a valid dma_fence subtype + T::signaled(unsafe { &mut *p }) +} + +unsafe extern "C" fn release_cb(fence: *mut bindings::dma_fence) { + // SAFETY: All of our fences are FenceObject. + let p = unsafe { crate::container_of!(fence, FenceObject, fence) as *mut FenceObject }; + + // SAFETY: p is never used after this + unsafe { + core::ptr::drop_in_place(&mut (*p).inner); + } + + // SAFETY: All of our fences are allocated using kmalloc, so this is safe. + unsafe { bindings::dma_fence_free(fence) }; +} + +/// A driver-specific DMA Fence Object +/// +/// # Invariants +/// ptr is a valid pointer to a dma_fence and we own a reference to it. +#[repr(C)] +pub struct FenceObject { + fence: bindings::dma_fence, + lock: Opaque, + inner: T, +} + +impl FenceObject { + const SIZE: usize = core::mem::size_of::(); + + const VTABLE: bindings::dma_fence_ops = bindings::dma_fence_ops { + get_driver_name: Some(get_driver_name_cb::), + get_timeline_name: Some(get_timeline_name_cb::), + enable_signaling: if T::HAS_ENABLE_SIGNALING { + Some(enable_signaling_cb::) + } else { + None + }, + signaled: if T::HAS_SIGNALED { + Some(signaled_cb::) + } else { + None + }, + wait: None, // Deprecated + release: Some(release_cb::), + set_deadline: None, + }; +} + +impl Deref for FenceObject { + type Target = T; + + fn deref(&self) -> &T { + &self.inner + } +} + +impl DerefMut for FenceObject { + fn deref_mut(&mut self) -> &mut T { + &mut self.inner + } +} + +impl private::Sealed for FenceObject {} +impl RawDmaFence for FenceObject { + fn raw(&self) -> *mut bindings::dma_fence { + &self.fence as *const _ as *mut _ + } +} + +/// A unique reference to a driver-specific fence object +pub struct UniqueFence(*mut FenceObject); + +impl Deref for UniqueFence { + type Target = FenceObject; + + fn deref(&self) -> &FenceObject { + // SAFETY: The pointer is always valid for UniqueFence objects + unsafe { &*self.0 } + } +} + +impl DerefMut for UniqueFence { + fn deref_mut(&mut self) -> &mut FenceObject { + // SAFETY: The pointer is always valid for UniqueFence objects + unsafe { &mut *self.0 } + } +} + +impl private::Sealed for UniqueFence {} +impl RawDmaFence for UniqueFence { + fn raw(&self) -> *mut bindings::dma_fence { + // SAFETY: The pointer is always valid for UniqueFence objects + unsafe { addr_of_mut!((*self.0).fence) } + } +} + +impl From> for UserFence { + fn from(value: UniqueFence) -> Self { + let ptr = value.0; + core::mem::forget(value); + + UserFence(ptr) + } +} + +impl Drop for UniqueFence { + fn drop(&mut self) { + // SAFETY: We own a reference to this fence. + unsafe { bindings::dma_fence_put(self.raw()) }; + } +} + +// SAFETY: The API for these objects is thread safe +unsafe impl Sync for UniqueFence {} +// SAFETY: The API for these objects is thread safe +unsafe impl Send for UniqueFence {} + +/// A shared reference to a driver-specific fence object +pub struct UserFence(*mut FenceObject); + +impl Deref for UserFence { + type Target = FenceObject; + + fn deref(&self) -> &FenceObject { + // SAFETY: The pointer is always valid for UserFence objects + unsafe { &*self.0 } + } +} + +impl Clone for UserFence { + fn clone(&self) -> Self { + // SAFETY: `ptr` is valid per the type invariant and we own a reference to it. + unsafe { + bindings::dma_fence_get(self.raw()); + Self(self.0) + } + } +} + +impl private::Sealed for UserFence {} +impl RawDmaFence for UserFence { + fn raw(&self) -> *mut bindings::dma_fence { + // SAFETY: The pointer is always valid for UserFence objects + unsafe { addr_of_mut!((*self.0).fence) } + } +} + +impl Drop for UserFence { + fn drop(&mut self) { + // SAFETY: We own a reference to this fence. + unsafe { bindings::dma_fence_put(self.raw()) }; + } +} + +// SAFETY: The API for these objects is thread safe +unsafe impl Sync for UserFence {} +// SAFETY: The API for these objects is thread safe +unsafe impl Send for UserFence {} + +/// An array of fence contexts, out of which fences can be created. +pub struct FenceContexts { + start: u64, + count: u32, + seqnos: KVec, + lock_name: &'static CStr, + lock_key: Pin<&'static LockClassKey>, +} + +impl FenceContexts { + /// Create a new set of fence contexts. + pub fn new( + count: u32, + name: &'static CStr, + key: Pin<&'static LockClassKey>, + ) -> Result { + let mut seqnos: KVec = KVec::new(); + + seqnos.reserve(count as usize, GFP_KERNEL)?; + + for _ in 0..count { + seqnos.push(Default::default(), GFP_KERNEL)?; + } + + // SAFETY: This is always safe to call + let start = unsafe { bindings::dma_fence_context_alloc(count as crate::ffi::c_uint) }; + + Ok(FenceContexts { + start, + count, + seqnos, + lock_name: name, + lock_key: key, + }) + } + + /// Create a new fence in a given context index. + pub fn new_fence(&self, context: u32, inner: T) -> Result> { + if context > self.count { + return Err(EINVAL); + } + + // SAFETY: krealloc is always safe to call like this + let p = unsafe { + bindings::krealloc_node_align( + core::ptr::null_mut(), + FenceObject::::SIZE, + 1, + bindings::GFP_KERNEL | bindings::__GFP_ZERO, + bindings::NUMA_NO_NODE, + ) as *mut FenceObject + }; + + if p.is_null() { + return Err(ENOMEM); + } + + let seqno = self.seqnos[context as usize].fetch_add(1, Ordering::Relaxed); + + // SAFETY: The pointer is valid, so pointers to members are too. + // After this, all fields are initialized. + unsafe { + addr_of_mut!((*p).inner).write(inner); + bindings::__spin_lock_init( + addr_of_mut!((*p).lock) as *mut _, + self.lock_name.as_char_ptr(), + self.lock_key.as_ptr(), + ); + bindings::dma_fence_init64( + addr_of_mut!((*p).fence), + &FenceObject::::VTABLE, + addr_of_mut!((*p).lock) as *mut _, + self.start + context as u64, + seqno, + ); + }; + + Ok(UniqueFence(p)) + } +} + +/// A DMA Fence Chain Object +/// +/// # Invariants +/// ptr is a valid pointer to a dma_fence_chain which we own. +pub struct FenceChain { + ptr: *mut bindings::dma_fence_chain, +} + +impl FenceChain { + /// Create a new DmaFenceChain object. + pub fn new() -> Result { + // SAFETY: This function is safe to call and takes no arguments. + let ptr = unsafe { bindings::dma_fence_chain_alloc() }; + + if ptr.is_null() { + Err(ENOMEM) + } else { + Ok(FenceChain { ptr }) + } + } + + /// Convert the DmaFenceChain into the underlying raw pointer. + /// + /// This assumes the caller will take ownership of the object. + pub(crate) fn into_raw(self) -> *mut bindings::dma_fence_chain { + let ptr = self.ptr; + core::mem::forget(self); + ptr + } +} + +impl Drop for FenceChain { + fn drop(&mut self) { + // SAFETY: We own this dma_fence_chain. + unsafe { bindings::dma_fence_chain_free(self.ptr) }; + } +} diff --git a/rust/kernel/driver.rs b/rust/kernel/driver.rs index bee3ae21a27b2a..36de8098754d05 100644 --- a/rust/kernel/driver.rs +++ b/rust/kernel/driver.rs @@ -94,10 +94,14 @@ //! [`device_id`]: kernel::device_id //! [`module_driver`]: kernel::module_driver -use crate::error::{Error, Result}; -use crate::{acpi, device, of, str::CStr, try_pin_init, types::Opaque, ThisModule}; -use core::pin::Pin; -use pin_init::{pin_data, pinned_drop, PinInit}; +use crate::{ + acpi, + device, + of, + prelude::*, + types::Opaque, + ThisModule, // +}; /// Trait describing the layout of a specific device driver. /// diff --git a/rust/kernel/drm/device.rs b/rust/kernel/drm/device.rs index 3ce8f62a005696..76625537f38023 100644 --- a/rust/kernel/drm/device.rs +++ b/rust/kernel/drm/device.rs @@ -86,7 +86,7 @@ impl Device { name: crate::str::as_char_ptr_in_const_context(T::INFO.name).cast_mut(), desc: crate::str::as_char_ptr_in_const_context(T::INFO.desc).cast_mut(), - driver_features: drm::driver::FEAT_GEM, + driver_features: T::FEATURES, ioctls: T::IOCTLS.as_ptr(), num_ioctls: T::IOCTLS.len() as i32, fops: &Self::GEM_FOPS, @@ -184,7 +184,10 @@ impl Device { // SAFETY: // - When `release` runs it is guaranteed that there is no further access to `this`. // - `this` is valid for dropping. - unsafe { core::ptr::drop_in_place(this) }; + // unsafe { core::ptr::drop_in_place(this) }; + // HACK: data might be uninitialized so leak the DRM device instead. The expected number + // of times the asahi device gets released is once at poweroff or reboot. + let _ = core::mem::ManuallyDrop::new(this); } } diff --git a/rust/kernel/drm/driver.rs b/rust/kernel/drm/driver.rs index f30ee4c6245cda..1d98a3f49c7346 100644 --- a/rust/kernel/drm/driver.rs +++ b/rust/kernel/drm/driver.rs @@ -13,7 +13,16 @@ use crate::{ use macros::vtable; /// Driver use the GEM memory manager. This should be set for all modern drivers. -pub(crate) const FEAT_GEM: u32 = bindings::drm_driver_feature_DRIVER_GEM; +pub const FEAT_GEM: u32 = bindings::drm_driver_feature_DRIVER_GEM; +/// Driver supports dedicated render nodes. +pub const FEAT_RENDER: u32 = bindings::drm_driver_feature_DRIVER_RENDER; +/// Driver supports DRM sync objects for explicit synchronization of command submission. +pub const FEAT_SYNCOBJ: u32 = bindings::drm_driver_feature_DRIVER_SYNCOBJ; +/// Driver supports the timeline flavor of DRM sync objects for explicit synchronization of command +/// submission. +pub const FEAT_SYNCOBJ_TIMELINE: u32 = bindings::drm_driver_feature_DRIVER_SYNCOBJ_TIMELINE; +/// Driver supports user defined GPU VA bindings for GEM objects. +pub const FEAT_GEM_GPUVA: u32 = bindings::drm_driver_feature_DRIVER_GEM_GPUVA; /// Information data for a DRM Driver. pub struct DriverInfo { @@ -111,6 +120,9 @@ pub trait Driver { /// Driver metadata const INFO: DriverInfo; + /// Feature flags + const FEATURES: u32; + /// IOCTL list. See `kernel::drm::ioctl::declare_drm_ioctls!{}`. const IOCTLS: &'static [drm::ioctl::DrmIoctlDescriptor]; } @@ -121,7 +133,6 @@ pub trait Driver { pub struct Registration(ARef>); impl Registration { - /// Creates a new [`Registration`] and registers it. fn new(drm: &drm::Device, flags: usize) -> Result { // SAFETY: `drm.as_raw()` is valid by the invariants of `drm::Device`. to_result(unsafe { bindings::drm_dev_register(drm.as_raw(), flags) })?; @@ -129,8 +140,9 @@ impl Registration { Ok(Self(drm.into())) } - /// Same as [`Registration::new`}, but transfers ownership of the [`Registration`] to - /// [`devres::register`]. + /// Registers a new [`Device`](drm::Device) with userspace. + /// + /// Ownership of the [`Registration`] object is passed to [`devres::register`]. pub fn new_foreign_owned( drm: &drm::Device, dev: &device::Device, diff --git a/rust/kernel/drm/file.rs b/rust/kernel/drm/file.rs index 8c46f8d519516a..9c7bcace70eef5 100644 --- a/rust/kernel/drm/file.rs +++ b/rust/kernel/drm/file.rs @@ -15,6 +15,9 @@ pub trait DriverFile { /// Open a new file (called when a client opens the DRM device). fn open(device: &drm::Device) -> Result>>; + + /// Get raw drm_file pointer + fn as_raw(&self) -> *mut bindings::drm_file; } /// An open DRM File. diff --git a/rust/kernel/drm/gem/mod.rs b/rust/kernel/drm/gem/mod.rs index a7f682e95c0184..381bd798f2199a 100644 --- a/rust/kernel/drm/gem/mod.rs +++ b/rust/kernel/drm/gem/mod.rs @@ -3,18 +3,58 @@ //! DRM GEM API //! //! C header: [`include/drm/drm_gem.h`](srctree/include/drm/drm_gem.h) +#[cfg(CONFIG_DRM_GEM_SHMEM_HELPER = "y")] +pub mod shmem; use crate::{ alloc::flags::*, - bindings, drm, + bindings, dma_buf, drm, drm::driver::{AllocImpl, AllocOps}, - error::{to_result, Result}, + error::{from_err_ptr, to_result, Result}, prelude::*, sync::aref::{ARef, AlwaysRefCounted}, types::Opaque, }; +use core::marker::PhantomData; use core::{ops::Deref, ptr::NonNull}; +/// A macro for implementing [`AlwaysRefCounted`] for any GEM object type. +/// +/// Since all GEM objects use the same refcounting scheme. +#[macro_export] +macro_rules! impl_aref_for_gem_obj { + ( + impl $( <$( $tparam_id:ident ),+> )? for $type:ty + $( + where + $( $bind_param:path : $bind_trait:path ),+ + )? + ) => { + // SAFETY: All gem objects are refcounted + unsafe impl $( <$( $tparam_id ),+> )? $crate::types::AlwaysRefCounted for $type + where + Self: IntoGEMObject, + $( $( $bind_param : $bind_trait ),+ )? + { + fn inc_ref(&self) { + // SAFETY: The existence of a shared reference guarantees that the refcount is + // non-zero. + unsafe { bindings::drm_gem_object_get(self.as_raw()) }; + } + + unsafe fn dec_ref(obj: core::ptr::NonNull) { + // SAFETY: `obj` is a valid pointer to an `Object`. + let obj = unsafe { obj.as_ref() }.as_raw(); + + // SAFETY: The safety requirements guarantee that the refcount is non-zero. + unsafe { bindings::drm_gem_object_put(obj) }; + } + } + }; +} + +pub(crate) use impl_aref_for_gem_obj; + /// A type alias for retrieving a [`Driver`]s [`DriverFile`] implementation from its /// [`DriverObject`] implementation. /// @@ -23,12 +63,20 @@ use core::{ops::Deref, ptr::NonNull}; pub type DriverFile = drm::File<<::Driver as drm::Driver>::File>; /// GEM object functions, which must be implemented by drivers. +#[vtable] pub trait DriverObject: Sync + Send + Sized { /// Parent `Driver` for this object. type Driver: drm::Driver; + /// The data type to use for passing arguments to [`DriverObject::new`]. + type Args; + /// Create a new driver data object for a GEM object of a given size. - fn new(dev: &drm::Device, size: usize) -> impl PinInit; + fn new( + dev: &drm::Device, + size: usize, + args: Self::Args, + ) -> impl PinInit; /// Open a new handle to an existing object, associated with a File. fn open(_obj: &::Object, _file: &DriverFile) -> Result { @@ -37,6 +85,14 @@ pub trait DriverObject: Sync + Send + Sized { /// Close a handle to an existing object, associated with a File. fn close(_obj: &::Object, _file: &DriverFile) {} + + /// Optional handle for exporting a gem object. + fn export( + _obj: &::Object, + _flags: u32, + ) -> Result::Object>> { + unimplemented!() + } } /// Trait that represents a GEM object subtype @@ -86,6 +142,21 @@ extern "C" fn close_callback( T::close(obj, file); } +extern "C" fn export_callback( + raw_obj: *mut bindings::drm_gem_object, + flags: i32, +) -> *mut bindings::dma_buf { + // SAFETY: `export_callback` is specified in the AllocOps structure for `Object`, ensuring + // that `raw_obj` is contained within a `Object`. + let obj = unsafe { <::Object as IntoGEMObject>::from_raw(raw_obj) }; + + match T::export(obj, flags as _) { + // DRM takes a hold of the reference + Ok(buf) => buf.into_raw(), + Err(e) => e.to_ptr(), + } +} + impl IntoGEMObject for Object { fn as_raw(&self) -> *mut bindings::drm_gem_object { self.obj.get() @@ -150,6 +221,28 @@ pub trait BaseObject: IntoGEMObject { Ok(unsafe { ARef::from_raw(obj.into()) }) } + /// Export a [`DmaBuf`] for this GEM object using the DRM prime helper library. + /// + /// `flags` should be a set of flags from [`fs::file::flags`](kernel::fs::file::flags). + fn prime_export(&self, flags: u32) -> Result> { + // SAFETY: + // - `as_raw()` always returns a valid pointer to a `drm_gem_object`. + // - `drm_gem_prime_export()` returns either an error pointer, or a valid pointer to an + // initialized `dma_buf` on success. + let dma_ptr = + from_err_ptr(unsafe { bindings::drm_gem_prime_export(self.as_raw(), flags as _) })?; + + // SAFETY: + // - We checked that dma_ptr is not an error, so it must point to an initialized dma_buf + // - We used drm_gem_prime_export(), so `dma_ptr` will remain valid until a call to + // `drm_gem_prime_release()` which we don't call here. + let dma_buf = unsafe { dma_buf::DmaBuf::as_ref(dma_ptr) }; + + // INVARIANT: We used drm_gem_prime_export() to create this dma_buf, fulfilling the + // invariant that this dma_buf came from a GEM object of type `Self`. + Ok(DmaBuf(dma_buf.into(), PhantomData)) + } + /// Creates an mmap offset to map the object from userspace. fn create_mmap_offset(&self) -> Result { // SAFETY: The arguments are valid per the type invariant. @@ -158,10 +251,35 @@ pub trait BaseObject: IntoGEMObject { // SAFETY: The arguments are valid per the type invariant. Ok(unsafe { bindings::drm_vma_node_offset_addr(&raw mut (*self.as_raw()).vma_node) }) } + + /// Lock the gpuva lock + fn lock_gpuva(&self) { + unsafe { + bindings::mutex_lock(&raw mut (*self.as_raw()).gpuva.lock); + } + } + + /// Lock the gpuva lock + fn unlock_gpuva(&self) { + unsafe { + bindings::mutex_unlock(&raw mut (*self.as_raw()).gpuva.lock); + } + } } impl BaseObject for T {} +/// Crate-private base operations shared by all GEM object classes. +pub(crate) trait BaseObjectPrivate: IntoGEMObject { + /// Return a pointer to this object's dma_resv. + fn raw_dma_resv(&self) -> *mut bindings::dma_resv { + // SAFETY: `as_gem_obj()` always returns a valid pointer to the base DRM gem object + unsafe { (*self.as_raw()).resv } + } +} + +impl BaseObjectPrivate for T {} + /// A base GEM object. /// /// # Invariants @@ -181,7 +299,11 @@ impl Object { open: Some(open_callback::), close: Some(close_callback::), print_info: None, - export: None, + export: if T::HAS_EXPORT { + Some(export_callback::) + } else { + None + }, pin: None, unpin: None, get_sg_table: None, @@ -195,11 +317,11 @@ impl Object { }; /// Create a new GEM object. - pub fn new(dev: &drm::Device, size: usize) -> Result> { + pub fn new(dev: &drm::Device, size: usize, args: T::Args) -> Result> { let obj: Pin> = KBox::pin_init( try_pin_init!(Self { obj: Opaque::new(bindings::drm_gem_object::default()), - data <- T::new(dev, size), + data <- T::new(dev, size, args), }), GFP_KERNEL, )?; @@ -252,21 +374,7 @@ impl Object { } } -// SAFETY: Instances of `Object` are always reference-counted. -unsafe impl crate::types::AlwaysRefCounted for Object { - fn inc_ref(&self) { - // SAFETY: The existence of a shared reference guarantees that the refcount is non-zero. - unsafe { bindings::drm_gem_object_get(self.as_raw()) }; - } - - unsafe fn dec_ref(obj: NonNull) { - // SAFETY: `obj` is a valid pointer to an `Object`. - let obj = unsafe { obj.as_ref() }; - - // SAFETY: The safety requirements guarantee that the refcount is non-zero. - unsafe { bindings::drm_gem_object_put(obj.as_raw()) } - } -} +impl_aref_for_gem_obj!(impl for Object where T: DriverObject); impl super::private::Sealed for Object {} @@ -292,6 +400,49 @@ impl AllocImpl for Object { }; } +/// A [`dma_buf::DmaBuf`] which has been exported from a GEM object. +/// +/// The [`dma_buf::DmaBuf`] will be released when this type is dropped. +/// +/// # Invariants +/// +/// - `self.0` points to a valid initialized [`dma_buf::DmaBuf`] for the lifetime of this object. +/// - The GEM object from which this [`dma_buf::DmaBuf`] was exported from is guaranteed to be of +/// type `T`. +pub struct DmaBuf(NonNull, PhantomData); + +impl Deref for DmaBuf { + type Target = dma_buf::DmaBuf; + + #[inline] + fn deref(&self) -> &Self::Target { + // SAFETY: This pointer is guaranteed to be valid by our type invariants. + unsafe { self.0.as_ref() } + } +} + +impl Drop for DmaBuf { + #[inline] + fn drop(&mut self) { + // SAFETY: + // - `dma_buf::DmaBuf` is guaranteed to have an identical layout to `struct dma_buf` + // by its type invariants. + // - We hold the last reference to this `DmaBuf`, making it safe to destroy. + unsafe { bindings::drm_gem_dmabuf_release(self.0.cast().as_ptr()) } + } +} + +impl DmaBuf { + /// Leak the reference for this [`DmaBuf`] and return a raw pointer to it. + #[inline] + pub(crate) fn into_raw(self) -> *mut bindings::dma_buf { + let dma_ptr = self.as_raw(); + + core::mem::forget(self); + dma_ptr + } +} + pub(super) const fn create_fops() -> bindings::file_operations { // SAFETY: As by the type invariant, it is safe to initialize `bindings::file_operations` // zeroed. diff --git a/rust/kernel/drm/gem/shmem.rs b/rust/kernel/drm/gem/shmem.rs new file mode 100644 index 00000000000000..bfb05e19cc4500 --- /dev/null +++ b/rust/kernel/drm/gem/shmem.rs @@ -0,0 +1,442 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! DRM GEM shmem helper objects +//! +//! C header: [`include/linux/drm/drm_gem_shmem_helper.h`](srctree/include/linux/drm/drm_gem_shmem_helper.h) + +// TODO: +// - There are a number of spots here that manually acquire/release the DMA reservation lock using +// dma_resv_(un)lock(). In the future we should add support for ww mutex, expose a method to +// acquire a reference to the WwMutex, and then use that directly instead of the C functions here. + +use crate::{ + container_of, + drm::{device, driver, gem, private::Sealed}, + error::{from_err_ptr, to_result}, + iosys_map::*, + prelude::*, + scatterlist, + transmute::*, + types::{ARef, Opaque}, +}; +use core::{ + mem::{self, MaybeUninit}, + ops::{Deref, DerefMut}, + ptr::NonNull, +}; +use gem::{BaseObject, BaseObjectPrivate, DriverObject, IntoGEMObject}; + +/// A struct for controlling the creation of shmem-backed GEM objects. +/// +/// This is used with [`Object::new()`] to control various properties that can only be set when +/// initially creating a shmem-backed GEM object. +#[derive(Default)] +pub struct ObjectConfig<'a, T: DriverObject> { + /// Whether to set the write-combine map flag. + pub map_wc: bool, + + /// Reuse the DMA reservation from another GEM object. + /// + /// The newly created [`Object`] will hold an owned refcount to `parent_resv_obj` if specified. + pub parent_resv_obj: Option<&'a Object>, +} + +/// A shmem-backed GEM object. +/// +/// # Invariants +/// +/// `obj` contains a valid initialized `struct drm_gem_shmem_object` for the lifetime of this +/// object. +#[repr(C)] +#[pin_data] +pub struct Object { + #[pin] + obj: Opaque, + // Parent object that owns this object's DMA reservation object + parent_resv_obj: Option>>, + #[pin] + inner: T, +} + +super::impl_aref_for_gem_obj!(impl for Object where T: DriverObject); + +// SAFETY: This object is thread-safe via our type invariants. +unsafe impl Send for Object {} +// SAFETY: This object is thread-safe via our type invariants. +unsafe impl Sync for Object {} + +impl Object { + /// `drm_gem_object_funcs` vtable suitable for GEM shmem objects. + const VTABLE: bindings::drm_gem_object_funcs = bindings::drm_gem_object_funcs { + free: Some(Self::free_callback), + open: Some(super::open_callback::), + close: Some(super::close_callback::), + print_info: Some(bindings::drm_gem_shmem_object_print_info), + export: if T::HAS_EXPORT { + Some(super::export_callback::) + } else { + None + }, + pin: Some(bindings::drm_gem_shmem_object_pin), + unpin: Some(bindings::drm_gem_shmem_object_unpin), + get_sg_table: Some(bindings::drm_gem_shmem_object_get_sg_table), + vmap: Some(bindings::drm_gem_shmem_object_vmap), + vunmap: Some(bindings::drm_gem_shmem_object_vunmap), + mmap: Some(bindings::drm_gem_shmem_object_mmap), + status: None, + rss: None, + // SAFETY: `drm_gem_shmem_vm_ops` is static const on the C side, so immutable references are + // safe here and such references shall be valid forever + vm_ops: unsafe { &bindings::drm_gem_shmem_vm_ops }, + evict: None, + }; + + /// Return a raw pointer to the embedded drm_gem_shmem_object. + fn as_shmem(&self) -> *mut bindings::drm_gem_shmem_object { + self.obj.get() + } + + /// Create a new shmem-backed DRM object of the given size. + /// + /// Additional config options can be specified using `config`. + pub fn new( + dev: &device::Device, + size: usize, + config: ObjectConfig<'_, T>, + args: T::Args, + ) -> Result> { + let new: Pin> = KBox::try_pin_init( + try_pin_init!(Self { + obj <- Opaque::init_zeroed(), + parent_resv_obj: config.parent_resv_obj.map(|p| p.into()), + inner <- T::new(dev, size, args), + }), + GFP_KERNEL, + )?; + + // SAFETY: `obj.as_raw()` is guaranteed to be valid by the initialization above. + unsafe { (*new.as_raw()).funcs = &Self::VTABLE }; + + // SAFETY: The arguments are all valid via the type invariants. + to_result(unsafe { bindings::drm_gem_shmem_init(dev.as_raw(), new.as_shmem(), size) })?; + + // SAFETY: We never move out of `self`. + let new = KBox::into_raw(unsafe { Pin::into_inner_unchecked(new) }); + + // SAFETY: We're taking over the owned refcount from `drm_gem_shmem_init`. + let obj = unsafe { ARef::from_raw(NonNull::new_unchecked(new)) }; + + // Start filling out values from `config` + if let Some(parent_resv) = config.parent_resv_obj { + // SAFETY: We have yet to expose the new gem object outside of this function, so it is + // safe to modify this field. + unsafe { (*obj.obj.get()).base.resv = parent_resv.raw_dma_resv() }; + } + + // SAFETY: We have yet to expose this object outside of this function, so we're guaranteed + // to have exclusive access - thus making this safe to hold a mutable reference to. + let shmem = unsafe { &mut *obj.as_shmem() }; + shmem.set_map_wc(config.map_wc); + + Ok(obj) + } + + /// Returns the `Device` that owns this GEM object. + pub fn dev(&self) -> &device::Device { + // SAFETY: `dev` will have been initialized in `Self::new()` by `drm_gem_shmem_init()`. + unsafe { device::Device::from_raw((*self.as_raw()).dev) } + } + + extern "C" fn free_callback(obj: *mut bindings::drm_gem_object) { + // SAFETY: + // - DRM always passes a valid gem object here + // - We used drm_gem_shmem_create() in our create_gem_object callback, so we know that + // `obj` is contained within a drm_gem_shmem_object + let this = unsafe { container_of!(obj, bindings::drm_gem_shmem_object, base) }; + + // SAFETY: + // - We're in free_callback - so this function is safe to call. + // - We won't be using the gem resources on `this` after this call. + unsafe { bindings::drm_gem_shmem_release(this) }; + + // SAFETY: + // - We verified above that `obj` is valid, which makes `this` valid + // - This function is set in AllocOps, so we know that `this` is contained within a + // `Object` + let this = unsafe { container_of!(Opaque::cast_from(this), Self, obj) }.cast_mut(); + + // SAFETY: We're recovering the Kbox<> we created in gem_create_object() + let _ = unsafe { KBox::from_raw(this) }; + } + + /// Creates (if necessary) and returns an immutable reference to a scatter-gather table of DMA + /// pages for this object. + /// + /// This will pin the object in memory. + #[inline] + pub fn sg_table(&self) -> Result<&scatterlist::SGTable> { + // SAFETY: + // - drm_gem_shmem_get_pages_sgt is thread-safe. + // - drm_gem_shmem_get_pages_sgt returns either a valid pointer to a scatterlist, or an + // error pointer. + let sgt = from_err_ptr(unsafe { bindings::drm_gem_shmem_get_pages_sgt(self.as_shmem()) })?; + + // SAFETY: We checked above that `sgt` is not an error pointer, so it must be a valid + // pointer to a scatterlist + Ok(unsafe { scatterlist::SGTable::from_raw(sgt) }) + } + + /// Creates (if necessary) and returns an owned reference to a scatter-gather table of DMA pages + /// for this object. + /// + /// This is the same as [`sg_table`](Self::sg_table), except that it instead returns an + /// [`shmem::SGTable`] which holds a reference to the associated gem object, instead of a + /// reference to an [`scatterlist::SGTable`]. + /// + /// This will pin the object in memory. + /// + /// [`shmem::SGTable`]: SGTable + pub fn owned_sg_table(&self) -> Result> { + Ok(SGTable { + sgt: self.sg_table()?.into(), + // INVARIANT: We take an owned refcount to `self` here, ensuring that `sgt` remains + // valid for as long as this `SGTable`. + _owner: self.into(), + }) + } + + /// Attempt to create a [`RawIoSysMap`] from the gem object. + fn raw_vmap(&self) -> Result> { + build_assert!( + mem::size_of::() > 0, + "It doesn't make sense for the mapping type to be a ZST" + ); + + let mut map: MaybeUninit = MaybeUninit::uninit(); + + // SAFETY: drm_gem_shmem_vmap can be called with the DMA reservation lock held + to_result(unsafe { + // TODO: see top of file + bindings::dma_resv_lock(self.raw_dma_resv(), core::ptr::null_mut()); + let ret = bindings::drm_gem_shmem_vmap_locked(self.as_shmem(), map.as_mut_ptr()); + bindings::dma_resv_unlock(self.raw_dma_resv()); + ret + })?; + + // SAFETY: if drm_gem_shmem_vmap did not fail, map is initialized now + Ok(unsafe { RawIoSysMap::from_raw(map.assume_init()) }) + } + + /// Unmap a [`RawIoSysMap`] from the gem object. + /// + /// # Safety + /// + /// - The caller promises that `map` came from a prior call to [`Self::raw_vmap`] on this gem + /// object. + /// - The caller promises that the memory pointed to by `map` will no longer be accesed through + /// this instance. + unsafe fn raw_vunmap(&self, map: &mut RawIoSysMap) { + let resv = self.raw_dma_resv(); + + // SAFETY: + // - This function is safe to call with the DMA reservation lock held + // - Our `ARef` is proof that the underlying gem object here is initialized and thus safe to + // dereference. + unsafe { + // TODO: see top of file + bindings::dma_resv_lock(resv, core::ptr::null_mut()); + bindings::drm_gem_shmem_vunmap_locked(self.as_shmem(), map.as_raw_mut()); + bindings::dma_resv_unlock(resv); + } + } + + /// Creates and returns a virtual kernel memory mapping for this object. + pub fn vmap(&self) -> Result> { + let map = self.raw_vmap()?; + + Ok(VMapRef { + // SAFETY: + // - The size of the vmap is the same as the size of the gem + // - The vmap will remain alive until this object is dropped. + map: unsafe { IoSysMapRef::new(map, self.size()) }, + owner: self, + }) + } + + /// Creates and returns an owned reference to a virtual kernel memory mapping for this object. + pub fn owned_vmap(&self) -> Result> { + Ok(VMap { + map: self.raw_vmap()?, + owner: self.into(), + }) + } +} + +impl Deref for Object { + type Target = T; + + fn deref(&self) -> &Self::Target { + &self.inner + } +} + +impl DerefMut for Object { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.inner + } +} + +impl Sealed for Object {} + +impl gem::IntoGEMObject for Object { + fn as_raw(&self) -> *mut bindings::drm_gem_object { + // SAFETY: + // - Our immutable reference is proof that this is safe to dereference. + // - `obj` is always a valid drm_gem_shmem_object via our type invariants. + unsafe { &raw mut (*self.obj.get()).base } + } + + unsafe fn from_raw<'a>(obj: *mut bindings::drm_gem_object) -> &'a Object { + // SAFETY: The safety contract of from_gem_obj() guarantees that `obj` is contained within + // `Self` + unsafe { + let obj = Opaque::cast_from(container_of!(obj, bindings::drm_gem_shmem_object, base)); + + &*container_of!(obj, Object, obj) + } + } +} + +impl driver::AllocImpl for Object { + type Driver = T::Driver; + + const ALLOC_OPS: driver::AllocOps = driver::AllocOps { + gem_create_object: None, + prime_handle_to_fd: None, + prime_fd_to_handle: None, + gem_prime_import: None, + gem_prime_import_sg_table: Some(bindings::drm_gem_shmem_prime_import_sg_table), + dumb_create: Some(bindings::drm_gem_shmem_dumb_create), + dumb_map_offset: None, + }; +} + +/// A borrowed reference to a virtual mapping for a shmem-based GEM object in kernel address space. +pub struct VMapRef<'a, D: DriverObject, T: AsBytes + FromBytes> { + map: IoSysMapRef<'a, T>, + owner: &'a Object, +} + +impl<'a, D: DriverObject, T: AsBytes + FromBytes> Clone for VMapRef<'a, D, T> { + fn clone(&self) -> Self { + // SAFETY: We have a successful vmap already, so this can't fail + unsafe { self.owner.vmap().unwrap_unchecked() } + } +} + +impl<'a, D: DriverObject, T: AsBytes + FromBytes> Deref for VMapRef<'a, D, T> { + type Target = IoSysMapRef<'a, T>; + + fn deref(&self) -> &Self::Target { + &self.map + } +} + +impl<'a, D: DriverObject, T: AsBytes + FromBytes> DerefMut for VMapRef<'a, D, T> { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.map + } +} + +impl<'a, D: DriverObject, T: AsBytes + FromBytes> Drop for VMapRef<'a, D, T> { + fn drop(&mut self) { + // SAFETY: Our existence is proof that this map was previously created using self.owner. + unsafe { self.owner.raw_vunmap(&mut self.map) }; + } +} + +/// An owned reference to a virtual mapping for a shmem-based GEM object in kernel address space. +/// +/// # Invariants +/// +/// - The memory pointed to by `map` is at least as large as `T`. +/// - The memory pointed to by `map` remains valid at least until this object is dropped. +pub struct VMap { + map: RawIoSysMap, + owner: ARef>, +} + +impl Clone for VMap { + fn clone(&self) -> Self { + // SAFETY: We have a successful vmap already, so this can't fail + unsafe { self.owner.owned_vmap().unwrap_unchecked() } + } +} + +impl<'a, D: DriverObject, T: AsBytes + FromBytes> From> for VMap { + fn from(value: VMapRef<'a, D, T>) -> Self { + let this = Self { + map: value.map.clone(), + owner: value.owner.into(), + }; + + mem::forget(value); + this + } +} + +impl VMap { + /// Return a reference to the iosys map for this `VMap`. + pub fn get(&self) -> IoSysMapRef<'_, T> { + // SAFETY: The size of the iosys_map is equivalent to the size of the gem object. + unsafe { IoSysMapRef::new(self.map.clone(), self.owner.size()) } + } + + /// Borrows a reference to the object that owns this virtual mapping. + pub fn owner(&self) -> &Object { + &self.owner + } +} + +impl Drop for VMap { + fn drop(&mut self) { + // SAFETY: Our existence is proof that this map was previously created using self.owner + unsafe { self.owner.raw_vunmap(&mut self.map) }; + } +} + +/// SAFETY: `iosys_map` objects are safe to send across threads. +unsafe impl Send for VMap {} +/// SAFETY: `iosys_map` objects are safe to send across threads. +unsafe impl Sync for VMap {} + +/// An owned reference to a scatter-gather table of DMA address spans for a GEM shmem object. +/// +/// This object holds an owned reference to the underlying GEM shmem object, ensuring that the +/// [`scatterlist::SGTable`] referenced by this type remains valid for the lifetime of this object. +/// +/// # Invariants +/// +/// - `sgt` is kept alive by `_owner`, ensuring it remains valid for as long as `Self`. +/// - `sgt` corresponds to the owned object in `_owner`. +/// - This object is only exposed in situations where we know the underlying `SGTable` will not be +/// modified for the lifetime of this object. Thus, it is safe to send/access this type across +/// threads. +pub struct SGTable { + sgt: NonNull, + _owner: ARef>, +} + +// SAFETY: This object is thread-safe via our type invariants. +unsafe impl Send for SGTable {} +// SAFETY: This object is thread-safe via our type invariants. +unsafe impl Sync for SGTable {} + +impl Deref for SGTable { + type Target = scatterlist::SGTable; + + fn deref(&self) -> &Self::Target { + // SAFETY: Creating an immutable reference to this is safe via our type invariants. + unsafe { self.sgt.as_ref() } + } +} diff --git a/rust/kernel/drm/gpuvm.rs b/rust/kernel/drm/gpuvm.rs new file mode 100644 index 00000000000000..04348b4e6d3933 --- /dev/null +++ b/rust/kernel/drm/gpuvm.rs @@ -0,0 +1,835 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT + +//! DRM Sync Objects +//! +//! C header: [`include/drm/drm_gpuvm.h`](../../../../include/drm/drm_gpuvm.h) + +#![allow(missing_docs)] + +use crate::{ + bindings, drm, + drm::{ + device, + gem::{ + BaseObject, + IntoGEMObject, // + }, + }, + error::{ + code::{EINVAL, ENOMEM}, + from_result, to_result, Error, Result, + }, + prelude::*, + types::{ARef, AlwaysRefCounted, Opaque}, +}; + +use core::cell::UnsafeCell; +use core::marker::{PhantomData, PhantomPinned}; +use core::mem::ManuallyDrop; +use core::ops::{Deref, DerefMut, Range}; +use core::ptr::NonNull; +use pin_init; + +/// GpuVaFlags to be used for a GpuVa. +/// +/// They can be combined with the operators `|`, `&`, and `!`. +#[derive(Clone, Copy, PartialEq, Default)] +pub struct GpuVaFlags(u32); + +impl GpuVaFlags { + /// No GpuVaFlags (zero) + pub const NONE: GpuVaFlags = GpuVaFlags(0); + + /// The backing GEM is invalidated. + pub const INVALIDATED: GpuVaFlags = GpuVaFlags(bindings::drm_gpuva_flags_DRM_GPUVA_INVALIDATED); + + /// The GpuVa is a sparse mapping. + pub const SPARSE: GpuVaFlags = GpuVaFlags(bindings::drm_gpuva_flags_DRM_GPUVA_SPARSE); + + /// The GpuVa is a repeat mapping. + pub const REPEAT: GpuVaFlags = GpuVaFlags(bindings::drm_gpuva_flags_DRM_GPUVA_REPEAT); + + /// Construct a driver-specific GpuVaFlag. + /// + /// The argument must be a flag index in the range [0..28]. + pub const fn user_flag(index: u32) -> GpuVaFlags { + let flags = bindings::drm_gpuva_flags_DRM_GPUVA_USERBITS << index; + assert!(flags != 0); + GpuVaFlags(flags) + } + + /// Get the raw representation of this flag. + pub(crate) fn as_raw(self) -> u32 { + self.0 + } + + /// Check whether `flags` is contained in `self`. + pub fn contains(self, flags: GpuVaFlags) -> bool { + (self & flags) == flags + } +} + +impl core::ops::BitOr for GpuVaFlags { + type Output = Self; + fn bitor(self, rhs: Self) -> Self::Output { + Self(self.0 | rhs.0) + } +} + +impl core::ops::BitAnd for GpuVaFlags { + type Output = Self; + fn bitand(self, rhs: Self) -> Self::Output { + Self(self.0 & rhs.0) + } +} + +impl core::ops::Not for GpuVaFlags { + type Output = Self; + fn not(self) -> Self::Output { + Self(!self.0) + } +} + +/// Trait that must be implemented by DRM drivers to represent a DRM GpuVm (a GPU address space). +pub trait DriverGpuVm: Sized { + /// The parent `Driver` implementation for this `DriverGpuVm`. + type Driver: drm::Driver; + type GpuVa: DriverGpuVa = (); + type GpuVmBo: DriverGpuVmBo = (); + type StepContext = (); + + fn step_map( + self: &mut UpdatingGpuVm<'_, Self>, + op: &mut OpMap, + ctx: &mut Self::StepContext, + ) -> Result; + fn step_unmap( + self: &mut UpdatingGpuVm<'_, Self>, + op: &mut OpUnMap, + ctx: &mut Self::StepContext, + ) -> Result; + fn step_remap( + self: &mut UpdatingGpuVm<'_, Self>, + op: &mut OpReMap, + vm_bo: &GpuVmBo, + ctx: &mut Self::StepContext, + ) -> Result; +} + +struct StepContext<'a, T: DriverGpuVm> { + gpuvm: &'a GpuVm, + ctx: &'a mut T::StepContext, +} + +/// Trait that must be implemented by DRM drivers to represent a DRM GpuVa (a mapping in GPU address space). +pub trait DriverGpuVa: Sized {} + +impl DriverGpuVa for () {} + +/// Trait that must be implemented by DRM drivers to represent a DRM GpuVmBo (a connection between a BO and a VM). +pub trait DriverGpuVmBo: Sized { + fn new() -> impl PinInit; +} + +/// Provide a default implementation for trivial types +impl DriverGpuVmBo for T { + fn new() -> impl PinInit { + pin_init::default() + } +} + +/// A convenience type for the driver's GEM object. +type Object = <::Driver as drm::driver::Driver>::Object; + +#[repr(transparent)] +pub struct OpMap(bindings::drm_gpuva_op_map, PhantomData); +#[repr(transparent)] +pub struct OpUnMap(bindings::drm_gpuva_op_unmap, PhantomData); +#[repr(transparent)] +pub struct OpReMap(bindings::drm_gpuva_op_remap, PhantomData); + +impl OpMap { + pub fn addr(&self) -> u64 { + self.0.va.addr + } + pub fn range(&self) -> u64 { + self.0.va.range + } + pub fn offset(&self) -> u64 { + self.0.gem.offset + } + pub fn flags(&self) -> GpuVaFlags { + GpuVaFlags(self.0.flags) + } + pub fn object(&self) -> &Object { + let p = unsafe { as IntoGEMObject>::from_raw(self.0.gem.obj) }; + // SAFETY: The GEM object has an active reference for the lifetime of this op + &*p + } + pub fn map_and_link_va( + &mut self, + gpuvm: &mut UpdatingGpuVm<'_, T>, + gpuva: Pin>>, + gpuvmbo: &GpuVmBo, + ) -> Result<(), Pin>>> { + // SAFETY: We are handing off the GpuVa ownership and it will not be moved. + let p = KBox::leak(unsafe { Pin::into_inner_unchecked(gpuva) }); + // SAFETY: These C functions are called with the correct invariants + unsafe { + bindings::drm_gpuva_init_from_op(&mut p.gpuva, &mut self.0); + if bindings::drm_gpuva_insert(gpuvm.0.gpuvm() as *mut _, &mut p.gpuva) != 0 { + // EEXIST, return the GpuVa to the caller as an error + return Err(Pin::new_unchecked(KBox::from_raw(p))); + }; + // SAFETY: This takes a new reference to the gpuvmbo. + gpuvmbo.lock_gpuva(); + bindings::drm_gpuva_link(&mut p.gpuva, &gpuvmbo.bo as *const _ as *mut _); + gpuvmbo.unlock_gpuva(); + } + Ok(()) + } +} + +impl OpUnMap { + pub fn va(&self) -> Option<&GpuVa> { + if self.0.va.is_null() { + return None; + } + // SAFETY: Container invariant is guaranteed for ops structs created for our types. + let p = unsafe { crate::container_of!(self.0.va, GpuVa, gpuva) as *mut GpuVa }; + // SAFETY: The GpuVa object reference is valid per the op_unmap contract + Some(unsafe { &*p }) + } + pub fn unmap_and_unlink_va(&mut self) -> Option>>> { + self.do_unmap_and_unlink_va(false) + } + pub fn unmap_and_unlink_va_defer(&mut self) -> Option>>> { + self.do_unmap_and_unlink_va(true) + } + fn do_unmap_and_unlink_va(&mut self, defer: bool) -> Option>>> { + if self.0.va.is_null() { + return None; + } + // SAFETY: Container invariant is guaranteed for ops structs created for our types. + let p = unsafe { crate::container_of!(self.0.va, GpuVa, gpuva) as *mut GpuVa }; + + // SAFETY: The GpuVa object reference is valid per the op_unmap contract + unsafe { + bindings::drm_gpuva_unmap(&mut self.0); + if defer { + bindings::drm_gpuva_unlink_defer(self.0.va); + } else { + bindings::drm_gpuva_unlink(self.0.va); + } + } + + // Unlinking/unmapping relinquishes ownership of the GpuVa object, + // so clear the pointer + self.0.va = core::ptr::null_mut(); + // SAFETY: The GpuVa object reference is valid per the op_unmap contract + Some(unsafe { Pin::new_unchecked(KBox::from_raw(p)) }) + } +} + +impl OpReMap { + pub fn prev_map(&mut self) -> Option<&mut OpMap> { + // SAFETY: The prev pointer must be valid if not-NULL per the op_remap contract + unsafe { (self.0.prev as *mut OpMap).as_mut() } + } + pub fn next_map(&mut self) -> Option<&mut OpMap> { + // SAFETY: The next pointer must be valid if not-NULL per the op_remap contract + unsafe { (self.0.next as *mut OpMap).as_mut() } + } + pub fn unmap(&mut self) -> &mut OpUnMap { + // SAFETY: The unmap pointer is always valid per the op_remap contract + unsafe { (self.0.unmap as *mut OpUnMap).as_mut().unwrap() } + } +} + +/// A base GPU VA. +#[repr(C)] +#[pin_data] +pub struct GpuVa { + #[pin] + gpuva: bindings::drm_gpuva, + #[pin] + inner: T::GpuVa, + #[pin] + _p: PhantomPinned, +} + +impl GpuVa { + pub fn new(inner: impl PinInit) -> Result>>> + where + Error: From, + { + KBox::try_pin_init( + try_pin_init!(Self { + gpuva <- pin_init::init_zeroed(), + inner <- inner, + _p: PhantomPinned + }), + GFP_KERNEL, + ) + } + + pub fn addr(&self) -> u64 { + self.gpuva.va.addr + } + pub fn range(&self) -> u64 { + self.gpuva.va.range + } + pub fn offset(&self) -> u64 { + self.gpuva.gem.offset + } + pub fn flags(&self) -> GpuVaFlags { + GpuVaFlags(self.gpuva.flags) + } +} + +/// A base GpuVm BO. +#[repr(C)] +#[pin_data] +pub struct GpuVmBo { + #[pin] + bo: bindings::drm_gpuvm_bo, + #[pin] + inner: T::GpuVmBo, + #[pin] + _p: PhantomPinned, +} + +impl GpuVmBo { + /// Return a reference to the inner driver data for this GpuVmBo + pub fn inner(&self) -> &T::GpuVmBo { + &self.inner + } + /// Lock the GpuVmBo's gem boject gpuva lock + pub fn lock_gpuva(&self) { + unsafe { + let lock = &raw mut (*self.bo.obj).gpuva.lock; + bindings::mutex_lock(lock); + } + } + /// Unlock the GpuVmBo's gem boject gpuva lock + pub fn unlock_gpuva(&self) { + unsafe { + let lock = &raw mut (*self.bo.obj).gpuva.lock; + bindings::mutex_unlock(lock); + } + } +} + +// SAFETY: DRM GpuVmBo objects are always reference counted and the get/put functions +// satisfy the requirements. +unsafe impl AlwaysRefCounted for GpuVmBo { + fn inc_ref(&self) { + // SAFETY: The drm_gpuvm_get function satisfies the requirements for inc_ref(). + unsafe { bindings::drm_gpuvm_bo_get(&self.bo as *const _ as *mut _) }; + } + + unsafe fn dec_ref(mut obj: NonNull) { + // SAFETY: drm_gpuvm_bo_put() requires holding the gpuva lock, which is the dma_resv lock by default. + // The drm_gpuvm_put function satisfies the requirements for dec_ref(). + // (We do not support custom locks yet.) + unsafe { + let resv = (*obj.as_mut().bo.obj).resv; + bindings::dma_resv_lock(resv, core::ptr::null_mut()); + bindings::drm_gpuvm_bo_put(&mut obj.as_mut().bo); + bindings::dma_resv_unlock(resv); + } + } +} + +/// A base GPU VM. +#[repr(C)] +#[pin_data] +pub struct GpuVm { + #[pin] + gpuvm: Opaque, + #[pin] + inner: UnsafeCell, + #[pin] + _p: PhantomPinned, +} + +pub(super) unsafe extern "C" fn vm_free_callback( + raw_gpuvm: *mut bindings::drm_gpuvm, +) { + // SAFETY: Container invariant is guaranteed for objects using our callback. + let p = unsafe { + crate::container_of!( + raw_gpuvm as *mut Opaque, + GpuVm, + gpuvm + ) as *mut GpuVm + }; + + // SAFETY: p is guaranteed to be valid for drm_gpuvm objects using this callback. + unsafe { drop(KBox::from_raw(p)) }; +} + +pub(super) unsafe extern "C" fn vm_bo_alloc_callback() -> *mut bindings::drm_gpuvm_bo +{ + let obj: Result>>> = KBox::try_pin_init( + try_pin_init!(GpuVmBo:: { + bo <- pin_init::default(), + inner <- T::GpuVmBo::new(), + _p: PhantomPinned + }), + GFP_KERNEL, + ); + + match obj { + Ok(obj) => + // SAFETY: The DRM core will keep this object pinned + unsafe { + let p = KBox::leak(Pin::into_inner_unchecked(obj)); + &mut p.bo + }, + Err(_) => core::ptr::null_mut(), + } +} + +pub(super) unsafe extern "C" fn vm_bo_free_callback( + raw_vm_bo: *mut bindings::drm_gpuvm_bo, +) { + // SAFETY: Container invariant is guaranteed for objects using this callback. + let p = unsafe { crate::container_of!(raw_vm_bo, GpuVmBo, bo) as *mut GpuVmBo }; + + // SAFETY: p is guaranteed to be valid for drm_gpuvm_bo objects using this callback. + unsafe { drop(KBox::from_raw(p)) }; +} + +pub(super) unsafe extern "C" fn step_map_callback( + op: *mut bindings::drm_gpuva_op, + _priv: *mut core::ffi::c_void, +) -> core::ffi::c_int { + // SAFETY: We know this is a map op, and OpMap is a transparent wrapper. + let map = unsafe { &mut *((&mut (*op).__bindgen_anon_1.map) as *mut _ as *mut OpMap) }; + // SAFETY: This is a pointer to a StepContext created inline in sm_map(), which is + // guaranteed to outlive this function. + let ctx = unsafe { &mut *(_priv as *mut StepContext<'_, T>) }; + + from_result(|| { + UpdatingGpuVm(ctx.gpuvm).step_map(map, ctx.ctx)?; + Ok(0) + }) +} + +pub(super) unsafe extern "C" fn step_remap_callback( + op: *mut bindings::drm_gpuva_op, + _priv: *mut core::ffi::c_void, +) -> core::ffi::c_int { + // SAFETY: We know this is a map op, and OpReMap is a transparent wrapper. + let remap = unsafe { &mut *((&mut (*op).__bindgen_anon_1.remap) as *mut _ as *mut OpReMap) }; + // SAFETY: This is a pointer to a StepContext created inline in sm_map(), which is + // guaranteed to outlive this function. + let ctx = unsafe { &mut *(_priv as *mut StepContext<'_, T>) }; + + let p_vm_bo = remap.unmap().va().unwrap().gpuva.vm_bo; + + let res = { + // SAFETY: vm_bo pointer must be valid and non-null by the step_remap invariants. + // Since we grab a ref, this reference's lifetime is until the decref. + let vm_bo_ref = unsafe { + bindings::drm_gpuvm_bo_get(p_vm_bo); + &*(crate::container_of!(p_vm_bo, GpuVmBo, bo) as *mut GpuVmBo) + }; + + from_result(|| { + UpdatingGpuVm(ctx.gpuvm).step_remap(remap, vm_bo_ref, ctx.ctx)?; + Ok(0) + }) + }; + + // SAFETY: We incremented the refcount above, and the Rust reference we took is + // no longer in scope. + unsafe { bindings::drm_gpuvm_bo_put_deferred(p_vm_bo) }; + + res +} +pub(super) unsafe extern "C" fn step_unmap_callback( + op: *mut bindings::drm_gpuva_op, + _priv: *mut core::ffi::c_void, +) -> core::ffi::c_int { + // SAFETY: We know this is a map op, and OpUnMap is a transparent wrapper. + let unmap = unsafe { &mut *((&mut (*op).__bindgen_anon_1.unmap) as *mut _ as *mut OpUnMap) }; + // SAFETY: This is a pointer to a StepContext created inline in sm_map(), which is + // guaranteed to outlive this function. + let ctx = unsafe { &mut *(_priv as *mut StepContext<'_, T>) }; + + from_result(|| { + UpdatingGpuVm(ctx.gpuvm).step_unmap(unmap, ctx.ctx)?; + Ok(0) + }) +} + +pub(super) unsafe extern "C" fn exec_lock_gem_object( + vm_exec: *mut bindings::drm_gpuvm_exec, +) -> core::ffi::c_int { + // SAFETY: The gpuvm_exec object is valid and priv_ is a GEM object pointer + // when this callback is used + unsafe { bindings::drm_exec_lock_obj(&mut (*vm_exec).exec, (*vm_exec).extra.priv_ as *mut _) } +} + +impl GpuVm { + const OPS: bindings::drm_gpuvm_ops = bindings::drm_gpuvm_ops { + vm_free: Some(vm_free_callback::), + op_alloc: None, + op_free: None, + vm_bo_alloc: Some(vm_bo_alloc_callback::), + vm_bo_free: Some(vm_bo_free_callback::), + vm_bo_validate: None, + sm_step_map: Some(step_map_callback::), + sm_step_remap: Some(step_remap_callback::), + sm_step_unmap: Some(step_unmap_callback::), + sm_can_merge_flags: None, + }; + + fn gpuvm(&self) -> *const bindings::drm_gpuvm { + self.gpuvm.get() + } + + pub fn new( + name: &'static CStr, + flags: bindings::drm_gpuvm_flags, + dev: &device::Device, + r_obj: ARef>, + range: Range, + reserve_range: Range, + inner: impl PinInit, + ) -> Result>> + where + Error: From, + { + let obj: Pin> = KBox::try_pin_init( + try_pin_init!(Self { + // SAFETY: drm_gpuvm_init cannot fail and always initializes the member + gpuvm <- unsafe { + pin_init::pin_init_from_closure(move |slot: *mut Opaque | { + // Zero-init required by drm_gpuvm_init + *slot = Opaque::zeroed(); + bindings::drm_gpuvm_init( + Opaque::cast_into(slot), + name.as_char_ptr(), + flags, + dev.as_raw(), + r_obj.as_raw() as *const _ as *mut _, + range.start, + range.end - range.start, + reserve_range.start, + reserve_range.end - reserve_range.start, + &Self::OPS + ); + Ok(()) + }) + }, + // SAFETY: Just passing through to the initializer argument + inner <- unsafe { + pin_init::pin_init_from_closure(move |slot: *mut UnsafeCell | { + inner.__pinned_init(slot as *mut _) + }) + }, + _p: PhantomPinned + }), + GFP_KERNEL, + )?; + + // SAFETY: We never move out of the object + let vm_ref = unsafe { + ARef::from_raw(NonNull::new_unchecked(KBox::leak( + Pin::into_inner_unchecked(obj), + ))) + }; + + Ok(vm_ref) + } + + pub fn exec_lock<'a, 'b>( + &'a self, + obj: Option<&'b Object>, + interruptible: bool, + ) -> Result> { + // Do not try to lock the object if it is internal (since it is already locked). + let is_ext = obj.map(|a| self.is_extobj(a)).unwrap_or(false); + + let mut guard = ManuallyDrop::new(LockedGpuVm { + gpuvm: self, + // vm_exec needs to be pinned, so stick it in a Box. + vm_exec: KBox::init( + init!(bindings::drm_gpuvm_exec { + vm: self.gpuvm() as *mut _, + flags: if interruptible { + bindings::DRM_EXEC_INTERRUPTIBLE_WAIT + } else { + 0 + }, + exec: Default::default(), + extra: match (is_ext, obj) { + (true, Some(obj)) => bindings::drm_gpuvm_exec__bindgen_ty_1 { + fn_: Some(exec_lock_gem_object), + priv_: obj.as_raw() as *const _ as *mut _, + }, + _ => Default::default(), + }, + num_fences: 0, + }), + GFP_KERNEL, + )?, + obj, + }); + + // SAFETY: The object is valid and was initialized above + to_result(unsafe { bindings::drm_gpuvm_exec_lock(&mut *guard.vm_exec) })?; + + Ok(ManuallyDrop::into_inner(guard)) + } + + /// Returns true if the given object is external to the GPUVM + /// (that is, if it does not share the DMA reservation object of the GPUVM). + pub fn is_extobj(&self, obj: &impl IntoGEMObject) -> bool { + let gem = obj.as_raw() as *const _ as *mut _; + // SAFETY: This is safe to call as long as the arguments are valid pointers. + unsafe { bindings::drm_gpuvm_is_extobj(self.gpuvm() as *mut _, gem) } + } + + pub fn bo_deferred_cleanup(&self) { + unsafe { bindings::drm_gpuvm_bo_deferred_cleanup(self.gpuvm() as *mut _) } + } + + pub fn find_bo(&self, obj: &Object) -> Option>> { + obj.lock_gpuva(); + // SAFETY: drm_gem_object.gpuva.lock was just locked. + let p = unsafe { + bindings::drm_gpuvm_bo_find(self.gpuvm() as *mut _, obj.as_raw() as *const _ as *mut _) + }; + obj.unlock_gpuva(); + if p.is_null() { + None + } else { + // SAFETY: All the drm_gpuvm_bo objects in this GpuVm are always allocated by us as GpuVmBo. + let p = unsafe { crate::container_of!(p, GpuVmBo, bo) as *mut GpuVmBo }; + // SAFETY: We checked for NULL above, and the types ensure that + // this object was created by vm_bo_alloc_callback. + Some(unsafe { ARef::from_raw(NonNull::new_unchecked(p)) }) + } + } + + pub fn obtain_bo(&self, obj: &Object) -> Result>> { + obj.lock_gpuva(); + // SAFETY: drm_gem_object.gpuva.lock was just locked. + let p = unsafe { + bindings::drm_gpuvm_bo_obtain( + self.gpuvm() as *mut _, + obj.as_raw() as *const _ as *mut _, + ) + }; + obj.unlock_gpuva(); + if p.is_null() { + Err(ENOMEM) + } else { + // SAFETY: Container invariant is guaranteed for GpuVmBo objects for this GpuVm. + let p = unsafe { crate::container_of!(p, GpuVmBo, bo) as *mut GpuVmBo }; + // SAFETY: We checked for NULL above, and the types ensure that + // this object was created by vm_bo_alloc_callback. + Ok(unsafe { ARef::from_raw(NonNull::new_unchecked(p)) }) + } + } + + pub fn bo_unmap(&self, ctx: &mut T::StepContext, bo: &GpuVmBo) -> Result { + let mut ctx = StepContext { ctx, gpuvm: self }; + // SAFETY: LockedGpuVm implies the right locks are held. + to_result(unsafe { + bindings::drm_gpuvm_bo_unmap(&bo.bo as *const _ as *mut _, &mut ctx as *mut _ as *mut _) + }) + } +} + +// SAFETY: DRM GpuVm objects are always reference counted and the get/put functions +// satisfy the requirements. +unsafe impl AlwaysRefCounted for GpuVm { + fn inc_ref(&self) { + // SAFETY: The drm_gpuvm_get function satisfies the requirements for inc_ref(). + unsafe { bindings::drm_gpuvm_get(&self.gpuvm as *const _ as *mut _) }; + } + + unsafe fn dec_ref(obj: NonNull) { + // SAFETY: The drm_gpuvm_put function satisfies the requirements for dec_ref(). + unsafe { bindings::drm_gpuvm_put(Opaque::cast_into(&(*obj.as_ptr()).gpuvm)) }; + } +} + +pub struct LockedGpuVm<'a, 'b, T: DriverGpuVm> { + gpuvm: &'a GpuVm, + vm_exec: KBox, + obj: Option<&'b Object>, +} + +impl LockedGpuVm<'_, '_, T> { + pub fn find_bo(&mut self) -> Option>> { + let obj = self.obj?; + // SAFETY: LockedGpuVm implies the right locks are held. + let p = unsafe { + bindings::drm_gpuvm_bo_find( + self.gpuvm.gpuvm() as *mut _, + obj.as_raw() as *const _ as *mut _, + ) + }; + if p.is_null() { + None + } else { + // SAFETY: All the drm_gpuvm_bo objects in this GpuVm are always allocated by us as GpuVmBo. + let p = unsafe { crate::container_of!(p, GpuVmBo, bo) as *mut GpuVmBo }; + // SAFETY: We checked for NULL above, and the types ensure that + // this object was created by vm_bo_alloc_callback. + Some(unsafe { ARef::from_raw(NonNull::new_unchecked(p)) }) + } + } + + pub fn obtain_bo(&mut self) -> Result>> { + let obj = self.obj.ok_or(EINVAL)?; + // SAFETY: LockedGpuVm implies the right locks are held. + let p = unsafe { + bindings::drm_gpuvm_bo_obtain( + self.gpuvm.gpuvm() as *mut _, + obj.as_raw() as *const _ as *mut _, + ) + }; + if p.is_null() { + Err(ENOMEM) + } else { + // SAFETY: Container invariant is guaranteed for GpuVmBo objects for this GpuVm. + let p = unsafe { crate::container_of!(p, GpuVmBo, bo) as *mut GpuVmBo }; + // SAFETY: We checked for NULL above, and the types ensure that + // this object was created by vm_bo_alloc_callback. + Ok(unsafe { ARef::from_raw(NonNull::new_unchecked(p)) }) + } + } + + pub fn sm_map( + &mut self, + ctx: &mut T::StepContext, + req_addr: u64, + req_range: u64, + req_offset: u64, + req_gem_range: u32, + flags: GpuVaFlags, + ) -> Result { + let obj = self.obj.ok_or(EINVAL)?; + let mut ctx = StepContext { + ctx, + gpuvm: self.gpuvm, + }; + + let req = bindings::drm_gpuvm_map_req { + map: bindings::drm_gpuva_op_map { + va: bindings::drm_gpuva_op_map__bindgen_ty_1 { + addr: req_addr, + range: req_range, + }, + gem: bindings::drm_gpuva_op_map__bindgen_ty_2 { + offset: req_offset, + range: req_gem_range, + obj: obj.as_raw(), + }, + flags: flags.as_raw(), + }, + }; + + // SAFETY: LockedGpuVm implies the right locks are held. + to_result(unsafe { + bindings::drm_gpuvm_sm_map( + self.gpuvm.gpuvm() as *mut _, + &mut ctx as *mut _ as *mut _, + &raw const req, + ) + }) + } + + pub fn sm_unmap(&mut self, ctx: &mut T::StepContext, req_addr: u64, req_range: u64) -> Result { + let mut ctx = StepContext { + ctx, + gpuvm: self.gpuvm, + }; + // SAFETY: LockedGpuVm implies the right locks are held. + to_result(unsafe { + bindings::drm_gpuvm_sm_unmap( + self.gpuvm.gpuvm() as *mut _, + &mut ctx as *mut _ as *mut _, + req_addr, + req_range, + ) + }) + } + + pub fn bo_unmap(&mut self, ctx: &mut T::StepContext, bo: &GpuVmBo) -> Result { + let mut ctx = StepContext { + ctx, + gpuvm: self.gpuvm, + }; + // SAFETY: LockedGpuVm implies the right locks are held. + to_result(unsafe { + bindings::drm_gpuvm_bo_unmap(&bo.bo as *const _ as *mut _, &mut ctx as *mut _ as *mut _) + }) + } +} + +impl Deref for LockedGpuVm<'_, '_, T> { + type Target = T; + + fn deref(&self) -> &T { + // SAFETY: The existence of this LockedGpuVm implies the lock is held, + // so this is the only reference + unsafe { &*self.gpuvm.inner.get() } + } +} + +impl DerefMut for LockedGpuVm<'_, '_, T> { + fn deref_mut(&mut self) -> &mut T { + // SAFETY: The existence of this UpdatingGpuVm implies the lock is held, + // so this is the only reference + unsafe { &mut *self.gpuvm.inner.get() } + } +} + +impl Drop for LockedGpuVm<'_, '_, T> { + fn drop(&mut self) { + // SAFETY: We hold the lock, so it's safe to unlock + unsafe { + bindings::drm_gpuvm_exec_unlock(&mut *self.vm_exec); + } + } +} + +pub struct UpdatingGpuVm<'a, T: DriverGpuVm>(&'a GpuVm); + +impl UpdatingGpuVm<'_, T> {} + +impl Deref for UpdatingGpuVm<'_, T> { + type Target = T; + + fn deref(&self) -> &T { + // SAFETY: The existence of this UpdatingGpuVm implies the lock is held, + // so this is the only reference + unsafe { &*self.0.inner.get() } + } +} + +impl DerefMut for UpdatingGpuVm<'_, T> { + fn deref_mut(&mut self) -> &mut T { + // SAFETY: The existence of this UpdatingGpuVm implies the lock is held, + // so this is the only reference + unsafe { &mut *self.0.inner.get() } + } +} + +// SAFETY: All our trait methods take locks +unsafe impl Sync for GpuVm {} +// SAFETY: All our trait methods take locks +unsafe impl Send for GpuVm {} + +// SAFETY: All our trait methods take locks +unsafe impl Sync for GpuVmBo {} +// SAFETY: All our trait methods take locks +unsafe impl Send for GpuVmBo {} diff --git a/rust/kernel/drm/mm.rs b/rust/kernel/drm/mm.rs new file mode 100644 index 00000000000000..7b13cfd7d53095 --- /dev/null +++ b/rust/kernel/drm/mm.rs @@ -0,0 +1,310 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT + +//! DRM MM range allocator +//! +//! C header: [`include/drm/drm_mm.h`](../../../../include/drm/drm_mm.h) + +use crate::{ + alloc::flags::*, + bindings, + error::{to_result, Result}, + sync::{new_mutex, Arc, Mutex, UniqueArc}, + types::Opaque, +}; + +use crate::init::InPlaceInit; +use crate::prelude::KBox; + +use core::{ + marker::{PhantomData, PhantomPinned}, + ops::Deref, + pin::Pin, +}; + +/// Type alias representing a DRM MM node. +pub type Node = Pin>>; + +/// Trait which must be implemented by the inner allocator state type provided by the user. +pub trait AllocInner { + /// Notification that a node was dropped from the allocator. + fn drop_object(&mut self, _start: u64, _size: u64, _color: usize, _object: &mut T) {} +} + +impl AllocInner for () {} + +/// Wrapper type for a `struct drm_mm` plus user AllocInner object. +/// +/// # Invariants +/// The `drm_mm` struct is valid and initialized. +struct MmInner, T>(Opaque, A, PhantomData); + +/// Represents a single allocated node in the MM allocator +pub struct NodeData, T> { + node: bindings::drm_mm_node, + mm: Arc>>, + valid: bool, + /// A drm_mm_node needs to be pinned because nodes reference each other in a linked list. + _pin: PhantomPinned, + inner: T, +} + +// SAFETY: Allocator ops take the mutex, and there are no mutable actions on the node. +unsafe impl, T: Send> Send for NodeData {} +// SAFETY: Allocator ops take the mutex, and there are no mutable actions on the node. +unsafe impl, T: Sync> Sync for NodeData {} + +/// Available MM node insertion modes +#[repr(u32)] +pub enum InsertMode { + /// Search for the smallest hole (within the search range) that fits the desired node. + /// + /// Allocates the node from the bottom of the found hole. + Best = bindings::drm_mm_insert_mode_DRM_MM_INSERT_BEST, + + /// Search for the lowest hole (address closest to 0, within the search range) that fits the + /// desired node. + /// + /// Allocates the node from the bottom of the found hole. + Low = bindings::drm_mm_insert_mode_DRM_MM_INSERT_LOW, + + /// Search for the highest hole (address closest to U64_MAX, within the search range) that fits + /// the desired node. + /// + /// Allocates the node from the top of the found hole. The specified alignment for the node is + /// applied to the base of the node (`Node.start()`). + High = bindings::drm_mm_insert_mode_DRM_MM_INSERT_HIGH, + + /// Search for the most recently evicted hole (within the search range) that fits the desired + /// node. This is appropriate for use immediately after performing an eviction scan and removing + /// the selected nodes to form a hole. + /// + /// Allocates the node from the bottom of the found hole. + Evict = bindings::drm_mm_insert_mode_DRM_MM_INSERT_EVICT, +} + +/// A clonable, interlocked reference to the allocator state. +/// +/// This is useful to perform actions on the user-supplied `AllocInner` type given just a Node, +/// without immediately taking the lock. +#[derive(Clone)] +pub struct InnerRef, T>(Arc>>); + +impl, T> InnerRef { + /// Operate on the user `AllocInner` implementation, taking the lock. + pub fn with(&self, cb: impl FnOnce(&mut A) -> RetVal) -> RetVal { + let mut l = self.0.lock(); + cb(&mut l.1) + } +} + +impl, T> NodeData { + /// Returns the color of the node (an opaque value) + pub fn color(&self) -> usize { + self.node.color as usize + } + + /// Returns the start address of the node + pub fn start(&self) -> u64 { + self.node.start + } + + /// Returns the size of the node in bytes + pub fn size(&self) -> u64 { + self.node.size + } + + /// Operate on the user `AllocInner` implementation associated with this node's allocator. + pub fn with_inner(&self, cb: impl FnOnce(&mut A) -> RetVal) -> RetVal { + let mut l = self.mm.lock(); + cb(&mut l.1) + } + + /// Return a clonable, detached reference to the allocator inner data. + pub fn alloc_ref(&self) -> InnerRef { + InnerRef(self.mm.clone()) + } + + /// Return a mutable reference to the inner data. + pub fn inner_mut(self: Pin<&mut Self>) -> &mut T { + // SAFETY: This is okay because inner is not structural + unsafe { &mut self.get_unchecked_mut().inner } + } +} + +impl, T> Deref for NodeData { + type Target = T; + + fn deref(&self) -> &Self::Target { + &self.inner + } +} + +impl, T> Drop for NodeData { + fn drop(&mut self) { + if self.valid { + let mut guard = self.mm.lock(); + + // Inform the user allocator that a node is being dropped. + guard + .1 + .drop_object(self.start(), self.size(), self.color(), &mut self.inner); + // SAFETY: The MM lock is still taken, so we can safely remove the node. + unsafe { bindings::drm_mm_remove_node(&mut self.node) }; + } + } +} + +/// An instance of a DRM MM range allocator. +pub struct Allocator, T> { + mm: Arc>>, + _p: PhantomData, +} + +impl, T> Allocator { + /// Create a new range allocator for the given start and size range of addresses. + /// + /// The user may optionally provide an inner object representing allocator state, which will + /// be protected by the same lock. If not required, `()` can be used. + #[track_caller] + pub fn new(start: u64, size: u64, inner: A) -> Result> { + // SAFETY: We call `Mutex::init_lock` below. + let mm = UniqueArc::pin_init( + new_mutex!(MmInner(Opaque::uninit(), inner, PhantomData)), + GFP_KERNEL, + )?; + + // SAFETY: The Opaque instance provides a valid pointer, and it is initialized after + // this call. + unsafe { + bindings::drm_mm_init(mm.lock().0.get(), start, size); + } + + Ok(Allocator { + mm: mm.into(), + _p: PhantomData, + }) + } + + /// Insert a new node into the allocator of a given size. + /// + /// `node` is the user `T` type data to store into the node. + pub fn insert_node(&mut self, node: T, size: u64) -> Result> { + self.insert_node_generic(node, size, 0, 0, InsertMode::Best) + } + + /// Insert a new node into the allocator of a given size, with configurable alignment, + /// color, and insertion mode. + /// + /// `node` is the user `T` type data to store into the node. + pub fn insert_node_generic( + &mut self, + node: T, + size: u64, + alignment: u64, + color: usize, + mode: InsertMode, + ) -> Result> { + self.insert_node_in_range(node, size, alignment, color, 0, u64::MAX, mode) + } + + /// Insert a new node into the allocator of a given size, with configurable alignment, + /// color, insertion mode, and sub-range to allocate from. + /// + /// `node` is the user `T` type data to store into the node. + #[allow(clippy::too_many_arguments)] + pub fn insert_node_in_range( + &mut self, + node: T, + size: u64, + alignment: u64, + color: usize, + start: u64, + end: u64, + mode: InsertMode, + ) -> Result> { + let mut mm_node = KBox::new( + NodeData { + // SAFETY: This C struct should be zero-initialized. + node: unsafe { core::mem::zeroed() }, + valid: false, + inner: node, + mm: self.mm.clone(), + _pin: PhantomPinned, + }, + GFP_KERNEL, + )?; + + let guard = self.mm.lock(); + // SAFETY: We hold the lock and all pointers are valid. + to_result(unsafe { + bindings::drm_mm_insert_node_in_range( + guard.0.get(), + &mut mm_node.node, + size, + alignment, + color, + start, + end, + mode as u32, + ) + })?; + + mm_node.valid = true; + + Ok(Pin::from(mm_node)) + } + + /// Insert a node into the allocator at a fixed start address. + /// + /// `node` is the user `T` type data to store into the node. + pub fn reserve_node( + &mut self, + node: T, + start: u64, + size: u64, + color: usize, + ) -> Result> { + let mut mm_node = KBox::new( + NodeData { + // SAFETY: This C struct should be zero-initialized. + node: unsafe { core::mem::zeroed() }, + valid: false, + inner: node, + mm: self.mm.clone(), + _pin: PhantomPinned, + }, + GFP_KERNEL, + )?; + + mm_node.node.start = start; + mm_node.node.size = size; + mm_node.node.color = color as crate::ffi::c_ulong; + + let guard = self.mm.lock(); + // SAFETY: We hold the lock and all pointers are valid. + to_result(unsafe { bindings::drm_mm_reserve_node(guard.0.get(), &mut mm_node.node) })?; + + mm_node.valid = true; + + Ok(Pin::from(mm_node)) + } + + /// Operate on the inner user type `A`, taking the allocator lock + pub fn with_inner(&self, cb: impl FnOnce(&mut A) -> RetVal) -> RetVal { + let mut guard = self.mm.lock(); + cb(&mut guard.1) + } +} + +impl, T> Drop for MmInner { + fn drop(&mut self) { + // SAFETY: If the MmInner is dropped then all nodes are gone (since they hold references), + // so it is safe to tear down the allocator. + unsafe { + bindings::drm_mm_takedown(self.0.get()); + } + } +} + +// SAFETY: MmInner is safely Send if the AllocInner user type is Send. +unsafe impl, T> Send for MmInner {} diff --git a/rust/kernel/drm/mod.rs b/rust/kernel/drm/mod.rs index 1b82b6945edf25..882841415aa414 100644 --- a/rust/kernel/drm/mod.rs +++ b/rust/kernel/drm/mod.rs @@ -6,7 +6,12 @@ pub mod device; pub mod driver; pub mod file; pub mod gem; +#[cfg(CONFIG_DRM_GPUVM = "y")] +pub mod gpuvm; pub mod ioctl; +pub mod mm; +pub mod sched; +pub mod syncobj; pub use self::device::Device; pub use self::driver::Driver; diff --git a/rust/kernel/drm/sched.rs b/rust/kernel/drm/sched.rs new file mode 100644 index 00000000000000..e2f5cd96014f93 --- /dev/null +++ b/rust/kernel/drm/sched.rs @@ -0,0 +1,391 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT + +//! DRM Scheduler +//! +//! C header: [`include/drm/gpu_scheduler.h`](../../../../include/drm/gpu_scheduler.h) + +use crate::{ + bindings, device, + dma_fence::*, + error::{to_result, Result}, + prelude::*, + sync::{Arc, UniqueArc}, + time::{self, msecs_to_jiffies}, +}; +use core::marker::PhantomData; +use core::mem::MaybeUninit; +use core::ops::{Deref, DerefMut}; +use core::ptr::{addr_of, addr_of_mut}; + +/// Scheduler status after timeout recovery +#[repr(u32)] +pub enum Status { + /// Device recovered from the timeout and can execute jobs again + Nominal = bindings::drm_gpu_sched_stat_DRM_GPU_SCHED_STAT_RESET, + /// Device is no longer available + NoDevice = bindings::drm_gpu_sched_stat_DRM_GPU_SCHED_STAT_ENODEV, +} + +/// Scheduler priorities +#[repr(u32)] +pub enum Priority { + /// Low userspace priority + Low = bindings::drm_sched_priority_DRM_SCHED_PRIORITY_LOW, + /// Normal userspace priority + Normal = bindings::drm_sched_priority_DRM_SCHED_PRIORITY_NORMAL, + /// High userspace priority + High = bindings::drm_sched_priority_DRM_SCHED_PRIORITY_HIGH, + /// Kernel priority (highest) + Kernel = bindings::drm_sched_priority_DRM_SCHED_PRIORITY_KERNEL, +} + +/// Trait to be implemented by driver job objects. +pub trait JobImpl: Sized { + /// Called when the scheduler is considering scheduling this job next, to get another Fence + /// for this job to block on. Once it returns None, run() may be called. + fn prepare(_job: &mut Job) -> Option { + None // Equivalent to NULL function pointer + } + + /// Called to execute the job once all of the dependencies have been resolved. This may be + /// called multiple times, if timed_out() has happened and drm_sched_job_recovery() decides + /// to try it again. + fn run(job: &mut Job) -> Result>; + + /// Called when a job has taken too long to execute, to trigger GPU recovery. + /// + /// This method is called in a workqueue context. + fn timed_out(job: &mut Job) -> Status; + + /// Called for remaining jobs in drm_sched_fini() to ensure the job's fences + /// get signalled before the scheduler is torn down. + fn cancel(job: &mut Job); +} + +unsafe extern "C" fn prepare_job_cb( + sched_job: *mut bindings::drm_sched_job, + _s_entity: *mut bindings::drm_sched_entity, +) -> *mut bindings::dma_fence { + // SAFETY: All of our jobs are Job. + let p = unsafe { crate::container_of!(sched_job, Job, job) as *mut Job }; + + // SAFETY: All of our jobs are Job. + match T::prepare(unsafe { &mut *p }) { + None => core::ptr::null_mut(), + Some(fence) => fence.into_raw(), + } +} + +unsafe extern "C" fn run_job_cb( + sched_job: *mut bindings::drm_sched_job, +) -> *mut bindings::dma_fence { + // SAFETY: All of our jobs are Job. + let p = unsafe { crate::container_of!(sched_job, Job, job) as *mut Job }; + + // SAFETY: All of our jobs are Job. + match T::run(unsafe { &mut *p }) { + Err(e) => e.to_ptr(), + Ok(None) => core::ptr::null_mut(), + Ok(Some(fence)) => fence.into_raw(), + } +} + +unsafe extern "C" fn timedout_job_cb( + sched_job: *mut bindings::drm_sched_job, +) -> bindings::drm_gpu_sched_stat { + // SAFETY: All of our jobs are Job. + let p = unsafe { crate::container_of!(sched_job, Job, job) as *mut Job }; + + // SAFETY: All of our jobs are Job. + T::timed_out(unsafe { &mut *p }) as bindings::drm_gpu_sched_stat +} + +unsafe extern "C" fn free_job_cb(sched_job: *mut bindings::drm_sched_job) { + // SAFETY: All of our jobs are Job. + let p = unsafe { crate::container_of!(sched_job, Job, job) as *mut Job }; + + // Convert the job back to a Box and drop it + // SAFETY: All of our Jobs are created inside a box. + unsafe { drop(KBox::from_raw(p)) }; +} + +unsafe extern "C" fn cancel_job_cb(sched_job: *mut bindings::drm_sched_job) { + // SAFETY: All of our jobs are Job. + let p = unsafe { crate::container_of!(sched_job, Job, job) as *mut Job }; + + // SAFETY: All of our jobs are Job. + T::cancel(unsafe { &mut *p }); + + let fence = unsafe { Fence::get_raw(&mut (*(*sched_job).s_fence).finished) }; + fence.set_error(ECANCELED); + let _ = fence.signal(); +} + +/// A DRM scheduler job. +pub struct Job { + job: bindings::drm_sched_job, + inner: T, +} + +impl Deref for Job { + type Target = T; + + fn deref(&self) -> &Self::Target { + &self.inner + } +} + +impl DerefMut for Job { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.inner + } +} + +impl Drop for Job { + fn drop(&mut self) { + // SAFETY: At this point the job has either been submitted and this is being called from + // `free_job_cb` above, or it hasn't and it is safe to call `drm_sched_job_cleanup`. + unsafe { bindings::drm_sched_job_cleanup(&mut self.job) }; + } +} + +/// A pending DRM scheduler job (not yet armed) +pub struct PendingJob<'a, T: JobImpl>(KBox>, PhantomData<&'a T>); + +impl<'a, T: JobImpl> PendingJob<'a, T> { + /// Add a fence as a dependency to the job + pub fn add_dependency(&mut self, fence: Fence) -> Result { + // SAFETY: C call with correct arguments + to_result(unsafe { + bindings::drm_sched_job_add_dependency(&mut self.0.job, fence.into_raw()) + }) + } + + /// Arm the job to make it ready for execution + pub fn arm(mut self) -> ArmedJob<'a, T> { + // SAFETY: C call with correct arguments + unsafe { bindings::drm_sched_job_arm(&mut self.0.job) }; + ArmedJob(self.0, PhantomData) + } +} + +impl<'a, T: JobImpl> Deref for PendingJob<'a, T> { + type Target = Job; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl<'a, T: JobImpl> DerefMut for PendingJob<'a, T> { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} + +/// An armed DRM scheduler job (not yet submitted) +pub struct ArmedJob<'a, T: JobImpl>(KBox>, PhantomData<&'a T>); + +impl<'a, T: JobImpl> ArmedJob<'a, T> { + /// Returns the job fences + pub fn fences(&mut self) -> JobFences<'_> { + // SAFETY: s_fence is always a valid drm_sched_fence pointer + JobFences(unsafe { &mut *self.0.job.s_fence }) + } + + /// Push the job for execution into the scheduler + pub fn push(self) { + // After this point, the job is submitted and owned by the scheduler + let ptr = match self { + ArmedJob(job, _) => KBox::>::into_raw(job), + }; + + // SAFETY: We are passing in ownership of a valid Box raw pointer. + unsafe { bindings::drm_sched_entity_push_job(addr_of_mut!((*ptr).job)) }; + } +} +impl<'a, T: JobImpl> Deref for ArmedJob<'a, T> { + type Target = Job; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl<'a, T: JobImpl> DerefMut for ArmedJob<'a, T> { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} + +/// Reference to the bundle of fences attached to a DRM scheduler job +pub struct JobFences<'a>(&'a mut bindings::drm_sched_fence); + +impl<'a> JobFences<'a> { + /// Returns a new reference to the job scheduled fence. + pub fn scheduled(&mut self) -> Fence { + // SAFETY: self.0.scheduled is always a valid fence + unsafe { Fence::get_raw(&mut self.0.scheduled) } + } + + /// Returns a new reference to the job finished fence. + pub fn finished(&mut self) -> Fence { + // SAFETY: self.0.finished is always a valid fence + unsafe { Fence::get_raw(&mut self.0.finished) } + } +} + +struct EntityInner { + entity: bindings::drm_sched_entity, + // TODO: Allow users to share guilty flag between entities + sched: Arc>, + guilty: bindings::atomic_t, + _p: PhantomData, +} + +impl Drop for EntityInner { + fn drop(&mut self) { + // SAFETY: The EntityInner is initialized. This will cancel/free all jobs. + unsafe { bindings::drm_sched_entity_destroy(&mut self.entity) }; + } +} + +// SAFETY: TODO +unsafe impl Sync for EntityInner {} +// SAFETY: TODO +unsafe impl Send for EntityInner {} + +/// A DRM scheduler entity. +pub struct Entity(Pin>>); + +impl Entity { + /// Create a new scheduler entity. + pub fn new(sched: &Scheduler, priority: Priority) -> Result { + let mut entity: KBox>> = + KBox::new_uninit(GFP_KERNEL | __GFP_ZERO)?; + + let mut sched_ptr = &sched.0.sched as *const _ as *mut _; + + // SAFETY: The Box is allocated above and valid. + unsafe { + bindings::drm_sched_entity_init( + addr_of_mut!((*entity.as_mut_ptr()).entity), + priority as _, + &mut sched_ptr, + 1, + addr_of_mut!((*entity.as_mut_ptr()).guilty), + ) + }; + + // SAFETY: The Box is allocated above and valid. + unsafe { addr_of_mut!((*entity.as_mut_ptr()).sched).write(sched.0.clone()) }; + + // SAFETY: entity is now initialized. + Ok(Self(Pin::from(unsafe { entity.assume_init() }))) + } + + /// Create a new job on this entity. + /// + /// The entity must outlive the pending job until it transitions into the submitted state, + /// after which the scheduler owns it. Since jobs must be submitted in creation order, + /// this requires a mutable reference to the entity, ensuring that only one new job can be + /// in flight at once. + pub fn new_job(&mut self, credits: u32, inner: T) -> Result> { + let mut job: KBox>> = Box::new_uninit(GFP_KERNEL | __GFP_ZERO)?; + + // SAFETY: We hold a reference to the entity (which is a valid pointer), + // and the job object was just allocated above. + to_result(unsafe { + bindings::drm_sched_job_init( + addr_of_mut!((*job.as_mut_ptr()).job), + &self.0.as_ref().get_ref().entity as *const _ as *mut _, + credits, + core::ptr::null_mut(), + 0, + ) + })?; + + // SAFETY: The Box pointer is valid, and this initializes the inner member. + unsafe { addr_of_mut!((*job.as_mut_ptr()).inner).write(inner) }; + + // SAFETY: All fields of the Job are now initialized. + Ok(PendingJob(unsafe { job.assume_init() }, PhantomData)) + } +} + +/// DRM scheduler inner data +pub struct SchedulerInner { + sched: bindings::drm_gpu_scheduler, + _p: PhantomData, +} + +impl Drop for SchedulerInner { + fn drop(&mut self) { + // SAFETY: The scheduler is valid. This assumes drm_sched_fini() will take care of + // freeing all in-progress jobs. + unsafe { bindings::drm_sched_stop(&mut self.sched, core::ptr::null_mut()) }; + unsafe { bindings::drm_sched_fini(&mut self.sched) }; + } +} + +// SAFETY: TODO +unsafe impl Sync for SchedulerInner {} +// SAFETY: TODO +unsafe impl Send for SchedulerInner {} + +/// A DRM Scheduler +pub struct Scheduler(Arc>); + +impl Scheduler { + const OPS: bindings::drm_sched_backend_ops = bindings::drm_sched_backend_ops { + prepare_job: Some(prepare_job_cb::), + run_job: Some(run_job_cb::), + timedout_job: Some(timedout_job_cb::), + free_job: Some(free_job_cb::), + cancel_job: Some(cancel_job_cb::), + }; + /// Creates a new DRM Scheduler object + // TODO: Shared timeout workqueues & scores + pub fn new( + device: &device::Device, + num_rqs: u32, + credit_limit: u32, + hang_limit: u32, + timeout_ms: time::Msecs, + name: &'static CStr, + ) -> Result> { + let mut sched: UniqueArc>> = + UniqueArc::new_uninit(GFP_KERNEL)?; + + // SAFETY: zero sched->sched_rq as drm_sched_init() uses it to exit early withoput initialisation + // TODO: allocate sched zzeroed instead + unsafe { + (*sched.as_mut_ptr()).sched.sched_rq = core::ptr::null_mut(); + }; + + let init_ops = bindings::drm_sched_init_args { + ops: &Self::OPS, + submit_wq: core::ptr::null_mut(), + timeout_wq: core::ptr::null_mut(), + num_rqs, + credit_limit, + hang_limit, + timeout: msecs_to_jiffies(timeout_ms).try_into()?, + score: core::ptr::null_mut(), + name: name.as_char_ptr(), + dev: device.as_raw(), + }; + + // SAFETY: The drm_sched pointer is valid and pinned as it was just allocated above. + // `device` is valid by its type invarants + to_result(unsafe { + bindings::drm_sched_init( + addr_of_mut!((*sched.as_mut_ptr()).sched), + addr_of!(init_ops), + ) + })?; + + // SAFETY: All fields of SchedulerInner are now initialized. + Ok(Scheduler(unsafe { sched.assume_init() }.into())) + } +} diff --git a/rust/kernel/drm/syncobj.rs b/rust/kernel/drm/syncobj.rs new file mode 100644 index 00000000000000..a022e08223588b --- /dev/null +++ b/rust/kernel/drm/syncobj.rs @@ -0,0 +1,83 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT + +//! DRM Sync Objects +//! +//! C header: [`include/drm/drm_syncobj.h`](../../../../include/drm/drm_syncobj.h) + +use crate::{bindings, dma_fence::*, drm, error::Result, prelude::*}; + +/// A DRM Sync Object +/// +/// # Invariants +/// ptr is a valid pointer to a drm_syncobj and we own a reference to it. +pub struct SyncObj { + ptr: *mut bindings::drm_syncobj, +} + +impl SyncObj { + /// Looks up a sync object by its handle for a given `File`. + pub fn lookup_handle( + file: &drm::File, + handle: u32, + ) -> Result { + // SAFETY: The arguments are all valid per the type invariants. + let ptr = unsafe { bindings::drm_syncobj_find(file.as_raw() as *mut _, handle) }; + + if ptr.is_null() { + Err(ENOENT) + } else { + Ok(SyncObj { ptr }) + } + } + + /// Returns the DMA fence associated with this sync object, if any. + pub fn fence_get(&self) -> Option { + // SAFETY: self.ptr is always valid + let fence = unsafe { bindings::drm_syncobj_fence_get(self.ptr) }; + if fence.is_null() { + None + } else { + // SAFETY: The pointer is non-NULL and drm_syncobj_fence_get acquired an + // additional reference. + Some(unsafe { Fence::from_raw(fence) }) + } + } + + /// Replaces the DMA fence with a new one, or removes it if fence is None. + pub fn replace_fence(&self, fence: Option<&Fence>) { + // SAFETY: All arguments should be valid per the respective type invariants. + unsafe { + bindings::drm_syncobj_replace_fence( + self.ptr, + fence.map_or(core::ptr::null_mut(), |a| a.raw()), + ) + }; + } + + /// Adds a new timeline point to the syncobj. + pub fn add_point(&self, chain: FenceChain, fence: &Fence, point: u64) { + // SAFETY: All arguments should be valid per the respective type invariants. + // This takes over the FenceChain ownership. + unsafe { bindings::drm_syncobj_add_point(self.ptr, chain.into_raw(), fence.raw(), point) }; + } +} + +impl Drop for SyncObj { + fn drop(&mut self) { + // SAFETY: We own a reference to this syncobj. + unsafe { bindings::drm_syncobj_put(self.ptr) }; + } +} + +impl Clone for SyncObj { + fn clone(&self) -> Self { + // SAFETY: `ptr` is valid per the type invariant and we own a reference to it. + unsafe { bindings::drm_syncobj_get(self.ptr) }; + SyncObj { ptr: self.ptr } + } +} + +// SAFETY: drm_syncobj operations are internally locked. +unsafe impl Sync for SyncObj {} +// SAFETY: drm_syncobj operations are internally locked. +unsafe impl Send for SyncObj {} diff --git a/rust/kernel/error.rs b/rust/kernel/error.rs index 258b12afdcba35..bc116561f6c09c 100644 --- a/rust/kernel/error.rs +++ b/rust/kernel/error.rs @@ -66,8 +66,11 @@ pub mod code { declare_err!(EPIPE, "Broken pipe."); declare_err!(EDOM, "Math argument out of domain of func."); declare_err!(ERANGE, "Math result not representable."); + declare_err!(ENOSYS, "Invalid system call number."); + declare_err!(ENODATA, "No data available."); declare_err!(EOVERFLOW, "Value too large for defined data type."); declare_err!(ETIMEDOUT, "Connection timed out."); + declare_err!(ECANCELED, "Operation Canceled."); declare_err!(ERESTARTSYS, "Restart the system call."); declare_err!(ERESTARTNOINTR, "System call was interrupted by a signal and will be restarted."); declare_err!(ERESTARTNOHAND, "Restart if no handler."); diff --git a/rust/kernel/iio/common/aop_sensors.rs b/rust/kernel/iio/common/aop_sensors.rs new file mode 100644 index 00000000000000..de0835784b66a8 --- /dev/null +++ b/rust/kernel/iio/common/aop_sensors.rs @@ -0,0 +1,167 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Apple AOP sensors common code +//! +//! Copyright (C) The Asahi Linux Contributors + +use core::marker::{PhantomData, PhantomPinned}; +use core::ptr; +use core::sync::atomic::{AtomicU32, Ordering}; + +use kernel::{ + bindings, device, + prelude::*, + soc::apple::aop::FakehidListener, + sync::Arc, + types::{ARef, ForeignOwnable}, + ThisModule, +}; + +/// TODO: add documentation +pub trait MessageProcessor { + /// TODO: add documentation + fn process(&self, message: &[u8]) -> u32; +} + +/// TODO: add documentation +pub struct AopSensorData { + dev: ARef, + ty: u32, + value: AtomicU32, + msg_proc: T, +} + +impl AopSensorData { + /// TODO: add documentation + pub fn new(dev: ARef, ty: u32, msg_proc: T) -> Result>> { + Ok(Arc::new( + AopSensorData { + dev, + ty, + value: AtomicU32::new(0), + msg_proc, + }, + GFP_KERNEL, + )?) + } +} + +impl FakehidListener for AopSensorData { + fn process_fakehid_report(&self, data: &[u8]) -> Result<()> { + self.value + .store(self.msg_proc.process(data), Ordering::Relaxed); + Ok(()) + } +} + +unsafe extern "C" fn aop_read_raw( + dev: *mut bindings::iio_dev, + chan: *const bindings::iio_chan_spec, + val: *mut i32, + _: *mut i32, + mask: isize, +) -> i32 { + let data = unsafe { Arc::>::borrow((*dev).priv_.cast()) }; + let ty = unsafe { (*chan).type_ }; + if mask != bindings::BINDINGS_IIO_CHAN_INFO_PROCESSED as isize + && mask != bindings::BINDINGS_IIO_CHAN_INFO_RAW as isize + { + return EINVAL.to_errno(); + } + if data.ty != ty { + return EINVAL.to_errno(); + } + let value = data.value.load(Ordering::Relaxed); + unsafe { + *val = value as i32; + } + bindings::IIO_VAL_INT as i32 +} + +struct IIOSpec { + spec: [bindings::iio_chan_spec; 1], + vtable: bindings::iio_info, + _p: PhantomPinned, +} + +/// TODO: add documentation +pub struct IIORegistration { + dev: *mut bindings::iio_dev, + spec: Pin>, + registered: bool, + _p: PhantomData>, +} + +impl IIORegistration { + /// TODO: add documentation + pub fn new( + data: Arc>, + name: &'static CStr, + ty: u32, + info_mask: usize, + module: &ThisModule, + ) -> Result { + let spec = KBox::pin( + IIOSpec { + spec: [bindings::iio_chan_spec { + type_: ty, + __bindgen_anon_1: bindings::iio_chan_spec__bindgen_ty_1 { + scan_type: bindings::iio_scan_type { + sign: b'u' as _, + realbits: 32, + storagebits: 32, + ..Default::default() + }, + }, + info_mask_separate: info_mask, + ..Default::default() + }], + vtable: bindings::iio_info { + read_raw: Some(aop_read_raw::), + ..Default::default() + }, + _p: PhantomPinned, + }, + GFP_KERNEL, + )?; + let mut this = IIORegistration { + dev: ptr::null_mut(), + spec, + registered: false, + _p: PhantomData, + }; + this.dev = unsafe { bindings::iio_device_alloc(data.dev.as_raw(), 0) }; + unsafe { + (*this.dev).priv_ = data.clone().into_foreign().cast(); + (*this.dev).name = name.as_ptr() as _; + // spec is now pinned + (*this.dev).channels = this.spec.spec.as_ptr(); + (*this.dev).num_channels = this.spec.spec.len() as i32; + (*this.dev).info = &this.spec.vtable; + } + let ret = unsafe { bindings::__iio_device_register(this.dev, module.as_ptr()) }; + if ret < 0 { + dev_err!(data.dev, "Unable to register iio sensor"); + return Err(Error::from_errno(ret)); + } + this.registered = true; + Ok(this) + } +} + +impl Drop for IIORegistration { + fn drop(&mut self) { + if self.dev != ptr::null_mut() { + unsafe { + if self.registered { + bindings::iio_device_unregister(self.dev); + } + Arc::>::from_foreign((*self.dev).priv_.cast()); + bindings::iio_device_free(self.dev); + } + } + } +} + +unsafe impl Send for IIORegistration {} +unsafe impl Sync for IIORegistration {} diff --git a/rust/kernel/iio/common/mod.rs b/rust/kernel/iio/common/mod.rs new file mode 100644 index 00000000000000..b789e9bf44c9bf --- /dev/null +++ b/rust/kernel/iio/common/mod.rs @@ -0,0 +1,6 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT + +//! IIO common modules + +#[cfg(any(CONFIG_IIO_AOP_SENSOR_LAS, CONFIG_IIO_AOP_SENSOR_ALS,))] +pub mod aop_sensors; diff --git a/rust/kernel/iio/mod.rs b/rust/kernel/iio/mod.rs new file mode 100644 index 00000000000000..b0cb308f0b454c --- /dev/null +++ b/rust/kernel/iio/mod.rs @@ -0,0 +1,5 @@ +// SPDX-License-Identifier: GPL-2.0 or MIT + +//! Industrial IO drivers + +pub mod common; diff --git a/rust/kernel/impl_flags.rs b/rust/kernel/impl_flags.rs new file mode 100644 index 00000000000000..e2bd7639da12e0 --- /dev/null +++ b/rust/kernel/impl_flags.rs @@ -0,0 +1,272 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Bitflag type generator. + +/// Common helper for declaring bitflag and bitmask types. +/// +/// This macro takes as input: +/// - A struct declaration representing a bitmask type +/// (e.g., `pub struct Permissions(u32)`). +/// - An enumeration declaration representing individual bit flags +/// (e.g., `pub enum Permission { ... }`). +/// +/// And generates: +/// - The struct and enum types with appropriate `#[repr]` attributes. +/// - Implementations of common bitflag operators +/// ([`::core::ops::BitOr`], [`::core::ops::BitAnd`], etc.). +/// - Utility methods such as `.contains()` to check flags. +/// +/// # Examples +/// +/// ``` +/// use kernel::impl_flags; +/// +/// impl_flags!( +/// /// Represents multiple permissions. +/// #[derive(Debug, Clone, Default, Copy, PartialEq, Eq)] +/// pub struct Permissions(u32); +/// +/// /// Represents a single permission. +/// #[derive(Debug, Clone, Copy, PartialEq, Eq)] +/// pub enum Permission { +/// /// Read permission. +/// Read = 1 << 0, +/// +/// /// Write permission. +/// Write = 1 << 1, +/// +/// /// Execute permission. +/// Execute = 1 << 2, +/// } +/// ); +/// +/// // Combine multiple permissions using the bitwise OR (`|`) operator. +/// let mut read_write: Permissions = Permission::Read | Permission::Write; +/// assert!(read_write.contains(Permission::Read)); +/// assert!(read_write.contains(Permission::Write)); +/// assert!(!read_write.contains(Permission::Execute)); +/// assert!(read_write.contains_any(Permission::Read | Permission::Execute)); +/// assert!(read_write.contains_all(Permission::Read | Permission::Write)); +/// +/// // Using the bitwise OR assignment (`|=`) operator. +/// read_write |= Permission::Execute; +/// assert!(read_write.contains(Permission::Execute)); +/// +/// // Masking a permission with the bitwise AND (`&`) operator. +/// let read_only: Permissions = read_write & Permission::Read; +/// assert!(read_only.contains(Permission::Read)); +/// assert!(!read_only.contains(Permission::Write)); +/// +/// // Toggling permissions with the bitwise XOR (`^`) operator. +/// let toggled: Permissions = read_only ^ Permission::Read; +/// assert!(!toggled.contains(Permission::Read)); +/// +/// // Inverting permissions with the bitwise NOT (`!`) operator. +/// let negated = !read_only; +/// assert!(negated.contains(Permission::Write)); +/// assert!(!negated.contains(Permission::Read)); +/// ``` +#[macro_export] +macro_rules! impl_flags { + ( + $(#[$outer_flags:meta])* + $vis_flags:vis struct $flags:ident($ty:ty); + + $(#[$outer_flag:meta])* + $vis_flag:vis enum $flag:ident { + $( + $(#[$inner_flag:meta])* + $name:ident = $value:expr + ),+ $( , )? + } + ) => { + $(#[$outer_flags])* + #[repr(transparent)] + $vis_flags struct $flags($ty); + + $(#[$outer_flag])* + #[repr($ty)] + $vis_flag enum $flag { + $( + $(#[$inner_flag])* + $name = $value + ),+ + } + + impl ::core::convert::From<$flag> for $flags { + #[inline] + fn from(value: $flag) -> Self { + Self(value as $ty) + } + } + + impl ::core::convert::From<$flags> for $ty { + #[inline] + fn from(value: $flags) -> Self { + value.0 + } + } + + impl ::core::ops::BitOr for $flags { + type Output = Self; + #[inline] + fn bitor(self, rhs: Self) -> Self::Output { + Self(self.0 | rhs.0) + } + } + + impl ::core::ops::BitOrAssign for $flags { + #[inline] + fn bitor_assign(&mut self, rhs: Self) { + *self = *self | rhs; + } + } + + impl ::core::ops::BitOr<$flag> for $flags { + type Output = Self; + #[inline] + fn bitor(self, rhs: $flag) -> Self::Output { + self | Self::from(rhs) + } + } + + impl ::core::ops::BitOrAssign<$flag> for $flags { + #[inline] + fn bitor_assign(&mut self, rhs: $flag) { + *self = *self | rhs; + } + } + + impl ::core::ops::BitAnd for $flags { + type Output = Self; + #[inline] + fn bitand(self, rhs: Self) -> Self::Output { + Self(self.0 & rhs.0) + } + } + + impl ::core::ops::BitAndAssign for $flags { + #[inline] + fn bitand_assign(&mut self, rhs: Self) { + *self = *self & rhs; + } + } + + impl ::core::ops::BitAnd<$flag> for $flags { + type Output = Self; + #[inline] + fn bitand(self, rhs: $flag) -> Self::Output { + self & Self::from(rhs) + } + } + + impl ::core::ops::BitAndAssign<$flag> for $flags { + #[inline] + fn bitand_assign(&mut self, rhs: $flag) { + *self = *self & rhs; + } + } + + impl ::core::ops::BitXor for $flags { + type Output = Self; + #[inline] + fn bitxor(self, rhs: Self) -> Self::Output { + Self((self.0 ^ rhs.0) & Self::all_bits()) + } + } + + impl ::core::ops::BitXorAssign for $flags { + #[inline] + fn bitxor_assign(&mut self, rhs: Self) { + *self = *self ^ rhs; + } + } + + impl ::core::ops::BitXor<$flag> for $flags { + type Output = Self; + #[inline] + fn bitxor(self, rhs: $flag) -> Self::Output { + self ^ Self::from(rhs) + } + } + + impl ::core::ops::BitXorAssign<$flag> for $flags { + #[inline] + fn bitxor_assign(&mut self, rhs: $flag) { + *self = *self ^ rhs; + } + } + + impl ::core::ops::Not for $flags { + type Output = Self; + #[inline] + fn not(self) -> Self::Output { + Self((!self.0) & Self::all_bits()) + } + } + + impl ::core::ops::BitOr for $flag { + type Output = $flags; + #[inline] + fn bitor(self, rhs: Self) -> Self::Output { + $flags(self as $ty | rhs as $ty) + } + } + + impl ::core::ops::BitAnd for $flag { + type Output = $flags; + #[inline] + fn bitand(self, rhs: Self) -> Self::Output { + $flags(self as $ty & rhs as $ty) + } + } + + impl ::core::ops::BitXor for $flag { + type Output = $flags; + #[inline] + fn bitxor(self, rhs: Self) -> Self::Output { + $flags((self as $ty ^ rhs as $ty) & $flags::all_bits()) + } + } + + impl ::core::ops::Not for $flag { + type Output = $flags; + #[inline] + fn not(self) -> Self::Output { + $flags((!(self as $ty)) & $flags::all_bits()) + } + } + + impl $flags { + /// Returns an empty instance where no flags are set. + #[inline] + pub const fn empty() -> Self { + Self(0) + } + + /// Returns a mask containing all valid flag bits. + #[inline] + pub const fn all_bits() -> $ty { + 0 $( | $value )+ + } + + /// Checks if a specific flag is set. + #[inline] + pub fn contains(self, flag: $flag) -> bool { + (self.0 & flag as $ty) == flag as $ty + } + + /// Checks if at least one of the provided flags is set. + #[inline] + pub fn contains_any(self, flags: $flags) -> bool { + (self.0 & flags.0) != 0 + } + + /// Checks if all of the provided flags are set. + #[inline] + pub fn contains_all(self, flags: $flags) -> bool { + (self.0 & flags.0) == flags.0 + } + } + }; +} diff --git a/rust/kernel/init.rs b/rust/kernel/init.rs index 899b9a96276235..1ee65c555a6dda 100644 --- a/rust/kernel/init.rs +++ b/rust/kernel/init.rs @@ -219,17 +219,17 @@ pub trait InPlaceInit: Sized { /// [`Error`]: crate::error::Error #[macro_export] macro_rules! try_init { - ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? { + ($(&$this:ident in)? $t:ident $(::$p:ident)* $(::<$($generics:ty),* $(,)?>)? { $($fields:tt)* }) => { - ::pin_init::try_init!($(&$this in)? $t $(::<$($generics),*>)? { + ::pin_init::try_init!($(&$this in)? $t $(::$p)* $(::<$($generics),*>)? { $($fields)* }? $crate::error::Error) }; - ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? { + ($(&$this:ident in)? $t:ident $(::$p:ident)* $(::<$($generics:ty),* $(,)?>)? { $($fields:tt)* }? $err:ty) => { - ::pin_init::try_init!($(&$this in)? $t $(::<$($generics),*>)? { + ::pin_init::try_init!($(&$this in)? $t $(::$p)* $(::<$($generics),*>)? { $($fields)* }? $err) }; @@ -279,17 +279,17 @@ macro_rules! try_init { /// [`Error`]: crate::error::Error #[macro_export] macro_rules! try_pin_init { - ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? { + ($(&$this:ident in)? $t:ident $(::$p:ident)* $(::<$($generics:ty),* $(,)?>)? { $($fields:tt)* }) => { - ::pin_init::try_pin_init!($(&$this in)? $t $(::<$($generics),*>)? { + ::pin_init::try_pin_init!($(&$this in)? $t $(::$p)* $(::<$($generics),*>)? { $($fields)* }? $crate::error::Error) }; - ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? { + ($(&$this:ident in)? $t:ident $(::$p:ident)* $(::<$($generics:ty),* $(,)?>)? { $($fields:tt)* }? $err:ty) => { - ::pin_init::try_pin_init!($(&$this in)? $t $(::<$($generics),*>)? { + ::pin_init::try_pin_init!($(&$this in)? $t $(::$p)* $(::<$($generics),*>)? { $($fields)* }? $err) }; diff --git a/rust/kernel/io.rs b/rust/kernel/io.rs index b64b11f75a353e..e58f7b18489f1e 100644 --- a/rust/kernel/io.rs +++ b/rust/kernel/io.rs @@ -230,6 +230,15 @@ impl Io { } } + #[inline] + const fn length_valid(offset: usize, length: usize, size: usize) -> bool { + if let Some(end) = offset.checked_add(length) { + end <= size + } else { + false + } + } + #[inline] fn io_addr(&self, offset: usize) -> Result { if !Self::offset_valid::(offset, self.maxsize()) { @@ -249,6 +258,67 @@ impl Io { self.addr() + offset } + /// Copy memory block from an i/o memory by filling the specified buffer with it. + /// + /// # Examples + /// ``` + /// use kernel::io::mem::IoMem; + /// use kernel::io::mem::Resource; + /// + /// fn test(device: &Device, res: Resource) -> Result { + /// // Create an i/o memory block of at least 100 bytes. + /// let devres_mem = IoMem::<100>::new(res, device)?; + /// // aquire access to memory block + /// let mem = devres_mem.try_access()?; + /// + /// let mut buffer: [u8; 32] = [0; 32]; + /// + /// // Memcpy 16 bytes from an offset 10 of i/o memory block into the buffer. + /// mem.try_memcpy_fromio(&mut buffer[..16], 10)?; + /// + /// Ok(()) + /// } + /// ``` + pub fn try_memcpy_fromio(&self, buffer: &mut [u8], offset: usize) -> Result { + if buffer.len() == 0 || !Self::length_valid(offset, buffer.len(), self.maxsize()) { + return Err(EINVAL); + } + let addr = self.io_addr::(offset)?; + + // SAFETY: + // - The type invariants guarantee that `adr` is a valid pointer. + // - The bounds of `buffer` are checked with a call to `length_valid`. + unsafe { + bindings::memcpy_fromio( + buffer.as_mut_ptr() as *mut _, + addr as *const _, + buffer.len() as _, + ) + }; + Ok(()) + } + + /// Copy memory block to i/o memory from the specified buffer. + pub fn try_memcpy_toio(&self, offset: usize, buffer: &[u8]) -> Result { + if buffer.len() == 0 || !Self::length_valid(offset, buffer.len(), self.maxsize()) { + return Err(EINVAL); + } + // no need to check since offset + buffer.len() - 1 is valid + let addr = self.io_addr::(offset)?; + + // SAFETY: + // - The type invariants guarantee that `adr` is a valid pointer. + // - The bounds of `buffer` are checked with a call to `length_valid`. + unsafe { + bindings::memcpy_toio( + addr as *mut _, + buffer.as_ptr() as *const _, + buffer.len() as _, + ) + }; + Ok(()) + } + define_read!(read8, try_read8, readb -> u8); define_read!(read16, try_read16, readw -> u16); define_read!(read32, try_read32, readl -> u32); diff --git a/rust/kernel/io/mem.rs b/rust/kernel/io/mem.rs index b03b82cd531b2d..40d9d193816ff7 100644 --- a/rust/kernel/io/mem.rs +++ b/rust/kernel/io/mem.rs @@ -3,6 +3,7 @@ //! Generic memory-mapped IO. use core::ops::Deref; +use core::ptr::NonNull; use crate::{ c_str, @@ -11,6 +12,7 @@ use crate::{ Device, // }, devres::Devres, + impl_flags, io::{ self, resource::{ @@ -285,3 +287,105 @@ impl Deref for IoMem { unsafe { Io::from_raw(&self.io) } } } + +impl_flags!( + /// Flags to be used when remapping memory. + #[derive(Debug, Clone, Default, Copy, PartialEq, Eq)] + pub struct MemFlags(usize); + + /// Enum mirroring the C MEMREMAP_* eum values + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + pub enum MemFlag { + /// Matches the default mapping for System RAM on the architecture. + /// + /// This is usually a read-allocate write-back cache. Moreover, if this flag is specified and + /// the requested remap region is RAM, memremap() will bypass establishing a new mapping and + /// instead return a pointer into the direct map. + WB = bindings::MEMREMAP_WB as usize, + + /// Establish a mapping whereby writes either bypass the cache or are written through to memory + /// and never exist in a cache-dirty state with respect to program visibility. + /// + /// Attempts to map System RAM with this mapping type will fail. + WT = bindings::MEMREMAP_WT as usize, + + /// Establish a writecombine mapping, whereby writes may be coalesced together (e.g. in the + /// CPU's write buffers), but is otherwise uncached. + /// + /// Attempts to map System RAM with this mapping type will fail. + WC = bindings::MEMREMAP_WC as usize, + // Note: Skipping MEMREMAP_ENC/DEC since they are under-documented and have zero + // users outside of arch/x86. + } +); + +/// Represents a non-MMIO memory block. This is like [`IoMem`], but for cases where it is known +/// that the resource being mapped does not have I/O side effects. +// Invariants: +// `ptr` is a non-null and valid address of at least `usize` bytes and returned by a `memremap` +// call. +// ``` +pub struct Mem { + ptr: NonNull, + size: usize, +} + +impl Mem { + /// Tries to create a new instance of a memory block from a Resource. + /// + /// The resource described by `res` is mapped into the CPU's address space so that it can be + /// accessed directly. It is also consumed by this function so that it can't be mapped again + /// to a different address. + /// + /// If multiple caching flags are specified, the different mapping types will be attempted in + /// the order [`MemFlag::WB`], [`MemFlag::WT`], [`MemFlag::WC`]. + /// + /// # Flags + /// + /// * [`MemFlag::WB`]: Matches the default mapping for System RAM on the architecture. + /// This is usually a read-allocate write-back cache. Moreover, if this flag is specified and + /// the requested remap region is RAM, memremap() will bypass establishing a new mapping and + /// instead return a pointer into the direct map. + /// + /// * [`MemFlag::WT`]: Establish a mapping whereby writes either bypass the cache or are written + /// through to memory and never exist in a cache-dirty state with respect to program visibility. + /// Attempts to map System RAM with this mapping type will fail. + /// * [`MemFlag::WC`]: Establish a writecombine mapping, whereby writes may be coalesced together + /// (e.g. in the CPU's write buffers), but is otherwise uncached. Attempts to map System RAM with + /// this mapping type will fail. + /// + /// # Safety + /// + /// Callers must ensure that either (a) the resulting interface cannot be used to initiate DMA + /// operations, or (b) that DMA operations initiated via the returned interface use DMA handles + /// allocated through the `dma` module. + pub unsafe fn try_new(res: Resource, flags: MemFlags) -> Result { + let size: usize = res.size().try_into()?; + + let addr = unsafe { bindings::memremap(res.start(), size, flags.into()) }; + let ptr = NonNull::new(addr).ok_or(ENOMEM)?; + // INVARIANT: `ptr` is non-null and was returned by `memremap`, so it is valid. + Ok(Self { ptr, size }) + } + + /// Returns the base address of the memory mapping as a raw pointer. + /// + /// It is up to the caller to use this pointer safely, depending on the requirements of the + /// hardware backing this memory block. + pub fn ptr(&self) -> *mut u8 { + self.ptr.cast().as_ptr() + } + + /// Returns the size of this mapped memory block. + pub fn size(&self) -> usize { + self.size + } +} + +impl Drop for Mem { + fn drop(&mut self) { + // SAFETY: By the type invariant, `self.ptr` is a value returned by a previous successful + // call to `memremap`. + unsafe { bindings::memunmap(self.ptr.as_ptr()) }; + } +} diff --git a/rust/kernel/io/resource.rs b/rust/kernel/io/resource.rs index b7ac9faf141d2e..2c7fe290b51440 100644 --- a/rust/kernel/io/resource.rs +++ b/rust/kernel/io/resource.rs @@ -79,6 +79,18 @@ unsafe impl Sync for Region {} pub struct Resource(Opaque); impl Resource { + /// Create a new zeroed [`Resource`] + pub(crate) fn zeroed() -> Self { + Resource { + 0: Opaque::::zeroed(), + } + } + + /// Gets the raw pointer to the wrapped `bindings::resource`. + pub(crate) fn as_raw(&self) -> *mut bindings::resource { + self.0.get() + } + /// Creates a reference to a [`Resource`] from a valid pointer. /// /// # Safety diff --git a/rust/kernel/iosys_map.rs b/rust/kernel/iosys_map.rs new file mode 100644 index 00000000000000..039137c051a2f1 --- /dev/null +++ b/rust/kernel/iosys_map.rs @@ -0,0 +1,647 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! IO-agnostic memory mapping interfaces. +//! +//! This crate provides bindings for the `struct iosys_map` type, which provides a common interface +//! for memory mappings which can reside within coherent memory, or within IO memory. +//! +//! C header: [`include/linux/iosys-map.h`](srctree/include/linux/pci.h) + +use crate::{ + prelude::*, + transmute::{AsBytes, FromBytes}, +}; +use bindings; +use core::{ + marker::PhantomData, + mem::{self, MaybeUninit}, + ops::{Deref, DerefMut, Range}, +}; + +/// Raw unsized representation of a `struct iosys_map`. +/// +/// This struct is a transparent wrapper around `struct iosys_map`. The C API does not provide the +/// size of the mapping by default, and thus this type also does not include the size of the +/// mapping. As such, it cannot be used for actually accessing the underlying data pointed to by the +/// mapping. +/// +/// With the exception of kernel crates which may provide their own wrappers around `RawIoSysMap`, +/// users will typically not interact with this type directly. +pub struct RawIoSysMap(bindings::iosys_map, PhantomData); + +impl RawIoSysMap { + /// Convert from a raw `bindings::iosys_map`. + #[allow(unused)] + #[inline] + pub(crate) fn from_raw(val: bindings::iosys_map) -> Self { + Self(val, PhantomData) + } + + /// Convert from a `RawIoSysMap` to a raw `bindings::iosys_map` ref. + #[inline] + pub(crate) fn as_raw(&self) -> &bindings::iosys_map { + &self.0 + } + + /// Convert from a `RawIoSysMap` to a raw mutable `bindings::iosys_map` ref. + #[inline] + pub(crate) fn as_raw_mut(&mut self) -> &mut bindings::iosys_map { + &mut self.0 + } + + /// Returns whether the mapping is within IO memory space or not. + #[inline] + pub fn is_iomem(&self) -> bool { + self.0.is_iomem + } + + /// Returns the size of a single item in this mapping. + pub const fn item_size(&self) -> usize { + mem::size_of::() + } + + /// Returns a mutable address to the memory pointed to by this iosys map. + /// + /// Note that this address is not guaranteed to reside in system memory, and may reside in IO + /// memory. + #[inline] + pub fn as_mut_ptr(&self) -> *mut T { + if self.is_iomem() { + // SAFETY: We confirmed above that this iosys map is contained within iomem, so it's + // safe to read vaddr_iomem + unsafe { self.0.__bindgen_anon_1.vaddr_iomem } + } else { + // SAFETY: We confirmed above that this iosys map is not contaned within iomem, so it's + // safe to read vaddr. + unsafe { self.0.__bindgen_anon_1.vaddr } + } + .cast() + } + + /// Returns an immutable address to the memory pointed to by this iosys map. + /// + /// Note that this address is not guaranteed to reside in system memory, and may reside in IO + /// memory. + #[inline] + pub fn as_ptr(&self) -> *const T { + self.as_mut_ptr().cast_const() + } +} + +// SAFETY: As we make no guarantees about the validity of the mapping, there's no issue with sending +// this type between threads. +unsafe impl Send for RawIoSysMap {} + +impl Clone for RawIoSysMap { + fn clone(&self) -> Self { + Self(self.0, PhantomData) + } +} + +/// A sized version of a [`RawIoSysMap`]. +/// +/// Since this type includes the size of the [`RawIoSysMap`], it can be used for accessing the +/// underlying data pointed to by it. +/// +/// # Invariants +/// +/// - The iosys mapping referenced by this type is guaranteed to be of at least `size` bytes in +/// size +/// - The iosys mapping referenced by this type is valid for the lifetime `'a`. +#[derive(Clone)] +pub struct IoSysMapRef<'a, T: AsBytes + FromBytes> { + map: RawIoSysMap, + size: usize, + _p: PhantomData<&'a T>, +} + +impl<'a, T: AsBytes + FromBytes> IoSysMapRef<'a, T> { + /// Create a new [`IoSysMapRef`] from a [`RawIoSysMap`]. + /// + /// # Safety + /// + /// - The caller guarantees that the mapping referenced by `map` is of at least `size` bytes in + /// size. + /// - The caller guarantees that the mapping referenced by `map` remains valid for the lifetime + /// of `'a`. + #[allow(unused)] + pub(crate) unsafe fn new(map: RawIoSysMap, size: usize) -> IoSysMapRef<'a, T> { + // INVARIANT: Our safety contract fulfills the type invariants of `IoSysMapRef`. + IoSysMapRef { + map, + size, + _p: PhantomData, + } + } + + /// Return the size of the `IoSysMapRef`. + #[inline] + pub fn size(&self) -> usize { + self.size + } + + /// Writes `src` to the region starting from `offset`. + /// + /// `offset` is in units of `T`, not the number of bytes. + /// + /// This function can return the following errors: + /// + /// * [`EOVERFLOW`] if calculating the length of the slice results in an overflow. + /// * [`EINVAL`] if the slice would go out of bounds of the memory region. + /// + /// # Examples + /// + /// ``` + /// use kernel::iosys_map::*; + /// + /// # fn test() -> Result { + /// # let mut map = tests::VecIoSysMap::new(&[0; 3])?; + /// # { + /// # let mut map = map.get(); + /// map.write(&[1, 2, 3], 0)?; // (now [1, 2, 3]) + /// map.write(&[4], 2)?; // (now [1, 2, 4]) + /// # } + /// # + /// # map.assert_eq(&[1, 2, 4]); + /// # + /// # Ok::<(), Error>(()) } + /// # assert!(test().is_ok()); + /// ``` + pub fn write(&mut self, src: &[T], offset: usize) -> Result { + let range = self.compute_range(offset, src.len())?; + + // SAFETY: + // - The address pointed to by this iosys_map is guaranteed to be valid via IoSysMapRef's + // type invariants. + // - We checked that this range of memory is within bounds above + unsafe { + bindings::iosys_map_memcpy_to( + self.as_raw_mut(), + range.start, + src.as_ptr().cast(), + range.len(), + ) + }; + + Ok(()) + } + + /// Memset the region starting from `offset`. + /// + /// `offset` and `len` are in units of `T`, not the number of bytes. + /// + /// This function can return the following errors: + /// + /// * [`EOVERFLOW`] if calculating the length of the slice results in an overflow. + /// * [`EINVAL`] if the slice would go out of bounds of the memory region. + /// + /// # Examples + /// + /// ``` + /// use kernel::iosys_map::*; + /// + /// # fn test() -> Result { + /// # let mut map = tests::VecIoSysMap::new(&[0u8; 3])?; + /// # { + /// # let mut map = map.get(); + /// map.memset(7)?; // (now [7, 7, 7]) + /// # } + /// # + /// # map.assert_eq(&[7, 7, 7]); + /// # + /// # Ok::<(), Error>(()) } + /// # assert!(test().is_ok()); + /// ``` + pub fn memset(&mut self, value: i32) { + // SAFETY: + // - The address pointed to by this iosys_map is guaranteed to be valid via IoSysMapRef's + // type invariants. + unsafe { bindings::iosys_map_memset(self.as_raw_mut(), 0, value, self.size()) }; + } + + /// Attempt to compute the offset of an item within the iosys map using its index. + /// + /// Returns an error if an overflow occurs. + /// + /// # Safety + /// + /// This function checks for overflows, but it explicitly does not check if the offset goes out + /// of bounds. It is the caller's responsibility to check for this before using the returned + /// offset with the iosys_map API. + unsafe fn item_from_index(&self, idx: usize) -> Result { + self.item_size().checked_mul(idx).ok_or(EOVERFLOW) + } + + /// Compute the range within this mapping a specific data type at a given offset would occupy. + /// + /// This function returns the computed range if it doesn't overflow, but does not check whether + /// or not the range is within the bounds of the allocated region pointed to by this iosys + /// mapping. + /// + /// On success, the range returned by this function is guaranteed: + /// + /// * To be a valid range of memory within the virtual mapping for this gem object. + /// * To be properly aligned to [`RawIoSysMap::item_size()`]. + fn compute_range(&self, offset: usize, count: usize) -> Result> { + // SAFETY: If the offset is out of bounds, we'll catch this via overflow checks or when + // checking range_end. + let offset = unsafe { self.item_from_index(offset)? }; + let range_size = count.checked_mul(self.item_size()).ok_or(EOVERFLOW)?; + let range_end = offset.checked_add(range_size).ok_or(EOVERFLOW)?; + + if range_end > self.size { + return Err(EINVAL); + } + + // INVARIANT: Since `offset` and `count` are both in units of `T`, we're guaranteed that the + // range returned here is properly aligned to `T`. + Ok(offset..range_end) + } + + /// Common helper to compute the memory address of an item within the iosys mapping. + /// + /// Public but hidden, since it should only be used from [`iosys_map_read`] and + /// [`iosys_map_write`]. + #[doc(hidden)] + pub fn ptr_from_index(&self, offset: usize) -> Result<*mut T> { + // SAFETY: We check if the resulting offset goes out of bounds below. + let offset = unsafe { self.item_from_index(offset)? }; + + if offset.checked_add(self.item_size()).ok_or(EOVERFLOW)? > self.size() { + return Err(EINVAL); + } + + // SAFETY: We confirmed that `offset` + the item size does not go out of bounds above. + Ok(unsafe { self.as_mut_ptr().byte_add(offset) }) + } + + // TODO: + // This function is currently needed for making the iosys_map_read!() and iosys_map_write!() + // macros work due to a combination of a few limitations: + // + // * The current C API for iosys_map requires that we use offsets for reading/writing + // iosys_maps. + // * Calculating the offset of a field within a struct requires that we either: + // * Use field projection for calculating the offset of the field. We don't have this yet. + // * Explicitly specify the type of the struct, which would be cumbersome to require in the + // read/write macros. + // * Provide a typed pointer (or other reference) to the struct in question, allowing the + // use of &raw const and &raw mut. + // * Keep in mind: we can't simply cast the offset of an item in the iosys map into a typed + // pointer to fulfill the third option. While having invalid memory addresses as pointers + // is ok, adding an offset to a pointer in rust requires that the resulting memory address + // is within the same allocation. Since an invalid pointer has no allocation, we can't + // make that guarantee. + // + // So, until we have field projection the way we workaround this: + // + // * Calculate the offset (self.item_from_index()) of the struct within the iosys map + // * Calculate the memory address of the struct using the offset from the last step + // (self.ptr_from_index()). + // * Use that memory address with &raw const/&raw mut in order to calculate the memory address + // of the desired field, ensuring it remains in the same allocation (happens within the + // macros). + // * Convert the address from the last step back into an offset within the iosys map + // (offset_from_ptr()). + // + // Once we do get field projection, this silly code should be removed. + // + /// Convert a pointer to an item within the iosys map back into an offset. + /// + /// # Safety + /// + /// `ptr` must be a valid pointer to data within the iosys map. + unsafe fn offset_from_ptr(&self, ptr: *const F) -> usize { + // SAFETY: `ptr` always points to data within the memory pointed to by the iosys map, + // meaning it is within the same memory allocation. + // + // Additionally, since `ptr` is within the iosys mapping, the offset here will always be + // positive and safe to cast to a usize. + // (TODO: replace this with byte_offset_from_unsigned once it's available in the kernel) + unsafe { ptr.byte_offset_from(self.as_ptr()) as usize } + } + + /// Reads the value of `field` and ensures that its type is [`FromBytes`]. + /// + /// # Safety + /// + /// This must be called from the [`iosys_map_read`] macro which ensures that the `field` + /// pointer is validated beforehand. + /// + /// Public but hidden since it should only be used from the [`iosys_map_read`] macro. + #[doc(hidden)] + pub unsafe fn field_read(&self, field: *const F) -> F { + let mut field_val = MaybeUninit::::uninit(); + + // SAFETY: `field` is guaranteed valid via our safety contract. + let offset = unsafe { self.offset_from_ptr(field) }; + + // SAFETY: Since we verified `field` is valid above, `offset_from_ptr` will always return a + // valid offset within the iosys map. + unsafe { + bindings::iosys_map_memcpy_from( + field_val.as_mut_ptr().cast(), + self.as_raw(), + offset, + mem::size_of::(), + ) + } + + // SAFETY: We just initialized `field_val` above. + unsafe { field_val.assume_init() } + } + + /// Writes the value of `field` and ensures that its type is [`AsBytes`]. + /// + /// # Safety + /// + /// This must be called from the [`iosys_map_write`] macro which ensures that the `field` + /// pointers validated beforehand. + /// + /// Public but hidden since it should only be used from the [`iosys_map_write`] macro. + #[doc(hidden)] + pub unsafe fn field_write(&mut self, field: *mut F, val: F) { + // SAFETY: `field` is guaranteed valid via our safety contract. + let offset = unsafe { self.offset_from_ptr(field) }; + + // SAFETY: `offset_from_ptr` always returns a valid offset within the iosys map. + unsafe { + bindings::iosys_map_memcpy_to( + self.as_raw_mut(), + offset, + core::ptr::from_ref(&val).cast(), + mem::size_of::(), + ) + } + } +} + +impl<'a, T: AsBytes + FromBytes> Deref for IoSysMapRef<'a, T> { + type Target = RawIoSysMap; + + fn deref(&self) -> &Self::Target { + &self.map + } +} + +impl<'a, T: AsBytes + FromBytes> DerefMut for IoSysMapRef<'a, T> { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.map + } +} + +/// Reads from a field of an item from an iosys map ref. +/// +/// # Examples +/// +/// ``` +/// use kernel::{iosys_map::*, transmute::*}; +/// +/// #[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// struct MyStruct { a: u32, b: u16 } +/// +/// // SAFETY: All bit patterns are acceptable values for `MyStruct`. +/// unsafe impl FromBytes for MyStruct {}; +/// // SAFETY: Instances of `MyStruct` have no uninitialized portions. +/// unsafe impl AsBytes for MyStruct {}; +/// +/// # fn test() -> Result { +/// # let mut map = tests::VecIoSysMap::new(&[MyStruct { a: 42, b: 2 }; 3])?; +/// # let map = map.get(); +/// let whole = kernel::iosys_map_read!(map[2])?; +/// assert_eq!(whole, MyStruct { a: 42, b: 2 }); +/// +/// let field = kernel::iosys_map_read!(map[1].b)?; +/// assert_eq!(field, 2); +/// # Ok::<(), Error>(()) } +/// # assert!(test().is_ok()); +/// ``` +#[macro_export] +macro_rules! iosys_map_read { + ($map:expr, $idx:expr, $($field:tt)*) => {{ + (|| -> ::core::result::Result<_, $crate::error::Error> { + let map = &$map; + let item = $crate::iosys_map::IoSysMapRef::ptr_from_index(map, $idx)?; + + // SAFETY: `ptr_from_index()` ensures that `item` is always a valid (although + // potentially not dereferenceable, which is fine here) pointer to within the iosys + // mapping. + unsafe { + let ptr_field = &raw const (*item) $($field)*; + ::core::result::Result::Ok( + $crate::iosys_map::IoSysMapRef::field_read(map, ptr_field) + ) + } + })() + }}; + ($map:ident [ $idx: expr ] $($field:tt)* ) => { + $crate::iosys_map_read!($map, $idx, $($field)*) + }; + ($($map:ident).* [ $idx:expr ] $($field:tt)* ) => { + $crate::iosys_map_read!($($map).*, $idx, $($field)*) + }; +} + +/// Writes to a field of an item from an iosys map ref. +/// +/// # Examples +/// +/// ``` +/// use kernel::{iosys_map::*, transmute::*}; +/// +/// #[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// struct MyStruct { a: u32, b: u16 }; +/// +/// // SAFETY: All bit patterns are acceptable values for `MyStruct`. +/// unsafe impl FromBytes for MyStruct {}; +/// // SAFETY: Instances of `MyStruct` have no uninitialized portions. +/// unsafe impl AsBytes for MyStruct {}; +/// +/// # fn test() -> Result { +/// # let mut map = tests::VecIoSysMap::new(&[MyStruct { a: 42, b: 2 }; 3])?; +/// # let mut map = map.get(); +/// kernel::iosys_map_write!(map[2].b = 1337)?; +/// # assert_eq!(kernel::iosys_map_read!(map[2].b)?, 1337); +/// +/// kernel::iosys_map_write!(map[1] = MyStruct { a: 10, b: 20 })?; +/// # assert_eq!(kernel::iosys_map_read!(map[1])?, MyStruct { a: 10, b: 20 }); +/// # Ok::<(), Error>(()) } +/// # assert!(test().is_ok()); +/// ``` +#[macro_export] +macro_rules! iosys_map_write { + ($map:ident [ $idx:expr ] $($field:tt)*) => {{ + $crate::iosys_map_write!($map, $idx, $($field)*) + }}; + ($($map:ident).* [ $idx:expr ] $($field:tt)* ) => {{ + $crate::iosys_map_write!($($map).*, $idx, $($field)*) + }}; + ($map:expr, $idx:expr, = $val:expr) => { + (|| -> ::core::result::Result<_, $crate::error::Error> { + // (expand these outside of the unsafe block (clippy::macro-metavars-in-unsafe) + let map = &mut $map; + let val = $val; + + let item = $crate::iosys_map::IoSysMapRef::ptr_from_index(map, $idx)?; + // SAFETY: `item_from_index` ensures that `item` is always a valid item. + unsafe { $crate::iosys_map::IoSysMapRef::field_write(map, item, val) }; + ::core::result::Result::Ok(()) + })() + }; + ($map:expr, $idx:expr, $(.$field:ident)* = $val:expr) => { + (|| -> ::core::result::Result<_, $crate::error::Error> { + // (expand these outside of the unsafe block (clippy::macro-metavars-in-unsafe) + let map = &mut $map; + let val = $val; + + let item = $crate::iosys_map::IoSysMapRef::ptr_from_index(map, $idx)?; + + // SAFETY: `ptr_from_index()` ensures that `item` is always a valid (although + // potentially not dereferenceable, which is fine here) pointer to within the iosys + // mapping. + unsafe { + let ptr_field = &raw mut (*item) $(.$field)*; + $crate::iosys_map::IoSysMapRef::field_write(map, ptr_field, val) + }; + ::core::result::Result::Ok(()) + })() + }; +} + +#[doc(hidden)] +#[kunit_tests(rust_iosys_map)] +pub mod tests { + use super::*; + + /// A helper struct for managed IoSysMapRef structs which point to a [`Vec`]. + pub struct VecIoSysMap { + map: RawIoSysMap, + vec: KVec, + } + + impl VecIoSysMap { + pub fn new(src: &[T]) -> Result { + let mut vec = KVec::::new(); + + vec.extend_from_slice(src, GFP_KERNEL)?; + + let map = RawIoSysMap( + bindings::iosys_map { + is_iomem: false, + __bindgen_anon_1: bindings::iosys_map__bindgen_ty_1 { + vaddr: vec.as_mut_ptr().cast(), + }, + }, + PhantomData, + ); + + Ok(Self { map, vec }) + } + + pub fn get(&mut self) -> IoSysMapRef<'_, T> { + // SAFETY: + // * `map` points to `vec`, so the size of `map` is the size of the `vec`. + unsafe { IoSysMapRef::new(self.map.clone(), self.vec.len() * self.map.item_size()) } + } + + /// Assert whether or not the contents of this struct match src. + pub fn assert_eq(&self, src: &[T]) { + assert_eq!(*self.vec.as_ref(), *src) + } + } + + #[test] + fn basic() -> Result { + let mut map = VecIoSysMap::new(&[0; 3])?; + + map.get().write(&[1, 2, 3], 0)?; + map.assert_eq(&[1, 2, 3]); + + map.get().write(&[42], 1)?; + map.assert_eq(&[1, 42, 3]); + + Ok(()) + } + + #[test] + fn oob_accesses() -> Result { + let mut map = VecIoSysMap::new(&[0; 3])?; + + assert!(map.get().write(&[1, 2, 3, 69], 0).is_err()); + assert!(map.get().write(&[1, 2, 3], 69).is_err()); + map.assert_eq(&[0; 3]); + + Ok(()) + } + + #[test] + fn overflows() -> Result { + let mut map = VecIoSysMap::new(&[0; 3])?; + + assert!(map.get().write(&[1], usize::MAX).is_err()); + map.assert_eq(&[0; 3]); + + Ok(()) + } + + #[derive(Copy, Clone, Debug, PartialEq, Eq)] + struct TestStruct { + a: u32, + b: u64, + } + + // SAFETY: All bit patterns are acceptable values for `TestStruct`. + unsafe impl FromBytes for TestStruct {} + // SAFETY: Instances of `TestStruct` have no uninitialized portions. + unsafe impl AsBytes for TestStruct {} + + #[test] + fn basic_macro() -> Result { + let mut expected = [TestStruct { a: 1, b: 2 }; 5]; + let mut map = VecIoSysMap::new(&expected)?; + + { + let mut map_ref = map.get(); + + iosys_map_write!(map_ref[3].a = u32::MAX)?; + expected[3].a = u32::MAX; + + assert_eq!(iosys_map_read!(map_ref[3].a)?, u32::MAX); + assert_eq!( + iosys_map_read!(map_ref[3])?, + TestStruct { a: u32::MAX, b: 2 } + ); + } + + // Compare the entire array, so that we catch any mis-sized writes. + map.assert_eq(&expected); + + Ok(()) + } + + #[test] + fn macro_oob_accesses() -> Result { + let mut map = VecIoSysMap::new(&[TestStruct { a: 1, b: 2 }; 3])?; + let mut map = map.get(); + + assert!(iosys_map_read!(map[5].b).is_err()); + assert!(iosys_map_read!(map[1000]).is_err()); + assert!(iosys_map_write!(map[6969].a = 999).is_err()); + assert!(iosys_map_write!(map[243] = TestStruct { a: 99, b: 22 }).is_err()); + + Ok(()) + } + + #[test] + fn macro_overflows() -> Result { + let mut map = VecIoSysMap::new(&[TestStruct { a: 1, b: 2 }; 3])?; + let mut map = map.get(); + + assert!(iosys_map_read!(map[usize::MAX]).is_err()); + assert!(iosys_map_read!(map[usize::MAX].b).is_err()); + assert!(iosys_map_write!(map[usize::MAX] = TestStruct { a: 1, b: 1 }).is_err()); + assert!(iosys_map_write!(map[usize::MAX].b = 1).is_err()); + + Ok(()) + } +} diff --git a/rust/kernel/irq/request.rs b/rust/kernel/irq/request.rs index b150563fdef809..2ceeaeb0543a4e 100644 --- a/rust/kernel/irq/request.rs +++ b/rust/kernel/irq/request.rs @@ -261,7 +261,10 @@ impl Registration { /// # Safety /// /// This function should be only used as the callback in `request_irq`. -unsafe extern "C" fn handle_irq_callback(_irq: i32, ptr: *mut c_void) -> c_uint { +unsafe extern "C" fn handle_irq_callback( + _irq: i32, + ptr: *mut c_void, +) -> c_uint { // SAFETY: `ptr` is a pointer to `Registration` set in `Registration::new` let registration = unsafe { &*(ptr as *const Registration) }; // SAFETY: The irq callback is removed before the device is unbound, so the fact that the irq @@ -480,7 +483,7 @@ impl ThreadedRegistration { /// # Safety /// /// This function should be only used as the callback in `request_threaded_irq`. -unsafe extern "C" fn handle_threaded_irq_callback( +unsafe extern "C" fn handle_threaded_irq_callback( _irq: i32, ptr: *mut c_void, ) -> c_uint { @@ -496,7 +499,10 @@ unsafe extern "C" fn handle_threaded_irq_callback( /// # Safety /// /// This function should be only used as the callback in `request_threaded_irq`. -unsafe extern "C" fn thread_fn_callback(_irq: i32, ptr: *mut c_void) -> c_uint { +unsafe extern "C" fn thread_fn_callback( + _irq: i32, + ptr: *mut c_void, +) -> c_uint { // SAFETY: `ptr` is a pointer to `ThreadedRegistration` set in `ThreadedRegistration::new` let registration = unsafe { &*(ptr as *const ThreadedRegistration) }; // SAFETY: The irq callback is removed before the device is unbound, so the fact that the irq diff --git a/rust/kernel/kunit.rs b/rust/kernel/kunit.rs index 79436509dd73df..8907b6f89ece5a 100644 --- a/rust/kernel/kunit.rs +++ b/rust/kernel/kunit.rs @@ -17,6 +17,10 @@ use crate::c_str; /// Public but hidden since it should only be used from KUnit generated code. #[doc(hidden)] pub fn err(args: fmt::Arguments<'_>) { + // `args` is unused if `CONFIG_PRINTK` is not set - this avoids a build-time warning. + #[cfg(not(CONFIG_PRINTK))] + let _ = args; + // SAFETY: The format string is null-terminated and the `%pA` specifier matches the argument we // are passing. #[cfg(CONFIG_PRINTK)] @@ -33,6 +37,10 @@ pub fn err(args: fmt::Arguments<'_>) { /// Public but hidden since it should only be used from KUnit generated code. #[doc(hidden)] pub fn info(args: fmt::Arguments<'_>) { + // `args` is unused if `CONFIG_PRINTK` is not set - this avoids a build-time warning. + #[cfg(not(CONFIG_PRINTK))] + let _ = args; + // SAFETY: The format string is null-terminated and the `%pA` specifier matches the argument we // are passing. #[cfg(CONFIG_PRINTK)] diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index f812cf12004286..a978d7f0065fae 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -16,6 +16,21 @@ // Please see https://github.com/Rust-for-Linux/linux/issues/2 for details on // the unstable features in use. // +// ============ start asahi downstream features =========== +#![feature(associated_type_defaults)] +// +#![feature(cfg_version)] +// +// Stable since Rust 1.87.0. +#![feature(ptr_sub_ptr)] +// +#![feature(sized_type_properties)] +// +#![feature(slice_range)] +// +#![cfg_attr(CONFIG_RUSTC_HAS_COERCE_POINTEE, feature(pin_coerce_unsized_trait))] +// ============ end asahi dowanstream features ============ +// // Stable since Rust 1.79.0. #![feature(generic_nonzero)] #![feature(inline_const)] @@ -65,6 +80,7 @@ extern crate self as kernel; pub use ffi; pub mod acpi; +pub mod addr; pub mod alloc; #[cfg(CONFIG_AUXILIARY_BUS)] pub mod auxiliary; @@ -84,10 +100,15 @@ pub mod cpufreq; pub mod cpumask; pub mod cred; pub mod debugfs; +#[cfg(CONFIG_DEV_COREDUMP)] +pub mod devcoredump; pub mod device; pub mod device_id; pub mod devres; pub mod dma; +pub mod dma_buf; +#[cfg(CONFIG_DMA_SHARED_BUFFER)] +pub mod dma_fence; pub mod driver; #[cfg(CONFIG_DRM = "y")] pub mod drm; @@ -100,9 +121,14 @@ pub mod fs; #[cfg(CONFIG_I2C = "y")] pub mod i2c; pub mod id_pool; +#[cfg(CONFIG_IIO)] +pub mod iio; +#[doc(hidden)] +pub mod impl_flags; pub mod init; pub mod io; pub mod ioctl; +pub mod iosys_map; pub mod iov; pub mod irq; pub mod jump_label; @@ -138,6 +164,7 @@ pub mod security; pub mod seq_file; pub mod sizes; pub mod slice; +pub mod soc; mod static_assert; #[doc(hidden)] pub mod std_vendor; diff --git a/rust/kernel/list/impl_list_item_mod.rs b/rust/kernel/list/impl_list_item_mod.rs index 202bc6f97c1326..ee53d0387e63dc 100644 --- a/rust/kernel/list/impl_list_item_mod.rs +++ b/rust/kernel/list/impl_list_item_mod.rs @@ -84,11 +84,12 @@ macro_rules! impl_has_list_links_self_ptr { // right type. unsafe impl$(<$($generics)*>)? $crate::list::HasSelfPtr<$item_type $(, $id)?> for $self {} + // SAFETY: TODO. unsafe impl$(<$($generics)*>)? $crate::list::HasListLinks$(<$id>)? for $self { #[inline] unsafe fn raw_get_list_links(ptr: *mut Self) -> *mut $crate::list::ListLinks$(<$id>)? { - // SAFETY: The caller promises that the pointer is not dangling. let ptr: *mut $crate::list::ListLinksSelfPtr<$item_type $(, $id)?> = + // SAFETY: The caller promises that the pointer is not dangling. unsafe { ::core::ptr::addr_of_mut!((*ptr)$(.$field)*) }; ptr.cast() } @@ -217,7 +218,7 @@ macro_rules! impl_list_item { // SAFETY: `me` originates from the most recent call to `prepare_to_insert`, so it // points at the field `$field` in a value of type `Self`. Thus, reversing that // operation is still in-bounds of the allocation. - $crate::container_of!(me, Self, $($field).*) + unsafe { $crate::container_of!(me, Self, $($field).*) } } // GUARANTEES: @@ -242,7 +243,7 @@ macro_rules! impl_list_item { // SAFETY: `me` originates from the most recent call to `prepare_to_insert`, so it // points at the field `$field` in a value of type `Self`. Thus, reversing that // operation is still in-bounds of the allocation. - $crate::container_of!(me, Self, $($field).*) + unsafe { $crate::container_of!(me, Self, $($field).*) } } } )*}; @@ -270,9 +271,12 @@ macro_rules! impl_list_item { // SAFETY: The caller promises that `me` points at a valid value of type `Self`. let links_field = unsafe { >::view_links(me) }; - let container = $crate::container_of!( - links_field, $crate::list::ListLinksSelfPtr, inner - ); + // SAFETY: TODO. + let container = unsafe { + $crate::container_of!( + links_field, $crate::list::ListLinksSelfPtr, inner + ) + }; // SAFETY: By the same reasoning above, `links_field` is a valid pointer. let self_ptr = unsafe { @@ -319,9 +323,12 @@ macro_rules! impl_list_item { // `ListArc` containing `Self` until the next call to `post_remove`. The value cannot // be destroyed while a `ListArc` reference exists. unsafe fn view_value(links_field: *mut $crate::list::ListLinks<$num>) -> *const Self { - let container = $crate::container_of!( - links_field, $crate::list::ListLinksSelfPtr, inner - ); + // SAFETY: TODO. + let container = unsafe { + $crate::container_of!( + links_field, $crate::list::ListLinksSelfPtr, inner + ) + }; // SAFETY: By the same reasoning above, `links_field` is a valid pointer. let self_ptr = unsafe { diff --git a/rust/kernel/of.rs b/rust/kernel/of.rs index 58b20c367f993f..80c604ee831452 100644 --- a/rust/kernel/of.rs +++ b/rust/kernel/of.rs @@ -8,6 +8,17 @@ use crate::{ prelude::*, }; +// Note: Most OF functions turn into inline dummies with CONFIG_OF(_*) disabled. +// We have to either add config conditionals to helpers.c or here; let's do it +// here for now. In the future, once bindgen can auto-generate static inline +// helpers, this can go away if desired. + +use core::marker::PhantomData; +use core::num::NonZeroU32; + +use crate::error::to_result; +use crate::io::resource::Resource; + /// IdTable type for OF drivers. pub type IdTable = &'static dyn kernel::device_id::IdTable; @@ -50,6 +61,518 @@ impl DeviceId { } } +/// Type alias for an OF phandle +pub type PHandle = bindings::phandle; + +/// An OF device tree node. +/// +/// # Invariants +/// +/// `raw_node` points to a valid OF node, and we hold a reference to it. +pub struct Node { + raw_node: *mut bindings::device_node, +} + +#[allow(dead_code)] +impl Node { + /// Creates a `Node` from a raw C pointer. The pointer must be owned (the caller + /// gives up its reference). If the pointer is NULL, returns None. + pub(crate) unsafe fn from_raw(raw_node: *mut bindings::device_node) -> Option { + if raw_node.is_null() { + None + } else { + // INVARIANT: `raw_node` is valid per the above contract, and non-null per the + // above check. + Some(Node { raw_node }) + } + } + + /// Creates a `Node` from a raw C pointer. The pointer must be borrowed (the caller + /// retains its reference, which must be valid for the duration of the call). If the + /// pointer is NULL, returns None. + pub(crate) unsafe fn get_from_raw(raw_node: *mut bindings::device_node) -> Option { + // SAFETY: `raw_node` is valid or NULL per the above contract. `of_node_get` can handle + // NULL. + unsafe { + #[cfg(CONFIG_OF_DYNAMIC)] + bindings::of_node_get(raw_node); + Node::from_raw(raw_node) + } + } + + /// Returns a reference to the underlying C `device_node` structure. + pub(crate) fn node(&self) -> &bindings::device_node { + // SAFETY: `raw_node` is valid per the type invariant. + unsafe { &*self.raw_node } + } + + /// Returns the name of the node. + pub(crate) fn name(&self) -> &CStr { + // SAFETY: The lifetime of the `CStr` is the same as the lifetime of this `Node`. + unsafe { CStr::from_char_ptr(self.node().name) } + } + + /// Returns the phandle for this node. + pub(crate) fn phandle(&self) -> PHandle { + self.node().phandle + } + + /// Returns the full name (with address) for this node. + pub(crate) fn full_name(&self) -> &CStr { + // SAFETY: The lifetime of the `CStr` is the same as the lifetime of this `Node`. + unsafe { CStr::from_char_ptr(self.node().full_name) } + } + + /// Returns `true` if the node is the root node. + pub(crate) fn is_root(&self) -> bool { + #[cfg(not(CONFIG_OF))] + { + false + } + #[cfg(CONFIG_OF)] + // SAFETY: `raw_node` is valid per the type invariant + unsafe { + bindings::of_node_is_root(self.raw_node) + } + } + + /// Returns the parent node, if any. + pub(crate) fn parent(&self) -> Option { + #[cfg(not(CONFIG_OF))] + { + None + } + #[cfg(CONFIG_OF)] + // SAFETY: `raw_node` is valid per the type invariant, and `of_get_parent()` takes a + // new reference to the parent (or returns NULL). + unsafe { + Node::from_raw(bindings::of_get_parent(self.raw_node)) + } + } + + /// Returns an iterator over the node's children. + // TODO: use type alias for return type once type_alias_impl_trait is stable + pub fn children( + &self, + ) -> NodeIterator<'_, impl Fn(*mut bindings::device_node) -> *mut bindings::device_node + '_> + { + #[cfg(not(CONFIG_OF))] + { + NodeIterator::new(|_prev| core::ptr::null_mut()) + } + #[cfg(CONFIG_OF)] + // SAFETY: `raw_node` is valid per the type invariant, and the lifetime of the `NodeIterator` + // does not exceed the lifetime of the `Node` so it can borrow its reference. + NodeIterator::new(|prev| unsafe { bindings::of_get_next_child(self.raw_node, prev) }) + } + + /// Find a child by its name and return it, or None if not found. + #[allow(unused_variables)] + pub(crate) fn get_child_by_name(&self, name: &CStr) -> Option { + #[cfg(not(CONFIG_OF))] + { + None + } + #[cfg(CONFIG_OF)] + // SAFETY: `raw_node` is valid per the type invariant. + unsafe { + Node::from_raw(bindings::of_get_child_by_name( + self.raw_node, + name.as_char_ptr(), + )) + } + } + + /// Checks whether the node is compatible with the given compatible string. + /// + /// Returns `None` if there is no match, or `Some` if there is, with the value + /// representing as match score (higher values for more specific compatible matches). + #[allow(unused_variables)] + pub(crate) fn is_compatible(&self, compatible: &CStr) -> Option { + #[cfg(not(CONFIG_OF))] + let ret = 0; + #[cfg(CONFIG_OF)] + let ret = + // SAFETY: `raw_node` is valid per the type invariant. + unsafe { bindings::of_device_is_compatible(self.raw_node, compatible.as_char_ptr()) }; + + NonZeroU32::new(ret.try_into().ok()?) + } + + /// Parse a phandle property and return the Node referenced at a given index, if any. + /// + /// Used only for phandle properties with no arguments. + #[allow(unused_variables)] + pub fn parse_phandle(&self, name: &CStr, index: usize) -> Option { + #[cfg(not(CONFIG_OF))] + { + None + } + #[cfg(CONFIG_OF)] + // SAFETY: `raw_node` is valid per the type invariant. `of_parse_phandle` returns an + // owned reference. + unsafe { + Node::from_raw(bindings::of_parse_phandle( + self.raw_node, + name.as_char_ptr(), + index.try_into().ok()?, + )) + } + } + + #[allow(unused_variables)] + /// Get a reserved memory region as a resource + pub fn reserved_mem_region_to_resource_byname(&self, name: &CStr) -> Result { + #[cfg(not(CONFIG_OF))] + { + Err(ENOENT) + } + #[cfg(CONFIG_OF)] + { + let res = Resource::zeroed(); + // SAFETY: This function is safe to call as long as the arguments are valid pointers. + let ret = unsafe { + bindings::of_reserved_mem_region_to_resource_byname( + self.raw_node, + name.as_char_ptr(), + res.as_raw(), + ) + }; + to_result(ret)?; + + Ok(res) + } + } + + #[allow(unused_variables)] + /// Look up a node property by name, returning a `Property` object if found. + pub(crate) fn find_property(&self, propname: &CStr) -> Option> { + #[cfg(not(CONFIG_OF))] + { + None + } + #[cfg(CONFIG_OF)] + // SAFETY: `raw_node` is valid per the type invariant. The property structure + // returned borrows the reference to the owning node, and so has the same + // lifetime. + unsafe { + Property::from_raw(bindings::of_find_property( + self.raw_node, + propname.as_char_ptr(), + core::ptr::null_mut(), + )) + } + } + + /// Look up a mandatory node property by name, and decode it into a value type. + /// + /// Returns `Err(ENOENT)` if the property is not found. + /// + /// The type `T` must implement `TryFrom>`. + pub fn get_property<'a, T: TryFrom>>(&'a self, propname: &CStr) -> Result + where + crate::error::Error: From<>>::Error>, + { + Ok(self.find_property(propname).ok_or(ENOENT)?.try_into()?) + } + + /// Look up an optional node property by name, and decode it into a value type. + /// + /// Returns `Ok(None)` if the property is not found. + /// + /// The type `T` must implement `TryFrom>`. + pub fn get_opt_property<'a, T: TryFrom>>( + &'a self, + propname: &CStr, + ) -> Result> + where + crate::error::Error: From<>>::Error>, + { + self.find_property(propname) + .map_or(Ok(None), |p| Ok(Some(p.try_into()?))) + } +} + +/// A property attached to a device tree `Node`. +/// +/// # Invariants +/// +/// `raw` must be valid and point to a property that outlives the lifetime of this object. +#[derive(Copy, Clone)] +pub struct Property<'a> { + raw: *mut bindings::property, + _p: PhantomData<&'a Node>, +} + +impl<'a> Property<'a> { + #[cfg(CONFIG_OF)] + /// Create a `Property` object from a raw C pointer. Returns `None` if NULL. + /// + /// The passed pointer must be valid and outlive the lifetime argument, or NULL. + unsafe fn from_raw(raw: *mut bindings::property) -> Option> { + if raw.is_null() { + None + } else { + Some(Property { + raw, + _p: PhantomData, + }) + } + } + + /// Returns the name of the property as a `CStr`. + pub fn name(&self) -> &CStr { + // SAFETY: `raw` is valid per the type invariant, and the lifetime of the `CStr` does not + // outlive it. + unsafe { CStr::from_char_ptr((*self.raw).name) } + } + + /// Returns the name of the property as a `&[u8]`. + pub fn value(&self) -> &[u8] { + // SAFETY: `raw` is valid per the type invariant, and the lifetime of the slice does not + // outlive it. + unsafe { core::slice::from_raw_parts((*self.raw).value as *const u8, self.len()) } + } + + /// Returns the length of the property in bytes. + pub fn len(&self) -> usize { + // SAFETY: `raw` is valid per the type invariant. + unsafe { (*self.raw).length.try_into().unwrap() } + } + + /// Returns true if the property is empty (zero-length), which typically represents boolean true. + pub fn is_empty(&self) -> bool { + self.len() == 0 + } +} + +/// A trait that represents a value decodable from a property with a fixed unit size. +/// +/// This allows us to auto-derive property decode implementations for `Vec`. +pub trait PropertyUnit: Sized { + /// The size in bytes of a single data unit. + const UNIT_SIZE: usize; + + /// Decode this data unit from a byte slice. The passed slice will have a length of `UNIT_SIZE`. + fn from_bytes(data: &[u8]) -> Result; +} + +// This doesn't work... +// impl<'a, T: PropertyUnit> TryFrom> for T { +// type Error = Error; +// +// fn try_from(p: Property<'_>) -> core::result::Result { +// if p.value().len() != T::UNIT_SIZE { +// Err(EINVAL) +// } else { +// Ok(T::from_bytes(p.value())?) +// } +// } +// } + +impl<'a, T: PropertyUnit> TryFrom> for KVec { + type Error = Error; + + fn try_from(p: Property<'_>) -> core::result::Result, Self::Error> { + if p.len() % T::UNIT_SIZE != 0 { + return Err(EINVAL); + } + + let mut v = Vec::new(); + let val = p.value(); + for off in (0..p.len()).step_by(T::UNIT_SIZE) { + v.push(T::from_bytes(&val[off..off + T::UNIT_SIZE])?, GFP_KERNEL)?; + } + Ok(v) + } +} + +macro_rules! prop_int_type ( + ($type:ty) => { + impl<'a> TryFrom> for $type { + type Error = Error; + + fn try_from(p: Property<'_>) -> core::result::Result<$type, Self::Error> { + Ok(<$type>::from_be_bytes(p.value().try_into().or(Err(EINVAL))?)) + } + } + + impl PropertyUnit for $type { + const UNIT_SIZE: usize = <$type>::BITS as usize / 8; + + fn from_bytes(data: &[u8]) -> Result { + Ok(<$type>::from_be_bytes(data.try_into().or(Err(EINVAL))?)) + } + } + } +); + +prop_int_type!(u8); +prop_int_type!(u16); +prop_int_type!(u32); +prop_int_type!(u64); +prop_int_type!(i8); +prop_int_type!(i16); +prop_int_type!(i32); +prop_int_type!(i64); + +/// An iterator across a collection of Node objects. +/// +/// # Invariants +/// +/// `cur` must be NULL or a valid node owned reference. If NULL, it represents either the first +/// or last position of the iterator. +/// +/// If `done` is true, `cur` must be NULL. +/// +/// fn_next must be a callback that iterates from one node to the next, and it must not capture +/// values that exceed the lifetime of the iterator. It must return owned references and also +/// take owned references. +pub struct NodeIterator<'a, T> +where + T: Fn(*mut bindings::device_node) -> *mut bindings::device_node, +{ + cur: *mut bindings::device_node, + done: bool, + fn_next: T, + _p: PhantomData<&'a T>, +} + +impl<'a, T> NodeIterator<'a, T> +where + T: Fn(*mut bindings::device_node) -> *mut bindings::device_node, +{ + fn new(next: T) -> NodeIterator<'a, T> { + // INVARIANT: `cur` is initialized to NULL to represent the initial state. + NodeIterator { + cur: core::ptr::null_mut(), + done: false, + fn_next: next, + _p: PhantomData, + } + } +} + +impl<'a, T> Iterator for NodeIterator<'a, T> +where + T: Fn(*mut bindings::device_node) -> *mut bindings::device_node, +{ + type Item = Node; + + fn next(&mut self) -> Option { + if self.done { + None + } else { + // INVARIANT: if the new `cur` is NULL, then the iterator has reached its end and we + // set `done` to `true`. + self.cur = (self.fn_next)(self.cur); + self.done = self.cur.is_null(); + // SAFETY: `fn_next` must return an owned reference per the iterator contract. + // The iterator itself is considered to own this reference, so we take another one. + unsafe { Node::get_from_raw(self.cur) } + } + } +} + +// Drop impl to ensure we drop the current node being iterated on, if any. +impl<'a, T> Drop for NodeIterator<'a, T> +where + T: Fn(*mut bindings::device_node) -> *mut bindings::device_node, +{ + fn drop(&mut self) { + // SAFETY: `cur` is valid or NULL, and `of_node_put()` can handle NULL. + #[cfg(CONFIG_OF_DYNAMIC)] + unsafe { + bindings::of_node_put(self.cur) + }; + } +} + +/// Returns the root node of the OF device tree (if any). +pub fn root() -> Option { + #[cfg(not(CONFIG_OF))] + { + None + } + #[cfg(CONFIG_OF)] + // SAFETY: bindings::of_root is always valid or NULL + unsafe { + Node::get_from_raw(bindings::of_root) + } +} + +/// Returns the /chosen node of the OF device tree (if any). +pub fn chosen() -> Option { + #[cfg(not(CONFIG_OF))] + { + None + } + #[cfg(CONFIG_OF)] + // SAFETY: bindings::of_chosen is always valid or NULL + unsafe { + Node::get_from_raw(bindings::of_chosen) + } +} + +/// Returns the /aliases node of the OF device tree (if any). +pub fn aliases() -> Option { + #[cfg(not(CONFIG_OF))] + { + None + } + #[cfg(CONFIG_OF)] + // SAFETY: bindings::of_aliases is always valid or NULL + unsafe { + Node::get_from_raw(bindings::of_aliases) + } +} + +/// Returns the system stdout node of the OF device tree (if any). +pub fn stdout() -> Option { + #[cfg(not(CONFIG_OF))] + { + None + } + #[cfg(CONFIG_OF)] + // SAFETY: bindings::of_stdout is always valid or NULL + unsafe { + Node::get_from_raw(bindings::of_stdout) + } +} + +#[allow(unused_variables)] +/// Looks up a node in the device tree by phandle. +pub fn find_node_by_phandle(handle: PHandle) -> Option { + #[cfg(not(CONFIG_OF))] + { + None + } + #[cfg(CONFIG_OF)] + // SAFETY: bindings::of_find_node_by_phandle always returns a valid pointer or NULL + unsafe { + #[allow(dead_code)] + Node::from_raw(bindings::of_find_node_by_phandle(handle)) + } +} + +impl Clone for Node { + fn clone(&self) -> Node { + // SAFETY: `raw_node` is valid and non-NULL per the type invariant, + // so this can never return None. + unsafe { Node::get_from_raw(self.raw_node).unwrap() } + } +} + +impl Drop for Node { + fn drop(&mut self) { + #[cfg(CONFIG_OF_DYNAMIC)] + // SAFETY: `raw_node` is valid per the type invariant. + unsafe { + bindings::of_node_put(self.raw_node) + }; + } +} + /// Create an OF `IdTable` with an "alias" for modpost. #[macro_export] macro_rules! of_device_table { diff --git a/rust/kernel/page.rs b/rust/kernel/page.rs index 432fc0297d4a85..c57311c2a64fed 100644 --- a/rust/kernel/page.rs +++ b/rust/kernel/page.rs @@ -3,10 +3,12 @@ //! Kernel page allocation and management. use crate::{ + addr::*, alloc::{AllocError, Flags}, bindings, error::code::*, error::Result, + types::{Opaque, Ownable, Owned}, uaccess::UserSliceReader, }; use core::{ @@ -77,7 +79,7 @@ pub const fn page_align(addr: usize) -> usize { /// /// [`VBox`]: kernel::alloc::VBox /// [`Vmalloc`]: kernel::alloc::allocator::Vmalloc -pub struct BorrowedPage<'a>(ManuallyDrop, PhantomData<&'a Page>); +pub struct BorrowedPage<'a>(ManuallyDrop>, PhantomData<&'a Page>); impl<'a> BorrowedPage<'a> { /// Constructs a [`BorrowedPage`] from a raw pointer to a `struct page`. @@ -87,7 +89,7 @@ impl<'a> BorrowedPage<'a> { /// - `ptr` must point to a valid `bindings::page`. /// - `ptr` must remain valid for the entire lifetime `'a`. pub unsafe fn from_raw(ptr: NonNull) -> Self { - let page = Page { page: ptr }; + let page = unsafe { Page::from_phys(bindings::page_to_phys(ptr.as_ptr())) }; // INVARIANT: The safety requirements guarantee that `ptr` is valid for the entire lifetime // `'a`. @@ -120,8 +122,9 @@ pub trait AsPageIter { /// # Invariants /// /// The pointer is valid, and has ownership over the page. +#[repr(transparent)] pub struct Page { - page: NonNull, + page: Opaque, } // SAFETY: Pages have no logic that relies on them staying on a given thread, so moving them across @@ -155,19 +158,20 @@ impl Page { /// # Ok::<(), kernel::alloc::AllocError>(()) /// ``` #[inline] - pub fn alloc_page(flags: Flags) -> Result { + pub fn alloc_page(flags: Flags) -> Result, AllocError> { // SAFETY: Depending on the value of `gfp_flags`, this call may sleep. Other than that, it // is always safe to call this method. let page = unsafe { bindings::alloc_pages(flags.as_raw(), 0) }; let page = NonNull::new(page).ok_or(AllocError)?; - // INVARIANT: We just successfully allocated a page, so we now have ownership of the newly - // allocated page. We transfer that ownership to the new `Page` object. - Ok(Self { page }) + // SAFETY: We just successfully allocated a page, so we now have ownership of the newly + // allocated page. We transfer that ownership to the new `Owned` object. + // Since `Page` is transparent, we can cast the pointer directly. + Ok(unsafe { Owned::from_raw(page.cast()) }) } /// Returns a raw pointer to the page. pub fn as_ptr(&self) -> *mut bindings::page { - self.page.as_ptr() + Opaque::cast_into(&self.page) } /// Get the node id containing this page. @@ -192,7 +196,7 @@ impl Page { /// different addresses. However, even if the addresses are different, the underlying memory is /// still the same for these purposes (e.g., it's still a data race if they both write to the /// same underlying byte at the same time). - fn with_page_mapped(&self, f: impl FnOnce(*mut u8) -> T) -> T { + pub fn with_page_mapped(&self, f: impl FnOnce(*mut u8) -> T) -> T { // SAFETY: `page` is valid due to the type invariants on `Page`. let mapped_addr = unsafe { bindings::kmap_local_page(self.as_ptr()) }; @@ -233,7 +237,7 @@ impl Page { /// different addresses. However, even if the addresses are different, the underlying memory is /// still the same for these purposes (e.g., it's still a data race if they both write to the /// same underlying byte at the same time). - fn with_pointer_into_page( + pub fn with_pointer_into_page( &self, off: usize, len: usize, @@ -340,12 +344,77 @@ impl Page { reader.read_raw(unsafe { core::slice::from_raw_parts_mut(dst.cast(), len) }) }) } + + /// Returns the physical address of this page. + pub fn phys(&self) -> PhysicalAddr { + // SAFETY: `page` is valid due to the type invariants on `Page`. + unsafe { bindings::page_to_phys(self.as_ptr()) } + } + + /// Converts a Rust-owned Page into its physical address. + /// The caller is responsible for calling `from_phys()` to avoid + /// leaking memory. + pub fn into_phys(this: Owned) -> PhysicalAddr { + ManuallyDrop::new(this).phys() + } + + /// Converts a physical address to a Rust-owned Page. + /// + /// SAFETY: + /// The caller must ensure that the physical address was previously returned + /// by a call to `Page::into_phys()`, and that the physical address is no + /// longer used after this call, nor is `from_phys()` called again on it. + pub unsafe fn from_phys(phys: PhysicalAddr) -> Owned { + // SAFETY: By the safety requirements, the physical address must be valid and + // have come from `into_phys()`, so phys_to_page() cannot fail and + // must return the original struct page pointer. + unsafe { Owned::from_raw(NonNull::new_unchecked(bindings::phys_to_page(phys)).cast()) } + } + + /// Borrows a Page from a physical address, without taking over ownership. + /// + /// If the physical address does not have a `struct page` entry or is not + /// part of the System RAM region, returns None. + /// + /// SAFETY: + /// The caller must ensure that the physical address, if it is backed by a + /// `struct page`, remains available for the duration of the borrowed + /// lifetime. + pub unsafe fn borrow_phys(phys: &PhysicalAddr) -> Option<&Self> { + // SAFETY: This is always safe, as it is just arithmetic + let pfn = unsafe { bindings::phys_to_pfn(*phys) }; + // SAFETY: This function is safe to call with any pfn + if !unsafe { bindings::pfn_valid(pfn) && bindings::page_is_ram(pfn) != 0 } { + None + } else { + // SAFETY: We have just checked that the pfn is valid above, so it must + // have a corresponding struct page. By the safety requirements, we can + // return a borrowed reference to it. + Some(unsafe { &*(bindings::pfn_to_page(pfn) as *mut Self as *const Self) }) + } + } + + /// Borrows a Page from a physical address, without taking over ownership + /// nor checking for validity. + /// + /// SAFETY: + /// The caller must ensure that the physical address is backed by a + /// `struct page` and corresponds to System RAM. + pub unsafe fn borrow_phys_unchecked(phys: &PhysicalAddr) -> &Self { + // SAFETY: This is always safe, as it is just arithmetic + let pfn = unsafe { bindings::phys_to_pfn(*phys) }; + // SAFETY: The caller guarantees that the pfn is valid. By the safety + // requirements, we can return a borrowed reference to it. + unsafe { &*(bindings::pfn_to_page(pfn) as *mut Self as *const Self) } + } } -impl Drop for Page { +// SAFETY: See below. +unsafe impl Ownable for Page { #[inline] - fn drop(&mut self) { + unsafe fn release(this: NonNull) { // SAFETY: By the type invariants, we have ownership of the page and can free it. - unsafe { bindings::__free_pages(self.page.as_ptr(), 0) }; + // Since Page is transparent, we can cast the raw pointer directly. + unsafe { bindings::__free_pages(this.cast().as_ptr(), 0) }; } } diff --git a/rust/kernel/platform.rs b/rust/kernel/platform.rs index 35a5813ffb33f1..54c7ede9e8d989 100644 --- a/rust/kernel/platform.rs +++ b/rust/kernel/platform.rs @@ -277,6 +277,20 @@ impl Device { // returned by `platform_get_resource`. Some(unsafe { Resource::from_raw(resource) }) } + + /// excute closure while the device is bound + pub fn while_bound_with(&self, f: F) -> Result + where + F: FnOnce(&Device) -> Result, + { + let _guard = self.as_ref().lock(); + if unsafe { !bindings::device_is_bound(self.as_ref().as_raw()) } { + return Err(ENODEV); + } + let ptr: *const Self = self; + let ptr = ptr.cast::>(); + f(unsafe { &*ptr }) + } } impl Device { diff --git a/rust/kernel/pwm.rs b/rust/kernel/pwm.rs index cb00f8a8765c8e..2ba9cfd02bfdbe 100644 --- a/rust/kernel/pwm.rs +++ b/rust/kernel/pwm.rs @@ -601,7 +601,11 @@ impl Chip { let drvdata_ptr = unsafe { bindings::pwmchip_get_drvdata(c_chip_ptr) }; // SAFETY: We construct the `T` object in-place in the allocated private memory. - unsafe { data.__pinned_init(drvdata_ptr.cast())? }; + unsafe { data.__pinned_init(drvdata_ptr.cast()) }.inspect_err(|_| { + // SAFETY: It is safe to call `pwmchip_put()` with a valid pointer obtained + // from `pwmchip_alloc()`. We will not use pointer after this. + unsafe { bindings::pwmchip_put(c_chip_ptr) } + })?; // SAFETY: `c_chip_ptr` points to a valid chip. unsafe { diff --git a/rust/kernel/regulator.rs b/rust/kernel/regulator.rs index 2c44827ad0b7e0..40c7f2209867d7 100644 --- a/rust/kernel/regulator.rs +++ b/rust/kernel/regulator.rs @@ -23,7 +23,10 @@ use crate::{ prelude::*, }; -use core::{marker::PhantomData, mem::ManuallyDrop, ptr::NonNull}; +use core::{ + marker::PhantomData, + mem::ManuallyDrop, // +}; mod private { pub trait Sealed {} @@ -232,15 +235,17 @@ pub fn devm_enable_optional(dev: &Device, name: &CStr) -> Result { /// /// # Invariants /// -/// - `inner` is a non-null wrapper over a pointer to a `struct -/// regulator` obtained from [`regulator_get()`]. +/// - `inner` is a pointer obtained from a successful call to +/// [`regulator_get()`]. It is treated as an opaque token that may only be +/// accessed using C API methods (e.g., it may be `NULL` if the C API returns +/// `NULL`). /// /// [`regulator_get()`]: https://docs.kernel.org/driver-api/regulator.html#c.regulator_get pub struct Regulator where State: RegulatorState, { - inner: NonNull, + inner: *mut bindings::regulator, _phantom: PhantomData, } @@ -252,7 +257,7 @@ impl Regulator { // SAFETY: Safe as per the type invariants of `Regulator`. to_result(unsafe { bindings::regulator_set_voltage( - self.inner.as_ptr(), + self.inner, min_voltage.as_microvolts(), max_voltage.as_microvolts(), ) @@ -262,7 +267,7 @@ impl Regulator { /// Gets the current voltage of the regulator. pub fn get_voltage(&self) -> Result { // SAFETY: Safe as per the type invariants of `Regulator`. - let voltage = unsafe { bindings::regulator_get_voltage(self.inner.as_ptr()) }; + let voltage = unsafe { bindings::regulator_get_voltage(self.inner) }; to_result(voltage).map(|()| Voltage::from_microvolts(voltage)) } @@ -273,10 +278,8 @@ impl Regulator { // received from the C code. from_err_ptr(unsafe { bindings::regulator_get(dev.as_raw(), name.as_char_ptr()) })?; - // SAFETY: We can safely trust `inner` to be a pointer to a valid - // regulator if `ERR_PTR` was not returned. - let inner = unsafe { NonNull::new_unchecked(inner) }; - + // INVARIANT: `inner` is a pointer obtained from `regulator_get()`, and + // the call was successful. Ok(Self { inner, _phantom: PhantomData, @@ -285,12 +288,12 @@ impl Regulator { fn enable_internal(&self) -> Result { // SAFETY: Safe as per the type invariants of `Regulator`. - to_result(unsafe { bindings::regulator_enable(self.inner.as_ptr()) }) + to_result(unsafe { bindings::regulator_enable(self.inner) }) } fn disable_internal(&self) -> Result { // SAFETY: Safe as per the type invariants of `Regulator`. - to_result(unsafe { bindings::regulator_disable(self.inner.as_ptr()) }) + to_result(unsafe { bindings::regulator_disable(self.inner) }) } } @@ -352,7 +355,7 @@ impl Regulator { /// Checks if the regulator is enabled. pub fn is_enabled(&self) -> bool { // SAFETY: Safe as per the type invariants of `Regulator`. - unsafe { bindings::regulator_is_enabled(self.inner.as_ptr()) != 0 } + unsafe { bindings::regulator_is_enabled(self.inner) != 0 } } } @@ -362,11 +365,11 @@ impl Drop for Regulator { // SAFETY: By the type invariants, we know that `self` owns a // reference on the enabled refcount, so it is safe to relinquish it // now. - unsafe { bindings::regulator_disable(self.inner.as_ptr()) }; + unsafe { bindings::regulator_disable(self.inner) }; } // SAFETY: By the type invariants, we know that `self` owns a reference, // so it is safe to relinquish it now. - unsafe { bindings::regulator_put(self.inner.as_ptr()) }; + unsafe { bindings::regulator_put(self.inner) }; } } diff --git a/rust/kernel/soc/apple/aop.rs b/rust/kernel/soc/apple/aop.rs new file mode 100644 index 00000000000000..da46ce0bcb027d --- /dev/null +++ b/rust/kernel/soc/apple/aop.rs @@ -0,0 +1,51 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Common code for AOP endpoint drivers + +use kernel::{prelude::*, sync::Arc}; + +/// Representation of an "EPIC" service. +#[derive(Clone, Copy, PartialEq, Eq)] +#[repr(C)] +pub struct EPICService { + /// Channel id + pub channel: u32, + /// RTKit endpoint + pub endpoint: u8, +} + +/// Listener for the "HID" events sent by aop +pub trait FakehidListener { + /// Process the event. + fn process_fakehid_report(&self, data: &[u8]) -> Result<()>; +} + +/// AOP communications manager. +pub trait AOP: Send + Sync { + /// Calls a method on a specified service + fn epic_call(&self, svc: &EPICService, subtype: u16, msg_bytes: &[u8]) -> Result; + /// Just like epic_call, but also returns a value + fn epic_call_ret( + &self, + svc: &EPICService, + subtype: u16, + msg_bytes: &[u8], + ret_len: usize, + ) -> Result<(u32, KVec)>; + + /// Adds the listener for the specified service + fn add_fakehid_listener( + &self, + svc: EPICService, + listener: Arc, + ) -> Result<()>; + /// Remove the listener for the specified service + fn remove_fakehid_listener(&self, svc: &EPICService) -> bool; + /// Internal method to detach the device. + fn remove(&self); +} + +/// Converts a text representation of a FourCC to u32 +pub const fn from_fourcc(b: &[u8]) -> u32 { + b[3] as u32 | (b[2] as u32) << 8 | (b[1] as u32) << 16 | (b[0] as u32) << 24 +} diff --git a/rust/kernel/soc/apple/mailbox.rs b/rust/kernel/soc/apple/mailbox.rs new file mode 100644 index 00000000000000..f477ddc0afb00e --- /dev/null +++ b/rust/kernel/soc/apple/mailbox.rs @@ -0,0 +1,111 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Support for Apple ASC Mailbox. +//! +//! C header: [`include/linux/soc/apple/mailbox.h`](../../../../include/linux/gpio/driver.h) + +use crate::{ + bindings, + device, + error::{ + from_err_ptr, + to_result, // + }, + prelude::*, + str::CStrExt, + types::{ + ForeignOwnable, + ScopeGuard, // + }, // +}; + +use core::marker::PhantomData; + +/// 96-bit message. What it means is up to the upper layer +pub type Message = bindings::apple_mbox_msg; + +/// Mailbox receive callback +pub trait MailCallback { + /// Callback context + type Data: ForeignOwnable + Send + Sync; + + /// The actual callback. Called in an interrupt context. + fn recv_message(data: ::Borrowed<'_>, msg: Message); +} + +/// Wrapper over `struct apple_mbox *` +#[repr(transparent)] +pub struct Mailbox { + mbox: *mut bindings::apple_mbox, + _p: PhantomData, +} + +extern "C" fn mailbox_rx_callback( + _mbox: *mut bindings::apple_mbox, + msg: Message, + cookie: *mut crate::ffi::c_void, +) { + // SAFETY: cookie came from a call to `into_foreign` + T::recv_message(unsafe { T::Data::borrow(cookie.cast()) }, msg); +} + +impl Mailbox { + /// Creates a mailbox for the specified name. + pub fn new_byname( + dev: &device::Device, + mbox_name: &'static CStr, + data: T::Data, + ) -> Result> { + let ptr: *mut crate::ffi::c_void = data.into_foreign().cast(); + let guard = ScopeGuard::new(|| { + // SAFETY: `ptr` came from a previous call to `into_foreign`. + unsafe { T::Data::from_foreign(ptr.cast()) }; + }); + // SAFETY: Just calling the c function, all values are valid. + let mbox = unsafe { + from_err_ptr(bindings::apple_mbox_get_byname( + dev.as_raw(), + mbox_name.as_char_ptr(), + ))? + }; + // SAFETY: mbox is a valid pointer + unsafe { + (*mbox).cookie = ptr; + (*mbox).rx = Some(mailbox_rx_callback::); + to_result(bindings::apple_mbox_start(mbox))?; + } + guard.dismiss(); + Ok(Mailbox { + mbox, + _p: PhantomData, + }) + } + /// Sends the specified message + pub fn send(&self, msg: Message, atomic: bool) -> Result<()> { + // SAFETY: Calling the c function, `mbox` is a valid pointer + to_result(unsafe { bindings::apple_mbox_send(self.mbox, msg, atomic) }) + } +} + +impl Drop for Mailbox { + fn drop(&mut self) { + // SAFETY: mbox is a valid pointer + unsafe { bindings::apple_mbox_stop(self.mbox) }; + // SAFETY: `cookie` came from `into_foreign` + unsafe { T::Data::from_foreign((*self.mbox).cookie.cast()) }; + } +} + +unsafe impl Sync for Mailbox +where + T: MailCallback, + T::Data: Sync, +{ +} + +unsafe impl Send for Mailbox +where + T: MailCallback, + T::Data: Send, +{ +} diff --git a/rust/kernel/soc/apple/mod.rs b/rust/kernel/soc/apple/mod.rs new file mode 100644 index 00000000000000..e77eba782a5867 --- /dev/null +++ b/rust/kernel/soc/apple/mod.rs @@ -0,0 +1,12 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Apple SoC drivers + +#[cfg(CONFIG_RUST_APPLE_RTKIT = "y")] +pub mod rtkit; + +#[cfg(any(CONFIG_APPLE_AOP = "y", CONFIG_APPLE_AOP = "m"))] +pub mod aop; + +#[cfg(CONFIG_RUST_APPLE_MAILBOX = "y")] +pub mod mailbox; diff --git a/rust/kernel/soc/apple/rtkit.rs b/rust/kernel/soc/apple/rtkit.rs new file mode 100644 index 00000000000000..a5bfcfb8fa7334 --- /dev/null +++ b/rust/kernel/soc/apple/rtkit.rs @@ -0,0 +1,293 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Support for Apple RTKit coprocessors. +//! +//! C header: [`include/linux/soc/apple/rtkit.h`](../../../../include/linux/gpio/driver.h) + +use crate::{ + alloc::flags::*, + bindings, + device, + error::{ + from_err_ptr, + from_result, + to_result, // + }, + iosys_map::IoSysMapRef, + prelude::*, + str::CStrExt, + types::{ + ForeignOwnable, + ScopeGuard, // + }, // +}; + +use core::marker::PhantomData; +use core::ptr; +use macros::vtable; + +/// Trait to represent allocatable buffers for the RTKit core. +/// +/// Users must implement this trait for their own representation of those allocations. +pub trait Buffer { + /// Returns the IOVA (virtual address) of the buffer from RTKit's point of view, or an error if + /// unavailable. + fn iova(&self) -> Result; + + /// Returns a mutable byte slice of the buffer contents, or an + /// error if unavailable. + fn buf(&mut self) -> Result>; +} + +/// Callback operations for an RTKit client. +#[vtable] +pub trait Operations { + /// Arbitrary user context type. + type Data: ForeignOwnable + Send + Sync; + + /// Type representing an allocated buffer for RTKit. + type Buffer: Buffer; + + /// Called when RTKit crashes. + fn crashed(_data: ::Borrowed<'_>, _crashlog: Option<&[u8]>) {} + + /// Called when a message was received on a non-system endpoint. Called in non-IRQ context. + fn recv_message( + _data: ::Borrowed<'_>, + _endpoint: u8, + _message: u64, + ) { + } + + /// Called in IRQ context when a message was received on a non-system endpoint. + /// + /// Must return `true` if the message is handled, or `false` to process it in + /// the handling thread. + fn recv_message_early( + _data: ::Borrowed<'_>, + _endpoint: u8, + _message: u64, + ) -> bool { + false + } + + /// Allocate a buffer for use by RTKit. + fn shmem_alloc( + _data: ::Borrowed<'_>, + _size: usize, + ) -> Result { + Err(EINVAL) + } + + /// Map an existing buffer used by RTKit at a device-specified virtual address. + fn shmem_map( + _data: ::Borrowed<'_>, + _iova: usize, + _size: usize, + ) -> Result { + Err(EINVAL) + } +} + +/// Represents `struct apple_rtkit *`. +/// +/// # Invariants +/// +/// The rtk pointer is valid. +/// The data pointer is a valid pointer from T::Data::into_foreign(). +pub struct RtKit { + rtk: *mut bindings::apple_rtkit, + data: *mut core::ffi::c_void, + _p: PhantomData, +} + +unsafe extern "C" fn crashed_callback( + cookie: *mut core::ffi::c_void, + crashlog: *const core::ffi::c_void, + crashlog_size: usize, +) { + let crashlog = if !crashlog.is_null() && crashlog_size > 0 { + // SAFETY: The crashlog is either missing or a byte buffer of the specified size + Some(unsafe { core::slice::from_raw_parts(crashlog as *const u8, crashlog_size) }) + } else { + None + }; + // SAFETY: cookie is always a T::Data in this API + T::crashed(unsafe { T::Data::borrow(cookie.cast()) }, crashlog); +} + +unsafe extern "C" fn recv_message_callback( + cookie: *mut core::ffi::c_void, + endpoint: u8, + message: u64, +) { + // SAFETY: cookie is always a T::Data in this API + T::recv_message(unsafe { T::Data::borrow(cookie.cast()) }, endpoint, message); +} + +unsafe extern "C" fn recv_message_early_callback( + cookie: *mut core::ffi::c_void, + endpoint: u8, + message: u64, +) -> bool { + // SAFETY: cookie is always a T::Data in this API + T::recv_message_early(unsafe { T::Data::borrow(cookie.cast()) }, endpoint, message) +} + +unsafe extern "C" fn shmem_setup_callback( + cookie: *mut core::ffi::c_void, + bfr: *mut bindings::apple_rtkit_shmem, +) -> core::ffi::c_int { + // SAFETY: `bfr` is a valid buffer + let bfr_mut = unsafe { &mut *bfr }; + + from_result(|| { + let mut buf = if bfr_mut.iova != 0 { + bfr_mut.is_mapped = true; + T::shmem_map( + // SAFETY: `cookie` came from a previous call to `into_foreign`. + unsafe { T::Data::borrow(cookie.cast()) }, + bfr_mut.iova as usize, + bfr_mut.size, + )? + } else { + bfr_mut.is_mapped = false; + // SAFETY: `cookie` came from a previous call to `into_foreign`. + T::shmem_alloc(unsafe { T::Data::borrow(cookie.cast()) }, bfr_mut.size)? + }; + + let iova = buf.iova()?; + let iosys_map = buf.buf()?; + + if iosys_map.size() < bfr_mut.size { + return Err(ENOMEM); + } + + bfr_mut.iova = iova as u64; + bfr_mut.buffer = iosys_map.as_mut_ptr() as *mut _; + + // Now box the returned buffer type and stash it in the private pointer of the + // `apple_rtkit_shmem` struct for safekeeping. + let boxed = KBox::new(buf, GFP_KERNEL)?; + bfr_mut.private = KBox::into_raw(boxed) as *mut _; + Ok(0) + }) +} + +unsafe extern "C" fn shmem_destroy_callback( + _cookie: *mut core::ffi::c_void, + bfr: *mut bindings::apple_rtkit_shmem, +) { + // SAFETY: `bfr` is a valid buffer + let bfr_mut = unsafe { &mut *bfr }; + if !bfr_mut.private.is_null() { + // SAFETY: Per shmem_setup_callback, this has to be a pointer to a Buffer if it is set. + unsafe { + core::mem::drop(KBox::from_raw(bfr_mut.private as *mut T::Buffer)); + } + bfr_mut.private = core::ptr::null_mut(); + } +} + +impl RtKit { + const VTABLE: bindings::apple_rtkit_ops = bindings::apple_rtkit_ops { + crashed: Some(crashed_callback::), + recv_message: Some(recv_message_callback::), + recv_message_early: Some(recv_message_early_callback::), + shmem_setup: if T::HAS_SHMEM_ALLOC || T::HAS_SHMEM_MAP { + Some(shmem_setup_callback::) + } else { + None + }, + shmem_destroy: if T::HAS_SHMEM_ALLOC || T::HAS_SHMEM_MAP { + Some(shmem_destroy_callback::) + } else { + None + }, + }; + + /// Creates a new RTKit client for a given device and optional mailbox name or index. + pub fn new( + dev: &device::Device, + mbox_name: Option<&'static CStr>, + mbox_idx: usize, + data: T::Data, + ) -> Result { + let ptr: *mut crate::ffi::c_void = data.into_foreign().cast(); + let guard = ScopeGuard::new(|| { + // SAFETY: `ptr` came from a previous call to `into_foreign`. + unsafe { T::Data::from_foreign(ptr.cast()) }; + }); + // SAFETY: `dev` is valid by its type invarants and otherwise his just + // calls the C init function. + let rtk = unsafe { + from_err_ptr(bindings::apple_rtkit_init( + dev.as_raw(), + ptr, + match mbox_name { + Some(s) => s.as_char_ptr(), + None => ptr::null(), + }, + mbox_idx.try_into()?, + &Self::VTABLE, + )) + }?; + + guard.dismiss(); + // INVARIANT: `rtk` and `data` are valid here. + Ok(Self { + rtk, + data: ptr, + _p: PhantomData, + }) + } + + /// Boots (wakes up) the RTKit coprocessor. + pub fn wake(self: Pin<&mut Self>) -> Result { + // SAFETY: `rtk` is valid per the type invariant. + to_result(unsafe { bindings::apple_rtkit_wake(self.rtk) }) + } + + /// Waits for the RTKit coprocessor to finish booting. + pub fn boot(self: Pin<&mut Self>) -> Result { + // SAFETY: `rtk` is valid per the type invariant. + to_result(unsafe { bindings::apple_rtkit_boot(self.rtk) }) + } + + /// Starts a non-system endpoint. + pub fn start_endpoint(self: Pin<&mut Self>, endpoint: u8) -> Result { + // SAFETY: `rtk` is valid per the type invariant. + to_result(unsafe { bindings::apple_rtkit_start_ep(self.rtk, endpoint) }) + } + + /// Sends a message to a given endpoint. + pub fn send_message(self: Pin<&mut Self>, endpoint: u8, message: u64) -> Result { + // SAFETY: `rtk` is valid per the type invariant. + to_result(unsafe { + bindings::apple_rtkit_send_message(self.rtk, endpoint, message, ptr::null_mut(), false) + }) + } + + /// Checks if an endpoint is present + pub fn has_endpoint(self: Pin<&mut Self>, endpoint: u8) -> bool { + unsafe { bindings::apple_rtkit_has_endpoint(self.rtk, endpoint) } + } +} + +// SAFETY: `RtKit` operations require a mutable reference +unsafe impl Sync for RtKit {} + +// SAFETY: `RtKit` operations require a mutable reference +unsafe impl Send for RtKit {} + +impl Drop for RtKit { + fn drop(&mut self) { + // SAFETY: The pointer is valid by the type invariant. + unsafe { bindings::apple_rtkit_free(self.rtk) }; + + // Free context data. + // + // SAFETY: This matches the call to `into_foreign` from `new` in the success case. + unsafe { T::Data::from_foreign(self.data.cast()) }; + } +} diff --git a/rust/kernel/soc/mod.rs b/rust/kernel/soc/mod.rs new file mode 100644 index 00000000000000..e3024042e74f0d --- /dev/null +++ b/rust/kernel/soc/mod.rs @@ -0,0 +1,5 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! SoC drivers + +pub mod apple; diff --git a/rust/kernel/str.rs b/rust/kernel/str.rs index fa87779d22539f..3f8918764640c6 100644 --- a/rust/kernel/str.rs +++ b/rust/kernel/str.rs @@ -664,13 +664,13 @@ impl fmt::Write for Formatter<'_> { /// /// * The first byte of `buffer` is always zero. /// * The length of `buffer` is at least 1. -pub(crate) struct NullTerminatedFormatter<'a> { +pub struct NullTerminatedFormatter<'a> { buffer: &'a mut [u8], } impl<'a> NullTerminatedFormatter<'a> { /// Create a new [`Self`] instance. - pub(crate) fn new(buffer: &'a mut [u8]) -> Option> { + pub fn new(buffer: &'a mut [u8]) -> Option> { *(buffer.first_mut()?) = 0; // INVARIANT: diff --git a/rust/kernel/sync/lock.rs b/rust/kernel/sync/lock.rs index 46a57d1fc309d3..961e252f5e137b 100644 --- a/rust/kernel/sync/lock.rs +++ b/rust/kernel/sync/lock.rs @@ -8,6 +8,7 @@ use super::LockClassKey; use crate::{ str::{CStr, CStrExt as _}, + try_pin_init, types::{NotThreadSafe, Opaque, ScopeGuard}, }; use core::{cell::UnsafeCell, marker::PhantomPinned, pin::Pin}; @@ -143,6 +144,31 @@ impl Lock { }), }) } + + /// Constructs a new lock initialiser taking an initialiser. + pub fn pin_init( + t: impl PinInit, + name: &'static CStr, + key: &'static LockClassKey, + ) -> impl PinInit + where + E: core::convert::From, + { + try_pin_init!(Self { + // SAFETY: We are just forwarding the initialization across a + // cast away from UnsafeCell, so the pin_init_from_closure and + // __pinned_init() requirements are in sync. + data <- unsafe { pin_init::pin_init_from_closure(move |slot: *mut UnsafeCell| { + t.__pinned_init(slot as *mut T) + })}, + _pin: PhantomPinned, + // SAFETY: `slot` is valid while the closure is called and both `name` and `key` have + // static lifetimes so they live indefinitely. + state <- Opaque::ffi_init(|slot| unsafe { + B::init(slot, name.as_char_ptr(), key.as_ptr()) + }), + }? E) + } } impl Lock<(), B> { @@ -281,10 +307,7 @@ impl core::ops::Deref for Guard<'_, T, B> { } } -impl core::ops::DerefMut for Guard<'_, T, B> -where - T: Unpin, -{ +impl core::ops::DerefMut for Guard<'_, T, B> { fn deref_mut(&mut self) -> &mut Self::Target { // SAFETY: The caller owns the lock, so it is safe to deref the protected data. unsafe { &mut *self.lock.data.get() } diff --git a/rust/kernel/sync/lock/global.rs b/rust/kernel/sync/lock/global.rs index eab48108a4aebe..79d0ef7fda8676 100644 --- a/rust/kernel/sync/lock/global.rs +++ b/rust/kernel/sync/lock/global.rs @@ -106,10 +106,7 @@ impl core::ops::Deref for GlobalGuard { } } -impl core::ops::DerefMut for GlobalGuard -where - B::Item: Unpin, -{ +impl core::ops::DerefMut for GlobalGuard { fn deref_mut(&mut self) -> &mut Self::Target { &mut self.inner } diff --git a/rust/kernel/sync/lock/mutex.rs b/rust/kernel/sync/lock/mutex.rs index 581cee7ab842ad..45a1c4c6483e90 100644 --- a/rust/kernel/sync/lock/mutex.rs +++ b/rust/kernel/sync/lock/mutex.rs @@ -17,6 +17,19 @@ macro_rules! new_mutex { } pub use new_mutex; +/// Creates a [`Mutex`] initialiser with the given name and a newly-created lock class, +/// given an initialiser for the inner type. +/// +/// It uses the name if one is given, otherwise it generates one based on the file name and line +/// number. +#[macro_export] +macro_rules! new_mutex_pinned { + ($inner:expr $(, $name:literal)? $(,)?) => { + $crate::sync::Mutex::pin_init( + $inner, $crate::optional_name!($($name)?), $crate::static_lock_class!()) + }; +} + /// A mutual exclusion primitive. /// /// Exposes the kernel's [`struct mutex`]. When multiple threads attempt to lock the same mutex, diff --git a/rust/kernel/task.rs b/rust/kernel/task.rs index 49fad6de06740a..cc907fb531bcee 100644 --- a/rust/kernel/task.rs +++ b/rust/kernel/task.rs @@ -204,18 +204,6 @@ impl Task { self.0.get() } - /// Returns the group leader of the given task. - pub fn group_leader(&self) -> &Task { - // SAFETY: The group leader of a task never changes after initialization, so reading this - // field is not a data race. - let ptr = unsafe { *ptr::addr_of!((*self.as_ptr()).group_leader) }; - - // SAFETY: The lifetime of the returned task reference is tied to the lifetime of `self`, - // and given that a task has a reference to its group leader, we know it must be valid for - // the lifetime of the returned task reference. - unsafe { &*ptr.cast() } - } - /// Returns the PID of the given task. pub fn pid(&self) -> Pid { // SAFETY: The pid of a task never changes after initialization, so reading this field is @@ -345,6 +333,18 @@ impl CurrentTask { // `release_task()` call. Some(unsafe { PidNamespace::from_ptr(active_ns) }) } + + /// Returns the group leader of the current task. + pub fn group_leader(&self) -> &Task { + // SAFETY: The group leader of a task never changes while the task is running, and `self` + // is the current task, which is guaranteed running. + let ptr = unsafe { (*self.as_ptr()).group_leader }; + + // SAFETY: `current->group_leader` stays valid for at least the duration in which `current` + // is running, and the signature of this function ensures that the returned `&Task` can + // only be used while `current` is still valid, thus still running. + unsafe { &*ptr.cast() } + } } // SAFETY: The type invariants guarantee that `Task` is always refcounted. diff --git a/rust/kernel/types.rs b/rust/kernel/types.rs index 9c5e7dbf16323f..4081f73f29f85c 100644 --- a/rust/kernel/types.rs +++ b/rust/kernel/types.rs @@ -6,8 +6,9 @@ use crate::ffi::c_void; use core::{ cell::UnsafeCell, marker::{PhantomData, PhantomPinned}, - mem::MaybeUninit, + mem::{ManuallyDrop, MaybeUninit}, ops::{Deref, DerefMut}, + ptr::NonNull, }; use pin_init::{PinInit, Wrapper, Zeroable}; @@ -422,6 +423,114 @@ impl Wrapper for Opaque { } } +/// Types that may be owned by Rust code or borrowed, but have a lifetime managed by C code. +/// +/// It allows such types to define their own custom destructor function to be called when +/// a Rust-owned reference is dropped. +/// +/// This is usually implemented by wrappers to existing structures on the C side of the code. +/// +/// # Safety +/// +/// Implementers must ensure that any objects borrowed directly stay alive for the duration +/// of the borrow lifetime, and that any objects deemed owned by Rust stay alive while +/// that owned reference exists, until the [`Ownable::release()`] function is called. +pub unsafe trait Ownable { + /// Releases the object (frees it or returns it to foreign ownership). + /// + /// # Safety + /// + /// Callers must ensure that the object is no longer referenced after this call. + unsafe fn release(this: NonNull); +} + +/// A subtrait of Ownable that asserts that an Owned Rust reference is not only unique +/// within Rust, but also follows the same rules in kernel C code. That is, the kernel +/// will never mutate the contents of the object while Rust owns it. +/// +/// When this type is implemented for an Ownable type, it allows Owned to be dereferenced +/// into a &mut T. + +/// # Safety +/// +/// Implementers must ensure that the kernel never mutates the underlying type while +/// Rust owns it. +pub unsafe trait OwnableMut: Ownable {} + +/// An owned reference to an ownable kernel object. +/// +/// The object is automatically freed or released when an instance of [`Owned`] is +/// dropped. +/// +/// # Invariants +/// +/// The pointer stored in `ptr` is non-null and valid for the lifetime of the [`Owned`] instance. +pub struct Owned { + ptr: NonNull, + _p: PhantomData, +} + +// SAFETY: It is safe to send `Owned` to another thread when the underlying `T` is `Send` because +// it effectively means sharing `&mut T` (which is safe because `T` is `Send`). +unsafe impl Send for Owned {} + +// SAFETY: It is safe to send `&Owned` to another thread when the underlying `T` is `Sync` +// because it effectively means sharing `&T` (which is safe because `T` is `Sync`). +unsafe impl Sync for Owned {} + +impl Owned { + /// Creates a new instance of [`Owned`]. + /// + /// It takes over ownership of the underlying object. + /// + /// # Safety + /// + /// Callers must ensure that the underlying object is acquired and can be considered owned by + /// Rust. + pub unsafe fn from_raw(ptr: NonNull) -> Self { + // INVARIANT: The safety requirements guarantee that the new instance now owns the + // reference. + Self { + ptr, + _p: PhantomData, + } + } + + /// Consumes the `Owned`, returning a raw pointer. + /// + /// This function does not actually relinquish ownership of the object. + /// After calling this function, the caller is responsible for ownership previously managed + /// by the `Owned`. + pub fn into_raw(me: Self) -> NonNull { + ManuallyDrop::new(me).ptr + } +} + +impl Deref for Owned { + type Target = T; + + fn deref(&self) -> &Self::Target { + // SAFETY: The type invariants guarantee that the object is valid. + unsafe { self.ptr.as_ref() } + } +} + +impl DerefMut for Owned { + fn deref_mut(&mut self) -> &mut Self::Target { + // SAFETY: The type invariants guarantee that the object is valid, + // and that we can safely return a mutable reference to it. + unsafe { self.ptr.as_mut() } + } +} + +impl Drop for Owned { + fn drop(&mut self) { + // SAFETY: The type invariants guarantee that the `Owned` owns the object we're about to + // release. + unsafe { T::release(self.ptr) }; + } +} + /// Zero-sized type to mark types not [`Send`]. /// /// Add this type as a field to your struct if your type should not be sent to a different task. @@ -442,3 +551,86 @@ pub type NotThreadSafe = PhantomData<*mut ()>; /// [`NotThreadSafe`]: type@NotThreadSafe #[allow(non_upper_case_globals)] pub const NotThreadSafe: NotThreadSafe = PhantomData; + +/// Helper macro to declare a bitfield style type. The type will automatically +/// gain boolean operator implementations, as well as the `as_raw()` and `contains()` +/// methods, Debug, Copy, Clone, and PartialEq implementations. +/// +/// Optionally, a default value can be specified with `= value` syntax, which +/// will add a Default trait implementation. +/// +/// # Examples +/// +/// ``` +/// declare_flags_type! { +/// /// Flags to be used for foo. +/// pub struct FooFlags(u32); +/// } +/// +/// declare_flags_type! { +/// /// Flags to be used for bar. +/// pub struct BarFlags(u32) = 0; +/// } +/// ``` +macro_rules! declare_flags_type ( + ( + $(#[$outer:meta])* + $v:vis struct $t:ident ( $base:ty ); + $($rest:tt)* + ) => { + $(#[$outer])* + #[derive(Debug, Clone, Copy, PartialEq)] + $v struct $t($base); + + impl $t { + /// Get the raw representation of this flag. + pub(crate) fn as_raw(self) -> $base { + self.0 + } + + /// Check whether `flags` is contained in `self`. + pub fn contains(self, flags: Self) -> bool { + (self & flags) == flags + } + } + + impl core::ops::BitOr for $t { + type Output = Self; + fn bitor(self, rhs: Self) -> Self::Output { + Self(self.0 | rhs.0) + } + } + + impl core::ops::BitAnd for $t { + type Output = Self; + fn bitand(self, rhs: Self) -> Self::Output { + Self(self.0 & rhs.0) + } + } + + impl core::ops::Not for $t { + type Output = Self; + fn not(self) -> Self::Output { + Self(!self.0) + } + } + }; + ( + $(#[$outer:meta])* + $v:vis struct $t:ident ( $base:ty ) = $default:expr; + $($rest:tt)* + ) => { + declare_flags_type! { + $(#[$outer])* + $v struct $t ($base); + $($rest)* + } + impl Default for $t { + fn default() -> Self { + Self($default) + } + } + }; +); + +pub(crate) use declare_flags_type; diff --git a/rust/kernel/xarray.rs b/rust/kernel/xarray.rs index a49d6db2884588..e1f7e2d9b629f0 100644 --- a/rust/kernel/xarray.rs +++ b/rust/kernel/xarray.rs @@ -5,17 +5,20 @@ //! C header: [`include/linux/xarray.h`](srctree/include/linux/xarray.h) use crate::{ - alloc, bindings, build_assert, - error::{Error, Result}, - ffi::c_void, - types::{ForeignOwnable, NotThreadSafe, Opaque}, + alloc, + prelude::*, + types::{ForeignOwnable, NotThreadSafe, Opaque, ScopeGuard}, +}; +use core::{ + fmt, iter, + marker::PhantomData, + mem, ops, + ptr::{null_mut, NonNull}, }; -use core::{iter, marker::PhantomData, pin::Pin, ptr::NonNull}; -use pin_init::{pin_data, pin_init, pinned_drop, PinInit}; /// An array which efficiently maps sparse integer indices to owned objects. /// -/// This is similar to a [`crate::alloc::kvec::Vec>`], but more efficient when there are +/// This is similar to a [`Vec>`], but more efficient when there are /// holes in the index space, and can be efficiently grown. /// /// # Invariants @@ -26,10 +29,11 @@ use pin_init::{pin_data, pin_init, pinned_drop, PinInit}; /// # Examples /// /// ```rust -/// use kernel::alloc::KBox; -/// use kernel::xarray::{AllocKind, XArray}; +/// # use kernel::alloc::KBox; +/// # use kernel::xarray::XArray; +/// # use pin_init::stack_pin_init; /// -/// let xa = KBox::pin_init(XArray::new(AllocKind::Alloc1), GFP_KERNEL)?; +/// stack_pin_init!(let xa = XArray::new(Default::default())); /// /// let dead = KBox::new(0xdead, GFP_KERNEL)?; /// let beef = KBox::new(0xbeef, GFP_KERNEL)?; @@ -77,8 +81,10 @@ impl PinnedDrop for XArray { } /// Flags passed to [`XArray::new`] to configure the array's allocation tracking behavior. +#[derive(Default)] pub enum AllocKind { /// Consider the first element to be at index 0. + #[default] Alloc, /// Consider the first element to be at index 1. Alloc1, @@ -105,19 +111,69 @@ impl XArray { fn iter(&self) -> impl Iterator> + '_ { let mut index = 0; - // SAFETY: `self.xa` is always valid by the type invariant. - iter::once(unsafe { - bindings::xa_find(self.xa.get(), &mut index, usize::MAX, bindings::XA_PRESENT) - }) - .chain(iter::from_fn(move || { + core::iter::Iterator::chain( // SAFETY: `self.xa` is always valid by the type invariant. - Some(unsafe { - bindings::xa_find_after(self.xa.get(), &mut index, usize::MAX, bindings::XA_PRESENT) - }) - })) + iter::once(unsafe { + bindings::xa_find(self.xa.get(), &mut index, usize::MAX, bindings::XA_PRESENT) + }), + iter::from_fn(move || { + // SAFETY: `self.xa` is always valid by the type invariant. + Some(unsafe { + bindings::xa_find_after( + self.xa.get(), + &mut index, + usize::MAX, + bindings::XA_PRESENT, + ) + }) + }), + ) .map_while(|ptr| NonNull::new(ptr.cast())) } + /// Looks up and returns a reference to the lowest entry in the array between index and max, + /// returning a tuple of its index and a `Guard` if one exists. + /// + /// This guard blocks all other actions on the `XArray`. Callers are expected to drop the + /// `Guard` eagerly to avoid blocking other users, such as by taking a clone of the value. + pub fn find(&self, index: usize, max: usize) -> Option<(usize, ValueGuard<'_, T>)> { + let mut index: usize = index; + + // SAFETY: `self.xa` is always valid by the type invariant. + unsafe { bindings::xa_lock(self.xa.get()) }; + + // SAFETY: `self.xa` is always valid by the type invariant. + let guard = ScopeGuard::new(|| unsafe { bindings::xa_unlock(self.xa.get()) }); + + // SAFETY: `self.xa` is always valid by the type invariant. + let p = unsafe { bindings::xa_find(self.xa.get(), &mut index, max, bindings::XA_PRESENT) }; + + NonNull::new(p as *mut T).map(|ptr| { + guard.dismiss(); + ( + index, + ValueGuard { + xa: self, + ptr, + _not_send: NotThreadSafe, + }, + ) + }) + } + + fn with_guard(&self, guard: Option<&mut Guard<'_, T>>, f: F) -> U + where + F: FnOnce(&mut Guard<'_, T>) -> U, + { + match guard { + None => f(&mut self.lock()), + Some(guard) => { + assert_eq!(guard.xa.xa.get(), self.xa.get()); + f(guard) + } + } + } + /// Attempts to lock the [`XArray`] for exclusive access. pub fn try_lock(&self) -> Option> { // SAFETY: `self.xa` is always valid by the type invariant. @@ -141,6 +197,12 @@ impl XArray { _not_send: NotThreadSafe, } } + + /// Removes and returns the element at the given index. + pub fn remove(&self, index: usize) -> Option { + let mut guard = self.lock(); + guard.remove(index) + } } /// A lock guard. @@ -161,9 +223,41 @@ impl Drop for Guard<'_, T> { } } +/// A lock guard. +/// +/// The lock is unlocked when the guard goes out of scope. +#[must_use = "the lock unlocks immediately when the guard is unused"] +pub struct ValueGuard<'a, T: ForeignOwnable> { + xa: &'a XArray, + ptr: NonNull, + _not_send: NotThreadSafe, +} + +impl<'a, T: ForeignOwnable> ValueGuard<'a, T> { + /// Borrow the underlying value wrapped by the `Guard`. + /// + /// Returns a `T::Borrowed` type for the owned `ForeignOwnable` type. + pub fn borrow(&self) -> T::Borrowed<'_> { + // SAFETY: The value is owned by the `XArray`, the lifetime it is borrowed for must not + // outlive the `XArray` itself, nor the Guard that holds the lock ensuring the value + // remains in the `XArray`. + unsafe { T::borrow(self.ptr.as_ptr() as _) } + } +} + +impl Drop for ValueGuard<'_, T> { + fn drop(&mut self) { + // SAFETY: + // - `self.xa.xa` is always valid by the type invariant. + // - The caller holds the lock, so it is safe to unlock it. + unsafe { bindings::xa_unlock(self.xa.xa.get()) }; + } +} + /// The error returned by [`store`](Guard::store). /// /// Contains the underlying error and the value that was not stored. +#[derive(Debug)] pub struct StoreError { /// The error that occurred. pub error: Error, @@ -177,6 +271,11 @@ impl From> for Error { } } +fn to_usize(i: u32) -> usize { + i.try_into() + .unwrap_or_else(|_| build_error!("cannot convert u32 to usize")) +} + impl<'a, T: ForeignOwnable> Guard<'a, T> { fn load(&self, index: usize, f: F) -> Option where @@ -211,7 +310,7 @@ impl<'a, T: ForeignOwnable> Guard<'a, T> { // - The caller holds the lock. let ptr = unsafe { bindings::__xa_erase(self.xa.xa.get(), index) }.cast(); // SAFETY: - // - `ptr` is either NULL or came from `T::into_foreign`. + // - `ptr` is either `NULL` or came from `T::into_foreign`. // - `&mut self` guarantees that the lifetimes of [`T::Borrowed`] and [`T::BorrowedMut`] // borrowed from `self` have ended. unsafe { T::try_from_foreign(ptr) } @@ -259,13 +358,272 @@ impl<'a, T: ForeignOwnable> Guard<'a, T> { }) } else { let old = old.cast(); - // SAFETY: `ptr` is either NULL or came from `T::into_foreign`. + // SAFETY: `ptr` is either `NULL` or came from `T::into_foreign`. // // NB: `XA_ZERO_ENTRY` is never returned by functions belonging to the Normal XArray // API; such entries present as `NULL`. Ok(unsafe { T::try_from_foreign(old) }) } } + + /// Stores an element at the given index if no entry is present. + /// + /// May drop the lock if needed to allocate memory, and then reacquire it afterwards. + /// + /// On failure, returns the element which was attempted to be stored. + pub fn insert( + &mut self, + index: usize, + value: T, + gfp: alloc::Flags, + ) -> Result<(), StoreError> { + build_assert!( + T::FOREIGN_ALIGN >= 4, + "pointers stored in XArray must be 4-byte aligned" + ); + let ptr = value.into_foreign(); + // SAFETY: `self.xa` is always valid by the type invariant. + // + // INVARIANT: `ptr` came from `T::into_foreign`. + match unsafe { bindings::__xa_insert(self.xa.xa.get(), index, ptr.cast(), gfp.as_raw()) } { + 0 => Ok(()), + errno => { + // SAFETY: `ptr` came from `T::into_foreign` and `__xa_insert` does not take + // ownership of the value on error. + let value = unsafe { T::from_foreign(ptr) }; + Err(StoreError { + value, + error: Error::from_errno(errno), + }) + } + } + } + + /// Wrapper around `__xa_alloc`. + /// + /// On success, takes ownership of pointers passed in `op`. + /// + /// On failure, ownership returns to the caller. + /// + /// # Safety + /// + /// `ptr` must be `NULL` or have come from a previous call to `T::into_foreign`. + unsafe fn alloc( + &mut self, + limit: impl ops::RangeBounds, + ptr: *mut c_void, + gfp: alloc::Flags, + ) -> Result { + // NB: `xa_limit::{max,min}` are inclusive. + let limit = bindings::xa_limit { + max: match limit.end_bound() { + ops::Bound::Included(&end) => end, + ops::Bound::Excluded(&end) => end - 1, + ops::Bound::Unbounded => u32::MAX, + }, + min: match limit.start_bound() { + ops::Bound::Included(&start) => start, + ops::Bound::Excluded(&start) => start + 1, + ops::Bound::Unbounded => 0, + }, + }; + + let mut index = u32::MAX; + + // SAFETY: + // - `self.xa` is always valid by the type invariant. + // - `self.xa` was initialized with `XA_FLAGS_ALLOC` or `XA_FLAGS_ALLOC1`. + // + // INVARIANT: `ptr` is either `NULL` or came from `T::into_foreign`. + match unsafe { + bindings::__xa_alloc( + self.xa.xa.get(), + &mut index, + ptr.cast(), + limit, + gfp.as_raw(), + ) + } { + 0 => Ok(to_usize(index)), + errno => Err(Error::from_errno(errno)), + } + } + + /// Allocates an entry somewhere in the array. + /// + /// On success, returns the index at which the entry was stored. + /// + /// On failure, returns the entry which was attempted to be stored. + pub fn insert_limit( + &mut self, + limit: impl ops::RangeBounds, + value: T, + gfp: alloc::Flags, + ) -> Result> { + build_assert!( + T::FOREIGN_ALIGN >= 4, + "pointers stored in XArray must be 4-byte aligned" + ); + let ptr = value.into_foreign(); + // SAFETY: `ptr` came from `T::into_foreign`. + unsafe { self.alloc(limit, ptr, gfp) }.map_err(|error| { + // SAFETY: `ptr` came from `T::into_foreign` and `self.alloc` does not take ownership of + // the value on error. + let value = unsafe { T::from_foreign(ptr) }; + StoreError { value, error } + }) + } + + /// Reserves an entry in the array. + pub fn reserve(&mut self, index: usize, gfp: alloc::Flags) -> Result> { + // NB: `__xa_insert` internally coerces `NULL` to `XA_ZERO_ENTRY` on ingress. + let ptr = null_mut(); + // SAFETY: `self.xa` is always valid by the type invariant. + // + // INVARIANT: `ptr` is `NULL`. + match unsafe { bindings::__xa_insert(self.xa.xa.get(), index, ptr, gfp.as_raw()) } { + 0 => Ok(Reservation { xa: self.xa, index }), + errno => Err(Error::from_errno(errno)), + } + } + + /// Reserves an entry somewhere in the array. + pub fn reserve_limit( + &mut self, + limit: impl ops::RangeBounds, + gfp: alloc::Flags, + ) -> Result> { + // NB: `__xa_alloc` internally coerces `NULL` to `XA_ZERO_ENTRY` on ingress. + let ptr = null_mut(); + // SAFETY: `ptr` is `NULL`. + unsafe { self.alloc(limit, ptr, gfp) }.map(|index| Reservation { xa: self.xa, index }) + } +} + +/// A reserved slot in an array. +/// +/// The slot is released when the reservation goes out of scope. +/// +/// Note that the array lock *must not* be held when the reservation is filled or dropped as this +/// will lead to deadlock. [`Reservation::fill_locked`] and [`Reservation::release_locked`] can be +/// used in context where the array lock is held. +#[must_use = "the reservation is released immediately when the reservation is unused"] +pub struct Reservation<'a, T: ForeignOwnable> { + xa: &'a XArray, + index: usize, +} + +impl fmt::Debug for Reservation<'_, T> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Reservation") + .field("index", &self.index()) + .finish() + } +} + +impl Reservation<'_, T> { + /// Returns the index of the reservation. + pub fn index(&self) -> usize { + self.index + } + + /// Replaces the reserved entry with the given entry. + /// + /// # Safety + /// + /// `ptr` must be `NULL` or have come from a previous call to `T::into_foreign`. + unsafe fn replace(guard: &mut Guard<'_, T>, index: usize, ptr: *mut c_void) -> Result { + // SAFETY: `xa_zero_entry` wraps `XA_ZERO_ENTRY` which is always safe to use. + let old = unsafe { bindings::xa_zero_entry() }; + + // NB: `__xa_cmpxchg_raw` is used over `__xa_cmpxchg` because the latter coerces + // `XA_ZERO_ENTRY` to `NULL` on egress, which would prevent us from determining whether a + // replacement was made. + // + // SAFETY: `self.xa` is always valid by the type invariant. + // + // INVARIANT: `ptr` is either `NULL` or came from `T::into_foreign` and `old` is + // `XA_ZERO_ENTRY`. + let ret = + unsafe { bindings::__xa_cmpxchg_raw(guard.xa.xa.get(), index, old, ptr.cast(), 0) }; + + // SAFETY: `__xa_cmpxchg_raw` returns the old entry at this index on success or `xa_err` if + // an error happened. + match unsafe { bindings::xa_err(ret) } { + 0 => { + if ret == old { + Ok(()) + } else { + Err(EBUSY) + } + } + errno => Err(Error::from_errno(errno)), + } + } + + fn fill_inner(&self, guard: Option<&mut Guard<'_, T>>, value: T) -> Result<(), StoreError> { + let Self { xa, index } = self; + let index = *index; + + let ptr = value.into_foreign(); + xa.with_guard(guard, |guard| { + // SAFETY: `ptr` came from `T::into_foreign`. + unsafe { Self::replace(guard, index, ptr) } + }) + .map_err(|error| { + // SAFETY: `ptr` came from `T::into_foreign` and `Self::replace` does not take ownership + // of the value on error. + let value = unsafe { T::from_foreign(ptr) }; + StoreError { value, error } + }) + } + + /// Fills the reservation. + pub fn fill(self, value: T) -> Result<(), StoreError> { + let result = self.fill_inner(None, value); + mem::forget(self); + result + } + + /// Fills the reservation without acquiring the array lock. + /// + /// # Panics + /// + /// Panics if the passed guard locks a different array. + pub fn fill_locked(self, guard: &mut Guard<'_, T>, value: T) -> Result<(), StoreError> { + let result = self.fill_inner(Some(guard), value); + mem::forget(self); + result + } + + fn release_inner(&self, guard: Option<&mut Guard<'_, T>>) -> Result { + let Self { xa, index } = self; + let index = *index; + + xa.with_guard(guard, |guard| { + let ptr = null_mut(); + // SAFETY: `ptr` is `NULL`. + unsafe { Self::replace(guard, index, ptr) } + }) + } + + /// Releases the reservation without acquiring the array lock. + /// + /// # Panics + /// + /// Panics if the passed guard locks a different array. + pub fn release_locked(self, guard: &mut Guard<'_, T>) -> Result { + let result = self.release_inner(Some(guard)); + mem::forget(self); + result + } +} + +impl Drop for Reservation<'_, T> { + fn drop(&mut self) { + // NB: Errors here are possible since `Guard::store` does not honor reservations. + let _: Result = self.release_inner(None); + } } // SAFETY: `XArray` has no shared mutable state so it is `Send` iff `T` is `Send`. @@ -274,3 +632,133 @@ unsafe impl Send for XArray {} // SAFETY: `XArray` serialises the interior mutability it provides so it is `Sync` iff `T` is // `Send`. unsafe impl Sync for XArray {} + +#[macros::kunit_tests(rust_xarray_kunit)] +mod tests { + use super::*; + use pin_init::stack_pin_init; + + fn new_kbox(value: T) -> Result> { + KBox::new(value, GFP_KERNEL).map_err(Into::into) + } + + #[test] + fn test_alloc_kind_alloc() -> Result { + test_alloc_kind(AllocKind::Alloc, 0) + } + + #[test] + fn test_alloc_kind_alloc1() -> Result { + test_alloc_kind(AllocKind::Alloc1, 1) + } + + fn test_alloc_kind(kind: AllocKind, expected_index: usize) -> Result { + stack_pin_init!(let xa = XArray::new(kind)); + let mut guard = xa.lock(); + + let reservation = guard.reserve_limit(.., GFP_KERNEL)?; + assert_eq!(reservation.index(), expected_index); + reservation.release_locked(&mut guard)?; + + let insertion = guard.insert_limit(.., new_kbox(0x1337)?, GFP_KERNEL); + assert!(insertion.is_ok()); + let insertion_index = insertion.unwrap(); + assert_eq!(insertion_index, expected_index); + + Ok(()) + } + + const IDX: usize = 0x1337; + + fn insert(guard: &mut Guard<'_, T>, value: T) -> Result<(), StoreError> { + guard.insert(IDX, value, GFP_KERNEL) + } + + fn reserve<'a, T: ForeignOwnable>(guard: &mut Guard<'a, T>) -> Result> { + guard.reserve(IDX, GFP_KERNEL) + } + + #[track_caller] + fn check_not_vacant<'a>(guard: &mut Guard<'a, KBox>) -> Result { + // Insertion fails. + { + let beef = new_kbox(0xbeef)?; + let ret = insert(guard, beef); + assert!(ret.is_err()); + let StoreError { error, value } = ret.unwrap_err(); + assert_eq!(error, EBUSY); + assert_eq!(*value, 0xbeef); + } + + // Reservation fails. + { + let ret = reserve(guard); + assert!(ret.is_err()); + assert_eq!(ret.unwrap_err(), EBUSY); + } + + Ok(()) + } + + #[test] + fn test_insert_and_reserve_interaction() -> Result { + stack_pin_init!(let xa = XArray::new(Default::default())); + let mut guard = xa.lock(); + + // Vacant. + assert_eq!(guard.get(IDX), None); + + // Reservation succeeds. + let reservation = { + let ret = reserve(&mut guard); + assert!(ret.is_ok()); + ret.unwrap() + }; + + // Reserved presents as vacant. + assert_eq!(guard.get(IDX), None); + + check_not_vacant(&mut guard)?; + + // Release reservation. + { + let ret = reservation.release_locked(&mut guard); + assert!(ret.is_ok()); + let () = ret.unwrap(); + } + + // Vacant again. + assert_eq!(guard.get(IDX), None); + + // Insert succeeds. + { + let dead = new_kbox(0xdead)?; + let ret = insert(&mut guard, dead); + assert!(ret.is_ok()); + let () = ret.unwrap(); + } + + check_not_vacant(&mut guard)?; + + // Remove. + assert_eq!(guard.remove(IDX).as_deref(), Some(&0xdead)); + + // Reserve and fill. + { + let beef = new_kbox(0xbeef)?; + let ret = reserve(&mut guard); + assert!(ret.is_ok()); + let reservation = ret.unwrap(); + let ret = reservation.fill_locked(&mut guard, beef); + assert!(ret.is_ok()); + let () = ret.unwrap(); + }; + + check_not_vacant(&mut guard)?; + + // Remove. + assert_eq!(guard.remove(IDX).as_deref(), Some(&0xbeef)); + + Ok(()) + } +} diff --git a/rust/macros/lib.rs b/rust/macros/lib.rs index 33f66e86418ab8..a6fc27ce5b2ee4 100644 --- a/rust/macros/lib.rs +++ b/rust/macros/lib.rs @@ -20,6 +20,7 @@ mod helpers; mod kunit; mod module; mod paste; +mod versions; mod vtable; use proc_macro::TokenStream; @@ -135,6 +136,12 @@ pub fn module(ts: TokenStream) -> TokenStream { module::module(ts) } +/// Declares multiple variants of a structure or impl code +#[proc_macro_attribute] +pub fn versions(attr: TokenStream, item: TokenStream) -> TokenStream { + versions::versions(attr, item) +} + /// Declares or implements a vtable trait. /// /// Linux's use of pure vtables is very close to Rust traits, but they differ diff --git a/rust/macros/versions.rs b/rust/macros/versions.rs new file mode 100644 index 00000000000000..b13a5d55c0e17b --- /dev/null +++ b/rust/macros/versions.rs @@ -0,0 +1,341 @@ +use proc_macro::{Delimiter, Group, Ident, Punct, Spacing, Span, TokenStream, TokenTree}; + +//use crate::helpers::expect_punct; + +fn expect_group(it: &mut impl Iterator) -> Group { + if let Some(TokenTree::Group(group)) = it.next() { + group + } else { + panic!("Expected Group") + } +} + +fn expect_punct(it: &mut impl Iterator) -> String { + if let Some(TokenTree::Punct(punct)) = it.next() { + punct.to_string() + } else { + panic!("Expected Group") + } +} + +fn drop_until_punct(it: &mut impl Iterator, delimiter: &str, is_struct: bool) { + let mut depth: isize = 0; + let mut colons: isize = 0; + for token in it.by_ref() { + if let TokenTree::Punct(punct) = token { + match punct.as_char() { + ':' => { + colons += 1; + } + '<' => { + if depth > 0 || colons == 2 || is_struct { + depth += 1; + } + colons = 0; + } + '>' => { + if depth > 0 { + depth -= 1; + } + colons = 0; + } + _ => { + colons = 0; + if depth == 0 && delimiter.contains(&punct.to_string()) { + break; + } + } + } + } + } +} + +fn drop_until_braces(it: &mut impl Iterator) { + let mut depth: isize = 0; + let mut colons: isize = 0; + for token in it.by_ref() { + match token { + TokenTree::Punct(punct) => match punct.as_char() { + ':' => { + colons += 1; + } + '<' => { + if depth > 0 || colons == 2 { + depth += 1; + } + colons = 0; + } + '>' => { + if depth > 0 { + depth -= 1; + } + colons = 0; + } + _ => colons = 0, + }, + TokenTree::Group(group) if group.delimiter() == Delimiter::Brace => { + if depth == 0 { + break; + } + } + _ => (), + } + } +} + +struct VersionConfig { + fields: &'static [&'static str], + enums: &'static [&'static [&'static str]], + versions: &'static [&'static [&'static str]], +} + +static AGX_VERSIONS: VersionConfig = VersionConfig { + fields: &["G", "V"], + enums: &[ + &["G13", "G14", "G14X"], + &["V12_3", "V12_4", "V13_0B4", "V13_2", "V13_3", "V13_5"], + ], + versions: &[ + &["G13", "V12_3"], + &["G14", "V12_4"], + &["G13", "V13_5"], + &["G14", "V13_5"], + &["G14X", "V13_5"], + ], +}; + +fn check_version( + config: &VersionConfig, + ver: &[usize], + it: &mut impl Iterator, +) -> bool { + let first = it.next().unwrap(); + let val: bool = match &first { + TokenTree::Group(group) => check_version(config, ver, &mut group.stream().into_iter()), + TokenTree::Ident(ident) => { + let key = config + .fields + .iter() + .position(|&r| r == ident.to_string()) + .unwrap_or_else(|| panic!("Unknown field {}", ident)); + let mut operator = expect_punct(it); + let mut rhs_token = it.next().unwrap(); + if let TokenTree::Punct(punct) = &rhs_token { + operator.extend(std::iter::once(punct.as_char())); + rhs_token = it.next().unwrap(); + } + let rhs_name = if let TokenTree::Ident(ident) = &rhs_token { + ident.to_string() + } else { + panic!("Unexpected token {}", ident) + }; + + let rhs = config.enums[key] + .iter() + .position(|&r| r == rhs_name) + .unwrap_or_else(|| panic!("Unknown value for {}:{}", ident, rhs_name)); + let lhs = ver[key]; + + match operator.as_str() { + "==" => lhs == rhs, + "!=" => lhs != rhs, + ">" => lhs > rhs, + ">=" => lhs >= rhs, + "<" => lhs < rhs, + "<=" => lhs <= rhs, + _ => panic!("Unknown operator {}", operator), + } + } + _ => { + panic!("Unknown token {}", first) + } + }; + + let boolop = it.next(); + match boolop { + Some(TokenTree::Punct(punct)) => { + let right = expect_punct(it); + if right != punct.to_string() { + panic!("Unexpected op {}{}", punct, right); + } + match punct.as_char() { + '&' => val && check_version(config, ver, it), + '|' => val || check_version(config, ver, it), + _ => panic!("Unexpected op {}{}", right, right), + } + } + Some(a) => panic!("Unexpected op {}", a), + None => val, + } +} + +fn filter_versions( + config: &VersionConfig, + tag: &str, + ver: &[usize], + tree: impl IntoIterator, + is_struct: bool, +) -> Vec { + let mut out = Vec::::new(); + let mut it = tree.into_iter(); + + while let Some(token) = it.next() { + let mut tail: Option = None; + match &token { + TokenTree::Punct(punct) if punct.to_string() == "#" => { + let group = expect_group(&mut it); + let mut grp_it = group.stream().into_iter(); + let attr = grp_it.next().unwrap(); + match attr { + TokenTree::Ident(ident) if ident.to_string() == "ver" => { + if check_version(config, ver, &mut grp_it) { + } else if is_struct { + drop_until_punct(&mut it, ",", true); + } else { + let first = it.next().unwrap(); + match &first { + TokenTree::Ident(ident) + if ["while", "for", "loop", "if", "match", "unsafe", "fn"] + .contains(&ident.to_string().as_str()) => + { + drop_until_braces(&mut it); + } + TokenTree::Group(_) => (), + _ => { + drop_until_punct(&mut it, ",;", false); + } + } + } + } + _ => { + out.push(token.clone()); + out.push(TokenTree::Group(group.clone())); + } + } + continue; + } + TokenTree::Punct(punct) if punct.to_string() == ":" => { + let next = it.next(); + match next { + Some(TokenTree::Punct(punct)) if punct.to_string() == ":" => { + let next = it.next(); + match next { + Some(TokenTree::Ident(idtag)) if idtag.to_string() == "ver" => { + let ident = match out.pop() { + Some(TokenTree::Ident(ident)) => ident, + a => panic!("$ver not following ident: {:?}", a), + }; + let name = ident.to_string() + tag; + let new_ident = Ident::new(name.as_str(), ident.span()); + out.push(TokenTree::Ident(new_ident)); + continue; + } + Some(a) => { + out.push(token.clone()); + out.push(token.clone()); + tail = Some(a); + } + None => { + out.push(token.clone()); + out.push(token.clone()); + } + } + } + Some(a) => { + out.push(token.clone()); + tail = Some(a); + } + None => { + out.push(token.clone()); + continue; + } + } + } + _ => { + tail = Some(token); + } + } + match &tail { + Some(TokenTree::Group(group)) => { + let new_body = + filter_versions(config, tag, ver, group.stream().into_iter(), is_struct); + let mut stream = TokenStream::new(); + stream.extend(new_body); + let mut filtered_group = Group::new(group.delimiter(), stream); + filtered_group.set_span(group.span()); + out.push(TokenTree::Group(filtered_group)); + } + Some(token) => { + out.push(token.clone()); + } + None => {} + } + } + + out +} + +pub(crate) fn versions(attr: TokenStream, item: TokenStream) -> TokenStream { + let config = match attr.to_string().as_str() { + "AGX" => &AGX_VERSIONS, + _ => panic!("Unknown version group {}", attr), + }; + + let mut it = item.into_iter(); + let mut out = TokenStream::new(); + let mut body: Vec = Vec::new(); + let mut is_struct = false; + + while let Some(token) = it.next() { + match token { + TokenTree::Punct(punct) if punct.to_string() == "#" => { + body.push(TokenTree::Punct(punct)); + body.push(it.next().unwrap()); + } + TokenTree::Ident(ident) + if ["struct", "enum", "union", "const", "type"] + .contains(&ident.to_string().as_str()) => + { + is_struct = ident.to_string() != "const"; + body.push(TokenTree::Ident(ident)); + body.push(it.next().unwrap()); + // This isn't valid syntax in a struct definition, so add it for the user + body.push(TokenTree::Punct(Punct::new(':', Spacing::Joint))); + body.push(TokenTree::Punct(Punct::new(':', Spacing::Alone))); + body.push(TokenTree::Ident(Ident::new("ver", Span::call_site()))); + break; + } + TokenTree::Ident(ident) if ident.to_string() == "impl" => { + body.push(TokenTree::Ident(ident)); + break; + } + TokenTree::Ident(ident) if ident.to_string() == "fn" => { + body.push(TokenTree::Ident(ident)); + break; + } + _ => { + body.push(token); + } + } + } + + body.extend(it); + + for ver in config.versions { + let tag = ver.join(""); + let mut ver_num = Vec::::new(); + for (i, comp) in ver.iter().enumerate() { + let idx = config.enums[i].iter().position(|&r| r == *comp).unwrap(); + ver_num.push(idx); + } + out.extend(filter_versions( + config, + &tag, + &ver_num, + body.clone(), + is_struct, + )); + } + + out +} diff --git a/rust/pin-init/src/lib.rs b/rust/pin-init/src/lib.rs index 8dc9dd5ac6fd32..e3c6ee3e8d57d9 100644 --- a/rust/pin-init/src/lib.rs +++ b/rust/pin-init/src/lib.rs @@ -780,10 +780,10 @@ macro_rules! stack_try_pin_init { // module `macros` inside of `macros.rs`. #[macro_export] macro_rules! pin_init { - ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? { + ($(&$this:ident in)? $t:ident $(::$p:ident)* $(::<$($generics:ty),* $(,)?>)? { $($fields:tt)* }) => { - $crate::try_pin_init!($(&$this in)? $t $(::<$($generics),*>)? { + $crate::try_pin_init!($(&$this in)? $t $(::$p)* $(::<$($generics),*>)? { $($fields)* }? ::core::convert::Infallible) }; @@ -831,12 +831,12 @@ macro_rules! pin_init { // module `macros` inside of `macros.rs`. #[macro_export] macro_rules! try_pin_init { - ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? { + ($(&$this:ident in)? $t:ident $(::$p:ident)* $(::<$($generics:ty),* $(,)?>)? { $($fields:tt)* }? $err:ty) => { $crate::__init_internal!( @this($($this)?), - @typ($t $(::<$($generics),*>)? ), + @typ($t $(::$p)* $(::<$($generics),*>)? ), @fields($($fields)*), @error($err), @data(PinData, use_data), @@ -887,10 +887,10 @@ macro_rules! try_pin_init { // module `macros` inside of `macros.rs`. #[macro_export] macro_rules! init { - ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? { + ($(&$this:ident in)? $t:ident $(::$p:ident)* $(::<$($generics:ty),* $(,)?>)? { $($fields:tt)* }) => { - $crate::try_init!($(&$this in)? $t $(::<$($generics),*>)? { + $crate::try_init!($(&$this in)? $t $(::$p)* $(::<$($generics),*>)? { $($fields)* }? ::core::convert::Infallible) } @@ -936,12 +936,12 @@ macro_rules! init { // module `macros` inside of `macros.rs`. #[macro_export] macro_rules! try_init { - ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? { + ($(&$this:ident in)? $t:ident $(::$p:ident)* $(::<$($generics:ty),* $(,)?>)? { $($fields:tt)* }? $err:ty) => { $crate::__init_internal!( @this($($this)?), - @typ($t $(::<$($generics),*>)?), + @typ($t $(::$p)* $(::<$($generics),*>)?), @fields($($fields)*), @error($err), @data(InitData, /*no use_data*/), @@ -1276,13 +1276,13 @@ pub const unsafe fn init_from_closure( /// /// - `*mut U` must be castable to `*mut T` and any value of type `T` written through such a /// pointer must result in a valid `U`. -#[expect(clippy::let_and_return)] pub const unsafe fn cast_pin_init(init: impl PinInit) -> impl PinInit { // SAFETY: initialization delegated to a valid initializer. Cast is valid by function safety // requirements. let res = unsafe { pin_init_from_closure(|ptr: *mut U| init.__pinned_init(ptr.cast::())) }; // FIXME: remove the let statement once the nightly-MSRV allows it (1.78 otherwise encounters a // cycle when computing the type returned by this function) + #[allow(clippy::let_and_return)] res } @@ -1292,13 +1292,13 @@ pub const unsafe fn cast_pin_init(init: impl PinInit) -> impl Pin /// /// - `*mut U` must be castable to `*mut T` and any value of type `T` written through such a /// pointer must result in a valid `U`. -#[expect(clippy::let_and_return)] pub const unsafe fn cast_init(init: impl Init) -> impl Init { // SAFETY: initialization delegated to a valid initializer. Cast is valid by function safety // requirements. let res = unsafe { init_from_closure(|ptr: *mut U| init.__init(ptr.cast::())) }; // FIXME: remove the let statement once the nightly-MSRV allows it (1.78 otherwise encounters a // cycle when computing the type returned by this function) + #[allow(clippy::let_and_return)] res } @@ -1574,6 +1574,21 @@ pub unsafe trait PinnedDrop: __internal::HasPinData { fn drop(self: Pin<&mut Self>, only_call_from_drop: __internal::OnlyCallFromDrop); } +/// Create a new default T. +/// +/// The returned initializer will use Default::default to initialize the `slot`. +#[inline] +pub fn default() -> impl Init { + // SAFETY: Because `T: Default`, T cannot require pinning and + // we can just move the data into the slot. + unsafe { + init_from_closure(|slot: *mut T| { + *slot = Default::default(); + Ok(()) + }) + } +} + /// Marker trait for types that can be initialized by writing just zeroes. /// /// # Safety diff --git a/rust/pin-init/src/macros.rs b/rust/pin-init/src/macros.rs index 682c61a587a0c1..1980584c442a9c 100644 --- a/rust/pin-init/src/macros.rs +++ b/rust/pin-init/src/macros.rs @@ -1312,6 +1312,10 @@ macro_rules! __init_internal { // return when an error/panic occurs. // We also use the `data` to require the correct trait (`Init` or `PinInit`) for `$field`. unsafe { $data.$field(::core::ptr::addr_of_mut!((*$slot).$field), init)? }; + // NOTE: the field accessor ensures that the initialized field is properly aligned. + // Unaligned fields will cause the compiler to emit E0793. We do not support + // unaligned fields since `Init::__init` requires an aligned pointer; the call to + // `ptr::write` below has the same requirement. // SAFETY: // - the project function does the correct field projection, // - the field has been initialized, @@ -1351,6 +1355,10 @@ macro_rules! __init_internal { // return when an error/panic occurs. unsafe { $crate::Init::__init(init, ::core::ptr::addr_of_mut!((*$slot).$field))? }; + // NOTE: the field accessor ensures that the initialized field is properly aligned. + // Unaligned fields will cause the compiler to emit E0793. We do not support + // unaligned fields since `Init::__init` requires an aligned pointer; the call to + // `ptr::write` below has the same requirement. // SAFETY: // - the field is not structurally pinned, since the line above must compile, // - the field has been initialized, @@ -1391,6 +1399,10 @@ macro_rules! __init_internal { unsafe { ::core::ptr::write(::core::ptr::addr_of_mut!((*$slot).$field), $field) }; } + // NOTE: the field accessor ensures that the initialized field is properly aligned. + // Unaligned fields will cause the compiler to emit E0793. We do not support + // unaligned fields since `Init::__init` requires an aligned pointer; the call to + // `ptr::write` below has the same requirement. #[allow(unused_variables)] // SAFETY: // - the field is not structurally pinned, since no `use_data` was required to create this @@ -1431,6 +1443,10 @@ macro_rules! __init_internal { // SAFETY: The memory at `slot` is uninitialized. unsafe { ::core::ptr::write(::core::ptr::addr_of_mut!((*$slot).$field), $field) }; } + // NOTE: the field accessor ensures that the initialized field is properly aligned. + // Unaligned fields will cause the compiler to emit E0793. We do not support + // unaligned fields since `Init::__init` requires an aligned pointer; the call to + // `ptr::write` below has the same requirement. // SAFETY: // - the project function does the correct field projection, // - the field has been initialized, diff --git a/rust/uapi/uapi_helper.h b/rust/uapi/uapi_helper.h index 06d7d1a2e8daba..512ac0aea08fde 100644 --- a/rust/uapi/uapi_helper.h +++ b/rust/uapi/uapi_helper.h @@ -7,10 +7,13 @@ */ #include +#include #include #include #include #include +#include +#include #include #include #include diff --git a/scripts/Makefile.build b/scripts/Makefile.build index 32e209bc7985cb..5d158072db95cb 100644 --- a/scripts/Makefile.build +++ b/scripts/Makefile.build @@ -321,6 +321,9 @@ $(obj)/%.lst: $(obj)/%.c FORCE # the unstable features in use. rust_allowed_features := asm_const,asm_goto,arbitrary_self_types,lint_reasons,offset_of_nested,raw_ref_op,used_with_arg +# additional rust features used by the downstream asahi kernel +rust_allowed_features := $(rust_allowed_features),ptr_sub_ptr + # `--out-dir` is required to avoid temporaries being created by `rustc` in the # current working directory, which may be not accessible in the out-of-tree # modules case. diff --git a/scripts/Makefile.package b/scripts/Makefile.package index 83bfcf7cb09fd2..1b56b1fa0d3f3e 100644 --- a/scripts/Makefile.package +++ b/scripts/Makefile.package @@ -195,7 +195,7 @@ tar%-pkg: linux-$(KERNELRELEASE)-$(ARCH).tar.% FORCE .tmp_modules_cpio: FORCE $(Q)$(MAKE) -f $(srctree)/Makefile $(Q)rm -rf $@ - $(Q)$(MAKE) -f $(srctree)/Makefile INSTALL_MOD_PATH=$@ modules_install + $(Q)$(MAKE) -f $(srctree)/Makefile INSTALL_MOD_PATH=$@/$(INSTALL_MOD_PATH) modules_install quiet_cmd_cpio = CPIO $@ cmd_cpio = $(CONFIG_SHELL) $(srctree)/usr/gen_initramfs.sh -o $@ $< @@ -265,6 +265,7 @@ help: @echo ' tarxz-pkg - Build the kernel as a xz compressed tarball' @echo ' tarzst-pkg - Build the kernel as a zstd compressed tarball' @echo ' modules-cpio-pkg - Build the kernel modules as cpio archive' + @echo ' (uses INSTALL_MOD_PATH inside the archive)' @echo ' perf-tar-src-pkg - Build the perf source tarball with no compression' @echo ' perf-targz-src-pkg - Build the perf source tarball with gzip compression' @echo ' perf-tarbz2-src-pkg - Build the perf source tarball with bz2 compression' diff --git a/scripts/cc-can-link.sh b/scripts/cc-can-link.sh index e67fd8d7b6841e..58dc7dd6d55689 100755 --- a/scripts/cc-can-link.sh +++ b/scripts/cc-can-link.sh @@ -5,7 +5,7 @@ cat << "END" | $@ -Werror -Wl,--fatal-warnings -x c - -o /dev/null >/dev/null 2> #include int main(void) { - printf(""); + printf("\n"); return 0; } END diff --git a/scripts/checkpatch.pl b/scripts/checkpatch.pl index c0250244cf7a3c..8bdec08cd12a6f 100755 --- a/scripts/checkpatch.pl +++ b/scripts/checkpatch.pl @@ -641,6 +641,7 @@ sub hash_show_words { Reviewed-by:| Reported-by:| Suggested-by:| + Assisted-by:| To:| Cc: )}; @@ -3091,6 +3092,15 @@ sub process { } } + # Assisted-by uses AGENT_NAME:MODEL_VERSION format, not email + if ($sign_off =~ /^Assisted-by:/i) { + if ($email !~ /^\S+:\S+/) { + WARN("BAD_SIGN_OFF", + "Assisted-by expects 'AGENT_NAME:MODEL_VERSION [TOOL1] [TOOL2]' format\n" . $herecurr); + } + next; + } + my ($email_name, $name_comment, $email_address, $comment) = parse_email($email); my $suggested_email = format_email(($email_name, $name_comment, $email_address, $comment)); if ($suggested_email eq "") { diff --git a/scripts/dtc/data.c b/scripts/dtc/data.c index 5b25aa06041613..ce449824c80a06 100644 --- a/scripts/dtc/data.c +++ b/scripts/dtc/data.c @@ -184,6 +184,33 @@ struct data data_append_integer(struct data d, uint64_t value, int bits) } } +struct data data_append_float(struct data d, double value, int bits) +{ + float f32; + uint32_t u32; + double f64; + uint64_t u64; + fdt32_t value_32; + fdt64_t value_64; + + switch (bits) { + case 32: + f32 = value; + memcpy(&u32, &f32, sizeof(u32)); + value_32 = cpu_to_fdt32(u32); + return data_append_data(d, &value_32, 4); + + case 64: + f64 = value; + memcpy(&u64, &f64, sizeof(u64)); + value_64 = cpu_to_fdt64(u64); + return data_append_data(d, &value_64, 8); + + default: + die("Invalid literal size (%d)\n", bits); + } +} + struct data data_append_re(struct data d, uint64_t address, uint64_t size) { struct fdt_reserve_entry re; diff --git a/scripts/dtc/dtc-lexer.l b/scripts/dtc/dtc-lexer.l index 15d585c8079802..bd750717aa3a54 100644 --- a/scripts/dtc/dtc-lexer.l +++ b/scripts/dtc/dtc-lexer.l @@ -151,6 +151,28 @@ static void PRINTF(1, 2) lexical_error(const char *fmt, ...); return DT_LABEL; } +[-+]?(([0-9]+\.[0-9]*)|([0-9]*\.[0-9]+))(e[-+]?[0-9]+)?f? { + char *e; + DPRINT("Floating-point Literal: '%s'\n", yytext); + + errno = 0; + yylval.floating = strtod(yytext, &e); + + if (*e && (*e != 'f' || e[1])) { + lexical_error("Bad floating-point literal '%s'", + yytext); + } + + if (errno == ERANGE) + lexical_error("Floating-point literal '%s' out of range", + yytext); + else + /* ERANGE is the only strtod error triggerable + * by strings matching the pattern */ + assert(errno == 0); + return DT_FP_LITERAL; + } + {LABEL} { /* Missed includes or macro definitions while * preprocessing can lead to unexpected identifiers in diff --git a/scripts/dtc/dtc-parser.y b/scripts/dtc/dtc-parser.y index 4d5eece5262434..225a6b41b14fcf 100644 --- a/scripts/dtc/dtc-parser.y +++ b/scripts/dtc/dtc-parser.y @@ -48,6 +48,7 @@ static bool is_ref_relative(const char *ref) struct node *nodelist; struct reserve_info *re; uint64_t integer; + double floating; unsigned int flags; } @@ -61,6 +62,7 @@ static bool is_ref_relative(const char *ref) %token DT_OMIT_NO_REF %token DT_PROPNODENAME %token DT_LITERAL +%token DT_FP_LITERAL %token DT_CHAR_LITERAL %token DT_BYTE %token DT_STRING @@ -86,6 +88,7 @@ static bool is_ref_relative(const char *ref) %type subnode %type subnodes +%type floating_prim %type integer_prim %type integer_unary %type integer_mul @@ -395,6 +398,15 @@ arrayprefix: $$.data = data_add_marker(empty_data, TYPE_UINT32, NULL); $$.bits = 32; } + | arrayprefix floating_prim + { + if ($1.bits < 32) { + ERROR(&@2, "Floating-point values must be" + " 32-bit or 64-bit"); + } + + $$.data = data_append_float($1.data, $2, $1.bits); + } | arrayprefix integer_prim { if ($1.bits < 64) { @@ -439,6 +451,10 @@ arrayprefix: } ; +floating_prim: + DT_FP_LITERAL + ; + integer_prim: DT_LITERAL | DT_CHAR_LITERAL diff --git a/scripts/dtc/dtc.h b/scripts/dtc/dtc.h index 3a220b9afc99f9..cad8be1440bd69 100644 --- a/scripts/dtc/dtc.h +++ b/scripts/dtc/dtc.h @@ -177,6 +177,7 @@ struct data data_insert_at_marker(struct data d, struct marker *m, struct data data_merge(struct data d1, struct data d2); struct data data_append_cell(struct data d, cell_t word); struct data data_append_integer(struct data d, uint64_t word, int bits); +struct data data_append_float(struct data d, double value, int bits); struct data data_append_re(struct data d, uint64_t address, uint64_t size); struct data data_append_addr(struct data d, uint64_t addr); struct data data_append_byte(struct data d, uint8_t byte); diff --git a/scripts/gdb/linux/symbols.py b/scripts/gdb/linux/symbols.py index d4308b7261838b..943ff1228b487e 100644 --- a/scripts/gdb/linux/symbols.py +++ b/scripts/gdb/linux/symbols.py @@ -298,7 +298,7 @@ def invoke(self, arg, from_tty): if p == "-bpf": monitor_bpf = True else: - p.append(os.path.abspath(os.path.expanduser(p))) + self.module_paths.append(os.path.abspath(os.path.expanduser(p))) self.module_paths.append(os.getcwd()) if self.breakpoint is not None: diff --git a/scripts/gendwarfksyms/dwarf.c b/scripts/gendwarfksyms/dwarf.c index 3538a7d9cb070b..e76d732f5f6022 100644 --- a/scripts/gendwarfksyms/dwarf.c +++ b/scripts/gendwarfksyms/dwarf.c @@ -750,6 +750,7 @@ static void process_enumerator_type(struct state *state, struct die *cache, Dwarf_Die *die) { bool overridden = false; + unsigned long override; Dwarf_Word value; if (stable) { @@ -761,7 +762,8 @@ static void process_enumerator_type(struct state *state, struct die *cache, return; overridden = kabi_get_enumerator_value( - state->expand.current_fqn, cache->fqn, &value); + state->expand.current_fqn, cache->fqn, &override); + value = override; } process_list_comma(state, cache); diff --git a/scripts/gendwarfksyms/symbols.c b/scripts/gendwarfksyms/symbols.c index ecddcb5ffcdfb9..42cd27c9cec4f6 100644 --- a/scripts/gendwarfksyms/symbols.c +++ b/scripts/gendwarfksyms/symbols.c @@ -3,6 +3,7 @@ * Copyright (C) 2024 Google LLC */ +#include #include "gendwarfksyms.h" #define SYMBOL_HASH_BITS 12 @@ -242,7 +243,7 @@ static void elf_for_each_global(int fd, elf_symbol_callback_t func, void *arg) error("elf_getdata failed: %s", elf_errmsg(-1)); if (shdr->sh_entsize != sym_size) - error("expected sh_entsize (%lu) to be %zu", + error("expected sh_entsize (%" PRIu64 ") to be %zu", shdr->sh_entsize, sym_size); nsyms = shdr->sh_size / shdr->sh_entsize; @@ -292,7 +293,7 @@ static void set_symbol_addr(struct symbol *sym, void *arg) hash_add(symbol_addrs, &sym->addr_hash, symbol_addr_hash(&sym->addr)); - debug("%s -> { %u, %lx }", sym->name, sym->addr.section, + debug("%s -> { %u, %" PRIx64 " }", sym->name, sym->addr.section, sym->addr.address); } else if (sym->addr.section != addr->section || sym->addr.address != addr->address) { diff --git a/scripts/generate_rust_analyzer.py b/scripts/generate_rust_analyzer.py index 766c2d91cd8111..91673d131d8cb9 100755 --- a/scripts/generate_rust_analyzer.py +++ b/scripts/generate_rust_analyzer.py @@ -190,9 +190,10 @@ def append_crate_with_generated( def is_root_crate(build_file, target): try: - return f"{target}.o" in open(build_file).read() + contents = build_file.read_text() except FileNotFoundError: return False + return f"{target}.o" in contents # Then, the rest outside of `rust/`. # diff --git a/scripts/kernel-doc.py b/scripts/kernel-doc.py index 7a1eaf986bcd4f..1ebb16b9bb087d 100755 --- a/scripts/kernel-doc.py +++ b/scripts/kernel-doc.py @@ -116,6 +116,8 @@ sys.path.insert(0, os.path.join(SRC_DIR, LIB_DIR)) +WERROR_RETURN_CODE = 3 + DESC = """ Read C language source or header FILEs, extract embedded documentation comments, and print formatted documentation to standard output. @@ -176,7 +178,21 @@ def format(self, record): return logging.Formatter.format(self, record) def main(): - """Main program""" + """ + Main program. + + By default, the return value is: + + - 0: success or Python version is not compatible with + kernel-doc. If -Werror is not used, it will also + return 0 if there are issues at kernel-doc markups; + + - 1: an abnormal condition happened; + + - 2: argparse issued an error; + + - 3: -Werror is used, and one or more unfiltered parse warnings happened. + """ parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter, description=DESC) @@ -323,16 +339,12 @@ def main(): if args.werror: print("%s warnings as errors" % error_count) # pylint: disable=C0209 - sys.exit(error_count) + sys.exit(WERROR_RETURN_CODE) if args.verbose: print("%s errors" % error_count) # pylint: disable=C0209 - if args.none: - sys.exit(0) - - sys.exit(error_count) - + sys.exit(0) # Call main method if __name__ == "__main__": diff --git a/scripts/livepatch/klp-build b/scripts/livepatch/klp-build index 809e198a561d54..7b82c7503c2bf0 100755 --- a/scripts/livepatch/klp-build +++ b/scripts/livepatch/klp-build @@ -285,15 +285,14 @@ set_module_name() { # application from appending it with '+' due to a dirty git working tree. set_kernelversion() { local file="$SRC/scripts/setlocalversion" - local localversion + local kernelrelease stash_file "$file" - localversion="$(cd "$SRC" && make --no-print-directory kernelversion)" - localversion="$(cd "$SRC" && KERNELVERSION="$localversion" ./scripts/setlocalversion)" - [[ -z "$localversion" ]] && die "setlocalversion failed" + kernelrelease="$(cd "$SRC" && make syncconfig &>/dev/null && make -s kernelrelease)" + [[ -z "$kernelrelease" ]] && die "failed to get kernel version" - sed -i "2i echo $localversion; exit 0" scripts/setlocalversion + sed -i "2i echo $kernelrelease; exit 0" scripts/setlocalversion } get_patch_files() { diff --git a/scripts/mod/modpost.c b/scripts/mod/modpost.c index 755b842f1f9b7a..e8680442968d8f 100644 --- a/scripts/mod/modpost.c +++ b/scripts/mod/modpost.c @@ -56,7 +56,7 @@ static bool allow_missing_ns_imports; static bool error_occurred; -static bool extra_warn; +static bool extra_warn __attribute__((unused)); bool target_is_big_endian; bool host_is_big_endian; @@ -602,6 +602,10 @@ static int ignore_undef_symbol(struct elf_info *info, const char *symname) /* Special register function linked on all modules during final link of .ko */ if (strstarts(symname, "_restgpr0_") || strstarts(symname, "_savegpr0_") || + strstarts(symname, "_restgpr1_") || + strstarts(symname, "_savegpr1_") || + strstarts(symname, "_restfpr_") || + strstarts(symname, "_savefpr_") || strstarts(symname, "_restvr_") || strstarts(symname, "_savevr_") || strcmp(symname, ".TOC.") == 0) diff --git a/scripts/package/install-extmod-build b/scripts/package/install-extmod-build index 2576cf7902dbbf..f12e1ffe409eb0 100755 --- a/scripts/package/install-extmod-build +++ b/scripts/package/install-extmod-build @@ -32,6 +32,10 @@ mkdir -p "${destdir}" echo tools/objtool/objtool fi + if is_enabled CONFIG_DEBUG_INFO_BTF_MODULES; then + echo tools/bpf/resolve_btfids/resolve_btfids + fi + echo Module.symvers echo "arch/${SRCARCH}/include/generated" echo include/config/auto.conf diff --git a/scripts/package/kernel.spec b/scripts/package/kernel.spec index 0f1c8de1bd95f8..b3c956205af001 100644 --- a/scripts/package/kernel.spec +++ b/scripts/package/kernel.spec @@ -2,8 +2,6 @@ %{!?_arch: %define _arch dummy} %{!?make: %define make make} %define makeflags %{?_smp_mflags} ARCH=%{ARCH} -%define __spec_install_post /usr/lib/rpm/brp-compress || : -%define debug_package %{nil} Name: kernel Summary: The Linux Kernel @@ -47,13 +45,49 @@ This package provides kernel headers and makefiles sufficient to build modules against the %{version} kernel package. %endif -%if %{with_debuginfo} +%if %{with_debuginfo_manual} %package debuginfo Summary: Debug information package for the Linux kernel +Group: Development/Debug +AutoReq: 0 +AutoProv: 1 %description debuginfo This package provides debug information for the kernel image and modules from the %{version} package. +%define install_mod_strip 1 +%endif + +%if %{with_debuginfo_rpm} +# list of debuginfo-related options taken from distribution kernel.spec +# files +%undefine _include_minidebuginfo +%undefine _find_debuginfo_dwz_opts +%undefine _unique_build_ids +%undefine _unique_debug_names +%undefine _unique_debug_srcs +%undefine _debugsource_packages +%undefine _debuginfo_subpackages +%global _find_debuginfo_opts -r +%global _missing_build_ids_terminate_build 1 +%global _no_recompute_build_ids 1 +%{debug_package} + +# later, we make all modules executable so that find-debuginfo.sh strips +# them up. but they don't actually need to be executable, so remove the +# executable bit, taking care to do it _after_ find-debuginfo.sh has run +%define __spec_install_post \ + %{?__debug_package:%{__debug_install_post}} \ + %{__arch_install_post} \ + %{__os_install_post} \ + find %{buildroot}/lib/modules/%{KERNELRELEASE} -name "*.ko" -type f \\\ + | xargs --no-run-if-empty chmod u-x +%else +%define __spec_install_post /usr/lib/rpm/brp-compress || : %endif +# some (but not all) versions of rpmbuild emit %%debug_package with +# %%install. since we've already emitted it manually, that would cause +# a package redefinition error. ensure that doesn't happen +%define debug_package %{nil} %prep %setup -q -n linux @@ -67,7 +101,7 @@ patch -p1 < %{SOURCE2} mkdir -p %{buildroot}/lib/modules/%{KERNELRELEASE} cp $(%{make} %{makeflags} -s image_name) %{buildroot}/lib/modules/%{KERNELRELEASE}/vmlinuz # DEPMOD=true makes depmod no-op. We do not package depmod-generated files. -%{make} %{makeflags} INSTALL_MOD_PATH=%{buildroot} INSTALL_MOD_STRIP=1 DEPMOD=true modules_install +%{make} %{makeflags} INSTALL_MOD_PATH=%{buildroot} %{?install_mod_strip:INSTALL_MOD_STRIP=1} DEPMOD=true modules_install %{make} %{makeflags} INSTALL_HDR_PATH=%{buildroot}/usr headers_install cp System.map %{buildroot}/lib/modules/%{KERNELRELEASE} cp .config %{buildroot}/lib/modules/%{KERNELRELEASE}/config @@ -98,22 +132,30 @@ ln -fns /usr/src/kernels/%{KERNELRELEASE} %{buildroot}/lib/modules/%{KERNELRELEA echo "%exclude /lib/modules/%{KERNELRELEASE}/build" } > %{buildroot}/kernel.list -%if %{with_debuginfo} +%if 0%{with_debuginfo_manual}%{with_debuginfo_rpm} > 0 # copying vmlinux directly to the debug directory means it will not get # stripped (but its source paths will still be collected + fixed up) mkdir -p %{buildroot}/usr/lib/debug/lib/modules/%{KERNELRELEASE} cp vmlinux %{buildroot}/usr/lib/debug/lib/modules/%{KERNELRELEASE} +%endif -echo /usr/lib/debug/lib/modules/%{KERNELRELEASE}/vmlinux > %{buildroot}/debuginfo.list +%if %{with_debuginfo_rpm} +# make modules executable so that find-debuginfo.sh strips them. this +# will be undone later in %%__spec_install_post +find %{buildroot}/lib/modules/%{KERNELRELEASE} -name "*.ko" -type f \ + | xargs --no-run-if-empty chmod u+x +%endif +%if %{with_debuginfo_manual} +echo /usr/lib/debug/lib/modules/%{KERNELRELEASE}/vmlinux > %{buildroot}/debuginfo.list while read -r mod; do mod="${mod%.o}.ko" dbg="%{buildroot}/usr/lib/debug/lib/modules/%{KERNELRELEASE}/kernel/${mod}" - buildid=$("${READELF}" -n "${mod}" | sed -n 's@^.*Build ID: \(..\)\(.*\)@\1/\2@p') + buildid=$("${READELF:-readelf}" -n "${mod}" | sed -n 's@^.*Build ID: \(..\)\(.*\)@\1/\2@p') link="%{buildroot}/usr/lib/debug/.build-id/${buildid}.debug" mkdir -p "${dbg%/*}" "${link%/*}" - "${OBJCOPY}" --only-keep-debug "${mod}" "${dbg}" + "${OBJCOPY:-objcopy}" --only-keep-debug "${mod}" "${dbg}" ln -sf --relative "${dbg}" "${link}" echo "${dbg#%{buildroot}}" >> %{buildroot}/debuginfo.list @@ -123,6 +165,10 @@ done < modules.order %clean rm -rf %{buildroot} +%if %{with_debuginfo_rpm} +rm -f debugfiles.list debuglinks.list debugsourcefiles.list debugsources.list \ + elfbins.list +%endif %post if [ -x /usr/bin/kernel-install ]; then @@ -162,7 +208,7 @@ fi /lib/modules/%{KERNELRELEASE}/build %endif -%if %{with_debuginfo} +%if %{with_debuginfo_manual} %files -f %{buildroot}/debuginfo.list debuginfo %defattr (-, root, root) %exclude /debuginfo.list diff --git a/scripts/package/mkspec b/scripts/package/mkspec index c7375bfc25a9ad..c604f8c174e2cd 100755 --- a/scripts/package/mkspec +++ b/scripts/package/mkspec @@ -23,15 +23,47 @@ else echo '%define with_devel 0' fi +# use %{debug_package} machinery to generate -debuginfo +with_debuginfo_rpm=0 +# manually generate -debuginfo package +with_debuginfo_manual=0 # debuginfo package generation uses find-debuginfo.sh under the hood, # which only works on uncompressed modules that contain debuginfo if grep -q CONFIG_DEBUG_INFO=y include/config/auto.conf && (! grep -q CONFIG_MODULE_COMPRESS=y include/config/auto.conf) && (! grep -q CONFIG_DEBUG_INFO_SPLIT=y include/config/auto.conf); then -echo '%define with_debuginfo %{?_without_debuginfo: 0} %{?!_without_debuginfo: 1}' -else -echo '%define with_debuginfo 0' + # If module signing is enabled (which may be required to boot with + # lockdown enabled), the find-debuginfo.sh machinery cannot be used + # because the signatures will be stripped off the modules. However, due + # to an rpm bug in versions prior to 4.20.0 + # + # https://github.com/rpm-software-management/rpm/issues/3057 + # https://github.com/rpm-software-management/rpm/commit/49f906998f3cf1f4152162ca61ac0869251c380f + # + # We cannot provide our own debuginfo package because it does not listen + # to our custom files list, failing the build due to unpackaged files. + # Manually generate the debug info package if using rpm 4.20.0. If not + # using rpm 4.20.0, avoid generating a -debuginfo package altogether, + # as it is not safe. + if grep -q CONFIG_MODULE_SIG=y include/config/auto.conf; then + rpm_ver_str=$(rpm --version 2>/dev/null) + # Split the version on spaces + IFS=' ' + set -- $rpm_ver_str + if [ "${1:-}" = RPM -a "${2:-}" = version ]; then + IFS=. + set -- $3 + rpm_ver=$(( 1000000 * $1 + 10000 * $2 + 100 * $3 + ${4:-0} )) + if [ "$rpm_ver" -ge 4200000 ]; then + with_debuginfo_manual='%{?_without_debuginfo:0}%{?!_without_debuginfo:1}' + fi + fi + else + with_debuginfo_rpm='%{?_without_debuginfo:0}%{?!_without_debuginfo:1}' + fi fi +echo "%define with_debuginfo_manual $with_debuginfo_manual" +echo "%define with_debuginfo_rpm $with_debuginfo_rpm" cat<loaddata); + aa_put_i_loaddata(private->loaddata); kvfree(private); } @@ -153,6 +155,71 @@ static int aafs_show_path(struct seq_file *seq, struct dentry *dentry) return 0; } +static struct aa_ns *get_ns_common_ref(struct aa_common_ref *ref) +{ + if (ref) { + struct aa_label *reflabel = container_of(ref, struct aa_label, + count); + return aa_get_ns(labels_ns(reflabel)); + } + + return NULL; +} + +static struct aa_proxy *get_proxy_common_ref(struct aa_common_ref *ref) +{ + if (ref) + return aa_get_proxy(container_of(ref, struct aa_proxy, count)); + + return NULL; +} + +static struct aa_loaddata *get_loaddata_common_ref(struct aa_common_ref *ref) +{ + if (ref) + return aa_get_i_loaddata(container_of(ref, struct aa_loaddata, + count)); + return NULL; +} + +static void aa_put_common_ref(struct aa_common_ref *ref) +{ + if (!ref) + return; + + switch (ref->reftype) { + case REF_RAWDATA: + aa_put_i_loaddata(container_of(ref, struct aa_loaddata, + count)); + break; + case REF_PROXY: + aa_put_proxy(container_of(ref, struct aa_proxy, + count)); + break; + case REF_NS: + /* ns count is held on its unconfined label */ + aa_put_ns(labels_ns(container_of(ref, struct aa_label, count))); + break; + default: + AA_BUG(true, "unknown refcount type"); + break; + } +} + +static void aa_get_common_ref(struct aa_common_ref *ref) +{ + kref_get(&ref->count); +} + +static void aafs_evict(struct inode *inode) +{ + struct aa_common_ref *ref = inode->i_private; + + clear_inode(inode); + aa_put_common_ref(ref); + inode->i_private = (void *) IREF_POISON; +} + static void aafs_free_inode(struct inode *inode) { if (S_ISLNK(inode->i_mode)) @@ -162,6 +229,7 @@ static void aafs_free_inode(struct inode *inode) static const struct super_operations aafs_super_ops = { .statfs = simple_statfs, + .evict_inode = aafs_evict, .free_inode = aafs_free_inode, .show_path = aafs_show_path, }; @@ -262,7 +330,8 @@ static int __aafs_setup_d_inode(struct inode *dir, struct dentry *dentry, * aafs_remove(). Will return ERR_PTR on failure. */ static struct dentry *aafs_create(const char *name, umode_t mode, - struct dentry *parent, void *data, void *link, + struct dentry *parent, + struct aa_common_ref *data, void *link, const struct file_operations *fops, const struct inode_operations *iops) { @@ -299,6 +368,9 @@ static struct dentry *aafs_create(const char *name, umode_t mode, goto fail_dentry; inode_unlock(dir); + if (data) + aa_get_common_ref(data); + return dentry; fail_dentry: @@ -323,7 +395,8 @@ static struct dentry *aafs_create(const char *name, umode_t mode, * see aafs_create */ static struct dentry *aafs_create_file(const char *name, umode_t mode, - struct dentry *parent, void *data, + struct dentry *parent, + struct aa_common_ref *data, const struct file_operations *fops) { return aafs_create(name, mode, parent, data, NULL, fops, NULL); @@ -409,7 +482,8 @@ static struct aa_loaddata *aa_simple_write_to_buffer(const char __user *userbuf, data->size = copy_size; if (copy_from_user(data->data, userbuf, copy_size)) { - aa_put_loaddata(data); + /* trigger free - don't need to put pcount */ + aa_put_i_loaddata(data); return ERR_PTR(-EFAULT); } @@ -417,7 +491,8 @@ static struct aa_loaddata *aa_simple_write_to_buffer(const char __user *userbuf, } static ssize_t policy_update(u32 mask, const char __user *buf, size_t size, - loff_t *pos, struct aa_ns *ns) + loff_t *pos, struct aa_ns *ns, + const struct cred *ocred) { struct aa_loaddata *data; struct aa_label *label; @@ -428,7 +503,7 @@ static ssize_t policy_update(u32 mask, const char __user *buf, size_t size, /* high level check about policy management - fine grained in * below after unpack */ - error = aa_may_manage_policy(current_cred(), label, ns, mask); + error = aa_may_manage_policy(current_cred(), label, ns, ocred, mask); if (error) goto end_section; @@ -436,7 +511,10 @@ static ssize_t policy_update(u32 mask, const char __user *buf, size_t size, error = PTR_ERR(data); if (!IS_ERR(data)) { error = aa_replace_profiles(ns, label, mask, data); - aa_put_loaddata(data); + /* put pcount, which will put count and free if no + * profiles referencing it. + */ + aa_put_profile_loaddata(data); } end_section: end_current_label_crit_section(label); @@ -448,8 +526,9 @@ static ssize_t policy_update(u32 mask, const char __user *buf, size_t size, static ssize_t profile_load(struct file *f, const char __user *buf, size_t size, loff_t *pos) { - struct aa_ns *ns = aa_get_ns(f->f_inode->i_private); - int error = policy_update(AA_MAY_LOAD_POLICY, buf, size, pos, ns); + struct aa_ns *ns = get_ns_common_ref(f->f_inode->i_private); + int error = policy_update(AA_MAY_LOAD_POLICY, buf, size, pos, ns, + f->f_cred); aa_put_ns(ns); @@ -465,9 +544,9 @@ static const struct file_operations aa_fs_profile_load = { static ssize_t profile_replace(struct file *f, const char __user *buf, size_t size, loff_t *pos) { - struct aa_ns *ns = aa_get_ns(f->f_inode->i_private); + struct aa_ns *ns = get_ns_common_ref(f->f_inode->i_private); int error = policy_update(AA_MAY_LOAD_POLICY | AA_MAY_REPLACE_POLICY, - buf, size, pos, ns); + buf, size, pos, ns, f->f_cred); aa_put_ns(ns); return error; @@ -485,14 +564,14 @@ static ssize_t profile_remove(struct file *f, const char __user *buf, struct aa_loaddata *data; struct aa_label *label; ssize_t error; - struct aa_ns *ns = aa_get_ns(f->f_inode->i_private); + struct aa_ns *ns = get_ns_common_ref(f->f_inode->i_private); label = begin_current_label_crit_section(); /* high level check about policy management - fine grained in * below after unpack */ error = aa_may_manage_policy(current_cred(), label, ns, - AA_MAY_REMOVE_POLICY); + f->f_cred, AA_MAY_REMOVE_POLICY); if (error) goto out; @@ -506,7 +585,7 @@ static ssize_t profile_remove(struct file *f, const char __user *buf, if (!IS_ERR(data)) { data->data[size] = 0; error = aa_remove_profiles(ns, label, data->data, size); - aa_put_loaddata(data); + aa_put_profile_loaddata(data); } out: end_current_label_crit_section(label); @@ -575,7 +654,7 @@ static int ns_revision_open(struct inode *inode, struct file *file) if (!rev) return -ENOMEM; - rev->ns = aa_get_ns(inode->i_private); + rev->ns = get_ns_common_ref(inode->i_private); if (!rev->ns) rev->ns = aa_get_current_ns(); file->private_data = rev; @@ -1061,7 +1140,7 @@ static const struct file_operations seq_profile_ ##NAME ##_fops = { \ static int seq_profile_open(struct inode *inode, struct file *file, int (*show)(struct seq_file *, void *)) { - struct aa_proxy *proxy = aa_get_proxy(inode->i_private); + struct aa_proxy *proxy = get_proxy_common_ref(inode->i_private); int error = single_open(file, show, proxy); if (error) { @@ -1253,18 +1332,17 @@ static const struct file_operations seq_rawdata_ ##NAME ##_fops = { \ static int seq_rawdata_open(struct inode *inode, struct file *file, int (*show)(struct seq_file *, void *)) { - struct aa_loaddata *data = __aa_get_loaddata(inode->i_private); + struct aa_loaddata *data = get_loaddata_common_ref(inode->i_private); int error; if (!data) - /* lost race this ent is being reaped */ return -ENOENT; error = single_open(file, show, data); if (error) { AA_BUG(file->private_data && ((struct seq_file *)file->private_data)->private); - aa_put_loaddata(data); + aa_put_i_loaddata(data); } return error; @@ -1275,7 +1353,7 @@ static int seq_rawdata_release(struct inode *inode, struct file *file) struct seq_file *seq = (struct seq_file *) file->private_data; if (seq) - aa_put_loaddata(seq->private); + aa_put_i_loaddata(seq->private); return single_release(inode, file); } @@ -1387,9 +1465,8 @@ static int rawdata_open(struct inode *inode, struct file *file) if (!aa_current_policy_view_capable(NULL)) return -EACCES; - loaddata = __aa_get_loaddata(inode->i_private); + loaddata = get_loaddata_common_ref(inode->i_private); if (!loaddata) - /* lost race: this entry is being reaped */ return -ENOENT; private = rawdata_f_data_alloc(loaddata->size); @@ -1414,7 +1491,7 @@ static int rawdata_open(struct inode *inode, struct file *file) return error; fail_private_alloc: - aa_put_loaddata(loaddata); + aa_put_i_loaddata(loaddata); return error; } @@ -1431,7 +1508,6 @@ static void remove_rawdata_dents(struct aa_loaddata *rawdata) for (i = 0; i < AAFS_LOADDATA_NDENTS; i++) { if (!IS_ERR_OR_NULL(rawdata->dents[i])) { - /* no refcounts on i_private */ aafs_remove(rawdata->dents[i]); rawdata->dents[i] = NULL; } @@ -1474,35 +1550,37 @@ int __aa_fs_create_rawdata(struct aa_ns *ns, struct aa_loaddata *rawdata) return PTR_ERR(dir); rawdata->dents[AAFS_LOADDATA_DIR] = dir; - dent = aafs_create_file("abi", S_IFREG | 0444, dir, rawdata, + dent = aafs_create_file("abi", S_IFREG | 0444, dir, &rawdata->count, &seq_rawdata_abi_fops); if (IS_ERR(dent)) goto fail; rawdata->dents[AAFS_LOADDATA_ABI] = dent; - dent = aafs_create_file("revision", S_IFREG | 0444, dir, rawdata, - &seq_rawdata_revision_fops); + dent = aafs_create_file("revision", S_IFREG | 0444, dir, + &rawdata->count, + &seq_rawdata_revision_fops); if (IS_ERR(dent)) goto fail; rawdata->dents[AAFS_LOADDATA_REVISION] = dent; if (aa_g_hash_policy) { dent = aafs_create_file("sha256", S_IFREG | 0444, dir, - rawdata, &seq_rawdata_hash_fops); + &rawdata->count, + &seq_rawdata_hash_fops); if (IS_ERR(dent)) goto fail; rawdata->dents[AAFS_LOADDATA_HASH] = dent; } dent = aafs_create_file("compressed_size", S_IFREG | 0444, dir, - rawdata, + &rawdata->count, &seq_rawdata_compressed_size_fops); if (IS_ERR(dent)) goto fail; rawdata->dents[AAFS_LOADDATA_COMPRESSED_SIZE] = dent; - dent = aafs_create_file("raw_data", S_IFREG | 0444, - dir, rawdata, &rawdata_fops); + dent = aafs_create_file("raw_data", S_IFREG | 0444, dir, + &rawdata->count, &rawdata_fops); if (IS_ERR(dent)) goto fail; rawdata->dents[AAFS_LOADDATA_DATA] = dent; @@ -1510,13 +1588,11 @@ int __aa_fs_create_rawdata(struct aa_ns *ns, struct aa_loaddata *rawdata) rawdata->ns = aa_get_ns(ns); list_add(&rawdata->list, &ns->rawdata_list); - /* no refcount on inode rawdata */ return 0; fail: remove_rawdata_dents(rawdata); - return PTR_ERR(dent); } #endif /* CONFIG_SECURITY_APPARMOR_EXPORT_BINARY */ @@ -1540,13 +1616,10 @@ void __aafs_profile_rmdir(struct aa_profile *profile) __aafs_profile_rmdir(child); for (i = AAFS_PROF_SIZEOF - 1; i >= 0; --i) { - struct aa_proxy *proxy; if (!profile->dents[i]) continue; - proxy = d_inode(profile->dents[i])->i_private; aafs_remove(profile->dents[i]); - aa_put_proxy(proxy); profile->dents[i] = NULL; } } @@ -1580,14 +1653,7 @@ static struct dentry *create_profile_file(struct dentry *dir, const char *name, struct aa_profile *profile, const struct file_operations *fops) { - struct aa_proxy *proxy = aa_get_proxy(profile->label.proxy); - struct dentry *dent; - - dent = aafs_create_file(name, S_IFREG | 0444, dir, proxy, fops); - if (IS_ERR(dent)) - aa_put_proxy(proxy); - - return dent; + return aafs_create_file(name, S_IFREG | 0444, dir, &profile->label.proxy->count, fops); } #ifdef CONFIG_SECURITY_APPARMOR_EXPORT_BINARY @@ -1633,7 +1699,8 @@ static const char *rawdata_get_link_base(struct dentry *dentry, struct delayed_call *done, const char *name) { - struct aa_proxy *proxy = inode->i_private; + struct aa_common_ref *ref = inode->i_private; + struct aa_proxy *proxy = container_of(ref, struct aa_proxy, count); struct aa_label *label; struct aa_profile *profile; char *target; @@ -1644,6 +1711,15 @@ static const char *rawdata_get_link_base(struct dentry *dentry, label = aa_get_label_rcu(&proxy->label); profile = labels_profile(label); + + /* rawdata can be null when aa_g_export_binary is unset during + * runtime and a profile is replaced + */ + if (!profile->rawdata) { + aa_put_label(label); + return ERR_PTR(-ENOENT); + } + depth = profile_depth(profile); target = gen_symlink_name(depth, profile->rawdata->name, name); aa_put_label(label); @@ -1766,27 +1842,24 @@ int __aafs_profile_mkdir(struct aa_profile *profile, struct dentry *parent) if (profile->rawdata) { if (aa_g_hash_policy) { dent = aafs_create("raw_sha256", S_IFLNK | 0444, dir, - profile->label.proxy, NULL, NULL, - &rawdata_link_sha256_iops); + &profile->label.proxy->count, NULL, + NULL, &rawdata_link_sha256_iops); if (IS_ERR(dent)) goto fail; - aa_get_proxy(profile->label.proxy); profile->dents[AAFS_PROF_RAW_HASH] = dent; } dent = aafs_create("raw_abi", S_IFLNK | 0444, dir, - profile->label.proxy, NULL, NULL, + &profile->label.proxy->count, NULL, NULL, &rawdata_link_abi_iops); if (IS_ERR(dent)) goto fail; - aa_get_proxy(profile->label.proxy); profile->dents[AAFS_PROF_RAW_ABI] = dent; dent = aafs_create("raw_data", S_IFLNK | 0444, dir, - profile->label.proxy, NULL, NULL, + &profile->label.proxy->count, NULL, NULL, &rawdata_link_data_iops); if (IS_ERR(dent)) goto fail; - aa_get_proxy(profile->label.proxy); profile->dents[AAFS_PROF_RAW_DATA] = dent; } #endif /*CONFIG_SECURITY_APPARMOR_EXPORT_BINARY */ @@ -1817,13 +1890,13 @@ static struct dentry *ns_mkdir_op(struct mnt_idmap *idmap, struct inode *dir, int error; label = begin_current_label_crit_section(); - error = aa_may_manage_policy(current_cred(), label, NULL, + error = aa_may_manage_policy(current_cred(), label, NULL, NULL, AA_MAY_LOAD_POLICY); end_current_label_crit_section(label); if (error) return ERR_PTR(error); - parent = aa_get_ns(dir->i_private); + parent = get_ns_common_ref(dir->i_private); AA_BUG(d_inode(ns_subns_dir(parent)) != dir); /* we have to unlock and then relock to get locking order right @@ -1867,13 +1940,13 @@ static int ns_rmdir_op(struct inode *dir, struct dentry *dentry) int error; label = begin_current_label_crit_section(); - error = aa_may_manage_policy(current_cred(), label, NULL, + error = aa_may_manage_policy(current_cred(), label, NULL, NULL, AA_MAY_LOAD_POLICY); end_current_label_crit_section(label); if (error) return error; - parent = aa_get_ns(dir->i_private); + parent = get_ns_common_ref(dir->i_private); /* rmdir calls the generic securityfs functions to remove files * from the apparmor dir. It is up to the apparmor ns locking * to avoid races. @@ -1943,27 +2016,6 @@ void __aafs_ns_rmdir(struct aa_ns *ns) __aa_fs_list_remove_rawdata(ns); - if (ns_subns_dir(ns)) { - sub = d_inode(ns_subns_dir(ns))->i_private; - aa_put_ns(sub); - } - if (ns_subload(ns)) { - sub = d_inode(ns_subload(ns))->i_private; - aa_put_ns(sub); - } - if (ns_subreplace(ns)) { - sub = d_inode(ns_subreplace(ns))->i_private; - aa_put_ns(sub); - } - if (ns_subremove(ns)) { - sub = d_inode(ns_subremove(ns))->i_private; - aa_put_ns(sub); - } - if (ns_subrevision(ns)) { - sub = d_inode(ns_subrevision(ns))->i_private; - aa_put_ns(sub); - } - for (i = AAFS_NS_SIZEOF - 1; i >= 0; --i) { aafs_remove(ns->dents[i]); ns->dents[i] = NULL; @@ -1988,40 +2040,40 @@ static int __aafs_ns_mkdir_entries(struct aa_ns *ns, struct dentry *dir) return PTR_ERR(dent); ns_subdata_dir(ns) = dent; - dent = aafs_create_file("revision", 0444, dir, ns, + dent = aafs_create_file("revision", 0444, dir, + &ns->unconfined->label.count, &aa_fs_ns_revision_fops); if (IS_ERR(dent)) return PTR_ERR(dent); - aa_get_ns(ns); ns_subrevision(ns) = dent; - dent = aafs_create_file(".load", 0640, dir, ns, - &aa_fs_profile_load); + dent = aafs_create_file(".load", 0640, dir, + &ns->unconfined->label.count, + &aa_fs_profile_load); if (IS_ERR(dent)) return PTR_ERR(dent); - aa_get_ns(ns); ns_subload(ns) = dent; - dent = aafs_create_file(".replace", 0640, dir, ns, - &aa_fs_profile_replace); + dent = aafs_create_file(".replace", 0640, dir, + &ns->unconfined->label.count, + &aa_fs_profile_replace); if (IS_ERR(dent)) return PTR_ERR(dent); - aa_get_ns(ns); ns_subreplace(ns) = dent; - dent = aafs_create_file(".remove", 0640, dir, ns, - &aa_fs_profile_remove); + dent = aafs_create_file(".remove", 0640, dir, + &ns->unconfined->label.count, + &aa_fs_profile_remove); if (IS_ERR(dent)) return PTR_ERR(dent); - aa_get_ns(ns); ns_subremove(ns) = dent; /* use create_dentry so we can supply private data */ - dent = aafs_create("namespaces", S_IFDIR | 0755, dir, ns, NULL, NULL, - &ns_dir_inode_operations); + dent = aafs_create("namespaces", S_IFDIR | 0755, dir, + &ns->unconfined->label.count, + NULL, NULL, &ns_dir_inode_operations); if (IS_ERR(dent)) return PTR_ERR(dent); - aa_get_ns(ns); ns_subns_dir(ns) = dent; return 0; diff --git a/security/apparmor/file.c b/security/apparmor/file.c index c758204028780f..7de23e85cd5d01 100644 --- a/security/apparmor/file.c +++ b/security/apparmor/file.c @@ -154,8 +154,12 @@ static int path_name(const char *op, const struct cred *subj_cred, const char *info = NULL; int error; - error = aa_path_name(path, flags, buffer, name, &info, - labels_profile(label)->disconnected); + /* don't reaudit files closed during inheritance */ + if (unlikely(path->dentry == aa_null.dentry)) + error = -EACCES; + else + error = aa_path_name(path, flags, buffer, name, &info, + labels_profile(label)->disconnected); if (error) { fn_for_each_confined(label, profile, aa_audit_file(subj_cred, @@ -578,6 +582,9 @@ static bool __unix_needs_revalidation(struct file *file, struct aa_label *label, return false; if (request & NET_PEER_MASK) return false; + /* sock and sock->sk can be NULL for sockets being set up or torn down */ + if (!sock || !sock->sk) + return false; if (sock->sk->sk_family == PF_UNIX) { struct aa_sk_ctx *ctx = aa_sock(sock->sk); @@ -613,6 +620,10 @@ int aa_file_perm(const char *op, const struct cred *subj_cred, AA_BUG(!label); AA_BUG(!file); + /* don't reaudit files closed during inheritance */ + if (unlikely(file->f_path.dentry == aa_null.dentry)) + return -EACCES; + fctx = file_ctx(file); rcu_read_lock(); diff --git a/security/apparmor/include/label.h b/security/apparmor/include/label.h index c0812dbc1b5b01..335f21930702aa 100644 --- a/security/apparmor/include/label.h +++ b/security/apparmor/include/label.h @@ -102,7 +102,7 @@ enum label_flags { struct aa_label; struct aa_proxy { - struct kref count; + struct aa_common_ref count; struct aa_label __rcu *label; }; @@ -125,7 +125,7 @@ struct label_it { * vec: vector of profiles comprising the compound label */ struct aa_label { - struct kref count; + struct aa_common_ref count; struct rb_node node; struct rcu_head rcu; struct aa_proxy *proxy; @@ -357,7 +357,7 @@ int aa_label_match(struct aa_profile *profile, struct aa_ruleset *rules, */ static inline struct aa_label *__aa_get_label(struct aa_label *l) { - if (l && kref_get_unless_zero(&l->count)) + if (l && kref_get_unless_zero(&l->count.count)) return l; return NULL; @@ -366,7 +366,7 @@ static inline struct aa_label *__aa_get_label(struct aa_label *l) static inline struct aa_label *aa_get_label(struct aa_label *l) { if (l) - kref_get(&(l->count)); + kref_get(&(l->count.count)); return l; } @@ -386,7 +386,7 @@ static inline struct aa_label *aa_get_label_rcu(struct aa_label __rcu **l) rcu_read_lock(); do { c = rcu_dereference(*l); - } while (c && !kref_get_unless_zero(&c->count)); + } while (c && !kref_get_unless_zero(&c->count.count)); rcu_read_unlock(); return c; @@ -426,7 +426,7 @@ static inline struct aa_label *aa_get_newest_label(struct aa_label *l) static inline void aa_put_label(struct aa_label *l) { if (l) - kref_put(&l->count, aa_label_kref); + kref_put(&l->count.count, aa_label_kref); } /* wrapper fn to indicate semantics of the check */ @@ -443,7 +443,7 @@ void aa_proxy_kref(struct kref *kref); static inline struct aa_proxy *aa_get_proxy(struct aa_proxy *proxy) { if (proxy) - kref_get(&(proxy->count)); + kref_get(&(proxy->count.count)); return proxy; } @@ -451,7 +451,7 @@ static inline struct aa_proxy *aa_get_proxy(struct aa_proxy *proxy) static inline void aa_put_proxy(struct aa_proxy *proxy) { if (proxy) - kref_put(&proxy->count, aa_proxy_kref); + kref_put(&proxy->count.count, aa_proxy_kref); } void __aa_proxy_redirect(struct aa_label *orig, struct aa_label *new); diff --git a/security/apparmor/include/lib.h b/security/apparmor/include/lib.h index 444197075fd6b5..26df19c1df4f8e 100644 --- a/security/apparmor/include/lib.h +++ b/security/apparmor/include/lib.h @@ -85,6 +85,18 @@ void aa_info_message(const char *str); /* Security blob offsets */ extern struct lsm_blob_sizes apparmor_blob_sizes; +enum reftype { + REF_NS, + REF_PROXY, + REF_RAWDATA, +}; + +/* common reference count used by data the shows up in aafs */ +struct aa_common_ref { + struct kref count; + enum reftype reftype; +}; + /** * aa_strneq - compare null terminated @str to a non null terminated substring * @str: a null terminated string diff --git a/security/apparmor/include/match.h b/security/apparmor/include/match.h index 1fbe82f5021b16..7accb1c39849aa 100644 --- a/security/apparmor/include/match.h +++ b/security/apparmor/include/match.h @@ -104,16 +104,18 @@ struct aa_dfa { struct table_header *tables[YYTD_ID_TSIZE]; }; -#define byte_to_byte(X) (X) - #define UNPACK_ARRAY(TABLE, BLOB, LEN, TTYPE, BTYPE, NTOHX) \ do { \ typeof(LEN) __i; \ TTYPE *__t = (TTYPE *) TABLE; \ BTYPE *__b = (BTYPE *) BLOB; \ - for (__i = 0; __i < LEN; __i++) { \ - __t[__i] = NTOHX(__b[__i]); \ - } \ + BUILD_BUG_ON(sizeof(TTYPE) != sizeof(BTYPE)); \ + if (IS_ENABLED(CONFIG_CPU_BIG_ENDIAN)) \ + memcpy(__t, __b, (LEN) * sizeof(BTYPE)); \ + else /* copy & convert from big-endian */ \ + for (__i = 0; __i < LEN; __i++) { \ + __t[__i] = NTOHX(&__b[__i]); \ + } \ } while (0) static inline size_t table_size(size_t len, size_t el_size) @@ -183,6 +185,7 @@ static inline void aa_put_dfa(struct aa_dfa *dfa) #define MATCH_FLAG_DIFF_ENCODE 0x80000000 #define MARK_DIFF_ENCODE 0x40000000 #define MATCH_FLAG_OOB_TRANSITION 0x20000000 +#define MARK_DIFF_ENCODE_VERIFIED 0x10000000 #define MATCH_FLAGS_MASK 0xff000000 #define MATCH_FLAGS_VALID (MATCH_FLAG_DIFF_ENCODE | MATCH_FLAG_OOB_TRANSITION) #define MATCH_FLAGS_INVALID (MATCH_FLAGS_MASK & ~MATCH_FLAGS_VALID) diff --git a/security/apparmor/include/policy.h b/security/apparmor/include/policy.h index 4c50875c9d13e8..bf105ae9019d3d 100644 --- a/security/apparmor/include/policy.h +++ b/security/apparmor/include/policy.h @@ -355,7 +355,7 @@ static inline bool profile_mediates_safe(struct aa_profile *profile, static inline struct aa_profile *aa_get_profile(struct aa_profile *p) { if (p) - kref_get(&(p->label.count)); + kref_get(&(p->label.count.count)); return p; } @@ -369,7 +369,7 @@ static inline struct aa_profile *aa_get_profile(struct aa_profile *p) */ static inline struct aa_profile *aa_get_profile_not0(struct aa_profile *p) { - if (p && kref_get_unless_zero(&p->label.count)) + if (p && kref_get_unless_zero(&p->label.count.count)) return p; return NULL; @@ -389,7 +389,7 @@ static inline struct aa_profile *aa_get_profile_rcu(struct aa_profile __rcu **p) rcu_read_lock(); do { c = rcu_dereference(*p); - } while (c && !kref_get_unless_zero(&c->label.count)); + } while (c && !kref_get_unless_zero(&c->label.count.count)); rcu_read_unlock(); return c; @@ -402,7 +402,7 @@ static inline struct aa_profile *aa_get_profile_rcu(struct aa_profile __rcu **p) static inline void aa_put_profile(struct aa_profile *p) { if (p) - kref_put(&p->label.count, aa_label_kref); + kref_put(&p->label.count.count, aa_label_kref); } static inline int AUDIT_MODE(struct aa_profile *profile) @@ -419,7 +419,7 @@ bool aa_policy_admin_capable(const struct cred *subj_cred, struct aa_label *label, struct aa_ns *ns); int aa_may_manage_policy(const struct cred *subj_cred, struct aa_label *label, struct aa_ns *ns, - u32 mask); + const struct cred *ocred, u32 mask); bool aa_current_policy_view_capable(struct aa_ns *ns); bool aa_current_policy_admin_capable(struct aa_ns *ns); diff --git a/security/apparmor/include/policy_ns.h b/security/apparmor/include/policy_ns.h index d646070fd966be..cc6e8415181209 100644 --- a/security/apparmor/include/policy_ns.h +++ b/security/apparmor/include/policy_ns.h @@ -18,6 +18,8 @@ #include "label.h" #include "policy.h" +/* Match max depth of user namespaces */ +#define MAX_NS_DEPTH 32 /* struct aa_ns_acct - accounting of profiles in namespace * @max_size: maximum space allowed for all profiles in namespace diff --git a/security/apparmor/include/policy_unpack.h b/security/apparmor/include/policy_unpack.h index a6f4611ee50cf9..e5a95dc4da1f76 100644 --- a/security/apparmor/include/policy_unpack.h +++ b/security/apparmor/include/policy_unpack.h @@ -87,17 +87,29 @@ struct aa_ext { u32 version; }; -/* - * struct aa_loaddata - buffer of policy raw_data set +/* struct aa_loaddata - buffer of policy raw_data set + * @count: inode/filesystem refcount - use aa_get_i_loaddata() + * @pcount: profile refcount - use aa_get_profile_loaddata() + * @list: list the loaddata is on + * @work: used to do a delayed cleanup + * @dents: refs to dents created in aafs + * @ns: the namespace this loaddata was loaded into + * @name: + * @size: the size of the data that was loaded + * @compressed_size: the size of the data when it is compressed + * @revision: unique revision count that this data was loaded as + * @abi: the abi number the loaddata uses + * @hash: a hash of the loaddata, used to help dedup data * - * there is no loaddata ref for being on ns list, nor a ref from - * d_inode(@dentry) when grab a ref from these, @ns->lock must be held - * && __aa_get_loaddata() needs to be used, and the return value - * checked, if NULL the loaddata is already being reaped and should be - * considered dead. + * There is no loaddata ref for being on ns->rawdata_list, so + * @ns->lock must be held when walking the list. Dentries and + * inode opens hold refs on @count; profiles hold refs on @pcount. + * When the last @pcount drops, do_ploaddata_rmfs() removes the + * fs entries and drops the associated @count ref. */ struct aa_loaddata { - struct kref count; + struct aa_common_ref count; + struct kref pcount; struct list_head list; struct work_struct work; struct dentry *dents[AAFS_LOADDATA_NDENTS]; @@ -119,50 +131,53 @@ struct aa_loaddata { int aa_unpack(struct aa_loaddata *udata, struct list_head *lh, const char **ns); /** - * __aa_get_loaddata - get a reference count to uncounted data reference + * aa_get_loaddata - get a reference count from a counted data reference * @data: reference to get a count on * - * Returns: pointer to reference OR NULL if race is lost and reference is - * being repeated. - * Requires: @data->ns->lock held, and the return code MUST be checked - * - * Use only from inode->i_private and @data->list found references + * Returns: pointer to reference + * Requires: @data to have a valid reference count on it. It is a bug + * if the race to reap can be encountered when it is used. */ static inline struct aa_loaddata * -__aa_get_loaddata(struct aa_loaddata *data) +aa_get_i_loaddata(struct aa_loaddata *data) { - if (data && kref_get_unless_zero(&(data->count))) - return data; - return NULL; + if (data) + kref_get(&(data->count.count)); + return data; } + /** - * aa_get_loaddata - get a reference count from a counted data reference + * aa_get_profile_loaddata - get a profile reference count on loaddata * @data: reference to get a count on * - * Returns: point to reference - * Requires: @data to have a valid reference count on it. It is a bug - * if the race to reap can be encountered when it is used. + * Returns: pointer to reference + * Requires: @data to have a valid reference count on it. */ static inline struct aa_loaddata * -aa_get_loaddata(struct aa_loaddata *data) +aa_get_profile_loaddata(struct aa_loaddata *data) { - struct aa_loaddata *tmp = __aa_get_loaddata(data); - - AA_BUG(data && !tmp); - - return tmp; + if (data) + kref_get(&(data->pcount)); + return data; } void __aa_loaddata_update(struct aa_loaddata *data, long revision); bool aa_rawdata_eq(struct aa_loaddata *l, struct aa_loaddata *r); void aa_loaddata_kref(struct kref *kref); +void aa_ploaddata_kref(struct kref *kref); struct aa_loaddata *aa_loaddata_alloc(size_t size); -static inline void aa_put_loaddata(struct aa_loaddata *data) +static inline void aa_put_i_loaddata(struct aa_loaddata *data) +{ + if (data) + kref_put(&data->count.count, aa_loaddata_kref); +} + +static inline void aa_put_profile_loaddata(struct aa_loaddata *data) { if (data) - kref_put(&data->count, aa_loaddata_kref); + kref_put(&data->pcount, aa_ploaddata_kref); } #if IS_ENABLED(CONFIG_KUNIT) diff --git a/security/apparmor/label.c b/security/apparmor/label.c index 913678f199c358..3bec1e33815e2f 100644 --- a/security/apparmor/label.c +++ b/security/apparmor/label.c @@ -52,7 +52,8 @@ static void free_proxy(struct aa_proxy *proxy) void aa_proxy_kref(struct kref *kref) { - struct aa_proxy *proxy = container_of(kref, struct aa_proxy, count); + struct aa_proxy *proxy = container_of(kref, struct aa_proxy, + count.count); free_proxy(proxy); } @@ -63,7 +64,8 @@ struct aa_proxy *aa_alloc_proxy(struct aa_label *label, gfp_t gfp) new = kzalloc(sizeof(struct aa_proxy), gfp); if (new) { - kref_init(&new->count); + kref_init(&new->count.count); + new->count.reftype = REF_PROXY; rcu_assign_pointer(new->label, aa_get_label(label)); } return new; @@ -375,7 +377,8 @@ static void label_free_rcu(struct rcu_head *head) void aa_label_kref(struct kref *kref) { - struct aa_label *label = container_of(kref, struct aa_label, count); + struct aa_label *label = container_of(kref, struct aa_label, + count.count); struct aa_ns *ns = labels_ns(label); if (!ns) { @@ -412,7 +415,8 @@ bool aa_label_init(struct aa_label *label, int size, gfp_t gfp) label->size = size; /* doesn't include null */ label->vec[size] = NULL; /* null terminate */ - kref_init(&label->count); + kref_init(&label->count.count); + label->count.reftype = REF_NS; /* for aafs purposes */ RB_CLEAR_NODE(&label->node); return true; @@ -1278,7 +1282,7 @@ static inline aa_state_t match_component(struct aa_profile *profile, * @request: permissions to request * @perms: perms struct to set * - * Returns: 0 on success else ERROR + * Returns: state match stopped at or DFA_NOMATCH if aborted early * * For the label A//&B//&C this does the perm match for A//&B//&C * @perms should be preinitialized with allperms OR a previous permission @@ -1305,7 +1309,7 @@ static int label_compound_match(struct aa_profile *profile, /* no component visible */ *perms = allperms; - return 0; + return state; next: label_for_each_cont(i, label, tp) { @@ -1317,15 +1321,11 @@ static int label_compound_match(struct aa_profile *profile, goto fail; } *perms = *aa_lookup_perms(rules->policy, state); - aa_apply_modes_to_perms(profile, perms); - if ((perms->allow & request) != request) - return -EACCES; - - return 0; + return state; fail: *perms = nullperms; - return state; + return DFA_NOMATCH; } /** @@ -1338,7 +1338,7 @@ static int label_compound_match(struct aa_profile *profile, * @request: permissions to request * @perms: an initialized perms struct to add accumulation to * - * Returns: 0 on success else ERROR + * Returns: the state the match finished in, may be the none matching state * * For the label A//&B//&C this does the perm match for each of A and B and C * @perms should be preinitialized with allperms OR a previous permission @@ -1366,11 +1366,10 @@ static int label_components_match(struct aa_profile *profile, } /* no subcomponents visible - no change in perms */ - return 0; + return state; next: tmp = *aa_lookup_perms(rules->policy, state); - aa_apply_modes_to_perms(profile, &tmp); aa_perms_accum(perms, &tmp); label_for_each_cont(i, label, tp) { if (!aa_ns_visible(profile->ns, tp->ns, subns)) @@ -1379,18 +1378,17 @@ static int label_components_match(struct aa_profile *profile, if (!state) goto fail; tmp = *aa_lookup_perms(rules->policy, state); - aa_apply_modes_to_perms(profile, &tmp); aa_perms_accum(perms, &tmp); } if ((perms->allow & request) != request) - return -EACCES; + return DFA_NOMATCH; - return 0; + return state; fail: *perms = nullperms; - return -EACCES; + return DFA_NOMATCH; } /** @@ -1409,11 +1407,12 @@ int aa_label_match(struct aa_profile *profile, struct aa_ruleset *rules, struct aa_label *label, aa_state_t state, bool subns, u32 request, struct aa_perms *perms) { - int error = label_compound_match(profile, rules, label, state, subns, - request, perms); - if (!error) - return error; + aa_state_t tmp = label_compound_match(profile, rules, label, state, subns, + request, perms); + if ((perms->allow & request) == request) + return tmp; + /* failed compound_match try component matches */ *perms = allperms; return label_components_match(profile, rules, label, state, subns, request, perms); diff --git a/security/apparmor/lsm.c b/security/apparmor/lsm.c index a87cd60ed20699..9175fd677ef3db 100644 --- a/security/apparmor/lsm.c +++ b/security/apparmor/lsm.c @@ -520,33 +520,26 @@ static void apparmor_file_free_security(struct file *file) aa_put_label(rcu_access_pointer(ctx->label)); } -static int common_file_perm(const char *op, struct file *file, u32 mask, - bool in_atomic) +static int common_file_perm(const char *op, struct file *file, u32 mask) { struct aa_label *label; int error = 0; - bool needput; - - /* don't reaudit files closed during inheritance */ - if (unlikely(file->f_path.dentry == aa_null.dentry)) - return -EACCES; - label = __begin_current_label_crit_section(&needput); - error = aa_file_perm(op, current_cred(), label, file, mask, in_atomic); - __end_current_label_crit_section(label, needput); + label = begin_current_label_crit_section(); + error = aa_file_perm(op, current_cred(), label, file, mask, false); + end_current_label_crit_section(label); return error; } static int apparmor_file_receive(struct file *file) { - return common_file_perm(OP_FRECEIVE, file, aa_map_file_to_perms(file), - false); + return common_file_perm(OP_FRECEIVE, file, aa_map_file_to_perms(file)); } static int apparmor_file_permission(struct file *file, int mask) { - return common_file_perm(OP_FPERM, file, mask, false); + return common_file_perm(OP_FPERM, file, mask); } static int apparmor_file_lock(struct file *file, unsigned int cmd) @@ -556,11 +549,11 @@ static int apparmor_file_lock(struct file *file, unsigned int cmd) if (cmd == F_WRLCK) mask |= MAY_WRITE; - return common_file_perm(OP_FLOCK, file, mask, false); + return common_file_perm(OP_FLOCK, file, mask); } static int common_mmap(const char *op, struct file *file, unsigned long prot, - unsigned long flags, bool in_atomic) + unsigned long flags) { int mask = 0; @@ -578,21 +571,20 @@ static int common_mmap(const char *op, struct file *file, unsigned long prot, if (prot & PROT_EXEC) mask |= AA_EXEC_MMAP; - return common_file_perm(op, file, mask, in_atomic); + return common_file_perm(op, file, mask); } static int apparmor_mmap_file(struct file *file, unsigned long reqprot, unsigned long prot, unsigned long flags) { - return common_mmap(OP_FMMAP, file, prot, flags, GFP_ATOMIC); + return common_mmap(OP_FMMAP, file, prot, flags); } static int apparmor_file_mprotect(struct vm_area_struct *vma, unsigned long reqprot, unsigned long prot) { return common_mmap(OP_FMPROT, vma->vm_file, prot, - !(vma->vm_flags & VM_SHARED) ? MAP_PRIVATE : 0, - false); + !(vma->vm_flags & VM_SHARED) ? MAP_PRIVATE : 0); } #ifdef CONFIG_IO_URING @@ -2145,7 +2137,8 @@ char *aa_get_buffer(bool in_atomic) if (!list_empty(&cache->head)) { aa_buf = list_first_entry(&cache->head, union aa_buffer, list); list_del(&aa_buf->list); - cache->hold--; + if (cache->hold) + cache->hold--; cache->count--; put_cpu_ptr(&aa_local_buffers); return &aa_buf->buffer[0]; diff --git a/security/apparmor/match.c b/security/apparmor/match.c index c5a91600842a16..0de249725efbf8 100644 --- a/security/apparmor/match.c +++ b/security/apparmor/match.c @@ -15,6 +15,7 @@ #include #include #include +#include #include "include/lib.h" #include "include/match.h" @@ -42,11 +43,11 @@ static struct table_header *unpack_table(char *blob, size_t bsize) /* loaded td_id's start at 1, subtract 1 now to avoid doing * it every time we use td_id as an index */ - th.td_id = be16_to_cpu(*(__be16 *) (blob)) - 1; + th.td_id = get_unaligned_be16(blob) - 1; if (th.td_id > YYTD_ID_MAX) goto out; - th.td_flags = be16_to_cpu(*(__be16 *) (blob + 2)); - th.td_lolen = be32_to_cpu(*(__be32 *) (blob + 8)); + th.td_flags = get_unaligned_be16(blob + 2); + th.td_lolen = get_unaligned_be32(blob + 8); blob += sizeof(struct table_header); if (!(th.td_flags == YYTD_DATA16 || th.td_flags == YYTD_DATA32 || @@ -66,14 +67,13 @@ static struct table_header *unpack_table(char *blob, size_t bsize) table->td_flags = th.td_flags; table->td_lolen = th.td_lolen; if (th.td_flags == YYTD_DATA8) - UNPACK_ARRAY(table->td_data, blob, th.td_lolen, - u8, u8, byte_to_byte); + memcpy(table->td_data, blob, th.td_lolen); else if (th.td_flags == YYTD_DATA16) UNPACK_ARRAY(table->td_data, blob, th.td_lolen, - u16, __be16, be16_to_cpu); + u16, __be16, get_unaligned_be16); else if (th.td_flags == YYTD_DATA32) UNPACK_ARRAY(table->td_data, blob, th.td_lolen, - u32, __be32, be32_to_cpu); + u32, __be32, get_unaligned_be32); else goto fail; /* if table was vmalloced make sure the page tables are synced @@ -160,9 +160,10 @@ static int verify_dfa(struct aa_dfa *dfa) if (state_count == 0) goto out; for (i = 0; i < state_count; i++) { - if (!(BASE_TABLE(dfa)[i] & MATCH_FLAG_DIFF_ENCODE) && - (DEFAULT_TABLE(dfa)[i] >= state_count)) + if (DEFAULT_TABLE(dfa)[i] >= state_count) { + pr_err("AppArmor DFA default state out of bounds"); goto out; + } if (BASE_TABLE(dfa)[i] & MATCH_FLAGS_INVALID) { pr_err("AppArmor DFA state with invalid match flags"); goto out; @@ -201,16 +202,31 @@ static int verify_dfa(struct aa_dfa *dfa) size_t j, k; for (j = i; - (BASE_TABLE(dfa)[j] & MATCH_FLAG_DIFF_ENCODE) && - !(BASE_TABLE(dfa)[j] & MARK_DIFF_ENCODE); + ((BASE_TABLE(dfa)[j] & MATCH_FLAG_DIFF_ENCODE) && + !(BASE_TABLE(dfa)[j] & MARK_DIFF_ENCODE_VERIFIED)); j = k) { + if (BASE_TABLE(dfa)[j] & MARK_DIFF_ENCODE) + /* loop in current chain */ + goto out; k = DEFAULT_TABLE(dfa)[j]; if (j == k) + /* self loop */ goto out; - if (k < j) - break; /* already verified */ BASE_TABLE(dfa)[j] |= MARK_DIFF_ENCODE; } + /* move mark to verified */ + for (j = i; + (BASE_TABLE(dfa)[j] & MATCH_FLAG_DIFF_ENCODE); + j = k) { + k = DEFAULT_TABLE(dfa)[j]; + if (j < i) + /* jumps to state/chain that has been + * verified + */ + break; + BASE_TABLE(dfa)[j] &= ~MARK_DIFF_ENCODE; + BASE_TABLE(dfa)[j] |= MARK_DIFF_ENCODE_VERIFIED; + } } error = 0; @@ -313,14 +329,14 @@ struct aa_dfa *aa_dfa_unpack(void *blob, size_t size, int flags) if (size < sizeof(struct table_set_header)) goto fail; - if (ntohl(*(__be32 *) data) != YYTH_MAGIC) + if (get_unaligned_be32(data) != YYTH_MAGIC) goto fail; - hsize = ntohl(*(__be32 *) (data + 4)); + hsize = get_unaligned_be32(data + 4); if (size < hsize) goto fail; - dfa->flags = ntohs(*(__be16 *) (data + 12)); + dfa->flags = get_unaligned_be16(data + 12); if (dfa->flags & ~(YYTH_FLAGS)) goto fail; @@ -329,7 +345,7 @@ struct aa_dfa *aa_dfa_unpack(void *blob, size_t size, int flags) * if (dfa->flags & YYTH_FLAGS_OOB_TRANS) { * if (hsize < 16 + 4) * goto fail; - * dfa->max_oob = ntol(*(__be32 *) (data + 16)); + * dfa->max_oob = get_unaligned_be32(data + 16); * if (dfa->max <= MAX_OOB_SUPPORTED) { * pr_err("AppArmor DFA OOB greater than supported\n"); * goto fail; @@ -463,13 +479,18 @@ aa_state_t aa_dfa_match_len(struct aa_dfa *dfa, aa_state_t start, if (dfa->tables[YYTD_ID_EC]) { /* Equivalence class table defined */ u8 *equiv = EQUIV_TABLE(dfa); - for (; len; len--) - match_char(state, def, base, next, check, - equiv[(u8) *str++]); + for (; len; len--) { + u8 c = equiv[(u8) *str]; + + match_char(state, def, base, next, check, c); + str++; + } } else { /* default is direct to next state */ - for (; len; len--) - match_char(state, def, base, next, check, (u8) *str++); + for (; len; len--) { + match_char(state, def, base, next, check, (u8) *str); + str++; + } } return state; @@ -503,13 +524,18 @@ aa_state_t aa_dfa_match(struct aa_dfa *dfa, aa_state_t start, const char *str) /* Equivalence class table defined */ u8 *equiv = EQUIV_TABLE(dfa); /* default is direct to next state */ - while (*str) - match_char(state, def, base, next, check, - equiv[(u8) *str++]); + while (*str) { + u8 c = equiv[(u8) *str]; + + match_char(state, def, base, next, check, c); + str++; + } } else { /* default is direct to next state */ - while (*str) - match_char(state, def, base, next, check, (u8) *str++); + while (*str) { + match_char(state, def, base, next, check, (u8) *str); + str++; + } } return state; diff --git a/security/apparmor/net.c b/security/apparmor/net.c index 45cf25605c345b..44c04102062f3d 100644 --- a/security/apparmor/net.c +++ b/security/apparmor/net.c @@ -326,8 +326,10 @@ int aa_sock_file_perm(const struct cred *subj_cred, struct aa_label *label, struct socket *sock = (struct socket *) file->private_data; AA_BUG(!label); - AA_BUG(!sock); - AA_BUG(!sock->sk); + + /* sock && sock->sk can be NULL for sockets being set up or torn down */ + if (!sock || !sock->sk) + return 0; if (sock->sk->sk_family == PF_UNIX) return aa_unix_file_perm(subj_cred, label, op, request, file); diff --git a/security/apparmor/policy.c b/security/apparmor/policy.c index 50d5345ff5cbab..b92db1b2f26e20 100644 --- a/security/apparmor/policy.c +++ b/security/apparmor/policy.c @@ -183,19 +183,43 @@ static void __list_remove_profile(struct aa_profile *profile) } /** - * __remove_profile - remove old profile, and children - * @profile: profile to be replaced (NOT NULL) + * __remove_profile - remove profile, and children + * @profile: profile to be removed (NOT NULL) * * Requires: namespace list lock be held, or list not be shared */ static void __remove_profile(struct aa_profile *profile) { + struct aa_profile *curr, *to_remove; + AA_BUG(!profile); AA_BUG(!profile->ns); AA_BUG(!mutex_is_locked(&profile->ns->lock)); /* release any children lists first */ - __aa_profile_list_release(&profile->base.profiles); + if (!list_empty(&profile->base.profiles)) { + curr = list_first_entry(&profile->base.profiles, struct aa_profile, base.list); + + while (curr != profile) { + + while (!list_empty(&curr->base.profiles)) + curr = list_first_entry(&curr->base.profiles, + struct aa_profile, base.list); + + to_remove = curr; + if (!list_is_last(&to_remove->base.list, + &aa_deref_parent(curr)->base.profiles)) + curr = list_next_entry(to_remove, base.list); + else + curr = aa_deref_parent(curr); + + /* released by free_profile */ + aa_label_remove(&to_remove->label); + __aafs_profile_rmdir(to_remove); + __list_remove_profile(to_remove); + } + } + /* released by free_profile */ aa_label_remove(&profile->label); __aafs_profile_rmdir(profile); @@ -312,7 +336,7 @@ void aa_free_profile(struct aa_profile *profile) } kfree_sensitive(profile->hash); - aa_put_loaddata(profile->rawdata); + aa_put_profile_loaddata(profile->rawdata); aa_label_destroy(&profile->label); kfree_sensitive(profile); @@ -901,17 +925,44 @@ bool aa_current_policy_admin_capable(struct aa_ns *ns) return res; } +static bool is_subset_of_obj_privilege(const struct cred *cred, + struct aa_label *label, + const struct cred *ocred) +{ + if (cred == ocred) + return true; + + if (!aa_label_is_subset(label, cred_label(ocred))) + return false; + /* don't allow crossing userns for now */ + if (cred->user_ns != ocred->user_ns) + return false; + if (!cap_issubset(cred->cap_inheritable, ocred->cap_inheritable)) + return false; + if (!cap_issubset(cred->cap_permitted, ocred->cap_permitted)) + return false; + if (!cap_issubset(cred->cap_effective, ocred->cap_effective)) + return false; + if (!cap_issubset(cred->cap_bset, ocred->cap_bset)) + return false; + if (!cap_issubset(cred->cap_ambient, ocred->cap_ambient)) + return false; + return true; +} + + /** * aa_may_manage_policy - can the current task manage policy * @subj_cred: subjects cred * @label: label to check if it can manage policy * @ns: namespace being managed by @label (may be NULL if @label's ns) + * @ocred: object cred if request is coming from an open object * @mask: contains the policy manipulation operation being done * * Returns: 0 if the task is allowed to manipulate policy else error */ int aa_may_manage_policy(const struct cred *subj_cred, struct aa_label *label, - struct aa_ns *ns, u32 mask) + struct aa_ns *ns, const struct cred *ocred, u32 mask) { const char *op; @@ -927,6 +978,11 @@ int aa_may_manage_policy(const struct cred *subj_cred, struct aa_label *label, return audit_policy(label, op, NULL, NULL, "policy_locked", -EACCES); + if (ocred && !is_subset_of_obj_privilege(subj_cred, label, ocred)) + return audit_policy(label, op, NULL, NULL, + "not privileged for target profile", + -EACCES); + if (!aa_policy_admin_capable(subj_cred, label, ns)) return audit_policy(label, op, NULL, NULL, "not policy admin", -EACCES); @@ -1098,7 +1154,7 @@ ssize_t aa_replace_profiles(struct aa_ns *policy_ns, struct aa_label *label, LIST_HEAD(lh); op = mask & AA_MAY_REPLACE_POLICY ? OP_PROF_REPL : OP_PROF_LOAD; - aa_get_loaddata(udata); + aa_get_profile_loaddata(udata); /* released below */ error = aa_unpack(udata, &lh, &ns_name); if (error) @@ -1125,6 +1181,7 @@ ssize_t aa_replace_profiles(struct aa_ns *policy_ns, struct aa_label *label, goto fail; } ns_name = ent->ns_name; + ent->ns_name = NULL; } else count++; } @@ -1149,10 +1206,10 @@ ssize_t aa_replace_profiles(struct aa_ns *policy_ns, struct aa_label *label, if (aa_rawdata_eq(rawdata_ent, udata)) { struct aa_loaddata *tmp; - tmp = __aa_get_loaddata(rawdata_ent); + tmp = aa_get_profile_loaddata(rawdata_ent); /* check we didn't fail the race */ if (tmp) { - aa_put_loaddata(udata); + aa_put_profile_loaddata(udata); udata = tmp; break; } @@ -1165,7 +1222,7 @@ ssize_t aa_replace_profiles(struct aa_ns *policy_ns, struct aa_label *label, struct aa_profile *p; if (aa_g_export_binary) - ent->new->rawdata = aa_get_loaddata(udata); + ent->new->rawdata = aa_get_profile_loaddata(udata); error = __lookup_replace(ns, ent->new->base.hname, !(mask & AA_MAY_REPLACE_POLICY), &ent->old, &info); @@ -1298,7 +1355,7 @@ ssize_t aa_replace_profiles(struct aa_ns *policy_ns, struct aa_label *label, out: aa_put_ns(ns); - aa_put_loaddata(udata); + aa_put_profile_loaddata(udata); kfree(ns_name); if (error) diff --git a/security/apparmor/policy_ns.c b/security/apparmor/policy_ns.c index 64783ca3b0f2a2..ff49a31ac27442 100644 --- a/security/apparmor/policy_ns.c +++ b/security/apparmor/policy_ns.c @@ -223,6 +223,8 @@ static struct aa_ns *__aa_create_ns(struct aa_ns *parent, const char *name, AA_BUG(!name); AA_BUG(!mutex_is_locked(&parent->lock)); + if (parent->level > MAX_NS_DEPTH) + return ERR_PTR(-ENOSPC); ns = alloc_ns(parent->base.hname, name); if (!ns) return ERR_PTR(-ENOMEM); diff --git a/security/apparmor/policy_unpack.c b/security/apparmor/policy_unpack.c index 7523971e37d9c0..b0e18dd8d512bf 100644 --- a/security/apparmor/policy_unpack.c +++ b/security/apparmor/policy_unpack.c @@ -109,34 +109,48 @@ bool aa_rawdata_eq(struct aa_loaddata *l, struct aa_loaddata *r) return memcmp(l->data, r->data, r->compressed_size ?: r->size) == 0; } +static void do_loaddata_free(struct aa_loaddata *d) +{ + kfree_sensitive(d->hash); + kfree_sensitive(d->name); + kvfree(d->data); + kfree_sensitive(d); +} + +void aa_loaddata_kref(struct kref *kref) +{ + struct aa_loaddata *d = container_of(kref, struct aa_loaddata, + count.count); + + do_loaddata_free(d); +} + /* * need to take the ns mutex lock which is NOT safe most places that * put_loaddata is called, so we have to delay freeing it */ -static void do_loaddata_free(struct work_struct *work) +static void do_ploaddata_rmfs(struct work_struct *work) { struct aa_loaddata *d = container_of(work, struct aa_loaddata, work); struct aa_ns *ns = aa_get_ns(d->ns); if (ns) { mutex_lock_nested(&ns->lock, ns->level); + /* remove fs ref to loaddata */ __aa_fs_remove_rawdata(d); mutex_unlock(&ns->lock); aa_put_ns(ns); } - - kfree_sensitive(d->hash); - kfree_sensitive(d->name); - kvfree(d->data); - kfree_sensitive(d); + /* called by dropping last pcount, so drop its associated icount */ + aa_put_i_loaddata(d); } -void aa_loaddata_kref(struct kref *kref) +void aa_ploaddata_kref(struct kref *kref) { - struct aa_loaddata *d = container_of(kref, struct aa_loaddata, count); + struct aa_loaddata *d = container_of(kref, struct aa_loaddata, pcount); if (d) { - INIT_WORK(&d->work, do_loaddata_free); + INIT_WORK(&d->work, do_ploaddata_rmfs); schedule_work(&d->work); } } @@ -153,7 +167,9 @@ struct aa_loaddata *aa_loaddata_alloc(size_t size) kfree(d); return ERR_PTR(-ENOMEM); } - kref_init(&d->count); + kref_init(&d->count.count); + d->count.reftype = REF_RAWDATA; + kref_init(&d->pcount); INIT_LIST_HEAD(&d->list); return d; @@ -687,8 +703,10 @@ static ssize_t unpack_perms_table(struct aa_ext *e, struct aa_perms **perms) if (!aa_unpack_array(e, NULL, &size)) goto fail_reset; *perms = kcalloc(size, sizeof(struct aa_perms), GFP_KERNEL); - if (!*perms) - goto fail_reset; + if (!*perms) { + e->pos = pos; + return -ENOMEM; + } for (i = 0; i < size; i++) { if (!unpack_perm(e, version, &(*perms)[i])) goto fail; @@ -768,7 +786,17 @@ static int unpack_pdb(struct aa_ext *e, struct aa_policydb **policy, if (!aa_unpack_u32(e, &pdb->start[AA_CLASS_FILE], "dfa_start")) { /* default start state for xmatch and file dfa */ pdb->start[AA_CLASS_FILE] = DFA_START; - } /* setup class index */ + } + + size_t state_count = pdb->dfa->tables[YYTD_ID_BASE]->td_lolen; + + if (pdb->start[0] >= state_count || + pdb->start[AA_CLASS_FILE] >= state_count) { + *info = "invalid dfa start state"; + goto fail; + } + + /* setup class index */ for (i = AA_CLASS_FILE + 1; i <= AA_CLASS_LAST; i++) { pdb->start[i] = aa_dfa_next(pdb->dfa, pdb->start[0], i); @@ -1165,7 +1193,6 @@ static int verify_header(struct aa_ext *e, int required, const char **ns) { int error = -EPROTONOSUPPORT; const char *name = NULL; - *ns = NULL; /* get the interface version */ if (!aa_unpack_u32(e, &e->version, "version")) { diff --git a/security/apparmor/resource.c b/security/apparmor/resource.c index 8e80db3ae21c09..64212b39ba4bbc 100644 --- a/security/apparmor/resource.c +++ b/security/apparmor/resource.c @@ -196,6 +196,11 @@ void __aa_transition_rlimits(struct aa_label *old_l, struct aa_label *new_l) rules->rlimits.limits[j].rlim_max); /* soft limit should not exceed hard limit */ rlim->rlim_cur = min(rlim->rlim_cur, rlim->rlim_max); + if (j == RLIMIT_CPU && + rlim->rlim_cur != RLIM_INFINITY && + IS_ENABLED(CONFIG_POSIX_TIMERS)) + (void) update_rlimit_cpu(current->group_leader, + rlim->rlim_cur); } } } diff --git a/security/integrity/evm/evm_crypto.c b/security/integrity/evm/evm_crypto.c index a5e730ffda57fb..5a8cef45bacf01 100644 --- a/security/integrity/evm/evm_crypto.c +++ b/security/integrity/evm/evm_crypto.c @@ -401,6 +401,7 @@ int evm_init_hmac(struct inode *inode, const struct xattr *xattrs, { struct shash_desc *desc; const struct xattr *xattr; + struct xattr_list *xattr_entry; desc = init_desc(EVM_XATTR_HMAC, HASH_ALGO_SHA1); if (IS_ERR(desc)) { @@ -408,11 +409,16 @@ int evm_init_hmac(struct inode *inode, const struct xattr *xattrs, return PTR_ERR(desc); } - for (xattr = xattrs; xattr->name; xattr++) { - if (!evm_protected_xattr(xattr->name)) - continue; + list_for_each_entry_lockless(xattr_entry, &evm_config_xattrnames, + list) { + for (xattr = xattrs; xattr->name; xattr++) { + if (strcmp(xattr_entry->name + + XATTR_SECURITY_PREFIX_LEN, xattr->name) != 0) + continue; - crypto_shash_update(desc, xattr->value, xattr->value_len); + crypto_shash_update(desc, xattr->value, + xattr->value_len); + } } hmac_add_misc(desc, inode, EVM_XATTR_HMAC, hmac_val); diff --git a/security/integrity/ima/ima.h b/security/integrity/ima/ima.h index e3d71d8d56e38c..89ebe98ffc5e57 100644 --- a/security/integrity/ima/ima.h +++ b/security/integrity/ima/ima.h @@ -441,7 +441,8 @@ int ima_check_blacklist(struct ima_iint_cache *iint, int ima_appraise_measurement(enum ima_hooks func, struct ima_iint_cache *iint, struct file *file, const unsigned char *filename, struct evm_ima_xattr_data *xattr_value, - int xattr_len, const struct modsig *modsig); + int xattr_len, const struct modsig *modsig, + bool bprm_is_check); int ima_must_appraise(struct mnt_idmap *idmap, struct inode *inode, int mask, enum ima_hooks func); void ima_update_xattr(struct ima_iint_cache *iint, struct file *file); @@ -466,7 +467,8 @@ static inline int ima_appraise_measurement(enum ima_hooks func, const unsigned char *filename, struct evm_ima_xattr_data *xattr_value, int xattr_len, - const struct modsig *modsig) + const struct modsig *modsig, + bool bprm_is_check) { return INTEGRITY_UNKNOWN; } diff --git a/security/integrity/ima/ima_appraise.c b/security/integrity/ima/ima_appraise.c index 5149ff4fd50d24..16c20c578ea878 100644 --- a/security/integrity/ima/ima_appraise.c +++ b/security/integrity/ima/ima_appraise.c @@ -470,17 +470,6 @@ int ima_check_blacklist(struct ima_iint_cache *iint, return rc; } -static bool is_bprm_creds_for_exec(enum ima_hooks func, struct file *file) -{ - struct linux_binprm *bprm; - - if (func == BPRM_CHECK) { - bprm = container_of(&file, struct linux_binprm, file); - return bprm->is_check; - } - return false; -} - /* * ima_appraise_measurement - appraise file measurement * @@ -492,7 +481,8 @@ static bool is_bprm_creds_for_exec(enum ima_hooks func, struct file *file) int ima_appraise_measurement(enum ima_hooks func, struct ima_iint_cache *iint, struct file *file, const unsigned char *filename, struct evm_ima_xattr_data *xattr_value, - int xattr_len, const struct modsig *modsig) + int xattr_len, const struct modsig *modsig, + bool bprm_is_check) { static const char op[] = "appraise_data"; int audit_msgno = AUDIT_INTEGRITY_DATA; @@ -514,7 +504,7 @@ int ima_appraise_measurement(enum ima_hooks func, struct ima_iint_cache *iint, * of the script interpreter(userspace). Differentiate kernel and * userspace enforced integrity audit messages. */ - if (is_bprm_creds_for_exec(func, file)) + if (bprm_is_check) audit_msgno = AUDIT_INTEGRITY_USERSPACE; /* If reading the xattr failed and there's no modsig, error out. */ diff --git a/security/integrity/ima/ima_kexec.c b/security/integrity/ima/ima_kexec.c index 5beb69edd12fd2..36a34c54de58ba 100644 --- a/security/integrity/ima/ima_kexec.c +++ b/security/integrity/ima/ima_kexec.c @@ -12,6 +12,8 @@ #include #include #include +#include +#include #include #include #include "ima.h" @@ -294,3 +296,36 @@ void __init ima_load_kexec_buffer(void) pr_debug("Error restoring the measurement list: %d\n", rc); } } + +/* + * ima_validate_range - verify a physical buffer lies in addressable RAM + * @phys: physical start address of the buffer from previous kernel + * @size: size of the buffer + * + * On success return 0. On failure returns -EINVAL so callers can skip + * restoring. + */ +int ima_validate_range(phys_addr_t phys, size_t size) +{ + unsigned long start_pfn, end_pfn; + phys_addr_t end_phys; + + if (check_add_overflow(phys, (phys_addr_t)size - 1, &end_phys)) + return -EINVAL; + + start_pfn = PHYS_PFN(phys); + end_pfn = PHYS_PFN(end_phys); + +#ifdef CONFIG_X86 + if (!pfn_range_is_mapped(start_pfn, end_pfn)) +#else + if (!page_is_ram(start_pfn) || !page_is_ram(end_pfn)) +#endif + { + pr_warn("IMA: previous kernel measurement buffer %pa (size 0x%zx) lies outside available memory\n", + &phys, size); + return -EINVAL; + } + + return 0; +} diff --git a/security/integrity/ima/ima_main.c b/security/integrity/ima/ima_main.c index 5770cf691912aa..1d6229b156fb17 100644 --- a/security/integrity/ima/ima_main.c +++ b/security/integrity/ima/ima_main.c @@ -236,7 +236,8 @@ static void ima_file_free(struct file *file) static int process_measurement(struct file *file, const struct cred *cred, struct lsm_prop *prop, char *buf, loff_t size, int mask, enum ima_hooks func, - enum kernel_read_file_id read_id) + enum kernel_read_file_id read_id, + bool bprm_is_check) { struct inode *real_inode, *inode = file_inode(file); struct ima_iint_cache *iint = NULL; @@ -426,7 +427,8 @@ static int process_measurement(struct file *file, const struct cred *cred, inode_lock(inode); rc = ima_appraise_measurement(func, iint, file, pathname, xattr_value, - xattr_len, modsig); + xattr_len, modsig, + bprm_is_check); inode_unlock(inode); } if (!rc) @@ -493,14 +495,15 @@ static int ima_file_mmap(struct file *file, unsigned long reqprot, if (reqprot & PROT_EXEC) { ret = process_measurement(file, current_cred(), &prop, NULL, - 0, MAY_EXEC, MMAP_CHECK_REQPROT, 0); + 0, MAY_EXEC, MMAP_CHECK_REQPROT, 0, + false); if (ret) return ret; } if (prot & PROT_EXEC) return process_measurement(file, current_cred(), &prop, NULL, - 0, MAY_EXEC, MMAP_CHECK, 0); + 0, MAY_EXEC, MMAP_CHECK, 0, false); return 0; } @@ -584,7 +587,8 @@ static int ima_bprm_check(struct linux_binprm *bprm) security_current_getlsmprop_subj(&prop); return process_measurement(bprm->file, current_cred(), - &prop, NULL, 0, MAY_EXEC, BPRM_CHECK, 0); + &prop, NULL, 0, MAY_EXEC, BPRM_CHECK, 0, + bprm->is_check); } /** @@ -614,7 +618,7 @@ static int ima_creds_check(struct linux_binprm *bprm, const struct file *file) security_current_getlsmprop_subj(&prop); return process_measurement((struct file *)file, bprm->cred, &prop, NULL, - 0, MAY_EXEC, CREDS_CHECK, 0); + 0, MAY_EXEC, CREDS_CHECK, 0, false); } /** @@ -662,7 +666,7 @@ static int ima_file_check(struct file *file, int mask) security_current_getlsmprop_subj(&prop); return process_measurement(file, current_cred(), &prop, NULL, 0, mask & (MAY_READ | MAY_WRITE | MAY_EXEC | - MAY_APPEND), FILE_CHECK, 0); + MAY_APPEND), FILE_CHECK, 0, false); } static int __ima_inode_hash(struct inode *inode, struct file *file, char *buf, @@ -881,7 +885,7 @@ static int ima_read_file(struct file *file, enum kernel_read_file_id read_id, func = read_idmap[read_id] ?: FILE_CHECK; security_current_getlsmprop_subj(&prop); return process_measurement(file, current_cred(), &prop, NULL, 0, - MAY_READ, func, 0); + MAY_READ, func, 0, false); } const int read_idmap[READING_MAX_ID] = { @@ -925,7 +929,7 @@ static int ima_post_read_file(struct file *file, char *buf, loff_t size, func = read_idmap[read_id] ?: FILE_CHECK; security_current_getlsmprop_subj(&prop); return process_measurement(file, current_cred(), &prop, buf, size, - MAY_READ, func, read_id); + MAY_READ, func, read_id, false); } /** diff --git a/security/security.c b/security/security.c index 31a688650601b6..7fd2868ef0f558 100644 --- a/security/security.c +++ b/security/security.c @@ -61,6 +61,7 @@ const char *const lockdown_reasons[LOCKDOWN_CONFIDENTIALITY_MAX + 1] = { [LOCKDOWN_BPF_WRITE_USER] = "use of bpf to write user RAM", [LOCKDOWN_DBG_WRITE_KERNEL] = "use of kgdb/kdb to write kernel RAM", [LOCKDOWN_RTAS_ERROR_INJECTION] = "RTAS error injection", + [LOCKDOWN_XEN_USER_ACTIONS] = "Xen guest user action", [LOCKDOWN_INTEGRITY_MAX] = "integrity", [LOCKDOWN_KCORE] = "/proc/kcore access", [LOCKDOWN_KPROBES] = "use of kprobes", diff --git a/security/smack/smackfs.c b/security/smack/smackfs.c index 2a9d3f2ebbe13c..8919e330d2f606 100644 --- a/security/smack/smackfs.c +++ b/security/smack/smackfs.c @@ -70,6 +70,7 @@ enum smk_inos { static DEFINE_MUTEX(smack_cipso_lock); static DEFINE_MUTEX(smack_ambient_lock); static DEFINE_MUTEX(smk_net4addr_lock); +static DEFINE_MUTEX(smk_cipso_doi_lock); #if IS_ENABLED(CONFIG_IPV6) static DEFINE_MUTEX(smk_net6addr_lock); #endif /* CONFIG_IPV6 */ @@ -141,7 +142,7 @@ struct smack_parsed_rule { int smk_access2; }; -static int smk_cipso_doi_value = SMACK_CIPSO_DOI_DEFAULT; +static u32 smk_cipso_doi_value = CIPSO_V4_DOI_UNKNOWN; /* * Values for parsing cipso rules @@ -663,43 +664,60 @@ static const struct file_operations smk_load_ops = { }; /** - * smk_cipso_doi - initialize the CIPSO domain + * smk_cipso_doi - set netlabel maps + * @ndoi: new value for our CIPSO DOI + * @gfp_flags: kmalloc allocation context */ -static void smk_cipso_doi(void) +static int +smk_cipso_doi(u32 ndoi, gfp_t gfp_flags) { - int rc; + int rc = 0; struct cipso_v4_doi *doip; struct netlbl_audit nai; - smk_netlabel_audit_set(&nai); + mutex_lock(&smk_cipso_doi_lock); - rc = netlbl_cfg_map_del(NULL, PF_INET, NULL, NULL, &nai); - if (rc != 0) - printk(KERN_WARNING "%s:%d remove rc = %d\n", - __func__, __LINE__, rc); + if (smk_cipso_doi_value == ndoi) + goto clr_doi_lock; + + smk_netlabel_audit_set(&nai); - doip = kmalloc(sizeof(struct cipso_v4_doi), GFP_KERNEL | __GFP_NOFAIL); + doip = kmalloc(sizeof(struct cipso_v4_doi), gfp_flags); + if (!doip) { + rc = -ENOMEM; + goto clr_doi_lock; + } doip->map.std = NULL; - doip->doi = smk_cipso_doi_value; + doip->doi = ndoi; doip->type = CIPSO_V4_MAP_PASS; doip->tags[0] = CIPSO_V4_TAG_RBITMAP; for (rc = 1; rc < CIPSO_V4_TAG_MAXCNT; rc++) doip->tags[rc] = CIPSO_V4_TAG_INVALID; rc = netlbl_cfg_cipsov4_add(doip, &nai); - if (rc != 0) { - printk(KERN_WARNING "%s:%d cipso add rc = %d\n", - __func__, __LINE__, rc); + if (rc) { kfree(doip); - return; + goto clr_doi_lock; } - rc = netlbl_cfg_cipsov4_map_add(doip->doi, NULL, NULL, NULL, &nai); - if (rc != 0) { - printk(KERN_WARNING "%s:%d map add rc = %d\n", - __func__, __LINE__, rc); - netlbl_cfg_cipsov4_del(doip->doi, &nai); - return; + + if (smk_cipso_doi_value != CIPSO_V4_DOI_UNKNOWN) { + rc = netlbl_cfg_map_del(NULL, PF_INET, NULL, NULL, &nai); + if (rc && rc != -ENOENT) + goto clr_ndoi_def; + + netlbl_cfg_cipsov4_del(smk_cipso_doi_value, &nai); } + + rc = netlbl_cfg_cipsov4_map_add(ndoi, NULL, NULL, NULL, &nai); + if (rc) { + smk_cipso_doi_value = CIPSO_V4_DOI_UNKNOWN; // no default map +clr_ndoi_def: netlbl_cfg_cipsov4_del(ndoi, &nai); + } else + smk_cipso_doi_value = ndoi; + +clr_doi_lock: + mutex_unlock(&smk_cipso_doi_lock); + return rc; } /** @@ -1562,7 +1580,7 @@ static ssize_t smk_read_doi(struct file *filp, char __user *buf, if (*ppos != 0) return 0; - sprintf(temp, "%d", smk_cipso_doi_value); + sprintf(temp, "%lu", (unsigned long)smk_cipso_doi_value); rc = simple_read_from_buffer(buf, count, ppos, temp, strlen(temp)); return rc; @@ -1581,7 +1599,7 @@ static ssize_t smk_write_doi(struct file *file, const char __user *buf, size_t count, loff_t *ppos) { char temp[80]; - int i; + unsigned long u; if (!smack_privileged(CAP_MAC_ADMIN)) return -EPERM; @@ -1594,14 +1612,13 @@ static ssize_t smk_write_doi(struct file *file, const char __user *buf, temp[count] = '\0'; - if (sscanf(temp, "%d", &i) != 1) + if (kstrtoul(temp, 10, &u)) return -EINVAL; - smk_cipso_doi_value = i; - - smk_cipso_doi(); + if (u == CIPSO_V4_DOI_UNKNOWN || u > U32_MAX) + return -EINVAL; - return count; + return smk_cipso_doi(u, GFP_KERNEL) ? : count; } static const struct file_operations smk_doi_ops = { @@ -2982,6 +2999,7 @@ int __init init_smk_fs(void) { int err; int rc; + struct netlbl_audit nai; if (smack_enabled == 0) return 0; @@ -3000,7 +3018,10 @@ int __init init_smk_fs(void) } } - smk_cipso_doi(); + smk_netlabel_audit_set(&nai); + (void) netlbl_cfg_map_del(NULL, PF_INET, NULL, NULL, &nai); + (void) smk_cipso_doi(SMACK_CIPSO_DOI_DEFAULT, + GFP_KERNEL | __GFP_NOFAIL); smk_unlbl_ambient(NULL); rc = smack_populate_secattr(&smack_known_floor); diff --git a/sound/core/compress_offload.c b/sound/core/compress_offload.c index da514fef45bca0..ed2eeb914c6dbf 100644 --- a/sound/core/compress_offload.c +++ b/sound/core/compress_offload.c @@ -514,12 +514,12 @@ static int snd_compr_get_codec_caps(struct snd_compr_stream *stream, unsigned long arg) { int retval; - struct snd_compr_codec_caps *caps __free(kfree) = NULL; if (!stream->ops->get_codec_caps) return -ENXIO; - caps = kzalloc(sizeof(*caps), GFP_KERNEL); + struct snd_compr_codec_caps *caps __free(kfree) = + kzalloc(sizeof(*caps), GFP_KERNEL); if (!caps) return -ENOMEM; @@ -647,7 +647,6 @@ snd_compress_check_input(struct snd_compr_stream *stream, struct snd_compr_param static int snd_compr_set_params(struct snd_compr_stream *stream, unsigned long arg) { - struct snd_compr_params *params __free(kfree) = NULL; int retval; if (stream->runtime->state == SNDRV_PCM_STATE_OPEN || stream->next_track) { @@ -655,7 +654,9 @@ snd_compr_set_params(struct snd_compr_stream *stream, unsigned long arg) * we should allow parameter change only when stream has been * opened not in other cases */ - params = memdup_user((void __user *)arg, sizeof(*params)); + struct snd_compr_params *params __free(kfree) = + memdup_user((void __user *)arg, sizeof(*params)); + if (IS_ERR(params)) return PTR_ERR(params); @@ -687,13 +688,13 @@ snd_compr_set_params(struct snd_compr_stream *stream, unsigned long arg) static int snd_compr_get_params(struct snd_compr_stream *stream, unsigned long arg) { - struct snd_codec *params __free(kfree) = NULL; int retval; if (!stream->ops->get_params) return -EBADFD; - params = kzalloc(sizeof(*params), GFP_KERNEL); + struct snd_codec *params __free(kfree) = + kzalloc(sizeof(*params), GFP_KERNEL); if (!params) return -ENOMEM; retval = stream->ops->get_params(stream, params); @@ -1104,12 +1105,13 @@ static int snd_compr_task_new(struct snd_compr_stream *stream, struct snd_compr_ static int snd_compr_task_create(struct snd_compr_stream *stream, unsigned long arg) { - struct snd_compr_task *task __free(kfree) = NULL; int retval; if (stream->runtime->state != SNDRV_PCM_STATE_SETUP) return -EPERM; - task = memdup_user((void __user *)arg, sizeof(*task)); + + struct snd_compr_task *task __free(kfree) = + memdup_user((void __user *)arg, sizeof(*task)); if (IS_ERR(task)) return PTR_ERR(task); retval = snd_compr_task_new(stream, task); @@ -1165,12 +1167,13 @@ static int snd_compr_task_start(struct snd_compr_stream *stream, struct snd_comp static int snd_compr_task_start_ioctl(struct snd_compr_stream *stream, unsigned long arg) { - struct snd_compr_task *task __free(kfree) = NULL; int retval; if (stream->runtime->state != SNDRV_PCM_STATE_SETUP) return -EPERM; - task = memdup_user((void __user *)arg, sizeof(*task)); + + struct snd_compr_task *task __free(kfree) = + memdup_user((void __user *)arg, sizeof(*task)); if (IS_ERR(task)) return PTR_ERR(task); retval = snd_compr_task_start(stream, task); @@ -1256,12 +1259,13 @@ static int snd_compr_task_status(struct snd_compr_stream *stream, static int snd_compr_task_status_ioctl(struct snd_compr_stream *stream, unsigned long arg) { - struct snd_compr_task_status *status __free(kfree) = NULL; int retval; if (stream->runtime->state != SNDRV_PCM_STATE_SETUP) return -EPERM; - status = memdup_user((void __user *)arg, sizeof(*status)); + + struct snd_compr_task_status *status __free(kfree) = + memdup_user((void __user *)arg, sizeof(*status)); if (IS_ERR(status)) return PTR_ERR(status); retval = snd_compr_task_status(stream, status); diff --git a/sound/core/control.c b/sound/core/control.c index 9c3fd5113a6173..a5f025d9bfba83 100644 --- a/sound/core/control.c +++ b/sound/core/control.c @@ -123,10 +123,12 @@ static int snd_ctl_release(struct inode *inode, struct file *file) scoped_guard(rwsem_write, &card->controls_rwsem) { list_for_each_entry(control, &card->controls, list) for (idx = 0; idx < control->count; idx++) - if (control->vd[idx].owner == ctl) + if (control->vd[idx].owner == ctl) { control->vd[idx].owner = NULL; + if (control->unlock) + control->unlock(control); + } } - snd_fasync_free(ctl->fasync); snd_ctl_empty_read_queue(ctl); put_pid(ctl->pid); @@ -303,6 +305,8 @@ struct snd_kcontrol *snd_ctl_new1(const struct snd_kcontrol_new *ncontrol, kctl->info = ncontrol->info; kctl->get = ncontrol->get; kctl->put = ncontrol->put; + kctl->lock = ncontrol->lock; + kctl->unlock = ncontrol->unlock; kctl->tlv.p = ncontrol->tlv.p; kctl->private_value = ncontrol->private_value; @@ -867,9 +871,9 @@ EXPORT_SYMBOL(snd_ctl_find_id); static int snd_ctl_card_info(struct snd_card *card, struct snd_ctl_file * ctl, unsigned int cmd, void __user *arg) { - struct snd_ctl_card_info *info __free(kfree) = NULL; + struct snd_ctl_card_info *info __free(kfree) = + kzalloc(sizeof(*info), GFP_KERNEL); - info = kzalloc(sizeof(*info), GFP_KERNEL); if (! info) return -ENOMEM; scoped_guard(rwsem_read, &snd_ioctl_rwsem) { @@ -1244,10 +1248,10 @@ static int snd_ctl_elem_read(struct snd_card *card, static int snd_ctl_elem_read_user(struct snd_card *card, struct snd_ctl_elem_value __user *_control) { - struct snd_ctl_elem_value *control __free(kfree) = NULL; int result; + struct snd_ctl_elem_value *control __free(kfree) = + memdup_user(_control, sizeof(*control)); - control = memdup_user(_control, sizeof(*control)); if (IS_ERR(control)) return PTR_ERR(control); @@ -1320,11 +1324,11 @@ static int snd_ctl_elem_write(struct snd_card *card, struct snd_ctl_file *file, static int snd_ctl_elem_write_user(struct snd_ctl_file *file, struct snd_ctl_elem_value __user *_control) { - struct snd_ctl_elem_value *control __free(kfree) = NULL; struct snd_card *card; int result; + struct snd_ctl_elem_value *control __free(kfree) = + memdup_user(_control, sizeof(*control)); - control = memdup_user(_control, sizeof(*control)); if (IS_ERR(control)) return PTR_ERR(control); @@ -1359,6 +1363,12 @@ static int snd_ctl_elem_lock(struct snd_ctl_file *file, vd = &kctl->vd[snd_ctl_get_ioff(kctl, &id)]; if (vd->owner) return -EBUSY; + + if (kctl->lock) { + int err = kctl->lock(kctl, file); + if (err < 0) + return err; + } vd->owner = file; return 0; } @@ -1383,6 +1393,8 @@ static int snd_ctl_elem_unlock(struct snd_ctl_file *file, if (vd->owner != file) return -EPERM; vd->owner = NULL; + if (kctl->unlock) + kctl->unlock(kctl); return 0; } diff --git a/sound/core/control_compat.c b/sound/core/control_compat.c index 6459809ed36486..b8988a4bcd9b5b 100644 --- a/sound/core/control_compat.c +++ b/sound/core/control_compat.c @@ -80,10 +80,10 @@ static int snd_ctl_elem_info_compat(struct snd_ctl_file *ctl, struct snd_ctl_elem_info32 __user *data32) { struct snd_card *card = ctl->card; - struct snd_ctl_elem_info *data __free(kfree) = NULL; int err; + struct snd_ctl_elem_info *data __free(kfree) = + kzalloc(sizeof(*data), GFP_KERNEL); - data = kzalloc(sizeof(*data), GFP_KERNEL); if (! data) return -ENOMEM; @@ -169,14 +169,15 @@ static int get_ctl_type(struct snd_card *card, struct snd_ctl_elem_id *id, int *countp) { struct snd_kcontrol *kctl; - struct snd_ctl_elem_info *info __free(kfree) = NULL; int err; guard(rwsem_read)(&card->controls_rwsem); kctl = snd_ctl_find_id(card, id); if (!kctl) return -ENOENT; - info = kzalloc(sizeof(*info), GFP_KERNEL); + + struct snd_ctl_elem_info *info __free(kfree) = + kzalloc(sizeof(*info), GFP_KERNEL); if (info == NULL) return -ENOMEM; info->id = *id; @@ -280,10 +281,10 @@ static int copy_ctl_value_to_user(void __user *userdata, static int __ctl_elem_read_user(struct snd_card *card, void __user *userdata, void __user *valuep) { - struct snd_ctl_elem_value *data __free(kfree) = NULL; int err, type, count; + struct snd_ctl_elem_value *data __free(kfree) = + kzalloc(sizeof(*data), GFP_KERNEL); - data = kzalloc(sizeof(*data), GFP_KERNEL); if (data == NULL) return -ENOMEM; @@ -314,11 +315,11 @@ static int ctl_elem_read_user(struct snd_card *card, static int __ctl_elem_write_user(struct snd_ctl_file *file, void __user *userdata, void __user *valuep) { - struct snd_ctl_elem_value *data __free(kfree) = NULL; struct snd_card *card = file->card; int err, type, count; + struct snd_ctl_elem_value *data __free(kfree) = + kzalloc(sizeof(*data), GFP_KERNEL); - data = kzalloc(sizeof(*data), GFP_KERNEL); if (data == NULL) return -ENOMEM; @@ -378,9 +379,9 @@ static int snd_ctl_elem_add_compat(struct snd_ctl_file *file, struct snd_ctl_elem_info32 __user *data32, int replace) { - struct snd_ctl_elem_info *data __free(kfree) = NULL; + struct snd_ctl_elem_info *data __free(kfree) = + kzalloc(sizeof(*data), GFP_KERNEL); - data = kzalloc(sizeof(*data), GFP_KERNEL); if (! data) return -ENOMEM; diff --git a/sound/core/control_led.c b/sound/core/control_led.c index e33dfcf863cf13..c7641d5084e7db 100644 --- a/sound/core/control_led.c +++ b/sound/core/control_led.c @@ -245,12 +245,12 @@ DEFINE_FREE(snd_card_unref, struct snd_card *, if (_T) snd_card_unref(_T)) static int snd_ctl_led_set_id(int card_number, struct snd_ctl_elem_id *id, unsigned int group, bool set) { - struct snd_card *card __free(snd_card_unref) = NULL; struct snd_kcontrol *kctl; struct snd_kcontrol_volatile *vd; unsigned int ioff, access, new_access; + struct snd_card *card __free(snd_card_unref) = + snd_card_ref(card_number); - card = snd_card_ref(card_number); if (!card) return -ENXIO; guard(rwsem_write)(&card->controls_rwsem); @@ -302,13 +302,13 @@ static void snd_ctl_led_clean(struct snd_card *card) static int snd_ctl_led_reset(int card_number, unsigned int group) { - struct snd_card *card __free(snd_card_unref) = NULL; struct snd_ctl_led_ctl *lctl, *_lctl; struct snd_ctl_led *led; struct snd_kcontrol_volatile *vd; bool change = false; + struct snd_card *card __free(snd_card_unref) = + snd_card_ref(card_number); - card = snd_card_ref(card_number); if (!card) return -ENXIO; @@ -598,11 +598,11 @@ static ssize_t list_show(struct device *dev, struct device_attribute *attr, char *buf) { struct snd_ctl_led_card *led_card = container_of(dev, struct snd_ctl_led_card, dev); - struct snd_card *card __free(snd_card_unref) = NULL; struct snd_ctl_led_ctl *lctl; size_t l = 0; + struct snd_card *card __free(snd_card_unref) = + snd_card_ref(led_card->number); - card = snd_card_ref(led_card->number); if (!card) return -ENXIO; guard(rwsem_read)(&card->controls_rwsem); diff --git a/sound/core/oss/mixer_oss.c b/sound/core/oss/mixer_oss.c index e839a4bb93f819..f4ad0bfb4dac60 100644 --- a/sound/core/oss/mixer_oss.c +++ b/sound/core/oss/mixer_oss.c @@ -517,19 +517,22 @@ static void snd_mixer_oss_get_volume1_vol(struct snd_mixer_oss_file *fmixer, unsigned int numid, int *left, int *right) { - struct snd_ctl_elem_info *uinfo __free(kfree) = NULL; - struct snd_ctl_elem_value *uctl __free(kfree) = NULL; struct snd_kcontrol *kctl; struct snd_card *card = fmixer->card; if (numid == ID_UNKNOWN) return; guard(rwsem_read)(&card->controls_rwsem); + if (card->shutdown) + return; kctl = snd_ctl_find_numid(card, numid); if (!kctl) return; - uinfo = kzalloc(sizeof(*uinfo), GFP_KERNEL); - uctl = kzalloc(sizeof(*uctl), GFP_KERNEL); + + struct snd_ctl_elem_info *uinfo __free(kfree) = + kzalloc(sizeof(*uinfo), GFP_KERNEL); + struct snd_ctl_elem_value *uctl __free(kfree) = + kzalloc(sizeof(*uctl), GFP_KERNEL); if (uinfo == NULL || uctl == NULL) return; if (kctl->info(kctl, uinfo)) @@ -550,19 +553,22 @@ static void snd_mixer_oss_get_volume1_sw(struct snd_mixer_oss_file *fmixer, int *left, int *right, int route) { - struct snd_ctl_elem_info *uinfo __free(kfree) = NULL; - struct snd_ctl_elem_value *uctl __free(kfree) = NULL; struct snd_kcontrol *kctl; struct snd_card *card = fmixer->card; if (numid == ID_UNKNOWN) return; guard(rwsem_read)(&card->controls_rwsem); + if (card->shutdown) + return; kctl = snd_ctl_find_numid(card, numid); if (!kctl) return; - uinfo = kzalloc(sizeof(*uinfo), GFP_KERNEL); - uctl = kzalloc(sizeof(*uctl), GFP_KERNEL); + + struct snd_ctl_elem_info *uinfo __free(kfree) = + kzalloc(sizeof(*uinfo), GFP_KERNEL); + struct snd_ctl_elem_value *uctl __free(kfree) = + kzalloc(sizeof(*uctl), GFP_KERNEL); if (uinfo == NULL || uctl == NULL) return; if (kctl->info(kctl, uinfo)) @@ -609,8 +615,6 @@ static void snd_mixer_oss_put_volume1_vol(struct snd_mixer_oss_file *fmixer, unsigned int numid, int left, int right) { - struct snd_ctl_elem_info *uinfo __free(kfree) = NULL; - struct snd_ctl_elem_value *uctl __free(kfree) = NULL; struct snd_kcontrol *kctl; struct snd_card *card = fmixer->card; int res; @@ -618,11 +622,16 @@ static void snd_mixer_oss_put_volume1_vol(struct snd_mixer_oss_file *fmixer, if (numid == ID_UNKNOWN) return; guard(rwsem_read)(&card->controls_rwsem); + if (card->shutdown) + return; kctl = snd_ctl_find_numid(card, numid); if (!kctl) return; - uinfo = kzalloc(sizeof(*uinfo), GFP_KERNEL); - uctl = kzalloc(sizeof(*uctl), GFP_KERNEL); + + struct snd_ctl_elem_info *uinfo __free(kfree) = + kzalloc(sizeof(*uinfo), GFP_KERNEL); + struct snd_ctl_elem_value *uctl __free(kfree) = + kzalloc(sizeof(*uctl), GFP_KERNEL); if (uinfo == NULL || uctl == NULL) return; if (kctl->info(kctl, uinfo)) @@ -646,8 +655,6 @@ static void snd_mixer_oss_put_volume1_sw(struct snd_mixer_oss_file *fmixer, int left, int right, int route) { - struct snd_ctl_elem_info *uinfo __free(kfree) = NULL; - struct snd_ctl_elem_value *uctl __free(kfree) = NULL; struct snd_kcontrol *kctl; struct snd_card *card = fmixer->card; int res; @@ -655,11 +662,16 @@ static void snd_mixer_oss_put_volume1_sw(struct snd_mixer_oss_file *fmixer, if (numid == ID_UNKNOWN) return; guard(rwsem_read)(&card->controls_rwsem); + if (card->shutdown) + return; kctl = snd_ctl_find_numid(card, numid); if (!kctl) return; - uinfo = kzalloc(sizeof(*uinfo), GFP_KERNEL); - uctl = kzalloc(sizeof(*uctl), GFP_KERNEL); + + struct snd_ctl_elem_info *uinfo __free(kfree) = + kzalloc(sizeof(*uinfo), GFP_KERNEL); + struct snd_ctl_elem_value *uctl __free(kfree) = + kzalloc(sizeof(*uctl), GFP_KERNEL); if (uinfo == NULL || uctl == NULL) return; if (kctl->info(kctl, uinfo)) @@ -783,15 +795,17 @@ static int snd_mixer_oss_get_recsrc2(struct snd_mixer_oss_file *fmixer, unsigned struct snd_kcontrol *kctl; struct snd_mixer_oss_slot *pslot; struct slot *slot; - struct snd_ctl_elem_info *uinfo __free(kfree) = NULL; - struct snd_ctl_elem_value *uctl __free(kfree) = NULL; int err, idx; - uinfo = kzalloc(sizeof(*uinfo), GFP_KERNEL); - uctl = kzalloc(sizeof(*uctl), GFP_KERNEL); + struct snd_ctl_elem_info *uinfo __free(kfree) = + kzalloc(sizeof(*uinfo), GFP_KERNEL); + struct snd_ctl_elem_value *uctl __free(kfree) = + kzalloc(sizeof(*uctl), GFP_KERNEL); if (uinfo == NULL || uctl == NULL) return -ENOMEM; guard(rwsem_read)(&card->controls_rwsem); + if (card->shutdown) + return -ENODEV; kctl = snd_mixer_oss_test_id(mixer, "Capture Source", 0); if (!kctl) return -ENOENT; @@ -825,16 +839,18 @@ static int snd_mixer_oss_put_recsrc2(struct snd_mixer_oss_file *fmixer, unsigned struct snd_kcontrol *kctl; struct snd_mixer_oss_slot *pslot; struct slot *slot = NULL; - struct snd_ctl_elem_info *uinfo __free(kfree) = NULL; - struct snd_ctl_elem_value *uctl __free(kfree) = NULL; int err; unsigned int idx; - uinfo = kzalloc(sizeof(*uinfo), GFP_KERNEL); - uctl = kzalloc(sizeof(*uctl), GFP_KERNEL); + struct snd_ctl_elem_info *uinfo __free(kfree) = + kzalloc(sizeof(*uinfo), GFP_KERNEL); + struct snd_ctl_elem_value *uctl __free(kfree) = + kzalloc(sizeof(*uctl), GFP_KERNEL); if (uinfo == NULL || uctl == NULL) return -ENOMEM; guard(rwsem_read)(&card->controls_rwsem); + if (card->shutdown) + return -ENODEV; kctl = snd_mixer_oss_test_id(mixer, "Capture Source", 0); if (!kctl) return -ENOENT; @@ -872,18 +888,20 @@ struct snd_mixer_oss_assign_table { static int snd_mixer_oss_build_test(struct snd_mixer_oss *mixer, struct slot *slot, const char *name, int index, int item) { - struct snd_ctl_elem_info *info __free(kfree) = NULL; struct snd_kcontrol *kcontrol; struct snd_card *card = mixer->card; int err; + struct snd_ctl_elem_info *info __free(kfree) = + kmalloc(sizeof(*info), GFP_KERNEL); + if (!info) + return -ENOMEM; scoped_guard(rwsem_read, &card->controls_rwsem) { + if (card->shutdown) + return -ENODEV; kcontrol = snd_mixer_oss_test_id(mixer, name, index); if (kcontrol == NULL) return 0; - info = kmalloc(sizeof(*info), GFP_KERNEL); - if (!info) - return -ENOMEM; err = kcontrol->info(kcontrol, info); if (err < 0) return err; @@ -1002,13 +1020,15 @@ static int snd_mixer_oss_build_input(struct snd_mixer_oss *mixer, if (snd_mixer_oss_build_test_all(mixer, ptr, &slot)) return 0; guard(rwsem_read)(&mixer->card->controls_rwsem); + if (mixer->card->shutdown) + return -ENODEV; kctl = NULL; if (!ptr->index) kctl = snd_mixer_oss_test_id(mixer, "Capture Source", 0); if (kctl) { - struct snd_ctl_elem_info *uinfo __free(kfree) = NULL; + struct snd_ctl_elem_info *uinfo __free(kfree) = + kzalloc(sizeof(*uinfo), GFP_KERNEL); - uinfo = kzalloc(sizeof(*uinfo), GFP_KERNEL); if (!uinfo) return -ENOMEM; diff --git a/sound/core/oss/pcm_oss.c b/sound/core/oss/pcm_oss.c index b12df5b5ddfc17..3bc94d34b35e76 100644 --- a/sound/core/oss/pcm_oss.c +++ b/sound/core/oss/pcm_oss.c @@ -377,7 +377,6 @@ static int snd_pcm_hw_param_near(struct snd_pcm_substream *pcm, snd_pcm_hw_param_t var, unsigned int best, int *dir) { - struct snd_pcm_hw_params *save __free(kfree) = NULL; int v; unsigned int saved_min; int last = 0; @@ -397,19 +396,22 @@ static int snd_pcm_hw_param_near(struct snd_pcm_substream *pcm, maxdir = 1; max--; } - save = kmalloc(sizeof(*save), GFP_KERNEL); + + struct snd_pcm_hw_params *save __free(kfree) = + kmalloc(sizeof(*save), GFP_KERNEL); if (save == NULL) return -ENOMEM; *save = *params; saved_min = min; min = snd_pcm_hw_param_min(pcm, params, var, min, &mindir); if (min >= 0) { - struct snd_pcm_hw_params *params1 __free(kfree) = NULL; if (max < 0) goto _end; if ((unsigned int)min == saved_min && mindir == valdir) goto _end; - params1 = kmalloc(sizeof(*params1), GFP_KERNEL); + + struct snd_pcm_hw_params *params1 __free(kfree) = + kmalloc(sizeof(*params1), GFP_KERNEL); if (params1 == NULL) return -ENOMEM; *params1 = *save; @@ -781,10 +783,10 @@ static int choose_rate(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params, unsigned int best_rate) { const struct snd_interval *it; - struct snd_pcm_hw_params *save __free(kfree) = NULL; unsigned int rate, prev; - save = kmalloc(sizeof(*save), GFP_KERNEL); + struct snd_pcm_hw_params *save __free(kfree) = + kmalloc(sizeof(*save), GFP_KERNEL); if (save == NULL) return -ENOMEM; *save = *params; @@ -1836,7 +1838,6 @@ static int snd_pcm_oss_get_formats(struct snd_pcm_oss_file *pcm_oss_file) struct snd_pcm_substream *substream; int err; int direct; - struct snd_pcm_hw_params *params __free(kfree) = NULL; unsigned int formats = 0; const struct snd_mask *format_mask; int fmt; @@ -1856,7 +1857,9 @@ static int snd_pcm_oss_get_formats(struct snd_pcm_oss_file *pcm_oss_file) AFMT_S32_LE | AFMT_S32_BE | AFMT_S24_LE | AFMT_S24_BE | AFMT_S24_PACKED; - params = kmalloc(sizeof(*params), GFP_KERNEL); + + struct snd_pcm_hw_params *params __free(kfree) = + kmalloc(sizeof(*params), GFP_KERNEL); if (!params) return -ENOMEM; _snd_pcm_hw_params_any(params); diff --git a/sound/core/pcm.c b/sound/core/pcm.c index 283aac441fa0a7..0b512085eb63f2 100644 --- a/sound/core/pcm.c +++ b/sound/core/pcm.c @@ -328,13 +328,13 @@ static const char *snd_pcm_oss_format_name(int format) static void snd_pcm_proc_info_read(struct snd_pcm_substream *substream, struct snd_info_buffer *buffer) { - struct snd_pcm_info *info __free(kfree) = NULL; int err; if (! substream) return; - info = kmalloc(sizeof(*info), GFP_KERNEL); + struct snd_pcm_info *info __free(kfree) = + kmalloc(sizeof(*info), GFP_KERNEL); if (!info) return; diff --git a/sound/core/pcm_compat.c b/sound/core/pcm_compat.c index 54eb9bd8eb2188..e86f68f1f23c1e 100644 --- a/sound/core/pcm_compat.c +++ b/sound/core/pcm_compat.c @@ -235,7 +235,6 @@ static int snd_pcm_ioctl_hw_params_compat(struct snd_pcm_substream *substream, int refine, struct snd_pcm_hw_params32 __user *data32) { - struct snd_pcm_hw_params *data __free(kfree) = NULL; struct snd_pcm_runtime *runtime; int err; @@ -243,7 +242,8 @@ static int snd_pcm_ioctl_hw_params_compat(struct snd_pcm_substream *substream, if (!runtime) return -ENOTTY; - data = kmalloc(sizeof(*data), GFP_KERNEL); + struct snd_pcm_hw_params *data __free(kfree) = + kmalloc(sizeof(*data), GFP_KERNEL); if (!data) return -ENOMEM; @@ -332,7 +332,6 @@ static int snd_pcm_ioctl_xfern_compat(struct snd_pcm_substream *substream, compat_caddr_t buf; compat_caddr_t __user *bufptr; u32 frames; - void __user **bufs __free(kfree) = NULL; int err, ch, i; if (! substream->runtime) @@ -349,7 +348,9 @@ static int snd_pcm_ioctl_xfern_compat(struct snd_pcm_substream *substream, get_user(frames, &data32->frames)) return -EFAULT; bufptr = compat_ptr(buf); - bufs = kmalloc_array(ch, sizeof(void __user *), GFP_KERNEL); + + void __user **bufs __free(kfree) = + kmalloc_array(ch, sizeof(void __user *), GFP_KERNEL); if (bufs == NULL) return -ENOMEM; for (i = 0; i < ch; i++) { diff --git a/sound/core/pcm_dmaengine.c b/sound/core/pcm_dmaengine.c index f0c17503df425d..805191876fc923 100644 --- a/sound/core/pcm_dmaengine.c +++ b/sound/core/pcm_dmaengine.c @@ -22,6 +22,8 @@ struct dmaengine_pcm_runtime_data { struct dma_chan *dma_chan; dma_cookie_t cookie; + struct work_struct complete_wq; /* for nonatomic PCM */ + struct snd_pcm_substream *substream; unsigned int pos; }; @@ -147,6 +149,21 @@ static void dmaengine_pcm_dma_complete(void *arg) snd_pcm_period_elapsed(substream); } +static void dmaengine_pcm_dma_complete_nonatomic(struct work_struct *wq) +{ + struct dmaengine_pcm_runtime_data *prtd = \ + container_of(wq, struct dmaengine_pcm_runtime_data, complete_wq); + struct snd_pcm_substream *substream = prtd->substream; + dmaengine_pcm_dma_complete(substream); +} + +static void dmaengine_pcm_dma_complete_nonatomic_callback(void *arg) +{ + struct snd_pcm_substream *substream = arg; + struct dmaengine_pcm_runtime_data *prtd = substream_to_prtd(substream); + schedule_work(&prtd->complete_wq); +} + static int dmaengine_pcm_prepare_and_submit(struct snd_pcm_substream *substream) { struct dmaengine_pcm_runtime_data *prtd = substream_to_prtd(substream); @@ -169,7 +186,11 @@ static int dmaengine_pcm_prepare_and_submit(struct snd_pcm_substream *substream) if (!desc) return -ENOMEM; - desc->callback = dmaengine_pcm_dma_complete; + if (substream->pcm->nonatomic) + desc->callback = dmaengine_pcm_dma_complete_nonatomic_callback; + else + desc->callback = dmaengine_pcm_dma_complete; + desc->callback_param = substream; prtd->cookie = dmaengine_submit(desc); @@ -322,6 +343,10 @@ int snd_dmaengine_pcm_open(struct snd_pcm_substream *substream, if (!prtd) return -ENOMEM; + if (substream->pcm->nonatomic) + INIT_WORK(&prtd->complete_wq, dmaengine_pcm_dma_complete_nonatomic); + + prtd->substream = substream; prtd->dma_chan = chan; substream->runtime->private_data = prtd; @@ -355,7 +380,14 @@ static void __snd_dmaengine_pcm_close(struct snd_pcm_substream *substream, if (status == DMA_PAUSED) dmaengine_terminate_async(prtd->dma_chan); + /* + * The PCM might have been closed while suspended, which would + * skip the STOP trigger. Make sure we terminate. + */ + dmaengine_terminate_async(prtd->dma_chan); dmaengine_synchronize(prtd->dma_chan); + if (substream->pcm->nonatomic) + flush_work(&prtd->complete_wq); if (release_channel) dma_release_channel(prtd->dma_chan); kfree(prtd); diff --git a/sound/core/pcm_lib.c b/sound/core/pcm_lib.c index 6eaa950504cfc0..d0df847152f0dd 100644 --- a/sound/core/pcm_lib.c +++ b/sound/core/pcm_lib.c @@ -1149,6 +1149,43 @@ static int snd_interval_step(struct snd_interval *i, unsigned int step) return changed; } +/** + * snd_interval_rate_bits - refine the rate interval from a rate bitmask + * @i: the rate interval to refine + * @mask: the rate bitmask + * + * Refines the interval value, assumed to be the sample rate, according to + * a bitmask of available rates (an ORed combination of SNDRV_PCM_RATE_*). + * + * Return: Positive if the value is changed, zero if it's not changed, or a + * negative error code. + */ +int snd_interval_rate_bits(struct snd_interval *i, unsigned int mask) +{ + unsigned int k; + struct snd_interval mask_range; + + if (!mask) + return -EINVAL; + + snd_interval_any(&mask_range); + mask_range.min = UINT_MAX; + mask_range.max = 0; + for (k = 0; k < snd_pcm_known_rates.count; k++) { + unsigned int rate = snd_pcm_known_rates.list[k]; + if (!(mask & (1 << k))) + continue; + + if (rate > mask_range.max) + mask_range.max = rate; + + if (rate < mask_range.min) + mask_range.min = rate; + } + return snd_interval_refine(i, &mask_range); +} +EXPORT_SYMBOL(snd_interval_rate_bits); + /* Info constraints helpers */ /** diff --git a/sound/core/pcm_native.c b/sound/core/pcm_native.c index 932a9bf98cbc09..3bce80416ecef6 100644 --- a/sound/core/pcm_native.c +++ b/sound/core/pcm_native.c @@ -242,10 +242,10 @@ int snd_pcm_info(struct snd_pcm_substream *substream, struct snd_pcm_info *info) int snd_pcm_info_user(struct snd_pcm_substream *substream, struct snd_pcm_info __user * _info) { - struct snd_pcm_info *info __free(kfree) = NULL; int err; + struct snd_pcm_info *info __free(kfree) = + kmalloc(sizeof(*info), GFP_KERNEL); - info = kmalloc(sizeof(*info), GFP_KERNEL); if (! info) return -ENOMEM; err = snd_pcm_info(substream, info); @@ -364,7 +364,6 @@ static int constrain_params_by_rules(struct snd_pcm_substream *substream, struct snd_pcm_hw_constraints *constrs = &substream->runtime->hw_constraints; unsigned int k; - unsigned int *rstamps __free(kfree) = NULL; unsigned int vstamps[SNDRV_PCM_HW_PARAM_LAST_INTERVAL + 1]; unsigned int stamp; struct snd_pcm_hw_rule *r; @@ -380,7 +379,8 @@ static int constrain_params_by_rules(struct snd_pcm_substream *substream, * Each member of 'rstamps' array represents the sequence number of * recent application of corresponding rule. */ - rstamps = kcalloc(constrs->rules_num, sizeof(unsigned int), GFP_KERNEL); + unsigned int *rstamps __free(kfree) = + kcalloc(constrs->rules_num, sizeof(unsigned int), GFP_KERNEL); if (!rstamps) return -ENOMEM; @@ -583,10 +583,10 @@ EXPORT_SYMBOL(snd_pcm_hw_refine); static int snd_pcm_hw_refine_user(struct snd_pcm_substream *substream, struct snd_pcm_hw_params __user * _params) { - struct snd_pcm_hw_params *params __free(kfree) = NULL; int err; + struct snd_pcm_hw_params *params __free(kfree) = + memdup_user(_params, sizeof(*params)); - params = memdup_user(_params, sizeof(*params)); if (IS_ERR(params)) return PTR_ERR(params); @@ -889,10 +889,10 @@ static int snd_pcm_hw_params(struct snd_pcm_substream *substream, static int snd_pcm_hw_params_user(struct snd_pcm_substream *substream, struct snd_pcm_hw_params __user * _params) { - struct snd_pcm_hw_params *params __free(kfree) = NULL; int err; + struct snd_pcm_hw_params *params __free(kfree) = + memdup_user(_params, sizeof(*params)); - params = memdup_user(_params, sizeof(*params)); if (IS_ERR(params)) return PTR_ERR(params); @@ -944,8 +944,9 @@ static int snd_pcm_hw_free(struct snd_pcm_substream *substream) goto unlock; result = do_hw_free(substream); snd_pcm_set_state(substream, SNDRV_PCM_STATE_OPEN); - cpu_latency_qos_remove_request(&substream->latency_pm_qos_req); - unlock: + if (cpu_latency_qos_request_active(&substream->latency_pm_qos_req)) + cpu_latency_qos_remove_request(&substream->latency_pm_qos_req); +unlock: snd_pcm_buffer_access_unlock(runtime); return result; } @@ -2144,6 +2145,10 @@ static int snd_pcm_drain(struct snd_pcm_substream *substream, for (;;) { long tout; struct snd_pcm_runtime *to_check; + unsigned int drain_rate; + snd_pcm_uframes_t drain_bufsz; + bool drain_no_period_wakeup; + if (signal_pending(current)) { result = -ERESTARTSYS; break; @@ -2163,16 +2168,25 @@ static int snd_pcm_drain(struct snd_pcm_substream *substream, snd_pcm_group_unref(group, substream); if (!to_check) break; /* all drained */ + /* + * Cache the runtime fields needed after unlock. + * A concurrent close() on the linked stream may free + * its runtime via snd_pcm_detach_substream() once we + * release the stream lock below. + */ + drain_no_period_wakeup = to_check->no_period_wakeup; + drain_rate = to_check->rate; + drain_bufsz = to_check->buffer_size; init_waitqueue_entry(&wait, current); set_current_state(TASK_INTERRUPTIBLE); add_wait_queue(&to_check->sleep, &wait); snd_pcm_stream_unlock_irq(substream); - if (runtime->no_period_wakeup) + if (drain_no_period_wakeup) tout = MAX_SCHEDULE_TIMEOUT; else { tout = 100; - if (runtime->rate) { - long t = runtime->buffer_size * 1100 / runtime->rate; + if (drain_rate) { + long t = drain_bufsz * 1100 / drain_rate; tout = max(t, tout); } tout = msecs_to_jiffies(tout); @@ -2267,7 +2281,6 @@ static int snd_pcm_link(struct snd_pcm_substream *substream, int fd) { struct snd_pcm_file *pcm_file; struct snd_pcm_substream *substream1; - struct snd_pcm_group *group __free(kfree) = NULL; struct snd_pcm_group *target_group; bool nonatomic = substream->pcm->nonatomic; CLASS(fd, f)(fd); @@ -2283,7 +2296,8 @@ static int snd_pcm_link(struct snd_pcm_substream *substream, int fd) if (substream == substream1) return -EINVAL; - group = kzalloc(sizeof(*group), GFP_KERNEL); + struct snd_pcm_group *group __free(kfree) = + kzalloc(sizeof(*group), GFP_KERNEL); if (!group) return -ENOMEM; snd_pcm_group_init(group); @@ -2457,6 +2471,7 @@ const struct snd_pcm_hw_constraint_list snd_pcm_known_rates = { .count = ARRAY_SIZE(rates), .list = rates, }; +EXPORT_SYMBOL_GPL(snd_pcm_known_rates); static int snd_pcm_hw_rule_rate(struct snd_pcm_hw_params *params, struct snd_pcm_hw_rule *rule) @@ -3577,7 +3592,6 @@ static ssize_t snd_pcm_readv(struct kiocb *iocb, struct iov_iter *to) struct snd_pcm_runtime *runtime; snd_pcm_sframes_t result; unsigned long i; - void __user **bufs __free(kfree) = NULL; snd_pcm_uframes_t frames; const struct iovec *iov = iter_iov(to); @@ -3596,7 +3610,9 @@ static ssize_t snd_pcm_readv(struct kiocb *iocb, struct iov_iter *to) if (!frame_aligned(runtime, iov->iov_len)) return -EINVAL; frames = bytes_to_samples(runtime, iov->iov_len); - bufs = kmalloc_array(to->nr_segs, sizeof(void *), GFP_KERNEL); + + void __user **bufs __free(kfree) = + kmalloc_array(to->nr_segs, sizeof(void *), GFP_KERNEL); if (bufs == NULL) return -ENOMEM; for (i = 0; i < to->nr_segs; ++i) { @@ -3616,7 +3632,6 @@ static ssize_t snd_pcm_writev(struct kiocb *iocb, struct iov_iter *from) struct snd_pcm_runtime *runtime; snd_pcm_sframes_t result; unsigned long i; - void __user **bufs __free(kfree) = NULL; snd_pcm_uframes_t frames; const struct iovec *iov = iter_iov(from); @@ -3634,7 +3649,9 @@ static ssize_t snd_pcm_writev(struct kiocb *iocb, struct iov_iter *from) !frame_aligned(runtime, iov->iov_len)) return -EINVAL; frames = bytes_to_samples(runtime, iov->iov_len); - bufs = kmalloc_array(from->nr_segs, sizeof(void *), GFP_KERNEL); + + void __user **bufs __free(kfree) = + kmalloc_array(from->nr_segs, sizeof(void *), GFP_KERNEL); if (bufs == NULL) return -ENOMEM; for (i = 0; i < from->nr_segs; ++i) { @@ -4106,15 +4123,15 @@ static void snd_pcm_hw_convert_to_old_params(struct snd_pcm_hw_params_old *opara static int snd_pcm_hw_refine_old_user(struct snd_pcm_substream *substream, struct snd_pcm_hw_params_old __user * _oparams) { - struct snd_pcm_hw_params *params __free(kfree) = NULL; - struct snd_pcm_hw_params_old *oparams __free(kfree) = NULL; int err; - params = kmalloc(sizeof(*params), GFP_KERNEL); + struct snd_pcm_hw_params *params __free(kfree) = + kmalloc(sizeof(*params), GFP_KERNEL); if (!params) return -ENOMEM; - oparams = memdup_user(_oparams, sizeof(*oparams)); + struct snd_pcm_hw_params_old *oparams __free(kfree) = + memdup_user(_oparams, sizeof(*oparams)); if (IS_ERR(oparams)) return PTR_ERR(oparams); snd_pcm_hw_convert_from_old_params(params, oparams); @@ -4135,15 +4152,15 @@ static int snd_pcm_hw_refine_old_user(struct snd_pcm_substream *substream, static int snd_pcm_hw_params_old_user(struct snd_pcm_substream *substream, struct snd_pcm_hw_params_old __user * _oparams) { - struct snd_pcm_hw_params *params __free(kfree) = NULL; - struct snd_pcm_hw_params_old *oparams __free(kfree) = NULL; int err; - params = kmalloc(sizeof(*params), GFP_KERNEL); + struct snd_pcm_hw_params *params __free(kfree) = + kmalloc(sizeof(*params), GFP_KERNEL); if (!params) return -ENOMEM; - oparams = memdup_user(_oparams, sizeof(*oparams)); + struct snd_pcm_hw_params_old *oparams __free(kfree) = + memdup_user(_oparams, sizeof(*oparams)); if (IS_ERR(oparams)) return PTR_ERR(oparams); diff --git a/sound/core/seq/oss/seq_oss_init.c b/sound/core/seq/oss/seq_oss_init.c index 973f057eb731f4..e0c368bd09cb66 100644 --- a/sound/core/seq/oss/seq_oss_init.c +++ b/sound/core/seq/oss/seq_oss_init.c @@ -63,10 +63,10 @@ int __init snd_seq_oss_create_client(void) { int rc; - struct snd_seq_port_info *port __free(kfree) = NULL; struct snd_seq_port_callback port_callback; + struct snd_seq_port_info *port __free(kfree) = + kzalloc(sizeof(*port), GFP_KERNEL); - port = kzalloc(sizeof(*port), GFP_KERNEL); if (!port) return -ENOMEM; diff --git a/sound/core/seq/oss/seq_oss_midi.c b/sound/core/seq/oss/seq_oss_midi.c index 023e5d0a4351da..2d48c25ff4df2f 100644 --- a/sound/core/seq/oss/seq_oss_midi.c +++ b/sound/core/seq/oss/seq_oss_midi.c @@ -65,11 +65,11 @@ static int send_midi_event(struct seq_oss_devinfo *dp, struct snd_seq_event *ev, int snd_seq_oss_midi_lookup_ports(int client) { - struct snd_seq_client_info *clinfo __free(kfree) = NULL; - struct snd_seq_port_info *pinfo __free(kfree) = NULL; + struct snd_seq_client_info *clinfo __free(kfree) = + kzalloc(sizeof(*clinfo), GFP_KERNEL); + struct snd_seq_port_info *pinfo __free(kfree) = + kzalloc(sizeof(*pinfo), GFP_KERNEL); - clinfo = kzalloc(sizeof(*clinfo), GFP_KERNEL); - pinfo = kzalloc(sizeof(*pinfo), GFP_KERNEL); if (!clinfo || !pinfo) return -ENOMEM; clinfo->client = -1; @@ -305,10 +305,10 @@ int snd_seq_oss_midi_open(struct seq_oss_devinfo *dp, int dev, int fmode) { int perm; - struct seq_oss_midi *mdev __free(seq_oss_midi) = NULL; struct snd_seq_port_subscribe subs; + struct seq_oss_midi *mdev __free(seq_oss_midi) = + get_mididev(dp, dev); - mdev = get_mididev(dp, dev); if (!mdev) return -ENODEV; @@ -364,10 +364,10 @@ snd_seq_oss_midi_open(struct seq_oss_devinfo *dp, int dev, int fmode) int snd_seq_oss_midi_close(struct seq_oss_devinfo *dp, int dev) { - struct seq_oss_midi *mdev __free(seq_oss_midi) = NULL; struct snd_seq_port_subscribe subs; + struct seq_oss_midi *mdev __free(seq_oss_midi) = + get_mididev(dp, dev); - mdev = get_mididev(dp, dev); if (!mdev) return -ENODEV; guard(mutex)(&mdev->open_mutex); @@ -399,10 +399,10 @@ snd_seq_oss_midi_close(struct seq_oss_devinfo *dp, int dev) int snd_seq_oss_midi_filemode(struct seq_oss_devinfo *dp, int dev) { - struct seq_oss_midi *mdev __free(seq_oss_midi) = NULL; int mode; + struct seq_oss_midi *mdev __free(seq_oss_midi) = + get_mididev(dp, dev); - mdev = get_mididev(dp, dev); if (!mdev) return 0; @@ -422,9 +422,9 @@ snd_seq_oss_midi_filemode(struct seq_oss_devinfo *dp, int dev) void snd_seq_oss_midi_reset(struct seq_oss_devinfo *dp, int dev) { - struct seq_oss_midi *mdev __free(seq_oss_midi) = NULL; + struct seq_oss_midi *mdev __free(seq_oss_midi) = + get_mididev(dp, dev); - mdev = get_mididev(dp, dev); if (!mdev) return; if (!mdev->opened) @@ -468,9 +468,9 @@ snd_seq_oss_midi_reset(struct seq_oss_devinfo *dp, int dev) void snd_seq_oss_midi_get_addr(struct seq_oss_devinfo *dp, int dev, struct snd_seq_addr *addr) { - struct seq_oss_midi *mdev __free(seq_oss_midi) = NULL; + struct seq_oss_midi *mdev __free(seq_oss_midi) = + get_mididev(dp, dev); - mdev = get_mididev(dp, dev); if (!mdev) return; addr->client = mdev->client; @@ -485,11 +485,11 @@ int snd_seq_oss_midi_input(struct snd_seq_event *ev, int direct, void *private_data) { struct seq_oss_devinfo *dp = (struct seq_oss_devinfo *)private_data; - struct seq_oss_midi *mdev __free(seq_oss_midi) = NULL; if (dp->readq == NULL) return 0; - mdev = find_slot(ev->source.client, ev->source.port); + struct seq_oss_midi *mdev __free(seq_oss_midi) = + find_slot(ev->source.client, ev->source.port); if (!mdev) return 0; if (!(mdev->opened & PERM_READ)) @@ -595,9 +595,9 @@ send_midi_event(struct seq_oss_devinfo *dp, struct snd_seq_event *ev, struct seq int snd_seq_oss_midi_putc(struct seq_oss_devinfo *dp, int dev, unsigned char c, struct snd_seq_event *ev) { - struct seq_oss_midi *mdev __free(seq_oss_midi) = NULL; + struct seq_oss_midi *mdev __free(seq_oss_midi) = + get_mididev(dp, dev); - mdev = get_mididev(dp, dev); if (!mdev) return -ENODEV; if (snd_midi_event_encode_byte(mdev->coder, c, ev)) { @@ -613,9 +613,9 @@ snd_seq_oss_midi_putc(struct seq_oss_devinfo *dp, int dev, unsigned char c, stru int snd_seq_oss_midi_make_info(struct seq_oss_devinfo *dp, int dev, struct midi_info *inf) { - struct seq_oss_midi *mdev __free(seq_oss_midi) = NULL; + struct seq_oss_midi *mdev __free(seq_oss_midi) = + get_mididev(dp, dev); - mdev = get_mididev(dp, dev); if (!mdev) return -ENXIO; inf->device = dev; @@ -651,10 +651,9 @@ snd_seq_oss_midi_info_read(struct snd_info_buffer *buf) snd_iprintf(buf, "\nNumber of MIDI devices: %d\n", max_midi_devs); for (i = 0; i < max_midi_devs; i++) { - struct seq_oss_midi *mdev __free(seq_oss_midi) = NULL; - snd_iprintf(buf, "\nmidi %d: ", i); - mdev = get_mdev(i); + struct seq_oss_midi *mdev __free(seq_oss_midi) = + get_mdev(i); if (mdev == NULL) { snd_iprintf(buf, "*empty*\n"); continue; diff --git a/sound/core/seq/oss/seq_oss_synth.c b/sound/core/seq/oss/seq_oss_synth.c index 8c4e5913c7e69a..beea37ed942cb1 100644 --- a/sound/core/seq/oss/seq_oss_synth.c +++ b/sound/core/seq/oss/seq_oss_synth.c @@ -368,7 +368,6 @@ reset_channels(struct seq_oss_synthinfo *info) void snd_seq_oss_synth_reset(struct seq_oss_devinfo *dp, int dev) { - struct seq_oss_synth *rec __free(seq_oss_synth) = NULL; struct seq_oss_synthinfo *info; info = get_synthinfo_nospec(dp, dev); @@ -391,7 +390,8 @@ snd_seq_oss_synth_reset(struct seq_oss_devinfo *dp, int dev) return; } - rec = get_sdev(dev); + struct seq_oss_synth *rec __free(seq_oss_synth) = + get_sdev(dev); if (rec == NULL) return; if (rec->oper.reset) { @@ -415,7 +415,6 @@ int snd_seq_oss_synth_load_patch(struct seq_oss_devinfo *dp, int dev, int fmt, const char __user *buf, int p, int c) { - struct seq_oss_synth *rec __free(seq_oss_synth) = NULL; struct seq_oss_synthinfo *info; info = get_synthinfo_nospec(dp, dev); @@ -424,7 +423,9 @@ snd_seq_oss_synth_load_patch(struct seq_oss_devinfo *dp, int dev, int fmt, if (info->is_midi) return 0; - rec = get_synthdev(dp, dev); + + struct seq_oss_synth *rec __free(seq_oss_synth) = + get_synthdev(dp, dev); if (!rec) return -ENXIO; @@ -440,9 +441,9 @@ snd_seq_oss_synth_load_patch(struct seq_oss_devinfo *dp, int dev, int fmt, struct seq_oss_synthinfo * snd_seq_oss_synth_info(struct seq_oss_devinfo *dp, int dev) { - struct seq_oss_synth *rec __free(seq_oss_synth) = NULL; + struct seq_oss_synth *rec __free(seq_oss_synth) = + get_synthdev(dp, dev); - rec = get_synthdev(dp, dev); if (rec) return get_synthinfo_nospec(dp, dev); return NULL; @@ -495,13 +496,14 @@ snd_seq_oss_synth_addr(struct seq_oss_devinfo *dp, int dev, struct snd_seq_event int snd_seq_oss_synth_ioctl(struct seq_oss_devinfo *dp, int dev, unsigned int cmd, unsigned long addr) { - struct seq_oss_synth *rec __free(seq_oss_synth) = NULL; struct seq_oss_synthinfo *info; info = get_synthinfo_nospec(dp, dev); if (!info || info->is_midi) return -ENXIO; - rec = get_synthdev(dp, dev); + + struct seq_oss_synth *rec __free(seq_oss_synth) = + get_synthdev(dp, dev); if (!rec) return -ENXIO; if (rec->oper.ioctl == NULL) @@ -575,10 +577,9 @@ snd_seq_oss_synth_info_read(struct snd_info_buffer *buf) snd_iprintf(buf, "\nNumber of synth devices: %d\n", max_synth_devs); for (i = 0; i < max_synth_devs; i++) { - struct seq_oss_synth *rec __free(seq_oss_synth) = NULL; - snd_iprintf(buf, "\nsynth %d: ", i); - rec = get_sdev(i); + struct seq_oss_synth *rec __free(seq_oss_synth) = + get_sdev(i); if (rec == NULL) { snd_iprintf(buf, "*empty*\n"); continue; diff --git a/sound/core/seq/seq_clientmgr.c b/sound/core/seq/seq_clientmgr.c index f9a6e497f997cc..75a7a2af9d8c96 100644 --- a/sound/core/seq/seq_clientmgr.c +++ b/sound/core/seq/seq_clientmgr.c @@ -494,9 +494,9 @@ static int check_port_perm(struct snd_seq_client_port *port, unsigned int flags) */ static struct snd_seq_client *get_event_dest_client(struct snd_seq_event *event) { - struct snd_seq_client *dest __free(snd_seq_client) = NULL; + struct snd_seq_client *dest __free(snd_seq_client) = + snd_seq_client_use_ptr(event->dest.client); - dest = snd_seq_client_use_ptr(event->dest.client); if (dest == NULL) return NULL; if (! dest->accept_input) @@ -565,9 +565,9 @@ static int bounce_error_event(struct snd_seq_client *client, static int update_timestamp_of_queue(struct snd_seq_event *event, int queue, int real_time) { - struct snd_seq_queue *q __free(snd_seq_queue) = NULL; + struct snd_seq_queue *q __free(snd_seq_queue) = + queueptr(queue); - q = queueptr(queue); if (! q) return 0; event->queue = queue; @@ -609,13 +609,13 @@ static int _snd_seq_deliver_single_event(struct snd_seq_client *client, struct snd_seq_event *event, int atomic, int hop) { - struct snd_seq_client *dest __free(snd_seq_client) = NULL; - struct snd_seq_client_port *dest_port __free(snd_seq_port) = NULL; - - dest = get_event_dest_client(event); + struct snd_seq_client *dest __free(snd_seq_client) = + get_event_dest_client(event); if (dest == NULL) return -ENOENT; - dest_port = snd_seq_port_use_ptr(dest, event->dest.port); + + struct snd_seq_client_port *dest_port __free(snd_seq_port) = + snd_seq_port_use_ptr(dest, event->dest.port); if (dest_port == NULL) return -ENOENT; @@ -672,7 +672,6 @@ static int __deliver_to_subscribers(struct snd_seq_client *client, struct snd_seq_event *event, int port, int atomic, int hop) { - struct snd_seq_client_port *src_port __free(snd_seq_port) = NULL; struct snd_seq_subscribers *subs; int err, result = 0, num_ev = 0; union __snd_seq_event event_saved; @@ -681,7 +680,9 @@ static int __deliver_to_subscribers(struct snd_seq_client *client, if (port < 0) return 0; - src_port = snd_seq_port_use_ptr(client, port); + + struct snd_seq_client_port *src_port __free(snd_seq_port) = + snd_seq_port_use_ptr(client, port); if (!src_port) return 0; @@ -801,13 +802,13 @@ static int snd_seq_deliver_event(struct snd_seq_client *client, struct snd_seq_e */ int snd_seq_dispatch_event(struct snd_seq_event_cell *cell, int atomic, int hop) { - struct snd_seq_client *client __free(snd_seq_client) = NULL; int result; if (snd_BUG_ON(!cell)) return -EINVAL; - client = snd_seq_client_use_ptr(cell->event.source.client); + struct snd_seq_client *client __free(snd_seq_client) = + snd_seq_client_use_ptr(cell->event.source.client); if (client == NULL) { snd_seq_cell_free(cell); /* release this cell */ return -EINVAL; @@ -1154,10 +1155,10 @@ static int snd_seq_ioctl_system_info(struct snd_seq_client *client, void *arg) static int snd_seq_ioctl_running_mode(struct snd_seq_client *client, void *arg) { struct snd_seq_running_info *info = arg; - struct snd_seq_client *cptr __free(snd_seq_client) = NULL; - /* requested client number */ - cptr = client_load_and_use_ptr(info->client); + struct snd_seq_client *cptr __free(snd_seq_client) = + client_load_and_use_ptr(info->client); + if (cptr == NULL) return -ENOENT; /* don't change !!! */ @@ -1207,10 +1208,10 @@ static int snd_seq_ioctl_get_client_info(struct snd_seq_client *client, void *arg) { struct snd_seq_client_info *client_info = arg; - struct snd_seq_client *cptr __free(snd_seq_client) = NULL; - /* requested client number */ - cptr = client_load_and_use_ptr(client_info->client); + struct snd_seq_client *cptr __free(snd_seq_client) = + client_load_and_use_ptr(client_info->client); + if (cptr == NULL) return -ENOENT; /* don't change !!! */ @@ -1344,14 +1345,14 @@ static int snd_seq_ioctl_delete_port(struct snd_seq_client *client, void *arg) static int snd_seq_ioctl_get_port_info(struct snd_seq_client *client, void *arg) { struct snd_seq_port_info *info = arg; - struct snd_seq_client *cptr __free(snd_seq_client) = NULL; - struct snd_seq_client_port *port __free(snd_seq_port) = NULL; - cptr = client_load_and_use_ptr(info->addr.client); + struct snd_seq_client *cptr __free(snd_seq_client) = + client_load_and_use_ptr(info->addr.client); if (cptr == NULL) return -ENXIO; - port = snd_seq_port_use_ptr(cptr, info->addr.port); + struct snd_seq_client_port *port __free(snd_seq_port) = + snd_seq_port_use_ptr(cptr, info->addr.port); if (port == NULL) return -ENOENT; /* don't change */ @@ -1367,11 +1368,12 @@ static int snd_seq_ioctl_get_port_info(struct snd_seq_client *client, void *arg) static int snd_seq_ioctl_set_port_info(struct snd_seq_client *client, void *arg) { struct snd_seq_port_info *info = arg; - struct snd_seq_client_port *port __free(snd_seq_port) = NULL; if (info->addr.client != client->number) /* only set our own ports ! */ return -EPERM; - port = snd_seq_port_use_ptr(client, info->addr.port); + + struct snd_seq_client_port *port __free(snd_seq_port) = + snd_seq_port_use_ptr(client, info->addr.port); if (port) { snd_seq_set_port_info(port, info); /* notify the change */ @@ -1444,22 +1446,22 @@ static int snd_seq_ioctl_subscribe_port(struct snd_seq_client *client, void *arg) { struct snd_seq_port_subscribe *subs = arg; - struct snd_seq_client *receiver __free(snd_seq_client) = NULL; - struct snd_seq_client *sender __free(snd_seq_client) = NULL; - struct snd_seq_client_port *sport __free(snd_seq_port) = NULL; - struct snd_seq_client_port *dport __free(snd_seq_port) = NULL; int result; - receiver = client_load_and_use_ptr(subs->dest.client); + struct snd_seq_client *receiver __free(snd_seq_client) = + client_load_and_use_ptr(subs->dest.client); if (!receiver) return -EINVAL; - sender = client_load_and_use_ptr(subs->sender.client); + struct snd_seq_client *sender __free(snd_seq_client) = + client_load_and_use_ptr(subs->sender.client); if (!sender) return -EINVAL; - sport = snd_seq_port_use_ptr(sender, subs->sender.port); + struct snd_seq_client_port *sport __free(snd_seq_port) = + snd_seq_port_use_ptr(sender, subs->sender.port); if (!sport) return -EINVAL; - dport = snd_seq_port_use_ptr(receiver, subs->dest.port); + struct snd_seq_client_port *dport __free(snd_seq_port) = + snd_seq_port_use_ptr(receiver, subs->dest.port); if (!dport) return -EINVAL; @@ -1483,22 +1485,22 @@ static int snd_seq_ioctl_unsubscribe_port(struct snd_seq_client *client, void *arg) { struct snd_seq_port_subscribe *subs = arg; - struct snd_seq_client *receiver __free(snd_seq_client) = NULL; - struct snd_seq_client *sender __free(snd_seq_client) = NULL; - struct snd_seq_client_port *sport __free(snd_seq_port) = NULL; - struct snd_seq_client_port *dport __free(snd_seq_port) = NULL; int result; - receiver = snd_seq_client_use_ptr(subs->dest.client); + struct snd_seq_client *receiver __free(snd_seq_client) = + snd_seq_client_use_ptr(subs->dest.client); if (!receiver) return -ENXIO; - sender = snd_seq_client_use_ptr(subs->sender.client); + struct snd_seq_client *sender __free(snd_seq_client) = + snd_seq_client_use_ptr(subs->sender.client); if (!sender) return -ENXIO; - sport = snd_seq_port_use_ptr(sender, subs->sender.port); + struct snd_seq_client_port *sport __free(snd_seq_port) = + snd_seq_port_use_ptr(sender, subs->sender.port); if (!sport) return -ENXIO; - dport = snd_seq_port_use_ptr(receiver, subs->dest.port); + struct snd_seq_client_port *dport __free(snd_seq_port) = + snd_seq_port_use_ptr(receiver, subs->dest.port); if (!dport) return -ENXIO; @@ -1518,9 +1520,9 @@ static int snd_seq_ioctl_unsubscribe_port(struct snd_seq_client *client, static int snd_seq_ioctl_create_queue(struct snd_seq_client *client, void *arg) { struct snd_seq_queue_info *info = arg; - struct snd_seq_queue *q __free(snd_seq_queue) = NULL; + struct snd_seq_queue *q __free(snd_seq_queue) = + snd_seq_queue_alloc(client->number, info->locked, info->flags); - q = snd_seq_queue_alloc(client->number, info->locked, info->flags); if (IS_ERR(q)) return PTR_ERR(q); @@ -1549,9 +1551,9 @@ static int snd_seq_ioctl_get_queue_info(struct snd_seq_client *client, void *arg) { struct snd_seq_queue_info *info = arg; - struct snd_seq_queue *q __free(snd_seq_queue) = NULL; + struct snd_seq_queue *q __free(snd_seq_queue) = + queueptr(info->queue); - q = queueptr(info->queue); if (q == NULL) return -EINVAL; @@ -1569,7 +1571,6 @@ static int snd_seq_ioctl_set_queue_info(struct snd_seq_client *client, void *arg) { struct snd_seq_queue_info *info = arg; - struct snd_seq_queue *q __free(snd_seq_queue) = NULL; if (info->owner != client->number) return -EINVAL; @@ -1584,7 +1585,8 @@ static int snd_seq_ioctl_set_queue_info(struct snd_seq_client *client, return -EPERM; } - q = queueptr(info->queue); + struct snd_seq_queue *q __free(snd_seq_queue) = + queueptr(info->queue); if (! q) return -EINVAL; if (q->owner != client->number) @@ -1599,9 +1601,9 @@ static int snd_seq_ioctl_get_named_queue(struct snd_seq_client *client, void *arg) { struct snd_seq_queue_info *info = arg; - struct snd_seq_queue *q __free(snd_seq_queue) = NULL; + struct snd_seq_queue *q __free(snd_seq_queue) = + snd_seq_queue_find_name(info->name); - q = snd_seq_queue_find_name(info->name); if (q == NULL) return -EINVAL; info->queue = q->queue; @@ -1616,10 +1618,10 @@ static int snd_seq_ioctl_get_queue_status(struct snd_seq_client *client, void *arg) { struct snd_seq_queue_status *status = arg; - struct snd_seq_queue *queue __free(snd_seq_queue) = NULL; struct snd_seq_timer *tmr; + struct snd_seq_queue *queue __free(snd_seq_queue) = + queueptr(status->queue); - queue = queueptr(status->queue); if (queue == NULL) return -EINVAL; memset(status, 0, sizeof(*status)); @@ -1644,10 +1646,10 @@ static int snd_seq_ioctl_get_queue_tempo(struct snd_seq_client *client, void *arg) { struct snd_seq_queue_tempo *tempo = arg; - struct snd_seq_queue *queue __free(snd_seq_queue) = NULL; struct snd_seq_timer *tmr; + struct snd_seq_queue *queue __free(snd_seq_queue) = + queueptr(tempo->queue); - queue = queueptr(tempo->queue); if (queue == NULL) return -EINVAL; memset(tempo, 0, sizeof(*tempo)); @@ -1693,10 +1695,10 @@ static int snd_seq_ioctl_get_queue_timer(struct snd_seq_client *client, void *arg) { struct snd_seq_queue_timer *timer = arg; - struct snd_seq_queue *queue __free(snd_seq_queue) = NULL; struct snd_seq_timer *tmr; + struct snd_seq_queue *queue __free(snd_seq_queue) = + queueptr(timer->queue); - queue = queueptr(timer->queue); if (queue == NULL) return -EINVAL; @@ -1726,10 +1728,10 @@ static int snd_seq_ioctl_set_queue_timer(struct snd_seq_client *client, return -EINVAL; if (snd_seq_queue_check_access(timer->queue, client->number)) { - struct snd_seq_queue *q __free(snd_seq_queue) = NULL; struct snd_seq_timer *tmr; + struct snd_seq_queue *q __free(snd_seq_queue) = + queueptr(timer->queue); - q = queueptr(timer->queue); if (q == NULL) return -ENXIO; guard(mutex)(&q->timer_mutex); @@ -1788,9 +1790,9 @@ static int snd_seq_ioctl_get_client_pool(struct snd_seq_client *client, void *arg) { struct snd_seq_client_pool *info = arg; - struct snd_seq_client *cptr __free(snd_seq_client) = NULL; + struct snd_seq_client *cptr __free(snd_seq_client) = + client_load_and_use_ptr(info->client); - cptr = client_load_and_use_ptr(info->client); if (cptr == NULL) return -ENOENT; memset(info, 0, sizeof(*info)); @@ -1888,13 +1890,13 @@ static int snd_seq_ioctl_get_subscription(struct snd_seq_client *client, void *arg) { struct snd_seq_port_subscribe *subs = arg; - struct snd_seq_client *sender __free(snd_seq_client) = NULL; - struct snd_seq_client_port *sport __free(snd_seq_port) = NULL; - sender = client_load_and_use_ptr(subs->sender.client); + struct snd_seq_client *sender __free(snd_seq_client) = + client_load_and_use_ptr(subs->sender.client); if (!sender) return -EINVAL; - sport = snd_seq_port_use_ptr(sender, subs->sender.port); + struct snd_seq_client_port *sport __free(snd_seq_port) = + snd_seq_port_use_ptr(sender, subs->sender.port); if (!sport) return -EINVAL; return snd_seq_port_get_subscription(&sport->c_src, &subs->dest, subs); @@ -1907,16 +1909,16 @@ static int snd_seq_ioctl_get_subscription(struct snd_seq_client *client, static int snd_seq_ioctl_query_subs(struct snd_seq_client *client, void *arg) { struct snd_seq_query_subs *subs = arg; - struct snd_seq_client *cptr __free(snd_seq_client) = NULL; - struct snd_seq_client_port *port __free(snd_seq_port) = NULL; struct snd_seq_port_subs_info *group; struct list_head *p; int i; - cptr = client_load_and_use_ptr(subs->root.client); + struct snd_seq_client *cptr __free(snd_seq_client) = + client_load_and_use_ptr(subs->root.client); if (!cptr) return -ENXIO; - port = snd_seq_port_use_ptr(cptr, subs->root.port); + struct snd_seq_client_port *port __free(snd_seq_port) = + snd_seq_port_use_ptr(cptr, subs->root.port); if (!port) return -ENXIO; @@ -1963,7 +1965,6 @@ static int snd_seq_ioctl_query_next_client(struct snd_seq_client *client, void *arg) { struct snd_seq_client_info *info = arg; - struct snd_seq_client *cptr __free(snd_seq_client) = NULL; /* search for next client */ if (info->client < INT_MAX) @@ -1971,7 +1972,8 @@ static int snd_seq_ioctl_query_next_client(struct snd_seq_client *client, if (info->client < 0) info->client = 0; for (; info->client < SNDRV_SEQ_MAX_CLIENTS; info->client++) { - cptr = client_load_and_use_ptr(info->client); + struct snd_seq_client *cptr __free(snd_seq_client) = + client_load_and_use_ptr(info->client); if (cptr) { get_client_info(cptr, info); return 0; /* found */ @@ -1987,16 +1989,16 @@ static int snd_seq_ioctl_query_next_port(struct snd_seq_client *client, void *arg) { struct snd_seq_port_info *info = arg; - struct snd_seq_client *cptr __free(snd_seq_client) = NULL; - struct snd_seq_client_port *port __free(snd_seq_port) = NULL; - cptr = client_load_and_use_ptr(info->addr.client); + struct snd_seq_client *cptr __free(snd_seq_client) = + client_load_and_use_ptr(info->addr.client); if (cptr == NULL) return -ENXIO; /* search for next port */ info->addr.port++; - port = snd_seq_port_query_nearest(cptr, info); + struct snd_seq_client_port *port __free(snd_seq_port) = + snd_seq_port_query_nearest(cptr, info); if (port == NULL) return -ENOENT; @@ -2067,7 +2069,6 @@ static int snd_seq_ioctl_client_ump_info(struct snd_seq_client *caller, { struct snd_seq_client_ump_info __user *argp = (struct snd_seq_client_ump_info __user *)arg; - struct snd_seq_client *cptr __free(snd_seq_client) = NULL; int client, type, err = 0; size_t size; void *p; @@ -2083,7 +2084,9 @@ static int snd_seq_ioctl_client_ump_info(struct snd_seq_client *caller, size = sizeof(struct snd_ump_endpoint_info); else size = sizeof(struct snd_ump_block_info); - cptr = client_load_and_use_ptr(client); + + struct snd_seq_client *cptr __free(snd_seq_client) = + client_load_and_use_ptr(client); if (!cptr) return -ENOENT; @@ -2342,8 +2345,6 @@ EXPORT_SYMBOL(snd_seq_delete_kernel_client); int snd_seq_kernel_client_enqueue(int client, struct snd_seq_event *ev, struct file *file, bool blocking) { - struct snd_seq_client *cptr __free(snd_seq_client) = NULL; - if (snd_BUG_ON(!ev)) return -EINVAL; @@ -2360,7 +2361,8 @@ int snd_seq_kernel_client_enqueue(int client, struct snd_seq_event *ev, if (check_event_type_and_length(ev)) return -EINVAL; - cptr = client_load_and_use_ptr(client); + struct snd_seq_client *cptr __free(snd_seq_client) = + client_load_and_use_ptr(client); if (cptr == NULL) return -EINVAL; @@ -2385,8 +2387,6 @@ EXPORT_SYMBOL(snd_seq_kernel_client_enqueue); int snd_seq_kernel_client_dispatch(int client, struct snd_seq_event * ev, int atomic, int hop) { - struct snd_seq_client *cptr __free(snd_seq_client) = NULL; - if (snd_BUG_ON(!ev)) return -EINVAL; @@ -2397,7 +2397,8 @@ int snd_seq_kernel_client_dispatch(int client, struct snd_seq_event * ev, if (check_event_type_and_length(ev)) return -EINVAL; - cptr = snd_seq_client_use_ptr(client); + struct snd_seq_client *cptr __free(snd_seq_client) = + snd_seq_client_use_ptr(client); if (cptr == NULL) return -EINVAL; @@ -2450,9 +2451,9 @@ EXPORT_SYMBOL(snd_seq_kernel_client_ctl); /* a similar like above but taking locks; used only from OSS sequencer layer */ int snd_seq_kernel_client_ioctl(int clientid, unsigned int cmd, void *arg) { - struct snd_seq_client *client __free(snd_seq_client) = NULL; + struct snd_seq_client *client __free(snd_seq_client) = + client_load_and_use_ptr(clientid); - client = client_load_and_use_ptr(clientid); if (!client) return -ENXIO; guard(mutex)(&client->ioctl_mutex); @@ -2597,9 +2598,9 @@ void snd_seq_info_clients_read(struct snd_info_entry *entry, /* list the client table */ for (c = 0; c < SNDRV_SEQ_MAX_CLIENTS; c++) { - struct snd_seq_client *client __free(snd_seq_client) = NULL; + struct snd_seq_client *client __free(snd_seq_client) = + client_load_and_use_ptr(c); - client = client_load_and_use_ptr(c); if (client == NULL) continue; if (client->type == NO_CLIENT) diff --git a/sound/core/seq/seq_compat.c b/sound/core/seq/seq_compat.c index 643af4c1e83866..260428747e337b 100644 --- a/sound/core/seq/seq_compat.c +++ b/sound/core/seq/seq_compat.c @@ -31,10 +31,10 @@ struct snd_seq_port_info32 { static int snd_seq_call_port_info_ioctl(struct snd_seq_client *client, unsigned int cmd, struct snd_seq_port_info32 __user *data32) { - struct snd_seq_port_info *data __free(kfree) = NULL; int err; + struct snd_seq_port_info *data __free(kfree) = + kmalloc(sizeof(*data), GFP_KERNEL); - data = kmalloc(sizeof(*data), GFP_KERNEL); if (!data) return -ENOMEM; diff --git a/sound/core/seq/seq_midi.c b/sound/core/seq/seq_midi.c index 581e138a311592..73dbef8fb2ac01 100644 --- a/sound/core/seq/seq_midi.c +++ b/sound/core/seq/seq_midi.c @@ -270,8 +270,6 @@ snd_seq_midisynth_probe(struct device *_dev) struct snd_seq_device *dev = to_seq_dev(_dev); struct seq_midisynth_client *client; struct seq_midisynth *msynth, *ms; - struct snd_seq_port_info *port __free(kfree) = NULL; - struct snd_rawmidi_info *info __free(kfree) = NULL; struct snd_rawmidi *rmidi = dev->private_data; int newclient = 0; unsigned int p, ports; @@ -282,7 +280,9 @@ snd_seq_midisynth_probe(struct device *_dev) if (snd_BUG_ON(!card || device < 0 || device >= SNDRV_RAWMIDI_DEVICES)) return -EINVAL; - info = kmalloc(sizeof(*info), GFP_KERNEL); + + struct snd_rawmidi_info *info __free(kfree) = + kmalloc(sizeof(*info), GFP_KERNEL); if (! info) return -ENOMEM; info->device = device; @@ -320,7 +320,9 @@ snd_seq_midisynth_probe(struct device *_dev) } msynth = kcalloc(ports, sizeof(struct seq_midisynth), GFP_KERNEL); - port = kmalloc(sizeof(*port), GFP_KERNEL); + + struct snd_seq_port_info *port __free(kfree) = + kmalloc(sizeof(*port), GFP_KERNEL); if (msynth == NULL || port == NULL) goto __nomem; diff --git a/sound/core/seq/seq_ports.c b/sound/core/seq/seq_ports.c index 40fa379847e573..bbec34bba4f998 100644 --- a/sound/core/seq/seq_ports.c +++ b/sound/core/seq/seq_ports.c @@ -211,14 +211,13 @@ static void clear_subscriber_list(struct snd_seq_client *client, list_for_each_safe(p, n, &grp->list_head) { struct snd_seq_subscribers *subs; - struct snd_seq_client *c __free(snd_seq_client) = NULL; - struct snd_seq_client_port *aport __free(snd_seq_port) = NULL; subs = get_subscriber(p, is_src); - if (is_src) - aport = get_client_port(&subs->info.dest, &c); - else - aport = get_client_port(&subs->info.sender, &c); + struct snd_seq_client *c __free(snd_seq_client) = NULL; + struct snd_seq_client_port *aport __free(snd_seq_port) = + is_src ? + get_client_port(&subs->info.dest, &c) : + get_client_port(&subs->info.sender, &c); delete_and_unsubscribe_port(client, port, subs, is_src, false); if (!aport) { diff --git a/sound/core/seq/seq_queue.c b/sound/core/seq/seq_queue.c index f5c0e401c8ae56..c0c5e1424c5a1c 100644 --- a/sound/core/seq/seq_queue.c +++ b/sound/core/seq/seq_queue.c @@ -211,8 +211,9 @@ struct snd_seq_queue *snd_seq_queue_find_name(char *name) int i; for (i = 0; i < SNDRV_SEQ_MAX_QUEUES; i++) { - struct snd_seq_queue *q __free(snd_seq_queue) = NULL; - q = queueptr(i); + struct snd_seq_queue *q __free(snd_seq_queue) = + queueptr(i); + if (q) { if (strncmp(q->name, name, sizeof(q->name)) == 0) return no_free_ptr(q); @@ -285,12 +286,13 @@ void snd_seq_check_queue(struct snd_seq_queue *q, int atomic, int hop) int snd_seq_enqueue_event(struct snd_seq_event_cell *cell, int atomic, int hop) { int dest, err; - struct snd_seq_queue *q __free(snd_seq_queue) = NULL; if (snd_BUG_ON(!cell)) return -EINVAL; dest = cell->event.queue; /* destination queue */ - q = queueptr(dest); + + struct snd_seq_queue *q __free(snd_seq_queue) = + queueptr(dest); if (q == NULL) return -EINVAL; /* handle relative time stamps, convert them into absolute */ @@ -403,10 +405,10 @@ int snd_seq_queue_set_owner(int queueid, int client, int locked) int snd_seq_queue_timer_open(int queueid) { int result = 0; - struct snd_seq_queue *queue __free(snd_seq_queue) = NULL; struct snd_seq_timer *tmr; + struct snd_seq_queue *queue __free(snd_seq_queue) = + queueptr(queueid); - queue = queueptr(queueid); if (queue == NULL) return -EINVAL; tmr = queue->timer; @@ -423,10 +425,10 @@ int snd_seq_queue_timer_open(int queueid) */ int snd_seq_queue_timer_close(int queueid) { - struct snd_seq_queue *queue __free(snd_seq_queue) = NULL; int result = 0; + struct snd_seq_queue *queue __free(snd_seq_queue) = + queueptr(queueid); - queue = queueptr(queueid); if (queue == NULL) return -EINVAL; snd_seq_timer_close(queue); @@ -479,9 +481,9 @@ static void queue_use(struct snd_seq_queue *queue, int client, int use) */ int snd_seq_queue_use(int queueid, int client, int use) { - struct snd_seq_queue *queue __free(snd_seq_queue) = NULL; + struct snd_seq_queue *queue __free(snd_seq_queue) = + queueptr(queueid); - queue = queueptr(queueid); if (queue == NULL) return -EINVAL; guard(mutex)(&queue->timer_mutex); @@ -496,9 +498,9 @@ int snd_seq_queue_use(int queueid, int client, int use) */ int snd_seq_queue_is_used(int queueid, int client) { - struct snd_seq_queue *q __free(snd_seq_queue) = NULL; + struct snd_seq_queue *q __free(snd_seq_queue) = + queueptr(queueid); - q = queueptr(queueid); if (q == NULL) return -EINVAL; /* invalid queue */ return test_bit(client, q->clients_bitmap) ? 1 : 0; @@ -642,11 +644,11 @@ static void snd_seq_queue_process_event(struct snd_seq_queue *q, */ int snd_seq_control_queue(struct snd_seq_event *ev, int atomic, int hop) { - struct snd_seq_queue *q __free(snd_seq_queue) = NULL; - if (snd_BUG_ON(!ev)) return -EINVAL; - q = queueptr(ev->data.queue.queue); + + struct snd_seq_queue *q __free(snd_seq_queue) = + queueptr(ev->data.queue.queue); if (q == NULL) return -EINVAL; diff --git a/sound/core/seq/seq_ump_client.c b/sound/core/seq/seq_ump_client.c index 27247babb16dee..14b82a31eb1952 100644 --- a/sound/core/seq/seq_ump_client.c +++ b/sound/core/seq/seq_ump_client.c @@ -214,13 +214,13 @@ static bool skip_group(struct seq_ump_client *client, struct snd_ump_group *grou static int seq_ump_group_init(struct seq_ump_client *client, int group_index) { struct snd_ump_group *group = &client->ump->groups[group_index]; - struct snd_seq_port_info *port __free(kfree) = NULL; struct snd_seq_port_callback pcallbacks; if (skip_group(client, group)) return 0; - port = kzalloc(sizeof(*port), GFP_KERNEL); + struct snd_seq_port_info *port __free(kfree) = + kzalloc(sizeof(*port), GFP_KERNEL); if (!port) return -ENOMEM; @@ -243,12 +243,12 @@ static int seq_ump_group_init(struct seq_ump_client *client, int group_index) /* update the sequencer ports; called from notify_fb_change callback */ static void update_port_infos(struct seq_ump_client *client) { - struct snd_seq_port_info *old __free(kfree) = NULL; - struct snd_seq_port_info *new __free(kfree) = NULL; int i, err; - old = kzalloc(sizeof(*old), GFP_KERNEL); - new = kzalloc(sizeof(*new), GFP_KERNEL); + struct snd_seq_port_info *old __free(kfree) = + kzalloc(sizeof(*old), GFP_KERNEL); + struct snd_seq_port_info *new __free(kfree) = + kzalloc(sizeof(*new), GFP_KERNEL); if (!old || !new) return; @@ -278,12 +278,12 @@ static void update_port_infos(struct seq_ump_client *client) /* create a UMP Endpoint port */ static int create_ump_endpoint_port(struct seq_ump_client *client) { - struct snd_seq_port_info *port __free(kfree) = NULL; struct snd_seq_port_callback pcallbacks; unsigned int rawmidi_info = client->ump->core.info_flags; int err; - port = kzalloc(sizeof(*port), GFP_KERNEL); + struct snd_seq_port_info *port __free(kfree) = + kzalloc(sizeof(*port), GFP_KERNEL); if (!port) return -ENOMEM; diff --git a/sound/core/seq/seq_virmidi.c b/sound/core/seq/seq_virmidi.c index 9e7fd4993a1085..574493fbd50d0e 100644 --- a/sound/core/seq/seq_virmidi.c +++ b/sound/core/seq/seq_virmidi.c @@ -361,13 +361,13 @@ static int snd_virmidi_dev_attach_seq(struct snd_virmidi_dev *rdev) { int client; struct snd_seq_port_callback pcallbacks; - struct snd_seq_port_info *pinfo __free(kfree) = NULL; int err; if (rdev->client >= 0) return 0; - pinfo = kzalloc(sizeof(*pinfo), GFP_KERNEL); + struct snd_seq_port_info *pinfo __free(kfree) = + kzalloc(sizeof(*pinfo), GFP_KERNEL); if (!pinfo) return -ENOMEM; diff --git a/sound/core/timer.c b/sound/core/timer.c index d9fff5c87613ef..9a4a1748ff80bb 100644 --- a/sound/core/timer.c +++ b/sound/core/timer.c @@ -1614,12 +1614,12 @@ static int snd_timer_user_next_device(struct snd_timer_id __user *_tid) static int snd_timer_user_ginfo(struct file *file, struct snd_timer_ginfo __user *_ginfo) { - struct snd_timer_ginfo *ginfo __free(kfree) = NULL; struct snd_timer_id tid; struct snd_timer *t; struct list_head *p; + struct snd_timer_ginfo *ginfo __free(kfree) = + memdup_user(_ginfo, sizeof(*ginfo)); - ginfo = memdup_user(_ginfo, sizeof(*ginfo)); if (IS_ERR(ginfo)) return PTR_ERR(ginfo); @@ -1756,7 +1756,6 @@ static int snd_timer_user_info(struct file *file, struct snd_timer_info __user *_info) { struct snd_timer_user *tu; - struct snd_timer_info *info __free(kfree) = NULL; struct snd_timer *t; tu = file->private_data; @@ -1766,7 +1765,8 @@ static int snd_timer_user_info(struct file *file, if (!t) return -EBADFD; - info = kzalloc(sizeof(*info), GFP_KERNEL); + struct snd_timer_info *info __free(kfree) = + kzalloc(sizeof(*info), GFP_KERNEL); if (! info) return -ENOMEM; info->card = t->card ? t->card->number : -1; @@ -2192,10 +2192,10 @@ static int snd_utimer_ioctl_create(struct file *file, struct snd_timer_uinfo __user *_utimer_info) { struct snd_utimer *utimer; - struct snd_timer_uinfo *utimer_info __free(kfree) = NULL; int err, timer_fd; + struct snd_timer_uinfo *utimer_info __free(kfree) = + memdup_user(_utimer_info, sizeof(*utimer_info)); - utimer_info = memdup_user(_utimer_info, sizeof(*utimer_info)); if (IS_ERR(utimer_info)) return PTR_ERR(utimer_info); diff --git a/sound/core/vmaster.c b/sound/core/vmaster.c index c657659b236c42..76cc64245f5df4 100644 --- a/sound/core/vmaster.c +++ b/sound/core/vmaster.c @@ -56,10 +56,10 @@ struct link_follower { static int follower_update(struct link_follower *follower) { - struct snd_ctl_elem_value *uctl __free(kfree) = NULL; int err, ch; + struct snd_ctl_elem_value *uctl __free(kfree) = + kzalloc(sizeof(*uctl), GFP_KERNEL); - uctl = kzalloc(sizeof(*uctl), GFP_KERNEL); if (!uctl) return -ENOMEM; uctl->id = follower->follower.id; @@ -74,7 +74,6 @@ static int follower_update(struct link_follower *follower) /* get the follower ctl info and save the initial values */ static int follower_init(struct link_follower *follower) { - struct snd_ctl_elem_info *uinfo __free(kfree) = NULL; int err; if (follower->info.count) { @@ -84,7 +83,8 @@ static int follower_init(struct link_follower *follower) return 0; } - uinfo = kmalloc(sizeof(*uinfo), GFP_KERNEL); + struct snd_ctl_elem_info *uinfo __free(kfree) = + kmalloc(sizeof(*uinfo), GFP_KERNEL); if (!uinfo) return -ENOMEM; uinfo->id = follower->follower.id; @@ -341,9 +341,9 @@ static int master_get(struct snd_kcontrol *kcontrol, static int sync_followers(struct link_master *master, int old_val, int new_val) { struct link_follower *follower; - struct snd_ctl_elem_value *uval __free(kfree) = NULL; + struct snd_ctl_elem_value *uval __free(kfree) = + kmalloc(sizeof(*uval), GFP_KERNEL); - uval = kmalloc(sizeof(*uval), GFP_KERNEL); if (!uval) return -ENOMEM; list_for_each_entry(follower, &master->followers, list) { diff --git a/sound/firewire/amdtp-stream.c b/sound/firewire/amdtp-stream.c index 5cdc34877fc191..76cc2b0801c9f6 100644 --- a/sound/firewire/amdtp-stream.c +++ b/sound/firewire/amdtp-stream.c @@ -1179,7 +1179,7 @@ static void process_rx_packets(struct fw_iso_context *context, u32 tstamp, size_ struct pkt_desc *desc = s->packet_descs_cursor; unsigned int pkt_header_length; unsigned int packets; - u32 curr_cycle_time; + u32 curr_cycle_time = 0; bool need_hw_irq; int i; diff --git a/sound/firewire/fireworks/fireworks_command.c b/sound/firewire/fireworks/fireworks_command.c index 2b595ee0bc357b..05550f36fac551 100644 --- a/sound/firewire/fireworks/fireworks_command.c +++ b/sound/firewire/fireworks/fireworks_command.c @@ -151,10 +151,13 @@ efw_transaction(struct snd_efw *efw, unsigned int category, (be32_to_cpu(header->category) != category) || (be32_to_cpu(header->command) != command) || (be32_to_cpu(header->status) != EFR_STATUS_OK)) { + u32 st = be32_to_cpu(header->status); + dev_err(&efw->unit->device, "EFW command failed [%u/%u]: %s\n", be32_to_cpu(header->category), be32_to_cpu(header->command), - efr_status_names[be32_to_cpu(header->status)]); + st < ARRAY_SIZE(efr_status_names) ? + efr_status_names[st] : "unknown"); err = -EIO; goto end; } diff --git a/sound/hda/codecs/conexant.c b/sound/hda/codecs/conexant.c index 5fcbc1312c6971..f71123a475464a 100644 --- a/sound/hda/codecs/conexant.c +++ b/sound/hda/codecs/conexant.c @@ -299,6 +299,7 @@ enum { CXT_PINCFG_SWS_JS201D, CXT_PINCFG_TOP_SPEAKER, CXT_FIXUP_HP_A_U, + CXT_FIXUP_ACER_SWIFT_HP, }; /* for hda_fixup_thinkpad_acpi() */ @@ -1024,6 +1025,14 @@ static const struct hda_fixup cxt_fixups[] = { .type = HDA_FIXUP_FUNC, .v.func = cxt_fixup_hp_a_u, }, + [CXT_FIXUP_ACER_SWIFT_HP] = { + .type = HDA_FIXUP_PINS, + .v.pins = (const struct hda_pintbl[]) { + { 0x16, 0x0321403f }, /* Headphone */ + { 0x19, 0x40f001f0 }, /* Mic */ + { } + }, + }, }; static const struct hda_quirk cxt5045_fixups[] = { @@ -1073,6 +1082,7 @@ static const struct hda_quirk cxt5066_fixups[] = { SND_PCI_QUIRK(0x1025, 0x0543, "Acer Aspire One 522", CXT_FIXUP_STEREO_DMIC), SND_PCI_QUIRK(0x1025, 0x054c, "Acer Aspire 3830TG", CXT_FIXUP_ASPIRE_DMIC), SND_PCI_QUIRK(0x1025, 0x054f, "Acer Aspire 4830T", CXT_FIXUP_ASPIRE_DMIC), + SND_PCI_QUIRK(0x1025, 0x136d, "Acer Swift SF314", CXT_FIXUP_ACER_SWIFT_HP), SND_PCI_QUIRK(0x103c, 0x8079, "HP EliteBook 840 G3", CXT_FIXUP_HP_DOCK), SND_PCI_QUIRK(0x103c, 0x807C, "HP EliteBook 820 G3", CXT_FIXUP_HP_DOCK), SND_PCI_QUIRK(0x103c, 0x80FD, "HP ProBook 640 G2", CXT_FIXUP_HP_DOCK), @@ -1081,6 +1091,7 @@ static const struct hda_quirk cxt5066_fixups[] = { SND_PCI_QUIRK(0x103c, 0x8174, "HP Spectre x360", CXT_FIXUP_HP_SPECTRE), SND_PCI_QUIRK(0x103c, 0x822e, "HP ProBook 440 G4", CXT_FIXUP_MUTE_LED_GPIO), SND_PCI_QUIRK(0x103c, 0x8231, "HP ProBook 450 G4", CXT_FIXUP_MUTE_LED_GPIO), + SND_PCI_QUIRK(0x103c, 0x826b, "HP ZBook Studio G4", CXT_FIXUP_MUTE_LED_GPIO), SND_PCI_QUIRK(0x103c, 0x828c, "HP EliteBook 840 G4", CXT_FIXUP_HP_DOCK), SND_PCI_QUIRK(0x103c, 0x8299, "HP 800 G3 SFF", CXT_FIXUP_HP_MIC_NO_PRESENCE), SND_PCI_QUIRK(0x103c, 0x829a, "HP 800 G3 DM", CXT_FIXUP_HP_MIC_NO_PRESENCE), @@ -1123,6 +1134,7 @@ static const struct hda_quirk cxt5066_fixups[] = { SND_PCI_QUIRK_VENDOR(0x17aa, "Thinkpad/Ideapad", CXT_FIXUP_LENOVO_XPAD_ACPI), SND_PCI_QUIRK(0x1c06, 0x2011, "Lemote A1004", CXT_PINCFG_LEMOTE_A1004), SND_PCI_QUIRK(0x1c06, 0x2012, "Lemote A1205", CXT_PINCFG_LEMOTE_A1205), + SND_PCI_QUIRK(0x1d05, 0x3012, "MECHREVO Wujie 15X Pro", CXT_FIXUP_HEADSET_MIC), HDA_CODEC_QUIRK(0x2782, 0x12c3, "Sirius Gen1", CXT_PINCFG_TOP_SPEAKER), HDA_CODEC_QUIRK(0x2782, 0x12c5, "Sirius Gen2", CXT_PINCFG_TOP_SPEAKER), {} diff --git a/sound/hda/codecs/generic.c b/sound/hda/codecs/generic.c index 7bcf9aef8275f9..443500a3518f41 100644 --- a/sound/hda/codecs/generic.c +++ b/sound/hda/codecs/generic.c @@ -1984,15 +1984,15 @@ static int parse_output_paths(struct hda_codec *codec) { struct hda_gen_spec *spec = codec->spec; struct auto_pin_cfg *cfg = &spec->autocfg; - struct auto_pin_cfg *best_cfg __free(kfree) = NULL; unsigned int val; int best_badness = INT_MAX; int badness; bool fill_hardwired = true, fill_mio_first = true; bool best_wired = true, best_mio = true; bool hp_spk_swapped = false; + struct auto_pin_cfg *best_cfg __free(kfree) = + kmalloc(sizeof(*best_cfg), GFP_KERNEL); - best_cfg = kmalloc(sizeof(*best_cfg), GFP_KERNEL); if (!best_cfg) return -ENOMEM; *best_cfg = *cfg; diff --git a/sound/hda/codecs/hdmi/hdmi.c b/sound/hda/codecs/hdmi/hdmi.c index 111c9b5335afcc..c2e3adc7b3c007 100644 --- a/sound/hda/codecs/hdmi/hdmi.c +++ b/sound/hda/codecs/hdmi/hdmi.c @@ -1557,6 +1557,7 @@ static const struct snd_pci_quirk force_connect_list[] = { SND_PCI_QUIRK(0x1043, 0x86ae, "ASUS", 1), /* Z170 PRO */ SND_PCI_QUIRK(0x1043, 0x86c7, "ASUS", 1), /* Z170M PLUS */ SND_PCI_QUIRK(0x1462, 0xec94, "MS-7C94", 1), + SND_PCI_QUIRK(0x1558, 0x14a1, "TUXEDO InfinityBook S 14 Gen6", 1), SND_PCI_QUIRK(0x8086, 0x2060, "Intel NUC5CPYB", 1), SND_PCI_QUIRK(0x8086, 0x2081, "Intel NUC 10", 1), {} diff --git a/sound/hda/codecs/hdmi/tegrahdmi.c b/sound/hda/codecs/hdmi/tegrahdmi.c index 5f6fe31aa20284..ebb6410a48313f 100644 --- a/sound/hda/codecs/hdmi/tegrahdmi.c +++ b/sound/hda/codecs/hdmi/tegrahdmi.c @@ -299,6 +299,7 @@ static const struct hda_device_id snd_hda_id_tegrahdmi[] = { HDA_CODEC_ID_MODEL(0x10de002f, "Tegra194 HDMI/DP2", MODEL_TEGRA), HDA_CODEC_ID_MODEL(0x10de0030, "Tegra194 HDMI/DP3", MODEL_TEGRA), HDA_CODEC_ID_MODEL(0x10de0031, "Tegra234 HDMI/DP", MODEL_TEGRA234), + HDA_CODEC_ID_MODEL(0x10de0032, "Tegra238 HDMI/DP", MODEL_TEGRA234), HDA_CODEC_ID_MODEL(0x10de0033, "SoC 33 HDMI/DP", MODEL_TEGRA234), HDA_CODEC_ID_MODEL(0x10de0034, "Tegra264 HDMI/DP", MODEL_TEGRA234), HDA_CODEC_ID_MODEL(0x10de0035, "SoC 35 HDMI/DP", MODEL_TEGRA234), diff --git a/sound/hda/codecs/realtek/alc269.c b/sound/hda/codecs/realtek/alc269.c index b66965a521076e..d954de3fd225e0 100644 --- a/sound/hda/codecs/realtek/alc269.c +++ b/sound/hda/codecs/realtek/alc269.c @@ -1017,6 +1017,50 @@ static int alc269_resume(struct hda_codec *codec) return 0; } +#define ALC233_STARFIGHTER_SPK_PIN 0x1b +#define ALC233_STARFIGHTER_GPIO2 0x04 + +static void alc233_starfighter_update_amp(struct hda_codec *codec, bool on) +{ + snd_hda_codec_write(codec, ALC233_STARFIGHTER_SPK_PIN, 0, + AC_VERB_SET_EAPD_BTLENABLE, + on ? AC_EAPDBTL_EAPD : 0); + alc_update_gpio_data(codec, ALC233_STARFIGHTER_GPIO2, on); +} + +static void alc233_starfighter_pcm_hook(struct hda_pcm_stream *hinfo, + struct hda_codec *codec, + struct snd_pcm_substream *substream, + int action) +{ + switch (action) { + case HDA_GEN_PCM_ACT_PREPARE: + alc233_starfighter_update_amp(codec, true); + break; + case HDA_GEN_PCM_ACT_CLEANUP: + alc233_starfighter_update_amp(codec, false); + break; + } +} + +static void alc233_fixup_starlabs_starfighter(struct hda_codec *codec, + const struct hda_fixup *fix, + int action) +{ + struct alc_spec *spec = codec->spec; + + switch (action) { + case HDA_FIXUP_ACT_PRE_PROBE: + spec->gpio_mask |= ALC233_STARFIGHTER_GPIO2; + spec->gpio_dir |= ALC233_STARFIGHTER_GPIO2; + spec->gpio_data &= ~ALC233_STARFIGHTER_GPIO2; + break; + case HDA_FIXUP_ACT_PROBE: + spec->gen.pcm_playback_hook = alc233_starfighter_pcm_hook; + break; + } +} + static void alc269_fixup_pincfg_no_hp_to_lineout(struct hda_codec *codec, const struct hda_fixup *fix, int action) { @@ -1551,6 +1595,22 @@ static void alc245_fixup_hp_mute_led_v1_coefbit(struct hda_codec *codec, } } +static void alc245_fixup_hp_mute_led_v2_coefbit(struct hda_codec *codec, + const struct hda_fixup *fix, + int action) +{ + struct alc_spec *spec = codec->spec; + + if (action == HDA_FIXUP_ACT_PRE_PROBE) { + spec->mute_led_polarity = 0; + spec->mute_led_coef.idx = 0x0b; + spec->mute_led_coef.mask = 1 << 3; + spec->mute_led_coef.on = 1 << 3; + spec->mute_led_coef.off = 0; + snd_hda_gen_add_mute_led_cdev(codec, coef_mute_led_set); + } +} + /* turn on/off mic-mute LED per capture hook by coef bit */ static int coef_micmute_led_set(struct led_classdev *led_cdev, enum led_brightness brightness) @@ -1616,6 +1676,20 @@ static void alc295_fixup_hp_mute_led_coefbit11(struct hda_codec *codec, } } +static void alc233_fixup_lenovo_coef_micmute_led(struct hda_codec *codec, + const struct hda_fixup *fix, int action) +{ + struct alc_spec *spec = codec->spec; + + if (action == HDA_FIXUP_ACT_PRE_PROBE) { + spec->mic_led_coef.idx = 0x10; + spec->mic_led_coef.mask = 1 << 13; + spec->mic_led_coef.on = 0; + spec->mic_led_coef.off = 1 << 13; + snd_hda_gen_add_micmute_led_cdev(codec, coef_micmute_led_set); + } +} + static void alc285_fixup_hp_mute_led(struct hda_codec *codec, const struct hda_fixup *fix, int action) { @@ -1630,6 +1704,13 @@ static void alc285_fixup_hp_spectre_x360_mute_led(struct hda_codec *codec, alc285_fixup_hp_gpio_micmute_led(codec, fix, action); } +static void alc245_fixup_hp_envy_x360_mute_led(struct hda_codec *codec, + const struct hda_fixup *fix, int action) +{ + alc245_fixup_hp_mute_led_v1_coefbit(codec, fix, action); + alc245_fixup_hp_gpio_led(codec, fix, action); +} + static void alc236_fixup_hp_mute_led(struct hda_codec *codec, const struct hda_fixup *fix, int action) { @@ -1824,6 +1905,163 @@ static void alc298_samsung_v2_init_amps(struct hda_codec *codec, spec->gen.pcm_playback_hook = alc298_samsung_v2_playback_hook; } +/* LG Gram Style 14: program vendor coef sequence used by HDA-verb workaround */ +struct alc298_lg_gram_style_seq { + unsigned short verb; + unsigned short idx; + unsigned short val; +}; + +static void alc298_lg_gram_style_coef_write(struct hda_codec *codec, + unsigned int verb, + unsigned int idx, + unsigned int val) +{ + snd_hda_codec_write(codec, 0x20, 0, AC_VERB_SET_COEF_INDEX, 0x23); + snd_hda_codec_write(codec, 0x20, 0, verb, idx); + snd_hda_codec_write(codec, 0x20, 0, AC_VERB_SET_PROC_COEF, 0x00); + snd_hda_codec_write(codec, 0x20, 0, AC_VERB_SET_PROC_COEF, val); + snd_hda_codec_write(codec, 0x20, 0, AC_VERB_SET_PROC_COEF, 0xb011); +} + +static void alc298_lg_gram_style_run_seq(struct hda_codec *codec, + const struct alc298_lg_gram_style_seq *seq, + int seq_size) +{ + int i; + + for (i = 0; i < seq_size; i++) + alc298_lg_gram_style_coef_write(codec, seq[i].verb, + seq[i].idx, seq[i].val); +} + +/* Coef sequences derived from the HDA-verb workaround for this model. */ +static const struct alc298_lg_gram_style_seq alc298_lg_gram_style_preinit_seq[] = { + { 0x420, 0x00, 0x01 }, +}; + +static const struct alc298_lg_gram_style_seq alc298_lg_gram_style_disable_seq[] = { + { 0x423, 0xff, 0x00 }, + { 0x420, 0x3a, 0x80 }, +}; + +static const struct alc298_lg_gram_style_seq alc298_lg_gram_style_enable_seq[] = { + { 0x420, 0x3a, 0x81 }, + { 0x423, 0xff, 0x01 }, +}; + +static const struct alc298_lg_gram_style_seq alc298_lg_gram_style_init_seq_38[] = { + { 0x423, 0xe1, 0x00 }, { 0x420, 0x12, 0x6f }, { 0x420, 0x14, 0x00 }, + { 0x420, 0x1b, 0x01 }, { 0x420, 0x1d, 0x01 }, { 0x420, 0x1f, 0xfe }, + { 0x420, 0x21, 0x00 }, { 0x420, 0x22, 0x10 }, { 0x420, 0x3d, 0x05 }, + { 0x420, 0x3f, 0x03 }, { 0x420, 0x50, 0x2c }, { 0x420, 0x76, 0x0e }, + { 0x420, 0x7c, 0x4a }, { 0x420, 0x81, 0x03 }, { 0x423, 0x99, 0x03 }, + { 0x423, 0xa4, 0xb5 }, { 0x423, 0xa5, 0x01 }, { 0x423, 0xba, 0x94 }, +}; + +static const struct alc298_lg_gram_style_seq alc298_lg_gram_style_init_seq_39[] = { + { 0x423, 0xe1, 0x00 }, { 0x420, 0x12, 0x6f }, { 0x420, 0x14, 0x00 }, + { 0x420, 0x1b, 0x02 }, { 0x420, 0x1d, 0x02 }, { 0x420, 0x1f, 0xfd }, + { 0x420, 0x21, 0x01 }, { 0x420, 0x22, 0x10 }, { 0x420, 0x3d, 0x05 }, + { 0x420, 0x3f, 0x03 }, { 0x420, 0x50, 0x2c }, { 0x420, 0x76, 0x0e }, + { 0x420, 0x7c, 0x4a }, { 0x420, 0x81, 0x03 }, { 0x423, 0x99, 0x03 }, + { 0x423, 0xa4, 0xb5 }, { 0x423, 0xa5, 0x01 }, { 0x423, 0xba, 0x94 }, +}; + +static const struct alc298_lg_gram_style_seq alc298_lg_gram_style_init_seq_3c[] = { + { 0x423, 0xe1, 0x00 }, { 0x420, 0x12, 0x6f }, { 0x420, 0x14, 0x00 }, + { 0x420, 0x1b, 0x01 }, { 0x420, 0x1d, 0x01 }, { 0x420, 0x1f, 0xfe }, + { 0x420, 0x21, 0x00 }, { 0x420, 0x22, 0x10 }, { 0x420, 0x3d, 0x05 }, + { 0x420, 0x3f, 0x03 }, { 0x420, 0x50, 0x2c }, { 0x420, 0x76, 0x0e }, + { 0x420, 0x7c, 0x4a }, { 0x420, 0x81, 0x03 }, { 0x423, 0xba, 0x8d }, +}; + +static const struct alc298_lg_gram_style_seq alc298_lg_gram_style_init_seq_3d[] = { + { 0x423, 0xe1, 0x00 }, { 0x420, 0x12, 0x6f }, { 0x420, 0x14, 0x00 }, + { 0x420, 0x1b, 0x02 }, { 0x420, 0x1d, 0x02 }, { 0x420, 0x1f, 0xfd }, + { 0x420, 0x21, 0x01 }, { 0x420, 0x22, 0x10 }, { 0x420, 0x3d, 0x05 }, + { 0x420, 0x3f, 0x03 }, { 0x420, 0x50, 0x2c }, { 0x420, 0x76, 0x0e }, + { 0x420, 0x7c, 0x4a }, { 0x420, 0x81, 0x03 }, { 0x423, 0xba, 0x8d }, +}; + +struct alc298_lg_gram_style_amp_desc { + unsigned char nid; + const struct alc298_lg_gram_style_seq *init_seq; + int init_seq_size; +}; + +static const struct alc298_lg_gram_style_amp_desc alc298_lg_gram_style_amps[] = { + { 0x38, alc298_lg_gram_style_init_seq_38, + ARRAY_SIZE(alc298_lg_gram_style_init_seq_38) }, + { 0x39, alc298_lg_gram_style_init_seq_39, + ARRAY_SIZE(alc298_lg_gram_style_init_seq_39) }, + { 0x3c, alc298_lg_gram_style_init_seq_3c, + ARRAY_SIZE(alc298_lg_gram_style_init_seq_3c) }, + { 0x3d, alc298_lg_gram_style_init_seq_3d, + ARRAY_SIZE(alc298_lg_gram_style_init_seq_3d) }, +}; + +static void alc298_lg_gram_style_enable_amps(struct hda_codec *codec) +{ + struct alc_spec *spec = codec->spec; + int i; + + for (i = 0; i < spec->num_speaker_amps; i++) { + alc_write_coef_idx(codec, 0x22, alc298_lg_gram_style_amps[i].nid); + alc298_lg_gram_style_run_seq(codec, + alc298_lg_gram_style_enable_seq, + ARRAY_SIZE(alc298_lg_gram_style_enable_seq)); + } +} + +static void alc298_lg_gram_style_disable_amps(struct hda_codec *codec) +{ + struct alc_spec *spec = codec->spec; + int i; + + for (i = 0; i < spec->num_speaker_amps; i++) { + alc_write_coef_idx(codec, 0x22, alc298_lg_gram_style_amps[i].nid); + alc298_lg_gram_style_run_seq(codec, + alc298_lg_gram_style_disable_seq, + ARRAY_SIZE(alc298_lg_gram_style_disable_seq)); + } +} + +static void alc298_lg_gram_style_playback_hook(struct hda_pcm_stream *hinfo, + struct hda_codec *codec, + struct snd_pcm_substream *substream, + int action) +{ + if (action == HDA_GEN_PCM_ACT_OPEN) + alc298_lg_gram_style_enable_amps(codec); + if (action == HDA_GEN_PCM_ACT_CLOSE) + alc298_lg_gram_style_disable_amps(codec); +} + +static void alc298_lg_gram_style_init_amps(struct hda_codec *codec) +{ + struct alc_spec *spec = codec->spec; + int i; + + spec->num_speaker_amps = ARRAY_SIZE(alc298_lg_gram_style_amps); + + for (i = 0; i < spec->num_speaker_amps; i++) { + alc_write_coef_idx(codec, 0x22, alc298_lg_gram_style_amps[i].nid); + alc298_lg_gram_style_run_seq(codec, + alc298_lg_gram_style_preinit_seq, + ARRAY_SIZE(alc298_lg_gram_style_preinit_seq)); + alc298_lg_gram_style_run_seq(codec, + alc298_lg_gram_style_disable_seq, + ARRAY_SIZE(alc298_lg_gram_style_disable_seq)); + alc298_lg_gram_style_run_seq(codec, + alc298_lg_gram_style_amps[i].init_seq, + alc298_lg_gram_style_amps[i].init_seq_size); + alc_write_coef_idx(codec, 0x89, 0x0); + } + + spec->gen.pcm_playback_hook = alc298_lg_gram_style_playback_hook; +} + static void alc298_fixup_samsung_amp_v2_2_amps(struct hda_codec *codec, const struct hda_fixup *fix, int action) { @@ -1838,6 +2076,13 @@ static void alc298_fixup_samsung_amp_v2_4_amps(struct hda_codec *codec, alc298_samsung_v2_init_amps(codec, 4); } +static void alc298_fixup_lg_gram_style_14(struct hda_codec *codec, + const struct hda_fixup *fix, int action) +{ + if (action == HDA_FIXUP_ACT_PROBE) + alc298_lg_gram_style_init_amps(codec); +} + static void gpio2_mic_hotkey_event(struct hda_codec *codec, struct hda_jack_callback *event) { @@ -1918,6 +2163,39 @@ static void alc280_fixup_hp_gpio2_mic_hotkey(struct hda_codec *codec, } } +/* GPIO2 = mic mute hotkey + * GPIO3 = mic mute LED + */ +static void alc233_fixup_lenovo_gpio2_mic_hotkey(struct hda_codec *codec, + const struct hda_fixup *fix, int action) +{ + struct alc_spec *spec = codec->spec; + + alc233_fixup_lenovo_coef_micmute_led(codec, fix, action); + if (action == HDA_FIXUP_ACT_PRE_PROBE) { + alc_update_coef_idx(codec, 0x10, 1<<2, 1<<2); + if (alc_register_micmute_input_device(codec) != 0) + return; + + spec->gpio_mask |= 0x04; + spec->gpio_dir |= 0x0; + snd_hda_codec_write_cache(codec, codec->core.afg, 0, + AC_VERB_SET_GPIO_UNSOLICITED_RSP_MASK, 0x04); + snd_hda_jack_detect_enable_callback(codec, codec->core.afg, + gpio2_mic_hotkey_event); + return; + } + + if (!spec->kb_dev) + return; + + switch (action) { + case HDA_FIXUP_ACT_FREE: + input_unregister_device(spec->kb_dev); + spec->kb_dev = NULL; + } +} + /* Line2 = mic mute hotkey * GPIO2 = mic mute LED */ @@ -2916,7 +3194,6 @@ static void find_cirrus_companion_amps(struct hda_codec *cdc) { struct device *dev = hda_codec_dev(cdc); struct acpi_device *adev; - struct fwnode_handle *fwnode __free(fwnode_handle) = NULL; const char *bus = NULL; static const struct { const char *hid; @@ -2946,7 +3223,8 @@ static void find_cirrus_companion_amps(struct hda_codec *cdc) bus = "spi"; } - fwnode = fwnode_handle_get(acpi_fwnode_handle(adev)); + struct fwnode_handle *fwnode __free(fwnode_handle) = + fwnode_handle_get(acpi_fwnode_handle(adev)); acpi_dev_put(adev); if (!bus) { @@ -3447,22 +3725,42 @@ static void alc245_tas2781_spi_hp_fixup_muteled(struct hda_codec *codec, alc_fixup_hp_gpio_led(codec, action, 0x04, 0x0); alc285_fixup_hp_coef_micmute_led(codec, fix, action); } + +static void alc245_hp_spk_mute_led_update(void *private_data, int enabled) +{ + struct hda_codec *codec = private_data; + unsigned int val; + + val = enabled ? 0x08 : 0x04; /* 0x08 led on, 0x04 led off */ + alc_update_coef_idx(codec, 0x0b, 0x0c, val); +} + /* JD2: mute led GPIO3: micmute led */ static void alc245_tas2781_i2c_hp_fixup_muteled(struct hda_codec *codec, const struct hda_fixup *fix, int action) { struct alc_spec *spec = codec->spec; + hda_nid_t hp_pin = alc_get_hp_pin(spec); static const hda_nid_t conn[] = { 0x02 }; switch (action) { case HDA_FIXUP_ACT_PRE_PROBE: + if (!hp_pin) { + spec->gen.vmaster_mute.hook = alc245_hp_spk_mute_led_update; + spec->gen.vmaster_mute_led = 1; + } spec->gen.auto_mute_via_amp = 1; snd_hda_override_conn_list(codec, 0x17, ARRAY_SIZE(conn), conn); break; + case HDA_FIXUP_ACT_INIT: + if (!hp_pin) + alc245_hp_spk_mute_led_update(codec, !spec->gen.master_mute); + break; } tas2781_fixup_txnw_i2c(codec, fix, action); - alc245_fixup_hp_mute_led_coefbit(codec, fix, action); + if (hp_pin) + alc245_fixup_hp_mute_led_coefbit(codec, fix, action); alc285_fixup_hp_coef_micmute_led(codec, fix, action); } /* @@ -3652,6 +3950,7 @@ enum { ALC294_FIXUP_ASUS_MIC, ALC294_FIXUP_ASUS_HEADSET_MIC, ALC294_FIXUP_ASUS_I2C_HEADSET_MIC, + ALC294_FIXUP_ASUS_SPI_HEADSET_MIC, ALC294_FIXUP_ASUS_SPK, ALC293_FIXUP_SYSTEM76_MIC_NO_PRESENCE, ALC285_FIXUP_LENOVO_PC_BEEP_IN_NOISE, @@ -3692,6 +3991,7 @@ enum { ALC285_FIXUP_HP_GPIO_LED, ALC285_FIXUP_HP_MUTE_LED, ALC285_FIXUP_HP_SPECTRE_X360_MUTE_LED, + ALC245_FIXUP_HP_ENVY_X360_MUTE_LED, ALC285_FIXUP_HP_BEEP_MICMUTE_LED, ALC236_FIXUP_HP_MUTE_LED_COEFBIT2, ALC236_FIXUP_HP_GPIO_LED, @@ -3701,6 +4001,7 @@ enum { ALC298_FIXUP_SAMSUNG_AMP, ALC298_FIXUP_SAMSUNG_AMP_V2_2_AMPS, ALC298_FIXUP_SAMSUNG_AMP_V2_4_AMPS, + ALC298_FIXUP_LG_GRAM_STYLE_14, ALC298_FIXUP_SAMSUNG_HEADPHONE_VERY_QUIET, ALC256_FIXUP_SAMSUNG_HEADPHONE_VERY_QUIET, ALC295_FIXUP_ASUS_MIC_NO_PRESENCE, @@ -3781,6 +4082,7 @@ enum { ALC287_FIXUP_YOGA7_14ARB7_I2C, ALC245_FIXUP_HP_MUTE_LED_COEFBIT, ALC245_FIXUP_HP_MUTE_LED_V1_COEFBIT, + ALC245_FIXUP_HP_MUTE_LED_V2_COEFBIT, ALC245_FIXUP_HP_X360_MUTE_LEDS, ALC287_FIXUP_THINKPAD_I2S_SPK, ALC287_FIXUP_MG_RTKC_CSAMP_CS35L41_I2C_THINKPAD, @@ -3802,6 +4104,7 @@ enum { ALC245_FIXUP_CLEVO_NOISY_MIC, ALC269_FIXUP_VAIO_VJFH52_MIC_NO_PRESENCE, ALC233_FIXUP_MEDION_MTL_SPK, + ALC233_FIXUP_STARLABS_STARFIGHTER, ALC294_FIXUP_BASS_SPEAKER_15, ALC283_FIXUP_DELL_HP_RESUME, ALC294_FIXUP_ASUS_CS35L41_SPI_2, @@ -3816,6 +4119,10 @@ enum { ALC245_FIXUP_HP_TAS2781_I2C_MUTE_LED, ALC288_FIXUP_SURFACE_SWAP_DACS, ALC236_FIXUP_HP_MUTE_LED_MICMUTE_GPIO, + ALC233_FIXUP_LENOVO_GPIO2_MIC_HOTKEY, + ALC245_FIXUP_BASS_HP_DAC, + ALC245_FIXUP_ACER_MICMUTE_LED, + ALC245_FIXUP_CS35L41_I2C_2_MUTE_LED, }; /* A special fixup for Lenovo C940 and Yoga Duet 7; @@ -4997,6 +5304,15 @@ static const struct hda_fixup alc269_fixups[] = { .chained = true, .chain_id = ALC287_FIXUP_CS35L41_I2C_2 }, + [ALC294_FIXUP_ASUS_SPI_HEADSET_MIC] = { + .type = HDA_FIXUP_PINS, + .v.pins = (const struct hda_pintbl[]) { + { 0x19, 0x04a11020 }, /* use as headset mic */ + { } + }, + .chained = true, + .chain_id = ALC245_FIXUP_CS35L41_SPI_2 + }, [ALC294_FIXUP_ASUS_SPK] = { .type = HDA_FIXUP_VERBS, .v.verbs = (const struct hda_verb[]) { @@ -5344,6 +5660,10 @@ static const struct hda_fixup alc269_fixups[] = { .type = HDA_FIXUP_FUNC, .v.func = alc285_fixup_hp_spectre_x360_mute_led, }, + [ALC245_FIXUP_HP_ENVY_X360_MUTE_LED] = { + .type = HDA_FIXUP_FUNC, + .v.func = alc245_fixup_hp_envy_x360_mute_led, + }, [ALC285_FIXUP_HP_BEEP_MICMUTE_LED] = { .type = HDA_FIXUP_FUNC, .v.func = alc285_fixup_hp_beep, @@ -5394,6 +5714,10 @@ static const struct hda_fixup alc269_fixups[] = { .type = HDA_FIXUP_FUNC, .v.func = alc298_fixup_samsung_amp_v2_4_amps }, + [ALC298_FIXUP_LG_GRAM_STYLE_14] = { + .type = HDA_FIXUP_FUNC, + .v.func = alc298_fixup_lg_gram_style_14 + }, [ALC298_FIXUP_SAMSUNG_HEADPHONE_VERY_QUIET] = { .type = HDA_FIXUP_VERBS, .v.verbs = (const struct hda_verb[]) { @@ -6117,6 +6441,10 @@ static const struct hda_fixup alc269_fixups[] = { .type = HDA_FIXUP_FUNC, .v.func = alc245_fixup_hp_mute_led_v1_coefbit, }, + [ALC245_FIXUP_HP_MUTE_LED_V2_COEFBIT] = { + .type = HDA_FIXUP_FUNC, + .v.func = alc245_fixup_hp_mute_led_v2_coefbit, + }, [ALC245_FIXUP_HP_X360_MUTE_LEDS] = { .type = HDA_FIXUP_FUNC, .v.func = alc245_fixup_hp_mute_led_coefbit, @@ -6238,6 +6566,10 @@ static const struct hda_fixup alc269_fixups[] = { { } }, }, + [ALC233_FIXUP_STARLABS_STARFIGHTER] = { + .type = HDA_FIXUP_FUNC, + .v.func = alc233_fixup_starlabs_starfighter, + }, [ALC294_FIXUP_BASS_SPEAKER_15] = { .type = HDA_FIXUP_FUNC, .v.func = alc294_fixup_bass_speaker_15, @@ -6306,6 +6638,27 @@ static const struct hda_fixup alc269_fixups[] = { .type = HDA_FIXUP_FUNC, .v.func = alc288_fixup_surface_swap_dacs, }, + [ALC233_FIXUP_LENOVO_GPIO2_MIC_HOTKEY] = { + .type = HDA_FIXUP_FUNC, + .v.func = alc233_fixup_lenovo_gpio2_mic_hotkey, + }, + [ALC245_FIXUP_BASS_HP_DAC] = { + .type = HDA_FIXUP_FUNC, + /* Borrow the DAC routing selected for those Thinkpads */ + .v.func = alc285_fixup_thinkpad_x1_gen7, + }, + [ALC245_FIXUP_ACER_MICMUTE_LED] = { + .type = HDA_FIXUP_FUNC, + .v.func = alc285_fixup_hp_coef_micmute_led, + .chained = true, + .chain_id = ALC2XX_FIXUP_HEADSET_MIC, + }, + [ALC245_FIXUP_CS35L41_I2C_2_MUTE_LED] = { + .type = HDA_FIXUP_FUNC, + .v.func = alc245_fixup_hp_mute_led_coefbit, + .chained = true, + .chain_id = ALC287_FIXUP_CS35L41_I2C_2, + } }; static const struct hda_quirk alc269_fixup_tbl[] = { @@ -6321,6 +6674,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x1025, 0x079b, "Acer Aspire V5-573G", ALC282_FIXUP_ASPIRE_V5_PINS), SND_PCI_QUIRK(0x1025, 0x080d, "Acer Aspire V5-122P", ALC269_FIXUP_ASPIRE_HEADSET_MIC), SND_PCI_QUIRK(0x1025, 0x0840, "Acer Aspire E1", ALC269VB_FIXUP_ASPIRE_E1_COEF), + SND_PCI_QUIRK(0x1025, 0x0943, "Acer Aspire V3-572G", ALC269_FIXUP_ASPIRE_HEADSET_MIC), SND_PCI_QUIRK(0x1025, 0x100c, "Acer Aspire E5-574G", ALC255_FIXUP_ACER_LIMIT_INT_MIC_BOOST), SND_PCI_QUIRK(0x1025, 0x101c, "Acer Veriton N2510G", ALC269_FIXUP_LIFEBOOK), SND_PCI_QUIRK(0x1025, 0x102b, "Acer Aspire C24-860", ALC286_FIXUP_ACER_AIO_MIC_NO_PRESENCE), @@ -6357,6 +6711,8 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x1025, 0x159c, "Acer Nitro 5 AN515-58", ALC2XX_FIXUP_HEADSET_MIC), SND_PCI_QUIRK(0x1025, 0x1597, "Acer Nitro 5 AN517-55", ALC2XX_FIXUP_HEADSET_MIC), SND_PCI_QUIRK(0x1025, 0x169a, "Acer Swift SFG16", ALC256_FIXUP_ACER_SFG16_MICMUTE_LED), + SND_PCI_QUIRK(0x1025, 0x171e, "Acer Nitro ANV15-51", ALC245_FIXUP_ACER_MICMUTE_LED), + SND_PCI_QUIRK(0x1025, 0x173a, "Acer Swift SFG14-73", ALC245_FIXUP_ACER_MICMUTE_LED), SND_PCI_QUIRK(0x1025, 0x1826, "Acer Helios ZPC", ALC287_FIXUP_PREDATOR_SPK_CS35L41_I2C_2), SND_PCI_QUIRK(0x1025, 0x182c, "Acer Helios ZPD", ALC287_FIXUP_PREDATOR_SPK_CS35L41_I2C_2), SND_PCI_QUIRK(0x1025, 0x1844, "Acer Helios ZPS", ALC287_FIXUP_PREDATOR_SPK_CS35L41_I2C_2), @@ -6552,6 +6908,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x103c, 0x8730, "HP ProBook 445 G7", ALC236_FIXUP_HP_MUTE_LED_MICMUTE_VREF), SND_PCI_QUIRK(0x103c, 0x8735, "HP ProBook 435 G7", ALC236_FIXUP_HP_MUTE_LED_MICMUTE_VREF), SND_PCI_QUIRK(0x103c, 0x8736, "HP", ALC285_FIXUP_HP_GPIO_AMP_INIT), + SND_PCI_QUIRK(0x103c, 0x8756, "HP ENVY Laptop 13-ba0xxx", ALC245_FIXUP_HP_X360_MUTE_LEDS), SND_PCI_QUIRK(0x103c, 0x8760, "HP EliteBook 8{4,5}5 G7", ALC285_FIXUP_HP_BEEP_MICMUTE_LED), SND_PCI_QUIRK(0x103c, 0x876e, "HP ENVY x360 Convertible 13-ay0xxx", ALC245_FIXUP_HP_X360_MUTE_LEDS), SND_PCI_QUIRK(0x103c, 0x877a, "HP", ALC285_FIXUP_HP_MUTE_LED), @@ -6565,6 +6922,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x103c, 0x8788, "HP OMEN 15", ALC285_FIXUP_HP_MUTE_LED), SND_PCI_QUIRK(0x103c, 0x87b7, "HP Laptop 14-fq0xxx", ALC236_FIXUP_HP_MUTE_LED_COEFBIT2), SND_PCI_QUIRK(0x103c, 0x87c8, "HP", ALC287_FIXUP_HP_GPIO_LED), + SND_PCI_QUIRK(0x103c, 0x87cb, "HP Pavilion 15-eg0xxx", ALC287_FIXUP_HP_GPIO_LED), SND_PCI_QUIRK(0x103c, 0x87cc, "HP Pavilion 15-eg0xxx", ALC287_FIXUP_HP_GPIO_LED), SND_PCI_QUIRK(0x103c, 0x87d3, "HP Laptop 15-gw0xxx", ALC236_FIXUP_HP_MUTE_LED_COEFBIT2), SND_PCI_QUIRK(0x103c, 0x87df, "HP ProBook 430 G8 Notebook PC", ALC236_FIXUP_HP_GPIO_LED), @@ -6600,8 +6958,11 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x103c, 0x8895, "HP EliteBook 855 G8 Notebook PC", ALC285_FIXUP_HP_SPEAKERS_MICMUTE_LED), SND_PCI_QUIRK(0x103c, 0x8896, "HP EliteBook 855 G8 Notebook PC", ALC285_FIXUP_HP_MUTE_LED), SND_PCI_QUIRK(0x103c, 0x8898, "HP EliteBook 845 G8 Notebook PC", ALC285_FIXUP_HP_LIMIT_INT_MIC_BOOST), + SND_PCI_QUIRK(0x103c, 0x88b3, "HP ENVY x360 Convertible 15-es0xxx", ALC245_FIXUP_HP_ENVY_X360_MUTE_LED), SND_PCI_QUIRK(0x103c, 0x88d0, "HP Pavilion 15-eh1xxx (mainboard 88D0)", ALC287_FIXUP_HP_GPIO_LED), + SND_PCI_QUIRK(0x103c, 0x88d1, "HP Pavilion 15-eh1xxx (mainboard 88D1)", ALC245_FIXUP_HP_MUTE_LED_V1_COEFBIT), SND_PCI_QUIRK(0x103c, 0x88dd, "HP Pavilion 15z-ec200", ALC285_FIXUP_HP_MUTE_LED), + SND_PCI_QUIRK(0x103c, 0x88eb, "HP Victus 16-e0xxx", ALC245_FIXUP_HP_MUTE_LED_V2_COEFBIT), SND_PCI_QUIRK(0x103c, 0x8902, "HP OMEN 16", ALC285_FIXUP_HP_MUTE_LED), SND_PCI_QUIRK(0x103c, 0x890e, "HP 255 G8 Notebook PC", ALC236_FIXUP_HP_MUTE_LED_COEFBIT2), SND_PCI_QUIRK(0x103c, 0x8919, "HP Pavilion Aero Laptop 13-be0xxx", ALC287_FIXUP_HP_GPIO_LED), @@ -6635,6 +6996,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x103c, 0x89da, "HP Spectre x360 14t-ea100", ALC245_FIXUP_HP_SPECTRE_X360_EU0XXX), SND_PCI_QUIRK(0x103c, 0x89e7, "HP Elite x2 G9", ALC245_FIXUP_CS35L41_SPI_2_HP_GPIO_LED), SND_PCI_QUIRK(0x103c, 0x8a0f, "HP Pavilion 14-ec1xxx", ALC287_FIXUP_HP_GPIO_LED), + SND_PCI_QUIRK(0x103c, 0x8a1f, "HP Laptop 14s-dr5xxx", ALC236_FIXUP_HP_MUTE_LED_COEFBIT2), SND_PCI_QUIRK(0x103c, 0x8a20, "HP Laptop 15s-fq5xxx", ALC236_FIXUP_HP_MUTE_LED_COEFBIT2), SND_PCI_QUIRK(0x103c, 0x8a25, "HP Victus 16-d1xxx (MB 8A25)", ALC245_FIXUP_HP_MUTE_LED_COEFBIT), SND_PCI_QUIRK(0x103c, 0x8a26, "HP Victus 16-d1xxx (MB 8A26)", ALC245_FIXUP_HP_MUTE_LED_COEFBIT), @@ -6648,6 +7010,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x103c, 0x8a30, "HP Envy 17", ALC287_FIXUP_CS35L41_I2C_2), SND_PCI_QUIRK(0x103c, 0x8a31, "HP Envy 15", ALC287_FIXUP_CS35L41_I2C_2), SND_PCI_QUIRK(0x103c, 0x8a34, "HP Pavilion x360 2-in-1 Laptop 14-ek0xxx", ALC245_FIXUP_HP_MUTE_LED_COEFBIT), + SND_PCI_QUIRK(0x103c, 0x8a3d, "HP Victus 15-fb0xxx (MB 8A3D)", ALC245_FIXUP_HP_MUTE_LED_V2_COEFBIT), SND_PCI_QUIRK(0x103c, 0x8a4f, "HP Victus 15-fa0xxx (MB 8A4F)", ALC245_FIXUP_HP_MUTE_LED_COEFBIT), SND_PCI_QUIRK(0x103c, 0x8a6e, "HP EDNA 360", ALC287_FIXUP_CS35L41_I2C_4), SND_PCI_QUIRK(0x103c, 0x8a74, "HP ProBook 440 G8 Notebook PC", ALC236_FIXUP_HP_GPIO_LED), @@ -6792,6 +7155,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x103c, 0x8da7, "HP 14 Enstrom OmniBook X", ALC287_FIXUP_CS35L41_I2C_2), SND_PCI_QUIRK(0x103c, 0x8da8, "HP 16 Piston OmniBook X", ALC287_FIXUP_CS35L41_I2C_2), SND_PCI_QUIRK(0x103c, 0x8dd4, "HP EliteStudio 8 AIO", ALC274_FIXUP_HP_AIO_BIND_DACS), + SND_PCI_QUIRK(0x103c, 0x8dd7, "HP Laptop 15-fd0xxx", ALC236_FIXUP_HP_MUTE_LED_COEFBIT2), SND_PCI_QUIRK(0x103c, 0x8de8, "HP Gemtree", ALC245_FIXUP_TAS2781_SPI_2), SND_PCI_QUIRK(0x103c, 0x8de9, "HP Gemtree", ALC245_FIXUP_TAS2781_SPI_2), SND_PCI_QUIRK(0x103c, 0x8dec, "HP EliteBook 640 G12", ALC236_FIXUP_HP_GPIO_LED), @@ -6822,7 +7186,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x103c, 0x8e37, "HP 16 Piston OmniBook X", ALC287_FIXUP_CS35L41_I2C_2), SND_PCI_QUIRK(0x103c, 0x8e3a, "HP Agusta", ALC287_FIXUP_CS35L41_I2C_2), SND_PCI_QUIRK(0x103c, 0x8e3b, "HP Agusta", ALC287_FIXUP_CS35L41_I2C_2), - SND_PCI_QUIRK(0x103c, 0x8e60, "HP Trekker ", ALC287_FIXUP_CS35L41_I2C_2), + SND_PCI_QUIRK(0x103c, 0x8e60, "HP OmniBook 7 Laptop 16-bh0xxx", ALC245_FIXUP_CS35L41_I2C_2_MUTE_LED), SND_PCI_QUIRK(0x103c, 0x8e61, "HP Trekker ", ALC287_FIXUP_CS35L41_I2C_2), SND_PCI_QUIRK(0x103c, 0x8e62, "HP Trekker ", ALC287_FIXUP_CS35L41_I2C_2), SND_PCI_QUIRK(0x103c, 0x8e8a, "HP NexusX", ALC245_FIXUP_HP_TAS2781_I2C_MUTE_LED), @@ -6871,6 +7235,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x1043, 0x115d, "Asus 1015E", ALC269_FIXUP_LIMIT_INT_MIC_BOOST), SND_PCI_QUIRK(0x1043, 0x1194, "ASUS UM3406KA", ALC287_FIXUP_CS35L41_I2C_2), SND_PCI_QUIRK(0x1043, 0x11c0, "ASUS X556UR", ALC255_FIXUP_ASUS_MIC_NO_PRESENCE), + HDA_CODEC_QUIRK(0x1043, 0x1204, "ASUS Strix G16 G615JMR", ALC287_FIXUP_TXNW2781_I2C_ASUS), SND_PCI_QUIRK(0x1043, 0x1204, "ASUS Strix G615JHR_JMR_JPR", ALC287_FIXUP_TAS2781_I2C), SND_PCI_QUIRK(0x1043, 0x1214, "ASUS Strix G615LH_LM_LP", ALC287_FIXUP_TAS2781_I2C), SND_PCI_QUIRK(0x1043, 0x125e, "ASUS Q524UQK", ALC255_FIXUP_ASUS_MIC_NO_PRESENCE), @@ -6900,10 +7265,12 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x1043, 0x14e3, "ASUS G513PI/PU/PV", ALC287_FIXUP_CS35L41_I2C_2), SND_PCI_QUIRK(0x1043, 0x14f2, "ASUS VivoBook X515JA", ALC256_FIXUP_ASUS_MIC_NO_PRESENCE), SND_PCI_QUIRK(0x1043, 0x1503, "ASUS G733PY/PZ/PZV/PYV", ALC287_FIXUP_CS35L41_I2C_2), + SND_PCI_QUIRK(0x1043, 0x1514, "ASUS ROG Flow Z13 GZ302EAC", ALC287_FIXUP_CS35L41_I2C_2), SND_PCI_QUIRK(0x1043, 0x1517, "Asus Zenbook UX31A", ALC269VB_FIXUP_ASUS_ZENBOOK_UX31A), SND_PCI_QUIRK(0x1043, 0x1533, "ASUS GV302XA/XJ/XQ/XU/XV/XI", ALC287_FIXUP_CS35L41_I2C_2), SND_PCI_QUIRK(0x1043, 0x1573, "ASUS GZ301VV/VQ/VU/VJ/VA/VC/VE/VVC/VQC/VUC/VJC/VEC/VCC", ALC285_FIXUP_ASUS_HEADSET_MIC), SND_PCI_QUIRK(0x1043, 0x1584, "ASUS UM3406GA ", ALC287_FIXUP_CS35L41_I2C_2), + SND_PCI_QUIRK(0x1043, 0x1602, "ASUS ROG Strix SCAR 15", ALC285_FIXUP_ASUS_G533Z_PINS), SND_PCI_QUIRK(0x1043, 0x1652, "ASUS ROG Zephyrus Do 15 SE", ALC289_FIXUP_ASUS_ZEPHYRUS_DUAL_SPK), SND_PCI_QUIRK(0x1043, 0x1662, "ASUS GV301QH", ALC294_FIXUP_ASUS_DUAL_SPK), SND_PCI_QUIRK(0x1043, 0x1663, "ASUS GU603ZI/ZJ/ZQ/ZU/ZV", ALC285_FIXUP_ASUS_HEADSET_MIC), @@ -6927,7 +7294,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x1043, 0x19ce, "ASUS B9450FA", ALC294_FIXUP_ASUS_HPE), SND_PCI_QUIRK(0x1043, 0x19e1, "ASUS UX581LV", ALC295_FIXUP_ASUS_MIC_NO_PRESENCE), SND_PCI_QUIRK(0x1043, 0x1a13, "Asus G73Jw", ALC269_FIXUP_ASUS_G73JW), - SND_PCI_QUIRK(0x1043, 0x1a63, "ASUS UX3405MA", ALC245_FIXUP_CS35L41_SPI_2), + SND_PCI_QUIRK(0x1043, 0x1a63, "ASUS UX3405MA", ALC294_FIXUP_ASUS_SPI_HEADSET_MIC), SND_PCI_QUIRK(0x1043, 0x1a83, "ASUS UM5302LA", ALC294_FIXUP_CS35L41_I2C_2), SND_PCI_QUIRK(0x1043, 0x1a8e, "ASUS G712LWS", ALC294_FIXUP_LENOVO_MIC_LOCATION), SND_PCI_QUIRK(0x1043, 0x1a8f, "ASUS UX582ZS", ALC245_FIXUP_CS35L41_SPI_2), @@ -6968,6 +7335,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x1043, 0x1e93, "ASUS ExpertBook B9403CVAR", ALC294_FIXUP_ASUS_HPE), SND_PCI_QUIRK(0x1043, 0x1eb3, "ASUS Ally RCLA72", ALC287_FIXUP_TAS2781_I2C), SND_PCI_QUIRK(0x1043, 0x1ed3, "ASUS HN7306W", ALC287_FIXUP_CS35L41_I2C_2), + HDA_CODEC_QUIRK(0x1043, 0x1ee2, "ASUS UM6702RA/RC", ALC285_FIXUP_ASUS_I2C_SPEAKER2_TO_DAC1), SND_PCI_QUIRK(0x1043, 0x1ee2, "ASUS UM6702RA/RC", ALC287_FIXUP_CS35L41_I2C_2), SND_PCI_QUIRK(0x1043, 0x1c52, "ASUS Zephyrus G15 2022", ALC289_FIXUP_ASUS_GA401), SND_PCI_QUIRK(0x1043, 0x1f11, "ASUS Zephyrus G14", ALC289_FIXUP_ASUS_GA401), @@ -7039,8 +7407,10 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x144d, 0xc109, "Samsung Ativ book 9 (NP900X3G)", ALC269_FIXUP_INV_DMIC), SND_PCI_QUIRK(0x144d, 0xc169, "Samsung Notebook 9 Pen (NP930SBE-K01US)", ALC298_FIXUP_SAMSUNG_AMP), SND_PCI_QUIRK(0x144d, 0xc176, "Samsung Notebook 9 Pro (NP930MBE-K04US)", ALC298_FIXUP_SAMSUNG_AMP), - SND_PCI_QUIRK(0x144d, 0xc189, "Samsung Galaxy Flex Book (NT950QCG-X716)", ALC298_FIXUP_SAMSUNG_AMP), + SND_PCI_QUIRK(0x144d, 0xc188, "Samsung Galaxy Book Flex (NT950QCT-A38A)", ALC298_FIXUP_SAMSUNG_AMP), + SND_PCI_QUIRK(0x144d, 0xc189, "Samsung Galaxy Book Flex (NT950QCG-X716)", ALC298_FIXUP_SAMSUNG_AMP), SND_PCI_QUIRK(0x144d, 0xc18a, "Samsung Galaxy Book Ion (NP930XCJ-K01US)", ALC298_FIXUP_SAMSUNG_AMP), + SND_PCI_QUIRK(0x144d, 0xc1ac, "Samsung Galaxy Book2 Pro 360 (NP950QED)", ALC298_FIXUP_SAMSUNG_AMP_V2_2_AMPS), SND_PCI_QUIRK(0x144d, 0xc1a3, "Samsung Galaxy Book Pro (NP935XDB-KC1SE)", ALC298_FIXUP_SAMSUNG_AMP), SND_PCI_QUIRK(0x144d, 0xc1a4, "Samsung Galaxy Book Pro 360 (NT935QBD)", ALC298_FIXUP_SAMSUNG_AMP), SND_PCI_QUIRK(0x144d, 0xc1a6, "Samsung Galaxy Book Pro 360 (NP930QBD)", ALC298_FIXUP_SAMSUNG_AMP), @@ -7056,7 +7426,9 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x144d, 0xc872, "Samsung Galaxy Book2 Pro (NP950XEE)", ALC298_FIXUP_SAMSUNG_AMP_V2_2_AMPS), SND_PCI_QUIRK(0x144d, 0xc886, "Samsung Galaxy Book3 Pro (NP964XFG)", ALC298_FIXUP_SAMSUNG_AMP_V2_4_AMPS), SND_PCI_QUIRK(0x144d, 0xc1ca, "Samsung Galaxy Book3 Pro 360 (NP960QFG)", ALC298_FIXUP_SAMSUNG_AMP_V2_4_AMPS), + SND_PCI_QUIRK(0x144d, 0xc1cb, "Samsung Galaxy Book3 Pro 360 (NP965QFG)", ALC298_FIXUP_SAMSUNG_AMP_V2_4_AMPS), SND_PCI_QUIRK(0x144d, 0xc1cc, "Samsung Galaxy Book3 Ultra (NT960XFH)", ALC298_FIXUP_SAMSUNG_AMP_V2_4_AMPS), + SND_PCI_QUIRK(0x1458, 0x900e, "Gigabyte G5 KF5 (2023)", ALC2XX_FIXUP_HEADSET_MIC), SND_PCI_QUIRK(0x1458, 0xfa53, "Gigabyte BXBT-2807", ALC283_FIXUP_HEADSET_MIC), SND_PCI_QUIRK(0x1462, 0xb120, "MSI Cubi MS-B120", ALC283_FIXUP_HEADSET_MIC), SND_PCI_QUIRK(0x1462, 0xb171, "Cubi N 8GL (MS-B171)", ALC283_FIXUP_HEADSET_MIC), @@ -7185,6 +7557,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x17aa, 0x224c, "Thinkpad", ALC298_FIXUP_TPT470_DOCK), SND_PCI_QUIRK(0x17aa, 0x224d, "Thinkpad", ALC298_FIXUP_TPT470_DOCK), SND_PCI_QUIRK(0x17aa, 0x225d, "Thinkpad T480", ALC269_FIXUP_LIMIT_INT_MIC_BOOST), + SND_PCI_QUIRK(0x17aa, 0x2288, "Thinkpad X390", ALC285_FIXUP_THINKPAD_NO_BASS_SPK_HEADSET_JACK), SND_PCI_QUIRK(0x17aa, 0x2292, "Thinkpad X1 Carbon 7th", ALC285_FIXUP_THINKPAD_HEADSET_JACK), SND_PCI_QUIRK(0x17aa, 0x22be, "Thinkpad X1 Carbon 8th", ALC285_FIXUP_THINKPAD_HEADSET_JACK), SND_PCI_QUIRK(0x17aa, 0x22c1, "Thinkpad P1 Gen 3", ALC285_FIXUP_THINKPAD_NO_BASS_SPK_HEADSET_JACK), @@ -7211,7 +7584,12 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x17aa, 0x3176, "ThinkCentre Station", ALC283_FIXUP_HEADSET_MIC), SND_PCI_QUIRK(0x17aa, 0x3178, "ThinkCentre Station", ALC283_FIXUP_HEADSET_MIC), SND_PCI_QUIRK(0x17aa, 0x31af, "ThinkCentre Station", ALC623_FIXUP_LENOVO_THINKSTATION_P340), + SND_PCI_QUIRK(0x17aa, 0x3341, "Lenovo ThinkCentre M90 Gen4", ALC233_FIXUP_LENOVO_GPIO2_MIC_HOTKEY), + SND_PCI_QUIRK(0x17aa, 0x3342, "Lenovo ThinkCentre M90 Gen4", ALC233_FIXUP_LENOVO_GPIO2_MIC_HOTKEY), + SND_PCI_QUIRK(0x17aa, 0x3343, "Lenovo ThinkCentre M70 Gen4", ALC233_FIXUP_LENOVO_GPIO2_MIC_HOTKEY), + SND_PCI_QUIRK(0x17aa, 0x3344, "Lenovo ThinkCentre M70 Gen4", ALC233_FIXUP_LENOVO_GPIO2_MIC_HOTKEY), SND_PCI_QUIRK(0x17aa, 0x334b, "Lenovo ThinkCentre M70 Gen5", ALC283_FIXUP_HEADSET_MIC), + SND_PCI_QUIRK(0x17aa, 0x334f, "Lenovo ThinkCentre M90a Gen5", ALC233_FIXUP_LENOVO_GPIO2_MIC_HOTKEY), SND_PCI_QUIRK(0x17aa, 0x3384, "ThinkCentre M90a PRO", ALC233_FIXUP_LENOVO_L2MH_LOW_ENLED), SND_PCI_QUIRK(0x17aa, 0x3386, "ThinkCentre M90a Gen6", ALC233_FIXUP_LENOVO_L2MH_LOW_ENLED), SND_PCI_QUIRK(0x17aa, 0x3387, "ThinkCentre M70a Gen6", ALC233_FIXUP_LENOVO_L2MH_LOW_ENLED), @@ -7228,6 +7606,10 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x17aa, 0x3834, "Lenovo IdeaPad Slim 9i 14ITL5", ALC287_FIXUP_YOGA7_14ITL_SPEAKERS), SND_PCI_QUIRK(0x17aa, 0x383d, "Legion Y9000X 2019", ALC285_FIXUP_LEGION_Y9000X_SPEAKERS), SND_PCI_QUIRK(0x17aa, 0x3843, "Lenovo Yoga 9i / Yoga Book 9i", ALC287_FIXUP_LENOVO_YOGA_BOOK_9I), + /* Yoga Pro 7 14IMH9 shares PCI SSID 17aa:3847 with Legion 7 16ACHG6; + * use codec SSID to distinguish them + */ + HDA_CODEC_QUIRK(0x17aa, 0x38cf, "Lenovo Yoga Pro 7 14IMH9", ALC287_FIXUP_YOGA9_14IMH9_BASS_SPK_PIN), SND_PCI_QUIRK(0x17aa, 0x3847, "Legion 7 16ACHG6", ALC287_FIXUP_LEGION_16ACHG6), SND_PCI_QUIRK(0x17aa, 0x384a, "Lenovo Yoga 7 15ITL5", ALC287_FIXUP_YOGA7_14ITL_SPEAKERS), SND_PCI_QUIRK(0x17aa, 0x3852, "Lenovo Yoga 7 14ITL5", ALC287_FIXUP_YOGA7_14ITL_SPEAKERS), @@ -7259,6 +7641,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x17aa, 0x38ab, "Thinkbook 16P", ALC287_FIXUP_MG_RTKC_CSAMP_CS35L41_I2C_THINKPAD), SND_PCI_QUIRK(0x17aa, 0x38b4, "Legion Slim 7 16IRH8", ALC287_FIXUP_CS35L41_I2C_2), HDA_CODEC_QUIRK(0x17aa, 0x391c, "Lenovo Yoga 7 2-in-1 14AKP10", ALC287_FIXUP_YOGA9_14IAP7_BASS_SPK_PIN), + HDA_CODEC_QUIRK(0x17aa, 0x391d, "Lenovo Yoga 7 2-in-1 16AKP10", ALC287_FIXUP_YOGA9_14IAP7_BASS_SPK_PIN), SND_PCI_QUIRK(0x17aa, 0x38b5, "Legion Slim 7 16IRH8", ALC287_FIXUP_CS35L41_I2C_2), SND_PCI_QUIRK(0x17aa, 0x38b6, "Legion Slim 7 16APH8", ALC287_FIXUP_CS35L41_I2C_2), SND_PCI_QUIRK(0x17aa, 0x38b7, "Legion Slim 7 16APH8", ALC287_FIXUP_CS35L41_I2C_2), @@ -7288,7 +7671,9 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x17aa, 0x38fd, "ThinkBook plus Gen5 Hybrid", ALC287_FIXUP_TAS2781_I2C), SND_PCI_QUIRK(0x17aa, 0x3902, "Lenovo E50-80", ALC269_FIXUP_DMIC_THINKPAD_ACPI), SND_PCI_QUIRK(0x17aa, 0x390d, "Lenovo Yoga Pro 7 14ASP10", ALC287_FIXUP_YOGA9_14IAP7_BASS_SPK_PIN), + SND_PCI_QUIRK(0x17aa, 0x3911, "Lenovo Yoga Pro 7 14IAH10", ALC287_FIXUP_YOGA9_14IAP7_BASS_SPK_PIN), SND_PCI_QUIRK(0x17aa, 0x3913, "Lenovo 145", ALC236_FIXUP_LENOVO_INV_DMIC), + SND_PCI_QUIRK(0x17aa, 0x391a, "Lenovo Yoga Slim 7 14AKP10", ALC287_FIXUP_YOGA9_14IAP7_BASS_SPK_PIN), SND_PCI_QUIRK(0x17aa, 0x391f, "Yoga S990-16 pro Quad YC Quad", ALC287_FIXUP_TXNW2781_I2C), SND_PCI_QUIRK(0x17aa, 0x3920, "Yoga S990-16 pro Quad VECO Quad", ALC287_FIXUP_TXNW2781_I2C), SND_PCI_QUIRK(0x17aa, 0x3929, "Thinkbook 13x Gen 5", ALC287_FIXUP_MG_RTKC_CSAMP_CS35L41_I2C_THINKPAD), @@ -7325,6 +7710,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x1854, 0x0488, "LG gram 16 (16Z90R)", ALC298_FIXUP_SAMSUNG_AMP_V2_4_AMPS), SND_PCI_QUIRK(0x1854, 0x0489, "LG gram 16 (16Z90R-A)", ALC298_FIXUP_SAMSUNG_AMP_V2_4_AMPS), SND_PCI_QUIRK(0x1854, 0x048a, "LG gram 17 (17ZD90R)", ALC298_FIXUP_SAMSUNG_AMP_V2_4_AMPS), + SND_PCI_QUIRK(0x1854, 0x0490, "LG Gram Style 14 (14Z90RS)", ALC298_FIXUP_LG_GRAM_STYLE_14), SND_PCI_QUIRK(0x19e5, 0x3204, "Huawei MACH-WX9", ALC256_FIXUP_HUAWEI_MACH_WX9_PINS), SND_PCI_QUIRK(0x19e5, 0x320f, "Huawei WRT-WX9 ", ALC256_FIXUP_ASUS_MIC_NO_PRESENCE), SND_PCI_QUIRK(0x19e5, 0x3212, "Huawei KLV-WX9 ", ALC256_FIXUP_ACER_HEADSET_MIC), @@ -7358,6 +7744,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x1d72, 0x1947, "RedmiBook Air", ALC255_FIXUP_XIAOMI_HEADSET_MIC), SND_PCI_QUIRK(0x1e39, 0xca14, "MEDION NM14LNL", ALC233_FIXUP_MEDION_MTL_SPK), SND_PCI_QUIRK(0x1ee7, 0x2078, "HONOR BRB-X M1010", ALC2XX_FIXUP_HEADSET_MIC), + SND_PCI_QUIRK(0x1f4c, 0xe001, "Minisforum V3 (SE)", ALC245_FIXUP_BASS_HP_DAC), SND_PCI_QUIRK(0x1f66, 0x0105, "Ayaneo Portable Game Player", ALC287_FIXUP_CS35L41_I2C_2), SND_PCI_QUIRK(0x2014, 0x800a, "Positivo ARN50", ALC269_FIXUP_LIMIT_INT_MIC_BOOST), SND_PCI_QUIRK(0x2039, 0x0001, "Inspur S14-G1", ALC295_FIXUP_CHROME_BOOK), @@ -7370,6 +7757,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x2782, 0x1705, "MEDION E15433", ALC269VC_FIXUP_INFINIX_Y4_MAX), SND_PCI_QUIRK(0x2782, 0x1707, "Vaio VJFE-ADL", ALC298_FIXUP_SPK_VOLUME), SND_PCI_QUIRK(0x2782, 0x4900, "MEDION E15443", ALC233_FIXUP_MEDION_MTL_SPK), + SND_PCI_QUIRK(0x7017, 0x2014, "Star Labs StarFighter", ALC233_FIXUP_STARLABS_STARFIGHTER), SND_PCI_QUIRK(0x8086, 0x2074, "Intel NUC 8", ALC233_FIXUP_INTEL_NUC8_DMIC), SND_PCI_QUIRK(0x8086, 0x2080, "Intel NUC 8 Rugged", ALC256_FIXUP_INTEL_NUC8_RUGGED), SND_PCI_QUIRK(0x8086, 0x2081, "Intel NUC 10", ALC256_FIXUP_INTEL_NUC10), @@ -7379,6 +7767,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0xf111, 0x0009, "Framework Laptop", ALC295_FIXUP_FRAMEWORK_LAPTOP_MIC_NO_PRESENCE), SND_PCI_QUIRK(0xf111, 0x000b, "Framework Laptop", ALC295_FIXUP_FRAMEWORK_LAPTOP_MIC_NO_PRESENCE), SND_PCI_QUIRK(0xf111, 0x000c, "Framework Laptop", ALC295_FIXUP_FRAMEWORK_LAPTOP_MIC_NO_PRESENCE), + SND_PCI_QUIRK(0xf111, 0x000f, "Framework Laptop", ALC295_FIXUP_FRAMEWORK_LAPTOP_MIC_NO_PRESENCE), #if 0 /* Below is a quirk table taken from the old code. @@ -7466,6 +7855,7 @@ static const struct hda_model_fixup alc269_fixup_models[] = { {.id = ALC298_FIXUP_TPT470_DOCK_FIX, .name = "tpt470-dock-fix"}, {.id = ALC298_FIXUP_TPT470_DOCK, .name = "tpt470-dock"}, {.id = ALC233_FIXUP_LENOVO_MULTI_CODECS, .name = "dual-codecs"}, + {.id = ALC233_FIXUP_STARLABS_STARFIGHTER, .name = "starlabs-starfighter"}, {.id = ALC700_FIXUP_INTEL_REFERENCE, .name = "alc700-ref"}, {.id = ALC269_FIXUP_SONY_VAIO, .name = "vaio"}, {.id = ALC269_FIXUP_DELL_M101Z, .name = "dell-m101z"}, @@ -7573,6 +7963,7 @@ static const struct hda_model_fixup alc269_fixup_models[] = { {.id = ALC285_FIXUP_HP_GPIO_AMP_INIT, .name = "alc285-hp-amp-init"}, {.id = ALC236_FIXUP_LENOVO_INV_DMIC, .name = "alc236-fixup-lenovo-inv-mic"}, {.id = ALC2XX_FIXUP_HEADSET_MIC, .name = "alc2xx-fixup-headset-mic"}, + {.id = ALC245_FIXUP_BASS_HP_DAC, .name = "alc245-fixup-bass-hp-dac"}, {} }; #define ALC225_STANDARD_PINS \ diff --git a/sound/hda/codecs/realtek/realtek.c b/sound/hda/codecs/realtek/realtek.c index ca377a5adadb5b..efe20b45052900 100644 --- a/sound/hda/codecs/realtek/realtek.c +++ b/sound/hda/codecs/realtek/realtek.c @@ -215,12 +215,13 @@ void alc_update_knob_master(struct hda_codec *codec, { unsigned int val; struct snd_kcontrol *kctl; - struct snd_ctl_elem_value *uctl __free(kfree) = NULL; kctl = snd_hda_find_mixer_ctl(codec, "Master Playback Volume"); if (!kctl) return; - uctl = kzalloc(sizeof(*uctl), GFP_KERNEL); + + struct snd_ctl_elem_value *uctl __free(kfree) = + kzalloc(sizeof(*uctl), GFP_KERNEL); if (!uctl) return; val = snd_hda_codec_read(codec, jack->nid, 0, diff --git a/sound/hda/codecs/senarytech.c b/sound/hda/codecs/senarytech.c index 63cda57cf7868e..f4732a8d7955da 100644 --- a/sound/hda/codecs/senarytech.c +++ b/sound/hda/codecs/senarytech.c @@ -28,6 +28,7 @@ struct senary_spec { /* extra EAPD pins */ unsigned int num_eapds; hda_nid_t eapds[4]; + bool dynamic_eapd; hda_nid_t mute_led_eapd; unsigned int parse_flags; /* flag for snd_hda_parse_pin_defcfg() */ @@ -134,8 +135,12 @@ static void senary_init_gpio_led(struct hda_codec *codec) static int senary_init(struct hda_codec *codec) { + struct senary_spec *spec = codec->spec; + snd_hda_gen_init(codec); senary_init_gpio_led(codec); + if (!spec->dynamic_eapd) + senary_auto_turn_eapd(codec, spec->num_eapds, spec->eapds, true); snd_hda_apply_fixup(codec, HDA_FIXUP_ACT_INIT); return 0; diff --git a/sound/hda/codecs/side-codecs/cs35l56_hda.c b/sound/hda/codecs/side-codecs/cs35l56_hda.c index f7ba92e119578a..32d734bf2fdf85 100644 --- a/sound/hda/codecs/side-codecs/cs35l56_hda.c +++ b/sound/hda/codecs/side-codecs/cs35l56_hda.c @@ -249,7 +249,7 @@ static int cs35l56_hda_posture_put(struct snd_kcontrol *kcontrol, struct snd_ctl_elem_value *ucontrol) { struct cs35l56_hda *cs35l56 = snd_kcontrol_chip(kcontrol); - unsigned long pos = ucontrol->value.integer.value[0]; + long pos = ucontrol->value.integer.value[0]; bool changed; int ret; diff --git a/sound/hda/codecs/side-codecs/tas2781_hda_spi.c b/sound/hda/codecs/side-codecs/tas2781_hda_spi.c index b9a55672bf15d5..488e35dac9524c 100644 --- a/sound/hda/codecs/side-codecs/tas2781_hda_spi.c +++ b/sound/hda/codecs/side-codecs/tas2781_hda_spi.c @@ -634,7 +634,7 @@ static void tasdev_fw_ready(const struct firmware *fmw, void *context) struct tasdevice_priv *tas_priv = context; struct tas2781_hda *tas_hda = dev_get_drvdata(tas_priv->dev); struct hda_codec *codec = tas_priv->codec; - int ret, val; + int ret; pm_runtime_get_sync(tas_priv->dev); guard(mutex)(&tas_priv->codec_lock); @@ -673,20 +673,14 @@ static void tasdev_fw_ready(const struct firmware *fmw, void *context) tas_priv->rcabin.profile_cfg_id = 0; tas_priv->fw_state = TASDEVICE_DSP_FW_ALL_OK; - ret = tasdevice_spi_dev_read(tas_priv, tas_priv->index, - TAS2781_REG_CLK_CONFIG, &val); - if (ret < 0) - goto out; - if (val == TAS2781_REG_CLK_CONFIG_RESET) { - ret = tasdevice_prmg_load(tas_priv, 0); - if (ret < 0) { - dev_err(tas_priv->dev, "FW download failed = %d\n", - ret); - goto out; - } - tas_priv->fw_state = TASDEVICE_DSP_FW_ALL_OK; + ret = tasdevice_prmg_load(tas_priv, 0); + if (ret < 0) { + dev_err(tas_priv->dev, "FW download failed = %d\n", ret); + goto out; } + tas_priv->fw_state = TASDEVICE_DSP_FW_ALL_OK; + if (tas_priv->fmw->nr_programs > 0) tas_priv->tasdevice[tas_priv->index].cur_prog = 0; if (tas_priv->fmw->nr_configurations > 0) diff --git a/sound/hda/common/codec.c b/sound/hda/common/codec.c index c6d44168c7f9de..ffe7c69d5a32c9 100644 --- a/sound/hda/common/codec.c +++ b/sound/hda/common/codec.c @@ -1854,9 +1854,9 @@ static int check_follower_present(struct hda_codec *codec, /* call kctl->put with the given value(s) */ static int put_kctl_with_value(struct snd_kcontrol *kctl, int val) { - struct snd_ctl_elem_value *ucontrol __free(kfree) = NULL; + struct snd_ctl_elem_value *ucontrol __free(kfree) = + kzalloc(sizeof(*ucontrol), GFP_KERNEL); - ucontrol = kzalloc(sizeof(*ucontrol), GFP_KERNEL); if (!ucontrol) return -ENOMEM; ucontrol->value.integer.value[0] = val; diff --git a/sound/hda/common/hda_jack.h b/sound/hda/common/hda_jack.h index ff7d289c034bfe..e9b9970c59ed4a 100644 --- a/sound/hda/common/hda_jack.h +++ b/sound/hda/common/hda_jack.h @@ -82,10 +82,10 @@ snd_hda_jack_detect_enable_callback_mst(struct hda_codec *codec, hda_nid_t nid, int dev_id, hda_jack_callback_fn func); /** - * snd_hda_jack_detect_enable - enable the jack-detection + * snd_hda_jack_detect_enable_callback - enable the jack-detection * @codec: the HDA codec * @nid: pin NID to enable - * @func: callback function to register + * @cb: callback function to register * * In the case of error, the return value will be a pointer embedded with * errno. Check and handle the return value appropriately with standard diff --git a/sound/hda/common/hda_local.h b/sound/hda/common/hda_local.h index a7e53277a0fea6..ab423f1cef5492 100644 --- a/sound/hda/common/hda_local.h +++ b/sound/hda/common/hda_local.h @@ -424,7 +424,7 @@ int _snd_hda_set_pin_ctl(struct hda_codec *codec, hda_nid_t pin, unsigned int val, bool cached); /** - * _snd_hda_set_pin_ctl - Set a pin-control value safely + * snd_hda_set_pin_ctl - Set a pin-control value safely * @codec: the codec instance * @pin: the pin NID to set the control * @val: the pin-control value (AC_PINCTL_* bits) diff --git a/sound/hda/common/sysfs.c b/sound/hda/common/sysfs.c index f8c8483fd5e5f3..bedf10b308850b 100644 --- a/sound/hda/common/sysfs.c +++ b/sound/hda/common/sysfs.c @@ -299,7 +299,6 @@ static void remove_trail_spaces(char *str) static int parse_hints(struct hda_codec *codec, const char *buf) { - char *key __free(kfree) = NULL; char *val; struct hda_hint *hint; @@ -308,7 +307,9 @@ static int parse_hints(struct hda_codec *codec, const char *buf) return 0; if (*buf == '=') return -EINVAL; - key = kstrndup_noeol(buf, 1024); + + char *key __free(kfree) = + kstrndup_noeol(buf, 1024); if (!key) return -ENOMEM; /* extract key and val */ diff --git a/sound/hda/controllers/intel.c b/sound/hda/controllers/intel.c index 1e8e3d61291a11..906be1660ac5fa 100644 --- a/sound/hda/controllers/intel.c +++ b/sound/hda/controllers/intel.c @@ -295,6 +295,9 @@ enum { #define AZX_DCAPS_INTEL_LNL \ (AZX_DCAPS_INTEL_SKYLAKE | AZX_DCAPS_PIO_COMMANDS) +#define AZX_DCAPS_INTEL_NVL \ + (AZX_DCAPS_INTEL_LNL & ~AZX_DCAPS_NO_ALIGN_BUFSIZE) + /* quirks for ATI SB / AMD Hudson */ #define AZX_DCAPS_PRESET_ATI_SB \ (AZX_DCAPS_NO_TCSEL | AZX_DCAPS_POSFIX_LPIB |\ @@ -1751,6 +1754,8 @@ static int default_bdl_pos_adj(struct azx *chip) return 1; case AZX_DRIVER_ZHAOXINHDMI: return 128; + case AZX_DRIVER_NVIDIA: + return 64; default: return 32; } @@ -2075,7 +2080,6 @@ static const struct pci_device_id driver_denylist[] = { { PCI_DEVICE_SUB(0x1022, 0x1487, 0x1043, 0x874f) }, /* ASUS ROG Zenith II / Strix */ { PCI_DEVICE_SUB(0x1022, 0x1487, 0x1462, 0xcb59) }, /* MSI TRX40 Creator */ { PCI_DEVICE_SUB(0x1022, 0x1487, 0x1462, 0xcb60) }, /* MSI TRX40 */ - { PCI_DEVICE_SUB(0x1022, 0x15e3, 0x1462, 0xee59) }, /* MSI X870E Tomahawk WiFi */ {} }; @@ -2551,7 +2555,8 @@ static const struct pci_device_id azx_ids[] = { /* Wildcat Lake */ { PCI_DEVICE_DATA(INTEL, HDA_WCL, AZX_DRIVER_SKL | AZX_DCAPS_INTEL_LNL) }, /* Nova Lake */ - { PCI_DEVICE_DATA(INTEL, HDA_NVL_S, AZX_DRIVER_SKL | AZX_DCAPS_INTEL_LNL) }, + { PCI_DEVICE_DATA(INTEL, HDA_NVL, AZX_DRIVER_SKL | AZX_DCAPS_INTEL_NVL) }, + { PCI_DEVICE_DATA(INTEL, HDA_NVL_S, AZX_DRIVER_SKL | AZX_DCAPS_INTEL_NVL) }, /* Apollolake (Broxton-P) */ { PCI_DEVICE_DATA(INTEL, HDA_APL, AZX_DRIVER_SKL | AZX_DCAPS_INTEL_BROXTON) }, /* Gemini-Lake */ diff --git a/sound/pci/asihpi/hpimsgx.c b/sound/pci/asihpi/hpimsgx.c index b68e6bfbbfbab5..ed1c7b77443614 100644 --- a/sound/pci/asihpi/hpimsgx.c +++ b/sound/pci/asihpi/hpimsgx.c @@ -581,8 +581,10 @@ static u16 adapter_prepare(u16 adapter) HPI_ADAPTER_OPEN); hm.adapter_index = adapter; hw_entry_point(&hm, &hr); - memcpy(&rESP_HPI_ADAPTER_OPEN[adapter], &hr, - sizeof(rESP_HPI_ADAPTER_OPEN[0])); + memcpy(&rESP_HPI_ADAPTER_OPEN[adapter].h, &hr, + sizeof(rESP_HPI_ADAPTER_OPEN[adapter].h)); + memcpy(&rESP_HPI_ADAPTER_OPEN[adapter].a, &hr.u.ax.info, + sizeof(rESP_HPI_ADAPTER_OPEN[adapter].a)); if (hr.error) return hr.error; diff --git a/sound/pci/ctxfi/ctatc.c b/sound/pci/ctxfi/ctatc.c index 227d8c8490e1f5..f50cff4654c22e 100644 --- a/sound/pci/ctxfi/ctatc.c +++ b/sound/pci/ctxfi/ctatc.c @@ -52,18 +52,19 @@ static const struct snd_pci_quirk subsys_20k1_list[] = { static const struct snd_pci_quirk subsys_20k2_list[] = { SND_PCI_QUIRK(PCI_VENDOR_ID_CREATIVE, PCI_SUBDEVICE_ID_CREATIVE_SB0760, "SB0760", CTSB0760), - SND_PCI_QUIRK(PCI_VENDOR_ID_CREATIVE, PCI_SUBDEVICE_ID_CREATIVE_SB1270, - "SB1270", CTSB1270), SND_PCI_QUIRK(PCI_VENDOR_ID_CREATIVE, PCI_SUBDEVICE_ID_CREATIVE_SB08801, "SB0880", CTSB0880), SND_PCI_QUIRK(PCI_VENDOR_ID_CREATIVE, PCI_SUBDEVICE_ID_CREATIVE_SB08802, "SB0880", CTSB0880), SND_PCI_QUIRK(PCI_VENDOR_ID_CREATIVE, PCI_SUBDEVICE_ID_CREATIVE_SB08803, "SB0880", CTSB0880), + SND_PCI_QUIRK(PCI_VENDOR_ID_CREATIVE, PCI_SUBDEVICE_ID_CREATIVE_SB1270, + "SB1270", CTSB1270), + SND_PCI_QUIRK(0x160b, 0x0101, "OK0010", CTOK0010), + SND_PCI_QUIRK(0x160b, 0x0102, "OK0010", CTOK0010), SND_PCI_QUIRK_MASK(PCI_VENDOR_ID_CREATIVE, 0xf000, PCI_SUBDEVICE_ID_CREATIVE_HENDRIX, "HENDRIX", CTHENDRIX), - SND_PCI_QUIRK(0x160b, 0x0101, "OK0010", CTOK0010), { } /* terminator */ }; @@ -78,8 +79,8 @@ static const char *ct_subsys_name[NUM_CTCARDS] = { [CTSB0760] = "SB076x", [CTHENDRIX] = "Hendrix", [CTSB0880] = "SB0880", - [CTSB1270] = "SB1270", - [CTOK0010] = "OK0010", + [CTSB1270] = "SB1270", + [CTOK0010] = "OK0010", [CT20K2_UNKNOWN] = "Unknown", }; @@ -1426,10 +1427,14 @@ static int atc_get_resources(struct ct_atc *atc) daio_mgr = (struct daio_mgr *)atc->rsc_mgrs[DAIO]; da_desc.msr = atc->msr; for (i = 0; i < NUM_DAIOTYP; i++) { - if (((i == MIC) && !cap.dedicated_mic) || ((i == RCA) && !cap.dedicated_rca)) + if (((i == MIC) && !cap.dedicated_mic) || + ((i == RCA) && !cap.dedicated_rca) || + i == SPDIFI1) continue; - da_desc.type = (atc->model != CTSB073X) ? i : - ((i == SPDIFIO) ? SPDIFI1 : i); + if (atc->model == CTSB073X && i == SPDIFIO) + da_desc.type = SPDIFI1; + else + da_desc.type = i; da_desc.output = (i < LINEIM) || (i == RCA); err = daio_mgr->get_daio(daio_mgr, &da_desc, (struct daio **)&atc->daios[i]); diff --git a/sound/pci/ctxfi/ctdaio.c b/sound/pci/ctxfi/ctdaio.c index 1c8f8efd836ceb..6166f07c010c11 100644 --- a/sound/pci/ctxfi/ctdaio.c +++ b/sound/pci/ctxfi/ctdaio.c @@ -99,7 +99,7 @@ static const struct rsc_ops daio_in_rsc_ops_20k2 = { .output_slot = daio_index, }; -static unsigned int daio_device_index(enum DAIOTYP type, struct hw *hw) +static int daio_device_index(enum DAIOTYP type, struct hw *hw) { switch (hw->chip_type) { case ATC20K1: @@ -112,12 +112,15 @@ static unsigned int daio_device_index(enum DAIOTYP type, struct hw *hw) case LINEO3: return 5; case LINEO4: return 6; case LINEIM: return 7; - default: return -EINVAL; + default: + pr_err("ctxfi: Invalid type %d for hw20k1\n", type); + return -EINVAL; } case ATC20K2: switch (type) { case SPDIFOO: return 0; case SPDIFIO: return 0; + case SPDIFI1: return 1; case LINEO1: return 4; case LINEO2: return 7; case LINEO3: return 5; @@ -125,9 +128,12 @@ static unsigned int daio_device_index(enum DAIOTYP type, struct hw *hw) case LINEIM: return 4; case MIC: return 5; case RCA: return 3; - default: return -EINVAL; + default: + pr_err("ctxfi: Invalid type %d for hw20k2\n", type); + return -EINVAL; } default: + pr_err("ctxfi: Invalid chip type %d\n", hw->chip_type); return -EINVAL; } } @@ -148,8 +154,11 @@ static int dao_spdif_set_spos(struct dao *dao, unsigned int spos) static int dao_commit_write(struct dao *dao) { - dao->hw->dao_commit_write(dao->hw, - daio_device_index(dao->daio.type, dao->hw), dao->ctrl_blk); + int idx = daio_device_index(dao->daio.type, dao->hw); + + if (idx < 0) + return idx; + dao->hw->dao_commit_write(dao->hw, idx, dao->ctrl_blk); return 0; } @@ -287,8 +296,11 @@ static int dai_set_enb_srt(struct dai *dai, unsigned int enb) static int dai_commit_write(struct dai *dai) { - dai->hw->dai_commit_write(dai->hw, - daio_device_index(dai->daio.type, dai->hw), dai->ctrl_blk); + int idx = daio_device_index(dai->daio.type, dai->hw); + + if (idx < 0) + return idx; + dai->hw->dai_commit_write(dai->hw, idx, dai->ctrl_blk); return 0; } @@ -367,7 +379,7 @@ static int dao_rsc_init(struct dao *dao, { struct hw *hw = mgr->mgr.hw; unsigned int conf; - int err; + int idx, err; err = daio_rsc_init(&dao->daio, desc, mgr->mgr.hw); if (err) @@ -386,15 +398,18 @@ static int dao_rsc_init(struct dao *dao, if (err) goto error2; - hw->daio_mgr_dsb_dao(mgr->mgr.ctrl_blk, - daio_device_index(dao->daio.type, hw)); + idx = daio_device_index(dao->daio.type, hw); + if (idx < 0) { + err = idx; + goto error2; + } + + hw->daio_mgr_dsb_dao(mgr->mgr.ctrl_blk, idx); hw->daio_mgr_commit_write(hw, mgr->mgr.ctrl_blk); conf = (desc->msr & 0x7) | (desc->passthru << 3); - hw->daio_mgr_dao_init(hw, mgr->mgr.ctrl_blk, - daio_device_index(dao->daio.type, hw), conf); - hw->daio_mgr_enb_dao(mgr->mgr.ctrl_blk, - daio_device_index(dao->daio.type, hw)); + hw->daio_mgr_dao_init(hw, mgr->mgr.ctrl_blk, idx, conf); + hw->daio_mgr_enb_dao(mgr->mgr.ctrl_blk, idx); hw->daio_mgr_commit_write(hw, mgr->mgr.ctrl_blk); return 0; @@ -443,7 +458,7 @@ static int dai_rsc_init(struct dai *dai, const struct daio_desc *desc, struct daio_mgr *mgr) { - int err; + int idx, err; struct hw *hw = mgr->mgr.hw; unsigned int rsr, msr; @@ -457,6 +472,12 @@ static int dai_rsc_init(struct dai *dai, if (err) goto error1; + idx = daio_device_index(dai->daio.type, dai->hw); + if (idx < 0) { + err = idx; + goto error1; + } + for (rsr = 0, msr = desc->msr; msr > 1; msr >>= 1) rsr++; @@ -465,8 +486,7 @@ static int dai_rsc_init(struct dai *dai, /* default to disabling control of a SRC */ hw->dai_srt_set_ec(dai->ctrl_blk, 0); hw->dai_srt_set_et(dai->ctrl_blk, 0); /* default to disabling SRT */ - hw->dai_commit_write(hw, - daio_device_index(dai->daio.type, dai->hw), dai->ctrl_blk); + hw->dai_commit_write(hw, idx, dai->ctrl_blk); return 0; @@ -581,28 +601,28 @@ static int put_daio_rsc(struct daio_mgr *mgr, struct daio *daio) static int daio_mgr_enb_daio(struct daio_mgr *mgr, struct daio *daio) { struct hw *hw = mgr->mgr.hw; - - if (daio->output) { - hw->daio_mgr_enb_dao(mgr->mgr.ctrl_blk, - daio_device_index(daio->type, hw)); - } else { - hw->daio_mgr_enb_dai(mgr->mgr.ctrl_blk, - daio_device_index(daio->type, hw)); - } + int idx = daio_device_index(daio->type, hw); + + if (idx < 0) + return idx; + if (daio->output) + hw->daio_mgr_enb_dao(mgr->mgr.ctrl_blk, idx); + else + hw->daio_mgr_enb_dai(mgr->mgr.ctrl_blk, idx); return 0; } static int daio_mgr_dsb_daio(struct daio_mgr *mgr, struct daio *daio) { struct hw *hw = mgr->mgr.hw; - - if (daio->output) { - hw->daio_mgr_dsb_dao(mgr->mgr.ctrl_blk, - daio_device_index(daio->type, hw)); - } else { - hw->daio_mgr_dsb_dai(mgr->mgr.ctrl_blk, - daio_device_index(daio->type, hw)); - } + int idx = daio_device_index(daio->type, hw); + + if (idx < 0) + return idx; + if (daio->output) + hw->daio_mgr_dsb_dao(mgr->mgr.ctrl_blk, idx); + else + hw->daio_mgr_dsb_dai(mgr->mgr.ctrl_blk, idx); return 0; } diff --git a/sound/pci/ctxfi/ctvmem.h b/sound/pci/ctxfi/ctvmem.h index da54cbcdb0be4d..43a0065b40c354 100644 --- a/sound/pci/ctxfi/ctvmem.h +++ b/sound/pci/ctxfi/ctvmem.h @@ -15,7 +15,7 @@ #ifndef CTVMEM_H #define CTVMEM_H -#define CT_PTP_NUM 4 /* num of device page table pages */ +#define CT_PTP_NUM 1 /* num of device page table pages */ #include #include diff --git a/sound/soc/amd/acp/acp-mach-common.c b/sound/soc/amd/acp/acp-mach-common.c index 4d99472c75baf0..09f6c9a2c04104 100644 --- a/sound/soc/amd/acp/acp-mach-common.c +++ b/sound/soc/amd/acp/acp-mach-common.c @@ -127,8 +127,13 @@ static int acp_card_rt5682_init(struct snd_soc_pcm_runtime *rtd) if (drvdata->hs_codec_id != RT5682) return -EINVAL; - drvdata->wclk = clk_get(component->dev, "rt5682-dai-wclk"); - drvdata->bclk = clk_get(component->dev, "rt5682-dai-bclk"); + drvdata->wclk = devm_clk_get(component->dev, "rt5682-dai-wclk"); + if (IS_ERR(drvdata->wclk)) + return PTR_ERR(drvdata->wclk); + + drvdata->bclk = devm_clk_get(component->dev, "rt5682-dai-bclk"); + if (IS_ERR(drvdata->bclk)) + return PTR_ERR(drvdata->bclk); ret = snd_soc_dapm_new_controls(dapm, rt5682_widgets, ARRAY_SIZE(rt5682_widgets)); @@ -370,8 +375,13 @@ static int acp_card_rt5682s_init(struct snd_soc_pcm_runtime *rtd) return -EINVAL; if (!drvdata->soc_mclk) { - drvdata->wclk = clk_get(component->dev, "rt5682-dai-wclk"); - drvdata->bclk = clk_get(component->dev, "rt5682-dai-bclk"); + drvdata->wclk = devm_clk_get(component->dev, "rt5682-dai-wclk"); + if (IS_ERR(drvdata->wclk)) + return PTR_ERR(drvdata->wclk); + + drvdata->bclk = devm_clk_get(component->dev, "rt5682-dai-bclk"); + if (IS_ERR(drvdata->bclk)) + return PTR_ERR(drvdata->bclk); } ret = snd_soc_dapm_new_controls(dapm, rt5682s_widgets, diff --git a/sound/soc/amd/acp/acp-sdw-legacy-mach.c b/sound/soc/amd/acp/acp-sdw-legacy-mach.c index fae94b9edd5a3f..9d674436727687 100644 --- a/sound/soc/amd/acp/acp-sdw-legacy-mach.c +++ b/sound/soc/amd/acp/acp-sdw-legacy-mach.c @@ -95,6 +95,30 @@ static const struct dmi_system_id soc_sdw_quirk_table[] = { }, .driver_data = (void *)(ASOC_SDW_CODEC_SPKR), }, + { + .callback = soc_sdw_quirk_cb, + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "LENOVO"), + DMI_MATCH(DMI_PRODUCT_SKU, "21YW"), + }, + .driver_data = (void *)((ASOC_SDW_CODEC_SPKR) | (ASOC_SDW_ACP_DMIC)), + }, + { + .callback = soc_sdw_quirk_cb, + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "LENOVO"), + DMI_MATCH(DMI_PRODUCT_SKU, "21YX"), + }, + .driver_data = (void *)((ASOC_SDW_CODEC_SPKR) | (ASOC_SDW_ACP_DMIC)), + }, + { + .callback = soc_sdw_quirk_cb, + .matches = { + DMI_MATCH(DMI_BOARD_VENDOR, "ASUSTeK COMPUTER INC."), + DMI_MATCH(DMI_PRODUCT_NAME, "HN7306EA"), + }, + .driver_data = (void *)(ASOC_SDW_ACP_DMIC), + }, {} }; diff --git a/sound/soc/amd/acp/amd-acp63-acpi-match.c b/sound/soc/amd/acp/amd-acp63-acpi-match.c index 9b6a49c051cda9..1dbbaba3c75b38 100644 --- a/sound/soc/amd/acp/amd-acp63-acpi-match.c +++ b/sound/soc/amd/acp/amd-acp63-acpi-match.c @@ -30,6 +30,20 @@ static const struct snd_soc_acpi_endpoint spk_r_endpoint = { .group_id = 1 }; +static const struct snd_soc_acpi_endpoint spk_2_endpoint = { + .num = 0, + .aggregated = 1, + .group_position = 2, + .group_id = 1 +}; + +static const struct snd_soc_acpi_endpoint spk_3_endpoint = { + .num = 0, + .aggregated = 1, + .group_position = 3, + .group_id = 1 +}; + static const struct snd_soc_acpi_adr_device rt711_rt1316_group_adr[] = { { .adr = 0x000030025D071101ull, @@ -103,6 +117,345 @@ static const struct snd_soc_acpi_adr_device rt722_0_single_adr[] = { } }; +static const struct snd_soc_acpi_endpoint cs42l43_endpoints[] = { + { /* Jack Playback Endpoint */ + .num = 0, + .aggregated = 0, + .group_position = 0, + .group_id = 0, + }, + { /* DMIC Capture Endpoint */ + .num = 1, + .aggregated = 0, + .group_position = 0, + .group_id = 0, + }, + { /* Jack Capture Endpoint */ + .num = 2, + .aggregated = 0, + .group_position = 0, + .group_id = 0, + }, + { /* Speaker Playback Endpoint */ + .num = 3, + .aggregated = 0, + .group_position = 0, + .group_id = 0, + }, +}; + +static const struct snd_soc_acpi_adr_device cs35l56x4_l1u3210_adr[] = { + { + .adr = 0x00013301FA355601ull, + .num_endpoints = 1, + .endpoints = &spk_l_endpoint, + .name_prefix = "AMP1" + }, + { + .adr = 0x00013201FA355601ull, + .num_endpoints = 1, + .endpoints = &spk_r_endpoint, + .name_prefix = "AMP2" + }, + { + .adr = 0x00013101FA355601ull, + .num_endpoints = 1, + .endpoints = &spk_2_endpoint, + .name_prefix = "AMP3" + }, + { + .adr = 0x00013001FA355601ull, + .num_endpoints = 1, + .endpoints = &spk_3_endpoint, + .name_prefix = "AMP4" + }, +}; + +static const struct snd_soc_acpi_adr_device cs35l63x2_l0u01_adr[] = { + { + .adr = 0x00003001FA356301ull, + .num_endpoints = 1, + .endpoints = &spk_l_endpoint, + .name_prefix = "AMP1" + }, + { + .adr = 0x00003101FA356301ull, + .num_endpoints = 1, + .endpoints = &spk_r_endpoint, + .name_prefix = "AMP2" + }, +}; + +static const struct snd_soc_acpi_adr_device cs35l63x2_l1u01_adr[] = { + { + .adr = 0x00013001FA356301ull, + .num_endpoints = 1, + .endpoints = &spk_l_endpoint, + .name_prefix = "AMP1" + }, + { + .adr = 0x00013101FA356301ull, + .num_endpoints = 1, + .endpoints = &spk_r_endpoint, + .name_prefix = "AMP2" + }, +}; + +static const struct snd_soc_acpi_adr_device cs35l63x2_l1u13_adr[] = { + { + .adr = 0x00013101FA356301ull, + .num_endpoints = 1, + .endpoints = &spk_l_endpoint, + .name_prefix = "AMP1" + }, + { + .adr = 0x00013301FA356301ull, + .num_endpoints = 1, + .endpoints = &spk_r_endpoint, + .name_prefix = "AMP2" + }, +}; + +static const struct snd_soc_acpi_adr_device cs35l63x4_l0u0246_adr[] = { + { + .adr = 0x00003001FA356301ull, + .num_endpoints = 1, + .endpoints = &spk_l_endpoint, + .name_prefix = "AMP1" + }, + { + .adr = 0x00003201FA356301ull, + .num_endpoints = 1, + .endpoints = &spk_r_endpoint, + .name_prefix = "AMP2" + }, + { + .adr = 0x00003401FA356301ull, + .num_endpoints = 1, + .endpoints = &spk_2_endpoint, + .name_prefix = "AMP3" + }, + { + .adr = 0x00003601FA356301ull, + .num_endpoints = 1, + .endpoints = &spk_3_endpoint, + .name_prefix = "AMP4" + }, +}; + +static const struct snd_soc_acpi_adr_device cs42l43_l0u0_adr[] = { + { + .adr = 0x00003001FA424301ull, + .num_endpoints = ARRAY_SIZE(cs42l43_endpoints), + .endpoints = cs42l43_endpoints, + .name_prefix = "cs42l43" + } +}; + +static const struct snd_soc_acpi_adr_device cs42l43_l0u1_adr[] = { + { + .adr = 0x00003101FA424301ull, + .num_endpoints = ARRAY_SIZE(cs42l43_endpoints), + .endpoints = cs42l43_endpoints, + .name_prefix = "cs42l43" + } +}; + +static const struct snd_soc_acpi_adr_device cs42l43b_l0u1_adr[] = { + { + .adr = 0x00003101FA2A3B01ull, + .num_endpoints = ARRAY_SIZE(cs42l43_endpoints), + .endpoints = cs42l43_endpoints, + .name_prefix = "cs42l43" + } +}; + +static const struct snd_soc_acpi_adr_device cs42l43_l1u0_cs35l56x4_l1u0123_adr[] = { + { + .adr = 0x00013001FA424301ull, + .num_endpoints = ARRAY_SIZE(cs42l43_endpoints), + .endpoints = cs42l43_endpoints, + .name_prefix = "cs42l43" + }, + { + .adr = 0x00013001FA355601ull, + .num_endpoints = 1, + .endpoints = &spk_l_endpoint, + .name_prefix = "AMP1" + }, + { + .adr = 0x00013101FA355601ull, + .num_endpoints = 1, + .endpoints = &spk_r_endpoint, + .name_prefix = "AMP2" + }, + { + .adr = 0x00013201FA355601ull, + .num_endpoints = 1, + .endpoints = &spk_2_endpoint, + .name_prefix = "AMP3" + }, + { + .adr = 0x00013301FA355601ull, + .num_endpoints = 1, + .endpoints = &spk_3_endpoint, + .name_prefix = "AMP4" + }, +}; + +static const struct snd_soc_acpi_adr_device cs42l45_l0u0_adr[] = { + { + .adr = 0x00003001FA424501ull, + /* Re-use endpoints, but cs42l45 has no speaker */ + .num_endpoints = ARRAY_SIZE(cs42l43_endpoints) - 1, + .endpoints = cs42l43_endpoints, + .name_prefix = "cs42l45" + } +}; + +static const struct snd_soc_acpi_adr_device cs42l45_l1u0_adr[] = { + { + .adr = 0x00013001FA424501ull, + /* Re-use endpoints, but cs42l45 has no speaker */ + .num_endpoints = ARRAY_SIZE(cs42l43_endpoints) - 1, + .endpoints = cs42l43_endpoints, + .name_prefix = "cs42l45" + } +}; + +static const struct snd_soc_acpi_link_adr acp63_cs35l56x4_l1u3210[] = { + { + .mask = BIT(1), + .num_adr = ARRAY_SIZE(cs35l56x4_l1u3210_adr), + .adr_d = cs35l56x4_l1u3210_adr, + }, + {} +}; + +static const struct snd_soc_acpi_link_adr acp63_cs35l63x4_l0u0246[] = { + { + .mask = BIT(0), + .num_adr = ARRAY_SIZE(cs35l63x4_l0u0246_adr), + .adr_d = cs35l63x4_l0u0246_adr, + }, + {} +}; + +static const struct snd_soc_acpi_link_adr acp63_cs42l43_l0u1[] = { + { + .mask = BIT(0), + .num_adr = ARRAY_SIZE(cs42l43_l0u1_adr), + .adr_d = cs42l43_l0u1_adr, + }, + {} +}; + +static const struct snd_soc_acpi_link_adr acp63_cs42l43b_l0u1[] = { + { + .mask = BIT(0), + .num_adr = ARRAY_SIZE(cs42l43b_l0u1_adr), + .adr_d = cs42l43b_l0u1_adr, + }, + {} +}; + +static const struct snd_soc_acpi_link_adr acp63_cs42l43_l0u0_cs35l56x4_l1u3210[] = { + { + .mask = BIT(0), + .num_adr = ARRAY_SIZE(cs42l43_l0u0_adr), + .adr_d = cs42l43_l0u0_adr, + }, + { + .mask = BIT(1), + .num_adr = ARRAY_SIZE(cs35l56x4_l1u3210_adr), + .adr_d = cs35l56x4_l1u3210_adr, + }, + {} +}; + +static const struct snd_soc_acpi_link_adr acp63_cs42l43_l1u0_cs35l56x4_l1u0123[] = { + { + .mask = BIT(1), + .num_adr = ARRAY_SIZE(cs42l43_l1u0_cs35l56x4_l1u0123_adr), + .adr_d = cs42l43_l1u0_cs35l56x4_l1u0123_adr, + }, + {} +}; + +static const struct snd_soc_acpi_link_adr acp63_cs42l45_l0u0[] = { + { + .mask = BIT(0), + .num_adr = ARRAY_SIZE(cs42l45_l0u0_adr), + .adr_d = cs42l45_l0u0_adr, + }, + {} +}; + +static const struct snd_soc_acpi_link_adr acp63_cs42l45_l0u0_cs35l63x2_l1u01[] = { + { + .mask = BIT(0), + .num_adr = ARRAY_SIZE(cs42l45_l0u0_adr), + .adr_d = cs42l45_l0u0_adr, + }, + { + .mask = BIT(1), + .num_adr = ARRAY_SIZE(cs35l63x2_l1u01_adr), + .adr_d = cs35l63x2_l1u01_adr, + }, + {} +}; + +static const struct snd_soc_acpi_link_adr acp63_cs42l45_l0u0_cs35l63x2_l1u13[] = { + { + .mask = BIT(0), + .num_adr = ARRAY_SIZE(cs42l45_l0u0_adr), + .adr_d = cs42l45_l0u0_adr, + }, + { + .mask = BIT(1), + .num_adr = ARRAY_SIZE(cs35l63x2_l1u13_adr), + .adr_d = cs35l63x2_l1u13_adr, + }, + {} +}; + +static const struct snd_soc_acpi_link_adr acp63_cs42l45_l1u0[] = { + { + .mask = BIT(1), + .num_adr = ARRAY_SIZE(cs42l45_l1u0_adr), + .adr_d = cs42l45_l1u0_adr, + }, + {} +}; + +static const struct snd_soc_acpi_link_adr acp63_cs42l45_l1u0_cs35l63x2_l0u01[] = { + { + .mask = BIT(1), + .num_adr = ARRAY_SIZE(cs42l45_l1u0_adr), + .adr_d = cs42l45_l1u0_adr, + }, + { + .mask = BIT(0), + .num_adr = ARRAY_SIZE(cs35l63x2_l0u01_adr), + .adr_d = cs35l63x2_l0u01_adr, + }, + {} +}; + +static const struct snd_soc_acpi_link_adr acp63_cs42l45_l1u0_cs35l63x4_l0u0246[] = { + { + .mask = BIT(1), + .num_adr = ARRAY_SIZE(cs42l45_l1u0_adr), + .adr_d = cs42l45_l1u0_adr, + }, + { + .mask = BIT(0), + .num_adr = ARRAY_SIZE(cs35l63x4_l0u0246_adr), + .adr_d = cs35l63x4_l0u0246_adr, + }, + {} +}; + static const struct snd_soc_acpi_link_adr acp63_rt722_only[] = { { .mask = BIT(0), @@ -135,6 +488,66 @@ struct snd_soc_acpi_mach snd_soc_acpi_amd_acp63_sdw_machines[] = { .links = acp63_4_in_1_sdca, .drv_name = "amd_sdw", }, + { + .link_mask = BIT(0) | BIT(1), + .links = acp63_cs42l43_l0u0_cs35l56x4_l1u3210, + .drv_name = "amd_sdw", + }, + { + .link_mask = BIT(0) | BIT(1), + .links = acp63_cs42l45_l1u0_cs35l63x4_l0u0246, + .drv_name = "amd_sdw", + }, + { + .link_mask = BIT(0) | BIT(1), + .links = acp63_cs42l45_l0u0_cs35l63x2_l1u01, + .drv_name = "amd_sdw", + }, + { + .link_mask = BIT(0) | BIT(1), + .links = acp63_cs42l45_l0u0_cs35l63x2_l1u13, + .drv_name = "amd_sdw", + }, + { + .link_mask = BIT(0) | BIT(1), + .links = acp63_cs42l45_l1u0_cs35l63x2_l0u01, + .drv_name = "amd_sdw", + }, + { + .link_mask = BIT(1), + .links = acp63_cs42l43_l1u0_cs35l56x4_l1u0123, + .drv_name = "amd_sdw", + }, + { + .link_mask = BIT(1), + .links = acp63_cs35l56x4_l1u3210, + .drv_name = "amd_sdw", + }, + { + .link_mask = BIT(0), + .links = acp63_cs35l63x4_l0u0246, + .drv_name = "amd_sdw", + }, + { + .link_mask = BIT(0), + .links = acp63_cs42l43_l0u1, + .drv_name = "amd_sdw", + }, + { + .link_mask = BIT(0), + .links = acp63_cs42l43b_l0u1, + .drv_name = "amd_sdw", + }, + { + .link_mask = BIT(0), + .links = acp63_cs42l45_l0u0, + .drv_name = "amd_sdw", + }, + { + .link_mask = BIT(1), + .links = acp63_cs42l45_l1u0, + .drv_name = "amd_sdw", + }, {}, }; EXPORT_SYMBOL(snd_soc_acpi_amd_acp63_sdw_machines); diff --git a/sound/soc/amd/acp3x-rt5682-max9836.c b/sound/soc/amd/acp3x-rt5682-max9836.c index 4ca1978020a962..d1eb6f12a18302 100644 --- a/sound/soc/amd/acp3x-rt5682-max9836.c +++ b/sound/soc/amd/acp3x-rt5682-max9836.c @@ -94,8 +94,13 @@ static int acp3x_5682_init(struct snd_soc_pcm_runtime *rtd) return ret; } - rt5682_dai_wclk = clk_get(component->dev, "rt5682-dai-wclk"); - rt5682_dai_bclk = clk_get(component->dev, "rt5682-dai-bclk"); + rt5682_dai_wclk = devm_clk_get(component->dev, "rt5682-dai-wclk"); + if (IS_ERR(rt5682_dai_wclk)) + return PTR_ERR(rt5682_dai_wclk); + + rt5682_dai_bclk = devm_clk_get(component->dev, "rt5682-dai-bclk"); + if (IS_ERR(rt5682_dai_bclk)) + return PTR_ERR(rt5682_dai_bclk); ret = snd_soc_card_jack_new_pins(card, "Headset Jack", SND_JACK_HEADSET | diff --git a/sound/soc/amd/yc/acp6x-mach.c b/sound/soc/amd/yc/acp6x-mach.c index 67f2fee1939804..4c0acdad13ea1c 100644 --- a/sound/soc/amd/yc/acp6x-mach.c +++ b/sound/soc/amd/yc/acp6x-mach.c @@ -45,6 +45,13 @@ static struct snd_soc_card acp6x_card = { }; static const struct dmi_system_id yc_acp_quirk_table[] = { + { + .driver_data = &acp6x_card, + .matches = { + DMI_MATCH(DMI_BOARD_VENDOR, "HP"), + DMI_MATCH(DMI_PRODUCT_NAME, "HP Laptop 15-fc0xxx"), + } + }, { .driver_data = &acp6x_card, .matches = { @@ -696,7 +703,41 @@ static const struct dmi_system_id yc_acp_quirk_table[] = { DMI_MATCH(DMI_BOARD_NAME, "XyloD5_RBU"), } }, - + { + .driver_data = &acp6x_card, + .matches = { + DMI_MATCH(DMI_BOARD_VENDOR, "ASUSTeK COMPUTER INC."), + DMI_MATCH(DMI_PRODUCT_NAME, "Vivobook_ASUSLaptop M6501RR_M6501RR"), + } + }, + { + .driver_data = &acp6x_card, + .matches = { + DMI_MATCH(DMI_BOARD_VENDOR, "ASUSTeK COMPUTER INC."), + DMI_MATCH(DMI_PRODUCT_NAME, "ASUS EXPERTBOOK BM1503CDA"), + } + }, + { + .driver_data = &acp6x_card, + .matches = { + DMI_MATCH(DMI_BOARD_VENDOR, "ASUSTeK COMPUTER INC."), + DMI_MATCH(DMI_BOARD_NAME, "PM1503CDA"), + } + }, + { + .driver_data = &acp6x_card, + .matches = { + DMI_MATCH(DMI_BOARD_VENDOR, "ASUSTeK COMPUTER INC."), + DMI_MATCH(DMI_BOARD_NAME, "BM1403CDA"), + } + }, + { + .driver_data = &acp6x_card, + .matches = { + DMI_MATCH(DMI_BOARD_VENDOR, "Micro-Star International Co., Ltd."), + DMI_MATCH(DMI_PRODUCT_NAME, "Thin A15 B7VE"), + } + }, {} }; diff --git a/sound/soc/apple/Kconfig b/sound/soc/apple/Kconfig index d8dc2f1ccc83e0..d47b189e0d611d 100644 --- a/sound/soc/apple/Kconfig +++ b/sound/soc/apple/Kconfig @@ -1,5 +1,16 @@ menu "Apple" +config SND_SOC_APPLE_AOP_AUDIO + tristate "AOP audio driver" + depends on ARCH_APPLE || COMPILE_TEST + depends on RUST + select APPLE_AOP + select SND_DMAENGINE_PCM + help + This option enables an ASoC driver for sound devices connected to the AOP + co-processor on ARM Macs. This includes the built-in microphone on those + machines. + config SND_SOC_APPLE_MCA tristate "Apple Silicon MCA driver" depends on ARCH_APPLE || COMPILE_TEST @@ -8,4 +19,21 @@ config SND_SOC_APPLE_MCA This option enables an ASoC platform driver for MCA peripherals found on Apple Silicon SoCs. +config SND_SOC_APPLE_MACAUDIO + tristate "Sound support for Apple Silicon Macs" + depends on ARCH_APPLE || COMPILE_TEST + select SND_SOC_APPLE_MCA + select SND_SIMPLE_CARD_UTILS + select APPLE_ADMAC if DMADEVICES + select COMMON_CLK_APPLE_NCO if COMMON_CLK + select SND_SOC_TAS2764 if I2C + select SND_SOC_TAS2770 if I2C + select SND_SOC_CS42L83 if I2C + select SND_SOC_CS42L84 if I2C + select REGULATOR_FIXED_VOLTAGE if REGULATOR + help + This option enables an ASoC machine-level driver for Apple Silicon Macs + and it also enables the required SoC and codec drivers for overall + sound support on these machines. + endmenu diff --git a/sound/soc/apple/Makefile b/sound/soc/apple/Makefile index 1eb8fbef60c617..7d4901f407014d 100644 --- a/sound/soc/apple/Makefile +++ b/sound/soc/apple/Makefile @@ -1,3 +1,10 @@ +snd-soc-aop-y := aop_audio.o +obj-$(CONFIG_SND_SOC_APPLE_AOP_AUDIO) += snd-soc-aop.o + snd-soc-apple-mca-y := mca.o obj-$(CONFIG_SND_SOC_APPLE_MCA) += snd-soc-apple-mca.o + +snd-soc-macaudio-objs := macaudio.o + +obj-$(CONFIG_SND_SOC_APPLE_MACAUDIO) += snd-soc-macaudio.o diff --git a/sound/soc/apple/aop_audio.rs b/sound/soc/apple/aop_audio.rs new file mode 100644 index 00000000000000..10b57a4518ab03 --- /dev/null +++ b/sound/soc/apple/aop_audio.rs @@ -0,0 +1,729 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#![recursion_limit = "2048"] + +//! Apple AOP audio driver +//! +//! Copyright (C) The Asahi Linux Contributors + +use core::fmt; +use core::sync::atomic::{AtomicU32, Ordering}; +use core::{mem, ptr, slice}; + +use kernel::{ + bindings, c_str, device, + device::property::FwNode, + device::Core, + error::from_err_ptr, + module_platform_driver, of, platform, + prelude::*, + soc::apple::aop::{from_fourcc, EPICService, AOP}, + str::CString, + sync::Arc, + types::{ARef, ForeignOwnable}, +}; + +use pin_init::Zeroable; + +/// An unaligned u32 type. +/// +/// This is useful to avoid having to pack firmware structures entirely, since that is incompatible +/// with `#[derive(Debug)]` and atomics. +#[derive(Copy, Clone, Default)] +#[repr(C, packed(1))] +pub(crate) struct U32(pub(crate) u32); + +// SAFETY: U32 is zeroable just like u32 +unsafe impl Zeroable for U32 {} + +impl fmt::Debug for U32 { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let v = self.0; + f.write_fmt(format_args!("{:#x}", v)) + } +} + +const EPIC_SUBTYPE_WRAPPED_CALL: u16 = 0x20; +const CALLTYPE_AUDIO_ATTACH_DEVICE: u32 = 0xc3000002; +const CALLTYPE_AUDIO_SET_PROP: u32 = 0xc3000005; +const PDM_NUM_COEFFS: usize = 120; +const DECIMATION_RATIOS: [u8; 3] = [0xf, 5, 2]; +const COEFFICIENTS: [u8; PDM_NUM_COEFFS * mem::size_of::()] = [ + 0x88, 0x03, 0x00, 0x00, 0x82, 0x08, 0x00, 0x00, 0x51, 0x12, 0x00, 0x00, 0x0a, 0x23, 0x00, 0x00, + 0xce, 0x3d, 0x00, 0x00, 0x97, 0x66, 0x00, 0x00, 0x43, 0xa2, 0x00, 0x00, 0x9c, 0xf6, 0x00, 0x00, + 0x53, 0x6a, 0x01, 0x00, 0xe6, 0x04, 0x02, 0x00, 0x7e, 0xce, 0x02, 0x00, 0xae, 0xcf, 0x03, 0x00, + 0x2e, 0x11, 0x05, 0x00, 0x7d, 0x9b, 0x06, 0x00, 0x75, 0x76, 0x08, 0x00, 0xd8, 0xa8, 0x0a, 0x00, + 0xd2, 0x37, 0x0d, 0x00, 0x82, 0x26, 0x10, 0x00, 0x86, 0x75, 0x13, 0x00, 0x97, 0x22, 0x17, 0x00, + 0x39, 0x28, 0x1b, 0x00, 0x89, 0x7d, 0x1f, 0x00, 0x2e, 0x16, 0x24, 0x00, 0x69, 0xe2, 0x28, 0x00, + 0x56, 0xcf, 0x2d, 0x00, 0x51, 0xc7, 0x32, 0x00, 0x80, 0xb2, 0x37, 0x00, 0x87, 0x77, 0x3c, 0x00, + 0x4c, 0xfc, 0x40, 0x00, 0xd9, 0x26, 0x45, 0x00, 0x47, 0xde, 0x48, 0x00, 0xa0, 0x0b, 0x4c, 0x00, + 0xc1, 0x9a, 0x4e, 0x00, 0x1f, 0x7b, 0x50, 0x00, 0x68, 0xa0, 0x51, 0x00, 0x06, 0x03, 0x52, 0x00, + 0x4a, 0x25, 0x00, 0x00, 0x4c, 0xaf, 0x00, 0x00, 0xc0, 0x07, 0x02, 0x00, 0x45, 0x99, 0x04, 0x00, + 0x9a, 0x84, 0x08, 0x00, 0x7d, 0x38, 0x0d, 0x00, 0x5f, 0x1a, 0x11, 0x00, 0xd9, 0x81, 0x11, 0x00, + 0x80, 0x44, 0x0b, 0x00, 0x8e, 0xe5, 0xfb, 0xff, 0xca, 0x32, 0xe3, 0xff, 0x52, 0xc7, 0xc4, 0xff, + 0xa6, 0xbc, 0xa8, 0xff, 0x83, 0xe6, 0x9a, 0xff, 0xb8, 0x5b, 0xa8, 0xff, 0x6b, 0xae, 0xdb, 0xff, + 0xe7, 0xd8, 0x38, 0x00, 0x24, 0x42, 0xba, 0x00, 0x33, 0x20, 0x50, 0x01, 0x6e, 0xdc, 0xe2, 0x01, + 0x42, 0x23, 0x58, 0x02, 0x2c, 0x50, 0x99, 0x02, 0xcf, 0xfa, 0xff, 0xff, 0x53, 0x0a, 0xff, 0xff, + 0x66, 0x23, 0xfb, 0xff, 0xa0, 0x3e, 0xf4, 0xff, 0xe6, 0x68, 0xf0, 0xff, 0xb8, 0x35, 0xf7, 0xff, + 0x56, 0xec, 0x04, 0x00, 0x37, 0xa3, 0x09, 0x00, 0x00, 0xd4, 0xfe, 0xff, 0x78, 0xa3, 0xf5, 0xff, + 0x03, 0xbf, 0xfe, 0xff, 0x84, 0xd5, 0x0b, 0x00, 0xbe, 0x0b, 0x04, 0x00, 0x52, 0x54, 0xf2, 0xff, + 0x6d, 0x3f, 0xf8, 0xff, 0xc5, 0x7f, 0x0f, 0x00, 0xe6, 0x9e, 0x0c, 0x00, 0x79, 0x03, 0xef, 0xff, + 0xd5, 0x33, 0xed, 0xff, 0xec, 0xd1, 0x11, 0x00, 0x7d, 0x69, 0x1a, 0x00, 0xd6, 0x55, 0xee, 0xff, + 0x88, 0x66, 0xdc, 0xff, 0x57, 0x26, 0x10, 0x00, 0xc7, 0x8d, 0x2e, 0x00, 0x82, 0x2e, 0xf3, 0xff, + 0x63, 0x69, 0xc4, 0xff, 0xcd, 0x08, 0x07, 0x00, 0x35, 0x34, 0x4b, 0x00, 0xaf, 0x21, 0x02, 0x00, + 0x83, 0xb6, 0xa1, 0xff, 0xe2, 0xd5, 0xef, 0xff, 0x94, 0x9b, 0x76, 0x00, 0xf3, 0xd7, 0x25, 0x00, + 0xff, 0xfc, 0x67, 0xff, 0xe3, 0xac, 0xb6, 0xff, 0x52, 0x1b, 0xcc, 0x00, 0x3c, 0x8a, 0x8b, 0x00, + 0x9f, 0x0c, 0xcd, 0xfe, 0x5c, 0x68, 0xcc, 0xfe, 0x4d, 0xc5, 0x98, 0x02, 0x82, 0xcf, 0xfb, 0x06, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +]; +const FILTER_LENGTHS: u32 = 0x542c47; +const AUDIO_DEV_PDM0: u32 = from_fourcc(b"pdm0"); +const AUDIO_DEV_LPAI: u32 = from_fourcc(b"lpai"); +const AUDIO_DEV_HPAI: u32 = from_fourcc(b"hpai"); +const POWER_STATE_OFF: u32 = from_fourcc(b"idle"); +const POWER_STATE_IDLE: u32 = from_fourcc(b"pw1 "); +const POWER_STATE_ON: u32 = from_fourcc(b"pwrd"); + +#[repr(C, packed)] +#[derive(Clone, Copy, Default)] +struct AudioAttachDevice { + _zero0: u32, + unk0: u32, + calltype: u32, + _zero1: u64, + _zero2: u64, + _pad0: u32, + len: u64, + dev_id: u32, + _pad1: u32, +} + +impl AudioAttachDevice { + fn new(dev_id: u32) -> AudioAttachDevice { + AudioAttachDevice { + unk0: 0xFFFFFFFF, + calltype: CALLTYPE_AUDIO_ATTACH_DEVICE, + dev_id, + len: 0x2c, + ..AudioAttachDevice::default() + } + } +} + +#[repr(C, packed(4))] +#[derive(Clone, Copy, Default)] +struct LpaiChannelConfig { + unk1: u32, + unk2: u32, + unk3: u32, + unk4: u32, +} + +#[repr(C, packed(4))] +#[derive(Debug, Copy, Clone)] +struct PDMConfig { + bytes_per_sample: u32, + clock_source: u32, + pdm_frequency: u32, + pdmc_frequency: u32, + slow_clock_speed: u32, + fast_clock_speed: u32, + channel_polarity_select: u32, + channel_phase_select: u32, + unk1: u32, + unk2: u16, + ratio1: u8, + ratio2: u8, + ratio3: u8, + _pad0: u8, + filter_lengths: U32, + coeff_bulk: U32, + coeffs: [u8; PDM_NUM_COEFFS * mem::size_of::()], + unk3: U32, + mic_turn_on_time_ms: U32, + _zero0: [u8; 8], + _zero1: [u8; 8], + unk4: U32, + mic_settle_time_ms: U32, + _zero2: [u8; 69], // ????? + _pad_extra: u8, // extra padding to increase the struct size to multiple of mem::size_of::() +} +// PDMConfig is intended to use `#[repr(C, packed)]` but this +// conflicts ith pin_init. Instead just ensure that it has the same size as if +// it where packed. +static_assert!(mem::size_of::() == 36 + 14 + (120 * 4) + 32 + 69 + 1); + +unsafe impl Zeroable for PDMConfig {} + +#[repr(C, packed(4))] +#[derive(Debug, Copy, Clone)] +struct DecimatorConfig { + latency: u32, + ratio1: u8, + ratio2: u8, + ratio3: u8, + _pad0: u8, + filter_lengths: u32, + coeff_bulk: u32, + coeffs: [u8; PDM_NUM_COEFFS * mem::size_of::()], +} +// DecimatorConfig is intended to use `#[repr(C, packed)]` but this +// conflicts ith pin_init. Instead just ensure that it has the same size as if +// it where packed. +static_assert!(mem::size_of::() == 16 + (120 * 4)); + +unsafe impl Zeroable for DecimatorConfig {} + +#[repr(C, packed(4))] +#[derive(Clone, Copy, Default, Debug)] +struct PowerSetting { + dev_id: u32, + cookie: u32, + _unk0: u32, + _zero0: [u8; 8], + target_pstate: u32, + unk1: u32, + _zero1: [u8; 20], +} + +impl PowerSetting { + fn new(dev_id: u32, cookie: u32, target_pstate: u32, unk1: u32) -> PowerSetting { + PowerSetting { + dev_id, + cookie, + target_pstate, + unk1, + ..PowerSetting::default() + } + } +} + +#[repr(C)] +#[derive(Clone, Copy, Default, Debug)] +struct AudioSetDeviceProp { + _zero0: u32, + unk0: u32, + calltype: u32, + _zero1: [u8; 8], + _zero2: [u8; 8], + _pad0: u32, + len: u32, + _pad1: u32, + dev_id: u32, + modifier: u32, + len2: u32, + data: T, +} +// AudioSetDeviceProp is intended to use `#[repr(C, packed)]` but this +// conflicts ith pin_init. Instead just ensure that it has the same size as if +// it where packed. +static_assert!(mem::size_of::>() == 52 + mem::size_of::()); +static_assert!( + mem::size_of::>() == 52 + mem::size_of::() +); +static_assert!( + mem::size_of::>() + == 52 + mem::size_of::() +); +static_assert!( + mem::size_of::>() == 52 + mem::size_of::() +); + +impl AudioSetDeviceProp { + fn new(dev_id: u32, modifier: u32, data: T) -> AudioSetDeviceProp { + AudioSetDeviceProp { + unk0: 0xFFFFFFFF, + calltype: CALLTYPE_AUDIO_SET_PROP, + dev_id, + modifier, + len: mem::size_of::() as u32 + 0x30, + len2: mem::size_of::() as u32, + data, + ..AudioSetDeviceProp::default() + } + } +} + +unsafe impl Zeroable for AudioSetDeviceProp {} + +impl AudioSetDeviceProp { + fn try_init( + dev_id: u32, + modifier: u32, + data: impl Init, + ) -> impl Init, Error> + where + Error: From, + { + try_init!( + AudioSetDeviceProp { + unk0: 0xFFFFFFFF, + calltype: CALLTYPE_AUDIO_SET_PROP, + dev_id, + modifier, + len: mem::size_of::() as u32 + 0x30, + len2: mem::size_of::() as u32, + data <- data, + ..Zeroable::init_zeroed() + } + ) + } +} + +struct SndSocAopData { + dev: ARef, + adata: Arc, + service: EPICService, + pstate_cookie: AtomicU32, + fwnode: ARef, +} + +impl SndSocAopData { + fn new( + dev: ARef, + adata: Arc, + service: EPICService, + fwnode: ARef, + ) -> Result> { + Ok(Arc::new( + SndSocAopData { + dev, + adata, + service, + fwnode, + pstate_cookie: AtomicU32::new(1), + }, + GFP_KERNEL, + )?) + } + fn set_pdm_config(&self) -> Result<()> { + let pdm_cfg = init!(PDMConfig { + bytes_per_sample: 2, + clock_source: 0x706c6c20, // 'pll ' + pdm_frequency: 2400000, + pdmc_frequency: 24000000, + slow_clock_speed: 24000000, + fast_clock_speed: 24000000, + channel_polarity_select: 256, + channel_phase_select: 0, + unk1: 0xf7600, + unk2: 0, + ratio1: DECIMATION_RATIOS[0], + ratio2: DECIMATION_RATIOS[1], + ratio3: DECIMATION_RATIOS[2], + filter_lengths: U32(FILTER_LENGTHS), + coeff_bulk: U32(PDM_NUM_COEFFS as u32), + coeffs: COEFFICIENTS, + unk3: U32(1), + mic_turn_on_time_ms: U32(20), + unk4: U32(1), + mic_settle_time_ms: U32(50), + ..Zeroable::init_zeroed() + }); + let set_prop = AudioSetDeviceProp::::try_init(AUDIO_DEV_PDM0, 200, pdm_cfg); + let msg = KBox::try_init(set_prop, GFP_KERNEL)?; + let ret = self.epic_wrapped_call(msg.as_ref())?; + if ret != 0 { + dev_err!(self.dev, "Unable to set pdm config, return code {}", ret); + return Err(EIO); + } else { + Ok(()) + } + } + fn set_decimator_config(&self) -> Result<()> { + let pdm_cfg = init!(DecimatorConfig { + latency: 15, + ratio1: DECIMATION_RATIOS[0], + ratio2: DECIMATION_RATIOS[1], + ratio3: DECIMATION_RATIOS[2], + filter_lengths: FILTER_LENGTHS, + coeff_bulk: PDM_NUM_COEFFS as u32, + coeffs: COEFFICIENTS, + ..Zeroable::init_zeroed() + }); + let set_prop = + AudioSetDeviceProp::::try_init(AUDIO_DEV_PDM0, 210, pdm_cfg); + let msg = KBox::try_init(set_prop, GFP_KERNEL)?; + let ret = self.epic_wrapped_call(msg.as_ref())?; + if ret != 0 { + dev_err!( + self.dev, + "Unable to set decimator config, return code {}", + ret + ); + return Err(EIO); + } else { + Ok(()) + } + } + fn set_lpai_channel_cfg(&self) -> Result<()> { + let cfg = LpaiChannelConfig { + unk1: 7, + unk2: 7, + unk3: 1, + unk4: 7, + }; + let msg = AudioSetDeviceProp::new(AUDIO_DEV_LPAI, 301, cfg); + let ret = self.epic_wrapped_call(&msg)?; + if ret != 0 { + dev_err!( + self.dev, + "Unable to set lpai channel config, return code {}", + ret + ); + return Err(EIO); + } else { + Ok(()) + } + } + fn audio_attach_device(&self, dev_id: u32) -> Result<()> { + let msg = AudioAttachDevice::new(dev_id); + let ret = self.epic_wrapped_call(&msg)?; + if ret != 0 { + dev_err!( + self.dev, + "Unable to attach device {:?}, return code {}", + dev_id, + ret + ); + return Err(EIO); + } else { + Ok(()) + } + } + fn set_audio_power(&self, pstate: u32, unk1: u32) -> Result<()> { + let set_pstate = PowerSetting::new( + AUDIO_DEV_HPAI, + self.pstate_cookie.fetch_add(1, Ordering::Relaxed), + pstate, + unk1, + ); + let msg = AudioSetDeviceProp::new(AUDIO_DEV_HPAI, 202, set_pstate); + let ret = self.epic_wrapped_call(&msg)?; + if ret != 0 { + dev_err!( + self.dev, + "Unable to set power state {:?}, return code {}", + pstate, + ret + ); + return Err(EIO); + } else { + Ok(()) + } + } + fn epic_wrapped_call(&self, data: &T) -> Result { + let msg_bytes = + unsafe { slice::from_raw_parts(data as *const T as *const u8, mem::size_of::()) }; + self.adata + .epic_call(&self.service, EPIC_SUBTYPE_WRAPPED_CALL, msg_bytes) + } + fn request_dma_channel(&self) -> Result<*mut bindings::dma_chan> { + let res = unsafe { + from_err_ptr(bindings::dma_request_chan( + self.dev.as_raw(), + c_str!("dma").as_ptr() as _, + )) + }; + if res.is_err() { + dev_err!(self.dev, "Unable to get dma channel"); + } + res + } +} + +#[repr(transparent)] +struct SndSocAopDriver(*mut bindings::snd_card); + +fn copy_str(target: &mut [u8], source: &[u8]) { + target[..source.len()].copy_from_slice(source) +} + +unsafe fn dmaengine_slave_config( + chan: *mut bindings::dma_chan, + config: *mut bindings::dma_slave_config, +) -> i32 { + unsafe { + match (*(*chan).device).device_config { + Some(dc) => dc(chan, config), + None => ENOSYS.to_errno(), + } + } +} + +unsafe extern "C" fn aop_hw_params( + substream: *mut bindings::snd_pcm_substream, + params: *mut bindings::snd_pcm_hw_params, +) -> i32 { + let chan = unsafe { bindings::snd_dmaengine_pcm_get_chan(substream) }; + let mut slave_config = bindings::dma_slave_config::default(); + let ret = + unsafe { bindings::snd_hwparams_to_dma_slave_config(substream, params, &mut slave_config) }; + if ret < 0 { + return ret; + } + slave_config.src_port_window_size = 4; + unsafe { dmaengine_slave_config(chan, &mut slave_config) } +} + +unsafe extern "C" fn aop_pcm_open(substream: *mut bindings::snd_pcm_substream) -> i32 { + let data = unsafe { Arc::::borrow((*substream).private_data.cast()) }; + if let Err(e) = data.set_audio_power(POWER_STATE_IDLE, 0) { + dev_err!(data.dev, "Unable to enter 'pw1 ' state"); + return e.to_errno(); + } + let mut hwparams = bindings::snd_pcm_hardware { + info: bindings::SNDRV_PCM_INFO_MMAP + | bindings::SNDRV_PCM_INFO_MMAP_VALID + | bindings::SNDRV_PCM_INFO_INTERLEAVED, + formats: bindings::BINDINGS_SNDRV_PCM_FMTBIT_FLOAT_LE, + subformats: 0, + rates: bindings::SNDRV_PCM_RATE_48000, + rate_min: 48000, + rate_max: 48000, + channels_min: 3, + channels_max: 3, + periods_min: 2, + buffer_bytes_max: usize::MAX, + period_bytes_max: 0x4000, + periods_max: u32::MAX, + period_bytes_min: 256, + fifo_size: 16, + }; + let dma_chan = match data.request_dma_channel() { + Ok(dc) => dc, + Err(e) => return e.to_errno(), + }; + + if unsafe { (*substream).dma_buffer.dev.type_ == bindings::SNDRV_DMA_TYPE_UNKNOWN as _ } { + let ret = unsafe { + bindings::snd_pcm_set_managed_buffer( + substream, + bindings::SNDRV_DMA_TYPE_DEV_IRAM as i32, + (*(*dma_chan).device).dev, + 0, + 0, + ) + }; + if ret < 0 { + dev_err!(data.dev, "Unable to allocate dma buffers"); + unsafe { + bindings::dma_release_channel(dma_chan); + } + return ret; + } + } + + let ret = unsafe { + let mut dai_data = bindings::snd_dmaengine_dai_dma_data::default(); + bindings::snd_dmaengine_pcm_refine_runtime_hwparams( + substream, + &mut dai_data, + &mut hwparams, + dma_chan, + ) + }; + if ret != 0 { + dev_err!(data.dev, "Unable to refine hwparams"); + return ret; + } + if let Err(e) = data.set_audio_power(POWER_STATE_ON, 1) { + dev_err!(data.dev, "Unable to power mic on"); + return e.to_errno(); + } + unsafe { + (*(*substream).runtime).hw = hwparams; + bindings::snd_dmaengine_pcm_open(substream, dma_chan) + } +} + +unsafe extern "C" fn aop_pcm_prepare(_: *mut bindings::snd_pcm_substream) -> i32 { + 0 +} + +unsafe extern "C" fn aop_pcm_close(substream: *mut bindings::snd_pcm_substream) -> i32 { + let data = unsafe { Arc::::borrow((*substream).private_data.cast()) }; + if let Err(e) = data.set_audio_power(POWER_STATE_IDLE, 1) { + dev_err!(data.dev, "Unable to power mic off"); + return e.to_errno(); + } + let ret = unsafe { bindings::snd_dmaengine_pcm_close_release_chan(substream) }; + if ret != 0 { + dev_err!(data.dev, "Unable to close channel"); + return ret; + } + if let Err(e) = data.set_audio_power(POWER_STATE_OFF, 0) { + dev_err!(data.dev, "Unable to enter 'idle' power state"); + return e.to_errno(); + } + 0 +} + +unsafe extern "C" fn aop_pcm_free_private(pcm: *mut bindings::snd_pcm) { + unsafe { + Arc::::from_foreign((*pcm).private_data.cast()); + } +} + +impl SndSocAopDriver { + const VTABLE: bindings::snd_pcm_ops = bindings::snd_pcm_ops { + open: Some(aop_pcm_open), + close: Some(aop_pcm_close), + prepare: Some(aop_pcm_prepare), + trigger: Some(bindings::snd_dmaengine_pcm_trigger), + pointer: Some(bindings::snd_dmaengine_pcm_pointer), + ioctl: None, + hw_params: Some(aop_hw_params), + hw_free: None, + sync_stop: None, + get_time_info: None, + fill_silence: None, + copy: None, + page: None, + mmap: None, + ack: None, + }; + fn new(data: Arc) -> Result { + let mut this = SndSocAopDriver(ptr::null_mut()); + let ret = unsafe { + bindings::snd_card_new( + data.dev.as_raw(), + -1, + ptr::null(), + THIS_MODULE.as_ptr(), + 0, + &mut this.0, + ) + }; + if ret < 0 { + dev_err!(data.dev, "Unable to allocate sound card"); + return Err(Error::from_errno(ret)); + } + let chassis = data + .fwnode + .property_read::(c_str!("apple,chassis-name")) + .required_by(&data.dev)?; + let machine_kind = data + .fwnode + .property_read::(c_str!("apple,machine-kind")) + .required_by(&data.dev)?; + unsafe { + copy_str(&mut (*this.0).driver, c"aop_audio".to_bytes_with_nul()); + } + let id_str = CString::try_from_fmt(fmt!("Apple{}HPAI", *chassis))?; + unsafe { + copy_str(&mut (*this.0).id, id_str.to_bytes_with_nul()); + } + let shortname = CString::try_from_fmt(fmt!("{} {} HPAI", *machine_kind, *chassis))?; + unsafe { + copy_str(&mut (*this.0).shortname, shortname.to_bytes_with_nul()); + } + let longname = CString::try_from_fmt(fmt!( + "{} {} High-Power Audio Interface", + *machine_kind, + *chassis + ))?; + unsafe { + copy_str(&mut (*this.0).longname, longname.to_bytes_with_nul()); + } + + let mut pcm = ptr::null_mut(); + let ret = + unsafe { bindings::snd_pcm_new(this.0, longname.as_ptr() as _, 0, 0, 1, &mut pcm) }; + if ret < 0 { + dev_err!(data.dev, "Unable to allocate PCM device"); + return Err(Error::from_errno(ret)); + } + + unsafe { + bindings::snd_pcm_set_ops( + pcm, + bindings::SNDRV_PCM_STREAM_CAPTURE as i32, + &Self::VTABLE, + ); + } + + unsafe { + (*pcm).private_data = data.clone().into_foreign() as _; + (*pcm).private_free = Some(aop_pcm_free_private); + (*pcm).info_flags = 0; + copy_str(&mut (*pcm).name, c"aop_audio".to_bytes_with_nul()); + } + + let ret = unsafe { bindings::snd_card_register(this.0) }; + if ret < 0 { + dev_err!(data.dev, "Unable to register sound card"); + return Err(Error::from_errno(ret)); + } + Ok(this) + } +} + +impl Drop for SndSocAopDriver { + fn drop(&mut self) { + if self.0 != ptr::null_mut() { + unsafe { + bindings::snd_card_free(self.0); + } + } + } +} + +unsafe impl Send for SndSocAopDriver {} +unsafe impl Sync for SndSocAopDriver {} + +kernel::of_device_table!( + OF_TABLE, + MODULE_OF_TABLE, + (), + [(of::DeviceId::new(c_str!("apple,aop-audio")), ())] +); + +impl platform::Driver for SndSocAopDriver { + type IdInfo = (); + + const OF_ID_TABLE: Option> = Some(&OF_TABLE); + + fn probe(pdev: &platform::Device, _info: Option<&()>) -> impl PinInit { + let dev = ARef::::from(pdev.as_ref()); + let parent = pdev.as_ref().parent().unwrap(); + // SAFETY: our parent is AOP, and AopDriver is repr(transparent) for Arc + let adata_ptr = unsafe { Pin::>>::borrow(parent.get_drvdata()) }; + let adata = (&*adata_ptr).clone(); + // SAFETY: AOP sets the platform data correctly + let svc = unsafe { *((*dev.as_raw()).platform_data as *const EPICService) }; + let parent_fwnode = parent.fwnode().ok_or(ENOENT)?; + let fwnode = parent_fwnode + .get_child_by_name(c_str!("audio")) + .ok_or(EIO)?; + let audio = *module_parameters::mic_check_123.value() != 0; + if !audio && parent_fwnode.property_present(c_str!("apple,no-beamforming")) { + return Err(ENODEV); + } + let data = SndSocAopData::new(dev, adata, svc, fwnode)?; + for dev in [AUDIO_DEV_PDM0, AUDIO_DEV_HPAI, AUDIO_DEV_LPAI] { + data.audio_attach_device(dev)?; + } + data.set_lpai_channel_cfg()?; + data.set_pdm_config()?; + data.set_decimator_config()?; + Ok(Self::new(data)?) + } +} + +module_platform_driver! { + type: SndSocAopDriver, + name: "snd_soc_apple_aop", + description: "AOP microphone capture driver", + license: "Dual MIT/GPL", + alias: ["platform:snd_soc_apple_aop"], + params: { + mic_check_123: u8 { + default: 0, + description: "Enable mics without user space handling", + }, + }, +} diff --git a/sound/soc/apple/macaudio.c b/sound/soc/apple/macaudio.c new file mode 100644 index 00000000000000..1a32679f41ea90 --- /dev/null +++ b/sound/soc/apple/macaudio.c @@ -0,0 +1,1684 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * ASoC machine driver for Apple Silicon Macs + * + * Copyright (C) The Asahi Linux Contributors + * + * Based on sound/soc/qcom/{sc7180.c|common.c} + * Copyright (c) 2018, Linaro Limited. + * Copyright (c) 2020, The Linux Foundation. All rights reserved. + * + * + * The platform driver has independent frontend and backend DAIs with the + * option of routing backends to any of the frontends. The platform + * driver configures the routing based on DPCM couplings in ASoC runtime + * structures, which in turn are determined from DAPM paths by ASoC. But the + * platform driver doesn't supply relevant DAPM paths and leaves that up for + * the machine driver to fill in. The filled-in virtual topology can be + * anything as long as any backend isn't connected to more than one frontend + * at any given time. (The limitation is due to the unsupported case of + * reparenting of live BEs.) + */ + +/* #define DEBUG */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define DRIVER_NAME "snd-soc-macaudio" + +/* + * CPU side is bit and frame clock provider + * I2S has both clocks inverted + */ +#define MACAUDIO_DAI_FMT (SND_SOC_DAIFMT_I2S | \ + SND_SOC_DAIFMT_CBC_CFC | \ + SND_SOC_DAIFMT_GATED | \ + SND_SOC_DAIFMT_IB_IF) +#define MACAUDIO_JACK_MASK (SND_JACK_HEADSET | SND_JACK_HEADPHONE) +#define MACAUDIO_SLOTWIDTH 32 +/* + * Maximum BCLK frequency + * + * Codec maximums: + * CS42L42 26.0 MHz + * TAS2770 27.1 MHz + * TAS2764 24.576 MHz + */ +#define MACAUDIO_MAX_BCLK_FREQ 24576000 + +#define SPEAKER_MAGIC_VALUE (s32)0xdec1be15 +/* milliseconds */ +#define SPEAKER_LOCK_TIMEOUT 250 + +enum macaudio_amp_type { + AMP_NONE, + AMP_TAS5770, + AMP_SN012776, + AMP_SSM3515, +}; + +enum macaudio_spkr_config { + SPKR_NONE, /* No speakers */ + SPKR_1W, /* 1 woofer / ch */ + SPKR_2W, /* 2 woofers / ch */ + SPKR_1W1T, /* 1 woofer + 1 tweeter / ch */ + SPKR_2W1T, /* 2 woofers + 1 tweeter / ch */ +}; + +struct macaudio_platform_cfg { + bool enable_speakers; + enum macaudio_amp_type amp; + enum macaudio_spkr_config speakers; + bool stereo; + int amp_gain; + int safe_vol; +}; + +static const char *volume_control_names[] = { + [AMP_TAS5770] = "* Speaker Playback Volume", + [AMP_SN012776] = "* Speaker Volume", + [AMP_SSM3515] = "* DAC Playback Volume", +}; + +#define SN012776_0DB 201 +#define SN012776_DB(x) (SN012776_0DB + 2 * (x)) +/* Same as SN012776 */ +#define TAS5770_0DB SN012776_0DB +#define TAS5770_DB(x) SN012776_DB(x) + +#define SSM3515_0DB (255 - 64) /* +24dB max, steps of 3/8 dB */ +#define SSM3515_DB(x) (SSM3515_0DB + (8 * (x) / 3)) + +struct macaudio_snd_data { + struct snd_soc_card card; + struct snd_soc_jack jack; + int jack_plugin_state; + + const struct macaudio_platform_cfg *cfg; + bool has_speakers; + bool has_sense; + bool has_safety; + unsigned int max_channels; + + struct macaudio_link_props { + /* frontend props */ + unsigned int bclk_ratio; + bool is_sense; + + /* backend props */ + bool is_speakers; + bool is_headphones; + unsigned int tdm_mask; + } *link_props; + + int speaker_sample_rate; + struct snd_kcontrol *speaker_sample_rate_kctl; + + struct mutex volume_lock_mutex; + bool speaker_volume_unlocked; + bool speaker_volume_was_locked; + struct snd_kcontrol *speaker_lock_kctl; + struct snd_ctl_file *speaker_lock_owner; + u64 bes_active; + bool speaker_lock_timeout_enabled; + ktime_t speaker_lock_timeout; + ktime_t speaker_lock_remain; + struct delayed_work lock_timeout_work; + struct work_struct lock_update_work; + +}; + +static int please_blow_up_my_speakers; +module_param(please_blow_up_my_speakers, int, 0644); +MODULE_PARM_DESC(please_blow_up_my_speakers, "Allow unsafe or untested operating configurations"); + +SND_SOC_DAILINK_DEFS(primary, + DAILINK_COMP_ARRAY(COMP_CPU("mca-pcm-0")), // CPU + DAILINK_COMP_ARRAY(COMP_DUMMY()), // CODEC + DAILINK_COMP_ARRAY(COMP_EMPTY())); // platform (filled at runtime) + +SND_SOC_DAILINK_DEFS(secondary, + DAILINK_COMP_ARRAY(COMP_CPU("mca-pcm-1")), // CPU + DAILINK_COMP_ARRAY(COMP_DUMMY()), // CODEC + DAILINK_COMP_ARRAY(COMP_EMPTY())); + +SND_SOC_DAILINK_DEFS(sense, + DAILINK_COMP_ARRAY(COMP_CPU("mca-pcm-2")), // CPU + DAILINK_COMP_ARRAY(COMP_DUMMY()), // CODEC + DAILINK_COMP_ARRAY(COMP_EMPTY())); + +static struct snd_soc_dai_link macaudio_fe_links[] = { + { + .name = "Primary", + .stream_name = "Primary", + .dynamic = 1, + .dpcm_merged_rate = 1, + .dpcm_merged_chan = 1, + .dpcm_merged_format = 1, + .dai_fmt = MACAUDIO_DAI_FMT, + SND_SOC_DAILINK_REG(primary), + }, + { + .name = "Secondary", + .stream_name = "Secondary", + .dynamic = 1, + .dpcm_merged_rate = 1, + .dpcm_merged_chan = 1, + .dpcm_merged_format = 1, + .dai_fmt = MACAUDIO_DAI_FMT, + .playback_only = 1, + SND_SOC_DAILINK_REG(secondary), + }, + { + .name = "Speaker Sense", + .stream_name = "Speaker Sense", + .capture_only = 1, + .dynamic = 1, + .dai_fmt = (SND_SOC_DAIFMT_I2S | \ + SND_SOC_DAIFMT_CBP_CFP | \ + SND_SOC_DAIFMT_GATED | \ + SND_SOC_DAIFMT_IB_IF), + SND_SOC_DAILINK_REG(sense), + }, +}; + +static struct macaudio_link_props macaudio_fe_link_props[] = { + { + /* + * Primary FE + * + * The bclk ratio at 64 for the primary frontend is important + * to ensure that the headphones codec's idea of left and right + * in a stereo stream over I2S fits in nicely with everyone else's. + * (This is until the headphones codec's driver supports + * set_tdm_slot.) + * + * The low bclk ratio precludes transmitting more than two + * channels over I2S, but that's okay since there is the secondary + * FE for speaker arrays anyway. + */ + .bclk_ratio = 64, + }, + { + /* + * Secondary FE + * + * Here we want frames plenty long to be able to drive all + * those fancy speaker arrays. + */ + .bclk_ratio = 256, + }, + { + .is_sense = 1, + } +}; + +static void macaudio_vlimit_unlock(struct macaudio_snd_data *ma, bool unlock) +{ + int ret, max; + const char *name = volume_control_names[ma->cfg->amp]; + + if (!name) { + WARN_ON_ONCE(1); + return; + } + + switch (ma->cfg->amp) { + case AMP_NONE: + WARN_ON_ONCE(1); + return; + case AMP_TAS5770: + if (unlock) + max = TAS5770_0DB; + else + max = 1; //TAS5770_DB(ma->cfg->safe_vol); + break; + case AMP_SN012776: + if (unlock) + max = SN012776_0DB; + else + max = 1; //SN012776_DB(ma->cfg->safe_vol); + break; + case AMP_SSM3515: + if (unlock) + max = SSM3515_0DB; + else + max = SSM3515_DB(ma->cfg->safe_vol); + break; + } + + ret = snd_soc_limit_volume(&ma->card, name, max); + if (ret < 0) + dev_err(ma->card.dev, "Failed to %slock volume %s: %d\n", + unlock ? "un" : "", name, ret); +} + +static void macaudio_vlimit_update(struct macaudio_snd_data *ma) +{ + int i; + bool unlock = true; + struct snd_kcontrol *kctl; + const char *reason; + + /* Do nothing if there is no safety configured */ + if (!ma->has_safety) + return; + + /* Check that someone is holding the main lock */ + if (!ma->speaker_lock_owner) { + reason = "Main control not locked"; + unlock = false; + } + + /* Check that the control has been pinged within the timeout */ + if (ma->speaker_lock_remain <= 0) { + reason = "Lock timeout"; + unlock = false; + } + + /* Check that *every* limited control is locked by the same owner */ + list_for_each_entry(kctl, &ma->card.snd_card->controls, list) { + if(!snd_soc_control_matches(kctl, volume_control_names[ma->cfg->amp])) + continue; + + for (i = 0; i < kctl->count; i++) { + if (kctl->vd[i].owner != ma->speaker_lock_owner) { + reason = "Not all child controls locked by the same process"; + unlock = false; + } + } + } + + + if (unlock != ma->speaker_volume_unlocked) { + if (unlock) { + dev_info(ma->card.dev, "Speaker volumes unlocked\n"); + } else { + dev_info(ma->card.dev, "Speaker volumes locked: %s\n", reason); + ma->speaker_volume_was_locked = true; + } + + macaudio_vlimit_unlock(ma, unlock); + ma->speaker_volume_unlocked = unlock; + snd_ctl_notify(ma->card.snd_card, SNDRV_CTL_EVENT_MASK_VALUE, + &ma->speaker_lock_kctl->id); + } +} + +static void macaudio_vlimit_enable_timeout(struct macaudio_snd_data *ma) +{ + mutex_lock(&ma->volume_lock_mutex); + + if (ma->speaker_lock_timeout_enabled) { + mutex_unlock(&ma->volume_lock_mutex); + return; + } + + if (ma->speaker_lock_remain > 0) { + ma->speaker_lock_timeout = ktime_add(ktime_get(), ma->speaker_lock_remain); + schedule_delayed_work(&ma->lock_timeout_work, usecs_to_jiffies(ktime_to_us(ma->speaker_lock_remain))); + dev_dbg(ma->card.dev, "Enabling volume limit timeout: %ld us left\n", + (long)ktime_to_us(ma->speaker_lock_remain)); + } + + macaudio_vlimit_update(ma); + + ma->speaker_lock_timeout_enabled = true; + mutex_unlock(&ma->volume_lock_mutex); +} + +static void macaudio_vlimit_disable_timeout(struct macaudio_snd_data *ma) +{ + ktime_t now; + + mutex_lock(&ma->volume_lock_mutex); + + if (!ma->speaker_lock_timeout_enabled) { + mutex_unlock(&ma->volume_lock_mutex); + return; + } + + now = ktime_get(); + + cancel_delayed_work(&ma->lock_timeout_work); + + if (ktime_after(now, ma->speaker_lock_timeout)) + ma->speaker_lock_remain = 0; + else if (ma->speaker_lock_remain > 0) + ma->speaker_lock_remain = ktime_sub(ma->speaker_lock_timeout, now); + + dev_dbg(ma->card.dev, "Disabling volume limit timeout: %ld us left\n", + (long)ktime_to_us(ma->speaker_lock_remain)); + + macaudio_vlimit_update(ma); + + ma->speaker_lock_timeout_enabled = false; + + mutex_unlock(&ma->volume_lock_mutex); +} + +static void macaudio_vlimit_timeout_work(struct work_struct *wrk) +{ + struct macaudio_snd_data *ma = container_of(to_delayed_work(wrk), + struct macaudio_snd_data, lock_timeout_work); + + mutex_lock(&ma->volume_lock_mutex); + + ma->speaker_lock_remain = 0; + macaudio_vlimit_update(ma); + + mutex_unlock(&ma->volume_lock_mutex); +} + +static void macaudio_vlimit_update_work(struct work_struct *wrk) +{ + struct macaudio_snd_data *ma = container_of(wrk, + struct macaudio_snd_data, lock_update_work); + + if (ma->bes_active) + macaudio_vlimit_enable_timeout(ma); + else + macaudio_vlimit_disable_timeout(ma); +} + +static int macaudio_copy_link(struct device *dev, struct snd_soc_dai_link *target, + struct snd_soc_dai_link *source) +{ + memcpy(target, source, sizeof(struct snd_soc_dai_link)); + + target->cpus = devm_kmemdup(dev, target->cpus, + sizeof(*target->cpus) * target->num_cpus, + GFP_KERNEL); + target->codecs = devm_kmemdup(dev, target->codecs, + sizeof(*target->codecs) * target->num_codecs, + GFP_KERNEL); + target->platforms = devm_kmemdup(dev, target->platforms, + sizeof(*target->platforms) * target->num_platforms, + GFP_KERNEL); + + if (!target->cpus || !target->codecs || !target->platforms) + return -ENOMEM; + + return 0; +} + +static int macaudio_parse_of_component(struct device_node *node, int index, + struct snd_soc_dai_link_component *comp) +{ + struct of_phandle_args args; + int ret; + + ret = of_parse_phandle_with_args(node, "sound-dai", "#sound-dai-cells", + index, &args); + if (ret) + return ret; + comp->of_node = args.np; + return snd_soc_get_dai_name(&args, &comp->dai_name); +} + +/* + * Parse one DPCM backend from the devicetree. This means taking one + * of the CPU DAIs and combining it with one or more CODEC DAIs. + */ +static int macaudio_parse_of_be_dai_link(struct macaudio_snd_data *ma, + struct snd_soc_dai_link *link, + int be_index, int ncodecs_per_be, + struct device_node *cpu, + struct device_node *codec) +{ + struct snd_soc_dai_link_component *comp; + struct device *dev = ma->card.dev; + int codec_base = be_index * ncodecs_per_be; + int ret, i; + + link->no_pcm = 1; + + link->dai_fmt = MACAUDIO_DAI_FMT; + + link->num_codecs = ncodecs_per_be; + link->codecs = devm_kcalloc(dev, ncodecs_per_be, + sizeof(*comp), GFP_KERNEL); + link->num_cpus = 1; + link->cpus = devm_kzalloc(dev, sizeof(*comp), GFP_KERNEL); + + if (!link->codecs || !link->cpus) + return -ENOMEM; + + link->num_platforms = 0; + + for_each_link_codecs(link, i, comp) { + ret = macaudio_parse_of_component(codec, codec_base + i, comp); + if (ret) + return dev_err_probe(ma->card.dev, ret, "parsing CODEC DAI of link '%s' at %pOF\n", + link->name, codec); + } + + ret = macaudio_parse_of_component(cpu, be_index, link->cpus); + if (ret) + return dev_err_probe(ma->card.dev, ret, "parsing CPU DAI of link '%s' at %pOF\n", + link->name, codec); + + link->name = link->cpus[0].dai_name; + + return 0; +} + +static int macaudio_parse_of(struct macaudio_snd_data *ma) +{ + struct device_node *codec = NULL; + struct device_node *cpu = NULL; + struct device_node *np = NULL; + struct device_node *platform = NULL; + struct snd_soc_dai_link *link = NULL; + struct snd_soc_card *card = &ma->card; + struct device *dev = card->dev; + struct macaudio_link_props *link_props; + int ret, num_links, i; + + ret = snd_soc_of_parse_card_name(card, "model"); + if (ret) { + dev_err_probe(dev, ret, "parsing card name\n"); + return ret; + } + /* + * Set long_name to prevent snd_soc_set_dmi_name() from setting one from + * make believe data u-boot provides in its SMBIOS emulation. + */ + card->long_name = card->name; + + /* Populate links, start with the fixed number of FE links */ + num_links = ARRAY_SIZE(macaudio_fe_links); + + /* Now add together the (dynamic) number of BE links */ + for_each_available_child_of_node(dev->of_node, np) { + int num_cpus; + + cpu = of_get_child_by_name(np, "cpu"); + if (!cpu) { + ret = dev_err_probe(dev, -EINVAL, + "missing CPU DAI node at %pOF\n", np); + goto err_free; + } + + num_cpus = of_count_phandle_with_args(cpu, "sound-dai", + "#sound-dai-cells"); + + if (num_cpus <= 0) { + ret = dev_err_probe(card->dev, -EINVAL, + "missing sound-dai property at %pOF\n", cpu); + goto err_free; + } + of_node_put(cpu); + cpu = NULL; + + /* Each CPU specified counts as one BE link */ + num_links += num_cpus; + } + + /* Allocate the DAI link array */ + card->dai_link = devm_kcalloc(dev, num_links, sizeof(*link), GFP_KERNEL); + ma->link_props = devm_kcalloc(dev, num_links, sizeof(*ma->link_props), GFP_KERNEL); + if (!card->dai_link || !ma->link_props) + return -ENOMEM; + + link = card->dai_link; + link_props = ma->link_props; + + for (i = 0; i < ARRAY_SIZE(macaudio_fe_links); i++) { + ret = macaudio_copy_link(dev, link, &macaudio_fe_links[i]); + if (ret) + goto err_free; + + memcpy(link_props, &macaudio_fe_link_props[i], sizeof(struct macaudio_link_props)); + link++; link_props++; + } + + for (i = 0; i < num_links; i++) + card->dai_link[i].id = i; + + /* We might disable the speakers, so count again */ + num_links = ARRAY_SIZE(macaudio_fe_links); + + /* Fill in the BEs */ + for_each_available_child_of_node(dev->of_node, np) { + const char *link_name; + bool speakers; + int be_index, num_codecs, num_bes, ncodecs_per_cpu, nchannels; + unsigned int left_mask, right_mask; + + ret = of_property_read_string(np, "link-name", &link_name); + if (ret) { + dev_err_probe(card->dev, ret, "missing link name\n"); + goto err_free; + } + + dev_dbg(ma->card.dev, "parsing link '%s'\n", link_name); + + speakers = !strcmp(link_name, "Speaker") + || !strcmp(link_name, "Speakers"); + if (speakers) { + if (!ma->cfg->enable_speakers && !please_blow_up_my_speakers) { + dev_err(card->dev, "driver can't assure safety on this model, disabling speakers\n"); + continue; + } + ma->has_speakers = 1; + if (ma->cfg->amp != AMP_SSM3515 && ma->cfg->safe_vol != 0) + ma->has_sense = 1; + } + + cpu = of_get_child_by_name(np, "cpu"); + codec = of_get_child_by_name(np, "codec"); + + if (!codec || !cpu) { + ret = dev_err_probe(dev, -EINVAL, + "missing DAI specifications for '%s'\n", link_name); + goto err_free; + } + + num_bes = of_count_phandle_with_args(cpu, "sound-dai", + "#sound-dai-cells"); + if (num_bes <= 0) { + ret = dev_err_probe(card->dev, -EINVAL, + "missing sound-dai property at %pOF\n", cpu); + goto err_free; + } + + num_codecs = of_count_phandle_with_args(codec, "sound-dai", + "#sound-dai-cells"); + if (num_codecs <= 0) { + ret = dev_err_probe(card->dev, -EINVAL, + "missing sound-dai property at %pOF\n", codec); + goto err_free; + } + + dev_dbg(ma->card.dev, "link '%s': %d CPUs %d CODECs\n", + link_name, num_bes, num_codecs); + + if (num_codecs % num_bes != 0) { + ret = dev_err_probe(card->dev, -EINVAL, + "bad combination of CODEC (%d) and CPU (%d) number at %pOF\n", + num_codecs, num_bes, np); + goto err_free; + } + + /* + * Now parse the cpu/codec lists into a number of DPCM backend links. + * In each link there will be one DAI from the cpu list paired with + * an evenly distributed number of DAIs from the codec list. (As is + * the binding semantics.) + */ + ncodecs_per_cpu = num_codecs / num_bes; + nchannels = num_codecs * (speakers ? 1 : 2); + + /* Save the max number of channels on the platform */ + if (nchannels > ma->max_channels) + ma->max_channels = nchannels; + + /* + * If there is a single speaker, assign two channels to it, because + * it can do downmix. + */ + if (nchannels < 2) + nchannels = 2; + + left_mask = 0; + for (i = 0; i < nchannels; i += 2) + left_mask = left_mask << 2 | 1; + right_mask = left_mask << 1; + + for (be_index = 0; be_index < num_bes; be_index++) { + /* + * Set initial link name to be overwritten by a BE-specific + * name later so that we can use at least use the provisional + * name in error messages. + */ + link->name = link_name; + + ret = macaudio_parse_of_be_dai_link(ma, link, be_index, + ncodecs_per_cpu, cpu, codec); + if (ret) + goto err_free; + + link_props->is_speakers = speakers; + link_props->is_headphones = !speakers; + + if (num_bes == 2) + /* This sound peripheral is split between left and right BE */ + link_props->tdm_mask = be_index ? right_mask : left_mask; + else + /* One BE covers all of the peripheral */ + link_props->tdm_mask = left_mask | right_mask; + + /* Steal platform OF reference for use in FE links later */ + platform = link->cpus->of_node; + + link++; link_props++; + } + + of_node_put(codec); + of_node_put(cpu); + cpu = codec = NULL; + + num_links += num_bes; + } + + for (i = 0; i < ARRAY_SIZE(macaudio_fe_links); i++) + card->dai_link[i].platforms->of_node = platform; + + /* Skip the speaker sense PCM link if this amp has no sense (or no speakers) */ + if (!ma->has_sense) { + for (i = 0; i < ARRAY_SIZE(macaudio_fe_links); i++) { + if (ma->link_props[i].is_sense) { + memmove(&card->dai_link[i], &card->dai_link[i + 1], + (num_links - i - 1) * sizeof (struct snd_soc_dai_link)); + num_links--; + break; + } + } + } + + card->num_links = num_links; + + return 0; + +err_free: + of_node_put(codec); + of_node_put(cpu); + of_node_put(np); + + if (!card->dai_link) + return ret; + + for (i = 0; i < num_links; i++) { + /* + * TODO: If we don't go through this path are the references + * freed inside ASoC? + */ + snd_soc_of_put_dai_link_codecs(&card->dai_link[i]); + snd_soc_of_put_dai_link_cpus(&card->dai_link[i]); + } + + return ret; +} + +static int macaudio_get_runtime_bclk_ratio(struct snd_pcm_substream *substream) +{ + struct snd_soc_pcm_runtime *rtd = snd_soc_substream_to_rtd(substream); + struct macaudio_snd_data *ma = snd_soc_card_get_drvdata(rtd->card); + struct snd_soc_dpcm *dpcm; + + /* + * If this is a FE, look it up in link_props directly. + * If this is a BE, look it up in the respective FE. + */ + if (!rtd->dai_link->no_pcm) + return ma->link_props[rtd->dai_link->id].bclk_ratio; + + for_each_dpcm_fe(rtd, substream->stream, dpcm) { + int fe_id = dpcm->fe->dai_link->id; + + return ma->link_props[fe_id].bclk_ratio; + } + + return 0; +} + +static int macaudio_dpcm_hw_params(struct snd_pcm_substream *substream, + struct snd_pcm_hw_params *params) +{ + struct snd_soc_pcm_runtime *rtd = snd_soc_substream_to_rtd(substream); + struct macaudio_snd_data *ma = snd_soc_card_get_drvdata(rtd->card); + struct macaudio_link_props *props = &ma->link_props[rtd->dai_link->id]; + struct snd_soc_dai *cpu_dai = snd_soc_rtd_to_cpu(rtd, 0); + struct snd_interval *rate = hw_param_interval(params, + SNDRV_PCM_HW_PARAM_RATE); + int bclk_ratio = macaudio_get_runtime_bclk_ratio(substream); + int i; + + if (props->is_sense) { + rate->min = rate->max = cpu_dai->symmetric_rate; + return 0; + } + + /* Speakers BE */ + if (props->is_speakers) { + if (substream->stream == SNDRV_PCM_STREAM_CAPTURE) { + /* Sense PCM: keep the existing BE rate (0 if not already running) */ + rate->min = rate->max = cpu_dai->symmetric_rate; + + return 0; + } else { + /* + * Set the sense PCM rate control to inform userspace of the + * new sample rate. + */ + ma->speaker_sample_rate = params_rate(params); + snd_ctl_notify(ma->card.snd_card, SNDRV_CTL_EVENT_MASK_VALUE, + &ma->speaker_sample_rate_kctl->id); + } + } + + if (bclk_ratio) { + struct snd_soc_dai *dai; + int mclk = params_rate(params) * bclk_ratio; + + for_each_rtd_codec_dais(rtd, i, dai) { + snd_soc_dai_set_sysclk(dai, 0, mclk, SND_SOC_CLOCK_IN); + snd_soc_dai_set_bclk_ratio(dai, bclk_ratio); + } + + snd_soc_dai_set_sysclk(cpu_dai, 0, mclk, SND_SOC_CLOCK_OUT); + snd_soc_dai_set_bclk_ratio(cpu_dai, bclk_ratio); + } + + return 0; +} + +static int macaudio_fe_startup(struct snd_pcm_substream *substream) +{ + + struct snd_soc_pcm_runtime *rtd = snd_soc_substream_to_rtd(substream); + struct macaudio_snd_data *ma = snd_soc_card_get_drvdata(rtd->card); + struct macaudio_link_props *props = &ma->link_props[rtd->dai_link->id]; + int max_rate, ret; + + if (props->is_sense) { + /* + * Sense stream will not return data while playback is inactive, + * so do not time out. + */ + substream->wait_time = MAX_SCHEDULE_TIMEOUT; + return 0; + } + + ret = snd_pcm_hw_constraint_minmax(substream->runtime, + SNDRV_PCM_HW_PARAM_CHANNELS, + 0, ma->max_channels); + if (ret < 0) + return ret; + + max_rate = MACAUDIO_MAX_BCLK_FREQ / props->bclk_ratio; + ret = snd_pcm_hw_constraint_minmax(substream->runtime, + SNDRV_PCM_HW_PARAM_RATE, + 0, max_rate); + if (ret < 0) + return ret; + + return 0; +} + +static int macaudio_fe_hw_params(struct snd_pcm_substream *substream, + struct snd_pcm_hw_params *params) +{ + struct snd_soc_pcm_runtime *rtd = snd_soc_substream_to_rtd(substream); + struct snd_soc_pcm_runtime *be; + struct snd_soc_dpcm *dpcm; + + be = NULL; + for_each_dpcm_be(rtd, substream->stream, dpcm) { + be = dpcm->be; + break; + } + + if (!be) { + dev_err(rtd->dev, "opening PCM device '%s' with no audio route configured by the user\n", + rtd->dai_link->name); + return -EINVAL; + } + + return macaudio_dpcm_hw_params(substream, params); +} + + +static void macaudio_dpcm_shutdown(struct snd_pcm_substream *substream) +{ + struct snd_soc_pcm_runtime *rtd = snd_soc_substream_to_rtd(substream); + struct snd_soc_dai *cpu_dai = snd_soc_rtd_to_cpu(rtd, 0); + struct snd_soc_dai *dai; + int bclk_ratio = macaudio_get_runtime_bclk_ratio(substream); + int i; + + if (bclk_ratio) { + for_each_rtd_codec_dais(rtd, i, dai) + snd_soc_dai_set_sysclk(dai, 0, 0, SND_SOC_CLOCK_IN); + + snd_soc_dai_set_sysclk(cpu_dai, 0, 0, SND_SOC_CLOCK_OUT); + } +} + +static int macaudio_be_hw_free(struct snd_pcm_substream *substream) +{ + struct snd_soc_pcm_runtime *rtd = snd_soc_substream_to_rtd(substream); + struct macaudio_snd_data *ma = snd_soc_card_get_drvdata(rtd->card); + struct macaudio_link_props *props = &ma->link_props[rtd->dai_link->id]; + struct snd_soc_dai *dai; + int i; + + if (props->is_speakers && substream->stream == SNDRV_PCM_STREAM_PLAYBACK) { + /* + * Clear the DAI rates, so the next open can change the sample rate. + * This won't happen automatically if the sense PCM is open. + */ + for_each_rtd_dais(rtd, i, dai) { + dai->symmetric_rate = 0; + } + + /* Notify userspace that the speakers are closed */ + ma->speaker_sample_rate = 0; + snd_ctl_notify(ma->card.snd_card, SNDRV_CTL_EVENT_MASK_VALUE, + &ma->speaker_sample_rate_kctl->id); + } + + return 0; +} + +static int macaudio_be_trigger(struct snd_pcm_substream *substream, int cmd) +{ + struct snd_soc_pcm_runtime *rtd = snd_soc_substream_to_rtd(substream); + struct macaudio_snd_data *ma = snd_soc_card_get_drvdata(rtd->card); + struct macaudio_link_props *props = &ma->link_props[rtd->dai_link->id]; + + if (props->is_speakers && substream->stream == SNDRV_PCM_STREAM_PLAYBACK) { + switch (cmd) { + case SNDRV_PCM_TRIGGER_START: + case SNDRV_PCM_TRIGGER_RESUME: + case SNDRV_PCM_TRIGGER_PAUSE_RELEASE: + ma->bes_active |= BIT(rtd->dai_link->id); + break; + case SNDRV_PCM_TRIGGER_SUSPEND: + case SNDRV_PCM_TRIGGER_PAUSE_PUSH: + case SNDRV_PCM_TRIGGER_STOP: + ma->bes_active &= ~BIT(rtd->dai_link->id); + break; + default: + return -EINVAL; + } + + schedule_work(&ma->lock_update_work); + } + + return 0; +} + +static const struct snd_soc_ops macaudio_fe_ops = { + .startup = macaudio_fe_startup, + .shutdown = macaudio_dpcm_shutdown, + .hw_params = macaudio_fe_hw_params, +}; + +static const struct snd_soc_ops macaudio_be_ops = { + .hw_free = macaudio_be_hw_free, + .shutdown = macaudio_dpcm_shutdown, + .hw_params = macaudio_dpcm_hw_params, + .trigger = macaudio_be_trigger, +}; + +static int macaudio_be_assign_tdm(struct snd_soc_pcm_runtime *rtd) +{ + struct snd_soc_card *card = rtd->card; + struct macaudio_snd_data *ma = snd_soc_card_get_drvdata(card); + struct macaudio_link_props *props = &ma->link_props[rtd->dai_link->id]; + struct snd_soc_dai *dai; + unsigned int mask; + int nslots, ret, i; + + if (!props->tdm_mask) + return 0; + + mask = props->tdm_mask; + nslots = __fls(mask) + 1; + + if (rtd->dai_link->num_codecs == 1) { + ret = snd_soc_dai_set_tdm_slot(snd_soc_rtd_to_codec(rtd, 0), mask, + 0, nslots, MACAUDIO_SLOTWIDTH); + + /* + * Headphones get a pass on -ENOTSUPP (see the comment + * around bclk_ratio value for primary FE). + */ + if (ret == -ENOTSUPP && props->is_headphones) + return 0; + + return ret; + } + + for_each_rtd_codec_dais(rtd, i, dai) { + int slot = __ffs(mask); + + mask &= ~(1 << slot); + ret = snd_soc_dai_set_tdm_slot(dai, 1 << slot, 0, nslots, + MACAUDIO_SLOTWIDTH); + if (ret) + return ret; + } + + return 0; +} + +static int macaudio_be_init(struct snd_soc_pcm_runtime *rtd) +{ + struct snd_soc_card *card = rtd->card; + struct macaudio_snd_data *ma = snd_soc_card_get_drvdata(card); + struct macaudio_link_props *props = &ma->link_props[rtd->dai_link->id]; + struct snd_soc_dai *dai; + int i, ret; + + ret = macaudio_be_assign_tdm(rtd); + if (ret < 0) + return ret; + + if (props->is_headphones) { + for_each_rtd_codec_dais(rtd, i, dai) + snd_soc_component_set_jack(dai->component, &ma->jack, NULL); + } + + return 0; +} + +static void macaudio_be_exit(struct snd_soc_pcm_runtime *rtd) +{ + struct snd_soc_card *card = rtd->card; + struct macaudio_snd_data *ma = snd_soc_card_get_drvdata(card); + struct macaudio_link_props *props = &ma->link_props[rtd->dai_link->id]; + struct snd_soc_dai *dai; + int i; + + if (props->is_headphones) { + for_each_rtd_codec_dais(rtd, i, dai) + snd_soc_component_set_jack(dai->component, NULL, NULL); + } +} + +static int macaudio_fe_init(struct snd_soc_pcm_runtime *rtd) +{ + struct snd_soc_card *card = rtd->card; + struct macaudio_snd_data *ma = snd_soc_card_get_drvdata(card); + struct macaudio_link_props *props = &ma->link_props[rtd->dai_link->id]; + int nslots = props->bclk_ratio / MACAUDIO_SLOTWIDTH; + + if (props->is_sense) + return snd_soc_dai_set_tdm_slot(snd_soc_rtd_to_cpu(rtd, 0), 0, 0xffff, 16, 16); + + return snd_soc_dai_set_tdm_slot(snd_soc_rtd_to_cpu(rtd, 0), (1 << nslots) - 1, + (1 << nslots) - 1, nslots, MACAUDIO_SLOTWIDTH); +} + +static struct snd_soc_jack_pin macaudio_jack_pins[] = { + { + .pin = "Headphone", + .mask = SND_JACK_HEADPHONE, + }, + { + .pin = "Headset Mic", + .mask = SND_JACK_MICROPHONE, + }, +}; + +static int macaudio_probe(struct snd_soc_card *card) +{ + struct macaudio_snd_data *ma = snd_soc_card_get_drvdata(card); + int ret; + + dev_dbg(card->dev, "%s!\n", __func__); + + ret = snd_soc_card_jack_new_pins(card, "Headphone Jack", + SND_JACK_HEADSET | SND_JACK_HEADPHONE, + &ma->jack, macaudio_jack_pins, + ARRAY_SIZE(macaudio_jack_pins)); + if (ret < 0) { + dev_err(card->dev, "jack creation failed: %d\n", ret); + return ret; + } + + return ret; +} + +static int macaudio_add_backend_dai_route(struct snd_soc_card *card, struct snd_soc_dai *dai, + bool is_speakers) +{ + struct snd_soc_dapm_route routes[2]; + struct snd_soc_dapm_route *r; + int nroutes = 0; + int ret; + + memset(routes, 0, sizeof(routes)); + + dev_dbg(card->dev, "adding routes for '%s'\n", dai->name); + + r = &routes[nroutes++]; + if (is_speakers) + r->source = "Speaker Playback"; + else + r->source = "Headphone Playback"; + r->sink = dai->stream[SNDRV_PCM_STREAM_PLAYBACK].widget->name; + + /* If headphone jack, add capture path */ + if (!is_speakers) { + r = &routes[nroutes++]; + r->source = dai->stream[SNDRV_PCM_STREAM_CAPTURE].widget->name; + r->sink = "Headset Capture"; + } + + /* If speakers, add sense capture path */ + if (is_speakers) { + r = &routes[nroutes++]; + r->source = dai->stream[SNDRV_PCM_STREAM_CAPTURE].widget->name; + r->sink = "Speaker Sense Capture"; + } + + ret = snd_soc_dapm_add_routes(&card->dapm, routes, nroutes); + if (ret) + dev_err(card->dev, "failed adding dynamic DAPM routes for %s\n", + dai->name); + return ret; +} + +static int macaudio_add_pin_routes(struct snd_soc_card *card, struct snd_soc_component *component, + bool is_speakers) +{ + struct snd_soc_dapm_route routes[2]; + struct snd_soc_dapm_route *r; + int nroutes = 0; + char buf[32]; + int ret; + + memset(routes, 0, sizeof(routes)); + + /* Connect the far ends of CODECs to pins */ + if (is_speakers) { + r = &routes[nroutes++]; + r->source = "OUT"; + if (component->name_prefix) { + snprintf(buf, sizeof(buf) - 1, "%s OUT", component->name_prefix); + r->source = buf; + } + r->sink = "Speaker"; + } else { + r = &routes[nroutes++]; + r->source = "Jack HP"; + r->sink = "Headphone"; + r = &routes[nroutes++]; + r->source = "Headset Mic"; + r->sink = "Jack HS"; + } + + ret = snd_soc_dapm_add_routes(&card->dapm, routes, nroutes); + if (ret) + dev_err(card->dev, "failed adding dynamic DAPM routes for %s\n", + component->name); + return ret; +} + +static int macaudio_late_probe(struct snd_soc_card *card) +{ + struct macaudio_snd_data *ma = snd_soc_card_get_drvdata(card); + struct snd_soc_pcm_runtime *rtd; + struct snd_soc_dai *dai; + int ret, i; + + /* Add the dynamic DAPM routes */ + for_each_card_rtds(card, rtd) { + struct macaudio_link_props *props = &ma->link_props[rtd->dai_link->id]; + + if (!rtd->dai_link->no_pcm) + continue; + + for_each_rtd_cpu_dais(rtd, i, dai) { + ret = macaudio_add_backend_dai_route(card, dai, props->is_speakers); + + if (ret) + return ret; + } + + for_each_rtd_codec_dais(rtd, i, dai) { + ret = macaudio_add_pin_routes(card, dai->component, + props->is_speakers); + + if (ret) + return ret; + } + } + + if (ma->has_speakers) + ma->speaker_sample_rate_kctl = snd_soc_card_get_kcontrol(card, + "Speaker Sample Rate"); + if (ma->has_safety) { + ma->speaker_lock_kctl = snd_soc_card_get_kcontrol(card, + "Speaker Volume Unlock"); + + mutex_lock(&ma->volume_lock_mutex); + macaudio_vlimit_unlock(ma, false); + mutex_unlock(&ma->volume_lock_mutex); + } + + return 0; +} + +#define CHECK(call, pattern, value, min) \ + { \ + int ret = call(card, pattern, value); \ + int err = (ret >= 0 && ret < min) ? -ERANGE : ret; \ + if (err < 0) { \ + dev_err(card->dev, "%s on '%s': %d\n", #call, pattern, \ + ret); \ + if (please_blow_up_my_speakers < 2) \ + return err; \ + } else { \ + dev_dbg(card->dev, "%s on '%s': %d hits\n", #call, \ + pattern, ret); \ + } \ + } + +#define CHECK_CONCAT(call, suffix, value) \ + { \ + snprintf(buf, sizeof(buf), "%s%s", prefix, suffix); \ + CHECK(call, buf, value, 1); \ + } + +static int macaudio_set_speaker(struct snd_soc_card *card, const char *prefix, bool tweeter) +{ + struct macaudio_snd_data *ma = snd_soc_card_get_drvdata(card); + char buf[256]; + + if (!ma->has_speakers) + return 0; + + switch (ma->cfg->amp) { + case AMP_TAS5770: + if (ma->cfg->stereo) { + CHECK_CONCAT(snd_soc_set_enum_kctl, "ASI1 Sel", "Left"); + CHECK_CONCAT(snd_soc_deactivate_kctl, "ASI1 Sel", 0); + } + + CHECK_CONCAT(snd_soc_limit_volume, "Amp Gain Volume", ma->cfg->amp_gain); + break; + case AMP_SN012776: + if (ma->cfg->stereo) { + CHECK_CONCAT(snd_soc_set_enum_kctl, "ASI1 Sel", "Left"); + CHECK_CONCAT(snd_soc_deactivate_kctl, "ASI1 Sel", 0); + } + + CHECK_CONCAT(snd_soc_limit_volume, "Amp Gain Volume", ma->cfg->amp_gain); + CHECK_CONCAT(snd_soc_set_enum_kctl, "HPF Corner Frequency", + tweeter ? "800 Hz" : "2 Hz"); + + if (please_blow_up_my_speakers < 2) + CHECK_CONCAT(snd_soc_deactivate_kctl, "HPF Corner Frequency", 0); + + CHECK_CONCAT(snd_soc_set_enum_kctl, "OCE Handling", "Retry"); + CHECK_CONCAT(snd_soc_deactivate_kctl, "OCE Handling", 0); + break; + case AMP_SSM3515: + /* TODO: check */ + CHECK_CONCAT(snd_soc_set_enum_kctl, "DAC Analog Gain Select", "8.4 V Span"); + + if (please_blow_up_my_speakers < 2) + CHECK_CONCAT(snd_soc_deactivate_kctl, "DAC Analog Gain Select", 0); + + /* TODO: HPF, needs new call to set */ + break; + default: + return -EINVAL; + } + + return 0; +} + +static int macaudio_fixup_controls(struct snd_soc_card *card) +{ + struct macaudio_snd_data *ma = snd_soc_card_get_drvdata(card); + const char *p; + + /* Set the card ID early to avoid races with udev */ + p = strrchr(card->name, ' '); + if (p) { + snprintf(card->snd_card->id, sizeof(card->snd_card->id), + "Apple%s", p + 1); + } + + if (!ma->has_speakers) + return 0; + + /* + * This needs some care to avoid matches against cs42l84's + * "Jack HPF Corner Frequency". + */ + switch(ma->cfg->speakers) { + case SPKR_NONE: + WARN_ON(please_blow_up_my_speakers < 2); + return please_blow_up_my_speakers >= 2 ? 0 : -EINVAL; + case SPKR_1W: + /* only 1W stereo system (J313) is uses cs42l83 */ + if (ma->cfg->stereo) { + CHECK(macaudio_set_speaker, "* ", false, 0); + } else { + CHECK(macaudio_set_speaker, "", false, 0); + } + break; + case SPKR_2W: + CHECK(macaudio_set_speaker, "* Front ", false, 0); + CHECK(macaudio_set_speaker, "* Rear ", false, 0); + break; + case SPKR_1W1T: + CHECK(macaudio_set_speaker, "* Tweeter ", true, 0); + CHECK(macaudio_set_speaker, "* Woofer ", false, 0); + break; + case SPKR_2W1T: + CHECK(macaudio_set_speaker, "* Tweeter ", true, 0); + CHECK(macaudio_set_speaker, "* Woofer 1 ", false, 0); + CHECK(macaudio_set_speaker, "* Woofer 2 ", false, 0); + break; + } + + return 0; +} + +static const char * const macaudio_spk_mux_texts[] = { + "Primary", + "Secondary" +}; + +SOC_ENUM_SINGLE_VIRT_DECL(macaudio_spk_mux_enum, macaudio_spk_mux_texts); + +static const struct snd_kcontrol_new macaudio_spk_mux = + SOC_DAPM_ENUM("Speaker Playback Mux", macaudio_spk_mux_enum); + +static const char * const macaudio_hp_mux_texts[] = { + "Primary", + "Secondary" +}; + +SOC_ENUM_SINGLE_VIRT_DECL(macaudio_hp_mux_enum, macaudio_hp_mux_texts); + +static const struct snd_kcontrol_new macaudio_hp_mux = + SOC_DAPM_ENUM("Headphones Playback Mux", macaudio_hp_mux_enum); + +static const struct snd_soc_dapm_widget macaudio_snd_widgets[] = { + SND_SOC_DAPM_SPK("Speaker", NULL), + SND_SOC_DAPM_SPK("Speaker (Static)", NULL), + SND_SOC_DAPM_HP("Headphone", NULL), + SND_SOC_DAPM_MIC("Headset Mic", NULL), + + SND_SOC_DAPM_MUX("Speaker Playback Mux", SND_SOC_NOPM, 0, 0, &macaudio_spk_mux), + SND_SOC_DAPM_MUX("Headphone Playback Mux", SND_SOC_NOPM, 0, 0, &macaudio_hp_mux), + + SND_SOC_DAPM_AIF_OUT("Speaker Playback", NULL, 0, SND_SOC_NOPM, 0, 0), + SND_SOC_DAPM_AIF_OUT("Headphone Playback", NULL, 0, SND_SOC_NOPM, 0, 0), + + SND_SOC_DAPM_AIF_IN("Headset Capture", NULL, 0, SND_SOC_NOPM, 0, 0), + SND_SOC_DAPM_AIF_IN("Speaker Sense Capture", NULL, 0, SND_SOC_NOPM, 0, 0), +}; + +static int macaudio_sss_info(struct snd_kcontrol *kcontrol, struct snd_ctl_elem_info *uinfo) +{ + uinfo->type = SNDRV_CTL_ELEM_TYPE_INTEGER; + uinfo->count = 1; + uinfo->value.integer.min = 0; + uinfo->value.integer.max = 192000; + + return 0; +} + +static int macaudio_sss_get(struct snd_kcontrol *kcontrol, struct snd_ctl_elem_value *uvalue) +{ + struct snd_soc_card *card = snd_kcontrol_chip(kcontrol); + struct macaudio_snd_data *ma = snd_soc_card_get_drvdata(card); + + /* + * TODO: Check if any locking is in order here. I would + * assume there is some ALSA-level lock, but DAPM implementations + * of kcontrol ops do explicit locking, so look into it. + */ + uvalue->value.integer.value[0] = ma->speaker_sample_rate; + + return 0; +} + +static int macaudio_slk_info(struct snd_kcontrol *kcontrol, struct snd_ctl_elem_info *uinfo) +{ + uinfo->type = SNDRV_CTL_ELEM_TYPE_INTEGER; + uinfo->count = 1; + uinfo->value.integer.min = INT_MIN; + uinfo->value.integer.max = INT_MAX; + + return 0; +} + +static int macaudio_slk_put(struct snd_kcontrol *kcontrol, struct snd_ctl_elem_value *uvalue) +{ + struct snd_soc_card *card = snd_kcontrol_chip(kcontrol); + struct macaudio_snd_data *ma = snd_soc_card_get_drvdata(card); + + if (!ma->speaker_lock_owner) + return -EPERM; + + if (uvalue->value.integer.value[0] != SPEAKER_MAGIC_VALUE) + return -EINVAL; + + /* Serves as a notification that the lock was lost at some point */ + if (ma->speaker_volume_was_locked) { + ma->speaker_volume_was_locked = false; + return -ETIMEDOUT; + } + + mutex_lock(&ma->volume_lock_mutex); + + cancel_delayed_work(&ma->lock_timeout_work); + + ma->speaker_lock_remain = ms_to_ktime(SPEAKER_LOCK_TIMEOUT); + ma->speaker_lock_timeout = ktime_add(ktime_get(), ma->speaker_lock_remain); + macaudio_vlimit_update(ma); + + if (ma->speaker_lock_timeout_enabled) { + dev_dbg(ma->card.dev, "Volume limit timeout ping: %ld us left\n", + (long)ktime_to_us(ma->speaker_lock_remain)); + schedule_delayed_work(&ma->lock_timeout_work, usecs_to_jiffies(ktime_to_us(ma->speaker_lock_remain))); + } + + mutex_unlock(&ma->volume_lock_mutex); + + return 0; +} + +static int macaudio_slk_get(struct snd_kcontrol *kcontrol, struct snd_ctl_elem_value *uvalue) +{ + struct snd_soc_card *card = snd_kcontrol_chip(kcontrol); + struct macaudio_snd_data *ma = snd_soc_card_get_drvdata(card); + + uvalue->value.integer.value[0] = ma->speaker_volume_unlocked ? 1 : 0; + + return 0; +} + +static int macaudio_slk_lock(struct snd_kcontrol *kcontrol, struct snd_ctl_file *owner) +{ + struct snd_soc_card *card = snd_kcontrol_chip(kcontrol); + struct macaudio_snd_data *ma = snd_soc_card_get_drvdata(card); + + mutex_lock(&ma->volume_lock_mutex); + ma->speaker_lock_owner = owner; + macaudio_vlimit_update(ma); + + /* + * Reset the unintended lock flag when the control is first locked. + * At this point the state is locked and cannot be unlocked until + * userspace writes to this control, so this cannot spuriously become + * true again until that point. + */ + ma->speaker_volume_was_locked = false; + + mutex_unlock(&ma->volume_lock_mutex); + + return 0; +} + +static void macaudio_slk_unlock(struct snd_kcontrol *kcontrol) +{ + struct snd_soc_card *card = snd_kcontrol_chip(kcontrol); + struct macaudio_snd_data *ma = snd_soc_card_get_drvdata(card); + + ma->speaker_lock_owner = NULL; + ma->speaker_lock_timeout = 0; + macaudio_vlimit_update(ma); +} + +/* + * Speaker limit controls go last. We only drop the unlock control, + * leaving sample rate, since that can be useful for safety + * bring-up before the kernel-side caps are ready. + */ +#define MACAUDIO_NUM_SPEAKER_LIMIT_CONTROLS 1 +/* + * If there are no speakers configured at all, we can drop both + * controls. + */ +#define MACAUDIO_NUM_SPEAKER_CONTROLS 2 + +static const struct snd_kcontrol_new macaudio_controls[] = { + SOC_DAPM_PIN_SWITCH("Speaker"), + SOC_DAPM_PIN_SWITCH("Headphone"), + SOC_DAPM_PIN_SWITCH("Headset Mic"), + { + .iface = SNDRV_CTL_ELEM_IFACE_MIXER, + .access = SNDRV_CTL_ELEM_ACCESS_READ | + SNDRV_CTL_ELEM_ACCESS_VOLATILE, + .name = "Speaker Sample Rate", + .info = macaudio_sss_info, .get = macaudio_sss_get, + }, + { + .iface = SNDRV_CTL_ELEM_IFACE_MIXER, + .access = SNDRV_CTL_ELEM_ACCESS_READ | + SNDRV_CTL_ELEM_ACCESS_WRITE | + SNDRV_CTL_ELEM_ACCESS_VOLATILE, + .name = "Speaker Volume Unlock", + .info = macaudio_slk_info, + .put = macaudio_slk_put, .get = macaudio_slk_get, + .lock = macaudio_slk_lock, .unlock = macaudio_slk_unlock, + }, +}; + +static const struct snd_soc_dapm_route macaudio_dapm_routes[] = { + /* Playback paths */ + { "Speaker Playback Mux", "Primary", "PCM0 TX" }, + { "Speaker Playback Mux", "Secondary", "PCM1 TX" }, + { "Speaker Playback", NULL, "Speaker Playback Mux"}, + + { "Headphone Playback Mux", "Primary", "PCM0 TX" }, + { "Headphone Playback Mux", "Secondary", "PCM1 TX" }, + { "Headphone Playback", NULL, "Headphone Playback Mux"}, + /* + * Additional paths (to specific I2S ports) are added dynamically. + */ + + /* Capture paths */ + { "PCM0 RX", NULL, "Headset Capture" }, + + /* Sense paths */ + { "PCM2 RX", NULL, "Speaker Sense Capture" }, +}; + +/* enable amp speakers stereo gain safe_vol */ +struct macaudio_platform_cfg macaudio_j180_cfg = { + false, AMP_SN012776, SPKR_1W1T, false, 10, -20, +}; +struct macaudio_platform_cfg macaudio_j274_cfg = { + true, AMP_TAS5770, SPKR_1W, false, 20, -20, +}; + +struct macaudio_platform_cfg macaudio_j293_cfg = { + true, AMP_TAS5770, SPKR_2W, true, 15, -20, +}; + +struct macaudio_platform_cfg macaudio_j313_cfg = { + true, AMP_TAS5770, SPKR_1W, true, 10, -20, +}; + +struct macaudio_platform_cfg macaudio_j314_cfg = { + true, AMP_SN012776, SPKR_2W1T, true, 15, -20, +}; + +struct macaudio_platform_cfg macaudio_j316_cfg = { + true, AMP_SN012776, SPKR_2W1T, true, 15, -20, +}; + +struct macaudio_platform_cfg macaudio_j37x_j47x_cfg = { + true, AMP_SN012776, SPKR_1W, false, 20, -20, +}; + +struct macaudio_platform_cfg macaudio_j413_cfg = { + true, AMP_SN012776, SPKR_1W1T, true, 15, -20, +}; + +struct macaudio_platform_cfg macaudio_j415_cfg = { + true, AMP_SN012776, SPKR_2W1T, true, 15, -20, +}; + +struct macaudio_platform_cfg macaudio_j45x_cfg = { + false, AMP_SSM3515, SPKR_1W1T, true, 9, -20, /* TODO: gain?? */ +}; + +struct macaudio_platform_cfg macaudio_j493_cfg = { + true, AMP_SN012776, SPKR_2W, true, 15, -20, +}; + +struct macaudio_platform_cfg macaudio_fallback_cfg = { + false, AMP_NONE, SPKR_NONE, false, 0, 0, +}; + +/* + * DT compatible/ID table rules: + * + * 1. Machines with **identical** speaker configurations (amps, models, chassis) + * are allowed to declare compatibility with the first model (chronologically), + * and are not enumerated in this array. + * + * 2. Machines with identical amps and speakers (=identical speaker protection + * rules) but a different chassis must use different compatibles, but may share + * the private data structure here. They are explicitly enumerated. + * + * 3. Machines with different amps or speaker layouts must use separate + * data structures. + * + * 4. Machines with identical speaker layouts and amps (but possibly different + * speaker models/chassis) may share the data structure, since only userspace + * cares about that (assuming our general -20dB safe level standard holds). + */ +static const struct of_device_id macaudio_snd_device_id[] = { + /* Model ID Amp Gain Speakers */ + /* j180 AID19 sn012776 10 1× 1W+1T */ + { .compatible = "apple,j180-macaudio", .data = &macaudio_j180_cfg }, + /* j274 AID6 tas5770 20 1× 1W */ + { .compatible = "apple,j274-macaudio", .data = &macaudio_j274_cfg }, + /* j293 AID3 tas5770 15 2× 2W */ + { .compatible = "apple,j293-macaudio", .data = &macaudio_j293_cfg }, + /* j313 AID4 tas5770 10 2× 1W */ + { .compatible = "apple,j313-macaudio", .data = &macaudio_j313_cfg }, + /* j314 AID8 sn012776 15 2× 2W+1T */ + { .compatible = "apple,j314-macaudio", .data = &macaudio_j314_cfg }, + /* j316 AID9 sn012776 15 2× 2W+1T */ + { .compatible = "apple,j316-macaudio", .data = &macaudio_j316_cfg }, + /* j375 AID10 sn012776 15 1× 1W */ + { .compatible = "apple,j375-macaudio", .data = &macaudio_j37x_j47x_cfg }, + /* j413 AID13 sn012776 15 2× 1W+1T */ + { .compatible = "apple,j413-macaudio", .data = &macaudio_j413_cfg }, + /* j414 AID14 sn012776 15 2× 2W+1T Compat: apple,j314-macaudio */ + /* j415 AID27 sn012776 15 2× 2W+1T */ + { .compatible = "apple,j415-macaudio", .data = &macaudio_j415_cfg }, + /* j416 AID15 sn012776 15 2× 2W+1T Compat: apple,j316-macaudio */ + /* j456 AID5 ssm3515 15 2× 1W+1T */ + { .compatible = "apple,j456-macaudio", .data = &macaudio_j45x_cfg }, + /* j457 AID7 ssm3515 15 2× 1W+1T Compat: apple,j456-macaudio */ + /* j473 AID12 sn012776 20 1× 1W */ + { .compatible = "apple,j473-macaudio", .data = &macaudio_j37x_j47x_cfg }, + /* j474 AID26 sn012776 20 1× 1W Compat: apple,j473-macaudio */ + /* j475 AID25 sn012776 20 1× 1W Compat: apple,j375-macaudio */ + /* j493 AID18 sn012776 15 2× 2W */ + { .compatible = "apple,j493-macaudio", .data = &macaudio_j493_cfg }, + /* Fallback, jack only */ + { .compatible = "apple,macaudio"}, + { } +}; +MODULE_DEVICE_TABLE(of, macaudio_snd_device_id); + +static int macaudio_snd_platform_probe(struct platform_device *pdev) +{ + struct snd_soc_card *card; + struct macaudio_snd_data *data; + struct device *dev = &pdev->dev; + struct snd_soc_dai_link *link; + const struct of_device_id *of_id; + int ret; + int i; + + of_id = of_match_device(macaudio_snd_device_id, dev); + if (!of_id) + return -EINVAL; + + data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL); + if (!data) + return -ENOMEM; + card = &data->card; + snd_soc_card_set_drvdata(card, data); + dev_set_drvdata(&pdev->dev, data); + mutex_init(&data->volume_lock_mutex); + + card->owner = THIS_MODULE; + card->driver_name = "macaudio"; + card->dev = dev; + card->dapm_widgets = macaudio_snd_widgets; + card->num_dapm_widgets = ARRAY_SIZE(macaudio_snd_widgets); + card->dapm_routes = macaudio_dapm_routes; + card->num_dapm_routes = ARRAY_SIZE(macaudio_dapm_routes); + card->controls = macaudio_controls; + card->num_controls = ARRAY_SIZE(macaudio_controls); + card->probe = macaudio_probe; + card->late_probe = macaudio_late_probe; + card->component_chaining = true; + card->fully_routed = true; + + if (of_id->data) + data->cfg = of_id->data; + else + data->cfg = &macaudio_fallback_cfg; + + card->fixup_controls = macaudio_fixup_controls; + + ret = macaudio_parse_of(data); + if (ret) + return ret; + + /* Remove useless controls */ + if (!data->has_speakers) /* No speakers, remove both */ + card->num_controls -= MACAUDIO_NUM_SPEAKER_CONTROLS; + else if (!data->cfg->safe_vol) /* No safety, remove unlock */ + card->num_controls -= MACAUDIO_NUM_SPEAKER_LIMIT_CONTROLS; + else /* Speakers with safety, mark us as such */ + data->has_safety = true; + + for_each_card_prelinks(card, i, link) { + if (link->no_pcm) { + link->ops = &macaudio_be_ops; + link->init = macaudio_be_init; + link->exit = macaudio_be_exit; + } else { + link->ops = &macaudio_fe_ops; + link->init = macaudio_fe_init; + } + } + + INIT_WORK(&data->lock_update_work, macaudio_vlimit_update_work); + INIT_DELAYED_WORK(&data->lock_timeout_work, macaudio_vlimit_timeout_work); + + return devm_snd_soc_register_card(dev, card); +} + +static void macaudio_snd_platform_remove(struct platform_device *pdev) +{ + struct macaudio_snd_data *ma = dev_get_drvdata(&pdev->dev); + + cancel_delayed_work_sync(&ma->lock_timeout_work); +} + +static struct platform_driver macaudio_snd_driver = { + .probe = macaudio_snd_platform_probe, + .remove = macaudio_snd_platform_remove, + .driver = { + .name = DRIVER_NAME, + .of_match_table = macaudio_snd_device_id, + .pm = &snd_soc_pm_ops, + }, +}; +module_platform_driver(macaudio_snd_driver); + +MODULE_AUTHOR("Martin Povišer "); +MODULE_DESCRIPTION("Apple Silicon Macs machine-level sound driver"); +MODULE_LICENSE("GPL"); diff --git a/sound/soc/apple/mca.c b/sound/soc/apple/mca.c index c4dcb2b545912c..01dacd10bd39ce 100644 --- a/sound/soc/apple/mca.c +++ b/sound/soc/apple/mca.c @@ -133,12 +133,17 @@ struct mca_cluster { struct clk *clk_parent; struct dma_chan *dma_chans[SNDRV_PCM_STREAM_LAST + 1]; - bool port_started[SNDRV_PCM_STREAM_LAST + 1]; - int port_driver; /* The cluster driving this cluster's port */ + bool clk_provider; + + bool port_clk_started[SNDRV_PCM_STREAM_LAST + 1]; + int port_clk_driver; /* The cluster driving this cluster's port */ bool clocks_in_use[SNDRV_PCM_STREAM_LAST + 1]; struct device_link *pd_link; + /* In case of clock consumer FE */ + int syncgen_in_use; + unsigned int bclk_ratio; /* Masks etc. picked up via the set_tdm_slot method */ @@ -157,7 +162,7 @@ struct mca_data { struct reset_control *rstc; struct device_link *pd_link; - /* Mutex for accessing port_driver of foreign clusters */ + /* Mutex for accessing port_clk_driver of foreign clusters */ struct mutex port_mutex; int nclusters; @@ -211,15 +216,21 @@ static void mca_fe_early_trigger(struct snd_pcm_substream *substream, int cmd, SERDES_STATUS_RST); /* * Experiments suggest that it takes at most ~1 us - * for the bit to clear, so wait 2 us for good measure. + * for the bit to clear, so wait 5 us for good measure. */ - udelay(2); + udelay(50); WARN_ON(readl_relaxed(cl->base + serdes_unit + REG_SERDES_STATUS) & SERDES_STATUS_RST); mca_modify(cl, serdes_conf, SERDES_CONF_SYNC_SEL, FIELD_PREP(SERDES_CONF_SYNC_SEL, 0)); mca_modify(cl, serdes_conf, SERDES_CONF_SYNC_SEL, FIELD_PREP(SERDES_CONF_SYNC_SEL, cl->no + 1)); + /* + * ADMAC gets started right after this. This delay seems + * to be needed for that to be reliable, e.g. ensure the + * clock is stable? + */ + udelay(100); break; default: break; @@ -256,11 +267,28 @@ static int mca_fe_trigger(struct snd_pcm_substream *substream, int cmd, return 0; } +static int mca_fe_get_portmask(struct snd_pcm_substream *substream) +{ + struct snd_soc_pcm_runtime *fe = snd_soc_substream_to_rtd(substream); + struct snd_soc_dpcm *dpcm; + int mask = 0; + + for_each_dpcm_be(fe, substream->stream, dpcm) { + int no = mca_dai_to_cluster(snd_soc_rtd_to_cpu(dpcm->be, 0))->no; + mask |= 1 << no; + } + + return mask; +} + static int mca_fe_enable_clocks(struct mca_cluster *cl) { struct mca_data *mca = cl->host; int ret; + if (!cl->clk_provider) + return -EINVAL; + ret = clk_prepare_enable(cl->clk_parent); if (ret) { dev_err(mca->dev, @@ -274,6 +302,7 @@ static int mca_fe_enable_clocks(struct mca_cluster *cl) * the power state driver would error out on seeing the device * as clock-gated. */ + WARN_ON(cl->pd_link); cl->pd_link = device_link_add(mca->dev, cl->pd_dev, DL_FLAG_STATELESS | DL_FLAG_PM_RUNTIME | DL_FLAG_RPM_ACTIVE); @@ -297,7 +326,11 @@ static void mca_fe_disable_clocks(struct mca_cluster *cl) mca_modify(cl, REG_SYNCGEN_STATUS, SYNCGEN_STATUS_EN, 0); mca_modify(cl, REG_STATUS, STATUS_MCLK_EN, 0); - device_link_del(cl->pd_link); + if (cl->pd_link) { + device_link_del(cl->pd_link); + cl->pd_link = NULL; + } + clk_disable_unprepare(cl->clk_parent); } @@ -311,7 +344,7 @@ static bool mca_fe_clocks_in_use(struct mca_cluster *cl) for (i = 0; i < mca->nclusters; i++) { be_cl = &mca->clusters[i]; - if (be_cl->port_driver != cl->no) + if (be_cl->port_clk_driver != cl->no) continue; for_each_pcm_streams(stream) { @@ -325,59 +358,55 @@ static bool mca_fe_clocks_in_use(struct mca_cluster *cl) return false; } -static int mca_be_prepare(struct snd_pcm_substream *substream, +static int mca_fe_prepare(struct snd_pcm_substream *substream, struct snd_soc_dai *dai) { struct mca_cluster *cl = mca_dai_to_cluster(dai); struct mca_data *mca = cl->host; - struct mca_cluster *fe_cl; - int ret; - if (cl->port_driver < 0) - return -EINVAL; + if (cl->clk_provider) + return 0; - fe_cl = &mca->clusters[cl->port_driver]; + if (!cl->syncgen_in_use) { + int port = ffs(mca_fe_get_portmask(substream)) - 1; - /* - * Typically the CODECs we are paired with will require clocks - * to be present at time of unmute with the 'mute_stream' op - * or at time of DAPM widget power-up. We need to enable clocks - * here at the latest (frontend prepare would be too late). - */ - if (!mca_fe_clocks_in_use(fe_cl)) { - ret = mca_fe_enable_clocks(fe_cl); - if (ret < 0) - return ret; - } + WARN_ON(cl->pd_link); + cl->pd_link = device_link_add(mca->dev, cl->pd_dev, + DL_FLAG_STATELESS | DL_FLAG_PM_RUNTIME | + DL_FLAG_RPM_ACTIVE); + if (!cl->pd_link) { + dev_err(mca->dev, + "cluster %d: unable to prop-up power domain\n", cl->no); + return -EINVAL; + } - cl->clocks_in_use[substream->stream] = true; + writel_relaxed(port + 6 + 1, + cl->base + REG_SYNCGEN_MCLK_SEL); + mca_modify(cl, REG_SYNCGEN_STATUS, SYNCGEN_STATUS_EN, + SYNCGEN_STATUS_EN); + } + cl->syncgen_in_use |= 1 << substream->stream; return 0; } -static int mca_be_hw_free(struct snd_pcm_substream *substream, +static int mca_fe_hw_free(struct snd_pcm_substream *substream, struct snd_soc_dai *dai) { struct mca_cluster *cl = mca_dai_to_cluster(dai); - struct mca_data *mca = cl->host; - struct mca_cluster *fe_cl; - - if (cl->port_driver < 0) - return -EINVAL; - /* - * We are operating on a foreign cluster here, but since we - * belong to the same PCM, accesses should have been - * synchronized at ASoC level. - */ - fe_cl = &mca->clusters[cl->port_driver]; - if (!mca_fe_clocks_in_use(fe_cl)) - return 0; /* Nothing to do */ + if (cl->clk_provider) + return 0; - cl->clocks_in_use[substream->stream] = false; + cl->syncgen_in_use &= ~(1 << substream->stream); + if (cl->syncgen_in_use) + return 0; - if (!mca_fe_clocks_in_use(fe_cl)) - mca_fe_disable_clocks(fe_cl); + mca_modify(cl, REG_SYNCGEN_STATUS, SYNCGEN_STATUS_EN, 0); + if (cl->pd_link) { + device_link_del(cl->pd_link); + cl->pd_link = NULL; + } return 0; } @@ -392,7 +421,7 @@ static unsigned int mca_crop_mask(unsigned int mask, int nchans) static int mca_configure_serdes(struct mca_cluster *cl, int serdes_unit, unsigned int mask, int slots, int nchans, - int slot_width, bool is_tx, int port) + int slot_width, bool is_tx, int portmask) { __iomem void *serdes_base = cl->base + serdes_unit; u32 serdes_conf, serdes_conf_mask; @@ -451,7 +480,7 @@ static int mca_configure_serdes(struct mca_cluster *cl, int serdes_unit, serdes_base + REG_RX_SERDES_SLOTMASK); writel_relaxed(~((u32)mca_crop_mask(mask, nchans)), serdes_base + REG_RX_SERDES_SLOTMASK + 0x4); - writel_relaxed(1 << port, + writel_relaxed(portmask, serdes_base + REG_RX_SERDES_PORT); } @@ -507,9 +536,18 @@ static int mca_fe_set_fmt(struct snd_soc_dai *dai, unsigned int fmt) u32 serdes_conf = 0; u32 bitstart; - if ((fmt & SND_SOC_DAIFMT_CLOCK_PROVIDER_MASK) != - SND_SOC_DAIFMT_BP_FP) + switch (fmt & SND_SOC_DAIFMT_CLOCK_PROVIDER_MASK) { + case SND_SOC_DAIFMT_BP_FP: + cl->clk_provider = true; + break; + + case SND_SOC_DAIFMT_BC_FC: + cl->clk_provider = false; + break; + + default: goto err; + } switch (fmt & SND_SOC_DAIFMT_FORMAT_MASK) { case SND_SOC_DAIFMT_I2S: @@ -566,24 +604,6 @@ static int mca_set_bclk_ratio(struct snd_soc_dai *dai, unsigned int ratio) return 0; } -static int mca_fe_get_port(struct snd_pcm_substream *substream) -{ - struct snd_soc_pcm_runtime *fe = snd_soc_substream_to_rtd(substream); - struct snd_soc_pcm_runtime *be; - struct snd_soc_dpcm *dpcm; - - be = NULL; - for_each_dpcm_be(fe, substream->stream, dpcm) { - be = dpcm->be; - break; - } - - if (!be) - return -EINVAL; - - return mca_dai_to_cluster(snd_soc_rtd_to_cpu(be, 0))->no; -} - static int mca_fe_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params, struct snd_soc_dai *dai) @@ -597,7 +617,7 @@ static int mca_fe_hw_params(struct snd_pcm_substream *substream, unsigned long bclk_ratio; unsigned int tdm_slots, tdm_slot_width, tdm_mask; u32 regval, pad; - int ret, port, nchans_ceiled; + int ret, portmask, nchans_ceiled; if (!cl->tdm_slot_width) { /* @@ -646,13 +666,13 @@ static int mca_fe_hw_params(struct snd_pcm_substream *substream, tdm_mask = (1 << tdm_slots) - 1; } - port = mca_fe_get_port(substream); - if (port < 0) - return port; + portmask = mca_fe_get_portmask(substream); + if (!portmask) + return -EINVAL; ret = mca_configure_serdes(cl, is_tx ? CLUSTER_TX_OFF : CLUSTER_RX_OFF, tdm_mask, tdm_slots, params_channels(params), - tdm_slot_width, is_tx, port); + tdm_slot_width, is_tx, portmask); if (ret) return ret; @@ -708,67 +728,123 @@ static const struct snd_soc_dai_ops mca_fe_ops = { .set_tdm_slot = mca_fe_set_tdm_slot, .hw_params = mca_fe_hw_params, .trigger = mca_fe_trigger, + .prepare = mca_fe_prepare, + .hw_free = mca_fe_hw_free, }; -static bool mca_be_started(struct mca_cluster *cl) +/* + * Is there a FE attached which will be feeding this port's clocks? + */ +static bool mca_be_clk_started(struct mca_cluster *cl) { int stream; for_each_pcm_streams(stream) - if (cl->port_started[stream]) + if (cl->port_clk_started[stream]) return true; return false; } -static int mca_be_startup(struct snd_pcm_substream *substream, +static struct snd_soc_pcm_runtime *mca_be_get_fe(struct snd_soc_pcm_runtime *be, + int stream) +{ + struct snd_soc_pcm_runtime *fe = NULL; + struct snd_soc_dpcm *dpcm; + + for_each_dpcm_fe(be, stream, dpcm) { + if (fe && dpcm->fe != fe) { + dev_err(be->dev, "many FE per one BE unsupported\n"); + return NULL; + } + + fe = dpcm->fe; + } + + return fe; +} + +static int mca_be_prepare(struct snd_pcm_substream *substream, struct snd_soc_dai *dai) { struct snd_soc_pcm_runtime *be = snd_soc_substream_to_rtd(substream); - struct snd_soc_pcm_runtime *fe; + struct snd_soc_pcm_runtime *fe = mca_be_get_fe(be, substream->stream); struct mca_cluster *cl = mca_dai_to_cluster(dai); - struct mca_cluster *fe_cl; struct mca_data *mca = cl->host; - struct snd_soc_dpcm *dpcm; + struct mca_cluster *fe_cl, *fe_clk_cl; + int ret; - fe = NULL; + fe_cl = mca_dai_to_cluster(snd_soc_rtd_to_cpu(fe, 0)); - for_each_dpcm_fe(be, substream->stream, dpcm) { - if (fe && dpcm->fe != fe) { - dev_err(mca->dev, "many FE per one BE unsupported\n"); - return -EINVAL; - } + if (!fe_cl->clk_provider) + return 0; - fe = dpcm->fe; + if (cl->port_clk_driver < 0) + return 0; + + fe_clk_cl = &mca->clusters[cl->port_clk_driver]; + + /* + * Typically the CODECs we are paired with will require clocks + * to be present at time of unmute with the 'mute_stream' op + * or at time of DAPM widget power-up. We need to enable clocks + * here at the latest (frontend prepare would be too late). + */ + if (!mca_fe_clocks_in_use(fe_clk_cl)) { + ret = mca_fe_enable_clocks(fe_clk_cl); + if (ret < 0) + return ret; } + cl->clocks_in_use[substream->stream] = true; + + return 0; +} + +static int mca_be_startup(struct snd_pcm_substream *substream, + struct snd_soc_dai *dai) +{ + struct snd_soc_pcm_runtime *be = snd_soc_substream_to_rtd(substream); + struct snd_soc_pcm_runtime *fe = mca_be_get_fe(be, substream->stream); + struct mca_cluster *cl = mca_dai_to_cluster(dai); + struct mca_cluster *fe_cl; + struct mca_data *mca = cl->host; + if (!fe) return -EINVAL; - fe_cl = mca_dai_to_cluster(snd_soc_rtd_to_cpu(fe, 0)); - if (mca_be_started(cl)) { + if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK) { + writel_relaxed(PORT_DATA_SEL_TXA(fe_cl->no), + cl->base + REG_PORT_DATA_SEL); + mca_modify(cl, REG_PORT_ENABLES, PORT_ENABLES_TX_DATA, + PORT_ENABLES_TX_DATA); + } + + if (!fe_cl->clk_provider) + return 0; + + if (mca_be_clk_started(cl)) { /* * Port is already started in the other direction. * Make sure there isn't a conflict with another cluster - * driving the port. + * driving the port clocks. */ - if (cl->port_driver != fe_cl->no) + if (cl->port_clk_driver != fe_cl->no) return -EINVAL; - cl->port_started[substream->stream] = true; + cl->port_clk_started[substream->stream] = true; return 0; } - writel_relaxed(PORT_ENABLES_CLOCKS | PORT_ENABLES_TX_DATA, - cl->base + REG_PORT_ENABLES); writel_relaxed(FIELD_PREP(PORT_CLOCK_SEL, fe_cl->no + 1), cl->base + REG_PORT_CLOCK_SEL); - writel_relaxed(PORT_DATA_SEL_TXA(fe_cl->no), - cl->base + REG_PORT_DATA_SEL); + mca_modify(cl, REG_PORT_ENABLES, PORT_ENABLES_CLOCKS, + PORT_ENABLES_CLOCKS); + mutex_lock(&mca->port_mutex); - cl->port_driver = fe_cl->no; + cl->port_clk_driver = fe_cl->no; mutex_unlock(&mca->port_mutex); - cl->port_started[substream->stream] = true; + cl->port_clk_started[substream->stream] = true; return 0; } @@ -776,27 +852,60 @@ static int mca_be_startup(struct snd_pcm_substream *substream, static void mca_be_shutdown(struct snd_pcm_substream *substream, struct snd_soc_dai *dai) { + struct snd_soc_pcm_runtime *be = snd_soc_substream_to_rtd(substream); + struct snd_soc_pcm_runtime *fe = mca_be_get_fe(be, substream->stream); struct mca_cluster *cl = mca_dai_to_cluster(dai); + struct mca_cluster *fe_cl; struct mca_data *mca = cl->host; - cl->port_started[substream->stream] = false; + if (cl->clocks_in_use[substream->stream] && + !WARN_ON(cl->port_clk_driver < 0)) { + struct mca_cluster *fe_cl = &mca->clusters[cl->port_clk_driver]; - if (!mca_be_started(cl)) { /* - * Were we the last direction to shutdown? - * Turn off the lights. + * Typically the CODECs we are paired with will require clocks + * to be present at time of mute with the 'mute_stream' op. + * We need to disable the clocks here at the earliest (hw_free + * would be too early). + * + * We are operating on a foreign cluster here, but since we + * belong to the same PCM, accesses should have been + * synchronized at ASoC level. */ - writel_relaxed(0, cl->base + REG_PORT_ENABLES); + cl->clocks_in_use[substream->stream] = false; + + if (!mca_fe_clocks_in_use(fe_cl)) + mca_fe_disable_clocks(fe_cl); + } + + if (!fe) + return; + fe_cl = mca_dai_to_cluster(snd_soc_rtd_to_cpu(fe, 0)); + + if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK) { + mca_modify(cl, REG_PORT_ENABLES, PORT_ENABLES_TX_DATA, 0); writel_relaxed(0, cl->base + REG_PORT_DATA_SEL); + } + + if (!fe_cl->clk_provider) + return; + + cl->port_clk_started[substream->stream] = false; + if (!mca_be_clk_started(cl)) { + /* + * Were we the last direction to shutdown? + * Turn off the lights (clocks). + */ + mca_modify(cl, REG_PORT_ENABLES, PORT_ENABLES_CLOCKS, 0); + writel_relaxed(0, cl->base + REG_PORT_CLOCK_SEL); mutex_lock(&mca->port_mutex); - cl->port_driver = -1; + cl->port_clk_driver = -1; mutex_unlock(&mca->port_mutex); } } static const struct snd_soc_dai_ops mca_be_ops = { .prepare = mca_be_prepare, - .hw_free = mca_be_hw_free, .startup = mca_be_startup, .shutdown = mca_be_shutdown, }; @@ -1020,8 +1129,10 @@ static void apple_mca_release(struct mca_data *mca) dev_pm_domain_detach(cl->pd_dev, true); } - if (mca->pd_link) + if (mca->pd_link) { device_link_del(mca->pd_link); + mca->pd_link = NULL; + } if (!IS_ERR_OR_NULL(mca->pd_dev)) dev_pm_domain_detach(mca->pd_dev, true); @@ -1096,7 +1207,7 @@ static int apple_mca_probe(struct platform_device *pdev) cl->host = mca; cl->no = i; cl->base = base + CLUSTER_STRIDE * i; - cl->port_driver = -1; + cl->port_clk_driver = -1; cl->clk_parent = of_clk_get(pdev->dev.of_node, i); if (IS_ERR(cl->clk_parent)) { dev_err(&pdev->dev, "unable to obtain clock %d: %ld\n", diff --git a/sound/soc/cirrus/ep93xx-i2s.c b/sound/soc/cirrus/ep93xx-i2s.c index cca01c03f04864..5dba741594fab9 100644 --- a/sound/soc/cirrus/ep93xx-i2s.c +++ b/sound/soc/cirrus/ep93xx-i2s.c @@ -91,16 +91,28 @@ static inline unsigned ep93xx_i2s_read_reg(struct ep93xx_i2s_info *info, return __raw_readl(info->regs + reg); } -static void ep93xx_i2s_enable(struct ep93xx_i2s_info *info, int stream) +static int ep93xx_i2s_enable(struct ep93xx_i2s_info *info, int stream) { unsigned base_reg; + int err; if ((ep93xx_i2s_read_reg(info, EP93XX_I2S_TX0EN) & 0x1) == 0 && (ep93xx_i2s_read_reg(info, EP93XX_I2S_RX0EN) & 0x1) == 0) { /* Enable clocks */ - clk_prepare_enable(info->mclk); - clk_prepare_enable(info->sclk); - clk_prepare_enable(info->lrclk); + err = clk_prepare_enable(info->mclk); + if (err) + return err; + err = clk_prepare_enable(info->sclk); + if (err) { + clk_disable_unprepare(info->mclk); + return err; + } + err = clk_prepare_enable(info->lrclk); + if (err) { + clk_disable_unprepare(info->sclk); + clk_disable_unprepare(info->mclk); + return err; + } /* Enable i2s */ ep93xx_i2s_write_reg(info, EP93XX_I2S_GLCTRL, 1); @@ -119,6 +131,8 @@ static void ep93xx_i2s_enable(struct ep93xx_i2s_info *info, int stream) ep93xx_i2s_write_reg(info, EP93XX_I2S_TXCTRL, EP93XX_I2S_TXCTRL_TXEMPTY_LVL | EP93XX_I2S_TXCTRL_TXUFIE); + + return 0; } static void ep93xx_i2s_disable(struct ep93xx_i2s_info *info, int stream) @@ -195,9 +209,7 @@ static int ep93xx_i2s_startup(struct snd_pcm_substream *substream, { struct ep93xx_i2s_info *info = snd_soc_dai_get_drvdata(dai); - ep93xx_i2s_enable(info, substream->stream); - - return 0; + return ep93xx_i2s_enable(info, substream->stream); } static void ep93xx_i2s_shutdown(struct snd_pcm_substream *substream, @@ -373,14 +385,16 @@ static int ep93xx_i2s_suspend(struct snd_soc_component *component) static int ep93xx_i2s_resume(struct snd_soc_component *component) { struct ep93xx_i2s_info *info = snd_soc_component_get_drvdata(component); + int err; if (!snd_soc_component_active(component)) return 0; - ep93xx_i2s_enable(info, SNDRV_PCM_STREAM_PLAYBACK); - ep93xx_i2s_enable(info, SNDRV_PCM_STREAM_CAPTURE); + err = ep93xx_i2s_enable(info, SNDRV_PCM_STREAM_PLAYBACK); + if (err) + return err; - return 0; + return ep93xx_i2s_enable(info, SNDRV_PCM_STREAM_CAPTURE); } #else #define ep93xx_i2s_suspend NULL diff --git a/sound/soc/codecs/adau1372.c b/sound/soc/codecs/adau1372.c index fdee689cae5383..d7363f9d53bb31 100644 --- a/sound/soc/codecs/adau1372.c +++ b/sound/soc/codecs/adau1372.c @@ -762,7 +762,7 @@ static int adau1372_startup(struct snd_pcm_substream *substream, struct snd_soc_ return 0; } -static void adau1372_enable_pll(struct adau1372 *adau1372) +static int adau1372_enable_pll(struct adau1372 *adau1372) { unsigned int val, timeout = 0; int ret; @@ -778,19 +778,26 @@ static void adau1372_enable_pll(struct adau1372 *adau1372) timeout++; } while (!(val & 1) && timeout < 3); - if (ret < 0 || !(val & 1)) + if (ret < 0 || !(val & 1)) { dev_err(adau1372->dev, "Failed to lock PLL\n"); + return ret < 0 ? ret : -ETIMEDOUT; + } + + return 0; } -static void adau1372_set_power(struct adau1372 *adau1372, bool enable) +static int adau1372_set_power(struct adau1372 *adau1372, bool enable) { if (adau1372->enabled == enable) - return; + return 0; if (enable) { unsigned int clk_ctrl = ADAU1372_CLK_CTRL_MCLK_EN; + int ret; - clk_prepare_enable(adau1372->mclk); + ret = clk_prepare_enable(adau1372->mclk); + if (ret) + return ret; if (adau1372->pd_gpio) gpiod_set_value(adau1372->pd_gpio, 0); @@ -804,7 +811,14 @@ static void adau1372_set_power(struct adau1372 *adau1372, bool enable) * accessed. */ if (adau1372->use_pll) { - adau1372_enable_pll(adau1372); + ret = adau1372_enable_pll(adau1372); + if (ret) { + regcache_cache_only(adau1372->regmap, true); + if (adau1372->pd_gpio) + gpiod_set_value(adau1372->pd_gpio, 1); + clk_disable_unprepare(adau1372->mclk); + return ret; + } clk_ctrl |= ADAU1372_CLK_CTRL_CLKSRC; } @@ -829,6 +843,8 @@ static void adau1372_set_power(struct adau1372 *adau1372, bool enable) } adau1372->enabled = enable; + + return 0; } static int adau1372_set_bias_level(struct snd_soc_component *component, @@ -842,11 +858,9 @@ static int adau1372_set_bias_level(struct snd_soc_component *component, case SND_SOC_BIAS_PREPARE: break; case SND_SOC_BIAS_STANDBY: - adau1372_set_power(adau1372, true); - break; + return adau1372_set_power(adau1372, true); case SND_SOC_BIAS_OFF: - adau1372_set_power(adau1372, false); - break; + return adau1372_set_power(adau1372, false); } return 0; diff --git a/sound/soc/codecs/aw88261.c b/sound/soc/codecs/aw88261.c index 8f37bfb974ae48..96b1a85c26aa72 100644 --- a/sound/soc/codecs/aw88261.c +++ b/sound/soc/codecs/aw88261.c @@ -423,9 +423,10 @@ static int aw88261_dev_reg_update(struct aw88261 *aw88261, if (ret) break; + /* keep all three bits from current hw status */ read_val &= (~AW88261_AMPPD_MASK) | (~AW88261_PWDN_MASK) | (~AW88261_HMUTE_MASK); - reg_val &= (AW88261_AMPPD_MASK | AW88261_PWDN_MASK | AW88261_HMUTE_MASK); + reg_val &= (AW88261_AMPPD_MASK & AW88261_PWDN_MASK & AW88261_HMUTE_MASK); reg_val |= read_val; /* enable uls hmute */ diff --git a/sound/soc/codecs/cs35l56-shared.c b/sound/soc/codecs/cs35l56-shared.c index 60100c8f8c952b..0ec6a96e80858c 100644 --- a/sound/soc/codecs/cs35l56-shared.c +++ b/sound/soc/codecs/cs35l56-shared.c @@ -23,7 +23,7 @@ #include "cs35l56.h" -static const struct reg_sequence cs35l56_patch[] = { +static const struct reg_sequence cs35l56_asp_patch[] = { /* * Firmware can change these to non-defaults to satisfy SDCA. * Ensure that they are at known defaults. @@ -40,6 +40,20 @@ static const struct reg_sequence cs35l56_patch[] = { { CS35L56_ASP1TX2_INPUT, 0x00000000 }, { CS35L56_ASP1TX3_INPUT, 0x00000000 }, { CS35L56_ASP1TX4_INPUT, 0x00000000 }, +}; + +int cs35l56_set_asp_patch(struct cs35l56_base *cs35l56_base) +{ + return regmap_register_patch(cs35l56_base->regmap, cs35l56_asp_patch, + ARRAY_SIZE(cs35l56_asp_patch)); +} +EXPORT_SYMBOL_NS_GPL(cs35l56_set_asp_patch, "SND_SOC_CS35L56_SHARED"); + +static const struct reg_sequence cs35l56_patch[] = { + /* + * Firmware can change these to non-defaults to satisfy SDCA. + * Ensure that they are at known defaults. + */ { CS35L56_SWIRE_DP3_CH1_INPUT, 0x00000018 }, { CS35L56_SWIRE_DP3_CH2_INPUT, 0x00000019 }, { CS35L56_SWIRE_DP3_CH3_INPUT, 0x00000029 }, diff --git a/sound/soc/codecs/cs35l56.c b/sound/soc/codecs/cs35l56.c index 55b4d0d55712a7..1c1924c6f40702 100644 --- a/sound/soc/codecs/cs35l56.c +++ b/sound/soc/codecs/cs35l56.c @@ -346,6 +346,13 @@ static int cs35l56_dsp_event(struct snd_soc_dapm_widget *w, return wm_adsp_event(w, kcontrol, event); } +static int cs35l56_asp_dai_probe(struct snd_soc_dai *codec_dai) +{ + struct cs35l56_private *cs35l56 = snd_soc_component_get_drvdata(codec_dai->component); + + return cs35l56_set_asp_patch(&cs35l56->base); +} + static int cs35l56_asp_dai_set_fmt(struct snd_soc_dai *codec_dai, unsigned int fmt) { struct cs35l56_private *cs35l56 = snd_soc_component_get_drvdata(codec_dai->component); @@ -550,6 +557,7 @@ static int cs35l56_asp_dai_set_sysclk(struct snd_soc_dai *dai, } static const struct snd_soc_dai_ops cs35l56_ops = { + .probe = cs35l56_asp_dai_probe, .set_fmt = cs35l56_asp_dai_set_fmt, .set_tdm_slot = cs35l56_asp_dai_set_tdm_slot, .hw_params = cs35l56_asp_dai_hw_params, diff --git a/sound/soc/codecs/cs4271.c b/sound/soc/codecs/cs4271.c index 77dfc83a3c0131..d8cdd37e9112b2 100644 --- a/sound/soc/codecs/cs4271.c +++ b/sound/soc/codecs/cs4271.c @@ -528,7 +528,7 @@ static int cs4271_soc_resume(struct snd_soc_component *component) ret = clk_prepare_enable(cs4271->clk); if (ret) { dev_err(component->dev, "Failed to enable clk: %d\n", ret); - return ret; + goto err_disable_regulators; } /* Do a proper reset after power up */ @@ -537,15 +537,21 @@ static int cs4271_soc_resume(struct snd_soc_component *component) /* Restore codec state */ ret = regcache_sync(cs4271->regmap); if (ret < 0) - return ret; + goto err_disable_clk; /* then disable the power-down bit */ ret = regmap_update_bits(cs4271->regmap, CS4271_MODE2, CS4271_MODE2_PDN, 0); if (ret < 0) - return ret; + goto err_disable_clk; return 0; + +err_disable_clk: + clk_disable_unprepare(cs4271->clk); +err_disable_regulators: + regulator_bulk_disable(ARRAY_SIZE(cs4271->supplies), cs4271->supplies); + return ret; } #else #define cs4271_soc_suspend NULL diff --git a/sound/soc/codecs/cs42l42.c b/sound/soc/codecs/cs42l42.c index 2652a639a79ad2..1f2efce071912d 100644 --- a/sound/soc/codecs/cs42l42.c +++ b/sound/soc/codecs/cs42l42.c @@ -1148,7 +1148,6 @@ struct snd_soc_dai_driver cs42l42_dai = { .formats = CS42L42_FORMATS, }, .symmetric_rate = 1, - .symmetric_sample_bits = 1, .ops = &cs42l42_ops, }; EXPORT_SYMBOL_NS_GPL(cs42l42_dai, "SND_SOC_CS42L42_CORE"); @@ -1676,7 +1675,7 @@ irqreturn_t cs42l42_irq_thread(int irq, void *data) return IRQ_NONE; } - /* Read sticky registers to clear interurpt */ + /* Read sticky registers to clear interrupt */ for (i = 0; i < ARRAY_SIZE(stickies); i++) { regmap_read(cs42l42->regmap, irq_params_table[i].status_addr, &(stickies[i])); @@ -2419,6 +2418,16 @@ int cs42l42_init(struct cs42l42_private *cs42l42) (1 << CS42L42_ADC_PDN_SHIFT) | (0 << CS42L42_PDN_ALL_SHIFT)); + /* + * Configure a faster digital ramp time, to avoid an audible + * fade-in when streams start. + */ + regmap_update_bits(cs42l42->regmap, CS42L42_SFTRAMP_RATE, + CS42L42_SFTRAMP_ASR_RATE_MASK | + CS42L42_SFTRAMP_DSR_RATE_MASK, + (10 << CS42L42_SFTRAMP_ASR_RATE_SHIFT) | + (1 << CS42L42_SFTRAMP_DSR_RATE_SHIFT)); + ret = cs42l42_handle_device_data(cs42l42->dev, cs42l42); if (ret != 0) goto err_shutdown; diff --git a/sound/soc/codecs/cs42l43-jack.c b/sound/soc/codecs/cs42l43-jack.c index b83bc4de1301d1..3e04e6897b1423 100644 --- a/sound/soc/codecs/cs42l43-jack.c +++ b/sound/soc/codecs/cs42l43-jack.c @@ -699,6 +699,7 @@ static int cs42l43_run_type_detect(struct cs42l43_codec *priv) switch (type & CS42L43_HSDET_TYPE_STS_MASK) { case 0x0: // CTIA case 0x1: // OMTP + case 0x4: return cs42l43_run_load_detect(priv, true); case 0x2: // 3-pole return cs42l43_run_load_detect(priv, false); diff --git a/sound/soc/codecs/cs42l84.c b/sound/soc/codecs/cs42l84.c index 1e1307a16f8152..eba2c900b8e4c8 100644 --- a/sound/soc/codecs/cs42l84.c +++ b/sound/soc/codecs/cs42l84.c @@ -357,8 +357,11 @@ struct cs42l84_pll_params { * Common PLL Settings for given BCLK */ static const struct cs42l84_pll_params pll_ratio_table[] = { + { 2822400, 1, 0, 0x40, 0x000000, 0x03, 0x10, 11289600}, { 3072000, 1, 0, 0x40, 0x000000, 0x03, 0x10, 12288000}, + { 5644800, 1, 0, 0x40, 0x000000, 0x03, 0x10, 11289600}, { 6144000, 1, 1, 0x40, 0x000000, 0x03, 0x10, 12288000}, + { 11289600, 0, 0, 0, 0, 0, 0, 11289600}, { 12288000, 0, 0, 0, 0, 0, 0, 12288000}, { 24576000, 1, 3, 0x40, 0x000000, 0x03, 0x10, 12288000}, }; @@ -670,14 +673,18 @@ static struct snd_soc_dai_driver cs42l84_dai = { .stream_name = "Playback", .channels_min = 1, .channels_max = 2, - .rates = SNDRV_PCM_RATE_48000 | SNDRV_PCM_RATE_96000, + .rates = SNDRV_PCM_RATE_44100 | SNDRV_PCM_RATE_48000 | + SNDRV_PCM_RATE_88200 | SNDRV_PCM_RATE_96000 | + SNDRV_PCM_RATE_176400 | SNDRV_PCM_RATE_192000, .formats = CS42L84_FORMATS, }, .capture = { .stream_name = "Capture", .channels_min = 1, .channels_max = 1, - .rates = SNDRV_PCM_RATE_48000 | SNDRV_PCM_RATE_96000, + .rates = SNDRV_PCM_RATE_44100 | SNDRV_PCM_RATE_48000 | + SNDRV_PCM_RATE_88200 | SNDRV_PCM_RATE_96000 | + SNDRV_PCM_RATE_176400 | SNDRV_PCM_RATE_192000, .formats = CS42L84_FORMATS, }, .symmetric_rate = 1, diff --git a/sound/soc/codecs/es8328.c b/sound/soc/codecs/es8328.c index 1e11175cfbbbf2..47c6b0c218b2c1 100644 --- a/sound/soc/codecs/es8328.c +++ b/sound/soc/codecs/es8328.c @@ -758,17 +758,23 @@ static int es8328_resume(struct snd_soc_component *component) es8328->supplies); if (ret) { dev_err(component->dev, "unable to enable regulators\n"); - return ret; + goto err_clk; } regcache_mark_dirty(regmap); ret = regcache_sync(regmap); if (ret) { dev_err(component->dev, "unable to sync regcache\n"); - return ret; + goto err_regulators; } return 0; + +err_regulators: + regulator_bulk_disable(ARRAY_SIZE(es8328->supplies), es8328->supplies); +err_clk: + clk_disable_unprepare(es8328->clk); + return ret; } static int es8328_component_probe(struct snd_soc_component *component) diff --git a/sound/soc/codecs/max98390.c b/sound/soc/codecs/max98390.c index 3dd4dd94bc371f..ff58805e97d17f 100644 --- a/sound/soc/codecs/max98390.c +++ b/sound/soc/codecs/max98390.c @@ -1067,6 +1067,9 @@ static int max98390_i2c_probe(struct i2c_client *i2c) reset_gpio = devm_gpiod_get_optional(&i2c->dev, "reset", GPIOD_OUT_HIGH); + if (IS_ERR(reset_gpio)) + return dev_err_probe(&i2c->dev, PTR_ERR(reset_gpio), + "Failed to get reset gpio\n"); /* Power on device */ if (reset_gpio) { diff --git a/sound/soc/codecs/nau8821.c b/sound/soc/codecs/nau8821.c index 3beb3c44dc2c03..58d2d5e77c8f44 100644 --- a/sound/soc/codecs/nau8821.c +++ b/sound/soc/codecs/nau8821.c @@ -1264,6 +1264,14 @@ static int nau8821_component_probe(struct snd_soc_component *component) return 0; } +static void nau8821_component_remove(struct snd_soc_component *component) +{ + struct nau8821 *nau8821 = snd_soc_component_get_drvdata(component); + + if (nau8821->jdet_active) + cancel_delayed_work_sync(&nau8821->jdet_work); +}; + /** * nau8821_calc_fll_param - Calculate FLL parameters. * @fll_in: external clock provided to codec. @@ -1597,6 +1605,10 @@ static int __maybe_unused nau8821_suspend(struct snd_soc_component *component) if (nau8821->irq) disable_irq(nau8821->irq); + + if (nau8821->jdet_active) + cancel_delayed_work_sync(&nau8821->jdet_work); + snd_soc_dapm_force_bias_level(nau8821->dapm, SND_SOC_BIAS_OFF); /* Power down codec power; don't support button wakeup */ snd_soc_dapm_disable_pin(nau8821->dapm, "MICBIAS"); @@ -1621,6 +1633,7 @@ static int __maybe_unused nau8821_resume(struct snd_soc_component *component) static const struct snd_soc_component_driver nau8821_component_driver = { .probe = nau8821_component_probe, + .remove = nau8821_component_remove, .set_sysclk = nau8821_set_sysclk, .set_pll = nau8821_set_fll, .set_bias_level = nau8821_set_bias_level, @@ -1655,8 +1668,13 @@ int nau8821_enable_jack_detect(struct snd_soc_component *component, int ret; nau8821->jack = jack; + + if (nau8821->jdet_active) + return 0; + /* Initiate jack detection work queue */ INIT_DELAYED_WORK(&nau8821->jdet_work, nau8821_jdet_work); + nau8821->jdet_active = true; ret = devm_request_threaded_irq(nau8821->dev, nau8821->irq, NULL, nau8821_interrupt, IRQF_TRIGGER_LOW | IRQF_ONESHOT, diff --git a/sound/soc/codecs/nau8821.h b/sound/soc/codecs/nau8821.h index 88602923780d85..f9d7cd8cbd2116 100644 --- a/sound/soc/codecs/nau8821.h +++ b/sound/soc/codecs/nau8821.h @@ -562,6 +562,7 @@ struct nau8821 { struct snd_soc_dapm_context *dapm; struct snd_soc_jack *jack; struct delayed_work jdet_work; + bool jdet_active; int irq; int clk_id; int micbias_voltage; diff --git a/sound/soc/codecs/rt1011.c b/sound/soc/codecs/rt1011.c index 9f34a6a3548763..03f31d9d916e6d 100644 --- a/sound/soc/codecs/rt1011.c +++ b/sound/soc/codecs/rt1011.c @@ -1047,7 +1047,7 @@ static int rt1011_recv_spk_mode_put(struct snd_kcontrol *kcontrol, struct snd_ctl_elem_value *ucontrol) { struct snd_soc_component *component = snd_kcontrol_chip(kcontrol); - struct snd_soc_dapm_context *dapm = snd_soc_dapm_kcontrol_to_dapm(kcontrol); + struct snd_soc_dapm_context *dapm = snd_soc_component_to_dapm(component); struct rt1011_priv *rt1011 = snd_soc_component_get_drvdata(component); diff --git a/sound/soc/codecs/rt1320-sdw.c b/sound/soc/codecs/rt1320-sdw.c index e6142645b90384..4d09dd06f2d839 100644 --- a/sound/soc/codecs/rt1320-sdw.c +++ b/sound/soc/codecs/rt1320-sdw.c @@ -1455,7 +1455,7 @@ static int rt1320_sdw_hw_params(struct snd_pcm_substream *substream, struct sdw_port_config port_config; struct sdw_port_config dmic_port_config[2]; struct sdw_stream_runtime *sdw_stream; - int retval; + int retval, num_channels; unsigned int sampling_rate; dev_dbg(dai->dev, "%s %s", __func__, dai->name); @@ -1487,7 +1487,8 @@ static int rt1320_sdw_hw_params(struct snd_pcm_substream *substream, dmic_port_config[1].num = 10; break; case RT1321_DEV_ID: - dmic_port_config[0].ch_mask = BIT(0) | BIT(1); + num_channels = params_channels(params); + dmic_port_config[0].ch_mask = GENMASK(num_channels - 1, 0); dmic_port_config[0].num = 8; break; default: diff --git a/sound/soc/codecs/rt721-sdca.c b/sound/soc/codecs/rt721-sdca.c index 8233532a1752a3..35960c22522491 100644 --- a/sound/soc/codecs/rt721-sdca.c +++ b/sound/soc/codecs/rt721-sdca.c @@ -245,12 +245,12 @@ static void rt721_sdca_jack_preset(struct rt721_sdca_priv *rt721) regmap_write(rt721->mbq_regmap, 0x5b10007, 0x2000); regmap_write(rt721->mbq_regmap, 0x5B10017, 0x1b0f); rt_sdca_index_write(rt721->mbq_regmap, RT721_CBJ_CTRL, - RT721_CBJ_A0_GAT_CTRL1, 0x2a02); + RT721_CBJ_A0_GAT_CTRL1, 0x2205); rt_sdca_index_write(rt721->mbq_regmap, RT721_CAP_PORT_CTRL, RT721_HP_AMP_2CH_CAL4, 0xa105); rt_sdca_index_write(rt721->mbq_regmap, RT721_VENDOR_ANA_CTL, RT721_UAJ_TOP_TCON14, 0x3b33); - regmap_write(rt721->mbq_regmap, 0x310400, 0x3023); + regmap_write(rt721->mbq_regmap, 0x310400, 0x3043); rt_sdca_index_write(rt721->mbq_regmap, RT721_VENDOR_ANA_CTL, RT721_UAJ_TOP_TCON14, 0x3f33); rt_sdca_index_write(rt721->mbq_regmap, RT721_VENDOR_ANA_CTL, diff --git a/sound/soc/codecs/sma1307.c b/sound/soc/codecs/sma1307.c index 4bb59e5c089153..5850bf6e71cade 100644 --- a/sound/soc/codecs/sma1307.c +++ b/sound/soc/codecs/sma1307.c @@ -1759,8 +1759,10 @@ static void sma1307_setting_loaded(struct sma1307_priv *sma1307, const char *fil sma1307->set.mode_size * 2 * sizeof(int), GFP_KERNEL); if (!sma1307->set.mode_set[i]) { - for (int j = 0; j < i; j++) - kfree(sma1307->set.mode_set[j]); + for (int j = 0; j < i; j++) { + devm_kfree(sma1307->dev, sma1307->set.mode_set[j]); + sma1307->set.mode_set[j] = NULL; + } sma1307->set.status = false; return; } diff --git a/sound/soc/codecs/tas2764.c b/sound/soc/codecs/tas2764.c index 36e25e48b35463..a458458128aa89 100644 --- a/sound/soc/codecs/tas2764.c +++ b/sound/soc/codecs/tas2764.c @@ -34,6 +34,7 @@ struct tas2764_priv { struct snd_soc_component *component; struct gpio_desc *reset_gpio; struct gpio_desc *sdz_gpio; + struct regulator *sdz_reg; struct regmap *regmap; struct device *dev; int irq; @@ -41,6 +42,7 @@ struct tas2764_priv { int v_sense_slot; int i_sense_slot; + u32 sdout_zero_mask; bool dac_powered; bool unmuted; @@ -152,6 +154,8 @@ static int tas2764_codec_suspend(struct snd_soc_component *component) if (tas2764->sdz_gpio) gpiod_set_value_cansleep(tas2764->sdz_gpio, 0); + regulator_disable(tas2764->sdz_reg); + regcache_cache_only(tas2764->regmap, true); regcache_mark_dirty(tas2764->regmap); @@ -165,19 +169,26 @@ static int tas2764_codec_resume(struct snd_soc_component *component) struct tas2764_priv *tas2764 = snd_soc_component_get_drvdata(component); int ret; + ret = regulator_enable(tas2764->sdz_reg); + + if (ret) { + dev_err(tas2764->dev, "Failed to enable regulator\n"); + return ret; + } + if (tas2764->sdz_gpio) { gpiod_set_value_cansleep(tas2764->sdz_gpio, 1); - usleep_range(1000, 2000); } - ret = tas2764_update_pwr_ctrl(tas2764); + usleep_range(1000, 2000); + regcache_cache_only(tas2764->regmap, false); + + ret = regcache_sync(tas2764->regmap); if (ret < 0) return ret; - regcache_cache_only(tas2764->regmap, false); - - return regcache_sync(tas2764->regmap); + return tas2764_update_pwr_ctrl(tas2764); } #else #define tas2764_codec_suspend NULL @@ -210,7 +221,7 @@ static const struct snd_soc_dapm_widget tas2764_dapm_widgets[] = { SND_SOC_DAPM_DAC("DAC", NULL, SND_SOC_NOPM, 0, 0), SND_SOC_DAPM_OUTPUT("OUT"), SND_SOC_DAPM_SIGGEN("VMON"), - SND_SOC_DAPM_SIGGEN("IMON") + SND_SOC_DAPM_SIGGEN("IMON"), }; static const struct snd_soc_dapm_route tas2764_audio_map[] = { @@ -261,7 +272,6 @@ static int tas2764_mute(struct snd_soc_dai *dai, int mute, int direction) static int tas2764_set_bitwidth(struct tas2764_priv *tas2764, int bitwidth) { struct snd_soc_component *component = tas2764->component; - int sense_en; int val; int ret; @@ -296,28 +306,6 @@ static int tas2764_set_bitwidth(struct tas2764_priv *tas2764, int bitwidth) if (val < 0) return val; - if (val & (1 << TAS2764_VSENSE_POWER_EN)) - sense_en = 0; - else - sense_en = TAS2764_TDM_CFG5_VSNS_ENABLE; - - ret = snd_soc_component_update_bits(tas2764->component, TAS2764_TDM_CFG5, - TAS2764_TDM_CFG5_VSNS_ENABLE, - sense_en); - if (ret < 0) - return ret; - - if (val & (1 << TAS2764_ISENSE_POWER_EN)) - sense_en = 0; - else - sense_en = TAS2764_TDM_CFG6_ISNS_ENABLE; - - ret = snd_soc_component_update_bits(tas2764->component, TAS2764_TDM_CFG6, - TAS2764_TDM_CFG6_ISNS_ENABLE, - sense_en); - if (ret < 0) - return ret; - return 0; } @@ -373,6 +361,44 @@ static int tas2764_hw_params(struct snd_pcm_substream *substream, return tas2764_set_samplerate(tas2764, params_rate(params)); } +static int tas2764_write_sdout_zero_mask(struct tas2764_priv *tas2764, int bclk_ratio) +{ + struct snd_soc_component *component = tas2764->component; + int nsense_slots = bclk_ratio / 8; + u32 cropped_mask; + int i, ret; + + if (!tas2764->sdout_zero_mask) + return 0; + + cropped_mask = tas2764->sdout_zero_mask & GENMASK(nsense_slots - 1, 0); + + for (i = 0; i < 4; i++) { + ret = snd_soc_component_write(component, TAS2764_SDOUT_HIZ_1 + i, + (cropped_mask >> (i * 8)) & 0xff); + + if (ret < 0) + return ret; + } + + ret = snd_soc_component_update_bits(component, TAS2764_SDOUT_HIZ_9, + TAS2764_SDOUT_HIZ_9_FORCE_0_EN, + TAS2764_SDOUT_HIZ_9_FORCE_0_EN); + + if (ret < 0) + return ret; + + return 0; +} + +static int tas2764_set_bclk_ratio(struct snd_soc_dai *dai, unsigned int ratio) +{ + struct snd_soc_component *component = dai->component; + struct tas2764_priv *tas2764 = snd_soc_component_get_drvdata(component); + + return tas2764_write_sdout_zero_mask(tas2764, ratio); +} + static int tas2764_set_fmt(struct snd_soc_dai *dai, unsigned int fmt) { struct snd_soc_component *component = dai->component; @@ -447,7 +473,6 @@ static int tas2764_set_dai_tdm_slot(struct snd_soc_dai *dai, int slots, int slot_width) { struct snd_soc_component *component = dai->component; - struct tas2764_priv *tas2764 = snd_soc_component_get_drvdata(component); int left_slot, right_slot; int slots_cfg; int slot_size; @@ -494,15 +519,26 @@ static int tas2764_set_dai_tdm_slot(struct snd_soc_dai *dai, if (ret < 0) return ret; - ret = snd_soc_component_update_bits(component, TAS2764_TDM_CFG5, + return 0; +} + +static int tas2764_set_ivsense_transmit(struct tas2764_priv *tas2764, int i_slot, int v_slot) +{ + int ret; + + ret = snd_soc_component_update_bits(tas2764->component, TAS2764_TDM_CFG5, + TAS2764_TDM_CFG5_VSNS_ENABLE | TAS2764_TDM_CFG5_50_MASK, - tas2764->v_sense_slot); + TAS2764_TDM_CFG5_VSNS_ENABLE | + v_slot); if (ret < 0) return ret; - ret = snd_soc_component_update_bits(component, TAS2764_TDM_CFG6, + ret = snd_soc_component_update_bits(tas2764->component, TAS2764_TDM_CFG6, + TAS2764_TDM_CFG6_ISNS_ENABLE | TAS2764_TDM_CFG6_50_MASK, - tas2764->i_sense_slot); + TAS2764_TDM_CFG6_ISNS_ENABLE | + i_slot); if (ret < 0) return ret; @@ -512,6 +548,7 @@ static int tas2764_set_dai_tdm_slot(struct snd_soc_dai *dai, static const struct snd_soc_dai_ops tas2764_dai_ops = { .mute_stream = tas2764_mute, .hw_params = tas2764_hw_params, + .set_bclk_ratio = tas2764_set_bclk_ratio, .set_fmt = tas2764_set_fmt, .set_tdm_slot = tas2764_set_dai_tdm_slot, .no_capture_mute = 1, @@ -659,11 +696,18 @@ static int tas2764_codec_probe(struct snd_soc_component *component) tas2764->component = component; + ret = regulator_enable(tas2764->sdz_reg); + if (ret != 0) { + dev_err(tas2764->dev, "Failed to enable regulator: %d\n", ret); + return ret; + } + if (tas2764->sdz_gpio) { gpiod_set_value_cansleep(tas2764->sdz_gpio, 1); - usleep_range(1000, 2000); } + usleep_range(1000, 2000); + tas2764_reset(tas2764); regmap_reinit_cache(tas2764->regmap, &tas2764_i2c_regmap); @@ -695,18 +739,33 @@ static int tas2764_codec_probe(struct snd_soc_component *component) dev_warn(tas2764->dev, "failed to request IRQ: %d\n", ret); } - ret = snd_soc_component_update_bits(tas2764->component, TAS2764_TDM_CFG5, - TAS2764_TDM_CFG5_VSNS_ENABLE, 0); - if (ret < 0) - return ret; + if (tas2764->i_sense_slot != -1 && tas2764->v_sense_slot != -1) { + ret = tas2764_set_ivsense_transmit(tas2764, tas2764->i_sense_slot, + tas2764->v_sense_slot); - ret = snd_soc_component_update_bits(tas2764->component, TAS2764_TDM_CFG6, - TAS2764_TDM_CFG6_ISNS_ENABLE, 0); - if (ret < 0) - return ret; + if (ret < 0) + return ret; + } switch (tas2764->devid) { case DEVID_SN012776: + if (tas2764->sdout_zero_mask) { + for (i = 0; i < 4; i++) { + ret = snd_soc_component_write(component, TAS2764_SDOUT_HIZ_1 + i, + (tas2764->sdout_zero_mask >> (i * 8)) & 0xff); + + if (ret < 0) + return ret; + } + + ret = snd_soc_component_update_bits(component, TAS2764_SDOUT_HIZ_9, + TAS2764_SDOUT_HIZ_9_FORCE_0_EN, + TAS2764_SDOUT_HIZ_9_FORCE_0_EN); + + if (ret < 0) + return ret; + } + ret = snd_soc_component_update_bits(component, TAS2764_PWR_CTRL, TAS2764_PWR_CTRL_BOP_SRC, TAS2764_PWR_CTRL_BOP_SRC); @@ -736,6 +795,13 @@ static int tas2764_codec_probe(struct snd_soc_component *component) return 0; } +static void tas2764_codec_remove(struct snd_soc_component *component) +{ + struct tas2764_priv *tas2764 = snd_soc_component_get_drvdata(component); + + regulator_disable(tas2764->sdz_reg); +} + static DECLARE_TLV_DB_SCALE(tas2764_digital_tlv, 1100, 50, 0); static DECLARE_TLV_DB_SCALE(tas2764_playback_volume, -10050, 50, 1); @@ -767,6 +833,7 @@ static const struct snd_kcontrol_new tas2764_snd_controls[] = { static const struct snd_soc_component_driver soc_component_driver_tas2764 = { .probe = tas2764_codec_probe, + .remove = tas2764_codec_remove, .suspend = tas2764_codec_suspend, .resume = tas2764_codec_resume, .controls = tas2764_snd_controls, @@ -836,6 +903,11 @@ static int tas2764_parse_dt(struct device *dev, struct tas2764_priv *tas2764) { int ret = 0; + tas2764->sdz_reg = devm_regulator_get(dev, "SDZ"); + if (IS_ERR(tas2764->sdz_reg)) + return dev_err_probe(dev, PTR_ERR(tas2764->sdz_reg), + "Failed to get SDZ supply\n"); + tas2764->reset_gpio = devm_gpiod_get_optional(tas2764->dev, "reset", GPIOD_OUT_HIGH); if (IS_ERR(tas2764->reset_gpio)) { @@ -856,12 +928,17 @@ static int tas2764_parse_dt(struct device *dev, struct tas2764_priv *tas2764) ret = fwnode_property_read_u32(dev->fwnode, "ti,imon-slot-no", &tas2764->i_sense_slot); if (ret) - tas2764->i_sense_slot = 0; + tas2764->i_sense_slot = -1; ret = fwnode_property_read_u32(dev->fwnode, "ti,vmon-slot-no", &tas2764->v_sense_slot); if (ret) - tas2764->v_sense_slot = 2; + tas2764->v_sense_slot = -1; + + ret = fwnode_property_read_u32(dev->fwnode, "ti,sdout-force-zero-mask", + &tas2764->sdout_zero_mask); + if (ret) + tas2764->sdout_zero_mask = 0; return 0; } diff --git a/sound/soc/codecs/tas2764.h b/sound/soc/codecs/tas2764.h index 538290ed3d92ac..4a419c11d4b08e 100644 --- a/sound/soc/codecs/tas2764.h +++ b/sound/soc/codecs/tas2764.h @@ -126,4 +126,15 @@ #define TAS2764_BOP_CFG0 TAS2764_REG(0X0, 0x1d) +#define TAS2764_SDOUT_HIZ_1 TAS2764_REG(0x1, 0x3d) +#define TAS2764_SDOUT_HIZ_2 TAS2764_REG(0x1, 0x3e) +#define TAS2764_SDOUT_HIZ_3 TAS2764_REG(0x1, 0x3f) +#define TAS2764_SDOUT_HIZ_4 TAS2764_REG(0x1, 0x40) +#define TAS2764_SDOUT_HIZ_5 TAS2764_REG(0x1, 0x41) +#define TAS2764_SDOUT_HIZ_6 TAS2764_REG(0x1, 0x42) +#define TAS2764_SDOUT_HIZ_7 TAS2764_REG(0x1, 0x43) +#define TAS2764_SDOUT_HIZ_8 TAS2764_REG(0x1, 0x44) +#define TAS2764_SDOUT_HIZ_9 TAS2764_REG(0x1, 0x45) +#define TAS2764_SDOUT_HIZ_9_FORCE_0_EN BIT(7) + #endif /* __TAS2764__ */ diff --git a/sound/soc/codecs/tas2770.c b/sound/soc/codecs/tas2770.c index 6f878b01716f72..e72027cf340bfe 100644 --- a/sound/soc/codecs/tas2770.c +++ b/sound/soc/codecs/tas2770.c @@ -71,23 +71,21 @@ static int tas2770_codec_suspend(struct snd_soc_component *component) struct tas2770_priv *tas2770 = snd_soc_component_get_drvdata(component); int ret = 0; - regcache_cache_only(tas2770->regmap, true); - regcache_mark_dirty(tas2770->regmap); + ret = snd_soc_component_update_bits(component, TAS2770_PWR_CTRL, + TAS2770_PWR_CTRL_MASK, + TAS2770_PWR_CTRL_SHUTDOWN); + if (ret < 0) + return ret; - if (tas2770->sdz_gpio) { + if (tas2770->sdz_gpio) gpiod_set_value_cansleep(tas2770->sdz_gpio, 0); - } else { - ret = snd_soc_component_update_bits(component, TAS2770_PWR_CTRL, - TAS2770_PWR_CTRL_MASK, - TAS2770_PWR_CTRL_SHUTDOWN); - if (ret < 0) { - regcache_cache_only(tas2770->regmap, false); - regcache_sync(tas2770->regmap); - return ret; - } - ret = 0; - } + regulator_disable(tas2770->sdz_reg); + + regcache_cache_only(tas2770->regmap, true); + regcache_mark_dirty(tas2770->regmap); + + usleep_range(6000, 7000); return ret; } @@ -97,18 +95,26 @@ static int tas2770_codec_resume(struct snd_soc_component *component) struct tas2770_priv *tas2770 = snd_soc_component_get_drvdata(component); int ret; - if (tas2770->sdz_gpio) { - gpiod_set_value_cansleep(tas2770->sdz_gpio, 1); - usleep_range(1000, 2000); - } else { - ret = tas2770_update_pwr_ctrl(tas2770); - if (ret < 0) - return ret; + ret = regulator_enable(tas2770->sdz_reg); + + if (ret) { + dev_err(tas2770->dev, "Failed to enable regulator\n"); + return ret; } + if (tas2770->sdz_gpio) + gpiod_set_value_cansleep(tas2770->sdz_gpio, 1); + + + usleep_range(1000, 2000); + regcache_cache_only(tas2770->regmap, false); - return regcache_sync(tas2770->regmap); + ret = regcache_sync(tas2770->regmap); + if (ret < 0) + return ret; + + return tas2770_update_pwr_ctrl(tas2770); } #else #define tas2770_codec_suspend NULL @@ -623,11 +629,18 @@ static int tas2770_codec_probe(struct snd_soc_component *component) tas2770->component = component; + ret = regulator_enable(tas2770->sdz_reg); + if (ret != 0) { + dev_err(tas2770->dev, "Failed to enable regulator: %d\n", ret); + return ret; + } + if (tas2770->sdz_gpio) { gpiod_set_value_cansleep(tas2770->sdz_gpio, 1); - usleep_range(1000, 2000); } + usleep_range(1000, 2000); + tas2770_reset(tas2770); regmap_reinit_cache(tas2770->regmap, &tas2770_i2c_regmap); @@ -641,14 +654,34 @@ static int tas2770_codec_probe(struct snd_soc_component *component) if (tas2770->pdm_slot != -1) { ret = tas2770_set_pdm_transmit(tas2770, tas2770->pdm_slot); - if (ret < 0) return ret; } + ret = snd_soc_component_update_bits(component, TAS2770_TDM_CFG_REG4, + TAS2770_TDM_CFG_REG4_TX_FILL, + tas2770->sdout_zfill ? 0 : + TAS2770_TDM_CFG_REG4_TX_FILL); + if (ret < 0) + return ret; + + ret = snd_soc_component_update_bits(component, TAS2770_DIN_PD, + TAS2770_DIN_PD_SDOUT, + tas2770->sdout_pd ? + TAS2770_DIN_PD_SDOUT : 0); + if (ret < 0) + return ret; + return 0; } +static void tas2770_codec_remove(struct snd_soc_component *component) +{ + struct tas2770_priv *tas2770 = snd_soc_component_get_drvdata(component); + + regulator_disable(tas2770->sdz_reg); +} + static DECLARE_TLV_DB_SCALE(tas2770_digital_tlv, 1100, 50, 0); static DECLARE_TLV_DB_SCALE(tas2770_playback_volume, -10050, 50, 0); @@ -661,6 +694,7 @@ static const struct snd_kcontrol_new tas2770_snd_controls[] = { static const struct snd_soc_component_driver soc_component_driver_tas2770 = { .probe = tas2770_codec_probe, + .remove = tas2770_codec_remove, .suspend = tas2770_codec_suspend, .resume = tas2770_codec_resume, .controls = tas2770_snd_controls, @@ -790,6 +824,14 @@ static int tas2770_parse_dt(struct device *dev, struct tas2770_priv *tas2770) if (rc) tas2770->pdm_slot = -1; + tas2770->sdout_pd = fwnode_property_read_bool(dev->fwnode, "ti,sdout-pull-down"); + tas2770->sdout_zfill = fwnode_property_read_bool(dev->fwnode, "ti,sdout-zero-fill"); + + tas2770->sdz_reg = devm_regulator_get(dev, "SDZ"); + if (IS_ERR(tas2770->sdz_reg)) + return dev_err_probe(dev, PTR_ERR(tas2770->sdz_reg), + "Failed to get SDZ supply\n"); + tas2770->sdz_gpio = devm_gpiod_get_optional(dev, "shutdown", GPIOD_OUT_HIGH); if (IS_ERR(tas2770->sdz_gpio)) { if (PTR_ERR(tas2770->sdz_gpio) == -EPROBE_DEFER) diff --git a/sound/soc/codecs/tas2770.h b/sound/soc/codecs/tas2770.h index 3fd2e7003c50b6..b309d19c58e1da 100644 --- a/sound/soc/codecs/tas2770.h +++ b/sound/soc/codecs/tas2770.h @@ -67,6 +67,14 @@ #define TAS2770_TDM_CFG_REG3_RXS_SHIFT 0x4 #define TAS2770_TDM_CFG_REG3_30_MASK GENMASK(3, 0) #define TAS2770_TDM_CFG_REG3_30_SHIFT 0 + /* TDM Configuration Reg4 */ +#define TAS2770_TDM_CFG_REG4 TAS2770_REG(0X0, 0x0E) +#define TAS2770_TDM_CFG_REG4_TX_LSB_CFG BIT(7) +#define TAS2770_TDM_CFG_REG4_TX_KEEPER_CFG BIT(6) +#define TAS2770_TDM_CFG_REG4_TX_KEEPER BIT(5) +#define TAS2770_TDM_CFG_REG4_TX_FILL BIT(4) +#define TAS2770_TDM_CFG_REG4_TX_OFFSET_MASK GENMASK(3, 1) +#define TAS2770_TDM_CFG_REG4_TX_EDGE_FALLING BIT(0) /* TDM Configuration Reg5 */ #define TAS2770_TDM_CFG_REG5 TAS2770_REG(0X0, 0x0F) #define TAS2770_TDM_CFG_REG5_VSNS_MASK BIT(6) @@ -115,6 +123,9 @@ #define TAS2770_TEMP_LSB TAS2770_REG(0X0, 0x2A) /* Interrupt Configuration */ #define TAS2770_INT_CFG TAS2770_REG(0X0, 0x30) + /* Data In Pull-Down */ +#define TAS2770_DIN_PD TAS2770_REG(0X0, 0x31) +#define TAS2770_DIN_PD_SDOUT BIT(7) /* Misc IRQ */ #define TAS2770_MISC_IRQ TAS2770_REG(0X0, 0x32) /* Clock Configuration */ @@ -139,11 +150,14 @@ struct tas2770_priv { struct snd_soc_component *component; struct gpio_desc *reset_gpio; struct gpio_desc *sdz_gpio; + struct regulator *sdz_reg; struct regmap *regmap; struct device *dev; int v_sense_slot; int i_sense_slot; int pdm_slot; + bool sdout_pd; + bool sdout_zfill; bool dac_powered; bool unmuted; }; diff --git a/sound/soc/codecs/wcd934x.c b/sound/soc/codecs/wcd934x.c index c8db33f78a1b50..bc41a1466c70f4 100644 --- a/sound/soc/codecs/wcd934x.c +++ b/sound/soc/codecs/wcd934x.c @@ -2172,7 +2172,7 @@ static int wcd934x_init_dmic(struct snd_soc_component *comp) u32 def_dmic_rate, dmic_clk_drv; int ret; - ret = wcd_dt_parse_mbhc_data(comp->dev, &wcd->mbhc_cfg); + ret = wcd_dt_parse_micbias_info(&wcd->common); if (ret) return ret; diff --git a/sound/soc/codecs/wm8962.c b/sound/soc/codecs/wm8962.c index e9e317ce689826..bff86446741631 100644 --- a/sound/soc/codecs/wm8962.c +++ b/sound/soc/codecs/wm8962.c @@ -67,6 +67,8 @@ struct wm8962_priv { struct mutex dsp2_ena_lock; u16 dsp2_ena; + int mic_status; + struct delayed_work mic_work; struct snd_soc_jack *jack; @@ -1760,7 +1762,7 @@ SND_SOC_BYTES("EQR Coefficients", WM8962_EQ24, 18), SOC_SINGLE("3D Switch", WM8962_THREED1, 0, 1, 0), -SND_SOC_BYTES_MASK("3D Coefficients", WM8962_THREED1, 4, WM8962_THREED_ENA), +SND_SOC_BYTES_MASK("3D Coefficients", WM8962_THREED1, 4, WM8962_THREED_ENA | WM8962_ADC_MONOMIX), SOC_SINGLE("DF1 Switch", WM8962_DF1, 0, 1, 0), SND_SOC_BYTES_MASK("DF1 Coefficients", WM8962_DF1, 7, WM8962_DF1_ENA), @@ -3081,8 +3083,16 @@ static void wm8962_mic_work(struct work_struct *work) if (reg & WM8962_MICSHORT_STS) { status |= SND_JACK_BTN_0; irq_pol |= WM8962_MICSCD_IRQ_POL; + + /* Don't report a microphone if it's shorted right after + * plugging in, as this may be a TRS plug in a TRRS socket. + */ + if (!(wm8962->mic_status & WM8962_MICDET_STS)) + status = 0; } + wm8962->mic_status = status; + snd_soc_jack_report(wm8962->jack, status, SND_JACK_MICROPHONE | SND_JACK_BTN_0); diff --git a/sound/soc/fsl/fsl_easrc.c b/sound/soc/fsl/fsl_easrc.c index e64a0d97afd0cb..6c56134c60cc81 100644 --- a/sound/soc/fsl/fsl_easrc.c +++ b/sound/soc/fsl/fsl_easrc.c @@ -52,10 +52,13 @@ static int fsl_easrc_iec958_put_bits(struct snd_kcontrol *kcontrol, struct soc_mreg_control *mc = (struct soc_mreg_control *)kcontrol->private_value; unsigned int regval = ucontrol->value.integer.value[0]; + int ret; + + ret = (easrc_priv->bps_iec958[mc->regbase] != regval); easrc_priv->bps_iec958[mc->regbase] = regval; - return 0; + return ret; } static int fsl_easrc_iec958_get_bits(struct snd_kcontrol *kcontrol, @@ -93,14 +96,17 @@ static int fsl_easrc_set_reg(struct snd_kcontrol *kcontrol, struct snd_soc_component *component = snd_kcontrol_chip(kcontrol); struct soc_mreg_control *mc = (struct soc_mreg_control *)kcontrol->private_value; + struct fsl_asrc *easrc = snd_soc_component_get_drvdata(component); unsigned int regval = ucontrol->value.integer.value[0]; + bool changed; int ret; - ret = snd_soc_component_write(component, mc->regbase, regval); - if (ret < 0) + ret = regmap_update_bits_check(easrc->regmap, mc->regbase, + GENMASK(31, 0), regval, &changed); + if (ret != 0) return ret; - return 0; + return changed; } #define SOC_SINGLE_REG_RW(xname, xreg) \ diff --git a/sound/soc/fsl/fsl_xcvr.c b/sound/soc/fsl/fsl_xcvr.c index 5de93f458b569d..a268fb81a2f868 100644 --- a/sound/soc/fsl/fsl_xcvr.c +++ b/sound/soc/fsl/fsl_xcvr.c @@ -223,13 +223,10 @@ static int fsl_xcvr_mode_put(struct snd_kcontrol *kcontrol, xcvr->mode = snd_soc_enum_item_to_val(e, item[0]); - down_read(&card->snd_card->controls_rwsem); fsl_xcvr_activate_ctl(dai, fsl_xcvr_arc_mode_kctl.name, (xcvr->mode == FSL_XCVR_MODE_ARC)); fsl_xcvr_activate_ctl(dai, fsl_xcvr_earc_capds_kctl.name, (xcvr->mode == FSL_XCVR_MODE_EARC)); - up_read(&card->snd_card->controls_rwsem); - /* Allow playback for SPDIF only */ rtd = snd_soc_get_pcm_runtime(card, card->dai_link); rtd->pcm->streams[SNDRV_PCM_STREAM_PLAYBACK].substream_count = diff --git a/sound/soc/fsl/imx-card.c b/sound/soc/fsl/imx-card.c index 05b4e971a36618..a4518fefad6906 100644 --- a/sound/soc/fsl/imx-card.c +++ b/sound/soc/fsl/imx-card.c @@ -710,6 +710,8 @@ static int imx_card_parse_of(struct imx_card_data *data) link->ops = &imx_aif_ops; } + playback_only = false; + capture_only = false; graph_util_parse_link_direction(np, &playback_only, &capture_only); link->playback_only = playback_only; link->capture_only = capture_only; diff --git a/sound/soc/fsl/imx-rpmsg.c b/sound/soc/fsl/imx-rpmsg.c index 53f04d1f32806d..76a8e68c1b620c 100644 --- a/sound/soc/fsl/imx-rpmsg.c +++ b/sound/soc/fsl/imx-rpmsg.c @@ -145,7 +145,7 @@ static int imx_rpmsg_probe(struct platform_device *pdev) data->dai.ignore_pmdown_time = 1; data->dai.cpus->dai_name = pdev->dev.platform_data; - cpu_dai = snd_soc_find_dai(data->dai.cpus); + cpu_dai = snd_soc_find_dai_with_mutex(data->dai.cpus); if (!cpu_dai) { ret = -EPROBE_DEFER; goto fail; diff --git a/sound/soc/generic/simple-card-utils.c b/sound/soc/generic/simple-card-utils.c index bdc02e85b089fe..89d694c2cbddae 100644 --- a/sound/soc/generic/simple-card-utils.c +++ b/sound/soc/generic/simple-card-utils.c @@ -1038,11 +1038,15 @@ int graph_util_is_ports0(struct device_node *np) else port = np; - struct device_node *ports __free(device_node) = of_get_parent(port); - struct device_node *top __free(device_node) = of_get_parent(ports); - struct device_node *ports0 __free(device_node) = of_get_child_by_name(top, "ports"); + struct device_node *ports __free(device_node) = of_get_parent(port); + const char *at = strchr(kbasename(ports->full_name), '@'); - return ports0 == ports; + /* + * Since child iteration order may differ + * between a base DT and DT overlays, + * string match "ports" or "ports@0" in the node name instead. + */ + return !at || !strcmp(at, "@0"); } EXPORT_SYMBOL_GPL(graph_util_is_ports0); @@ -1179,9 +1183,9 @@ void graph_util_parse_link_direction(struct device_node *np, bool is_playback_only = of_property_read_bool(np, "playback-only"); bool is_capture_only = of_property_read_bool(np, "capture-only"); - if (np && playback_only) + if (playback_only && is_playback_only) *playback_only = is_playback_only; - if (np && capture_only) + if (capture_only && is_capture_only) *capture_only = is_capture_only; } EXPORT_SYMBOL_GPL(graph_util_parse_link_direction); diff --git a/sound/soc/intel/avs/board_selection.c b/sound/soc/intel/avs/board_selection.c index 52e6266a7cb86f..96dc637ccb20c8 100644 --- a/sound/soc/intel/avs/board_selection.c +++ b/sound/soc/intel/avs/board_selection.c @@ -520,7 +520,8 @@ static int avs_register_i2s_test_boards(struct avs_dev *adev) if (num_elems > max_ssps) { dev_err(adev->dev, "board supports only %d SSP, %d specified\n", max_ssps, num_elems); - return -EINVAL; + ret = -EINVAL; + goto exit; } for (ssp_port = 0; ssp_port < num_elems; ssp_port++) { @@ -528,11 +529,13 @@ static int avs_register_i2s_test_boards(struct avs_dev *adev) for_each_set_bit(tdm_slot, &tdm_slots, 16) { ret = avs_register_i2s_test_board(adev, ssp_port, tdm_slot); if (ret) - return ret; + goto exit; } } - return 0; +exit: + kfree(array); + return ret; } static int avs_register_i2s_board(struct avs_dev *adev, struct snd_soc_acpi_mach *mach) diff --git a/sound/soc/intel/boards/Kconfig b/sound/soc/intel/boards/Kconfig index c23fdb6aad4ca4..1031d6497f55e7 100644 --- a/sound/soc/intel/boards/Kconfig +++ b/sound/soc/intel/boards/Kconfig @@ -525,8 +525,6 @@ config SND_SOC_INTEL_SOUNDWIRE_SOF_MACH select SND_SOC_CS42L43_SDW select MFD_CS42L43 select MFD_CS42L43_SDW - select PINCTRL_CS42L43 - select SPI_CS42L43 select SND_SOC_CS35L56_SPI select SND_SOC_CS35L56_SDW select SND_SOC_DMIC diff --git a/sound/soc/intel/boards/sof_sdw.c b/sound/soc/intel/boards/sof_sdw.c index 50b838be24e95c..0186c281296ecb 100644 --- a/sound/soc/intel/boards/sof_sdw.c +++ b/sound/soc/intel/boards/sof_sdw.c @@ -763,6 +763,14 @@ static const struct dmi_system_id sof_sdw_quirk_table[] = { }, .driver_data = (void *)(SOC_SDW_CODEC_SPKR), }, + { + .callback = sof_sdw_quirk_cb, + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "Alienware"), + DMI_EXACT_MATCH(DMI_PRODUCT_SKU, "0CCD") + }, + .driver_data = (void *)(SOC_SDW_CODEC_SPKR), + }, /* Pantherlake devices*/ { .callback = sof_sdw_quirk_cb, diff --git a/sound/soc/intel/catpt/device.c b/sound/soc/intel/catpt/device.c index d13062c8e907cc..fcc7a7342a4abc 100644 --- a/sound/soc/intel/catpt/device.c +++ b/sound/soc/intel/catpt/device.c @@ -281,7 +281,15 @@ static int catpt_acpi_probe(struct platform_device *pdev) if (IS_ERR(cdev->pci_ba)) return PTR_ERR(cdev->pci_ba); - /* alloc buffer for storing DRAM context during dx transitions */ + /* + * As per design HOST is responsible for preserving firmware's runtime + * context during D0 -> D3 -> D0 transitions. Addresses used for DMA + * to/from HOST memory shall be outside the reserved range of 0xFFFxxxxx. + */ + ret = dma_coerce_mask_and_coherent(cdev->dev, DMA_BIT_MASK(31)); + if (ret) + return ret; + cdev->dxbuf_vaddr = dmam_alloc_coherent(dev, catpt_dram_size(cdev), &cdev->dxbuf_paddr, GFP_KERNEL); if (!cdev->dxbuf_vaddr) diff --git a/sound/soc/intel/catpt/dsp.c b/sound/soc/intel/catpt/dsp.c index 008a20a2acbda7..677f348909c8f1 100644 --- a/sound/soc/intel/catpt/dsp.c +++ b/sound/soc/intel/catpt/dsp.c @@ -125,9 +125,6 @@ int catpt_dmac_probe(struct catpt_dev *cdev) dmac->dev = cdev->dev; dmac->irq = cdev->irq; - ret = dma_coerce_mask_and_coherent(cdev->dev, DMA_BIT_MASK(31)); - if (ret) - return ret; /* * Caller is responsible for putting device in D0 to allow * for I/O and memory access before probing DW. diff --git a/sound/soc/intel/common/soc-acpi-intel-arl-match.c b/sound/soc/intel/common/soc-acpi-intel-arl-match.c index 6bf7a6250ddc32..c952f7d2b2c0e0 100644 --- a/sound/soc/intel/common/soc-acpi-intel-arl-match.c +++ b/sound/soc/intel/common/soc-acpi-intel-arl-match.c @@ -45,23 +45,22 @@ static const struct snd_soc_acpi_endpoint spk_3_endpoint = { .group_id = 1, }; -/* - * RT722 is a multi-function codec, three endpoints are created for - * its headset, amp and dmic functions. - */ -static const struct snd_soc_acpi_endpoint rt722_endpoints[] = { +static const struct snd_soc_acpi_endpoint jack_amp_g1_dmic_endpoints[] = { + /* Jack Endpoint */ { .num = 0, .aggregated = 0, .group_position = 0, .group_id = 0, }, + /* Amp Endpoint, work as spk_l_endpoint */ { .num = 1, - .aggregated = 0, + .aggregated = 1, .group_position = 0, - .group_id = 0, + .group_id = 1, }, + /* DMIC Endpoint */ { .num = 2, .aggregated = 0, @@ -229,11 +228,11 @@ static const struct snd_soc_acpi_adr_device rt711_sdca_0_adr[] = { } }; -static const struct snd_soc_acpi_adr_device rt722_0_single_adr[] = { +static const struct snd_soc_acpi_adr_device rt722_0_agg_adr[] = { { .adr = 0x000030025D072201ull, - .num_endpoints = ARRAY_SIZE(rt722_endpoints), - .endpoints = rt722_endpoints, + .num_endpoints = ARRAY_SIZE(jack_amp_g1_dmic_endpoints), + .endpoints = jack_amp_g1_dmic_endpoints, .name_prefix = "rt722" } }; @@ -394,8 +393,8 @@ static const struct snd_soc_acpi_link_adr arl_rt711_l0_rt1316_l3[] = { static const struct snd_soc_acpi_link_adr arl_rt722_l0_rt1320_l2[] = { { .mask = BIT(0), - .num_adr = ARRAY_SIZE(rt722_0_single_adr), - .adr_d = rt722_0_single_adr, + .num_adr = ARRAY_SIZE(rt722_0_agg_adr), + .adr_d = rt722_0_agg_adr, }, { .mask = BIT(2), diff --git a/sound/soc/intel/common/soc-acpi-intel-ptl-match.c b/sound/soc/intel/common/soc-acpi-intel-ptl-match.c index e297c8ecedb726..1055fb4838f612 100644 --- a/sound/soc/intel/common/soc-acpi-intel-ptl-match.c +++ b/sound/soc/intel/common/soc-acpi-intel-ptl-match.c @@ -383,6 +383,15 @@ static const struct snd_soc_acpi_link_adr ptl_rt721_l3[] = { {}, }; +static const struct snd_soc_acpi_adr_device rt722_0_agg_adr[] = { + { + .adr = 0x000030025d072201ull, + .num_endpoints = ARRAY_SIZE(jack_amp_g1_dmic_endpoints), + .endpoints = jack_amp_g1_dmic_endpoints, + .name_prefix = "rt722" + } +}; + static const struct snd_soc_acpi_adr_device rt722_0_single_adr[] = { { .adr = 0x000030025d072201ull, @@ -536,8 +545,8 @@ static const struct snd_soc_acpi_link_adr ptl_rt722_l3[] = { static const struct snd_soc_acpi_link_adr ptl_rt722_l0_rt1320_l23[] = { { .mask = BIT(0), - .num_adr = ARRAY_SIZE(rt722_0_single_adr), - .adr_d = rt722_0_single_adr, + .num_adr = ARRAY_SIZE(rt722_0_agg_adr), + .adr_d = rt722_0_agg_adr, }, { .mask = BIT(2), diff --git a/sound/soc/mediatek/mt8188/mt8188-mt6359.c b/sound/soc/mediatek/mt8188/mt8188-mt6359.c index 55ebac0c3cef2a..5a69378545b861 100644 --- a/sound/soc/mediatek/mt8188/mt8188-mt6359.c +++ b/sound/soc/mediatek/mt8188/mt8188-mt6359.c @@ -1280,7 +1280,7 @@ static struct snd_soc_dai_link mt8188_mt6359_dai_links[] = { }, }; -static void mt8188_fixup_controls(struct snd_soc_card *card) +static int mt8188_fixup_controls(struct snd_soc_card *card) { struct mtk_soc_card_data *soc_card_data = snd_soc_card_get_drvdata(card); struct mtk_platform_card_data *card_data = soc_card_data->card_data; @@ -1302,6 +1302,8 @@ static void mt8188_fixup_controls(struct snd_soc_card *card) else dev_warn(card->dev, "Cannot find ctl : Headphone Switch\n"); } + + return 0; } static struct snd_soc_card mt8188_mt6359_soc_card = { diff --git a/sound/soc/qcom/qdsp6/q6apm-dai.c b/sound/soc/qcom/qdsp6/q6apm-dai.c index aaeeadded7aace..7e9d0b393bf986 100644 --- a/sound/soc/qcom/qdsp6/q6apm-dai.c +++ b/sound/soc/qcom/qdsp6/q6apm-dai.c @@ -838,6 +838,7 @@ static const struct snd_soc_component_driver q6apm_fe_dai_component = { .ack = q6apm_dai_ack, .compress_ops = &q6apm_dai_compress_ops, .use_dai_pcm_id = true, + .remove_order = SND_SOC_COMP_ORDER_EARLY, }; static int q6apm_dai_probe(struct platform_device *pdev) diff --git a/sound/soc/qcom/qdsp6/q6apm-lpass-dais.c b/sound/soc/qcom/qdsp6/q6apm-lpass-dais.c index 528756f1332bcf..5be37eeea329fc 100644 --- a/sound/soc/qcom/qdsp6/q6apm-lpass-dais.c +++ b/sound/soc/qcom/qdsp6/q6apm-lpass-dais.c @@ -278,6 +278,7 @@ static const struct snd_soc_component_driver q6apm_lpass_dai_component = { .of_xlate_dai_name = q6dsp_audio_ports_of_xlate_dai_name, .be_pcm_base = AUDIOREACH_BE_PCM_BASE, .use_dai_pcm_id = true, + .remove_order = SND_SOC_COMP_ORDER_FIRST, }; static int q6apm_lpass_dai_dev_probe(struct platform_device *pdev) diff --git a/sound/soc/qcom/qdsp6/q6apm.c b/sound/soc/qcom/qdsp6/q6apm.c index 94cc6376a36762..e337a259bb5874 100644 --- a/sound/soc/qcom/qdsp6/q6apm.c +++ b/sound/soc/qcom/qdsp6/q6apm.c @@ -712,6 +712,7 @@ static const struct snd_soc_component_driver q6apm_audio_component = { .name = APM_AUDIO_DRV_NAME, .probe = q6apm_audio_probe, .remove = q6apm_audio_remove, + .remove_order = SND_SOC_COMP_ORDER_LAST, }; static int apm_probe(gpr_device_t *gdev) @@ -743,13 +744,22 @@ static int apm_probe(gpr_device_t *gdev) q6apm_get_apm_state(apm); - ret = devm_snd_soc_register_component(dev, &q6apm_audio_component, NULL, 0); + ret = snd_soc_register_component(dev, &q6apm_audio_component, NULL, 0); if (ret < 0) { dev_err(dev, "failed to register q6apm: %d\n", ret); return ret; } - return of_platform_populate(dev->of_node, NULL, NULL, dev); + ret = of_platform_populate(dev->of_node, NULL, NULL, dev); + if (ret) + snd_soc_unregister_component(dev); + + return ret; +} + +static void apm_remove(gpr_device_t *gdev) +{ + snd_soc_unregister_component(&gdev->dev); } struct audioreach_module *q6apm_find_module_by_mid(struct q6apm_graph *graph, uint32_t mid) @@ -816,6 +826,7 @@ MODULE_DEVICE_TABLE(of, apm_device_id); static gpr_driver_t apm_driver = { .probe = apm_probe, + .remove = apm_remove, .gpr_callback = apm_callback, .driver = { .name = "qcom-apm", diff --git a/sound/soc/qcom/qdsp6/q6asm.c b/sound/soc/qcom/qdsp6/q6asm.c index e7295b7b246105..3c4a24c9dba223 100644 --- a/sound/soc/qcom/qdsp6/q6asm.c +++ b/sound/soc/qcom/qdsp6/q6asm.c @@ -638,7 +638,6 @@ static int32_t q6asm_stream_callback(struct apr_device *adev, client_event = ASM_CLIENT_EVENT_CMD_OUT_FLUSH_DONE; break; case ASM_STREAM_CMD_OPEN_WRITE_V3: - case ASM_DATA_CMD_WRITE_V2: case ASM_STREAM_CMD_OPEN_READ_V3: case ASM_STREAM_CMD_OPEN_READWRITE_V2: case ASM_STREAM_CMD_SET_ENCDEC_PARAM: @@ -657,8 +656,9 @@ static int32_t q6asm_stream_callback(struct apr_device *adev, break; case ASM_DATA_CMD_EOS: case ASM_DATA_CMD_READ_V2: + case ASM_DATA_CMD_WRITE_V2: /* response as result of close stream */ - break; + goto done; default: dev_err(ac->dev, "command[0x%x] not expecting rsp\n", result->opcode); diff --git a/sound/soc/rockchip/rockchip_i2s_tdm.c b/sound/soc/rockchip/rockchip_i2s_tdm.c index 770b9bfbb384ae..fc52149ed6ae3e 100644 --- a/sound/soc/rockchip/rockchip_i2s_tdm.c +++ b/sound/soc/rockchip/rockchip_i2s_tdm.c @@ -22,6 +22,7 @@ #define DRV_NAME "rockchip-i2s-tdm" +#define DEFAULT_MCLK_FS 256 #define CH_GRP_MAX 4 /* The max channel 8 / 2 */ #define MULTIPLEX_CH_MAX 10 @@ -665,6 +666,15 @@ static int rockchip_i2s_tdm_hw_params(struct snd_pcm_substream *substream, mclk_rate = i2s_tdm->mclk_rx_freq; } + /* + * When the dai/component driver doesn't need to set mclk-fs for a specific + * clock, it can skip the call to set_sysclk() for that clock. + * In that case, simply use the clock rate from the params and multiply it by + * the default mclk-fs value. + */ + if (!mclk_rate) + mclk_rate = DEFAULT_MCLK_FS * params_rate(params); + err = clk_set_rate(mclk, mclk_rate); if (err) return err; diff --git a/sound/soc/samsung/i2s.c b/sound/soc/samsung/i2s.c index e9964f0e010aee..140907a41a70df 100644 --- a/sound/soc/samsung/i2s.c +++ b/sound/soc/samsung/i2s.c @@ -1360,10 +1360,10 @@ static int i2s_create_secondary_device(struct samsung_i2s_priv *priv) if (!pdev_sec) return -ENOMEM; - pdev_sec->driver_override = kstrdup("samsung-i2s", GFP_KERNEL); - if (!pdev_sec->driver_override) { + ret = device_set_driver_override(&pdev_sec->dev, "samsung-i2s"); + if (ret) { platform_device_put(pdev_sec); - return -ENOMEM; + return ret; } ret = platform_device_add(pdev_sec); diff --git a/sound/soc/sdca/Kconfig b/sound/soc/sdca/Kconfig index fabb69a3450d3b..87ab2895096c1d 100644 --- a/sound/soc/sdca/Kconfig +++ b/sound/soc/sdca/Kconfig @@ -46,12 +46,14 @@ config SND_SOC_SDCA_CLASS select SND_SOC_SDCA_FDL select SND_SOC_SDCA_HID select SND_SOC_SDCA_IRQ + select REGMAP_SOUNDWIRE help This option enables support for the SDCA Class driver which should support any class compliant SDCA part. config SND_SOC_SDCA_CLASS_FUNCTION tristate + select REGMAP_SOUNDWIRE_MBQ help This option enables support for the SDCA Class Function drivers, these implement the individual functions of the SDCA Class driver. diff --git a/sound/soc/sdca/Makefile b/sound/soc/sdca/Makefile index f6b73275d96493..b3b0f5d94c8de8 100644 --- a/sound/soc/sdca/Makefile +++ b/sound/soc/sdca/Makefile @@ -3,7 +3,7 @@ snd-soc-sdca-y := sdca_functions.o sdca_device.o sdca_function_device.o \ sdca_regmap.o sdca_asoc.o sdca_ump.o snd-soc-sdca-$(CONFIG_SND_SOC_SDCA_HID) += sdca_hid.o -snd-soc-sdca-$(CONFIG_SND_SOC_SDCA_IRQ) += sdca_interrupts.o +snd-soc-sdca-$(CONFIG_SND_SOC_SDCA_IRQ) += sdca_interrupts.o sdca_jack.o snd-soc-sdca-$(CONFIG_SND_SOC_SDCA_FDL) += sdca_fdl.o snd-soc-sdca-class-y := sdca_class.o diff --git a/sound/soc/sdca/sdca_asoc.c b/sound/soc/sdca/sdca_asoc.c index 2d328bbb95b945..9685281529e9f7 100644 --- a/sound/soc/sdca/sdca_asoc.c +++ b/sound/soc/sdca/sdca_asoc.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -792,6 +793,48 @@ static int control_limit_kctl(struct device *dev, return 0; } +static int volatile_get_volsw(struct snd_kcontrol *kcontrol, + struct snd_ctl_elem_value *ucontrol) +{ + struct snd_soc_component *component = snd_kcontrol_chip(kcontrol); + struct device *dev = component->dev; + int ret; + + ret = pm_runtime_resume_and_get(dev); + if (ret < 0) { + dev_err(dev, "failed to resume reading %s: %d\n", + kcontrol->id.name, ret); + return ret; + } + + ret = snd_soc_get_volsw(kcontrol, ucontrol); + + pm_runtime_put(dev); + + return ret; +} + +static int volatile_put_volsw(struct snd_kcontrol *kcontrol, + struct snd_ctl_elem_value *ucontrol) +{ + struct snd_soc_component *component = snd_kcontrol_chip(kcontrol); + struct device *dev = component->dev; + int ret; + + ret = pm_runtime_resume_and_get(dev); + if (ret < 0) { + dev_err(dev, "failed to resume writing %s: %d\n", + kcontrol->id.name, ret); + return ret; + } + + ret = snd_soc_put_volsw(kcontrol, ucontrol); + + pm_runtime_put(dev); + + return ret; +} + static int populate_control(struct device *dev, struct sdca_function_data *function, struct sdca_entity *entity, @@ -849,8 +892,13 @@ static int populate_control(struct device *dev, (*kctl)->private_value = (unsigned long)mc; (*kctl)->iface = SNDRV_CTL_ELEM_IFACE_MIXER; (*kctl)->info = snd_soc_info_volsw; - (*kctl)->get = snd_soc_get_volsw; - (*kctl)->put = snd_soc_put_volsw; + if (control->is_volatile) { + (*kctl)->get = volatile_get_volsw; + (*kctl)->put = volatile_put_volsw; + } else { + (*kctl)->get = snd_soc_get_volsw; + (*kctl)->put = snd_soc_put_volsw; + } if (readonly_control(control)) (*kctl)->access = SNDRV_CTL_ELEM_ACCESS_READ; @@ -1478,7 +1526,7 @@ static int set_usage(struct device *dev, struct regmap *regmap, unsigned int rate = sdca_range(range, SDCA_USAGE_SAMPLE_RATE, i); unsigned int width = sdca_range(range, SDCA_USAGE_SAMPLE_WIDTH, i); - if ((!rate || rate == target_rate) && width == target_width) { + if ((!rate || rate == target_rate) && (!width || width == target_width)) { unsigned int usage = sdca_range(range, SDCA_USAGE_NUMBER, i); unsigned int reg = SDW_SDCA_CTL(function->desc->adr, entity->id, sel, 0); diff --git a/sound/soc/sdca/sdca_class_function.c b/sound/soc/sdca/sdca_class_function.c index 0028482a1e7520..92600f419db43c 100644 --- a/sound/soc/sdca/sdca_class_function.c +++ b/sound/soc/sdca/sdca_class_function.c @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -195,8 +196,26 @@ static int class_function_component_probe(struct snd_soc_component *component) return sdca_irq_populate(drv->function, component, core->irq_info); } +static void class_function_component_remove(struct snd_soc_component *component) +{ + struct class_function_drv *drv = snd_soc_component_get_drvdata(component); + struct sdca_class_drv *core = drv->core; + + sdca_irq_cleanup(component->dev, drv->function, core->irq_info); +} + +static int class_function_set_jack(struct snd_soc_component *component, + struct snd_soc_jack *jack, void *d) +{ + struct class_function_drv *drv = snd_soc_component_get_drvdata(component); + struct sdca_class_drv *core = drv->core; + + return sdca_jack_set_jack(core->irq_info, jack); +} + static const struct snd_soc_component_driver class_function_component_drv = { .probe = class_function_component_probe, + .remove = class_function_component_remove, .endianness = 1, }; @@ -351,6 +370,9 @@ static int class_function_probe(struct auxiliary_device *auxdev, return dev_err_probe(dev, PTR_ERR(drv->regmap), "failed to create regmap"); + if (desc->type == SDCA_FUNCTION_TYPE_UAJ) + cmp_drv->set_jack = class_function_set_jack; + ret = sdca_asoc_populate_component(dev, drv->function, cmp_drv, &dais, &num_dais, &class_function_sdw_ops); @@ -380,6 +402,13 @@ static int class_function_probe(struct auxiliary_device *auxdev, return 0; } +static void class_function_remove(struct auxiliary_device *auxdev) +{ + struct class_function_drv *drv = auxiliary_get_drvdata(auxdev); + + sdca_irq_cleanup(drv->dev, drv->function, drv->core->irq_info); +} + static int class_function_runtime_suspend(struct device *dev) { struct auxiliary_device *auxdev = to_auxiliary_dev(dev); @@ -451,6 +480,7 @@ static struct auxiliary_driver class_function_drv = { }, .probe = class_function_probe, + .remove = class_function_remove, .id_table = class_function_id_table }; module_auxiliary_driver(class_function_drv); diff --git a/sound/soc/sdca/sdca_functions.c b/sound/soc/sdca/sdca_functions.c index 5a1f120487ef05..a1f6f931a80816 100644 --- a/sound/soc/sdca/sdca_functions.c +++ b/sound/soc/sdca/sdca_functions.c @@ -911,10 +911,6 @@ static int find_sdca_control_value(struct device *dev, struct sdca_entity *entit return 0; } -/* - * TODO: Add support for -cn- properties, allowing different channels to have - * different defaults etc. - */ static int find_sdca_entity_control(struct device *dev, struct sdca_entity *entity, struct fwnode_handle *control_node, struct sdca_control *control) @@ -1124,9 +1120,12 @@ static int find_sdca_entity_iot(struct device *dev, if (!terminal->is_dataport) { const char *type_name = sdca_find_terminal_name(terminal->type); - if (type_name) + if (type_name) { entity->label = devm_kasprintf(dev, GFP_KERNEL, "%s %s", entity->label, type_name); + if (!entity->label) + return -ENOMEM; + } } ret = fwnode_property_read_u32(entity_node, @@ -1569,10 +1568,19 @@ static int find_sdca_entities(struct device *dev, struct sdw_slave *sdw, static struct sdca_entity *find_sdca_entity_by_label(struct sdca_function_data *function, const char *entity_label) { + struct sdca_entity *entity = NULL; int i; for (i = 0; i < function->num_entities; i++) { - struct sdca_entity *entity = &function->entities[i]; + entity = &function->entities[i]; + + /* check whole string first*/ + if (!strcmp(entity->label, entity_label)) + return entity; + } + + for (i = 0; i < function->num_entities; i++) { + entity = &function->entities[i]; if (!strncmp(entity->label, entity_label, strlen(entity_label))) return entity; diff --git a/sound/soc/sdca/sdca_interrupts.c b/sound/soc/sdca/sdca_interrupts.c index 8f6a2adfb6fbea..76f50a0f6b0efc 100644 --- a/sound/soc/sdca/sdca_interrupts.c +++ b/sound/soc/sdca/sdca_interrupts.c @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -116,9 +117,7 @@ static irqreturn_t function_status_handler(int irq, void *data) status = val; for_each_set_bit(mask, &status, BITS_PER_BYTE) { - mask = 1 << mask; - - switch (mask) { + switch (BIT(mask)) { case SDCA_CTL_ENTITY_0_FUNCTION_NEEDS_INITIALIZATION: //FIXME: Add init writes break; @@ -155,14 +154,7 @@ static irqreturn_t detected_mode_handler(int irq, void *data) { struct sdca_interrupt *interrupt = data; struct device *dev = interrupt->dev; - struct snd_soc_component *component = interrupt->component; - struct snd_soc_card *card = component->card; - struct rw_semaphore *rwsem = &card->snd_card->controls_rwsem; - struct snd_kcontrol *kctl = interrupt->priv; - struct snd_ctl_elem_value *ucontrol __free(kfree) = NULL; - struct soc_enum *soc_enum; irqreturn_t irqret = IRQ_NONE; - unsigned int reg, val; int ret; ret = pm_runtime_get_sync(dev); @@ -171,76 +163,9 @@ static irqreturn_t detected_mode_handler(int irq, void *data) goto error; } - if (!kctl) { - const char *name __free(kfree) = kasprintf(GFP_KERNEL, "%s %s", - interrupt->entity->label, - SDCA_CTL_SELECTED_MODE_NAME); - - if (!name) - goto error; - - kctl = snd_soc_component_get_kcontrol(component, name); - if (!kctl) { - dev_dbg(dev, "control not found: %s\n", name); - goto error; - } - - interrupt->priv = kctl; - } - - soc_enum = (struct soc_enum *)kctl->private_value; - - reg = SDW_SDCA_CTL(interrupt->function->desc->adr, interrupt->entity->id, - interrupt->control->sel, 0); - - ret = regmap_read(interrupt->function_regmap, reg, &val); - if (ret < 0) { - dev_err(dev, "failed to read detected mode: %d\n", ret); - goto error; - } - - switch (val) { - case SDCA_DETECTED_MODE_DETECTION_IN_PROGRESS: - case SDCA_DETECTED_MODE_JACK_UNKNOWN: - reg = SDW_SDCA_CTL(interrupt->function->desc->adr, - interrupt->entity->id, - SDCA_CTL_GE_SELECTED_MODE, 0); - - /* - * Selected mode is not normally marked as volatile register - * (RW), but here force a read from the hardware. If the - * detected mode is unknown we need to see what the device - * selected as a "safe" option. - */ - regcache_drop_region(interrupt->function_regmap, reg, reg); - - ret = regmap_read(interrupt->function_regmap, reg, &val); - if (ret) { - dev_err(dev, "failed to re-check selected mode: %d\n", ret); - goto error; - } - break; - default: - break; - } - - dev_dbg(dev, "%s: %#x\n", interrupt->name, val); - - ucontrol = kzalloc(sizeof(*ucontrol), GFP_KERNEL); - if (!ucontrol) - goto error; - - ucontrol->value.enumerated.item[0] = snd_soc_enum_val_to_item(soc_enum, val); - - down_write(rwsem); - ret = kctl->put(kctl, ucontrol); - up_write(rwsem); - if (ret < 0) { - dev_err(dev, "failed to update selected mode: %d\n", ret); + ret = sdca_jack_process(interrupt); + if (ret) goto error; - } - - snd_ctl_notify(card->snd_card, SNDRV_CTL_EVENT_MASK_VALUE, &kctl->id); irqret = IRQ_HANDLED; error: @@ -306,8 +231,7 @@ static int sdca_irq_request_locked(struct device *dev, if (irq < 0) return irq; - ret = devm_request_threaded_irq(dev, irq, NULL, handler, - IRQF_ONESHOT, name, data); + ret = request_threaded_irq(irq, NULL, handler, IRQF_ONESHOT, name, data); if (ret) return ret; @@ -318,10 +242,26 @@ static int sdca_irq_request_locked(struct device *dev, return 0; } +static void sdca_irq_free_locked(struct device *dev, struct sdca_interrupt_info *info, + int sdca_irq, const char *name, void *data) +{ + int irq; + + irq = regmap_irq_get_virq(info->irq_data, sdca_irq); + if (irq < 0) + return; + + free_irq(irq, data); + + info->irqs[sdca_irq].irq = 0; + + dev_dbg(dev, "freed irq %d for %s\n", irq, name); +} + /** - * sdca_request_irq - request an individual SDCA interrupt + * sdca_irq_request - request an individual SDCA interrupt * @dev: Pointer to the struct device against which things should be allocated. - * @interrupt_info: Pointer to the interrupt information structure. + * @info: Pointer to the interrupt information structure. * @sdca_irq: SDCA interrupt position. * @name: Name to be given to the IRQ. * @handler: A callback thread function to be called for the IRQ. @@ -356,6 +296,30 @@ int sdca_irq_request(struct device *dev, struct sdca_interrupt_info *info, } EXPORT_SYMBOL_NS_GPL(sdca_irq_request, "SND_SOC_SDCA"); +/** + * sdca_irq_free - free an individual SDCA interrupt + * @dev: Pointer to the struct device. + * @info: Pointer to the interrupt information structure. + * @sdca_irq: SDCA interrupt position. + * @name: Name to be given to the IRQ. + * @data: Private data pointer that will be passed to the handler. + * + * Typically this is handled internally by sdca_irq_cleanup, however if + * a device requires custom IRQ handling this can be called manually before + * calling sdca_irq_cleanup, which will then skip that IRQ whilst processing. + */ +void sdca_irq_free(struct device *dev, struct sdca_interrupt_info *info, + int sdca_irq, const char *name, void *data) +{ + if (sdca_irq < 0 || sdca_irq >= SDCA_MAX_INTERRUPTS) + return; + + guard(mutex)(&info->irq_lock); + + sdca_irq_free_locked(dev, info, sdca_irq, name, data); +} +EXPORT_SYMBOL_NS_GPL(sdca_irq_free, "SND_SOC_SDCA"); + /** * sdca_irq_data_populate - Populate common interrupt data * @dev: Pointer to the Function device. @@ -382,8 +346,8 @@ int sdca_irq_data_populate(struct device *dev, struct regmap *regmap, if (!dev) return -ENODEV; - name = devm_kasprintf(dev, GFP_KERNEL, "%s %s %s", function->desc->name, - entity->label, control->label); + name = kasprintf(GFP_KERNEL, "%s %s %s", function->desc->name, + entity->label, control->label); if (!name) return -ENOMEM; @@ -536,6 +500,10 @@ int sdca_irq_populate(struct sdca_function_data *function, handler = function_status_handler; break; case SDCA_CTL_TYPE_S(GE, DETECTED_MODE): + ret = sdca_jack_alloc_state(interrupt); + if (ret) + return ret; + handler = detected_mode_handler; break; case SDCA_CTL_TYPE_S(XU, FDL_CURRENTOWNER): @@ -566,6 +534,35 @@ int sdca_irq_populate(struct sdca_function_data *function, } EXPORT_SYMBOL_NS_GPL(sdca_irq_populate, "SND_SOC_SDCA"); +/** + * sdca_irq_cleanup - Free all the individual IRQs for an SDCA Function + * @sdev: Device pointer against which the sdca_interrupt_info was allocated. + * @function: Pointer to the SDCA Function. + * @info: Pointer to the SDCA interrupt info for this device. + * + * Typically this would be called from the driver for a single SDCA Function. + */ +void sdca_irq_cleanup(struct device *dev, + struct sdca_function_data *function, + struct sdca_interrupt_info *info) +{ + int i; + + guard(mutex)(&info->irq_lock); + + for (i = 0; i < SDCA_MAX_INTERRUPTS; i++) { + struct sdca_interrupt *interrupt = &info->irqs[i]; + + if (interrupt->function != function || !interrupt->irq) + continue; + + sdca_irq_free_locked(dev, info, i, interrupt->name, interrupt); + + kfree(interrupt->name); + } +} +EXPORT_SYMBOL_NS_GPL(sdca_irq_cleanup, "SND_SOC_SDCA"); + /** * sdca_irq_allocate - allocate an SDCA interrupt structure for a device * @sdev: Device pointer against which things should be allocated. diff --git a/sound/soc/sdca/sdca_jack.c b/sound/soc/sdca/sdca_jack.c new file mode 100644 index 00000000000000..bfa621b744e1a2 --- /dev/null +++ b/sound/soc/sdca/sdca_jack.c @@ -0,0 +1,248 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (C) 2025 Cirrus Logic, Inc. and +// Cirrus Logic International Semiconductor Ltd. + +/* + * The MIPI SDCA specification is available for public downloads at + * https://www.mipi.org/mipi-sdca-v1-0-download + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/** + * sdca_jack_process - Process an SDCA jack event + * @interrupt: SDCA interrupt structure + * + * Return: Zero on success or a negative error code. + */ +int sdca_jack_process(struct sdca_interrupt *interrupt) +{ + struct device *dev = interrupt->dev; + struct snd_soc_component *component = interrupt->component; + struct snd_soc_card *card = component->card; + struct rw_semaphore *rwsem = &card->snd_card->controls_rwsem; + struct jack_state *state = interrupt->priv; + struct snd_kcontrol *kctl = state->kctl; + struct snd_ctl_elem_value *ucontrol __free(kfree) = NULL; + unsigned int reg, val; + int ret; + + guard(rwsem_write)(rwsem); + + if (!kctl) { + const char *name __free(kfree) = kasprintf(GFP_KERNEL, "%s %s", + interrupt->entity->label, + SDCA_CTL_SELECTED_MODE_NAME); + + if (!name) + return -ENOMEM; + + kctl = snd_soc_component_get_kcontrol(component, name); + if (!kctl) + dev_dbg(dev, "control not found: %s\n", name); + else + state->kctl = kctl; + } + + reg = SDW_SDCA_CTL(interrupt->function->desc->adr, interrupt->entity->id, + interrupt->control->sel, 0); + + ret = regmap_read(interrupt->function_regmap, reg, &val); + if (ret < 0) { + dev_err(dev, "failed to read detected mode: %d\n", ret); + return ret; + } + + reg = SDW_SDCA_CTL(interrupt->function->desc->adr, interrupt->entity->id, + SDCA_CTL_GE_SELECTED_MODE, 0); + + switch (val) { + case SDCA_DETECTED_MODE_DETECTION_IN_PROGRESS: + case SDCA_DETECTED_MODE_JACK_UNKNOWN: + /* + * Selected mode is not normally marked as volatile register + * (RW), but here force a read from the hardware. If the + * detected mode is unknown we need to see what the device + * selected as a "safe" option. + */ + regcache_drop_region(interrupt->function_regmap, reg, reg); + + ret = regmap_read(interrupt->function_regmap, reg, &val); + if (ret) { + dev_err(dev, "failed to re-check selected mode: %d\n", ret); + return ret; + } + break; + default: + break; + } + + dev_dbg(dev, "%s: %#x\n", interrupt->name, val); + + if (kctl) { + struct soc_enum *soc_enum = (struct soc_enum *)kctl->private_value; + + ucontrol = kzalloc(sizeof(*ucontrol), GFP_KERNEL); + if (!ucontrol) + return -ENOMEM; + + ucontrol->value.enumerated.item[0] = snd_soc_enum_val_to_item(soc_enum, val); + + ret = kctl->put(kctl, ucontrol); + if (ret < 0) { + dev_err(dev, "failed to update selected mode: %d\n", ret); + return ret; + } + + snd_ctl_notify(card->snd_card, SNDRV_CTL_EVENT_MASK_VALUE, &kctl->id); + } else { + ret = regmap_write(interrupt->function_regmap, reg, val); + if (ret) { + dev_err(dev, "failed to write selected mode: %d\n", ret); + return ret; + } + } + + return sdca_jack_report(interrupt); +} +EXPORT_SYMBOL_NS_GPL(sdca_jack_process, "SND_SOC_SDCA"); + +/** + * sdca_jack_alloc_state - allocate state for a jack interrupt + * @interrupt: SDCA interrupt structure. + * + * Return: Zero on success or a negative error code. + */ +int sdca_jack_alloc_state(struct sdca_interrupt *interrupt) +{ + struct device *dev = interrupt->dev; + struct jack_state *jack_state; + + jack_state = devm_kzalloc(dev, sizeof(*jack_state), GFP_KERNEL); + if (!jack_state) + return -ENOMEM; + + interrupt->priv = jack_state; + + return 0; +} +EXPORT_SYMBOL_NS_GPL(sdca_jack_alloc_state, "SND_SOC_SDCA"); + +/** + * sdca_jack_set_jack - attach an ASoC jack to SDCA + * @info: SDCA interrupt information. + * @jack: ASoC jack to be attached. + * + * Return: Zero on success or a negative error code. + */ +int sdca_jack_set_jack(struct sdca_interrupt_info *info, struct snd_soc_jack *jack) +{ + int i, ret; + + guard(mutex)(&info->irq_lock); + + for (i = 0; i < SDCA_MAX_INTERRUPTS; i++) { + struct sdca_interrupt *interrupt = &info->irqs[i]; + struct sdca_control *control = interrupt->control; + struct sdca_entity *entity = interrupt->entity; + struct jack_state *jack_state; + + if (!interrupt->irq) + continue; + + switch (SDCA_CTL_TYPE(entity->type, control->sel)) { + case SDCA_CTL_TYPE_S(GE, DETECTED_MODE): + jack_state = interrupt->priv; + jack_state->jack = jack; + + /* Report initial state in case IRQ was already handled */ + ret = sdca_jack_report(interrupt); + if (ret) + return ret; + break; + default: + break; + } + } + + return 0; +} +EXPORT_SYMBOL_NS_GPL(sdca_jack_set_jack, "SND_SOC_SDCA"); + +int sdca_jack_report(struct sdca_interrupt *interrupt) +{ + struct jack_state *jack_state = interrupt->priv; + struct sdca_control_range *range; + enum sdca_terminal_type type; + unsigned int report = 0; + unsigned int reg, val; + int ret; + + reg = SDW_SDCA_CTL(interrupt->function->desc->adr, interrupt->entity->id, + SDCA_CTL_GE_SELECTED_MODE, 0); + + ret = regmap_read(interrupt->function_regmap, reg, &val); + if (ret) { + dev_err(interrupt->dev, "failed to read selected mode: %d\n", ret); + return ret; + } + + range = sdca_selector_find_range(interrupt->dev, interrupt->entity, + SDCA_CTL_GE_SELECTED_MODE, + SDCA_SELECTED_MODE_NCOLS, 0); + if (!range) + return -EINVAL; + + type = sdca_range_search(range, SDCA_SELECTED_MODE_INDEX, + val, SDCA_SELECTED_MODE_TERM_TYPE); + + switch (type) { + case SDCA_TERM_TYPE_LINEIN_STEREO: + case SDCA_TERM_TYPE_LINEIN_FRONT_LR: + case SDCA_TERM_TYPE_LINEIN_CENTER_LFE: + case SDCA_TERM_TYPE_LINEIN_SURROUND_LR: + case SDCA_TERM_TYPE_LINEIN_REAR_LR: + report = SND_JACK_LINEIN; + break; + case SDCA_TERM_TYPE_LINEOUT_STEREO: + case SDCA_TERM_TYPE_LINEOUT_FRONT_LR: + case SDCA_TERM_TYPE_LINEOUT_CENTER_LFE: + case SDCA_TERM_TYPE_LINEOUT_SURROUND_LR: + case SDCA_TERM_TYPE_LINEOUT_REAR_LR: + report = SND_JACK_LINEOUT; + break; + case SDCA_TERM_TYPE_MIC_JACK: + report = SND_JACK_MICROPHONE; + break; + case SDCA_TERM_TYPE_HEADPHONE_JACK: + report = SND_JACK_HEADPHONE; + break; + case SDCA_TERM_TYPE_HEADSET_JACK: + report = SND_JACK_HEADSET; + break; + default: + break; + } + + snd_soc_jack_report(jack_state->jack, report, 0xFFFF); + + return 0; +} +EXPORT_SYMBOL_NS_GPL(sdca_jack_report, "SND_SOC_SDCA"); diff --git a/sound/soc/sdw_utils/soc_sdw_utils.c b/sound/soc/sdw_utils/soc_sdw_utils.c index ccf149f949e8f6..d03072cd13cb9f 100644 --- a/sound/soc/sdw_utils/soc_sdw_utils.c +++ b/sound/soc/sdw_utils/soc_sdw_utils.c @@ -1421,29 +1421,14 @@ static int is_sdca_endpoint_present(struct device *dev, const struct snd_soc_acpi_adr_device *adr_dev = &adr_link->adr_d[adr_index]; const struct snd_soc_acpi_endpoint *adr_end; const struct asoc_sdw_dai_info *dai_info; - struct snd_soc_dai_link_component *dlc; - struct snd_soc_dai *codec_dai; struct sdw_slave *slave; struct device *sdw_dev; const char *sdw_codec_name; int ret, i; - dlc = kzalloc(sizeof(*dlc), GFP_KERNEL); - if (!dlc) - return -ENOMEM; - adr_end = &adr_dev->endpoints[end_index]; dai_info = &codec_info->dais[adr_end->num]; - dlc->dai_name = dai_info->dai_name; - codec_dai = snd_soc_find_dai_with_mutex(dlc); - if (!codec_dai) { - dev_warn(dev, "codec dai %s not registered yet\n", dlc->dai_name); - kfree(dlc); - return -EPROBE_DEFER; - } - kfree(dlc); - sdw_codec_name = _asoc_sdw_get_codec_name(dev, adr_link, adr_index); if (!sdw_codec_name) return -ENOMEM; diff --git a/sound/soc/soc-card.c b/sound/soc/soc-card.c index 235427d6906173..bc02c7b864e295 100644 --- a/sound/soc/soc-card.c +++ b/sound/soc/soc-card.c @@ -184,10 +184,16 @@ int snd_soc_card_late_probe(struct snd_soc_card *card) return 0; } -void snd_soc_card_fixup_controls(struct snd_soc_card *card) +int snd_soc_card_fixup_controls(struct snd_soc_card *card) { - if (card->fixup_controls) - card->fixup_controls(card); + if (card->fixup_controls) { + int ret = card->fixup_controls(card); + + if (ret < 0) + return soc_card_ret(card, ret); + } + + return 0; } int snd_soc_card_remove(struct snd_soc_card *card) diff --git a/sound/soc/soc-core.c b/sound/soc/soc-core.c index e4b21bf39e59f6..6fec5e080fffba 100644 --- a/sound/soc/soc-core.c +++ b/sound/soc/soc-core.c @@ -462,8 +462,7 @@ static void soc_free_pcm_runtime(struct snd_soc_pcm_runtime *rtd) list_del(&rtd->list); - if (delayed_work_pending(&rtd->delayed_work)) - flush_delayed_work(&rtd->delayed_work); + flush_delayed_work(&rtd->delayed_work); snd_soc_pcm_component_free(rtd); /* @@ -1864,12 +1863,15 @@ static void cleanup_dmi_name(char *name) /* * Check if a DMI field is valid, i.e. not containing any string - * in the black list. + * in the black list and not the empty string. */ static int is_dmi_valid(const char *field) { int i = 0; + if (!field[0]) + return 0; + while (dmi_blacklist[i]) { if (strstr(field, dmi_blacklist[i])) return 0; @@ -2122,6 +2124,9 @@ static void soc_cleanup_card_resources(struct snd_soc_card *card) for_each_card_rtds(card, rtd) if (rtd->initialized) snd_soc_link_exit(rtd); + /* flush delayed work before removing DAIs and DAPM widgets */ + snd_soc_flush_all_delayed_work(card); + /* remove and free each DAI */ soc_remove_link_dais(card); soc_remove_link_components(card); @@ -2288,7 +2293,10 @@ static int snd_soc_bind_card(struct snd_soc_card *card) goto probe_end; snd_soc_dapm_new_widgets(card); - snd_soc_card_fixup_controls(card); + + ret = snd_soc_card_fixup_controls(card); + if (ret < 0) + goto probe_end; ret = snd_card_register(card->snd_card); if (ret < 0) { @@ -2844,6 +2852,7 @@ int snd_soc_component_initialize(struct snd_soc_component *component, INIT_LIST_HEAD(&component->dobj_list); INIT_LIST_HEAD(&component->card_list); INIT_LIST_HEAD(&component->list); + INIT_LIST_HEAD(&component->card_aux_list); mutex_init(&component->io_mutex); if (!component->name) { diff --git a/sound/soc/soc-dapm.c b/sound/soc/soc-dapm.c index 4d920a59da3c49..15de858c5533ee 100644 --- a/sound/soc/soc-dapm.c +++ b/sound/soc/soc-dapm.c @@ -2526,6 +2526,141 @@ static const struct file_operations dapm_bias_fops = { .llseek = default_llseek, }; +static ssize_t dapm_graph_read_file(struct file *file, char __user *user_buf, + size_t count, loff_t *ppos) +{ + struct snd_soc_card *card = file->private_data; + struct snd_soc_dapm_context *dapm; + struct snd_soc_dapm_path *p; + struct snd_soc_dapm_widget *w; + struct snd_soc_pcm_runtime *rtd; + struct snd_soc_dapm_widget *wdone[16]; + struct snd_soc_dai *dai; + int i, num_wdone = 0, cluster = 0; + char *buf; + ssize_t bufsize; + ssize_t ret = 0; + + bufsize = 1024 * card->num_dapm_widgets; + buf = kmalloc(bufsize, GFP_KERNEL); + if (!buf) + return -ENOMEM; + + mutex_lock(&card->dapm_mutex); + +#define bufprintf(...) \ + ret += scnprintf(buf + ret, bufsize - ret, __VA_ARGS__) + + bufprintf("digraph dapm {\n"); + + /* + * Print the user-visible devices of the card. + */ + bufprintf("subgraph cluster_%d {\n", cluster++); + bufprintf("label=\"Devices\";style=filled;fillcolor=gray;\n"); + for_each_card_rtds(card, rtd) { + if (rtd->dai_link->no_pcm) + continue; + + bufprintf("w%pK [label=\"%d: %s\"];\n", rtd, + rtd->pcm->device, rtd->dai_link->name); + } + bufprintf("};\n"); + + /* + * Print the playback/capture widgets of DAIs just next to + * the user-visible devices. Keep the list of already printed + * widgets in 'wdone', so they will be skipped later. + */ + for_each_card_rtds(card, rtd) { + for_each_rtd_cpu_dais(rtd, i, dai) { + if (dai->stream[SNDRV_PCM_STREAM_PLAYBACK].widget) { + w = dai->stream[SNDRV_PCM_STREAM_PLAYBACK].widget; + bufprintf("w%pK [label=\"%s\"];\n", w, w->name); + if (!rtd->dai_link->no_pcm) + bufprintf("w%pK -> w%pK;\n", rtd, w); + if (num_wdone < ARRAY_SIZE(wdone)) { + wdone[num_wdone] = w; + num_wdone++; + } + } + + if (dai->stream[SNDRV_PCM_STREAM_CAPTURE].widget) { + w = dai->stream[SNDRV_PCM_STREAM_CAPTURE].widget; + bufprintf("w%pK [label=\"%s\"];\n", w, w->name); + if (!rtd->dai_link->no_pcm) + bufprintf("w%pK -> w%pK;\n", w, rtd); + if (num_wdone < ARRAY_SIZE(wdone)) { + wdone[num_wdone] = w; + num_wdone++; + } + } + } + } + + for_each_card_dapms(card, dapm) { + const char *prefix = soc_dapm_prefix(dapm); + + if (dapm != &card->dapm) { + bufprintf("subgraph cluster_%d {\n", cluster++); + if (prefix) + bufprintf("label=\"%s\";\n", prefix); + else if (dapm->component) + bufprintf("label=\"%s\";\n", + dapm->component->name); + } + + for_each_card_widgets(dapm->card, w) { + const char *name = w->name; + bool skip = false; + + if (w->dapm != dapm) + continue; + + if (list_empty(&w->edges[0]) && list_empty(&w->edges[1])) + continue; + + for (i = 0; i < num_wdone; i++) + if (wdone[i] == w) + skip = true; + if (skip) + continue; + + if (prefix && strlen(name) > strlen(prefix) + 1) + name += strlen(prefix) + 1; + + bufprintf("w%pK [label=\"%s\"];\n", w, name); + } + + if (dapm != &card->dapm) + bufprintf("}\n"); + } + + list_for_each_entry(p, &card->paths, list) { + if (p->name) + bufprintf("w%pK -> w%pK [label=\"%s\"];\n", + p->source, p->sink, p->name); + else + bufprintf("w%pK -> w%pK;\n", p->source, p->sink); + } + + bufprintf("}\n"); +#undef bufprintf + + mutex_unlock(&card->dapm_mutex); + + ret = simple_read_from_buffer(user_buf, count, ppos, buf, ret); + + kfree(buf); + return ret; +} + +static const struct file_operations dapm_graph_fops = { + .open = simple_open, + .read = dapm_graph_read_file, + .llseek = default_llseek, +}; + void snd_soc_dapm_debugfs_init(struct snd_soc_dapm_context *dapm, struct dentry *parent) { @@ -2536,6 +2671,10 @@ void snd_soc_dapm_debugfs_init(struct snd_soc_dapm_context *dapm, debugfs_create_file("bias_level", 0444, dapm->debugfs_dapm, dapm, &dapm_bias_fops); + + if (dapm == &dapm->card->dapm) + debugfs_create_file("graph.dot", 0444, dapm->debugfs_dapm, + dapm->card, &dapm_graph_fops); } static void dapm_debugfs_add_widget(struct snd_soc_dapm_widget *w) diff --git a/sound/soc/soc-ops.c b/sound/soc/soc-ops.c index ba42939d5f013c..15d53c1345fead 100644 --- a/sound/soc/soc-ops.c +++ b/sound/soc/soc-ops.c @@ -445,6 +445,30 @@ int snd_soc_put_volsw_sx(struct snd_kcontrol *kcontrol, } EXPORT_SYMBOL_GPL(snd_soc_put_volsw_sx); +bool snd_soc_control_matches(struct snd_kcontrol *kctl, + const char *pattern) +{ + const char *name = kctl->id.name; + + if (pattern[0] == '*') { + int namelen; + int patternlen; + + pattern++; + if (pattern[0] == ' ') + pattern++; + + namelen = strlen(name); + patternlen = strlen(pattern); + + if (namelen > patternlen) + name += namelen - patternlen; + } + + return !strcmp(name, pattern); +} +EXPORT_SYMBOL_GPL(snd_soc_control_matches); + static int snd_soc_clip_to_platform_max(struct snd_kcontrol *kctl) { struct soc_mixer_control *mc = (struct soc_mixer_control *)kctl->private_value; @@ -476,39 +500,165 @@ static int snd_soc_clip_to_platform_max(struct snd_kcontrol *kctl) return ret; } +static int soc_limit_volume(struct snd_kcontrol *kctl, int max) +{ + struct soc_mixer_control *mc = (struct soc_mixer_control *)kctl->private_value; + + if (max <= 0 || max > mc->max - mc->min) + return -EINVAL; + mc->platform_max = max; + return snd_soc_clip_to_platform_max(kctl); +} + /** - * snd_soc_limit_volume - Set new limit to an existing volume control. + * snd_soc_limit_volume - Set new limit to existing volume controls * * @card: where to look for the control - * @name: Name of the control + * @name: name pattern * @max: new maximum limit + * + * Finds controls matching the given name (which can be either a name + * verbatim, or a pattern starting with the wildcard '*') and sets + * a platform volume limit on them. * - * Return 0 for success, else error. + * Return number of matching controls on success, else error. At least + * one control needs to match the pattern. */ int snd_soc_limit_volume(struct snd_soc_card *card, const char *name, int max) { struct snd_kcontrol *kctl; - int ret = -EINVAL; + int hits = 0; + int ret; - /* Sanity check for name and max */ - if (unlikely(!name || max <= 0)) + /* Sanity check for name */ + if (unlikely(!name)) return -EINVAL; - kctl = snd_soc_card_get_kcontrol(card, name); - if (kctl) { - struct soc_mixer_control *mc = - (struct soc_mixer_control *)kctl->private_value; + list_for_each_entry(kctl, &card->snd_card->controls, list) { + if (!snd_soc_control_matches(kctl, name)) + continue; - if (max <= mc->max - mc->min) { - mc->platform_max = max; - ret = snd_soc_clip_to_platform_max(kctl); - } + ret = soc_limit_volume(kctl, max); + if (ret < 0) + return ret; + hits++; } - return ret; + if (!hits) + return -EINVAL; + + return hits; } EXPORT_SYMBOL_GPL(snd_soc_limit_volume); +/** + * snd_soc_deactivate_kctl - Activate/deactive controls matching a pattern + * + * @card: where to look for the controls + * @name: name pattern + * @active: non-zero to activate, zero to deactivate + * + * Return number of matching controls on success, else error. + * No controls need to match. + */ +int snd_soc_deactivate_kctl(struct snd_soc_card *card, + const char *name, int active) +{ + struct snd_kcontrol *kctl; + int hits = 0; + int ret; + + /* Sanity check for name */ + if (unlikely(!name)) + return -EINVAL; + + list_for_each_entry(kctl, &card->snd_card->controls, list) { + if (!snd_soc_control_matches(kctl, name)) + continue; + + ret = snd_ctl_activate_id(card->snd_card, &kctl->id, active); + if (ret < 0) + return ret; + hits++; + } + + if (!hits) + return -EINVAL; + + return hits; +} +EXPORT_SYMBOL_GPL(snd_soc_deactivate_kctl); + +static int soc_set_enum_kctl(struct snd_kcontrol *kctl, const char *strval) +{ + struct snd_ctl_elem_value value; + struct snd_ctl_elem_info info; + int sel, i, ret; + + ret = kctl->info(kctl, &info); + if (ret < 0) + return ret; + + if (info.type != SNDRV_CTL_ELEM_TYPE_ENUMERATED) + return -EINVAL; + + for (sel = 0; sel < info.value.enumerated.items; sel++) { + info.value.enumerated.item = sel; + ret = kctl->info(kctl, &info); + if (ret < 0) + return ret; + + if (!strcmp(strval, info.value.enumerated.name)) + break; + } + + if (sel == info.value.enumerated.items) + return -EINVAL; + + for (i = 0; i < info.count; i++) + value.value.enumerated.item[i] = sel; + + return kctl->put(kctl, &value); +} + +/** + * snd_soc_set_enum_kctl - Set enumerated controls matching a pattern + * + * @card: where to look for the controls + * @name: name pattern + * @value: string value to set the controls to + * + * Return number of matching and set controls on success, else error. + * No controls need to match. + */ +int snd_soc_set_enum_kctl(struct snd_soc_card *card, + const char *name, const char *value) +{ + struct snd_kcontrol *kctl; + int hits = 0; + int ret; + + /* Sanity check for name */ + if (unlikely(!name)) + return -EINVAL; + + list_for_each_entry(kctl, &card->snd_card->controls, list) { + if (!snd_soc_control_matches(kctl, name)) + continue; + + ret = soc_set_enum_kctl(kctl, value); + if (ret < 0) + return ret; + hits++; + } + + if (!hits) + return -EINVAL; + + return hits; +} +EXPORT_SYMBOL_GPL(snd_soc_set_enum_kctl); + int snd_soc_bytes_info(struct snd_kcontrol *kcontrol, struct snd_ctl_elem_info *uinfo) { diff --git a/sound/soc/sof/intel/hda-dai.c b/sound/soc/sof/intel/hda-dai.c index 883d0d3bae9ec2..3c742d53513337 100644 --- a/sound/soc/sof/intel/hda-dai.c +++ b/sound/soc/sof/intel/hda-dai.c @@ -70,12 +70,22 @@ static const struct hda_dai_widget_dma_ops * hda_dai_get_ops(struct snd_pcm_substream *substream, struct snd_soc_dai *cpu_dai) { struct snd_soc_dapm_widget *w = snd_soc_dai_get_widget(cpu_dai, substream->stream); - struct snd_sof_widget *swidget = w->dobj.private; + struct snd_sof_widget *swidget; struct snd_sof_dev *sdev; struct snd_sof_dai *sdai; - sdev = widget_to_sdev(w); + /* + * this is unlikely if the topology and the machine driver DAI links match. + * But if there's a missing DAI link in topology, this will prevent a NULL pointer + * dereference later on. + */ + if (!w) { + dev_err(cpu_dai->dev, "%s: widget is NULL\n", __func__); + return NULL; + } + sdev = widget_to_sdev(w); + swidget = w->dobj.private; if (!swidget) { dev_err(sdev->dev, "%s: swidget is NULL\n", __func__); return NULL; diff --git a/sound/soc/sof/intel/hda-pcm.c b/sound/soc/sof/intel/hda-pcm.c index da6c1e7263cde1..16a3640728210b 100644 --- a/sound/soc/sof/intel/hda-pcm.c +++ b/sound/soc/sof/intel/hda-pcm.c @@ -219,6 +219,7 @@ EXPORT_SYMBOL_NS(hda_dsp_pcm_pointer, "SND_SOC_SOF_INTEL_HDA_COMMON"); int hda_dsp_pcm_open(struct snd_sof_dev *sdev, struct snd_pcm_substream *substream) { + const struct sof_intel_dsp_desc *chip_info = get_chip_info(sdev->pdata); struct snd_soc_pcm_runtime *rtd = snd_soc_substream_to_rtd(substream); struct snd_pcm_runtime *runtime = substream->runtime; struct snd_soc_component *scomp = sdev->component; @@ -268,8 +269,17 @@ int hda_dsp_pcm_open(struct snd_sof_dev *sdev, return -ENODEV; } - /* minimum as per HDA spec */ - snd_pcm_hw_constraint_step(substream->runtime, 0, SNDRV_PCM_HW_PARAM_PERIOD_BYTES, 4); + /* + * Set period size constraint to ensure BDLE buffer length and + * start address alignment requirements are met. Align to 128 + * bytes for newer Intel platforms, with older ones using 4 byte alignment. + */ + if (chip_info->hw_ip_version >= SOF_INTEL_ACE_4_0) + snd_pcm_hw_constraint_step(substream->runtime, 0, + SNDRV_PCM_HW_PARAM_PERIOD_BYTES, 128); + else + snd_pcm_hw_constraint_step(substream->runtime, 0, + SNDRV_PCM_HW_PARAM_PERIOD_BYTES, 4); /* avoid circular buffer wrap in middle of period */ snd_pcm_hw_constraint_integer(substream->runtime, diff --git a/sound/soc/sof/intel/hda.c b/sound/soc/sof/intel/hda.c index 686ecc040867a2..b039306454da26 100644 --- a/sound/soc/sof/intel/hda.c +++ b/sound/soc/sof/intel/hda.c @@ -1133,13 +1133,12 @@ static void hda_generic_machine_select(struct snd_sof_dev *sdev, #if IS_ENABLED(CONFIG_SND_SOC_SOF_INTEL_SOUNDWIRE) -static bool is_endpoint_present(struct sdw_slave *sdw_device, - struct asoc_sdw_codec_info *dai_info, int dai_type) +static bool is_endpoint_present(struct sdw_slave *sdw_device, int dai_type) { int i; for (i = 0; i < sdw_device->sdca_data.num_functions; i++) { - if (dai_type == dai_info->dais[i].dai_type) + if (dai_type == asoc_sdw_get_dai_type(sdw_device->sdca_data.function[i].type)) return true; } dev_dbg(&sdw_device->dev, "Endpoint DAI type %d not found\n", dai_type); @@ -1193,11 +1192,10 @@ static struct snd_soc_acpi_adr_device *find_acpi_adr_device(struct device *dev, } for (j = 0; j < codec_info_list[i].dai_num; j++) { /* Check if the endpoint is present by the SDCA DisCo table */ - if (!is_endpoint_present(sdw_device, &codec_info_list[i], - codec_info_list[i].dais[j].dai_type)) + if (!is_endpoint_present(sdw_device, codec_info_list[i].dais[j].dai_type)) continue; - endpoints[ep_index].num = ep_index; + endpoints[ep_index].num = j; if (codec_info_list[i].dais[j].dai_type == SOC_SDW_DAI_TYPE_AMP) { /* Assume all amp are aggregated */ endpoints[ep_index].aggregated = 1; diff --git a/sound/soc/sof/ipc4-control.c b/sound/soc/sof/ipc4-control.c index 976a4794d61000..453ed1643b89c9 100644 --- a/sound/soc/sof/ipc4-control.c +++ b/sound/soc/sof/ipc4-control.c @@ -66,7 +66,7 @@ static int sof_ipc4_set_get_kcontrol_data(struct snd_sof_control *scontrol, * configuration */ memcpy(scontrol->ipc_control_data, scontrol->old_ipc_control_data, - scontrol->max_size); + scontrol->size); kfree(scontrol->old_ipc_control_data); scontrol->old_ipc_control_data = NULL; /* Send the last known good configuration to firmware */ @@ -412,19 +412,35 @@ static int sof_ipc4_set_get_bytes_data(struct snd_sof_dev *sdev, int ret = 0; /* Send the new data to the firmware only if it is powered up */ - if (set && !pm_runtime_active(sdev->dev)) - return 0; + if (set) { + if (!pm_runtime_active(sdev->dev)) + return 0; + + if (!data->size) { + dev_dbg(sdev->dev, "%s: No data to be sent.\n", + scontrol->name); + return 0; + } + } msg->extension = SOF_IPC4_MOD_EXT_MSG_PARAM_ID(data->type); msg->data_ptr = data->data; - msg->data_size = data->size; + if (set) + msg->data_size = data->size; + else + msg->data_size = scontrol->max_size - sizeof(*data); ret = sof_ipc4_set_get_kcontrol_data(scontrol, set, lock); - if (ret < 0) + if (ret < 0) { dev_err(sdev->dev, "Failed to %s for %s\n", set ? "set bytes update" : "get bytes", scontrol->name); + } else if (!set) { + /* Update the sizes according to the received payload data */ + data->size = msg->data_size; + scontrol->size = sizeof(*cdata) + sizeof(*data) + data->size; + } msg->data_ptr = NULL; msg->data_size = 0; @@ -440,6 +456,7 @@ static int sof_ipc4_bytes_put(struct snd_sof_control *scontrol, struct snd_sof_dev *sdev = snd_soc_component_get_drvdata(scomp); struct sof_abi_hdr *data = cdata->data; size_t size; + int ret; if (scontrol->max_size > sizeof(ucontrol->value.bytes.data)) { dev_err_ratelimited(scomp->dev, @@ -461,9 +478,12 @@ static int sof_ipc4_bytes_put(struct snd_sof_control *scontrol, /* copy from kcontrol */ memcpy(data, ucontrol->value.bytes.data, size); - sof_ipc4_set_get_bytes_data(sdev, scontrol, true, true); + ret = sof_ipc4_set_get_bytes_data(sdev, scontrol, true, true); + if (!ret) + /* Update the cdata size */ + scontrol->size = sizeof(*cdata) + size; - return 0; + return ret; } static int sof_ipc4_bytes_get(struct snd_sof_control *scontrol, @@ -559,7 +579,7 @@ static int sof_ipc4_bytes_ext_put(struct snd_sof_control *scontrol, if (!scontrol->old_ipc_control_data) { /* Create a backup of the current, valid bytes control */ scontrol->old_ipc_control_data = kmemdup(scontrol->ipc_control_data, - scontrol->max_size, GFP_KERNEL); + scontrol->size, GFP_KERNEL); if (!scontrol->old_ipc_control_data) return -ENOMEM; } @@ -567,12 +587,15 @@ static int sof_ipc4_bytes_ext_put(struct snd_sof_control *scontrol, /* Copy the whole binary data which includes the ABI header and the payload */ if (copy_from_user(data, tlvd->tlv, header.length)) { memcpy(scontrol->ipc_control_data, scontrol->old_ipc_control_data, - scontrol->max_size); + scontrol->size); kfree(scontrol->old_ipc_control_data); scontrol->old_ipc_control_data = NULL; return -EFAULT; } + /* Update the cdata size */ + scontrol->size = sizeof(*cdata) + header.length; + return sof_ipc4_set_get_bytes_data(sdev, scontrol, true, true); } diff --git a/sound/soc/sof/ipc4-topology.c b/sound/soc/sof/ipc4-topology.c index d621e7914a73c0..f1486e7bb524fa 100644 --- a/sound/soc/sof/ipc4-topology.c +++ b/sound/soc/sof/ipc4-topology.c @@ -2870,22 +2870,41 @@ static int sof_ipc4_control_load_bytes(struct snd_sof_dev *sdev, struct snd_sof_ struct sof_ipc4_msg *msg; int ret; - if (scontrol->max_size < (sizeof(*control_data) + sizeof(struct sof_abi_hdr))) { - dev_err(sdev->dev, "insufficient size for a bytes control %s: %zu.\n", + /* + * The max_size is coming from topology and indicates the maximum size + * of sof_abi_hdr plus the payload, which excludes the local only + * 'struct sof_ipc4_control_data' + */ + if (scontrol->max_size < sizeof(struct sof_abi_hdr)) { + dev_err(sdev->dev, + "insufficient maximum size for a bytes control %s: %zu.\n", scontrol->name, scontrol->max_size); return -EINVAL; } - if (scontrol->priv_size > scontrol->max_size - sizeof(*control_data)) { - dev_err(sdev->dev, "scontrol %s bytes data size %zu exceeds max %zu.\n", - scontrol->name, scontrol->priv_size, - scontrol->max_size - sizeof(*control_data)); + if (scontrol->priv_size > scontrol->max_size) { + dev_err(sdev->dev, + "bytes control %s initial data size %zu exceeds max %zu.\n", + scontrol->name, scontrol->priv_size, scontrol->max_size); + return -EINVAL; + } + + if (scontrol->priv_size && scontrol->priv_size < sizeof(struct sof_abi_hdr)) { + dev_err(sdev->dev, + "bytes control %s initial data size %zu is insufficient.\n", + scontrol->name, scontrol->priv_size); return -EINVAL; } - scontrol->size = sizeof(struct sof_ipc4_control_data) + scontrol->priv_size; + /* + * The used size behind the cdata pointer, which can be smaller than + * the maximum size + */ + scontrol->size = sizeof(*control_data) + scontrol->priv_size; - scontrol->ipc_control_data = kzalloc(scontrol->max_size, GFP_KERNEL); + /* Allocate the cdata: local struct size + maximum payload size */ + scontrol->ipc_control_data = kzalloc(sizeof(*control_data) + scontrol->max_size, + GFP_KERNEL); if (!scontrol->ipc_control_data) return -ENOMEM; diff --git a/sound/soc/sof/ipc4.c b/sound/soc/sof/ipc4.c index a4a090e6724a63..20d723f48fff0c 100644 --- a/sound/soc/sof/ipc4.c +++ b/sound/soc/sof/ipc4.c @@ -15,6 +15,7 @@ #include "sof-audio.h" #include "ipc4-fw-reg.h" #include "ipc4-priv.h" +#include "ipc4-topology.h" #include "ipc4-telemetry.h" #include "ops.h" @@ -433,6 +434,23 @@ static int sof_ipc4_tx_msg(struct snd_sof_dev *sdev, void *msg_data, size_t msg_ return ret; } +static bool sof_ipc4_tx_payload_for_get_data(struct sof_ipc4_msg *tx) +{ + /* + * Messages that require TX payload with LARGE_CONFIG_GET. + * The TX payload is placed into the IPC message data section by caller, + * which needs to be copied to temporary buffer since the received data + * will overwrite it. + */ + switch (tx->extension & SOF_IPC4_MOD_EXT_MSG_PARAM_ID_MASK) { + case SOF_IPC4_MOD_EXT_MSG_PARAM_ID(SOF_IPC4_SWITCH_CONTROL_PARAM_ID): + case SOF_IPC4_MOD_EXT_MSG_PARAM_ID(SOF_IPC4_ENUM_CONTROL_PARAM_ID): + return true; + default: + return false; + } +} + static int sof_ipc4_set_get_data(struct snd_sof_dev *sdev, void *data, size_t payload_bytes, bool set) { @@ -444,6 +462,8 @@ static int sof_ipc4_set_get_data(struct snd_sof_dev *sdev, void *data, struct sof_ipc4_msg tx = {{ 0 }}; struct sof_ipc4_msg rx = {{ 0 }}; size_t remaining = payload_bytes; + void *tx_payload_for_get = NULL; + size_t tx_data_size = 0; size_t offset = 0; size_t chunk_size; int ret; @@ -469,10 +489,20 @@ static int sof_ipc4_set_get_data(struct snd_sof_dev *sdev, void *data, tx.extension |= SOF_IPC4_MOD_EXT_MSG_FIRST_BLOCK(1); + if (sof_ipc4_tx_payload_for_get_data(&tx)) { + tx_data_size = min(ipc4_msg->data_size, payload_limit); + tx_payload_for_get = kmemdup(ipc4_msg->data_ptr, tx_data_size, + GFP_KERNEL); + if (!tx_payload_for_get) + return -ENOMEM; + } + /* ensure the DSP is in D0i0 before sending IPC */ ret = snd_sof_dsp_set_power_state(sdev, &target_state); - if (ret < 0) + if (ret < 0) { + kfree(tx_payload_for_get); return ret; + } /* Serialise IPC TX */ mutex_lock(&sdev->ipc->tx_mutex); @@ -506,7 +536,15 @@ static int sof_ipc4_set_get_data(struct snd_sof_dev *sdev, void *data, rx.data_size = chunk_size; rx.data_ptr = ipc4_msg->data_ptr + offset; - tx_size = 0; + if (tx_payload_for_get) { + tx_size = tx_data_size; + tx.data_size = tx_size; + tx.data_ptr = tx_payload_for_get; + } else { + tx_size = 0; + tx.data_size = 0; + tx.data_ptr = NULL; + } rx_size = chunk_size; } @@ -553,6 +591,8 @@ static int sof_ipc4_set_get_data(struct snd_sof_dev *sdev, void *data, mutex_unlock(&sdev->ipc->tx_mutex); + kfree(tx_payload_for_get); + return ret; } diff --git a/sound/soc/sof/topology.c b/sound/soc/sof/topology.c index 9bf8ab610a7ea4..8880ac5d8d6ff8 100644 --- a/sound/soc/sof/topology.c +++ b/sound/soc/sof/topology.c @@ -736,7 +736,7 @@ static int sof_parse_token_sets(struct snd_soc_component *scomp, asize = le32_to_cpu(array->size); /* validate asize */ - if (asize < 0) { /* FIXME: A zero-size array makes no sense */ + if (asize < sizeof(*array)) { dev_err(scomp->dev, "error: invalid array size 0x%x\n", asize); return -EINVAL; diff --git a/sound/soc/stm/stm32_sai_sub.c b/sound/soc/stm/stm32_sai_sub.c index 450e1585edeee0..3e82fa90e719af 100644 --- a/sound/soc/stm/stm32_sai_sub.c +++ b/sound/soc/stm/stm32_sai_sub.c @@ -802,6 +802,7 @@ static int stm32_sai_set_dai_fmt(struct snd_soc_dai *cpu_dai, unsigned int fmt) break; /* Left justified */ case SND_SOC_DAIFMT_MSB: + cr1 |= SAI_XCR1_CKSTR; frcr |= SAI_XFRCR_FSPOL | SAI_XFRCR_FSDEF; break; /* Right justified */ @@ -809,9 +810,11 @@ static int stm32_sai_set_dai_fmt(struct snd_soc_dai *cpu_dai, unsigned int fmt) frcr |= SAI_XFRCR_FSPOL | SAI_XFRCR_FSDEF; break; case SND_SOC_DAIFMT_DSP_A: + cr1 |= SAI_XCR1_CKSTR; frcr |= SAI_XFRCR_FSPOL | SAI_XFRCR_FSOFF; break; case SND_SOC_DAIFMT_DSP_B: + cr1 |= SAI_XCR1_CKSTR; frcr |= SAI_XFRCR_FSPOL; break; default: diff --git a/sound/soc/sunxi/sun50i-dmic.c b/sound/soc/sunxi/sun50i-dmic.c index bab1e29c99887c..eddfebe166169a 100644 --- a/sound/soc/sunxi/sun50i-dmic.c +++ b/sound/soc/sunxi/sun50i-dmic.c @@ -358,6 +358,9 @@ static int sun50i_dmic_probe(struct platform_device *pdev) host->regmap = devm_regmap_init_mmio(&pdev->dev, base, &sun50i_dmic_regmap_config); + if (IS_ERR(host->regmap)) + return dev_err_probe(&pdev->dev, PTR_ERR(host->regmap), + "failed to initialise regmap\n"); /* Clocks */ host->bus_clk = devm_clk_get(&pdev->dev, "bus"); diff --git a/sound/soc/tegra/tegra210_ahub.c b/sound/soc/tegra/tegra210_ahub.c index e795907a3963af..fc5892056f832f 100644 --- a/sound/soc/tegra/tegra210_ahub.c +++ b/sound/soc/tegra/tegra210_ahub.c @@ -2049,6 +2049,61 @@ static const struct snd_soc_component_driver tegra264_ahub_component = { .num_dapm_routes = ARRAY_SIZE(tegra264_ahub_routes), }; +static bool tegra210_ahub_wr_reg(struct device *dev, unsigned int reg) +{ + int part; + + if (reg % TEGRA210_XBAR_RX_STRIDE) + return false; + + for (part = 0; part < TEGRA210_XBAR_UPDATE_MAX_REG; part++) { + switch (reg & ~(part * TEGRA210_XBAR_PART1_RX)) { + case TEGRA210_AXBAR_PART_0_ADMAIF_RX1_0 ... TEGRA210_AXBAR_PART_0_ADMAIF_RX10_0: + case TEGRA210_AXBAR_PART_0_I2S1_RX1_0 ... TEGRA210_AXBAR_PART_0_I2S5_RX1_0: + case TEGRA210_AXBAR_PART_0_SFC1_RX1_0 ... TEGRA210_AXBAR_PART_0_SFC4_RX1_0: + case TEGRA210_AXBAR_PART_0_MIXER1_RX1_0 ... TEGRA210_AXBAR_PART_0_MIXER1_RX10_0: + case TEGRA210_AXBAR_PART_0_SPDIF1_RX1_0 ... TEGRA210_AXBAR_PART_0_SPDIF1_RX2_0: + case TEGRA210_AXBAR_PART_0_AFC1_RX1_0 ... TEGRA210_AXBAR_PART_0_AFC6_RX1_0: + case TEGRA210_AXBAR_PART_0_OPE1_RX1_0 ... TEGRA210_AXBAR_PART_0_OPE2_RX1_0: + case TEGRA210_AXBAR_PART_0_SPKPROT1_RX1_0: + case TEGRA210_AXBAR_PART_0_MVC1_RX1_0 ... TEGRA210_AXBAR_PART_0_MVC2_RX1_0: + case TEGRA210_AXBAR_PART_0_AMX1_RX1_0 ... TEGRA210_AXBAR_PART_0_ADX2_RX1_0: + return true; + default: + break; + } + } + + return false; +} + +static bool tegra186_ahub_wr_reg(struct device *dev, unsigned int reg) +{ + int part; + + if (reg % TEGRA210_XBAR_RX_STRIDE) + return false; + + for (part = 0; part < TEGRA186_XBAR_UPDATE_MAX_REG; part++) { + switch (reg & ~(part * TEGRA210_XBAR_PART1_RX)) { + case TEGRA210_AXBAR_PART_0_ADMAIF_RX1_0 ... TEGRA186_AXBAR_PART_0_I2S6_RX1_0: + case TEGRA210_AXBAR_PART_0_SFC1_RX1_0 ... TEGRA210_AXBAR_PART_0_SFC4_RX1_0: + case TEGRA210_AXBAR_PART_0_MIXER1_RX1_0 ... TEGRA210_AXBAR_PART_0_MIXER1_RX10_0: + case TEGRA186_AXBAR_PART_0_DSPK1_RX1_0 ... TEGRA186_AXBAR_PART_0_DSPK2_RX1_0: + case TEGRA210_AXBAR_PART_0_AFC1_RX1_0 ... TEGRA210_AXBAR_PART_0_AFC6_RX1_0: + case TEGRA210_AXBAR_PART_0_OPE1_RX1_0: + case TEGRA186_AXBAR_PART_0_MVC1_RX1_0 ... TEGRA186_AXBAR_PART_0_MVC2_RX1_0: + case TEGRA186_AXBAR_PART_0_AMX1_RX1_0 ... TEGRA186_AXBAR_PART_0_AMX3_RX4_0: + case TEGRA210_AXBAR_PART_0_ADX1_RX1_0 ... TEGRA186_AXBAR_PART_0_ASRC1_RX7_0: + return true; + default: + break; + } + } + + return false; +} + static bool tegra264_ahub_wr_reg(struct device *dev, unsigned int reg) { int part; @@ -2076,6 +2131,7 @@ static const struct regmap_config tegra210_ahub_regmap_config = { .reg_bits = 32, .val_bits = 32, .reg_stride = 4, + .writeable_reg = tegra210_ahub_wr_reg, .max_register = TEGRA210_MAX_REGISTER_ADDR, .cache_type = REGCACHE_FLAT, }; @@ -2084,6 +2140,7 @@ static const struct regmap_config tegra186_ahub_regmap_config = { .reg_bits = 32, .val_bits = 32, .reg_stride = 4, + .writeable_reg = tegra186_ahub_wr_reg, .max_register = TEGRA186_MAX_REGISTER_ADDR, .cache_type = REGCACHE_FLAT, }; diff --git a/sound/soc/tegra/tegra210_ahub.h b/sound/soc/tegra/tegra210_ahub.h index f355b2cfd19b22..acbe640dd3b572 100644 --- a/sound/soc/tegra/tegra210_ahub.h +++ b/sound/soc/tegra/tegra210_ahub.h @@ -68,6 +68,36 @@ #define TEGRA210_MAX_REGISTER_ADDR (TEGRA210_XBAR_PART2_RX + \ (TEGRA210_XBAR_RX_STRIDE * (TEGRA210_XBAR_AUDIO_RX_COUNT - 1))) +/* AXBAR register offsets */ +#define TEGRA186_AXBAR_PART_0_AMX1_RX1_0 0x120 +#define TEGRA186_AXBAR_PART_0_AMX3_RX4_0 0x14c +#define TEGRA186_AXBAR_PART_0_ASRC1_RX7_0 0x1a8 +#define TEGRA186_AXBAR_PART_0_DSPK1_RX1_0 0xc0 +#define TEGRA186_AXBAR_PART_0_DSPK2_RX1_0 0xc4 +#define TEGRA186_AXBAR_PART_0_I2S6_RX1_0 0x54 +#define TEGRA186_AXBAR_PART_0_MVC1_RX1_0 0x110 +#define TEGRA186_AXBAR_PART_0_MVC2_RX1_0 0x114 +#define TEGRA210_AXBAR_PART_0_ADMAIF_RX10_0 0x24 +#define TEGRA210_AXBAR_PART_0_ADMAIF_RX1_0 0x0 +#define TEGRA210_AXBAR_PART_0_ADX1_RX1_0 0x160 +#define TEGRA210_AXBAR_PART_0_ADX2_RX1_0 0x164 +#define TEGRA210_AXBAR_PART_0_AFC1_RX1_0 0xd0 +#define TEGRA210_AXBAR_PART_0_AFC6_RX1_0 0xe4 +#define TEGRA210_AXBAR_PART_0_AMX1_RX1_0 0x140 +#define TEGRA210_AXBAR_PART_0_I2S1_RX1_0 0x40 +#define TEGRA210_AXBAR_PART_0_I2S5_RX1_0 0x50 +#define TEGRA210_AXBAR_PART_0_MIXER1_RX10_0 0xa4 +#define TEGRA210_AXBAR_PART_0_MIXER1_RX1_0 0x80 +#define TEGRA210_AXBAR_PART_0_MVC1_RX1_0 0x120 +#define TEGRA210_AXBAR_PART_0_MVC2_RX1_0 0x124 +#define TEGRA210_AXBAR_PART_0_OPE1_RX1_0 0x100 +#define TEGRA210_AXBAR_PART_0_OPE2_RX1_0 0x104 +#define TEGRA210_AXBAR_PART_0_SFC1_RX1_0 0x60 +#define TEGRA210_AXBAR_PART_0_SFC4_RX1_0 0x6c +#define TEGRA210_AXBAR_PART_0_SPDIF1_RX1_0 0xc0 +#define TEGRA210_AXBAR_PART_0_SPDIF1_RX2_0 0xc4 +#define TEGRA210_AXBAR_PART_0_SPKPROT1_RX1_0 0x110 + #define MUX_REG(id) (TEGRA210_XBAR_RX_STRIDE * (id)) #define MUX_VALUE(npart, nbit) (1 + (nbit) + (npart) * 32) diff --git a/sound/usb/6fire/chip.c b/sound/usb/6fire/chip.c index 5ff78814e68750..874f6cd503ca59 100644 --- a/sound/usb/6fire/chip.c +++ b/sound/usb/6fire/chip.c @@ -53,11 +53,6 @@ static void usb6fire_chip_abort(struct sfire_chip *chip) usb6fire_comm_abort(chip); if (chip->control) usb6fire_control_abort(chip); - if (chip->card) { - snd_card_disconnect(chip->card); - snd_card_free_when_closed(chip->card); - chip->card = NULL; - } } } @@ -168,6 +163,7 @@ static int usb6fire_chip_probe(struct usb_interface *intf, static void usb6fire_chip_disconnect(struct usb_interface *intf) { struct sfire_chip *chip; + struct snd_card *card; chip = usb_get_intfdata(intf); if (chip) { /* if !chip, fw upload has been performed */ @@ -178,8 +174,19 @@ static void usb6fire_chip_disconnect(struct usb_interface *intf) chips[chip->regidx] = NULL; } + /* + * Save card pointer before teardown. + * snd_card_free_when_closed() may free card (and + * the embedded chip) immediately, so it must be + * called last and chip must not be accessed after. + */ + card = chip->card; chip->shutdown = true; + if (card) + snd_card_disconnect(card); usb6fire_chip_abort(chip); + if (card) + snd_card_free_when_closed(card); } } } diff --git a/sound/usb/Kconfig b/sound/usb/Kconfig index 9b890abd96d34c..b4588915efa114 100644 --- a/sound/usb/Kconfig +++ b/sound/usb/Kconfig @@ -192,6 +192,7 @@ config SND_USB_AUDIO_QMI tristate "Qualcomm Audio Offload driver" depends on QCOM_QMI_HELPERS && SND_USB_AUDIO && SND_SOC_USB depends on USB_XHCI_HCD && USB_XHCI_SIDEBAND + select AUXILIARY_BUS help Say Y here to enable the Qualcomm USB audio offloading feature. diff --git a/sound/usb/caiaq/device.c b/sound/usb/caiaq/device.c index dfd820483849eb..3a71bab8a47749 100644 --- a/sound/usb/caiaq/device.c +++ b/sound/usb/caiaq/device.c @@ -488,7 +488,7 @@ static int init_card(struct snd_usb_caiaqdev *cdev) memset(id, 0, sizeof(id)); for (c = card->shortname, len = 0; - *c && len < sizeof(card->id); c++) + *c && len < sizeof(card->id) - 1; c++) if (*c != ' ') id[len++] = *c; diff --git a/sound/usb/endpoint.c b/sound/usb/endpoint.c index 8f9313857ee9da..1a020ea5587550 100644 --- a/sound/usb/endpoint.c +++ b/sound/usb/endpoint.c @@ -160,8 +160,8 @@ int snd_usb_endpoint_implicit_feedback_sink(struct snd_usb_endpoint *ep) * This won't be used for implicit feedback which takes the packet size * returned from the sync source */ -static int slave_next_packet_size(struct snd_usb_endpoint *ep, - unsigned int avail) +static int synced_next_packet_size(struct snd_usb_endpoint *ep, + unsigned int avail) { unsigned int phase; int ret; @@ -221,13 +221,14 @@ int snd_usb_endpoint_next_packet_size(struct snd_usb_endpoint *ep, packet = ctx->packet_size[idx]; if (packet) { + packet = min(packet, ep->maxframesize); if (avail && packet >= avail) return -EAGAIN; return packet; } if (ep->sync_source) - return slave_next_packet_size(ep, avail); + return synced_next_packet_size(ep, avail); else return next_packet_size(ep, avail); } @@ -275,8 +276,8 @@ static inline bool has_tx_length_quirk(struct snd_usb_audio *chip) return chip->quirk_flags & QUIRK_FLAG_TX_LENGTH; } -static void prepare_silent_urb(struct snd_usb_endpoint *ep, - struct snd_urb_ctx *ctx) +static int prepare_silent_urb(struct snd_usb_endpoint *ep, + struct snd_urb_ctx *ctx) { struct urb *urb = ctx->urb; unsigned int offs = 0; @@ -289,28 +290,34 @@ static void prepare_silent_urb(struct snd_usb_endpoint *ep, extra = sizeof(packet_length); for (i = 0; i < ctx->packets; ++i) { - unsigned int offset; - unsigned int length; - int counts; - - counts = snd_usb_endpoint_next_packet_size(ep, ctx, i, 0); - length = counts * ep->stride; /* number of silent bytes */ - offset = offs * ep->stride + extra * i; - urb->iso_frame_desc[i].offset = offset; + int length; + + length = snd_usb_endpoint_next_packet_size(ep, ctx, i, 0); + if (length < 0) + return length; + length *= ep->stride; /* number of silent bytes */ + if (offs + length + extra > ctx->buffer_size) + break; + urb->iso_frame_desc[i].offset = offs; urb->iso_frame_desc[i].length = length + extra; if (extra) { packet_length = cpu_to_le32(length); - memcpy(urb->transfer_buffer + offset, + memcpy(urb->transfer_buffer + offs, &packet_length, sizeof(packet_length)); + offs += extra; } - memset(urb->transfer_buffer + offset + extra, + memset(urb->transfer_buffer + offs, ep->silence_value, length); - offs += counts; + offs += length; } - urb->number_of_packets = ctx->packets; - urb->transfer_buffer_length = offs * ep->stride + ctx->packets * extra; + if (!offs) + return -EPIPE; + + urb->number_of_packets = i; + urb->transfer_buffer_length = offs; ctx->queued = 0; + return 0; } /* @@ -332,8 +339,7 @@ static int prepare_outbound_urb(struct snd_usb_endpoint *ep, if (data_subs && ep->prepare_data_urb) return ep->prepare_data_urb(data_subs, urb, in_stream_lock); /* no data provider, so send silence */ - prepare_silent_urb(ep, ctx); - break; + return prepare_silent_urb(ep, ctx); case SND_USB_ENDPOINT_TYPE_SYNC: if (snd_usb_get_speed(ep->chip->dev) >= USB_SPEED_HIGH) { @@ -481,6 +487,7 @@ int snd_usb_queue_pending_output_urbs(struct snd_usb_endpoint *ep, /* copy over the length information */ if (implicit_fb) { + ctx->packets = packet->packets; for (i = 0; i < packet->packets; i++) ctx->packet_size[i] = packet->packet_size[i]; } @@ -1368,6 +1375,9 @@ int snd_usb_endpoint_set_params(struct snd_usb_audio *chip, return -EINVAL; } + ep->packsize[0] = min(ep->packsize[0], ep->maxframesize); + ep->packsize[1] = min(ep->packsize[1], ep->maxframesize); + /* calculate the frequency in 16.16 format */ ep->freqm = ep->freqn; ep->freqshift = INT_MIN; diff --git a/sound/usb/fcp.c b/sound/usb/fcp.c index 11e9a96b46ffe5..1f4595d1e217f9 100644 --- a/sound/usb/fcp.c +++ b/sound/usb/fcp.c @@ -182,10 +182,6 @@ static int fcp_usb(struct usb_mixer_interface *mixer, u32 opcode, { struct fcp_data *private = mixer->private_data; struct usb_device *dev = mixer->chip->dev; - struct fcp_usb_packet *req __free(kfree) = NULL; - struct fcp_usb_packet *resp __free(kfree) = NULL; - size_t req_buf_size = struct_size(req, data, req_size); - size_t resp_buf_size = struct_size(resp, data, resp_size); int retries = 0; const int max_retries = 5; int err; @@ -193,10 +189,14 @@ static int fcp_usb(struct usb_mixer_interface *mixer, u32 opcode, if (!mixer->urb) return -ENODEV; + struct fcp_usb_packet *req __free(kfree) = NULL; + size_t req_buf_size = struct_size(req, data, req_size); req = kmalloc(req_buf_size, GFP_KERNEL); if (!req) return -ENOMEM; + struct fcp_usb_packet *resp __free(kfree) = NULL; + size_t resp_buf_size = struct_size(resp, data, resp_size); resp = kmalloc(resp_buf_size, GFP_KERNEL); if (!resp) return -ENOMEM; @@ -300,16 +300,17 @@ static int fcp_usb(struct usb_mixer_interface *mixer, u32 opcode, static int fcp_reinit(struct usb_mixer_interface *mixer) { struct fcp_data *private = mixer->private_data; - void *step0_resp __free(kfree) = NULL; - void *step2_resp __free(kfree) = NULL; if (mixer->urb) return 0; - step0_resp = kmalloc(private->step0_resp_size, GFP_KERNEL); + void *step0_resp __free(kfree) = + kmalloc(private->step0_resp_size, GFP_KERNEL); if (!step0_resp) return -ENOMEM; - step2_resp = kmalloc(private->step2_resp_size, GFP_KERNEL); + + void *step2_resp __free(kfree) = + kmalloc(private->step2_resp_size, GFP_KERNEL); if (!step2_resp) return -ENOMEM; @@ -464,7 +465,6 @@ static int fcp_ioctl_init(struct usb_mixer_interface *mixer, struct fcp_init init; struct usb_device *dev = mixer->chip->dev; struct fcp_data *private = mixer->private_data; - void *resp __free(kfree) = NULL; void *step2_resp; int err, buf_size; @@ -485,7 +485,8 @@ static int fcp_ioctl_init(struct usb_mixer_interface *mixer, /* Allocate response buffer */ buf_size = init.step0_resp_size + init.step2_resp_size; - resp = kmalloc(buf_size, GFP_KERNEL); + void *resp __free(kfree) = + kmalloc(buf_size, GFP_KERNEL); if (!resp) return -ENOMEM; @@ -619,7 +620,6 @@ static int fcp_ioctl_set_meter_map(struct usb_mixer_interface *mixer, { struct fcp_meter_map map; struct fcp_data *private = mixer->private_data; - s16 *tmp_map __free(kfree) = NULL; int err; if (copy_from_user(&map, arg, sizeof(map))) @@ -641,7 +641,8 @@ static int fcp_ioctl_set_meter_map(struct usb_mixer_interface *mixer, return -EINVAL; /* Allocate and copy the map data */ - tmp_map = memdup_array_user(arg->map, map.map_size, sizeof(s16)); + s16 *tmp_map __free(kfree) = + memdup_array_user(arg->map, map.map_size, sizeof(s16)); if (IS_ERR(tmp_map)) return PTR_ERR(tmp_map); @@ -651,17 +652,16 @@ static int fcp_ioctl_set_meter_map(struct usb_mixer_interface *mixer, /* If the control doesn't exist, create it */ if (!private->meter_ctl) { - s16 *new_map __free(kfree) = NULL; - __le32 *meter_levels __free(kfree) = NULL; - /* Allocate buffer for the map */ - new_map = kmalloc_array(map.map_size, sizeof(s16), GFP_KERNEL); + s16 *new_map __free(kfree) = + kmalloc_array(map.map_size, sizeof(s16), GFP_KERNEL); if (!new_map) return -ENOMEM; /* Allocate buffer for reading meter levels */ - meter_levels = kmalloc_array(map.meter_slots, sizeof(__le32), - GFP_KERNEL); + __le32 *meter_levels __free(kfree) = + kmalloc_array(map.meter_slots, sizeof(__le32), + GFP_KERNEL); if (!meter_levels) return -ENOMEM; diff --git a/sound/usb/format.c b/sound/usb/format.c index 64cfe4a9d8cdf2..1207c507882add 100644 --- a/sound/usb/format.c +++ b/sound/usb/format.c @@ -305,17 +305,48 @@ static bool s1810c_valid_sample_rate(struct audioformat *fp, } /* - * Many Focusrite devices supports a limited set of sampling rates per - * altsetting. Maximum rate is exposed in the last 4 bytes of Format Type - * descriptor which has a non-standard bLength = 10. + * Focusrite devices use rate pairs: 44100/48000, 88200/96000, and + * 176400/192000. Return true if rate is in the pair for max_rate. + */ +static bool focusrite_rate_pair(unsigned int rate, + unsigned int max_rate) +{ + switch (max_rate) { + case 48000: return rate == 44100 || rate == 48000; + case 96000: return rate == 88200 || rate == 96000; + case 192000: return rate == 176400 || rate == 192000; + default: return true; + } +} + +/* + * Focusrite devices report all supported rates in a single clock + * source but only a subset is valid per altsetting. + * + * Detection uses two descriptor features: + * + * 1. Format Type descriptor bLength == 10: non-standard extension + * with max sample rate in bytes 6..9. + * + * 2. bmControls VAL_ALT_SETTINGS readable bit: when set, the device + * only supports the highest rate pair for that altsetting, and when + * clear, all rates up to max_rate are valid. + * + * For devices without the bLength == 10 extension but with + * VAL_ALT_SETTINGS readable and multiple altsettings (only seen in + * Scarlett 18i8 3rd Gen playback), fall back to the Focusrite + * convention: alt 1 = 48kHz, alt 2 = 96kHz, alt 3 = 192kHz. */ static bool focusrite_valid_sample_rate(struct snd_usb_audio *chip, struct audioformat *fp, unsigned int rate) { + struct usb_interface *iface; struct usb_host_interface *alts; + struct uac2_as_header_descriptor *as; unsigned char *fmt; unsigned int max_rate; + bool val_alt; alts = snd_usb_get_host_interface(chip, fp->iface, fp->altsetting); if (!alts) @@ -326,9 +357,21 @@ static bool focusrite_valid_sample_rate(struct snd_usb_audio *chip, if (!fmt) return true; + as = snd_usb_find_csint_desc(alts->extra, alts->extralen, + NULL, UAC_AS_GENERAL); + if (!as) + return true; + + val_alt = uac_v2v3_control_is_readable(as->bmControls, + UAC2_AS_VAL_ALT_SETTINGS); + if (fmt[0] == 10) { /* bLength */ max_rate = combine_quad(&fmt[6]); + if (val_alt) + return focusrite_rate_pair(rate, max_rate); + + /* No val_alt: rates fall through from higher */ switch (max_rate) { case 192000: if (rate == 176400 || rate == 192000) @@ -344,12 +387,29 @@ static bool focusrite_valid_sample_rate(struct snd_usb_audio *chip, usb_audio_info(chip, "%u:%d : unexpected max rate: %u\n", fp->iface, fp->altsetting, max_rate); - return true; } } - return true; + if (!val_alt) + return true; + + /* Multi-altsetting device with val_alt but no max_rate + * in the format descriptor. Use Focusrite convention: + * alt 1 = 48kHz, alt 2 = 96kHz, alt 3 = 192kHz. + */ + iface = usb_ifnum_to_if(chip->dev, fp->iface); + if (!iface || iface->num_altsetting <= 2) + return true; + + switch (fp->altsetting) { + case 1: max_rate = 48000; break; + case 2: max_rate = 96000; break; + case 3: max_rate = 192000; break; + default: return true; + } + + return focusrite_rate_pair(rate, max_rate); } /* diff --git a/sound/usb/mixer_s1810c.c b/sound/usb/mixer_s1810c.c index 6e09e074c0e7fa..93510aa0dc5ef5 100644 --- a/sound/usb/mixer_s1810c.c +++ b/sound/usb/mixer_s1810c.c @@ -82,13 +82,13 @@ * mixer and output but a different set for device. */ struct s1810c_ctl_packet { - u32 a; - u32 b; - u32 fixed1; - u32 fixed2; - u32 c; - u32 d; - u32 e; + __le32 a; + __le32 b; + __le32 fixed1; + __le32 fixed2; + __le32 c; + __le32 d; + __le32 e; }; #define SC1810C_CTL_LINE_SW 0 @@ -118,7 +118,7 @@ struct s1810c_ctl_packet { * being zero and different f1/f2. */ struct s1810c_state_packet { - u32 fields[63]; + __le32 fields[63]; }; #define SC1810C_STATE_48V_SW 58 @@ -140,14 +140,14 @@ snd_s1810c_send_ctl_packet(struct usb_device *dev, u32 a, struct s1810c_ctl_packet pkt = { 0 }; int ret = 0; - pkt.fixed1 = SC1810C_CMD_F1; - pkt.fixed2 = SC1810C_CMD_F2; + pkt.fixed1 = __cpu_to_le32(SC1810C_CMD_F1); + pkt.fixed2 = __cpu_to_le32(SC1810C_CMD_F2); - pkt.a = a; - pkt.b = b; - pkt.c = c; - pkt.d = d; - pkt.e = e; + pkt.a = __cpu_to_le32(a); + pkt.b = __cpu_to_le32(b); + pkt.c = __cpu_to_le32(c); + pkt.d = __cpu_to_le32(d); + pkt.e = __cpu_to_le32(e); ret = snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0), SC1810C_CMD_REQ, @@ -176,8 +176,8 @@ snd_sc1810c_get_status_field(struct usb_device *dev, struct s1810c_state_packet pkt_in = { { 0 } }; int ret = 0; - pkt_out.fields[SC1810C_STATE_F1_IDX] = SC1810C_SET_STATE_F1; - pkt_out.fields[SC1810C_STATE_F2_IDX] = SC1810C_SET_STATE_F2; + pkt_out.fields[SC1810C_STATE_F1_IDX] = __cpu_to_le32(SC1810C_SET_STATE_F1); + pkt_out.fields[SC1810C_STATE_F2_IDX] = __cpu_to_le32(SC1810C_SET_STATE_F2); ret = snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0), SC1810C_SET_STATE_REQ, SC1810C_SET_STATE_REQTYPE, @@ -197,7 +197,7 @@ snd_sc1810c_get_status_field(struct usb_device *dev, return ret; } - (*field) = pkt_in.fields[field_idx]; + (*field) = __le32_to_cpu(pkt_in.fields[field_idx]); (*seqnum)++; return 0; } diff --git a/sound/usb/mixer_scarlett2.c b/sound/usb/mixer_scarlett2.c index bef8c9e544dd3d..4342d93ab7718a 100644 --- a/sound/usb/mixer_scarlett2.c +++ b/sound/usb/mixer_scarlett2.c @@ -1328,8 +1328,6 @@ struct scarlett2_data { struct snd_kcontrol *mux_ctls[SCARLETT2_MUX_MAX]; struct snd_kcontrol *mix_ctls[SCARLETT2_MIX_MAX]; struct snd_kcontrol *compressor_ctls[SCARLETT2_COMPRESSOR_CTLS_MAX]; - struct snd_kcontrol *precomp_flt_ctls[SCARLETT2_PRECOMP_FLT_CTLS_MAX]; - struct snd_kcontrol *peq_flt_ctls[SCARLETT2_PEQ_FLT_CTLS_MAX]; struct snd_kcontrol *precomp_flt_switch_ctls[SCARLETT2_DSP_SWITCH_MAX]; struct snd_kcontrol *peq_flt_switch_ctls[SCARLETT2_DSP_SWITCH_MAX]; struct snd_kcontrol *direct_monitor_ctl; @@ -2377,18 +2375,18 @@ static int scarlett2_usb( { struct scarlett2_data *private = mixer->private_data; struct usb_device *dev = mixer->chip->dev; - struct scarlett2_usb_packet *req __free(kfree) = NULL; - struct scarlett2_usb_packet *resp __free(kfree) = NULL; - size_t req_buf_size = struct_size(req, data, req_size); - size_t resp_buf_size = struct_size(resp, data, resp_size); int retries = 0; const int max_retries = 5; int err; + struct scarlett2_usb_packet *req __free(kfree) = NULL; + size_t req_buf_size = struct_size(req, data, req_size); req = kmalloc(req_buf_size, GFP_KERNEL); if (!req) return -ENOMEM; + struct scarlett2_usb_packet *resp __free(kfree) = NULL; + size_t resp_buf_size = struct_size(resp, data, resp_size); resp = kmalloc(resp_buf_size, GFP_KERNEL); if (!resp) return -ENOMEM; @@ -3447,7 +3445,6 @@ static int scarlett2_update_autogain(struct usb_mixer_interface *mixer) private->autogain_status[i] = private->num_autogain_status_texts - 1; - for (i = 0; i < SCARLETT2_AG_TARGET_COUNT; i++) if (scarlett2_has_config_item(private, scarlett2_ag_target_configs[i])) { @@ -3919,9 +3916,9 @@ static int scarlett2_input_select_ctl_info( struct scarlett2_data *private = mixer->private_data; int inputs = private->info->gain_input_count; int i, err; - char **values __free(kfree) = NULL; + char **values __free(kfree) = + kcalloc(inputs, sizeof(char *), GFP_KERNEL); - values = kcalloc(inputs, sizeof(char *), GFP_KERNEL); if (!values) return -ENOMEM; @@ -5372,8 +5369,7 @@ static int scarlett2_update_filter_values(struct usb_mixer_interface *mixer) err = scarlett2_usb_get_config( mixer, SCARLETT2_CONFIG_PEQ_FLT_SWITCH, - info->dsp_input_count * info->peq_flt_count, - private->peq_flt_switch); + info->dsp_input_count, private->peq_flt_switch); if (err < 0) return err; @@ -6546,7 +6542,7 @@ static int scarlett2_add_dsp_ctls(struct usb_mixer_interface *mixer, int i) err = scarlett2_add_new_ctl( mixer, &scarlett2_precomp_flt_ctl, i * info->precomp_flt_count + j, - 1, s, &private->precomp_flt_switch_ctls[j]); + 1, s, NULL); if (err < 0) return err; } @@ -6556,7 +6552,7 @@ static int scarlett2_add_dsp_ctls(struct usb_mixer_interface *mixer, int i) err = scarlett2_add_new_ctl( mixer, &scarlett2_peq_flt_ctl, i * info->peq_flt_count + j, - 1, s, &private->peq_flt_switch_ctls[j]); + 1, s, NULL); if (err < 0) return err; } @@ -8255,6 +8251,8 @@ static int scarlett2_find_fc_interface(struct usb_device *dev, if (desc->bInterfaceClass != 255) continue; + if (desc->bNumEndpoints < 1) + continue; epd = get_endpoint(intf->altsetting, 0); private->bInterfaceNumber = desc->bInterfaceNumber; @@ -9083,8 +9081,6 @@ static long scarlett2_hwdep_read(struct snd_hwdep *hw, __le32 len; } __packed req; - u8 *resp __free(kfree) = NULL; - /* Flash segment must first be selected */ if (private->flash_write_state != SCARLETT2_FLASH_WRITE_STATE_SELECTED) return -EINVAL; @@ -9122,7 +9118,8 @@ static long scarlett2_hwdep_read(struct snd_hwdep *hw, req.offset = cpu_to_le32(*offset); req.len = cpu_to_le32(count); - resp = kzalloc(count, GFP_KERNEL); + u8 *resp __free(kfree) = + kzalloc(count, GFP_KERNEL); if (!resp) return -ENOMEM; @@ -9267,7 +9264,6 @@ static ssize_t scarlett2_devmap_read( loff_t pos) { struct usb_mixer_interface *mixer = entry->private_data; - u8 *resp_buf __free(kfree) = NULL; const size_t block_size = SCARLETT2_DEVMAP_BLOCK_SIZE; size_t copied = 0; @@ -9277,7 +9273,8 @@ static ssize_t scarlett2_devmap_read( if (pos + count > entry->size) count = entry->size - pos; - resp_buf = kmalloc(block_size, GFP_KERNEL); + u8 *resp_buf __free(kfree) = + kmalloc(block_size, GFP_KERNEL); if (!resp_buf) return -ENOMEM; diff --git a/sound/usb/qcom/qc_audio_offload.c b/sound/usb/qcom/qc_audio_offload.c index cfb30a195364a4..542eae3a57d9d8 100644 --- a/sound/usb/qcom/qc_audio_offload.c +++ b/sound/usb/qcom/qc_audio_offload.c @@ -699,6 +699,7 @@ static void uaudio_event_ring_cleanup_free(struct uaudio_dev *dev) uaudio_iommu_unmap(MEM_EVENT_RING, IOVA_BASE, PAGE_SIZE, PAGE_SIZE); xhci_sideband_remove_interrupter(uadev[dev->chip->card->number].sb); + usb_offload_put(dev->udev); } } @@ -1007,7 +1008,7 @@ static int enable_audio_stream(struct snd_usb_substream *subs, /** * uaudio_transfer_buffer_setup() - fetch and populate xfer buffer params * @subs: usb substream - * @xfer_buf: xfer buf to be allocated + * @xfer_buf_cpu: xfer buf to be allocated * @xfer_buf_len: size of allocation * @mem_info: QMI response info * @@ -1182,12 +1183,16 @@ static int uaudio_event_ring_setup(struct snd_usb_substream *subs, dma_coherent = dev_is_dma_coherent(subs->dev->bus->sysdev); er_pa = 0; + ret = usb_offload_get(subs->dev); + if (ret < 0) + goto exit; + /* event ring */ ret = xhci_sideband_create_interrupter(uadev[card_num].sb, 1, false, 0, uaudio_qdev->data->intr_num); if (ret < 0) { dev_err(&subs->dev->dev, "failed to fetch interrupter\n"); - goto exit; + goto put_offload; } sgt = xhci_sideband_get_event_buffer(uadev[card_num].sb); @@ -1219,6 +1224,8 @@ static int uaudio_event_ring_setup(struct snd_usb_substream *subs, mem_info->dma = 0; remove_interrupter: xhci_sideband_remove_interrupter(uadev[card_num].sb); +put_offload: + usb_offload_put(subs->dev); exit: return ret; } @@ -1483,6 +1490,7 @@ static int prepare_qmi_response(struct snd_usb_substream *subs, uaudio_iommu_unmap(MEM_EVENT_RING, IOVA_BASE, PAGE_SIZE, PAGE_SIZE); free_sec_ring: xhci_sideband_remove_interrupter(uadev[card_num].sb); + usb_offload_put(subs->dev); drop_sync_ep: if (subs->sync_endpoint) { uaudio_iommu_unmap(MEM_XFER_RING, diff --git a/sound/usb/quirks.c b/sound/usb/quirks.c index 2d9f28558874c0..1686022db0adff 100644 --- a/sound/usb/quirks.c +++ b/sound/usb/quirks.c @@ -555,7 +555,6 @@ int snd_usb_create_quirk(struct snd_usb_audio *chip, static int snd_usb_extigy_boot_quirk(struct usb_device *dev, struct usb_interface *intf) { struct usb_host_config *config = dev->actconfig; - struct usb_device_descriptor *new_device_descriptor __free(kfree) = NULL; int err; if (le16_to_cpu(get_cfg_desc(config)->wTotalLength) == EXTIGY_FIRMWARE_SIZE_OLD || @@ -566,8 +565,8 @@ static int snd_usb_extigy_boot_quirk(struct usb_device *dev, struct usb_interfac 0x10, 0x43, 0x0001, 0x000a, NULL, 0); if (err < 0) dev_dbg(&dev->dev, "error sending boot message: %d\n", err); - - new_device_descriptor = kmalloc(sizeof(*new_device_descriptor), GFP_KERNEL); + struct usb_device_descriptor *new_device_descriptor __free(kfree) = + kmalloc(sizeof(*new_device_descriptor), GFP_KERNEL); if (!new_device_descriptor) return -ENOMEM; err = usb_get_descriptor(dev, USB_DT_DEVICE, 0, @@ -910,7 +909,6 @@ static void mbox2_setup_48_24_magic(struct usb_device *dev) static int snd_usb_mbox2_boot_quirk(struct usb_device *dev) { struct usb_host_config *config = dev->actconfig; - struct usb_device_descriptor *new_device_descriptor __free(kfree) = NULL; int err; u8 bootresponse[0x12]; int fwsize; @@ -945,7 +943,8 @@ static int snd_usb_mbox2_boot_quirk(struct usb_device *dev) dev_dbg(&dev->dev, "device initialised!\n"); - new_device_descriptor = kmalloc(sizeof(*new_device_descriptor), GFP_KERNEL); + struct usb_device_descriptor *new_device_descriptor __free(kfree) = + kmalloc(sizeof(*new_device_descriptor), GFP_KERNEL); if (!new_device_descriptor) return -ENOMEM; @@ -1267,7 +1266,6 @@ static void mbox3_setup_defaults(struct usb_device *dev) static int snd_usb_mbox3_boot_quirk(struct usb_device *dev) { struct usb_host_config *config = dev->actconfig; - struct usb_device_descriptor *new_device_descriptor __free(kfree) = NULL; int err; int descriptor_size; @@ -1280,7 +1278,8 @@ static int snd_usb_mbox3_boot_quirk(struct usb_device *dev) dev_dbg(&dev->dev, "MBOX3: device initialised!\n"); - new_device_descriptor = kmalloc(sizeof(*new_device_descriptor), GFP_KERNEL); + struct usb_device_descriptor *new_device_descriptor __free(kfree) = + kmalloc(sizeof(*new_device_descriptor), GFP_KERNEL); if (!new_device_descriptor) return -ENOMEM; @@ -2147,6 +2146,8 @@ struct usb_audio_quirk_flags_table { static const struct usb_audio_quirk_flags_table quirk_flags_table[] = { /* Device matches */ + DEVICE_FLG(0x001f, 0x0b21, /* AB13X USB Audio */ + QUIRK_FLAG_FORCE_IFACE_RESET | QUIRK_FLAG_IFACE_DELAY), DEVICE_FLG(0x03f0, 0x654a, /* HP 320 FHD Webcam */ QUIRK_FLAG_GET_SAMPLE_RATE | QUIRK_FLAG_MIC_RES_16), DEVICE_FLG(0x041e, 0x3000, /* Creative SB Extigy */ @@ -2236,6 +2237,10 @@ static const struct usb_audio_quirk_flags_table quirk_flags_table[] = { DEVICE_FLG(0x0644, 0x806c, /* Esoteric XD */ QUIRK_FLAG_ITF_USB_DSD_DAC | QUIRK_FLAG_CTL_MSG_DELAY | QUIRK_FLAG_IFACE_DELAY | QUIRK_FLAG_FORCE_IFACE_RESET), + DEVICE_FLG(0x0661, 0x0883, /* iBasso DC04 Ultra */ + QUIRK_FLAG_DSD_RAW), + DEVICE_FLG(0x0666, 0x0880, /* SPACETOUCH USB Audio */ + QUIRK_FLAG_FORCE_IFACE_RESET | QUIRK_FLAG_IFACE_DELAY), DEVICE_FLG(0x06f8, 0xb000, /* Hercules DJ Console (Windows Edition) */ QUIRK_FLAG_IGNORE_CTL_ERROR), DEVICE_FLG(0x06f8, 0xd002, /* Hercules DJ Console (Macintosh Edition) */ @@ -2294,6 +2299,8 @@ static const struct usb_audio_quirk_flags_table quirk_flags_table[] = { QUIRK_FLAG_PLAYBACK_FIRST | QUIRK_FLAG_GENERIC_IMPLICIT_FB), DEVICE_FLG(0x13e5, 0x0001, /* Serato Phono */ QUIRK_FLAG_IGNORE_CTL_ERROR), + DEVICE_FLG(0x152a, 0x880a, /* NeuralDSP Quad Cortex */ + 0), /* Doesn't have the vendor quirk which would otherwise apply */ DEVICE_FLG(0x154e, 0x1002, /* Denon DCD-1500RE */ QUIRK_FLAG_ITF_USB_DSD_DAC | QUIRK_FLAG_CTL_MSG_DELAY), DEVICE_FLG(0x154e, 0x1003, /* Denon DA-300USB */ @@ -2360,6 +2367,8 @@ static const struct usb_audio_quirk_flags_table quirk_flags_table[] = { QUIRK_FLAG_SHARE_MEDIA_DEVICE | QUIRK_FLAG_ALIGN_TRANSFER), DEVICE_FLG(0x2040, 0x7281, /* Hauppauge HVR-950Q-MXL */ QUIRK_FLAG_SHARE_MEDIA_DEVICE | QUIRK_FLAG_ALIGN_TRANSFER), + DEVICE_FLG(0x20b1, 0x2009, /* XMOS Ltd DIYINHK USB Audio 2.0 */ + QUIRK_FLAG_SKIP_IMPLICIT_FB | QUIRK_FLAG_DSD_RAW), DEVICE_FLG(0x2040, 0x8200, /* Hauppauge Woodbury */ QUIRK_FLAG_SHARE_MEDIA_DEVICE | QUIRK_FLAG_ALIGN_TRANSFER), DEVICE_FLG(0x21b4, 0x0081, /* AudioQuest DragonFly */ @@ -2418,8 +2427,12 @@ static const struct usb_audio_quirk_flags_table quirk_flags_table[] = { QUIRK_FLAG_CTL_MSG_DELAY | QUIRK_FLAG_IFACE_DELAY), VENDOR_FLG(0x07fd, /* MOTU */ QUIRK_FLAG_VALIDATE_RATES), + DEVICE_FLG(0x1235, 0x8006, 0), /* Focusrite Scarlett 2i2 1st Gen */ + DEVICE_FLG(0x1235, 0x800a, 0), /* Focusrite Scarlett 2i4 1st Gen */ + DEVICE_FLG(0x1235, 0x8016, 0), /* Focusrite Scarlett 2i2 1st Gen */ + DEVICE_FLG(0x1235, 0x801c, 0), /* Focusrite Scarlett Solo 1st Gen */ VENDOR_FLG(0x1235, /* Focusrite Novation */ - QUIRK_FLAG_VALIDATE_RATES), + QUIRK_FLAG_SKIP_IFACE_SETUP), VENDOR_FLG(0x1511, /* AURALiC */ QUIRK_FLAG_DSD_RAW), VENDOR_FLG(0x152a, /* Thesycon devices */ @@ -2501,6 +2514,7 @@ static const char *const snd_usb_audio_quirk_flag_names[] = { QUIRK_STRING_ENTRY(MIC_RES_384), QUIRK_STRING_ENTRY(MIXER_PLAYBACK_MIN_MUTE), QUIRK_STRING_ENTRY(MIXER_CAPTURE_MIN_MUTE), + QUIRK_STRING_ENTRY(SKIP_IFACE_SETUP), NULL }; diff --git a/sound/usb/stream.c b/sound/usb/stream.c index ec7d756d78d178..421e94b233e175 100644 --- a/sound/usb/stream.c +++ b/sound/usb/stream.c @@ -1259,6 +1259,9 @@ static int __snd_usb_parse_audio_interface(struct snd_usb_audio *chip, set_iface_first = true; /* try to set the interface... */ + if (chip->quirk_flags & QUIRK_FLAG_SKIP_IFACE_SETUP) + continue; + usb_set_interface(chip->dev, iface_no, 0); if (set_iface_first) usb_set_interface(chip->dev, iface_no, altno); diff --git a/sound/usb/usbaudio.h b/sound/usb/usbaudio.h index 79978cae9799cd..085530cf62d924 100644 --- a/sound/usb/usbaudio.h +++ b/sound/usb/usbaudio.h @@ -224,6 +224,10 @@ extern bool snd_usb_skip_validation; * playback value represents muted state instead of minimum audible volume * QUIRK_FLAG_MIXER_CAPTURE_MIN_MUTE * Similar to QUIRK_FLAG_MIXER_PLAYBACK_MIN_MUTE, but for capture streams + * QUIRK_FLAG_SKIP_IFACE_SETUP + * Skip the probe-time interface setup (usb_set_interface, + * init_pitch, init_sample_rate); redundant with + * snd_usb_endpoint_prepare() at stream-open time */ enum { @@ -253,6 +257,7 @@ enum { QUIRK_TYPE_MIC_RES_384 = 23, QUIRK_TYPE_MIXER_PLAYBACK_MIN_MUTE = 24, QUIRK_TYPE_MIXER_CAPTURE_MIN_MUTE = 25, + QUIRK_TYPE_SKIP_IFACE_SETUP = 26, /* Please also edit snd_usb_audio_quirk_flag_names */ }; @@ -284,5 +289,6 @@ enum { #define QUIRK_FLAG_MIC_RES_384 QUIRK_FLAG(MIC_RES_384) #define QUIRK_FLAG_MIXER_PLAYBACK_MIN_MUTE QUIRK_FLAG(MIXER_PLAYBACK_MIN_MUTE) #define QUIRK_FLAG_MIXER_CAPTURE_MIN_MUTE QUIRK_FLAG(MIXER_CAPTURE_MIN_MUTE) +#define QUIRK_FLAG_SKIP_IFACE_SETUP QUIRK_FLAG(SKIP_IFACE_SETUP) #endif /* __USBAUDIO_H */ diff --git a/sound/usb/usx2y/us144mkii.c b/sound/usb/usx2y/us144mkii.c index f6572a576c150e..fd028715e9ec04 100644 --- a/sound/usb/usx2y/us144mkii.c +++ b/sound/usb/usx2y/us144mkii.c @@ -412,7 +412,6 @@ static int tascam_probe(struct usb_interface *intf, struct snd_card *card; struct tascam_card *tascam; int err; - char *handshake_buf __free(kfree) = NULL; if (dev->speed != USB_SPEED_HIGH) dev_info( @@ -421,7 +420,11 @@ static int tascam_probe(struct usb_interface *intf, /* The device has two interfaces; we drive both from this driver. */ if (intf->cur_altsetting->desc.bInterfaceNumber == 1) { - tascam = usb_get_intfdata(usb_ifnum_to_if(dev, 0)); + struct usb_interface *intf_zero = usb_ifnum_to_if(dev, 0); + + if (!intf_zero) + return -ENODEV; + tascam = usb_get_intfdata(intf_zero); if (tascam) { usb_set_intfdata(intf, tascam); tascam->iface1 = intf; @@ -439,7 +442,8 @@ static int tascam_probe(struct usb_interface *intf, return -ENOENT; } - handshake_buf = kmalloc(1, GFP_KERNEL); + char *handshake_buf __free(kfree) = + kmalloc(1, GFP_KERNEL); if (!handshake_buf) return -ENOMEM; diff --git a/sound/usb/usx2y/us144mkii_controls.c b/sound/usb/usx2y/us144mkii_controls.c index 5d69441ef414b9..62055fb8e7bace 100644 --- a/sound/usb/usx2y/us144mkii_controls.c +++ b/sound/usb/usx2y/us144mkii_controls.c @@ -373,7 +373,6 @@ static int tascam_samplerate_get(struct snd_kcontrol *kcontrol, { struct tascam_card *tascam = (struct tascam_card *)snd_kcontrol_chip(kcontrol); - u8 *buf __free(kfree) = NULL; int err; u32 rate = 0; @@ -384,7 +383,8 @@ static int tascam_samplerate_get(struct snd_kcontrol *kcontrol, } } - buf = kmalloc(3, GFP_KERNEL); + u8 *buf __free(kfree) = + kmalloc(3, GFP_KERNEL); if (!buf) return -ENOMEM; diff --git a/sound/usb/usx2y/us144mkii_pcm.c b/sound/usb/usx2y/us144mkii_pcm.c index 0c84304d462469..03dfb1f3880122 100644 --- a/sound/usb/usx2y/us144mkii_pcm.c +++ b/sound/usb/usx2y/us144mkii_pcm.c @@ -115,7 +115,6 @@ void process_capture_routing_us144mkii(struct tascam_card *tascam, int us144mkii_configure_device_for_rate(struct tascam_card *tascam, int rate) { struct usb_device *dev = tascam->dev; - u8 *rate_payload_buf __free(kfree) = NULL; u16 rate_vendor_wValue; int err = 0; const u8 *current_payload_src; @@ -148,7 +147,8 @@ int us144mkii_configure_device_for_rate(struct tascam_card *tascam, int rate) return -EINVAL; } - rate_payload_buf = kmemdup(current_payload_src, 3, GFP_KERNEL); + u8 *rate_payload_buf __free(kfree) = + kmemdup(current_payload_src, 3, GFP_KERNEL); if (!rate_payload_buf) return -ENOMEM; diff --git a/sound/usb/validate.c b/sound/usb/validate.c index 4bb4893f6e74f7..f62b7cc041dc9b 100644 --- a/sound/usb/validate.c +++ b/sound/usb/validate.c @@ -281,7 +281,7 @@ static const struct usb_desc_validator audio_validators[] = { /* UAC_VERSION_2, UAC2_SAMPLE_RATE_CONVERTER: not implemented yet */ /* UAC3 */ - FIXED(UAC_VERSION_2, UAC_HEADER, struct uac3_ac_header_descriptor), + FIXED(UAC_VERSION_3, UAC_HEADER, struct uac3_ac_header_descriptor), FIXED(UAC_VERSION_3, UAC_INPUT_TERMINAL, struct uac3_input_terminal_descriptor), FIXED(UAC_VERSION_3, UAC_OUTPUT_TERMINAL, diff --git a/tools/arch/arm64/include/uapi/asm/unistd.h b/tools/arch/arm64/include/uapi/asm/unistd.h index df36f23876e863..9306726337fe00 100644 --- a/tools/arch/arm64/include/uapi/asm/unistd.h +++ b/tools/arch/arm64/include/uapi/asm/unistd.h @@ -1,2 +1,24 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -#include +/* + * Copyright (C) 2012 ARM Ltd. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#define __ARCH_WANT_RENAMEAT +#define __ARCH_WANT_NEW_STAT +#define __ARCH_WANT_SET_GET_RLIMIT +#define __ARCH_WANT_TIME32_SYSCALLS +#define __ARCH_WANT_MEMFD_SECRET + +#include diff --git a/tools/bootconfig/main.c b/tools/bootconfig/main.c index 55d59ed507d541..643f707b8f1da1 100644 --- a/tools/bootconfig/main.c +++ b/tools/bootconfig/main.c @@ -162,8 +162,11 @@ static int load_xbc_file(const char *path, char **buf) if (fd < 0) return -errno; ret = fstat(fd, &stat); - if (ret < 0) - return -errno; + if (ret < 0) { + ret = -errno; + close(fd); + return ret; + } ret = load_xbc_fd(fd, buf, stat.st_size); diff --git a/tools/bpf/bpftool/Makefile b/tools/bpf/bpftool/Makefile index 5442073a2e4286..519ea5cb8ab1c0 100644 --- a/tools/bpf/bpftool/Makefile +++ b/tools/bpf/bpftool/Makefile @@ -130,8 +130,8 @@ include $(FEATURES_DUMP) endif endif -LIBS = $(LIBBPF) -lelf -lz -lcrypto -LIBS_BOOTSTRAP = $(LIBBPF_BOOTSTRAP) -lelf -lz -lcrypto +LIBS = $(LIBBPF) -lelf -lcrypto -lz +LIBS_BOOTSTRAP = $(LIBBPF_BOOTSTRAP) -lelf -lcrypto -lz ifeq ($(feature-libelf-zstd),1) LIBS += -lzstd diff --git a/tools/bpf/bpftool/net.c b/tools/bpf/bpftool/net.c index cfc6f944f7c33a..1a06b0b5eef350 100644 --- a/tools/bpf/bpftool/net.c +++ b/tools/bpf/bpftool/net.c @@ -156,7 +156,7 @@ static int netlink_recv(int sock, __u32 nl_pid, __u32 seq, bool multipart = true; struct nlmsgerr *err; struct nlmsghdr *nh; - char buf[4096]; + char buf[8192]; int len, ret; while (multipart) { @@ -201,6 +201,9 @@ static int netlink_recv(int sock, __u32 nl_pid, __u32 seq, return ret; } } + + if (len) + p_err("Invalid message or trailing data in Netlink response: %d bytes left", len); } ret = 0; done: diff --git a/tools/docs/find-unused-docs.sh b/tools/docs/find-unused-docs.sh index 05552dbda5bcb2..ca4e607ec3f720 100755 --- a/tools/docs/find-unused-docs.sh +++ b/tools/docs/find-unused-docs.sh @@ -28,7 +28,7 @@ if ! [ -d "$1" ]; then fi cd "$( dirname "${BASH_SOURCE[0]}" )" -cd .. +cd ../.. cd Documentation/ diff --git a/tools/include/linux/bitfield.h b/tools/include/linux/bitfield.h index 6093fa6db2600b..ddf81f24956ba0 100644 --- a/tools/include/linux/bitfield.h +++ b/tools/include/linux/bitfield.h @@ -8,6 +8,7 @@ #define _LINUX_BITFIELD_H #include +#include #include /* diff --git a/tools/include/nolibc/Makefile b/tools/include/nolibc/Makefile index 8118e22844f199..afb6a20ac1336c 100644 --- a/tools/include/nolibc/Makefile +++ b/tools/include/nolibc/Makefile @@ -103,9 +103,12 @@ headers_standalone: headers $(Q)$(MAKE) -C $(srctree) headers $(Q)$(MAKE) -C $(srctree) headers_install INSTALL_HDR_PATH=$(OUTPUT)sysroot +CFLAGS_s390 := -m64 +CFLAGS := $(CFLAGS_$(ARCH)) + headers_check: headers_standalone $(Q)for header in $(filter-out crt.h std.h,$(all_files)); do \ - $(CC) $(CLANG_CROSS_FLAGS) -Wall -Werror -nostdinc -fsyntax-only -x c /dev/null \ + $(CC) $(CFLAGS) $(CLANG_CROSS_FLAGS) -Wall -Werror -nostdinc -fsyntax-only -x c /dev/null \ -I$(or $(objtree),$(srctree))/usr/include -include $$header -include $$header || exit 1; \ done diff --git a/tools/lib/bpf/btf_dump.c b/tools/lib/bpf/btf_dump.c index 6388392f49a0b5..53c6624161d797 100644 --- a/tools/lib/bpf/btf_dump.c +++ b/tools/lib/bpf/btf_dump.c @@ -1762,9 +1762,18 @@ static int btf_dump_get_bitfield_value(struct btf_dump *d, __u16 left_shift_bits, right_shift_bits; const __u8 *bytes = data; __u8 nr_copy_bits; + __u8 start_bit, nr_bytes; __u64 num = 0; int i; + /* Calculate how many bytes cover the bitfield */ + start_bit = bits_offset % 8; + nr_bytes = (start_bit + bit_sz + 7) / 8; + + /* Bound check */ + if (data + nr_bytes > d->typed_dump->data_end) + return -E2BIG; + /* Maximum supported bitfield size is 64 bits */ if (t->size > 8) { pr_warn("unexpected bitfield size %d\n", t->size); diff --git a/tools/lib/bpf/linker.c b/tools/lib/bpf/linker.c index f4403e3cf99464..78f92c39290af5 100644 --- a/tools/lib/bpf/linker.c +++ b/tools/lib/bpf/linker.c @@ -581,7 +581,7 @@ int bpf_linker__add_buf(struct bpf_linker *linker, void *buf, size_t buf_sz, written = 0; while (written < buf_sz) { - ret = write(fd, buf, buf_sz); + ret = write(fd, buf + written, buf_sz - written); if (ret < 0) { ret = -errno; pr_warn("failed to write '%s': %s\n", filename, errstr(ret)); diff --git a/tools/lib/bpf/netlink.c b/tools/lib/bpf/netlink.c index c997e69d507fef..c9a78fb16f115a 100644 --- a/tools/lib/bpf/netlink.c +++ b/tools/lib/bpf/netlink.c @@ -143,7 +143,7 @@ static int libbpf_netlink_recv(int sock, __u32 nl_pid, int seq, struct nlmsghdr *nh; int len, ret; - ret = alloc_iov(&iov, 4096); + ret = alloc_iov(&iov, 8192); if (ret) goto done; @@ -212,6 +212,8 @@ static int libbpf_netlink_recv(int sock, __u32 nl_pid, int seq, } } } + if (len) + pr_warn("Invalid message or trailing data in Netlink response: %d bytes left\n", len); } ret = 0; done: diff --git a/tools/lib/perf/Makefile b/tools/lib/perf/Makefile index 7fbb50b74c00b3..5c64122bf5374a 100644 --- a/tools/lib/perf/Makefile +++ b/tools/lib/perf/Makefile @@ -51,9 +51,9 @@ INCLUDES = \ -I$(srctree)/tools/include/uapi # Append required CFLAGS +override CFLAGS := $(INCLUDES) $(CFLAGS) override CFLAGS += -g -Werror -Wall override CFLAGS += -fPIC -override CFLAGS += $(INCLUDES) override CFLAGS += -fvisibility=hidden override CFLAGS += $(EXTRA_WARNINGS) override CFLAGS += $(EXTRA_CFLAGS) diff --git a/tools/lib/python/kdoc/kdoc_parser.py b/tools/lib/python/kdoc/kdoc_parser.py index 500aafc500322c..2168d623f78675 100644 --- a/tools/lib/python/kdoc/kdoc_parser.py +++ b/tools/lib/python/kdoc/kdoc_parser.py @@ -295,7 +295,7 @@ def contents(self): # TODO: rename to emit_message after removal of kernel-doc.pl def emit_msg(self, ln, msg, *, warning=True): - """Emit a message""" + """Emit a message.""" log_msg = f"{self.fname}:{ln} {msg}" @@ -448,18 +448,37 @@ def output_declaration(self, dtype, name, **args): self.config.log.debug("Output: %s:%s = %s", dtype, name, pformat(args)) + def emit_unused_warnings(self): + """ + When the parser fails to produce a valid entry, it places some + warnings under `entry.warnings` that will be discarded when resetting + the state. + + Ensure that those warnings are not lost. + + .. note:: + + Because we are calling `config.warning()` here, those + warnings are not filtered by the `-W` parameters: they will all + be produced even when `-Wreturn`, `-Wshort-desc`, and/or + `-Wcontents-before-sections` are used. + + Allowing those warnings to be filtered is complex, because it + would require storing them in a buffer and then filtering them + during the output step of the code, depending on the + selected symbols. + """ + if self.entry and self.entry not in self.entries: + for log_msg in self.entry.warnings: + self.config.warning(log_msg) + def reset_state(self, ln): """ Ancillary routine to create a new entry. It initializes all variables used by the state machine. """ - # - # Flush the warnings out before we proceed further - # - if self.entry and self.entry not in self.entries: - for log_msg in self.entry.warnings: - self.config.log.warning(log_msg) + self.emit_unused_warnings() self.entry = KernelEntry(self.config, self.fname, ln) @@ -1664,6 +1683,8 @@ def parse_kdoc(self): # Hand this line to the appropriate state handler self.state_actions[self.state](self, ln, line) + self.emit_unused_warnings() + except OSError: self.config.log.error(f"Error: Cannot open file {self.fname}") diff --git a/tools/lib/subcmd/help.c b/tools/lib/subcmd/help.c index ddaeb4eb3e2497..db94aa685b73b8 100644 --- a/tools/lib/subcmd/help.c +++ b/tools/lib/subcmd/help.c @@ -97,11 +97,13 @@ void exclude_cmds(struct cmdnames *cmds, struct cmdnames *excludes) ei++; } } - if (ci != cj) { - while (ci < cmds->cnt) { - cmds->names[cj++] = cmds->names[ci]; - cmds->names[ci++] = NULL; + while (ci < cmds->cnt) { + if (ci != cj) { + cmds->names[cj] = cmds->names[ci]; + cmds->names[ci] = NULL; } + ci++; + cj++; } for (ci = cj; ci < cmds->cnt; ci++) assert(cmds->names[ci] == NULL); diff --git a/tools/net/sunrpc/xdrgen/generators/__init__.py b/tools/net/sunrpc/xdrgen/generators/__init__.py index e22632cf38fbec..1d577a986c6cf8 100644 --- a/tools/net/sunrpc/xdrgen/generators/__init__.py +++ b/tools/net/sunrpc/xdrgen/generators/__init__.py @@ -6,7 +6,7 @@ from jinja2 import Environment, FileSystemLoader, Template from xdr_ast import _XdrAst, Specification, _RpcProgram, _XdrTypeSpecifier -from xdr_ast import public_apis, pass_by_reference, get_header_name +from xdr_ast import public_apis, pass_by_reference, structs, get_header_name from xdr_parse import get_xdr_annotate @@ -25,6 +25,7 @@ def create_jinja2_environment(language: str, xdr_type: str) -> Environment: environment.globals["annotate"] = get_xdr_annotate() environment.globals["public_apis"] = public_apis environment.globals["pass_by_reference"] = pass_by_reference + environment.globals["structs"] = structs return environment case _: raise NotImplementedError("Language not supported") diff --git a/tools/net/sunrpc/xdrgen/templates/C/program/decoder/argument.j2 b/tools/net/sunrpc/xdrgen/templates/C/program/decoder/argument.j2 index 0b1709cca0d4a4..19b219dd276d3e 100644 --- a/tools/net/sunrpc/xdrgen/templates/C/program/decoder/argument.j2 +++ b/tools/net/sunrpc/xdrgen/templates/C/program/decoder/argument.j2 @@ -14,7 +14,11 @@ bool {{ program }}_svc_decode_{{ argument }}(struct svc_rqst *rqstp, struct xdr_ {% if argument == 'void' %} return xdrgen_decode_void(xdr); {% else %} +{% if argument in structs %} struct {{ argument }} *argp = rqstp->rq_argp; +{% else %} + {{ argument }} *argp = rqstp->rq_argp; +{% endif %} return xdrgen_decode_{{ argument }}(xdr, argp); {% endif %} diff --git a/tools/net/sunrpc/xdrgen/templates/C/program/encoder/result.j2 b/tools/net/sunrpc/xdrgen/templates/C/program/encoder/result.j2 index 6fc61a5d47b7f5..746592cfda5628 100644 --- a/tools/net/sunrpc/xdrgen/templates/C/program/encoder/result.j2 +++ b/tools/net/sunrpc/xdrgen/templates/C/program/encoder/result.j2 @@ -14,8 +14,14 @@ bool {{ program }}_svc_encode_{{ result }}(struct svc_rqst *rqstp, struct xdr_st {% if result == 'void' %} return xdrgen_encode_void(xdr); {% else %} +{% if result in structs %} struct {{ result }} *resp = rqstp->rq_resp; return xdrgen_encode_{{ result }}(xdr, resp); +{% else %} + {{ result }} *resp = rqstp->rq_resp; + + return xdrgen_encode_{{ result }}(xdr, *resp); +{% endif %} {% endif %} } diff --git a/tools/net/sunrpc/xdrgen/templates/C/source_top/client.j2 b/tools/net/sunrpc/xdrgen/templates/C/source_top/client.j2 index c5518c519854a0..df3598c38b2c29 100644 --- a/tools/net/sunrpc/xdrgen/templates/C/source_top/client.j2 +++ b/tools/net/sunrpc/xdrgen/templates/C/source_top/client.j2 @@ -8,6 +8,5 @@ #include #include #include -#include #include diff --git a/tools/objtool/Makefile b/tools/objtool/Makefile index a40f3023292910..b71d1886022e9b 100644 --- a/tools/objtool/Makefile +++ b/tools/objtool/Makefile @@ -13,7 +13,7 @@ endif ifeq ($(ARCH_HAS_KLP),y) HAVE_XXHASH = $(shell printf "$(pound)include \nXXH3_state_t *state;int main() {}" | \ - $(HOSTCC) -xc - -o /dev/null -lxxhash 2> /dev/null && echo y || echo n) + $(HOSTCC) $(HOSTCFLAGS) -xc - -o /dev/null -lxxhash 2> /dev/null && echo y || echo n) ifeq ($(HAVE_XXHASH),y) BUILD_KLP := y LIBXXHASH_CFLAGS := $(shell $(HOSTPKG_CONFIG) libxxhash --cflags 2>/dev/null) \ @@ -29,6 +29,8 @@ srctree := $(patsubst %/,%,$(dir $(CURDIR))) srctree := $(patsubst %/,%,$(dir $(srctree))) endif +RM ?= rm -f + LIBSUBCMD_DIR = $(srctree)/tools/lib/subcmd/ ifneq ($(OUTPUT),) LIBSUBCMD_OUTPUT = $(abspath $(OUTPUT))/libsubcmd @@ -140,13 +142,15 @@ $(LIBSUBCMD)-clean: $(Q)$(RM) -r -- $(LIBSUBCMD_OUTPUT) clean: $(LIBSUBCMD)-clean - $(call QUIET_CLEAN, objtool) $(RM) $(OBJTOOL) - $(Q)find $(OUTPUT) -name '*.o' -delete -o -name '\.*.cmd' -delete -o -name '\.*.d' -delete + $(Q)find $(OUTPUT) \( -name '*.o' -o -name '\.*.cmd' -o -name '\.*.d' \) -type f -print | xargs $(RM) $(Q)$(RM) $(OUTPUT)arch/x86/lib/cpu-feature-names.c $(OUTPUT)fixdep $(Q)$(RM) $(OUTPUT)arch/x86/lib/inat-tables.c $(OUTPUT)fixdep $(Q)$(RM) -- $(OUTPUT)FEATURE-DUMP.objtool $(Q)$(RM) -r -- $(OUTPUT)feature +mrproper: clean + $(call QUIET_CLEAN, objtool) $(RM) $(OBJTOOL) + FORCE: -.PHONY: clean FORCE +.PHONY: clean mrproper FORCE diff --git a/tools/objtool/arch/x86/decode.c b/tools/objtool/arch/x86/decode.c index f4af825082284f..4544c2cb44400e 100644 --- a/tools/objtool/arch/x86/decode.c +++ b/tools/objtool/arch/x86/decode.c @@ -395,52 +395,36 @@ int arch_decode_instruction(struct objtool_file *file, const struct section *sec if (!rex_w) break; - if (modrm_reg == CFI_SP) { - - if (mod_is_reg()) { - /* mov %rsp, reg */ - ADD_OP(op) { - op->src.type = OP_SRC_REG; - op->src.reg = CFI_SP; - op->dest.type = OP_DEST_REG; - op->dest.reg = modrm_rm; - } - break; - - } else { - /* skip RIP relative displacement */ - if (is_RIP()) - break; - - /* skip nontrivial SIB */ - if (have_SIB()) { - modrm_rm = sib_base; - if (sib_index != CFI_SP) - break; - } - - /* mov %rsp, disp(%reg) */ - ADD_OP(op) { - op->src.type = OP_SRC_REG; - op->src.reg = CFI_SP; - op->dest.type = OP_DEST_REG_INDIRECT; - op->dest.reg = modrm_rm; - op->dest.offset = ins.displacement.value; - } - break; + if (mod_is_reg()) { + /* mov reg, reg */ + ADD_OP(op) { + op->src.type = OP_SRC_REG; + op->src.reg = modrm_reg; + op->dest.type = OP_DEST_REG; + op->dest.reg = modrm_rm; } - break; } - if (rm_is_reg(CFI_SP)) { + /* skip RIP relative displacement */ + if (is_RIP()) + break; - /* mov reg, %rsp */ + /* skip nontrivial SIB */ + if (have_SIB()) { + modrm_rm = sib_base; + if (sib_index != CFI_SP) + break; + } + + /* mov %rsp, disp(%reg) */ + if (modrm_reg == CFI_SP) { ADD_OP(op) { op->src.type = OP_SRC_REG; - op->src.reg = modrm_reg; - op->dest.type = OP_DEST_REG; - op->dest.reg = CFI_SP; + op->src.reg = CFI_SP; + op->dest.type = OP_DEST_REG_INDIRECT; + op->dest.reg = modrm_rm; + op->dest.offset = ins.displacement.value; } break; } diff --git a/tools/objtool/check.c b/tools/objtool/check.c index 3fd98c5b6e1a88..948e0cb3141d3c 100644 --- a/tools/objtool/check.c +++ b/tools/objtool/check.c @@ -197,7 +197,8 @@ static bool is_rust_noreturn(const struct symbol *func) * as well as changes to the source code itself between versions (since * these come from the Rust standard library). */ - return str_ends_with(func->name, "_4core3num22from_ascii_radix_panic") || + return str_ends_with(func->name, "_4core3num20from_str_radix_panic") || + str_ends_with(func->name, "_4core3num22from_ascii_radix_panic") || str_ends_with(func->name, "_4core5sliceSp15copy_from_slice17len_mismatch_fail") || str_ends_with(func->name, "_4core6option13expect_failed") || str_ends_with(func->name, "_4core6option13unwrap_failed") || @@ -1260,7 +1261,7 @@ static const char *uaccess_safe_builtin[] = { "copy_mc_enhanced_fast_string", "rep_stos_alternative", "rep_movs_alternative", - "__copy_user_nocache", + "copy_to_nontemporal", NULL }; @@ -2143,12 +2144,11 @@ static void mark_func_jump_tables(struct objtool_file *file, last = insn; /* - * Store back-pointers for unconditional forward jumps such + * Store back-pointers for forward jumps such * that find_jump_table() can back-track using those and * avoid some potentially confusing code. */ - if (insn->type == INSN_JUMP_UNCONDITIONAL && insn->jump_dest && - insn->offset > last->offset && + if (insn->jump_dest && insn->jump_dest->offset > insn->offset && !insn->jump_dest->first_jump_src) { @@ -2959,6 +2959,20 @@ static int update_cfi_state(struct instruction *insn, cfi->stack_size += 8; } + else if (cfi->vals[op->src.reg].base == CFI_CFA) { + /* + * Clang RSP musical chairs: + * + * mov %rsp, %rdx [handled above] + * ... + * mov %rdx, %rbx [handled here] + * ... + * mov %rbx, %rsp [handled above] + */ + cfi->vals[op->dest.reg].base = CFI_CFA; + cfi->vals[op->dest.reg].offset = cfi->vals[op->src.reg].offset; + } + break; @@ -3693,7 +3707,7 @@ static void checksum_update_insn(struct objtool_file *file, struct symbol *func, static int validate_branch(struct objtool_file *file, struct symbol *func, struct instruction *insn, struct insn_state state); static int do_validate_branch(struct objtool_file *file, struct symbol *func, - struct instruction *insn, struct insn_state state); + struct instruction *insn, struct insn_state *state); static int validate_insn(struct objtool_file *file, struct symbol *func, struct instruction *insn, struct insn_state *statep, @@ -3958,7 +3972,7 @@ static int validate_insn(struct objtool_file *file, struct symbol *func, * tools/objtool/Documentation/objtool.txt. */ static int do_validate_branch(struct objtool_file *file, struct symbol *func, - struct instruction *insn, struct insn_state state) + struct instruction *insn, struct insn_state *state) { struct instruction *next_insn, *prev_insn = NULL; bool dead_end; @@ -3989,7 +4003,7 @@ static int do_validate_branch(struct objtool_file *file, struct symbol *func, return 1; } - ret = validate_insn(file, func, insn, &state, prev_insn, next_insn, + ret = validate_insn(file, func, insn, state, prev_insn, next_insn, &dead_end); if (!insn->trace) { @@ -4000,7 +4014,7 @@ static int do_validate_branch(struct objtool_file *file, struct symbol *func, } if (!dead_end && !next_insn) { - if (state.cfi.cfa.base == CFI_UNDEFINED) + if (state->cfi.cfa.base == CFI_UNDEFINED) return 0; if (file->ignore_unreachables) return 0; @@ -4025,7 +4039,7 @@ static int validate_branch(struct objtool_file *file, struct symbol *func, int ret; trace_depth_inc(); - ret = do_validate_branch(file, func, insn, state); + ret = do_validate_branch(file, func, insn, &state); trace_depth_dec(); return ret; diff --git a/tools/objtool/elf.c b/tools/objtool/elf.c index 2c02c7b492658c..2ffe3ebfbe37c1 100644 --- a/tools/objtool/elf.c +++ b/tools/objtool/elf.c @@ -16,7 +16,6 @@ #include #include #include -#include #include #include #include @@ -1189,7 +1188,7 @@ struct elf *elf_open_read(const char *name, int flags) struct elf *elf_create_file(GElf_Ehdr *ehdr, const char *name) { struct section *null, *symtab, *strtab, *shstrtab; - char *dir, *base, *tmp_name; + char *tmp_name; struct symbol *sym; struct elf *elf; @@ -1203,29 +1202,13 @@ struct elf *elf_create_file(GElf_Ehdr *ehdr, const char *name) INIT_LIST_HEAD(&elf->sections); - dir = strdup(name); - if (!dir) { - ERROR_GLIBC("strdup"); - return NULL; - } - - dir = dirname(dir); - - base = strdup(name); - if (!base) { - ERROR_GLIBC("strdup"); - return NULL; - } - - base = basename(base); - - tmp_name = malloc(256); + tmp_name = malloc(strlen(name) + 8); if (!tmp_name) { ERROR_GLIBC("malloc"); return NULL; } - snprintf(tmp_name, 256, "%s/%s.XXXXXX", dir, base); + sprintf(tmp_name, "%s.XXXXXX", name); elf->fd = mkstemp(tmp_name); if (elf->fd == -1) { @@ -1375,7 +1358,7 @@ void *elf_add_data(struct elf *elf, struct section *sec, const void *data, size_ memcpy(sec->data->d_buf, data, size); sec->data->d_size = size; - sec->data->d_align = 1; + sec->data->d_align = sec->sh.sh_addralign; offset = ALIGN(sec->sh.sh_size, sec->sh.sh_addralign); sec->sh.sh_size = offset + size; diff --git a/tools/objtool/klp-diff.c b/tools/objtool/klp-diff.c index 9f1f4011eb9cda..b9340ef8d370c2 100644 --- a/tools/objtool/klp-diff.c +++ b/tools/objtool/klp-diff.c @@ -14,6 +14,7 @@ #include #include +#include #include #include #include @@ -560,7 +561,7 @@ static struct symbol *__clone_symbol(struct elf *elf, struct symbol *patched_sym } if (!is_sec_sym(patched_sym)) - offset = sec_size(out_sec); + offset = ALIGN(sec_size(out_sec), out_sec->sh.sh_addralign); if (patched_sym->len || is_sec_sym(patched_sym)) { void *data = NULL; @@ -1334,18 +1335,18 @@ static bool should_keep_special_sym(struct elf *elf, struct symbol *sym) * be applied after static branch/call init, resulting in code corruption. * * Validate a special section entry to avoid that. Note that an inert - * tracepoint is harmless enough, in that case just skip the entry and print a - * warning. Otherwise, return an error. + * tracepoint or pr_debug() is harmless enough, in that case just skip the + * entry and print a warning. Otherwise, return an error. * - * This is only a temporary limitation which will be fixed when livepatch adds - * support for submodules: fully self-contained modules which are embedded in - * the top-level livepatch module's data and which can be loaded on demand when - * their corresponding to-be-patched module gets loaded. Then klp relocs can - * be retired. + * TODO: This is only a temporary limitation which will be fixed when livepatch + * adds support for submodules: fully self-contained modules which are embedded + * in the top-level livepatch module's data and which can be loaded on demand + * when their corresponding to-be-patched module gets loaded. Then klp relocs + * can be retired. * * Return: * -1: error: validation failed - * 1: warning: tracepoint skipped + * 1: warning: disabled tracepoint or pr_debug() * 0: success */ static int validate_special_section_klp_reloc(struct elfs *e, struct symbol *sym) @@ -1364,6 +1365,9 @@ static int validate_special_section_klp_reloc(struct elfs *e, struct symbol *sym const char *sym_modname; struct export *export; + if (convert_reloc_sym(e->patched, reloc)) + continue; + /* Static branch/call keys are always STT_OBJECT */ if (reloc->sym->type != STT_OBJECT) { @@ -1400,6 +1404,13 @@ static int validate_special_section_klp_reloc(struct elfs *e, struct symbol *sym continue; } + if (strstr(reloc->sym->name, "__UNIQUE_ID_ddebug_")) { + WARN("%s: disabling unsupported pr_debug()", + code_sym->name); + ret = 1; + continue; + } + ERROR("%s+0x%lx: unsupported static branch key %s. Use static_key_enabled() instead", code_sym->name, code_offset, reloc->sym->name); return -1; diff --git a/tools/perf/Makefile.perf b/tools/perf/Makefile.perf index b3f481a626afa3..fbeb5c81c80727 100644 --- a/tools/perf/Makefile.perf +++ b/tools/perf/Makefile.perf @@ -86,8 +86,6 @@ include ../scripts/utilities.mak # # Define NO_LIBBPF if you do not want BPF support # -# Define NO_LIBCAP if you do not want process capabilities considered by perf -# # Define NO_SDT if you do not want to define SDT event in perf tools, # note that it doesn't disable SDT scanning support. # @@ -251,11 +249,12 @@ else endif # shellcheck is using in tools/perf/tests/Build with option -a/--check-sourced ( -# introduced in v0.4.7) and -S/--severity (introduced in v0.6.0). So make the -# minimal shellcheck version as v0.6.0. +# introduced in v0.4.7) and -S/--severity (introduced in v0.6.0) as well as +# dynamic source inclusions (properly handled since v0.7.2). +# So make the minimal shellcheck version as v0.7.2. ifneq ($(SHELLCHECK),) ifeq ($(shell expr $(shell $(SHELLCHECK) --version | grep version: | \ - sed -e 's/.\+ \([0-9]\+\).\([0-9]\+\).\([0-9]\+\)/\1\2\3/g') \< 060), 1) + sed -e 's/.\+ \([0-9]\+\).\([0-9]\+\).\([0-9]\+\)/\1\2\3/g') \< 072), 1) SHELLCHECK := else SHELLCHECK := $(SHELLCHECK) -s bash -a -S warning diff --git a/tools/perf/arch/loongarch/annotate/instructions.c b/tools/perf/arch/loongarch/annotate/instructions.c index 70262d5f144424..1c3abb43c8d721 100644 --- a/tools/perf/arch/loongarch/annotate/instructions.c +++ b/tools/perf/arch/loongarch/annotate/instructions.c @@ -10,9 +10,7 @@ static int loongarch_call__parse(struct arch *arch, struct ins_operands *ops, st { char *c, *endptr, *tok, *name; struct map *map = ms->map; - struct addr_map_symbol target = { - .ms = { .map = map, }, - }; + struct addr_map_symbol target; c = strchr(ops->raw, '#'); if (c++ == NULL) @@ -38,12 +36,16 @@ static int loongarch_call__parse(struct arch *arch, struct ins_operands *ops, st if (ops->target.name == NULL) return -1; - target.addr = map__objdump_2mem(map, ops->target.addr); + target = (struct addr_map_symbol) { + .ms = { .map = map__get(map), }, + .addr = map__objdump_2mem(map, ops->target.addr), + }; if (maps__find_ams(ms->maps, &target) == 0 && map__rip_2objdump(target.ms.map, map__map_ip(target.ms.map, target.addr)) == ops->target.addr) ops->target.sym = target.ms.sym; + addr_map_symbol__exit(&target); return 0; } @@ -58,7 +60,7 @@ static int loongarch_jump__parse(struct arch *arch, struct ins_operands *ops, st struct map *map = ms->map; struct symbol *sym = ms->sym; struct addr_map_symbol target = { - .ms = { .map = map, }, + .ms = { .map = map__get(map), }, }; const char *c = strchr(ops->raw, '#'); u64 start, end; @@ -90,7 +92,7 @@ static int loongarch_jump__parse(struct arch *arch, struct ins_operands *ops, st } else { ops->target.offset_avail = false; } - + addr_map_symbol__exit(&target); return 0; } diff --git a/tools/perf/arch/s390/annotate/instructions.c b/tools/perf/arch/s390/annotate/instructions.c index c61193f1e09640..626e6d2cbc81a1 100644 --- a/tools/perf/arch/s390/annotate/instructions.c +++ b/tools/perf/arch/s390/annotate/instructions.c @@ -6,9 +6,7 @@ static int s390_call__parse(struct arch *arch, struct ins_operands *ops, { char *endptr, *tok, *name; struct map *map = ms->map; - struct addr_map_symbol target = { - .ms = { .map = map, }, - }; + struct addr_map_symbol target; tok = strchr(ops->raw, ','); if (!tok) @@ -36,12 +34,17 @@ static int s390_call__parse(struct arch *arch, struct ins_operands *ops, if (ops->target.name == NULL) return -1; - target.addr = map__objdump_2mem(map, ops->target.addr); + + target = (struct addr_map_symbol) { + .ms = { .map = map__get(map), }, + .addr = map__objdump_2mem(map, ops->target.addr), + }; if (maps__find_ams(ms->maps, &target) == 0 && map__rip_2objdump(target.ms.map, map__map_ip(target.ms.map, target.addr)) == ops->target.addr) ops->target.sym = target.ms.sym; + addr_map_symbol__exit(&target); return 0; } diff --git a/tools/perf/builtin-ftrace.c b/tools/perf/builtin-ftrace.c index 6b6eec65f93f5c..4cc33452d79b62 100644 --- a/tools/perf/builtin-ftrace.c +++ b/tools/perf/builtin-ftrace.c @@ -18,6 +18,7 @@ #include #include #include +#include #include #include @@ -1209,8 +1210,12 @@ static int prepare_func_profile(struct perf_ftrace *ftrace) ftrace->graph_verbose = 0; ftrace->profile_hash = hashmap__new(profile_hash, profile_equal, NULL); - if (ftrace->profile_hash == NULL) - return -ENOMEM; + if (IS_ERR(ftrace->profile_hash)) { + int err = PTR_ERR(ftrace->profile_hash); + + ftrace->profile_hash = NULL; + return err; + } return 0; } diff --git a/tools/perf/pmu-events/arch/x86/amdzen5/load-store.json b/tools/perf/pmu-events/arch/x86/amdzen5/load-store.json index ff6627a7780579..06bbaea159259f 100644 --- a/tools/perf/pmu-events/arch/x86/amdzen5/load-store.json +++ b/tools/perf/pmu-events/arch/x86/amdzen5/load-store.json @@ -70,19 +70,19 @@ "EventName": "ls_mab_alloc.load_store_allocations", "EventCode": "0x41", "BriefDescription": "Miss Address Buffer (MAB) entries allocated by a Load-Store (LS) pipe for load-store allocations.", - "UMask": "0x3f" + "UMask": "0x07" }, { "EventName": "ls_mab_alloc.hardware_prefetcher_allocations", "EventCode": "0x41", "BriefDescription": "Miss Address Buffer (MAB) entries allocated by a Load-Store (LS) pipe for hardware prefetcher allocations.", - "UMask": "0x40" + "UMask": "0x08" }, { "EventName": "ls_mab_alloc.all_allocations", "EventCode": "0x41", "BriefDescription": "Miss Address Buffer (MAB) entries allocated by a Load-Store (LS) pipe for all types of allocations.", - "UMask": "0x7f" + "UMask": "0x0f" }, { "EventName": "ls_dmnd_fills_from_sys.local_l2", diff --git a/tools/perf/tests/kallsyms-split.c b/tools/perf/tests/kallsyms-split.c index bbbc66957e5d04..117ed3b70f630a 100644 --- a/tools/perf/tests/kallsyms-split.c +++ b/tools/perf/tests/kallsyms-split.c @@ -148,6 +148,7 @@ static int test__kallsyms_split(struct test_suite *test __maybe_unused, ret = TEST_OK; out: + map__put(map); remove_proc_dir(0); machine__exit(&m); return ret; diff --git a/tools/perf/tests/make b/tools/perf/tests/make index 6641701e482856..c721cd1bcaa9a4 100644 --- a/tools/perf/tests/make +++ b/tools/perf/tests/make @@ -122,7 +122,7 @@ make_minimal += NO_DEMANGLE=1 NO_LIBELF=1 NO_BACKTRACE=1 make_minimal += NO_LIBNUMA=1 NO_LIBBIONIC=1 NO_LIBDW=1 make_minimal += NO_LIBDW_DWARF_UNWIND=1 NO_LIBBPF=1 make_minimal += NO_SDT=1 NO_JVMTI=1 NO_LIBZSTD=1 -make_minimal += NO_LIBCAP=1 NO_CAPSTONE=1 +make_minimal += NO_CAPSTONE=1 # $(run) contains all available tests run := make_pure diff --git a/tools/perf/tests/parse-metric.c b/tools/perf/tests/parse-metric.c index 6bbc209a5c6af8..7c7f489a5eb0a6 100644 --- a/tools/perf/tests/parse-metric.c +++ b/tools/perf/tests/parse-metric.c @@ -41,6 +41,8 @@ static void load_runtime_stat(struct evlist *evlist, struct value *vals) count = find_value(evsel->name, vals); evsel->supported = true; evsel->stats->aggr->counts.val = count; + evsel->stats->aggr->counts.ena = 1; + evsel->stats->aggr->counts.run = 1; } } diff --git a/tools/perf/tests/shell/evlist.sh b/tools/perf/tests/shell/evlist.sh index 140f099e75c1ea..8a22f4171c07c2 100755 --- a/tools/perf/tests/shell/evlist.sh +++ b/tools/perf/tests/shell/evlist.sh @@ -21,13 +21,13 @@ trap trap_cleanup EXIT TERM INT test_evlist_simple() { echo "Simple evlist test" - if ! perf record -e cycles -o "${perfdata}" true 2> /dev/null + if ! perf record -e cpu-clock -o "${perfdata}" true 2> /dev/null then echo "Simple evlist [Failed record]" err=1 return fi - if ! perf evlist -i "${perfdata}" | grep -q "cycles" + if ! perf evlist -i "${perfdata}" | grep -q "cpu-clock" then echo "Simple evlist [Failed to list event]" err=1 @@ -38,13 +38,14 @@ test_evlist_simple() { test_evlist_group() { echo "Group evlist test" - if ! perf record -e "{cycles,instructions}" -o "${perfdata}" true 2> /dev/null + if ! perf record -e "{cpu-clock,task-clock}" -o "${perfdata}" \ + -- perf test -w noploop 2> /dev/null then echo "Group evlist [Skipped event group recording failed]" return fi - if ! perf evlist -i "${perfdata}" -g | grep -q "{.*cycles.*,.*instructions.*}" + if ! perf evlist -i "${perfdata}" -g | grep -q "{.*cpu-clock.*,.*task-clock.*}" then echo "Group evlist [Failed to list event group]" err=1 diff --git a/tools/perf/tests/shell/sched.sh b/tools/perf/tests/shell/sched.sh index b9b81eaf856e65..b9637069adb1f0 100755 --- a/tools/perf/tests/shell/sched.sh +++ b/tools/perf/tests/shell/sched.sh @@ -53,7 +53,7 @@ start_noploops() { } cleanup_noploops() { - kill "$PID1" "$PID2" + kill "$PID1" "$PID2" || true } test_sched_record() { diff --git a/tools/perf/tests/shell/stat.sh b/tools/perf/tests/shell/stat.sh index 0b2f0f88ca1667..792a0b79f6b86b 100755 --- a/tools/perf/tests/shell/stat.sh +++ b/tools/perf/tests/shell/stat.sh @@ -233,7 +233,7 @@ test_hybrid() { fi # Run default Perf stat - cycles_events=$(perf stat -a -- sleep 0.1 2>&1 | grep -E "/cpu-cycles/[uH]*| cpu-cycles[:uH]* " -c) + cycles_events=$(perf stat -a -- sleep 0.1 2>&1 | grep -E "/cpu-cycles/[uH]*| cpu-cycles[:uH]* " | wc -l) # The expectation is that default output will have a cycles events on each # hybrid PMU. In situations with no cycles PMU events, like virtualized, this diff --git a/tools/perf/util/addr2line.c b/tools/perf/util/addr2line.c index f2d94a3272d713..a8b39f4f202b67 100644 --- a/tools/perf/util/addr2line.c +++ b/tools/perf/util/addr2line.c @@ -18,8 +18,8 @@ #define MAX_INLINE_NEST 1024 -/* If addr2line doesn't return data for 1 second then timeout. */ -int addr2line_timeout_ms = 1 * 1000; +/* If addr2line doesn't return data for 5 seconds then timeout. */ +int addr2line_timeout_ms = 5 * 1000; static int filename_split(char *filename, unsigned int *line_nr) { diff --git a/tools/perf/util/annotate.c b/tools/perf/util/annotate.c index cc7764455faf66..df7b7e70c19fed 100644 --- a/tools/perf/util/annotate.c +++ b/tools/perf/util/annotate.c @@ -44,6 +44,7 @@ #include "strbuf.h" #include #include +#include #include #include #include @@ -137,8 +138,10 @@ static int annotated_source__alloc_histograms(struct annotated_source *src, return -1; src->samples = hashmap__new(sym_hist_hash, sym_hist_equal, NULL); - if (src->samples == NULL) + if (IS_ERR(src->samples)) { zfree(&src->histograms); + src->samples = NULL; + } return src->histograms ? 0 : -1; } @@ -1031,7 +1034,7 @@ int symbol__annotate(struct map_symbol *ms, struct evsel *evsel, return 0; args.arch = arch; - args.ms = *ms; + args.ms = ms; if (notes->src == NULL) { notes->src = annotated_source__new(); diff --git a/tools/perf/util/capstone.c b/tools/perf/util/capstone.c index be5fd44b1f9dc3..2c7feab61b7bf1 100644 --- a/tools/perf/util/capstone.c +++ b/tools/perf/util/capstone.c @@ -143,7 +143,7 @@ static void print_capstone_detail(cs_insn *insn, char *buf, size_t len, struct annotate_args *args, u64 addr) { int i; - struct map *map = args->ms.map; + struct map *map = args->ms->map; struct symbol *sym; /* TODO: support more architectures */ @@ -222,7 +222,7 @@ int symbol__disassemble_capstone(const char *filename __maybe_unused, { #ifdef HAVE_LIBCAPSTONE_SUPPORT struct annotation *notes = symbol__annotation(sym); - struct map *map = args->ms.map; + struct map *map = args->ms->map; struct dso *dso = map__dso(map); u64 start = map__rip_2objdump(map, sym->start); u64 offset; @@ -256,7 +256,7 @@ int symbol__disassemble_capstone(const char *filename __maybe_unused, args->line = disasm_buf; args->line_nr = 0; args->fileloc = NULL; - args->ms.sym = sym; + args->ms->sym = sym; dl = disasm_line__new(args); if (dl == NULL) @@ -268,7 +268,7 @@ int symbol__disassemble_capstone(const char *filename __maybe_unused, !strcmp(args->options->disassembler_style, "att")) disassembler_style = true; - if (capstone_init(maps__machine(args->ms.maps), &handle, is_64bit, disassembler_style) < 0) + if (capstone_init(maps__machine(args->ms->maps), &handle, is_64bit, disassembler_style) < 0) goto err; needs_cs_close = true; @@ -345,7 +345,7 @@ int symbol__disassemble_capstone_powerpc(const char *filename __maybe_unused, { #ifdef HAVE_LIBCAPSTONE_SUPPORT struct annotation *notes = symbol__annotation(sym); - struct map *map = args->ms.map; + struct map *map = args->ms->map; struct dso *dso = map__dso(map); struct nscookie nsc; u64 start = map__rip_2objdump(map, sym->start); @@ -382,7 +382,7 @@ int symbol__disassemble_capstone_powerpc(const char *filename __maybe_unused, !strcmp(args->options->disassembler_style, "att")) disassembler_style = true; - if (capstone_init(maps__machine(args->ms.maps), &handle, is_64bit, disassembler_style) < 0) + if (capstone_init(maps__machine(args->ms->maps), &handle, is_64bit, disassembler_style) < 0) goto err; needs_cs_close = true; @@ -408,7 +408,7 @@ int symbol__disassemble_capstone_powerpc(const char *filename __maybe_unused, args->line = disasm_buf; args->line_nr = 0; args->fileloc = NULL; - args->ms.sym = sym; + args->ms->sym = sym; dl = disasm_line__new(args); if (dl == NULL) diff --git a/tools/perf/util/cs-etm.c b/tools/perf/util/cs-etm.c index 25d56e0f1c078f..12b55c2bc2ca4f 100644 --- a/tools/perf/util/cs-etm.c +++ b/tools/perf/util/cs-etm.c @@ -3086,7 +3086,7 @@ static int cs_etm__queue_aux_fragment(struct perf_session *session, off_t file_o if (aux_offset >= auxtrace_event->offset && aux_offset + aux_size <= auxtrace_event->offset + auxtrace_event->size) { - struct cs_etm_queue *etmq = etm->queues.queue_array[auxtrace_event->idx].priv; + struct cs_etm_queue *etmq = cs_etm__get_queue(etm, auxtrace_event->cpu); /* * If this AUX event was inside this buffer somewhere, create a new auxtrace event @@ -3095,6 +3095,7 @@ static int cs_etm__queue_aux_fragment(struct perf_session *session, off_t file_o auxtrace_fragment.auxtrace = *auxtrace_event; auxtrace_fragment.auxtrace.size = aux_size; auxtrace_fragment.auxtrace.offset = aux_offset; + auxtrace_fragment.auxtrace.idx = etmq->queue_nr; file_offset += aux_offset - auxtrace_event->offset + auxtrace_event->header.size; pr_debug3("CS ETM: Queue buffer size: %#"PRI_lx64" offset: %#"PRI_lx64 diff --git a/tools/perf/util/disasm.c b/tools/perf/util/disasm.c index 50b9433f3f8e60..b1be847446fea0 100644 --- a/tools/perf/util/disasm.c +++ b/tools/perf/util/disasm.c @@ -81,7 +81,7 @@ static int arch__grow_instructions(struct arch *arch) if (new_instructions == NULL) return -1; - memcpy(new_instructions, arch->instructions, arch->nr_instructions); + memcpy(new_instructions, arch->instructions, arch->nr_instructions * sizeof(struct ins)); goto out_update_instructions; } @@ -269,9 +269,7 @@ static int call__parse(struct arch *arch, struct ins_operands *ops, struct map_s { char *endptr, *tok, *name; struct map *map = ms->map; - struct addr_map_symbol target = { - .ms = { .map = map, }, - }; + struct addr_map_symbol target; ops->target.addr = strtoull(ops->raw, &endptr, 16); @@ -296,12 +294,16 @@ static int call__parse(struct arch *arch, struct ins_operands *ops, struct map_s if (ops->target.name == NULL) return -1; find_target: - target.addr = map__objdump_2mem(map, ops->target.addr); + target = (struct addr_map_symbol) { + .ms = { .map = map__get(map), }, + .addr = map__objdump_2mem(map, ops->target.addr), + }; if (maps__find_ams(ms->maps, &target) == 0 && map__rip_2objdump(target.ms.map, map__map_ip(target.ms.map, target.addr)) == ops->target.addr) ops->target.sym = target.ms.sym; + addr_map_symbol__exit(&target); return 0; indirect_call: @@ -366,7 +368,7 @@ static int jump__parse(struct arch *arch, struct ins_operands *ops, struct map_s struct map *map = ms->map; struct symbol *sym = ms->sym; struct addr_map_symbol target = { - .ms = { .map = map, }, + .ms = { .map = map__get(map), }, }; const char *c = strchr(ops->raw, ','); u64 start, end; @@ -410,7 +412,7 @@ static int jump__parse(struct arch *arch, struct ins_operands *ops, struct map_s start = map__unmap_ip(map, sym->start); end = map__unmap_ip(map, sym->end); - ops->target.outside = target.addr < start || target.addr > end; + ops->target.outside = target.addr < start || target.addr >= end; /* * FIXME: things like this in _cpp_lex_token (gcc's cc1 program): @@ -440,7 +442,7 @@ static int jump__parse(struct arch *arch, struct ins_operands *ops, struct map_s } else { ops->target.offset_avail = false; } - + addr_map_symbol__exit(&target); return 0; } @@ -1046,7 +1048,7 @@ static size_t disasm_line_size(int nr) struct disasm_line *disasm_line__new(struct annotate_args *args) { struct disasm_line *dl = NULL; - struct annotation *notes = symbol__annotation(args->ms.sym); + struct annotation *notes = symbol__annotation(args->ms->sym); int nr = notes->src->nr_events; dl = zalloc(disasm_line_size(nr)); @@ -1064,7 +1066,7 @@ struct disasm_line *disasm_line__new(struct annotate_args *args) } else if (disasm_line__parse(dl->al.line, &dl->ins.name, &dl->ops.raw) < 0) goto out_free_line; - disasm_line__init_ins(dl, args->arch, &args->ms); + disasm_line__init_ins(dl, args->arch, args->ms); } return dl; @@ -1119,7 +1121,7 @@ static int symbol__parse_objdump_line(struct symbol *sym, struct annotate_args *args, char *parsed_line, int *line_nr, char **fileloc) { - struct map *map = args->ms.map; + struct map *map = args->ms->map; struct annotation *notes = symbol__annotation(sym); struct disasm_line *dl; char *tmp; @@ -1151,7 +1153,7 @@ static int symbol__parse_objdump_line(struct symbol *sym, args->line = parsed_line; args->line_nr = *line_nr; args->fileloc = *fileloc; - args->ms.sym = sym; + args->ms->sym = sym; dl = disasm_line__new(args); (*line_nr)++; @@ -1169,12 +1171,14 @@ static int symbol__parse_objdump_line(struct symbol *sym, if (dl->ins.ops && ins__is_call(&dl->ins) && !dl->ops.target.sym) { struct addr_map_symbol target = { .addr = dl->ops.target.addr, - .ms = { .map = map, }, + .ms = { .map = map__get(map), }, }; - if (!maps__find_ams(args->ms.maps, &target) && + if (!maps__find_ams(args->ms->maps, &target) && target.ms.sym->start == target.al_addr) dl->ops.target.sym = target.ms.sym; + + addr_map_symbol__exit(&target); } annotation_line__add(&dl->al, ¬es->src->source); @@ -1338,7 +1342,7 @@ static int symbol__disassemble_raw(char *filename, struct symbol *sym, struct annotate_args *args) { struct annotation *notes = symbol__annotation(sym); - struct map *map = args->ms.map; + struct map *map = args->ms->map; struct dso *dso = map__dso(map); u64 start = map__rip_2objdump(map, sym->start); u64 end = map__rip_2objdump(map, sym->end); @@ -1375,7 +1379,7 @@ static int symbol__disassemble_raw(char *filename, struct symbol *sym, args->line = disasm_buf; args->line_nr = 0; args->fileloc = NULL; - args->ms.sym = sym; + args->ms->sym = sym; dl = disasm_line__new(args); if (dl == NULL) @@ -1501,7 +1505,7 @@ static int symbol__disassemble_objdump(const char *filename, struct symbol *sym, struct annotate_args *args) { struct annotation_options *opts = &annotate_opts; - struct map *map = args->ms.map; + struct map *map = args->ms->map; struct dso *dso = map__dso(map); char *command; FILE *file; @@ -1644,7 +1648,7 @@ static int symbol__disassemble_objdump(const char *filename, struct symbol *sym, int symbol__disassemble(struct symbol *sym, struct annotate_args *args) { struct annotation_options *options = args->options; - struct map *map = args->ms.map; + struct map *map = args->ms->map; struct dso *dso = map__dso(map); char symfs_filename[PATH_MAX]; bool delete_extract = false; diff --git a/tools/perf/util/disasm.h b/tools/perf/util/disasm.h index d2cb555e4a3be5..a3ea9d67628162 100644 --- a/tools/perf/util/disasm.h +++ b/tools/perf/util/disasm.h @@ -97,7 +97,7 @@ struct ins_ops { struct annotate_args { struct arch *arch; - struct map_symbol ms; + struct map_symbol *ms; struct annotation_options *options; s64 offset; char *line; diff --git a/tools/perf/util/dso.c b/tools/perf/util/dso.c index 344e689567ee1a..dc202d49437217 100644 --- a/tools/perf/util/dso.c +++ b/tools/perf/util/dso.c @@ -111,7 +111,7 @@ bool dso__is_object_file(const struct dso *dso) int dso__read_binary_type_filename(const struct dso *dso, enum dso_binary_type type, - char *root_dir, char *filename, size_t size) + const char *root_dir, char *filename, size_t size) { char build_id_hex[SBUILD_ID_SIZE]; int ret = 0; @@ -563,20 +563,15 @@ char *dso__filename_with_chroot(const struct dso *dso, const char *filename) return filename_with_chroot(nsinfo__pid(dso__nsinfo_const(dso)), filename); } -static int __open_dso(struct dso *dso, struct machine *machine) - EXCLUSIVE_LOCKS_REQUIRED(_dso__data_open_lock) +static char *dso__get_filename(struct dso *dso, const char *root_dir, + bool *decomp) { - int fd = -EINVAL; - char *root_dir = (char *)""; char *name = malloc(PATH_MAX); - bool decomp = false; - if (!name) - return -ENOMEM; + *decomp = false; - mutex_lock(dso__lock(dso)); - if (machine) - root_dir = machine->root_dir; + if (name == NULL) + return NULL; if (dso__read_binary_type_filename(dso, dso__binary_type(dso), root_dir, name, PATH_MAX)) @@ -601,20 +596,38 @@ static int __open_dso(struct dso *dso, struct machine *machine) size_t len = sizeof(newpath); if (dso__decompress_kmodule_path(dso, name, newpath, len) < 0) { - fd = -(*dso__load_errno(dso)); + errno = *dso__load_errno(dso); goto out; } - decomp = true; + *decomp = true; strcpy(name, newpath); } + return name; + +out: + free(name); + return NULL; +} - fd = do_open(name); +static int __open_dso(struct dso *dso, struct machine *machine) + EXCLUSIVE_LOCKS_REQUIRED(_dso__data_open_lock) +{ + int fd = -EINVAL; + char *name; + bool decomp = false; + + mutex_lock(dso__lock(dso)); + + name = dso__get_filename(dso, machine ? machine->root_dir : "", &decomp); + if (name) + fd = do_open(name); + else + fd = -errno; if (decomp) unlink(name); -out: mutex_unlock(dso__lock(dso)); free(name); return fd; @@ -1910,3 +1923,23 @@ const u8 *dso__read_symbol(struct dso *dso, const char *symfs_filename, return __dso__read_symbol(dso, symfs_filename, start, len, out_buf, out_buf_len, is_64bit); } + +struct debuginfo *dso__debuginfo(struct dso *dso) +{ + char *name; + bool decomp = false; + struct debuginfo *dinfo = NULL; + + mutex_lock(dso__lock(dso)); + + name = dso__get_filename(dso, "", &decomp); + if (name) + dinfo = debuginfo__new(name); + + if (decomp) + unlink(name); + + mutex_unlock(dso__lock(dso)); + free(name); + return dinfo; +} diff --git a/tools/perf/util/dso.h b/tools/perf/util/dso.h index f8ccb9816b89c3..54e470dd073055 100644 --- a/tools/perf/util/dso.h +++ b/tools/perf/util/dso.h @@ -766,7 +766,7 @@ int dso__kernel_module_get_build_id(struct dso *dso, const char *root_dir); char dso__symtab_origin(const struct dso *dso); int dso__read_binary_type_filename(const struct dso *dso, enum dso_binary_type type, - char *root_dir, char *filename, size_t size); + const char *root_dir, char *filename, size_t size); bool is_kernel_module(const char *pathname, int cpumode); bool dso__needs_decompress(struct dso *dso); int dso__decompress_kmodule_fd(struct dso *dso, const char *name); @@ -915,14 +915,7 @@ u64 dso__findnew_global_type(struct dso *dso, u64 addr, u64 offset); bool perf_pid_map_tid(const char *dso_name, int *tid); bool is_perf_pid_map_name(const char *dso_name); -/* - * In the future, we may get debuginfo using build-ID (w/o path). - * Add this helper is for the smooth conversion. - */ -static inline struct debuginfo *dso__debuginfo(struct dso *dso) -{ - return debuginfo__new(dso__long_name(dso)); -} +struct debuginfo *dso__debuginfo(struct dso *dso); const u8 *dso__read_symbol(struct dso *dso, const char *symfs_filename, const struct map *map, const struct symbol *sym, diff --git a/tools/perf/util/evsel_fprintf.c b/tools/perf/util/evsel_fprintf.c index 10f1a03c28601e..5521d00bff2c0f 100644 --- a/tools/perf/util/evsel_fprintf.c +++ b/tools/perf/util/evsel_fprintf.c @@ -185,8 +185,12 @@ int sample__fprintf_callchain(struct perf_sample *sample, int left_alignment, if (print_dso && (!sym || !sym->inlined)) printed += map__fprintf_dsoname_dsoff(map, print_dsoff, addr, fp); - if (print_srcline) - printed += map__fprintf_srcline(map, addr, "\n ", fp); + if (print_srcline) { + if (node->srcline) + printed += fprintf(fp, "\n %s", node->srcline); + else + printed += map__fprintf_srcline(map, addr, "\n ", fp); + } if (sym && sym->inlined) printed += fprintf(fp, " (inlined)"); diff --git a/tools/perf/util/libbfd.c b/tools/perf/util/libbfd.c index 79f4528234a9d6..63ea3fb53e77d5 100644 --- a/tools/perf/util/libbfd.c +++ b/tools/perf/util/libbfd.c @@ -501,7 +501,7 @@ int symbol__disassemble_bpf_libbfd(struct symbol *sym __maybe_unused, struct bpf_prog_info_node *info_node; int len = sym->end - sym->start; disassembler_ftype disassemble; - struct map *map = args->ms.map; + struct map *map = args->ms->map; struct perf_bpil *info_linear; struct disassemble_info info; struct dso *dso = map__dso(map); @@ -612,7 +612,7 @@ int symbol__disassemble_bpf_libbfd(struct symbol *sym __maybe_unused, args->line = strdup(srcline); args->line_nr = 0; args->fileloc = NULL; - args->ms.sym = sym; + args->ms->sym = sym; dl = disasm_line__new(args); if (dl) { annotation_line__add(&dl->al, @@ -624,7 +624,7 @@ int symbol__disassemble_bpf_libbfd(struct symbol *sym __maybe_unused, args->line = buf + prev_buf_size; args->line_nr = 0; args->fileloc = NULL; - args->ms.sym = sym; + args->ms->sym = sym; dl = disasm_line__new(args); if (dl) annotation_line__add(&dl->al, ¬es->src->source); diff --git a/tools/perf/util/llvm.c b/tools/perf/util/llvm.c index 2ebf1f5f65bf77..4ada9a10bd93f9 100644 --- a/tools/perf/util/llvm.c +++ b/tools/perf/util/llvm.c @@ -118,7 +118,7 @@ int symbol__disassemble_llvm(const char *filename, struct symbol *sym, { #ifdef HAVE_LIBLLVM_SUPPORT struct annotation *notes = symbol__annotation(sym); - struct map *map = args->ms.map; + struct map *map = args->ms->map; struct dso *dso = map__dso(map); u64 start = map__rip_2objdump(map, sym->start); /* Malloc-ed buffer containing instructions read from disk. */ @@ -184,7 +184,7 @@ int symbol__disassemble_llvm(const char *filename, struct symbol *sym, args->line = disasm_buf; args->line_nr = 0; args->fileloc = NULL; - args->ms.sym = sym; + args->ms->sym = sym; dl = disasm_line__new(args); if (dl == NULL) @@ -242,7 +242,7 @@ int symbol__disassemble_llvm(const char *filename, struct symbol *sym, &line_storage_len); args->line_nr = 0; args->fileloc = NULL; - args->ms.sym = sym; + args->ms->sym = sym; llvm_addr2line(filename, pc, &args->fileloc, (unsigned int *)&args->line_nr, false, NULL); diff --git a/tools/perf/util/maps.c b/tools/perf/util/maps.c index c321d4f4d84669..8885c95f02b3ea 100644 --- a/tools/perf/util/maps.c +++ b/tools/perf/util/maps.c @@ -676,6 +676,7 @@ int maps__find_ams(struct maps *maps, struct addr_map_symbol *ams) if (ams->addr < map__start(ams->ms.map) || ams->addr >= map__end(ams->ms.map)) { if (maps == NULL) return -1; + map__put(ams->ms.map); ams->ms.map = maps__find(maps, ams->addr); if (ams->ms.map == NULL) return -1; diff --git a/tools/perf/util/metricgroup.c b/tools/perf/util/metricgroup.c index 25c75fdbfc525f..45bb94da97b99e 100644 --- a/tools/perf/util/metricgroup.c +++ b/tools/perf/util/metricgroup.c @@ -1563,8 +1563,6 @@ int metricgroup__parse_groups(struct evlist *perf_evlist, { const struct pmu_metrics_table *table = pmu_metrics_table__find(); - if (!table) - return -EINVAL; if (hardware_aware_grouping) pr_debug("Use hardware aware grouping instead of traditional metric grouping method\n"); @@ -1602,22 +1600,16 @@ static int metricgroup__has_metric_or_groups_callback(const struct pmu_metric *p bool metricgroup__has_metric_or_groups(const char *pmu, const char *metric_or_groups) { - const struct pmu_metrics_table *tables[2] = { - pmu_metrics_table__find(), - pmu_metrics_table__default(), - }; + const struct pmu_metrics_table *table = pmu_metrics_table__find(); struct metricgroup__has_metric_data data = { .pmu = pmu, .metric_or_groups = metric_or_groups, }; - for (size_t i = 0; i < ARRAY_SIZE(tables); i++) { - if (pmu_metrics_table__for_each_metric(tables[i], - metricgroup__has_metric_or_groups_callback, - &data)) - return true; - } - return false; + return metricgroup__for_each_metric(table, + metricgroup__has_metric_or_groups_callback, + &data) + ? true : false; } static int metricgroup__topdown_max_level_callback(const struct pmu_metric *pm, diff --git a/tools/perf/util/stat-display.c b/tools/perf/util/stat-display.c index 6d02f84c5691a4..f4bd579908b438 100644 --- a/tools/perf/util/stat-display.c +++ b/tools/perf/util/stat-display.c @@ -820,12 +820,6 @@ static void printout(struct perf_stat_config *config, struct outstate *os, } if (run == 0 || ena == 0 || counter->counts->scaled == -1) { - if (config->metric_only) { - pm(config, os, METRIC_THRESHOLD_UNKNOWN, /*format=*/NULL, - /*unit=*/NULL, /*val=*/0); - return; - } - ok = false; if (counter->supported) { @@ -848,33 +842,32 @@ static void printout(struct perf_stat_config *config, struct outstate *os, print_running(config, os, run, ena, /*before_metric=*/true); } - if (ok) { - if (!config->metric_only && counter->default_metricgroup && !counter->default_show_events) { - void *from = NULL; - - aggr_printout(config, os, os->evsel, os->id, os->aggr_nr); - /* Print out all the metricgroup with the same metric event. */ - do { - int num = 0; - - /* Print out the new line for the next new metricgroup. */ - if (from) { - if (config->json_output) - new_line_json(config, (void *)os); - else - __new_line_std_csv(config, os); - } - - print_noise(config, os, counter, noise, /*before_metric=*/true); - print_running(config, os, run, ena, /*before_metric=*/true); - from = perf_stat__print_shadow_stats_metricgroup(config, counter, aggr_idx, - &num, from, &out); - } while (from != NULL); - } else { - perf_stat__print_shadow_stats(config, counter, aggr_idx, &out); - } + if (!config->metric_only && counter->default_metricgroup && + !counter->default_show_events) { + void *from = NULL; + + aggr_printout(config, os, os->evsel, os->id, os->aggr_nr); + /* Print out all the metricgroup with the same metric event. */ + do { + int num = 0; + + /* Print out the new line for the next new metricgroup. */ + if (from) { + if (config->json_output) + new_line_json(config, (void *)os); + else + __new_line_std_csv(config, os); + } + + print_noise(config, os, counter, noise, + /*before_metric=*/true); + print_running(config, os, run, ena, + /*before_metric=*/true); + from = perf_stat__print_shadow_stats_metricgroup( + config, counter, aggr_idx, &num, from, &out); + } while (from != NULL); } else { - pm(config, os, METRIC_THRESHOLD_UNKNOWN, /*format=*/NULL, /*unit=*/NULL, /*val=*/0); + perf_stat__print_shadow_stats(config, counter, aggr_idx, &out); } if (!config->metric_only) { @@ -987,7 +980,7 @@ static void print_counter_aggrdata(struct perf_stat_config *config, ena = aggr->counts.ena; run = aggr->counts.run; - if (perf_stat__skip_metric_event(counter, ena, run)) + if (perf_stat__skip_metric_event(counter)) return; if (val == 0 && should_skip_zero_counter(config, counter, &id)) diff --git a/tools/perf/util/stat-shadow.c b/tools/perf/util/stat-shadow.c index 9c83f7d96caa4e..59d2cd4f2188de 100644 --- a/tools/perf/util/stat-shadow.c +++ b/tools/perf/util/stat-shadow.c @@ -57,7 +57,6 @@ static int prepare_metric(struct perf_stat_config *config, bool is_tool_time = tool_pmu__is_time_event(config, metric_events[i], &tool_aggr_idx); struct perf_stat_evsel *ps = metric_events[i]->stats; - struct perf_stat_aggr *aggr; char *n; double val; @@ -82,8 +81,7 @@ static int prepare_metric(struct perf_stat_config *config, } } /* Time events are always on CPU0, the first aggregation index. */ - aggr = &ps->aggr[is_tool_time ? tool_aggr_idx : aggr_idx]; - if (!aggr || !metric_events[i]->supported) { + if (!ps || !metric_events[i]->supported) { /* * Not supported events will have a count of 0, which * can be confusing in a metric. Explicitly set the @@ -93,11 +91,21 @@ static int prepare_metric(struct perf_stat_config *config, val = NAN; source_count = 0; } else { - val = aggr->counts.val; - if (is_tool_time) - val *= 1e-9; /* Convert time event nanoseconds to seconds. */ - if (!source_count) - source_count = evsel__source_count(metric_events[i]); + struct perf_stat_aggr *aggr = + &ps->aggr[is_tool_time ? tool_aggr_idx : aggr_idx]; + + if (aggr->counts.run == 0) { + val = NAN; + source_count = 0; + } else { + val = aggr->counts.val; + if (is_tool_time) { + /* Convert time event nanoseconds to seconds. */ + val *= 1e-9; + } + if (!source_count) + source_count = evsel__source_count(metric_events[i]); + } } n = strdup(evsel__metric_id(metric_events[i])); if (!n) @@ -335,14 +343,10 @@ void perf_stat__print_shadow_stats(struct perf_stat_config *config, * perf_stat__skip_metric_event - Skip the evsel in the Default metricgroup, * if it's not running or not the metric event. */ -bool perf_stat__skip_metric_event(struct evsel *evsel, - u64 ena, u64 run) +bool perf_stat__skip_metric_event(struct evsel *evsel) { if (!evsel->default_metricgroup) return false; - if (!ena || !run) - return true; - return !metricgroup__lookup(&evsel->evlist->metric_events, evsel, false); } diff --git a/tools/perf/util/stat.h b/tools/perf/util/stat.h index f986911c9296e7..4bced233d2fc00 100644 --- a/tools/perf/util/stat.h +++ b/tools/perf/util/stat.h @@ -163,7 +163,7 @@ void perf_stat__print_shadow_stats(struct perf_stat_config *config, struct evsel *evsel, int aggr_idx, struct perf_stat_output_ctx *out); -bool perf_stat__skip_metric_event(struct evsel *evsel, u64 ena, u64 run); +bool perf_stat__skip_metric_event(struct evsel *evsel); void *perf_stat__print_shadow_stats_metricgroup(struct perf_stat_config *config, struct evsel *evsel, int aggr_idx, diff --git a/tools/perf/util/symbol-elf.c b/tools/perf/util/symbol-elf.c index d1dcafa4b3b808..439f252937b890 100644 --- a/tools/perf/util/symbol-elf.c +++ b/tools/perf/util/symbol-elf.c @@ -1173,7 +1173,7 @@ int symsrc__init(struct symsrc *ss, struct dso *dso, const char *name, Elf *embedded = read_gnu_debugdata(dso, elf, name, &new_fd); if (!embedded) - goto out_close; + goto out_elf_end; elf_end(elf); close(fd); diff --git a/tools/perf/util/synthetic-events.c b/tools/perf/util/synthetic-events.c index 2ba9fa25e00a68..7a47e21c6704fb 100644 --- a/tools/perf/util/synthetic-events.c +++ b/tools/perf/util/synthetic-events.c @@ -703,6 +703,11 @@ static int perf_event__synthesize_modules_maps_cb(struct map *map, void *data) memcpy(event->mmap2.filename, dso__long_name(dso), dso__long_name_len(dso) + 1); + /* Clear stale build ID from previous module iteration */ + event->mmap2.header.misc &= ~PERF_RECORD_MISC_MMAP_BUILD_ID; + memset(event->mmap2.build_id, 0, sizeof(event->mmap2.build_id)); + event->mmap2.build_id_size = 0; + perf_record_mmap2__read_build_id(&event->mmap2, args->machine, false); } else { size = PERF_ALIGN(dso__long_name_len(dso) + 1, sizeof(u64)); diff --git a/tools/perf/util/unwind-libdw.c b/tools/perf/util/unwind-libdw.c index ae70fb56a05729..3ff427a49e4c5e 100644 --- a/tools/perf/util/unwind-libdw.c +++ b/tools/perf/util/unwind-libdw.c @@ -136,8 +136,8 @@ static int entry(u64 ip, struct unwind_info *ui) } e->ip = ip; - e->ms.maps = al.maps; - e->ms.map = al.map; + e->ms.maps = maps__get(al.maps); + e->ms.map = map__get(al.map); e->ms.sym = al.sym; pr_debug("unwind: %s:ip = 0x%" PRIx64 " (0x%" PRIx64 ")\n", @@ -325,6 +325,9 @@ int unwind__get_entries(unwind_entry_cb_t cb, void *arg, if (err) pr_debug("unwind: failed with '%s'\n", dwfl_errmsg(-1)); + for (i = 0; i < ui->idx; i++) + map_symbol__exit(&ui->entries[i].ms); + dwfl_end(ui->dwfl); free(ui); return 0; diff --git a/tools/power/cpupower/lib/cpuidle.c b/tools/power/cpupower/lib/cpuidle.c index f2c1139adf7169..bd857ee7541a70 100644 --- a/tools/power/cpupower/lib/cpuidle.c +++ b/tools/power/cpupower/lib/cpuidle.c @@ -150,6 +150,7 @@ unsigned long long cpuidle_state_get_one_value(unsigned int cpu, if (len == 0) return 0; + errno = 0; value = strtoull(linebuf, &endp, 0); if (endp == linebuf || errno == ERANGE) diff --git a/tools/power/cpupower/utils/cpufreq-info.c b/tools/power/cpupower/utils/cpufreq-info.c index 7d3732f5f2f6f5..5fe01e516817e8 100644 --- a/tools/power/cpupower/utils/cpufreq-info.c +++ b/tools/power/cpupower/utils/cpufreq-info.c @@ -270,7 +270,7 @@ static int get_freq_hardware(unsigned int cpu, unsigned int human) { unsigned long freq; - if (cpupower_cpu_info.caps & CPUPOWER_CAP_APERF) + if (!(cpupower_cpu_info.caps & CPUPOWER_CAP_APERF)) return -EINVAL; freq = cpufreq_get_freq_hardware(cpu); diff --git a/tools/power/x86/intel-speed-select/isst-config.c b/tools/power/x86/intel-speed-select/isst-config.c index 558138eea75e97..d00d15490a98c2 100644 --- a/tools/power/x86/intel-speed-select/isst-config.c +++ b/tools/power/x86/intel-speed-select/isst-config.c @@ -950,9 +950,11 @@ int isolate_cpus(struct isst_id *id, int mask_size, cpu_set_t *cpu_mask, int lev ret = write(fd, "member", strlen("member")); if (ret == -1) { printf("Can't update to member\n"); + close(fd); return ret; } + close(fd); return 0; } diff --git a/tools/power/x86/turbostat/turbostat.c b/tools/power/x86/turbostat/turbostat.c index 5ad45c2ac5bd8f..c6060f65eaaf14 100644 --- a/tools/power/x86/turbostat/turbostat.c +++ b/tools/power/x86/turbostat/turbostat.c @@ -2135,7 +2135,7 @@ off_t idx_to_offset(int idx) switch (idx) { case IDX_PKG_ENERGY: - if (valid_rapl_msrs & RAPL_AMD_F17H) + if (platform->plat_rapl_msrs & RAPL_AMD_F17H) offset = MSR_PKG_ENERGY_STAT; else offset = MSR_PKG_ENERGY_STATUS; @@ -2732,30 +2732,29 @@ static inline int print_name(int width, int *printed, char *delim, char *name, e UNUSED(type); if (format == FORMAT_RAW && width >= 64) - return (sprintf(outp, "%s%-8s", (*printed++ ? delim : ""), name)); + return (sprintf(outp, "%s%-8s", ((*printed)++ ? delim : ""), name)); else - return (sprintf(outp, "%s%s", (*printed++ ? delim : ""), name)); + return (sprintf(outp, "%s%s", ((*printed)++ ? delim : ""), name)); } static inline int print_hex_value(int width, int *printed, char *delim, unsigned long long value) { if (width <= 32) - return (sprintf(outp, "%s%08x", (*printed++ ? delim : ""), (unsigned int)value)); + return (sprintf(outp, "%s%08x", ((*printed)++ ? delim : ""), (unsigned int)value)); else - return (sprintf(outp, "%s%016llx", (*printed++ ? delim : ""), value)); + return (sprintf(outp, "%s%016llx", ((*printed)++ ? delim : ""), value)); } static inline int print_decimal_value(int width, int *printed, char *delim, unsigned long long value) { - if (width <= 32) - return (sprintf(outp, "%s%d", (*printed++ ? delim : ""), (unsigned int)value)); - else - return (sprintf(outp, "%s%-8lld", (*printed++ ? delim : ""), value)); + UNUSED(width); + + return (sprintf(outp, "%s%lld", ((*printed)++ ? delim : ""), value)); } static inline int print_float_value(int *printed, char *delim, double value) { - return (sprintf(outp, "%s%0.2f", (*printed++ ? delim : ""), value)); + return (sprintf(outp, "%s%0.2f", ((*printed)++ ? delim : ""), value)); } void print_header(char *delim) @@ -3001,22 +3000,30 @@ void print_header(char *delim) } /* - * pct() + * pct(numerator, denominator) * - * If absolute value is < 1.1, return percentage - * otherwise, return nan + * Return sanity checked percentage (100.0 * numerator/denominotor) * - * return value is appropriate for printing percentages with %f - * while flagging some obvious erroneous values. + * n < 0: nan + * d <= 0: nan + * n/d > 1.1: nan */ -double pct(double d) +double pct(double numerator, double denominator) { + double retval; + + if (numerator < 0) + return nan(""); - double abs = fabs(d); + if (denominator <= 0) + return nan(""); - if (abs < 1.10) - return (100.0 * d); - return nan(""); + retval = 100.0 * numerator / denominator; + + if (retval > 110.0) + return nan(""); + + return retval; } int dump_counters(PER_THREAD_PARAMS) @@ -3046,7 +3053,7 @@ int dump_counters(PER_THREAD_PARAMS) outp += sprintf(outp, "LLC refs: %lld", t->llc.references); outp += sprintf(outp, "LLC miss: %lld", t->llc.misses); - outp += sprintf(outp, "LLC Hit%%: %.2f", pct((t->llc.references - t->llc.misses) / t->llc.references)); + outp += sprintf(outp, "LLC Hit%%: %.2f", pct((t->llc.references - t->llc.misses), t->llc.references)); for (i = 0, mp = sys.tp; mp; i++, mp = mp->next) { outp += sprintf(outp, "tADDED [%d] %8s msr0x%x: %08llX %s\n", i, mp->name, mp->msr_num, t->counter[i], mp->sp->path); @@ -3261,7 +3268,7 @@ int format_counters(PER_THREAD_PARAMS) outp += sprintf(outp, "%s%.0f", (printed++ ? delim : ""), 1.0 / units * t->aperf / interval_float); if (DO_BIC(BIC_Busy)) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(t->mperf / tsc)); + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(t->mperf, tsc)); if (DO_BIC(BIC_Bzy_MHz)) { if (has_base_hz) @@ -3302,7 +3309,7 @@ int format_counters(PER_THREAD_PARAMS) outp += sprintf(outp, "%s%.0f", (printed++ ? delim : ""), t->llc.references / interval_float / 1000); if (DO_BIC(BIC_LLC_HIT)) - outp += sprintf(outp, fmt8, (printed++ ? delim : ""), pct((t->llc.references - t->llc.misses) / t->llc.references)); + outp += sprintf(outp, fmt8, (printed++ ? delim : ""), pct((t->llc.references - t->llc.misses), t->llc.references)); } /* Added Thread Counters */ @@ -3315,7 +3322,7 @@ int format_counters(PER_THREAD_PARAMS) if (mp->type == COUNTER_USEC) outp += print_float_value(&printed, delim, t->counter[i] / interval_float / 10000); else - outp += print_float_value(&printed, delim, pct(t->counter[i] / tsc)); + outp += print_float_value(&printed, delim, pct(t->counter[i], tsc)); } } @@ -3323,13 +3330,13 @@ int format_counters(PER_THREAD_PARAMS) for (i = 0, pp = sys.perf_tp; pp; ++i, pp = pp->next) { if (pp->format == FORMAT_RAW) outp += print_hex_value(pp->width, &printed, delim, t->perf_counter[i]); - else if (pp->format == FORMAT_DELTA || mp->format == FORMAT_AVERAGE) + else if (pp->format == FORMAT_DELTA || pp->format == FORMAT_AVERAGE) outp += print_decimal_value(pp->width, &printed, delim, t->perf_counter[i]); else if (pp->format == FORMAT_PERCENT) { if (pp->type == COUNTER_USEC) outp += print_float_value(&printed, delim, t->perf_counter[i] / interval_float / 10000); else - outp += print_float_value(&printed, delim, pct(t->perf_counter[i] / tsc)); + outp += print_float_value(&printed, delim, pct(t->perf_counter[i], tsc)); } } @@ -3343,34 +3350,34 @@ int format_counters(PER_THREAD_PARAMS) break; case PMT_TYPE_XTAL_TIME: - value_converted = pct(value_raw / crystal_hz / interval_float); + value_converted = pct(value_raw / crystal_hz, interval_float); outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), value_converted); break; case PMT_TYPE_TCORE_CLOCK: - value_converted = pct(value_raw / tcore_clock_freq_hz / interval_float); + value_converted = pct(value_raw / tcore_clock_freq_hz, interval_float); outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), value_converted); } } /* C1 */ if (DO_BIC(BIC_CPU_c1)) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(t->c1 / tsc)); + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(t->c1, tsc)); /* print per-core data only for 1st thread in core */ if (!is_cpu_first_thread_in_core(t, c)) goto done; if (DO_BIC(BIC_CPU_c3)) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(c->c3 / tsc)); + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(c->c3, tsc)); if (DO_BIC(BIC_CPU_c6)) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(c->c6 / tsc)); + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(c->c6, tsc)); if (DO_BIC(BIC_CPU_c7)) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(c->c7 / tsc)); + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(c->c7, tsc)); /* Mod%c6 */ if (DO_BIC(BIC_Mod_c6)) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(c->mc6_us / tsc)); + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(c->mc6_us, tsc)); if (DO_BIC(BIC_CoreTmp)) outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), c->core_temp_c); @@ -3386,17 +3393,17 @@ int format_counters(PER_THREAD_PARAMS) else if (mp->format == FORMAT_DELTA || mp->format == FORMAT_AVERAGE) outp += print_decimal_value(mp->width, &printed, delim, c->counter[i]); else if (mp->format == FORMAT_PERCENT) - outp += print_float_value(&printed, delim, pct(c->counter[i] / tsc)); + outp += print_float_value(&printed, delim, pct(c->counter[i], tsc)); } /* Added perf Core counters */ for (i = 0, pp = sys.perf_cp; pp; i++, pp = pp->next) { if (pp->format == FORMAT_RAW) outp += print_hex_value(pp->width, &printed, delim, c->perf_counter[i]); - else if (pp->format == FORMAT_DELTA || mp->format == FORMAT_AVERAGE) + else if (pp->format == FORMAT_DELTA || pp->format == FORMAT_AVERAGE) outp += print_decimal_value(pp->width, &printed, delim, c->perf_counter[i]); else if (pp->format == FORMAT_PERCENT) - outp += print_float_value(&printed, delim, pct(c->perf_counter[i] / tsc)); + outp += print_float_value(&printed, delim, pct(c->perf_counter[i], tsc)); } /* Added PMT Core counters */ @@ -3409,12 +3416,12 @@ int format_counters(PER_THREAD_PARAMS) break; case PMT_TYPE_XTAL_TIME: - value_converted = pct(value_raw / crystal_hz / interval_float); + value_converted = pct(value_raw / crystal_hz, interval_float); outp += print_float_value(&printed, delim, value_converted); break; case PMT_TYPE_TCORE_CLOCK: - value_converted = pct(value_raw / tcore_clock_freq_hz / interval_float); + value_converted = pct(value_raw / tcore_clock_freq_hz, interval_float); outp += print_float_value(&printed, delim, value_converted); } } @@ -3470,39 +3477,39 @@ int format_counters(PER_THREAD_PARAMS) if (DO_BIC(BIC_Totl_c0)) outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100 * p->pkg_wtd_core_c0 / tsc); /* can exceed 100% */ if (DO_BIC(BIC_Any_c0)) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pkg_any_core_c0 / tsc)); + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pkg_any_core_c0, tsc)); if (DO_BIC(BIC_GFX_c0)) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pkg_any_gfxe_c0 / tsc)); + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pkg_any_gfxe_c0, tsc)); if (DO_BIC(BIC_CPUGFX)) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pkg_both_core_gfxe_c0 / tsc)); + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pkg_both_core_gfxe_c0, tsc)); if (DO_BIC(BIC_Pkgpc2)) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pc2 / tsc)); + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pc2, tsc)); if (DO_BIC(BIC_Pkgpc3)) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pc3 / tsc)); + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pc3, tsc)); if (DO_BIC(BIC_Pkgpc6)) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pc6 / tsc)); + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pc6, tsc)); if (DO_BIC(BIC_Pkgpc7)) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pc7 / tsc)); + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pc7, tsc)); if (DO_BIC(BIC_Pkgpc8)) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pc8 / tsc)); + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pc8, tsc)); if (DO_BIC(BIC_Pkgpc9)) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pc9 / tsc)); + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pc9, tsc)); if (DO_BIC(BIC_Pkgpc10)) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pc10 / tsc)); + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pc10, tsc)); if (DO_BIC(BIC_Diec6)) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->die_c6 / crystal_hz / interval_float)); + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->die_c6 / crystal_hz, interval_float)); if (DO_BIC(BIC_CPU_LPI)) { if (p->cpu_lpi >= 0) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->cpu_lpi / 1000000.0 / interval_float)); + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->cpu_lpi / 1000000.0, interval_float)); else outp += sprintf(outp, "%s(neg)", (printed++ ? delim : "")); } if (DO_BIC(BIC_SYS_LPI)) { if (p->sys_lpi >= 0) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->sys_lpi / 1000000.0 / interval_float)); + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->sys_lpi / 1000000.0, interval_float)); else outp += sprintf(outp, "%s(neg)", (printed++ ? delim : "")); } @@ -3542,7 +3549,7 @@ int format_counters(PER_THREAD_PARAMS) else if (mp->format == FORMAT_DELTA || mp->format == FORMAT_AVERAGE) outp += print_decimal_value(mp->width, &printed, delim, p->counter[i]); else if (mp->format == FORMAT_PERCENT) - outp += print_float_value(&printed, delim, pct(p->counter[i] / tsc)); + outp += print_float_value(&printed, delim, pct(p->counter[i], tsc)); } /* Added perf Package Counters */ @@ -3551,10 +3558,10 @@ int format_counters(PER_THREAD_PARAMS) outp += print_hex_value(pp->width, &printed, delim, p->perf_counter[i]); else if (pp->type == COUNTER_K2M) outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), (unsigned int)p->perf_counter[i] / 1000); - else if (pp->format == FORMAT_DELTA || mp->format == FORMAT_AVERAGE) + else if (pp->format == FORMAT_DELTA || pp->format == FORMAT_AVERAGE) outp += print_decimal_value(pp->width, &printed, delim, p->perf_counter[i]); else if (pp->format == FORMAT_PERCENT) - outp += print_float_value(&printed, delim, pct(p->perf_counter[i] / tsc)); + outp += print_float_value(&printed, delim, pct(p->perf_counter[i], tsc)); } /* Added PMT Package Counters */ @@ -3567,12 +3574,12 @@ int format_counters(PER_THREAD_PARAMS) break; case PMT_TYPE_XTAL_TIME: - value_converted = pct(value_raw / crystal_hz / interval_float); + value_converted = pct(value_raw / crystal_hz, interval_float); outp += print_float_value(&printed, delim, value_converted); break; case PMT_TYPE_TCORE_CLOCK: - value_converted = pct(value_raw / tcore_clock_freq_hz / interval_float); + value_converted = pct(value_raw / tcore_clock_freq_hz, interval_float); outp += print_float_value(&printed, delim, value_converted); } } @@ -8805,10 +8812,13 @@ void process_cpuid() edx_flags = edx; if (!no_msr) { - if (get_msr(sched_getcpu(), MSR_IA32_UCODE_REV, &ucode_patch)) + if (get_msr(sched_getcpu(), MSR_IA32_UCODE_REV, &ucode_patch)) { warnx("get_msr(UCODE)"); - else + } else { ucode_patch_valid = true; + if (!authentic_amd && !hygon_genuine) + ucode_patch >>= 32; + } } /* @@ -8822,7 +8832,7 @@ void process_cpuid() if (!quiet) { fprintf(outf, "CPUID(1): family:model:stepping 0x%x:%x:%x (%d:%d:%d)", family, model, stepping, family, model, stepping); if (ucode_patch_valid) - fprintf(outf, " microcode 0x%x", (unsigned int)((ucode_patch >> 32) & 0xFFFFFFFF)); + fprintf(outf, " microcode 0x%x", (unsigned int)ucode_patch); fputc('\n', outf); fprintf(outf, "CPUID(0x80000000): max_extended_levels: 0x%x\n", max_extended_level); @@ -10898,6 +10908,14 @@ void probe_cpuidle_residency(void) } } +static bool cpuidle_counter_wanted(char *name) +{ + if (is_deferred_skip(name)) + return false; + + return DO_BIC(BIC_cpuidle) || is_deferred_add(name); +} + void probe_cpuidle_counts(void) { char path[64]; @@ -10907,7 +10925,7 @@ void probe_cpuidle_counts(void) int min_state = 1024, max_state = 0; char *sp; - if (!DO_BIC(BIC_cpuidle)) + if (!DO_BIC(BIC_cpuidle) && !deferred_add_index) return; for (state = 10; state >= 0; --state) { @@ -10922,12 +10940,6 @@ void probe_cpuidle_counts(void) remove_underbar(name_buf); - if (!DO_BIC(BIC_cpuidle) && !is_deferred_add(name_buf)) - continue; - - if (is_deferred_skip(name_buf)) - continue; - /* truncate "C1-HSW\n" to "C1", or truncate "C1\n" to "C1" */ sp = strchr(name_buf, '-'); if (!sp) @@ -10942,16 +10954,19 @@ void probe_cpuidle_counts(void) * Add 'C1+' for C1, and so on. The 'below' sysfs file always contains 0 for * the last state, so do not add it. */ - *sp = '+'; *(sp + 1) = '\0'; - sprintf(path, "cpuidle/state%d/below", state); - add_counter(0, path, name_buf, 64, SCOPE_CPU, COUNTER_ITEMS, FORMAT_DELTA, SYSFS_PERCPU, 0); + if (cpuidle_counter_wanted(name_buf)) { + sprintf(path, "cpuidle/state%d/below", state); + add_counter(0, path, name_buf, 64, SCOPE_CPU, COUNTER_ITEMS, FORMAT_DELTA, SYSFS_PERCPU, 0); + } } *sp = '\0'; - sprintf(path, "cpuidle/state%d/usage", state); - add_counter(0, path, name_buf, 64, SCOPE_CPU, COUNTER_ITEMS, FORMAT_DELTA, SYSFS_PERCPU, 0); + if (cpuidle_counter_wanted(name_buf)) { + sprintf(path, "cpuidle/state%d/usage", state); + add_counter(0, path, name_buf, 64, SCOPE_CPU, COUNTER_ITEMS, FORMAT_DELTA, SYSFS_PERCPU, 0); + } /* * The 'above' sysfs file always contains 0 for the shallowest state (smallest @@ -10960,8 +10975,10 @@ void probe_cpuidle_counts(void) if (state != min_state) { *sp = '-'; *(sp + 1) = '\0'; - sprintf(path, "cpuidle/state%d/above", state); - add_counter(0, path, name_buf, 64, SCOPE_CPU, COUNTER_ITEMS, FORMAT_DELTA, SYSFS_PERCPU, 0); + if (cpuidle_counter_wanted(name_buf)) { + sprintf(path, "cpuidle/state%d/above", state); + add_counter(0, path, name_buf, 64, SCOPE_CPU, COUNTER_ITEMS, FORMAT_DELTA, SYSFS_PERCPU, 0); + } } } } diff --git a/tools/spi/.gitignore b/tools/spi/.gitignore index 14ddba3d21957b..038261b34ed83c 100644 --- a/tools/spi/.gitignore +++ b/tools/spi/.gitignore @@ -1,3 +1,4 @@ # SPDX-License-Identifier: GPL-2.0-only spidev_fdx spidev_test +include/ diff --git a/tools/testing/kunit/kunit_kernel.py b/tools/testing/kunit/kunit_kernel.py index 260d8d9aa1db4a..2998e1bc088b28 100644 --- a/tools/testing/kunit/kunit_kernel.py +++ b/tools/testing/kunit/kunit_kernel.py @@ -346,8 +346,10 @@ def build_kernel(self, jobs: int, build_dir: str, make_options: Optional[List[st return self.validate_config(build_dir) def run_kernel(self, args: Optional[List[str]]=None, build_dir: str='', filter_glob: str='', filter: str='', filter_action: Optional[str]=None, timeout: Optional[int]=None) -> Iterator[str]: - if not args: - args = [] + # Copy to avoid mutating the caller-supplied list. exec_tests() reuses + # the same args across repeated run_kernel() calls (e.g. --run_isolated), + # so appending to the original would accumulate stale flags on each call. + args = list(args) if args else [] if filter_glob: args.append('kunit.filter_glob=' + filter_glob) if filter: diff --git a/tools/testing/kunit/kunit_tool_test.py b/tools/testing/kunit/kunit_tool_test.py index bbba921e0eacb1..ed45bac1548d9d 100755 --- a/tools/testing/kunit/kunit_tool_test.py +++ b/tools/testing/kunit/kunit_tool_test.py @@ -489,6 +489,32 @@ def fake_start(unused_args, unused_build_dir): with open(kunit_kernel.get_outfile_path(build_dir), 'rt') as outfile: self.assertEqual(outfile.read(), 'hi\nbye\n', msg='Missing some output') + def test_run_kernel_args_not_mutated(self): + """Verify run_kernel() copies args so callers can reuse them.""" + start_calls = [] + + def fake_start(start_args, unused_build_dir): + start_calls.append(list(start_args)) + return subprocess.Popen(['printf', 'KTAP version 1\n'], + text=True, stdout=subprocess.PIPE) + + with tempfile.TemporaryDirectory('') as build_dir: + tree = kunit_kernel.LinuxSourceTree(build_dir, + kunitconfig_paths=[os.devnull]) + with mock.patch.object(tree._ops, 'start', side_effect=fake_start), \ + mock.patch.object(kunit_kernel.subprocess, 'call'): + kernel_args = ['mem=1G'] + for _ in tree.run_kernel(args=kernel_args, build_dir=build_dir, + filter_glob='suite.test1'): + pass + for _ in tree.run_kernel(args=kernel_args, build_dir=build_dir, + filter_glob='suite.test2'): + pass + self.assertEqual(kernel_args, ['mem=1G'], + 'run_kernel() should not modify caller args') + self.assertIn('kunit.filter_glob=suite.test1', start_calls[0]) + self.assertIn('kunit.filter_glob=suite.test2', start_calls[1]) + def test_build_reconfig_no_config(self): with tempfile.TemporaryDirectory('') as build_dir: with open(kunit_kernel.get_kunitconfig_path(build_dir), 'w') as f: diff --git a/tools/testing/selftests/arm64/abi/hwcap.c b/tools/testing/selftests/arm64/abi/hwcap.c index c41640f18e4ecb..62ea450f2ccc05 100644 --- a/tools/testing/selftests/arm64/abi/hwcap.c +++ b/tools/testing/selftests/arm64/abi/hwcap.c @@ -473,8 +473,8 @@ static void sve2_sigill(void) static void sve2p1_sigill(void) { - /* BFADD Z0.H, Z0.H, Z0.H */ - asm volatile(".inst 0x65000000" : : : "z0"); + /* LD1Q {Z0.Q}, P0/Z, [Z0.D, X0] */ + asm volatile(".inst 0xC400A000" : : : "z0"); } static void sve2p2_sigill(void) diff --git a/tools/testing/selftests/bpf/prog_tests/reg_bounds.c b/tools/testing/selftests/bpf/prog_tests/reg_bounds.c index d93a0c7b1786f1..04938d0d431b38 100644 --- a/tools/testing/selftests/bpf/prog_tests/reg_bounds.c +++ b/tools/testing/selftests/bpf/prog_tests/reg_bounds.c @@ -422,15 +422,69 @@ static bool is_valid_range(enum num_t t, struct range x) } } -static struct range range_improve(enum num_t t, struct range old, struct range new) +static struct range range_intersection(enum num_t t, struct range old, struct range new) { return range(t, max_t(t, old.a, new.a), min_t(t, old.b, new.b)); } +/* + * Result is precise when 'x' and 'y' overlap or form a continuous range, + * result is an over-approximation if 'x' and 'y' do not overlap. + */ +static struct range range_union(enum num_t t, struct range x, struct range y) +{ + if (!is_valid_range(t, x)) + return y; + if (!is_valid_range(t, y)) + return x; + return range(t, min_t(t, x.a, y.a), max_t(t, x.b, y.b)); +} + +/* + * This function attempts to improve x range intersecting it with y. + * range_cast(... to_t ...) looses precision for ranges that pass to_t + * min/max boundaries. To avoid such precision loses this function + * splits both x and y into halves corresponding to non-overflowing + * sub-ranges: [0, smin] and [smax, -1]. + * Final result is computed as follows: + * + * ((x ∩ [0, smax]) ∩ (y ∩ [0, smax])) ∪ + * ((x ∩ [smin,-1]) ∩ (y ∩ [smin,-1])) + * + * Precision might still be lost if final union is not a continuous range. + */ +static struct range range_refine_in_halves(enum num_t x_t, struct range x, + enum num_t y_t, struct range y) +{ + struct range x_pos, x_neg, y_pos, y_neg, r_pos, r_neg; + u64 smax, smin, neg_one; + + if (t_is_32(x_t)) { + smax = (u64)(u32)S32_MAX; + smin = (u64)(u32)S32_MIN; + neg_one = (u64)(u32)(s32)(-1); + } else { + smax = (u64)S64_MAX; + smin = (u64)S64_MIN; + neg_one = U64_MAX; + } + x_pos = range_intersection(x_t, x, range(x_t, 0, smax)); + x_neg = range_intersection(x_t, x, range(x_t, smin, neg_one)); + y_pos = range_intersection(y_t, y, range(x_t, 0, smax)); + y_neg = range_intersection(y_t, y, range(y_t, smin, neg_one)); + r_pos = range_intersection(x_t, x_pos, range_cast(y_t, x_t, y_pos)); + r_neg = range_intersection(x_t, x_neg, range_cast(y_t, x_t, y_neg)); + return range_union(x_t, r_pos, r_neg); + +} + static struct range range_refine(enum num_t x_t, struct range x, enum num_t y_t, struct range y) { struct range y_cast; + if (t_is_32(x_t) == t_is_32(y_t)) + x = range_refine_in_halves(x_t, x, y_t, y); + y_cast = range_cast(y_t, x_t, y); /* If we know that @@ -444,7 +498,7 @@ static struct range range_refine(enum num_t x_t, struct range x, enum num_t y_t, */ if (x_t == S64 && y_t == S32 && y_cast.a <= S32_MAX && y_cast.b <= S32_MAX && (s64)x.a >= S32_MIN && (s64)x.b <= S32_MAX) - return range_improve(x_t, x, y_cast); + return range_intersection(x_t, x, y_cast); /* the case when new range knowledge, *y*, is a 32-bit subregister * range, while previous range knowledge, *x*, is a full register @@ -462,7 +516,7 @@ static struct range range_refine(enum num_t x_t, struct range x, enum num_t y_t, x_swap = range(x_t, swap_low32(x.a, y_cast.a), swap_low32(x.b, y_cast.b)); if (!is_valid_range(x_t, x_swap)) return x; - return range_improve(x_t, x, x_swap); + return range_intersection(x_t, x, x_swap); } if (!t_is_32(x_t) && !t_is_32(y_t) && x_t != y_t) { @@ -480,7 +534,7 @@ static struct range range_refine(enum num_t x_t, struct range x, enum num_t y_t, } /* otherwise, plain range cast and intersection works */ - return range_improve(x_t, x, y_cast); + return range_intersection(x_t, x, y_cast); } /* ======================= @@ -2091,7 +2145,7 @@ static struct subtest_case crafted_cases[] = { {U64, S64, {0, 0xffffffffULL}, {0x7fffffff, 0x7fffffff}}, {U64, U32, {0, 0x100000000}, {0, 0}}, - {U64, U32, {0xfffffffe, 0x100000000}, {0x80000000, 0x80000000}}, + {U64, U32, {0xfffffffe, 0x300000000}, {0x80000000, 0x80000000}}, {U64, S32, {0, 0xffffffff00000000ULL}, {0, 0}}, /* these are tricky cases where lower 32 bits allow to tighten 64 diff --git a/tools/testing/selftests/bpf/prog_tests/stacktrace_ips.c b/tools/testing/selftests/bpf/prog_tests/stacktrace_ips.c index c9efdd2a5b18a5..c93718dafd9b63 100644 --- a/tools/testing/selftests/bpf/prog_tests/stacktrace_ips.c +++ b/tools/testing/selftests/bpf/prog_tests/stacktrace_ips.c @@ -74,11 +74,20 @@ static void test_stacktrace_ips_kprobe_multi(bool retprobe) load_kallsyms(); - check_stacktrace_ips(bpf_map__fd(skel->maps.stackmap), skel->bss->stack_key, 4, - ksym_get_addr("bpf_testmod_stacktrace_test_3"), - ksym_get_addr("bpf_testmod_stacktrace_test_2"), - ksym_get_addr("bpf_testmod_stacktrace_test_1"), - ksym_get_addr("bpf_testmod_test_read")); + if (retprobe) { + check_stacktrace_ips(bpf_map__fd(skel->maps.stackmap), skel->bss->stack_key, 4, + ksym_get_addr("bpf_testmod_stacktrace_test_3"), + ksym_get_addr("bpf_testmod_stacktrace_test_2"), + ksym_get_addr("bpf_testmod_stacktrace_test_1"), + ksym_get_addr("bpf_testmod_test_read")); + } else { + check_stacktrace_ips(bpf_map__fd(skel->maps.stackmap), skel->bss->stack_key, 5, + ksym_get_addr("bpf_testmod_stacktrace_test"), + ksym_get_addr("bpf_testmod_stacktrace_test_3"), + ksym_get_addr("bpf_testmod_stacktrace_test_2"), + ksym_get_addr("bpf_testmod_stacktrace_test_1"), + ksym_get_addr("bpf_testmod_test_read")); + } cleanup: stacktrace_ips__destroy(skel); diff --git a/tools/testing/selftests/bpf/prog_tests/test_xsk.c b/tools/testing/selftests/bpf/prog_tests/test_xsk.c index 5af28f359cfda4..bab4a31621c751 100644 --- a/tools/testing/selftests/bpf/prog_tests/test_xsk.c +++ b/tools/testing/selftests/bpf/prog_tests/test_xsk.c @@ -433,7 +433,7 @@ static u32 pkt_nb_frags(u32 frame_size, struct pkt_stream *pkt_stream, struct pk } /* Search for the end of the packet in verbatim mode */ - if (!pkt_continues(pkt->options)) + if (!pkt_continues(pkt->options) || !pkt->valid) return nb_frags; next_frag = pkt_stream->current_pkt_nb; @@ -1090,6 +1090,8 @@ static int __receive_pkts(struct test_spec *test, struct xsk_socket_info *xsk) xsk_ring_prod__cancel(&umem->fq, nb_frags); } frags_processed -= nb_frags; + pkt_stream_cancel(pkt_stream); + pkts_sent--; } if (ifobj->use_fill_ring) diff --git a/tools/testing/selftests/bpf/prog_tests/wq.c b/tools/testing/selftests/bpf/prog_tests/wq.c index 15c67d23128b24..84831eecc935bd 100644 --- a/tools/testing/selftests/bpf/prog_tests/wq.c +++ b/tools/testing/selftests/bpf/prog_tests/wq.c @@ -16,12 +16,12 @@ void serial_test_wq(void) /* re-run the success test to check if the timer was actually executed */ wq_skel = wq__open_and_load(); - if (!ASSERT_OK_PTR(wq_skel, "wq_skel_load")) + if (!ASSERT_OK_PTR(wq_skel, "wq__open_and_load")) return; err = wq__attach(wq_skel); if (!ASSERT_OK(err, "wq_attach")) - return; + goto clean_up; prog_fd = bpf_program__fd(wq_skel->progs.test_syscall_array_sleepable); err = bpf_prog_test_run_opts(prog_fd, &topts); @@ -31,6 +31,7 @@ void serial_test_wq(void) usleep(50); /* 10 usecs should be enough, but give it extra */ ASSERT_EQ(wq_skel->bss->ok_sleepable, (1 << 1), "ok_sleepable"); +clean_up: wq__destroy(wq_skel); } diff --git a/tools/testing/selftests/bpf/progs/dmabuf_iter.c b/tools/testing/selftests/bpf/progs/dmabuf_iter.c index 13cdb11fdeb2bf..9cbb7442646e58 100644 --- a/tools/testing/selftests/bpf/progs/dmabuf_iter.c +++ b/tools/testing/selftests/bpf/progs/dmabuf_iter.c @@ -48,7 +48,7 @@ int dmabuf_collector(struct bpf_iter__dmabuf *ctx) /* Buffers are not required to be named */ if (pname) { - if (bpf_probe_read_kernel(name, sizeof(name), pname)) + if (bpf_probe_read_kernel_str(name, sizeof(name), pname) < 0) return 1; /* Name strings can be provided by userspace */ diff --git a/tools/testing/selftests/bpf/progs/exceptions_assert.c b/tools/testing/selftests/bpf/progs/exceptions_assert.c index a01c2736890f94..858af5988a38a9 100644 --- a/tools/testing/selftests/bpf/progs/exceptions_assert.c +++ b/tools/testing/selftests/bpf/progs/exceptions_assert.c @@ -18,43 +18,43 @@ return *(u64 *)num; \ } -__msg(": R0=0xffffffff80000000") +__msg("R{{.}}=0xffffffff80000000") check_assert(s64, ==, eq_int_min, INT_MIN); -__msg(": R0=0x7fffffff") +__msg("R{{.}}=0x7fffffff") check_assert(s64, ==, eq_int_max, INT_MAX); -__msg(": R0=0") +__msg("R{{.}}=0") check_assert(s64, ==, eq_zero, 0); -__msg(": R0=0x8000000000000000 R1=0x8000000000000000") +__msg("R{{.}}=0x8000000000000000") check_assert(s64, ==, eq_llong_min, LLONG_MIN); -__msg(": R0=0x7fffffffffffffff R1=0x7fffffffffffffff") +__msg("R{{.}}=0x7fffffffffffffff") check_assert(s64, ==, eq_llong_max, LLONG_MAX); -__msg(": R0=scalar(id=1,smax=0x7ffffffe)") +__msg("R{{.}}=scalar(id=1,smax=0x7ffffffe)") check_assert(s64, <, lt_pos, INT_MAX); -__msg(": R0=scalar(id=1,smax=-1,umin=0x8000000000000000,var_off=(0x8000000000000000; 0x7fffffffffffffff))") +__msg("R{{.}}=scalar(id=1,smax=-1,umin=0x8000000000000000,var_off=(0x8000000000000000; 0x7fffffffffffffff))") check_assert(s64, <, lt_zero, 0); -__msg(": R0=scalar(id=1,smax=0xffffffff7fffffff") +__msg("R{{.}}=scalar(id=1,smax=0xffffffff7fffffff") check_assert(s64, <, lt_neg, INT_MIN); -__msg(": R0=scalar(id=1,smax=0x7fffffff)") +__msg("R{{.}}=scalar(id=1,smax=0x7fffffff)") check_assert(s64, <=, le_pos, INT_MAX); -__msg(": R0=scalar(id=1,smax=0)") +__msg("R{{.}}=scalar(id=1,smax=0)") check_assert(s64, <=, le_zero, 0); -__msg(": R0=scalar(id=1,smax=0xffffffff80000000") +__msg("R{{.}}=scalar(id=1,smax=0xffffffff80000000") check_assert(s64, <=, le_neg, INT_MIN); -__msg(": R0=scalar(id=1,smin=umin=0x80000000,umax=0x7fffffffffffffff,var_off=(0x0; 0x7fffffffffffffff))") +__msg("R{{.}}=scalar(id=1,smin=umin=0x80000000,umax=0x7fffffffffffffff,var_off=(0x0; 0x7fffffffffffffff))") check_assert(s64, >, gt_pos, INT_MAX); -__msg(": R0=scalar(id=1,smin=umin=1,umax=0x7fffffffffffffff,var_off=(0x0; 0x7fffffffffffffff))") +__msg("R{{.}}=scalar(id=1,smin=umin=1,umax=0x7fffffffffffffff,var_off=(0x0; 0x7fffffffffffffff))") check_assert(s64, >, gt_zero, 0); -__msg(": R0=scalar(id=1,smin=0xffffffff80000001") +__msg("R{{.}}=scalar(id=1,smin=0xffffffff80000001") check_assert(s64, >, gt_neg, INT_MIN); -__msg(": R0=scalar(id=1,smin=umin=0x7fffffff,umax=0x7fffffffffffffff,var_off=(0x0; 0x7fffffffffffffff))") +__msg("R{{.}}=scalar(id=1,smin=umin=0x7fffffff,umax=0x7fffffffffffffff,var_off=(0x0; 0x7fffffffffffffff))") check_assert(s64, >=, ge_pos, INT_MAX); -__msg(": R0=scalar(id=1,smin=0,umax=0x7fffffffffffffff,var_off=(0x0; 0x7fffffffffffffff))") +__msg("R{{.}}=scalar(id=1,smin=0,umax=0x7fffffffffffffff,var_off=(0x0; 0x7fffffffffffffff))") check_assert(s64, >=, ge_zero, 0); -__msg(": R0=scalar(id=1,smin=0xffffffff80000000") +__msg("R{{.}}=scalar(id=1,smin=0xffffffff80000000") check_assert(s64, >=, ge_neg, INT_MIN); SEC("?tc") diff --git a/tools/testing/selftests/bpf/progs/exceptions_fail.c b/tools/testing/selftests/bpf/progs/exceptions_fail.c index 8a0fdff899271d..d7f1c492e3dd33 100644 --- a/tools/testing/selftests/bpf/progs/exceptions_fail.c +++ b/tools/testing/selftests/bpf/progs/exceptions_fail.c @@ -8,6 +8,7 @@ #include "bpf_experimental.h" extern void bpf_rcu_read_lock(void) __ksym; +extern void bpf_rcu_read_unlock(void) __ksym; #define private(name) SEC(".bss." #name) __hidden __attribute__((aligned(8))) @@ -131,7 +132,7 @@ int reject_subprog_with_lock(void *ctx) } SEC("?tc") -__failure __msg("BPF_EXIT instruction in main prog cannot be used inside bpf_rcu_read_lock-ed region") +__failure __msg("bpf_throw cannot be used inside bpf_rcu_read_lock-ed region") int reject_with_rcu_read_lock(void *ctx) { bpf_rcu_read_lock(); @@ -147,11 +148,13 @@ __noinline static int throwing_subprog(struct __sk_buff *ctx) } SEC("?tc") -__failure __msg("BPF_EXIT instruction in main prog cannot be used inside bpf_rcu_read_lock-ed region") +__failure __msg("bpf_throw cannot be used inside bpf_rcu_read_lock-ed region") int reject_subprog_with_rcu_read_lock(void *ctx) { bpf_rcu_read_lock(); - return throwing_subprog(ctx); + throwing_subprog(ctx); + bpf_rcu_read_unlock(); + return 0; } static bool rbless(struct bpf_rb_node *n1, const struct bpf_rb_node *n2) diff --git a/tools/testing/selftests/bpf/progs/verifier_bounds.c b/tools/testing/selftests/bpf/progs/verifier_bounds.c index 411a18437d7eab..74d4985800523d 100644 --- a/tools/testing/selftests/bpf/progs/verifier_bounds.c +++ b/tools/testing/selftests/bpf/progs/verifier_bounds.c @@ -1863,4 +1863,141 @@ l1_%=: r0 = 1; \ : __clobber_all); } +/* This test covers the bounds deduction when the u64 range and the tnum + * overlap only at umax. After instruction 3, the ranges look as follows: + * + * 0 umin=0xe01 umax=0xf00 U64_MAX + * | [xxxxxxxxxxxxxx] | + * |----------------------------|------------------------------| + * | x x | tnum values + * + * The verifier can therefore deduce that the R0=0xf0=240. + */ +SEC("socket") +__description("bounds refinement with single-value tnum on umax") +__msg("3: (15) if r0 == 0xe0 {{.*}} R0=240") +__success __log_level(2) +__flag(BPF_F_TEST_REG_INVARIANTS) +__naked void bounds_refinement_tnum_umax(void *ctx) +{ + asm volatile(" \ + call %[bpf_get_prandom_u32]; \ + r0 |= 0xe0; \ + r0 &= 0xf0; \ + if r0 == 0xe0 goto +2; \ + if r0 == 0xf0 goto +1; \ + r10 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* This test covers the bounds deduction when the u64 range and the tnum + * overlap only at umin. After instruction 3, the ranges look as follows: + * + * 0 umin=0xe00 umax=0xeff U64_MAX + * | [xxxxxxxxxxxxxx] | + * |----------------------------|------------------------------| + * | x x | tnum values + * + * The verifier can therefore deduce that the R0=0xe0=224. + */ +SEC("socket") +__description("bounds refinement with single-value tnum on umin") +__msg("3: (15) if r0 == 0xf0 {{.*}} R0=224") +__success __log_level(2) +__flag(BPF_F_TEST_REG_INVARIANTS) +__naked void bounds_refinement_tnum_umin(void *ctx) +{ + asm volatile(" \ + call %[bpf_get_prandom_u32]; \ + r0 |= 0xe0; \ + r0 &= 0xf0; \ + if r0 == 0xf0 goto +2; \ + if r0 == 0xe0 goto +1; \ + r10 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* This test covers the bounds deduction when the only possible tnum value is + * in the middle of the u64 range. After instruction 3, the ranges look as + * follows: + * + * 0 umin=0x7cf umax=0x7df U64_MAX + * | [xxxxxxxxxxxx] | + * |----------------------------|------------------------------| + * | x x x x x | tnum values + * | +--- 0x7e0 + * +--- 0x7d0 + * + * Since the lower four bits are zero, the tnum and the u64 range only overlap + * in R0=0x7d0=2000. Instruction 5 is therefore dead code. + */ +SEC("socket") +__description("bounds refinement with single-value tnum in middle of range") +__msg("3: (a5) if r0 < 0x7cf {{.*}} R0=2000") +__success __log_level(2) +__naked void bounds_refinement_tnum_middle(void *ctx) +{ + asm volatile(" \ + call %[bpf_get_prandom_u32]; \ + if r0 & 0x0f goto +4; \ + if r0 > 0x7df goto +3; \ + if r0 < 0x7cf goto +2; \ + if r0 == 0x7d0 goto +1; \ + r10 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* This test cover the negative case for the tnum/u64 overlap. Since + * they contain the same two values (i.e., {0, 1}), we can't deduce + * anything more. + */ +SEC("socket") +__description("bounds refinement: several overlaps between tnum and u64") +__msg("2: (25) if r0 > 0x1 {{.*}} R0=scalar(smin=smin32=0,smax=umax=smax32=umax32=1,var_off=(0x0; 0x1))") +__failure __log_level(2) +__naked void bounds_refinement_several_overlaps(void *ctx) +{ + asm volatile(" \ + call %[bpf_get_prandom_u32]; \ + if r0 < 0 goto +3; \ + if r0 > 1 goto +2; \ + if r0 == 1 goto +1; \ + r10 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* This test cover the negative case for the tnum/u64 overlap. Since + * they overlap in the two values contained by the u64 range (i.e., + * {0xf, 0x10}), we can't deduce anything more. + */ +SEC("socket") +__description("bounds refinement: multiple overlaps between tnum and u64") +__msg("2: (25) if r0 > 0x10 {{.*}} R0=scalar(smin=umin=smin32=umin32=15,smax=umax=smax32=umax32=16,var_off=(0x0; 0x1f))") +__failure __log_level(2) +__naked void bounds_refinement_multiple_overlaps(void *ctx) +{ + asm volatile(" \ + call %[bpf_get_prandom_u32]; \ + if r0 < 0xf goto +3; \ + if r0 > 0x10 goto +2; \ + if r0 == 0x10 goto +1; \ + r10 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_scalar_ids.c b/tools/testing/selftests/bpf/progs/verifier_scalar_ids.c index c0ce690ddb68a7..1fdd85b4b84432 100644 --- a/tools/testing/selftests/bpf/progs/verifier_scalar_ids.c +++ b/tools/testing/selftests/bpf/progs/verifier_scalar_ids.c @@ -40,6 +40,9 @@ __naked void linked_regs_bpf_k(void) */ "r3 = r10;" "r3 += r0;" + /* Mark r1 and r2 as alive. */ + "r1 = r1;" + "r2 = r2;" "r0 = 0;" "exit;" : @@ -73,6 +76,9 @@ __naked void linked_regs_bpf_x_src(void) */ "r4 = r10;" "r4 += r0;" + /* Mark r1 and r2 as alive. */ + "r1 = r1;" + "r2 = r2;" "r0 = 0;" "exit;" : @@ -106,6 +112,10 @@ __naked void linked_regs_bpf_x_dst(void) */ "r4 = r10;" "r4 += r3;" + /* Mark r1 and r2 as alive. */ + "r0 = r0;" + "r1 = r1;" + "r2 = r2;" "r0 = 0;" "exit;" : @@ -143,6 +153,9 @@ __naked void linked_regs_broken_link(void) */ "r3 = r10;" "r3 += r0;" + /* Mark r1 and r2 as alive. */ + "r1 = r1;" + "r2 = r2;" "r0 = 0;" "exit;" : @@ -156,16 +169,16 @@ __naked void linked_regs_broken_link(void) */ SEC("socket") __success __log_level(2) -__msg("12: (0f) r2 += r1") +__msg("17: (0f) r2 += r1") /* Current state */ -__msg("frame2: last_idx 12 first_idx 11 subseq_idx -1 ") -__msg("frame2: regs=r1 stack= before 11: (bf) r2 = r10") +__msg("frame2: last_idx 17 first_idx 14 subseq_idx -1 ") +__msg("frame2: regs=r1 stack= before 16: (bf) r2 = r10") __msg("frame2: parent state regs=r1 stack=") __msg("frame1: parent state regs= stack=") __msg("frame0: parent state regs= stack=") /* Parent state */ -__msg("frame2: last_idx 10 first_idx 10 subseq_idx 11 ") -__msg("frame2: regs=r1 stack= before 10: (25) if r1 > 0x7 goto pc+0") +__msg("frame2: last_idx 13 first_idx 13 subseq_idx 14 ") +__msg("frame2: regs=r1 stack= before 13: (25) if r1 > 0x7 goto pc+0") __msg("frame2: parent state regs=r1 stack=") /* frame1.r{6,7} are marked because mark_precise_scalar_ids() * looks for all registers with frame2.r1.id in the current state @@ -173,20 +186,20 @@ __msg("frame2: parent state regs=r1 stack=") __msg("frame1: parent state regs=r6,r7 stack=") __msg("frame0: parent state regs=r6 stack=") /* Parent state */ -__msg("frame2: last_idx 8 first_idx 8 subseq_idx 10") -__msg("frame2: regs=r1 stack= before 8: (85) call pc+1") +__msg("frame2: last_idx 9 first_idx 9 subseq_idx 13") +__msg("frame2: regs=r1 stack= before 9: (85) call pc+3") /* frame1.r1 is marked because of backtracking of call instruction */ __msg("frame1: parent state regs=r1,r6,r7 stack=") __msg("frame0: parent state regs=r6 stack=") /* Parent state */ -__msg("frame1: last_idx 7 first_idx 6 subseq_idx 8") -__msg("frame1: regs=r1,r6,r7 stack= before 7: (bf) r7 = r1") -__msg("frame1: regs=r1,r6 stack= before 6: (bf) r6 = r1") +__msg("frame1: last_idx 8 first_idx 7 subseq_idx 9") +__msg("frame1: regs=r1,r6,r7 stack= before 8: (bf) r7 = r1") +__msg("frame1: regs=r1,r6 stack= before 7: (bf) r6 = r1") __msg("frame1: parent state regs=r1 stack=") __msg("frame0: parent state regs=r6 stack=") /* Parent state */ -__msg("frame1: last_idx 4 first_idx 4 subseq_idx 6") -__msg("frame1: regs=r1 stack= before 4: (85) call pc+1") +__msg("frame1: last_idx 4 first_idx 4 subseq_idx 7") +__msg("frame1: regs=r1 stack= before 4: (85) call pc+2") __msg("frame0: parent state regs=r1,r6 stack=") /* Parent state */ __msg("frame0: last_idx 3 first_idx 1 subseq_idx 4") @@ -204,6 +217,7 @@ __naked void precision_many_frames(void) "r1 = r0;" "r6 = r0;" "call precision_many_frames__foo;" + "r6 = r6;" /* mark r6 as live */ "exit;" : : __imm(bpf_ktime_get_ns) @@ -220,6 +234,8 @@ void precision_many_frames__foo(void) "r6 = r1;" "r7 = r1;" "call precision_many_frames__bar;" + "r6 = r6;" /* mark r6 as live */ + "r7 = r7;" /* mark r7 as live */ "exit" ::: __clobber_all); } @@ -229,6 +245,8 @@ void precision_many_frames__bar(void) { asm volatile ( "if r1 > 7 goto +0;" + "r6 = 0;" /* mark r6 as live */ + "r7 = 0;" /* mark r7 as live */ /* force r1 to be precise, this eventually marks: * - bar frame r1 * - foo frame r{1,6,7} @@ -340,6 +358,8 @@ __naked void precision_two_ids(void) "r3 += r7;" /* force r9 to be precise, this also marks r8 */ "r3 += r9;" + "r6 = r6;" /* mark r6 as live */ + "r8 = r8;" /* mark r8 as live */ "exit;" : : __imm(bpf_ktime_get_ns) @@ -353,7 +373,7 @@ __flag(BPF_F_TEST_STATE_FREQ) * collect_linked_regs() can't tie more than 6 registers for a single insn. */ __msg("8: (25) if r0 > 0x7 goto pc+0 ; R0=scalar(id=1") -__msg("9: (bf) r6 = r6 ; R6=scalar(id=2") +__msg("14: (bf) r6 = r6 ; R6=scalar(id=2") /* check that r{0-5} are marked precise after 'if' */ __msg("frame0: regs=r0 stack= before 8: (25) if r0 > 0x7 goto pc+0") __msg("frame0: parent state regs=r0,r1,r2,r3,r4,r5 stack=:") @@ -372,6 +392,12 @@ __naked void linked_regs_too_many_regs(void) "r6 = r0;" /* propagate range for r{0-6} */ "if r0 > 7 goto +0;" + /* keep r{1-5} live */ + "r1 = r1;" + "r2 = r2;" + "r3 = r3;" + "r4 = r4;" + "r5 = r5;" /* make r6 appear in the log */ "r6 = r6;" /* force r0 to be precise, @@ -517,7 +543,7 @@ __naked void check_ids_in_regsafe_2(void) "*(u64*)(r10 - 8) = r1;" /* r9 = pointer to stack */ "r9 = r10;" - "r9 += -8;" + "r9 += -16;" /* r8 = ktime_get_ns() */ "call %[bpf_ktime_get_ns];" "r8 = r0;" @@ -538,6 +564,8 @@ __naked void check_ids_in_regsafe_2(void) "if r7 > 4 goto l2_%=;" /* Access memory at r9[r6] */ "r9 += r6;" + "r9 += r7;" + "r9 += r8;" "r0 = *(u8*)(r9 + 0);" "l2_%=:" "r0 = 0;" diff --git a/tools/testing/selftests/bpf/verifier/precise.c b/tools/testing/selftests/bpf/verifier/precise.c index 59a020c3564742..ef3ec56672c221 100644 --- a/tools/testing/selftests/bpf/verifier/precise.c +++ b/tools/testing/selftests/bpf/verifier/precise.c @@ -44,9 +44,9 @@ mark_precise: frame0: regs=r2 stack= before 23\ mark_precise: frame0: regs=r2 stack= before 22\ mark_precise: frame0: regs=r2 stack= before 20\ - mark_precise: frame0: parent state regs=r2,r9 stack=:\ + mark_precise: frame0: parent state regs=r2 stack=:\ mark_precise: frame0: last_idx 19 first_idx 10\ - mark_precise: frame0: regs=r2,r9 stack= before 19\ + mark_precise: frame0: regs=r2 stack= before 19\ mark_precise: frame0: regs=r9 stack= before 18\ mark_precise: frame0: regs=r8,r9 stack= before 17\ mark_precise: frame0: regs=r0,r9 stack= before 15\ @@ -107,9 +107,9 @@ mark_precise: frame0: parent state regs=r2 stack=:\ mark_precise: frame0: last_idx 20 first_idx 20\ mark_precise: frame0: regs=r2 stack= before 20\ - mark_precise: frame0: parent state regs=r2,r9 stack=:\ + mark_precise: frame0: parent state regs=r2 stack=:\ mark_precise: frame0: last_idx 19 first_idx 17\ - mark_precise: frame0: regs=r2,r9 stack= before 19\ + mark_precise: frame0: regs=r2 stack= before 19\ mark_precise: frame0: regs=r9 stack= before 18\ mark_precise: frame0: regs=r8,r9 stack= before 17\ mark_precise: frame0: parent state regs= stack=:", diff --git a/tools/testing/selftests/bpf/veristat.c b/tools/testing/selftests/bpf/veristat.c index e962f133250c7d..1be1e353d40a7b 100644 --- a/tools/testing/selftests/bpf/veristat.c +++ b/tools/testing/selftests/bpf/veristat.c @@ -2580,7 +2580,7 @@ static void output_stats(const struct verif_stats *s, enum resfmt fmt, bool last if (last && fmt == RESFMT_TABLE) { output_header_underlines(); printf("Done. Processed %d files, %d programs. Skipped %d files, %d programs.\n", - env.files_processed, env.files_skipped, env.progs_processed, env.progs_skipped); + env.files_processed, env.progs_processed, env.files_skipped, env.progs_skipped); } } diff --git a/tools/testing/selftests/cgroup/lib/cgroup_util.c b/tools/testing/selftests/cgroup/lib/cgroup_util.c index 44c52f620fda17..4b0f2c46d43221 100644 --- a/tools/testing/selftests/cgroup/lib/cgroup_util.c +++ b/tools/testing/selftests/cgroup/lib/cgroup_util.c @@ -123,6 +123,21 @@ int cg_read_strcmp(const char *cgroup, const char *control, return ret; } +int cg_read_strcmp_wait(const char *cgroup, const char *control, + const char *expected) +{ + int i, ret; + + for (i = 0; i < 100; i++) { + ret = cg_read_strcmp(cgroup, control, expected); + if (!ret) + return ret; + usleep(10000); + } + + return ret; +} + int cg_read_strstr(const char *cgroup, const char *control, const char *needle) { char buf[PAGE_SIZE]; diff --git a/tools/testing/selftests/cgroup/lib/include/cgroup_util.h b/tools/testing/selftests/cgroup/lib/include/cgroup_util.h index 7ab2824ed7b54d..1cbe3b0ac6f737 100644 --- a/tools/testing/selftests/cgroup/lib/include/cgroup_util.h +++ b/tools/testing/selftests/cgroup/lib/include/cgroup_util.h @@ -59,6 +59,8 @@ extern int cg_read(const char *cgroup, const char *control, char *buf, size_t len); extern int cg_read_strcmp(const char *cgroup, const char *control, const char *expected); +extern int cg_read_strcmp_wait(const char *cgroup, const char *control, + const char *expected); extern int cg_read_strstr(const char *cgroup, const char *control, const char *needle); extern long cg_read_long(const char *cgroup, const char *control); diff --git a/tools/testing/selftests/cgroup/test_core.c b/tools/testing/selftests/cgroup/test_core.c index 102262555a599b..7b83c7e7c9d4fa 100644 --- a/tools/testing/selftests/cgroup/test_core.c +++ b/tools/testing/selftests/cgroup/test_core.c @@ -233,7 +233,8 @@ static int test_cgcore_populated(const char *root) if (err) goto cleanup; - if (cg_read_strcmp(cg_test_d, "cgroup.events", "populated 0\n")) + if (cg_read_strcmp_wait(cg_test_d, "cgroup.events", + "populated 0\n")) goto cleanup; /* Remove cgroup. */ diff --git a/tools/testing/selftests/cgroup/test_kill.c b/tools/testing/selftests/cgroup/test_kill.c index c8c9d306925b63..f6cd23a8ecc71c 100644 --- a/tools/testing/selftests/cgroup/test_kill.c +++ b/tools/testing/selftests/cgroup/test_kill.c @@ -86,7 +86,7 @@ static int test_cgkill_simple(const char *root) wait_for_pid(pids[i]); if (ret == KSFT_PASS && - cg_read_strcmp(cgroup, "cgroup.events", "populated 0\n")) + cg_read_strcmp_wait(cgroup, "cgroup.events", "populated 0\n")) ret = KSFT_FAIL; if (cgroup) @@ -190,7 +190,8 @@ static int test_cgkill_tree(const char *root) wait_for_pid(pids[i]); if (ret == KSFT_PASS && - cg_read_strcmp(cgroup[0], "cgroup.events", "populated 0\n")) + cg_read_strcmp_wait(cgroup[0], "cgroup.events", + "populated 0\n")) ret = KSFT_FAIL; for (i = 9; i >= 0 && cgroup[i]; i--) { @@ -251,7 +252,7 @@ static int test_cgkill_forkbomb(const char *root) wait_for_pid(pid); if (ret == KSFT_PASS && - cg_read_strcmp(cgroup, "cgroup.events", "populated 0\n")) + cg_read_strcmp_wait(cgroup, "cgroup.events", "populated 0\n")) ret = KSFT_FAIL; if (cgroup) diff --git a/tools/testing/selftests/drivers/net/lib/sh/lib_netcons.sh b/tools/testing/selftests/drivers/net/lib/sh/lib_netcons.sh index ae8abff4be409e..64d3941576d5db 100644 --- a/tools/testing/selftests/drivers/net/lib/sh/lib_netcons.sh +++ b/tools/testing/selftests/drivers/net/lib/sh/lib_netcons.sh @@ -247,8 +247,8 @@ function listen_port_and_save_to() { SOCAT_MODE="UDP6-LISTEN" fi - # Just wait for 2 seconds - timeout 2 ip netns exec "${NAMESPACE}" \ + # Just wait for 3 seconds + timeout 3 ip netns exec "${NAMESPACE}" \ socat "${SOCAT_MODE}":"${PORT}",fork "${OUTPUT}" 2> /dev/null } diff --git a/tools/testing/selftests/drivers/net/mlxsw/tc_restrictions.sh b/tools/testing/selftests/drivers/net/mlxsw/tc_restrictions.sh index 0441a18f098b10..aac8ef490feb80 100755 --- a/tools/testing/selftests/drivers/net/mlxsw/tc_restrictions.sh +++ b/tools/testing/selftests/drivers/net/mlxsw/tc_restrictions.sh @@ -317,7 +317,7 @@ police_limits_test() tc filter add dev $swp1 ingress pref 1 proto ip handle 101 \ flower skip_sw \ - action police rate 0.5kbit burst 1m conform-exceed drop/ok + action police rate 0.5kbit burst 2k conform-exceed drop/ok check_fail $? "Incorrect success to add police action with too low rate" tc filter add dev $swp1 ingress pref 1 proto ip handle 101 \ @@ -327,7 +327,7 @@ police_limits_test() tc filter add dev $swp1 ingress pref 1 proto ip handle 101 \ flower skip_sw \ - action police rate 1.5kbit burst 1m conform-exceed drop/ok + action police rate 1.5kbit burst 2k conform-exceed drop/ok check_err $? "Failed to add police action with low rate" tc filter del dev $swp1 ingress protocol ip pref 1 handle 101 flower diff --git a/tools/testing/selftests/filesystems/nsfs/iterate_mntns.c b/tools/testing/selftests/filesystems/nsfs/iterate_mntns.c index 61e55dfbf121da..e19ff8168baffe 100644 --- a/tools/testing/selftests/filesystems/nsfs/iterate_mntns.c +++ b/tools/testing/selftests/filesystems/nsfs/iterate_mntns.c @@ -37,17 +37,20 @@ FIXTURE(iterate_mount_namespaces) { __u64 mnt_ns_id[MNT_NS_COUNT]; }; +static inline bool mntns_in_list(__u64 *mnt_ns_id, struct mnt_ns_info *info) +{ + for (int i = 0; i < MNT_NS_COUNT; i++) { + if (mnt_ns_id[i] == info->mnt_ns_id) + return true; + } + return false; +} + FIXTURE_SETUP(iterate_mount_namespaces) { for (int i = 0; i < MNT_NS_COUNT; i++) self->fd_mnt_ns[i] = -EBADF; - /* - * Creating a new user namespace let's us guarantee that we only see - * mount namespaces that we did actually create. - */ - ASSERT_EQ(unshare(CLONE_NEWUSER), 0); - for (int i = 0; i < MNT_NS_COUNT; i++) { struct mnt_ns_info info = {}; @@ -75,13 +78,15 @@ TEST_F(iterate_mount_namespaces, iterate_all_forward) fd_mnt_ns_cur = fcntl(self->fd_mnt_ns[0], F_DUPFD_CLOEXEC); ASSERT_GE(fd_mnt_ns_cur, 0); - for (;; count++) { + for (;;) { struct mnt_ns_info info = {}; int fd_mnt_ns_next; fd_mnt_ns_next = ioctl(fd_mnt_ns_cur, NS_MNT_GET_NEXT, &info); if (fd_mnt_ns_next < 0 && errno == ENOENT) break; + if (mntns_in_list(self->mnt_ns_id, &info)) + count++; ASSERT_GE(fd_mnt_ns_next, 0); ASSERT_EQ(close(fd_mnt_ns_cur), 0); fd_mnt_ns_cur = fd_mnt_ns_next; @@ -96,13 +101,15 @@ TEST_F(iterate_mount_namespaces, iterate_all_backwards) fd_mnt_ns_cur = fcntl(self->fd_mnt_ns[MNT_NS_LAST_INDEX], F_DUPFD_CLOEXEC); ASSERT_GE(fd_mnt_ns_cur, 0); - for (;; count++) { + for (;;) { struct mnt_ns_info info = {}; int fd_mnt_ns_prev; fd_mnt_ns_prev = ioctl(fd_mnt_ns_cur, NS_MNT_GET_PREV, &info); if (fd_mnt_ns_prev < 0 && errno == ENOENT) break; + if (mntns_in_list(self->mnt_ns_id, &info)) + count++; ASSERT_GE(fd_mnt_ns_prev, 0); ASSERT_EQ(close(fd_mnt_ns_cur), 0); fd_mnt_ns_cur = fd_mnt_ns_prev; @@ -125,7 +132,6 @@ TEST_F(iterate_mount_namespaces, iterate_forward) ASSERT_GE(fd_mnt_ns_next, 0); ASSERT_EQ(close(fd_mnt_ns_cur), 0); fd_mnt_ns_cur = fd_mnt_ns_next; - ASSERT_EQ(info.mnt_ns_id, self->mnt_ns_id[i]); } } @@ -144,7 +150,6 @@ TEST_F(iterate_mount_namespaces, iterate_backward) ASSERT_GE(fd_mnt_ns_prev, 0); ASSERT_EQ(close(fd_mnt_ns_cur), 0); fd_mnt_ns_cur = fd_mnt_ns_prev; - ASSERT_EQ(info.mnt_ns_id, self->mnt_ns_id[i]); } } diff --git a/tools/testing/selftests/hid/progs/hid_bpf_helpers.h b/tools/testing/selftests/hid/progs/hid_bpf_helpers.h index 531228b849daeb..125a975f32f9e2 100644 --- a/tools/testing/selftests/hid/progs/hid_bpf_helpers.h +++ b/tools/testing/selftests/hid/progs/hid_bpf_helpers.h @@ -6,8 +6,10 @@ #define __HID_BPF_HELPERS_H /* "undefine" structs and enums in vmlinux.h, because we "override" them below */ +#define bpf_wq bpf_wq___not_used #define hid_bpf_ctx hid_bpf_ctx___not_used #define hid_bpf_ops hid_bpf_ops___not_used +#define hid_device hid_device___not_used #define hid_report_type hid_report_type___not_used #define hid_class_request hid_class_request___not_used #define hid_bpf_attach_flags hid_bpf_attach_flags___not_used @@ -27,8 +29,10 @@ #include "vmlinux.h" +#undef bpf_wq #undef hid_bpf_ctx #undef hid_bpf_ops +#undef hid_device #undef hid_report_type #undef hid_class_request #undef hid_bpf_attach_flags @@ -55,6 +59,14 @@ enum hid_report_type { HID_REPORT_TYPES, }; +struct hid_device { + unsigned int id; +} __attribute__((preserve_access_index)); + +struct bpf_wq { + __u64 __opaque[2]; +}; + struct hid_bpf_ctx { struct hid_device *hid; __u32 allocated_size; diff --git a/tools/testing/selftests/kselftest_harness.h b/tools/testing/selftests/kselftest_harness.h index 16a119a4656c7e..4afaef01c22e97 100644 --- a/tools/testing/selftests/kselftest_harness.h +++ b/tools/testing/selftests/kselftest_harness.h @@ -76,6 +76,9 @@ static inline void __kselftest_memset_safe(void *s, int c, size_t n) memset(s, c, n); } +#define KSELFTEST_PRIO_TEST_F 20000 +#define KSELFTEST_PRIO_XFAIL 20001 + #define TEST_TIMEOUT_DEFAULT 30 /* Utilities exposed to the test definitions */ @@ -465,7 +468,7 @@ static inline void __kselftest_memset_safe(void *s, int c, size_t n) fixture_name##_teardown(_metadata, self, variant); \ } \ static struct __test_metadata *_##fixture_name##_##test_name##_object; \ - static void __attribute__((constructor)) \ + static void __attribute__((constructor(KSELFTEST_PRIO_TEST_F))) \ _register_##fixture_name##_##test_name(void) \ { \ struct __test_metadata *object = mmap(NULL, sizeof(*object), \ @@ -880,7 +883,7 @@ struct __test_xfail { .fixture = &_##fixture_name##_fixture_object, \ .variant = &_##fixture_name##_##variant_name##_object, \ }; \ - static void __attribute__((constructor)) \ + static void __attribute__((constructor(KSELFTEST_PRIO_XFAIL))) \ _register_##fixture_name##_##variant_name##_##test_name##_xfail(void) \ { \ _##fixture_name##_##variant_name##_##test_name##_xfail.test = \ diff --git a/tools/testing/selftests/kvm/x86/sev_migrate_tests.c b/tools/testing/selftests/kvm/x86/sev_migrate_tests.c index 0a6dfba3905b68..6b0928e69051d5 100644 --- a/tools/testing/selftests/kvm/x86/sev_migrate_tests.c +++ b/tools/testing/selftests/kvm/x86/sev_migrate_tests.c @@ -36,8 +36,6 @@ static struct kvm_vm *sev_vm_create(bool es) sev_vm_launch(vm, es ? SEV_POLICY_ES : 0); - if (es) - vm_sev_ioctl(vm, KVM_SEV_LAUNCH_UPDATE_VMSA, NULL); return vm; } diff --git a/tools/testing/selftests/memfd/memfd_test.c b/tools/testing/selftests/memfd/memfd_test.c index 5b993924cc3f5b..2ca07ea7202a5f 100644 --- a/tools/testing/selftests/memfd/memfd_test.c +++ b/tools/testing/selftests/memfd/memfd_test.c @@ -18,6 +18,9 @@ #include #include #include +#include +#include +#include #include #include @@ -39,6 +42,20 @@ F_SEAL_EXEC) #define MFD_NOEXEC_SEAL 0x0008U +union semun { + int val; + struct semid_ds *buf; + unsigned short int *array; + struct seminfo *__buf; +}; + +/* + * we use semaphores on nested wait tasks due the use of CLONE_NEWPID: the + * child will be PID 1 and can't send SIGSTOP to themselves due special + * treatment of the init task, so the SIGSTOP/SIGCONT synchronization + * approach can't be used here. + */ +#define SEM_KEY 0xdeadbeef /* * Default is not to test hugetlbfs @@ -1333,8 +1350,22 @@ static int sysctl_nested(void *arg) static int sysctl_nested_wait(void *arg) { - /* Wait for a SIGCONT. */ - kill(getpid(), SIGSTOP); + int sem = semget(SEM_KEY, 1, 0600); + struct sembuf sembuf; + + if (sem < 0) { + perror("semget:"); + abort(); + } + sembuf.sem_num = 0; + sembuf.sem_flg = 0; + sembuf.sem_op = 0; + + if (semop(sem, &sembuf, 1) < 0) { + perror("semop:"); + abort(); + } + return sysctl_nested(arg); } @@ -1355,7 +1386,9 @@ static void test_sysctl_sysctl2_failset(void) static int sysctl_nested_child(void *arg) { - int pid; + int pid, sem; + union semun semun; + struct sembuf sembuf; printf("%s nested sysctl 0\n", memfd_str); sysctl_assert_write("0"); @@ -1389,23 +1422,53 @@ static int sysctl_nested_child(void *arg) test_sysctl_sysctl2_failset); join_thread(pid); + sem = semget(SEM_KEY, 1, IPC_CREAT | 0600); + if (sem < 0) { + perror("semget:"); + return 1; + } + semun.val = 1; + sembuf.sem_op = -1; + sembuf.sem_flg = 0; + sembuf.sem_num = 0; + /* Verify that the rules are actually inherited after fork. */ printf("%s nested sysctl 0 -> 1 after fork\n", memfd_str); sysctl_assert_write("0"); + if (semctl(sem, 0, SETVAL, semun) < 0) { + perror("semctl:"); + return 1; + } + pid = spawn_thread(CLONE_NEWPID, sysctl_nested_wait, test_sysctl_sysctl1_failset); sysctl_assert_write("1"); - kill(pid, SIGCONT); + + /* Allow child to continue */ + if (semop(sem, &sembuf, 1) < 0) { + perror("semop:"); + return 1; + } join_thread(pid); printf("%s nested sysctl 0 -> 2 after fork\n", memfd_str); sysctl_assert_write("0"); + if (semctl(sem, 0, SETVAL, semun) < 0) { + perror("semctl:"); + return 1; + } + pid = spawn_thread(CLONE_NEWPID, sysctl_nested_wait, test_sysctl_sysctl2_failset); sysctl_assert_write("2"); - kill(pid, SIGCONT); + + /* Allow child to continue */ + if (semop(sem, &sembuf, 1) < 0) { + perror("semop:"); + return 1; + } join_thread(pid); /* @@ -1415,28 +1478,62 @@ static int sysctl_nested_child(void *arg) */ printf("%s nested sysctl 2 -> 1 after fork\n", memfd_str); sysctl_assert_write("2"); + + if (semctl(sem, 0, SETVAL, semun) < 0) { + perror("semctl:"); + return 1; + } + pid = spawn_thread(CLONE_NEWPID, sysctl_nested_wait, test_sysctl_sysctl2); sysctl_assert_write("1"); - kill(pid, SIGCONT); + + /* Allow child to continue */ + if (semop(sem, &sembuf, 1) < 0) { + perror("semop:"); + return 1; + } join_thread(pid); printf("%s nested sysctl 2 -> 0 after fork\n", memfd_str); sysctl_assert_write("2"); + + if (semctl(sem, 0, SETVAL, semun) < 0) { + perror("semctl:"); + return 1; + } + pid = spawn_thread(CLONE_NEWPID, sysctl_nested_wait, test_sysctl_sysctl2); sysctl_assert_write("0"); - kill(pid, SIGCONT); + + /* Allow child to continue */ + if (semop(sem, &sembuf, 1) < 0) { + perror("semop:"); + return 1; + } join_thread(pid); printf("%s nested sysctl 1 -> 0 after fork\n", memfd_str); sysctl_assert_write("1"); + + if (semctl(sem, 0, SETVAL, semun) < 0) { + perror("semctl:"); + return 1; + } + pid = spawn_thread(CLONE_NEWPID, sysctl_nested_wait, test_sysctl_sysctl1); sysctl_assert_write("0"); - kill(pid, SIGCONT); + /* Allow child to continue */ + if (semop(sem, &sembuf, 1) < 0) { + perror("semop:"); + return 1; + } join_thread(pid); + semctl(sem, 0, IPC_RMID); + return 0; } diff --git a/tools/testing/selftests/mm/charge_reserved_hugetlb.sh b/tools/testing/selftests/mm/charge_reserved_hugetlb.sh index e1fe16bcbbe880..fa6713892d82d8 100755 --- a/tools/testing/selftests/mm/charge_reserved_hugetlb.sh +++ b/tools/testing/selftests/mm/charge_reserved_hugetlb.sh @@ -290,7 +290,7 @@ function run_test() { setup_cgroup "hugetlb_cgroup_test" "$cgroup_limit" "$reservation_limit" mkdir -p /mnt/huge - mount -t hugetlbfs -o pagesize=${MB}M,size=256M none /mnt/huge + mount -t hugetlbfs -o pagesize=${MB}M none /mnt/huge write_hugetlbfs_and_get_usage "hugetlb_cgroup_test" "$size" "$populate" \ "$write" "/mnt/huge/test" "$method" "$private" "$expect_failure" \ @@ -344,7 +344,7 @@ function run_multiple_cgroup_test() { setup_cgroup "hugetlb_cgroup_test2" "$cgroup_limit2" "$reservation_limit2" mkdir -p /mnt/huge - mount -t hugetlbfs -o pagesize=${MB}M,size=256M none /mnt/huge + mount -t hugetlbfs -o pagesize=${MB}M none /mnt/huge write_hugetlbfs_and_get_usage "hugetlb_cgroup_test1" "$size1" \ "$populate1" "$write1" "/mnt/huge/test1" "$method" "$private" \ diff --git a/tools/testing/selftests/mm/cow.c b/tools/testing/selftests/mm/cow.c index accfd198dbda84..83b3563be26b65 100644 --- a/tools/testing/selftests/mm/cow.c +++ b/tools/testing/selftests/mm/cow.c @@ -1612,8 +1612,8 @@ static void run_with_huge_zeropage(non_anon_test_fn fn, const char *desc) * the first sub-page and test if we get another sub-page populated * automatically. */ - FORCE_READ(mem); - FORCE_READ(smem); + FORCE_READ(*mem); + FORCE_READ(*smem); if (!pagemap_is_populated(pagemap_fd, mem + pagesize) || !pagemap_is_populated(pagemap_fd, smem + pagesize)) { ksft_test_result_skip("Did not get THPs populated\n"); @@ -1663,8 +1663,8 @@ static void run_with_memfd(non_anon_test_fn fn, const char *desc) } /* Fault the page in. */ - FORCE_READ(mem); - FORCE_READ(smem); + FORCE_READ(*mem); + FORCE_READ(*smem); fn(mem, smem, pagesize); munmap: @@ -1719,8 +1719,8 @@ static void run_with_tmpfile(non_anon_test_fn fn, const char *desc) } /* Fault the page in. */ - FORCE_READ(mem); - FORCE_READ(smem); + FORCE_READ(*mem); + FORCE_READ(*smem); fn(mem, smem, pagesize); munmap: @@ -1773,8 +1773,8 @@ static void run_with_memfd_hugetlb(non_anon_test_fn fn, const char *desc, } /* Fault the page in. */ - FORCE_READ(mem); - FORCE_READ(smem); + FORCE_READ(*mem); + FORCE_READ(*smem); fn(mem, smem, hugetlbsize); munmap: diff --git a/tools/testing/selftests/mm/hmm-tests.c b/tools/testing/selftests/mm/hmm-tests.c index e8328c89d855ef..788689497e92a4 100644 --- a/tools/testing/selftests/mm/hmm-tests.c +++ b/tools/testing/selftests/mm/hmm-tests.c @@ -34,6 +34,7 @@ */ #include #include +#include struct hmm_buffer { void *ptr; @@ -548,7 +549,7 @@ TEST_F(hmm, anon_write_child) for (migrate = 0; migrate < 2; ++migrate) { for (use_thp = 0; use_thp < 2; ++use_thp) { - npages = ALIGN(use_thp ? TWOMEG : HMM_BUFFER_SIZE, + npages = ALIGN(use_thp ? read_pmd_pagesize() : HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; ASSERT_NE(npages, 0); size = npages << self->page_shift; @@ -728,7 +729,7 @@ TEST_F(hmm, anon_write_huge) int *ptr; int ret; - size = 2 * TWOMEG; + size = 2 * read_pmd_pagesize(); buffer = malloc(sizeof(*buffer)); ASSERT_NE(buffer, NULL); @@ -744,7 +745,7 @@ TEST_F(hmm, anon_write_huge) buffer->fd, 0); ASSERT_NE(buffer->ptr, MAP_FAILED); - size = TWOMEG; + size /= 2; npages = size >> self->page_shift; map = (void *)ALIGN((uintptr_t)buffer->ptr, size); ret = madvise(map, size, MADV_HUGEPAGE); @@ -770,54 +771,6 @@ TEST_F(hmm, anon_write_huge) hmm_buffer_free(buffer); } -/* - * Read numeric data from raw and tagged kernel status files. Used to read - * /proc and /sys data (without a tag) and from /proc/meminfo (with a tag). - */ -static long file_read_ulong(char *file, const char *tag) -{ - int fd; - char buf[2048]; - int len; - char *p, *q; - long val; - - fd = open(file, O_RDONLY); - if (fd < 0) { - /* Error opening the file */ - return -1; - } - - len = read(fd, buf, sizeof(buf)); - close(fd); - if (len < 0) { - /* Error in reading the file */ - return -1; - } - if (len == sizeof(buf)) { - /* Error file is too large */ - return -1; - } - buf[len] = '\0'; - - /* Search for a tag if provided */ - if (tag) { - p = strstr(buf, tag); - if (!p) - return -1; /* looks like the line we want isn't there */ - p += strlen(tag); - } else - p = buf; - - val = strtol(p, &q, 0); - if (*q != ' ') { - /* Error parsing the file */ - return -1; - } - - return val; -} - /* * Write huge TLBFS page. */ @@ -826,15 +779,13 @@ TEST_F(hmm, anon_write_hugetlbfs) struct hmm_buffer *buffer; unsigned long npages; unsigned long size; - unsigned long default_hsize; + unsigned long default_hsize = default_huge_page_size(); unsigned long i; int *ptr; int ret; - default_hsize = file_read_ulong("/proc/meminfo", "Hugepagesize:"); - if (default_hsize < 0 || default_hsize*1024 < default_hsize) + if (!default_hsize) SKIP(return, "Huge page size could not be determined"); - default_hsize = default_hsize*1024; /* KB to B */ size = ALIGN(TWOMEG, default_hsize); npages = size >> self->page_shift; @@ -1606,7 +1557,7 @@ TEST_F(hmm, compound) struct hmm_buffer *buffer; unsigned long npages; unsigned long size; - unsigned long default_hsize; + unsigned long default_hsize = default_huge_page_size(); int *ptr; unsigned char *m; int ret; @@ -1614,10 +1565,8 @@ TEST_F(hmm, compound) /* Skip test if we can't allocate a hugetlbfs page. */ - default_hsize = file_read_ulong("/proc/meminfo", "Hugepagesize:"); - if (default_hsize < 0 || default_hsize*1024 < default_hsize) + if (!default_hsize) SKIP(return, "Huge page size could not be determined"); - default_hsize = default_hsize*1024; /* KB to B */ size = ALIGN(TWOMEG, default_hsize); npages = size >> self->page_shift; @@ -2106,7 +2055,7 @@ TEST_F(hmm, migrate_anon_huge_empty) int *ptr; int ret; - size = TWOMEG; + size = read_pmd_pagesize(); buffer = malloc(sizeof(*buffer)); ASSERT_NE(buffer, NULL); @@ -2158,7 +2107,7 @@ TEST_F(hmm, migrate_anon_huge_zero) int ret; int val; - size = TWOMEG; + size = read_pmd_pagesize(); buffer = malloc(sizeof(*buffer)); ASSERT_NE(buffer, NULL); @@ -2221,7 +2170,7 @@ TEST_F(hmm, migrate_anon_huge_free) int *ptr; int ret; - size = TWOMEG; + size = read_pmd_pagesize(); buffer = malloc(sizeof(*buffer)); ASSERT_NE(buffer, NULL); @@ -2280,7 +2229,7 @@ TEST_F(hmm, migrate_anon_huge_fault) int *ptr; int ret; - size = TWOMEG; + size = read_pmd_pagesize(); buffer = malloc(sizeof(*buffer)); ASSERT_NE(buffer, NULL); @@ -2332,7 +2281,7 @@ TEST_F(hmm, migrate_partial_unmap_fault) { struct hmm_buffer *buffer; unsigned long npages; - unsigned long size = TWOMEG; + unsigned long size = read_pmd_pagesize(); unsigned long i; void *old_ptr; void *map; @@ -2398,7 +2347,7 @@ TEST_F(hmm, migrate_remap_fault) { struct hmm_buffer *buffer; unsigned long npages; - unsigned long size = TWOMEG; + unsigned long size = read_pmd_pagesize(); unsigned long i; void *old_ptr, *new_ptr = NULL; void *map; @@ -2498,7 +2447,7 @@ TEST_F(hmm, migrate_anon_huge_err) int *ptr; int ret; - size = TWOMEG; + size = read_pmd_pagesize(); buffer = malloc(sizeof(*buffer)); ASSERT_NE(buffer, NULL); @@ -2593,7 +2542,7 @@ TEST_F(hmm, migrate_anon_huge_zero_err) int *ptr; int ret; - size = TWOMEG; + size = read_pmd_pagesize(); buffer = malloc(sizeof(*buffer)); ASSERT_NE(buffer, NULL); diff --git a/tools/testing/selftests/mount_setattr/mount_setattr_test.c b/tools/testing/selftests/mount_setattr/mount_setattr_test.c index 7aec3ae82a4466..c6dafb3cc11639 100644 --- a/tools/testing/selftests/mount_setattr/mount_setattr_test.c +++ b/tools/testing/selftests/mount_setattr/mount_setattr_test.c @@ -1020,7 +1020,7 @@ FIXTURE_SETUP(mount_setattr_idmapped) "size=100000,mode=700"), 0); ASSERT_EQ(mount("testing", "/mnt", "tmpfs", MS_NOATIME | MS_NODEV, - "size=2m,mode=700"), 0); + "size=256m,mode=700"), 0); ASSERT_EQ(mkdir("/mnt/A", 0777), 0); diff --git a/tools/testing/selftests/net/forwarding/bridge_vlan_mcast.sh b/tools/testing/selftests/net/forwarding/bridge_vlan_mcast.sh index 72dfbeaf56b923..e8031f68200adf 100755 --- a/tools/testing/selftests/net/forwarding/bridge_vlan_mcast.sh +++ b/tools/testing/selftests/net/forwarding/bridge_vlan_mcast.sh @@ -414,6 +414,7 @@ vlmc_querier_intvl_test() bridge vlan add vid 10 dev br1 self pvid untagged ip link set dev $h1 master br1 ip link set dev br1 up + setup_wait_dev $h1 0 bridge vlan add vid 10 dev $h1 master bridge vlan global set vid 10 dev br1 mcast_snooping 1 mcast_querier 1 sleep 2 diff --git a/tools/testing/selftests/net/forwarding/pedit_dsfield.sh b/tools/testing/selftests/net/forwarding/pedit_dsfield.sh index af008fbf2725eb..eb2d8034de9c74 100755 --- a/tools/testing/selftests/net/forwarding/pedit_dsfield.sh +++ b/tools/testing/selftests/net/forwarding/pedit_dsfield.sh @@ -98,12 +98,20 @@ setup_prepare() h1_create h2_create switch_create + + if [ -f /proc/sys/net/bridge/bridge-nf-call-iptables ]; then + sysctl_set net.bridge.bridge-nf-call-iptables 0 + fi } cleanup() { pre_cleanup + if [ -f /proc/sys/net/bridge/bridge-nf-call-iptables ]; then + sysctl_restore net.bridge.bridge-nf-call-iptables + fi + switch_destroy h2_destroy h1_destroy diff --git a/tools/testing/selftests/net/forwarding/pedit_ip.sh b/tools/testing/selftests/net/forwarding/pedit_ip.sh index d14efb2d23b2e4..9235674627abd5 100755 --- a/tools/testing/selftests/net/forwarding/pedit_ip.sh +++ b/tools/testing/selftests/net/forwarding/pedit_ip.sh @@ -91,12 +91,20 @@ setup_prepare() h1_create h2_create switch_create + + if [ -f /proc/sys/net/bridge/bridge-nf-call-iptables ]; then + sysctl_set net.bridge.bridge-nf-call-iptables 0 + fi } cleanup() { pre_cleanup + if [ -f /proc/sys/net/bridge/bridge-nf-call-iptables ]; then + sysctl_restore net.bridge.bridge-nf-call-iptables + fi + switch_destroy h2_destroy h1_destroy diff --git a/tools/testing/selftests/net/forwarding/tc_actions.sh b/tools/testing/selftests/net/forwarding/tc_actions.sh index ea89e558672db0..86edbc7e2489b3 100755 --- a/tools/testing/selftests/net/forwarding/tc_actions.sh +++ b/tools/testing/selftests/net/forwarding/tc_actions.sh @@ -223,7 +223,7 @@ mirred_egress_to_ingress_tcp_test() ip_proto icmp \ action drop - ip vrf exec v$h1 ncat --recv-only -w10 -l -p 12345 -o $mirred_e2i_tf2 & + ip vrf exec v$h1 ncat --recv-only -w10 -l -p 12345 > $mirred_e2i_tf2 & local rpid=$! ip vrf exec v$h1 ncat -w1 --send-only 192.0.2.2 12345 <$mirred_e2i_tf1 wait -n $rpid diff --git a/tools/testing/selftests/net/forwarding/vxlan_bridge_1d.sh b/tools/testing/selftests/net/forwarding/vxlan_bridge_1d.sh index b43816dd998cab..457f41d5e584b5 100755 --- a/tools/testing/selftests/net/forwarding/vxlan_bridge_1d.sh +++ b/tools/testing/selftests/net/forwarding/vxlan_bridge_1d.sh @@ -567,6 +567,21 @@ vxlan_encapped_ping_do() local inner_tos=$1; shift local outer_tos=$1; shift + local ipv4hdr=$(: + )"45:"$( : IP version + IHL + )"$inner_tos:"$( : IP TOS + )"00:54:"$( : IP total length + )"99:83:"$( : IP identification + )"40:00:"$( : IP flags + frag off + )"40:"$( : IP TTL + )"01:"$( : IP proto + )"CHECKSUM:"$( : IP header csum + )"c0:00:02:03:"$( : IP saddr: 192.0.2.3 + )"c0:00:02:01"$( : IP daddr: 192.0.2.1 + ) + local checksum=$(payload_template_calc_checksum "$ipv4hdr") + ipv4hdr=$(payload_template_expand_checksum "$ipv4hdr" $checksum) + $MZ $dev -c $count -d 100msec -q \ -b $next_hop_mac -B $dest_ip \ -t udp tos=$outer_tos,sp=23456,dp=$VXPORT,p=$(: @@ -577,16 +592,7 @@ vxlan_encapped_ping_do() )"$dest_mac:"$( : ETH daddr )"$(mac_get w2):"$( : ETH saddr )"08:00:"$( : ETH type - )"45:"$( : IP version + IHL - )"$inner_tos:"$( : IP TOS - )"00:54:"$( : IP total length - )"99:83:"$( : IP identification - )"40:00:"$( : IP flags + frag off - )"40:"$( : IP TTL - )"01:"$( : IP proto - )"00:00:"$( : IP header csum - )"c0:00:02:03:"$( : IP saddr: 192.0.2.3 - )"c0:00:02:01:"$( : IP daddr: 192.0.2.1 + )"$ipv4hdr:"$( : IPv4 header )"08:"$( : ICMP type )"00:"$( : ICMP code )"8b:f2:"$( : ICMP csum diff --git a/tools/testing/selftests/net/forwarding/vxlan_bridge_1d_ipv6.sh b/tools/testing/selftests/net/forwarding/vxlan_bridge_1d_ipv6.sh index a603f7b0a08f07..e642feeada0e7e 100755 --- a/tools/testing/selftests/net/forwarding/vxlan_bridge_1d_ipv6.sh +++ b/tools/testing/selftests/net/forwarding/vxlan_bridge_1d_ipv6.sh @@ -695,7 +695,7 @@ vxlan_encapped_ping_do() )"6"$( : IP version )"$inner_tos"$( : Traffic class )"0:00:00:"$( : Flow label - )"00:08:"$( : Payload length + )"00:03:"$( : Payload length )"3a:"$( : Next header )"04:"$( : Hop limit )"$saddr:"$( : IP saddr diff --git a/tools/testing/selftests/net/lib.sh b/tools/testing/selftests/net/lib.sh index 0ec131b339bc42..b40694573f4c7d 100644 --- a/tools/testing/selftests/net/lib.sh +++ b/tools/testing/selftests/net/lib.sh @@ -577,7 +577,7 @@ ip_link_has_flag() local flag=$1; shift local state=$(ip -j link show "$name" | - jq --arg flag "$flag" 'any(.[].flags.[]; . == $flag)') + jq --arg flag "$flag" 'any(.[].flags[]; . == $flag)') [[ $state == true ]] } diff --git a/tools/testing/selftests/net/mptcp/mptcp_join.sh b/tools/testing/selftests/net/mptcp/mptcp_join.sh index e70d3420954fc1..c739e0185f7fd9 100755 --- a/tools/testing/selftests/net/mptcp/mptcp_join.sh +++ b/tools/testing/selftests/net/mptcp/mptcp_join.sh @@ -104,6 +104,24 @@ CBPF_MPTCP_SUBOPTION_ADD_ADDR="14, 6 0 0 65535, 6 0 0 0" +# IPv4: TCP hdr of 48B, a first suboption of 12B (DACK8), the RM_ADDR suboption +# generated using "nfbpf_compile '(ip[32] & 0xf0) == 0xc0 && ip[53] == 0x0c && +# (ip[66] & 0xf0) == 0x40'" +CBPF_MPTCP_SUBOPTION_RM_ADDR="13, + 48 0 0 0, + 84 0 0 240, + 21 0 9 64, + 48 0 0 32, + 84 0 0 240, + 21 0 6 192, + 48 0 0 53, + 21 0 4 12, + 48 0 0 66, + 84 0 0 240, + 21 0 1 64, + 6 0 0 65535, + 6 0 0 0" + init_partial() { capout=$(mktemp) @@ -2619,6 +2637,19 @@ remove_tests() chk_rst_nr 0 0 fi + # signal+subflow with limits, remove + if reset "remove signal+subflow with limits"; then + pm_nl_set_limits $ns1 0 0 + pm_nl_add_endpoint $ns1 10.0.2.1 flags signal,subflow + pm_nl_set_limits $ns2 0 0 + addr_nr_ns1=-1 speed=slow \ + run_tests $ns1 $ns2 10.0.1.1 + chk_join_nr 0 0 0 + chk_add_nr 1 1 + chk_rm_nr 1 0 invert + chk_rst_nr 0 0 + fi + # addresses remove if reset "remove addresses"; then pm_nl_set_limits $ns1 3 3 @@ -4222,6 +4253,14 @@ endpoint_tests() chk_subflow_nr "after no reject" 3 chk_mptcp_info subflows 2 subflows 2 + # To make sure RM_ADDR are sent over a different subflow, but + # allow the rest to quickly and cleanly close the subflow + local ipt=1 + ip netns exec "${ns2}" ${iptables} -I OUTPUT -s "10.0.1.2" \ + -p tcp -m tcp --tcp-option 30 \ + -m bpf --bytecode \ + "$CBPF_MPTCP_SUBOPTION_RM_ADDR" \ + -j DROP || ipt=0 local i for i in $(seq 3); do pm_nl_del_endpoint $ns2 1 10.0.1.2 @@ -4234,6 +4273,7 @@ endpoint_tests() chk_subflow_nr "after re-add id 0 ($i)" 3 chk_mptcp_info subflows 3 subflows 3 done + [ ${ipt} = 1 ] && ip netns exec "${ns2}" ${iptables} -D OUTPUT 1 mptcp_lib_kill_group_wait $tests_pid @@ -4293,11 +4333,20 @@ endpoint_tests() chk_mptcp_info subflows 2 subflows 2 chk_mptcp_info add_addr_signal 2 add_addr_accepted 2 + # To make sure RM_ADDR are sent over a different subflow, but + # allow the rest to quickly and cleanly close the subflow + local ipt=1 + ip netns exec "${ns1}" ${iptables} -I OUTPUT -s "10.0.1.1" \ + -p tcp -m tcp --tcp-option 30 \ + -m bpf --bytecode \ + "$CBPF_MPTCP_SUBOPTION_RM_ADDR" \ + -j DROP || ipt=0 pm_nl_del_endpoint $ns1 42 10.0.1.1 sleep 0.5 chk_subflow_nr "after delete ID 0" 2 chk_mptcp_info subflows 2 subflows 2 chk_mptcp_info add_addr_signal 2 add_addr_accepted 2 + [ ${ipt} = 1 ] && ip netns exec "${ns1}" ${iptables} -D OUTPUT 1 pm_nl_add_endpoint $ns1 10.0.1.1 id 99 flags signal wait_mpj $ns2 diff --git a/tools/testing/selftests/net/mptcp/simult_flows.sh b/tools/testing/selftests/net/mptcp/simult_flows.sh index 806aaa7d2d61dc..d11a8b949aab5c 100755 --- a/tools/testing/selftests/net/mptcp/simult_flows.sh +++ b/tools/testing/selftests/net/mptcp/simult_flows.sh @@ -237,10 +237,13 @@ run_test() for dev in ns2eth1 ns2eth2; do tc -n $ns2 qdisc del dev $dev root >/dev/null 2>&1 done - tc -n $ns1 qdisc add dev ns1eth1 root netem rate ${rate1}mbit $delay1 - tc -n $ns1 qdisc add dev ns1eth2 root netem rate ${rate2}mbit $delay2 - tc -n $ns2 qdisc add dev ns2eth1 root netem rate ${rate1}mbit $delay1 - tc -n $ns2 qdisc add dev ns2eth2 root netem rate ${rate2}mbit $delay2 + + # keep the queued pkts number low, or the RTT estimator will see + # increasing latency over time. + tc -n $ns1 qdisc add dev ns1eth1 root netem rate ${rate1}mbit $delay1 limit 50 + tc -n $ns1 qdisc add dev ns1eth2 root netem rate ${rate2}mbit $delay2 limit 50 + tc -n $ns2 qdisc add dev ns2eth1 root netem rate ${rate1}mbit $delay1 limit 50 + tc -n $ns2 qdisc add dev ns2eth2 root netem rate ${rate2}mbit $delay2 limit 50 # time is measured in ms, account for transfer size, aggregated link speed # and header overhead (10%) diff --git a/tools/testing/selftests/resctrl/resctrlfs.c b/tools/testing/selftests/resctrl/resctrlfs.c index 195f04c4d15868..b9c1bfb6cc029a 100644 --- a/tools/testing/selftests/resctrl/resctrlfs.c +++ b/tools/testing/selftests/resctrl/resctrlfs.c @@ -243,6 +243,16 @@ int snc_nodes_per_l3_cache(void) } snc_mode = cache_cpus / node_cpus; + /* + * On some platforms (e.g. Hygon), + * cache_cpus < node_cpus, the calculated snc_mode is 0. + * + * Set snc_mode = 1 to indicate that SNC mode is not + * supported on the platform. + */ + if (!snc_mode) + snc_mode = 1; + if (snc_mode > 1) ksft_print_msg("SNC-%d mode discovered.\n", snc_mode); } diff --git a/tools/testing/selftests/ublk/kublk.h b/tools/testing/selftests/ublk/kublk.h index 8a83b90ec603ad..cae2e30f0cdd57 100644 --- a/tools/testing/selftests/ublk/kublk.h +++ b/tools/testing/selftests/ublk/kublk.h @@ -223,7 +223,7 @@ static inline __u64 build_user_data(unsigned tag, unsigned op, unsigned tgt_data, unsigned q_id, unsigned is_target_io) { /* we only have 7 bits to encode q_id */ - _Static_assert(UBLK_MAX_QUEUES_SHIFT <= 7); + _Static_assert(UBLK_MAX_QUEUES_SHIFT <= 7, "UBLK_MAX_QUEUES_SHIFT must be <= 7"); assert(!(tag >> 16) && !(op >> 8) && !(tgt_data >> 16) && !(q_id >> 7)); return tag | (op << 16) | (tgt_data << 24) | diff --git a/tools/testing/vsock/util.c b/tools/testing/vsock/util.c index 9430ef5b8bc3ea..1fe1338c79cd15 100644 --- a/tools/testing/vsock/util.c +++ b/tools/testing/vsock/util.c @@ -344,7 +344,9 @@ void send_buf(int fd, const void *buf, size_t len, int flags, ret = send(fd, buf + nwritten, len - nwritten, flags); timeout_check("send"); - if (ret == 0 || (ret < 0 && errno != EINTR)) + if (ret < 0 && errno == EINTR) + continue; + if (ret <= 0) break; nwritten += ret; @@ -396,7 +398,9 @@ void recv_buf(int fd, void *buf, size_t len, int flags, ssize_t expected_ret) ret = recv(fd, buf + nread, len - nread, flags); timeout_check("recv"); - if (ret == 0 || (ret < 0 && errno != EINTR)) + if (ret < 0 && errno == EINTR) + continue; + if (ret <= 0) break; nread += ret; diff --git a/tools/tracing/rtla/src/actions.c b/tools/tracing/rtla/src/actions.c index 8945aee58d511f..15986505b43760 100644 --- a/tools/tracing/rtla/src/actions.c +++ b/tools/tracing/rtla/src/actions.c @@ -141,6 +141,8 @@ actions_parse(struct actions *self, const char *trigger, const char *tracefn) strcpy(trigger_c, trigger); token = strtok(trigger_c, ","); + if (!token) + return -1; if (strcmp(token, "trace") == 0) type = ACTION_TRACE_OUTPUT; diff --git a/virt/kvm/binary_stats.c b/virt/kvm/binary_stats.c index eefca6c69f519e..76ce697c773bf5 100644 --- a/virt/kvm/binary_stats.c +++ b/virt/kvm/binary_stats.c @@ -50,7 +50,7 @@ * Return: the number of bytes that has been successfully read */ ssize_t kvm_stats_read(char *id, const struct kvm_stats_header *header, - const struct _kvm_stats_desc *desc, + const struct kvm_stats_desc *desc, void *stats, size_t size_stats, char __user *user_buffer, size_t size, loff_t *offset) { diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index 5b5b69c97665ec..cf65fd82d36dd7 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -983,9 +983,9 @@ static void kvm_free_memslots(struct kvm *kvm, struct kvm_memslots *slots) kvm_free_memslot(kvm, memslot); } -static umode_t kvm_stats_debugfs_mode(const struct _kvm_stats_desc *pdesc) +static umode_t kvm_stats_debugfs_mode(const struct kvm_stats_desc *desc) { - switch (pdesc->desc.flags & KVM_STATS_TYPE_MASK) { + switch (desc->flags & KVM_STATS_TYPE_MASK) { case KVM_STATS_TYPE_INSTANT: return 0444; case KVM_STATS_TYPE_CUMULATIVE: @@ -1020,7 +1020,7 @@ static int kvm_create_vm_debugfs(struct kvm *kvm, const char *fdname) struct dentry *dent; char dir_name[ITOA_MAX_LEN * 2]; struct kvm_stat_data *stat_data; - const struct _kvm_stats_desc *pdesc; + const struct kvm_stats_desc *pdesc; int i, ret = -ENOMEM; int kvm_debugfs_num_entries = kvm_vm_stats_header.num_desc + kvm_vcpu_stats_header.num_desc; @@ -6186,11 +6186,11 @@ static int kvm_stat_data_get(void *data, u64 *val) switch (stat_data->kind) { case KVM_STAT_VM: r = kvm_get_stat_per_vm(stat_data->kvm, - stat_data->desc->desc.offset, val); + stat_data->desc->offset, val); break; case KVM_STAT_VCPU: r = kvm_get_stat_per_vcpu(stat_data->kvm, - stat_data->desc->desc.offset, val); + stat_data->desc->offset, val); break; } @@ -6208,11 +6208,11 @@ static int kvm_stat_data_clear(void *data, u64 val) switch (stat_data->kind) { case KVM_STAT_VM: r = kvm_clear_stat_per_vm(stat_data->kvm, - stat_data->desc->desc.offset); + stat_data->desc->offset); break; case KVM_STAT_VCPU: r = kvm_clear_stat_per_vcpu(stat_data->kvm, - stat_data->desc->desc.offset); + stat_data->desc->offset); break; } @@ -6360,7 +6360,7 @@ static void kvm_uevent_notify_change(unsigned int type, struct kvm *kvm) static void kvm_init_debug(void) { const struct file_operations *fops; - const struct _kvm_stats_desc *pdesc; + const struct kvm_stats_desc *pdesc; int i; kvm_debugfs_dir = debugfs_create_dir("kvm", NULL); @@ -6373,7 +6373,7 @@ static void kvm_init_debug(void) fops = &vm_stat_readonly_fops; debugfs_create_file(pdesc->name, kvm_stats_debugfs_mode(pdesc), kvm_debugfs_dir, - (void *)(long)pdesc->desc.offset, fops); + (void *)(long)pdesc->offset, fops); } for (i = 0; i < kvm_vcpu_stats_header.num_desc; ++i) { @@ -6384,7 +6384,7 @@ static void kvm_init_debug(void) fops = &vcpu_stat_readonly_fops; debugfs_create_file(pdesc->name, kvm_stats_debugfs_mode(pdesc), kvm_debugfs_dir, - (void *)(long)pdesc->desc.offset, fops); + (void *)(long)pdesc->offset, fops); } }