Accessibility Inspector: Allow navigation in all possible directions

ios/accessibility/accessibility_control.go

@@ -1,6 +1,8 @@
 package accessibility
 
 import (
+	"context"
+	"encoding/base64"
 	"fmt"
 
 	dtx "github.com/danielpaulus/go-ios/ios/dtx_codec"
@@ -14,6 +16,21 @@ type ControlInterface struct {
 	channel *dtx.Channel
 }
 
+// Direction represents navigation direction values used by AX service
+type MoveDirection int32
+
+const (
+	DirectionPrevious MoveDirection = 3
+	DirectionNext     MoveDirection = 4
+	DirectionFirst    MoveDirection = 5
+	DirectionLast     MoveDirection = 6
+)
+
+// AXElementData represents the data returned from Move operations
+type AXElementData struct {
+	PlatformElementValue string `json:"platformElementValue"` // Base64-encoded platform element data
+}
+
 func (a ControlInterface) readhostAppStateChanged() {
 	for {
 		msg := a.channel.ReceiveMethodCall("hostAppStateChanged:")
@@ -115,7 +132,10 @@ func (a ControlInterface) SwitchToDevice() {
 	a.deviceInspectorShowIgnoredElements(false)
 	a.deviceSetAuditTargetPid(0)
 	a.deviceInspectorFocusOnElement()
-	a.awaitHostInspectorCurrentElementChanged()
+	_, err := a.awaitHostInspectorCurrentElementChanged(context.Background())
+	if err != nil {
+		log.Warnf("await element change failed during SwitchToDevice: %v", err)
+	}
 	a.deviceInspectorPreviewOnElement()
 	a.deviceHighlightIssue()
 }
@@ -125,20 +145,72 @@ func (a ControlInterface) TurnOff() {
 	a.deviceInspectorSetMonitoredEventType(0)
 	a.awaitHostInspectorMonitoredEventTypeChanged()
 	a.deviceInspectorFocusOnElement()
-	a.awaitHostInspectorCurrentElementChanged()
+	_, err := a.awaitHostInspectorCurrentElementChanged(context.Background())
+	if err != nil {
+		log.Warnf("await element change failed during TurnOff: %v", err)
+	}
 	a.deviceInspectorPreviewOnElement()
 	a.deviceHighlightIssue()
 	a.deviceInspectorShowVisuals(false)
 }
 
-// GetElement moves the green selection rectangle one element further
-func (a ControlInterface) GetElement() {
+// Move navigates focus using the given direction and returns selected element data.
+func (a ControlInterface) Move(ctx context.Context, direction MoveDirection) (AXElementData, error) {
 	log.Info("changing")
-	a.deviceInspectorMoveWithOptions()
-	// a.deviceInspectorMoveWithOptions()
+	a.deviceInspectorMoveWithOptions(direction)
+	log.Info("before changed")
 
-	resp := a.awaitHostInspectorCurrentElementChanged()
-	log.Info("item changed", resp)
+	resp, err := a.awaitHostInspectorCurrentElementChanged(ctx)
+	if err != nil {
+		return AXElementData{}, err
+	}
+
+	// Extraction path for platform element bytes:
+	// Value -> Value -> ElementValue_v1 -> Value -> Value -> PlatformElementValue_v1 -> Value ([]byte)
+	value, ok := resp["Value"].(map[string]interface{})
+	if !ok {
+		return AXElementData{}, fmt.Errorf("resp[\"Value\"] is not a map, got %T", resp["Value"])
+	}
+
+	innerValue, ok := value["Value"].(map[string]interface{})
+	if !ok {
+		return AXElementData{}, fmt.Errorf("Value[\"Value\"] is not a map, got %T", value["Value"])
+	}
+
+	elementValue, ok := innerValue["ElementValue_v1"].(map[string]interface{})
+	if !ok {
+		return AXElementData{}, fmt.Errorf("ElementValue_v1 is not a map, got %T", innerValue["ElementValue_v1"])
+	}
+
+	axElement, ok := elementValue["Value"].(map[string]interface{})
+	if !ok {
+		return AXElementData{}, fmt.Errorf("ElementValue_v1[\"Value\"] is not a map, got %T", elementValue["Value"])
+	}
+
+	valMap, ok := axElement["Value"].(map[string]interface{})
+	if !ok {
+		return AXElementData{}, fmt.Errorf("AX element inner \"Value\" is not a map, got %T", axElement["Value"])
+	}
+
+	platformElement, ok := valMap["PlatformElementValue_v1"].(map[string]interface{})
+	if !ok {
+		return AXElementData{}, fmt.Errorf("PlatformElementValue_v1 is not a map, got %T", valMap["PlatformElementValue_v1"])
+	}
+
+	byteArray, ok := platformElement["Value"].([]byte)
+	if !ok {
+		return AXElementData{}, fmt.Errorf("PlatformElementValue_v1[\"Value\"] is not a []byte, got %T", platformElement["Value"])
+	}
+	encoded := base64.StdEncoding.EncodeToString(byteArray)
+
+	return AXElementData{
+		PlatformElementValue: encoded,
+	}, nil
+}
+
+// GetElement moves the green selection rectangle one element further
+func (a ControlInterface) GetElement(ctx context.Context) (AXElementData, error) {
+	return a.Move(ctx, DirectionNext)
 }
 
 func (a ControlInterface) UpdateAccessibilitySetting(name string, val interface{}) {
@@ -159,14 +231,18 @@ func (a ControlInterface) ResetToDefaultAccessibilitySettings() error {
 	return nil
 }
 
-func (a ControlInterface) awaitHostInspectorCurrentElementChanged() map[string]interface{} {
-	msg := a.channel.ReceiveMethodCall("hostInspectorCurrentElementChanged:")
+func (a ControlInterface) awaitHostInspectorCurrentElementChanged(ctx context.Context) (map[string]interface{}, error) {
+	msg, err := a.channel.ReceiveMethodCallWithTimeout(ctx, "hostInspectorCurrentElementChanged:")
+	if err != nil {
+		log.Errorf("Failed to receive hostInspectorCurrentElementChanged: %v", err)
+		return nil, fmt.Errorf("failed to receive hostInspectorCurrentElementChanged: %w", err)
+	}
 	log.Info("received hostInspectorCurrentElementChanged")
 	result, err := nskeyedarchiver.Unarchive(msg.Auxiliary.GetArguments()[0].([]byte))
 	if err != nil {
 		panic(fmt.Sprintf("Failed unarchiving: %s this is a bug and should not happen", err))
 	}
-	return result[0].(map[string]interface{})
+	return result[0].(map[string]interface{}), nil
 }
 
 func (a ControlInterface) awaitHostInspectorMonitoredEventTypeChanged() {
@@ -175,13 +251,13 @@ func (a ControlInterface) awaitHostInspectorMonitoredEventTypeChanged() {
 	log.Infof("hostInspectorMonitoredEventTypeChanged: was set to %d by the device", n[0])
 }
 
-func (a ControlInterface) deviceInspectorMoveWithOptions() {
+func (a ControlInterface) deviceInspectorMoveWithOptions(direction MoveDirection) {
 	method := "deviceInspectorMoveWithOptions:"
 	options := nskeyedarchiver.NewNSMutableDictionary(map[string]interface{}{
 		"ObjectType": "passthrough",
 		"Value": nskeyedarchiver.NewNSMutableDictionary(map[string]interface{}{
 			"allowNonAX":        nskeyedarchiver.NewNSMutableDictionary(map[string]interface{}{"ObjectType": "passthrough", "Value": false}),
-			"direction":         nskeyedarchiver.NewNSMutableDictionary(map[string]interface{}{"ObjectType": "passthrough", "Value": int32(4)}),
+			"direction":         nskeyedarchiver.NewNSMutableDictionary(map[string]interface{}{"ObjectType": "passthrough", "Value": int32(direction)}),
 			"includeContainers": nskeyedarchiver.NewNSMutableDictionary(map[string]interface{}{"ObjectType": "passthrough", "Value": true}),
 		}),
 	})

ios/accessibility/accessibility_integration_test.go

@@ -10,7 +10,7 @@ import (
 	"github.com/danielpaulus/go-ios/ios/accessibility"
 )
 
-func TestIT(t *testing.T) {
+func TestMove(t *testing.T) {
 	device, err := ios.GetDevice("")
 	if err != nil {
 		t.Fatal(err)
@@ -20,15 +20,31 @@ func TestIT(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
+	defer conn.TurnOff()
 
 	conn.SwitchToDevice()
 	if err != nil {
 		t.Fatal(err)
 	}
 	conn.EnableSelectionMode()
-	conn.GetElement()
-	conn.GetElement()
-	conn.TurnOff()
 
-	// conn.EnableSelectionMode()
+	t.Run("Test Move directions", func(t *testing.T) {
+		directions := []accessibility.MoveDirection{
+			// newer ios(18+) devices sometimes doesn't bring focus to first element, so we need to move twice
+			accessibility.DirectionNext,
+			accessibility.DirectionNext,
+			accessibility.DirectionPrevious,
+		}
+
+		for _, direction := range directions {
+			t.Logf("Testing direction: %v", direction)
+			element, err := conn.Move(direction)
+			if err != nil {
+				t.Logf("Move %v failed (expected on some devices): %v", direction, err)
+				continue
+			}
+
+			t.Logf("Move %v succeeded: %+v", direction, element)
+		}
+	})
 }

ios/dtx_codec/channel.go

@@ -1,6 +1,7 @@
 package dtx
 
 import (
+	"context"
 	"fmt"
 	"sync"
 	"time"
@@ -46,6 +47,19 @@ func (d *Channel) ReceiveMethodCall(selector string) Message {
 	return <-channel
 }
 
+func (d *Channel) ReceiveMethodCallWithTimeout(ctx context.Context, selector string) (Message, error) {
+	d.mutex.Lock()
+	channel := d.registeredMethods[selector]
+	d.mutex.Unlock()
+	select {
+	case msg := <-channel:
+		return msg, nil
+	// context is cancelled because the timeout is exceeded
+	case <-ctx.Done():
+		return Message{}, ctx.Err()
+	}
+}
+
 // MethodCall is the standard DTX style remote method invocation pattern. The ObjectiveC Selector goes as a NSKeyedArchiver.archived NSString into the
 // DTXMessage payload, and the arguments are separately NSKeyArchiver.archived and put into the Auxiliary DTXPrimitiveDictionary. It returns the response message and an error.
 func (d *Channel) MethodCall(selector string, args ...interface{}) (Message, error) {

main.go

@@ -1798,7 +1798,7 @@ func startAx(device ios.DeviceEntry, arguments docopt.Opts) {
 		}
 
 		for i := 0; i < 3; i++ {
-			conn.GetElement()
+			conn.GetElement(context.Background())
 			time.Sleep(time.Second)
 		}
 		/*	conn.GetElement()