NoMaD项目复现#

1
    # ...
2

3
    def navigate(self,
4
                 single_img = None,# priority
5
                 num_samples: int = 8,
6
                 waypoint_index: int = 2,
7
                 return_traj: bool = False) -> np.ndarray:
8
        """
9
        Main navigation method to generate navigation actions.
10

11
        Args:
12
            context_images (List[str]): List of image paths for context
13
            num_samples (int, optional): Number of action samples. Defaults to 8.
14
            waypoint_index (int, optional): Waypoint to use for navigation. Defaults to 2.
15
            return_traj (bool, optional): Whether to return full trajectory. Defaults to False.
16

17
        Returns:
18
            np.ndarray: Chosen navigation waypoint
19
        """
20
        # Check context size
21

22
        # Load and transform images
23

24
        if len(self.context_queue) < 4:
25
            self.context_queue = [single_img,single_img,single_img,single_img] # 4imgs !!
26
        else:
27
            self.context_queue = self.context_queue[1:] + [single_img]
28

29
        context_queue = self.context_queue
30

31
        obs_images = transform_images(context_queue, self.model_params["image_size"], center_crop=False)
32
        obs_images = obs_images.to(self.device)
33

34
        # Prepare goal and mask
35
        fake_goal = torch.randn((1, 3, *self.model_params["image_size"])).to(self.device)
36
        mask = torch.ones(1).long().to(self.device)
37

38
        # Inference
39
        with torch.no_grad():
40
            # Encode vision features
41
            obs_cond = self.model('vision_encoder', obs_img=obs_images, goal_img=fake_goal, input_goal_mask=mask)
42

43
            # Repeat condition for multiple samples
44
            obs_cond = obs_cond.repeat(num_samples, 1) if len(obs_cond.shape) == 2 else obs_cond.repeat(num_samples, 1, 1)
45

46
            # Initialize noisy action
47
            noisy_action = torch.randn(
48
                (num_samples, self.model_params["len_traj_pred"], 2), device=self.device)
49
            naction = noisy_action
50

51
            # Run noise scheduler
52
            self.noise_scheduler.set_timesteps(self.num_diffusion_iters)
53
            for k in self.noise_scheduler.timesteps[:]:
54
                noise_pred = self.model(
55
                    'noise_pred_net',
56
                    sample=naction,
57
                    timestep=k,
58
                    global_cond=obs_cond
59
                )
60
                naction = self.noise_scheduler.step(
61
                    model_output=noise_pred,
62
                    timestep=k,
63
                    sample=naction
64
                ).prev_sample
65

66
        # Process actions
67
        naction = to_numpy(get_action(naction))
68

69
        if return_traj:
70
            return naction  # (num_samples, T, 2)
71

72
        naction = naction[0]  # Select first sample
73
        chosen_waypoint = naction[waypoint_index]
74

75
        # Normalize if required
76
        if self.model_params.get("normalize", False):
77
            chosen_waypoint *= (self.MAX_V / self.RATE)
78

79
        return chosen_waypoint
80

81
    # ...

1
import sys
2
sys.path.append('../vint_release/deployment')
3

4
from deployment.src.api_explore_RGB import VisualNavigationAPI
5
import PIL.Image as PILImage
6
import numpy as np
7

8
def main():
9
    single_img_path = 'path/to/0.jpg'
10
    single_img = PILImage.open(single_img_path).convert("RGB")
11

12
    nav_api = VisualNavigationAPI(model_name="nomad")
13

14
    # 核心：返回整条轨迹
15
    naction = nav_api.navigate(
16
        single_img=single_img,
17
        num_samples=8,
18
        return_traj=True
19
    )
20

21
    print("naction shape:", naction.shape)  # (8, 8, 2)
22
    np.save("naction.npy", naction)
23
    print("saved naction.npy")
24

25
if __name__ == "__main__":
26
    main()

1
import numpy as np
2
import matplotlib.pyplot as plt
3

4
naction = np.load("naction.npy")
5

6
plt.figure(figsize=(6, 6))
7
for i in range(naction.shape[0]):
8
    x = naction[i, :, 0]
9
    y = naction[i, :, 1]
10
    plt.plot(x, y, marker='o', alpha=0.8)
11

12
plt.axis("equal")
13
plt.grid(True)
14
plt.title("Predicted Trajectories")
15
plt.show()