echonet / echo_clip Goto Github PK

model, _, preprocess_val = create_model_and_transforms(
    "mkaichristensen/echo-clip-r", cache_dir="~/.cache/huggingface/hub",pretrained="...open_clip_pytorch_model.bin",precision="bf16", device="cuda"
)
# wrapping code for the model etc.
...
def test_step(self, batch, batch_idx):
        # this is the test loop
        x = batch["video"].to(torch.bfloat16)
        y = batch[self.gt_fieldname]
        bs,c,T,h,w = x.shape
        x = x.reshape(-1,c,h,w)
        x = F.normalize(self.model.encode_image(x),dim=-1)
        # now the c is not here - the embedding dimension is 512
        x = x.reshape(bs,T,-1)
        x_hat,*_ = compute_regression_metric(
                video_embeddings=x,
                prompt_embeddings=self.query_embeddings,
                prompt_values=self.prompt_values
            )
        
        # now the logic how to convert the similarity into prediction and than loss
        # check if the model is correct. what is the loss here?
        self.log("test_xhat",x_hat)

list_of_batchs,list_of_xy_pairs,list_of_minmaxs = zip(*results)
mat = torch.stack(list_of_xy_pairs)
offsets = torch.stack(list_of_minmaxs,dim=1)
summary_writer.add_pr_curve(labels=mat[:,0],predictions=mat[:,1],tag="foo")
summary_writer.add_embedding(
    mat=mat
)


# plot errors


# plt.style.use('_mpl-gallery')

# make data:
x = mat[:,0]
y = mat[:,1]
yerr = offsets.detach().cpu().numpy()

fig,ax1 = plt.subplots()
# plot:
plt.figure()
plt.errorbar(x, y, yerr, fmt='o',ecolor="cornflowerblue",capthick=1,lw=2)
# for EF the scale is 1 to 100 percents of efficiency
ax1.set(xlim=(0, 100), xticks=np.arange(0, 100,10),
       ylim=(0, 100), yticks=np.arange(0, 100,10))
ax1.set_xlabel("real EF")
ax1.set_title("EF plot")
plt.axline((0, 0), (100, 100), color='gray',linestyle="--",linewidth=0.9)
ax1.plot([0, 0], [100, 100], transform=ax1.transAxes)
plt.show()
plt.savefig("docs/assets/img/plot")