diff --git a/arch/arm/cpu/armv7/omap5/config.mk b/arch/arm/cpu/armv7/omap5/config.mk
index d245572ef04344113bc9a910e3586828d235e167..286ca8688d761b31d08303387f70d3c68d0c7a45 100644
--- a/arch/arm/cpu/armv7/omap5/config.mk
+++ b/arch/arm/cpu/armv7/omap5/config.mk
@@ -16,7 +16,7 @@ ALL-y	+= MLO
 endif
 else
 ifeq ($(CONFIG_TI_SECURE_DEVICE),y)
-ALL-y   += u-boot_HS.img
+ALL-$(CONFIG_SPL_LOAD_FIT) += u-boot_HS.img
 endif
 ALL-y	+= u-boot.img
 endif