diff --git a/arch/xtensa/include/asm/unaligned.h b/arch/xtensa/include/asm/unaligned.h
index 8f3424fc5d18..8e7ed046bfed 100644
--- a/arch/xtensa/include/asm/unaligned.h
+++ b/arch/xtensa/include/asm/unaligned.h
@@ -10,20 +10,20 @@
 #ifndef _ASM_XTENSA_UNALIGNED_H
 #define _ASM_XTENSA_UNALIGNED_H
 
-#ifdef __XTENSA_EL__
-# include <linux/unaligned/le_memmove.h>
+#include <asm/byteorder.h>
+
+#ifdef __LITTLE_ENDIAN
+# include <linux/unaligned/le_struct.h>
 # include <linux/unaligned/be_byteshift.h>
 # include <linux/unaligned/generic.h>
 # define get_unaligned	__get_unaligned_le
 # define put_unaligned	__put_unaligned_le
-#elif defined(__XTENSA_EB__)
-# include <linux/unaligned/be_memmove.h>
+#else
+# include <linux/unaligned/be_struct.h>
 # include <linux/unaligned/le_byteshift.h>
 # include <linux/unaligned/generic.h>
 # define get_unaligned	__get_unaligned_be
 # define put_unaligned	__put_unaligned_be
-#else
-# error processor byte order undefined!
 #endif
 
 #endif	/* _ASM_XTENSA_UNALIGNED_H */