Cuda 12 sm86

ocelot/CMakeLists.txt

@@ -420,6 +420,10 @@ endif()
 
 add_library(${PROJECT_NAME}_executive STATIC ${${PROJECT_NAME}_executive_sources})
 
+if(CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang")
+set_source_files_properties(src/executive/CooperativeThreadArray.cpp PROPERTIES COMPILE_OPTIONS -frounding-math)
+endif()
+
 set_property(TARGET ${PROJECT_NAME}_executive PROPERTY CXX_STANDARD 14)
 set_property(TARGET ${PROJECT_NAME}_executive PROPERTY POSITION_INDEPENDENT_CODE ON)
 target_compile_definitions(${PROJECT_NAME}_executive PRIVATE ${${PROJECT_NAME}_DEFINITIONS})

ocelot/include/ocelot/executive/CooperativeThreadArray.h

@@ -456,9 +456,12 @@ namespace executive {
 		void eval_AddC(CTAContext &context, const ir::PTXInstruction &instr);
 		void eval_And(CTAContext &context, const ir::PTXInstruction &instr);
 		void eval_Atom(CTAContext &context, const ir::PTXInstruction &instr);
+		void evalAtomicRMW(CTAContext &context, const ir::PTXInstruction &instr,
+			ir::PTXInstruction::AtomicOperation operation, bool writeback);
 		void eval_Bar(CTAContext &context, const ir::PTXInstruction &instr);
 		void eval_Bfi(CTAContext &context, const ir::PTXInstruction &instr);
 		void eval_Bfind(CTAContext &context, const ir::PTXInstruction &instr);
+		void eval_Bmsk(CTAContext &context, const ir::PTXInstruction &instr);
 		void eval_Bfe(CTAContext &context, const ir::PTXInstruction &instr);
 		void eval_Bra(CTAContext &context, const ir::PTXInstruction &instr);
 		void eval_Brev(CTAContext &context, const ir::PTXInstruction &instr);
@@ -472,18 +475,23 @@ namespace executive {
 		void eval_Cvt(CTAContext &context, const ir::PTXInstruction &instr);
 		void eval_Cvta(CTAContext &context, const ir::PTXInstruction &instr);
 		void eval_Div(CTAContext &context, const ir::PTXInstruction &instr);
+		void eval_Dp(CTAContext &context, const ir::PTXInstruction &instr);
 		void eval_Ex2(CTAContext &context, const ir::PTXInstruction &instr);
 		void eval_Exit(CTAContext &context, const ir::PTXInstruction &instr);
 		void eval_Fma(CTAContext &context, const ir::PTXInstruction &instr);
+		void eval_Fns(CTAContext &context, const ir::PTXInstruction &instr);
+		void eval_Mma(CTAContext &context, const ir::PTXInstruction &instr);
 		void eval_Isspacep(CTAContext &context,
 			const ir::PTXInstruction &instr);
 		void eval_Ld(CTAContext &context, const ir::PTXInstruction &instr);
 		void eval_Ldu(CTAContext &context, const ir::PTXInstruction &instr);
 		void eval_Lg2(CTAContext &context, const ir::PTXInstruction &instr);
+		void eval_Lop3(CTAContext &context, const ir::PTXInstruction &instr);
 		void eval_Mad24(CTAContext &context, const ir::PTXInstruction &instr);
 		void eval_Mad(CTAContext &context, const ir::PTXInstruction &instr);
 		void eval_Max(CTAContext &context, const ir::PTXInstruction &instr);
 		void eval_Membar(CTAContext &context, const ir::PTXInstruction &instr);
+		void eval_Fence(CTAContext &context, const ir::PTXInstruction &instr);
 		void eval_Min(CTAContext &context, const ir::PTXInstruction &instr);
 		void eval_Mov(CTAContext &context, const ir::PTXInstruction &instr);
 		void eval_Mul24(CTAContext &context, const ir::PTXInstruction &instr);
@@ -520,6 +528,8 @@ namespace executive {
 		void eval_Sured(CTAContext &context, const ir::PTXInstruction &instr);
 		void eval_Sust(CTAContext &context, const ir::PTXInstruction &instr);
 		void eval_Suq(CTAContext &context, const ir::PTXInstruction &instr);
+		void eval_Szext(CTAContext &context, const ir::PTXInstruction &instr);
+		void eval_Tanh(CTAContext &context, const ir::PTXInstruction &instr);
 		void eval_TestP(CTAContext &context, const ir::PTXInstruction &instr);
 		void eval_Tex(CTAContext &context, const ir::PTXInstruction &instr);
 		void eval_Trap(CTAContext &context, const ir::PTXInstruction &instr);
@@ -562,4 +572,3 @@ namespace executive {
 }
 
 #endif
-

ocelot/include/ocelot/executive/EmulatedKernel.h

@@ -233,7 +233,7 @@ namespace executive {
 		TextureVector textures;
 
 		/*! A handle to the current scheduler, or 0 if none is executing */
-		EmulatedKernelScheduler* scheduler;
+		EmulatedKernelScheduler* scheduler = nullptr;
 
 	private:
 		/*! Maps program counter to the kernel that begins there */

ocelot/include/ocelot/executive/EmulatedKernelScheduler.h

@@ -49,6 +49,8 @@ class EmulatedKernelScheduler
 		const ir::Dim3& ctaDim, ir::PTXU32 sharedMemory, ir::PTXU64 stream);
 	/*! \brief Get the argument memory for the current context */
 	ir::PTXU64 argumentMemory() const;
+	/*! \brief Get the argument memory size for the current context */
+	ir::PTXU64 argumentMemorySize() const;
 
 private:
 	class Context
@@ -123,4 +125,3 @@ class EmulatedKernelScheduler
 
 }
 
-

ocelot/include/ocelot/executive/EmulatorCallStack.h

@@ -89,6 +89,9 @@ class EmulatorCallStack
 		const RegisterType* registerFilePointer(unsigned int thread) const;
 		/*! \brief Get a pointer to local memory for a given thread */
 		void* localMemoryPointer(unsigned int thread);
+		/*! \brief Test whether an address belongs to any active local frame */
+		bool isLocalMemoryAddress(unsigned long long address,
+			unsigned int thread, unsigned int addressBits) const;
 		/*! \brief Get a pointer to shared memory */
 		void* sharedMemoryPointer();
 		/*! \brief Get a pointer to global local memory */

ocelot/include/ocelot/ir/PTXInstruction.h

@@ -22,6 +22,16 @@ namespace ir {
 			Level_Invalid
 		};
 
+		enum Semantics {
+			Sc,
+			AcqRel,
+			Acquire,
+			Release,
+			Relaxed,
+			Weak,
+			Semantics_Invalid
+		};
+
 		/*!	List of opcodes for PTX instructions */
 		enum Opcode {
 			Abs = 0,
@@ -33,6 +43,7 @@ namespace ir {
 			Bfe,
 			Bfi,
 			Bfind,
+			Bmsk,
 			Bra,
 			Brev,
 			Brkpt,
@@ -44,18 +55,24 @@ namespace ir {
 			Cvt,
 			Cvta,
 			Div,
+			Dp2a,
+			Dp4a,
 			Ex2,
 			Exit,
 			Fma,
+			Fns,
 			Isspacep,
 			Ld,
 			Ldu,
 			Lg2,
+			Lop3,
 			Mad24,
 			Mad,
 			MadC,
+			Mma,
 			Max,
 			Membar,
+			Fence,
 			Min,
 			Mov,
 			Mul24,
@@ -91,6 +108,8 @@ namespace ir {
 			Sured,
 			Sust,
 			Suq,
+			Szext,
+			Tanh,
 			TestP,
 			Tex,
 			Tld4,
@@ -136,13 +155,34 @@ namespace ir {
 			approx = 8192,//< identify an approximate instruction
 			ftz = 16384,	//< flush to zero
 			full = 32768,	//< full division
+			nan = 65536,	//< return NaN if either input is NaN
+			xorsign = 131072,	//< XOR input sign bits
+			abs = 262144,	//< compare absolute input values
+			relu = 524288,	//< clamp negative floating-point results to zero
+			rna = 1048576,	//< round to nearest, ties away from zero
+			satfinite = 2097152,	//< clamp integer mma result to representable range
 			Modifier_invalid = 0
 		};
 
 		enum CarryFlag {
 			None = 0,
 			CC = 1
 		};
+
+		enum MmaShape {
+			MmaM16N8K8,
+			MmaM16N8K16,
+			MmaM8N8K16,
+			MmaM16N8K32,
+			MmaM16N8K4,
+			MmaM8N8K4,
+			MmaM8N8K32,
+			MmaM16N8K64,
+			MmaM8N8K128,
+			MmaM16N8K128,
+			MmaM16N8K256,
+			MmaShape_Invalid
+		};
 
 		enum Volatility {
 			Nonvolatile = 0,
@@ -200,6 +240,7 @@ namespace ir {
 			Cg = 2,
 			Cs = 3,
 			Nc = 4,
+			Lu = 5,
 			Wb = 0,
 			Wt = 1,
 			CacheOperation_Invalid
@@ -344,6 +385,7 @@ namespace ir {
 
 	public:
 		static std::string toString( Level );
+		static std::string toString( Semantics );
 		static std::string toString( CacheLevel cache );
 		static std::string toStringLoad( CacheOperation op );
 		static std::string toStringStore( CacheOperation op );
@@ -425,6 +467,13 @@ namespace ir {
 
 		/*! indicates data type of instruction */
 		PTXOperand::DataType type;
+		/*! Second input type for packed dot-product instructions */
+		PTXOperand::DataType bType;
+
+		/*! Shape for MMA instructions */
+		MmaShape mmaShape;
+		bool mmaAColumnMajor;
+		bool mmaBColumnMajor;
 
 		/*! Flag containing one or more floating-point modifiers */
 		unsigned int modifier;
@@ -441,7 +490,7 @@ namespace ir {
 
 			/*! For membar, the visibility level in the thread hierarchy */
 			Level level;
-			
+
 			/*! Shift amount flag for bfind instructions */
 			bool shiftAmount;
 
@@ -468,8 +517,30 @@ namespace ir {
 
 		};
 
+		/*! For fence, the memory ordering semantics -- deliberately outside the
+			level/shuffleMode/etc. union above: fence needs both a semantics
+			AND a level (scope) simultaneously, unlike the other union members
+			which are each only ever used by instructions with no need for
+			the others at the same time. */
+		Semantics semantics;
+
+		/*! For atom/red, the optional memory-ordering scope (.cta/.gpu/.sys)
+			-- deliberately its own field rather than reusing the level/
+			addressSpace union member above: atom/red already use addressSpace
+			for their .global/.shared qualifier, so writing scope through the
+			shared level union slot would alias and corrupt it. */
+		Level scope;
+
 		/*! For call instructions, indicates a tail call */
 		bool tailCall;
+
+		/*! For ld/st, indicates the .mmio memory-mapped-I/O form
+			(always .sem.sys{.global}) -- deliberately a standalone flag
+			rather than inferred from semantics/scope alone: a plain
+			ld.acquire.sys and ld.mmio.acquire.sys produce the same
+			semantics/scope pair but are syntactically distinct and must
+			round-trip through the printer differently. */
+		bool mmio;
 
 		/*! If the instruction is predicated, the guard */
 		PTXOperand pg;
@@ -503,9 +574,6 @@ namespace ir {
 			/*! Indicates whether the target address space is volatile */
 			Volatility volatility;
 
-			/*! Is this a divide full instruction? */
-			bool divideFull;
-
 			/*! If cvta instruction, indicates whether destination is 
 				generic address or if source is generic address - true if 
 				segmented address space, false if generic */
@@ -554,6 +622,10 @@ namespace ir {
 		/*! Source operand c */
 		PTXOperand c;
 
+		/*! Lookup table and predicate input for lop3 */
+		PTXOperand immLut;
+		PTXOperand q;
+
 		/*  Runtime annotations 
 
 			The following members are used to annotate the instruction 
@@ -602,4 +674,3 @@ namespace ir {
 }
 
 #endif
-

ocelot/include/ocelot/ir/PTXOperand.h

@@ -25,6 +25,7 @@ namespace ir {
 	typedef int32_t PTXS32;
 	typedef int64_t PTXS64;
 
+	typedef _Float16 PTXF16;
 	typedef float PTXF32;
 	typedef double PTXF64;
 
@@ -62,12 +63,17 @@ namespace ir {
 			u64,
 			f16,
 			f32,
+			bf16,
+			tf32,
 			f64,
 			b8,
 			b16,
 			b32,
 			b64,
-			pred
+			pred,
+			f16x2,
+			bf16x2,
+			s4, u4, b1 // MMA element types; stored in packed b32 registers.
 		};
 
 		/*!	Special register names */
@@ -139,7 +145,8 @@ namespace ir {
 		enum Vec {
 			v1 = 1, 			//< scalar
 			v2 = 2,				//< vector2
-			v4 = 4				//< vector4
+			v4 = 4,				//< vector4
+			v8 = 8                  //< eight-register MMA fragment
 		};
 
 		enum VectorIndex {
@@ -285,4 +292,3 @@ namespace std {
 }
 
 #endif
-

ocelot/include/ocelot/parser/PTXParser.h

@@ -137,6 +137,7 @@ namespace parser
 
 				private:
 					void _setImmediateTypes();
+					void _setMovVectorImmediateTypes();
 					std::string _nameInContext( const std::string& name );
 
 					OperandWrapper* _getOperand( const std::string& name );
@@ -236,6 +237,7 @@ namespace parser
 					void constantOperand( double value );
 					void indexedOperand( const std::string& name, 
 						YYLTYPE& location, long long int value );
+					void vectorOperand( unsigned int elements );
 					void addressableOperand( const std::string& name, 
 						long long int value, YYLTYPE& location, 
 						bool invert );
@@ -259,6 +261,10 @@ namespace parser
 					void vote( int token );
 					void shuffle( int token );
 					void level( int token );
+					void semantics( int token );
+					void scope( int token );
+					void mmio( bool condition );
+					void finalizeMmioAddressSpace();
 					void permute( int token );
 					void floatingPointMode( int token );
 					void defaultPermute();
@@ -267,6 +273,9 @@ namespace parser
 					void instruction();
 					void instruction( const std::string& opcode, int dataType );
 					void instruction( const std::string& opcode );
+					void dotType( int token );
+					void lop3();
+					void mma( int shape, int accumulatorType, int aType, int bType, int cType, bool aColumnMajor = false, bool bColumnMajor = true );
 					void tex( int dataType );
 					void tld4( int dataType );
 					void callPrototypeName( const std::string& identifier );
@@ -345,6 +354,7 @@ namespace parser
 			static ir::PTXInstruction::VoteMode tokenToVoteMode( int );
 			static ir::PTXInstruction::ShuffleMode tokenToShuffleMode( int );
 			static ir::PTXInstruction::Level tokenToLevel( int );
+			static ir::PTXInstruction::Semantics tokenToSemantics( int );
 			static ir::PTXInstruction::PermuteMode tokenToPermuteMode( int );
 			static ir::PTXInstruction::FloatingPointMode
 				tokenToFloatingPointMode( int);
@@ -366,4 +376,3 @@ namespace parser
 }
 
 #endif
-