using ChocolArm64.Decoders; using ChocolArm64.IntermediateRepresentation; using ChocolArm64.Memory; using ChocolArm64.State; using ChocolArm64.Translation; using System; using System.Reflection.Emit; using System.Threading; using static ChocolArm64.Instructions.InstEmitMemoryHelper; namespace ChocolArm64.Instructions { static partial class InstEmit { [Flags] private enum AccessType { None = 0, Ordered = 1, Exclusive = 2, OrderedEx = Ordered | Exclusive } public static void Clrex(ILEmitterCtx context) { context.EmitLdarg(TranslatedSub.StateArgIdx); context.EmitPrivateCall(typeof(CpuThreadState), nameof(CpuThreadState.ClearExclusiveAddress)); } public static void Dmb(ILEmitterCtx context) => EmitBarrier(context); public static void Dsb(ILEmitterCtx context) => EmitBarrier(context); public static void Ldar(ILEmitterCtx context) => EmitLdr(context, AccessType.Ordered); public static void Ldaxr(ILEmitterCtx context) => EmitLdr(context, AccessType.OrderedEx); public static void Ldxr(ILEmitterCtx context) => EmitLdr(context, AccessType.Exclusive); public static void Ldxp(ILEmitterCtx context) => EmitLdp(context, AccessType.Exclusive); public static void Ldaxp(ILEmitterCtx context) => EmitLdp(context, AccessType.OrderedEx); private static void EmitLdr(ILEmitterCtx context, AccessType accType) { EmitLoad(context, accType, pair: false); } private static void EmitLdp(ILEmitterCtx context, AccessType accType) { EmitLoad(context, accType, pair: true); } private static void EmitLoad(ILEmitterCtx context, AccessType accType, bool pair) { OpCodeMemEx64 op = (OpCodeMemEx64)context.CurrOp; bool ordered = (accType & AccessType.Ordered) != 0; bool exclusive = (accType & AccessType.Exclusive) != 0; if (ordered) { EmitBarrier(context); } context.EmitLdint(op.Rn); context.EmitSttmp(); if (exclusive) { context.EmitLdarg(TranslatedSub.StateArgIdx); context.EmitLdtmp(); context.EmitPrivateCall(typeof(CpuThreadState), nameof(CpuThreadState.SetExclusiveAddress)); } void WriteExclusiveValue(string propName) { context.Emit(OpCodes.Dup); if (op.Size < 3) { context.Emit(OpCodes.Conv_U8); } context.EmitSttmp2(); context.EmitLdarg(TranslatedSub.StateArgIdx); context.EmitLdtmp2(); context.EmitCallPrivatePropSet(typeof(CpuThreadState), propName); } if (pair) { // Exclusive loads should be atomic. For pairwise loads, we need to // read all the data at once. For a 32-bits pairwise load, we do a // simple 64-bits load, for a 128-bits load, we need to call a special // method to read 128-bits atomically. if (op.Size == 2) { context.EmitLdtmp(); EmitReadZxCall(context, 3); context.Emit(OpCodes.Dup); // Mask low half. context.Emit(OpCodes.Conv_U4); if (exclusive) { WriteExclusiveValue(nameof(CpuThreadState.ExclusiveValueLow)); } context.EmitStintzr(op.Rt); // Shift high half. context.EmitLsr(32); context.Emit(OpCodes.Conv_U4); if (exclusive) { WriteExclusiveValue(nameof(CpuThreadState.ExclusiveValueHigh)); } context.EmitStintzr(op.Rt2); } else if (op.Size == 3) { context.EmitLdarg(TranslatedSub.MemoryArgIdx); context.EmitLdtmp(); context.EmitPrivateCall(typeof(MemoryManager), nameof(MemoryManager.AtomicReadInt128)); context.Emit(OpCodes.Dup); // Load low part of the vector. context.EmitLdc_I4(0); context.EmitLdc_I4(3); VectorHelper.EmitCall(context, nameof(VectorHelper.VectorExtractIntZx)); if (exclusive) { WriteExclusiveValue(nameof(CpuThreadState.ExclusiveValueLow)); } context.EmitStintzr(op.Rt); // Load high part of the vector. context.EmitLdc_I4(1); context.EmitLdc_I4(3); VectorHelper.EmitCall(context, nameof(VectorHelper.VectorExtractIntZx)); if (exclusive) { WriteExclusiveValue(nameof(CpuThreadState.ExclusiveValueHigh)); } context.EmitStintzr(op.Rt2); } else { throw new InvalidOperationException($"Invalid load size of {1 << op.Size} bytes."); } } else { // 8, 16, 32 or 64-bits (non-pairwise) load. context.EmitLdtmp(); EmitReadZxCall(context, op.Size); if (exclusive) { WriteExclusiveValue(nameof(CpuThreadState.ExclusiveValueLow)); } context.EmitStintzr(op.Rt); } } public static void Pfrm(ILEmitterCtx context) { // Memory Prefetch, execute as no-op. } public static void Stlr(ILEmitterCtx context) => EmitStr(context, AccessType.Ordered); public static void Stlxr(ILEmitterCtx context) => EmitStr(context, AccessType.OrderedEx); public static void Stxr(ILEmitterCtx context) => EmitStr(context, AccessType.Exclusive); public static void Stxp(ILEmitterCtx context) => EmitStp(context, AccessType.Exclusive); public static void Stlxp(ILEmitterCtx context) => EmitStp(context, AccessType.OrderedEx); private static void EmitStr(ILEmitterCtx context, AccessType accType) { EmitStore(context, accType, pair: false); } private static void EmitStp(ILEmitterCtx context, AccessType accType) { EmitStore(context, accType, pair: true); } private static void EmitStore(ILEmitterCtx context, AccessType accType, bool pair) { OpCodeMemEx64 op = (OpCodeMemEx64)context.CurrOp; bool ordered = (accType & AccessType.Ordered) != 0; bool exclusive = (accType & AccessType.Exclusive) != 0; if (ordered) { EmitBarrier(context); } if (exclusive) { ILLabel lblEx = new ILLabel(); ILLabel lblEnd = new ILLabel(); context.EmitLdarg(TranslatedSub.StateArgIdx); context.EmitLdint(op.Rn); context.EmitPrivateCall(typeof(CpuThreadState), nameof(CpuThreadState.CheckExclusiveAddress)); context.Emit(OpCodes.Brtrue_S, lblEx); // Address check failed, set error right away and do not store anything. context.EmitLdc_I4(1); context.EmitStintzr(op.Rs); context.Emit(OpCodes.Br, lblEnd); // Address check passed. context.MarkLabel(lblEx); context.EmitLdarg(TranslatedSub.MemoryArgIdx); context.EmitLdint(op.Rn); context.EmitLdarg(TranslatedSub.StateArgIdx); context.EmitCallPrivatePropGet(typeof(CpuThreadState), nameof(CpuThreadState.ExclusiveValueLow)); void EmitCast() { // The input should be always int64. switch (op.Size) { case 0: context.Emit(OpCodes.Conv_U1); break; case 1: context.Emit(OpCodes.Conv_U2); break; case 2: context.Emit(OpCodes.Conv_U4); break; } } EmitCast(); if (pair) { context.EmitLdarg(TranslatedSub.StateArgIdx); context.EmitCallPrivatePropGet(typeof(CpuThreadState), nameof(CpuThreadState.ExclusiveValueHigh)); EmitCast(); context.EmitLdintzr(op.Rt); EmitCast(); context.EmitLdintzr(op.Rt2); EmitCast(); switch (op.Size) { case 2: context.EmitPrivateCall(typeof(MemoryManager), nameof(MemoryManager.AtomicCompareExchange2xInt32)); break; case 3: context.EmitPrivateCall(typeof(MemoryManager), nameof(MemoryManager.AtomicCompareExchangeInt128)); break; default: throw new InvalidOperationException($"Invalid store size of {1 << op.Size} bytes."); } } else { context.EmitLdintzr(op.Rt); EmitCast(); switch (op.Size) { case 0: context.EmitCall(typeof(MemoryManager), nameof(MemoryManager.AtomicCompareExchangeByte)); break; case 1: context.EmitCall(typeof(MemoryManager), nameof(MemoryManager.AtomicCompareExchangeInt16)); break; case 2: context.EmitCall(typeof(MemoryManager), nameof(MemoryManager.AtomicCompareExchangeInt32)); break; case 3: context.EmitCall(typeof(MemoryManager), nameof(MemoryManager.AtomicCompareExchangeInt64)); break; default: throw new InvalidOperationException($"Invalid store size of {1 << op.Size} bytes."); } } // The value returned is a bool, true if the values compared // were equal and the new value was written, false otherwise. // We need to invert this result, as on ARM 1 indicates failure, // and 0 success on those instructions. context.EmitLdc_I4(1); context.Emit(OpCodes.Xor); context.Emit(OpCodes.Dup); context.Emit(OpCodes.Conv_U8); context.EmitStintzr(op.Rs); // Only clear the exclusive monitor if the store was successful (Rs = false). context.Emit(OpCodes.Brtrue_S, lblEnd); Clrex(context); context.MarkLabel(lblEnd); } else { void EmitWriteCall(int rt, long offset) { context.EmitLdint(op.Rn); if (offset != 0) { context.EmitLdc_I8(offset); context.Emit(OpCodes.Add); } context.EmitLdintzr(rt); InstEmitMemoryHelper.EmitWriteCall(context, op.Size); } EmitWriteCall(op.Rt, 0); if (pair) { EmitWriteCall(op.Rt2, 1 << op.Size); } } } private static void EmitBarrier(ILEmitterCtx context) { // Note: This barrier is most likely not necessary, and probably // doesn't make any difference since we need to do a ton of stuff // (software MMU emulation) to read or write anything anyway. context.EmitCall(typeof(Thread), nameof(Thread.MemoryBarrier)); } } }